diff --git a/README.md b/README.md index 14cf9893..b0faa26b 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,13 @@ Once you have a valid disk, and optionally partition, you can access filesystems * `CreateFilesystem()` - create a filesystem in an individual partition or the entire disk * `GetFilesystem()` - access an existing filesystem in a partition or the entire disk -As of this writing, supported filesystems include `FAT32` and `ISO9660` (a.k.a. `.iso`). +As of this writing, supported filesystems include `FAT32`, `ISO9660` (a.k.a. `.iso`), `SquashFS`, `ext4`, and read-only `EROFS`. + +EROFS support targets a read-only subset of the EROFS on-disk format: basic and +LZMA-compressed files, compact and full compression indexes, fragment tails, +and large logical clusters. Extended attributes are not exposed. See the +[filesystem/erofs](./filesystem/erofs) package documentation for details, +including how to read an image nested inside another backend. With a filesystem in hand, you can create, access and modify directories and files. @@ -84,7 +90,7 @@ With a `File` in hand, you then can: * `Seek(offset int64, whence int)` to set the next read or write to an offset in the file ### Read-Only Filesystems -Some filesystem types are intended to be created once, after which they are read-only, for example `ISO9660`/`.iso` and `squashfs`. +Some filesystem types are intended to be created once, after which they are read-only, for example `ISO9660`/`.iso` and `squashfs`. EROFS is currently read-only with no creation support. `godiskfs` recognizes read-only filesystems and limits working with them to the following: diff --git a/disk/disk.go b/disk/disk.go index dd0851f2..632be7c2 100644 --- a/disk/disk.go +++ b/disk/disk.go @@ -11,6 +11,7 @@ import ( "github.com/diskfs/go-diskfs/backend" "github.com/diskfs/go-diskfs/filesystem" + "github.com/diskfs/go-diskfs/filesystem/erofs" "github.com/diskfs/go-diskfs/filesystem/ext4" "github.com/diskfs/go-diskfs/filesystem/fat12" "github.com/diskfs/go-diskfs/filesystem/fat16" @@ -178,6 +179,8 @@ func (d *Disk) CreateFilesystem(spec FilesystemSpec) (filesystem.FileSystem, err return ext4.Create(d.Backend, size, start, d.LogicalBlocksize, &ext4.Params{VolumeName: spec.VolumeLabel}) case filesystem.TypeSquashfs: return squashfs.Create(d.Backend, size, start, d.LogicalBlocksize) + case filesystem.TypeErofs: + return nil, filesystem.ErrNotImplemented default: return nil, errors.New("unknown filesystem type requested") } @@ -246,6 +249,12 @@ func (d *Disk) GetFilesystem(partIndex int) (filesystem.FileSystem, error) { if err == nil { return squashFS, nil } + log.Debug("trying erofs") + erofsFS, err := erofs.Read(d.Backend, size, start, 0) + if err == nil { + return erofsFS, nil + } + log.Debugf("erofs failed: %v", err) log.Debug("trying ext4") ext4FS, err := ext4.Read(d.Backend, size, start, d.LogicalBlocksize) if err == nil { diff --git a/disk/disk_test.go b/disk/disk_test.go index 37353f4b..0f2ac3f3 100644 --- a/disk/disk_test.go +++ b/disk/disk_test.go @@ -616,4 +616,53 @@ func TestGetFilesystem(t *testing.T) { t.Errorf("returned filesystem was unexpectedly nil") } }) + t.Run("erofs partition", func(t *testing.T) { + image, err := os.ReadFile("../filesystem/erofs/testdata/compressed.erofs") + if err != nil { + t.Fatalf("error reading EROFS fixture: %v", err) + } + f, err := os.CreateTemp("", "erofs_disk_test") + if err != nil { + t.Fatalf("error creating temporary disk: %v", err) + } + defer f.Close() + if _, err := f.Write(make([]byte, 4096)); err != nil { + t.Fatalf("error writing partition prefix: %v", err) + } + if _, err := f.Write(image); err != nil { + t.Fatalf("error writing EROFS image: %v", err) + } + fileInfo, err := f.Stat() + if err != nil { + t.Fatalf("error reading temporary disk info: %v", err) + } + + table := &mbr.Table{ + Partitions: []*mbr.Partition{ + {Index: 1, Start: 8, Size: uint32(len(image) / 512)}, + }, + LogicalSectorSize: 512, + } + d := &disk.Disk{ + Backend: file.New(f, true), + LogicalBlocksize: 512, + PhysicalBlocksize: 512, + Size: fileInfo.Size(), + Table: table, + } + fs, err := d.GetFilesystem(1) + if err != nil { + t.Fatalf("GetFilesystem() error = %v", err) + } + if fs.Type() != filesystem.TypeErofs { + t.Fatalf("filesystem type = %v, want %v", fs.Type(), filesystem.TypeErofs) + } + data, err := fs.ReadFile("sub/data.bin") + if err != nil { + t.Fatalf("ReadFile() error = %v", err) + } + if len(data) != 96*1024 { + t.Fatalf("ReadFile() length = %d, want %d", len(data), 96*1024) + } + }) } diff --git a/filesystem/erofs/cache_internal_test.go b/filesystem/erofs/cache_internal_test.go new file mode 100644 index 00000000..2043084a --- /dev/null +++ b/filesystem/erofs/cache_internal_test.go @@ -0,0 +1,43 @@ +package erofs + +import ( + "bytes" + "io" + "os" + "sync" + "testing" +) + +func TestConcurrentCompressedReadAt(t *testing.T) { + filesystem := readTestImage(t, fullFixture) + opened, err := filesystem.OpenFile("data.bin", os.O_RDONLY) + if err != nil { + t.Fatal(err) + } + defer opened.Close() + reader, ok := opened.(io.ReaderAt) + if !ok { + t.Fatalf("OpenFile() returned %T, want io.ReaderAt", opened) + } + payload := fullFixturePayload() + var workers sync.WaitGroup + for worker := 0; worker < 8; worker++ { + workers.Go(func() { + buffer := make([]byte, 1024) + for i := 0; i < 32; i++ { + offset := (worker*7919 + i*4093) % (len(payload) - len(buffer)) + n, err := reader.ReadAt(buffer, int64(offset)) + if err != nil || n != len(buffer) || !bytes.Equal(buffer, payload[offset:offset+n]) { + t.Errorf("ReadAt(%d) = %d, %v or incorrect data", offset, n, err) + return + } + } + }) + } + workers.Wait() + // ReaderAt must not change the sequential cursor. + buffer := make([]byte, 32) + if _, err := io.ReadFull(opened, buffer); err != nil || !bytes.Equal(buffer, payload[:len(buffer)]) { + t.Fatalf("Read after ReadAt = %x, %v", buffer, err) + } +} diff --git a/filesystem/erofs/compatibility_test.go b/filesystem/erofs/compatibility_test.go new file mode 100644 index 00000000..a6efa1bb --- /dev/null +++ b/filesystem/erofs/compatibility_test.go @@ -0,0 +1,292 @@ +package erofs + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "math" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/diskfs/go-diskfs/backend/file" +) + +const ( + uncompressedFixture = "uncompressed.erofs" + compressedFixture = "compressed.erofs" + fullFixture = "full.erofs" + largeFixture = "large.erofs" +) + +func testPayload() []byte { + payload := make([]byte, 96*1024) + seed := uint32(0x45524f46) + for offset := 0; offset < len(payload); offset += 4 { + seed = seed*1664525 + 1013904223 + binary.LittleEndian.PutUint32(payload[offset:offset+4], seed) + } + return payload +} + +func readTestImage(tb testing.TB, name string) *FileSystem { + tb.Helper() + path := filepath.Join("testdata", name) + f, err := os.Open(path) + if err != nil { + tb.Fatalf("os.Open(%q) error = %v", path, err) + } + tb.Cleanup(func() { _ = f.Close() }) + info, err := f.Stat() + if err != nil { + tb.Fatalf("f.Stat() error = %v", err) + } + filesystem, err := Read(file.New(f, true), info.Size(), 0, 0) + if err != nil { + tb.Fatalf("Read() error = %v", err) + } + return filesystem +} + +func readFixtureBytes(t *testing.T, name string) []byte { + t.Helper() + image, err := os.ReadFile(filepath.Join("testdata", name)) + if err != nil { + t.Fatalf("os.ReadFile(%q) error = %v", name, err) + } + return image +} + +func TestReadUncompressedFixture(t *testing.T) { + filesystem := readTestImage(t, uncompressedFixture) + entries, err := filesystem.ReadDir(".") + if err != nil { + t.Fatalf("ReadDir() error = %v", err) + } + if len(entries) != 5 { + t.Fatalf("ReadDir() returned %d entries, want 5", len(entries)) + } + if data, err := filesystem.ReadFile("hello.txt"); err != nil || string(data) != "hello world\n" { + t.Fatalf("ReadFile(hello.txt) = %q, %v; want hello world\\n, ", data, err) + } + if target, err := filesystem.ReadFile("link"); err != nil || string(target) != "hello.txt" { + t.Fatalf("ReadFile(link) = %q, %v; want hello.txt, ", target, err) + } + data, err := filesystem.ReadFile("sub/data.bin") + if err != nil { + t.Fatalf("ReadFile(sub/data.bin) error = %v", err) + } + expected := testPayload() + expectedHash := sha256.Sum256(expected) + dataHash := sha256.Sum256(data) + if dataHash != expectedHash { + t.Fatalf("ReadFile(sub/data.bin) SHA-256 = %x, want %x", dataHash, expectedHash) + } +} + +func TestReadCompressedFixture(t *testing.T) { + filesystem := readTestImage(t, compressedFixture) + data, err := filesystem.ReadFile("sub/data.bin") + if err != nil { + t.Fatalf("ReadFile(sub/data.bin) error = %v", err) + } + expected := testPayload() + if len(data) != len(expected) { + t.Fatalf("ReadFile(sub/data.bin) length = %d, want %d", len(data), len(expected)) + } + if !bytes.Equal(data, expected) { + t.Fatal("ReadFile(sub/data.bin) does not match the source payload") + } +} + +func fullFixturePayload() []byte { + pattern := []byte("EROFS deterministic fixture payload\n") + payload := bytes.Repeat(pattern, 2622) + return append(payload, make([]byte, 96*1024-len(payload))...) +} + +func updateSuperblockChecksum(image []byte) { + binary.LittleEndian.PutUint32(image[superOffset+4:superOffset+8], superblockChecksum(image[superOffset:4096])) +} + +func TestReadFullCompressedFixture(t *testing.T) { + filesystem := readTestImage(t, fullFixture) + data, err := filesystem.ReadFile("data.bin") + if err != nil { + t.Fatalf("ReadFile(data.bin) error = %v", err) + } + if !bytes.Equal(data, fullFixturePayload()) { + t.Fatal("ReadFile(data.bin) does not match the source payload") + } +} + +func compressionHeaderOffset(t *testing.T, filesystem *FileSystem, path string) int64 { + t.Helper() + node, err := filesystem.stat(path) + if err != nil { + t.Fatalf("stat(%q) error = %v", path, err) + } + inodeOffset := int64(filesystem.superblock.metaBlockAddress)*filesystem.blockSize() + int64(node.nid)*inodeSlotSize + return roundUp8(inodeOffset + int64(node.inodeSize+node.xattrSize)) +} + +func TestCompressedFixtureCorruption(t *testing.T) { + filesystem := readTestImage(t, compressedFixture) + headerOffset := compressionHeaderOffset(t, filesystem, "sub/data.bin") + indexOffset := headerOffset + 8 + + tests := []struct { + name string + mutate func([]byte) + expected string + }{ + { + name: "invalid compact head type", + mutate: func(image []byte) { + binary.LittleEndian.PutUint16(image[indexOffset:], 0x3000) + }, + expected: "unsupported compression algorithm 0", + }, + { + name: "extent outside image", + mutate: func(image []byte) { + binary.LittleEndian.PutUint32(image[indexOffset+4:], math.MaxUint32) + }, + expected: "outside the image", + }, + { + name: "fragment outside packed inode", + mutate: func(image []byte) { + binary.LittleEndian.PutUint32(image[headerOffset:], math.MaxUint32) + }, + expected: "outside packed inode", + }, + { + name: "invalid packed inode", + mutate: func(image []byte) { + binary.LittleEndian.PutUint64(image[superOffset+96:], 999999) + updateSuperblockChecksum(image) + }, + expected: "inode 999999 offset", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + image := readFixtureBytes(t, compressedFixture) + tt.mutate(image) + updateSuperblockChecksum(image) + corrupted, err := Read(testBackend(t, image), int64(len(image)), 0, 0) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + _, err = corrupted.ReadFile("sub/data.bin") + if err == nil || !strings.Contains(err.Error(), tt.expected) { + t.Fatalf("ReadFile() error = %v, want containing %q", err, tt.expected) + } + }) + } +} + +func TestFullCompressedFixtureCorruption(t *testing.T) { + filesystem := readTestImage(t, fullFixture) + headerOffset := compressionHeaderOffset(t, filesystem, "data.bin") + indexOffset := headerOffset + 16 + + tests := []struct { + name string + mutate func([]byte) + expected string + }{ + { + name: "unsupported algorithm", + mutate: func(image []byte) { image[headerOffset+6] = compressionLZ4 }, + expected: "unsupported compression algorithm 0", + }, + { + name: "invalid full head type", + mutate: func(image []byte) { + binary.LittleEndian.PutUint16(image[indexOffset:], uint16(zClusterTypeHead2)) + }, + expected: "unsupported compression algorithm 0", + }, + { + name: "extent outside image", + mutate: func(image []byte) { + binary.LittleEndian.PutUint32(image[indexOffset+4:], math.MaxUint32) + }, + expected: "outside the image", + }, + { + name: "malformed LZMA", + mutate: func(image []byte) { + physicalStart := int64(binary.LittleEndian.Uint32(image[indexOffset+4:])) * filesystem.blockSize() + for offset := physicalStart; offset < physicalStart+filesystem.blockSize(); offset++ { + image[offset] ^= 0xff + } + }, + expected: "unable to initialize LZMA decoder", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + image := readFixtureBytes(t, fullFixture) + tt.mutate(image) + updateSuperblockChecksum(image) + corrupted, err := Read(testBackend(t, image), int64(len(image)), 0, 0) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + _, err = corrupted.ReadFile("data.bin") + if err == nil || !strings.Contains(err.Error(), tt.expected) { + t.Fatalf("ReadFile() error = %v, want containing %q", err, tt.expected) + } + }) + } +} + +func largeFixturePayload() []byte { + payload := make([]byte, 15<<20) + seed := uint32(0x45524f46) + for record := 0; record < len(payload); record += 64 << 10 { + for offset := record; offset < record+(16<<10); offset += 4 { + seed = seed*1664525 + 1013904223 + binary.LittleEndian.PutUint32(payload[offset:], seed) + } + } + return payload +} + +func TestReadLargeCompressedFixture(t *testing.T) { + filesystem := readTestImage(t, largeFixture) + node, err := filesystem.stat("test/path/file") + if err != nil { + t.Fatalf("stat(test/path/file) error = %v", err) + } + if err := filesystem.initCompressed(node); err != nil { + t.Fatalf("initCompressed() error = %v", err) + } + first, err := filesystem.mapCompressed(node, 0, false) + if err != nil { + t.Fatalf("mapCompressed(0) error = %v", err) + } + if first.physicalLength != 1<<20 || first.logicalLength <= 1<<20 { + t.Fatalf("first extent = %+v, want 1 MiB encoded and more than 1 MiB decoded", first) + } + tail, err := filesystem.mapCompressed(node, node.size-1, false) + if err != nil { + t.Fatalf("mapCompressed(tail) error = %v", err) + } + if !tail.fragment { + t.Fatal("last extent is not a packed fragment") + } + data, err := filesystem.ReadFile("test/path/file") + if err != nil { + t.Fatalf("ReadFile() error = %v", err) + } + sourceHash := sha256.Sum256(largeFixturePayload()) + readHash := sha256.Sum256(data) + if sourceHash != readHash { + t.Fatalf("ReadFile() SHA-256 = %x, want %x", readHash, sourceHash) + } +} diff --git a/filesystem/erofs/compressedmap.go b/filesystem/erofs/compressedmap.go new file mode 100644 index 00000000..e8d5d4db --- /dev/null +++ b/filesystem/erofs/compressedmap.go @@ -0,0 +1,482 @@ +package erofs + +import ( + "encoding/binary" + "fmt" + "math" +) + +const ( + // Limit each physical cluster to 1 MiB encoded and 12 MiB decoded, matching + // Z_EROFS_PCLUSTER_MAX_SIZE and Z_EROFS_PCLUSTER_MAX_DSIZE in the EROFS + // on-disk definitions (fs/erofs/erofs_fs.h). These bounds cap input/output + // allocations and the decoded extent retained in the cache. + maxEncodedExtentSize = 1 << 20 + maxDecodedExtentSize = 12 << 20 +) + +type compressedInfo struct { + advise uint16 + // Logical clusters are 1<> 4 + if info.advise&zAdviseFragment != 0 { + info.fragmentOffset = int64(binary.LittleEndian.Uint32(header[0:4])) + } else if info.advise&zAdviseInlineData != 0 && binary.LittleEndian.Uint16(header[2:4]) != 0 { + return fmt.Errorf("inline compressed tails are not supported for inode %d", node.nid) + } + if node.dataLayout == dataLayoutCompressedCompact && + ((info.advise&zAdviseBigPcluster1 == 0) != (info.advise&zAdviseBigPcluster2 == 0)) { + return fmt.Errorf("inconsistent big pcluster advice for inode %d", node.nid) + } + if info.lclusterBits > 27 { + return fmt.Errorf("invalid logical cluster bits %d for inode %d", info.lclusterBits, node.nid) + } + node.compressed = info + if info.advise&zAdviseFragment != 0 { + extent, err := f.mapCompressed(node, node.size-1, true) + if err != nil { + return err + } + info.tailExtentHeadLCN = uint64(extent.logicalStart >> info.lclusterBits) + if node.dataLayout == dataLayoutCompressedFull { + fragmentOffset := uint64(info.fragmentOffset) | uint64(extent.physicalBlock)<<32 + if fragmentOffset > math.MaxInt64 { + return fmt.Errorf("fragment offset for inode %d is too large", node.nid) + } + info.fragmentOffset = int64(fragmentOffset) + } + } + info.initialized = true + node.compressed = info + return nil +} + +func (f *FileSystem) mapCompressed(node *inode, offset int64, findTail bool) (*compressedExtent, error) { + if offset < 0 || offset >= node.size { + return nil, fmt.Errorf("compressed offset %d is outside inode %d size %d", offset, node.nid, node.size) + } + info := node.compressed + if info == nil { + return nil, fmt.Errorf("compression metadata is not initialized for inode %d", node.nid) + } + if info.advise&zAdviseFragment != 0 && !findTail && info.tailExtentHeadLCN == 0 { + return &compressedExtent{logicalStart: 0, logicalLength: node.size, fragment: true}, nil + } + if node.dataLayout == dataLayoutCompressedFull && info.advise&1 != 0 { + return nil, fmt.Errorf("compressed extent metadata is not supported for inode %d", node.nid) + } + + initialLCN := uint64(offset >> info.lclusterBits) + endOffset := int(offset & ((1 << info.lclusterBits) - 1)) + recorder := mapRecorder{} + var err error + if node.dataLayout == dataLayoutCompressedCompact { + err = f.loadCompactLCluster(node, &recorder, initialLCN, false) + } else { + err = f.loadFullLCluster(node, &recorder, initialLCN) + } + if err != nil { + return nil, err + } + extent := &compressedExtent{} + switch recorder.clusterType { + case zClusterTypePlain, zClusterTypeHead1, zClusterTypeHead2: + if endOffset >= recorder.clusterOffset { + recorder.headType = recorder.clusterType + } else { + if recorder.lcn == 0 { + return nil, fmt.Errorf("invalid logical cluster zero for inode %d", node.nid) + } + recorder.delta[0] = 1 + if err := f.lookBack(node, &recorder, recorder.delta[0]); err != nil { + return nil, err + } + } + case zClusterTypeNonHead: + if err := f.lookBack(node, &recorder, recorder.delta[0]); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("invalid logical cluster type %d for inode %d", recorder.clusterType, node.nid) + } + extent.logicalStart = int64(recorder.lcn)<= compressionInterlaced+1 || + (extent.algorithm < compressionShifted && f.superblock.availableComprAlgs&(1< maxEncodedExtentSize || extent.logicalLength <= 0 || extent.logicalLength > maxDecodedExtentSize { + return fmt.Errorf("invalid compressed extent sizes for inode %d", node.nid) + } + if extent.physicalStart < 0 || extent.physicalStart+extent.physicalLength > f.size { + return fmt.Errorf("compressed extent for inode %d is outside the image", node.nid) + } + return nil +} + +func (f *FileSystem) compressedExtentEnd(node *inode, headLCN uint64) (int64, error) { + info := node.compressed + lcn := headLCN + for { + if int64(lcn)<= node.size { + return node.size, nil + } + recorder := mapRecorder{} + var err error + if node.dataLayout == dataLayoutCompressedCompact { + err = f.loadCompactLCluster(node, &recorder, lcn, true) + } else { + err = f.loadFullLCluster(node, &recorder, lcn) + } + if err != nil { + return 0, err + } + delta := recorder.delta[1] + if recorder.clusterType == zClusterTypeNonHead { + if delta == 0 { + delta = 1 + } + } else { + if lcn != headLCN { + return int64(lcn)<> (bitPosition & 7) + return uint16(value & ((1 << bits) - 1)), uint8(value >> bits & 3) +} + +// loadCompactLCluster decodes the EROFS compacted compression-index layout. +// With COMPACTED_2B enabled, initial 4-byte records align the 2-byte records to +// 32-byte packs, followed by trailing 4-byte records. Each pack ends with a base +// physical block number; the head/non-head records encode logical offsets or +// lookback distances and counts. +func (f *FileSystem) loadCompactLCluster(node *inode, recorder *mapRecorder, requestedLCN uint64, lookahead bool) error { + info := node.compressed + inodeOffset, err := f.inodeOffset(node.nid) + if err != nil { + return err + } + ebase := 8 + roundUp8(inodeOffset+int64(node.inodeSize+node.xattrSize)) + totalIndex := uint64((node.size + f.blockSize() - 1) / f.blockSize()) + if requestedLCN >= totalIndex || info.lclusterBits > 14 { + return fmt.Errorf("invalid logical cluster %d for inode %d", requestedLCN, node.nid) + } + lcn := requestedLCN + compacted4Initial := uint64(((32 - ebase%32) / 4) & 7) + compacted2B := uint64(0) + if info.advise&zAdviseCompacted2B != 0 && compacted4Initial < totalIndex { + compacted2B = (totalIndex - compacted4Initial) / 16 * 16 + } + pos := ebase + shift := uint(2) + if lcn >= compacted4Initial { + pos += int64(compacted4Initial) * 4 + lcn -= compacted4Initial + if lcn < compacted2B { + shift = 1 + } else { + pos += int64(compacted2B) * 2 + lcn -= compacted2B + } + } + pos += int64(lcn) << shift + var vcnt uint + switch { + case shift == 2 && info.lclusterBits <= 14: + vcnt = 2 + case shift == 1 && info.lclusterBits <= 12: + vcnt = 16 + default: + return fmt.Errorf("unsupported compact index packing for inode %d", node.nid) + } + packSize := int64(vcnt) << shift + packStart := pos &^ (packSize - 1) + pack := make([]byte, packSize) + if err := f.readAt(pack, packStart); err != nil { + return fmt.Errorf("unable to read compact compression index for inode %d: %w", node.nid, err) + } + lobits := info.lclusterBits + if lobits < 12 { + lobits = 12 + } + encodeBits := int(((packSize - 4) * 8) / int64(vcnt)) + index := int((pos - packStart) >> shift) + lo, clusterType := decodeCompactBits(lobits, pack, encodeBits*index) + recorder.lcn = requestedLCN + recorder.clusterType = clusterType + if clusterType == zClusterTypeNonHead { + recorder.clusterOffset = 1 << info.lclusterBits + if lookahead { + recorder.delta[1] = compactLookahead(pack, index, int(vcnt), encodeBits, int(lobits)) + } + if lo&zD0ClusterCount != 0 { + if info.advise&zAdviseBigPcluster1 == 0 { + return fmt.Errorf("invalid big pcluster count for inode %d", node.nid) + } + recorder.compressedBlks = uint32(lo &^ zD0ClusterCount) + recorder.delta[0] = 1 + return f.validateLCluster(node, recorder) + } + if index+1 != int(vcnt) { + recorder.delta[0] = lo + return f.validateLCluster(node, recorder) + } + previous, previousType := decodeCompactBits(lobits, pack, encodeBits*(index-1)) + if previousType != zClusterTypeNonHead { + previous = 0 + } else if previous&zD0ClusterCount != 0 { + previous = 1 + } + recorder.delta[0] = previous + 1 + return f.validateLCluster(node, recorder) + } + recorder.clusterOffset = int(lo) + recorder.delta[0] = 0 + bigPcluster := info.advise&zAdviseBigPcluster1 != 0 + nblk := uint32(1) + if bigPcluster { + nblk = 0 + } + for index > 0 { + index-- + lo, clusterType = decodeCompactBits(lobits, pack, encodeBits*index) + if clusterType == zClusterTypeNonHead { + switch { + case !bigPcluster: + index -= int(lo) + case lo&zD0ClusterCount != 0: + index-- + nblk += uint32(lo &^ zD0ClusterCount) + continue + case lo <= 1: + return fmt.Errorf("invalid big pcluster delta for inode %d", node.nid) + default: + index -= int(lo) - 2 + continue + } + } + if index >= 0 { + nblk++ + } + } + recorder.physicalBlock = binary.LittleEndian.Uint32(pack[packSize-4:]) + nblk + return f.validateLCluster(node, recorder) +} + +func compactLookahead(pack []byte, index, vcnt, encodeBits, lobits int) uint16 { + var distance uint16 + var last uint16 + for i := index; i < vcnt; i++ { + lo, clusterType := decodeCompactBits(uint(lobits), pack, encodeBits*i) + if clusterType != zClusterTypeNonHead { + return distance + } + distance++ + last = lo + } + if last&zD0ClusterCount == 0 { + distance += last - 1 + } + return distance +} + +func (f *FileSystem) validateLCluster(node *inode, recorder *mapRecorder) error { + if recorder.clusterType > zClusterTypeHead2 { + return fmt.Errorf("invalid compression index type %d for inode %d", recorder.clusterType, node.nid) + } + if recorder.clusterType != zClusterTypeNonHead && recorder.clusterOffset >= 1<= uint64(distance) { + lcn := recorder.lcn - uint64(distance) + var err error + if node.dataLayout == dataLayoutCompressedCompact { + err = f.loadCompactLCluster(node, recorder, lcn, false) + } else { + err = f.loadFullLCluster(node, recorder, lcn) + } + if err != nil { + return err + } + if recorder.clusterType == zClusterTypeNonHead { + distance = recorder.delta[0] + if distance == 0 { + break + } + continue + } + recorder.headType = recorder.clusterType + return nil + } + return fmt.Errorf("invalid compression lookback distance for inode %d", node.nid) +} + +func (f *FileSystem) compressedLength(node *inode, recorder *mapRecorder) error { + info := node.compressed + if (recorder.headType == zClusterTypeHead1 && info.advise&zAdviseBigPcluster1 == 0) || + ((recorder.headType == zClusterTypePlain || recorder.headType == zClusterTypeHead2) && info.advise&zAdviseBigPcluster2 == 0) || + (recorder.lcn+1)<= uint64(node.size) { + recorder.compressedBlks = 1 + } + if recorder.compressedBlks != 0 { + return nil + } + lcn := recorder.lcn + 1 + var err error + if node.dataLayout == dataLayoutCompressedCompact { + err = f.loadCompactLCluster(node, recorder, lcn, false) + } else { + err = f.loadFullLCluster(node, recorder, lcn) + } + if err != nil { + return err + } + if recorder.clusterType != zClusterTypeNonHead { + recorder.compressedBlks = 1 + return nil + } + if recorder.clusterType != zClusterTypeNonHead || recorder.delta[0] != 1 || recorder.compressedBlks == 0 { + return fmt.Errorf("unable to determine compressed length for inode %d", node.nid) + } + return nil +} diff --git a/filesystem/erofs/compressedmap_internal_test.go b/filesystem/erofs/compressedmap_internal_test.go new file mode 100644 index 00000000..23b8d0e9 --- /dev/null +++ b/filesystem/erofs/compressedmap_internal_test.go @@ -0,0 +1,52 @@ +package erofs + +import ( + "encoding/binary" + "testing" +) + +func TestInitCompressedFullFragmentHighOffset(t *testing.T) { + image := make([]byte, 16384) + binary.LittleEndian.PutUint32(image[4160:], 7) + binary.LittleEndian.PutUint16(image[4164:], zAdviseFragment) + + indexes := []struct { + offset int + physicalBlock uint32 + }{ + {offset: 4176, physicalBlock: 10}, + {offset: 4184, physicalBlock: 11}, + {offset: 4192, physicalBlock: 1}, + } + for _, index := range indexes { + binary.LittleEndian.PutUint16(image[index.offset:], uint16(zClusterTypeHead1)) + binary.LittleEndian.PutUint16(image[index.offset+2:], 0) + binary.LittleEndian.PutUint32(image[index.offset+4:], index.physicalBlock) + } + + filesystem := &FileSystem{ + backend: testBackend(t, image), + size: int64(len(image)), + blocksize: 4096, + superblock: &superblock{ + blockSizeBits: 12, + metaBlockAddress: 1, + }, + } + node := &inode{ + nid: 1, + inodeSize: inodeCompactSize, + size: 12288, + dataLayout: dataLayoutCompressedFull, + } + if err := filesystem.initCompressed(node); err != nil { + t.Fatalf("initCompressed() error = %v", err) + } + want := int64(uint64(1)<<32 | 7) + if node.compressed.fragmentOffset != want { + t.Fatalf("fragmentOffset = %d, want %d", node.compressed.fragmentOffset, want) + } + if node.compressed.fragmentOffset < 0 { + t.Fatalf("fragmentOffset = %d is negative", node.compressed.fragmentOffset) + } +} diff --git a/filesystem/erofs/compression.go b/filesystem/erofs/compression.go new file mode 100644 index 00000000..bc053475 --- /dev/null +++ b/filesystem/erofs/compression.go @@ -0,0 +1,99 @@ +package erofs + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + + "github.com/ulikunitz/xz/lzma" +) + +// decodedExtent holds one LZMA extent in inode-relative byte coordinates. A +// File owns one entry, including when reading a fragment through the packed NID. +type decodedExtent struct { + nid uint64 + logicalStart int64 + data []byte +} + +func (f *FileSystem) parseCompressionConfig(s *superblock) error { + if s.featureIncompat&featureIncompatComprCfgs == 0 { + return nil + } + const allCompressionAlgorithms uint16 = (1 << 4) - 1 + if unknown := s.availableComprAlgs &^ allCompressionAlgorithms; unknown != 0 { + return fmt.Errorf("unsupported compression algorithms 0x%04x", unknown) + } + offset := int64(superOffset + s.superblockSize()) + for algorithm := uint8(0); algorithm < 4; algorithm++ { + if s.availableComprAlgs&(1< 8*1024*1024 || format != 0 { + return fmt.Errorf("invalid LZMA configuration dictionary size %d format %d", dictSize, format) + } + for _, reserved := range config[6:] { + if reserved != 0 { + return fmt.Errorf("nonzero reserved byte in LZMA configuration") + } + } + f.lzmaDictSize = dictSize + } + offset += int64(2 + length) + } + return nil +} + +func (f *FileSystem) decompressLZMA(encoded []byte, decodedLength int64) ([]byte, error) { + // MicroLZMA replaces the initial zero range-coder byte with the bitwise + // complement of the LZMA properties byte, as documented in the MicroLZMA + // decompressor section of https://docs.kernel.org/staging/xz.html. + // Restore the zero byte and build a classic .lzma header from the properties, + // dictionary size and exact output length so the LZMA decoder can read it. + input := encoded + for len(input) > 0 && input[0] == 0 { + input = input[1:] + } + if len(input) == 0 || decodedLength <= 0 { + return nil, fmt.Errorf("invalid LZMA compressed data") + } + dictSize := f.lzmaDictSize + if dictSize == 0 { + dictSize = 8 * 1024 * 1024 + } + headerSize := decodedLength + header := make([]byte, lzma.HeaderLen) + header[0] = ^input[0] + binary.LittleEndian.PutUint32(header[1:5], dictSize) + binary.LittleEndian.PutUint64(header[5:13], uint64(headerSize)) + classic := append([]byte{0}, input[1:]...) + reader, err := lzma.ReaderConfig{DictCap: int(dictSize)}.NewReader(io.MultiReader(bytes.NewReader(header), bytes.NewReader(classic))) + if err != nil { + return nil, fmt.Errorf("unable to initialize LZMA decoder: %w", err) + } + decoded := make([]byte, decodedLength) + if n, err := io.ReadFull(reader, decoded); err != nil { + return nil, fmt.Errorf("unable to decode LZMA data (%d/%d bytes): %w", n, decodedLength, err) + } + return decoded, nil +} diff --git a/filesystem/erofs/constants.go b/filesystem/erofs/constants.go new file mode 100644 index 00000000..1de90dbc --- /dev/null +++ b/filesystem/erofs/constants.go @@ -0,0 +1,67 @@ +package erofs + +const superMagic uint32 = 0xE0F5E1E2 + +const ( + superOffset = 1024 + superblockBaseSize = 128 + superblockExtSlotSize = 16 + inodeSlotSize = 32 + inodeCompactSize = 32 + inodeExtendedSize = 64 + direntSize = 12 + maxNameLength = 255 +) + +const ( + featureCompatChecksum uint32 = 1 << 0 + featureCompatMtime uint32 = 1 << 1 + featureCompatXattr uint32 = 1 << 2 + + featureIncompatLZ4Padding uint32 = 1 << 0 + featureIncompatComprCfgs uint32 = 1 << 1 + featureIncompatFragments uint32 = 1 << 5 +) + +const ( + dataLayoutFlatPlain uint8 = iota + dataLayoutCompressedFull + dataLayoutFlatInline + dataLayoutCompressedCompact + dataLayoutChunked +) + +const inodeFormatReservedMask uint16 = 0xffe0 + +const ( + zAdviseCompacted2B uint16 = 1 << 0 + zAdviseBigPcluster1 uint16 = 1 << 1 + zAdviseBigPcluster2 uint16 = 1 << 2 + zAdviseInlineData uint16 = 1 << 3 + zAdviseInterlaced uint16 = 1 << 4 + zAdviseFragment uint16 = 1 << 5 +) + +const ( + zClusterTypePlain uint8 = iota + zClusterTypeHead1 + zClusterTypeNonHead + zClusterTypeHead2 +) + +const ( + zD0ClusterCount uint16 = 1 << 11 + zPartialRef uint16 = 1 << 15 +) + +const ( + compressionLZ4 uint8 = iota + compressionLZMA + compressionDeflate + compressionZstd +) + +const ( + compressionShifted uint8 = 4 + compressionInterlaced uint8 = 5 +) diff --git a/filesystem/erofs/directory.go b/filesystem/erofs/directory.go new file mode 100644 index 00000000..36be1dca --- /dev/null +++ b/filesystem/erofs/directory.go @@ -0,0 +1,112 @@ +package erofs + +import ( + "bytes" + "encoding/binary" + "fmt" + iofs "io/fs" +) + +type directoryEntry struct { + nid uint64 + name string + fileType uint8 +} + +func parseDirectory(b []byte) ([]*directoryEntry, error) { + if len(b) == 0 { + return nil, fmt.Errorf("directory block is empty") + } + if len(b)%direntSize != 0 && len(b) < direntSize { + return nil, fmt.Errorf("directory block is %d bytes, less than entry size %d", len(b), direntSize) + } + firstNameOffset := int(binary.LittleEndian.Uint16(b[8:10])) + if firstNameOffset < direntSize || firstNameOffset >= len(b) || firstNameOffset%direntSize != 0 { + return nil, fmt.Errorf("invalid first directory name offset %d", firstNameOffset) + } + entryCount := firstNameOffset / direntSize + if entryCount == 0 { + return nil, fmt.Errorf("directory block has no entries") + } + + entries := make([]*directoryEntry, 0, entryCount) + previousName := "" + for index := 0; index < entryCount; index++ { + offset := index * direntSize + entry := b[offset : offset+direntSize] + nameOffset := int(binary.LittleEndian.Uint16(entry[8:10])) + nextNameOffset := len(b) + if index+1 < entryCount { + nextNameOffset = int(binary.LittleEndian.Uint16(b[offset+direntSize+8 : offset+direntSize+10])) + } + if nextNameOffset > len(b) { + return nil, fmt.Errorf("directory entry %d has invalid next name offset %d", index+1, nextNameOffset) + } + if nameOffset < firstNameOffset || nameOffset >= len(b) || nameOffset >= nextNameOffset { + return nil, fmt.Errorf("directory entry %d has invalid name offset %d", index, nameOffset) + } + if entry[11] != 0 { + return nil, fmt.Errorf("directory entry %d has nonzero reserved byte", index) + } + if entry[10] == 0 || entry[10] > fileTypeSymlink { + return nil, fmt.Errorf("directory entry %d has invalid file type %d", index, entry[10]) + } + if index+1 == entryCount { + if nul := bytes.IndexByte(b[nameOffset:], 0); nul >= 0 { + nextNameOffset = nameOffset + nul + } + } + nameBytes := b[nameOffset:nextNameOffset] + if len(nameBytes) == 0 || len(nameBytes) > maxNameLength || bytes.IndexByte(nameBytes, 0) >= 0 || + bytes.IndexByte(nameBytes, '/') >= 0 { + return nil, fmt.Errorf("directory entry %d has invalid name length %d", index, len(nameBytes)) + } + name := string(nameBytes) + if index > 0 && string(nameBytes) <= previousName { + return nil, fmt.Errorf("directory names are not strictly ordered: %q after %q", name, previousName) + } + nid := binary.LittleEndian.Uint64(entry[0:8]) + if nid>>63 != 0 { + return nil, fmt.Errorf("directory entry %d has metabox inode number", index) + } + entries = append(entries, &directoryEntry{nid: nid, name: name, fileType: entry[10]}) + previousName = name + } + return entries, nil +} + +func (f *FileSystem) readDirectory(i *inode) ([]*directoryEntry, error) { + if !i.isDir() { + return nil, fmt.Errorf("inode %d is not a directory", i.nid) + } + var entries []*directoryEntry + for offset := int64(0); offset < i.size; offset += f.blockSize() { + length := f.blockSize() + if remaining := i.size - offset; remaining < length { + length = remaining + } + b := make([]byte, length) + if _, err := f.readInodeData(i, b, offset, nil); err != nil { + return nil, err + } + blockEntries, err := parseDirectory(b) + if err != nil { + return nil, fmt.Errorf("invalid directory block at inode %d offset %d: %w", i.nid, offset, err) + } + entries = append(entries, blockEntries...) + } + return entries, nil +} + +func (f *FileSystem) lookup(parent *inode, name string) (*inode, error) { + entries, err := f.readDirectory(parent) + if err != nil { + return nil, err + } + for _, entry := range entries { + if entry.name == name { + return f.getInode(entry.nid) + } + } + return nil, iofs.ErrNotExist +} diff --git a/filesystem/erofs/directory_file_internal_test.go b/filesystem/erofs/directory_file_internal_test.go new file mode 100644 index 00000000..6677be42 --- /dev/null +++ b/filesystem/erofs/directory_file_internal_test.go @@ -0,0 +1,33 @@ +package erofs + +import ( + "errors" + "io" + "io/fs" + "math" + "testing" +) + +func TestReadDirLargeCount(t *testing.T) { + filesystem := readTestImage(t, uncompressedFixture) + opened, err := filesystem.Open(".") + if err != nil { + t.Fatal(err) + } + defer opened.Close() + directory, ok := opened.(fs.ReadDirFile) + if !ok { + t.Fatalf("Open() returned %T, want fs.ReadDirFile", opened) + } + first, err := directory.ReadDir(1) + if err != nil || len(first) != 1 { + t.Fatalf("ReadDir(1) = %v, %v", first, err) + } + rest, err := directory.ReadDir(math.MaxInt) + if err != nil || len(rest) != 4 { + t.Fatalf("ReadDir(MaxInt) returned %d entries, %v; want 4, nil", len(rest), err) + } + if entries, err := directory.ReadDir(math.MaxInt); len(entries) != 0 || !errors.Is(err, io.EOF) { + t.Fatalf("exhausted ReadDir(MaxInt) = %v, %v", entries, err) + } +} diff --git a/filesystem/erofs/directory_internal_test.go b/filesystem/erofs/directory_internal_test.go new file mode 100644 index 00000000..b1e05bd2 --- /dev/null +++ b/filesystem/erofs/directory_internal_test.go @@ -0,0 +1,123 @@ +package erofs + +import ( + "encoding/binary" + "strings" + "testing" +) + +func validDirectoryBlock() []byte { + b := make([]byte, 17) + binary.LittleEndian.PutUint64(b[0:], 1) + binary.LittleEndian.PutUint16(b[8:], 12) + b[10] = fileTypeRegular + copy(b[12:], "hello") + return b +} + +func TestParseDirectory(t *testing.T) { + entries, err := parseDirectory(validDirectoryBlock()) + if err != nil { + t.Fatalf("parseDirectory() error = %v", err) + } + if len(entries) != 1 || entries[0].name != "hello" || entries[0].nid != 1 { + t.Fatalf("parseDirectory() entries = %#v", entries) + } +} + +func TestParseDirectorySymlink(t *testing.T) { + b := validDirectoryBlock() + b[10] = fileTypeSymlink + entries, err := parseDirectory(b) + if err != nil { + t.Fatalf("parseDirectory() error = %v", err) + } + if entries[0].fileType != fileTypeSymlink { + t.Fatalf("fileType = %d, want %d", entries[0].fileType, fileTypeSymlink) + } +} + +func TestParseDirectoryErrors(t *testing.T) { + tests := []struct { + name string + mutate func([]byte) + }{ + {"first name offset", func(b []byte) { binary.LittleEndian.PutUint16(b[8:], 11) }}, + {"file type", func(b []byte) { b[10] = 0 }}, + {"reserved", func(b []byte) { b[11] = 1 }}, + {"unaligned name offset", func(b []byte) { binary.LittleEndian.PutUint16(b[8:], 13) }}, + {"traversal name", func(b []byte) { copy(b[12:], "../hi") }}, + {"nested name", func(b []byte) { copy(b[12:], "a/b") }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := validDirectoryBlock() + tt.mutate(b) + if _, err := parseDirectory(b); err == nil { + t.Fatal("parseDirectory() error = nil, want error") + } + }) + } +} + +func TestParseDirectoryNameOrder(t *testing.T) { + b := make([]byte, 28) + binary.LittleEndian.PutUint64(b[0:], 1) + binary.LittleEndian.PutUint16(b[8:], 24) + b[10] = fileTypeRegular + binary.LittleEndian.PutUint64(b[12:], 2) + binary.LittleEndian.PutUint16(b[20:], 26) + b[22] = fileTypeRegular + copy(b[24:], "b") + copy(b[26:], "a") + if _, err := parseDirectory(b); err == nil { + t.Fatal("parseDirectory() error = nil for unsorted names") + } +} + +func TestParseDirectoryMaximumName(t *testing.T) { + b := make([]byte, direntSize+maxNameLength) + binary.LittleEndian.PutUint64(b[0:], 1) + binary.LittleEndian.PutUint16(b[8:], direntSize) + b[10] = fileTypeRegular + copy(b[direntSize:], strings.Repeat("a", maxNameLength)) + entries, err := parseDirectory(b) + if err != nil { + t.Fatalf("parseDirectory() error = %v", err) + } + if len(entries) != 1 || len(entries[0].name) != maxNameLength { + t.Fatalf("parseDirectory() entries = %#v", entries) + } +} + +func TestParseDirectoryPaddedFinalName(t *testing.T) { + b := make([]byte, 27) + binary.LittleEndian.PutUint64(b[0:], 1) + binary.LittleEndian.PutUint16(b[8:], 24) + b[10] = fileTypeRegular + binary.LittleEndian.PutUint64(b[12:], 2) + binary.LittleEndian.PutUint16(b[20:], 25) + b[22] = fileTypeRegular + copy(b[24:], "ab") + entries, err := parseDirectory(b) + if err != nil { + t.Fatalf("parseDirectory() error = %v", err) + } + if len(entries) != 2 || entries[0].name != "a" || entries[1].name != "b" { + t.Fatalf("parseDirectory() entries = %#v", entries) + } +} + +func TestParseDirectoryNextNameOffsetOutsideBlock(t *testing.T) { + b := make([]byte, 26) + binary.LittleEndian.PutUint64(b[0:], 1) + binary.LittleEndian.PutUint16(b[8:], 24) + b[10] = fileTypeRegular + binary.LittleEndian.PutUint64(b[12:], 2) + binary.LittleEndian.PutUint16(b[20:], 65535) + b[22] = fileTypeRegular + copy(b[24:], "a") + if _, err := parseDirectory(b); err == nil { + t.Fatal("parseDirectory() error = nil for next name offset outside block") + } +} diff --git a/filesystem/erofs/doc.go b/filesystem/erofs/doc.go new file mode 100644 index 00000000..5ccbdfe4 --- /dev/null +++ b/filesystem/erofs/doc.go @@ -0,0 +1,47 @@ +// Package erofs provides read-only access to EROFS filesystem images. +// +// The implementation follows the [EROFS on-disk documentation], especially the +// [core format specification]. Compression structures follow the [Linux EROFS +// on-disk definitions], with decoding based on the [MicroLZMA format documentation]. +// +// It supports an EROFS on-disk-format subset with block sizes from 512 through +// 65536 bytes, compact and extended inodes, uncompressed FLAT_PLAIN and +// FLAT_INLINE data, and LZMA-compressed compact and full index layouts. +// Compressed reads support logical clusters larger than the filesystem block, +// fragment tails, and shifted or interlaced incompressible extents. +// +// LZ4, Zstd, DEFLATE, chunked files, multi-device images, compressed inline +// tails, partial compressed references, 48-bit layouts, metaboxes, and long +// xattr prefixes are unsupported. Recognizing the fragments/dedupe feature bit +// does not imply support for every deduplication encoding. Physical compressed +// extents are limited to 1 MiB and decoded extents to 12 MiB. Each open File +// caches one decoded extent, including packed-inode fragments, until replaced +// or closed. ReadAt calls on the same File may run concurrently. +// +// Symlinks are exposed as link objects: Stat reports ModeSymlink and ReadFile +// or Open reads the stored target name. Paths are not resolved through symlinks, +// including intermediate components. Consumers extracting a binary must use its +// regular-file deployment path, rather than a symlink to it. +// +// Extended attributes are skipped when locating inode data but are not exposed +// through this API. Unsupported layouts and compression algorithms return +// descriptive errors rather than returning invalid data. +// +// Read borrows its backend. The caller must keep it open and unchanged while +// using the filesystem. FileSystem.Close is a no-op; it does not close the +// backend or open files. File.Close releases that file's decoded extent. +// Existing images reject mutations with filesystem.ErrReadonlyFilesystem; +// image creation is not implemented. +// +// For an EROFS image embedded at a known offset, create an offset backend with +// backend.Sub, then pass that backend and the image size to Read. The EROFS +// reader enforces this size; backend.Sub only translates offsets: +// +// nested := backend.Sub(backend, offset, size) +// filesystem, err := erofs.Read(nested, size, 0, 0) +// +// [EROFS on-disk documentation]: https://erofs.docs.kernel.org/en/latest/ondisk/index.html +// [core format specification]: https://erofs.docs.kernel.org/en/latest/ondisk/core_ondisk.html +// [Linux EROFS on-disk definitions]: https://raw.githubusercontent.com/torvalds/linux/master/fs/erofs/erofs_fs.h +// [MicroLZMA format documentation]: https://docs.kernel.org/staging/xz.html +package erofs diff --git a/filesystem/erofs/erofs.go b/filesystem/erofs/erofs.go new file mode 100644 index 00000000..bbd87f8a --- /dev/null +++ b/filesystem/erofs/erofs.go @@ -0,0 +1,405 @@ +package erofs + +import ( + "fmt" + "io" + iofs "io/fs" + "math" + "os" + "path" + "strings" + "time" + + "github.com/diskfs/go-diskfs/backend" + "github.com/diskfs/go-diskfs/filesystem" +) + +// FileSystem implements read-only access to EROFS images. +type FileSystem struct { + superblock *superblock + backend backend.Storage + size int64 + blocksize int64 + lzmaDictSize uint32 +} + +var _ filesystem.FileSystem = (*FileSystem)(nil) + +// Read opens an existing image. start is a byte offset into b and size is the +// filesystem length in bytes. blocksize is zero to use the image's block size, +// or its expected size in bytes. +// The caller owns b and must keep it open and unchanged while using the reader. +func Read(b backend.Storage, size, start, blocksize int64) (*FileSystem, error) { + if size <= 0 { + return nil, fmt.Errorf("invalid filesystem size %d", size) + } + if start < 0 { + return nil, fmt.Errorf("invalid filesystem start %d", start) + } + if start > math.MaxInt64-size { + return nil, fmt.Errorf("filesystem start %d and size %d exceed the supported address range", start, size) + } + if blocksize < 0 || (blocksize != 0 && blocksize&(blocksize-1) != 0) { + return nil, fmt.Errorf("block size %d is not a power of two", blocksize) + } + if start != 0 { + b = backend.Sub(b, start, size) + } + result := &FileSystem{backend: b, size: size} + + header := make([]byte, 14) + if err := result.readAt(header, superOffset); err != nil { + return nil, fmt.Errorf("unable to read EROFS superblock header: %w", err) + } + superblockLength := superblockBaseSize + int(header[13])*superblockExtSlotSize + superblockBytes := make([]byte, superblockLength) + if err := result.readAt(superblockBytes, superOffset); err != nil { + return nil, fmt.Errorf("unable to read EROFS superblock: %w", err) + } + s, err := parseSuperblock(superblockBytes, size) + if err != nil { + return nil, err + } + if s.featureCompat&featureCompatChecksum != 0 { + checksumLength := superblockChecksumLength(s.blockSize()) + checksumBytes := make([]byte, checksumLength) + if err := result.readAt(checksumBytes, superOffset); err != nil { + return nil, fmt.Errorf("unable to read EROFS checksum area: %w", err) + } + if err := verifySuperblockChecksum(s, checksumBytes); err != nil { + return nil, err + } + } + if blocksize != 0 && blocksize != s.blockSize() { + return nil, fmt.Errorf("requested block size %d does not match superblock block size %d", blocksize, s.blockSize()) + } + + result.superblock = s + result.blocksize = s.blockSize() + if err := result.parseCompressionConfig(s); err != nil { + return nil, err + } + if _, err := result.getInode(uint64(s.rootNID)); err != nil { + return nil, fmt.Errorf("unable to read root inode: %w", err) + } + return result, nil +} + +func Create(_ backend.Storage, _, _, _ int64) (*FileSystem, error) { + return nil, filesystem.ErrNotImplemented +} + +func (f *FileSystem) blockSize() int64 { return f.blocksize } + +func (f *FileSystem) Type() filesystem.Type { return filesystem.TypeErofs } +func (f *FileSystem) Label() string { + if f.superblock == nil { + return "" + } + return strings.TrimRight(string(f.superblock.volumeName[:]), "\x00") +} +func (f *FileSystem) SetLabel(string) error { return filesystem.ErrReadonlyFilesystem } + +// Close is a no-op. It does not close the caller-owned backend or open files; +// close each File to release its decoded extent and close the backend separately. +func (f *FileSystem) Close() error { + return nil +} + +func (f *FileSystem) stat(p string) (*inode, error) { + if !iofs.ValidPath(p) { + return nil, iofs.ErrInvalid + } + node, err := f.getInode(uint64(f.superblock.rootNID)) + if err != nil { + return nil, err + } + if p == "." { + if !node.isDir() { + return nil, fmt.Errorf("root inode is not a directory") + } + return node, nil + } + for _, component := range strings.Split(p, "/") { + if !node.isDir() { + return nil, fmt.Errorf("%s is not a directory", component) + } + node, err = f.lookup(node, component) + if err != nil { + return nil, err + } + } + return node, nil +} + +func (f *FileSystem) Open(p string) (iofs.File, error) { + file, err := f.open(p) + if err != nil { + return nil, &iofs.PathError{Op: "open", Path: p, Err: err} + } + return file, nil +} + +// open leaves errors unwrapped so each public method can supply its operation +// and the original caller path in a PathError. +func (f *FileSystem) open(p string) (iofs.File, error) { + node, err := f.stat(p) + if err != nil { + return nil, err + } + if node.isDir() { + entries, err := f.readDirectory(node) + if err != nil { + return nil, err + } + visibleEntries := make([]*directoryEntry, 0, len(entries)) + for _, entry := range entries { + if entry.name == "." || entry.name == ".." { + continue + } + visibleEntries = append(visibleEntries, entry) + } + return &directoryFile{filesystem: f, node: node, name: path.Base(p), entries: visibleEntries}, nil + } + return &File{filesystem: f, node: node, name: path.Base(p)}, nil +} + +func (f *FileSystem) OpenFile(p string, flag int) (filesystem.File, error) { + if flag != os.O_RDONLY { + return nil, &iofs.PathError{Op: "openfile", Path: p, Err: filesystem.ErrReadonlyFilesystem} + } + file, err := f.open(p) + if err != nil { + return nil, &iofs.PathError{Op: "openfile", Path: p, Err: err} + } + if regular, ok := file.(*File); ok { + return regular, nil + } + return nil, &iofs.PathError{Op: "openfile", Path: p, Err: fmt.Errorf("%s is a directory", p)} +} + +func (f *FileSystem) Stat(p string) (iofs.FileInfo, error) { + node, err := f.stat(p) + if err != nil { + return nil, &iofs.PathError{Op: "stat", Path: p, Err: err} + } + return &fileInfo{name: path.Base(p), node: node}, nil +} + +func (f *FileSystem) ReadDir(p string) ([]iofs.DirEntry, error) { + node, err := f.stat(p) + if err != nil { + return nil, &iofs.PathError{Op: "readdir", Path: p, Err: err} + } + if !node.isDir() { + return nil, &iofs.PathError{Op: "readdir", Path: p, Err: fmt.Errorf("%s is not a directory", p)} + } + entries, err := f.readDirectory(node) + if err != nil { + return nil, &iofs.PathError{Op: "readdir", Path: p, Err: err} + } + result := make([]iofs.DirEntry, 0, len(entries)) + for _, entry := range entries { + if entry.name == "." || entry.name == ".." { + continue + } + result = append(result, &dirEntry{entry: entry, filesystem: f}) + } + return result, nil +} + +func (f *FileSystem) ReadFile(p string) ([]byte, error) { + file, err := f.open(p) + if err != nil { + return nil, &iofs.PathError{Op: "readfile", Path: p, Err: err} + } + defer file.Close() + if _, ok := file.(*File); !ok { + return nil, &iofs.PathError{Op: "readfile", Path: p, Err: fmt.Errorf("%s is a directory", p)} + } + data, err := io.ReadAll(file) + if err != nil { + return data, &iofs.PathError{Op: "readfile", Path: p, Err: err} + } + return data, nil +} + +// cache belongs to the open File, whose mutex is held by the caller. Passing it +// through fragment reads retains the packed inode's decoded extent as well. +// Directory reads do not need a cache and pass nil. +func (f *FileSystem) readInodeData(node *inode, b []byte, offset int64, cache *decodedExtent) (int, error) { + if len(b) == 0 { + return 0, nil + } + if node.dataLayout == dataLayoutCompressedFull || node.dataLayout == dataLayoutCompressedCompact { + return f.readCompressedData(node, b, offset, cache) + } + if node.isDir() || node.mode&0xF000 == linuxModeRegular || node.isSymlink() { + total := 0 + nonInlineSize := node.size + if node.dataLayout == dataLayoutFlatInline { + tailSize := node.size % f.blockSize() + nonInlineSize = node.size - tailSize + } + if offset < nonInlineSize { + location := int64(node.startBlock)*f.blockSize() + offset + readLength := min(int64(len(b)), nonInlineSize-offset) + if location < 0 || location+readLength > f.size { + return 0, fmt.Errorf("inode %d data range is outside the image", node.nid) + } + if err := f.readAt(b[:readLength], location); err != nil { + return total, err + } + total += int(readLength) + if readLength == int64(len(b)) { + return total, nil + } + b = b[readLength:] + offset += readLength + } + if node.dataLayout != dataLayoutFlatInline { + return 0, fmt.Errorf("inode %d data range is outside the image", node.nid) + } + inodeOffset, err := f.inodeOffset(node.nid) + if err != nil { + return total, err + } + inlineStart := inodeOffset + int64(node.inodeSize+node.xattrSize) + (offset - nonInlineSize) + if inlineStart < inodeOffset || inlineStart+int64(len(b)) > f.size || + inlineStart/f.blockSize() != (inlineStart+int64(len(b))-1)/f.blockSize() { + return total, fmt.Errorf("inode %d inline data crosses or exceeds a block boundary", node.nid) + } + if err := f.readAt(b, inlineStart); err != nil { + return total, err + } + return total + len(b), nil + } + return 0, fmt.Errorf("inode %d has no readable data", node.nid) +} + +func (f *FileSystem) readCompressedData(node *inode, b []byte, offset int64, cache *decodedExtent) (int, error) { + read := 0 + for read < len(b) { + currentOffset := offset + int64(read) + if cache != nil && cache.nid == node.nid && currentOffset >= cache.logicalStart { + start := currentOffset - cache.logicalStart + if start < int64(len(cache.data)) { + read += copy(b[read:], cache.data[start:]) + continue + } + } + extent, err := f.mapCompressed(node, currentOffset, false) + if err != nil { + return read, err + } + available := extent.logicalStart + extent.logicalLength - currentOffset + if available <= 0 { + return read, fmt.Errorf("invalid compressed extent for inode %d", node.nid) + } + if available > int64(len(b)-read) { + available = int64(len(b) - read) + } + target := b[read : read+int(available)] + if extent.fragment { + n, err := f.readFragment(node, extent, target, currentOffset, cache) + read += n + if err != nil { + return read, err + } + continue + } + encoded := make([]byte, extent.physicalLength) + if err := f.readAt(encoded, extent.physicalStart); err != nil { + return read, fmt.Errorf("unable to read compressed data for inode %d: %w", node.nid, err) + } + start := currentOffset - extent.logicalStart + if extent.algorithm == compressionShifted { + if extent.logicalLength > extent.physicalLength || start+int64(len(target)) > extent.logicalLength { + return read, fmt.Errorf("invalid shifted extent for inode %d", node.nid) + } + copy(target, encoded[start:start+int64(len(target))]) + read += len(target) + continue + } + if extent.algorithm == compressionInterlaced { + if extent.logicalLength > extent.physicalLength || start+int64(len(target)) > extent.logicalLength { + return read, fmt.Errorf("invalid interlaced extent for inode %d", node.nid) + } + skip := int((extent.logicalStart + start) % f.blockSize()) + count := len(target) + right := int(f.blockSize()) - skip + if right > count { + right = count + } + copy(target, encoded[skip:skip+right]) + copy(target[right:], encoded[:count-right]) + read += count + continue + } + if extent.algorithm != compressionLZMA { + return read, fmt.Errorf("unsupported compression algorithm %d for inode %d", extent.algorithm, node.nid) + } + decoded, err := f.decompressLZMA(encoded, extent.logicalLength) + if err != nil { + return read, err + } + if cache != nil { + *cache = decodedExtent{nid: node.nid, logicalStart: extent.logicalStart, data: decoded} + } + copy(target, decoded[start:start+int64(len(target))]) + read += len(target) + } + return read, nil +} + +func (f *FileSystem) readFragment(node *inode, extent *compressedExtent, target []byte, currentOffset int64, cache *decodedExtent) (int, error) { + if f.superblock.packedNID == 0 || f.superblock.packedNID == node.nid { + return 0, fmt.Errorf("invalid packed inode for fragment of inode %d", node.nid) + } + packedNode, err := f.getInode(f.superblock.packedNID) + if err != nil { + return 0, err + } + packedOffset := node.compressed.fragmentOffset + (currentOffset - extent.logicalStart) + if packedOffset < 0 || packedOffset+int64(len(target)) > packedNode.size { + return 0, fmt.Errorf("fragment range is outside packed inode for inode %d", node.nid) + } + return f.readInodeData(packedNode, target, packedOffset, cache) +} + +// readAt validates filesystem-relative bounds before accessing the backend, +// which may contain other partitions and does not enforce the filesystem size. +func (f *FileSystem) readAt(p []byte, offset int64) error { + if offset < 0 || offset > f.size || int64(len(p)) > f.size-offset { + return fmt.Errorf("read at offset %d with length %d is outside the filesystem size %d", offset, len(p), f.size) + } + if len(p) == 0 { + return nil + } + read, err := f.backend.ReadAt(p, offset) + if err != nil { + return err + } + if read != len(p) { + return io.ErrUnexpectedEOF + } + return nil +} + +func (f *FileSystem) Mkdir(string) error { return filesystem.ErrReadonlyFilesystem } +func (f *FileSystem) Mknod(string, uint32, int) error { + return filesystem.ErrReadonlyFilesystem +} +func (f *FileSystem) Link(string, string) error { return filesystem.ErrReadonlyFilesystem } +func (f *FileSystem) Symlink(string, string) error { + return filesystem.ErrReadonlyFilesystem +} +func (f *FileSystem) Chmod(string, os.FileMode) error { return filesystem.ErrReadonlyFilesystem } +func (f *FileSystem) Chown(string, int, int) error { + return filesystem.ErrReadonlyFilesystem +} +func (f *FileSystem) Chtimes(string, time.Time, time.Time, time.Time) error { + return filesystem.ErrReadonlyFilesystem +} +func (f *FileSystem) Rename(string, string) error { return filesystem.ErrReadonlyFilesystem } +func (f *FileSystem) Remove(string) error { return filesystem.ErrReadonlyFilesystem } diff --git a/filesystem/erofs/erofs_test.go b/filesystem/erofs/erofs_test.go new file mode 100644 index 00000000..f0647292 --- /dev/null +++ b/filesystem/erofs/erofs_test.go @@ -0,0 +1,586 @@ +package erofs + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + iofs "io/fs" + "os" + "strings" + "testing" + "testing/fstest" + + "github.com/diskfs/go-diskfs/backend" + backendfile "github.com/diskfs/go-diskfs/backend/file" + fsapi "github.com/diskfs/go-diskfs/filesystem" +) + +func testBackend(t *testing.T, image []byte) backend.Storage { + t.Helper() + f, err := os.CreateTemp(t.TempDir(), "image-*.erofs") + if err != nil { + t.Fatalf("os.CreateTemp() error = %v", err) + } + t.Cleanup(func() { _ = f.Close() }) + if _, err := f.Write(image); err != nil { + t.Fatalf("Write() error = %v", err) + } + return backendfile.New(f, true) +} + +func setCompactInode(b []byte, offset int, format, mode uint16, size, startBlock uint32) { + binary.LittleEndian.PutUint16(b[offset:], format) + binary.LittleEndian.PutUint16(b[offset+2:], 0) + binary.LittleEndian.PutUint16(b[offset+4:], mode) + binary.LittleEndian.PutUint16(b[offset+6:], 0) + binary.LittleEndian.PutUint32(b[offset+8:], size) + binary.LittleEndian.PutUint32(b[offset+12:], 1) + binary.LittleEndian.PutUint32(b[offset+16:], startBlock) + binary.LittleEndian.PutUint32(b[offset+20:], uint32(offset/inodeSlotSize)) + binary.LittleEndian.PutUint16(b[offset+24:], 0) + binary.LittleEndian.PutUint16(b[offset+26:], 0) +} + +func setDirectoryEntry(b []byte, offset int, nid uint64, fileType byte) { + binary.LittleEndian.PutUint64(b[offset:], nid) + binary.LittleEndian.PutUint16(b[offset+8:], 12) + b[offset+10] = fileType + b[offset+11] = 0 +} + +func buildImage(inline bool) []byte { + image := make([]byte, 16384) + copy(image[superOffset:], validSuperblockBytes()) + binary.LittleEndian.PutUint32(image[superOffset+36:], 3) + if !inline { + binary.LittleEndian.PutUint32(image[superOffset+36:], 4) + } + + setCompactInode(image, 4096, 1<<4, linuxModeDir, 17, 2) + setCompactInode(image, 4128, 1<<4|(uint16(dataLayoutFlatInline)<<1), linuxModeRegular, 11, 0) + if !inline { + format := 1<<4 | uint16(dataLayoutFlatPlain)<<1 + setCompactInode(image, 4128, format, linuxModeRegular, 11, 3) + copy(image[12288:], "hello world") + } else { + copy(image[4160:], "hello world") + } + + setDirectoryEntry(image, 8192, 1, fileTypeRegular) + copy(image[8204:], "hello") + return image +} + +func TestReadCoreFilesystem(t *testing.T) { + image := buildImage(false) + filesystem, err := Read(testBackend(t, image), int64(len(image)), 0, 0) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + entries, err := filesystem.ReadDir(".") + if err != nil { + t.Fatalf("ReadDir() error = %v", err) + } + if len(entries) != 1 || entries[0].Name() != "hello" { + t.Fatalf("ReadDir() entries = %#v, want hello", entries) + } + info, err := filesystem.Stat("hello") + if err != nil { + t.Fatalf("Stat() error = %v", err) + } + if info.Size() != 11 || !info.Mode().IsRegular() { + t.Fatalf("Stat() = size %d mode %v", info.Size(), info.Mode()) + } + data, err := filesystem.ReadFile("hello") + if err != nil { + t.Fatalf("ReadFile() error = %v", err) + } + if string(data) != "hello world" { + t.Fatalf("ReadFile() = %q", data) + } +} + +func TestReadInlineFilesystem(t *testing.T) { + image := buildImage(true) + filesystem, err := Read(testBackend(t, image), int64(len(image)), 0, 0) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + data, err := filesystem.ReadFile("hello") + if err != nil { + t.Fatalf("ReadFile() error = %v", err) + } + if string(data) != "hello world" { + t.Fatalf("ReadFile() = %q", data) + } +} + +func TestReadCompactDirectoryExternalData(t *testing.T) { + filesystem := readTestImage(t, largeFixture) + node, err := filesystem.stat("dir") + if err != nil { + t.Fatalf("stat(dir) error = %v", err) + } + if node.inodeSize != inodeCompactSize || node.size <= filesystem.blockSize() { + t.Fatalf("dir inode size = %d, data size = %d; want a compact inode with external data", node.inodeSize, node.size) + } + entries, err := filesystem.ReadDir("dir") + if err != nil { + t.Fatalf("ReadDir() error = %v", err) + } + if len(entries) != 300 { + t.Fatalf("ReadDir() returned %d entries, want 300", len(entries)) + } +} + +func TestReadRejectsUnknownCompressionAlgorithms(t *testing.T) { + image := readFixtureBytes(t, compressedFixture) + superblockLength := superblockBaseSize + int(image[superOffset+13])*superblockExtSlotSize + superblock, err := parseSuperblock(image[superOffset:superOffset+superblockLength], int64(len(image))) + if err != nil { + t.Fatalf("parseSuperblock() error = %v", err) + } + if superblock.featureIncompat&featureIncompatComprCfgs == 0 { + t.Fatal("compressed fixture does not enable compression configurations") + } + const firstUnknownCompressionAlgorithm = compressionZstd + 1 + if superblock.availableComprAlgs&(1< int64(len(expected)) && !errors.Is(err, io.EOF) { + t.Fatalf("ReadAt() error = %v, want %v", err, io.EOF) + } + if tt.offset+int64(tt.length) <= int64(len(expected)) && err != nil { + t.Fatalf("ReadAt() error = %v", err) + } + }) + } + + if _, err := file.Seek(61439, io.SeekStart); err != nil { + t.Fatalf("Seek() error = %v", err) + } + b := make([]byte, 2) + if _, err := io.ReadFull(file, b); err != nil { + t.Fatalf("io.ReadFull() error = %v", err) + } + if !bytes.Equal(b, expected[61439:61441]) { + t.Fatalf("read after seek = %q, want %q", b, expected[61439:61441]) + } +} + +func buildMetadataImage() []byte { + image := make([]byte, 16384) + copy(image[superOffset:], validSuperblockBytes()) + binary.LittleEndian.PutUint32(image[superOffset+36:], 4) + + setCompactInode(image, 4096, 1<<4, linuxModeDir, 127, 2) + + type metadata struct { + name string + format uint16 + mode uint16 + size uint32 + start uint32 + fileType byte + } + nodes := []metadata{ + {name: "block", format: uint16(dataLayoutFlatPlain) << 1, mode: linuxModeBlock | 0o644, start: 0x0802, fileType: fileTypeBlockDevice}, + {name: "char", format: uint16(dataLayoutFlatPlain) << 1, mode: linuxModeChar | 0o644, start: 0x0301, fileType: fileTypeCharDevice}, + {name: "fifo", format: uint16(dataLayoutFlatPlain) << 1, mode: linuxModeFIFO | 0o644, fileType: fileTypeFIFO}, + {name: "hardlink", format: uint16(dataLayoutFlatPlain) << 1, mode: linuxModeRegular | 0o644, size: 11, start: 3, fileType: fileTypeRegular}, + {name: "socket", format: uint16(dataLayoutFlatPlain) << 1, mode: linuxModeSocket | 0o644, fileType: fileTypeSocket}, + {name: "symlink", format: 1<<4 | uint16(dataLayoutFlatInline)<<1, mode: linuxModeSymlink | 0o777, size: 9, fileType: fileTypeSymlink}, + } + for index, node := range nodes { + inodeOffset := 4128 + index*32 + setCompactInode(image, inodeOffset, node.format, node.mode, node.size, node.start) + if node.name == "hardlink" { + binary.LittleEndian.PutUint16(image[inodeOffset+6:], 2) + } + } + copy(image[4320:], "hello.txt") + copy(image[12288:], "hello world") + + directory := make([]byte, 127) + names := []struct { + name string + nid uint64 + }{ + {name: "block", nid: 1}, + {name: "char", nid: 2}, + {name: "fifo", nid: 3}, + {name: "hardlink", nid: 4}, + {name: "hardlink2", nid: 4}, + {name: "socket", nid: 5}, + {name: "symlink", nid: 6}, + } + nameOffset := 84 + for index, entry := range names { + fileType := fileTypeRegular + switch entry.name { + case "block": + fileType = fileTypeBlockDevice + case "char": + fileType = fileTypeCharDevice + case "fifo": + fileType = fileTypeFIFO + case "socket": + fileType = fileTypeSocket + case "symlink": + fileType = fileTypeSymlink + } + setDirectoryEntry(directory, index*direntSize, entry.nid, fileType) + binary.LittleEndian.PutUint16(directory[index*direntSize+8:], uint16(nameOffset)) + copy(directory[nameOffset:], entry.name) + nameOffset += len(entry.name) + } + binary.LittleEndian.PutUint16(directory[8:], 84) + copy(image[8192:], directory) + return image +} + +func TestSpecialFileMetadata(t *testing.T) { + filesystem, err := Read(testBackend(t, buildMetadataImage()), 16384, 0, 0) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + expectedModes := map[string]iofs.FileMode{ + "block": iofs.ModeDevice | 0o644, + "char": iofs.ModeDevice | iofs.ModeCharDevice | 0o644, + "fifo": iofs.ModeNamedPipe | 0o644, + "socket": iofs.ModeSocket | 0o644, + "symlink": iofs.ModeSymlink | 0o777, + } + for name, mode := range expectedModes { + info, err := filesystem.Stat(name) + if err != nil { + t.Fatalf("Stat(%q) error = %v", name, err) + } + if info.Mode() != mode { + t.Fatalf("Stat(%q) mode = %v, want %v", name, info.Mode(), mode) + } + } + rootInfo, err := filesystem.Stat(".") + if err != nil { + t.Fatalf("Stat(.) error = %v", err) + } + if !rootInfo.IsDir() || !rootInfo.Mode().IsDir() { + t.Fatal("Stat(.) is not classified as a directory") + } + for _, name := range []string{"block", "char", "fifo", "hardlink", "socket", "symlink"} { + info, err := filesystem.Stat(name) + if err != nil { + t.Fatalf("Stat(%q) error = %v", name, err) + } + if info.IsDir() || info.Mode().IsDir() { + t.Fatalf("Stat(%q) is classified as a directory", name) + } + file, err := filesystem.Open(name) + if err != nil { + t.Fatalf("Open(%q) error = %v", name, err) + } + if _, ok := file.(*directoryFile); ok { + t.Fatalf("Open(%q) returned a directory handle", name) + } + if err := file.Close(); err != nil { + t.Fatalf("Close(%q) error = %v", name, err) + } + if _, err := filesystem.ReadDir(name); err == nil { + t.Fatalf("ReadDir(%q) error = nil, want non-directory error", name) + } + } + blockInfo, err := filesystem.Stat("block") + if err != nil { + t.Fatalf("Stat(block) error = %v", err) + } + blockStat, ok := blockInfo.Sys().(*StatT) + if !ok || blockStat.DeviceID != 0x0802 { + t.Fatalf("Stat(block).Sys() = %#v, want device ID 0x0802", blockInfo.Sys()) + } + charInfo, err := filesystem.Stat("char") + if err != nil { + t.Fatalf("Stat(char) error = %v", err) + } + charStat, ok := charInfo.Sys().(*StatT) + if !ok || charStat.DeviceID != 0x0301 { + t.Fatalf("Stat(char).Sys() = %#v, want device ID 0x0301", charInfo.Sys()) + } + if target, err := filesystem.ReadFile("symlink"); err != nil || string(target) != "hello.txt" { + t.Fatalf("ReadFile(symlink) = %q, %v", target, err) + } +} + +func TestHardLinks(t *testing.T) { + filesystem, err := Read(testBackend(t, buildMetadataImage()), 16384, 0, 0) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + for _, name := range []string{"hardlink", "hardlink2"} { + info, err := filesystem.Stat(name) + if err != nil { + t.Fatalf("Stat(%q) error = %v", name, err) + } + stat, ok := info.Sys().(*StatT) + if !ok || stat.Inode != 4 || stat.LinkCount != 2 { + t.Fatalf("Stat(%q).Sys() = %#v, want inode 4 with two links", name, info.Sys()) + } + data, err := filesystem.ReadFile(name) + if err != nil || string(data) != "hello world" { + t.Fatalf("ReadFile(%q) = %q, %v", name, data, err) + } + } +} diff --git a/filesystem/erofs/errors_internal_test.go b/filesystem/erofs/errors_internal_test.go new file mode 100644 index 00000000..5b6f9bda --- /dev/null +++ b/filesystem/erofs/errors_internal_test.go @@ -0,0 +1,143 @@ +package erofs + +import ( + "errors" + iofs "io/fs" + "os" + "testing" + + "github.com/diskfs/go-diskfs/backend" + fsapi "github.com/diskfs/go-diskfs/filesystem" +) + +func checkPathError(t *testing.T, err error, op, path string, cause error) { + t.Helper() + var pathError *iofs.PathError + if !errors.As(err, &pathError) { + t.Fatalf("error = %T %v, want *fs.PathError", err, err) + } + if pathError.Op != op || pathError.Path != path { + t.Errorf("PathError = {Op: %q, Path: %q}, want {%q, %q}", pathError.Op, pathError.Path, op, path) + } + if cause != nil && !errors.Is(err, cause) { + t.Errorf("error = %v, want errors.Is(err, %v)", err, cause) + } +} + +func pathErrorOperations(filesystem *FileSystem) []struct { + op string + call func(string) error +} { + return []struct { + op string + call func(string) error + }{ + {op: "open", call: func(p string) error { + file, err := filesystem.Open(p) + if err == nil { + _ = file.Close() + } + return err + }}, + {op: "stat", call: func(p string) error { _, err := filesystem.Stat(p); return err }}, + {op: "readdir", call: func(p string) error { _, err := filesystem.ReadDir(p); return err }}, + {op: "readfile", call: func(p string) error { _, err := filesystem.ReadFile(p); return err }}, + {op: "openfile", call: func(p string) error { + file, err := filesystem.OpenFile(p, os.O_RDONLY) + if err == nil { + _ = file.Close() + } + return err + }}, + } +} + +func TestPublicPathErrors(t *testing.T) { + filesystem := readTestImage(t, uncompressedFixture) + for _, operation := range pathErrorOperations(filesystem) { + for _, tt := range []struct { + path string + cause error + }{ + {path: "missing", cause: iofs.ErrNotExist}, + {path: "missing/child", cause: iofs.ErrNotExist}, + {path: "../hello.txt", cause: iofs.ErrInvalid}, + {path: "/hello.txt", cause: iofs.ErrInvalid}, + {path: "", cause: iofs.ErrInvalid}, + } { + t.Run(operation.op+"/"+tt.path, func(t *testing.T) { + checkPathError(t, operation.call(tt.path), operation.op, tt.path, tt.cause) + }) + } + } +} + +func TestPathErrorsForFileTypesAndFlags(t *testing.T) { + filesystem := readTestImage(t, uncompressedFixture) + t.Run("readfile directory", func(t *testing.T) { + _, err := filesystem.ReadFile(".") + checkPathError(t, err, "readfile", ".", nil) + }) + t.Run("openfile directory", func(t *testing.T) { + _, err := filesystem.OpenFile(".", os.O_RDONLY) + checkPathError(t, err, "openfile", ".", nil) + }) + t.Run("readdir file", func(t *testing.T) { + _, err := filesystem.ReadDir("hello.txt") + checkPathError(t, err, "readdir", "hello.txt", nil) + }) + t.Run("openfile write", func(t *testing.T) { + _, err := filesystem.OpenFile("hello.txt", os.O_WRONLY) + checkPathError(t, err, "openfile", "hello.txt", fsapi.ErrReadonlyFilesystem) + }) +} + +type failingReadBackend struct { + backend.Storage + offset int64 + err error +} + +func (b failingReadBackend) ReadAt(p []byte, offset int64) (int, error) { + if offset == b.offset { + return 0, b.err + } + return b.Storage.ReadAt(p, offset) +} + +func TestPathErrorsPreserveBackendFailures(t *testing.T) { + filesystem, err := Read(testBackend(t, buildImage(false)), 16384, 0, 0) + if err != nil { + t.Fatal(err) + } + cause := errors.New("injected backend failure") + original := filesystem.backend + filesystem.backend = failingReadBackend{Storage: original, offset: 4128, err: cause} + for _, operation := range pathErrorOperations(filesystem) { + t.Run(operation.op, func(t *testing.T) { + checkPathError(t, operation.call("hello"), operation.op, "hello", cause) + }) + } + + // ReadFile must also wrap failures encountered after Open succeeds. + filesystem.backend = failingReadBackend{Storage: original, offset: 12288, err: cause} + _, err = filesystem.ReadFile("hello") + checkPathError(t, err, "readfile", "hello", cause) +} + +func TestPathErrorsForMalformedDirectory(t *testing.T) { + image := buildImage(false) + image[8202] = 0 // Invalid directory entry type. + filesystem, err := Read(testBackend(t, image), int64(len(image)), 0, 0) + if err != nil { + t.Fatal(err) + } + for _, operation := range pathErrorOperations(filesystem) { + if operation.op == "stat" { + continue // Stat does not read the root directory's contents. + } + t.Run(operation.op, func(t *testing.T) { + checkPathError(t, operation.call("."), operation.op, ".", nil) + }) + } +} diff --git a/filesystem/erofs/file.go b/filesystem/erofs/file.go new file mode 100644 index 00000000..8a5dc44f --- /dev/null +++ b/filesystem/erofs/file.go @@ -0,0 +1,221 @@ +package erofs + +import ( + "errors" + "fmt" + "io" + "io/fs" + "os" + "sync" + + "github.com/diskfs/go-diskfs/filesystem" +) + +const ( + fileTypeRegular uint8 = iota + 1 + fileTypeDirectory + fileTypeCharDevice + fileTypeBlockDevice + fileTypeFIFO + fileTypeSocket + fileTypeSymlink +) + +var _ fs.File = (*File)(nil) +var _ fs.ReadDirFile = (*directoryFile)(nil) + +type File struct { + // mu protects the cursor and the single decoded extent, including concurrent + // ReadAt calls. Each open file retains at most maxDecodedExtentSize bytes. + mu sync.Mutex + decoded decodedExtent + filesystem *FileSystem + node *inode + name string + offset int64 +} + +func (f *File) Read(b []byte) (int, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.filesystem == nil { + return 0, os.ErrClosed + } + if f.offset >= f.node.size { + return 0, io.EOF + } + if len(b) == 0 { + return 0, nil + } + length := int64(len(b)) + if f.node.size-f.offset < length { + length = f.node.size - f.offset + } + read, err := f.filesystem.readInodeData(f.node, b[:length], f.offset, &f.decoded) + f.offset += int64(read) + if err != nil { + return read, err + } + if f.offset >= f.node.size { + err = io.EOF + } + return read, err +} + +func (f *File) ReadAt(b []byte, offset int64) (int, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.filesystem == nil { + return 0, os.ErrClosed + } + if offset < 0 { + return 0, fmt.Errorf("read offset %d is outside file size %d", offset, f.node.size) + } + if len(b) == 0 { + return 0, nil + } + if offset >= f.node.size { + return 0, io.EOF + } + length := int64(len(b)) + if f.node.size-offset < length { + length = f.node.size - offset + } + read, err := f.filesystem.readInodeData(f.node, b[:length], offset, &f.decoded) + if err != nil { + return read, err + } + if read < len(b) { + return read, io.EOF + } + return read, nil +} + +func (f *File) Write([]byte) (int, error) { return 0, filesystem.ErrReadonlyFilesystem } + +func (f *File) Seek(offset int64, whence int) (int64, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.filesystem == nil { + return 0, os.ErrClosed + } + var newOffset int64 + switch whence { + case io.SeekStart: + newOffset = offset + case io.SeekCurrent: + newOffset = f.offset + offset + case io.SeekEnd: + newOffset = f.node.size + offset + default: + return f.offset, fmt.Errorf("invalid seek whence %d", whence) + } + if newOffset < 0 { + return f.offset, fmt.Errorf("cannot seek before start of file") + } + f.offset = newOffset + return newOffset, nil +} + +func (f *File) Close() error { + f.mu.Lock() + defer f.mu.Unlock() + f.decoded = decodedExtent{} + f.filesystem = nil + return nil +} + +func (f *File) Stat() (fs.FileInfo, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.filesystem == nil { + return nil, os.ErrClosed + } + return &fileInfo{name: f.name, node: f.node}, nil +} + +type directoryFile struct { + filesystem *FileSystem + node *inode + name string + entries []*directoryEntry + offset int +} + +func (f *directoryFile) Read([]byte) (int, error) { + return 0, errors.New("cannot read from a directory") +} + +func (f *directoryFile) Close() error { + f.filesystem = nil + return nil +} + +func (f *directoryFile) Stat() (fs.FileInfo, error) { + if f.filesystem == nil { + return nil, os.ErrClosed + } + return &fileInfo{name: f.name, node: f.node}, nil +} + +func (f *directoryFile) ReadDir(n int) ([]fs.DirEntry, error) { + if f.filesystem == nil { + return nil, os.ErrClosed + } + if n <= 0 { + entries := make([]fs.DirEntry, 0, len(f.entries)-f.offset) + for _, entry := range f.entries[f.offset:] { + entries = append(entries, &dirEntry{entry: entry, filesystem: f.filesystem}) + } + f.offset = len(f.entries) + return entries, nil + } + if f.offset >= len(f.entries) { + return nil, io.EOF + } + n = min(n, len(f.entries)-f.offset) + end := f.offset + n + entries := make([]fs.DirEntry, 0, end-f.offset) + for _, entry := range f.entries[f.offset:end] { + entries = append(entries, &dirEntry{entry: entry, filesystem: f.filesystem}) + } + f.offset = end + return entries, nil +} + +type dirEntry struct { + entry *directoryEntry + filesystem *FileSystem +} + +var _ fs.DirEntry = (*dirEntry)(nil) + +func (d *dirEntry) Name() string { return d.entry.name } +func (d *dirEntry) IsDir() bool { + return d.entry.fileType == fileTypeDirectory +} +func (d *dirEntry) Type() fs.FileMode { + switch d.entry.fileType { + case fileTypeDirectory: + return fs.ModeDir + case fileTypeCharDevice: + return fs.ModeDevice | fs.ModeCharDevice + case fileTypeBlockDevice: + return fs.ModeDevice + case fileTypeFIFO: + return fs.ModeNamedPipe + case fileTypeSymlink: + return fs.ModeSymlink + case fileTypeSocket: + return fs.ModeSocket + default: + return 0 + } +} +func (d *dirEntry) Info() (fs.FileInfo, error) { + node, err := d.filesystem.getInode(d.entry.nid) + if err != nil { + return nil, err + } + return &fileInfo{name: d.entry.name, node: node}, nil +} diff --git a/filesystem/erofs/file_internal_test.go b/filesystem/erofs/file_internal_test.go new file mode 100644 index 00000000..697a3a9b --- /dev/null +++ b/filesystem/erofs/file_internal_test.go @@ -0,0 +1,52 @@ +package erofs + +import ( + "errors" + "io" + "math" + "os" + "testing" +) + +func TestFileReadAtEOF(t *testing.T) { + filesystem := readTestImage(t, uncompressedFixture) + for _, path := range []string{"hello.txt", "empty.txt"} { + t.Run(path, func(t *testing.T) { + opened, err := filesystem.OpenFile(path, os.O_RDONLY) + if err != nil { + t.Fatal(err) + } + defer opened.Close() + info, err := opened.Stat() + if err != nil { + t.Fatal(err) + } + reader, ok := opened.(io.ReaderAt) + if !ok { + t.Fatalf("OpenFile() returned %T, want io.ReaderAt", opened) + } + for _, offset := range []int64{info.Size(), info.Size() + 1, math.MaxInt64} { + for _, length := range []int{0, 1} { + n, err := reader.ReadAt(make([]byte, length), offset) + var want error + if length != 0 { + want = io.EOF + } + if n != 0 || !errors.Is(err, want) { + t.Errorf("ReadAt(length=%d, offset=%d) = %d, %v; want 0, %v", length, offset, n, err, want) + } + } + } + for _, length := range []int{0, 1} { + if _, err := reader.ReadAt(make([]byte, length), -1); err == nil || errors.Is(err, io.EOF) { + t.Errorf("ReadAt(length=%d, -1) error = %v; want invalid offset", length, err) + } + } + if info.Size() > 0 { + if n, err := reader.ReadAt(make([]byte, 2), info.Size()-1); n != 1 || !errors.Is(err, io.EOF) { + t.Errorf("ReadAt crossing EOF = %d, %v; want 1, EOF", n, err) + } + } + }) + } +} diff --git a/filesystem/erofs/fileinfo.go b/filesystem/erofs/fileinfo.go new file mode 100644 index 00000000..5acdfdf2 --- /dev/null +++ b/filesystem/erofs/fileinfo.go @@ -0,0 +1,58 @@ +package erofs + +import ( + "io/fs" + "time" +) + +// StatT is the EROFS-specific metadata returned by FileInfo.Sys(). +type StatT struct { + UID uint32 + GID uint32 + Inode uint64 + LinkCount uint32 + // DeviceID preserves the EROFS/Linux new_encode_dev value for device nodes. + // Major is (DeviceID >> 8) & 0xfff; minor is (DeviceID & 0xff) | + // ((DeviceID >> 12) & 0xfff00). It is zero for other inode types. + DeviceID uint32 +} + +type fileInfo struct { + name string + node *inode +} + +var _ fs.FileInfo = (*fileInfo)(nil) + +func (fi *fileInfo) Name() string { return fi.name } +func (fi *fileInfo) Size() int64 { return fi.node.size } +func (fi *fileInfo) Mode() fs.FileMode { + mode := goFileMode(fi.node.mode) + return mode +} +func (fi *fileInfo) ModTime() time.Time { return fi.node.mtime } +func (fi *fileInfo) IsDir() bool { return fi.node.isDir() } +func (fi *fileInfo) Sys() any { + return &StatT{ + UID: fi.node.uid, + GID: fi.node.gid, + Inode: fi.node.nid, + LinkCount: fi.node.nlink, + DeviceID: fi.node.rdev, + } +} + +func (fi *fileInfo) Equal(other fs.FileInfo) bool { + if fi.Name() != other.Name() || fi.Size() != other.Size() || fi.Mode() != other.Mode() || + !fi.ModTime().Equal(other.ModTime()) || fi.IsDir() != other.IsDir() { + return false + } + var stat *StatT + switch sys := other.Sys().(type) { + case *StatT: + stat = sys + default: + return false + } + return fi.node.uid == stat.UID && fi.node.gid == stat.GID && fi.node.nid == stat.Inode +} diff --git a/filesystem/erofs/fileinfo_internal_test.go b/filesystem/erofs/fileinfo_internal_test.go new file mode 100644 index 00000000..28c6086f --- /dev/null +++ b/filesystem/erofs/fileinfo_internal_test.go @@ -0,0 +1,42 @@ +package erofs + +import ( + "encoding/binary" + "fmt" + "math" + "testing" +) + +func TestDeviceIDPreservesEncoding(t *testing.T) { + for _, extended := range []bool{false, true} { + for _, mode := range []uint16{linuxModeChar, linuxModeBlock} { + // Include the colliding pair from the review and both maximum fields. + for _, encoded := range []uint32{0x100, 0x100000, 0x10000, math.MaxUint32} { + t.Run(fmt.Sprintf("extended=%t/mode=%x/device=%x", extended, mode, encoded), func(t *testing.T) { + image := buildImage(false) + inodeBytes := image[4128:] + if extended { + binary.LittleEndian.PutUint16(inodeBytes, 1) + } + binary.LittleEndian.PutUint16(inodeBytes[4:], mode|0o600) + binary.LittleEndian.PutUint32(inodeBytes[16:], encoded) + filesystem, err := Read(testBackend(t, image), int64(len(image)), 0, 0) + if err != nil { + t.Fatal(err) + } + info, err := filesystem.Stat("hello") + if err != nil { + t.Fatal(err) + } + stat, ok := info.Sys().(*StatT) + if !ok { + t.Fatalf("Sys() returned %T, want *StatT", info.Sys()) + } + if stat.DeviceID != encoded { + t.Fatalf("DeviceID = %#x, want %#x", stat.DeviceID, encoded) + } + }) + } + } + } +} diff --git a/filesystem/erofs/inode.go b/filesystem/erofs/inode.go new file mode 100644 index 00000000..eed91de6 --- /dev/null +++ b/filesystem/erofs/inode.go @@ -0,0 +1,180 @@ +package erofs + +import ( + "encoding/binary" + "fmt" + "os" + "time" +) + +const ( + linuxModeSocket uint16 = 0xC000 + linuxModeSymlink uint16 = 0xA000 + linuxModeRegular uint16 = 0x8000 + linuxModeBlock uint16 = 0x6000 + linuxModeDir uint16 = 0x4000 + linuxModeChar uint16 = 0x2000 + linuxModeFIFO uint16 = 0x1000 +) + +type inode struct { + nid uint64 + mode uint16 + size int64 + uid uint32 + gid uint32 + nlink uint32 + mtime time.Time + inodeSize int + xattrSize int + dataLayout uint8 + // startBlock is a filesystem block number for flat data (the on-disk union + // instead stores compressed block counts or a device number for other types). + startBlock uint64 + rdev uint32 + compressed *compressedInfo +} + +func (i *inode) isDir() bool { + return i.mode&0xF000 == linuxModeDir +} + +func (i *inode) isSymlink() bool { + return i.mode&0xF000 == linuxModeSymlink +} + +func (f *FileSystem) inodeOffset(nid uint64) (int64, error) { + if nid>>63 != 0 { + return 0, fmt.Errorf("invalid inode number 0x%016x", nid) + } + if f.size < inodeCompactSize { + return 0, fmt.Errorf("image is too small for inode %d", nid) + } + // NIDs count 32-byte inode slots from the metadata area's block address. + metadataBase := int64(f.superblock.metaBlockAddress) * f.blockSize() + available := f.size - inodeCompactSize - metadataBase + if available < 0 || nid > uint64(available)/inodeSlotSize { + return 0, fmt.Errorf("inode %d offset is outside the image", nid) + } + return metadataBase + int64(nid)*inodeSlotSize, nil +} + +func (f *FileSystem) getInode(nid uint64) (*inode, error) { + offset, err := f.inodeOffset(nid) + if err != nil { + return nil, err + } + + readLength := inodeExtendedSize + if offset+inodeExtendedSize > f.size { + readLength = inodeCompactSize + } + b := make([]byte, readLength) + if err := f.readAt(b, offset); err != nil { + return nil, fmt.Errorf("unable to read inode %d: %w", nid, err) + } + format := binary.LittleEndian.Uint16(b[0:2]) + if format&inodeFormatReservedMask != 0 { + return nil, fmt.Errorf("inode %d has unsupported format 0x%04x", nid, format) + } + version := format & 1 + layout := uint8((format >> 1) & 0x7) + if layout > dataLayoutChunked { + return nil, fmt.Errorf("inode %d has unsupported data layout %d", nid, layout) + } + + i := &inode{ + nid: nid, + dataLayout: layout, + } + xattrCount := binary.LittleEndian.Uint16(b[2:4]) + i.mode = binary.LittleEndian.Uint16(b[4:6]) + i.xattrSize = 0 + if xattrCount != 0 { + i.xattrSize = 12 + 4*(int(xattrCount)-1) + } + + if version == 0 { + i.inodeSize = inodeCompactSize + i.size = int64(binary.LittleEndian.Uint32(b[8:12])) + i.mtime = time.Unix(int64(f.superblock.epoch)+int64(binary.LittleEndian.Uint32(b[12:16])), int64(f.superblock.fixedNsec)).UTC() + i.uid = uint32(binary.LittleEndian.Uint16(b[24:26])) + i.gid = uint32(binary.LittleEndian.Uint16(b[26:28])) + nb := binary.LittleEndian.Uint16(b[6:8]) + if !i.isDir() && format&(1<<4) != 0 { + i.nlink = 1 + } else { + i.nlink = uint32(nb) + nb = 0 + } + i.startBlock = uint64(binary.LittleEndian.Uint32(b[16:20])) | uint64(nb)<<32 + } else { + if readLength < inodeExtendedSize { + return nil, fmt.Errorf("inode %d extended inode extends beyond the image", nid) + } + i.inodeSize = inodeExtendedSize + i.size = int64(binary.LittleEndian.Uint64(b[8:16])) + i.startBlock = uint64(binary.LittleEndian.Uint32(b[16:20])) | + uint64(binary.LittleEndian.Uint16(b[6:8]))<<32 + i.uid = binary.LittleEndian.Uint32(b[24:28]) + i.gid = binary.LittleEndian.Uint32(b[28:32]) + i.mtime = time.Unix(int64(binary.LittleEndian.Uint64(b[32:40])), int64(binary.LittleEndian.Uint32(b[40:44]))).UTC() + i.nlink = binary.LittleEndian.Uint32(b[44:48]) + } + + switch i.mode & 0xF000 { + case linuxModeDir, linuxModeRegular, linuxModeSymlink: + if i.startBlock>>48 != 0 { + return nil, fmt.Errorf("inode %d uses unsupported 48-bit block addresses", nid) + } + case linuxModeChar, linuxModeBlock: + i.rdev = binary.LittleEndian.Uint32(b[16:20]) + case linuxModeFIFO, linuxModeSocket: + i.rdev = 0 + default: + return nil, fmt.Errorf("inode %d has invalid mode 0o%o", nid, i.mode) + } + if i.size < 0 { + return nil, fmt.Errorf("inode %d has negative size %d", nid, i.size) + } + if offset+int64(i.inodeSize+i.xattrSize) > f.size { + return nil, fmt.Errorf("inode %d inline metadata extends beyond the image", nid) + } + if layout == dataLayoutChunked { + return nil, fmt.Errorf("inode %d uses unsupported data layout %d", nid, layout) + } + if layout == dataLayoutCompressedFull || layout == dataLayoutCompressedCompact { + if err := f.initCompressed(i); err != nil { + return nil, err + } + } + return i, nil +} + +func goFileMode(mode uint16) os.FileMode { + var result os.FileMode + switch mode & 0xF000 { + case linuxModeDir: + result |= os.ModeDir + case linuxModeSymlink: + result |= os.ModeSymlink + case linuxModeChar: + result |= os.ModeDevice | os.ModeCharDevice + case linuxModeBlock: + result |= os.ModeDevice + case linuxModeFIFO: + result |= os.ModeNamedPipe + case linuxModeSocket: + result |= os.ModeSocket + } + if mode&0o4000 != 0 { + result |= os.ModeSetuid + } + if mode&0o2000 != 0 { + result |= os.ModeSetgid + } + if mode&0o1000 != 0 { + result |= os.ModeSticky + } + return result | os.FileMode(mode&0o777) +} diff --git a/filesystem/erofs/inode_internal_test.go b/filesystem/erofs/inode_internal_test.go new file mode 100644 index 00000000..aa67f1e6 --- /dev/null +++ b/filesystem/erofs/inode_internal_test.go @@ -0,0 +1,38 @@ +package erofs + +import ( + "encoding/binary" + "testing" +) + +func TestGetInodeErrors(t *testing.T) { + tests := []struct { + name string + mutate func([]byte) + }{ + {"reserved format", func(b []byte) { binary.LittleEndian.PutUint16(b[4128:], 1<<5) }}, + {"invalid mode", func(b []byte) { binary.LittleEndian.PutUint16(b[4132:], 0) }}, + {"inline compressed tail", func(b []byte) { + binary.LittleEndian.PutUint16(b[4128:], uint16(dataLayoutCompressedCompact)<<1) + binary.LittleEndian.PutUint16(b[4162:], 1) + binary.LittleEndian.PutUint16(b[4164:], zAdviseInlineData) + }}, + {"xattrs outside image", func(b []byte) { binary.LittleEndian.PutUint16(b[4130:], 10000) }}, + {"large inode number wraps", func(b []byte) { + binary.LittleEndian.PutUint64(b[8192:], uint64(1)<<59|1) + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + image := buildImage(false) + tt.mutate(image) + filesystem, err := Read(testBackend(t, image), int64(len(image)), 0, 0) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + if _, err := filesystem.ReadFile("hello"); err == nil { + t.Fatal("Read() error = nil, want error") + } + }) + } +} diff --git a/filesystem/erofs/read_internal_test.go b/filesystem/erofs/read_internal_test.go new file mode 100644 index 00000000..72cd70f1 --- /dev/null +++ b/filesystem/erofs/read_internal_test.go @@ -0,0 +1,152 @@ +package erofs + +import ( + "encoding/binary" + "fmt" + "math" + "testing" + + "github.com/diskfs/go-diskfs/backend" +) + +// The backing file deliberately extends past the filesystem. Recording reads +// catches boundary violations even if a later validation rejects the image. +type checkedReadBackend struct { + backend.Storage + t *testing.T + start, size int64 +} + +func (b checkedReadBackend) ReadAt(p []byte, offset int64) (int, error) { + if offset < b.start || offset > b.start+b.size || int64(len(p)) > b.start+b.size-offset { + b.t.Errorf("backend read at %d, length %d exceeds filesystem [%d, %d)", offset, len(p), b.start, b.start+b.size) + } + return b.Storage.ReadAt(p, offset) +} + +func TestReadSuperblockBounds(t *testing.T) { + tests := []struct { + name string + size int64 + extensions byte + }{ + {name: "header", size: superOffset + 13}, + {name: "base", size: superOffset + 64}, + {name: "extension", size: superOffset + superblockBaseSize, extensions: 1}, + } + for _, start := range []int64{0, 4096} { + for _, tt := range tests { + t.Run(fmt.Sprintf("%s/start=%d", tt.name, start), func(t *testing.T) { + image := buildImage(false) + image[superOffset+13] = tt.extensions + whole := append(make([]byte, start), image...) + b := checkedReadBackend{Storage: testBackend(t, whole), t: t, start: start, size: tt.size} + if _, err := Read(b, tt.size, start, 0); err == nil { + t.Fatal("Read() accepted an out-of-bounds superblock") + } + }) + } + } +} + +func TestReadCompressionMetadataBounds(t *testing.T) { + const size = int64(12288) + tests := []struct { + name string + nid uint64 + layout uint8 + xattrs uint16 + headerOffset int + header uint64 + }{ + {name: "header", nid: 255, layout: dataLayoutCompressedFull, headerOffset: 12288, header: uint64(1) << 63}, + {name: "full index", nid: 254, layout: dataLayoutCompressedFull, xattrs: 1, headerOffset: 12272, header: uint64(zAdviseFragment) << 32}, + {name: "compact index", nid: 254, layout: dataLayoutCompressedCompact, xattrs: 3, headerOffset: 12280, header: uint64(zAdviseFragment) << 32}, + } + for _, start := range []int64{0, 4096} { + for _, tt := range tests { + t.Run(fmt.Sprintf("%s/start=%d", tt.name, start), func(t *testing.T) { + image := buildImage(true) + binary.LittleEndian.PutUint64(image[superOffset+96:], 1) + binary.LittleEndian.PutUint64(image[8192:], tt.nid) + inodeOffset := 4096 + int(tt.nid)*inodeSlotSize + setCompactInode(image, inodeOffset, uint16(tt.layout)<<1, linuxModeRegular, 11, 0) + binary.LittleEndian.PutUint16(image[inodeOffset+2:], tt.xattrs) + binary.LittleEndian.PutUint64(image[tt.headerOffset:], tt.header) + // Zeroed indexes encode a plain cluster at logical offset zero. + // Without bounds checks these indexes resolve to the packed file. + whole := append(make([]byte, start), image...) + b := checkedReadBackend{Storage: testBackend(t, whole), t: t, start: start, size: size} + filesystem, err := Read(b, size, start, 0) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + if data, err := filesystem.ReadFile("hello"); err == nil { + t.Fatalf("ReadFile() = %q, nil; accepted out-of-bounds metadata", data) + } + }) + } + } +} + +func TestReadCompressionConfigBounds(t *testing.T) { + const size = int64(12288) + for _, start := range []int64{0, 4096} { + for _, lengthOutside := range []bool{false, true} { + t.Run(fmt.Sprintf("lengthOutside=%t/start=%d", lengthOutside, start), func(t *testing.T) { + image := buildImage(true) + binary.LittleEndian.PutUint32(image[superOffset+80:], featureIncompatComprCfgs) + binary.LittleEndian.PutUint16(image[superOffset+84:], 1< b.start { + b.reads++ + } + return b.Storage.ReadAt(p, offset) +} + +func TestAdjacentReadsReuseDecodedExtent(t *testing.T) { + for _, fragment := range []bool{false, true} { + t.Run(fmt.Sprintf("fragment=%t", fragment), func(t *testing.T) { + image := readFixtureBytes(t, fullFixture) + if fragment { + // Keep the existing file's directory entry, but turn its inode + // into an all-fragment file. Move the original full-index inode + // and indexes to a free metadata slot as the packed inode. This + // exercises a compressed packed inode without requiring mkfs. + const originalOffset, packedOffset, packedNID = 40 * inodeSlotSize, 2048, 64 + const metadataLength = inodeCompactSize + 16 + 24*8 + copy(image[packedOffset:], image[originalOffset:originalOffset+metadataLength]) + binary.LittleEndian.PutUint64(image[originalOffset+inodeCompactSize:], uint64(1)<<63) + binary.LittleEndian.PutUint64(image[superOffset+96:], packedNID) + updateSuperblockChecksum(image) + } + filesystem, err := Read(testBackend(t, image), int64(len(image)), 0, 0) + if err != nil { + t.Fatal(err) + } + counter := &extentReadBackend{Storage: filesystem.backend, start: 4096, length: 4096} + filesystem.backend = counter + opened, err := filesystem.OpenFile("data.bin", os.O_RDONLY) + if err != nil { + t.Fatal(err) + } + defer opened.Close() + reader, ok := opened.(io.ReaderAt) + if !ok { + t.Fatalf("OpenFile() returned %T, want io.ReaderAt", opened) + } + payload := fullFixturePayload() + buffer := make([]byte, 4096) + for offset := 0; offset < len(payload); offset += len(buffer) { + // Alternate Read and ReadAt so both public APIs share the cache. + var n int + if offset%8192 == 0 { + _, err = opened.Seek(int64(offset), io.SeekStart) + if err != nil { + t.Fatal(err) + } + n, err = opened.Read(buffer) + } else { + n, err = reader.ReadAt(buffer, int64(offset)) + } + if (err != nil && !errors.Is(err, io.EOF)) || n != len(buffer) || !bytes.Equal(buffer, payload[offset:offset+n]) { + t.Fatalf("read at %d = %d, %v or incorrect data", offset, n, err) + } + } + if _, err := reader.ReadAt(buffer, 0); err != nil || !bytes.Equal(buffer, payload[:len(buffer)]) { + t.Fatalf("read back into cached extent: %v or incorrect data", err) + } + if counter.reads != 1 { + t.Fatalf("compressed data fetched %d times, want once", counter.reads) + } + }) + } +} + +// BenchmarkLargeExtentStreaming measures a fresh open/extraction each time, +// including first decode, so it cannot benchmark an already warmed cache. The +// image uses 1 MiB physical clusters and multi-megabyte decoded extents, with a +// fragment tail. The image and expected payload are loaded outside the timed region. +func BenchmarkLargeExtentStreaming(b *testing.B) { + filesystem := readTestImage(b, largeFixture) + payload := largeFixturePayload() + for _, bufferSize := range []int{32 << 10, len(payload)} { + b.Run(fmt.Sprintf("buffer=%d", bufferSize), func(b *testing.B) { + buffer := make([]byte, bufferSize) + b.SetBytes(int64(len(payload))) + b.ReportAllocs() + for b.Loop() { + opened, err := filesystem.Open("test/path/file") + if err != nil { + b.Fatal(err) + } + total := 0 + for { + n, err := opened.Read(buffer) + if !bytes.Equal(buffer[:n], payload[total:total+n]) { + b.Fatal("incorrect payload") + } + total += n + if errors.Is(err, io.EOF) { + break + } + if err != nil { + b.Fatal(err) + } + } + if total != len(payload) { + b.Fatalf("read %d bytes, want %d", total, len(payload)) + } + if err := opened.Close(); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/filesystem/erofs/superblock.go b/filesystem/erofs/superblock.go new file mode 100644 index 00000000..c2cc8a6d --- /dev/null +++ b/filesystem/erofs/superblock.go @@ -0,0 +1,175 @@ +package erofs + +import ( + "encoding/binary" + "fmt" + "hash/crc32" +) + +type superblock struct { + checksum uint32 + featureCompat uint32 + blockSizeBits uint8 + superblockExtSlots uint8 + rootNID uint16 + inos uint64 + epoch uint64 + fixedNsec uint32 + blocks uint32 + metaBlockAddress uint32 + xattrBlockAddress uint32 + uuid [16]byte + volumeName [16]byte + featureIncompat uint32 + availableComprAlgs uint16 + extraDevices uint16 + deviceSlotOffset uint16 + dirBlockBits uint8 + xattrPrefixCount uint8 + xattrPrefixStart uint32 + packedNID uint64 + xattrFilterReserved uint8 + ishareXattrPrefixID uint8 + buildTime uint32 + rootNID8 uint64 + metaboxNID uint64 +} + +func (s *superblock) blockSize() int64 { + return int64(1) << s.blockSizeBits +} + +func (s *superblock) superblockSize() int { + return superblockBaseSize + int(s.superblockExtSlots)*superblockExtSlotSize +} + +func parseSuperblock(b []byte, imageSize int64) (*superblock, error) { + if len(b) < superblockBaseSize { + return nil, fmt.Errorf("superblock is %d bytes, expected at least %d", len(b), superblockBaseSize) + } + if imageSize < superOffset+int64(superblockBaseSize) { + return nil, fmt.Errorf("image is %d bytes, too small for an EROFS superblock", imageSize) + } + if binary.LittleEndian.Uint32(b[0:4]) != superMagic { + return nil, fmt.Errorf("invalid EROFS magic 0x%08x", binary.LittleEndian.Uint32(b[0:4])) + } + + s := &superblock{ + checksum: binary.LittleEndian.Uint32(b[4:8]), + featureCompat: binary.LittleEndian.Uint32(b[8:12]), + blockSizeBits: b[12], + superblockExtSlots: b[13], + rootNID: binary.LittleEndian.Uint16(b[14:16]), + inos: binary.LittleEndian.Uint64(b[16:24]), + epoch: binary.LittleEndian.Uint64(b[24:32]), + fixedNsec: binary.LittleEndian.Uint32(b[32:36]), + blocks: binary.LittleEndian.Uint32(b[36:40]), + metaBlockAddress: binary.LittleEndian.Uint32(b[40:44]), + xattrBlockAddress: binary.LittleEndian.Uint32(b[44:48]), + featureIncompat: binary.LittleEndian.Uint32(b[80:84]), + availableComprAlgs: binary.LittleEndian.Uint16(b[84:86]), + extraDevices: binary.LittleEndian.Uint16(b[86:88]), + deviceSlotOffset: binary.LittleEndian.Uint16(b[88:90]), + dirBlockBits: b[90], + xattrPrefixCount: b[91], + xattrPrefixStart: binary.LittleEndian.Uint32(b[92:96]), + packedNID: binary.LittleEndian.Uint64(b[96:104]), + xattrFilterReserved: b[104], + ishareXattrPrefixID: b[105], + buildTime: binary.LittleEndian.Uint32(b[108:112]), + rootNID8: binary.LittleEndian.Uint64(b[112:120]), + } + if s.superblockSize() > 128 && len(b) >= 136 { + s.metaboxNID = binary.LittleEndian.Uint64(b[128:136]) + } + copy(s.uuid[:], b[48:64]) + copy(s.volumeName[:], b[64:80]) + + if s.blockSizeBits < 9 || s.blockSizeBits > 16 { + return nil, fmt.Errorf("invalid block size 2^%d", s.blockSizeBits) + } + if s.superblockSize() > len(b) || int64(superOffset)+int64(s.superblockSize()) > imageSize { + return nil, fmt.Errorf("invalid superblock extension count %d", s.superblockExtSlots) + } + if err := s.validateFeatures(b); err != nil { + return nil, err + } + if s.blocks == 0 || s.metaBlockAddress >= s.blocks { + return nil, fmt.Errorf("invalid metadata block address %d with %d blocks", s.metaBlockAddress, s.blocks) + } + if s.packedNID != 0 && s.packedNID>>63 != 0 { + return nil, fmt.Errorf("invalid packed inode number 0x%016x", s.packedNID) + } + rootOffset := int64(s.metaBlockAddress)*s.blockSize() + int64(s.rootNID)*inodeSlotSize + if rootOffset < 0 || rootOffset+inodeCompactSize > imageSize { + return nil, fmt.Errorf("root inode offset %d is outside the image", rootOffset) + } + if s.blocks > uint32(imageSize/s.blockSize()) { + return nil, fmt.Errorf("filesystem has %d blocks but image only has %d", s.blocks, imageSize/s.blockSize()) + } + return s, nil +} + +func (s *superblock) validateFeatures(b []byte) error { + allowedCompat := featureCompatChecksum | featureCompatMtime | featureCompatXattr + if unknown := s.featureCompat &^ allowedCompat; unknown != 0 { + return fmt.Errorf("unsupported compatible feature bits 0x%08x", unknown) + } + allowedIncompat := featureIncompatLZ4Padding | featureIncompatComprCfgs | featureIncompatFragments + if unknown := s.featureIncompat &^ allowedIncompat; unknown != 0 { + return fmt.Errorf("unsupported incompatible feature bits 0x%08x", unknown) + } + if s.extraDevices != 0 { + return fmt.Errorf("multi-device images are not supported") + } + if s.dirBlockBits != 0 { + return fmt.Errorf("directory block size 2^%d is not supported", s.dirBlockBits) + } + if s.xattrPrefixCount != 0 || s.xattrPrefixStart != 0 { + return fmt.Errorf("long xattr prefixes are not supported") + } + if s.xattrFilterReserved != 0 { + return fmt.Errorf("reserved superblock byte 112 is nonzero") + } + if s.ishareXattrPrefixID != 0 { + return fmt.Errorf("shared xattr prefixes are not supported") + } + if b[106] != 0 || b[107] != 0 { + return fmt.Errorf("reserved superblock bytes 106-107 are nonzero") + } + reserved3 := uint64(0) + if s.superblockSize() > 136 && len(b) >= 144 { + reserved3 = binary.LittleEndian.Uint64(b[136:144]) + } + if s.rootNID8 != 0 || s.metaboxNID != 0 || binary.LittleEndian.Uint64(b[120:128]) != 0 || reserved3 != 0 { + return fmt.Errorf("48-bit and metabox layouts are not supported") + } + return nil +} + +func superblockChecksumLength(blockSize int64) int { + if blockSize > superOffset { + return int(blockSize - superOffset) + } + return int(blockSize) +} + +func verifySuperblockChecksum(s *superblock, b []byte) error { + checksumLength := superblockChecksumLength(s.blockSize()) + if len(b) < checksumLength { + return fmt.Errorf("superblock checksum area is %d bytes, expected %d", len(b), checksumLength) + } + if actual := superblockChecksum(b[:checksumLength]); actual != s.checksum { + return fmt.Errorf("invalid superblock checksum 0x%08x, want 0x%08x", s.checksum, actual) + } + return nil +} + +func superblockChecksum(b []byte) uint32 { + if len(b) < superblockBaseSize { + return 0 + } + checksummed := append([]byte(nil), b...) + binary.LittleEndian.PutUint32(checksummed[4:8], 0) + return ^crc32.Update(0, crc32.MakeTable(crc32.Castagnoli), checksummed) +} diff --git a/filesystem/erofs/superblock_internal_test.go b/filesystem/erofs/superblock_internal_test.go new file mode 100644 index 00000000..c5b34ce5 --- /dev/null +++ b/filesystem/erofs/superblock_internal_test.go @@ -0,0 +1,81 @@ +package erofs + +import ( + "encoding/binary" + "testing" +) + +func validSuperblockBytes() []byte { + b := make([]byte, 148) + binary.LittleEndian.PutUint32(b[0:4], superMagic) + b[12] = 12 + binary.LittleEndian.PutUint16(b[14:16], 0) + binary.LittleEndian.PutUint32(b[36:40], 2) + binary.LittleEndian.PutUint32(b[40:44], 1) + copy(b[64:80], "test volume") + return b +} + +func TestParseSuperblock(t *testing.T) { + b := validSuperblockBytes() + s, err := parseSuperblock(b, 8192) + if err != nil { + t.Fatalf("parseSuperblock() error = %v", err) + } + if s.blockSize() != 4096 { + t.Errorf("blockSize() = %d, want 4096", s.blockSize()) + } + if s.rootNID != 0 || s.metaBlockAddress != 1 { + t.Errorf("unexpected root location: nid=%d meta=%d", s.rootNID, s.metaBlockAddress) + } +} + +func TestParseSuperblockErrors(t *testing.T) { + tests := []struct { + name string + mutate func([]byte) + }{ + {"magic", func(b []byte) { binary.LittleEndian.PutUint32(b[0:4], 0) }}, + {"block size", func(b []byte) { b[12] = 8 }}, + {"compat feature", func(b []byte) { binary.LittleEndian.PutUint32(b[8:12], 1<<3) }}, + {"incompat feature", func(b []byte) { binary.LittleEndian.PutUint32(b[80:84], 1<<2) }}, + {"extra devices", func(b []byte) { binary.LittleEndian.PutUint16(b[86:88], 1) }}, + {"directory blocks", func(b []byte) { b[90] = 1 }}, + {"xattr prefixes", func(b []byte) { b[91] = 1 }}, + {"shared xattr prefixes", func(b []byte) { b[105] = 1 }}, + {"reserved", func(b []byte) { b[106] = 1 }}, + {"48-bit", func(b []byte) { binary.LittleEndian.PutUint64(b[112:120], 1) }}, + {"metabox", func(b []byte) { + b[13] = 1 + binary.LittleEndian.PutUint64(b[128:136], 1) + }}, + {"root outside image", func(b []byte) { binary.LittleEndian.PutUint16(b[14:16], 1000) }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := validSuperblockBytes() + tt.mutate(b) + if _, err := parseSuperblock(b, 8192); err == nil { + t.Fatal("parseSuperblock() error = nil, want error") + } + }) + } +} + +func TestSuperblockChecksum(t *testing.T) { + b := make([]byte, 3072) + copy(b, validSuperblockBytes()) + binary.LittleEndian.PutUint32(b[8:12], featureCompatChecksum) + binary.LittleEndian.PutUint32(b[4:8], superblockChecksum(b[:3072])) + s, err := parseSuperblock(b, 8192) + if err != nil { + t.Fatalf("parseSuperblock() error = %v", err) + } + if err := verifySuperblockChecksum(s, b); err != nil { + t.Fatalf("verifySuperblockChecksum() error = %v", err) + } + b[100] ^= 1 + if err := verifySuperblockChecksum(s, b); err == nil { + t.Fatal("verifySuperblockChecksum() error = nil after corruption") + } +} diff --git a/filesystem/erofs/testdata/README.md b/filesystem/erofs/testdata/README.md new file mode 100644 index 00000000..9fdfeb1b --- /dev/null +++ b/filesystem/erofs/testdata/README.md @@ -0,0 +1,57 @@ +# EROFS Test Fixtures + +The fixtures were generated with `erofs-utils` 1.9.4 and can be read by tests +without external tools. To regenerate them, install Python 3 and +`mkfs.erofs` from `erofs-utils` 1.9.4 with LZMA support, then run: + +```sh +./filesystem/erofs/testdata/generate.sh +``` + +The script works from any directory, creates and cleans up its source trees, +and fixes permissions, ownership, timestamps, UUIDs, and block sizes. An optional +output directory lets you compare regenerated images with the committed ones: + +```sh +./filesystem/erofs/testdata/generate.sh /tmp/erofs-fixtures +``` + +The source tree contains an empty directory, an empty file, `hello.txt` +containing `hello world\n`, a `link` symlink to `hello.txt`, and +`sub/data.bin`. The 96 KiB data payload is the little-endian output of the +pseudo-random sequence `seed = seed * 1664525 + 1013904223`, starting with +`0x45524f46`. + +`full.erofs` contains a single 96 KiB file filled with +`EROFS deterministic fixture payload\n` and truncated to length. It uses the +full compressed index layout without storing the whole file in the packed +inode. + +`large.erofs` contains `dir/file-0` through `dir/file-299` and +`test/path/file`. The directory spans more than one block and uses a compact +inode. The file contains a 15 MiB payload made of 64 KiB records, each with +16 KiB generated by the same pseudo-random sequence followed by 48 KiB of zeros. +It uses 1 MiB physical compressed clusters, multi-megabyte decoded extents, and +a packed fragment tail. The read tests and streaming benchmark share this +fixture so they run without `mkfs.erofs`, including in CI. Only regenerating +fixtures requires external tools. + +SHA-256: + +* `uncompressed.erofs`: `0528acf617874fc58673a6dc9cbba677b750c7e285fb40d353628bc7e319d55a` +* `compressed.erofs`: `22faedd65c01aa91baa76aca52c523dd746d5585ffb5b0c6a02b93d0b913f279` +* `full.erofs`: `352775b0e0b6561c6b51a8a9c5ca5924df7e4447325b183d8f1f670c77371e44` +* `large.erofs`: `6bcf1b0eeaa8a65a3d439f16c751432575c750a8a1ed7984bd42e0d146d669a7` + +## Streaming benchmark + +Run: + +```sh +go test ./filesystem/erofs -run '^$' -bench BenchmarkLargeExtentStreaming -benchmem +``` + +The benchmark reads `large.erofs`, loading the image and expected payload +outside the timed region. Each extraction opens a fresh file and +checks the output against the source; the 32 KiB buffer and whole-file buffer +cases both include initial decoding. diff --git a/filesystem/erofs/testdata/compressed.erofs b/filesystem/erofs/testdata/compressed.erofs new file mode 100644 index 00000000..82120d07 Binary files /dev/null and b/filesystem/erofs/testdata/compressed.erofs differ diff --git a/filesystem/erofs/testdata/full.erofs b/filesystem/erofs/testdata/full.erofs new file mode 100644 index 00000000..74b23d4c Binary files /dev/null and b/filesystem/erofs/testdata/full.erofs differ diff --git a/filesystem/erofs/testdata/generate.sh b/filesystem/erofs/testdata/generate.sh new file mode 100755 index 00000000..0f53250c --- /dev/null +++ b/filesystem/erofs/testdata/generate.sh @@ -0,0 +1,61 @@ +#!/bin/sh +set -eu + +# Keep source permissions independent of the caller's umask. +umask 022 +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +output_dir=${1:-$script_dir} +mkdir -p "$output_dir" +output_dir=$(CDPATH= cd -- "$output_dir" && pwd) +work_dir=$(mktemp -d) +trap 'rm -rf "$work_dir"' 0 +trap 'exit 1' HUP INT TERM + +python3 - "$work_dir" <<'PY' +import pathlib +import sys + +work = pathlib.Path(sys.argv[1]) +source = work / "source" +(source / "empty").mkdir(parents=True) +(source / "sub").mkdir() +(source / "empty.txt").touch() +(source / "hello.txt").write_bytes(b"hello world\n") +(source / "link").symlink_to("hello.txt") + +payload = bytearray() +seed = 0x45524F46 +for _ in range(96 * 1024 // 4): + seed = (seed * 1664525 + 1013904223) & 0xFFFFFFFF + payload += seed.to_bytes(4, "little") +(source / "sub" / "data.bin").write_bytes(payload) + +full = work / "full" +full.mkdir() +payload = b"EROFS deterministic fixture payload\n" * 2622 +(full / "data.bin").write_bytes(payload.ljust(96 * 1024, b"\0")) + +large = work / "large" +(large / "dir").mkdir(parents=True) +for index in range(300): + (large / "dir" / f"file-{index}").touch() +(large / "test" / "path").mkdir(parents=True) +payload = bytearray(15 * 1024 * 1024) +seed = 0x45524F46 +for record in range(0, len(payload), 64 * 1024): + for offset in range(record, record + 16 * 1024, 4): + seed = (seed * 1664525 + 1013904223) & 0xFFFFFFFF + payload[offset:offset + 4] = seed.to_bytes(4, "little") +(large / "test" / "path" / "file").write_bytes(payload) +PY + +# Store common mkfs options for reuse through "$@" below. +set -- --quiet -b4096 -x-1 --all-root -T1700000000 \ + -U00000000-0000-0000-0000-000000000000 +mkfs.erofs "$@" "$output_dir/uncompressed.erofs" "$work_dir/source" +mkfs.erofs "$@" -zlzma -C65536 -Efragments \ + "$output_dir/compressed.erofs" "$work_dir/source" +mkfs.erofs "$@" -zlzma -Elegacy-compress \ + "$output_dir/full.erofs" "$work_dir/full" +mkfs.erofs "$@" -zlzma -C1048576 -Efragments --workers=1 \ + "$output_dir/large.erofs" "$work_dir/large" diff --git a/filesystem/erofs/testdata/large.erofs b/filesystem/erofs/testdata/large.erofs new file mode 100644 index 00000000..9a61b95b Binary files /dev/null and b/filesystem/erofs/testdata/large.erofs differ diff --git a/filesystem/erofs/testdata/uncompressed.erofs b/filesystem/erofs/testdata/uncompressed.erofs new file mode 100644 index 00000000..081191fc Binary files /dev/null and b/filesystem/erofs/testdata/uncompressed.erofs differ diff --git a/filesystem/filesystem.go b/filesystem/filesystem.go index 63e58cb5..61e89b69 100644 --- a/filesystem/filesystem.go +++ b/filesystem/filesystem.go @@ -74,4 +74,6 @@ const ( TypeFat12 // TypeFat16 is a FAT16 compatible filesystem TypeFat16 + // TypeErofs is an EROFS compatible filesystem + TypeErofs )