diff --git a/README.md b/README.md index a0d4cc4..1bd113b 100644 --- a/README.md +++ b/README.md @@ -109,14 +109,16 @@ All the code is in the `pkg/dcf` package. The repo has no commands. each one. - `helpers.go` finds the mount points that hold a DCF store. - `parse.go` reads one DCF store through an `fs.FS` rooted at its mount point. +- `filegroup.go` sorts the files of one DCF directory into file groups. The main types are: - `Store` is one DCF store on one mounted filesystem. -- `Directory` is one DCF directory. It has a directory number, a name and the - objects inside it. +- `Directory` is one DCF directory. It has a directory number, a name, the + objects inside it and their file groups. - `Object` is one file in a store. A file in a DCF directory also has a directory number and a file number. +- `FileGroup` is one file group in a DCF directory. - `Image` and `Video` embed `Object`. The file extension decides which one a file becomes. A file with any other extension, such as a `THM` thumbnail, is neither. @@ -149,11 +151,37 @@ The naming rules do not apply under `PRIVATE/M4ROOT`. The images and videos there are in `Store.Images` and `Store.Videos`, and their DCF numbers are 0. `dcf` skips any symlink there. +A file group is made of the DCF files in one DCF directory that share a file +number. The specification calls it a DCF object. A RAW file and its JPEG form a +file group, and so do a video and its `THM` thumbnail. + +- The first four characters and the extension can differ within a group. +- A DCF file that shares its number with no other file is a group of one. +- Files in different DCF directories are never in the same group. +- Only DCF files are grouped. Files in `Store.NonConforming` and under + `PRIVATE/M4ROOT` are in no group. + +`Directory.FileGroups` lists the groups in file number order. Each `FileGroup` +has: + +- `DirectoryNumber` and `FileNumber` +- `Members`, which is every file in the group +- `Primary`, which is the member that stands for the whole group +- `MembersCount`, which returns the number of members +- `TotalSize`, which returns the total size of the members in bytes + +The primary member is the first of these that the group holds: + +1. a `JPG` file +2. any other image +3. a video +4. any other file except a `THM` +5. a `THM` + +`Store.Images` and `Store.Videos` list each file on its own, whatever its group. + ## TODO -- Handle DCF file groups, where multiple objects share the same index number. - `dcf` does not handle these yet. If you need this, send me a link to a zip or - tar of a memory card that uses them, and I'll see what I can do. - Take a `context.Context` to cancel the walk of a store. - Return a plain slice from `GetDCFStores`, not a pointer to one. - Copy files off a card, and verify each copy against its SHA256 hash. diff --git a/pkg/dcf/dcf.go b/pkg/dcf/dcf.go index 69d5917..7265e68 100644 --- a/pkg/dcf/dcf.go +++ b/pkg/dcf/dcf.go @@ -2,7 +2,8 @@ // Camera File system" specification (JEITA CP-3461). It finds the cards among // the mounted filesystems and parses the DCF directories and files on each one // into their directory numbers and file numbers. It lists the images and videos -// on each card, and everything under DCIM that is not a DCF directory or file. +// on each card, and everything under DCIM that is not a DCF directory or file. It +// groups the files in each DCF directory that share a file number. package dcf import ( @@ -46,6 +47,22 @@ type Directory struct { // Objects holds every DCF file in the directory, including those that are // neither images nor videos. Objects []*Object + // FileGroups holds the same files, grouped by file number, in file number + // order. + FileGroups []*FileGroup +} + +// FileGroup represents one file group: the files in one DCF directory that share +// a file number. The DCF specification calls this a DCF object. A file that +// shares its number with no other file is a group of one. +type FileGroup struct { + DirectoryNumber int + FileNumber int + // Members holds every file in the group, including Primary, in name order. + // Two members can have the same extension. + Members []*Object + // Primary is the member that stands for the whole group. + Primary *Object } // Object represents a file in a DCF store. It is embedded in Image and Video @@ -114,6 +131,21 @@ func (d *Store) TotalSize() int64 { return d.TotalImageSize() + d.TotalVideoSize() } +// MembersCount returns the number of files in the file group. +func (g *FileGroup) MembersCount() int { + return len(g.Members) +} + +// TotalSize returns the total size in bytes of the files in the file group. +func (g *FileGroup) TotalSize() int64 { + totalSize := int64(0) + for _, member := range g.Members { + totalSize += member.Size + } + + return totalSize +} + func (d *Store) String() string { // A caller can set a negative Size, and uint64 would make that huge. totalSize := uint64(max(d.TotalSize(), 0)) diff --git a/pkg/dcf/filegroup.go b/pkg/dcf/filegroup.go new file mode 100644 index 0000000..213caf3 --- /dev/null +++ b/pkg/dcf/filegroup.go @@ -0,0 +1,65 @@ +package dcf + +import ( + "cmp" + "path" + "slices" + "strings" +) + +// newFileGroups sorts the objects of one DCF directory into file groups, in file +// number order. Section 4.3.2.2 of the specification says the files in a DCF +// directory that share a file number are one group. Their first four characters +// and their extensions can differ. +func newFileGroups(objects []*Object) []*FileGroup { + var groups []*FileGroup + + groupWithNumber := map[int]*FileGroup{} + + for _, object := range objects { + group, found := groupWithNumber[object.FileNumber] + if !found { + group = &FileGroup{ + DirectoryNumber: object.DirectoryNumber, + FileNumber: object.FileNumber, + } + groupWithNumber[object.FileNumber] = group + groups = append(groups, group) + } + + group.Members = append(group.Members, object) + } + + for _, group := range groups { + group.Primary = primaryMember(group.Members) + } + + slices.SortFunc(groups, func(a, b *FileGroup) int { + return cmp.Compare(a.FileNumber, b.FileNumber) + }) + + return groups +} + +// primaryMember returns the member that stands for the whole group. Section +// 6.2.1.3 of the specification gives a JPG priority over the other files. Section +// 4.6 makes a THM only the thumbnail of another file, so it comes last. The +// specification leaves the rest open, so an image comes before a video, and a +// video before any other file. Among equals, the first in name order wins. +func primaryMember(members []*Object) *Object { + choices := []func(string) bool{isJPG, isImageFile, isVideoFile, isNotTHM} + + for _, choice := range choices { + for _, member := range members { + if choice(member.Path) { + return member + } + } + } + + return members[0] +} + +func isNotTHM(name string) bool { + return !strings.EqualFold(path.Ext(name), ".THM") +} diff --git a/pkg/dcf/filegroup_internal_test.go b/pkg/dcf/filegroup_internal_test.go new file mode 100644 index 0000000..26fc8ee --- /dev/null +++ b/pkg/dcf/filegroup_internal_test.go @@ -0,0 +1,230 @@ +package dcf + +import ( + "slices" + "testing" + "testing/fstest" +) + +// mustFileGroups returns the file groups of directory, and stops the test unless +// there are exactly count of them. +func mustFileGroups(t *testing.T, directory *Directory, count int) []*FileGroup { + t.Helper() + + if len(directory.FileGroups) != count { + t.Fatalf("%s has %d file groups, want %d", + directory.Name, len(directory.FileGroups), count) + } + + return directory.FileGroups +} + +// assertFileGroup checks the numbers, the primary member and the members of group. +func assertFileGroup( + t *testing.T, + group *FileGroup, + directoryNumber, fileNumber int, + primary string, + members ...string, +) { + t.Helper() + + if group.DirectoryNumber != directoryNumber || group.FileNumber != fileNumber { + t.Errorf("group numbers = %d and %d, want %d and %d", + group.DirectoryNumber, group.FileNumber, directoryNumber, fileNumber) + } + + if group.Primary.Path != primary { + t.Errorf("Primary = %q, want %q", group.Primary.Path, primary) + } + + paths := make([]string, 0, len(group.Members)) + + for _, member := range group.Members { + paths = append(paths, member.Path) + } + + assertPaths(t, "members", paths, members...) + + if got := group.MembersCount(); got != len(members) { + t.Errorf("MembersCount() = %d, want %d", got, len(members)) + } +} + +// assertSizes checks the total size of group and the size of each member. +func assertSizes(t *testing.T, group *FileGroup, total int64, sizes ...int64) { + t.Helper() + + if got := group.TotalSize(); got != total { + t.Errorf("TotalSize() = %d, want %d", got, total) + } + + got := make([]int64, 0, len(group.Members)) + + for _, member := range group.Members { + got = append(got, member.Size) + } + + if !slices.Equal(got, sizes) { + t.Errorf("member sizes = %v, want %v", got, sizes) + } +} + +// assertPrimaryMember checks which of the named members primaryMember picks. +func assertPrimaryMember(t *testing.T, want string, names ...string) { + t.Helper() + + members := make([]*Object, 0, len(names)) + + for _, name := range names { + members = append(members, &Object{Path: name}) + } + + if got := primaryMember(members).Path; got != want { + t.Errorf("primaryMember of %q = %q, want %q", names, got, want) + } +} + +func TestParseStoreGroupsFilesSharingAFileNumber(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, fstest.MapFS{ + "DCIM/100ABCDE/DSC00001.ARW": {Data: make([]byte, 2000)}, + "DCIM/100ABCDE/DSC00001.JPG": {Data: make([]byte, 500)}, + "DCIM/100ABCDE/MVI_0002.MP4": {Data: make([]byte, 9000)}, + "DCIM/100ABCDE/MVI_0002.THM": {Data: make([]byte, 100)}, + "DCIM/100ABCDE/DSC00003.JPG": {Data: make([]byte, 700)}, + "DCIM/101ABCDE/DSC00001.JPG": {Data: make([]byte, 300)}, + }) + + first := mustFileGroups(t, store.Directories[0], 3) + second := mustFileGroups(t, store.Directories[1], 1) + + assertFileGroup(t, first[0], 100, 1, "DCIM/100ABCDE/DSC00001.JPG", + "DCIM/100ABCDE/DSC00001.ARW", + "DCIM/100ABCDE/DSC00001.JPG", + ) + assertSizes(t, first[0], 2500, 2000, 500) + + assertFileGroup(t, first[1], 100, 2, "DCIM/100ABCDE/MVI_0002.MP4", + "DCIM/100ABCDE/MVI_0002.MP4", + "DCIM/100ABCDE/MVI_0002.THM", + ) + assertSizes(t, first[1], 9100, 9000, 100) + + assertFileGroup(t, first[2], 100, 3, "DCIM/100ABCDE/DSC00003.JPG", + "DCIM/100ABCDE/DSC00003.JPG", + ) + assertSizes(t, first[2], 700, 700) + + assertFileGroup(t, second[0], 101, 1, "DCIM/101ABCDE/DSC00001.JPG", + "DCIM/101ABCDE/DSC00001.JPG", + ) + assertSizes(t, second[0], 300, 300) + + // The images, the videos and the totals still count each file on its own. + if got := store.ImagesCount(); got != 4 { + t.Errorf("ImagesCount() = %d, want 4", got) + } + + if got := store.VideosCount(); got != 1 { + t.Errorf("VideosCount() = %d, want 1", got) + } + + if got := store.TotalSize(); got != 12500 { + t.Errorf("TotalSize() = %d, want 12500", got) + } +} + +func TestParseStoreGroupsFilesWhoseFirstFourCharactersDiffer(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, newCard( + "DCIM/100ABCDE/ABCD0004.TIF", + "DCIM/100ABCDE/THMB0004.THM", + )) + + groups := mustFileGroups(t, store.Directories[0], 1) + + assertFileGroup(t, groups[0], 100, 4, "DCIM/100ABCDE/ABCD0004.TIF", + "DCIM/100ABCDE/ABCD0004.TIF", + "DCIM/100ABCDE/THMB0004.THM", + ) +} + +func TestParseStoreOrdersFileGroupsByFileNumber(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, newCard( + "DCIM/100ABCDE/ABCD0002.JPG", + "DCIM/100ABCDE/WXYZ0001.JPG", + )) + + groups := mustFileGroups(t, store.Directories[0], 2) + + assertFileGroup(t, groups[0], 100, 1, "DCIM/100ABCDE/WXYZ0001.JPG", + "DCIM/100ABCDE/WXYZ0001.JPG", + ) + assertFileGroup(t, groups[1], 100, 2, "DCIM/100ABCDE/ABCD0002.JPG", + "DCIM/100ABCDE/ABCD0002.JPG", + ) +} + +func TestParseStoreLeavesOtherFilesOutOfFileGroups(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, newCard( + "DCIM/100ABCDE/ABCD0004.ARW", + "DCIM/100ABCDE/ABCD0004.JPG", + "DCIM/100ABCDE/PQRS0004.JPG", + "DCIM/100ABCDE/README.TXT", + "DCIM/MISC/ABCD0005.JPG", + "PRIVATE/M4ROOT/CLIP/C0006.MP4", + )) + + if len(store.Directories) != 1 { + t.Fatalf("got %d directories, want 1", len(store.Directories)) + } + + groups := mustFileGroups(t, store.Directories[0], 1) + + assertFileGroup(t, groups[0], 100, 4, "DCIM/100ABCDE/ABCD0004.ARW", + "DCIM/100ABCDE/ABCD0004.ARW", + ) +} + +func TestParseStoreGivesDirectoryWithoutDCFFilesNoFileGroups(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, newCard("DCIM/100ABCDE/README.TXT")) + + mustFileGroups(t, store.Directories[0], 0) +} + +func TestPrimaryMember(t *testing.T) { + t.Parallel() + + // A JPG comes first, in either case. + assertPrimaryMember(t, "A0010001.JPG", "A0010001.ARW", "A0010001.JPG") + assertPrimaryMember(t, "A0010001.JPG", "A0010001.JPG", "A0010001.MOV") + assertPrimaryMember(t, "a0010001.jpg", "a0010001.arw", "a0010001.jpg") + + // Then any other image. + assertPrimaryMember(t, "Z0010001.ARW", "A0010001.MP4", "Z0010001.ARW") + assertPrimaryMember(t, "Z0010001.ARW", "A0010001.WAV", "Z0010001.ARW") + + // Then a video. + assertPrimaryMember(t, "A0010001.MP4", "A0010001.AAC", "A0010001.MP4") + assertPrimaryMember(t, "Z0010001.MP4", "A0010001.THM", "Z0010001.MP4") + + // Then any other file except a THM, in either case. + assertPrimaryMember(t, "Z0010001.TIF", "A0010001.THM", "Z0010001.TIF") + assertPrimaryMember(t, "a0010001.tif", "a0010001.thm", "a0010001.tif") + + // A THM is primary only when the group holds nothing else. + assertPrimaryMember(t, "A0010001.THM", "A0010001.THM") + + // Among equals, the first in name order wins. + assertPrimaryMember(t, "A0010001.THM", "A0010001.THM", "Z0010001.THM") + assertPrimaryMember(t, "A0010001.WAV", "A0010001.WAV", "Z0010001.XML") +} diff --git a/pkg/dcf/parse.go b/pkg/dcf/parse.go index 08c6b38..ffef610 100644 --- a/pkg/dcf/parse.go +++ b/pkg/dcf/parse.go @@ -157,6 +157,7 @@ func parseDirectory(fsys fs.FS, store *Store, name string, number int) error { addMedia(store, object) } + directory.FileGroups = newFileGroups(directory.Objects) store.Directories = append(store.Directories, directory) return nil diff --git a/pkg/dcf/parse_internal_test.go b/pkg/dcf/parse_internal_test.go index 633dd79..7925c9b 100644 --- a/pkg/dcf/parse_internal_test.go +++ b/pkg/dcf/parse_internal_test.go @@ -113,6 +113,16 @@ func directoryNumbers(store *Store) []int { return numbers } +// wantDirectory returns the Directory that parseStore should build for objects. +func wantDirectory(number int, name string, objects ...*Object) *Directory { + return &Directory{ + Number: number, + Name: name, + Objects: objects, + FileGroups: newFileGroups(objects), + } +} + func assertPaths(t *testing.T, what string, got []string, want ...string) { t.Helper() @@ -172,8 +182,8 @@ func TestParseStorePutsEachFileWhereItBelongs(t *testing.T) { } wantDirectories := []*Directory{ - {Number: 100, Name: "100MSDCF", Objects: []*Object{&arw}}, - {Number: 101, Name: "101CANON", Objects: []*Object{&mp4, &thm}}, + wantDirectory(100, "100MSDCF", &arw), + wantDirectory(101, "101CANON", &mp4, &thm), } if !reflect.DeepEqual(store.Directories, wantDirectories) { t.Errorf("Directories = %+v, want %+v", store.Directories, wantDirectories) @@ -305,17 +315,13 @@ func TestParseStoreTreatsLowerCaseNamesAsDCFNames(t *testing.T) { store := mustParseStore(t, newCard("DCIM/101abcde/abcd0002.jpg")) - want := []*Directory{{ - Number: 101, - Name: "101abcde", - Objects: []*Object{{ - DCFStoreRoot: testRoot, - Path: "DCIM/101abcde/abcd0002.jpg", - Extension: ".jpg", - DirectoryNumber: 101, - FileNumber: 2, - }}, - }} + want := []*Directory{wantDirectory(101, "101abcde", &Object{ + DCFStoreRoot: testRoot, + Path: "DCIM/101abcde/abcd0002.jpg", + Extension: ".jpg", + DirectoryNumber: 101, + FileNumber: 2, + })} if !reflect.DeepEqual(store.Directories, want) { t.Errorf("Directories = %+v, want %+v", store.Directories, want) }