group DCF files that share a file number (closes #3)

Files in one DCF directory that share a file number are one file group,
as section 4.3.2.2 of the specification says. `Directory.FileGroups`
lists them, each with its numbers, its members and its primary member. A
lone file is a group of one. A group reports its member count and total
size.

The primary member follows section 6.2.1.3, which gives a JPG priority. A
`THM` comes last. `Store.Images`, `Store.Videos` and the totals are
unchanged.

Model: opus-5-5

README.md

@@ -109,14 +109,16 @@ All the code is in the `pkg/dcf` package. The repo has no commands.
   each one.
 - `helpers.go` finds the mount points that hold a DCF store.
 - `parse.go` reads one DCF store through an `fs.FS` rooted at its mount point.
+- `filegroup.go` sorts the files of one DCF directory into file groups.
 
 The main types are:
 
 - `Store` is one DCF store on one mounted filesystem.
-- `Directory` is one DCF directory. It has a directory number, a name and the
+- `Directory` is one DCF directory. It has a directory number, a name, the
-  objects inside it.
+  objects inside it and their file groups.
 - `Object` is one file in a store. A file in a DCF directory also has a
   directory number and a file number.
+- `FileGroup` is one file group in a DCF directory.
 - `Image` and `Video` embed `Object`. The file extension decides which one a
   file becomes. A file with any other extension, such as a `THM` thumbnail, is
   neither.
@@ -149,11 +151,37 @@ The naming rules do not apply under `PRIVATE/M4ROOT`. The images and videos
 there are in `Store.Images` and `Store.Videos`, and their DCF numbers are 0.
 `dcf` skips any symlink there.
 
+A file group is made of the DCF files in one DCF directory that share a file
+number. The specification calls it a DCF object. A RAW file and its JPEG form a
+file group, and so do a video and its `THM` thumbnail.
+
+- The first four characters and the extension can differ within a group.
+- A DCF file that shares its number with no other file is a group of one.
+- Files in different DCF directories are never in the same group.
+- Only DCF files are grouped. Files in `Store.NonConforming` and under
+  `PRIVATE/M4ROOT` are in no group.
+
+`Directory.FileGroups` lists the groups in file number order. Each `FileGroup`
+has:
+
+- `DirectoryNumber` and `FileNumber`
+- `Members`, which is every file in the group
+- `Primary`, which is the member that stands for the whole group
+- `MembersCount`, which returns the number of members
+- `TotalSize`, which returns the total size of the members in bytes
+
+The primary member is the first of these that the group holds:
+
+1. a `JPG` file
+2. any other image
+3. a video
+4. any other file except a `THM`
+5. a `THM`
+
+`Store.Images` and `Store.Videos` list each file on its own, whatever its group.
+
 ## TODO
 
-- Handle DCF file groups, where multiple objects share the same index number.
-  `dcf` does not handle these yet. If you need this, send me a link to a zip or
-  tar of a memory card that uses them, and I'll see what I can do.
 - Take a `context.Context` to cancel the walk of a store.
 - Return a plain slice from `GetDCFStores`, not a pointer to one.
 - Copy files off a card, and verify each copy against its SHA256 hash.

pkg/dcf/dcf.go

@@ -2,7 +2,8 @@
 // Camera File system" specification (JEITA CP-3461). It finds the cards among
 // the mounted filesystems and parses the DCF directories and files on each one
 // into their directory numbers and file numbers. It lists the images and videos
-// on each card, and everything under DCIM that is not a DCF directory or file.
+// on each card, and everything under DCIM that is not a DCF directory or file. It
+// groups the files in each DCF directory that share a file number.
 package dcf
 
 import (
@@ -46,6 +47,22 @@ type Directory struct {
 	// Objects holds every DCF file in the directory, including those that are
 	// neither images nor videos.
 	Objects []*Object
+	// FileGroups holds the same files, grouped by file number, in file number
+	// order.
+	FileGroups []*FileGroup
+}
+
+// FileGroup represents one file group: the files in one DCF directory that share
+// a file number. The DCF specification calls this a DCF object. A file that
+// shares its number with no other file is a group of one.
+type FileGroup struct {
+	DirectoryNumber int
+	FileNumber      int
+	// Members holds every file in the group, including Primary, in name order.
+	// Two members can have the same extension.
+	Members []*Object
+	// Primary is the member that stands for the whole group.
+	Primary *Object
 }
 
 // Object represents a file in a DCF store. It is embedded in Image and Video
@@ -114,6 +131,21 @@ func (d *Store) TotalSize() int64 {
 	return d.TotalImageSize() + d.TotalVideoSize()
 }
 
+// MembersCount returns the number of files in the file group.
+func (g *FileGroup) MembersCount() int {
+	return len(g.Members)
+}
+
+// TotalSize returns the total size in bytes of the files in the file group.
+func (g *FileGroup) TotalSize() int64 {
+	totalSize := int64(0)
+	for _, member := range g.Members {
+		totalSize += member.Size
+	}
+
+	return totalSize
+}
+
 func (d *Store) String() string {
 	// A caller can set a negative Size, and uint64 would make that huge.
 	totalSize := uint64(max(d.TotalSize(), 0))

pkg/dcf/filegroup.go

@@ -0,0 +1,65 @@
+package dcf
+
+import (
+	"cmp"
+	"path"
+	"slices"
+	"strings"
+)
+
+// newFileGroups sorts the objects of one DCF directory into file groups, in file
+// number order. Section 4.3.2.2 of the specification says the files in a DCF
+// directory that share a file number are one group. Their first four characters
+// and their extensions can differ.
+func newFileGroups(objects []*Object) []*FileGroup {
+	var groups []*FileGroup
+
+	groupWithNumber := map[int]*FileGroup{}
+
+	for _, object := range objects {
+		group, found := groupWithNumber[object.FileNumber]
+		if !found {
+			group = &FileGroup{
+				DirectoryNumber: object.DirectoryNumber,
+				FileNumber:      object.FileNumber,
+			}
+			groupWithNumber[object.FileNumber] = group
+			groups = append(groups, group)
+		}
+
+		group.Members = append(group.Members, object)
+	}
+
+	for _, group := range groups {
+		group.Primary = primaryMember(group.Members)
+	}
+
+	slices.SortFunc(groups, func(a, b *FileGroup) int {
+		return cmp.Compare(a.FileNumber, b.FileNumber)
+	})
+
+	return groups
+}
+
+// primaryMember returns the member that stands for the whole group. Section
+// 6.2.1.3 of the specification gives a JPG priority over the other files. Section
+// 4.6 makes a THM only the thumbnail of another file, so it comes last. The
+// specification leaves the rest open, so an image comes before a video, and a
+// video before any other file. Among equals, the first in name order wins.
+func primaryMember(members []*Object) *Object {
+	choices := []func(string) bool{isJPG, isImageFile, isVideoFile, isNotTHM}
+
+	for _, choice := range choices {
+		for _, member := range members {
+			if choice(member.Path) {
+				return member
+			}
+		}
+	}
+
+	return members[0]
+}
+
+func isNotTHM(name string) bool {
+	return !strings.EqualFold(path.Ext(name), ".THM")
+}

pkg/dcf/filegroup_internal_test.go

@@ -0,0 +1,230 @@
+package dcf
+
+import (
+	"slices"
+	"testing"
+	"testing/fstest"
+)
+
+// mustFileGroups returns the file groups of directory, and stops the test unless
+// there are exactly count of them.
+func mustFileGroups(t *testing.T, directory *Directory, count int) []*FileGroup {
+	t.Helper()
+
+	if len(directory.FileGroups) != count {
+		t.Fatalf("%s has %d file groups, want %d",
+			directory.Name, len(directory.FileGroups), count)
+	}
+
+	return directory.FileGroups
+}
+
+// assertFileGroup checks the numbers, the primary member and the members of group.
+func assertFileGroup(
+	t *testing.T,
+	group *FileGroup,
+	directoryNumber, fileNumber int,
+	primary string,
+	members ...string,
+) {
+	t.Helper()
+
+	if group.DirectoryNumber != directoryNumber || group.FileNumber != fileNumber {
+		t.Errorf("group numbers = %d and %d, want %d and %d",
+			group.DirectoryNumber, group.FileNumber, directoryNumber, fileNumber)
+	}
+
+	if group.Primary.Path != primary {
+		t.Errorf("Primary = %q, want %q", group.Primary.Path, primary)
+	}
+
+	paths := make([]string, 0, len(group.Members))
+
+	for _, member := range group.Members {
+		paths = append(paths, member.Path)
+	}
+
+	assertPaths(t, "members", paths, members...)
+
+	if got := group.MembersCount(); got != len(members) {
+		t.Errorf("MembersCount() = %d, want %d", got, len(members))
+	}
+}
+
+// assertSizes checks the total size of group and the size of each member.
+func assertSizes(t *testing.T, group *FileGroup, total int64, sizes ...int64) {
+	t.Helper()
+
+	if got := group.TotalSize(); got != total {
+		t.Errorf("TotalSize() = %d, want %d", got, total)
+	}
+
+	got := make([]int64, 0, len(group.Members))
+
+	for _, member := range group.Members {
+		got = append(got, member.Size)
+	}
+
+	if !slices.Equal(got, sizes) {
+		t.Errorf("member sizes = %v, want %v", got, sizes)
+	}
+}
+
+// assertPrimaryMember checks which of the named members primaryMember picks.
+func assertPrimaryMember(t *testing.T, want string, names ...string) {
+	t.Helper()
+
+	members := make([]*Object, 0, len(names))
+
+	for _, name := range names {
+		members = append(members, &Object{Path: name})
+	}
+
+	if got := primaryMember(members).Path; got != want {
+		t.Errorf("primaryMember of %q = %q, want %q", names, got, want)
+	}
+}
+
+func TestParseStoreGroupsFilesSharingAFileNumber(t *testing.T) {
+	t.Parallel()
+
+	store := mustParseStore(t, fstest.MapFS{
+		"DCIM/100ABCDE/DSC00001.ARW": {Data: make([]byte, 2000)},
+		"DCIM/100ABCDE/DSC00001.JPG": {Data: make([]byte, 500)},
+		"DCIM/100ABCDE/MVI_0002.MP4": {Data: make([]byte, 9000)},
+		"DCIM/100ABCDE/MVI_0002.THM": {Data: make([]byte, 100)},
+		"DCIM/100ABCDE/DSC00003.JPG": {Data: make([]byte, 700)},
+		"DCIM/101ABCDE/DSC00001.JPG": {Data: make([]byte, 300)},
+	})
+
+	first := mustFileGroups(t, store.Directories[0], 3)
+	second := mustFileGroups(t, store.Directories[1], 1)
+
+	assertFileGroup(t, first[0], 100, 1, "DCIM/100ABCDE/DSC00001.JPG",
+		"DCIM/100ABCDE/DSC00001.ARW",
+		"DCIM/100ABCDE/DSC00001.JPG",
+	)
+	assertSizes(t, first[0], 2500, 2000, 500)
+
+	assertFileGroup(t, first[1], 100, 2, "DCIM/100ABCDE/MVI_0002.MP4",
+		"DCIM/100ABCDE/MVI_0002.MP4",
+		"DCIM/100ABCDE/MVI_0002.THM",
+	)
+	assertSizes(t, first[1], 9100, 9000, 100)
+
+	assertFileGroup(t, first[2], 100, 3, "DCIM/100ABCDE/DSC00003.JPG",
+		"DCIM/100ABCDE/DSC00003.JPG",
+	)
+	assertSizes(t, first[2], 700, 700)
+
+	assertFileGroup(t, second[0], 101, 1, "DCIM/101ABCDE/DSC00001.JPG",
+		"DCIM/101ABCDE/DSC00001.JPG",
+	)
+	assertSizes(t, second[0], 300, 300)
+
+	// The images, the videos and the totals still count each file on its own.
+	if got := store.ImagesCount(); got != 4 {
+		t.Errorf("ImagesCount() = %d, want 4", got)
+	}
+
+	if got := store.VideosCount(); got != 1 {
+		t.Errorf("VideosCount() = %d, want 1", got)
+	}
+
+	if got := store.TotalSize(); got != 12500 {
+		t.Errorf("TotalSize() = %d, want 12500", got)
+	}
+}
+
+func TestParseStoreGroupsFilesWhoseFirstFourCharactersDiffer(t *testing.T) {
+	t.Parallel()
+
+	store := mustParseStore(t, newCard(
+		"DCIM/100ABCDE/ABCD0004.TIF",
+		"DCIM/100ABCDE/THMB0004.THM",
+	))
+
+	groups := mustFileGroups(t, store.Directories[0], 1)
+
+	assertFileGroup(t, groups[0], 100, 4, "DCIM/100ABCDE/ABCD0004.TIF",
+		"DCIM/100ABCDE/ABCD0004.TIF",
+		"DCIM/100ABCDE/THMB0004.THM",
+	)
+}
+
+func TestParseStoreOrdersFileGroupsByFileNumber(t *testing.T) {
+	t.Parallel()
+
+	store := mustParseStore(t, newCard(
+		"DCIM/100ABCDE/ABCD0002.JPG",
+		"DCIM/100ABCDE/WXYZ0001.JPG",
+	))
+
+	groups := mustFileGroups(t, store.Directories[0], 2)
+
+	assertFileGroup(t, groups[0], 100, 1, "DCIM/100ABCDE/WXYZ0001.JPG",
+		"DCIM/100ABCDE/WXYZ0001.JPG",
+	)
+	assertFileGroup(t, groups[1], 100, 2, "DCIM/100ABCDE/ABCD0002.JPG",
+		"DCIM/100ABCDE/ABCD0002.JPG",
+	)
+}
+
+func TestParseStoreLeavesOtherFilesOutOfFileGroups(t *testing.T) {
+	t.Parallel()
+
+	store := mustParseStore(t, newCard(
+		"DCIM/100ABCDE/ABCD0004.ARW",
+		"DCIM/100ABCDE/ABCD0004.JPG",
+		"DCIM/100ABCDE/PQRS0004.JPG",
+		"DCIM/100ABCDE/README.TXT",
+		"DCIM/MISC/ABCD0005.JPG",
+		"PRIVATE/M4ROOT/CLIP/C0006.MP4",
+	))
+
+	if len(store.Directories) != 1 {
+		t.Fatalf("got %d directories, want 1", len(store.Directories))
+	}
+
+	groups := mustFileGroups(t, store.Directories[0], 1)
+
+	assertFileGroup(t, groups[0], 100, 4, "DCIM/100ABCDE/ABCD0004.ARW",
+		"DCIM/100ABCDE/ABCD0004.ARW",
+	)
+}
+
+func TestParseStoreGivesDirectoryWithoutDCFFilesNoFileGroups(t *testing.T) {
+	t.Parallel()
+
+	store := mustParseStore(t, newCard("DCIM/100ABCDE/README.TXT"))
+
+	mustFileGroups(t, store.Directories[0], 0)
+}
+
+func TestPrimaryMember(t *testing.T) {
+	t.Parallel()
+
+	// A JPG comes first, in either case.
+	assertPrimaryMember(t, "A0010001.JPG", "A0010001.ARW", "A0010001.JPG")
+	assertPrimaryMember(t, "A0010001.JPG", "A0010001.JPG", "A0010001.MOV")
+	assertPrimaryMember(t, "a0010001.jpg", "a0010001.arw", "a0010001.jpg")
+
+	// Then any other image.
+	assertPrimaryMember(t, "Z0010001.ARW", "A0010001.MP4", "Z0010001.ARW")
+	assertPrimaryMember(t, "Z0010001.ARW", "A0010001.WAV", "Z0010001.ARW")
+
+	// Then a video.
+	assertPrimaryMember(t, "A0010001.MP4", "A0010001.AAC", "A0010001.MP4")
+	assertPrimaryMember(t, "Z0010001.MP4", "A0010001.THM", "Z0010001.MP4")
+
+	// Then any other file except a THM, in either case.
+	assertPrimaryMember(t, "Z0010001.TIF", "A0010001.THM", "Z0010001.TIF")
+	assertPrimaryMember(t, "a0010001.tif", "a0010001.thm", "a0010001.tif")
+
+	// A THM is primary only when the group holds nothing else.
+	assertPrimaryMember(t, "A0010001.THM", "A0010001.THM")
+
+	// Among equals, the first in name order wins.
+	assertPrimaryMember(t, "A0010001.THM", "A0010001.THM", "Z0010001.THM")
+	assertPrimaryMember(t, "A0010001.WAV", "A0010001.WAV", "Z0010001.XML")
+}

pkg/dcf/parse.go

@@ -157,6 +157,7 @@ func parseDirectory(fsys fs.FS, store *Store, name string, number int) error {
 		addMedia(store, object)
 	}
 
+	directory.FileGroups = newFileGroups(directory.Objects)
 	store.Directories = append(store.Directories, directory)
 
 	return nil

pkg/dcf/parse_internal_test.go

@@ -113,6 +113,16 @@ func directoryNumbers(store *Store) []int {
 	return numbers
 }
 
+// wantDirectory returns the Directory that parseStore should build for objects.
+func wantDirectory(number int, name string, objects ...*Object) *Directory {
+	return &Directory{
+		Number:     number,
+		Name:       name,
+		Objects:    objects,
+		FileGroups: newFileGroups(objects),
+	}
+}
+
 func assertPaths(t *testing.T, what string, got []string, want ...string) {
 	t.Helper()
 
@@ -172,8 +182,8 @@ func TestParseStorePutsEachFileWhereItBelongs(t *testing.T) {
 	}
 
 	wantDirectories := []*Directory{
-		{Number: 100, Name: "100MSDCF", Objects: []*Object{&arw}},
+		wantDirectory(100, "100MSDCF", &arw),
-		{Number: 101, Name: "101CANON", Objects: []*Object{&mp4, &thm}},
+		wantDirectory(101, "101CANON", &mp4, &thm),
 	}
 	if !reflect.DeepEqual(store.Directories, wantDirectories) {
 		t.Errorf("Directories = %+v, want %+v", store.Directories, wantDirectories)
@@ -305,17 +315,13 @@ func TestParseStoreTreatsLowerCaseNamesAsDCFNames(t *testing.T) {
 
 	store := mustParseStore(t, newCard("DCIM/101abcde/abcd0002.jpg"))
 
-	want := []*Directory{{
+	want := []*Directory{wantDirectory(101, "101abcde", &Object{
-		Number: 101,
-		Name:   "101abcde",
-		Objects: []*Object{{
 		DCFStoreRoot:    testRoot,
 		Path:            "DCIM/101abcde/abcd0002.jpg",
 		Extension:       ".jpg",
 		DirectoryNumber: 101,
 		FileNumber:      2,
-		}},
+	})}
-	}}
 	if !reflect.DeepEqual(store.Directories, want) {
 		t.Errorf("Directories = %+v, want %+v", store.Directories, want)
 	}