diff --git a/README.md b/README.md index 39b33b0..a0d4cc4 100644 --- a/README.md +++ b/README.md @@ -105,30 +105,57 @@ overengineering and reusable code. All the code is in the `pkg/dcf` package. The repo has no commands. -- `dcf.go` holds the public API. `GetDCFStores` finds the DCF stores and lists - the files on them. +- `dcf.go` holds the public API. `GetDCFStores` finds the DCF stores and reads + each one. - `helpers.go` finds the mount points that hold a DCF store. +- `parse.go` reads one DCF store through an `fs.FS` rooted at its mount point. The main types are: - `Store` is one DCF store on one mounted filesystem. -- `Object` is one file in a store. +- `Directory` is one DCF directory. It has a directory number, a name and the + objects inside it. +- `Object` is one file in a store. A file in a DCF directory also has a + directory number and a file number. - `Image` and `Video` embed `Object`. The file extension decides which one a - file becomes. + file becomes. A file with any other extension, such as a `THM` thumbnail, is + neither. A mounted filesystem is a DCF store when its root holds a `DCIM` or -`PRIVATE/M4ROOT` directory. `dcf` never mounts a filesystem. +`PRIVATE/M4ROOT` directory. It needs only one of them. `dcf` never mounts a +filesystem. Media anywhere else on the filesystem is not part of the store. + +`dcf` never follows a symlink. + +`dcf` applies the naming rules of the DCF specification, CIPA DC-009-2010, under +`DCIM`: + +- A DCF directory sits directly under `DCIM`. Its name is a directory number + from 100 to 999, then five characters, such as `100MSDCF`. +- A DCF file sits directly in a DCF directory. Its name is four characters, then + a file number from 0001 to 9999, then a dot and a three-character extension, + such as `DSC00001.ARW`. +- The characters allowed are `A` to `Z`, `0` to `9` and `_`. Lower-case letters + count as upper-case. +- Directories that share a directory number are not DCF directories. +- `JPG` files that share a file number in one directory are not DCF files. + +`Store.NonConforming` lists the path of everything else under `DCIM`. It +includes everything inside a directory that breaks the rules. It also includes +every symlink, whatever its name. These entries have no DCF numbers, and they +are not counted as images or videos. + +The naming rules do not apply under `PRIVATE/M4ROOT`. The images and videos +there are in `Store.Images` and `Store.Videos`, and their DCF numbers are 0. +`dcf` skips any symlink there. ## TODO - Handle DCF file groups, where multiple objects share the same index number. `dcf` does not handle these yet. If you need this, send me a link to a zip or tar of a memory card that uses them, and I'll see what I can do. -- Parse the DCF directory and file names, rather than listing every media file - on the filesystem. -- Read stores through `fs.FS`, with a `context.Context` to cancel a walk. +- Take a `context.Context` to cancel the walk of a store. - Return a plain slice from `GetDCFStores`, not a pointer to one. -- Test the walk of a store. It only runs against real mounted media today. - Copy files off a card, and verify each copy against its SHA256 hash. ## License diff --git a/pkg/dcf/dcf.go b/pkg/dcf/dcf.go index 94484ee..69d5917 100644 --- a/pkg/dcf/dcf.go +++ b/pkg/dcf/dcf.go @@ -1,6 +1,8 @@ // Package dcf reads DCF cards: memory cards laid out to the "Design rule for // Camera File system" specification (JEITA CP-3461). It finds the cards among -// the mounted filesystems and lists the images and videos on each one. +// the mounted filesystems and parses the DCF directories and files on each one +// into their directory numbers and file numbers. It lists the images and videos +// on each card, and everything under DCIM that is not a DCF directory or file. package dcf import ( @@ -20,8 +22,30 @@ import ( // images and videos in the root of a mounted filesystem. type Store struct { RootDirectory string - Images []*Image - Videos []*Video + // Directories holds the DCF directories under DCIM. + Directories []*Directory + // Images and Videos hold the images and videos among the objects in + // Directories, and those under PRIVATE/M4ROOT. + Images []*Image + Videos []*Video + // NonConforming holds the path, relative to RootDirectory, of everything + // under DCIM that is not a DCF directory or a DCF file. It includes every + // symlink, and everything inside a directory that breaks the DCF naming + // rules. A directory whose name is not valid UTF-8 is listed without what is + // inside it, because fs.FS refuses to open one. + NonConforming []string +} + +// Directory represents a DCF directory: a directory directly under DCIM whose +// name follows the DCF naming rules. +type Directory struct { + // Number is the directory number, from 100 to 999. + Number int + // Name is the name of the directory as it appears on the card. + Name string + // Objects holds every DCF file in the directory, including those that are + // neither images nor videos. + Objects []*Object } // Object represents a file in a DCF store. It is embedded in Image and Video @@ -31,6 +55,10 @@ type Object struct { Path string Size int64 Extension string + // DirectoryNumber and FileNumber are the DCF numbers of a file in a DCF + // directory. Both are 0 for a file under PRIVATE/M4ROOT. + DirectoryNumber int + FileNumber int } // Image represents an image file on a DCF store. @@ -140,58 +168,37 @@ func pathToSHA256(path string) (string, error) { // any mounted filesystems. The integer argument is the number of DCF stores to // return. If the argument is 0, all DCF stores are returned. It does not mount // filesystems or search outside of filesystem root directories. It does, -// however, walk the filesystems to find images and videos and populate the -// returned Stores. +// however, read the DCIM and PRIVATE/M4ROOT directories of each filesystem to +// populate the returned Stores. It never follows a symlink. If a card cannot be +// read, it returns an error that names the card's mount point. func GetDCFStores(requestedCount int) (*[]Store, error) { dcfStorePaths, err := findDCFMountPoints(requestedCount) if err != nil { return nil, err } + dcfStores, err := parseStores(dcfStorePaths) + if err != nil { + return nil, err + } + + return &dcfStores, nil +} + +// parseStores reads the card at each of the given mount points. +func parseStores(mountpoints []string) ([]Store, error) { dcfStores := []Store{} - for _, dcfStorePath := range dcfStorePaths { - dcfStore := Store{ - RootDirectory: dcfStorePath, - Images: make([]*Image, 0), - Videos: make([]*Video, 0), - } - - addFile := func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if !info.IsDir() && isImageFile(path) { - image := Image{} - image.Path = strings.TrimPrefix(path, dcfStorePath+"/") - image.Size = info.Size() - image.Extension = filepath.Ext(path) - image.DCFStoreRoot = dcfStorePath - dcfStore.Images = append(dcfStore.Images, &image) - } - - if !info.IsDir() && isVideoFile(path) { - video := Video{} - video.Path = strings.TrimPrefix(path, dcfStorePath+"/") - video.Size = info.Size() - video.Extension = filepath.Ext(path) - video.DCFStoreRoot = dcfStorePath - dcfStore.Videos = append(dcfStore.Videos, &video) - } - - return nil - } - - err := filepath.Walk(dcfStorePath, addFile) + for _, mountpoint := range mountpoints { + dcfStore, err := parseStore(os.DirFS(mountpoint), mountpoint) if err != nil { return nil, err } - dcfStores = append(dcfStores, dcfStore) + dcfStores = append(dcfStores, *dcfStore) } - return &dcfStores, nil + return dcfStores, nil } func isImageFile(path string) bool { diff --git a/pkg/dcf/dcf_internal_test.go b/pkg/dcf/dcf_internal_test.go index 46cead7..8d91c6e 100644 --- a/pkg/dcf/dcf_internal_test.go +++ b/pkg/dcf/dcf_internal_test.go @@ -1,6 +1,60 @@ package dcf -import "testing" +import ( + "path/filepath" + "strings" + "testing" +) + +func TestParseStoresReadsEachMountPoint(t *testing.T) { + t.Parallel() + + first := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG") + second := newCardOnDisk(t, "PRIVATE/M4ROOT/CLIP/C0001.MP4") + + stores, err := parseStores([]string{first, second}) + if err != nil { + t.Fatalf("parseStores: %v", err) + } + + if len(stores) != 2 { + t.Fatalf("parseStores returned %d stores, want 2", len(stores)) + } + + cases := map[string]Object{ + filepath.Join(first, "DCIM/100ABCDE/ABCD0001.JPG"): stores[0].Images[0].Object, + filepath.Join(second, "PRIVATE/M4ROOT/CLIP/C0001.MP4"): stores[1].Videos[0].Object, + } + + for want, object := range cases { + if got := object.FullFilePath(); got != want { + t.Errorf("FullFilePath() = %q, want %q", got, want) + } + + got, err := object.Hash() + if err != nil { + t.Fatalf("Hash() returned error: %v", err) + } + + // The published SHA-256 test vector for "abc". + wantHash := "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + if got != wantHash { + t.Errorf("Hash() = %q, want %q", got, wantHash) + } + } +} + +func TestParseStoresReturnsErrorNamingTheCard(t *testing.T) { + t.Parallel() + + // Every lookup under a plain file fails. + mountpoint := filepath.Join(newCardOnDisk(t, "FILE"), "FILE") + + _, err := parseStores([]string{mountpoint}) + if err == nil || !strings.Contains(err.Error(), mountpoint) { + t.Fatalf("parseStores error = %v, want one naming %s", err, mountpoint) + } +} func TestIsImageFile(t *testing.T) { t.Parallel() diff --git a/pkg/dcf/helpers.go b/pkg/dcf/helpers.go index 5725697..35558de 100644 --- a/pkg/dcf/helpers.go +++ b/pkg/dcf/helpers.go @@ -4,7 +4,6 @@ import ( "fmt" "log/slog" "os" - "path/filepath" "slices" "github.com/shirou/gopsutil/disk" @@ -63,12 +62,12 @@ func filterDCFMountPoints(mountpoints []string, requestedCount int) []string { // isDCFMountPoint reports whether the root of mountpoint holds a DCIM or // PRIVATE/M4ROOT directory, which is how we detect DCF stores. func isDCFMountPoint(mountpoint string) bool { - for _, directory := range []string{"DCIM", "PRIVATE/M4ROOT"} { - path := filepath.Join(mountpoint, directory) + fsys := os.DirFS(mountpoint) - info, err := os.Stat(path) - if err == nil && info.IsDir() { - slog.Debug(fmt.Sprintf("Found %s directory at %s\n", directory, path)) + for _, directory := range []string{dcimDirectory, m4RootDirectory} { + found, err := hasDirectory(fsys, directory) + if err == nil && found { + slog.Debug(fmt.Sprintf("Found %s directory at %s\n", directory, mountpoint)) return true } diff --git a/pkg/dcf/helpers_internal_test.go b/pkg/dcf/helpers_internal_test.go index 1d6bef9..7670e3d 100644 --- a/pkg/dcf/helpers_internal_test.go +++ b/pkg/dcf/helpers_internal_test.go @@ -57,6 +57,21 @@ func TestIsDCFMountPointIgnoresPlainFile(t *testing.T) { } } +func TestIsDCFMountPointIgnoresSymlink(t *testing.T) { + t.Parallel() + + outside := newMountPoint(t, "M4ROOT") + + for _, link := range []string{dcimDirectory, privateDirectory, m4RootDirectory} { + root := newMountPoint(t, filepath.Dir(link)) + addSymlink(t, root, link, outside) + + if isDCFMountPoint(root) { + t.Errorf("isDCFMountPoint counted a symlink at %s as a directory", link) + } + } +} + func TestFilterDCFMountPointsReturnsRequestedCount(t *testing.T) { t.Parallel() diff --git a/pkg/dcf/parse.go b/pkg/dcf/parse.go new file mode 100644 index 0000000..08c6b38 --- /dev/null +++ b/pkg/dcf/parse.go @@ -0,0 +1,275 @@ +package dcf + +import ( + "errors" + "fmt" + "io/fs" + "path" + "regexp" + "strconv" + "strings" +) + +// The two directories at the root of a card that hold its files. +const ( + dcimDirectory = "DCIM" + m4RootDirectory = "PRIVATE/M4ROOT" +) + +// The naming rules of the DCF specification, CIPA DC-009-2010, sections 2.3, +// 4.2.2 and 4.3.1. Names use A-Z, 0-9 and "_", and lower-case letters count as +// upper-case. Each pattern captures the number in the name. +var ( + dcfDirectoryName = regexp.MustCompile(`^([1-9][0-9]{2})[0-9A-Za-z_]{5}$`) + dcfFileName = regexp.MustCompile(`^[0-9A-Za-z_]{4}([0-9]{4})\.[0-9A-Za-z_]{3}$`) +) + +// parseStore reads one card from fsys, which is rooted at the card's mount +// point. The root argument is the path of that mount point. Only DCIM and +// PRIVATE/M4ROOT are read. A card can lack either one. A symlink is never +// followed. An error names the mount point, because fsys reports only paths +// inside it. +func parseStore(fsys fs.FS, root string) (*Store, error) { + store := &Store{RootDirectory: root} + + err := parseDCIM(fsys, store) + if err == nil { + err = parseM4Root(fsys, store) + } + + if err != nil { + return nil, fmt.Errorf("reading the card at %s: %w", root, err) + } + + return store, nil +} + +// hasDirectory reports whether the card holds the named directory. A plain file +// or a symlink at any step of the path counts as missing. Any other failure to +// look is an error. +func hasDirectory(fsys fs.FS, name string) (bool, error) { + parent := path.Dir(name) + if parent != "." { + found, err := hasDirectory(fsys, parent) + if err != nil || !found { + return false, err + } + } + + info, err := fs.Lstat(fsys, name) + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + + if err != nil { + return false, err + } + + return info.IsDir(), nil +} + +// parseDCIM adds the DCF directories under DCIM to store. Everything else there +// is not conforming. That includes directories that share a directory number, +// which section 7.1.2 of the specification treats as non-DCF directories. +func parseDCIM(fsys fs.FS, store *Store) error { + found, err := hasDirectory(fsys, dcimDirectory) + if err != nil || !found { + return err + } + + entries, err := fs.ReadDir(fsys, dcimDirectory) + if err != nil { + return err + } + + directoriesWithNumber := map[int]int{} + + for _, entry := range entries { + if entry.IsDir() { + directoriesWithNumber[parseNumber(dcfDirectoryName, entry.Name())]++ + } + } + + for _, entry := range entries { + number := parseNumber(dcfDirectoryName, entry.Name()) + + if entry.IsDir() && number != 0 && directoriesWithNumber[number] == 1 { + err = parseDirectory(fsys, store, entry.Name(), number) + } else { + err = addNonConforming(fsys, store, dcimDirectory, entry) + } + + if err != nil { + return err + } + } + + return nil +} + +// parseDirectory adds one DCF directory and the DCF files directly inside it to +// store. A DCF file is a regular file, so a symlink with a DCF name is not one. +// Everything else there is not conforming. That includes JPG files that share a +// file number, which section 7.2.2 of the specification says must not show DCF +// numbers. Other files that share a file number do conform. +func parseDirectory(fsys fs.FS, store *Store, name string, number int) error { + directoryPath := path.Join(dcimDirectory, name) + + entries, err := fs.ReadDir(fsys, directoryPath) + if err != nil { + return err + } + + jpgsWithNumber := map[int]int{} + + for _, entry := range entries { + if entry.Type().IsRegular() && isJPG(entry.Name()) { + jpgsWithNumber[parseNumber(dcfFileName, entry.Name())]++ + } + } + + directory := &Directory{Number: number, Name: name} + + for _, entry := range entries { + fileNumber := parseNumber(dcfFileName, entry.Name()) + sharedJPG := isJPG(entry.Name()) && jpgsWithNumber[fileNumber] > 1 + + if !entry.Type().IsRegular() || fileNumber == 0 || sharedJPG { + err = addNonConforming(fsys, store, directoryPath, entry) + if err != nil { + return err + } + + continue + } + + entryPath := path.Join(directoryPath, entry.Name()) + + object, err := newObject(store, entryPath, entry) + if err != nil { + return err + } + + object.DirectoryNumber = number + object.FileNumber = fileNumber + directory.Objects = append(directory.Objects, &object) + + addMedia(store, object) + } + + store.Directories = append(store.Directories, directory) + + return nil +} + +// parseM4Root adds the images and videos under PRIVATE/M4ROOT to store. The DCF +// naming rules do not apply there, so these files have no DCF numbers. It skips +// a symlink. It also skips a name that is not valid UTF-8, because fs.FS refuses +// to open one. +func parseM4Root(fsys fs.FS, store *Store) error { + found, err := hasDirectory(fsys, m4RootDirectory) + if err != nil || !found { + return err + } + + return fs.WalkDir(fsys, m4RootDirectory, + func(filePath string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + + validName := fs.ValidPath(filePath) + + if entry.IsDir() && !validName { + return fs.SkipDir + } + + if !entry.Type().IsRegular() || !validName { + return nil + } + + object, err := newObject(store, filePath, entry) + if err != nil { + return err + } + + addMedia(store, object) + + return nil + }) +} + +// addNonConforming lists one entry of the directory dir on store as not +// conforming. If the entry is a directory, it lists everything inside it too. It +// does not follow a symlink. It does not read a directory whose name is not +// valid UTF-8, because fs.FS refuses to open one. +func addNonConforming(fsys fs.FS, store *Store, dir string, entry fs.DirEntry) error { + entryPath := path.Join(dir, entry.Name()) + store.NonConforming = append(store.NonConforming, entryPath) + + if !entry.IsDir() || !fs.ValidPath(entryPath) { + return nil + } + + entries, err := fs.ReadDir(fsys, entryPath) + if err != nil { + return err + } + + for _, inner := range entries { + err = addNonConforming(fsys, store, entryPath, inner) + if err != nil { + return err + } + } + + return nil +} + +// newObject returns the Object for the file at filePath, with no DCF numbers. +func newObject(store *Store, filePath string, entry fs.DirEntry) (Object, error) { + info, err := entry.Info() + if err != nil { + return Object{}, err + } + + return Object{ + DCFStoreRoot: store.RootDirectory, + Path: filePath, + Size: info.Size(), + Extension: path.Ext(filePath), + }, nil +} + +// addMedia adds object to the images or videos of store. A file with any other +// extension is neither, and is not added. +func addMedia(store *Store, object Object) { + if isImageFile(object.Path) { + store.Images = append(store.Images, &Image{Object: object}) + } + + if isVideoFile(object.Path) { + store.Videos = append(store.Videos, &Video{Object: object}) + } +} + +// parseNumber returns the number that pattern captures in name. It returns 0 +// when name does not match. No DCF number is 0, so this also rejects the file +// number 0000, which the specification forbids. +func parseNumber(pattern *regexp.Regexp, name string) int { + match := pattern.FindStringSubmatch(name) + if match == nil { + return 0 + } + + number, err := strconv.Atoi(match[1]) + if err != nil { + return 0 + } + + return number +} + +func isJPG(name string) bool { + return strings.EqualFold(path.Ext(name), ".JPG") +} diff --git a/pkg/dcf/parse_internal_test.go b/pkg/dcf/parse_internal_test.go new file mode 100644 index 0000000..633dd79 --- /dev/null +++ b/pkg/dcf/parse_internal_test.go @@ -0,0 +1,809 @@ +package dcf + +import ( + "errors" + "io/fs" + "os" + "path" + "path/filepath" + "reflect" + "slices" + "testing" + "testing/fstest" +) + +const ( + testRoot = "/mnt/card" + privateDirectory = "PRIVATE" +) + +// newCard returns a card holding an empty file at each of the given paths. +func newCard(paths ...string) fstest.MapFS { + card := fstest.MapFS{} + + for _, filePath := range paths { + card[filePath] = &fstest.MapFile{} + } + + return card +} + +// newCardOnDisk returns a temporary directory holding the three bytes "abc" in a +// file at each of the given paths. It is for tests that fstest.MapFS cannot do. +func newCardOnDisk(t *testing.T, paths ...string) string { + t.Helper() + + root := t.TempDir() + + for _, filePath := range paths { + fullPath := filepath.Join(root, filePath) + + err := os.MkdirAll(filepath.Dir(fullPath), 0o750) + if err != nil { + t.Fatalf("MkdirAll: %v", err) + } + + err = os.WriteFile(fullPath, []byte("abc"), 0o600) + if err != nil { + t.Fatalf("WriteFile: %v", err) + } + } + + return root +} + +// addSymlink creates a symlink to target at the named path under root. +func addSymlink(t *testing.T, root, name, target string) { + t.Helper() + + err := os.Symlink(target, filepath.Join(root, name)) + if err != nil { + t.Fatalf("Symlink: %v", err) + } +} + +func mustParseStore(t *testing.T, fsys fs.FS) *Store { + t.Helper() + + store, err := parseStore(fsys, testRoot) + if err != nil { + t.Fatalf("parseStore: %v", err) + } + + return store +} + +func imagePaths(store *Store) []string { + paths := make([]string, 0, len(store.Images)) + + for _, image := range store.Images { + paths = append(paths, image.Path) + } + + return paths +} + +func videoPaths(store *Store) []string { + paths := make([]string, 0, len(store.Videos)) + + for _, video := range store.Videos { + paths = append(paths, video.Path) + } + + return paths +} + +func objectPaths(directory *Directory) []string { + paths := make([]string, 0, len(directory.Objects)) + + for _, object := range directory.Objects { + paths = append(paths, object.Path) + } + + return paths +} + +func directoryNumbers(store *Store) []int { + numbers := make([]int, 0, len(store.Directories)) + + for _, directory := range store.Directories { + numbers = append(numbers, directory.Number) + } + + return numbers +} + +func assertPaths(t *testing.T, what string, got []string, want ...string) { + t.Helper() + + if !slices.Equal(got, want) { + t.Errorf("%s = %q, want %q", what, got, want) + } +} + +func TestParseStorePutsEachFileWhereItBelongs(t *testing.T) { + t.Parallel() + + card := fstest.MapFS{ + "DCIM/100MSDCF/ABCD0001.ARW": {Data: make([]byte, 2000)}, + "DCIM/100MSDCF/readme.txt": {Data: make([]byte, 7)}, + "DCIM/101CANON/MVI_0002.MP4": {Data: make([]byte, 9000)}, + "DCIM/101CANON/MVI_0002.THM": {Data: make([]byte, 100)}, + "DCIM/MISC/DSC00003.JPG": {Data: make([]byte, 7)}, + "DCIM/DSC00004.JPG": {Data: make([]byte, 7)}, + "PRIVATE/M4ROOT/CLIP/C0002.MP4": {Data: make([]byte, 400)}, + "PRIVATE/M4ROOT/CLIP/C0002M01.XML": {Data: make([]byte, 7)}, + "PRIVATE/OTHER/C0003.MP4": {Data: make([]byte, 7)}, + "BACKUP/DSC00005.JPG": {Data: make([]byte, 7)}, + "DSC00006.JPG": {Data: make([]byte, 7)}, + } + + store := mustParseStore(t, card) + + arw := Object{ + DCFStoreRoot: testRoot, + Path: "DCIM/100MSDCF/ABCD0001.ARW", + Size: 2000, + Extension: ".ARW", + DirectoryNumber: 100, + FileNumber: 1, + } + mp4 := Object{ + DCFStoreRoot: testRoot, + Path: "DCIM/101CANON/MVI_0002.MP4", + Size: 9000, + Extension: ".MP4", + DirectoryNumber: 101, + FileNumber: 2, + } + thm := Object{ + DCFStoreRoot: testRoot, + Path: "DCIM/101CANON/MVI_0002.THM", + Size: 100, + Extension: ".THM", + DirectoryNumber: 101, + FileNumber: 2, + } + clip := Object{ + DCFStoreRoot: testRoot, + Path: "PRIVATE/M4ROOT/CLIP/C0002.MP4", + Size: 400, + Extension: ".MP4", + } + + wantDirectories := []*Directory{ + {Number: 100, Name: "100MSDCF", Objects: []*Object{&arw}}, + {Number: 101, Name: "101CANON", Objects: []*Object{&mp4, &thm}}, + } + if !reflect.DeepEqual(store.Directories, wantDirectories) { + t.Errorf("Directories = %+v, want %+v", store.Directories, wantDirectories) + } + + wantImages := []*Image{{Object: arw}} + if !reflect.DeepEqual(store.Images, wantImages) { + t.Errorf("Images = %+v, want %+v", store.Images, wantImages) + } + + wantVideos := []*Video{{Object: mp4}, {Object: clip}} + if !reflect.DeepEqual(store.Videos, wantVideos) { + t.Errorf("Videos = %+v, want %+v", store.Videos, wantVideos) + } + + assertPaths(t, "NonConforming", store.NonConforming, + "DCIM/100MSDCF/readme.txt", + "DCIM/DSC00004.JPG", + "DCIM/MISC", + "DCIM/MISC/DSC00003.JPG", + ) +} + +func TestParseStoreCountsOnlyImagesAndVideos(t *testing.T) { + t.Parallel() + + card := fstest.MapFS{ + "DCIM/100ABCDE/ABCD0001.ARW": {Data: make([]byte, 2000)}, + "DCIM/100ABCDE/ABCD0002.THM": {Data: make([]byte, 100)}, + "DCIM/100ABCDE/ABCD0003.MP4": {Data: make([]byte, 9000)}, + "DCIM/MISC/ABCD0004.JPG": {Data: make([]byte, 50)}, + "PRIVATE/M4ROOT/CLIP/C0004.MP4": {Data: make([]byte, 400)}, + "ABCD0005.JPG": {Data: make([]byte, 30)}, + } + + store := mustParseStore(t, card) + + if got := store.ImagesCount(); got != 1 { + t.Errorf("ImagesCount() = %d, want 1", got) + } + + if got := store.VideosCount(); got != 2 { + t.Errorf("VideosCount() = %d, want 2", got) + } + + if got := store.TotalSize(); got != 11400 { + t.Errorf("TotalSize() = %d, want 11400", got) + } +} + +func TestParseStoreSetsFullFilePath(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, newCard("DCIM/100MSDCF/DSC00001.JPG")) + + want := testRoot + "/DCIM/100MSDCF/DSC00001.JPG" + if got := store.Images[0].FullFilePath(); got != want { + t.Fatalf("FullFilePath() = %q, want %q", got, want) + } +} + +func TestParseNumberOfDirectoryName(t *testing.T) { + t.Parallel() + + cases := map[string]int{ + "100MSDCF": 100, + "999VWXYZ": 999, + "100D3500": 100, + "101abcde": 101, + "102A_B_C": 102, + "099ABCDE": 0, + "000ABCDE": 0, + "103A": 0, + "IM104ABC": 0, + "105_abcdefg": 0, + "107@-%+]": 0, + "IM08ABCD": 0, + "100ABC-E": 0, + "100ABCD": 0, + "100ABCDEF": 0, + "1000ABCDE": 0, + "X100ABCDE": 0, + "TEMP": 0, + } + + for name, want := range cases { + if got := parseNumber(dcfDirectoryName, name); got != want { + t.Errorf("directory %q parsed to %d, want %d", name, got, want) + } + } +} + +func TestParseNumberOfFileName(t *testing.T) { + t.Parallel() + + cases := map[string]int{ + "DSC00001.ARW": 1, + "ABCD9999.JPG": 9999, + "abcd0002.jpg": 2, + "_ABC0001.JPG": 1, + "IMG_0001.JPG": 1, + "12340005.JPG": 5, + "ABCD0000.JPG": 0, + "A0004.JPG": 0, + "ABCDE0005.JPG": 0, + "ABCD10000.JPG": 0, + "abcdefg0006.JPG": 0, + "+-@]0007.JPG": 0, + "ABCD00~1.JPG": 0, + "ABCDEFGH.JPG": 0, + "ABCD0001": 0, + "ABCD0001.": 0, + "ABCD0001.JP": 0, + "ABCD0001.JPEG": 0, + "ABCD0001.J-G": 0, + "._DSC0001.JPG": 0, + ".DS_Store": 0, + } + + for name, want := range cases { + if got := parseNumber(dcfFileName, name); got != want { + t.Errorf("file %q parsed to %d, want %d", name, got, want) + } + } +} + +func TestParseStoreTreatsLowerCaseNamesAsDCFNames(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, newCard("DCIM/101abcde/abcd0002.jpg")) + + want := []*Directory{{ + Number: 101, + Name: "101abcde", + Objects: []*Object{{ + DCFStoreRoot: testRoot, + Path: "DCIM/101abcde/abcd0002.jpg", + Extension: ".jpg", + DirectoryNumber: 101, + FileNumber: 2, + }}, + }} + if !reflect.DeepEqual(store.Directories, want) { + t.Errorf("Directories = %+v, want %+v", store.Directories, want) + } + + assertPaths(t, "Images", imagePaths(store), "DCIM/101abcde/abcd0002.jpg") + assertPaths(t, "NonConforming", store.NonConforming) +} + +func TestParseStoreAcceptsUnderscoreInNames(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, newCard("DCIM/100A_B_C/_ABC0001.JPG")) + + assertPaths(t, "Images", imagePaths(store), "DCIM/100A_B_C/_ABC0001.JPG") + assertPaths(t, "NonConforming", store.NonConforming) +} + +func TestParseStoreRejectsDirectoriesSharingANumber(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, newCard( + "DCIM/102ABCDE/ABCD0001.JPG", + "DCIM/102PQRST/ABCD0002.JPG", + "DCIM/103ABCDE/ABCD0003.JPG", + )) + + if got, want := directoryNumbers(store), []int{103}; !slices.Equal(got, want) { + t.Errorf("directory numbers = %v, want %v", got, want) + } + + assertPaths(t, "Images", imagePaths(store), "DCIM/103ABCDE/ABCD0003.JPG") + assertPaths(t, "NonConforming", store.NonConforming, + "DCIM/102ABCDE", + "DCIM/102ABCDE/ABCD0001.JPG", + "DCIM/102PQRST", + "DCIM/102PQRST/ABCD0002.JPG", + ) +} + +func TestParseStoreDoesNotCountPlainFileAsSharingADirectoryNumber(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, newCard( + "DCIM/100ABCDE", + "DCIM/100PQRST/ABCD0001.JPG", + )) + + if got, want := directoryNumbers(store), []int{100}; !slices.Equal(got, want) { + t.Errorf("directory numbers = %v, want %v", got, want) + } + + assertPaths(t, "NonConforming", store.NonConforming, "DCIM/100ABCDE") +} + +func TestParseStoreRejectsJPGsSharingAFileNumber(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, newCard( + "DCIM/100ABCDE/ABCD0003.JPG", + "DCIM/100ABCDE/PQRS0003.jpg", + "DCIM/101ABCDE/ABCD0003.JPG", + )) + + assertPaths(t, "objects", objectPaths(store.Directories[0])) + assertPaths(t, "objects", objectPaths(store.Directories[1]), + "DCIM/101ABCDE/ABCD0003.JPG", + ) + assertPaths(t, "Images", imagePaths(store), "DCIM/101ABCDE/ABCD0003.JPG") + assertPaths(t, "NonConforming", store.NonConforming, + "DCIM/100ABCDE/ABCD0003.JPG", + "DCIM/100ABCDE/PQRS0003.jpg", + ) +} + +func TestParseStoreKeepsOtherFilesSharingAFileNumber(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, newCard( + "DCIM/100ABCDE/DSC00004.ARW", + "DCIM/100ABCDE/DSC00004.JPG", + "DCIM/100ABCDE/MVI_0005.MP4", + "DCIM/100ABCDE/MVI_0005.THM", + )) + + assertPaths(t, "objects", objectPaths(store.Directories[0]), + "DCIM/100ABCDE/DSC00004.ARW", + "DCIM/100ABCDE/DSC00004.JPG", + "DCIM/100ABCDE/MVI_0005.MP4", + "DCIM/100ABCDE/MVI_0005.THM", + ) + assertPaths(t, "Images", imagePaths(store), + "DCIM/100ABCDE/DSC00004.ARW", + "DCIM/100ABCDE/DSC00004.JPG", + ) + assertPaths(t, "Videos", videoPaths(store), "DCIM/100ABCDE/MVI_0005.MP4") + assertPaths(t, "NonConforming", store.NonConforming) +} + +func TestParseStoreKeepsARWSharingANumberWithTwoJPGs(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, newCard( + "DCIM/100ABCDE/ABCD0004.ARW", + "DCIM/100ABCDE/ABCD0004.JPG", + "DCIM/100ABCDE/PQRS0004.JPG", + )) + + assertPaths(t, "objects", objectPaths(store.Directories[0]), + "DCIM/100ABCDE/ABCD0004.ARW", + ) + assertPaths(t, "NonConforming", store.NonConforming, + "DCIM/100ABCDE/ABCD0004.JPG", + "DCIM/100ABCDE/PQRS0004.JPG", + ) +} + +func TestParseStoreDoesNotCountDirectoryAsSharingAFileNumber(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, newCard( + "DCIM/100ABCDE/ABCD0003.JPG", + "DCIM/100ABCDE/PQRS0003.JPG/README.TXT", + )) + + assertPaths(t, "objects", objectPaths(store.Directories[0]), + "DCIM/100ABCDE/ABCD0003.JPG", + ) + assertPaths(t, "NonConforming", store.NonConforming, + "DCIM/100ABCDE/PQRS0003.JPG", + "DCIM/100ABCDE/PQRS0003.JPG/README.TXT", + ) +} + +func TestParseStoreDoesNotCountDirectoryNamedLikeAVideo(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, newCard("PRIVATE/M4ROOT/CLIP.MP4/C0001.XML")) + + assertPaths(t, "Videos", videoPaths(store)) +} + +func TestParseStoreRejectsSubdirectoryOfDCFDirectory(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, newCard( + "DCIM/100ABCDE/SUB/ABCD0001.JPG", + "DCIM/100ABCDE/ABCD0002.JPG/ABCD0003.JPG", + )) + + assertPaths(t, "objects", objectPaths(store.Directories[0])) + assertPaths(t, "Images", imagePaths(store)) + assertPaths(t, "NonConforming", store.NonConforming, + "DCIM/100ABCDE/ABCD0002.JPG", + "DCIM/100ABCDE/ABCD0002.JPG/ABCD0003.JPG", + "DCIM/100ABCDE/SUB", + "DCIM/100ABCDE/SUB/ABCD0001.JPG", + ) +} + +func TestParseStoreListsEmptyDirectories(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, fstest.MapFS{ + "DCIM/100EMPTY": {Mode: fs.ModeDir}, + "DCIM/MISC": {Mode: fs.ModeDir}, + }) + + if got, want := directoryNumbers(store), []int{100}; !slices.Equal(got, want) { + t.Errorf("directory numbers = %v, want %v", got, want) + } + + assertPaths(t, "NonConforming", store.NonConforming, "DCIM/MISC") +} + +func TestParseStoreWithoutDCIM(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, newCard("PRIVATE/M4ROOT/CLIP/C0001.MP4")) + + if len(store.Directories) != 0 { + t.Errorf("Directories = %+v, want none", store.Directories) + } + + assertPaths(t, "Videos", videoPaths(store), "PRIVATE/M4ROOT/CLIP/C0001.MP4") +} + +func TestParseStoreWithoutM4Root(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, newCard("DCIM/100ABCDE/ABCD0001.JPG")) + + assertPaths(t, "Images", imagePaths(store), "DCIM/100ABCDE/ABCD0001.JPG") + assertPaths(t, "Videos", videoPaths(store)) +} + +func TestParseStoreTreatsPlainFilesAsMissingDirectories(t *testing.T) { + t.Parallel() + + store := mustParseStore(t, newCard("DCIM", "PRIVATE/M4ROOT")) + + assertPaths(t, "Images", imagePaths(store)) + assertPaths(t, "Videos", videoPaths(store)) + assertPaths(t, "NonConforming", store.NonConforming) +} + +// A real filesystem reports a lookup under a plain file differently from a +// missing path, which fstest.MapFS does not reproduce. +func TestParseStoreReadsRealFilesystemWithPlainFileNamedPrivate(t *testing.T) { + t.Parallel() + + root := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG", "PRIVATE") + + store := mustParseStore(t, os.DirFS(root)) + + assertPaths(t, "Images", imagePaths(store), "DCIM/100ABCDE/ABCD0001.JPG") + + if got := store.TotalImageSize(); got != 3 { + t.Errorf("TotalImageSize() = %d, want 3", got) + } +} + +// A real filesystem can hold a name with the byte 0xff, which is not valid UTF-8. +// fs.FS refuses to open such a name. +func TestParseStoreListsNamesThatFSRefuses(t *testing.T) { + t.Parallel() + + root := newCardOnDisk(t, + "DCIM/100ABCDE/ABCD0001.JPG", + "DCIM/100ABCDE/A\xffDIR/INNER", + "DCIM/100ABCDE/A\xffFILE", + "DCIM/A\xffDIR/INNER", + "DCIM/A\xffFILE", + "DCIM/MISC/A\xffDIR/INNER", + "DCIM/MISC/A\xffFILE", + ) + + store := mustParseStore(t, os.DirFS(root)) + + assertPaths(t, "Images", imagePaths(store), "DCIM/100ABCDE/ABCD0001.JPG") + assertPaths(t, "NonConforming", store.NonConforming, + "DCIM/100ABCDE/A\xffDIR", + "DCIM/100ABCDE/A\xffFILE", + "DCIM/A\xffDIR", + "DCIM/A\xffFILE", + "DCIM/MISC", + "DCIM/MISC/A\xffDIR", + "DCIM/MISC/A\xffFILE", + ) +} + +func TestParseStoreListsDanglingSymlinks(t *testing.T) { + t.Parallel() + + root := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG", "DCIM/MISC/README.TXT") + + addSymlink(t, root, "DCIM/100ABCDE/ABCD0002.JPG", "absent") + addSymlink(t, root, "DCIM/100ABCDE/LINK", "absent") + addSymlink(t, root, "DCIM/101ABCDE", "absent") + addSymlink(t, root, "DCIM/LINK", "absent") + addSymlink(t, root, "DCIM/MISC/LINK", "absent") + + store := mustParseStore(t, os.DirFS(root)) + + if got, want := directoryNumbers(store), []int{100}; !slices.Equal(got, want) { + t.Errorf("directory numbers = %v, want %v", got, want) + } + + assertPaths(t, "Images", imagePaths(store), "DCIM/100ABCDE/ABCD0001.JPG") + assertPaths(t, "NonConforming", store.NonConforming, + "DCIM/100ABCDE/ABCD0002.JPG", + "DCIM/100ABCDE/LINK", + "DCIM/101ABCDE", + "DCIM/LINK", + "DCIM/MISC", + "DCIM/MISC/LINK", + "DCIM/MISC/README.TXT", + ) +} + +func TestParseStoreDoesNotFollowSymlinksUnderDCIM(t *testing.T) { + t.Parallel() + + outside := newCardOnDisk(t, "OUTSIDE.JPG", "DIRECTORY/ABCD0009.JPG") + outsideFile := filepath.Join(outside, "OUTSIDE.JPG") + outsideDirectory := filepath.Join(outside, "DIRECTORY") + + root := newCardOnDisk(t, + "DCIM/100ABCDE/ABCD0001.JPG", + "DCIM/102ABCDE/ABCD0002.JPG", + "DCIM/MISC/README.TXT", + ) + + // These two have DCF file names. The second has the number of a real JPG, + // which must not make that JPG shared. + addSymlink(t, root, "DCIM/100ABCDE/ABCD0003.JPG", outsideFile) + addSymlink(t, root, "DCIM/100ABCDE/PQRS0001.JPG", outsideFile) + + // These two have DCF directory names. The second has the number of a real + // directory, which must not make that directory shared. + addSymlink(t, root, "DCIM/101ABCDE", outsideDirectory) + addSymlink(t, root, "DCIM/102PQRST", outsideDirectory) + + addSymlink(t, root, "DCIM/LINK", outsideDirectory) + addSymlink(t, root, "DCIM/MISC/LINK", outsideDirectory) + + store := mustParseStore(t, os.DirFS(root)) + + if got, want := directoryNumbers(store), []int{100, 102}; !slices.Equal(got, want) { + t.Errorf("directory numbers = %v, want %v", got, want) + } + + assertPaths(t, "Images", imagePaths(store), + "DCIM/100ABCDE/ABCD0001.JPG", + "DCIM/102ABCDE/ABCD0002.JPG", + ) + + if got := store.TotalImageSize(); got != 6 { + t.Errorf("TotalImageSize() = %d, want 6", got) + } + + assertPaths(t, "NonConforming", store.NonConforming, + "DCIM/100ABCDE/ABCD0003.JPG", + "DCIM/100ABCDE/PQRS0001.JPG", + "DCIM/101ABCDE", + "DCIM/102PQRST", + "DCIM/LINK", + "DCIM/MISC", + "DCIM/MISC/LINK", + "DCIM/MISC/README.TXT", + ) +} + +func TestParseStoreSkipsSymlinksAndNamesThatFSRefusesUnderM4Root(t *testing.T) { + t.Parallel() + + outside := newCardOnDisk(t, "OUTSIDE.MP4") + + // The names with the byte 0xff sort before CLIP, so a walk that stops at one + // of them does not reach it. + root := newCardOnDisk(t, + "PRIVATE/M4ROOT/A\xffDIR/C0002.MP4", + "PRIVATE/M4ROOT/A\xffFILE.MP4", + "PRIVATE/M4ROOT/CLIP/C0001.MP4", + ) + + addSymlink(t, root, "PRIVATE/M4ROOT/CLIP/C0003.MP4", + filepath.Join(outside, "OUTSIDE.MP4")) + addSymlink(t, root, "PRIVATE/M4ROOT/CLIP/C0004.MP4", "absent") + addSymlink(t, root, "PRIVATE/M4ROOT/LINK", outside) + + store := mustParseStore(t, os.DirFS(root)) + + assertPaths(t, "Videos", videoPaths(store), "PRIVATE/M4ROOT/CLIP/C0001.MP4") + + if got := store.TotalVideoSize(); got != 3 { + t.Errorf("TotalVideoSize() = %d, want 3", got) + } +} + +func TestParseStoreTreatsSymlinksAsMissingDirectories(t *testing.T) { + t.Parallel() + + outside := newCardOnDisk(t, + "100ABCDE/ABCD0001.JPG", + "M4ROOT/C0001.MP4", + "C0002.MP4", + ) + + cases := []struct{ link, target string }{ + {dcimDirectory, outside}, + {dcimDirectory, dcimDirectory}, + {privateDirectory, outside}, + {privateDirectory, privateDirectory}, + {m4RootDirectory, outside}, + } + + for _, testCase := range cases { + root := newMountPoint(t, filepath.Dir(testCase.link)) + addSymlink(t, root, testCase.link, testCase.target) + + store := mustParseStore(t, os.DirFS(root)) + + if !reflect.DeepEqual(store, &Store{RootDirectory: testRoot}) { + t.Errorf("symlink at %s to %s read as %+v, want an empty store", + testCase.link, testCase.target, store) + } + } +} + +// failingFS is a card on which one call fails for one path, the way a failing +// card does. The call is Lstat, ReadDir or Info. +type failingFS struct { + fstest.MapFS + + call string + name string +} + +func (f failingFS) Lstat(name string) (fs.FileInfo, error) { + err := f.failure("Lstat", name) + if err != nil { + return nil, err + } + + return f.MapFS.Lstat(name) +} + +func (f failingFS) ReadDir(name string) ([]fs.DirEntry, error) { + err := f.failure("ReadDir", name) + if err != nil { + return nil, err + } + + entries, err := f.MapFS.ReadDir(name) + if err != nil { + return nil, err + } + + for i, entry := range entries { + err = f.failure("Info", path.Join(name, entry.Name())) + if err != nil { + entries[i] = failingEntry{DirEntry: entry, err: err} + } + } + + return entries, nil +} + +func (f failingFS) failure(call, name string) error { + if call == f.call && name == f.name { + return &fs.PathError{Op: call, Path: name, Err: fs.ErrPermission} + } + + return nil +} + +// failingEntry is a directory entry whose Info always returns err. +type failingEntry struct { + fs.DirEntry + + err error +} + +func (e failingEntry) Info() (fs.FileInfo, error) { + return nil, e.err +} + +func TestParseStoreReturnsReadErrorsNamingTheCard(t *testing.T) { + t.Parallel() + + card := newCard( + "DCIM/100ABCDE/ABCD0001.JPG", + "DCIM/100ABCDE/SUB/README.TXT", + "DCIM/MISC/SUB/README.TXT", + "PRIVATE/M4ROOT/CLIP/C0001.MP4", + ) + + cases := map[string][]string{ + "Lstat": {dcimDirectory, privateDirectory, m4RootDirectory}, + "ReadDir": { + dcimDirectory, + "DCIM/100ABCDE", + "DCIM/100ABCDE/SUB", + "DCIM/MISC", + "DCIM/MISC/SUB", + m4RootDirectory, + "PRIVATE/M4ROOT/CLIP", + }, + "Info": {"DCIM/100ABCDE/ABCD0001.JPG", "PRIVATE/M4ROOT/CLIP/C0001.MP4"}, + } + + for call, names := range cases { + for _, name := range names { + fsys := failingFS{MapFS: card, call: call, name: name} + + _, err := parseStore(fsys, testRoot) + if !errors.Is(err, fs.ErrPermission) { + t.Errorf("%s of %s: error = %v, want %v", + call, name, err, fs.ErrPermission) + + continue + } + + want := "reading the card at " + testRoot + ": " + + call + " " + name + ": permission denied" + if err.Error() != want { + t.Errorf("error = %q, want %q", err.Error(), want) + } + } + } +}