Parse the DCF structure under DCIM #11

Merged
clawbot merged 1 commits from issue-2-parse-dcf-structure into next 2026-11-03 19:25:34 +01:00
7 changed files with 1243 additions and 57 deletions

No files matched your search

+36 -9
View File
@@ -105,30 +105,57 @@ overengineering and reusable code.
All the code is in the `pkg/dcf` package. The repo has no commands. All the code is in the `pkg/dcf` package. The repo has no commands.
- `dcf.go` holds the public API. `GetDCFStores` finds the DCF stores and lists - `dcf.go` holds the public API. `GetDCFStores` finds the DCF stores and reads
the files on them. each one.
- `helpers.go` finds the mount points that hold a DCF store. - `helpers.go` finds the mount points that hold a DCF store.
- `parse.go` reads one DCF store through an `fs.FS` rooted at its mount point.
The main types are: The main types are:
- `Store` is one DCF store on one mounted filesystem. - `Store` is one DCF store on one mounted filesystem.
- `Object` is one file in a store. - `Directory` is one DCF directory. It has a directory number, a name and the
objects inside it.
- `Object` is one file in a store. A file in a DCF directory also has a
directory number and a file number.
- `Image` and `Video` embed `Object`. The file extension decides which one a - `Image` and `Video` embed `Object`. The file extension decides which one a
file becomes. file becomes. A file with any other extension, such as a `THM` thumbnail, is
neither.
A mounted filesystem is a DCF store when its root holds a `DCIM` or A mounted filesystem is a DCF store when its root holds a `DCIM` or
`PRIVATE/M4ROOT` directory. `dcf` never mounts a filesystem. `PRIVATE/M4ROOT` directory. It needs only one of them. `dcf` never mounts a
filesystem. Media anywhere else on the filesystem is not part of the store.
`dcf` never follows a symlink.
`dcf` applies the naming rules of the DCF specification, CIPA DC-009-2010, under
`DCIM`:
- A DCF directory sits directly under `DCIM`. Its name is a directory number
from 100 to 999, then five characters, such as `100MSDCF`.
- A DCF file sits directly in a DCF directory. Its name is four characters, then
a file number from 0001 to 9999, then a dot and a three-character extension,
such as `DSC00001.ARW`.
- The characters allowed are `A` to `Z`, `0` to `9` and `_`. Lower-case letters
count as upper-case.
- Directories that share a directory number are not DCF directories.
- `JPG` files that share a file number in one directory are not DCF files.
`Store.NonConforming` lists the path of everything else under `DCIM`. It
includes everything inside a directory that breaks the rules. It also includes
every symlink, whatever its name. These entries have no DCF numbers, and they
are not counted as images or videos.
The naming rules do not apply under `PRIVATE/M4ROOT`. The images and videos
there are in `Store.Images` and `Store.Videos`, and their DCF numbers are 0.
`dcf` skips any symlink there.
## TODO ## TODO
- Handle DCF file groups, where multiple objects share the same index number. - Handle DCF file groups, where multiple objects share the same index number.
`dcf` does not handle these yet. If you need this, send me a link to a zip or `dcf` does not handle these yet. If you need this, send me a link to a zip or
tar of a memory card that uses them, and I'll see what I can do. tar of a memory card that uses them, and I'll see what I can do.
- Parse the DCF directory and file names, rather than listing every media file - Take a `context.Context` to cancel the walk of a store.
on the filesystem.
- Read stores through `fs.FS`, with a `context.Context` to cancel a walk.
- Return a plain slice from `GetDCFStores`, not a pointer to one. - Return a plain slice from `GetDCFStores`, not a pointer to one.
- Test the walk of a store. It only runs against real mounted media today.
- Copy files off a card, and verify each copy against its SHA256 hash. - Copy files off a card, and verify each copy against its SHA256 hash.
## License ## License
+48 -41
View File
@@ -1,6 +1,8 @@
// Package dcf reads DCF cards: memory cards laid out to the "Design rule for // Package dcf reads DCF cards: memory cards laid out to the "Design rule for
// Camera File system" specification (JEITA CP-3461). It finds the cards among // Camera File system" specification (JEITA CP-3461). It finds the cards among
// the mounted filesystems and lists the images and videos on each one. // the mounted filesystems and parses the DCF directories and files on each one
// into their directory numbers and file numbers. It lists the images and videos
// on each card, and everything under DCIM that is not a DCF directory or file.
package dcf package dcf
import ( import (
@@ -20,8 +22,30 @@ import (
// images and videos in the root of a mounted filesystem. // images and videos in the root of a mounted filesystem.
type Store struct { type Store struct {
RootDirectory string RootDirectory string
Images []*Image // Directories holds the DCF directories under DCIM.
Videos []*Video Directories []*Directory
// Images and Videos hold the images and videos among the objects in
// Directories, and those under PRIVATE/M4ROOT.
Images []*Image
Videos []*Video
// NonConforming holds the path, relative to RootDirectory, of everything
// under DCIM that is not a DCF directory or a DCF file. It includes every
// symlink, and everything inside a directory that breaks the DCF naming
// rules. A directory whose name is not valid UTF-8 is listed without what is
// inside it, because fs.FS refuses to open one.
NonConforming []string
}
// Directory represents a DCF directory: a directory directly under DCIM whose
// name follows the DCF naming rules.
type Directory struct {
// Number is the directory number, from 100 to 999.
Number int
// Name is the name of the directory as it appears on the card.
Name string
// Objects holds every DCF file in the directory, including those that are
// neither images nor videos.
Objects []*Object
} }
// Object represents a file in a DCF store. It is embedded in Image and Video // Object represents a file in a DCF store. It is embedded in Image and Video
@@ -31,6 +55,10 @@ type Object struct {
Path string Path string
Size int64 Size int64
Extension string Extension string
// DirectoryNumber and FileNumber are the DCF numbers of a file in a DCF
// directory. Both are 0 for a file under PRIVATE/M4ROOT.
DirectoryNumber int
FileNumber int
} }
// Image represents an image file on a DCF store. // Image represents an image file on a DCF store.
@@ -140,58 +168,37 @@ func pathToSHA256(path string) (string, error) {
// any mounted filesystems. The integer argument is the number of DCF stores to // any mounted filesystems. The integer argument is the number of DCF stores to
// return. If the argument is 0, all DCF stores are returned. It does not mount // return. If the argument is 0, all DCF stores are returned. It does not mount
// filesystems or search outside of filesystem root directories. It does, // filesystems or search outside of filesystem root directories. It does,
// however, walk the filesystems to find images and videos and populate the // however, read the DCIM and PRIVATE/M4ROOT directories of each filesystem to
// returned Stores. // populate the returned Stores. It never follows a symlink. If a card cannot be
// read, it returns an error that names the card's mount point.
func GetDCFStores(requestedCount int) (*[]Store, error) { func GetDCFStores(requestedCount int) (*[]Store, error) {
dcfStorePaths, err := findDCFMountPoints(requestedCount) dcfStorePaths, err := findDCFMountPoints(requestedCount)
if err != nil { if err != nil {
return nil, err return nil, err
} }
dcfStores, err := parseStores(dcfStorePaths)
if err != nil {
return nil, err
}
return &dcfStores, nil
}
// parseStores reads the card at each of the given mount points.
func parseStores(mountpoints []string) ([]Store, error) {
dcfStores := []Store{} dcfStores := []Store{}
for _, dcfStorePath := range dcfStorePaths { for _, mountpoint := range mountpoints {
dcfStore := Store{ dcfStore, err := parseStore(os.DirFS(mountpoint), mountpoint)
RootDirectory: dcfStorePath,
Images: make([]*Image, 0),
Videos: make([]*Video, 0),
}
addFile := func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() && isImageFile(path) {
image := Image{}
image.Path = strings.TrimPrefix(path, dcfStorePath+"/")
image.Size = info.Size()
image.Extension = filepath.Ext(path)
image.DCFStoreRoot = dcfStorePath
dcfStore.Images = append(dcfStore.Images, &image)
}
if !info.IsDir() && isVideoFile(path) {
video := Video{}
video.Path = strings.TrimPrefix(path, dcfStorePath+"/")
video.Size = info.Size()
video.Extension = filepath.Ext(path)
video.DCFStoreRoot = dcfStorePath
dcfStore.Videos = append(dcfStore.Videos, &video)
}
return nil
}
err := filepath.Walk(dcfStorePath, addFile)
if err != nil { if err != nil {
return nil, err return nil, err
} }
dcfStores = append(dcfStores, dcfStore) dcfStores = append(dcfStores, *dcfStore)
} }
return &dcfStores, nil return dcfStores, nil
} }
func isImageFile(path string) bool { func isImageFile(path string) bool {
+55 -1
View File
@@ -1,6 +1,60 @@
package dcf package dcf
import "testing" import (
"path/filepath"
"strings"
"testing"
)
func TestParseStoresReadsEachMountPoint(t *testing.T) {
t.Parallel()
first := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG")
second := newCardOnDisk(t, "PRIVATE/M4ROOT/CLIP/C0001.MP4")
stores, err := parseStores([]string{first, second})
if err != nil {
t.Fatalf("parseStores: %v", err)
}
if len(stores) != 2 {
t.Fatalf("parseStores returned %d stores, want 2", len(stores))
}
cases := map[string]Object{
filepath.Join(first, "DCIM/100ABCDE/ABCD0001.JPG"): stores[0].Images[0].Object,
filepath.Join(second, "PRIVATE/M4ROOT/CLIP/C0001.MP4"): stores[1].Videos[0].Object,
}
for want, object := range cases {
if got := object.FullFilePath(); got != want {
t.Errorf("FullFilePath() = %q, want %q", got, want)
}
got, err := object.Hash()
if err != nil {
t.Fatalf("Hash() returned error: %v", err)
}
// The published SHA-256 test vector for "abc".
wantHash := "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
if got != wantHash {
t.Errorf("Hash() = %q, want %q", got, wantHash)
}
}
}
func TestParseStoresReturnsErrorNamingTheCard(t *testing.T) {
t.Parallel()
// Every lookup under a plain file fails.
mountpoint := filepath.Join(newCardOnDisk(t, "FILE"), "FILE")
_, err := parseStores([]string{mountpoint})
if err == nil || !strings.Contains(err.Error(), mountpoint) {
t.Fatalf("parseStores error = %v, want one naming %s", err, mountpoint)
}
}
func TestIsImageFile(t *testing.T) { func TestIsImageFile(t *testing.T) {
t.Parallel() t.Parallel()
+5 -6
View File
@@ -4,7 +4,6 @@ import (
"fmt" "fmt"
"log/slog" "log/slog"
"os" "os"
"path/filepath"
"slices" "slices"
"github.com/shirou/gopsutil/disk" "github.com/shirou/gopsutil/disk"
@@ -63,12 +62,12 @@ func filterDCFMountPoints(mountpoints []string, requestedCount int) []string {
// isDCFMountPoint reports whether the root of mountpoint holds a DCIM or // isDCFMountPoint reports whether the root of mountpoint holds a DCIM or
// PRIVATE/M4ROOT directory, which is how we detect DCF stores. // PRIVATE/M4ROOT directory, which is how we detect DCF stores.
func isDCFMountPoint(mountpoint string) bool { func isDCFMountPoint(mountpoint string) bool {
for _, directory := range []string{"DCIM", "PRIVATE/M4ROOT"} { fsys := os.DirFS(mountpoint)
path := filepath.Join(mountpoint, directory)
info, err := os.Stat(path) for _, directory := range []string{dcimDirectory, m4RootDirectory} {
if err == nil && info.IsDir() { found, err := hasDirectory(fsys, directory)
slog.Debug(fmt.Sprintf("Found %s directory at %s\n", directory, path)) if err == nil && found {
slog.Debug(fmt.Sprintf("Found %s directory at %s\n", directory, mountpoint))
return true return true
} }
+15
View File
@@ -57,6 +57,21 @@ func TestIsDCFMountPointIgnoresPlainFile(t *testing.T) {
} }
} }
func TestIsDCFMountPointIgnoresSymlink(t *testing.T) {
t.Parallel()
outside := newMountPoint(t, "M4ROOT")
for _, link := range []string{dcimDirectory, privateDirectory, m4RootDirectory} {
root := newMountPoint(t, filepath.Dir(link))
addSymlink(t, root, link, outside)
if isDCFMountPoint(root) {
t.Errorf("isDCFMountPoint counted a symlink at %s as a directory", link)
}
}
}
func TestFilterDCFMountPointsReturnsRequestedCount(t *testing.T) { func TestFilterDCFMountPointsReturnsRequestedCount(t *testing.T) {
t.Parallel() t.Parallel()
+275
View File
@@ -0,0 +1,275 @@
package dcf
import (
"errors"
"fmt"
"io/fs"
"path"
"regexp"
"strconv"
"strings"
)
// The two directories at the root of a card that hold its files.
const (
dcimDirectory = "DCIM"
m4RootDirectory = "PRIVATE/M4ROOT"
)
// The naming rules of the DCF specification, CIPA DC-009-2010, sections 2.3,
// 4.2.2 and 4.3.1. Names use A-Z, 0-9 and "_", and lower-case letters count as
// upper-case. Each pattern captures the number in the name.
var (
dcfDirectoryName = regexp.MustCompile(`^([1-9][0-9]{2})[0-9A-Za-z_]{5}$`)
dcfFileName = regexp.MustCompile(`^[0-9A-Za-z_]{4}([0-9]{4})\.[0-9A-Za-z_]{3}$`)
)
// parseStore reads one card from fsys, which is rooted at the card's mount
// point. The root argument is the path of that mount point. Only DCIM and
// PRIVATE/M4ROOT are read. A card can lack either one. A symlink is never
// followed. An error names the mount point, because fsys reports only paths
// inside it.
func parseStore(fsys fs.FS, root string) (*Store, error) {
store := &Store{RootDirectory: root}
err := parseDCIM(fsys, store)
if err == nil {
err = parseM4Root(fsys, store)
}
if err != nil {
return nil, fmt.Errorf("reading the card at %s: %w", root, err)
}
return store, nil
}
// hasDirectory reports whether the card holds the named directory. A plain file
// or a symlink at any step of the path counts as missing. Any other failure to
// look is an error.
func hasDirectory(fsys fs.FS, name string) (bool, error) {
parent := path.Dir(name)
if parent != "." {
found, err := hasDirectory(fsys, parent)
if err != nil || !found {
return false, err
}
}
info, err := fs.Lstat(fsys, name)
if errors.Is(err, fs.ErrNotExist) {
return false, nil
}
if err != nil {
return false, err
}
return info.IsDir(), nil
}
// parseDCIM adds the DCF directories under DCIM to store. Everything else there
// is not conforming. That includes directories that share a directory number,
// which section 7.1.2 of the specification treats as non-DCF directories.
func parseDCIM(fsys fs.FS, store *Store) error {
found, err := hasDirectory(fsys, dcimDirectory)
if err != nil || !found {
return err
}
entries, err := fs.ReadDir(fsys, dcimDirectory)
if err != nil {
return err
}
directoriesWithNumber := map[int]int{}
for _, entry := range entries {
if entry.IsDir() {
directoriesWithNumber[parseNumber(dcfDirectoryName, entry.Name())]++
}
}
for _, entry := range entries {
number := parseNumber(dcfDirectoryName, entry.Name())
if entry.IsDir() && number != 0 && directoriesWithNumber[number] == 1 {
err = parseDirectory(fsys, store, entry.Name(), number)
} else {
err = addNonConforming(fsys, store, dcimDirectory, entry)
}
if err != nil {
return err
}
}
return nil
}
// parseDirectory adds one DCF directory and the DCF files directly inside it to
// store. A DCF file is a regular file, so a symlink with a DCF name is not one.
// Everything else there is not conforming. That includes JPG files that share a
// file number, which section 7.2.2 of the specification says must not show DCF
// numbers. Other files that share a file number do conform.
func parseDirectory(fsys fs.FS, store *Store, name string, number int) error {
directoryPath := path.Join(dcimDirectory, name)
entries, err := fs.ReadDir(fsys, directoryPath)
if err != nil {
return err
}
jpgsWithNumber := map[int]int{}
for _, entry := range entries {
if entry.Type().IsRegular() && isJPG(entry.Name()) {
jpgsWithNumber[parseNumber(dcfFileName, entry.Name())]++
}
}
directory := &Directory{Number: number, Name: name}
for _, entry := range entries {
fileNumber := parseNumber(dcfFileName, entry.Name())
sharedJPG := isJPG(entry.Name()) && jpgsWithNumber[fileNumber] > 1
if !entry.Type().IsRegular() || fileNumber == 0 || sharedJPG {
err = addNonConforming(fsys, store, directoryPath, entry)
if err != nil {
return err
}
continue
}
entryPath := path.Join(directoryPath, entry.Name())
object, err := newObject(store, entryPath, entry)
if err != nil {
return err
}
object.DirectoryNumber = number
object.FileNumber = fileNumber
directory.Objects = append(directory.Objects, &object)
addMedia(store, object)
}
store.Directories = append(store.Directories, directory)
return nil
}
// parseM4Root adds the images and videos under PRIVATE/M4ROOT to store. The DCF
// naming rules do not apply there, so these files have no DCF numbers. It skips
// a symlink. It also skips a name that is not valid UTF-8, because fs.FS refuses
// to open one.
func parseM4Root(fsys fs.FS, store *Store) error {
found, err := hasDirectory(fsys, m4RootDirectory)
if err != nil || !found {
return err
}
return fs.WalkDir(fsys, m4RootDirectory,
func(filePath string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
validName := fs.ValidPath(filePath)
if entry.IsDir() && !validName {
return fs.SkipDir
}
if !entry.Type().IsRegular() || !validName {
return nil
}
object, err := newObject(store, filePath, entry)
if err != nil {
return err
}
addMedia(store, object)
return nil
})
}
// addNonConforming lists one entry of the directory dir on store as not
// conforming. If the entry is a directory, it lists everything inside it too. It
// does not follow a symlink. It does not read a directory whose name is not
// valid UTF-8, because fs.FS refuses to open one.
func addNonConforming(fsys fs.FS, store *Store, dir string, entry fs.DirEntry) error {
entryPath := path.Join(dir, entry.Name())
store.NonConforming = append(store.NonConforming, entryPath)
if !entry.IsDir() || !fs.ValidPath(entryPath) {
return nil
}
entries, err := fs.ReadDir(fsys, entryPath)
if err != nil {
return err
}
for _, inner := range entries {
err = addNonConforming(fsys, store, entryPath, inner)
if err != nil {
return err
}
}
return nil
}
// newObject returns the Object for the file at filePath, with no DCF numbers.
func newObject(store *Store, filePath string, entry fs.DirEntry) (Object, error) {
info, err := entry.Info()
if err != nil {
return Object{}, err
}
return Object{
DCFStoreRoot: store.RootDirectory,
Path: filePath,
Size: info.Size(),
Extension: path.Ext(filePath),
}, nil
}
// addMedia adds object to the images or videos of store. A file with any other
// extension is neither, and is not added.
func addMedia(store *Store, object Object) {
if isImageFile(object.Path) {
store.Images = append(store.Images, &Image{Object: object})
}
if isVideoFile(object.Path) {
store.Videos = append(store.Videos, &Video{Object: object})
}
}
// parseNumber returns the number that pattern captures in name. It returns 0
// when name does not match. No DCF number is 0, so this also rejects the file
// number 0000, which the specification forbids.
func parseNumber(pattern *regexp.Regexp, name string) int {
match := pattern.FindStringSubmatch(name)
if match == nil {
return 0
}
number, err := strconv.Atoi(match[1])
if err != nil {
return 0
}
return number
}
func isJPG(name string) bool {
return strings.EqualFold(path.Ext(name), ".JPG")
}
+809
View File
@@ -0,0 +1,809 @@
package dcf
import (
"errors"
"io/fs"
"os"
"path"
"path/filepath"
"reflect"
"slices"
"testing"
"testing/fstest"
)
const (
testRoot = "/mnt/card"
privateDirectory = "PRIVATE"
)
// newCard returns a card holding an empty file at each of the given paths.
func newCard(paths ...string) fstest.MapFS {
card := fstest.MapFS{}
for _, filePath := range paths {
card[filePath] = &fstest.MapFile{}
}
return card
}
// newCardOnDisk returns a temporary directory holding the three bytes "abc" in a
// file at each of the given paths. It is for tests that fstest.MapFS cannot do.
func newCardOnDisk(t *testing.T, paths ...string) string {
t.Helper()
root := t.TempDir()
for _, filePath := range paths {
fullPath := filepath.Join(root, filePath)
err := os.MkdirAll(filepath.Dir(fullPath), 0o750)
if err != nil {
t.Fatalf("MkdirAll: %v", err)
}
err = os.WriteFile(fullPath, []byte("abc"), 0o600)
if err != nil {
t.Fatalf("WriteFile: %v", err)
}
}
return root
}
// addSymlink creates a symlink to target at the named path under root.
func addSymlink(t *testing.T, root, name, target string) {
t.Helper()
err := os.Symlink(target, filepath.Join(root, name))
if err != nil {
t.Fatalf("Symlink: %v", err)
}
}
func mustParseStore(t *testing.T, fsys fs.FS) *Store {
t.Helper()
store, err := parseStore(fsys, testRoot)
if err != nil {
t.Fatalf("parseStore: %v", err)
}
return store
}
func imagePaths(store *Store) []string {
paths := make([]string, 0, len(store.Images))
for _, image := range store.Images {
paths = append(paths, image.Path)
}
return paths
}
func videoPaths(store *Store) []string {
paths := make([]string, 0, len(store.Videos))
for _, video := range store.Videos {
paths = append(paths, video.Path)
}
return paths
}
func objectPaths(directory *Directory) []string {
paths := make([]string, 0, len(directory.Objects))
for _, object := range directory.Objects {
paths = append(paths, object.Path)
}
return paths
}
func directoryNumbers(store *Store) []int {
numbers := make([]int, 0, len(store.Directories))
for _, directory := range store.Directories {
numbers = append(numbers, directory.Number)
}
return numbers
}
func assertPaths(t *testing.T, what string, got []string, want ...string) {
t.Helper()
if !slices.Equal(got, want) {
t.Errorf("%s = %q, want %q", what, got, want)
}
}
func TestParseStorePutsEachFileWhereItBelongs(t *testing.T) {
t.Parallel()
card := fstest.MapFS{
"DCIM/100MSDCF/ABCD0001.ARW": {Data: make([]byte, 2000)},
"DCIM/100MSDCF/readme.txt": {Data: make([]byte, 7)},
"DCIM/101CANON/MVI_0002.MP4": {Data: make([]byte, 9000)},
"DCIM/101CANON/MVI_0002.THM": {Data: make([]byte, 100)},
"DCIM/MISC/DSC00003.JPG": {Data: make([]byte, 7)},
"DCIM/DSC00004.JPG": {Data: make([]byte, 7)},
"PRIVATE/M4ROOT/CLIP/C0002.MP4": {Data: make([]byte, 400)},
"PRIVATE/M4ROOT/CLIP/C0002M01.XML": {Data: make([]byte, 7)},
"PRIVATE/OTHER/C0003.MP4": {Data: make([]byte, 7)},
"BACKUP/DSC00005.JPG": {Data: make([]byte, 7)},
"DSC00006.JPG": {Data: make([]byte, 7)},
}
store := mustParseStore(t, card)
arw := Object{
DCFStoreRoot: testRoot,
Path: "DCIM/100MSDCF/ABCD0001.ARW",
Size: 2000,
Extension: ".ARW",
DirectoryNumber: 100,
FileNumber: 1,
}
mp4 := Object{
DCFStoreRoot: testRoot,
Path: "DCIM/101CANON/MVI_0002.MP4",
Size: 9000,
Extension: ".MP4",
DirectoryNumber: 101,
FileNumber: 2,
}
thm := Object{
DCFStoreRoot: testRoot,
Path: "DCIM/101CANON/MVI_0002.THM",
Size: 100,
Extension: ".THM",
DirectoryNumber: 101,
FileNumber: 2,
}
clip := Object{
DCFStoreRoot: testRoot,
Path: "PRIVATE/M4ROOT/CLIP/C0002.MP4",
Size: 400,
Extension: ".MP4",
}
wantDirectories := []*Directory{
{Number: 100, Name: "100MSDCF", Objects: []*Object{&arw}},
{Number: 101, Name: "101CANON", Objects: []*Object{&mp4, &thm}},
}
if !reflect.DeepEqual(store.Directories, wantDirectories) {
t.Errorf("Directories = %+v, want %+v", store.Directories, wantDirectories)
}
wantImages := []*Image{{Object: arw}}
if !reflect.DeepEqual(store.Images, wantImages) {
t.Errorf("Images = %+v, want %+v", store.Images, wantImages)
}
wantVideos := []*Video{{Object: mp4}, {Object: clip}}
if !reflect.DeepEqual(store.Videos, wantVideos) {
t.Errorf("Videos = %+v, want %+v", store.Videos, wantVideos)
}
assertPaths(t, "NonConforming", store.NonConforming,
"DCIM/100MSDCF/readme.txt",
"DCIM/DSC00004.JPG",
"DCIM/MISC",
"DCIM/MISC/DSC00003.JPG",
)
}
func TestParseStoreCountsOnlyImagesAndVideos(t *testing.T) {
t.Parallel()
card := fstest.MapFS{
"DCIM/100ABCDE/ABCD0001.ARW": {Data: make([]byte, 2000)},
"DCIM/100ABCDE/ABCD0002.THM": {Data: make([]byte, 100)},
"DCIM/100ABCDE/ABCD0003.MP4": {Data: make([]byte, 9000)},
"DCIM/MISC/ABCD0004.JPG": {Data: make([]byte, 50)},
"PRIVATE/M4ROOT/CLIP/C0004.MP4": {Data: make([]byte, 400)},
"ABCD0005.JPG": {Data: make([]byte, 30)},
}
store := mustParseStore(t, card)
if got := store.ImagesCount(); got != 1 {
t.Errorf("ImagesCount() = %d, want 1", got)
}
if got := store.VideosCount(); got != 2 {
t.Errorf("VideosCount() = %d, want 2", got)
}
if got := store.TotalSize(); got != 11400 {
t.Errorf("TotalSize() = %d, want 11400", got)
}
}
func TestParseStoreSetsFullFilePath(t *testing.T) {
t.Parallel()
store := mustParseStore(t, newCard("DCIM/100MSDCF/DSC00001.JPG"))
want := testRoot + "/DCIM/100MSDCF/DSC00001.JPG"
if got := store.Images[0].FullFilePath(); got != want {
t.Fatalf("FullFilePath() = %q, want %q", got, want)
}
}
func TestParseNumberOfDirectoryName(t *testing.T) {
t.Parallel()
cases := map[string]int{
"100MSDCF": 100,
"999VWXYZ": 999,
"100D3500": 100,
"101abcde": 101,
"102A_B_C": 102,
"099ABCDE": 0,
"000ABCDE": 0,
"103A": 0,
"IM104ABC": 0,
"105_abcdefg": 0,
"107@-%+]": 0,
"IM08ABCD": 0,
"100ABC-E": 0,
"100ABCD": 0,
"100ABCDEF": 0,
"1000ABCDE": 0,
"X100ABCDE": 0,
"TEMP": 0,
}
for name, want := range cases {
if got := parseNumber(dcfDirectoryName, name); got != want {
t.Errorf("directory %q parsed to %d, want %d", name, got, want)
}
}
}
func TestParseNumberOfFileName(t *testing.T) {
t.Parallel()
cases := map[string]int{
"DSC00001.ARW": 1,
"ABCD9999.JPG": 9999,
"abcd0002.jpg": 2,
"_ABC0001.JPG": 1,
"IMG_0001.JPG": 1,
"12340005.JPG": 5,
"ABCD0000.JPG": 0,
"A0004.JPG": 0,
"ABCDE0005.JPG": 0,
"ABCD10000.JPG": 0,
"abcdefg0006.JPG": 0,
"+-@]0007.JPG": 0,
"ABCD00~1.JPG": 0,
"ABCDEFGH.JPG": 0,
"ABCD0001": 0,
"ABCD0001.": 0,
"ABCD0001.JP": 0,
"ABCD0001.JPEG": 0,
"ABCD0001.J-G": 0,
"._DSC0001.JPG": 0,
".DS_Store": 0,
}
for name, want := range cases {
if got := parseNumber(dcfFileName, name); got != want {
t.Errorf("file %q parsed to %d, want %d", name, got, want)
}
}
}
func TestParseStoreTreatsLowerCaseNamesAsDCFNames(t *testing.T) {
t.Parallel()
store := mustParseStore(t, newCard("DCIM/101abcde/abcd0002.jpg"))
want := []*Directory{{
Number: 101,
Name: "101abcde",
Objects: []*Object{{
DCFStoreRoot: testRoot,
Path: "DCIM/101abcde/abcd0002.jpg",
Extension: ".jpg",
DirectoryNumber: 101,
FileNumber: 2,
}},
}}
if !reflect.DeepEqual(store.Directories, want) {
t.Errorf("Directories = %+v, want %+v", store.Directories, want)
}
assertPaths(t, "Images", imagePaths(store), "DCIM/101abcde/abcd0002.jpg")
assertPaths(t, "NonConforming", store.NonConforming)
}
func TestParseStoreAcceptsUnderscoreInNames(t *testing.T) {
t.Parallel()
store := mustParseStore(t, newCard("DCIM/100A_B_C/_ABC0001.JPG"))
assertPaths(t, "Images", imagePaths(store), "DCIM/100A_B_C/_ABC0001.JPG")
assertPaths(t, "NonConforming", store.NonConforming)
}
func TestParseStoreRejectsDirectoriesSharingANumber(t *testing.T) {
t.Parallel()
store := mustParseStore(t, newCard(
"DCIM/102ABCDE/ABCD0001.JPG",
"DCIM/102PQRST/ABCD0002.JPG",
"DCIM/103ABCDE/ABCD0003.JPG",
))
if got, want := directoryNumbers(store), []int{103}; !slices.Equal(got, want) {
t.Errorf("directory numbers = %v, want %v", got, want)
}
assertPaths(t, "Images", imagePaths(store), "DCIM/103ABCDE/ABCD0003.JPG")
assertPaths(t, "NonConforming", store.NonConforming,
"DCIM/102ABCDE",
"DCIM/102ABCDE/ABCD0001.JPG",
"DCIM/102PQRST",
"DCIM/102PQRST/ABCD0002.JPG",
)
}
func TestParseStoreDoesNotCountPlainFileAsSharingADirectoryNumber(t *testing.T) {
t.Parallel()
store := mustParseStore(t, newCard(
"DCIM/100ABCDE",
"DCIM/100PQRST/ABCD0001.JPG",
))
if got, want := directoryNumbers(store), []int{100}; !slices.Equal(got, want) {
t.Errorf("directory numbers = %v, want %v", got, want)
}
assertPaths(t, "NonConforming", store.NonConforming, "DCIM/100ABCDE")
}
func TestParseStoreRejectsJPGsSharingAFileNumber(t *testing.T) {
t.Parallel()
store := mustParseStore(t, newCard(
"DCIM/100ABCDE/ABCD0003.JPG",
"DCIM/100ABCDE/PQRS0003.jpg",
"DCIM/101ABCDE/ABCD0003.JPG",
))
assertPaths(t, "objects", objectPaths(store.Directories[0]))
assertPaths(t, "objects", objectPaths(store.Directories[1]),
"DCIM/101ABCDE/ABCD0003.JPG",
)
assertPaths(t, "Images", imagePaths(store), "DCIM/101ABCDE/ABCD0003.JPG")
assertPaths(t, "NonConforming", store.NonConforming,
"DCIM/100ABCDE/ABCD0003.JPG",
"DCIM/100ABCDE/PQRS0003.jpg",
)
}
func TestParseStoreKeepsOtherFilesSharingAFileNumber(t *testing.T) {
t.Parallel()
store := mustParseStore(t, newCard(
"DCIM/100ABCDE/DSC00004.ARW",
"DCIM/100ABCDE/DSC00004.JPG",
"DCIM/100ABCDE/MVI_0005.MP4",
"DCIM/100ABCDE/MVI_0005.THM",
))
assertPaths(t, "objects", objectPaths(store.Directories[0]),
"DCIM/100ABCDE/DSC00004.ARW",
"DCIM/100ABCDE/DSC00004.JPG",
"DCIM/100ABCDE/MVI_0005.MP4",
"DCIM/100ABCDE/MVI_0005.THM",
)
assertPaths(t, "Images", imagePaths(store),
"DCIM/100ABCDE/DSC00004.ARW",
"DCIM/100ABCDE/DSC00004.JPG",
)
assertPaths(t, "Videos", videoPaths(store), "DCIM/100ABCDE/MVI_0005.MP4")
assertPaths(t, "NonConforming", store.NonConforming)
}
func TestParseStoreKeepsARWSharingANumberWithTwoJPGs(t *testing.T) {
t.Parallel()
store := mustParseStore(t, newCard(
"DCIM/100ABCDE/ABCD0004.ARW",
"DCIM/100ABCDE/ABCD0004.JPG",
"DCIM/100ABCDE/PQRS0004.JPG",
))
assertPaths(t, "objects", objectPaths(store.Directories[0]),
"DCIM/100ABCDE/ABCD0004.ARW",
)
assertPaths(t, "NonConforming", store.NonConforming,
"DCIM/100ABCDE/ABCD0004.JPG",
"DCIM/100ABCDE/PQRS0004.JPG",
)
}
func TestParseStoreDoesNotCountDirectoryAsSharingAFileNumber(t *testing.T) {
t.Parallel()
store := mustParseStore(t, newCard(
"DCIM/100ABCDE/ABCD0003.JPG",
"DCIM/100ABCDE/PQRS0003.JPG/README.TXT",
))
assertPaths(t, "objects", objectPaths(store.Directories[0]),
"DCIM/100ABCDE/ABCD0003.JPG",
)
assertPaths(t, "NonConforming", store.NonConforming,
"DCIM/100ABCDE/PQRS0003.JPG",
"DCIM/100ABCDE/PQRS0003.JPG/README.TXT",
)
}
func TestParseStoreDoesNotCountDirectoryNamedLikeAVideo(t *testing.T) {
t.Parallel()
store := mustParseStore(t, newCard("PRIVATE/M4ROOT/CLIP.MP4/C0001.XML"))
assertPaths(t, "Videos", videoPaths(store))
}
func TestParseStoreRejectsSubdirectoryOfDCFDirectory(t *testing.T) {
t.Parallel()
store := mustParseStore(t, newCard(
"DCIM/100ABCDE/SUB/ABCD0001.JPG",
"DCIM/100ABCDE/ABCD0002.JPG/ABCD0003.JPG",
))
assertPaths(t, "objects", objectPaths(store.Directories[0]))
assertPaths(t, "Images", imagePaths(store))
assertPaths(t, "NonConforming", store.NonConforming,
"DCIM/100ABCDE/ABCD0002.JPG",
"DCIM/100ABCDE/ABCD0002.JPG/ABCD0003.JPG",
"DCIM/100ABCDE/SUB",
"DCIM/100ABCDE/SUB/ABCD0001.JPG",
)
}
func TestParseStoreListsEmptyDirectories(t *testing.T) {
t.Parallel()
store := mustParseStore(t, fstest.MapFS{
"DCIM/100EMPTY": {Mode: fs.ModeDir},
"DCIM/MISC": {Mode: fs.ModeDir},
})
if got, want := directoryNumbers(store), []int{100}; !slices.Equal(got, want) {
t.Errorf("directory numbers = %v, want %v", got, want)
}
assertPaths(t, "NonConforming", store.NonConforming, "DCIM/MISC")
}
func TestParseStoreWithoutDCIM(t *testing.T) {
t.Parallel()
store := mustParseStore(t, newCard("PRIVATE/M4ROOT/CLIP/C0001.MP4"))
if len(store.Directories) != 0 {
t.Errorf("Directories = %+v, want none", store.Directories)
}
assertPaths(t, "Videos", videoPaths(store), "PRIVATE/M4ROOT/CLIP/C0001.MP4")
}
func TestParseStoreWithoutM4Root(t *testing.T) {
t.Parallel()
store := mustParseStore(t, newCard("DCIM/100ABCDE/ABCD0001.JPG"))
assertPaths(t, "Images", imagePaths(store), "DCIM/100ABCDE/ABCD0001.JPG")
assertPaths(t, "Videos", videoPaths(store))
}
func TestParseStoreTreatsPlainFilesAsMissingDirectories(t *testing.T) {
t.Parallel()
store := mustParseStore(t, newCard("DCIM", "PRIVATE/M4ROOT"))
assertPaths(t, "Images", imagePaths(store))
assertPaths(t, "Videos", videoPaths(store))
assertPaths(t, "NonConforming", store.NonConforming)
}
// A real filesystem reports a lookup under a plain file differently from a
// missing path, which fstest.MapFS does not reproduce.
func TestParseStoreReadsRealFilesystemWithPlainFileNamedPrivate(t *testing.T) {
t.Parallel()
root := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG", "PRIVATE")
store := mustParseStore(t, os.DirFS(root))
assertPaths(t, "Images", imagePaths(store), "DCIM/100ABCDE/ABCD0001.JPG")
if got := store.TotalImageSize(); got != 3 {
t.Errorf("TotalImageSize() = %d, want 3", got)
}
}
// A real filesystem can hold a name with the byte 0xff, which is not valid UTF-8.
// fs.FS refuses to open such a name.
func TestParseStoreListsNamesThatFSRefuses(t *testing.T) {
t.Parallel()
root := newCardOnDisk(t,
"DCIM/100ABCDE/ABCD0001.JPG",
"DCIM/100ABCDE/A\xffDIR/INNER",
"DCIM/100ABCDE/A\xffFILE",
"DCIM/A\xffDIR/INNER",
"DCIM/A\xffFILE",
"DCIM/MISC/A\xffDIR/INNER",
"DCIM/MISC/A\xffFILE",
)
store := mustParseStore(t, os.DirFS(root))
assertPaths(t, "Images", imagePaths(store), "DCIM/100ABCDE/ABCD0001.JPG")
assertPaths(t, "NonConforming", store.NonConforming,
"DCIM/100ABCDE/A\xffDIR",
"DCIM/100ABCDE/A\xffFILE",
"DCIM/A\xffDIR",
"DCIM/A\xffFILE",
"DCIM/MISC",
"DCIM/MISC/A\xffDIR",
"DCIM/MISC/A\xffFILE",
)
}
func TestParseStoreListsDanglingSymlinks(t *testing.T) {
t.Parallel()
root := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG", "DCIM/MISC/README.TXT")
addSymlink(t, root, "DCIM/100ABCDE/ABCD0002.JPG", "absent")
addSymlink(t, root, "DCIM/100ABCDE/LINK", "absent")
addSymlink(t, root, "DCIM/101ABCDE", "absent")
addSymlink(t, root, "DCIM/LINK", "absent")
addSymlink(t, root, "DCIM/MISC/LINK", "absent")
store := mustParseStore(t, os.DirFS(root))
if got, want := directoryNumbers(store), []int{100}; !slices.Equal(got, want) {
t.Errorf("directory numbers = %v, want %v", got, want)
}
assertPaths(t, "Images", imagePaths(store), "DCIM/100ABCDE/ABCD0001.JPG")
assertPaths(t, "NonConforming", store.NonConforming,
"DCIM/100ABCDE/ABCD0002.JPG",
"DCIM/100ABCDE/LINK",
"DCIM/101ABCDE",
"DCIM/LINK",
"DCIM/MISC",
"DCIM/MISC/LINK",
"DCIM/MISC/README.TXT",
)
}
func TestParseStoreDoesNotFollowSymlinksUnderDCIM(t *testing.T) {
t.Parallel()
outside := newCardOnDisk(t, "OUTSIDE.JPG", "DIRECTORY/ABCD0009.JPG")
outsideFile := filepath.Join(outside, "OUTSIDE.JPG")
outsideDirectory := filepath.Join(outside, "DIRECTORY")
root := newCardOnDisk(t,
"DCIM/100ABCDE/ABCD0001.JPG",
"DCIM/102ABCDE/ABCD0002.JPG",
"DCIM/MISC/README.TXT",
)
// These two have DCF file names. The second has the number of a real JPG,
// which must not make that JPG shared.
addSymlink(t, root, "DCIM/100ABCDE/ABCD0003.JPG", outsideFile)
addSymlink(t, root, "DCIM/100ABCDE/PQRS0001.JPG", outsideFile)
// These two have DCF directory names. The second has the number of a real
// directory, which must not make that directory shared.
addSymlink(t, root, "DCIM/101ABCDE", outsideDirectory)
addSymlink(t, root, "DCIM/102PQRST", outsideDirectory)
addSymlink(t, root, "DCIM/LINK", outsideDirectory)
addSymlink(t, root, "DCIM/MISC/LINK", outsideDirectory)
store := mustParseStore(t, os.DirFS(root))
if got, want := directoryNumbers(store), []int{100, 102}; !slices.Equal(got, want) {
t.Errorf("directory numbers = %v, want %v", got, want)
}
assertPaths(t, "Images", imagePaths(store),
"DCIM/100ABCDE/ABCD0001.JPG",
"DCIM/102ABCDE/ABCD0002.JPG",
)
if got := store.TotalImageSize(); got != 6 {
t.Errorf("TotalImageSize() = %d, want 6", got)
}
assertPaths(t, "NonConforming", store.NonConforming,
"DCIM/100ABCDE/ABCD0003.JPG",
"DCIM/100ABCDE/PQRS0001.JPG",
"DCIM/101ABCDE",
"DCIM/102PQRST",
"DCIM/LINK",
"DCIM/MISC",
"DCIM/MISC/LINK",
"DCIM/MISC/README.TXT",
)
}
func TestParseStoreSkipsSymlinksAndNamesThatFSRefusesUnderM4Root(t *testing.T) {
t.Parallel()
outside := newCardOnDisk(t, "OUTSIDE.MP4")
// The names with the byte 0xff sort before CLIP, so a walk that stops at one
// of them does not reach it.
root := newCardOnDisk(t,
"PRIVATE/M4ROOT/A\xffDIR/C0002.MP4",
"PRIVATE/M4ROOT/A\xffFILE.MP4",
"PRIVATE/M4ROOT/CLIP/C0001.MP4",
)
addSymlink(t, root, "PRIVATE/M4ROOT/CLIP/C0003.MP4",
filepath.Join(outside, "OUTSIDE.MP4"))
addSymlink(t, root, "PRIVATE/M4ROOT/CLIP/C0004.MP4", "absent")
addSymlink(t, root, "PRIVATE/M4ROOT/LINK", outside)
store := mustParseStore(t, os.DirFS(root))
assertPaths(t, "Videos", videoPaths(store), "PRIVATE/M4ROOT/CLIP/C0001.MP4")
if got := store.TotalVideoSize(); got != 3 {
t.Errorf("TotalVideoSize() = %d, want 3", got)
}
}
func TestParseStoreTreatsSymlinksAsMissingDirectories(t *testing.T) {
t.Parallel()
outside := newCardOnDisk(t,
"100ABCDE/ABCD0001.JPG",
"M4ROOT/C0001.MP4",
"C0002.MP4",
)
cases := []struct{ link, target string }{
{dcimDirectory, outside},
{dcimDirectory, dcimDirectory},
{privateDirectory, outside},
{privateDirectory, privateDirectory},
{m4RootDirectory, outside},
}
for _, testCase := range cases {
root := newMountPoint(t, filepath.Dir(testCase.link))
addSymlink(t, root, testCase.link, testCase.target)
store := mustParseStore(t, os.DirFS(root))
if !reflect.DeepEqual(store, &Store{RootDirectory: testRoot}) {
t.Errorf("symlink at %s to %s read as %+v, want an empty store",
testCase.link, testCase.target, store)
}
}
}
// failingFS is a card on which one call fails for one path, the way a failing
// card does. The call is Lstat, ReadDir or Info.
type failingFS struct {
fstest.MapFS
call string
name string
}
func (f failingFS) Lstat(name string) (fs.FileInfo, error) {
err := f.failure("Lstat", name)
if err != nil {
return nil, err
}
return f.MapFS.Lstat(name)
}
func (f failingFS) ReadDir(name string) ([]fs.DirEntry, error) {
err := f.failure("ReadDir", name)
if err != nil {
return nil, err
}
entries, err := f.MapFS.ReadDir(name)
if err != nil {
return nil, err
}
for i, entry := range entries {
err = f.failure("Info", path.Join(name, entry.Name()))
if err != nil {
entries[i] = failingEntry{DirEntry: entry, err: err}
}
}
return entries, nil
}
func (f failingFS) failure(call, name string) error {
if call == f.call && name == f.name {
return &fs.PathError{Op: call, Path: name, Err: fs.ErrPermission}
}
return nil
}
// failingEntry is a directory entry whose Info always returns err.
type failingEntry struct {
fs.DirEntry
err error
}
func (e failingEntry) Info() (fs.FileInfo, error) {
return nil, e.err
}
func TestParseStoreReturnsReadErrorsNamingTheCard(t *testing.T) {
t.Parallel()
card := newCard(
"DCIM/100ABCDE/ABCD0001.JPG",
"DCIM/100ABCDE/SUB/README.TXT",
"DCIM/MISC/SUB/README.TXT",
"PRIVATE/M4ROOT/CLIP/C0001.MP4",
)
cases := map[string][]string{
"Lstat": {dcimDirectory, privateDirectory, m4RootDirectory},
"ReadDir": {
dcimDirectory,
"DCIM/100ABCDE",
"DCIM/100ABCDE/SUB",
"DCIM/MISC",
"DCIM/MISC/SUB",
m4RootDirectory,
"PRIVATE/M4ROOT/CLIP",
},
"Info": {"DCIM/100ABCDE/ABCD0001.JPG", "PRIVATE/M4ROOT/CLIP/C0001.MP4"},
}
for call, names := range cases {
for _, name := range names {
fsys := failingFS{MapFS: card, call: call, name: name}
_, err := parseStore(fsys, testRoot)
if !errors.Is(err, fs.ErrPermission) {
t.Errorf("%s of %s: error = %v, want %v",
call, name, err, fs.ErrPermission)
continue
}
want := "reading the card at " + testRoot + ": " +
call + " " + name + ": permission denied"
if err.Error() != want {
t.Errorf("error = %q, want %q", err.Error(), want)
}
}
}
}