Compare commits

..
1 Commits
Author SHA1 Message Date
clawbot 25c6aca864 parse the DCF structure under DCIM (closes #2)
A card is now read by the naming rules in CIPA DC-009-2010, not by
looking for media extensions anywhere on the volume.

Store.Directories holds the DCF directories and their objects, with
directory and file numbers. Store.NonConforming lists everything else
under DCIM, including symlinks and names fs.FS refuses. Media under
PRIVATE/M4ROOT stays, with no DCF numbers. Media anywhere else is no
longer part of the store. A symlink is never followed, and a read error
names the card's mount point.

Model: opus-5-5
2026-11-03 18:04:25 +00:00
7 changed files with 532 additions and 98 deletions

No files matched your search

+7 -3
View File
@@ -125,6 +125,8 @@ A mounted filesystem is a DCF store when its root holds a `DCIM` or
`PRIVATE/M4ROOT` directory. It needs only one of them. `dcf` never mounts a `PRIVATE/M4ROOT` directory. It needs only one of them. `dcf` never mounts a
filesystem. Media anywhere else on the filesystem is not part of the store. filesystem. Media anywhere else on the filesystem is not part of the store.
`dcf` never follows a symlink.
`dcf` applies the naming rules of the DCF specification, CIPA DC-009-2010, under `dcf` applies the naming rules of the DCF specification, CIPA DC-009-2010, under
`DCIM`: `DCIM`:
@@ -138,12 +140,14 @@ filesystem. Media anywhere else on the filesystem is not part of the store.
- Directories that share a directory number are not DCF directories. - Directories that share a directory number are not DCF directories.
- `JPG` files that share a file number in one directory are not DCF files. - `JPG` files that share a file number in one directory are not DCF files.
`Store.NonConforming` lists the path of every other file and directory under `Store.NonConforming` lists the path of everything else under `DCIM`. It
`DCIM`. It includes everything inside a directory that breaks the rules. These includes everything inside a directory that breaks the rules. It also includes
files have no DCF numbers, and they are not counted as images or videos. every symlink, whatever its name. These entries have no DCF numbers, and they
are not counted as images or videos.
The naming rules do not apply under `PRIVATE/M4ROOT`. The images and videos The naming rules do not apply under `PRIVATE/M4ROOT`. The images and videos
there are in `Store.Images` and `Store.Videos`, and their DCF numbers are 0. there are in `Store.Images` and `Store.Videos`, and their DCF numbers are 0.
`dcf` skips any symlink there.
## TODO ## TODO
+21 -8
View File
@@ -2,7 +2,7 @@
// Camera File system" specification (JEITA CP-3461). It finds the cards among // Camera File system" specification (JEITA CP-3461). It finds the cards among
// the mounted filesystems and parses the DCF directories and files on each one // the mounted filesystems and parses the DCF directories and files on each one
// into their directory numbers and file numbers. It lists the images and videos // into their directory numbers and file numbers. It lists the images and videos
// on each card, and the names under DCIM that break the DCF naming rules. // on each card, and everything under DCIM that is not a DCF directory or file.
package dcf package dcf
import ( import (
@@ -28,9 +28,11 @@ type Store struct {
// Directories, and those under PRIVATE/M4ROOT. // Directories, and those under PRIVATE/M4ROOT.
Images []*Image Images []*Image
Videos []*Video Videos []*Video
// NonConforming holds the path, relative to RootDirectory, of every file // NonConforming holds the path, relative to RootDirectory, of everything
// and directory under DCIM that breaks the DCF naming rules. It includes // under DCIM that is not a DCF directory or a DCF file. It includes every
// everything inside a directory that breaks them. // symlink, and everything inside a directory that breaks the DCF naming
// rules. A directory whose name is not valid UTF-8 is listed without what is
// inside it, because fs.FS refuses to open one.
NonConforming []string NonConforming []string
} }
@@ -167,17 +169,28 @@ func pathToSHA256(path string) (string, error) {
// return. If the argument is 0, all DCF stores are returned. It does not mount // return. If the argument is 0, all DCF stores are returned. It does not mount
// filesystems or search outside of filesystem root directories. It does, // filesystems or search outside of filesystem root directories. It does,
// however, read the DCIM and PRIVATE/M4ROOT directories of each filesystem to // however, read the DCIM and PRIVATE/M4ROOT directories of each filesystem to
// populate the returned Stores. // populate the returned Stores. It never follows a symlink. If a card cannot be
// read, it returns an error that names the card's mount point.
func GetDCFStores(requestedCount int) (*[]Store, error) { func GetDCFStores(requestedCount int) (*[]Store, error) {
dcfStorePaths, err := findDCFMountPoints(requestedCount) dcfStorePaths, err := findDCFMountPoints(requestedCount)
if err != nil { if err != nil {
return nil, err return nil, err
} }
dcfStores, err := parseStores(dcfStorePaths)
if err != nil {
return nil, err
}
return &dcfStores, nil
}
// parseStores reads the card at each of the given mount points.
func parseStores(mountpoints []string) ([]Store, error) {
dcfStores := []Store{} dcfStores := []Store{}
for _, dcfStorePath := range dcfStorePaths { for _, mountpoint := range mountpoints {
dcfStore, err := parseStore(os.DirFS(dcfStorePath), dcfStorePath) dcfStore, err := parseStore(os.DirFS(mountpoint), mountpoint)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -185,7 +198,7 @@ func GetDCFStores(requestedCount int) (*[]Store, error) {
dcfStores = append(dcfStores, *dcfStore) dcfStores = append(dcfStores, *dcfStore)
} }
return &dcfStores, nil return dcfStores, nil
} }
func isImageFile(path string) bool { func isImageFile(path string) bool {
+55 -1
View File
@@ -1,6 +1,60 @@
package dcf package dcf
import "testing" import (
"path/filepath"
"strings"
"testing"
)
func TestParseStoresReadsEachMountPoint(t *testing.T) {
t.Parallel()
first := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG")
second := newCardOnDisk(t, "PRIVATE/M4ROOT/CLIP/C0001.MP4")
stores, err := parseStores([]string{first, second})
if err != nil {
t.Fatalf("parseStores: %v", err)
}
if len(stores) != 2 {
t.Fatalf("parseStores returned %d stores, want 2", len(stores))
}
cases := map[string]Object{
filepath.Join(first, "DCIM/100ABCDE/ABCD0001.JPG"): stores[0].Images[0].Object,
filepath.Join(second, "PRIVATE/M4ROOT/CLIP/C0001.MP4"): stores[1].Videos[0].Object,
}
for want, object := range cases {
if got := object.FullFilePath(); got != want {
t.Errorf("FullFilePath() = %q, want %q", got, want)
}
got, err := object.Hash()
if err != nil {
t.Fatalf("Hash() returned error: %v", err)
}
// The published SHA-256 test vector for "abc".
wantHash := "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
if got != wantHash {
t.Errorf("Hash() = %q, want %q", got, wantHash)
}
}
}
func TestParseStoresReturnsErrorNamingTheCard(t *testing.T) {
t.Parallel()
// Every lookup under a plain file fails.
mountpoint := filepath.Join(newCardOnDisk(t, "FILE"), "FILE")
_, err := parseStores([]string{mountpoint})
if err == nil || !strings.Contains(err.Error(), mountpoint) {
t.Fatalf("parseStores error = %v, want one naming %s", err, mountpoint)
}
}
func TestIsImageFile(t *testing.T) { func TestIsImageFile(t *testing.T) {
t.Parallel() t.Parallel()
+5 -6
View File
@@ -4,7 +4,6 @@ import (
"fmt" "fmt"
"log/slog" "log/slog"
"os" "os"
"path/filepath"
"slices" "slices"
"github.com/shirou/gopsutil/disk" "github.com/shirou/gopsutil/disk"
@@ -63,12 +62,12 @@ func filterDCFMountPoints(mountpoints []string, requestedCount int) []string {
// isDCFMountPoint reports whether the root of mountpoint holds a DCIM or // isDCFMountPoint reports whether the root of mountpoint holds a DCIM or
// PRIVATE/M4ROOT directory, which is how we detect DCF stores. // PRIVATE/M4ROOT directory, which is how we detect DCF stores.
func isDCFMountPoint(mountpoint string) bool { func isDCFMountPoint(mountpoint string) bool {
for _, directory := range []string{dcimDirectory, m4RootDirectory} { fsys := os.DirFS(mountpoint)
path := filepath.Join(mountpoint, directory)
info, err := os.Stat(path) for _, directory := range []string{dcimDirectory, m4RootDirectory} {
if err == nil && info.IsDir() { found, err := hasDirectory(fsys, directory)
slog.Debug(fmt.Sprintf("Found %s directory at %s\n", directory, path)) if err == nil && found {
slog.Debug(fmt.Sprintf("Found %s directory at %s\n", directory, mountpoint))
return true return true
} }
+15
View File
@@ -57,6 +57,21 @@ func TestIsDCFMountPointIgnoresPlainFile(t *testing.T) {
} }
} }
func TestIsDCFMountPointIgnoresSymlink(t *testing.T) {
t.Parallel()
outside := newMountPoint(t, "M4ROOT")
for _, link := range []string{dcimDirectory, privateDirectory, m4RootDirectory} {
root := newMountPoint(t, filepath.Dir(link))
addSymlink(t, root, link, outside)
if isDCFMountPoint(root) {
t.Errorf("isDCFMountPoint counted a symlink at %s as a directory", link)
}
}
}
func TestFilterDCFMountPointsReturnsRequestedCount(t *testing.T) { func TestFilterDCFMountPointsReturnsRequestedCount(t *testing.T) {
t.Parallel() t.Parallel()
+76 -50
View File
@@ -2,12 +2,12 @@ package dcf
import ( import (
"errors" "errors"
"fmt"
"io/fs" "io/fs"
"path" "path"
"regexp" "regexp"
"strconv" "strconv"
"strings" "strings"
"syscall"
) )
// The two directories at the root of a card that hold its files. // The two directories at the root of a card that hold its files.
@@ -26,43 +26,38 @@ var (
// parseStore reads one card from fsys, which is rooted at the card's mount // parseStore reads one card from fsys, which is rooted at the card's mount
// point. The root argument is the path of that mount point. Only DCIM and // point. The root argument is the path of that mount point. Only DCIM and
// PRIVATE/M4ROOT are read. A card can lack either one. // PRIVATE/M4ROOT are read. A card can lack either one. A symlink is never
// followed. An error names the mount point, because fsys reports only paths
// inside it.
func parseStore(fsys fs.FS, root string) (*Store, error) { func parseStore(fsys fs.FS, root string) (*Store, error) {
store := &Store{RootDirectory: root} store := &Store{RootDirectory: root}
hasDCIM, err := hasDirectory(fsys, dcimDirectory) err := parseDCIM(fsys, store)
if err != nil { if err == nil {
return nil, err
}
if hasDCIM {
err = parseDCIM(fsys, store)
if err != nil {
return nil, err
}
}
hasM4Root, err := hasDirectory(fsys, m4RootDirectory)
if err != nil {
return nil, err
}
if hasM4Root {
err = parseM4Root(fsys, store) err = parseM4Root(fsys, store)
if err != nil { }
return nil, err
} if err != nil {
return nil, fmt.Errorf("reading the card at %s: %w", root, err)
} }
return store, nil return store, nil
} }
// hasDirectory reports whether the card holds the named directory. Only a // hasDirectory reports whether the card holds the named directory. A plain file
// missing directory counts as false: any other failure to look is an error. A // or a symlink at any step of the path counts as missing. Any other failure to
// plain file named PRIVATE makes a lookup of PRIVATE/M4ROOT fail with ENOTDIR. // look is an error.
func hasDirectory(fsys fs.FS, name string) (bool, error) { func hasDirectory(fsys fs.FS, name string) (bool, error) {
info, err := fs.Stat(fsys, name) parent := path.Dir(name)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { if parent != "." {
found, err := hasDirectory(fsys, parent)
if err != nil || !found {
return false, err
}
}
info, err := fs.Lstat(fsys, name)
if errors.Is(err, fs.ErrNotExist) {
return false, nil return false, nil
} }
@@ -77,6 +72,11 @@ func hasDirectory(fsys fs.FS, name string) (bool, error) {
// is not conforming. That includes directories that share a directory number, // is not conforming. That includes directories that share a directory number,
// which section 7.1.2 of the specification treats as non-DCF directories. // which section 7.1.2 of the specification treats as non-DCF directories.
func parseDCIM(fsys fs.FS, store *Store) error { func parseDCIM(fsys fs.FS, store *Store) error {
found, err := hasDirectory(fsys, dcimDirectory)
if err != nil || !found {
return err
}
entries, err := fs.ReadDir(fsys, dcimDirectory) entries, err := fs.ReadDir(fsys, dcimDirectory)
if err != nil { if err != nil {
return err return err
@@ -96,8 +96,7 @@ func parseDCIM(fsys fs.FS, store *Store) error {
if entry.IsDir() && number != 0 && directoriesWithNumber[number] == 1 { if entry.IsDir() && number != 0 && directoriesWithNumber[number] == 1 {
err = parseDirectory(fsys, store, entry.Name(), number) err = parseDirectory(fsys, store, entry.Name(), number)
} else { } else {
entryPath := path.Join(dcimDirectory, entry.Name()) err = addNonConforming(fsys, store, dcimDirectory, entry)
err = addNonConforming(fsys, store, entryPath)
} }
if err != nil { if err != nil {
@@ -109,9 +108,10 @@ func parseDCIM(fsys fs.FS, store *Store) error {
} }
// parseDirectory adds one DCF directory and the DCF files directly inside it to // parseDirectory adds one DCF directory and the DCF files directly inside it to
// store. Everything else there is not conforming. That includes JPG files that // store. A DCF file is a regular file, so a symlink with a DCF name is not one.
// share a file number, which section 7.2.2 of the specification says must not // Everything else there is not conforming. That includes JPG files that share a
// show DCF numbers. Other files that share a file number do conform. // file number, which section 7.2.2 of the specification says must not show DCF
// numbers. Other files that share a file number do conform.
func parseDirectory(fsys fs.FS, store *Store, name string, number int) error { func parseDirectory(fsys fs.FS, store *Store, name string, number int) error {
directoryPath := path.Join(dcimDirectory, name) directoryPath := path.Join(dcimDirectory, name)
@@ -123,7 +123,7 @@ func parseDirectory(fsys fs.FS, store *Store, name string, number int) error {
jpgsWithNumber := map[int]int{} jpgsWithNumber := map[int]int{}
for _, entry := range entries { for _, entry := range entries {
if !entry.IsDir() && isJPG(entry.Name()) { if entry.Type().IsRegular() && isJPG(entry.Name()) {
jpgsWithNumber[parseNumber(dcfFileName, entry.Name())]++ jpgsWithNumber[parseNumber(dcfFileName, entry.Name())]++
} }
} }
@@ -131,12 +131,11 @@ func parseDirectory(fsys fs.FS, store *Store, name string, number int) error {
directory := &Directory{Number: number, Name: name} directory := &Directory{Number: number, Name: name}
for _, entry := range entries { for _, entry := range entries {
entryPath := path.Join(directoryPath, entry.Name())
fileNumber := parseNumber(dcfFileName, entry.Name()) fileNumber := parseNumber(dcfFileName, entry.Name())
sharedJPG := isJPG(entry.Name()) && jpgsWithNumber[fileNumber] > 1 sharedJPG := isJPG(entry.Name()) && jpgsWithNumber[fileNumber] > 1
if entry.IsDir() || fileNumber == 0 || sharedJPG { if !entry.Type().IsRegular() || fileNumber == 0 || sharedJPG {
err = addNonConforming(fsys, store, entryPath) err = addNonConforming(fsys, store, directoryPath, entry)
if err != nil { if err != nil {
return err return err
} }
@@ -144,6 +143,8 @@ func parseDirectory(fsys fs.FS, store *Store, name string, number int) error {
continue continue
} }
entryPath := path.Join(directoryPath, entry.Name())
object, err := newObject(store, entryPath, entry) object, err := newObject(store, entryPath, entry)
if err != nil { if err != nil {
return err return err
@@ -162,15 +163,28 @@ func parseDirectory(fsys fs.FS, store *Store, name string, number int) error {
} }
// parseM4Root adds the images and videos under PRIVATE/M4ROOT to store. The DCF // parseM4Root adds the images and videos under PRIVATE/M4ROOT to store. The DCF
// naming rules do not apply there, so these files have no DCF numbers. // naming rules do not apply there, so these files have no DCF numbers. It skips
// a symlink. It also skips a name that is not valid UTF-8, because fs.FS refuses
// to open one.
func parseM4Root(fsys fs.FS, store *Store) error { func parseM4Root(fsys fs.FS, store *Store) error {
found, err := hasDirectory(fsys, m4RootDirectory)
if err != nil || !found {
return err
}
return fs.WalkDir(fsys, m4RootDirectory, return fs.WalkDir(fsys, m4RootDirectory,
func(filePath string, entry fs.DirEntry, err error) error { func(filePath string, entry fs.DirEntry, err error) error {
if err != nil { if err != nil {
return err return err
} }
if entry.IsDir() { validName := fs.ValidPath(filePath)
if entry.IsDir() && !validName {
return fs.SkipDir
}
if !entry.Type().IsRegular() || !validName {
return nil return nil
} }
@@ -185,19 +199,31 @@ func parseM4Root(fsys fs.FS, store *Store) error {
}) })
} }
// addNonConforming lists the named file or directory on store as not // addNonConforming lists one entry of the directory dir on store as not
// conforming, along with everything inside it. // conforming. If the entry is a directory, it lists everything inside it too. It
func addNonConforming(fsys fs.FS, store *Store, name string) error { // does not follow a symlink. It does not read a directory whose name is not
return fs.WalkDir(fsys, name, // valid UTF-8, because fs.FS refuses to open one.
func(entryPath string, _ fs.DirEntry, err error) error { func addNonConforming(fsys fs.FS, store *Store, dir string, entry fs.DirEntry) error {
if err != nil { entryPath := path.Join(dir, entry.Name())
return err store.NonConforming = append(store.NonConforming, entryPath)
}
store.NonConforming = append(store.NonConforming, entryPath) if !entry.IsDir() || !fs.ValidPath(entryPath) {
return nil
}
return nil entries, err := fs.ReadDir(fsys, entryPath)
}) if err != nil {
return err
}
for _, inner := range entries {
err = addNonConforming(fsys, store, entryPath, inner)
if err != nil {
return err
}
}
return nil
} }
// newObject returns the Object for the file at filePath, with no DCF numbers. // newObject returns the Object for the file at filePath, with no DCF numbers.
+353 -30
View File
@@ -4,6 +4,7 @@ import (
"errors" "errors"
"io/fs" "io/fs"
"os" "os"
"path"
"path/filepath" "path/filepath"
"reflect" "reflect"
"slices" "slices"
@@ -11,7 +12,10 @@ import (
"testing/fstest" "testing/fstest"
) )
const testRoot = "/mnt/card" const (
testRoot = "/mnt/card"
privateDirectory = "PRIVATE"
)
// newCard returns a card holding an empty file at each of the given paths. // newCard returns a card holding an empty file at each of the given paths.
func newCard(paths ...string) fstest.MapFS { func newCard(paths ...string) fstest.MapFS {
@@ -24,6 +28,40 @@ func newCard(paths ...string) fstest.MapFS {
return card return card
} }
// newCardOnDisk returns a temporary directory holding the three bytes "abc" in a
// file at each of the given paths. It is for tests that fstest.MapFS cannot do.
func newCardOnDisk(t *testing.T, paths ...string) string {
t.Helper()
root := t.TempDir()
for _, filePath := range paths {
fullPath := filepath.Join(root, filePath)
err := os.MkdirAll(filepath.Dir(fullPath), 0o750)
if err != nil {
t.Fatalf("MkdirAll: %v", err)
}
err = os.WriteFile(fullPath, []byte("abc"), 0o600)
if err != nil {
t.Fatalf("WriteFile: %v", err)
}
}
return root
}
// addSymlink creates a symlink to target at the named path under root.
func addSymlink(t *testing.T, root, name, target string) {
t.Helper()
err := os.Symlink(target, filepath.Join(root, name))
if err != nil {
t.Fatalf("Symlink: %v", err)
}
}
func mustParseStore(t *testing.T, fsys fs.FS) *Store { func mustParseStore(t *testing.T, fsys fs.FS) *Store {
t.Helper() t.Helper()
@@ -216,6 +254,8 @@ func TestParseNumberOfDirectoryName(t *testing.T) {
"100ABC-E": 0, "100ABC-E": 0,
"100ABCD": 0, "100ABCD": 0,
"100ABCDEF": 0, "100ABCDEF": 0,
"1000ABCDE": 0,
"X100ABCDE": 0,
"TEMP": 0, "TEMP": 0,
} }
@@ -239,12 +279,14 @@ func TestParseNumberOfFileName(t *testing.T) {
"ABCD0000.JPG": 0, "ABCD0000.JPG": 0,
"A0004.JPG": 0, "A0004.JPG": 0,
"ABCDE0005.JPG": 0, "ABCDE0005.JPG": 0,
"ABCD10000.JPG": 0,
"abcdefg0006.JPG": 0, "abcdefg0006.JPG": 0,
"+-@]0007.JPG": 0, "+-@]0007.JPG": 0,
"ABCD00~1.JPG": 0, "ABCD00~1.JPG": 0,
"ABCDEFGH.JPG": 0, "ABCDEFGH.JPG": 0,
"ABCD0001": 0, "ABCD0001": 0,
"ABCD0001.": 0, "ABCD0001.": 0,
"ABCD0001.JP": 0,
"ABCD0001.JPEG": 0, "ABCD0001.JPEG": 0,
"ABCD0001.J-G": 0, "ABCD0001.J-G": 0,
"._DSC0001.JPG": 0, "._DSC0001.JPG": 0,
@@ -372,6 +414,49 @@ func TestParseStoreKeepsOtherFilesSharingAFileNumber(t *testing.T) {
assertPaths(t, "NonConforming", store.NonConforming) assertPaths(t, "NonConforming", store.NonConforming)
} }
func TestParseStoreKeepsARWSharingANumberWithTwoJPGs(t *testing.T) {
t.Parallel()
store := mustParseStore(t, newCard(
"DCIM/100ABCDE/ABCD0004.ARW",
"DCIM/100ABCDE/ABCD0004.JPG",
"DCIM/100ABCDE/PQRS0004.JPG",
))
assertPaths(t, "objects", objectPaths(store.Directories[0]),
"DCIM/100ABCDE/ABCD0004.ARW",
)
assertPaths(t, "NonConforming", store.NonConforming,
"DCIM/100ABCDE/ABCD0004.JPG",
"DCIM/100ABCDE/PQRS0004.JPG",
)
}
func TestParseStoreDoesNotCountDirectoryAsSharingAFileNumber(t *testing.T) {
t.Parallel()
store := mustParseStore(t, newCard(
"DCIM/100ABCDE/ABCD0003.JPG",
"DCIM/100ABCDE/PQRS0003.JPG/README.TXT",
))
assertPaths(t, "objects", objectPaths(store.Directories[0]),
"DCIM/100ABCDE/ABCD0003.JPG",
)
assertPaths(t, "NonConforming", store.NonConforming,
"DCIM/100ABCDE/PQRS0003.JPG",
"DCIM/100ABCDE/PQRS0003.JPG/README.TXT",
)
}
func TestParseStoreDoesNotCountDirectoryNamedLikeAVideo(t *testing.T) {
t.Parallel()
store := mustParseStore(t, newCard("PRIVATE/M4ROOT/CLIP.MP4/C0001.XML"))
assertPaths(t, "Videos", videoPaths(store))
}
func TestParseStoreRejectsSubdirectoryOfDCFDirectory(t *testing.T) { func TestParseStoreRejectsSubdirectoryOfDCFDirectory(t *testing.T) {
t.Parallel() t.Parallel()
@@ -441,24 +526,7 @@ func TestParseStoreTreatsPlainFilesAsMissingDirectories(t *testing.T) {
func TestParseStoreReadsRealFilesystemWithPlainFileNamedPrivate(t *testing.T) { func TestParseStoreReadsRealFilesystemWithPlainFileNamedPrivate(t *testing.T) {
t.Parallel() t.Parallel()
root := t.TempDir() root := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG", "PRIVATE")
err := os.MkdirAll(filepath.Join(root, "DCIM", "100ABCDE"), 0o750)
if err != nil {
t.Fatalf("MkdirAll: %v", err)
}
jpg := filepath.Join(root, "DCIM", "100ABCDE", "ABCD0001.JPG")
err = os.WriteFile(jpg, []byte("abc"), 0o600)
if err != nil {
t.Fatalf("WriteFile: %v", err)
}
err = os.WriteFile(filepath.Join(root, "PRIVATE"), nil, 0o600)
if err != nil {
t.Fatalf("WriteFile: %v", err)
}
store := mustParseStore(t, os.DirFS(root)) store := mustParseStore(t, os.DirFS(root))
@@ -469,18 +537,273 @@ func TestParseStoreReadsRealFilesystemWithPlainFileNamedPrivate(t *testing.T) {
} }
} }
// unreadableFS fails every read the way a failing card does. // A real filesystem can hold a name with the byte 0xff, which is not valid UTF-8.
type unreadableFS struct{} // fs.FS refuses to open such a name.
func TestParseStoreListsNamesThatFSRefuses(t *testing.T) {
func (unreadableFS) Open(string) (fs.File, error) {
return nil, fs.ErrPermission
}
func TestParseStoreReturnsReadErrors(t *testing.T) {
t.Parallel() t.Parallel()
_, err := parseStore(unreadableFS{}, testRoot) root := newCardOnDisk(t,
if !errors.Is(err, fs.ErrPermission) { "DCIM/100ABCDE/ABCD0001.JPG",
t.Fatalf("parseStore error = %v, want %v", err, fs.ErrPermission) "DCIM/100ABCDE/A\xffDIR/INNER",
"DCIM/100ABCDE/A\xffFILE",
"DCIM/A\xffDIR/INNER",
"DCIM/A\xffFILE",
"DCIM/MISC/A\xffDIR/INNER",
"DCIM/MISC/A\xffFILE",
)
store := mustParseStore(t, os.DirFS(root))
assertPaths(t, "Images", imagePaths(store), "DCIM/100ABCDE/ABCD0001.JPG")
assertPaths(t, "NonConforming", store.NonConforming,
"DCIM/100ABCDE/A\xffDIR",
"DCIM/100ABCDE/A\xffFILE",
"DCIM/A\xffDIR",
"DCIM/A\xffFILE",
"DCIM/MISC",
"DCIM/MISC/A\xffDIR",
"DCIM/MISC/A\xffFILE",
)
}
func TestParseStoreListsDanglingSymlinks(t *testing.T) {
t.Parallel()
root := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG", "DCIM/MISC/README.TXT")
addSymlink(t, root, "DCIM/100ABCDE/ABCD0002.JPG", "absent")
addSymlink(t, root, "DCIM/100ABCDE/LINK", "absent")
addSymlink(t, root, "DCIM/101ABCDE", "absent")
addSymlink(t, root, "DCIM/LINK", "absent")
addSymlink(t, root, "DCIM/MISC/LINK", "absent")
store := mustParseStore(t, os.DirFS(root))
if got, want := directoryNumbers(store), []int{100}; !slices.Equal(got, want) {
t.Errorf("directory numbers = %v, want %v", got, want)
}
assertPaths(t, "Images", imagePaths(store), "DCIM/100ABCDE/ABCD0001.JPG")
assertPaths(t, "NonConforming", store.NonConforming,
"DCIM/100ABCDE/ABCD0002.JPG",
"DCIM/100ABCDE/LINK",
"DCIM/101ABCDE",
"DCIM/LINK",
"DCIM/MISC",
"DCIM/MISC/LINK",
"DCIM/MISC/README.TXT",
)
}
func TestParseStoreDoesNotFollowSymlinksUnderDCIM(t *testing.T) {
t.Parallel()
outside := newCardOnDisk(t, "OUTSIDE.JPG", "DIRECTORY/ABCD0009.JPG")
outsideFile := filepath.Join(outside, "OUTSIDE.JPG")
outsideDirectory := filepath.Join(outside, "DIRECTORY")
root := newCardOnDisk(t,
"DCIM/100ABCDE/ABCD0001.JPG",
"DCIM/102ABCDE/ABCD0002.JPG",
"DCIM/MISC/README.TXT",
)
// These two have DCF file names. The second has the number of a real JPG,
// which must not make that JPG shared.
addSymlink(t, root, "DCIM/100ABCDE/ABCD0003.JPG", outsideFile)
addSymlink(t, root, "DCIM/100ABCDE/PQRS0001.JPG", outsideFile)
// These two have DCF directory names. The second has the number of a real
// directory, which must not make that directory shared.
addSymlink(t, root, "DCIM/101ABCDE", outsideDirectory)
addSymlink(t, root, "DCIM/102PQRST", outsideDirectory)
addSymlink(t, root, "DCIM/LINK", outsideDirectory)
addSymlink(t, root, "DCIM/MISC/LINK", outsideDirectory)
store := mustParseStore(t, os.DirFS(root))
if got, want := directoryNumbers(store), []int{100, 102}; !slices.Equal(got, want) {
t.Errorf("directory numbers = %v, want %v", got, want)
}
assertPaths(t, "Images", imagePaths(store),
"DCIM/100ABCDE/ABCD0001.JPG",
"DCIM/102ABCDE/ABCD0002.JPG",
)
if got := store.TotalImageSize(); got != 6 {
t.Errorf("TotalImageSize() = %d, want 6", got)
}
assertPaths(t, "NonConforming", store.NonConforming,
"DCIM/100ABCDE/ABCD0003.JPG",
"DCIM/100ABCDE/PQRS0001.JPG",
"DCIM/101ABCDE",
"DCIM/102PQRST",
"DCIM/LINK",
"DCIM/MISC",
"DCIM/MISC/LINK",
"DCIM/MISC/README.TXT",
)
}
func TestParseStoreSkipsSymlinksAndNamesThatFSRefusesUnderM4Root(t *testing.T) {
t.Parallel()
outside := newCardOnDisk(t, "OUTSIDE.MP4")
// The names with the byte 0xff sort before CLIP, so a walk that stops at one
// of them does not reach it.
root := newCardOnDisk(t,
"PRIVATE/M4ROOT/A\xffDIR/C0002.MP4",
"PRIVATE/M4ROOT/A\xffFILE.MP4",
"PRIVATE/M4ROOT/CLIP/C0001.MP4",
)
addSymlink(t, root, "PRIVATE/M4ROOT/CLIP/C0003.MP4",
filepath.Join(outside, "OUTSIDE.MP4"))
addSymlink(t, root, "PRIVATE/M4ROOT/CLIP/C0004.MP4", "absent")
addSymlink(t, root, "PRIVATE/M4ROOT/LINK", outside)
store := mustParseStore(t, os.DirFS(root))
assertPaths(t, "Videos", videoPaths(store), "PRIVATE/M4ROOT/CLIP/C0001.MP4")
if got := store.TotalVideoSize(); got != 3 {
t.Errorf("TotalVideoSize() = %d, want 3", got)
}
}
func TestParseStoreTreatsSymlinksAsMissingDirectories(t *testing.T) {
t.Parallel()
outside := newCardOnDisk(t,
"100ABCDE/ABCD0001.JPG",
"M4ROOT/C0001.MP4",
"C0002.MP4",
)
cases := []struct{ link, target string }{
{dcimDirectory, outside},
{dcimDirectory, dcimDirectory},
{privateDirectory, outside},
{privateDirectory, privateDirectory},
{m4RootDirectory, outside},
}
for _, testCase := range cases {
root := newMountPoint(t, filepath.Dir(testCase.link))
addSymlink(t, root, testCase.link, testCase.target)
store := mustParseStore(t, os.DirFS(root))
if !reflect.DeepEqual(store, &Store{RootDirectory: testRoot}) {
t.Errorf("symlink at %s to %s read as %+v, want an empty store",
testCase.link, testCase.target, store)
}
}
}
// failingFS is a card on which one call fails for one path, the way a failing
// card does. The call is Lstat, ReadDir or Info.
type failingFS struct {
fstest.MapFS
call string
name string
}
func (f failingFS) Lstat(name string) (fs.FileInfo, error) {
err := f.failure("Lstat", name)
if err != nil {
return nil, err
}
return f.MapFS.Lstat(name)
}
func (f failingFS) ReadDir(name string) ([]fs.DirEntry, error) {
err := f.failure("ReadDir", name)
if err != nil {
return nil, err
}
entries, err := f.MapFS.ReadDir(name)
if err != nil {
return nil, err
}
for i, entry := range entries {
err = f.failure("Info", path.Join(name, entry.Name()))
if err != nil {
entries[i] = failingEntry{DirEntry: entry, err: err}
}
}
return entries, nil
}
func (f failingFS) failure(call, name string) error {
if call == f.call && name == f.name {
return &fs.PathError{Op: call, Path: name, Err: fs.ErrPermission}
}
return nil
}
// failingEntry is a directory entry whose Info always returns err.
type failingEntry struct {
fs.DirEntry
err error
}
func (e failingEntry) Info() (fs.FileInfo, error) {
return nil, e.err
}
func TestParseStoreReturnsReadErrorsNamingTheCard(t *testing.T) {
t.Parallel()
card := newCard(
"DCIM/100ABCDE/ABCD0001.JPG",
"DCIM/100ABCDE/SUB/README.TXT",
"DCIM/MISC/SUB/README.TXT",
"PRIVATE/M4ROOT/CLIP/C0001.MP4",
)
cases := map[string][]string{
"Lstat": {dcimDirectory, privateDirectory, m4RootDirectory},
"ReadDir": {
dcimDirectory,
"DCIM/100ABCDE",
"DCIM/100ABCDE/SUB",
"DCIM/MISC",
"DCIM/MISC/SUB",
m4RootDirectory,
"PRIVATE/M4ROOT/CLIP",
},
"Info": {"DCIM/100ABCDE/ABCD0001.JPG", "PRIVATE/M4ROOT/CLIP/C0001.MP4"},
}
for call, names := range cases {
for _, name := range names {
fsys := failingFS{MapFS: card, call: call, name: name}
_, err := parseStore(fsys, testRoot)
if !errors.Is(err, fs.ErrPermission) {
t.Errorf("%s of %s: error = %v, want %v",
call, name, err, fs.ErrPermission)
continue
}
want := "reading the card at " + testRoot + ": " +
call + " " + name + ": permission denied"
if err.Error() != want {
t.Errorf("error = %q, want %q", err.Error(), want)
}
}
} }
} }