Compare commits

..
1 Commits
Author SHA1 Message Date
clawbot caa73fbcca group DCF files that share a file number (closes #3)
Files in one DCF directory that share a file number are one file group,
as section 4.3.2.2 of the specification says. `Directory.FileGroups`
lists them, each with its numbers, its members and its primary member. A
lone file is a group of one. A group reports its member count and total
size.

Section 6.2.1.3 gives a DCF basic file priority over a DCF extended image
file. It says nothing about a DCF optional file, so ranking every `JPG`
first is a choice. A `THM` comes last.

`isImageFile` now accepts `.tif`, so a `TIF` counts as an image.
`Store.Images`, `Store.Videos` and the totals still count each file on
its own.

Model: opus-5-5
2026-11-03 18:51:48 +00:00
8 changed files with 113 additions and 350 deletions

No files matched your search

+7 -16
View File
@@ -29,7 +29,6 @@ Then list the DCF stores on the mounted filesystems:
package main package main
import ( import (
"context"
"fmt" "fmt"
"log" "log"
@@ -37,21 +36,17 @@ import (
) )
func main() { func main() {
stores, err := dcf.GetStores(context.Background()) stores, err := dcf.GetDCFStores(0)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
for _, store := range stores { for _, store := range *stores {
fmt.Println(store.String()) fmt.Println(store.String())
} }
} }
``` ```
Call `dcf.GetStore` instead when you expect exactly one DCF store. It returns
`dcf.ErrNoStore` when none is mounted, and `dcf.ErrMultipleStores` when more
than one is.
To work on the library itself, you need docker and git: To work on the library itself, you need docker and git:
```sh ```sh
@@ -110,8 +105,8 @@ overengineering and reusable code.
All the code is in the `pkg/dcf` package. The repo has no commands. All the code is in the `pkg/dcf` package. The repo has no commands.
- `dcf.go` holds the public API. `GetStores` finds the DCF stores and reads each - `dcf.go` holds the public API. `GetDCFStores` finds the DCF stores and reads
one. `GetStore` returns the one DCF store when exactly one is mounted. each one.
- `helpers.go` finds the mount points that hold a DCF store. - `helpers.go` finds the mount points that hold a DCF store.
- `parse.go` reads one DCF store through an `fs.FS` rooted at its mount point. - `parse.go` reads one DCF store through an `fs.FS` rooted at its mount point.
- `filegroup.go` sorts the files of one DCF directory into file groups. - `filegroup.go` sorts the files of one DCF directory into file groups.
@@ -122,9 +117,7 @@ The main types are:
- `Directory` is one DCF directory. It has a directory number, a name, the - `Directory` is one DCF directory. It has a directory number, a name, the
objects inside it and their file groups. objects inside it and their file groups.
- `Object` is one file in a store. A file in a DCF directory also has a - `Object` is one file in a store. A file in a DCF directory also has a
directory number and a file number. `Object.Hash` reads the file through the directory number and a file number.
`fs.FS` of its store. An `Object` you build yourself has none, so its `Hash`
returns an error.
- `FileGroup` is one file group in a DCF directory. - `FileGroup` is one file group in a DCF directory.
- `Image` and `Video` embed `Object`. The file extension decides which one a - `Image` and `Video` embed `Object`. The file extension decides which one a
file becomes. A file with any other extension, such as a `THM` thumbnail, is file becomes. A file with any other extension, such as a `THM` thumbnail, is
@@ -136,10 +129,6 @@ filesystem. Media anywhere else on the filesystem is not part of the store.
`dcf` never follows a symlink. `dcf` never follows a symlink.
`GetStores` and `GetStore` take a `context.Context`. When it is cancelled, the
read stops before the next directory. The returned error matches
`context.Canceled`.
`dcf` applies the naming rules of the DCF specification, CIPA DC-009-2010, under `dcf` applies the naming rules of the DCF specification, CIPA DC-009-2010, under
`DCIM`: `DCIM`:
@@ -193,6 +182,8 @@ The primary member is the first of these that the group holds:
## TODO ## TODO
- Take a `context.Context` to cancel the walk of a store.
- Return a plain slice from `GetDCFStores`, not a pointer to one.
- Copy files off a card, and verify each copy against its SHA256 hash. - Copy files off a card, and verify each copy against its SHA256 hash.
## License ## License
+21 -57
View File
@@ -7,13 +7,10 @@
package dcf package dcf
import ( import (
"context"
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
"errors"
"fmt" "fmt"
"io" "io"
"io/fs"
"os" "os"
"path/filepath" "path/filepath"
"slices" "slices"
@@ -22,14 +19,6 @@ import (
"github.com/dustin/go-humanize" "github.com/dustin/go-humanize"
) )
var (
// ErrNoStore is the error from GetStore when no DCF store is mounted.
ErrNoStore = errors.New("no DCF store is mounted")
// ErrMultipleStores is the error from GetStore when more than one DCF store
// is mounted.
ErrMultipleStores = errors.New("more than one DCF store is mounted")
)
// Store represents a DCF store, which is a directory structure that contains // Store represents a DCF store, which is a directory structure that contains
// images and videos in the root of a mounted filesystem. // images and videos in the root of a mounted filesystem.
type Store struct { type Store struct {
@@ -87,9 +76,6 @@ type Object struct {
// directory. Both are 0 for a file under PRIVATE/M4ROOT. // directory. Both are 0 for a file under PRIVATE/M4ROOT.
DirectoryNumber int DirectoryNumber int
FileNumber int FileNumber int
// fsys is the fs.FS the store was read through. Hash reads the file through
// it.
fsys fs.FS
} }
// Image represents an image file on a DCF store. // Image represents an image file on a DCF store.
@@ -187,17 +173,15 @@ func (i *Image) String() string {
) )
} }
// Hash returns the SHA256 hash of the file as a hex string. It reads the file // Hash returns the SHA256 hash of the file as a hex string.
// through the fs.FS of its store. An Object built by hand has none, so Hash
// returns an error that matches fs.ErrInvalid.
func (d *Object) Hash() (string, error) { func (d *Object) Hash() (string, error) {
if d.fsys == nil { return pathToSHA256(d.FullFilePath())
return "", fmt.Errorf("hashing %s: %w", d.FullFilePath(), fs.ErrInvalid)
} }
file, err := d.fsys.Open(d.Path) func pathToSHA256(path string) (string, error) {
file, err := os.Open(filepath.Clean(path))
if err != nil { if err != nil {
return "", fmt.Errorf("hashing %s: %w", d.FullFilePath(), err) return "", err
} }
defer func() { _ = file.Close() }() defer func() { _ = file.Close() }()
@@ -206,64 +190,44 @@ func (d *Object) Hash() (string, error) {
_, err = io.Copy(hash, file) _, err = io.Copy(hash, file)
if err != nil { if err != nil {
return "", fmt.Errorf("hashing %s: %w", d.FullFilePath(), err) return "", err
} }
return hex.EncodeToString(hash.Sum(nil)), nil return hex.EncodeToString(hash.Sum(nil)), nil
} }
// GetStores finds the DCF stores in the root of the mounted filesystems and reads // GetDCFStores returns a list of DCF stores found on the system in the root of
// each one. It does not mount filesystems or search outside of filesystem root // any mounted filesystems. The integer argument is the number of DCF stores to
// directories. It reads the DCIM and PRIVATE/M4ROOT directories of each // return. If the argument is 0, all DCF stores are returned. It does not mount
// filesystem to populate the returned Stores. It never follows a symlink. If a // filesystems or search outside of filesystem root directories. It does,
// card cannot be read, it returns an error that names the card's mount point. // however, read the DCIM and PRIVATE/M4ROOT directories of each filesystem to
// When ctx is cancelled, the read stops before the next directory, and the error // populate the returned Stores. It never follows a symlink. If a card cannot be
// matches ctx.Err(). // read, it returns an error that names the card's mount point.
func GetStores(ctx context.Context) ([]*Store, error) { func GetDCFStores(requestedCount int) (*[]Store, error) {
dcfStorePaths, err := findDCFMountPoints(ctx) dcfStorePaths, err := findDCFMountPoints(requestedCount)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return parseStores(ctx, dcfStorePaths) dcfStores, err := parseStores(dcfStorePaths)
}
// GetStore returns the one DCF store when exactly one is mounted. It returns
// ErrNoStore when none is mounted, and ErrMultipleStores when more than one is.
// It reads every card first, the way GetStores does, so a card that cannot be
// read returns that error instead.
func GetStore(ctx context.Context) (*Store, error) {
stores, err := GetStores(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return onlyStore(stores) return &dcfStores, nil
}
// onlyStore returns the store when stores holds exactly one.
func onlyStore(stores []*Store) (*Store, error) {
switch len(stores) {
case 0:
return nil, ErrNoStore
case 1:
return stores[0], nil
default:
return nil, ErrMultipleStores
}
} }
// parseStores reads the card at each of the given mount points. // parseStores reads the card at each of the given mount points.
func parseStores(ctx context.Context, mountpoints []string) ([]*Store, error) { func parseStores(mountpoints []string) ([]Store, error) {
dcfStores := []*Store{} dcfStores := []Store{}
for _, mountpoint := range mountpoints { for _, mountpoint := range mountpoints {
dcfStore, err := parseStore(ctx, os.DirFS(mountpoint), mountpoint) dcfStore, err := parseStore(os.DirFS(mountpoint), mountpoint)
if err != nil { if err != nil {
return nil, err return nil, err
} }
dcfStores = append(dcfStores, dcfStore) dcfStores = append(dcfStores, *dcfStore)
} }
return dcfStores, nil return dcfStores, nil
+2 -86
View File
@@ -1,13 +1,9 @@
package dcf package dcf
import ( import (
"context"
"errors"
"io/fs"
"path/filepath" "path/filepath"
"strings" "strings"
"testing" "testing"
"testing/fstest"
) )
func TestParseStoresReadsEachMountPoint(t *testing.T) { func TestParseStoresReadsEachMountPoint(t *testing.T) {
@@ -16,7 +12,7 @@ func TestParseStoresReadsEachMountPoint(t *testing.T) {
first := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG") first := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG")
second := newCardOnDisk(t, "PRIVATE/M4ROOT/CLIP/C0001.MP4") second := newCardOnDisk(t, "PRIVATE/M4ROOT/CLIP/C0001.MP4")
stores, err := parseStores(t.Context(), []string{first, second}) stores, err := parseStores([]string{first, second})
if err != nil { if err != nil {
t.Fatalf("parseStores: %v", err) t.Fatalf("parseStores: %v", err)
} }
@@ -54,92 +50,12 @@ func TestParseStoresReturnsErrorNamingTheCard(t *testing.T) {
// Every lookup under a plain file fails. // Every lookup under a plain file fails.
mountpoint := filepath.Join(newCardOnDisk(t, "FILE"), "FILE") mountpoint := filepath.Join(newCardOnDisk(t, "FILE"), "FILE")
_, err := parseStores(t.Context(), []string{mountpoint}) _, err := parseStores([]string{mountpoint})
if err == nil || !strings.Contains(err.Error(), mountpoint) { if err == nil || !strings.Contains(err.Error(), mountpoint) {
t.Fatalf("parseStores error = %v, want one naming %s", err, mountpoint) t.Fatalf("parseStores error = %v, want one naming %s", err, mountpoint)
} }
} }
func TestParseStoresReturnsErrorWhenCancelled(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(t.Context())
cancel()
mountpoint := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG")
_, err := parseStores(ctx, []string{mountpoint})
if !errors.Is(err, context.Canceled) {
t.Fatalf("parseStores error = %v, want %v", err, context.Canceled)
}
}
func TestOnlyStore(t *testing.T) {
t.Parallel()
first := newMountPoint(t, dcimDirectory)
second := newMountPoint(t, m4RootDirectory)
stores, err := parseStores(t.Context(), []string{first, second})
if err != nil {
t.Fatalf("parseStores: %v", err)
}
_, err = onlyStore(stores[:0])
if !errors.Is(err, ErrNoStore) {
t.Errorf("onlyStore of no cards: error = %v, want %v", err, ErrNoStore)
}
store, err := onlyStore(stores[:1])
if err != nil {
t.Fatalf("onlyStore of one card: %v", err)
}
if store.RootDirectory != first {
t.Errorf("onlyStore returned the card at %s, want %s",
store.RootDirectory, first)
}
_, err = onlyStore(stores)
if !errors.Is(err, ErrMultipleStores) {
t.Errorf("onlyStore of two cards: error = %v, want %v",
err, ErrMultipleStores)
}
}
func TestObjectHashReadsThroughTheCard(t *testing.T) {
t.Parallel()
card := fstest.MapFS{"DCIM/100ABCDE/ABCD0001.JPG": {Data: []byte("abc")}}
got, err := mustParseStore(t, card).Images[0].Hash()
if err != nil {
t.Fatalf("Hash() returned error: %v", err)
}
// The published SHA-256 test vector for "abc".
want := "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
if got != want {
t.Fatalf("Hash() = %q, want %q", got, want)
}
}
func TestObjectHashOfMissingFileReturnsError(t *testing.T) {
t.Parallel()
const filePath = "DCIM/100ABCDE/ABCD0001.JPG"
card := newCard(filePath)
image := mustParseStore(t, card).Images[0]
delete(card, filePath)
_, err := image.Hash()
if !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("Hash() error = %v, want %v", err, fs.ErrNotExist)
}
}
func TestIsImageFile(t *testing.T) { func TestIsImageFile(t *testing.T) {
t.Parallel() t.Parallel()
+30 -24
View File
@@ -1,9 +1,8 @@
package dcf_test package dcf_test
import ( import (
"context" "os"
"errors" "path/filepath"
"io/fs"
"testing" "testing"
"sneak.berlin/go/dcf/pkg/dcf" "sneak.berlin/go/dcf/pkg/dcf"
@@ -126,30 +125,37 @@ func TestVideoString(t *testing.T) {
} }
} }
func TestObjectHashOfObjectBuiltByHandReturnsError(t *testing.T) { func TestObjectHashIsSHA256OfFile(t *testing.T) {
t.Parallel() t.Parallel()
object := dcf.Object{DCFStoreRoot: cardRoot, Path: "a.jpg"} root := t.TempDir()
err := os.WriteFile(filepath.Join(root, "a.jpg"), []byte("abc"), 0o600)
if err != nil {
t.Fatalf("WriteFile: %v", err)
}
object := dcf.Object{DCFStoreRoot: root, Path: "a.jpg"}
got, err := object.Hash()
if err != nil {
t.Fatalf("Hash() returned error: %v", err)
}
// The published SHA-256 test vector for "abc".
want := "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
if got != want {
t.Fatalf("Hash() = %q, want %q", got, want)
}
}
func TestObjectHashOfMissingFileReturnsError(t *testing.T) {
t.Parallel()
object := dcf.Object{DCFStoreRoot: t.TempDir(), Path: "absent.jpg"}
_, err := object.Hash() _, err := object.Hash()
if !errors.Is(err, fs.ErrInvalid) { if err == nil {
t.Fatalf("Hash() error = %v, want %v", err, fs.ErrInvalid) t.Fatal("Hash() of a missing file returned no error")
}
}
func TestGetStoresAndGetStoreReturnCancellation(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(t.Context())
cancel()
_, err := dcf.GetStores(ctx)
if !errors.Is(err, context.Canceled) {
t.Errorf("GetStores error = %v, want %v", err, context.Canceled)
}
_, err = dcf.GetStore(ctx)
if !errors.Is(err, context.Canceled) {
t.Errorf("GetStore error = %v, want %v", err, context.Canceled)
} }
} }
+15 -17
View File
@@ -1,7 +1,6 @@
package dcf package dcf
import ( import (
"context"
"fmt" "fmt"
"log/slog" "log/slog"
"os" "os"
@@ -10,19 +9,12 @@ import (
"github.com/shirou/gopsutil/disk" "github.com/shirou/gopsutil/disk"
) )
func findAllMountPoints(ctx context.Context) ([]string, error) { func findAllMountPoints() ([]string, error) {
mountpoints := []string{} mountpoints := []string{}
// disk.PartitionsWithContext does not check ctx on Linux. partitions, err := disk.Partitions(false) // physical devices only, so false
err := ctx.Err()
if err != nil { if err != nil {
return nil, fmt.Errorf("listing the mounted filesystems: %w", err) return nil, err
}
// physical devices only, so false
partitions, err := disk.PartitionsWithContext(ctx, false)
if err != nil {
return nil, fmt.Errorf("listing the mounted filesystems: %w", err)
} }
for _, partition := range partitions { for _, partition := range partitions {
@@ -34,18 +26,21 @@ func findAllMountPoints(ctx context.Context) ([]string, error) {
return mountpoints, nil return mountpoints, nil
} }
// findDCFMountPoints returns the paths of the mountpoints that hold a DCF store. // findDCFMountPoints returns a list of strings of the paths of mountpoints
func findDCFMountPoints(ctx context.Context) ([]string, error) { // that hold a DCF store. Its only argument is an integer of how many mountpoints
mountpoints, err := findAllMountPoints(ctx) // to return. If the argument is 0, all mountpoints are returned.
func findDCFMountPoints(requestedCount int) ([]string, error) {
mountpoints, err := findAllMountPoints()
if err != nil { if err != nil {
return nil, err return nil, err
} }
return filterDCFMountPoints(mountpoints), nil return filterDCFMountPoints(mountpoints, requestedCount), nil
} }
// filterDCFMountPoints returns the mountpoints that hold a DCF store. // filterDCFMountPoints returns the first requestedCount of the mountpoints that
func filterDCFMountPoints(mountpoints []string) []string { // hold a DCF store. If requestedCount is 0, it returns all of them.
func filterDCFMountPoints(mountpoints []string, requestedCount int) []string {
filteredMountpoints := []string{} filteredMountpoints := []string{}
for _, mountpoint := range mountpoints { for _, mountpoint := range mountpoints {
@@ -56,6 +51,9 @@ func filterDCFMountPoints(mountpoints []string) []string {
slog.Debug(fmt.Sprintf("Keeping mountpoint %s\n", mountpoint)) slog.Debug(fmt.Sprintf("Keeping mountpoint %s\n", mountpoint))
filteredMountpoints = append(filteredMountpoints, mountpoint) filteredMountpoints = append(filteredMountpoints, mountpoint)
if (requestedCount > 0) && (len(filteredMountpoints) >= requestedCount) {
break
}
} }
return filteredMountpoints return filteredMountpoints
+17 -4
View File
@@ -72,16 +72,29 @@ func TestIsDCFMountPointIgnoresSymlink(t *testing.T) {
} }
} }
func TestFilterDCFMountPointsKeepsThoseHoldingAStore(t *testing.T) { func TestFilterDCFMountPointsReturnsRequestedCount(t *testing.T) {
t.Parallel() t.Parallel()
first := newMountPoint(t, "DCIM") first := newMountPoint(t, "DCIM")
second := newMountPoint(t, "PRIVATE/M4ROOT") second := newMountPoint(t, "PRIVATE/M4ROOT")
third := newMountPoint(t, "DCIM")
got := filterDCFMountPoints([]string{newMountPoint(t), first, second}) // This one holds no DCF store, so it must not use up the requested count.
mountpoints := []string{newMountPoint(t), first, second, third}
want := []string{first, second} cases := map[int][]string{
0: {first, second, third},
1: {first},
2: {first, second},
3: {first, second, third},
4: {first, second, third},
}
for requestedCount, want := range cases {
got := filterDCFMountPoints(mountpoints, requestedCount)
if !slices.Equal(got, want) { if !slices.Equal(got, want) {
t.Errorf("filterDCFMountPoints = %v, want %v", got, want) t.Errorf("filterDCFMountPoints(%d) = %v, want %v",
requestedCount, got, want)
}
} }
} }
+18 -52
View File
@@ -1,7 +1,6 @@
package dcf package dcf
import ( import (
"context"
"errors" "errors"
"fmt" "fmt"
"io/fs" "io/fs"
@@ -29,13 +28,13 @@ var (
// point. The root argument is the path of that mount point. Only DCIM and // point. The root argument is the path of that mount point. Only DCIM and
// PRIVATE/M4ROOT are read. A card can lack either one. A symlink is never // PRIVATE/M4ROOT are read. A card can lack either one. A symlink is never
// followed. An error names the mount point, because fsys reports only paths // followed. An error names the mount point, because fsys reports only paths
// inside it. When ctx is cancelled, the read stops before the next directory. // inside it.
func parseStore(ctx context.Context, fsys fs.FS, root string) (*Store, error) { func parseStore(fsys fs.FS, root string) (*Store, error) {
store := &Store{RootDirectory: root} store := &Store{RootDirectory: root}
err := parseDCIM(ctx, fsys, store) err := parseDCIM(fsys, store)
if err == nil { if err == nil {
err = parseM4Root(ctx, fsys, store) err = parseM4Root(fsys, store)
} }
if err != nil { if err != nil {
@@ -69,26 +68,16 @@ func hasDirectory(fsys fs.FS, name string) (bool, error) {
return info.IsDir(), nil return info.IsDir(), nil
} }
// readDir returns the entries of the named directory, unless ctx is cancelled.
func readDir(ctx context.Context, fsys fs.FS, name string) ([]fs.DirEntry, error) {
err := ctx.Err()
if err != nil {
return nil, err
}
return fs.ReadDir(fsys, name)
}
// parseDCIM adds the DCF directories under DCIM to store. Everything else there // parseDCIM adds the DCF directories under DCIM to store. Everything else there
// is not conforming. That includes directories that share a directory number, // is not conforming. That includes directories that share a directory number,
// which section 7.1.2 of the specification treats as non-DCF directories. // which section 7.1.2 of the specification treats as non-DCF directories.
func parseDCIM(ctx context.Context, fsys fs.FS, store *Store) error { func parseDCIM(fsys fs.FS, store *Store) error {
found, err := hasDirectory(fsys, dcimDirectory) found, err := hasDirectory(fsys, dcimDirectory)
if err != nil || !found { if err != nil || !found {
return err return err
} }
entries, err := readDir(ctx, fsys, dcimDirectory) entries, err := fs.ReadDir(fsys, dcimDirectory)
if err != nil { if err != nil {
return err return err
} }
@@ -105,9 +94,9 @@ func parseDCIM(ctx context.Context, fsys fs.FS, store *Store) error {
number := parseNumber(dcfDirectoryName, entry.Name()) number := parseNumber(dcfDirectoryName, entry.Name())
if entry.IsDir() && number != 0 && directoriesWithNumber[number] == 1 { if entry.IsDir() && number != 0 && directoriesWithNumber[number] == 1 {
err = parseDirectory(ctx, fsys, store, entry.Name(), number) err = parseDirectory(fsys, store, entry.Name(), number)
} else { } else {
err = addNonConforming(ctx, fsys, store, dcimDirectory, entry) err = addNonConforming(fsys, store, dcimDirectory, entry)
} }
if err != nil { if err != nil {
@@ -123,16 +112,10 @@ func parseDCIM(ctx context.Context, fsys fs.FS, store *Store) error {
// Everything else there is not conforming. That includes JPG files that share a // Everything else there is not conforming. That includes JPG files that share a
// file number, which section 7.2.2 of the specification says must not show DCF // file number, which section 7.2.2 of the specification says must not show DCF
// numbers. Other files that share a file number do conform. // numbers. Other files that share a file number do conform.
func parseDirectory( func parseDirectory(fsys fs.FS, store *Store, name string, number int) error {
ctx context.Context,
fsys fs.FS,
store *Store,
name string,
number int,
) error {
directoryPath := path.Join(dcimDirectory, name) directoryPath := path.Join(dcimDirectory, name)
entries, err := readDir(ctx, fsys, directoryPath) entries, err := fs.ReadDir(fsys, directoryPath)
if err != nil { if err != nil {
return err return err
} }
@@ -152,7 +135,7 @@ func parseDirectory(
sharedJPG := isJPG(entry.Name()) && jpgsWithNumber[fileNumber] > 1 sharedJPG := isJPG(entry.Name()) && jpgsWithNumber[fileNumber] > 1
if !entry.Type().IsRegular() || fileNumber == 0 || sharedJPG { if !entry.Type().IsRegular() || fileNumber == 0 || sharedJPG {
err = addNonConforming(ctx, fsys, store, directoryPath, entry) err = addNonConforming(fsys, store, directoryPath, entry)
if err != nil { if err != nil {
return err return err
} }
@@ -162,7 +145,7 @@ func parseDirectory(
entryPath := path.Join(directoryPath, entry.Name()) entryPath := path.Join(directoryPath, entry.Name())
object, err := newObject(fsys, store, entryPath, entry) object, err := newObject(store, entryPath, entry)
if err != nil { if err != nil {
return err return err
} }
@@ -184,7 +167,7 @@ func parseDirectory(
// naming rules do not apply there, so these files have no DCF numbers. It skips // naming rules do not apply there, so these files have no DCF numbers. It skips
// a symlink. It also skips a name that is not valid UTF-8, because fs.FS refuses // a symlink. It also skips a name that is not valid UTF-8, because fs.FS refuses
// to open one. // to open one.
func parseM4Root(ctx context.Context, fsys fs.FS, store *Store) error { func parseM4Root(fsys fs.FS, store *Store) error {
found, err := hasDirectory(fsys, m4RootDirectory) found, err := hasDirectory(fsys, m4RootDirectory)
if err != nil || !found { if err != nil || !found {
return err return err
@@ -202,16 +185,11 @@ func parseM4Root(ctx context.Context, fsys fs.FS, store *Store) error {
return fs.SkipDir return fs.SkipDir
} }
// fs.WalkDir reads a directory right after this call for it.
if entry.IsDir() {
return ctx.Err()
}
if !entry.Type().IsRegular() || !validName { if !entry.Type().IsRegular() || !validName {
return nil return nil
} }
object, err := newObject(fsys, store, filePath, entry) object, err := newObject(store, filePath, entry)
if err != nil { if err != nil {
return err return err
} }
@@ -226,13 +204,7 @@ func parseM4Root(ctx context.Context, fsys fs.FS, store *Store) error {
// conforming. If the entry is a directory, it lists everything inside it too. It // conforming. If the entry is a directory, it lists everything inside it too. It
// does not follow a symlink. It does not read a directory whose name is not // does not follow a symlink. It does not read a directory whose name is not
// valid UTF-8, because fs.FS refuses to open one. // valid UTF-8, because fs.FS refuses to open one.
func addNonConforming( func addNonConforming(fsys fs.FS, store *Store, dir string, entry fs.DirEntry) error {
ctx context.Context,
fsys fs.FS,
store *Store,
dir string,
entry fs.DirEntry,
) error {
entryPath := path.Join(dir, entry.Name()) entryPath := path.Join(dir, entry.Name())
store.NonConforming = append(store.NonConforming, entryPath) store.NonConforming = append(store.NonConforming, entryPath)
@@ -240,13 +212,13 @@ func addNonConforming(
return nil return nil
} }
entries, err := readDir(ctx, fsys, entryPath) entries, err := fs.ReadDir(fsys, entryPath)
if err != nil { if err != nil {
return err return err
} }
for _, inner := range entries { for _, inner := range entries {
err = addNonConforming(ctx, fsys, store, entryPath, inner) err = addNonConforming(fsys, store, entryPath, inner)
if err != nil { if err != nil {
return err return err
} }
@@ -256,12 +228,7 @@ func addNonConforming(
} }
// newObject returns the Object for the file at filePath, with no DCF numbers. // newObject returns the Object for the file at filePath, with no DCF numbers.
func newObject( func newObject(store *Store, filePath string, entry fs.DirEntry) (Object, error) {
fsys fs.FS,
store *Store,
filePath string,
entry fs.DirEntry,
) (Object, error) {
info, err := entry.Info() info, err := entry.Info()
if err != nil { if err != nil {
return Object{}, err return Object{}, err
@@ -272,7 +239,6 @@ func newObject(
Path: filePath, Path: filePath,
Size: info.Size(), Size: info.Size(),
Extension: path.Ext(filePath), Extension: path.Ext(filePath),
fsys: fsys,
}, nil }, nil
} }
+3 -94
View File
@@ -1,7 +1,6 @@
package dcf package dcf
import ( import (
"context"
"errors" "errors"
"io/fs" "io/fs"
"os" "os"
@@ -66,7 +65,7 @@ func addSymlink(t *testing.T, root, name, target string) {
func mustParseStore(t *testing.T, fsys fs.FS) *Store { func mustParseStore(t *testing.T, fsys fs.FS) *Store {
t.Helper() t.Helper()
store, err := parseStore(t.Context(), fsys, testRoot) store, err := parseStore(fsys, testRoot)
if err != nil { if err != nil {
t.Fatalf("parseStore: %v", err) t.Fatalf("parseStore: %v", err)
} }
@@ -158,7 +157,6 @@ func TestParseStorePutsEachFileWhereItBelongs(t *testing.T) {
Extension: ".ARW", Extension: ".ARW",
DirectoryNumber: 100, DirectoryNumber: 100,
FileNumber: 1, FileNumber: 1,
fsys: card,
} }
mp4 := Object{ mp4 := Object{
DCFStoreRoot: testRoot, DCFStoreRoot: testRoot,
@@ -167,7 +165,6 @@ func TestParseStorePutsEachFileWhereItBelongs(t *testing.T) {
Extension: ".MP4", Extension: ".MP4",
DirectoryNumber: 101, DirectoryNumber: 101,
FileNumber: 2, FileNumber: 2,
fsys: card,
} }
thm := Object{ thm := Object{
DCFStoreRoot: testRoot, DCFStoreRoot: testRoot,
@@ -176,14 +173,12 @@ func TestParseStorePutsEachFileWhereItBelongs(t *testing.T) {
Extension: ".THM", Extension: ".THM",
DirectoryNumber: 101, DirectoryNumber: 101,
FileNumber: 2, FileNumber: 2,
fsys: card,
} }
clip := Object{ clip := Object{
DCFStoreRoot: testRoot, DCFStoreRoot: testRoot,
Path: "PRIVATE/M4ROOT/CLIP/C0002.MP4", Path: "PRIVATE/M4ROOT/CLIP/C0002.MP4",
Size: 400, Size: 400,
Extension: ".MP4", Extension: ".MP4",
fsys: card,
} }
wantDirectories := []*Directory{ wantDirectories := []*Directory{
@@ -318,9 +313,7 @@ func TestParseNumberOfFileName(t *testing.T) {
func TestParseStoreTreatsLowerCaseNamesAsDCFNames(t *testing.T) { func TestParseStoreTreatsLowerCaseNamesAsDCFNames(t *testing.T) {
t.Parallel() t.Parallel()
card := newCard("DCIM/101abcde/abcd0002.jpg") store := mustParseStore(t, newCard("DCIM/101abcde/abcd0002.jpg"))
store := mustParseStore(t, card)
want := []*Directory{wantDirectory(101, "101abcde", &Object{ want := []*Directory{wantDirectory(101, "101abcde", &Object{
DCFStoreRoot: testRoot, DCFStoreRoot: testRoot,
@@ -328,7 +321,6 @@ func TestParseStoreTreatsLowerCaseNamesAsDCFNames(t *testing.T) {
Extension: ".jpg", Extension: ".jpg",
DirectoryNumber: 101, DirectoryNumber: 101,
FileNumber: 2, FileNumber: 2,
fsys: card,
})} })}
if !reflect.DeepEqual(store.Directories, want) { if !reflect.DeepEqual(store.Directories, want) {
t.Errorf("Directories = %+v, want %+v", store.Directories, want) t.Errorf("Directories = %+v, want %+v", store.Directories, want)
@@ -805,7 +797,7 @@ func TestParseStoreReturnsReadErrorsNamingTheCard(t *testing.T) {
for _, name := range names { for _, name := range names {
fsys := failingFS{MapFS: card, call: call, name: name} fsys := failingFS{MapFS: card, call: call, name: name}
_, err := parseStore(t.Context(), fsys, testRoot) _, err := parseStore(fsys, testRoot)
if !errors.Is(err, fs.ErrPermission) { if !errors.Is(err, fs.ErrPermission) {
t.Errorf("%s of %s: error = %v, want %v", t.Errorf("%s of %s: error = %v, want %v",
call, name, err, fs.ErrPermission) call, name, err, fs.ErrPermission)
@@ -821,86 +813,3 @@ func TestParseStoreReturnsReadErrorsNamingTheCard(t *testing.T) {
} }
} }
} }
// cancelingFS is a card that cancels the read when the read reaches the named
// directory. It records every directory read in read.
type cancelingFS struct {
fstest.MapFS
name string
cancel context.CancelFunc
read *[]string
}
func (f cancelingFS) ReadDir(name string) ([]fs.DirEntry, error) {
*f.read = append(*f.read, name)
if name == f.name {
f.cancel()
}
return f.MapFS.ReadDir(name)
}
func TestParseStoreStopsBeforeTheNextDirectoryWhenCancelled(t *testing.T) {
t.Parallel()
card := newCard(
"DCIM/100ABCDE/SUB/README.TXT",
"DCIM/OTHER/SUB/README.TXT",
"PRIVATE/M4ROOT/CLIP/SUB/C0001.MP4",
)
// Each of these directories holds another directory, which must not be read.
names := []string{
dcimDirectory,
"DCIM/100ABCDE",
"DCIM/OTHER",
m4RootDirectory,
"PRIVATE/M4ROOT/CLIP",
}
for _, name := range names {
ctx, cancel := context.WithCancel(t.Context())
var read []string
fsys := cancelingFS{MapFS: card, name: name, cancel: cancel, read: &read}
_, err := parseStore(ctx, fsys, testRoot)
if !errors.Is(err, context.Canceled) {
t.Errorf("cancelled at %s: error = %v, want %v",
name, err, context.Canceled)
}
if read[len(read)-1] != name {
t.Errorf("cancelled at %s: read %q", name, read)
}
}
}
func TestParseStoreReadsNoDirectoryWhenAlreadyCancelled(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(t.Context())
cancel()
cards := []fstest.MapFS{
newCard("DCIM/README.TXT"),
newCard("PRIVATE/M4ROOT/C0001.MP4"),
}
for _, card := range cards {
var read []string
// The context is already cancelled, so this needs no name to cancel at.
fsys := cancelingFS{MapFS: card, read: &read}
_, err := parseStore(ctx, fsys, testRoot)
if !errors.Is(err, context.Canceled) {
t.Errorf("error = %v, want %v", err, context.Canceled)
}
assertPaths(t, "directories read", read)
}
}