Compare commits

..
1 Commits
Author SHA1 Message Date
clawbot c32c6db3ae group DCF files that share a file number (closes #3)
Files in one DCF directory that share a file number are one file group,
as section 4.3.2.2 of the specification says. `Directory.FileGroups`
lists them, each with its numbers, its members and its primary member. A
lone file is a group of one. A group reports its member count and total
size.

The primary member follows section 6.2.1.3, which gives a JPG priority. A
`THM` comes last. `Store.Images`, `Store.Videos` and the totals are
unchanged.

Model: opus-5-5
2026-11-03 18:35:47 +00:00
10 changed files with 124 additions and 367 deletions

No files matched your search

+9 -18
View File
@@ -29,7 +29,6 @@ Then list the DCF stores on the mounted filesystems:
package main
import (
"context"
"fmt"
"log"
@@ -37,21 +36,17 @@ import (
)
func main() {
stores, err := dcf.GetStores(context.Background())
stores, err := dcf.GetDCFStores(0)
if err != nil {
log.Fatal(err)
}
for _, store := range stores {
for _, store := range *stores {
fmt.Println(store.String())
}
}
```
Call `dcf.GetStore` instead when you expect exactly one DCF store. It returns
`dcf.ErrNoStore` when none is mounted, and `dcf.ErrMultipleStores` when more
than one is.
To work on the library itself, you need docker and git:
```sh
@@ -110,8 +105,8 @@ overengineering and reusable code.
All the code is in the `pkg/dcf` package. The repo has no commands.
- `dcf.go` holds the public API. `GetStores` finds the DCF stores and reads each
one. `GetStore` returns the one DCF store when exactly one is mounted.
- `dcf.go` holds the public API. `GetDCFStores` finds the DCF stores and reads
each one.
- `helpers.go` finds the mount points that hold a DCF store.
- `parse.go` reads one DCF store through an `fs.FS` rooted at its mount point.
- `filegroup.go` sorts the files of one DCF directory into file groups.
@@ -122,9 +117,7 @@ The main types are:
- `Directory` is one DCF directory. It has a directory number, a name, the
objects inside it and their file groups.
- `Object` is one file in a store. A file in a DCF directory also has a
directory number and a file number. `Object.Hash` reads the file through the
`fs.FS` of its store. An `Object` you build yourself has none, so its `Hash`
returns an error.
directory number and a file number.
- `FileGroup` is one file group in a DCF directory.
- `Image` and `Video` embed `Object`. The file extension decides which one a
file becomes. A file with any other extension, such as a `THM` thumbnail, is
@@ -136,10 +129,6 @@ filesystem. Media anywhere else on the filesystem is not part of the store.
`dcf` never follows a symlink.
`GetStores` and `GetStore` take a `context.Context`. When it is cancelled, the
read stops before the next directory. The returned error matches
`context.Canceled`.
`dcf` applies the naming rules of the DCF specification, CIPA DC-009-2010, under
`DCIM`:
@@ -184,8 +173,8 @@ has:
The primary member is the first of these that the group holds:
1. a `JPG` file
2. any other file in `Store.Images`
3. a file in `Store.Videos`
2. any other image
3. a video
4. any other file except a `THM`
5. a `THM`
@@ -193,6 +182,8 @@ The primary member is the first of these that the group holds:
## TODO
- Take a `context.Context` to cancel the walk of a store.
- Return a plain slice from `GetDCFStores`, not a pointer to one.
- Copy files off a card, and verify each copy against its SHA256 hash.
## License
+23 -59
View File
@@ -7,13 +7,10 @@
package dcf
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"slices"
@@ -22,14 +19,6 @@ import (
"github.com/dustin/go-humanize"
)
var (
// ErrNoStore is the error from GetStore when no DCF store is mounted.
ErrNoStore = errors.New("no DCF store is mounted")
// ErrMultipleStores is the error from GetStore when more than one DCF store
// is mounted.
ErrMultipleStores = errors.New("more than one DCF store is mounted")
)
// Store represents a DCF store, which is a directory structure that contains
// images and videos in the root of a mounted filesystem.
type Store struct {
@@ -87,9 +76,6 @@ type Object struct {
// directory. Both are 0 for a file under PRIVATE/M4ROOT.
DirectoryNumber int
FileNumber int
// fsys is the fs.FS the store was read through. Hash reads the file through
// it.
fsys fs.FS
}
// Image represents an image file on a DCF store.
@@ -187,17 +173,15 @@ func (i *Image) String() string {
)
}
// Hash returns the SHA256 hash of the file as a hex string. It reads the file
// through the fs.FS of its store. An Object built by hand has none, so Hash
// returns an error that matches fs.ErrInvalid.
// Hash returns the SHA256 hash of the file as a hex string.
func (d *Object) Hash() (string, error) {
if d.fsys == nil {
return "", fmt.Errorf("hashing %s: %w", d.FullFilePath(), fs.ErrInvalid)
}
return pathToSHA256(d.FullFilePath())
}
file, err := d.fsys.Open(d.Path)
func pathToSHA256(path string) (string, error) {
file, err := os.Open(filepath.Clean(path))
if err != nil {
return "", fmt.Errorf("hashing %s: %w", d.FullFilePath(), err)
return "", err
}
defer func() { _ = file.Close() }()
@@ -206,64 +190,44 @@ func (d *Object) Hash() (string, error) {
_, err = io.Copy(hash, file)
if err != nil {
return "", fmt.Errorf("hashing %s: %w", d.FullFilePath(), err)
return "", err
}
return hex.EncodeToString(hash.Sum(nil)), nil
}
// GetStores finds the DCF stores in the root of the mounted filesystems and reads
// each one. It does not mount filesystems or search outside of filesystem root
// directories. It reads the DCIM and PRIVATE/M4ROOT directories of each
// filesystem to populate the returned Stores. It never follows a symlink. If a
// card cannot be read, it returns an error that names the card's mount point.
// When ctx is cancelled, the read stops before the next directory, and the error
// matches ctx.Err().
func GetStores(ctx context.Context) ([]*Store, error) {
dcfStorePaths, err := findDCFMountPoints(ctx)
// GetDCFStores returns a list of DCF stores found on the system in the root of
// any mounted filesystems. The integer argument is the number of DCF stores to
// return. If the argument is 0, all DCF stores are returned. It does not mount
// filesystems or search outside of filesystem root directories. It does,
// however, read the DCIM and PRIVATE/M4ROOT directories of each filesystem to
// populate the returned Stores. It never follows a symlink. If a card cannot be
// read, it returns an error that names the card's mount point.
func GetDCFStores(requestedCount int) (*[]Store, error) {
dcfStorePaths, err := findDCFMountPoints(requestedCount)
if err != nil {
return nil, err
}
return parseStores(ctx, dcfStorePaths)
}
// GetStore returns the one DCF store when exactly one is mounted. It returns
// ErrNoStore when none is mounted, and ErrMultipleStores when more than one is.
// It reads every card first, the way GetStores does, so a card that cannot be
// read returns that error instead.
func GetStore(ctx context.Context) (*Store, error) {
stores, err := GetStores(ctx)
dcfStores, err := parseStores(dcfStorePaths)
if err != nil {
return nil, err
}
return onlyStore(stores)
}
// onlyStore returns the store when stores holds exactly one.
func onlyStore(stores []*Store) (*Store, error) {
switch len(stores) {
case 0:
return nil, ErrNoStore
case 1:
return stores[0], nil
default:
return nil, ErrMultipleStores
}
return &dcfStores, nil
}
// parseStores reads the card at each of the given mount points.
func parseStores(ctx context.Context, mountpoints []string) ([]*Store, error) {
dcfStores := []*Store{}
func parseStores(mountpoints []string) ([]Store, error) {
dcfStores := []Store{}
for _, mountpoint := range mountpoints {
dcfStore, err := parseStore(ctx, os.DirFS(mountpoint), mountpoint)
dcfStore, err := parseStore(os.DirFS(mountpoint), mountpoint)
if err != nil {
return nil, err
}
dcfStores = append(dcfStores, dcfStore)
dcfStores = append(dcfStores, *dcfStore)
}
return dcfStores, nil
@@ -271,7 +235,7 @@ func parseStores(ctx context.Context, mountpoints []string) ([]*Store, error) {
func isImageFile(path string) bool {
imageExtensions := []string{
".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tif", ".tiff", ".raw", ".arw",
".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".raw", ".arw",
}
return slices.Contains(imageExtensions, strings.ToLower(filepath.Ext(path)))
+2 -88
View File
@@ -1,13 +1,9 @@
package dcf
import (
"context"
"errors"
"io/fs"
"path/filepath"
"strings"
"testing"
"testing/fstest"
)
func TestParseStoresReadsEachMountPoint(t *testing.T) {
@@ -16,7 +12,7 @@ func TestParseStoresReadsEachMountPoint(t *testing.T) {
first := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG")
second := newCardOnDisk(t, "PRIVATE/M4ROOT/CLIP/C0001.MP4")
stores, err := parseStores(t.Context(), []string{first, second})
stores, err := parseStores([]string{first, second})
if err != nil {
t.Fatalf("parseStores: %v", err)
}
@@ -54,100 +50,18 @@ func TestParseStoresReturnsErrorNamingTheCard(t *testing.T) {
// Every lookup under a plain file fails.
mountpoint := filepath.Join(newCardOnDisk(t, "FILE"), "FILE")
_, err := parseStores(t.Context(), []string{mountpoint})
_, err := parseStores([]string{mountpoint})
if err == nil || !strings.Contains(err.Error(), mountpoint) {
t.Fatalf("parseStores error = %v, want one naming %s", err, mountpoint)
}
}
func TestParseStoresReturnsErrorWhenCancelled(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(t.Context())
cancel()
mountpoint := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG")
_, err := parseStores(ctx, []string{mountpoint})
if !errors.Is(err, context.Canceled) {
t.Fatalf("parseStores error = %v, want %v", err, context.Canceled)
}
}
func TestOnlyStore(t *testing.T) {
t.Parallel()
first := newMountPoint(t, dcimDirectory)
second := newMountPoint(t, m4RootDirectory)
stores, err := parseStores(t.Context(), []string{first, second})
if err != nil {
t.Fatalf("parseStores: %v", err)
}
_, err = onlyStore(stores[:0])
if !errors.Is(err, ErrNoStore) {
t.Errorf("onlyStore of no cards: error = %v, want %v", err, ErrNoStore)
}
store, err := onlyStore(stores[:1])
if err != nil {
t.Fatalf("onlyStore of one card: %v", err)
}
if store.RootDirectory != first {
t.Errorf("onlyStore returned the card at %s, want %s",
store.RootDirectory, first)
}
_, err = onlyStore(stores)
if !errors.Is(err, ErrMultipleStores) {
t.Errorf("onlyStore of two cards: error = %v, want %v",
err, ErrMultipleStores)
}
}
func TestObjectHashReadsThroughTheCard(t *testing.T) {
t.Parallel()
card := fstest.MapFS{"DCIM/100ABCDE/ABCD0001.JPG": {Data: []byte("abc")}}
got, err := mustParseStore(t, card).Images[0].Hash()
if err != nil {
t.Fatalf("Hash() returned error: %v", err)
}
// The published SHA-256 test vector for "abc".
want := "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
if got != want {
t.Fatalf("Hash() = %q, want %q", got, want)
}
}
func TestObjectHashOfMissingFileReturnsError(t *testing.T) {
t.Parallel()
const filePath = "DCIM/100ABCDE/ABCD0001.JPG"
card := newCard(filePath)
image := mustParseStore(t, card).Images[0]
delete(card, filePath)
_, err := image.Hash()
if !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("Hash() error = %v, want %v", err, fs.ErrNotExist)
}
}
func TestIsImageFile(t *testing.T) {
t.Parallel()
cases := map[string]bool{
"DCIM/100MSDCF/DSC00001.jpg": true,
"DCIM/100MSDCF/DSC00001.ARW": true,
"DCIM/100MSDCF/DSC00001.TIF": true,
"DCIM/100MSDCF/DSC00001.tif": true,
"PRIVATE/M4ROOT/CLIP/C0001.MP4": false,
"DCIM/100MSDCF/notes.txt": false,
"DCIM/100MSDCF/noextension": false,
+30 -24
View File
@@ -1,9 +1,8 @@
package dcf_test
import (
"context"
"errors"
"io/fs"
"os"
"path/filepath"
"testing"
"sneak.berlin/go/dcf/pkg/dcf"
@@ -126,30 +125,37 @@ func TestVideoString(t *testing.T) {
}
}
func TestObjectHashOfObjectBuiltByHandReturnsError(t *testing.T) {
func TestObjectHashIsSHA256OfFile(t *testing.T) {
t.Parallel()
object := dcf.Object{DCFStoreRoot: cardRoot, Path: "a.jpg"}
root := t.TempDir()
err := os.WriteFile(filepath.Join(root, "a.jpg"), []byte("abc"), 0o600)
if err != nil {
t.Fatalf("WriteFile: %v", err)
}
object := dcf.Object{DCFStoreRoot: root, Path: "a.jpg"}
got, err := object.Hash()
if err != nil {
t.Fatalf("Hash() returned error: %v", err)
}
// The published SHA-256 test vector for "abc".
want := "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
if got != want {
t.Fatalf("Hash() = %q, want %q", got, want)
}
}
func TestObjectHashOfMissingFileReturnsError(t *testing.T) {
t.Parallel()
object := dcf.Object{DCFStoreRoot: t.TempDir(), Path: "absent.jpg"}
_, err := object.Hash()
if !errors.Is(err, fs.ErrInvalid) {
t.Fatalf("Hash() error = %v, want %v", err, fs.ErrInvalid)
}
}
func TestGetStoresAndGetStoreReturnCancellation(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(t.Context())
cancel()
_, err := dcf.GetStores(ctx)
if !errors.Is(err, context.Canceled) {
t.Errorf("GetStores error = %v, want %v", err, context.Canceled)
}
_, err = dcf.GetStore(ctx)
if !errors.Is(err, context.Canceled) {
t.Errorf("GetStore error = %v, want %v", err, context.Canceled)
if err == nil {
t.Fatal("Hash() of a missing file returned no error")
}
}
+4 -6
View File
@@ -42,12 +42,10 @@ func newFileGroups(objects []*Object) []*FileGroup {
}
// primaryMember returns the member that stands for the whole group. Section
// 6.2.1.3 of the specification gives a DCF basic file priority over a DCF extended
// image file. It says nothing about a DCF optional file, the other kind of JPG, so
// ranking every JPG first is a choice. Section 4.6 makes a THM only the thumbnail
// of another file, so it comes last. The specification leaves the rest open, so an
// image comes before a video, and a video before any other file. Among equals, the
// first in name order wins.
// 6.2.1.3 of the specification gives a JPG priority over the other files. Section
// 4.6 makes a THM only the thumbnail of another file, so it comes last. The
// specification leaves the rest open, so an image comes before a video, and a
// video before any other file. Among equals, the first in name order wins.
func primaryMember(members []*Object) *Object {
choices := []func(string) bool{isJPG, isImageFile, isVideoFile, isNotTHM}
+2 -4
View File
@@ -212,16 +212,14 @@ func TestPrimaryMember(t *testing.T) {
// Then any other image.
assertPrimaryMember(t, "Z0010001.ARW", "A0010001.MP4", "Z0010001.ARW")
assertPrimaryMember(t, "Z0010001.ARW", "A0010001.WAV", "Z0010001.ARW")
assertPrimaryMember(t, "A0010001.TIF", "A0010001.MOV", "A0010001.TIF")
assertPrimaryMember(t, "Z0010001.TIF", "A0010001.WAV", "Z0010001.TIF")
// Then a video.
assertPrimaryMember(t, "A0010001.MP4", "A0010001.AAC", "A0010001.MP4")
assertPrimaryMember(t, "Z0010001.MP4", "A0010001.THM", "Z0010001.MP4")
// Then any other file except a THM, in either case.
assertPrimaryMember(t, "Z0010001.WAV", "A0010001.THM", "Z0010001.WAV")
assertPrimaryMember(t, "a0010001.wav", "a0010001.thm", "a0010001.wav")
assertPrimaryMember(t, "Z0010001.TIF", "A0010001.THM", "Z0010001.TIF")
assertPrimaryMember(t, "a0010001.tif", "a0010001.thm", "a0010001.tif")
// A THM is primary only when the group holds nothing else.
assertPrimaryMember(t, "A0010001.THM", "A0010001.THM")
+15 -17
View File
@@ -1,7 +1,6 @@
package dcf
import (
"context"
"fmt"
"log/slog"
"os"
@@ -10,19 +9,12 @@ import (
"github.com/shirou/gopsutil/disk"
)
func findAllMountPoints(ctx context.Context) ([]string, error) {
func findAllMountPoints() ([]string, error) {
mountpoints := []string{}
// disk.PartitionsWithContext does not check ctx on Linux.
err := ctx.Err()
partitions, err := disk.Partitions(false) // physical devices only, so false
if err != nil {
return nil, fmt.Errorf("listing the mounted filesystems: %w", err)
}
// physical devices only, so false
partitions, err := disk.PartitionsWithContext(ctx, false)
if err != nil {
return nil, fmt.Errorf("listing the mounted filesystems: %w", err)
return nil, err
}
for _, partition := range partitions {
@@ -34,18 +26,21 @@ func findAllMountPoints(ctx context.Context) ([]string, error) {
return mountpoints, nil
}
// findDCFMountPoints returns the paths of the mountpoints that hold a DCF store.
func findDCFMountPoints(ctx context.Context) ([]string, error) {
mountpoints, err := findAllMountPoints(ctx)
// findDCFMountPoints returns a list of strings of the paths of mountpoints
// that hold a DCF store. Its only argument is an integer of how many mountpoints
// to return. If the argument is 0, all mountpoints are returned.
func findDCFMountPoints(requestedCount int) ([]string, error) {
mountpoints, err := findAllMountPoints()
if err != nil {
return nil, err
}
return filterDCFMountPoints(mountpoints), nil
return filterDCFMountPoints(mountpoints, requestedCount), nil
}
// filterDCFMountPoints returns the mountpoints that hold a DCF store.
func filterDCFMountPoints(mountpoints []string) []string {
// filterDCFMountPoints returns the first requestedCount of the mountpoints that
// hold a DCF store. If requestedCount is 0, it returns all of them.
func filterDCFMountPoints(mountpoints []string, requestedCount int) []string {
filteredMountpoints := []string{}
for _, mountpoint := range mountpoints {
@@ -56,6 +51,9 @@ func filterDCFMountPoints(mountpoints []string) []string {
slog.Debug(fmt.Sprintf("Keeping mountpoint %s\n", mountpoint))
filteredMountpoints = append(filteredMountpoints, mountpoint)
if (requestedCount > 0) && (len(filteredMountpoints) >= requestedCount) {
break
}
}
return filteredMountpoints
+18 -5
View File
@@ -72,16 +72,29 @@ func TestIsDCFMountPointIgnoresSymlink(t *testing.T) {
}
}
func TestFilterDCFMountPointsKeepsThoseHoldingAStore(t *testing.T) {
func TestFilterDCFMountPointsReturnsRequestedCount(t *testing.T) {
t.Parallel()
first := newMountPoint(t, "DCIM")
second := newMountPoint(t, "PRIVATE/M4ROOT")
third := newMountPoint(t, "DCIM")
got := filterDCFMountPoints([]string{newMountPoint(t), first, second})
// This one holds no DCF store, so it must not use up the requested count.
mountpoints := []string{newMountPoint(t), first, second, third}
want := []string{first, second}
if !slices.Equal(got, want) {
t.Errorf("filterDCFMountPoints = %v, want %v", got, want)
cases := map[int][]string{
0: {first, second, third},
1: {first},
2: {first, second},
3: {first, second, third},
4: {first, second, third},
}
for requestedCount, want := range cases {
got := filterDCFMountPoints(mountpoints, requestedCount)
if !slices.Equal(got, want) {
t.Errorf("filterDCFMountPoints(%d) = %v, want %v",
requestedCount, got, want)
}
}
}
+18 -52
View File
@@ -1,7 +1,6 @@
package dcf
import (
"context"
"errors"
"fmt"
"io/fs"
@@ -29,13 +28,13 @@ var (
// point. The root argument is the path of that mount point. Only DCIM and
// PRIVATE/M4ROOT are read. A card can lack either one. A symlink is never
// followed. An error names the mount point, because fsys reports only paths
// inside it. When ctx is cancelled, the read stops before the next directory.
func parseStore(ctx context.Context, fsys fs.FS, root string) (*Store, error) {
// inside it.
func parseStore(fsys fs.FS, root string) (*Store, error) {
store := &Store{RootDirectory: root}
err := parseDCIM(ctx, fsys, store)
err := parseDCIM(fsys, store)
if err == nil {
err = parseM4Root(ctx, fsys, store)
err = parseM4Root(fsys, store)
}
if err != nil {
@@ -69,26 +68,16 @@ func hasDirectory(fsys fs.FS, name string) (bool, error) {
return info.IsDir(), nil
}
// readDir returns the entries of the named directory, unless ctx is cancelled.
func readDir(ctx context.Context, fsys fs.FS, name string) ([]fs.DirEntry, error) {
err := ctx.Err()
if err != nil {
return nil, err
}
return fs.ReadDir(fsys, name)
}
// parseDCIM adds the DCF directories under DCIM to store. Everything else there
// is not conforming. That includes directories that share a directory number,
// which section 7.1.2 of the specification treats as non-DCF directories.
func parseDCIM(ctx context.Context, fsys fs.FS, store *Store) error {
func parseDCIM(fsys fs.FS, store *Store) error {
found, err := hasDirectory(fsys, dcimDirectory)
if err != nil || !found {
return err
}
entries, err := readDir(ctx, fsys, dcimDirectory)
entries, err := fs.ReadDir(fsys, dcimDirectory)
if err != nil {
return err
}
@@ -105,9 +94,9 @@ func parseDCIM(ctx context.Context, fsys fs.FS, store *Store) error {
number := parseNumber(dcfDirectoryName, entry.Name())
if entry.IsDir() && number != 0 && directoriesWithNumber[number] == 1 {
err = parseDirectory(ctx, fsys, store, entry.Name(), number)
err = parseDirectory(fsys, store, entry.Name(), number)
} else {
err = addNonConforming(ctx, fsys, store, dcimDirectory, entry)
err = addNonConforming(fsys, store, dcimDirectory, entry)
}
if err != nil {
@@ -123,16 +112,10 @@ func parseDCIM(ctx context.Context, fsys fs.FS, store *Store) error {
// Everything else there is not conforming. That includes JPG files that share a
// file number, which section 7.2.2 of the specification says must not show DCF
// numbers. Other files that share a file number do conform.
func parseDirectory(
ctx context.Context,
fsys fs.FS,
store *Store,
name string,
number int,
) error {
func parseDirectory(fsys fs.FS, store *Store, name string, number int) error {
directoryPath := path.Join(dcimDirectory, name)
entries, err := readDir(ctx, fsys, directoryPath)
entries, err := fs.ReadDir(fsys, directoryPath)
if err != nil {
return err
}
@@ -152,7 +135,7 @@ func parseDirectory(
sharedJPG := isJPG(entry.Name()) && jpgsWithNumber[fileNumber] > 1
if !entry.Type().IsRegular() || fileNumber == 0 || sharedJPG {
err = addNonConforming(ctx, fsys, store, directoryPath, entry)
err = addNonConforming(fsys, store, directoryPath, entry)
if err != nil {
return err
}
@@ -162,7 +145,7 @@ func parseDirectory(
entryPath := path.Join(directoryPath, entry.Name())
object, err := newObject(fsys, store, entryPath, entry)
object, err := newObject(store, entryPath, entry)
if err != nil {
return err
}
@@ -184,7 +167,7 @@ func parseDirectory(
// naming rules do not apply there, so these files have no DCF numbers. It skips
// a symlink. It also skips a name that is not valid UTF-8, because fs.FS refuses
// to open one.
func parseM4Root(ctx context.Context, fsys fs.FS, store *Store) error {
func parseM4Root(fsys fs.FS, store *Store) error {
found, err := hasDirectory(fsys, m4RootDirectory)
if err != nil || !found {
return err
@@ -202,16 +185,11 @@ func parseM4Root(ctx context.Context, fsys fs.FS, store *Store) error {
return fs.SkipDir
}
// fs.WalkDir reads a directory right after this call for it.
if entry.IsDir() {
return ctx.Err()
}
if !entry.Type().IsRegular() || !validName {
return nil
}
object, err := newObject(fsys, store, filePath, entry)
object, err := newObject(store, filePath, entry)
if err != nil {
return err
}
@@ -226,13 +204,7 @@ func parseM4Root(ctx context.Context, fsys fs.FS, store *Store) error {
// conforming. If the entry is a directory, it lists everything inside it too. It
// does not follow a symlink. It does not read a directory whose name is not
// valid UTF-8, because fs.FS refuses to open one.
func addNonConforming(
ctx context.Context,
fsys fs.FS,
store *Store,
dir string,
entry fs.DirEntry,
) error {
func addNonConforming(fsys fs.FS, store *Store, dir string, entry fs.DirEntry) error {
entryPath := path.Join(dir, entry.Name())
store.NonConforming = append(store.NonConforming, entryPath)
@@ -240,13 +212,13 @@ func addNonConforming(
return nil
}
entries, err := readDir(ctx, fsys, entryPath)
entries, err := fs.ReadDir(fsys, entryPath)
if err != nil {
return err
}
for _, inner := range entries {
err = addNonConforming(ctx, fsys, store, entryPath, inner)
err = addNonConforming(fsys, store, entryPath, inner)
if err != nil {
return err
}
@@ -256,12 +228,7 @@ func addNonConforming(
}
// newObject returns the Object for the file at filePath, with no DCF numbers.
func newObject(
fsys fs.FS,
store *Store,
filePath string,
entry fs.DirEntry,
) (Object, error) {
func newObject(store *Store, filePath string, entry fs.DirEntry) (Object, error) {
info, err := entry.Info()
if err != nil {
return Object{}, err
@@ -272,7 +239,6 @@ func newObject(
Path: filePath,
Size: info.Size(),
Extension: path.Ext(filePath),
fsys: fsys,
}, nil
}
+3 -94
View File
@@ -1,7 +1,6 @@
package dcf
import (
"context"
"errors"
"io/fs"
"os"
@@ -66,7 +65,7 @@ func addSymlink(t *testing.T, root, name, target string) {
func mustParseStore(t *testing.T, fsys fs.FS) *Store {
t.Helper()
store, err := parseStore(t.Context(), fsys, testRoot)
store, err := parseStore(fsys, testRoot)
if err != nil {
t.Fatalf("parseStore: %v", err)
}
@@ -158,7 +157,6 @@ func TestParseStorePutsEachFileWhereItBelongs(t *testing.T) {
Extension: ".ARW",
DirectoryNumber: 100,
FileNumber: 1,
fsys: card,
}
mp4 := Object{
DCFStoreRoot: testRoot,
@@ -167,7 +165,6 @@ func TestParseStorePutsEachFileWhereItBelongs(t *testing.T) {
Extension: ".MP4",
DirectoryNumber: 101,
FileNumber: 2,
fsys: card,
}
thm := Object{
DCFStoreRoot: testRoot,
@@ -176,14 +173,12 @@ func TestParseStorePutsEachFileWhereItBelongs(t *testing.T) {
Extension: ".THM",
DirectoryNumber: 101,
FileNumber: 2,
fsys: card,
}
clip := Object{
DCFStoreRoot: testRoot,
Path: "PRIVATE/M4ROOT/CLIP/C0002.MP4",
Size: 400,
Extension: ".MP4",
fsys: card,
}
wantDirectories := []*Directory{
@@ -318,9 +313,7 @@ func TestParseNumberOfFileName(t *testing.T) {
func TestParseStoreTreatsLowerCaseNamesAsDCFNames(t *testing.T) {
t.Parallel()
card := newCard("DCIM/101abcde/abcd0002.jpg")
store := mustParseStore(t, card)
store := mustParseStore(t, newCard("DCIM/101abcde/abcd0002.jpg"))
want := []*Directory{wantDirectory(101, "101abcde", &Object{
DCFStoreRoot: testRoot,
@@ -328,7 +321,6 @@ func TestParseStoreTreatsLowerCaseNamesAsDCFNames(t *testing.T) {
Extension: ".jpg",
DirectoryNumber: 101,
FileNumber: 2,
fsys: card,
})}
if !reflect.DeepEqual(store.Directories, want) {
t.Errorf("Directories = %+v, want %+v", store.Directories, want)
@@ -805,7 +797,7 @@ func TestParseStoreReturnsReadErrorsNamingTheCard(t *testing.T) {
for _, name := range names {
fsys := failingFS{MapFS: card, call: call, name: name}
_, err := parseStore(t.Context(), fsys, testRoot)
_, err := parseStore(fsys, testRoot)
if !errors.Is(err, fs.ErrPermission) {
t.Errorf("%s of %s: error = %v, want %v",
call, name, err, fs.ErrPermission)
@@ -821,86 +813,3 @@ func TestParseStoreReturnsReadErrorsNamingTheCard(t *testing.T) {
}
}
}
// cancelingFS is a card that cancels the read when the read reaches the named
// directory. It records every directory read in read.
type cancelingFS struct {
fstest.MapFS
name string
cancel context.CancelFunc
read *[]string
}
func (f cancelingFS) ReadDir(name string) ([]fs.DirEntry, error) {
*f.read = append(*f.read, name)
if name == f.name {
f.cancel()
}
return f.MapFS.ReadDir(name)
}
func TestParseStoreStopsBeforeTheNextDirectoryWhenCancelled(t *testing.T) {
t.Parallel()
card := newCard(
"DCIM/100ABCDE/SUB/README.TXT",
"DCIM/OTHER/SUB/README.TXT",
"PRIVATE/M4ROOT/CLIP/SUB/C0001.MP4",
)
// Each of these directories holds another directory, which must not be read.
names := []string{
dcimDirectory,
"DCIM/100ABCDE",
"DCIM/OTHER",
m4RootDirectory,
"PRIVATE/M4ROOT/CLIP",
}
for _, name := range names {
ctx, cancel := context.WithCancel(t.Context())
var read []string
fsys := cancelingFS{MapFS: card, name: name, cancel: cancel, read: &read}
_, err := parseStore(ctx, fsys, testRoot)
if !errors.Is(err, context.Canceled) {
t.Errorf("cancelled at %s: error = %v, want %v",
name, err, context.Canceled)
}
if read[len(read)-1] != name {
t.Errorf("cancelled at %s: read %q", name, read)
}
}
}
func TestParseStoreReadsNoDirectoryWhenAlreadyCancelled(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(t.Context())
cancel()
cards := []fstest.MapFS{
newCard("DCIM/README.TXT"),
newCard("PRIVATE/M4ROOT/C0001.MP4"),
}
for _, card := range cards {
var read []string
// The context is already cancelled, so this needs no name to cancel at.
fsys := cancelingFS{MapFS: card, read: &read}
_, err := parseStore(ctx, fsys, testRoot)
if !errors.Is(err, context.Canceled) {
t.Errorf("error = %v, want %v", err, context.Canceled)
}
assertPaths(t, "directories read", read)
}
}