Compare commits
1
Commits
d991df0b24
...
next
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
acb108a82e |
No files matched your search
@@ -29,6 +29,7 @@ Then list the DCF stores on the mounted filesystems:
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
@@ -36,17 +37,21 @@ import (
|
||||
)
|
||||
|
||||
func main() {
|
||||
stores, err := dcf.GetDCFStores(0)
|
||||
stores, err := dcf.GetStores(context.Background())
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
for _, store := range *stores {
|
||||
for _, store := range stores {
|
||||
fmt.Println(store.String())
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Call `dcf.GetStore` instead when you expect exactly one DCF store. It returns
|
||||
`dcf.ErrNoStore` when none is mounted, and `dcf.ErrMultipleStores` when more
|
||||
than one is.
|
||||
|
||||
To work on the library itself, you need docker and git:
|
||||
|
||||
```sh
|
||||
@@ -105,8 +110,8 @@ overengineering and reusable code.
|
||||
|
||||
All the code is in the `pkg/dcf` package. The repo has no commands.
|
||||
|
||||
- `dcf.go` holds the public API. `GetDCFStores` finds the DCF stores and reads
|
||||
each one.
|
||||
- `dcf.go` holds the public API. `GetStores` finds the DCF stores and reads each
|
||||
one. `GetStore` returns the one DCF store when exactly one is mounted.
|
||||
- `helpers.go` finds the mount points that hold a DCF store.
|
||||
- `parse.go` reads one DCF store through an `fs.FS` rooted at its mount point.
|
||||
- `filegroup.go` sorts the files of one DCF directory into file groups.
|
||||
@@ -117,7 +122,9 @@ The main types are:
|
||||
- `Directory` is one DCF directory. It has a directory number, a name, the
|
||||
objects inside it and their file groups.
|
||||
- `Object` is one file in a store. A file in a DCF directory also has a
|
||||
directory number and a file number.
|
||||
directory number and a file number. `Object.Hash` reads the file through the
|
||||
`fs.FS` of its store. An `Object` you build yourself has none, so its `Hash`
|
||||
returns an error.
|
||||
- `FileGroup` is one file group in a DCF directory.
|
||||
- `Image` and `Video` embed `Object`. The file extension decides which one a
|
||||
file becomes. A file with any other extension, such as a `THM` thumbnail, is
|
||||
@@ -129,6 +136,10 @@ filesystem. Media anywhere else on the filesystem is not part of the store.
|
||||
|
||||
`dcf` never follows a symlink.
|
||||
|
||||
`GetStores` and `GetStore` take a `context.Context`. When it is cancelled, the
|
||||
read stops before the next directory. The returned error matches
|
||||
`context.Canceled`.
|
||||
|
||||
`dcf` applies the naming rules of the DCF specification, CIPA DC-009-2010, under
|
||||
`DCIM`:
|
||||
|
||||
@@ -182,8 +193,6 @@ The primary member is the first of these that the group holds:
|
||||
|
||||
## TODO
|
||||
|
||||
- Take a `context.Context` to cancel the walk of a store.
|
||||
- Return a plain slice from `GetDCFStores`, not a pointer to one.
|
||||
- Copy files off a card, and verify each copy against its SHA256 hash.
|
||||
|
||||
## License
|
||||
|
||||
+58
-22
@@ -7,10 +7,13 @@
|
||||
package dcf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
@@ -19,6 +22,14 @@ import (
|
||||
"github.com/dustin/go-humanize"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrNoStore is the error from GetStore when no DCF store is mounted.
|
||||
ErrNoStore = errors.New("no DCF store is mounted")
|
||||
// ErrMultipleStores is the error from GetStore when more than one DCF store
|
||||
// is mounted.
|
||||
ErrMultipleStores = errors.New("more than one DCF store is mounted")
|
||||
)
|
||||
|
||||
// Store represents a DCF store, which is a directory structure that contains
|
||||
// images and videos in the root of a mounted filesystem.
|
||||
type Store struct {
|
||||
@@ -76,6 +87,9 @@ type Object struct {
|
||||
// directory. Both are 0 for a file under PRIVATE/M4ROOT.
|
||||
DirectoryNumber int
|
||||
FileNumber int
|
||||
// fsys is the fs.FS the store was read through. Hash reads the file through
|
||||
// it.
|
||||
fsys fs.FS
|
||||
}
|
||||
|
||||
// Image represents an image file on a DCF store.
|
||||
@@ -173,15 +187,17 @@ func (i *Image) String() string {
|
||||
)
|
||||
}
|
||||
|
||||
// Hash returns the SHA256 hash of the file as a hex string.
|
||||
// Hash returns the SHA256 hash of the file as a hex string. It reads the file
|
||||
// through the fs.FS of its store. An Object built by hand has none, so Hash
|
||||
// returns an error that matches fs.ErrInvalid.
|
||||
func (d *Object) Hash() (string, error) {
|
||||
return pathToSHA256(d.FullFilePath())
|
||||
}
|
||||
if d.fsys == nil {
|
||||
return "", fmt.Errorf("hashing %s: %w", d.FullFilePath(), fs.ErrInvalid)
|
||||
}
|
||||
|
||||
func pathToSHA256(path string) (string, error) {
|
||||
file, err := os.Open(filepath.Clean(path))
|
||||
file, err := d.fsys.Open(d.Path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", fmt.Errorf("hashing %s: %w", d.FullFilePath(), err)
|
||||
}
|
||||
|
||||
defer func() { _ = file.Close() }()
|
||||
@@ -190,44 +206,64 @@ func pathToSHA256(path string) (string, error) {
|
||||
|
||||
_, err = io.Copy(hash, file)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", fmt.Errorf("hashing %s: %w", d.FullFilePath(), err)
|
||||
}
|
||||
|
||||
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// GetDCFStores returns a list of DCF stores found on the system in the root of
|
||||
// any mounted filesystems. The integer argument is the number of DCF stores to
|
||||
// return. If the argument is 0, all DCF stores are returned. It does not mount
|
||||
// filesystems or search outside of filesystem root directories. It does,
|
||||
// however, read the DCIM and PRIVATE/M4ROOT directories of each filesystem to
|
||||
// populate the returned Stores. It never follows a symlink. If a card cannot be
|
||||
// read, it returns an error that names the card's mount point.
|
||||
func GetDCFStores(requestedCount int) (*[]Store, error) {
|
||||
dcfStorePaths, err := findDCFMountPoints(requestedCount)
|
||||
// GetStores finds the DCF stores in the root of the mounted filesystems and reads
|
||||
// each one. It does not mount filesystems or search outside of filesystem root
|
||||
// directories. It reads the DCIM and PRIVATE/M4ROOT directories of each
|
||||
// filesystem to populate the returned Stores. It never follows a symlink. If a
|
||||
// card cannot be read, it returns an error that names the card's mount point.
|
||||
// When ctx is cancelled, the read stops before the next directory, and the error
|
||||
// matches ctx.Err().
|
||||
func GetStores(ctx context.Context) ([]*Store, error) {
|
||||
dcfStorePaths, err := findDCFMountPoints(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dcfStores, err := parseStores(dcfStorePaths)
|
||||
return parseStores(ctx, dcfStorePaths)
|
||||
}
|
||||
|
||||
// GetStore returns the one DCF store when exactly one is mounted. It returns
|
||||
// ErrNoStore when none is mounted, and ErrMultipleStores when more than one is.
|
||||
// It reads every card first, the way GetStores does, so a card that cannot be
|
||||
// read returns that error instead.
|
||||
func GetStore(ctx context.Context) (*Store, error) {
|
||||
stores, err := GetStores(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &dcfStores, nil
|
||||
return onlyStore(stores)
|
||||
}
|
||||
|
||||
// onlyStore returns the store when stores holds exactly one.
|
||||
func onlyStore(stores []*Store) (*Store, error) {
|
||||
switch len(stores) {
|
||||
case 0:
|
||||
return nil, ErrNoStore
|
||||
case 1:
|
||||
return stores[0], nil
|
||||
default:
|
||||
return nil, ErrMultipleStores
|
||||
}
|
||||
}
|
||||
|
||||
// parseStores reads the card at each of the given mount points.
|
||||
func parseStores(mountpoints []string) ([]Store, error) {
|
||||
dcfStores := []Store{}
|
||||
func parseStores(ctx context.Context, mountpoints []string) ([]*Store, error) {
|
||||
dcfStores := []*Store{}
|
||||
|
||||
for _, mountpoint := range mountpoints {
|
||||
dcfStore, err := parseStore(os.DirFS(mountpoint), mountpoint)
|
||||
dcfStore, err := parseStore(ctx, os.DirFS(mountpoint), mountpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dcfStores = append(dcfStores, *dcfStore)
|
||||
dcfStores = append(dcfStores, dcfStore)
|
||||
}
|
||||
|
||||
return dcfStores, nil
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
package dcf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
)
|
||||
|
||||
func TestParseStoresReadsEachMountPoint(t *testing.T) {
|
||||
@@ -12,7 +16,7 @@ func TestParseStoresReadsEachMountPoint(t *testing.T) {
|
||||
first := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG")
|
||||
second := newCardOnDisk(t, "PRIVATE/M4ROOT/CLIP/C0001.MP4")
|
||||
|
||||
stores, err := parseStores([]string{first, second})
|
||||
stores, err := parseStores(t.Context(), []string{first, second})
|
||||
if err != nil {
|
||||
t.Fatalf("parseStores: %v", err)
|
||||
}
|
||||
@@ -50,12 +54,92 @@ func TestParseStoresReturnsErrorNamingTheCard(t *testing.T) {
|
||||
// Every lookup under a plain file fails.
|
||||
mountpoint := filepath.Join(newCardOnDisk(t, "FILE"), "FILE")
|
||||
|
||||
_, err := parseStores([]string{mountpoint})
|
||||
_, err := parseStores(t.Context(), []string{mountpoint})
|
||||
if err == nil || !strings.Contains(err.Error(), mountpoint) {
|
||||
t.Fatalf("parseStores error = %v, want one naming %s", err, mountpoint)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseStoresReturnsErrorWhenCancelled(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
|
||||
mountpoint := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG")
|
||||
|
||||
_, err := parseStores(ctx, []string{mountpoint})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("parseStores error = %v, want %v", err, context.Canceled)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyStore(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
first := newMountPoint(t, dcimDirectory)
|
||||
second := newMountPoint(t, m4RootDirectory)
|
||||
|
||||
stores, err := parseStores(t.Context(), []string{first, second})
|
||||
if err != nil {
|
||||
t.Fatalf("parseStores: %v", err)
|
||||
}
|
||||
|
||||
_, err = onlyStore(stores[:0])
|
||||
if !errors.Is(err, ErrNoStore) {
|
||||
t.Errorf("onlyStore of no cards: error = %v, want %v", err, ErrNoStore)
|
||||
}
|
||||
|
||||
store, err := onlyStore(stores[:1])
|
||||
if err != nil {
|
||||
t.Fatalf("onlyStore of one card: %v", err)
|
||||
}
|
||||
|
||||
if store.RootDirectory != first {
|
||||
t.Errorf("onlyStore returned the card at %s, want %s",
|
||||
store.RootDirectory, first)
|
||||
}
|
||||
|
||||
_, err = onlyStore(stores)
|
||||
if !errors.Is(err, ErrMultipleStores) {
|
||||
t.Errorf("onlyStore of two cards: error = %v, want %v",
|
||||
err, ErrMultipleStores)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectHashReadsThroughTheCard(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
card := fstest.MapFS{"DCIM/100ABCDE/ABCD0001.JPG": {Data: []byte("abc")}}
|
||||
|
||||
got, err := mustParseStore(t, card).Images[0].Hash()
|
||||
if err != nil {
|
||||
t.Fatalf("Hash() returned error: %v", err)
|
||||
}
|
||||
|
||||
// The published SHA-256 test vector for "abc".
|
||||
want := "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
|
||||
if got != want {
|
||||
t.Fatalf("Hash() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectHashOfMissingFileReturnsError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const filePath = "DCIM/100ABCDE/ABCD0001.JPG"
|
||||
|
||||
card := newCard(filePath)
|
||||
image := mustParseStore(t, card).Images[0]
|
||||
|
||||
delete(card, filePath)
|
||||
|
||||
_, err := image.Hash()
|
||||
if !errors.Is(err, fs.ErrNotExist) {
|
||||
t.Fatalf("Hash() error = %v, want %v", err, fs.ErrNotExist)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsImageFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
+24
-30
@@ -1,8 +1,9 @@
|
||||
package dcf_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/dcf/pkg/dcf"
|
||||
@@ -125,37 +126,30 @@ func TestVideoString(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectHashIsSHA256OfFile(t *testing.T) {
|
||||
func TestObjectHashOfObjectBuiltByHandReturnsError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
|
||||
err := os.WriteFile(filepath.Join(root, "a.jpg"), []byte("abc"), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("WriteFile: %v", err)
|
||||
}
|
||||
|
||||
object := dcf.Object{DCFStoreRoot: root, Path: "a.jpg"}
|
||||
|
||||
got, err := object.Hash()
|
||||
if err != nil {
|
||||
t.Fatalf("Hash() returned error: %v", err)
|
||||
}
|
||||
|
||||
// The published SHA-256 test vector for "abc".
|
||||
want := "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
|
||||
if got != want {
|
||||
t.Fatalf("Hash() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectHashOfMissingFileReturnsError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
object := dcf.Object{DCFStoreRoot: t.TempDir(), Path: "absent.jpg"}
|
||||
object := dcf.Object{DCFStoreRoot: cardRoot, Path: "a.jpg"}
|
||||
|
||||
_, err := object.Hash()
|
||||
if err == nil {
|
||||
t.Fatal("Hash() of a missing file returned no error")
|
||||
if !errors.Is(err, fs.ErrInvalid) {
|
||||
t.Fatalf("Hash() error = %v, want %v", err, fs.ErrInvalid)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetStoresAndGetStoreReturnCancellation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
|
||||
_, err := dcf.GetStores(ctx)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("GetStores error = %v, want %v", err, context.Canceled)
|
||||
}
|
||||
|
||||
_, err = dcf.GetStore(ctx)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("GetStore error = %v, want %v", err, context.Canceled)
|
||||
}
|
||||
}
|
||||
+17
-15
@@ -1,6 +1,7 @@
|
||||
package dcf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
@@ -9,12 +10,19 @@ import (
|
||||
"github.com/shirou/gopsutil/disk"
|
||||
)
|
||||
|
||||
func findAllMountPoints() ([]string, error) {
|
||||
func findAllMountPoints(ctx context.Context) ([]string, error) {
|
||||
mountpoints := []string{}
|
||||
|
||||
partitions, err := disk.Partitions(false) // physical devices only, so false
|
||||
// disk.PartitionsWithContext does not check ctx on Linux.
|
||||
err := ctx.Err()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("listing the mounted filesystems: %w", err)
|
||||
}
|
||||
|
||||
// physical devices only, so false
|
||||
partitions, err := disk.PartitionsWithContext(ctx, false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listing the mounted filesystems: %w", err)
|
||||
}
|
||||
|
||||
for _, partition := range partitions {
|
||||
@@ -26,21 +34,18 @@ func findAllMountPoints() ([]string, error) {
|
||||
return mountpoints, nil
|
||||
}
|
||||
|
||||
// findDCFMountPoints returns a list of strings of the paths of mountpoints
|
||||
// that hold a DCF store. Its only argument is an integer of how many mountpoints
|
||||
// to return. If the argument is 0, all mountpoints are returned.
|
||||
func findDCFMountPoints(requestedCount int) ([]string, error) {
|
||||
mountpoints, err := findAllMountPoints()
|
||||
// findDCFMountPoints returns the paths of the mountpoints that hold a DCF store.
|
||||
func findDCFMountPoints(ctx context.Context) ([]string, error) {
|
||||
mountpoints, err := findAllMountPoints(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return filterDCFMountPoints(mountpoints, requestedCount), nil
|
||||
return filterDCFMountPoints(mountpoints), nil
|
||||
}
|
||||
|
||||
// filterDCFMountPoints returns the first requestedCount of the mountpoints that
|
||||
// hold a DCF store. If requestedCount is 0, it returns all of them.
|
||||
func filterDCFMountPoints(mountpoints []string, requestedCount int) []string {
|
||||
// filterDCFMountPoints returns the mountpoints that hold a DCF store.
|
||||
func filterDCFMountPoints(mountpoints []string) []string {
|
||||
filteredMountpoints := []string{}
|
||||
|
||||
for _, mountpoint := range mountpoints {
|
||||
@@ -51,9 +56,6 @@ func filterDCFMountPoints(mountpoints []string, requestedCount int) []string {
|
||||
slog.Debug(fmt.Sprintf("Keeping mountpoint %s\n", mountpoint))
|
||||
|
||||
filteredMountpoints = append(filteredMountpoints, mountpoint)
|
||||
if (requestedCount > 0) && (len(filteredMountpoints) >= requestedCount) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return filteredMountpoints
|
||||
|
||||
@@ -72,29 +72,16 @@ func TestIsDCFMountPointIgnoresSymlink(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterDCFMountPointsReturnsRequestedCount(t *testing.T) {
|
||||
func TestFilterDCFMountPointsKeepsThoseHoldingAStore(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
first := newMountPoint(t, "DCIM")
|
||||
second := newMountPoint(t, "PRIVATE/M4ROOT")
|
||||
third := newMountPoint(t, "DCIM")
|
||||
|
||||
// This one holds no DCF store, so it must not use up the requested count.
|
||||
mountpoints := []string{newMountPoint(t), first, second, third}
|
||||
got := filterDCFMountPoints([]string{newMountPoint(t), first, second})
|
||||
|
||||
cases := map[int][]string{
|
||||
0: {first, second, third},
|
||||
1: {first},
|
||||
2: {first, second},
|
||||
3: {first, second, third},
|
||||
4: {first, second, third},
|
||||
}
|
||||
|
||||
for requestedCount, want := range cases {
|
||||
got := filterDCFMountPoints(mountpoints, requestedCount)
|
||||
if !slices.Equal(got, want) {
|
||||
t.Errorf("filterDCFMountPoints(%d) = %v, want %v",
|
||||
requestedCount, got, want)
|
||||
}
|
||||
want := []string{first, second}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Errorf("filterDCFMountPoints = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
+52
-18
@@ -1,6 +1,7 @@
|
||||
package dcf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
@@ -28,13 +29,13 @@ var (
|
||||
// point. The root argument is the path of that mount point. Only DCIM and
|
||||
// PRIVATE/M4ROOT are read. A card can lack either one. A symlink is never
|
||||
// followed. An error names the mount point, because fsys reports only paths
|
||||
// inside it.
|
||||
func parseStore(fsys fs.FS, root string) (*Store, error) {
|
||||
// inside it. When ctx is cancelled, the read stops before the next directory.
|
||||
func parseStore(ctx context.Context, fsys fs.FS, root string) (*Store, error) {
|
||||
store := &Store{RootDirectory: root}
|
||||
|
||||
err := parseDCIM(fsys, store)
|
||||
err := parseDCIM(ctx, fsys, store)
|
||||
if err == nil {
|
||||
err = parseM4Root(fsys, store)
|
||||
err = parseM4Root(ctx, fsys, store)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -68,16 +69,26 @@ func hasDirectory(fsys fs.FS, name string) (bool, error) {
|
||||
return info.IsDir(), nil
|
||||
}
|
||||
|
||||
// readDir returns the entries of the named directory, unless ctx is cancelled.
|
||||
func readDir(ctx context.Context, fsys fs.FS, name string) ([]fs.DirEntry, error) {
|
||||
err := ctx.Err()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return fs.ReadDir(fsys, name)
|
||||
}
|
||||
|
||||
// parseDCIM adds the DCF directories under DCIM to store. Everything else there
|
||||
// is not conforming. That includes directories that share a directory number,
|
||||
// which section 7.1.2 of the specification treats as non-DCF directories.
|
||||
func parseDCIM(fsys fs.FS, store *Store) error {
|
||||
func parseDCIM(ctx context.Context, fsys fs.FS, store *Store) error {
|
||||
found, err := hasDirectory(fsys, dcimDirectory)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
|
||||
entries, err := fs.ReadDir(fsys, dcimDirectory)
|
||||
entries, err := readDir(ctx, fsys, dcimDirectory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -94,9 +105,9 @@ func parseDCIM(fsys fs.FS, store *Store) error {
|
||||
number := parseNumber(dcfDirectoryName, entry.Name())
|
||||
|
||||
if entry.IsDir() && number != 0 && directoriesWithNumber[number] == 1 {
|
||||
err = parseDirectory(fsys, store, entry.Name(), number)
|
||||
err = parseDirectory(ctx, fsys, store, entry.Name(), number)
|
||||
} else {
|
||||
err = addNonConforming(fsys, store, dcimDirectory, entry)
|
||||
err = addNonConforming(ctx, fsys, store, dcimDirectory, entry)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -112,10 +123,16 @@ func parseDCIM(fsys fs.FS, store *Store) error {
|
||||
// Everything else there is not conforming. That includes JPG files that share a
|
||||
// file number, which section 7.2.2 of the specification says must not show DCF
|
||||
// numbers. Other files that share a file number do conform.
|
||||
func parseDirectory(fsys fs.FS, store *Store, name string, number int) error {
|
||||
func parseDirectory(
|
||||
ctx context.Context,
|
||||
fsys fs.FS,
|
||||
store *Store,
|
||||
name string,
|
||||
number int,
|
||||
) error {
|
||||
directoryPath := path.Join(dcimDirectory, name)
|
||||
|
||||
entries, err := fs.ReadDir(fsys, directoryPath)
|
||||
entries, err := readDir(ctx, fsys, directoryPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -135,7 +152,7 @@ func parseDirectory(fsys fs.FS, store *Store, name string, number int) error {
|
||||
sharedJPG := isJPG(entry.Name()) && jpgsWithNumber[fileNumber] > 1
|
||||
|
||||
if !entry.Type().IsRegular() || fileNumber == 0 || sharedJPG {
|
||||
err = addNonConforming(fsys, store, directoryPath, entry)
|
||||
err = addNonConforming(ctx, fsys, store, directoryPath, entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -145,7 +162,7 @@ func parseDirectory(fsys fs.FS, store *Store, name string, number int) error {
|
||||
|
||||
entryPath := path.Join(directoryPath, entry.Name())
|
||||
|
||||
object, err := newObject(store, entryPath, entry)
|
||||
object, err := newObject(fsys, store, entryPath, entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -167,7 +184,7 @@ func parseDirectory(fsys fs.FS, store *Store, name string, number int) error {
|
||||
// naming rules do not apply there, so these files have no DCF numbers. It skips
|
||||
// a symlink. It also skips a name that is not valid UTF-8, because fs.FS refuses
|
||||
// to open one.
|
||||
func parseM4Root(fsys fs.FS, store *Store) error {
|
||||
func parseM4Root(ctx context.Context, fsys fs.FS, store *Store) error {
|
||||
found, err := hasDirectory(fsys, m4RootDirectory)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
@@ -185,11 +202,16 @@ func parseM4Root(fsys fs.FS, store *Store) error {
|
||||
return fs.SkipDir
|
||||
}
|
||||
|
||||
// fs.WalkDir reads a directory right after this call for it.
|
||||
if entry.IsDir() {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if !entry.Type().IsRegular() || !validName {
|
||||
return nil
|
||||
}
|
||||
|
||||
object, err := newObject(store, filePath, entry)
|
||||
object, err := newObject(fsys, store, filePath, entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -204,7 +226,13 @@ func parseM4Root(fsys fs.FS, store *Store) error {
|
||||
// conforming. If the entry is a directory, it lists everything inside it too. It
|
||||
// does not follow a symlink. It does not read a directory whose name is not
|
||||
// valid UTF-8, because fs.FS refuses to open one.
|
||||
func addNonConforming(fsys fs.FS, store *Store, dir string, entry fs.DirEntry) error {
|
||||
func addNonConforming(
|
||||
ctx context.Context,
|
||||
fsys fs.FS,
|
||||
store *Store,
|
||||
dir string,
|
||||
entry fs.DirEntry,
|
||||
) error {
|
||||
entryPath := path.Join(dir, entry.Name())
|
||||
store.NonConforming = append(store.NonConforming, entryPath)
|
||||
|
||||
@@ -212,13 +240,13 @@ func addNonConforming(fsys fs.FS, store *Store, dir string, entry fs.DirEntry) e
|
||||
return nil
|
||||
}
|
||||
|
||||
entries, err := fs.ReadDir(fsys, entryPath)
|
||||
entries, err := readDir(ctx, fsys, entryPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, inner := range entries {
|
||||
err = addNonConforming(fsys, store, entryPath, inner)
|
||||
err = addNonConforming(ctx, fsys, store, entryPath, inner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -228,7 +256,12 @@ func addNonConforming(fsys fs.FS, store *Store, dir string, entry fs.DirEntry) e
|
||||
}
|
||||
|
||||
// newObject returns the Object for the file at filePath, with no DCF numbers.
|
||||
func newObject(store *Store, filePath string, entry fs.DirEntry) (Object, error) {
|
||||
func newObject(
|
||||
fsys fs.FS,
|
||||
store *Store,
|
||||
filePath string,
|
||||
entry fs.DirEntry,
|
||||
) (Object, error) {
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return Object{}, err
|
||||
@@ -239,6 +272,7 @@ func newObject(store *Store, filePath string, entry fs.DirEntry) (Object, error)
|
||||
Path: filePath,
|
||||
Size: info.Size(),
|
||||
Extension: path.Ext(filePath),
|
||||
fsys: fsys,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package dcf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
@@ -65,7 +66,7 @@ func addSymlink(t *testing.T, root, name, target string) {
|
||||
func mustParseStore(t *testing.T, fsys fs.FS) *Store {
|
||||
t.Helper()
|
||||
|
||||
store, err := parseStore(fsys, testRoot)
|
||||
store, err := parseStore(t.Context(), fsys, testRoot)
|
||||
if err != nil {
|
||||
t.Fatalf("parseStore: %v", err)
|
||||
}
|
||||
@@ -157,6 +158,7 @@ func TestParseStorePutsEachFileWhereItBelongs(t *testing.T) {
|
||||
Extension: ".ARW",
|
||||
DirectoryNumber: 100,
|
||||
FileNumber: 1,
|
||||
fsys: card,
|
||||
}
|
||||
mp4 := Object{
|
||||
DCFStoreRoot: testRoot,
|
||||
@@ -165,6 +167,7 @@ func TestParseStorePutsEachFileWhereItBelongs(t *testing.T) {
|
||||
Extension: ".MP4",
|
||||
DirectoryNumber: 101,
|
||||
FileNumber: 2,
|
||||
fsys: card,
|
||||
}
|
||||
thm := Object{
|
||||
DCFStoreRoot: testRoot,
|
||||
@@ -173,12 +176,14 @@ func TestParseStorePutsEachFileWhereItBelongs(t *testing.T) {
|
||||
Extension: ".THM",
|
||||
DirectoryNumber: 101,
|
||||
FileNumber: 2,
|
||||
fsys: card,
|
||||
}
|
||||
clip := Object{
|
||||
DCFStoreRoot: testRoot,
|
||||
Path: "PRIVATE/M4ROOT/CLIP/C0002.MP4",
|
||||
Size: 400,
|
||||
Extension: ".MP4",
|
||||
fsys: card,
|
||||
}
|
||||
|
||||
wantDirectories := []*Directory{
|
||||
@@ -313,7 +318,9 @@ func TestParseNumberOfFileName(t *testing.T) {
|
||||
func TestParseStoreTreatsLowerCaseNamesAsDCFNames(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store := mustParseStore(t, newCard("DCIM/101abcde/abcd0002.jpg"))
|
||||
card := newCard("DCIM/101abcde/abcd0002.jpg")
|
||||
|
||||
store := mustParseStore(t, card)
|
||||
|
||||
want := []*Directory{wantDirectory(101, "101abcde", &Object{
|
||||
DCFStoreRoot: testRoot,
|
||||
@@ -321,6 +328,7 @@ func TestParseStoreTreatsLowerCaseNamesAsDCFNames(t *testing.T) {
|
||||
Extension: ".jpg",
|
||||
DirectoryNumber: 101,
|
||||
FileNumber: 2,
|
||||
fsys: card,
|
||||
})}
|
||||
if !reflect.DeepEqual(store.Directories, want) {
|
||||
t.Errorf("Directories = %+v, want %+v", store.Directories, want)
|
||||
@@ -797,7 +805,7 @@ func TestParseStoreReturnsReadErrorsNamingTheCard(t *testing.T) {
|
||||
for _, name := range names {
|
||||
fsys := failingFS{MapFS: card, call: call, name: name}
|
||||
|
||||
_, err := parseStore(fsys, testRoot)
|
||||
_, err := parseStore(t.Context(), fsys, testRoot)
|
||||
if !errors.Is(err, fs.ErrPermission) {
|
||||
t.Errorf("%s of %s: error = %v, want %v",
|
||||
call, name, err, fs.ErrPermission)
|
||||
@@ -813,3 +821,86 @@ func TestParseStoreReturnsReadErrorsNamingTheCard(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cancelingFS is a card that cancels the read when the read reaches the named
|
||||
// directory. It records every directory read in read.
|
||||
type cancelingFS struct {
|
||||
fstest.MapFS
|
||||
|
||||
name string
|
||||
cancel context.CancelFunc
|
||||
read *[]string
|
||||
}
|
||||
|
||||
func (f cancelingFS) ReadDir(name string) ([]fs.DirEntry, error) {
|
||||
*f.read = append(*f.read, name)
|
||||
|
||||
if name == f.name {
|
||||
f.cancel()
|
||||
}
|
||||
|
||||
return f.MapFS.ReadDir(name)
|
||||
}
|
||||
|
||||
func TestParseStoreStopsBeforeTheNextDirectoryWhenCancelled(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
card := newCard(
|
||||
"DCIM/100ABCDE/SUB/README.TXT",
|
||||
"DCIM/OTHER/SUB/README.TXT",
|
||||
"PRIVATE/M4ROOT/CLIP/SUB/C0001.MP4",
|
||||
)
|
||||
|
||||
// Each of these directories holds another directory, which must not be read.
|
||||
names := []string{
|
||||
dcimDirectory,
|
||||
"DCIM/100ABCDE",
|
||||
"DCIM/OTHER",
|
||||
m4RootDirectory,
|
||||
"PRIVATE/M4ROOT/CLIP",
|
||||
}
|
||||
|
||||
for _, name := range names {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
|
||||
var read []string
|
||||
|
||||
fsys := cancelingFS{MapFS: card, name: name, cancel: cancel, read: &read}
|
||||
|
||||
_, err := parseStore(ctx, fsys, testRoot)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("cancelled at %s: error = %v, want %v",
|
||||
name, err, context.Canceled)
|
||||
}
|
||||
|
||||
if read[len(read)-1] != name {
|
||||
t.Errorf("cancelled at %s: read %q", name, read)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseStoreReadsNoDirectoryWhenAlreadyCancelled(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
|
||||
cards := []fstest.MapFS{
|
||||
newCard("DCIM/README.TXT"),
|
||||
newCard("PRIVATE/M4ROOT/C0001.MP4"),
|
||||
}
|
||||
|
||||
for _, card := range cards {
|
||||
var read []string
|
||||
|
||||
// The context is already cancelled, so this needs no name to cancel at.
|
||||
fsys := cancelingFS{MapFS: card, read: &read}
|
||||
|
||||
_, err := parseStore(ctx, fsys, testRoot)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("error = %v, want %v", err, context.Canceled)
|
||||
}
|
||||
|
||||
assertPaths(t, "directories read", read)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user