Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d2728b1cea | ||
|
|
acb108a82e | ||
|
|
cf7611ea1b |
No files matched your search
@@ -3,7 +3,7 @@
|
|||||||
## Description
|
## Description
|
||||||
|
|
||||||
`dcf` is a WTFPL-licensed Go library by [@sneak](https://sneak.berlin) that
|
`dcf` is a WTFPL-licensed Go library by [@sneak](https://sneak.berlin) that
|
||||||
reads DCF memory cards from digital cameras.
|
reads DCF memory cards from digital cameras and copies the files off them.
|
||||||
|
|
||||||
DCF is the "Design rule for Camera File system". It is also JEITA specification
|
DCF is the "Design rule for Camera File system". It is also JEITA specification
|
||||||
number CP-3461. JEITA is the Japan Electronics and Information Technology
|
number CP-3461. JEITA is the Japan Electronics and Information Technology
|
||||||
@@ -29,6 +29,7 @@ Then list the DCF stores on the mounted filesystems:
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
|
|
||||||
@@ -36,17 +37,53 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
stores, err := dcf.GetDCFStores(0)
|
stores, err := dcf.GetStores(context.Background())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, store := range *stores {
|
for _, store := range stores {
|
||||||
fmt.Println(store.String())
|
fmt.Println(store.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Call `dcf.GetStore` instead when you expect exactly one DCF store. It returns
|
||||||
|
`dcf.ErrNoStore` when none is mounted, and `dcf.ErrMultipleStores` when more
|
||||||
|
than one is.
|
||||||
|
|
||||||
|
To copy the files off a card, call `Import` with a destination directory:
|
||||||
|
|
||||||
|
```go
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
|
||||||
|
"sneak.berlin/go/dcf/pkg/dcf"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
store, err := dcf.GetStore(ctx)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := store.Import(ctx, "/path/to/photos")
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println("copied:", result.Copied)
|
||||||
|
fmt.Println("skipped:", result.Skipped)
|
||||||
|
fmt.Println("conflicts:", result.Conflicts)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
To work on the library itself, you need docker and git:
|
To work on the library itself, you need docker and git:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
@@ -105,11 +142,13 @@ overengineering and reusable code.
|
|||||||
|
|
||||||
All the code is in the `pkg/dcf` package. The repo has no commands.
|
All the code is in the `pkg/dcf` package. The repo has no commands.
|
||||||
|
|
||||||
- `dcf.go` holds the public API. `GetDCFStores` finds the DCF stores and reads
|
- `dcf.go` holds the public API for reading a card. `GetStores` finds the DCF
|
||||||
each one.
|
stores and reads each one. `GetStore` returns the one DCF store when exactly
|
||||||
|
one is mounted.
|
||||||
- `helpers.go` finds the mount points that hold a DCF store.
|
- `helpers.go` finds the mount points that hold a DCF store.
|
||||||
- `parse.go` reads one DCF store through an `fs.FS` rooted at its mount point.
|
- `parse.go` reads one DCF store through an `fs.FS` rooted at its mount point.
|
||||||
- `filegroup.go` sorts the files of one DCF directory into file groups.
|
- `filegroup.go` sorts the files of one DCF directory into file groups.
|
||||||
|
- `import.go` holds `Store.Import`, which copies files off a card.
|
||||||
|
|
||||||
The main types are:
|
The main types are:
|
||||||
|
|
||||||
@@ -117,11 +156,14 @@ The main types are:
|
|||||||
- `Directory` is one DCF directory. It has a directory number, a name, the
|
- `Directory` is one DCF directory. It has a directory number, a name, the
|
||||||
objects inside it and their file groups.
|
objects inside it and their file groups.
|
||||||
- `Object` is one file in a store. A file in a DCF directory also has a
|
- `Object` is one file in a store. A file in a DCF directory also has a
|
||||||
directory number and a file number.
|
directory number and a file number. `Object.ModTime` is its modification time.
|
||||||
|
`Object.Hash` reads the file through the `fs.FS` of its store. An `Object` you
|
||||||
|
build yourself has none, so its `Hash` returns an error.
|
||||||
- `FileGroup` is one file group in a DCF directory.
|
- `FileGroup` is one file group in a DCF directory.
|
||||||
- `Image` and `Video` embed `Object`. The file extension decides which one a
|
- `Image` and `Video` embed `Object`. The file extension decides which one a
|
||||||
file becomes. A file with any other extension, such as a `THM` thumbnail, is
|
file becomes. A file with any other extension, such as a `THM` thumbnail, is
|
||||||
neither.
|
neither.
|
||||||
|
- `ImportResult` lists what `Store.Import` did with each file.
|
||||||
|
|
||||||
A mounted filesystem is a DCF store when its root holds a `DCIM` or
|
A mounted filesystem is a DCF store when its root holds a `DCIM` or
|
||||||
`PRIVATE/M4ROOT` directory. It needs only one of them. `dcf` never mounts a
|
`PRIVATE/M4ROOT` directory. It needs only one of them. `dcf` never mounts a
|
||||||
@@ -129,6 +171,10 @@ filesystem. Media anywhere else on the filesystem is not part of the store.
|
|||||||
|
|
||||||
`dcf` never follows a symlink.
|
`dcf` never follows a symlink.
|
||||||
|
|
||||||
|
`GetStores` and `GetStore` take a `context.Context`. When it is cancelled, the
|
||||||
|
read stops before the next directory. The returned error matches
|
||||||
|
`context.Canceled`.
|
||||||
|
|
||||||
`dcf` applies the naming rules of the DCF specification, CIPA DC-009-2010, under
|
`dcf` applies the naming rules of the DCF specification, CIPA DC-009-2010, under
|
||||||
`DCIM`:
|
`DCIM`:
|
||||||
|
|
||||||
@@ -180,11 +226,49 @@ The primary member is the first of these that the group holds:
|
|||||||
|
|
||||||
`Store.Images` and `Store.Videos` list each file on its own, whatever its group.
|
`Store.Images` and `Store.Videos` list each file on its own, whatever its group.
|
||||||
|
|
||||||
|
`Store.Import` copies files off a card to a destination directory. It verifies
|
||||||
|
each copy against the SHA256 hash of its source. It only reads the card, so it
|
||||||
|
never deletes anything there.
|
||||||
|
|
||||||
|
It copies:
|
||||||
|
|
||||||
|
- every member of every file group
|
||||||
|
- the images and videos under `PRIVATE/M4ROOT`
|
||||||
|
|
||||||
|
It does not copy the files in `Store.NonConforming`.
|
||||||
|
|
||||||
|
Each file goes to `YYYY-MM-DD/<file name>` under the destination directory. The
|
||||||
|
date is the day of the file's modification time, in the computer's time zone.
|
||||||
|
Every member of a file group goes to the day of its primary member, so a group
|
||||||
|
never splits across two folders. Each copy keeps the modification time of its
|
||||||
|
source. Only the user who ran the import can read the copies.
|
||||||
|
|
||||||
|
`ImportResult` lists each destination path in one of three lists:
|
||||||
|
|
||||||
|
- `Copied` holds the copies this import wrote and verified.
|
||||||
|
- `Skipped` holds the paths that already held a file with the same SHA256 hash.
|
||||||
|
- `Conflicts` holds the paths that already held anything else. `dcf` leaves that
|
||||||
|
alone and does not copy the file.
|
||||||
|
|
||||||
|
A conflict is not an error, so a caller must check `Conflicts`. These are the
|
||||||
|
usual causes:
|
||||||
|
|
||||||
|
- Two different files have the same name and the same day. Two cards can reuse a
|
||||||
|
file number, and so can two DCF directories on one card. The first file is
|
||||||
|
copied, and the second is a conflict.
|
||||||
|
- A partial file remains because the process died during a copy. Delete it and
|
||||||
|
import again.
|
||||||
|
|
||||||
|
A copy that fails or does not match the hash of its source is removed. One file
|
||||||
|
that fails does not stop the others. The returned error joins the failures, each
|
||||||
|
naming its file. The result still lists what was done.
|
||||||
|
|
||||||
|
When the context is cancelled, the import stops at once, even inside a file, and
|
||||||
|
removes the partial copy. The returned error matches `context.Canceled`.
|
||||||
|
|
||||||
## TODO
|
## TODO
|
||||||
|
|
||||||
- Take a `context.Context` to cancel the walk of a store.
|
No open items.
|
||||||
- Return a plain slice from `GetDCFStores`, not a pointer to one.
|
|
||||||
- Copy files off a card, and verify each copy against its SHA256 hash.
|
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
|
|||||||
+68
-34
@@ -3,22 +3,32 @@
|
|||||||
// the mounted filesystems and parses the DCF directories and files on each one
|
// the mounted filesystems and parses the DCF directories and files on each one
|
||||||
// into their directory numbers and file numbers. It lists the images and videos
|
// into their directory numbers and file numbers. It lists the images and videos
|
||||||
// on each card, and everything under DCIM that is not a DCF directory or file. It
|
// on each card, and everything under DCIM that is not a DCF directory or file. It
|
||||||
// groups the files in each DCF directory that share a file number.
|
// groups the files in each DCF directory that share a file number. It copies the
|
||||||
|
// files off a card, and verifies each copy against its SHA256 hash.
|
||||||
package dcf
|
package dcf
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"crypto/sha256"
|
"context"
|
||||||
"encoding/hex"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
// ErrNoStore is the error from GetStore when no DCF store is mounted.
|
||||||
|
ErrNoStore = errors.New("no DCF store is mounted")
|
||||||
|
// ErrMultipleStores is the error from GetStore when more than one DCF store
|
||||||
|
// is mounted.
|
||||||
|
ErrMultipleStores = errors.New("more than one DCF store is mounted")
|
||||||
|
)
|
||||||
|
|
||||||
// Store represents a DCF store, which is a directory structure that contains
|
// Store represents a DCF store, which is a directory structure that contains
|
||||||
// images and videos in the root of a mounted filesystem.
|
// images and videos in the root of a mounted filesystem.
|
||||||
type Store struct {
|
type Store struct {
|
||||||
@@ -72,10 +82,15 @@ type Object struct {
|
|||||||
Path string
|
Path string
|
||||||
Size int64
|
Size int64
|
||||||
Extension string
|
Extension string
|
||||||
|
// ModTime is the modification time of the file.
|
||||||
|
ModTime time.Time
|
||||||
// DirectoryNumber and FileNumber are the DCF numbers of a file in a DCF
|
// DirectoryNumber and FileNumber are the DCF numbers of a file in a DCF
|
||||||
// directory. Both are 0 for a file under PRIVATE/M4ROOT.
|
// directory. Both are 0 for a file under PRIVATE/M4ROOT.
|
||||||
DirectoryNumber int
|
DirectoryNumber int
|
||||||
FileNumber int
|
FileNumber int
|
||||||
|
// fsys is the fs.FS the store was read through. Hash reads the file through
|
||||||
|
// it.
|
||||||
|
fsys fs.FS
|
||||||
}
|
}
|
||||||
|
|
||||||
// Image represents an image file on a DCF store.
|
// Image represents an image file on a DCF store.
|
||||||
@@ -173,61 +188,80 @@ func (i *Image) String() string {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Hash returns the SHA256 hash of the file as a hex string.
|
// Hash returns the SHA256 hash of the file as a hex string. It reads the file
|
||||||
|
// through the fs.FS of its store. An Object built by hand has none, so Hash
|
||||||
|
// returns an error that matches fs.ErrInvalid.
|
||||||
func (d *Object) Hash() (string, error) {
|
func (d *Object) Hash() (string, error) {
|
||||||
return pathToSHA256(d.FullFilePath())
|
hash, err := d.hash(context.Background())
|
||||||
}
|
|
||||||
|
|
||||||
func pathToSHA256(path string) (string, error) {
|
|
||||||
file, err := os.Open(filepath.Clean(path))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", fmt.Errorf("hashing %s: %w", d.FullFilePath(), err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() { _ = file.Close() }()
|
return hash, nil
|
||||||
|
|
||||||
hash := sha256.New()
|
|
||||||
|
|
||||||
_, err = io.Copy(hash, file)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return hex.EncodeToString(hash.Sum(nil)), nil
|
// hash returns the SHA256 hash of the file as a hex string, and stops when ctx is
|
||||||
|
// cancelled.
|
||||||
|
func (d *Object) hash(ctx context.Context) (string, error) {
|
||||||
|
if d.fsys == nil {
|
||||||
|
return "", fs.ErrInvalid
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetDCFStores returns a list of DCF stores found on the system in the root of
|
return hashFile(ctx, d.fsys, d.Path)
|
||||||
// any mounted filesystems. The integer argument is the number of DCF stores to
|
}
|
||||||
// return. If the argument is 0, all DCF stores are returned. It does not mount
|
|
||||||
// filesystems or search outside of filesystem root directories. It does,
|
// GetStores finds the DCF stores in the root of the mounted filesystems and reads
|
||||||
// however, read the DCIM and PRIVATE/M4ROOT directories of each filesystem to
|
// each one. It does not mount filesystems or search outside of filesystem root
|
||||||
// populate the returned Stores. It never follows a symlink. If a card cannot be
|
// directories. It reads the DCIM and PRIVATE/M4ROOT directories of each
|
||||||
// read, it returns an error that names the card's mount point.
|
// filesystem to populate the returned Stores. It never follows a symlink. If a
|
||||||
func GetDCFStores(requestedCount int) (*[]Store, error) {
|
// card cannot be read, it returns an error that names the card's mount point.
|
||||||
dcfStorePaths, err := findDCFMountPoints(requestedCount)
|
// When ctx is cancelled, the read stops before the next directory, and the error
|
||||||
|
// matches ctx.Err().
|
||||||
|
func GetStores(ctx context.Context) ([]*Store, error) {
|
||||||
|
dcfStorePaths, err := findDCFMountPoints(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
dcfStores, err := parseStores(dcfStorePaths)
|
return parseStores(ctx, dcfStorePaths)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetStore returns the one DCF store when exactly one is mounted. It returns
|
||||||
|
// ErrNoStore when none is mounted, and ErrMultipleStores when more than one is.
|
||||||
|
// It reads every card first, the way GetStores does, so a card that cannot be
|
||||||
|
// read returns that error instead.
|
||||||
|
func GetStore(ctx context.Context) (*Store, error) {
|
||||||
|
stores, err := GetStores(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return &dcfStores, nil
|
return onlyStore(stores)
|
||||||
|
}
|
||||||
|
|
||||||
|
// onlyStore returns the store when stores holds exactly one.
|
||||||
|
func onlyStore(stores []*Store) (*Store, error) {
|
||||||
|
switch len(stores) {
|
||||||
|
case 0:
|
||||||
|
return nil, ErrNoStore
|
||||||
|
case 1:
|
||||||
|
return stores[0], nil
|
||||||
|
default:
|
||||||
|
return nil, ErrMultipleStores
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseStores reads the card at each of the given mount points.
|
// parseStores reads the card at each of the given mount points.
|
||||||
func parseStores(mountpoints []string) ([]Store, error) {
|
func parseStores(ctx context.Context, mountpoints []string) ([]*Store, error) {
|
||||||
dcfStores := []Store{}
|
dcfStores := []*Store{}
|
||||||
|
|
||||||
for _, mountpoint := range mountpoints {
|
for _, mountpoint := range mountpoints {
|
||||||
dcfStore, err := parseStore(os.DirFS(mountpoint), mountpoint)
|
dcfStore, err := parseStore(ctx, os.DirFS(mountpoint), mountpoint)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
dcfStores = append(dcfStores, *dcfStore)
|
dcfStores = append(dcfStores, dcfStore)
|
||||||
}
|
}
|
||||||
|
|
||||||
return dcfStores, nil
|
return dcfStores, nil
|
||||||
|
|||||||
@@ -1,9 +1,13 @@
|
|||||||
package dcf
|
package dcf
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io/fs"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestParseStoresReadsEachMountPoint(t *testing.T) {
|
func TestParseStoresReadsEachMountPoint(t *testing.T) {
|
||||||
@@ -12,7 +16,7 @@ func TestParseStoresReadsEachMountPoint(t *testing.T) {
|
|||||||
first := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG")
|
first := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG")
|
||||||
second := newCardOnDisk(t, "PRIVATE/M4ROOT/CLIP/C0001.MP4")
|
second := newCardOnDisk(t, "PRIVATE/M4ROOT/CLIP/C0001.MP4")
|
||||||
|
|
||||||
stores, err := parseStores([]string{first, second})
|
stores, err := parseStores(t.Context(), []string{first, second})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("parseStores: %v", err)
|
t.Fatalf("parseStores: %v", err)
|
||||||
}
|
}
|
||||||
@@ -50,12 +54,92 @@ func TestParseStoresReturnsErrorNamingTheCard(t *testing.T) {
|
|||||||
// Every lookup under a plain file fails.
|
// Every lookup under a plain file fails.
|
||||||
mountpoint := filepath.Join(newCardOnDisk(t, "FILE"), "FILE")
|
mountpoint := filepath.Join(newCardOnDisk(t, "FILE"), "FILE")
|
||||||
|
|
||||||
_, err := parseStores([]string{mountpoint})
|
_, err := parseStores(t.Context(), []string{mountpoint})
|
||||||
if err == nil || !strings.Contains(err.Error(), mountpoint) {
|
if err == nil || !strings.Contains(err.Error(), mountpoint) {
|
||||||
t.Fatalf("parseStores error = %v, want one naming %s", err, mountpoint)
|
t.Fatalf("parseStores error = %v, want one naming %s", err, mountpoint)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseStoresReturnsErrorWhenCancelled(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
mountpoint := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG")
|
||||||
|
|
||||||
|
_, err := parseStores(ctx, []string{mountpoint})
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("parseStores error = %v, want %v", err, context.Canceled)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOnlyStore(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
first := newMountPoint(t, dcimDirectory)
|
||||||
|
second := newMountPoint(t, m4RootDirectory)
|
||||||
|
|
||||||
|
stores, err := parseStores(t.Context(), []string{first, second})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parseStores: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = onlyStore(stores[:0])
|
||||||
|
if !errors.Is(err, ErrNoStore) {
|
||||||
|
t.Errorf("onlyStore of no cards: error = %v, want %v", err, ErrNoStore)
|
||||||
|
}
|
||||||
|
|
||||||
|
store, err := onlyStore(stores[:1])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("onlyStore of one card: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if store.RootDirectory != first {
|
||||||
|
t.Errorf("onlyStore returned the card at %s, want %s",
|
||||||
|
store.RootDirectory, first)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = onlyStore(stores)
|
||||||
|
if !errors.Is(err, ErrMultipleStores) {
|
||||||
|
t.Errorf("onlyStore of two cards: error = %v, want %v",
|
||||||
|
err, ErrMultipleStores)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestObjectHashReadsThroughTheCard(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
card := fstest.MapFS{"DCIM/100ABCDE/ABCD0001.JPG": {Data: []byte("abc")}}
|
||||||
|
|
||||||
|
got, err := mustParseStore(t, card).Images[0].Hash()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Hash() returned error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The published SHA-256 test vector for "abc".
|
||||||
|
want := "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("Hash() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestObjectHashOfMissingFileReturnsError(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const filePath = "DCIM/100ABCDE/ABCD0001.JPG"
|
||||||
|
|
||||||
|
card := newCard(filePath)
|
||||||
|
image := mustParseStore(t, card).Images[0]
|
||||||
|
|
||||||
|
delete(card, filePath)
|
||||||
|
|
||||||
|
_, err := image.Hash()
|
||||||
|
if !errors.Is(err, fs.ErrNotExist) {
|
||||||
|
t.Fatalf("Hash() error = %v, want %v", err, fs.ErrNotExist)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestIsImageFile(t *testing.T) {
|
func TestIsImageFile(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
+24
-30
@@ -1,8 +1,9 @@
|
|||||||
package dcf_test
|
package dcf_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"os"
|
"context"
|
||||||
"path/filepath"
|
"errors"
|
||||||
|
"io/fs"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"sneak.berlin/go/dcf/pkg/dcf"
|
"sneak.berlin/go/dcf/pkg/dcf"
|
||||||
@@ -125,37 +126,30 @@ func TestVideoString(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestObjectHashIsSHA256OfFile(t *testing.T) {
|
func TestObjectHashOfObjectBuiltByHandReturnsError(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
root := t.TempDir()
|
object := dcf.Object{DCFStoreRoot: cardRoot, Path: "a.jpg"}
|
||||||
|
|
||||||
err := os.WriteFile(filepath.Join(root, "a.jpg"), []byte("abc"), 0o600)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("WriteFile: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
object := dcf.Object{DCFStoreRoot: root, Path: "a.jpg"}
|
|
||||||
|
|
||||||
got, err := object.Hash()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Hash() returned error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The published SHA-256 test vector for "abc".
|
|
||||||
want := "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
|
|
||||||
if got != want {
|
|
||||||
t.Fatalf("Hash() = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestObjectHashOfMissingFileReturnsError(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
object := dcf.Object{DCFStoreRoot: t.TempDir(), Path: "absent.jpg"}
|
|
||||||
|
|
||||||
_, err := object.Hash()
|
_, err := object.Hash()
|
||||||
if err == nil {
|
if !errors.Is(err, fs.ErrInvalid) {
|
||||||
t.Fatal("Hash() of a missing file returned no error")
|
t.Fatalf("Hash() error = %v, want %v", err, fs.ErrInvalid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetStoresAndGetStoreReturnCancellation(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
_, err := dcf.GetStores(ctx)
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Errorf("GetStores error = %v, want %v", err, context.Canceled)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = dcf.GetStore(ctx)
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Errorf("GetStore error = %v, want %v", err, context.Canceled)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+17
-15
@@ -1,6 +1,7 @@
|
|||||||
package dcf
|
package dcf
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
@@ -9,12 +10,19 @@ import (
|
|||||||
"github.com/shirou/gopsutil/disk"
|
"github.com/shirou/gopsutil/disk"
|
||||||
)
|
)
|
||||||
|
|
||||||
func findAllMountPoints() ([]string, error) {
|
func findAllMountPoints(ctx context.Context) ([]string, error) {
|
||||||
mountpoints := []string{}
|
mountpoints := []string{}
|
||||||
|
|
||||||
partitions, err := disk.Partitions(false) // physical devices only, so false
|
// disk.PartitionsWithContext does not check ctx on Linux.
|
||||||
|
err := ctx.Err()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("listing the mounted filesystems: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// physical devices only, so false
|
||||||
|
partitions, err := disk.PartitionsWithContext(ctx, false)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("listing the mounted filesystems: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, partition := range partitions {
|
for _, partition := range partitions {
|
||||||
@@ -26,21 +34,18 @@ func findAllMountPoints() ([]string, error) {
|
|||||||
return mountpoints, nil
|
return mountpoints, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// findDCFMountPoints returns a list of strings of the paths of mountpoints
|
// findDCFMountPoints returns the paths of the mountpoints that hold a DCF store.
|
||||||
// that hold a DCF store. Its only argument is an integer of how many mountpoints
|
func findDCFMountPoints(ctx context.Context) ([]string, error) {
|
||||||
// to return. If the argument is 0, all mountpoints are returned.
|
mountpoints, err := findAllMountPoints(ctx)
|
||||||
func findDCFMountPoints(requestedCount int) ([]string, error) {
|
|
||||||
mountpoints, err := findAllMountPoints()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return filterDCFMountPoints(mountpoints, requestedCount), nil
|
return filterDCFMountPoints(mountpoints), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// filterDCFMountPoints returns the first requestedCount of the mountpoints that
|
// filterDCFMountPoints returns the mountpoints that hold a DCF store.
|
||||||
// hold a DCF store. If requestedCount is 0, it returns all of them.
|
func filterDCFMountPoints(mountpoints []string) []string {
|
||||||
func filterDCFMountPoints(mountpoints []string, requestedCount int) []string {
|
|
||||||
filteredMountpoints := []string{}
|
filteredMountpoints := []string{}
|
||||||
|
|
||||||
for _, mountpoint := range mountpoints {
|
for _, mountpoint := range mountpoints {
|
||||||
@@ -51,9 +56,6 @@ func filterDCFMountPoints(mountpoints []string, requestedCount int) []string {
|
|||||||
slog.Debug(fmt.Sprintf("Keeping mountpoint %s\n", mountpoint))
|
slog.Debug(fmt.Sprintf("Keeping mountpoint %s\n", mountpoint))
|
||||||
|
|
||||||
filteredMountpoints = append(filteredMountpoints, mountpoint)
|
filteredMountpoints = append(filteredMountpoints, mountpoint)
|
||||||
if (requestedCount > 0) && (len(filteredMountpoints) >= requestedCount) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return filteredMountpoints
|
return filteredMountpoints
|
||||||
|
|||||||
@@ -72,29 +72,16 @@ func TestIsDCFMountPointIgnoresSymlink(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFilterDCFMountPointsReturnsRequestedCount(t *testing.T) {
|
func TestFilterDCFMountPointsKeepsThoseHoldingAStore(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
first := newMountPoint(t, "DCIM")
|
first := newMountPoint(t, "DCIM")
|
||||||
second := newMountPoint(t, "PRIVATE/M4ROOT")
|
second := newMountPoint(t, "PRIVATE/M4ROOT")
|
||||||
third := newMountPoint(t, "DCIM")
|
|
||||||
|
|
||||||
// This one holds no DCF store, so it must not use up the requested count.
|
got := filterDCFMountPoints([]string{newMountPoint(t), first, second})
|
||||||
mountpoints := []string{newMountPoint(t), first, second, third}
|
|
||||||
|
|
||||||
cases := map[int][]string{
|
want := []string{first, second}
|
||||||
0: {first, second, third},
|
|
||||||
1: {first},
|
|
||||||
2: {first, second},
|
|
||||||
3: {first, second, third},
|
|
||||||
4: {first, second, third},
|
|
||||||
}
|
|
||||||
|
|
||||||
for requestedCount, want := range cases {
|
|
||||||
got := filterDCFMountPoints(mountpoints, requestedCount)
|
|
||||||
if !slices.Equal(got, want) {
|
if !slices.Equal(got, want) {
|
||||||
t.Errorf("filterDCFMountPoints(%d) = %v, want %v",
|
t.Errorf("filterDCFMountPoints = %v, want %v", got, want)
|
||||||
requestedCount, got, want)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,279 @@
|
|||||||
|
package dcf
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The modes of what Import creates. Only the user who ran it can read the copies.
|
||||||
|
const (
|
||||||
|
importDirectoryMode = 0o750
|
||||||
|
importFileMode = 0o600
|
||||||
|
)
|
||||||
|
|
||||||
|
var errHashMismatch = errors.New("the SHA256 hash of the copy differs from the source")
|
||||||
|
|
||||||
|
// ImportResult lists what Store.Import did with each file, by destination path.
|
||||||
|
type ImportResult struct {
|
||||||
|
// Copied holds the copies this import wrote and verified.
|
||||||
|
Copied []string
|
||||||
|
// Skipped holds the paths that already held a file with the same SHA256 hash.
|
||||||
|
Skipped []string
|
||||||
|
// Conflicts holds the paths that already held anything else. Import left it
|
||||||
|
// alone and did not copy the file.
|
||||||
|
Conflicts []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// importFile is one file for Import to copy, and the path to copy it to.
|
||||||
|
type importFile struct {
|
||||||
|
source *Object
|
||||||
|
target string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Import copies the files of the store to dest, and verifies each copy against
|
||||||
|
// the SHA256 hash of its source. It copies every member of every file group, and
|
||||||
|
// the images and videos under PRIVATE/M4ROOT. It only reads the card.
|
||||||
|
//
|
||||||
|
// Each file goes to dest/YYYY-MM-DD/<file name>, from its modification time. Every
|
||||||
|
// member of a file group goes to the day of its primary member, so a group never
|
||||||
|
// splits across two folders.
|
||||||
|
//
|
||||||
|
// A file with the same hash already at the path is skipped. Anything else there is
|
||||||
|
// a conflict, which is not an error. Import never overwrites anything.
|
||||||
|
//
|
||||||
|
// One file that fails does not stop the others. The returned error joins the
|
||||||
|
// failures, each naming its file, and the result still lists what was done. When
|
||||||
|
// ctx is cancelled, the import stops at once and removes the partial copy, and the
|
||||||
|
// error matches ctx.Err().
|
||||||
|
func (d *Store) Import(ctx context.Context, dest string) (ImportResult, error) {
|
||||||
|
var (
|
||||||
|
result ImportResult
|
||||||
|
failures []error
|
||||||
|
)
|
||||||
|
|
||||||
|
for _, file := range d.importFiles(dest) {
|
||||||
|
err := ctx.Err()
|
||||||
|
if err != nil {
|
||||||
|
failures = append(failures, fmt.Errorf("import stopped: %w", err))
|
||||||
|
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
err = copyFile(ctx, file, &result)
|
||||||
|
if err != nil {
|
||||||
|
failures = append(failures,
|
||||||
|
fmt.Errorf("importing %s: %w", file.source.FullFilePath(), err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result, errors.Join(failures...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// importFiles lists the files of the store that Import copies to dest.
|
||||||
|
func (d *Store) importFiles(dest string) []importFile {
|
||||||
|
var files []importFile
|
||||||
|
|
||||||
|
add := func(source *Object, day time.Time) {
|
||||||
|
name := path.Base(source.Path)
|
||||||
|
|
||||||
|
files = append(files, importFile{
|
||||||
|
source: source,
|
||||||
|
target: filepath.Join(dest, day.Format(time.DateOnly), name),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, directory := range d.Directories {
|
||||||
|
for _, group := range directory.FileGroups {
|
||||||
|
for _, member := range group.Members {
|
||||||
|
add(member, group.Primary.ModTime)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The files under PRIVATE/M4ROOT are the ones with no DCF numbers.
|
||||||
|
for _, image := range d.Images {
|
||||||
|
if image.DirectoryNumber == 0 {
|
||||||
|
add(&image.Object, image.ModTime)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, video := range d.Videos {
|
||||||
|
if video.DirectoryNumber == 0 {
|
||||||
|
add(&video.Object, video.ModTime)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return files
|
||||||
|
}
|
||||||
|
|
||||||
|
// copyFile copies one file, and adds its target to the list in result that says
|
||||||
|
// what happened. It adds the target to no list when it returns an error.
|
||||||
|
func copyFile(ctx context.Context, file importFile, result *ImportResult) error {
|
||||||
|
sourceHash, err := file.source.hash(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = os.MkdirAll(filepath.Dir(file.target), importDirectoryMode)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// O_EXCL fails on anything already at the path, even a symlink, so nothing
|
||||||
|
// there can be overwritten.
|
||||||
|
target, err := os.OpenFile(
|
||||||
|
file.target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, importFileMode,
|
||||||
|
)
|
||||||
|
if errors.Is(err, fs.ErrExist) {
|
||||||
|
return checkExisting(ctx, file.target, sourceHash, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = writeCopy(ctx, file, target, sourceHash)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Join(err, os.Remove(file.target))
|
||||||
|
}
|
||||||
|
|
||||||
|
result.Copied = append(result.Copied, file.target)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkExisting adds target to the skipped files if it is a file with the given
|
||||||
|
// hash, and to the conflicts if it is anything else.
|
||||||
|
func checkExisting(
|
||||||
|
ctx context.Context,
|
||||||
|
target, sourceHash string,
|
||||||
|
result *ImportResult,
|
||||||
|
) error {
|
||||||
|
info, err := os.Lstat(target)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
result.Conflicts = append(result.Conflicts, target)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
targetHash, err := hashOnDisk(ctx, target)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if targetHash == sourceHash {
|
||||||
|
result.Skipped = append(result.Skipped, target)
|
||||||
|
} else {
|
||||||
|
result.Conflicts = append(result.Conflicts, target)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeCopy fills the new file target from the source, and closes it. It then
|
||||||
|
// reads the written file back to compare its hash with the hash of the source.
|
||||||
|
func writeCopy(
|
||||||
|
ctx context.Context,
|
||||||
|
file importFile,
|
||||||
|
target *os.File,
|
||||||
|
sourceHash string,
|
||||||
|
) error {
|
||||||
|
err := writeAndClose(ctx, file.source, target)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
targetHash, err := hashOnDisk(ctx, file.target)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if targetHash != sourceHash {
|
||||||
|
return errHashMismatch
|
||||||
|
}
|
||||||
|
|
||||||
|
// The zero time leaves the access time as it is.
|
||||||
|
return os.Chtimes(file.target, time.Time{}, file.source.ModTime)
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeAndClose writes the bytes of source to target, syncs target and closes it.
|
||||||
|
func writeAndClose(ctx context.Context, source *Object, target *os.File) error {
|
||||||
|
defer func() { _ = target.Close() }()
|
||||||
|
|
||||||
|
sourceFile, err := source.fsys.Open(source.Path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() { _ = sourceFile.Close() }()
|
||||||
|
|
||||||
|
_, err = io.Copy(target, contextReader(ctx, sourceFile))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = target.Sync()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return target.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// hashOnDisk returns the SHA256 hash of the file at filePath on the local
|
||||||
|
// filesystem.
|
||||||
|
func hashOnDisk(ctx context.Context, filePath string) (string, error) {
|
||||||
|
return hashFile(ctx, os.DirFS(filepath.Dir(filePath)), filepath.Base(filePath))
|
||||||
|
}
|
||||||
|
|
||||||
|
// hashFile returns the SHA256 hash of the named file as a hex string.
|
||||||
|
func hashFile(ctx context.Context, fsys fs.FS, name string) (string, error) {
|
||||||
|
file, err := fsys.Open(name)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() { _ = file.Close() }()
|
||||||
|
|
||||||
|
hash := sha256.New()
|
||||||
|
|
||||||
|
_, err = io.Copy(hash, contextReader(ctx, file))
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readerFunc is a function that works as an io.Reader.
|
||||||
|
type readerFunc func([]byte) (int, error)
|
||||||
|
|
||||||
|
func (f readerFunc) Read(p []byte) (int, error) {
|
||||||
|
return f(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
// contextReader returns a reader that fails with ctx.Err() once ctx is cancelled,
|
||||||
|
// so a copy stops inside a file.
|
||||||
|
func contextReader(ctx context.Context, reader io.Reader) io.Reader {
|
||||||
|
return readerFunc(func(p []byte) (int, error) {
|
||||||
|
err := ctx.Err()
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return reader.Read(p)
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,521 @@
|
|||||||
|
package dcf
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io/fs"
|
||||||
|
"maps"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// testDay is the folder that a file from cardFile goes to. The other constants
|
||||||
|
// are the paths and contents of the files on most cards in these tests.
|
||||||
|
const (
|
||||||
|
testDay = "2026-10-03"
|
||||||
|
|
||||||
|
firstJPG = "DCIM/100ABCDE/DSC00011.JPG"
|
||||||
|
firstJPGData = "first jpeg"
|
||||||
|
secondJPG = "DCIM/100ABCDE/DSC00012.JPG"
|
||||||
|
secondJPGData = "second jpeg"
|
||||||
|
testClip = "PRIVATE/M4ROOT/CLIP/C0011.MP4"
|
||||||
|
testClipData = "clip"
|
||||||
|
)
|
||||||
|
|
||||||
|
// cardFile returns a file on a card, modified at noon on testDay.
|
||||||
|
func cardFile(contents string) *fstest.MapFile {
|
||||||
|
return &fstest.MapFile{
|
||||||
|
Data: []byte(contents),
|
||||||
|
ModTime: time.Date(2026, 10, 3, 12, 0, 0, 0, time.UTC),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// inTestDay returns the paths of the named files in the testDay folder of dest.
|
||||||
|
func inTestDay(dest string, names ...string) []string {
|
||||||
|
paths := make([]string, 0, len(names))
|
||||||
|
|
||||||
|
for _, name := range names {
|
||||||
|
paths = append(paths, filepath.Join(dest, testDay, name))
|
||||||
|
}
|
||||||
|
|
||||||
|
return paths
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertDest checks that dest holds exactly the given files, as a map from each
|
||||||
|
// file's path under dest to its contents.
|
||||||
|
func assertDest(t *testing.T, dest string, want map[string]string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
fsys := os.DirFS(dest)
|
||||||
|
got := map[string]string{}
|
||||||
|
|
||||||
|
err := fs.WalkDir(fsys, ".",
|
||||||
|
func(name string, entry fs.DirEntry, err error) error {
|
||||||
|
if err != nil || !entry.Type().IsRegular() {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := fs.ReadFile(fsys, name)
|
||||||
|
got[name] = string(data)
|
||||||
|
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reading %s: %v", dest, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !maps.Equal(got, want) {
|
||||||
|
t.Errorf("destination holds %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertEmptyDirectory(t *testing.T, directory string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
entries, err := os.ReadDir(directory)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadDir: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(entries) != 0 {
|
||||||
|
t.Errorf("%s holds %v, want nothing", directory, entries)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// addDestFile writes a file at the named path under dest.
|
||||||
|
func addDestFile(t *testing.T, dest, name, contents string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
fullPath := filepath.Join(dest, name)
|
||||||
|
|
||||||
|
err := os.MkdirAll(filepath.Dir(fullPath), 0o750)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MkdirAll: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = os.WriteFile(fullPath, []byte(contents), 0o600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("WriteFile: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImportCopiesEveryFileAndKeepsItsModificationTime(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dest := t.TempDir()
|
||||||
|
|
||||||
|
store := mustParseStore(t, fstest.MapFS{
|
||||||
|
"DCIM/100ABCDE/DSC00011.ARW": cardFile("raw"),
|
||||||
|
firstJPG: cardFile(firstJPGData),
|
||||||
|
"DCIM/101ABCDE/MVI_0012.MP4": cardFile("video"),
|
||||||
|
"DCIM/101ABCDE/MVI_0012.THM": cardFile("thumbnail"),
|
||||||
|
"DCIM/MISC/DSC00013.JPG": cardFile("not conforming"),
|
||||||
|
testClip: cardFile(testClipData),
|
||||||
|
"PRIVATE/M4ROOT/CLIP/C0011M01.XML": cardFile("neither image nor video"),
|
||||||
|
"PRIVATE/M4ROOT/THMBNL/C0011T01.JPG": cardFile("clip thumbnail"),
|
||||||
|
})
|
||||||
|
|
||||||
|
result, err := store.Import(t.Context(), dest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Import: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "Copied", result.Copied, inTestDay(dest,
|
||||||
|
"DSC00011.ARW",
|
||||||
|
"DSC00011.JPG",
|
||||||
|
"MVI_0012.MP4",
|
||||||
|
"MVI_0012.THM",
|
||||||
|
"C0011T01.JPG",
|
||||||
|
"C0011.MP4",
|
||||||
|
)...)
|
||||||
|
assertPaths(t, "Skipped", result.Skipped)
|
||||||
|
assertPaths(t, "Conflicts", result.Conflicts)
|
||||||
|
|
||||||
|
assertDest(t, dest, map[string]string{
|
||||||
|
testDay + "/DSC00011.ARW": "raw",
|
||||||
|
testDay + "/DSC00011.JPG": firstJPGData,
|
||||||
|
testDay + "/MVI_0012.MP4": "video",
|
||||||
|
testDay + "/MVI_0012.THM": "thumbnail",
|
||||||
|
testDay + "/C0011.MP4": testClipData,
|
||||||
|
testDay + "/C0011T01.JPG": "clip thumbnail",
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, copied := range result.Copied {
|
||||||
|
info, err := os.Stat(copied)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Stat: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if want := cardFile("").ModTime; !info.ModTime().Equal(want) {
|
||||||
|
t.Errorf("%s modified at %v, want %v", copied, info.ModTime(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImportTwiceSkipsEveryFileTheSecondTime(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dest := t.TempDir()
|
||||||
|
|
||||||
|
store := mustParseStore(t, fstest.MapFS{
|
||||||
|
firstJPG: cardFile(firstJPGData),
|
||||||
|
testClip: cardFile(testClipData),
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := store.Import(t.Context(), dest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first Import: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := store.Import(t.Context(), dest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second Import: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "Copied", result.Copied)
|
||||||
|
assertPaths(t, "Skipped", result.Skipped,
|
||||||
|
inTestDay(dest, "DSC00011.JPG", "C0011.MP4")...)
|
||||||
|
assertPaths(t, "Conflicts", result.Conflicts)
|
||||||
|
|
||||||
|
assertDest(t, dest, map[string]string{
|
||||||
|
testDay + "/DSC00011.JPG": firstJPGData,
|
||||||
|
testDay + "/C0011.MP4": testClipData,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImportLeavesADifferentFileAloneAndReportsAConflict(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dest := t.TempDir()
|
||||||
|
addDestFile(t, dest, testDay+"/DSC00011.JPG", "different")
|
||||||
|
|
||||||
|
store := mustParseStore(t, fstest.MapFS{
|
||||||
|
firstJPG: cardFile(firstJPGData),
|
||||||
|
secondJPG: cardFile(secondJPGData),
|
||||||
|
})
|
||||||
|
|
||||||
|
result, err := store.Import(t.Context(), dest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Import: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "Copied", result.Copied, inTestDay(dest, "DSC00012.JPG")...)
|
||||||
|
assertPaths(t, "Skipped", result.Skipped)
|
||||||
|
assertPaths(t, "Conflicts", result.Conflicts, inTestDay(dest, "DSC00011.JPG")...)
|
||||||
|
|
||||||
|
assertDest(t, dest, map[string]string{
|
||||||
|
testDay + "/DSC00011.JPG": "different",
|
||||||
|
testDay + "/DSC00012.JPG": secondJPGData,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImportReportsAConflictBetweenTwoFilesWithOneName(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dest := t.TempDir()
|
||||||
|
|
||||||
|
store := mustParseStore(t, fstest.MapFS{
|
||||||
|
firstJPG: cardFile(firstJPGData),
|
||||||
|
"DCIM/101ABCDE/DSC00011.JPG": cardFile(secondJPGData),
|
||||||
|
})
|
||||||
|
|
||||||
|
result, err := store.Import(t.Context(), dest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Import: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "Copied", result.Copied, inTestDay(dest, "DSC00011.JPG")...)
|
||||||
|
assertPaths(t, "Conflicts", result.Conflicts, inTestDay(dest, "DSC00011.JPG")...)
|
||||||
|
assertDest(t, dest, map[string]string{testDay + "/DSC00011.JPG": firstJPGData})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImportReportsAConflictAtADirectoryOrASymlink(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dest := t.TempDir()
|
||||||
|
|
||||||
|
// The symlink points at a file with the same bytes, which must not count.
|
||||||
|
outside := newCardOnDisk(t, "OUTSIDE.JPG")
|
||||||
|
|
||||||
|
err := os.MkdirAll(filepath.Join(dest, testDay, "DSC00011.JPG"), 0o750)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MkdirAll: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
addSymlink(t, dest, testDay+"/DSC00012.JPG", filepath.Join(outside, "OUTSIDE.JPG"))
|
||||||
|
|
||||||
|
store := mustParseStore(t, fstest.MapFS{
|
||||||
|
firstJPG: cardFile("abc"),
|
||||||
|
secondJPG: cardFile("abc"),
|
||||||
|
})
|
||||||
|
|
||||||
|
result, err := store.Import(t.Context(), dest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Import: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "Copied", result.Copied)
|
||||||
|
assertPaths(t, "Skipped", result.Skipped)
|
||||||
|
assertPaths(t, "Conflicts", result.Conflicts,
|
||||||
|
inTestDay(dest, "DSC00011.JPG", "DSC00012.JPG")...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImportPutsAFileGroupInTheFolderOfItsPrimaryMember(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dest := t.TempDir()
|
||||||
|
|
||||||
|
beforeMidnight := time.Date(2026, 10, 3, 23, 59, 59, 0, time.UTC)
|
||||||
|
afterMidnight := time.Date(2026, 10, 4, 0, 0, 1, 0, time.UTC)
|
||||||
|
|
||||||
|
store := mustParseStore(t, fstest.MapFS{
|
||||||
|
"DCIM/100ABCDE/DSC00011.ARW": {Data: []byte("raw"), ModTime: afterMidnight},
|
||||||
|
firstJPG: {Data: []byte("jpeg"), ModTime: beforeMidnight},
|
||||||
|
secondJPG: {Data: []byte("next"), ModTime: afterMidnight},
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := store.Import(t.Context(), dest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Import: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertDest(t, dest, map[string]string{
|
||||||
|
"2026-10-03/DSC00011.ARW": "raw",
|
||||||
|
"2026-10-03/DSC00011.JPG": "jpeg",
|
||||||
|
"2026-10-04/DSC00012.JPG": "next",
|
||||||
|
})
|
||||||
|
|
||||||
|
// The copy keeps its own modification time, whichever folder it is in.
|
||||||
|
info, err := os.Stat(filepath.Join(dest, "2026-10-03/DSC00011.ARW"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Stat: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !info.ModTime().Equal(afterMidnight) {
|
||||||
|
t.Errorf("modified at %v, want %v", info.ModTime(), afterMidnight)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// changingFS is a card whose file at name holds other bytes from its second read
|
||||||
|
// on. It counts the reads in opens.
|
||||||
|
type changingFS struct {
|
||||||
|
fstest.MapFS
|
||||||
|
|
||||||
|
name string
|
||||||
|
opens *int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f changingFS) Open(name string) (fs.File, error) {
|
||||||
|
if name == f.name {
|
||||||
|
*f.opens++
|
||||||
|
if *f.opens > 1 {
|
||||||
|
return fstest.MapFS{name: cardFile("other bytes")}.Open(name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return f.MapFS.Open(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImportRemovesACopyThatDoesNotMatchItsSource(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dest := t.TempDir()
|
||||||
|
opens := 0
|
||||||
|
|
||||||
|
store := mustParseStore(t, changingFS{
|
||||||
|
MapFS: fstest.MapFS{
|
||||||
|
firstJPG: cardFile(firstJPGData),
|
||||||
|
secondJPG: cardFile(secondJPGData),
|
||||||
|
},
|
||||||
|
name: firstJPG,
|
||||||
|
opens: &opens,
|
||||||
|
})
|
||||||
|
|
||||||
|
result, err := store.Import(t.Context(), dest)
|
||||||
|
if !errors.Is(err, errHashMismatch) {
|
||||||
|
t.Fatalf("Import error = %v, want %v", err, errHashMismatch)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(err.Error(), firstJPG) {
|
||||||
|
t.Errorf("Import error = %v, want one naming %s", err, firstJPG)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "Copied", result.Copied, inTestDay(dest, "DSC00012.JPG")...)
|
||||||
|
assertDest(t, dest, map[string]string{testDay + "/DSC00012.JPG": secondJPGData})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImportCopiesTheOtherFilesWhenOneCannotBeRead(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dest := t.TempDir()
|
||||||
|
|
||||||
|
store := mustParseStore(t, failingFS{
|
||||||
|
MapFS: fstest.MapFS{
|
||||||
|
firstJPG: cardFile(firstJPGData),
|
||||||
|
secondJPG: cardFile(secondJPGData),
|
||||||
|
testClip: cardFile(testClipData),
|
||||||
|
},
|
||||||
|
call: "Open",
|
||||||
|
name: firstJPG,
|
||||||
|
})
|
||||||
|
|
||||||
|
result, err := store.Import(t.Context(), dest)
|
||||||
|
if !errors.Is(err, fs.ErrPermission) {
|
||||||
|
t.Fatalf("Import error = %v, want %v", err, fs.ErrPermission)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(err.Error(), firstJPG) {
|
||||||
|
t.Errorf("Import error = %v, want one naming %s", err, firstJPG)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "Copied", result.Copied,
|
||||||
|
inTestDay(dest, "DSC00012.JPG", "C0011.MP4")...)
|
||||||
|
assertDest(t, dest, map[string]string{
|
||||||
|
testDay + "/DSC00012.JPG": secondJPGData,
|
||||||
|
testDay + "/C0011.MP4": testClipData,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImportNamesEveryFileThatFails(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dest := t.TempDir()
|
||||||
|
|
||||||
|
// A plain file where the day folder belongs fails every file of that day.
|
||||||
|
addDestFile(t, dest, testDay, "not a folder")
|
||||||
|
|
||||||
|
store := mustParseStore(t, fstest.MapFS{
|
||||||
|
firstJPG: cardFile(firstJPGData),
|
||||||
|
secondJPG: cardFile(secondJPGData),
|
||||||
|
})
|
||||||
|
|
||||||
|
result, err := store.Import(t.Context(), dest)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Import returned no error")
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, name := range []string{firstJPG, secondJPG} {
|
||||||
|
if !strings.Contains(err.Error(), name) {
|
||||||
|
t.Errorf("Import error = %v, want one naming %s", err, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "Copied", result.Copied)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImportOfObjectBuiltByHandReturnsError(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dest := t.TempDir()
|
||||||
|
|
||||||
|
store := &Store{Images: []*Image{{Object: Object{Path: "DSC00011.JPG"}}}}
|
||||||
|
|
||||||
|
_, err := store.Import(t.Context(), dest)
|
||||||
|
if !errors.Is(err, fs.ErrInvalid) {
|
||||||
|
t.Fatalf("Import error = %v, want %v", err, fs.ErrInvalid)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertEmptyDirectory(t, dest)
|
||||||
|
}
|
||||||
|
|
||||||
|
// cancelingFile is an open file that cancels the import after each read.
|
||||||
|
type cancelingFile struct {
|
||||||
|
fs.File
|
||||||
|
|
||||||
|
cancel context.CancelFunc
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f cancelingFile) Read(p []byte) (int, error) {
|
||||||
|
n, err := f.File.Read(p)
|
||||||
|
|
||||||
|
f.cancel()
|
||||||
|
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// cancelingReadFS is a card that cancels the import during the read of the named
|
||||||
|
// file, the time that file is opened for the openCount-th time. It counts the
|
||||||
|
// reads in opens.
|
||||||
|
type cancelingReadFS struct {
|
||||||
|
fstest.MapFS
|
||||||
|
|
||||||
|
name string
|
||||||
|
openCount int
|
||||||
|
opens *int
|
||||||
|
cancel context.CancelFunc
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f cancelingReadFS) Open(name string) (fs.File, error) {
|
||||||
|
file, err := f.MapFS.Open(name)
|
||||||
|
if err != nil || name != f.name {
|
||||||
|
return file, err
|
||||||
|
}
|
||||||
|
|
||||||
|
*f.opens++
|
||||||
|
if *f.opens != f.openCount {
|
||||||
|
return file, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return cancelingFile{File: file, cancel: f.cancel}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImportStopsInsideAFileWhenCancelledAndRemovesThePartialCopy(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// The first read of a file is hashing it, and the second is copying it.
|
||||||
|
for _, openCount := range []int{1, 2} {
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
dest := t.TempDir()
|
||||||
|
opens := 0
|
||||||
|
|
||||||
|
store := mustParseStore(t, cancelingReadFS{
|
||||||
|
MapFS: fstest.MapFS{
|
||||||
|
// Larger than one read, so the cancellation falls inside the file.
|
||||||
|
firstJPG: {Data: make([]byte, 1<<20)},
|
||||||
|
secondJPG: cardFile(secondJPGData),
|
||||||
|
},
|
||||||
|
name: firstJPG,
|
||||||
|
openCount: openCount,
|
||||||
|
opens: &opens,
|
||||||
|
cancel: cancel,
|
||||||
|
})
|
||||||
|
|
||||||
|
result, err := store.Import(ctx, dest)
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Errorf("cancelled in read %d: error = %v, want %v",
|
||||||
|
openCount, err, context.Canceled)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "Copied", result.Copied)
|
||||||
|
assertDest(t, dest, map[string]string{})
|
||||||
|
|
||||||
|
if opens != openCount {
|
||||||
|
t.Errorf("cancelled in read %d: file opened %d times", openCount, opens)
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImportCopiesNothingWhenAlreadyCancelled(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
dest := t.TempDir()
|
||||||
|
|
||||||
|
store := mustParseStore(t, fstest.MapFS{firstJPG: cardFile(firstJPGData)})
|
||||||
|
|
||||||
|
result, err := store.Import(ctx, dest)
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("Import error = %v, want %v", err, context.Canceled)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "Copied", result.Copied)
|
||||||
|
assertEmptyDirectory(t, dest)
|
||||||
|
}
|
||||||
+53
-18
@@ -1,6 +1,7 @@
|
|||||||
package dcf
|
package dcf
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
@@ -28,13 +29,13 @@ var (
|
|||||||
// point. The root argument is the path of that mount point. Only DCIM and
|
// point. The root argument is the path of that mount point. Only DCIM and
|
||||||
// PRIVATE/M4ROOT are read. A card can lack either one. A symlink is never
|
// PRIVATE/M4ROOT are read. A card can lack either one. A symlink is never
|
||||||
// followed. An error names the mount point, because fsys reports only paths
|
// followed. An error names the mount point, because fsys reports only paths
|
||||||
// inside it.
|
// inside it. When ctx is cancelled, the read stops before the next directory.
|
||||||
func parseStore(fsys fs.FS, root string) (*Store, error) {
|
func parseStore(ctx context.Context, fsys fs.FS, root string) (*Store, error) {
|
||||||
store := &Store{RootDirectory: root}
|
store := &Store{RootDirectory: root}
|
||||||
|
|
||||||
err := parseDCIM(fsys, store)
|
err := parseDCIM(ctx, fsys, store)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
err = parseM4Root(fsys, store)
|
err = parseM4Root(ctx, fsys, store)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -68,16 +69,26 @@ func hasDirectory(fsys fs.FS, name string) (bool, error) {
|
|||||||
return info.IsDir(), nil
|
return info.IsDir(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// readDir returns the entries of the named directory, unless ctx is cancelled.
|
||||||
|
func readDir(ctx context.Context, fsys fs.FS, name string) ([]fs.DirEntry, error) {
|
||||||
|
err := ctx.Err()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return fs.ReadDir(fsys, name)
|
||||||
|
}
|
||||||
|
|
||||||
// parseDCIM adds the DCF directories under DCIM to store. Everything else there
|
// parseDCIM adds the DCF directories under DCIM to store. Everything else there
|
||||||
// is not conforming. That includes directories that share a directory number,
|
// is not conforming. That includes directories that share a directory number,
|
||||||
// which section 7.1.2 of the specification treats as non-DCF directories.
|
// which section 7.1.2 of the specification treats as non-DCF directories.
|
||||||
func parseDCIM(fsys fs.FS, store *Store) error {
|
func parseDCIM(ctx context.Context, fsys fs.FS, store *Store) error {
|
||||||
found, err := hasDirectory(fsys, dcimDirectory)
|
found, err := hasDirectory(fsys, dcimDirectory)
|
||||||
if err != nil || !found {
|
if err != nil || !found {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
entries, err := fs.ReadDir(fsys, dcimDirectory)
|
entries, err := readDir(ctx, fsys, dcimDirectory)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -94,9 +105,9 @@ func parseDCIM(fsys fs.FS, store *Store) error {
|
|||||||
number := parseNumber(dcfDirectoryName, entry.Name())
|
number := parseNumber(dcfDirectoryName, entry.Name())
|
||||||
|
|
||||||
if entry.IsDir() && number != 0 && directoriesWithNumber[number] == 1 {
|
if entry.IsDir() && number != 0 && directoriesWithNumber[number] == 1 {
|
||||||
err = parseDirectory(fsys, store, entry.Name(), number)
|
err = parseDirectory(ctx, fsys, store, entry.Name(), number)
|
||||||
} else {
|
} else {
|
||||||
err = addNonConforming(fsys, store, dcimDirectory, entry)
|
err = addNonConforming(ctx, fsys, store, dcimDirectory, entry)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -112,10 +123,16 @@ func parseDCIM(fsys fs.FS, store *Store) error {
|
|||||||
// Everything else there is not conforming. That includes JPG files that share a
|
// Everything else there is not conforming. That includes JPG files that share a
|
||||||
// file number, which section 7.2.2 of the specification says must not show DCF
|
// file number, which section 7.2.2 of the specification says must not show DCF
|
||||||
// numbers. Other files that share a file number do conform.
|
// numbers. Other files that share a file number do conform.
|
||||||
func parseDirectory(fsys fs.FS, store *Store, name string, number int) error {
|
func parseDirectory(
|
||||||
|
ctx context.Context,
|
||||||
|
fsys fs.FS,
|
||||||
|
store *Store,
|
||||||
|
name string,
|
||||||
|
number int,
|
||||||
|
) error {
|
||||||
directoryPath := path.Join(dcimDirectory, name)
|
directoryPath := path.Join(dcimDirectory, name)
|
||||||
|
|
||||||
entries, err := fs.ReadDir(fsys, directoryPath)
|
entries, err := readDir(ctx, fsys, directoryPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -135,7 +152,7 @@ func parseDirectory(fsys fs.FS, store *Store, name string, number int) error {
|
|||||||
sharedJPG := isJPG(entry.Name()) && jpgsWithNumber[fileNumber] > 1
|
sharedJPG := isJPG(entry.Name()) && jpgsWithNumber[fileNumber] > 1
|
||||||
|
|
||||||
if !entry.Type().IsRegular() || fileNumber == 0 || sharedJPG {
|
if !entry.Type().IsRegular() || fileNumber == 0 || sharedJPG {
|
||||||
err = addNonConforming(fsys, store, directoryPath, entry)
|
err = addNonConforming(ctx, fsys, store, directoryPath, entry)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -145,7 +162,7 @@ func parseDirectory(fsys fs.FS, store *Store, name string, number int) error {
|
|||||||
|
|
||||||
entryPath := path.Join(directoryPath, entry.Name())
|
entryPath := path.Join(directoryPath, entry.Name())
|
||||||
|
|
||||||
object, err := newObject(store, entryPath, entry)
|
object, err := newObject(fsys, store, entryPath, entry)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -167,7 +184,7 @@ func parseDirectory(fsys fs.FS, store *Store, name string, number int) error {
|
|||||||
// naming rules do not apply there, so these files have no DCF numbers. It skips
|
// naming rules do not apply there, so these files have no DCF numbers. It skips
|
||||||
// a symlink. It also skips a name that is not valid UTF-8, because fs.FS refuses
|
// a symlink. It also skips a name that is not valid UTF-8, because fs.FS refuses
|
||||||
// to open one.
|
// to open one.
|
||||||
func parseM4Root(fsys fs.FS, store *Store) error {
|
func parseM4Root(ctx context.Context, fsys fs.FS, store *Store) error {
|
||||||
found, err := hasDirectory(fsys, m4RootDirectory)
|
found, err := hasDirectory(fsys, m4RootDirectory)
|
||||||
if err != nil || !found {
|
if err != nil || !found {
|
||||||
return err
|
return err
|
||||||
@@ -185,11 +202,16 @@ func parseM4Root(fsys fs.FS, store *Store) error {
|
|||||||
return fs.SkipDir
|
return fs.SkipDir
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// fs.WalkDir reads a directory right after this call for it.
|
||||||
|
if entry.IsDir() {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
if !entry.Type().IsRegular() || !validName {
|
if !entry.Type().IsRegular() || !validName {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
object, err := newObject(store, filePath, entry)
|
object, err := newObject(fsys, store, filePath, entry)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -204,7 +226,13 @@ func parseM4Root(fsys fs.FS, store *Store) error {
|
|||||||
// conforming. If the entry is a directory, it lists everything inside it too. It
|
// conforming. If the entry is a directory, it lists everything inside it too. It
|
||||||
// does not follow a symlink. It does not read a directory whose name is not
|
// does not follow a symlink. It does not read a directory whose name is not
|
||||||
// valid UTF-8, because fs.FS refuses to open one.
|
// valid UTF-8, because fs.FS refuses to open one.
|
||||||
func addNonConforming(fsys fs.FS, store *Store, dir string, entry fs.DirEntry) error {
|
func addNonConforming(
|
||||||
|
ctx context.Context,
|
||||||
|
fsys fs.FS,
|
||||||
|
store *Store,
|
||||||
|
dir string,
|
||||||
|
entry fs.DirEntry,
|
||||||
|
) error {
|
||||||
entryPath := path.Join(dir, entry.Name())
|
entryPath := path.Join(dir, entry.Name())
|
||||||
store.NonConforming = append(store.NonConforming, entryPath)
|
store.NonConforming = append(store.NonConforming, entryPath)
|
||||||
|
|
||||||
@@ -212,13 +240,13 @@ func addNonConforming(fsys fs.FS, store *Store, dir string, entry fs.DirEntry) e
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
entries, err := fs.ReadDir(fsys, entryPath)
|
entries, err := readDir(ctx, fsys, entryPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, inner := range entries {
|
for _, inner := range entries {
|
||||||
err = addNonConforming(fsys, store, entryPath, inner)
|
err = addNonConforming(ctx, fsys, store, entryPath, inner)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -228,7 +256,12 @@ func addNonConforming(fsys fs.FS, store *Store, dir string, entry fs.DirEntry) e
|
|||||||
}
|
}
|
||||||
|
|
||||||
// newObject returns the Object for the file at filePath, with no DCF numbers.
|
// newObject returns the Object for the file at filePath, with no DCF numbers.
|
||||||
func newObject(store *Store, filePath string, entry fs.DirEntry) (Object, error) {
|
func newObject(
|
||||||
|
fsys fs.FS,
|
||||||
|
store *Store,
|
||||||
|
filePath string,
|
||||||
|
entry fs.DirEntry,
|
||||||
|
) (Object, error) {
|
||||||
info, err := entry.Info()
|
info, err := entry.Info()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Object{}, err
|
return Object{}, err
|
||||||
@@ -239,6 +272,8 @@ func newObject(store *Store, filePath string, entry fs.DirEntry) (Object, error)
|
|||||||
Path: filePath,
|
Path: filePath,
|
||||||
Size: info.Size(),
|
Size: info.Size(),
|
||||||
Extension: path.Ext(filePath),
|
Extension: path.Ext(filePath),
|
||||||
|
ModTime: info.ModTime(),
|
||||||
|
fsys: fsys,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package dcf
|
package dcf
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
@@ -65,7 +66,7 @@ func addSymlink(t *testing.T, root, name, target string) {
|
|||||||
func mustParseStore(t *testing.T, fsys fs.FS) *Store {
|
func mustParseStore(t *testing.T, fsys fs.FS) *Store {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
store, err := parseStore(fsys, testRoot)
|
store, err := parseStore(t.Context(), fsys, testRoot)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("parseStore: %v", err)
|
t.Fatalf("parseStore: %v", err)
|
||||||
}
|
}
|
||||||
@@ -157,6 +158,7 @@ func TestParseStorePutsEachFileWhereItBelongs(t *testing.T) {
|
|||||||
Extension: ".ARW",
|
Extension: ".ARW",
|
||||||
DirectoryNumber: 100,
|
DirectoryNumber: 100,
|
||||||
FileNumber: 1,
|
FileNumber: 1,
|
||||||
|
fsys: card,
|
||||||
}
|
}
|
||||||
mp4 := Object{
|
mp4 := Object{
|
||||||
DCFStoreRoot: testRoot,
|
DCFStoreRoot: testRoot,
|
||||||
@@ -165,6 +167,7 @@ func TestParseStorePutsEachFileWhereItBelongs(t *testing.T) {
|
|||||||
Extension: ".MP4",
|
Extension: ".MP4",
|
||||||
DirectoryNumber: 101,
|
DirectoryNumber: 101,
|
||||||
FileNumber: 2,
|
FileNumber: 2,
|
||||||
|
fsys: card,
|
||||||
}
|
}
|
||||||
thm := Object{
|
thm := Object{
|
||||||
DCFStoreRoot: testRoot,
|
DCFStoreRoot: testRoot,
|
||||||
@@ -173,12 +176,14 @@ func TestParseStorePutsEachFileWhereItBelongs(t *testing.T) {
|
|||||||
Extension: ".THM",
|
Extension: ".THM",
|
||||||
DirectoryNumber: 101,
|
DirectoryNumber: 101,
|
||||||
FileNumber: 2,
|
FileNumber: 2,
|
||||||
|
fsys: card,
|
||||||
}
|
}
|
||||||
clip := Object{
|
clip := Object{
|
||||||
DCFStoreRoot: testRoot,
|
DCFStoreRoot: testRoot,
|
||||||
Path: "PRIVATE/M4ROOT/CLIP/C0002.MP4",
|
Path: "PRIVATE/M4ROOT/CLIP/C0002.MP4",
|
||||||
Size: 400,
|
Size: 400,
|
||||||
Extension: ".MP4",
|
Extension: ".MP4",
|
||||||
|
fsys: card,
|
||||||
}
|
}
|
||||||
|
|
||||||
wantDirectories := []*Directory{
|
wantDirectories := []*Directory{
|
||||||
@@ -313,7 +318,9 @@ func TestParseNumberOfFileName(t *testing.T) {
|
|||||||
func TestParseStoreTreatsLowerCaseNamesAsDCFNames(t *testing.T) {
|
func TestParseStoreTreatsLowerCaseNamesAsDCFNames(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
store := mustParseStore(t, newCard("DCIM/101abcde/abcd0002.jpg"))
|
card := newCard("DCIM/101abcde/abcd0002.jpg")
|
||||||
|
|
||||||
|
store := mustParseStore(t, card)
|
||||||
|
|
||||||
want := []*Directory{wantDirectory(101, "101abcde", &Object{
|
want := []*Directory{wantDirectory(101, "101abcde", &Object{
|
||||||
DCFStoreRoot: testRoot,
|
DCFStoreRoot: testRoot,
|
||||||
@@ -321,6 +328,7 @@ func TestParseStoreTreatsLowerCaseNamesAsDCFNames(t *testing.T) {
|
|||||||
Extension: ".jpg",
|
Extension: ".jpg",
|
||||||
DirectoryNumber: 101,
|
DirectoryNumber: 101,
|
||||||
FileNumber: 2,
|
FileNumber: 2,
|
||||||
|
fsys: card,
|
||||||
})}
|
})}
|
||||||
if !reflect.DeepEqual(store.Directories, want) {
|
if !reflect.DeepEqual(store.Directories, want) {
|
||||||
t.Errorf("Directories = %+v, want %+v", store.Directories, want)
|
t.Errorf("Directories = %+v, want %+v", store.Directories, want)
|
||||||
@@ -712,7 +720,7 @@ func TestParseStoreTreatsSymlinksAsMissingDirectories(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// failingFS is a card on which one call fails for one path, the way a failing
|
// failingFS is a card on which one call fails for one path, the way a failing
|
||||||
// card does. The call is Lstat, ReadDir or Info.
|
// card does. The call is Open, Lstat, ReadDir or Info.
|
||||||
type failingFS struct {
|
type failingFS struct {
|
||||||
fstest.MapFS
|
fstest.MapFS
|
||||||
|
|
||||||
@@ -720,6 +728,15 @@ type failingFS struct {
|
|||||||
name string
|
name string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f failingFS) Open(name string) (fs.File, error) {
|
||||||
|
err := f.failure("Open", name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return f.MapFS.Open(name)
|
||||||
|
}
|
||||||
|
|
||||||
func (f failingFS) Lstat(name string) (fs.FileInfo, error) {
|
func (f failingFS) Lstat(name string) (fs.FileInfo, error) {
|
||||||
err := f.failure("Lstat", name)
|
err := f.failure("Lstat", name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -797,7 +814,7 @@ func TestParseStoreReturnsReadErrorsNamingTheCard(t *testing.T) {
|
|||||||
for _, name := range names {
|
for _, name := range names {
|
||||||
fsys := failingFS{MapFS: card, call: call, name: name}
|
fsys := failingFS{MapFS: card, call: call, name: name}
|
||||||
|
|
||||||
_, err := parseStore(fsys, testRoot)
|
_, err := parseStore(t.Context(), fsys, testRoot)
|
||||||
if !errors.Is(err, fs.ErrPermission) {
|
if !errors.Is(err, fs.ErrPermission) {
|
||||||
t.Errorf("%s of %s: error = %v, want %v",
|
t.Errorf("%s of %s: error = %v, want %v",
|
||||||
call, name, err, fs.ErrPermission)
|
call, name, err, fs.ErrPermission)
|
||||||
@@ -813,3 +830,86 @@ func TestParseStoreReturnsReadErrorsNamingTheCard(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// cancelingFS is a card that cancels the read when the read reaches the named
|
||||||
|
// directory. It records every directory read in read.
|
||||||
|
type cancelingFS struct {
|
||||||
|
fstest.MapFS
|
||||||
|
|
||||||
|
name string
|
||||||
|
cancel context.CancelFunc
|
||||||
|
read *[]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f cancelingFS) ReadDir(name string) ([]fs.DirEntry, error) {
|
||||||
|
*f.read = append(*f.read, name)
|
||||||
|
|
||||||
|
if name == f.name {
|
||||||
|
f.cancel()
|
||||||
|
}
|
||||||
|
|
||||||
|
return f.MapFS.ReadDir(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreStopsBeforeTheNextDirectoryWhenCancelled(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
card := newCard(
|
||||||
|
"DCIM/100ABCDE/SUB/README.TXT",
|
||||||
|
"DCIM/OTHER/SUB/README.TXT",
|
||||||
|
"PRIVATE/M4ROOT/CLIP/SUB/C0001.MP4",
|
||||||
|
)
|
||||||
|
|
||||||
|
// Each of these directories holds another directory, which must not be read.
|
||||||
|
names := []string{
|
||||||
|
dcimDirectory,
|
||||||
|
"DCIM/100ABCDE",
|
||||||
|
"DCIM/OTHER",
|
||||||
|
m4RootDirectory,
|
||||||
|
"PRIVATE/M4ROOT/CLIP",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, name := range names {
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
|
||||||
|
var read []string
|
||||||
|
|
||||||
|
fsys := cancelingFS{MapFS: card, name: name, cancel: cancel, read: &read}
|
||||||
|
|
||||||
|
_, err := parseStore(ctx, fsys, testRoot)
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Errorf("cancelled at %s: error = %v, want %v",
|
||||||
|
name, err, context.Canceled)
|
||||||
|
}
|
||||||
|
|
||||||
|
if read[len(read)-1] != name {
|
||||||
|
t.Errorf("cancelled at %s: read %q", name, read)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreReadsNoDirectoryWhenAlreadyCancelled(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
cards := []fstest.MapFS{
|
||||||
|
newCard("DCIM/README.TXT"),
|
||||||
|
newCard("PRIVATE/M4ROOT/C0001.MP4"),
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, card := range cards {
|
||||||
|
var read []string
|
||||||
|
|
||||||
|
// The context is already cancelled, so this needs no name to cancel at.
|
||||||
|
fsys := cancelingFS{MapFS: card, read: &read}
|
||||||
|
|
||||||
|
_, err := parseStore(ctx, fsys, testRoot)
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Errorf("error = %v, want %v", err, context.Canceled)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "directories read", read)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user