diff --git a/README.md b/README.md index 3f658a5..af872c9 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ## Description `dcf` is a WTFPL-licensed Go library by [@sneak](https://sneak.berlin) that -reads DCF memory cards from digital cameras. +reads DCF memory cards from digital cameras and copies the files off them. DCF is the "Design rule for Camera File system". It is also JEITA specification number CP-3461. JEITA is the Japan Electronics and Information Technology @@ -52,6 +52,38 @@ Call `dcf.GetStore` instead when you expect exactly one DCF store. It returns `dcf.ErrNoStore` when none is mounted, and `dcf.ErrMultipleStores` when more than one is. +To copy the files off a card, call `Import` with a destination directory: + +```go +package main + +import ( + "context" + "fmt" + "log" + + "sneak.berlin/go/dcf/pkg/dcf" +) + +func main() { + ctx := context.Background() + + store, err := dcf.GetStore(ctx) + if err != nil { + log.Fatal(err) + } + + result, err := store.Import(ctx, "/path/to/photos") + if err != nil { + log.Fatal(err) + } + + fmt.Println("copied:", result.Copied) + fmt.Println("skipped:", result.Skipped) + fmt.Println("conflicts:", result.Conflicts) +} +``` + To work on the library itself, you need docker and git: ```sh @@ -110,11 +142,13 @@ overengineering and reusable code. All the code is in the `pkg/dcf` package. The repo has no commands. -- `dcf.go` holds the public API. `GetStores` finds the DCF stores and reads each - one. `GetStore` returns the one DCF store when exactly one is mounted. +- `dcf.go` holds the public API for reading a card. `GetStores` finds the DCF + stores and reads each one. `GetStore` returns the one DCF store when exactly + one is mounted. - `helpers.go` finds the mount points that hold a DCF store. - `parse.go` reads one DCF store through an `fs.FS` rooted at its mount point. - `filegroup.go` sorts the files of one DCF directory into file groups. +- `import.go` holds `Store.Import`, which copies files off a card. The main types are: @@ -122,13 +156,14 @@ The main types are: - `Directory` is one DCF directory. It has a directory number, a name, the objects inside it and their file groups. - `Object` is one file in a store. A file in a DCF directory also has a - directory number and a file number. `Object.Hash` reads the file through the - `fs.FS` of its store. An `Object` you build yourself has none, so its `Hash` - returns an error. + directory number and a file number. `Object.ModTime` is its modification time. + `Object.Hash` reads the file through the `fs.FS` of its store. An `Object` you + build yourself has none, so its `Hash` returns an error. - `FileGroup` is one file group in a DCF directory. - `Image` and `Video` embed `Object`. The file extension decides which one a file becomes. A file with any other extension, such as a `THM` thumbnail, is neither. +- `ImportResult` lists what `Store.Import` did with each file. A mounted filesystem is a DCF store when its root holds a `DCIM` or `PRIVATE/M4ROOT` directory. It needs only one of them. `dcf` never mounts a @@ -191,9 +226,49 @@ The primary member is the first of these that the group holds: `Store.Images` and `Store.Videos` list each file on its own, whatever its group. +`Store.Import` copies files off a card to a destination directory. It verifies +each copy against the SHA256 hash of its source. It only reads the card, so it +never deletes anything there. + +It copies: + +- every member of every file group +- the images and videos under `PRIVATE/M4ROOT` + +It does not copy the files in `Store.NonConforming`. + +Each file goes to `YYYY-MM-DD/` under the destination directory. The +date is the day of the file's modification time, in the computer's time zone. +Every member of a file group goes to the day of its primary member, so a group +never splits across two folders. Each copy keeps the modification time of its +source. Only the user who ran the import can read the copies. + +`ImportResult` lists each destination path in one of three lists: + +- `Copied` holds the copies this import wrote and verified. +- `Skipped` holds the paths that already held a file with the same SHA256 hash. +- `Conflicts` holds the paths that already held anything else. `dcf` leaves that + alone and does not copy the file. + +A conflict is not an error, so a caller must check `Conflicts`. These are the +usual causes: + +- Two different files have the same name and the same day. Two cards can reuse a + file number, and so can two DCF directories on one card. The first file is + copied, and the second is a conflict. +- A partial file remains because the process died during a copy. Delete it and + import again. + +A copy that fails or does not match the hash of its source is removed. One file +that fails does not stop the others. The returned error joins the failures, each +naming its file. The result still lists what was done. + +When the context is cancelled, the import stops at once, even inside a file, and +removes the partial copy. The returned error matches `context.Canceled`. + ## TODO -- Copy files off a card, and verify each copy against its SHA256 hash. +No open items. ## License diff --git a/pkg/dcf/dcf.go b/pkg/dcf/dcf.go index 6df82f3..1f8bb59 100644 --- a/pkg/dcf/dcf.go +++ b/pkg/dcf/dcf.go @@ -3,21 +3,20 @@ // the mounted filesystems and parses the DCF directories and files on each one // into their directory numbers and file numbers. It lists the images and videos // on each card, and everything under DCIM that is not a DCF directory or file. It -// groups the files in each DCF directory that share a file number. +// groups the files in each DCF directory that share a file number. It copies the +// files off a card, and verifies each copy against its SHA256 hash. package dcf import ( "context" - "crypto/sha256" - "encoding/hex" "errors" "fmt" - "io" "io/fs" "os" "path/filepath" "slices" "strings" + "time" "github.com/dustin/go-humanize" ) @@ -83,6 +82,8 @@ type Object struct { Path string Size int64 Extension string + // ModTime is the modification time of the file. + ModTime time.Time // DirectoryNumber and FileNumber are the DCF numbers of a file in a DCF // directory. Both are 0 for a file under PRIVATE/M4ROOT. DirectoryNumber int @@ -191,25 +192,22 @@ func (i *Image) String() string { // through the fs.FS of its store. An Object built by hand has none, so Hash // returns an error that matches fs.ErrInvalid. func (d *Object) Hash() (string, error) { + hash, err := d.hash(context.Background()) + if err != nil { + return "", fmt.Errorf("hashing %s: %w", d.FullFilePath(), err) + } + + return hash, nil +} + +// hash returns the SHA256 hash of the file as a hex string, and stops when ctx is +// cancelled. +func (d *Object) hash(ctx context.Context) (string, error) { if d.fsys == nil { - return "", fmt.Errorf("hashing %s: %w", d.FullFilePath(), fs.ErrInvalid) + return "", fs.ErrInvalid } - file, err := d.fsys.Open(d.Path) - if err != nil { - return "", fmt.Errorf("hashing %s: %w", d.FullFilePath(), err) - } - - defer func() { _ = file.Close() }() - - hash := sha256.New() - - _, err = io.Copy(hash, file) - if err != nil { - return "", fmt.Errorf("hashing %s: %w", d.FullFilePath(), err) - } - - return hex.EncodeToString(hash.Sum(nil)), nil + return hashFile(ctx, d.fsys, d.Path) } // GetStores finds the DCF stores in the root of the mounted filesystems and reads diff --git a/pkg/dcf/import.go b/pkg/dcf/import.go new file mode 100644 index 0000000..63c98b6 --- /dev/null +++ b/pkg/dcf/import.go @@ -0,0 +1,279 @@ +package dcf + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "time" +) + +// The modes of what Import creates. Only the user who ran it can read the copies. +const ( + importDirectoryMode = 0o750 + importFileMode = 0o600 +) + +var errHashMismatch = errors.New("the SHA256 hash of the copy differs from the source") + +// ImportResult lists what Store.Import did with each file, by destination path. +type ImportResult struct { + // Copied holds the copies this import wrote and verified. + Copied []string + // Skipped holds the paths that already held a file with the same SHA256 hash. + Skipped []string + // Conflicts holds the paths that already held anything else. Import left it + // alone and did not copy the file. + Conflicts []string +} + +// importFile is one file for Import to copy, and the path to copy it to. +type importFile struct { + source *Object + target string +} + +// Import copies the files of the store to dest, and verifies each copy against +// the SHA256 hash of its source. It copies every member of every file group, and +// the images and videos under PRIVATE/M4ROOT. It only reads the card. +// +// Each file goes to dest/YYYY-MM-DD/, from its modification time. Every +// member of a file group goes to the day of its primary member, so a group never +// splits across two folders. +// +// A file with the same hash already at the path is skipped. Anything else there is +// a conflict, which is not an error. Import never overwrites anything. +// +// One file that fails does not stop the others. The returned error joins the +// failures, each naming its file, and the result still lists what was done. When +// ctx is cancelled, the import stops at once and removes the partial copy, and the +// error matches ctx.Err(). +func (d *Store) Import(ctx context.Context, dest string) (ImportResult, error) { + var ( + result ImportResult + failures []error + ) + + for _, file := range d.importFiles(dest) { + err := ctx.Err() + if err != nil { + failures = append(failures, fmt.Errorf("import stopped: %w", err)) + + break + } + + err = copyFile(ctx, file, &result) + if err != nil { + failures = append(failures, + fmt.Errorf("importing %s: %w", file.source.FullFilePath(), err)) + } + } + + return result, errors.Join(failures...) +} + +// importFiles lists the files of the store that Import copies to dest. +func (d *Store) importFiles(dest string) []importFile { + var files []importFile + + add := func(source *Object, day time.Time) { + name := path.Base(source.Path) + + files = append(files, importFile{ + source: source, + target: filepath.Join(dest, day.Format(time.DateOnly), name), + }) + } + + for _, directory := range d.Directories { + for _, group := range directory.FileGroups { + for _, member := range group.Members { + add(member, group.Primary.ModTime) + } + } + } + + // The files under PRIVATE/M4ROOT are the ones with no DCF numbers. + for _, image := range d.Images { + if image.DirectoryNumber == 0 { + add(&image.Object, image.ModTime) + } + } + + for _, video := range d.Videos { + if video.DirectoryNumber == 0 { + add(&video.Object, video.ModTime) + } + } + + return files +} + +// copyFile copies one file, and adds its target to the list in result that says +// what happened. It adds the target to no list when it returns an error. +func copyFile(ctx context.Context, file importFile, result *ImportResult) error { + sourceHash, err := file.source.hash(ctx) + if err != nil { + return err + } + + err = os.MkdirAll(filepath.Dir(file.target), importDirectoryMode) + if err != nil { + return err + } + + // O_EXCL fails on anything already at the path, even a symlink, so nothing + // there can be overwritten. + target, err := os.OpenFile( + file.target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, importFileMode, + ) + if errors.Is(err, fs.ErrExist) { + return checkExisting(ctx, file.target, sourceHash, result) + } + + if err != nil { + return err + } + + err = writeCopy(ctx, file, target, sourceHash) + if err != nil { + return errors.Join(err, os.Remove(file.target)) + } + + result.Copied = append(result.Copied, file.target) + + return nil +} + +// checkExisting adds target to the skipped files if it is a file with the given +// hash, and to the conflicts if it is anything else. +func checkExisting( + ctx context.Context, + target, sourceHash string, + result *ImportResult, +) error { + info, err := os.Lstat(target) + if err != nil { + return err + } + + if !info.Mode().IsRegular() { + result.Conflicts = append(result.Conflicts, target) + + return nil + } + + targetHash, err := hashOnDisk(ctx, target) + if err != nil { + return err + } + + if targetHash == sourceHash { + result.Skipped = append(result.Skipped, target) + } else { + result.Conflicts = append(result.Conflicts, target) + } + + return nil +} + +// writeCopy fills the new file target from the source, and closes it. It then +// reads the written file back to compare its hash with the hash of the source. +func writeCopy( + ctx context.Context, + file importFile, + target *os.File, + sourceHash string, +) error { + err := writeAndClose(ctx, file.source, target) + if err != nil { + return err + } + + targetHash, err := hashOnDisk(ctx, file.target) + if err != nil { + return err + } + + if targetHash != sourceHash { + return errHashMismatch + } + + // The zero time leaves the access time as it is. + return os.Chtimes(file.target, time.Time{}, file.source.ModTime) +} + +// writeAndClose writes the bytes of source to target, syncs target and closes it. +func writeAndClose(ctx context.Context, source *Object, target *os.File) error { + defer func() { _ = target.Close() }() + + sourceFile, err := source.fsys.Open(source.Path) + if err != nil { + return err + } + + defer func() { _ = sourceFile.Close() }() + + _, err = io.Copy(target, contextReader(ctx, sourceFile)) + if err != nil { + return err + } + + err = target.Sync() + if err != nil { + return err + } + + return target.Close() +} + +// hashOnDisk returns the SHA256 hash of the file at filePath on the local +// filesystem. +func hashOnDisk(ctx context.Context, filePath string) (string, error) { + return hashFile(ctx, os.DirFS(filepath.Dir(filePath)), filepath.Base(filePath)) +} + +// hashFile returns the SHA256 hash of the named file as a hex string. +func hashFile(ctx context.Context, fsys fs.FS, name string) (string, error) { + file, err := fsys.Open(name) + if err != nil { + return "", err + } + + defer func() { _ = file.Close() }() + + hash := sha256.New() + + _, err = io.Copy(hash, contextReader(ctx, file)) + if err != nil { + return "", err + } + + return hex.EncodeToString(hash.Sum(nil)), nil +} + +// readerFunc is a function that works as an io.Reader. +type readerFunc func([]byte) (int, error) + +func (f readerFunc) Read(p []byte) (int, error) { + return f(p) +} + +// contextReader returns a reader that fails with ctx.Err() once ctx is cancelled, +// so a copy stops inside a file. +func contextReader(ctx context.Context, reader io.Reader) io.Reader { + return readerFunc(func(p []byte) (int, error) { + err := ctx.Err() + if err != nil { + return 0, err + } + + return reader.Read(p) + }) +} diff --git a/pkg/dcf/import_internal_test.go b/pkg/dcf/import_internal_test.go new file mode 100644 index 0000000..55ed6d3 --- /dev/null +++ b/pkg/dcf/import_internal_test.go @@ -0,0 +1,521 @@ +package dcf + +import ( + "context" + "errors" + "io/fs" + "maps" + "os" + "path/filepath" + "strings" + "testing" + "testing/fstest" + "time" +) + +// testDay is the folder that a file from cardFile goes to. The other constants +// are the paths and contents of the files on most cards in these tests. +const ( + testDay = "2026-10-03" + + firstJPG = "DCIM/100ABCDE/DSC00011.JPG" + firstJPGData = "first jpeg" + secondJPG = "DCIM/100ABCDE/DSC00012.JPG" + secondJPGData = "second jpeg" + testClip = "PRIVATE/M4ROOT/CLIP/C0011.MP4" + testClipData = "clip" +) + +// cardFile returns a file on a card, modified at noon on testDay. +func cardFile(contents string) *fstest.MapFile { + return &fstest.MapFile{ + Data: []byte(contents), + ModTime: time.Date(2026, 10, 3, 12, 0, 0, 0, time.UTC), + } +} + +// inTestDay returns the paths of the named files in the testDay folder of dest. +func inTestDay(dest string, names ...string) []string { + paths := make([]string, 0, len(names)) + + for _, name := range names { + paths = append(paths, filepath.Join(dest, testDay, name)) + } + + return paths +} + +// assertDest checks that dest holds exactly the given files, as a map from each +// file's path under dest to its contents. +func assertDest(t *testing.T, dest string, want map[string]string) { + t.Helper() + + fsys := os.DirFS(dest) + got := map[string]string{} + + err := fs.WalkDir(fsys, ".", + func(name string, entry fs.DirEntry, err error) error { + if err != nil || !entry.Type().IsRegular() { + return err + } + + data, err := fs.ReadFile(fsys, name) + got[name] = string(data) + + return err + }) + if err != nil { + t.Fatalf("reading %s: %v", dest, err) + } + + if !maps.Equal(got, want) { + t.Errorf("destination holds %q, want %q", got, want) + } +} + +func assertEmptyDirectory(t *testing.T, directory string) { + t.Helper() + + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatalf("ReadDir: %v", err) + } + + if len(entries) != 0 { + t.Errorf("%s holds %v, want nothing", directory, entries) + } +} + +// addDestFile writes a file at the named path under dest. +func addDestFile(t *testing.T, dest, name, contents string) { + t.Helper() + + fullPath := filepath.Join(dest, name) + + err := os.MkdirAll(filepath.Dir(fullPath), 0o750) + if err != nil { + t.Fatalf("MkdirAll: %v", err) + } + + err = os.WriteFile(fullPath, []byte(contents), 0o600) + if err != nil { + t.Fatalf("WriteFile: %v", err) + } +} + +func TestImportCopiesEveryFileAndKeepsItsModificationTime(t *testing.T) { + t.Parallel() + + dest := t.TempDir() + + store := mustParseStore(t, fstest.MapFS{ + "DCIM/100ABCDE/DSC00011.ARW": cardFile("raw"), + firstJPG: cardFile(firstJPGData), + "DCIM/101ABCDE/MVI_0012.MP4": cardFile("video"), + "DCIM/101ABCDE/MVI_0012.THM": cardFile("thumbnail"), + "DCIM/MISC/DSC00013.JPG": cardFile("not conforming"), + testClip: cardFile(testClipData), + "PRIVATE/M4ROOT/CLIP/C0011M01.XML": cardFile("neither image nor video"), + "PRIVATE/M4ROOT/THMBNL/C0011T01.JPG": cardFile("clip thumbnail"), + }) + + result, err := store.Import(t.Context(), dest) + if err != nil { + t.Fatalf("Import: %v", err) + } + + assertPaths(t, "Copied", result.Copied, inTestDay(dest, + "DSC00011.ARW", + "DSC00011.JPG", + "MVI_0012.MP4", + "MVI_0012.THM", + "C0011T01.JPG", + "C0011.MP4", + )...) + assertPaths(t, "Skipped", result.Skipped) + assertPaths(t, "Conflicts", result.Conflicts) + + assertDest(t, dest, map[string]string{ + testDay + "/DSC00011.ARW": "raw", + testDay + "/DSC00011.JPG": firstJPGData, + testDay + "/MVI_0012.MP4": "video", + testDay + "/MVI_0012.THM": "thumbnail", + testDay + "/C0011.MP4": testClipData, + testDay + "/C0011T01.JPG": "clip thumbnail", + }) + + for _, copied := range result.Copied { + info, err := os.Stat(copied) + if err != nil { + t.Fatalf("Stat: %v", err) + } + + if want := cardFile("").ModTime; !info.ModTime().Equal(want) { + t.Errorf("%s modified at %v, want %v", copied, info.ModTime(), want) + } + } +} + +func TestImportTwiceSkipsEveryFileTheSecondTime(t *testing.T) { + t.Parallel() + + dest := t.TempDir() + + store := mustParseStore(t, fstest.MapFS{ + firstJPG: cardFile(firstJPGData), + testClip: cardFile(testClipData), + }) + + _, err := store.Import(t.Context(), dest) + if err != nil { + t.Fatalf("first Import: %v", err) + } + + result, err := store.Import(t.Context(), dest) + if err != nil { + t.Fatalf("second Import: %v", err) + } + + assertPaths(t, "Copied", result.Copied) + assertPaths(t, "Skipped", result.Skipped, + inTestDay(dest, "DSC00011.JPG", "C0011.MP4")...) + assertPaths(t, "Conflicts", result.Conflicts) + + assertDest(t, dest, map[string]string{ + testDay + "/DSC00011.JPG": firstJPGData, + testDay + "/C0011.MP4": testClipData, + }) +} + +func TestImportLeavesADifferentFileAloneAndReportsAConflict(t *testing.T) { + t.Parallel() + + dest := t.TempDir() + addDestFile(t, dest, testDay+"/DSC00011.JPG", "different") + + store := mustParseStore(t, fstest.MapFS{ + firstJPG: cardFile(firstJPGData), + secondJPG: cardFile(secondJPGData), + }) + + result, err := store.Import(t.Context(), dest) + if err != nil { + t.Fatalf("Import: %v", err) + } + + assertPaths(t, "Copied", result.Copied, inTestDay(dest, "DSC00012.JPG")...) + assertPaths(t, "Skipped", result.Skipped) + assertPaths(t, "Conflicts", result.Conflicts, inTestDay(dest, "DSC00011.JPG")...) + + assertDest(t, dest, map[string]string{ + testDay + "/DSC00011.JPG": "different", + testDay + "/DSC00012.JPG": secondJPGData, + }) +} + +func TestImportReportsAConflictBetweenTwoFilesWithOneName(t *testing.T) { + t.Parallel() + + dest := t.TempDir() + + store := mustParseStore(t, fstest.MapFS{ + firstJPG: cardFile(firstJPGData), + "DCIM/101ABCDE/DSC00011.JPG": cardFile(secondJPGData), + }) + + result, err := store.Import(t.Context(), dest) + if err != nil { + t.Fatalf("Import: %v", err) + } + + assertPaths(t, "Copied", result.Copied, inTestDay(dest, "DSC00011.JPG")...) + assertPaths(t, "Conflicts", result.Conflicts, inTestDay(dest, "DSC00011.JPG")...) + assertDest(t, dest, map[string]string{testDay + "/DSC00011.JPG": firstJPGData}) +} + +func TestImportReportsAConflictAtADirectoryOrASymlink(t *testing.T) { + t.Parallel() + + dest := t.TempDir() + + // The symlink points at a file with the same bytes, which must not count. + outside := newCardOnDisk(t, "OUTSIDE.JPG") + + err := os.MkdirAll(filepath.Join(dest, testDay, "DSC00011.JPG"), 0o750) + if err != nil { + t.Fatalf("MkdirAll: %v", err) + } + + addSymlink(t, dest, testDay+"/DSC00012.JPG", filepath.Join(outside, "OUTSIDE.JPG")) + + store := mustParseStore(t, fstest.MapFS{ + firstJPG: cardFile("abc"), + secondJPG: cardFile("abc"), + }) + + result, err := store.Import(t.Context(), dest) + if err != nil { + t.Fatalf("Import: %v", err) + } + + assertPaths(t, "Copied", result.Copied) + assertPaths(t, "Skipped", result.Skipped) + assertPaths(t, "Conflicts", result.Conflicts, + inTestDay(dest, "DSC00011.JPG", "DSC00012.JPG")...) +} + +func TestImportPutsAFileGroupInTheFolderOfItsPrimaryMember(t *testing.T) { + t.Parallel() + + dest := t.TempDir() + + beforeMidnight := time.Date(2026, 10, 3, 23, 59, 59, 0, time.UTC) + afterMidnight := time.Date(2026, 10, 4, 0, 0, 1, 0, time.UTC) + + store := mustParseStore(t, fstest.MapFS{ + "DCIM/100ABCDE/DSC00011.ARW": {Data: []byte("raw"), ModTime: afterMidnight}, + firstJPG: {Data: []byte("jpeg"), ModTime: beforeMidnight}, + secondJPG: {Data: []byte("next"), ModTime: afterMidnight}, + }) + + _, err := store.Import(t.Context(), dest) + if err != nil { + t.Fatalf("Import: %v", err) + } + + assertDest(t, dest, map[string]string{ + "2026-10-03/DSC00011.ARW": "raw", + "2026-10-03/DSC00011.JPG": "jpeg", + "2026-10-04/DSC00012.JPG": "next", + }) + + // The copy keeps its own modification time, whichever folder it is in. + info, err := os.Stat(filepath.Join(dest, "2026-10-03/DSC00011.ARW")) + if err != nil { + t.Fatalf("Stat: %v", err) + } + + if !info.ModTime().Equal(afterMidnight) { + t.Errorf("modified at %v, want %v", info.ModTime(), afterMidnight) + } +} + +// changingFS is a card whose file at name holds other bytes from its second read +// on. It counts the reads in opens. +type changingFS struct { + fstest.MapFS + + name string + opens *int +} + +func (f changingFS) Open(name string) (fs.File, error) { + if name == f.name { + *f.opens++ + if *f.opens > 1 { + return fstest.MapFS{name: cardFile("other bytes")}.Open(name) + } + } + + return f.MapFS.Open(name) +} + +func TestImportRemovesACopyThatDoesNotMatchItsSource(t *testing.T) { + t.Parallel() + + dest := t.TempDir() + opens := 0 + + store := mustParseStore(t, changingFS{ + MapFS: fstest.MapFS{ + firstJPG: cardFile(firstJPGData), + secondJPG: cardFile(secondJPGData), + }, + name: firstJPG, + opens: &opens, + }) + + result, err := store.Import(t.Context(), dest) + if !errors.Is(err, errHashMismatch) { + t.Fatalf("Import error = %v, want %v", err, errHashMismatch) + } + + if !strings.Contains(err.Error(), firstJPG) { + t.Errorf("Import error = %v, want one naming %s", err, firstJPG) + } + + assertPaths(t, "Copied", result.Copied, inTestDay(dest, "DSC00012.JPG")...) + assertDest(t, dest, map[string]string{testDay + "/DSC00012.JPG": secondJPGData}) +} + +func TestImportCopiesTheOtherFilesWhenOneCannotBeRead(t *testing.T) { + t.Parallel() + + dest := t.TempDir() + + store := mustParseStore(t, failingFS{ + MapFS: fstest.MapFS{ + firstJPG: cardFile(firstJPGData), + secondJPG: cardFile(secondJPGData), + testClip: cardFile(testClipData), + }, + call: "Open", + name: firstJPG, + }) + + result, err := store.Import(t.Context(), dest) + if !errors.Is(err, fs.ErrPermission) { + t.Fatalf("Import error = %v, want %v", err, fs.ErrPermission) + } + + if !strings.Contains(err.Error(), firstJPG) { + t.Errorf("Import error = %v, want one naming %s", err, firstJPG) + } + + assertPaths(t, "Copied", result.Copied, + inTestDay(dest, "DSC00012.JPG", "C0011.MP4")...) + assertDest(t, dest, map[string]string{ + testDay + "/DSC00012.JPG": secondJPGData, + testDay + "/C0011.MP4": testClipData, + }) +} + +func TestImportNamesEveryFileThatFails(t *testing.T) { + t.Parallel() + + dest := t.TempDir() + + // A plain file where the day folder belongs fails every file of that day. + addDestFile(t, dest, testDay, "not a folder") + + store := mustParseStore(t, fstest.MapFS{ + firstJPG: cardFile(firstJPGData), + secondJPG: cardFile(secondJPGData), + }) + + result, err := store.Import(t.Context(), dest) + if err == nil { + t.Fatal("Import returned no error") + } + + for _, name := range []string{firstJPG, secondJPG} { + if !strings.Contains(err.Error(), name) { + t.Errorf("Import error = %v, want one naming %s", err, name) + } + } + + assertPaths(t, "Copied", result.Copied) +} + +func TestImportOfObjectBuiltByHandReturnsError(t *testing.T) { + t.Parallel() + + dest := t.TempDir() + + store := &Store{Images: []*Image{{Object: Object{Path: "DSC00011.JPG"}}}} + + _, err := store.Import(t.Context(), dest) + if !errors.Is(err, fs.ErrInvalid) { + t.Fatalf("Import error = %v, want %v", err, fs.ErrInvalid) + } + + assertEmptyDirectory(t, dest) +} + +// cancelingFile is an open file that cancels the import after each read. +type cancelingFile struct { + fs.File + + cancel context.CancelFunc +} + +func (f cancelingFile) Read(p []byte) (int, error) { + n, err := f.File.Read(p) + + f.cancel() + + return n, err +} + +// cancelingReadFS is a card that cancels the import during the read of the named +// file, the time that file is opened for the openCount-th time. It counts the +// reads in opens. +type cancelingReadFS struct { + fstest.MapFS + + name string + openCount int + opens *int + cancel context.CancelFunc +} + +func (f cancelingReadFS) Open(name string) (fs.File, error) { + file, err := f.MapFS.Open(name) + if err != nil || name != f.name { + return file, err + } + + *f.opens++ + if *f.opens != f.openCount { + return file, nil + } + + return cancelingFile{File: file, cancel: f.cancel}, nil +} + +func TestImportStopsInsideAFileWhenCancelledAndRemovesThePartialCopy(t *testing.T) { + t.Parallel() + + // The first read of a file is hashing it, and the second is copying it. + for _, openCount := range []int{1, 2} { + ctx, cancel := context.WithCancel(t.Context()) + dest := t.TempDir() + opens := 0 + + store := mustParseStore(t, cancelingReadFS{ + MapFS: fstest.MapFS{ + // Larger than one read, so the cancellation falls inside the file. + firstJPG: {Data: make([]byte, 1<<20)}, + secondJPG: cardFile(secondJPGData), + }, + name: firstJPG, + openCount: openCount, + opens: &opens, + cancel: cancel, + }) + + result, err := store.Import(ctx, dest) + if !errors.Is(err, context.Canceled) { + t.Errorf("cancelled in read %d: error = %v, want %v", + openCount, err, context.Canceled) + } + + assertPaths(t, "Copied", result.Copied) + assertDest(t, dest, map[string]string{}) + + if opens != openCount { + t.Errorf("cancelled in read %d: file opened %d times", openCount, opens) + } + + cancel() + } +} + +func TestImportCopiesNothingWhenAlreadyCancelled(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + dest := t.TempDir() + + store := mustParseStore(t, fstest.MapFS{firstJPG: cardFile(firstJPGData)}) + + result, err := store.Import(ctx, dest) + if !errors.Is(err, context.Canceled) { + t.Fatalf("Import error = %v, want %v", err, context.Canceled) + } + + assertPaths(t, "Copied", result.Copied) + assertEmptyDirectory(t, dest) +} diff --git a/pkg/dcf/parse.go b/pkg/dcf/parse.go index c818b2d..992242a 100644 --- a/pkg/dcf/parse.go +++ b/pkg/dcf/parse.go @@ -272,6 +272,7 @@ func newObject( Path: filePath, Size: info.Size(), Extension: path.Ext(filePath), + ModTime: info.ModTime(), fsys: fsys, }, nil } diff --git a/pkg/dcf/parse_internal_test.go b/pkg/dcf/parse_internal_test.go index ce5a4af..925c7ee 100644 --- a/pkg/dcf/parse_internal_test.go +++ b/pkg/dcf/parse_internal_test.go @@ -720,7 +720,7 @@ func TestParseStoreTreatsSymlinksAsMissingDirectories(t *testing.T) { } // failingFS is a card on which one call fails for one path, the way a failing -// card does. The call is Lstat, ReadDir or Info. +// card does. The call is Open, Lstat, ReadDir or Info. type failingFS struct { fstest.MapFS @@ -728,6 +728,15 @@ type failingFS struct { name string } +func (f failingFS) Open(name string) (fs.File, error) { + err := f.failure("Open", name) + if err != nil { + return nil, err + } + + return f.MapFS.Open(name) +} + func (f failingFS) Lstat(name string) (fs.FileInfo, error) { err := f.failure("Lstat", name) if err != nil {