copy files off a card, verified by SHA256 (closes #5)
Store.Import copies every member of every file group and the images and videos under PRIVATE/M4ROOT to dest/YYYY-MM-DD/<file name>. The day comes from the new Object.ModTime. A file group stays in the folder of its primary member. Each copy is read back and compared by SHA256. A file with the same hash already there is skipped, and anything else there is a conflict and left alone. One failed file does not stop the others. A cancelled context stops at once and removes the partial copy. Object.Hash now shares its reading code with the import. Model: opus-5-5
This commit is contained in:
1 parent
acb108a82e
commit
d2728b1cea
6 files changed
+911
-28
No files matched your search
+18
-20
@@ -3,21 +3,20 @@
|
||||
// the mounted filesystems and parses the DCF directories and files on each one
|
||||
// into their directory numbers and file numbers. It lists the images and videos
|
||||
// on each card, and everything under DCIM that is not a DCF directory or file. It
|
||||
// groups the files in each DCF directory that share a file number.
|
||||
// groups the files in each DCF directory that share a file number. It copies the
|
||||
// files off a card, and verifies each copy against its SHA256 hash.
|
||||
package dcf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
)
|
||||
@@ -83,6 +82,8 @@ type Object struct {
|
||||
Path string
|
||||
Size int64
|
||||
Extension string
|
||||
// ModTime is the modification time of the file.
|
||||
ModTime time.Time
|
||||
// DirectoryNumber and FileNumber are the DCF numbers of a file in a DCF
|
||||
// directory. Both are 0 for a file under PRIVATE/M4ROOT.
|
||||
DirectoryNumber int
|
||||
@@ -191,25 +192,22 @@ func (i *Image) String() string {
|
||||
// through the fs.FS of its store. An Object built by hand has none, so Hash
|
||||
// returns an error that matches fs.ErrInvalid.
|
||||
func (d *Object) Hash() (string, error) {
|
||||
hash, err := d.hash(context.Background())
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("hashing %s: %w", d.FullFilePath(), err)
|
||||
}
|
||||
|
||||
return hash, nil
|
||||
}
|
||||
|
||||
// hash returns the SHA256 hash of the file as a hex string, and stops when ctx is
|
||||
// cancelled.
|
||||
func (d *Object) hash(ctx context.Context) (string, error) {
|
||||
if d.fsys == nil {
|
||||
return "", fmt.Errorf("hashing %s: %w", d.FullFilePath(), fs.ErrInvalid)
|
||||
return "", fs.ErrInvalid
|
||||
}
|
||||
|
||||
file, err := d.fsys.Open(d.Path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("hashing %s: %w", d.FullFilePath(), err)
|
||||
}
|
||||
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
hash := sha256.New()
|
||||
|
||||
_, err = io.Copy(hash, file)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("hashing %s: %w", d.FullFilePath(), err)
|
||||
}
|
||||
|
||||
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||
return hashFile(ctx, d.fsys, d.Path)
|
||||
}
|
||||
|
||||
// GetStores finds the DCF stores in the root of the mounted filesystems and reads
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
package dcf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The modes of what Import creates. Only the user who ran it can read the copies.
|
||||
const (
|
||||
importDirectoryMode = 0o750
|
||||
importFileMode = 0o600
|
||||
)
|
||||
|
||||
var errHashMismatch = errors.New("the SHA256 hash of the copy differs from the source")
|
||||
|
||||
// ImportResult lists what Store.Import did with each file, by destination path.
|
||||
type ImportResult struct {
|
||||
// Copied holds the copies this import wrote and verified.
|
||||
Copied []string
|
||||
// Skipped holds the paths that already held a file with the same SHA256 hash.
|
||||
Skipped []string
|
||||
// Conflicts holds the paths that already held anything else. Import left it
|
||||
// alone and did not copy the file.
|
||||
Conflicts []string
|
||||
}
|
||||
|
||||
// importFile is one file for Import to copy, and the path to copy it to.
|
||||
type importFile struct {
|
||||
source *Object
|
||||
target string
|
||||
}
|
||||
|
||||
// Import copies the files of the store to dest, and verifies each copy against
|
||||
// the SHA256 hash of its source. It copies every member of every file group, and
|
||||
// the images and videos under PRIVATE/M4ROOT. It only reads the card.
|
||||
//
|
||||
// Each file goes to dest/YYYY-MM-DD/<file name>, from its modification time. Every
|
||||
// member of a file group goes to the day of its primary member, so a group never
|
||||
// splits across two folders.
|
||||
//
|
||||
// A file with the same hash already at the path is skipped. Anything else there is
|
||||
// a conflict, which is not an error. Import never overwrites anything.
|
||||
//
|
||||
// One file that fails does not stop the others. The returned error joins the
|
||||
// failures, each naming its file, and the result still lists what was done. When
|
||||
// ctx is cancelled, the import stops at once and removes the partial copy, and the
|
||||
// error matches ctx.Err().
|
||||
func (d *Store) Import(ctx context.Context, dest string) (ImportResult, error) {
|
||||
var (
|
||||
result ImportResult
|
||||
failures []error
|
||||
)
|
||||
|
||||
for _, file := range d.importFiles(dest) {
|
||||
err := ctx.Err()
|
||||
if err != nil {
|
||||
failures = append(failures, fmt.Errorf("import stopped: %w", err))
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
err = copyFile(ctx, file, &result)
|
||||
if err != nil {
|
||||
failures = append(failures,
|
||||
fmt.Errorf("importing %s: %w", file.source.FullFilePath(), err))
|
||||
}
|
||||
}
|
||||
|
||||
return result, errors.Join(failures...)
|
||||
}
|
||||
|
||||
// importFiles lists the files of the store that Import copies to dest.
|
||||
func (d *Store) importFiles(dest string) []importFile {
|
||||
var files []importFile
|
||||
|
||||
add := func(source *Object, day time.Time) {
|
||||
name := path.Base(source.Path)
|
||||
|
||||
files = append(files, importFile{
|
||||
source: source,
|
||||
target: filepath.Join(dest, day.Format(time.DateOnly), name),
|
||||
})
|
||||
}
|
||||
|
||||
for _, directory := range d.Directories {
|
||||
for _, group := range directory.FileGroups {
|
||||
for _, member := range group.Members {
|
||||
add(member, group.Primary.ModTime)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The files under PRIVATE/M4ROOT are the ones with no DCF numbers.
|
||||
for _, image := range d.Images {
|
||||
if image.DirectoryNumber == 0 {
|
||||
add(&image.Object, image.ModTime)
|
||||
}
|
||||
}
|
||||
|
||||
for _, video := range d.Videos {
|
||||
if video.DirectoryNumber == 0 {
|
||||
add(&video.Object, video.ModTime)
|
||||
}
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// copyFile copies one file, and adds its target to the list in result that says
|
||||
// what happened. It adds the target to no list when it returns an error.
|
||||
func copyFile(ctx context.Context, file importFile, result *ImportResult) error {
|
||||
sourceHash, err := file.source.hash(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = os.MkdirAll(filepath.Dir(file.target), importDirectoryMode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// O_EXCL fails on anything already at the path, even a symlink, so nothing
|
||||
// there can be overwritten.
|
||||
target, err := os.OpenFile(
|
||||
file.target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, importFileMode,
|
||||
)
|
||||
if errors.Is(err, fs.ErrExist) {
|
||||
return checkExisting(ctx, file.target, sourceHash, result)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = writeCopy(ctx, file, target, sourceHash)
|
||||
if err != nil {
|
||||
return errors.Join(err, os.Remove(file.target))
|
||||
}
|
||||
|
||||
result.Copied = append(result.Copied, file.target)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkExisting adds target to the skipped files if it is a file with the given
|
||||
// hash, and to the conflicts if it is anything else.
|
||||
func checkExisting(
|
||||
ctx context.Context,
|
||||
target, sourceHash string,
|
||||
result *ImportResult,
|
||||
) error {
|
||||
info, err := os.Lstat(target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !info.Mode().IsRegular() {
|
||||
result.Conflicts = append(result.Conflicts, target)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
targetHash, err := hashOnDisk(ctx, target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if targetHash == sourceHash {
|
||||
result.Skipped = append(result.Skipped, target)
|
||||
} else {
|
||||
result.Conflicts = append(result.Conflicts, target)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeCopy fills the new file target from the source, and closes it. It then
|
||||
// reads the written file back to compare its hash with the hash of the source.
|
||||
func writeCopy(
|
||||
ctx context.Context,
|
||||
file importFile,
|
||||
target *os.File,
|
||||
sourceHash string,
|
||||
) error {
|
||||
err := writeAndClose(ctx, file.source, target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
targetHash, err := hashOnDisk(ctx, file.target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if targetHash != sourceHash {
|
||||
return errHashMismatch
|
||||
}
|
||||
|
||||
// The zero time leaves the access time as it is.
|
||||
return os.Chtimes(file.target, time.Time{}, file.source.ModTime)
|
||||
}
|
||||
|
||||
// writeAndClose writes the bytes of source to target, syncs target and closes it.
|
||||
func writeAndClose(ctx context.Context, source *Object, target *os.File) error {
|
||||
defer func() { _ = target.Close() }()
|
||||
|
||||
sourceFile, err := source.fsys.Open(source.Path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() { _ = sourceFile.Close() }()
|
||||
|
||||
_, err = io.Copy(target, contextReader(ctx, sourceFile))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = target.Sync()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return target.Close()
|
||||
}
|
||||
|
||||
// hashOnDisk returns the SHA256 hash of the file at filePath on the local
|
||||
// filesystem.
|
||||
func hashOnDisk(ctx context.Context, filePath string) (string, error) {
|
||||
return hashFile(ctx, os.DirFS(filepath.Dir(filePath)), filepath.Base(filePath))
|
||||
}
|
||||
|
||||
// hashFile returns the SHA256 hash of the named file as a hex string.
|
||||
func hashFile(ctx context.Context, fsys fs.FS, name string) (string, error) {
|
||||
file, err := fsys.Open(name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
hash := sha256.New()
|
||||
|
||||
_, err = io.Copy(hash, contextReader(ctx, file))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// readerFunc is a function that works as an io.Reader.
|
||||
type readerFunc func([]byte) (int, error)
|
||||
|
||||
func (f readerFunc) Read(p []byte) (int, error) {
|
||||
return f(p)
|
||||
}
|
||||
|
||||
// contextReader returns a reader that fails with ctx.Err() once ctx is cancelled,
|
||||
// so a copy stops inside a file.
|
||||
func contextReader(ctx context.Context, reader io.Reader) io.Reader {
|
||||
return readerFunc(func(p []byte) (int, error) {
|
||||
err := ctx.Err()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return reader.Read(p)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,521 @@
|
||||
package dcf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
)
|
||||
|
||||
// testDay is the folder that a file from cardFile goes to. The other constants
|
||||
// are the paths and contents of the files on most cards in these tests.
|
||||
const (
|
||||
testDay = "2026-10-03"
|
||||
|
||||
firstJPG = "DCIM/100ABCDE/DSC00011.JPG"
|
||||
firstJPGData = "first jpeg"
|
||||
secondJPG = "DCIM/100ABCDE/DSC00012.JPG"
|
||||
secondJPGData = "second jpeg"
|
||||
testClip = "PRIVATE/M4ROOT/CLIP/C0011.MP4"
|
||||
testClipData = "clip"
|
||||
)
|
||||
|
||||
// cardFile returns a file on a card, modified at noon on testDay.
|
||||
func cardFile(contents string) *fstest.MapFile {
|
||||
return &fstest.MapFile{
|
||||
Data: []byte(contents),
|
||||
ModTime: time.Date(2026, 10, 3, 12, 0, 0, 0, time.UTC),
|
||||
}
|
||||
}
|
||||
|
||||
// inTestDay returns the paths of the named files in the testDay folder of dest.
|
||||
func inTestDay(dest string, names ...string) []string {
|
||||
paths := make([]string, 0, len(names))
|
||||
|
||||
for _, name := range names {
|
||||
paths = append(paths, filepath.Join(dest, testDay, name))
|
||||
}
|
||||
|
||||
return paths
|
||||
}
|
||||
|
||||
// assertDest checks that dest holds exactly the given files, as a map from each
|
||||
// file's path under dest to its contents.
|
||||
func assertDest(t *testing.T, dest string, want map[string]string) {
|
||||
t.Helper()
|
||||
|
||||
fsys := os.DirFS(dest)
|
||||
got := map[string]string{}
|
||||
|
||||
err := fs.WalkDir(fsys, ".",
|
||||
func(name string, entry fs.DirEntry, err error) error {
|
||||
if err != nil || !entry.Type().IsRegular() {
|
||||
return err
|
||||
}
|
||||
|
||||
data, err := fs.ReadFile(fsys, name)
|
||||
got[name] = string(data)
|
||||
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("reading %s: %v", dest, err)
|
||||
}
|
||||
|
||||
if !maps.Equal(got, want) {
|
||||
t.Errorf("destination holds %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertEmptyDirectory(t *testing.T, directory string) {
|
||||
t.Helper()
|
||||
|
||||
entries, err := os.ReadDir(directory)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir: %v", err)
|
||||
}
|
||||
|
||||
if len(entries) != 0 {
|
||||
t.Errorf("%s holds %v, want nothing", directory, entries)
|
||||
}
|
||||
}
|
||||
|
||||
// addDestFile writes a file at the named path under dest.
|
||||
func addDestFile(t *testing.T, dest, name, contents string) {
|
||||
t.Helper()
|
||||
|
||||
fullPath := filepath.Join(dest, name)
|
||||
|
||||
err := os.MkdirAll(filepath.Dir(fullPath), 0o750)
|
||||
if err != nil {
|
||||
t.Fatalf("MkdirAll: %v", err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(fullPath, []byte(contents), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("WriteFile: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportCopiesEveryFileAndKeepsItsModificationTime(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dest := t.TempDir()
|
||||
|
||||
store := mustParseStore(t, fstest.MapFS{
|
||||
"DCIM/100ABCDE/DSC00011.ARW": cardFile("raw"),
|
||||
firstJPG: cardFile(firstJPGData),
|
||||
"DCIM/101ABCDE/MVI_0012.MP4": cardFile("video"),
|
||||
"DCIM/101ABCDE/MVI_0012.THM": cardFile("thumbnail"),
|
||||
"DCIM/MISC/DSC00013.JPG": cardFile("not conforming"),
|
||||
testClip: cardFile(testClipData),
|
||||
"PRIVATE/M4ROOT/CLIP/C0011M01.XML": cardFile("neither image nor video"),
|
||||
"PRIVATE/M4ROOT/THMBNL/C0011T01.JPG": cardFile("clip thumbnail"),
|
||||
})
|
||||
|
||||
result, err := store.Import(t.Context(), dest)
|
||||
if err != nil {
|
||||
t.Fatalf("Import: %v", err)
|
||||
}
|
||||
|
||||
assertPaths(t, "Copied", result.Copied, inTestDay(dest,
|
||||
"DSC00011.ARW",
|
||||
"DSC00011.JPG",
|
||||
"MVI_0012.MP4",
|
||||
"MVI_0012.THM",
|
||||
"C0011T01.JPG",
|
||||
"C0011.MP4",
|
||||
)...)
|
||||
assertPaths(t, "Skipped", result.Skipped)
|
||||
assertPaths(t, "Conflicts", result.Conflicts)
|
||||
|
||||
assertDest(t, dest, map[string]string{
|
||||
testDay + "/DSC00011.ARW": "raw",
|
||||
testDay + "/DSC00011.JPG": firstJPGData,
|
||||
testDay + "/MVI_0012.MP4": "video",
|
||||
testDay + "/MVI_0012.THM": "thumbnail",
|
||||
testDay + "/C0011.MP4": testClipData,
|
||||
testDay + "/C0011T01.JPG": "clip thumbnail",
|
||||
})
|
||||
|
||||
for _, copied := range result.Copied {
|
||||
info, err := os.Stat(copied)
|
||||
if err != nil {
|
||||
t.Fatalf("Stat: %v", err)
|
||||
}
|
||||
|
||||
if want := cardFile("").ModTime; !info.ModTime().Equal(want) {
|
||||
t.Errorf("%s modified at %v, want %v", copied, info.ModTime(), want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportTwiceSkipsEveryFileTheSecondTime(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dest := t.TempDir()
|
||||
|
||||
store := mustParseStore(t, fstest.MapFS{
|
||||
firstJPG: cardFile(firstJPGData),
|
||||
testClip: cardFile(testClipData),
|
||||
})
|
||||
|
||||
_, err := store.Import(t.Context(), dest)
|
||||
if err != nil {
|
||||
t.Fatalf("first Import: %v", err)
|
||||
}
|
||||
|
||||
result, err := store.Import(t.Context(), dest)
|
||||
if err != nil {
|
||||
t.Fatalf("second Import: %v", err)
|
||||
}
|
||||
|
||||
assertPaths(t, "Copied", result.Copied)
|
||||
assertPaths(t, "Skipped", result.Skipped,
|
||||
inTestDay(dest, "DSC00011.JPG", "C0011.MP4")...)
|
||||
assertPaths(t, "Conflicts", result.Conflicts)
|
||||
|
||||
assertDest(t, dest, map[string]string{
|
||||
testDay + "/DSC00011.JPG": firstJPGData,
|
||||
testDay + "/C0011.MP4": testClipData,
|
||||
})
|
||||
}
|
||||
|
||||
func TestImportLeavesADifferentFileAloneAndReportsAConflict(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dest := t.TempDir()
|
||||
addDestFile(t, dest, testDay+"/DSC00011.JPG", "different")
|
||||
|
||||
store := mustParseStore(t, fstest.MapFS{
|
||||
firstJPG: cardFile(firstJPGData),
|
||||
secondJPG: cardFile(secondJPGData),
|
||||
})
|
||||
|
||||
result, err := store.Import(t.Context(), dest)
|
||||
if err != nil {
|
||||
t.Fatalf("Import: %v", err)
|
||||
}
|
||||
|
||||
assertPaths(t, "Copied", result.Copied, inTestDay(dest, "DSC00012.JPG")...)
|
||||
assertPaths(t, "Skipped", result.Skipped)
|
||||
assertPaths(t, "Conflicts", result.Conflicts, inTestDay(dest, "DSC00011.JPG")...)
|
||||
|
||||
assertDest(t, dest, map[string]string{
|
||||
testDay + "/DSC00011.JPG": "different",
|
||||
testDay + "/DSC00012.JPG": secondJPGData,
|
||||
})
|
||||
}
|
||||
|
||||
func TestImportReportsAConflictBetweenTwoFilesWithOneName(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dest := t.TempDir()
|
||||
|
||||
store := mustParseStore(t, fstest.MapFS{
|
||||
firstJPG: cardFile(firstJPGData),
|
||||
"DCIM/101ABCDE/DSC00011.JPG": cardFile(secondJPGData),
|
||||
})
|
||||
|
||||
result, err := store.Import(t.Context(), dest)
|
||||
if err != nil {
|
||||
t.Fatalf("Import: %v", err)
|
||||
}
|
||||
|
||||
assertPaths(t, "Copied", result.Copied, inTestDay(dest, "DSC00011.JPG")...)
|
||||
assertPaths(t, "Conflicts", result.Conflicts, inTestDay(dest, "DSC00011.JPG")...)
|
||||
assertDest(t, dest, map[string]string{testDay + "/DSC00011.JPG": firstJPGData})
|
||||
}
|
||||
|
||||
func TestImportReportsAConflictAtADirectoryOrASymlink(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dest := t.TempDir()
|
||||
|
||||
// The symlink points at a file with the same bytes, which must not count.
|
||||
outside := newCardOnDisk(t, "OUTSIDE.JPG")
|
||||
|
||||
err := os.MkdirAll(filepath.Join(dest, testDay, "DSC00011.JPG"), 0o750)
|
||||
if err != nil {
|
||||
t.Fatalf("MkdirAll: %v", err)
|
||||
}
|
||||
|
||||
addSymlink(t, dest, testDay+"/DSC00012.JPG", filepath.Join(outside, "OUTSIDE.JPG"))
|
||||
|
||||
store := mustParseStore(t, fstest.MapFS{
|
||||
firstJPG: cardFile("abc"),
|
||||
secondJPG: cardFile("abc"),
|
||||
})
|
||||
|
||||
result, err := store.Import(t.Context(), dest)
|
||||
if err != nil {
|
||||
t.Fatalf("Import: %v", err)
|
||||
}
|
||||
|
||||
assertPaths(t, "Copied", result.Copied)
|
||||
assertPaths(t, "Skipped", result.Skipped)
|
||||
assertPaths(t, "Conflicts", result.Conflicts,
|
||||
inTestDay(dest, "DSC00011.JPG", "DSC00012.JPG")...)
|
||||
}
|
||||
|
||||
func TestImportPutsAFileGroupInTheFolderOfItsPrimaryMember(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dest := t.TempDir()
|
||||
|
||||
beforeMidnight := time.Date(2026, 10, 3, 23, 59, 59, 0, time.UTC)
|
||||
afterMidnight := time.Date(2026, 10, 4, 0, 0, 1, 0, time.UTC)
|
||||
|
||||
store := mustParseStore(t, fstest.MapFS{
|
||||
"DCIM/100ABCDE/DSC00011.ARW": {Data: []byte("raw"), ModTime: afterMidnight},
|
||||
firstJPG: {Data: []byte("jpeg"), ModTime: beforeMidnight},
|
||||
secondJPG: {Data: []byte("next"), ModTime: afterMidnight},
|
||||
})
|
||||
|
||||
_, err := store.Import(t.Context(), dest)
|
||||
if err != nil {
|
||||
t.Fatalf("Import: %v", err)
|
||||
}
|
||||
|
||||
assertDest(t, dest, map[string]string{
|
||||
"2026-10-03/DSC00011.ARW": "raw",
|
||||
"2026-10-03/DSC00011.JPG": "jpeg",
|
||||
"2026-10-04/DSC00012.JPG": "next",
|
||||
})
|
||||
|
||||
// The copy keeps its own modification time, whichever folder it is in.
|
||||
info, err := os.Stat(filepath.Join(dest, "2026-10-03/DSC00011.ARW"))
|
||||
if err != nil {
|
||||
t.Fatalf("Stat: %v", err)
|
||||
}
|
||||
|
||||
if !info.ModTime().Equal(afterMidnight) {
|
||||
t.Errorf("modified at %v, want %v", info.ModTime(), afterMidnight)
|
||||
}
|
||||
}
|
||||
|
||||
// changingFS is a card whose file at name holds other bytes from its second read
|
||||
// on. It counts the reads in opens.
|
||||
type changingFS struct {
|
||||
fstest.MapFS
|
||||
|
||||
name string
|
||||
opens *int
|
||||
}
|
||||
|
||||
func (f changingFS) Open(name string) (fs.File, error) {
|
||||
if name == f.name {
|
||||
*f.opens++
|
||||
if *f.opens > 1 {
|
||||
return fstest.MapFS{name: cardFile("other bytes")}.Open(name)
|
||||
}
|
||||
}
|
||||
|
||||
return f.MapFS.Open(name)
|
||||
}
|
||||
|
||||
func TestImportRemovesACopyThatDoesNotMatchItsSource(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dest := t.TempDir()
|
||||
opens := 0
|
||||
|
||||
store := mustParseStore(t, changingFS{
|
||||
MapFS: fstest.MapFS{
|
||||
firstJPG: cardFile(firstJPGData),
|
||||
secondJPG: cardFile(secondJPGData),
|
||||
},
|
||||
name: firstJPG,
|
||||
opens: &opens,
|
||||
})
|
||||
|
||||
result, err := store.Import(t.Context(), dest)
|
||||
if !errors.Is(err, errHashMismatch) {
|
||||
t.Fatalf("Import error = %v, want %v", err, errHashMismatch)
|
||||
}
|
||||
|
||||
if !strings.Contains(err.Error(), firstJPG) {
|
||||
t.Errorf("Import error = %v, want one naming %s", err, firstJPG)
|
||||
}
|
||||
|
||||
assertPaths(t, "Copied", result.Copied, inTestDay(dest, "DSC00012.JPG")...)
|
||||
assertDest(t, dest, map[string]string{testDay + "/DSC00012.JPG": secondJPGData})
|
||||
}
|
||||
|
||||
func TestImportCopiesTheOtherFilesWhenOneCannotBeRead(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dest := t.TempDir()
|
||||
|
||||
store := mustParseStore(t, failingFS{
|
||||
MapFS: fstest.MapFS{
|
||||
firstJPG: cardFile(firstJPGData),
|
||||
secondJPG: cardFile(secondJPGData),
|
||||
testClip: cardFile(testClipData),
|
||||
},
|
||||
call: "Open",
|
||||
name: firstJPG,
|
||||
})
|
||||
|
||||
result, err := store.Import(t.Context(), dest)
|
||||
if !errors.Is(err, fs.ErrPermission) {
|
||||
t.Fatalf("Import error = %v, want %v", err, fs.ErrPermission)
|
||||
}
|
||||
|
||||
if !strings.Contains(err.Error(), firstJPG) {
|
||||
t.Errorf("Import error = %v, want one naming %s", err, firstJPG)
|
||||
}
|
||||
|
||||
assertPaths(t, "Copied", result.Copied,
|
||||
inTestDay(dest, "DSC00012.JPG", "C0011.MP4")...)
|
||||
assertDest(t, dest, map[string]string{
|
||||
testDay + "/DSC00012.JPG": secondJPGData,
|
||||
testDay + "/C0011.MP4": testClipData,
|
||||
})
|
||||
}
|
||||
|
||||
func TestImportNamesEveryFileThatFails(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dest := t.TempDir()
|
||||
|
||||
// A plain file where the day folder belongs fails every file of that day.
|
||||
addDestFile(t, dest, testDay, "not a folder")
|
||||
|
||||
store := mustParseStore(t, fstest.MapFS{
|
||||
firstJPG: cardFile(firstJPGData),
|
||||
secondJPG: cardFile(secondJPGData),
|
||||
})
|
||||
|
||||
result, err := store.Import(t.Context(), dest)
|
||||
if err == nil {
|
||||
t.Fatal("Import returned no error")
|
||||
}
|
||||
|
||||
for _, name := range []string{firstJPG, secondJPG} {
|
||||
if !strings.Contains(err.Error(), name) {
|
||||
t.Errorf("Import error = %v, want one naming %s", err, name)
|
||||
}
|
||||
}
|
||||
|
||||
assertPaths(t, "Copied", result.Copied)
|
||||
}
|
||||
|
||||
func TestImportOfObjectBuiltByHandReturnsError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dest := t.TempDir()
|
||||
|
||||
store := &Store{Images: []*Image{{Object: Object{Path: "DSC00011.JPG"}}}}
|
||||
|
||||
_, err := store.Import(t.Context(), dest)
|
||||
if !errors.Is(err, fs.ErrInvalid) {
|
||||
t.Fatalf("Import error = %v, want %v", err, fs.ErrInvalid)
|
||||
}
|
||||
|
||||
assertEmptyDirectory(t, dest)
|
||||
}
|
||||
|
||||
// cancelingFile is an open file that cancels the import after each read.
|
||||
type cancelingFile struct {
|
||||
fs.File
|
||||
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (f cancelingFile) Read(p []byte) (int, error) {
|
||||
n, err := f.File.Read(p)
|
||||
|
||||
f.cancel()
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
// cancelingReadFS is a card that cancels the import during the read of the named
|
||||
// file, the time that file is opened for the openCount-th time. It counts the
|
||||
// reads in opens.
|
||||
type cancelingReadFS struct {
|
||||
fstest.MapFS
|
||||
|
||||
name string
|
||||
openCount int
|
||||
opens *int
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (f cancelingReadFS) Open(name string) (fs.File, error) {
|
||||
file, err := f.MapFS.Open(name)
|
||||
if err != nil || name != f.name {
|
||||
return file, err
|
||||
}
|
||||
|
||||
*f.opens++
|
||||
if *f.opens != f.openCount {
|
||||
return file, nil
|
||||
}
|
||||
|
||||
return cancelingFile{File: file, cancel: f.cancel}, nil
|
||||
}
|
||||
|
||||
func TestImportStopsInsideAFileWhenCancelledAndRemovesThePartialCopy(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// The first read of a file is hashing it, and the second is copying it.
|
||||
for _, openCount := range []int{1, 2} {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
dest := t.TempDir()
|
||||
opens := 0
|
||||
|
||||
store := mustParseStore(t, cancelingReadFS{
|
||||
MapFS: fstest.MapFS{
|
||||
// Larger than one read, so the cancellation falls inside the file.
|
||||
firstJPG: {Data: make([]byte, 1<<20)},
|
||||
secondJPG: cardFile(secondJPGData),
|
||||
},
|
||||
name: firstJPG,
|
||||
openCount: openCount,
|
||||
opens: &opens,
|
||||
cancel: cancel,
|
||||
})
|
||||
|
||||
result, err := store.Import(ctx, dest)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("cancelled in read %d: error = %v, want %v",
|
||||
openCount, err, context.Canceled)
|
||||
}
|
||||
|
||||
assertPaths(t, "Copied", result.Copied)
|
||||
assertDest(t, dest, map[string]string{})
|
||||
|
||||
if opens != openCount {
|
||||
t.Errorf("cancelled in read %d: file opened %d times", openCount, opens)
|
||||
}
|
||||
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportCopiesNothingWhenAlreadyCancelled(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
|
||||
dest := t.TempDir()
|
||||
|
||||
store := mustParseStore(t, fstest.MapFS{firstJPG: cardFile(firstJPGData)})
|
||||
|
||||
result, err := store.Import(ctx, dest)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("Import error = %v, want %v", err, context.Canceled)
|
||||
}
|
||||
|
||||
assertPaths(t, "Copied", result.Copied)
|
||||
assertEmptyDirectory(t, dest)
|
||||
}
|
||||
@@ -272,6 +272,7 @@ func newObject(
|
||||
Path: filePath,
|
||||
Size: info.Size(),
|
||||
Extension: path.Ext(filePath),
|
||||
ModTime: info.ModTime(),
|
||||
fsys: fsys,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -720,7 +720,7 @@ func TestParseStoreTreatsSymlinksAsMissingDirectories(t *testing.T) {
|
||||
}
|
||||
|
||||
// failingFS is a card on which one call fails for one path, the way a failing
|
||||
// card does. The call is Lstat, ReadDir or Info.
|
||||
// card does. The call is Open, Lstat, ReadDir or Info.
|
||||
type failingFS struct {
|
||||
fstest.MapFS
|
||||
|
||||
@@ -728,6 +728,15 @@ type failingFS struct {
|
||||
name string
|
||||
}
|
||||
|
||||
func (f failingFS) Open(name string) (fs.File, error) {
|
||||
err := f.failure("Open", name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return f.MapFS.Open(name)
|
||||
}
|
||||
|
||||
func (f failingFS) Lstat(name string) (fs.FileInfo, error) {
|
||||
err := f.failure("Lstat", name)
|
||||
if err != nil {
|
||||
|
||||
Reference in new issue
Block a user