Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
25c6aca864 |
No files matched your search
@@ -105,30 +105,57 @@ overengineering and reusable code.
|
|||||||
|
|
||||||
All the code is in the `pkg/dcf` package. The repo has no commands.
|
All the code is in the `pkg/dcf` package. The repo has no commands.
|
||||||
|
|
||||||
- `dcf.go` holds the public API. `GetDCFStores` finds the DCF stores and lists
|
- `dcf.go` holds the public API. `GetDCFStores` finds the DCF stores and reads
|
||||||
the files on them.
|
each one.
|
||||||
- `helpers.go` finds the mount points that hold a DCF store.
|
- `helpers.go` finds the mount points that hold a DCF store.
|
||||||
|
- `parse.go` reads one DCF store through an `fs.FS` rooted at its mount point.
|
||||||
|
|
||||||
The main types are:
|
The main types are:
|
||||||
|
|
||||||
- `Store` is one DCF store on one mounted filesystem.
|
- `Store` is one DCF store on one mounted filesystem.
|
||||||
- `Object` is one file in a store.
|
- `Directory` is one DCF directory. It has a directory number, a name and the
|
||||||
|
objects inside it.
|
||||||
|
- `Object` is one file in a store. A file in a DCF directory also has a
|
||||||
|
directory number and a file number.
|
||||||
- `Image` and `Video` embed `Object`. The file extension decides which one a
|
- `Image` and `Video` embed `Object`. The file extension decides which one a
|
||||||
file becomes.
|
file becomes. A file with any other extension, such as a `THM` thumbnail, is
|
||||||
|
neither.
|
||||||
|
|
||||||
A mounted filesystem is a DCF store when its root holds a `DCIM` or
|
A mounted filesystem is a DCF store when its root holds a `DCIM` or
|
||||||
`PRIVATE/M4ROOT` directory. `dcf` never mounts a filesystem.
|
`PRIVATE/M4ROOT` directory. It needs only one of them. `dcf` never mounts a
|
||||||
|
filesystem. Media anywhere else on the filesystem is not part of the store.
|
||||||
|
|
||||||
|
`dcf` never follows a symlink.
|
||||||
|
|
||||||
|
`dcf` applies the naming rules of the DCF specification, CIPA DC-009-2010, under
|
||||||
|
`DCIM`:
|
||||||
|
|
||||||
|
- A DCF directory sits directly under `DCIM`. Its name is a directory number
|
||||||
|
from 100 to 999, then five characters, such as `100MSDCF`.
|
||||||
|
- A DCF file sits directly in a DCF directory. Its name is four characters, then
|
||||||
|
a file number from 0001 to 9999, then a dot and a three-character extension,
|
||||||
|
such as `DSC00001.ARW`.
|
||||||
|
- The characters allowed are `A` to `Z`, `0` to `9` and `_`. Lower-case letters
|
||||||
|
count as upper-case.
|
||||||
|
- Directories that share a directory number are not DCF directories.
|
||||||
|
- `JPG` files that share a file number in one directory are not DCF files.
|
||||||
|
|
||||||
|
`Store.NonConforming` lists the path of everything else under `DCIM`. It
|
||||||
|
includes everything inside a directory that breaks the rules. It also includes
|
||||||
|
every symlink, whatever its name. These entries have no DCF numbers, and they
|
||||||
|
are not counted as images or videos.
|
||||||
|
|
||||||
|
The naming rules do not apply under `PRIVATE/M4ROOT`. The images and videos
|
||||||
|
there are in `Store.Images` and `Store.Videos`, and their DCF numbers are 0.
|
||||||
|
`dcf` skips any symlink there.
|
||||||
|
|
||||||
## TODO
|
## TODO
|
||||||
|
|
||||||
- Handle DCF file groups, where multiple objects share the same index number.
|
- Handle DCF file groups, where multiple objects share the same index number.
|
||||||
`dcf` does not handle these yet. If you need this, send me a link to a zip or
|
`dcf` does not handle these yet. If you need this, send me a link to a zip or
|
||||||
tar of a memory card that uses them, and I'll see what I can do.
|
tar of a memory card that uses them, and I'll see what I can do.
|
||||||
- Parse the DCF directory and file names, rather than listing every media file
|
- Take a `context.Context` to cancel the walk of a store.
|
||||||
on the filesystem.
|
|
||||||
- Read stores through `fs.FS`, with a `context.Context` to cancel a walk.
|
|
||||||
- Return a plain slice from `GetDCFStores`, not a pointer to one.
|
- Return a plain slice from `GetDCFStores`, not a pointer to one.
|
||||||
- Test the walk of a store. It only runs against real mounted media today.
|
|
||||||
- Copy files off a card, and verify each copy against its SHA256 hash.
|
- Copy files off a card, and verify each copy against its SHA256 hash.
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|||||||
+48
-41
@@ -1,6 +1,8 @@
|
|||||||
// Package dcf reads DCF cards: memory cards laid out to the "Design rule for
|
// Package dcf reads DCF cards: memory cards laid out to the "Design rule for
|
||||||
// Camera File system" specification (JEITA CP-3461). It finds the cards among
|
// Camera File system" specification (JEITA CP-3461). It finds the cards among
|
||||||
// the mounted filesystems and lists the images and videos on each one.
|
// the mounted filesystems and parses the DCF directories and files on each one
|
||||||
|
// into their directory numbers and file numbers. It lists the images and videos
|
||||||
|
// on each card, and everything under DCIM that is not a DCF directory or file.
|
||||||
package dcf
|
package dcf
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -20,8 +22,30 @@ import (
|
|||||||
// images and videos in the root of a mounted filesystem.
|
// images and videos in the root of a mounted filesystem.
|
||||||
type Store struct {
|
type Store struct {
|
||||||
RootDirectory string
|
RootDirectory string
|
||||||
|
// Directories holds the DCF directories under DCIM.
|
||||||
|
Directories []*Directory
|
||||||
|
// Images and Videos hold the images and videos among the objects in
|
||||||
|
// Directories, and those under PRIVATE/M4ROOT.
|
||||||
Images []*Image
|
Images []*Image
|
||||||
Videos []*Video
|
Videos []*Video
|
||||||
|
// NonConforming holds the path, relative to RootDirectory, of everything
|
||||||
|
// under DCIM that is not a DCF directory or a DCF file. It includes every
|
||||||
|
// symlink, and everything inside a directory that breaks the DCF naming
|
||||||
|
// rules. A directory whose name is not valid UTF-8 is listed without what is
|
||||||
|
// inside it, because fs.FS refuses to open one.
|
||||||
|
NonConforming []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Directory represents a DCF directory: a directory directly under DCIM whose
|
||||||
|
// name follows the DCF naming rules.
|
||||||
|
type Directory struct {
|
||||||
|
// Number is the directory number, from 100 to 999.
|
||||||
|
Number int
|
||||||
|
// Name is the name of the directory as it appears on the card.
|
||||||
|
Name string
|
||||||
|
// Objects holds every DCF file in the directory, including those that are
|
||||||
|
// neither images nor videos.
|
||||||
|
Objects []*Object
|
||||||
}
|
}
|
||||||
|
|
||||||
// Object represents a file in a DCF store. It is embedded in Image and Video
|
// Object represents a file in a DCF store. It is embedded in Image and Video
|
||||||
@@ -31,6 +55,10 @@ type Object struct {
|
|||||||
Path string
|
Path string
|
||||||
Size int64
|
Size int64
|
||||||
Extension string
|
Extension string
|
||||||
|
// DirectoryNumber and FileNumber are the DCF numbers of a file in a DCF
|
||||||
|
// directory. Both are 0 for a file under PRIVATE/M4ROOT.
|
||||||
|
DirectoryNumber int
|
||||||
|
FileNumber int
|
||||||
}
|
}
|
||||||
|
|
||||||
// Image represents an image file on a DCF store.
|
// Image represents an image file on a DCF store.
|
||||||
@@ -140,58 +168,37 @@ func pathToSHA256(path string) (string, error) {
|
|||||||
// any mounted filesystems. The integer argument is the number of DCF stores to
|
// any mounted filesystems. The integer argument is the number of DCF stores to
|
||||||
// return. If the argument is 0, all DCF stores are returned. It does not mount
|
// return. If the argument is 0, all DCF stores are returned. It does not mount
|
||||||
// filesystems or search outside of filesystem root directories. It does,
|
// filesystems or search outside of filesystem root directories. It does,
|
||||||
// however, walk the filesystems to find images and videos and populate the
|
// however, read the DCIM and PRIVATE/M4ROOT directories of each filesystem to
|
||||||
// returned Stores.
|
// populate the returned Stores. It never follows a symlink. If a card cannot be
|
||||||
|
// read, it returns an error that names the card's mount point.
|
||||||
func GetDCFStores(requestedCount int) (*[]Store, error) {
|
func GetDCFStores(requestedCount int) (*[]Store, error) {
|
||||||
dcfStorePaths, err := findDCFMountPoints(requestedCount)
|
dcfStorePaths, err := findDCFMountPoints(requestedCount)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
dcfStores := []Store{}
|
dcfStores, err := parseStores(dcfStorePaths)
|
||||||
|
|
||||||
for _, dcfStorePath := range dcfStorePaths {
|
|
||||||
dcfStore := Store{
|
|
||||||
RootDirectory: dcfStorePath,
|
|
||||||
Images: make([]*Image, 0),
|
|
||||||
Videos: make([]*Video, 0),
|
|
||||||
}
|
|
||||||
|
|
||||||
addFile := func(path string, info os.FileInfo, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if !info.IsDir() && isImageFile(path) {
|
|
||||||
image := Image{}
|
|
||||||
image.Path = strings.TrimPrefix(path, dcfStorePath+"/")
|
|
||||||
image.Size = info.Size()
|
|
||||||
image.Extension = filepath.Ext(path)
|
|
||||||
image.DCFStoreRoot = dcfStorePath
|
|
||||||
dcfStore.Images = append(dcfStore.Images, &image)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !info.IsDir() && isVideoFile(path) {
|
|
||||||
video := Video{}
|
|
||||||
video.Path = strings.TrimPrefix(path, dcfStorePath+"/")
|
|
||||||
video.Size = info.Size()
|
|
||||||
video.Extension = filepath.Ext(path)
|
|
||||||
video.DCFStoreRoot = dcfStorePath
|
|
||||||
dcfStore.Videos = append(dcfStore.Videos, &video)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
err := filepath.Walk(dcfStorePath, addFile)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
dcfStores = append(dcfStores, dcfStore)
|
return &dcfStores, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return &dcfStores, nil
|
// parseStores reads the card at each of the given mount points.
|
||||||
|
func parseStores(mountpoints []string) ([]Store, error) {
|
||||||
|
dcfStores := []Store{}
|
||||||
|
|
||||||
|
for _, mountpoint := range mountpoints {
|
||||||
|
dcfStore, err := parseStore(os.DirFS(mountpoint), mountpoint)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
dcfStores = append(dcfStores, *dcfStore)
|
||||||
|
}
|
||||||
|
|
||||||
|
return dcfStores, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func isImageFile(path string) bool {
|
func isImageFile(path string) bool {
|
||||||
|
|||||||
@@ -1,6 +1,60 @@
|
|||||||
package dcf
|
package dcf
|
||||||
|
|
||||||
import "testing"
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseStoresReadsEachMountPoint(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
first := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG")
|
||||||
|
second := newCardOnDisk(t, "PRIVATE/M4ROOT/CLIP/C0001.MP4")
|
||||||
|
|
||||||
|
stores, err := parseStores([]string{first, second})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parseStores: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(stores) != 2 {
|
||||||
|
t.Fatalf("parseStores returned %d stores, want 2", len(stores))
|
||||||
|
}
|
||||||
|
|
||||||
|
cases := map[string]Object{
|
||||||
|
filepath.Join(first, "DCIM/100ABCDE/ABCD0001.JPG"): stores[0].Images[0].Object,
|
||||||
|
filepath.Join(second, "PRIVATE/M4ROOT/CLIP/C0001.MP4"): stores[1].Videos[0].Object,
|
||||||
|
}
|
||||||
|
|
||||||
|
for want, object := range cases {
|
||||||
|
if got := object.FullFilePath(); got != want {
|
||||||
|
t.Errorf("FullFilePath() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := object.Hash()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Hash() returned error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The published SHA-256 test vector for "abc".
|
||||||
|
wantHash := "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
|
||||||
|
if got != wantHash {
|
||||||
|
t.Errorf("Hash() = %q, want %q", got, wantHash)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoresReturnsErrorNamingTheCard(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// Every lookup under a plain file fails.
|
||||||
|
mountpoint := filepath.Join(newCardOnDisk(t, "FILE"), "FILE")
|
||||||
|
|
||||||
|
_, err := parseStores([]string{mountpoint})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), mountpoint) {
|
||||||
|
t.Fatalf("parseStores error = %v, want one naming %s", err, mountpoint)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestIsImageFile(t *testing.T) {
|
func TestIsImageFile(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|||||||
+5
-6
@@ -4,7 +4,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"slices"
|
"slices"
|
||||||
|
|
||||||
"github.com/shirou/gopsutil/disk"
|
"github.com/shirou/gopsutil/disk"
|
||||||
@@ -63,12 +62,12 @@ func filterDCFMountPoints(mountpoints []string, requestedCount int) []string {
|
|||||||
// isDCFMountPoint reports whether the root of mountpoint holds a DCIM or
|
// isDCFMountPoint reports whether the root of mountpoint holds a DCIM or
|
||||||
// PRIVATE/M4ROOT directory, which is how we detect DCF stores.
|
// PRIVATE/M4ROOT directory, which is how we detect DCF stores.
|
||||||
func isDCFMountPoint(mountpoint string) bool {
|
func isDCFMountPoint(mountpoint string) bool {
|
||||||
for _, directory := range []string{"DCIM", "PRIVATE/M4ROOT"} {
|
fsys := os.DirFS(mountpoint)
|
||||||
path := filepath.Join(mountpoint, directory)
|
|
||||||
|
|
||||||
info, err := os.Stat(path)
|
for _, directory := range []string{dcimDirectory, m4RootDirectory} {
|
||||||
if err == nil && info.IsDir() {
|
found, err := hasDirectory(fsys, directory)
|
||||||
slog.Debug(fmt.Sprintf("Found %s directory at %s\n", directory, path))
|
if err == nil && found {
|
||||||
|
slog.Debug(fmt.Sprintf("Found %s directory at %s\n", directory, mountpoint))
|
||||||
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -57,6 +57,21 @@ func TestIsDCFMountPointIgnoresPlainFile(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestIsDCFMountPointIgnoresSymlink(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
outside := newMountPoint(t, "M4ROOT")
|
||||||
|
|
||||||
|
for _, link := range []string{dcimDirectory, privateDirectory, m4RootDirectory} {
|
||||||
|
root := newMountPoint(t, filepath.Dir(link))
|
||||||
|
addSymlink(t, root, link, outside)
|
||||||
|
|
||||||
|
if isDCFMountPoint(root) {
|
||||||
|
t.Errorf("isDCFMountPoint counted a symlink at %s as a directory", link)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFilterDCFMountPointsReturnsRequestedCount(t *testing.T) {
|
func TestFilterDCFMountPointsReturnsRequestedCount(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,275 @@
|
|||||||
|
package dcf
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"path"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The two directories at the root of a card that hold its files.
|
||||||
|
const (
|
||||||
|
dcimDirectory = "DCIM"
|
||||||
|
m4RootDirectory = "PRIVATE/M4ROOT"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The naming rules of the DCF specification, CIPA DC-009-2010, sections 2.3,
|
||||||
|
// 4.2.2 and 4.3.1. Names use A-Z, 0-9 and "_", and lower-case letters count as
|
||||||
|
// upper-case. Each pattern captures the number in the name.
|
||||||
|
var (
|
||||||
|
dcfDirectoryName = regexp.MustCompile(`^([1-9][0-9]{2})[0-9A-Za-z_]{5}$`)
|
||||||
|
dcfFileName = regexp.MustCompile(`^[0-9A-Za-z_]{4}([0-9]{4})\.[0-9A-Za-z_]{3}$`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// parseStore reads one card from fsys, which is rooted at the card's mount
|
||||||
|
// point. The root argument is the path of that mount point. Only DCIM and
|
||||||
|
// PRIVATE/M4ROOT are read. A card can lack either one. A symlink is never
|
||||||
|
// followed. An error names the mount point, because fsys reports only paths
|
||||||
|
// inside it.
|
||||||
|
func parseStore(fsys fs.FS, root string) (*Store, error) {
|
||||||
|
store := &Store{RootDirectory: root}
|
||||||
|
|
||||||
|
err := parseDCIM(fsys, store)
|
||||||
|
if err == nil {
|
||||||
|
err = parseM4Root(fsys, store)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("reading the card at %s: %w", root, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return store, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// hasDirectory reports whether the card holds the named directory. A plain file
|
||||||
|
// or a symlink at any step of the path counts as missing. Any other failure to
|
||||||
|
// look is an error.
|
||||||
|
func hasDirectory(fsys fs.FS, name string) (bool, error) {
|
||||||
|
parent := path.Dir(name)
|
||||||
|
if parent != "." {
|
||||||
|
found, err := hasDirectory(fsys, parent)
|
||||||
|
if err != nil || !found {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := fs.Lstat(fsys, name)
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return info.IsDir(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseDCIM adds the DCF directories under DCIM to store. Everything else there
|
||||||
|
// is not conforming. That includes directories that share a directory number,
|
||||||
|
// which section 7.1.2 of the specification treats as non-DCF directories.
|
||||||
|
func parseDCIM(fsys fs.FS, store *Store) error {
|
||||||
|
found, err := hasDirectory(fsys, dcimDirectory)
|
||||||
|
if err != nil || !found {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
entries, err := fs.ReadDir(fsys, dcimDirectory)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
directoriesWithNumber := map[int]int{}
|
||||||
|
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.IsDir() {
|
||||||
|
directoriesWithNumber[parseNumber(dcfDirectoryName, entry.Name())]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, entry := range entries {
|
||||||
|
number := parseNumber(dcfDirectoryName, entry.Name())
|
||||||
|
|
||||||
|
if entry.IsDir() && number != 0 && directoriesWithNumber[number] == 1 {
|
||||||
|
err = parseDirectory(fsys, store, entry.Name(), number)
|
||||||
|
} else {
|
||||||
|
err = addNonConforming(fsys, store, dcimDirectory, entry)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseDirectory adds one DCF directory and the DCF files directly inside it to
|
||||||
|
// store. A DCF file is a regular file, so a symlink with a DCF name is not one.
|
||||||
|
// Everything else there is not conforming. That includes JPG files that share a
|
||||||
|
// file number, which section 7.2.2 of the specification says must not show DCF
|
||||||
|
// numbers. Other files that share a file number do conform.
|
||||||
|
func parseDirectory(fsys fs.FS, store *Store, name string, number int) error {
|
||||||
|
directoryPath := path.Join(dcimDirectory, name)
|
||||||
|
|
||||||
|
entries, err := fs.ReadDir(fsys, directoryPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
jpgsWithNumber := map[int]int{}
|
||||||
|
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.Type().IsRegular() && isJPG(entry.Name()) {
|
||||||
|
jpgsWithNumber[parseNumber(dcfFileName, entry.Name())]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
directory := &Directory{Number: number, Name: name}
|
||||||
|
|
||||||
|
for _, entry := range entries {
|
||||||
|
fileNumber := parseNumber(dcfFileName, entry.Name())
|
||||||
|
sharedJPG := isJPG(entry.Name()) && jpgsWithNumber[fileNumber] > 1
|
||||||
|
|
||||||
|
if !entry.Type().IsRegular() || fileNumber == 0 || sharedJPG {
|
||||||
|
err = addNonConforming(fsys, store, directoryPath, entry)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
entryPath := path.Join(directoryPath, entry.Name())
|
||||||
|
|
||||||
|
object, err := newObject(store, entryPath, entry)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
object.DirectoryNumber = number
|
||||||
|
object.FileNumber = fileNumber
|
||||||
|
directory.Objects = append(directory.Objects, &object)
|
||||||
|
|
||||||
|
addMedia(store, object)
|
||||||
|
}
|
||||||
|
|
||||||
|
store.Directories = append(store.Directories, directory)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseM4Root adds the images and videos under PRIVATE/M4ROOT to store. The DCF
|
||||||
|
// naming rules do not apply there, so these files have no DCF numbers. It skips
|
||||||
|
// a symlink. It also skips a name that is not valid UTF-8, because fs.FS refuses
|
||||||
|
// to open one.
|
||||||
|
func parseM4Root(fsys fs.FS, store *Store) error {
|
||||||
|
found, err := hasDirectory(fsys, m4RootDirectory)
|
||||||
|
if err != nil || !found {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return fs.WalkDir(fsys, m4RootDirectory,
|
||||||
|
func(filePath string, entry fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
validName := fs.ValidPath(filePath)
|
||||||
|
|
||||||
|
if entry.IsDir() && !validName {
|
||||||
|
return fs.SkipDir
|
||||||
|
}
|
||||||
|
|
||||||
|
if !entry.Type().IsRegular() || !validName {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
object, err := newObject(store, filePath, entry)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
addMedia(store, object)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// addNonConforming lists one entry of the directory dir on store as not
|
||||||
|
// conforming. If the entry is a directory, it lists everything inside it too. It
|
||||||
|
// does not follow a symlink. It does not read a directory whose name is not
|
||||||
|
// valid UTF-8, because fs.FS refuses to open one.
|
||||||
|
func addNonConforming(fsys fs.FS, store *Store, dir string, entry fs.DirEntry) error {
|
||||||
|
entryPath := path.Join(dir, entry.Name())
|
||||||
|
store.NonConforming = append(store.NonConforming, entryPath)
|
||||||
|
|
||||||
|
if !entry.IsDir() || !fs.ValidPath(entryPath) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
entries, err := fs.ReadDir(fsys, entryPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, inner := range entries {
|
||||||
|
err = addNonConforming(fsys, store, entryPath, inner)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// newObject returns the Object for the file at filePath, with no DCF numbers.
|
||||||
|
func newObject(store *Store, filePath string, entry fs.DirEntry) (Object, error) {
|
||||||
|
info, err := entry.Info()
|
||||||
|
if err != nil {
|
||||||
|
return Object{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return Object{
|
||||||
|
DCFStoreRoot: store.RootDirectory,
|
||||||
|
Path: filePath,
|
||||||
|
Size: info.Size(),
|
||||||
|
Extension: path.Ext(filePath),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// addMedia adds object to the images or videos of store. A file with any other
|
||||||
|
// extension is neither, and is not added.
|
||||||
|
func addMedia(store *Store, object Object) {
|
||||||
|
if isImageFile(object.Path) {
|
||||||
|
store.Images = append(store.Images, &Image{Object: object})
|
||||||
|
}
|
||||||
|
|
||||||
|
if isVideoFile(object.Path) {
|
||||||
|
store.Videos = append(store.Videos, &Video{Object: object})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseNumber returns the number that pattern captures in name. It returns 0
|
||||||
|
// when name does not match. No DCF number is 0, so this also rejects the file
|
||||||
|
// number 0000, which the specification forbids.
|
||||||
|
func parseNumber(pattern *regexp.Regexp, name string) int {
|
||||||
|
match := pattern.FindStringSubmatch(name)
|
||||||
|
if match == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
number, err := strconv.Atoi(match[1])
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
return number
|
||||||
|
}
|
||||||
|
|
||||||
|
func isJPG(name string) bool {
|
||||||
|
return strings.EqualFold(path.Ext(name), ".JPG")
|
||||||
|
}
|
||||||
@@ -0,0 +1,809 @@
|
|||||||
|
package dcf
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
testRoot = "/mnt/card"
|
||||||
|
privateDirectory = "PRIVATE"
|
||||||
|
)
|
||||||
|
|
||||||
|
// newCard returns a card holding an empty file at each of the given paths.
|
||||||
|
func newCard(paths ...string) fstest.MapFS {
|
||||||
|
card := fstest.MapFS{}
|
||||||
|
|
||||||
|
for _, filePath := range paths {
|
||||||
|
card[filePath] = &fstest.MapFile{}
|
||||||
|
}
|
||||||
|
|
||||||
|
return card
|
||||||
|
}
|
||||||
|
|
||||||
|
// newCardOnDisk returns a temporary directory holding the three bytes "abc" in a
|
||||||
|
// file at each of the given paths. It is for tests that fstest.MapFS cannot do.
|
||||||
|
func newCardOnDisk(t *testing.T, paths ...string) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
root := t.TempDir()
|
||||||
|
|
||||||
|
for _, filePath := range paths {
|
||||||
|
fullPath := filepath.Join(root, filePath)
|
||||||
|
|
||||||
|
err := os.MkdirAll(filepath.Dir(fullPath), 0o750)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MkdirAll: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = os.WriteFile(fullPath, []byte("abc"), 0o600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("WriteFile: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return root
|
||||||
|
}
|
||||||
|
|
||||||
|
// addSymlink creates a symlink to target at the named path under root.
|
||||||
|
func addSymlink(t *testing.T, root, name, target string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
err := os.Symlink(target, filepath.Join(root, name))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Symlink: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustParseStore(t *testing.T, fsys fs.FS) *Store {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
store, err := parseStore(fsys, testRoot)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parseStore: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return store
|
||||||
|
}
|
||||||
|
|
||||||
|
func imagePaths(store *Store) []string {
|
||||||
|
paths := make([]string, 0, len(store.Images))
|
||||||
|
|
||||||
|
for _, image := range store.Images {
|
||||||
|
paths = append(paths, image.Path)
|
||||||
|
}
|
||||||
|
|
||||||
|
return paths
|
||||||
|
}
|
||||||
|
|
||||||
|
func videoPaths(store *Store) []string {
|
||||||
|
paths := make([]string, 0, len(store.Videos))
|
||||||
|
|
||||||
|
for _, video := range store.Videos {
|
||||||
|
paths = append(paths, video.Path)
|
||||||
|
}
|
||||||
|
|
||||||
|
return paths
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectPaths(directory *Directory) []string {
|
||||||
|
paths := make([]string, 0, len(directory.Objects))
|
||||||
|
|
||||||
|
for _, object := range directory.Objects {
|
||||||
|
paths = append(paths, object.Path)
|
||||||
|
}
|
||||||
|
|
||||||
|
return paths
|
||||||
|
}
|
||||||
|
|
||||||
|
func directoryNumbers(store *Store) []int {
|
||||||
|
numbers := make([]int, 0, len(store.Directories))
|
||||||
|
|
||||||
|
for _, directory := range store.Directories {
|
||||||
|
numbers = append(numbers, directory.Number)
|
||||||
|
}
|
||||||
|
|
||||||
|
return numbers
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertPaths(t *testing.T, what string, got []string, want ...string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("%s = %q, want %q", what, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStorePutsEachFileWhereItBelongs(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
card := fstest.MapFS{
|
||||||
|
"DCIM/100MSDCF/ABCD0001.ARW": {Data: make([]byte, 2000)},
|
||||||
|
"DCIM/100MSDCF/readme.txt": {Data: make([]byte, 7)},
|
||||||
|
"DCIM/101CANON/MVI_0002.MP4": {Data: make([]byte, 9000)},
|
||||||
|
"DCIM/101CANON/MVI_0002.THM": {Data: make([]byte, 100)},
|
||||||
|
"DCIM/MISC/DSC00003.JPG": {Data: make([]byte, 7)},
|
||||||
|
"DCIM/DSC00004.JPG": {Data: make([]byte, 7)},
|
||||||
|
"PRIVATE/M4ROOT/CLIP/C0002.MP4": {Data: make([]byte, 400)},
|
||||||
|
"PRIVATE/M4ROOT/CLIP/C0002M01.XML": {Data: make([]byte, 7)},
|
||||||
|
"PRIVATE/OTHER/C0003.MP4": {Data: make([]byte, 7)},
|
||||||
|
"BACKUP/DSC00005.JPG": {Data: make([]byte, 7)},
|
||||||
|
"DSC00006.JPG": {Data: make([]byte, 7)},
|
||||||
|
}
|
||||||
|
|
||||||
|
store := mustParseStore(t, card)
|
||||||
|
|
||||||
|
arw := Object{
|
||||||
|
DCFStoreRoot: testRoot,
|
||||||
|
Path: "DCIM/100MSDCF/ABCD0001.ARW",
|
||||||
|
Size: 2000,
|
||||||
|
Extension: ".ARW",
|
||||||
|
DirectoryNumber: 100,
|
||||||
|
FileNumber: 1,
|
||||||
|
}
|
||||||
|
mp4 := Object{
|
||||||
|
DCFStoreRoot: testRoot,
|
||||||
|
Path: "DCIM/101CANON/MVI_0002.MP4",
|
||||||
|
Size: 9000,
|
||||||
|
Extension: ".MP4",
|
||||||
|
DirectoryNumber: 101,
|
||||||
|
FileNumber: 2,
|
||||||
|
}
|
||||||
|
thm := Object{
|
||||||
|
DCFStoreRoot: testRoot,
|
||||||
|
Path: "DCIM/101CANON/MVI_0002.THM",
|
||||||
|
Size: 100,
|
||||||
|
Extension: ".THM",
|
||||||
|
DirectoryNumber: 101,
|
||||||
|
FileNumber: 2,
|
||||||
|
}
|
||||||
|
clip := Object{
|
||||||
|
DCFStoreRoot: testRoot,
|
||||||
|
Path: "PRIVATE/M4ROOT/CLIP/C0002.MP4",
|
||||||
|
Size: 400,
|
||||||
|
Extension: ".MP4",
|
||||||
|
}
|
||||||
|
|
||||||
|
wantDirectories := []*Directory{
|
||||||
|
{Number: 100, Name: "100MSDCF", Objects: []*Object{&arw}},
|
||||||
|
{Number: 101, Name: "101CANON", Objects: []*Object{&mp4, &thm}},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(store.Directories, wantDirectories) {
|
||||||
|
t.Errorf("Directories = %+v, want %+v", store.Directories, wantDirectories)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantImages := []*Image{{Object: arw}}
|
||||||
|
if !reflect.DeepEqual(store.Images, wantImages) {
|
||||||
|
t.Errorf("Images = %+v, want %+v", store.Images, wantImages)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantVideos := []*Video{{Object: mp4}, {Object: clip}}
|
||||||
|
if !reflect.DeepEqual(store.Videos, wantVideos) {
|
||||||
|
t.Errorf("Videos = %+v, want %+v", store.Videos, wantVideos)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "NonConforming", store.NonConforming,
|
||||||
|
"DCIM/100MSDCF/readme.txt",
|
||||||
|
"DCIM/DSC00004.JPG",
|
||||||
|
"DCIM/MISC",
|
||||||
|
"DCIM/MISC/DSC00003.JPG",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreCountsOnlyImagesAndVideos(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
card := fstest.MapFS{
|
||||||
|
"DCIM/100ABCDE/ABCD0001.ARW": {Data: make([]byte, 2000)},
|
||||||
|
"DCIM/100ABCDE/ABCD0002.THM": {Data: make([]byte, 100)},
|
||||||
|
"DCIM/100ABCDE/ABCD0003.MP4": {Data: make([]byte, 9000)},
|
||||||
|
"DCIM/MISC/ABCD0004.JPG": {Data: make([]byte, 50)},
|
||||||
|
"PRIVATE/M4ROOT/CLIP/C0004.MP4": {Data: make([]byte, 400)},
|
||||||
|
"ABCD0005.JPG": {Data: make([]byte, 30)},
|
||||||
|
}
|
||||||
|
|
||||||
|
store := mustParseStore(t, card)
|
||||||
|
|
||||||
|
if got := store.ImagesCount(); got != 1 {
|
||||||
|
t.Errorf("ImagesCount() = %d, want 1", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := store.VideosCount(); got != 2 {
|
||||||
|
t.Errorf("VideosCount() = %d, want 2", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := store.TotalSize(); got != 11400 {
|
||||||
|
t.Errorf("TotalSize() = %d, want 11400", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreSetsFullFilePath(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := mustParseStore(t, newCard("DCIM/100MSDCF/DSC00001.JPG"))
|
||||||
|
|
||||||
|
want := testRoot + "/DCIM/100MSDCF/DSC00001.JPG"
|
||||||
|
if got := store.Images[0].FullFilePath(); got != want {
|
||||||
|
t.Fatalf("FullFilePath() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseNumberOfDirectoryName(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cases := map[string]int{
|
||||||
|
"100MSDCF": 100,
|
||||||
|
"999VWXYZ": 999,
|
||||||
|
"100D3500": 100,
|
||||||
|
"101abcde": 101,
|
||||||
|
"102A_B_C": 102,
|
||||||
|
"099ABCDE": 0,
|
||||||
|
"000ABCDE": 0,
|
||||||
|
"103A": 0,
|
||||||
|
"IM104ABC": 0,
|
||||||
|
"105_abcdefg": 0,
|
||||||
|
"107@-%+]": 0,
|
||||||
|
"IM08ABCD": 0,
|
||||||
|
"100ABC-E": 0,
|
||||||
|
"100ABCD": 0,
|
||||||
|
"100ABCDEF": 0,
|
||||||
|
"1000ABCDE": 0,
|
||||||
|
"X100ABCDE": 0,
|
||||||
|
"TEMP": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, want := range cases {
|
||||||
|
if got := parseNumber(dcfDirectoryName, name); got != want {
|
||||||
|
t.Errorf("directory %q parsed to %d, want %d", name, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseNumberOfFileName(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cases := map[string]int{
|
||||||
|
"DSC00001.ARW": 1,
|
||||||
|
"ABCD9999.JPG": 9999,
|
||||||
|
"abcd0002.jpg": 2,
|
||||||
|
"_ABC0001.JPG": 1,
|
||||||
|
"IMG_0001.JPG": 1,
|
||||||
|
"12340005.JPG": 5,
|
||||||
|
"ABCD0000.JPG": 0,
|
||||||
|
"A0004.JPG": 0,
|
||||||
|
"ABCDE0005.JPG": 0,
|
||||||
|
"ABCD10000.JPG": 0,
|
||||||
|
"abcdefg0006.JPG": 0,
|
||||||
|
"+-@]0007.JPG": 0,
|
||||||
|
"ABCD00~1.JPG": 0,
|
||||||
|
"ABCDEFGH.JPG": 0,
|
||||||
|
"ABCD0001": 0,
|
||||||
|
"ABCD0001.": 0,
|
||||||
|
"ABCD0001.JP": 0,
|
||||||
|
"ABCD0001.JPEG": 0,
|
||||||
|
"ABCD0001.J-G": 0,
|
||||||
|
"._DSC0001.JPG": 0,
|
||||||
|
".DS_Store": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, want := range cases {
|
||||||
|
if got := parseNumber(dcfFileName, name); got != want {
|
||||||
|
t.Errorf("file %q parsed to %d, want %d", name, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreTreatsLowerCaseNamesAsDCFNames(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := mustParseStore(t, newCard("DCIM/101abcde/abcd0002.jpg"))
|
||||||
|
|
||||||
|
want := []*Directory{{
|
||||||
|
Number: 101,
|
||||||
|
Name: "101abcde",
|
||||||
|
Objects: []*Object{{
|
||||||
|
DCFStoreRoot: testRoot,
|
||||||
|
Path: "DCIM/101abcde/abcd0002.jpg",
|
||||||
|
Extension: ".jpg",
|
||||||
|
DirectoryNumber: 101,
|
||||||
|
FileNumber: 2,
|
||||||
|
}},
|
||||||
|
}}
|
||||||
|
if !reflect.DeepEqual(store.Directories, want) {
|
||||||
|
t.Errorf("Directories = %+v, want %+v", store.Directories, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "Images", imagePaths(store), "DCIM/101abcde/abcd0002.jpg")
|
||||||
|
assertPaths(t, "NonConforming", store.NonConforming)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreAcceptsUnderscoreInNames(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := mustParseStore(t, newCard("DCIM/100A_B_C/_ABC0001.JPG"))
|
||||||
|
|
||||||
|
assertPaths(t, "Images", imagePaths(store), "DCIM/100A_B_C/_ABC0001.JPG")
|
||||||
|
assertPaths(t, "NonConforming", store.NonConforming)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreRejectsDirectoriesSharingANumber(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := mustParseStore(t, newCard(
|
||||||
|
"DCIM/102ABCDE/ABCD0001.JPG",
|
||||||
|
"DCIM/102PQRST/ABCD0002.JPG",
|
||||||
|
"DCIM/103ABCDE/ABCD0003.JPG",
|
||||||
|
))
|
||||||
|
|
||||||
|
if got, want := directoryNumbers(store), []int{103}; !slices.Equal(got, want) {
|
||||||
|
t.Errorf("directory numbers = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "Images", imagePaths(store), "DCIM/103ABCDE/ABCD0003.JPG")
|
||||||
|
assertPaths(t, "NonConforming", store.NonConforming,
|
||||||
|
"DCIM/102ABCDE",
|
||||||
|
"DCIM/102ABCDE/ABCD0001.JPG",
|
||||||
|
"DCIM/102PQRST",
|
||||||
|
"DCIM/102PQRST/ABCD0002.JPG",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreDoesNotCountPlainFileAsSharingADirectoryNumber(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := mustParseStore(t, newCard(
|
||||||
|
"DCIM/100ABCDE",
|
||||||
|
"DCIM/100PQRST/ABCD0001.JPG",
|
||||||
|
))
|
||||||
|
|
||||||
|
if got, want := directoryNumbers(store), []int{100}; !slices.Equal(got, want) {
|
||||||
|
t.Errorf("directory numbers = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "NonConforming", store.NonConforming, "DCIM/100ABCDE")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreRejectsJPGsSharingAFileNumber(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := mustParseStore(t, newCard(
|
||||||
|
"DCIM/100ABCDE/ABCD0003.JPG",
|
||||||
|
"DCIM/100ABCDE/PQRS0003.jpg",
|
||||||
|
"DCIM/101ABCDE/ABCD0003.JPG",
|
||||||
|
))
|
||||||
|
|
||||||
|
assertPaths(t, "objects", objectPaths(store.Directories[0]))
|
||||||
|
assertPaths(t, "objects", objectPaths(store.Directories[1]),
|
||||||
|
"DCIM/101ABCDE/ABCD0003.JPG",
|
||||||
|
)
|
||||||
|
assertPaths(t, "Images", imagePaths(store), "DCIM/101ABCDE/ABCD0003.JPG")
|
||||||
|
assertPaths(t, "NonConforming", store.NonConforming,
|
||||||
|
"DCIM/100ABCDE/ABCD0003.JPG",
|
||||||
|
"DCIM/100ABCDE/PQRS0003.jpg",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreKeepsOtherFilesSharingAFileNumber(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := mustParseStore(t, newCard(
|
||||||
|
"DCIM/100ABCDE/DSC00004.ARW",
|
||||||
|
"DCIM/100ABCDE/DSC00004.JPG",
|
||||||
|
"DCIM/100ABCDE/MVI_0005.MP4",
|
||||||
|
"DCIM/100ABCDE/MVI_0005.THM",
|
||||||
|
))
|
||||||
|
|
||||||
|
assertPaths(t, "objects", objectPaths(store.Directories[0]),
|
||||||
|
"DCIM/100ABCDE/DSC00004.ARW",
|
||||||
|
"DCIM/100ABCDE/DSC00004.JPG",
|
||||||
|
"DCIM/100ABCDE/MVI_0005.MP4",
|
||||||
|
"DCIM/100ABCDE/MVI_0005.THM",
|
||||||
|
)
|
||||||
|
assertPaths(t, "Images", imagePaths(store),
|
||||||
|
"DCIM/100ABCDE/DSC00004.ARW",
|
||||||
|
"DCIM/100ABCDE/DSC00004.JPG",
|
||||||
|
)
|
||||||
|
assertPaths(t, "Videos", videoPaths(store), "DCIM/100ABCDE/MVI_0005.MP4")
|
||||||
|
assertPaths(t, "NonConforming", store.NonConforming)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreKeepsARWSharingANumberWithTwoJPGs(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := mustParseStore(t, newCard(
|
||||||
|
"DCIM/100ABCDE/ABCD0004.ARW",
|
||||||
|
"DCIM/100ABCDE/ABCD0004.JPG",
|
||||||
|
"DCIM/100ABCDE/PQRS0004.JPG",
|
||||||
|
))
|
||||||
|
|
||||||
|
assertPaths(t, "objects", objectPaths(store.Directories[0]),
|
||||||
|
"DCIM/100ABCDE/ABCD0004.ARW",
|
||||||
|
)
|
||||||
|
assertPaths(t, "NonConforming", store.NonConforming,
|
||||||
|
"DCIM/100ABCDE/ABCD0004.JPG",
|
||||||
|
"DCIM/100ABCDE/PQRS0004.JPG",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreDoesNotCountDirectoryAsSharingAFileNumber(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := mustParseStore(t, newCard(
|
||||||
|
"DCIM/100ABCDE/ABCD0003.JPG",
|
||||||
|
"DCIM/100ABCDE/PQRS0003.JPG/README.TXT",
|
||||||
|
))
|
||||||
|
|
||||||
|
assertPaths(t, "objects", objectPaths(store.Directories[0]),
|
||||||
|
"DCIM/100ABCDE/ABCD0003.JPG",
|
||||||
|
)
|
||||||
|
assertPaths(t, "NonConforming", store.NonConforming,
|
||||||
|
"DCIM/100ABCDE/PQRS0003.JPG",
|
||||||
|
"DCIM/100ABCDE/PQRS0003.JPG/README.TXT",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreDoesNotCountDirectoryNamedLikeAVideo(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := mustParseStore(t, newCard("PRIVATE/M4ROOT/CLIP.MP4/C0001.XML"))
|
||||||
|
|
||||||
|
assertPaths(t, "Videos", videoPaths(store))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreRejectsSubdirectoryOfDCFDirectory(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := mustParseStore(t, newCard(
|
||||||
|
"DCIM/100ABCDE/SUB/ABCD0001.JPG",
|
||||||
|
"DCIM/100ABCDE/ABCD0002.JPG/ABCD0003.JPG",
|
||||||
|
))
|
||||||
|
|
||||||
|
assertPaths(t, "objects", objectPaths(store.Directories[0]))
|
||||||
|
assertPaths(t, "Images", imagePaths(store))
|
||||||
|
assertPaths(t, "NonConforming", store.NonConforming,
|
||||||
|
"DCIM/100ABCDE/ABCD0002.JPG",
|
||||||
|
"DCIM/100ABCDE/ABCD0002.JPG/ABCD0003.JPG",
|
||||||
|
"DCIM/100ABCDE/SUB",
|
||||||
|
"DCIM/100ABCDE/SUB/ABCD0001.JPG",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreListsEmptyDirectories(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := mustParseStore(t, fstest.MapFS{
|
||||||
|
"DCIM/100EMPTY": {Mode: fs.ModeDir},
|
||||||
|
"DCIM/MISC": {Mode: fs.ModeDir},
|
||||||
|
})
|
||||||
|
|
||||||
|
if got, want := directoryNumbers(store), []int{100}; !slices.Equal(got, want) {
|
||||||
|
t.Errorf("directory numbers = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "NonConforming", store.NonConforming, "DCIM/MISC")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreWithoutDCIM(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := mustParseStore(t, newCard("PRIVATE/M4ROOT/CLIP/C0001.MP4"))
|
||||||
|
|
||||||
|
if len(store.Directories) != 0 {
|
||||||
|
t.Errorf("Directories = %+v, want none", store.Directories)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "Videos", videoPaths(store), "PRIVATE/M4ROOT/CLIP/C0001.MP4")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreWithoutM4Root(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := mustParseStore(t, newCard("DCIM/100ABCDE/ABCD0001.JPG"))
|
||||||
|
|
||||||
|
assertPaths(t, "Images", imagePaths(store), "DCIM/100ABCDE/ABCD0001.JPG")
|
||||||
|
assertPaths(t, "Videos", videoPaths(store))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreTreatsPlainFilesAsMissingDirectories(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := mustParseStore(t, newCard("DCIM", "PRIVATE/M4ROOT"))
|
||||||
|
|
||||||
|
assertPaths(t, "Images", imagePaths(store))
|
||||||
|
assertPaths(t, "Videos", videoPaths(store))
|
||||||
|
assertPaths(t, "NonConforming", store.NonConforming)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A real filesystem reports a lookup under a plain file differently from a
|
||||||
|
// missing path, which fstest.MapFS does not reproduce.
|
||||||
|
func TestParseStoreReadsRealFilesystemWithPlainFileNamedPrivate(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
root := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG", "PRIVATE")
|
||||||
|
|
||||||
|
store := mustParseStore(t, os.DirFS(root))
|
||||||
|
|
||||||
|
assertPaths(t, "Images", imagePaths(store), "DCIM/100ABCDE/ABCD0001.JPG")
|
||||||
|
|
||||||
|
if got := store.TotalImageSize(); got != 3 {
|
||||||
|
t.Errorf("TotalImageSize() = %d, want 3", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A real filesystem can hold a name with the byte 0xff, which is not valid UTF-8.
|
||||||
|
// fs.FS refuses to open such a name.
|
||||||
|
func TestParseStoreListsNamesThatFSRefuses(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
root := newCardOnDisk(t,
|
||||||
|
"DCIM/100ABCDE/ABCD0001.JPG",
|
||||||
|
"DCIM/100ABCDE/A\xffDIR/INNER",
|
||||||
|
"DCIM/100ABCDE/A\xffFILE",
|
||||||
|
"DCIM/A\xffDIR/INNER",
|
||||||
|
"DCIM/A\xffFILE",
|
||||||
|
"DCIM/MISC/A\xffDIR/INNER",
|
||||||
|
"DCIM/MISC/A\xffFILE",
|
||||||
|
)
|
||||||
|
|
||||||
|
store := mustParseStore(t, os.DirFS(root))
|
||||||
|
|
||||||
|
assertPaths(t, "Images", imagePaths(store), "DCIM/100ABCDE/ABCD0001.JPG")
|
||||||
|
assertPaths(t, "NonConforming", store.NonConforming,
|
||||||
|
"DCIM/100ABCDE/A\xffDIR",
|
||||||
|
"DCIM/100ABCDE/A\xffFILE",
|
||||||
|
"DCIM/A\xffDIR",
|
||||||
|
"DCIM/A\xffFILE",
|
||||||
|
"DCIM/MISC",
|
||||||
|
"DCIM/MISC/A\xffDIR",
|
||||||
|
"DCIM/MISC/A\xffFILE",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreListsDanglingSymlinks(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
root := newCardOnDisk(t, "DCIM/100ABCDE/ABCD0001.JPG", "DCIM/MISC/README.TXT")
|
||||||
|
|
||||||
|
addSymlink(t, root, "DCIM/100ABCDE/ABCD0002.JPG", "absent")
|
||||||
|
addSymlink(t, root, "DCIM/100ABCDE/LINK", "absent")
|
||||||
|
addSymlink(t, root, "DCIM/101ABCDE", "absent")
|
||||||
|
addSymlink(t, root, "DCIM/LINK", "absent")
|
||||||
|
addSymlink(t, root, "DCIM/MISC/LINK", "absent")
|
||||||
|
|
||||||
|
store := mustParseStore(t, os.DirFS(root))
|
||||||
|
|
||||||
|
if got, want := directoryNumbers(store), []int{100}; !slices.Equal(got, want) {
|
||||||
|
t.Errorf("directory numbers = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "Images", imagePaths(store), "DCIM/100ABCDE/ABCD0001.JPG")
|
||||||
|
assertPaths(t, "NonConforming", store.NonConforming,
|
||||||
|
"DCIM/100ABCDE/ABCD0002.JPG",
|
||||||
|
"DCIM/100ABCDE/LINK",
|
||||||
|
"DCIM/101ABCDE",
|
||||||
|
"DCIM/LINK",
|
||||||
|
"DCIM/MISC",
|
||||||
|
"DCIM/MISC/LINK",
|
||||||
|
"DCIM/MISC/README.TXT",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreDoesNotFollowSymlinksUnderDCIM(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
outside := newCardOnDisk(t, "OUTSIDE.JPG", "DIRECTORY/ABCD0009.JPG")
|
||||||
|
outsideFile := filepath.Join(outside, "OUTSIDE.JPG")
|
||||||
|
outsideDirectory := filepath.Join(outside, "DIRECTORY")
|
||||||
|
|
||||||
|
root := newCardOnDisk(t,
|
||||||
|
"DCIM/100ABCDE/ABCD0001.JPG",
|
||||||
|
"DCIM/102ABCDE/ABCD0002.JPG",
|
||||||
|
"DCIM/MISC/README.TXT",
|
||||||
|
)
|
||||||
|
|
||||||
|
// These two have DCF file names. The second has the number of a real JPG,
|
||||||
|
// which must not make that JPG shared.
|
||||||
|
addSymlink(t, root, "DCIM/100ABCDE/ABCD0003.JPG", outsideFile)
|
||||||
|
addSymlink(t, root, "DCIM/100ABCDE/PQRS0001.JPG", outsideFile)
|
||||||
|
|
||||||
|
// These two have DCF directory names. The second has the number of a real
|
||||||
|
// directory, which must not make that directory shared.
|
||||||
|
addSymlink(t, root, "DCIM/101ABCDE", outsideDirectory)
|
||||||
|
addSymlink(t, root, "DCIM/102PQRST", outsideDirectory)
|
||||||
|
|
||||||
|
addSymlink(t, root, "DCIM/LINK", outsideDirectory)
|
||||||
|
addSymlink(t, root, "DCIM/MISC/LINK", outsideDirectory)
|
||||||
|
|
||||||
|
store := mustParseStore(t, os.DirFS(root))
|
||||||
|
|
||||||
|
if got, want := directoryNumbers(store), []int{100, 102}; !slices.Equal(got, want) {
|
||||||
|
t.Errorf("directory numbers = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "Images", imagePaths(store),
|
||||||
|
"DCIM/100ABCDE/ABCD0001.JPG",
|
||||||
|
"DCIM/102ABCDE/ABCD0002.JPG",
|
||||||
|
)
|
||||||
|
|
||||||
|
if got := store.TotalImageSize(); got != 6 {
|
||||||
|
t.Errorf("TotalImageSize() = %d, want 6", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPaths(t, "NonConforming", store.NonConforming,
|
||||||
|
"DCIM/100ABCDE/ABCD0003.JPG",
|
||||||
|
"DCIM/100ABCDE/PQRS0001.JPG",
|
||||||
|
"DCIM/101ABCDE",
|
||||||
|
"DCIM/102PQRST",
|
||||||
|
"DCIM/LINK",
|
||||||
|
"DCIM/MISC",
|
||||||
|
"DCIM/MISC/LINK",
|
||||||
|
"DCIM/MISC/README.TXT",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreSkipsSymlinksAndNamesThatFSRefusesUnderM4Root(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
outside := newCardOnDisk(t, "OUTSIDE.MP4")
|
||||||
|
|
||||||
|
// The names with the byte 0xff sort before CLIP, so a walk that stops at one
|
||||||
|
// of them does not reach it.
|
||||||
|
root := newCardOnDisk(t,
|
||||||
|
"PRIVATE/M4ROOT/A\xffDIR/C0002.MP4",
|
||||||
|
"PRIVATE/M4ROOT/A\xffFILE.MP4",
|
||||||
|
"PRIVATE/M4ROOT/CLIP/C0001.MP4",
|
||||||
|
)
|
||||||
|
|
||||||
|
addSymlink(t, root, "PRIVATE/M4ROOT/CLIP/C0003.MP4",
|
||||||
|
filepath.Join(outside, "OUTSIDE.MP4"))
|
||||||
|
addSymlink(t, root, "PRIVATE/M4ROOT/CLIP/C0004.MP4", "absent")
|
||||||
|
addSymlink(t, root, "PRIVATE/M4ROOT/LINK", outside)
|
||||||
|
|
||||||
|
store := mustParseStore(t, os.DirFS(root))
|
||||||
|
|
||||||
|
assertPaths(t, "Videos", videoPaths(store), "PRIVATE/M4ROOT/CLIP/C0001.MP4")
|
||||||
|
|
||||||
|
if got := store.TotalVideoSize(); got != 3 {
|
||||||
|
t.Errorf("TotalVideoSize() = %d, want 3", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreTreatsSymlinksAsMissingDirectories(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
outside := newCardOnDisk(t,
|
||||||
|
"100ABCDE/ABCD0001.JPG",
|
||||||
|
"M4ROOT/C0001.MP4",
|
||||||
|
"C0002.MP4",
|
||||||
|
)
|
||||||
|
|
||||||
|
cases := []struct{ link, target string }{
|
||||||
|
{dcimDirectory, outside},
|
||||||
|
{dcimDirectory, dcimDirectory},
|
||||||
|
{privateDirectory, outside},
|
||||||
|
{privateDirectory, privateDirectory},
|
||||||
|
{m4RootDirectory, outside},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, testCase := range cases {
|
||||||
|
root := newMountPoint(t, filepath.Dir(testCase.link))
|
||||||
|
addSymlink(t, root, testCase.link, testCase.target)
|
||||||
|
|
||||||
|
store := mustParseStore(t, os.DirFS(root))
|
||||||
|
|
||||||
|
if !reflect.DeepEqual(store, &Store{RootDirectory: testRoot}) {
|
||||||
|
t.Errorf("symlink at %s to %s read as %+v, want an empty store",
|
||||||
|
testCase.link, testCase.target, store)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// failingFS is a card on which one call fails for one path, the way a failing
|
||||||
|
// card does. The call is Lstat, ReadDir or Info.
|
||||||
|
type failingFS struct {
|
||||||
|
fstest.MapFS
|
||||||
|
|
||||||
|
call string
|
||||||
|
name string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f failingFS) Lstat(name string) (fs.FileInfo, error) {
|
||||||
|
err := f.failure("Lstat", name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return f.MapFS.Lstat(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f failingFS) ReadDir(name string) ([]fs.DirEntry, error) {
|
||||||
|
err := f.failure("ReadDir", name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
entries, err := f.MapFS.ReadDir(name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, entry := range entries {
|
||||||
|
err = f.failure("Info", path.Join(name, entry.Name()))
|
||||||
|
if err != nil {
|
||||||
|
entries[i] = failingEntry{DirEntry: entry, err: err}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return entries, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f failingFS) failure(call, name string) error {
|
||||||
|
if call == f.call && name == f.name {
|
||||||
|
return &fs.PathError{Op: call, Path: name, Err: fs.ErrPermission}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// failingEntry is a directory entry whose Info always returns err.
|
||||||
|
type failingEntry struct {
|
||||||
|
fs.DirEntry
|
||||||
|
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e failingEntry) Info() (fs.FileInfo, error) {
|
||||||
|
return nil, e.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStoreReturnsReadErrorsNamingTheCard(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
card := newCard(
|
||||||
|
"DCIM/100ABCDE/ABCD0001.JPG",
|
||||||
|
"DCIM/100ABCDE/SUB/README.TXT",
|
||||||
|
"DCIM/MISC/SUB/README.TXT",
|
||||||
|
"PRIVATE/M4ROOT/CLIP/C0001.MP4",
|
||||||
|
)
|
||||||
|
|
||||||
|
cases := map[string][]string{
|
||||||
|
"Lstat": {dcimDirectory, privateDirectory, m4RootDirectory},
|
||||||
|
"ReadDir": {
|
||||||
|
dcimDirectory,
|
||||||
|
"DCIM/100ABCDE",
|
||||||
|
"DCIM/100ABCDE/SUB",
|
||||||
|
"DCIM/MISC",
|
||||||
|
"DCIM/MISC/SUB",
|
||||||
|
m4RootDirectory,
|
||||||
|
"PRIVATE/M4ROOT/CLIP",
|
||||||
|
},
|
||||||
|
"Info": {"DCIM/100ABCDE/ABCD0001.JPG", "PRIVATE/M4ROOT/CLIP/C0001.MP4"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for call, names := range cases {
|
||||||
|
for _, name := range names {
|
||||||
|
fsys := failingFS{MapFS: card, call: call, name: name}
|
||||||
|
|
||||||
|
_, err := parseStore(fsys, testRoot)
|
||||||
|
if !errors.Is(err, fs.ErrPermission) {
|
||||||
|
t.Errorf("%s of %s: error = %v, want %v",
|
||||||
|
call, name, err, fs.ErrPermission)
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
want := "reading the card at " + testRoot + ": " +
|
||||||
|
call + " " + name + ": permission denied"
|
||||||
|
if err.Error() != want {
|
||||||
|
t.Errorf("error = %q, want %q", err.Error(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user