'cause it splits and hashes, avi.
This commit is contained in:
		
							parent
							
								
									14a446f4d9
								
							
						
					
					
						commit
						bf8525fa04
					
				
							
								
								
									
										1
									
								
								.gitignore
									
									
									
									
										vendored
									
									
										Normal file
									
								
							
							
						
						
									
										1
									
								
								.gitignore
									
									
									
									
										vendored
									
									
										Normal file
									
								
							@ -0,0 +1 @@
 | 
				
			|||||||
 | 
					hashingsplitter
 | 
				
			||||||
							
								
								
									
										192
									
								
								main.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										192
									
								
								main.go
									
									
									
									
									
										Normal file
									
								
							@ -0,0 +1,192 @@
 | 
				
			|||||||
 | 
					package main
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import (
 | 
				
			||||||
 | 
					    "crypto/sha1"
 | 
				
			||||||
 | 
					    "encoding/hex"
 | 
				
			||||||
 | 
					    "errors"
 | 
				
			||||||
 | 
					    "flag"
 | 
				
			||||||
 | 
					    "fmt"
 | 
				
			||||||
 | 
					    "io"
 | 
				
			||||||
 | 
					    "os"
 | 
				
			||||||
 | 
					    "path/filepath"
 | 
				
			||||||
 | 
					    "strconv"
 | 
				
			||||||
 | 
					    "strings"
 | 
				
			||||||
 | 
					    "unicode"
 | 
				
			||||||
 | 
					)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					// parseByteSize parses a string like "100MB", "1M", "10MiB", or "1024" into a byte count.
 | 
				
			||||||
 | 
					//
 | 
				
			||||||
 | 
					// Supported suffixes:
 | 
				
			||||||
 | 
					//   • Single-letter SI: K=1e3, M=1e6, G=1e9, T=1e12
 | 
				
			||||||
 | 
					//   • Two-letter SI: KB=1e3, MB=1e6, GB=1e9, TB=1e12
 | 
				
			||||||
 | 
					//   • Binary suffixes: KiB=2^10, MiB=2^20, GiB=2^30, TiB=2^40
 | 
				
			||||||
 | 
					// If no suffix is present, it assumes bytes.
 | 
				
			||||||
 | 
					func parseByteSize(s string) (int64, error) {
 | 
				
			||||||
 | 
					    s = strings.TrimSpace(s)
 | 
				
			||||||
 | 
					    if s == "" {
 | 
				
			||||||
 | 
					        return 0, errors.New("empty size string")
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    // Find the first non-digit character (we’ll allow digits only).
 | 
				
			||||||
 | 
					    // That remainder is our suffix. Everything before that is the numeric portion.
 | 
				
			||||||
 | 
					    idx := 0
 | 
				
			||||||
 | 
					    for idx < len(s) && unicode.IsDigit(rune(s[idx])) {
 | 
				
			||||||
 | 
					        idx++
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    numStr := s[:idx]
 | 
				
			||||||
 | 
					    suffixStr := strings.ToUpper(strings.TrimSpace(s[idx:]))
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    // Parse the numeric portion as an integer
 | 
				
			||||||
 | 
					    baseVal, err := strconv.ParseInt(numStr, 10, 64)
 | 
				
			||||||
 | 
					    if err != nil {
 | 
				
			||||||
 | 
					        return 0, fmt.Errorf("invalid numeric portion %q: %v", numStr, err)
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    // If there is no suffix, it’s just bytes
 | 
				
			||||||
 | 
					    if suffixStr == "" {
 | 
				
			||||||
 | 
					        return baseVal, nil
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    // Known multipliers (include single-letter SI).
 | 
				
			||||||
 | 
					    multipliers := map[string]int64{
 | 
				
			||||||
 | 
					        "":    1,                 // no suffix (handled above, but let's keep for completeness)
 | 
				
			||||||
 | 
					        "B":   1,
 | 
				
			||||||
 | 
					        // Single-letter SI:
 | 
				
			||||||
 | 
					        "K":   1000,
 | 
				
			||||||
 | 
					        "M":   1000 * 1000,
 | 
				
			||||||
 | 
					        "G":   1000 * 1000 * 1000,
 | 
				
			||||||
 | 
					        "T":   1000 * 1000 * 1000 * 1000,
 | 
				
			||||||
 | 
					        // Two-letter SI:
 | 
				
			||||||
 | 
					        "KB":  1000,
 | 
				
			||||||
 | 
					        "MB":  1000 * 1000,
 | 
				
			||||||
 | 
					        "GB":  1000 * 1000 * 1000,
 | 
				
			||||||
 | 
					        "TB":  1000 * 1000 * 1000 * 1000,
 | 
				
			||||||
 | 
					        // Binary:
 | 
				
			||||||
 | 
					        "KIB": 1024,
 | 
				
			||||||
 | 
					        "MIB": 1024 * 1024,
 | 
				
			||||||
 | 
					        "GIB": 1024 * 1024 * 1024,
 | 
				
			||||||
 | 
					        "TIB": 1024 * 1024 * 1024 * 1024,
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    factor, found := multipliers[suffixStr]
 | 
				
			||||||
 | 
					    if !found {
 | 
				
			||||||
 | 
					        return 0, fmt.Errorf("unrecognized size suffix %q", suffixStr)
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					    return baseVal * factor, nil
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					func main() {
 | 
				
			||||||
 | 
					    // Command-line flags
 | 
				
			||||||
 | 
					    sizeStr := flag.String("size", "", "Split size (e.g. 64MiB, 128KB, 1M, 65536)")
 | 
				
			||||||
 | 
					    dirFlag := flag.String("dir", ".", "Destination directory for output files")
 | 
				
			||||||
 | 
					    prefixFlag := flag.String("prefix", "outfile", "Prefix (base name) for split files")
 | 
				
			||||||
 | 
					    flag.Parse()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    // Parse the size from the -size argument
 | 
				
			||||||
 | 
					    chunkSize, err := parseByteSize(*sizeStr)
 | 
				
			||||||
 | 
					    if err != nil {
 | 
				
			||||||
 | 
					        fmt.Fprintf(os.Stderr, "Error parsing -size: %v\n", err)
 | 
				
			||||||
 | 
					        os.Exit(1)
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					    if chunkSize <= 0 {
 | 
				
			||||||
 | 
					        fmt.Fprintf(os.Stderr, "Error: parsed size must be a positive integer.\n")
 | 
				
			||||||
 | 
					        os.Exit(1)
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    var (
 | 
				
			||||||
 | 
					        chunkIndex int
 | 
				
			||||||
 | 
					        sumFiles   []string
 | 
				
			||||||
 | 
					    )
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    // Read stdin and split into chunks
 | 
				
			||||||
 | 
					    for {
 | 
				
			||||||
 | 
					        chunkIndex++
 | 
				
			||||||
 | 
					        chunkName := fmt.Sprintf("%s.%05d", *prefixFlag, chunkIndex)
 | 
				
			||||||
 | 
					        chunkPath := filepath.Join(*dirFlag, chunkName)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        outFile, err := os.Create(chunkPath)
 | 
				
			||||||
 | 
					        if err != nil {
 | 
				
			||||||
 | 
					            fmt.Fprintf(os.Stderr, "Failed to create file '%s': %v\n", chunkPath, err)
 | 
				
			||||||
 | 
					            os.Exit(1)
 | 
				
			||||||
 | 
					        }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        // Use SHA-1 hasher (same default algorithm used by many "shasum" tools)
 | 
				
			||||||
 | 
					        hasher := sha1.New()
 | 
				
			||||||
 | 
					        writer := io.MultiWriter(outFile, hasher)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        var writtenThisChunk int64
 | 
				
			||||||
 | 
					        for writtenThisChunk < chunkSize {
 | 
				
			||||||
 | 
					            remaining := chunkSize - writtenThisChunk
 | 
				
			||||||
 | 
					            copied, copyErr := io.CopyN(writer, os.Stdin, remaining)
 | 
				
			||||||
 | 
					            writtenThisChunk += copied
 | 
				
			||||||
 | 
					            if copyErr == io.EOF {
 | 
				
			||||||
 | 
					                break // No more data from stdin
 | 
				
			||||||
 | 
					            } else if copyErr != nil {
 | 
				
			||||||
 | 
					                fmt.Fprintf(os.Stderr, "Error reading stdin: %v\n", copyErr)
 | 
				
			||||||
 | 
					                os.Exit(1)
 | 
				
			||||||
 | 
					            }
 | 
				
			||||||
 | 
					            if writtenThisChunk == chunkSize {
 | 
				
			||||||
 | 
					                break
 | 
				
			||||||
 | 
					            }
 | 
				
			||||||
 | 
					        }
 | 
				
			||||||
 | 
					        outFile.Close()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        // If empty chunk, remove file and end
 | 
				
			||||||
 | 
					        if writtenThisChunk == 0 {
 | 
				
			||||||
 | 
					            os.Remove(chunkPath)
 | 
				
			||||||
 | 
					            break
 | 
				
			||||||
 | 
					        }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        // Compute the SHA-1 sum in "shasum" format: <hex_digest>  filename
 | 
				
			||||||
 | 
					        sumHex := hex.EncodeToString(hasher.Sum(nil))
 | 
				
			||||||
 | 
					        line := fmt.Sprintf("%s  %s\n", sumHex, chunkName)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        shasumPath := chunkPath + ".shasum.txt"
 | 
				
			||||||
 | 
					        sumFile, err := os.Create(shasumPath)
 | 
				
			||||||
 | 
					        if err != nil {
 | 
				
			||||||
 | 
					            fmt.Fprintf(os.Stderr, "Failed to create shasum file '%s': %v\n", shasumPath, err)
 | 
				
			||||||
 | 
					            os.Exit(1)
 | 
				
			||||||
 | 
					        }
 | 
				
			||||||
 | 
					        if _, err := sumFile.WriteString(line); err != nil {
 | 
				
			||||||
 | 
					            fmt.Fprintf(os.Stderr, "Failed to write to '%s': %v\n", shasumPath, err)
 | 
				
			||||||
 | 
					            os.Exit(1)
 | 
				
			||||||
 | 
					        }
 | 
				
			||||||
 | 
					        sumFile.Close()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        // Collect this .shasum.txt filename so we can reference it in check.sh
 | 
				
			||||||
 | 
					        sumFiles = append(sumFiles, filepath.Base(shasumPath))
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        // If we wrote less than chunkSize, we're at EOF
 | 
				
			||||||
 | 
					        if writtenThisChunk < chunkSize {
 | 
				
			||||||
 | 
					            break
 | 
				
			||||||
 | 
					        }
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    // Create a "check.sh" script in the destination directory
 | 
				
			||||||
 | 
					    checkScriptPath := filepath.Join(*dirFlag, "check.sh")
 | 
				
			||||||
 | 
					    checkScriptFile, err := os.Create(checkScriptPath)
 | 
				
			||||||
 | 
					    if err != nil {
 | 
				
			||||||
 | 
					        fmt.Fprintf(os.Stderr, "Failed to create check.sh script: %v\n", err)
 | 
				
			||||||
 | 
					        os.Exit(1)
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    // Instead of wildcards, list all .shasum.txt files by name
 | 
				
			||||||
 | 
					    checkCommand := "shasum -c"
 | 
				
			||||||
 | 
					    for _, f := range sumFiles {
 | 
				
			||||||
 | 
					        checkCommand += " " + f
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    script := "#!/bin/sh\n\n" +
 | 
				
			||||||
 | 
					        "echo \"Checking all generated .shasum.txt files...\"\n" +
 | 
				
			||||||
 | 
					        checkCommand + "\n"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    if _, err := checkScriptFile.WriteString(script); err != nil {
 | 
				
			||||||
 | 
					        fmt.Fprintf(os.Stderr, "Failed to write check.sh: %v\n", err)
 | 
				
			||||||
 | 
					        os.Exit(1)
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					    checkScriptFile.Close()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    // Attempt to make check.sh executable
 | 
				
			||||||
 | 
					    _ = os.Chmod(checkScriptPath, 0755)
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
		Loading…
	
		Reference in New Issue
	
	Block a user