// 2>/dev/null||/usr/bin/env go run "$0" "$@"; exit $? package main import ( "bytes" "flag" "io" "log" "os" "regexp" ) var removePtr = flag.Bool("r", false, "Don't move, but remove doubles") var noActionPtr = flag.Bool("n", false, "Don't actually move or remove files") var sources = []string{} var targetDir = "" var rePath = regexp.MustCompile("^.*/") func cmdline() { flag.Parse() // log.Println("remove only: ", *removePtr) // log.Println("no action: ", *noActionPtr) // log.Println("tail:", flag.Args()) sources = flag.Args() targetDir, sources = sources[len(sources)-1], sources[:len(sources)-1] // log.Println("sources: ", sources) } func checkTargetDir() { // log.Println("target dir: ", targetDir) // Check if targetDir is a dir and writable if stat, err := os.Stat(targetDir); err == nil && stat.IsDir() { // path is a directory // log.Println("is dir") } else { log.Println("Target not a directory: ", targetDir) os.Exit(10) } } func processSources() { // log.Println("Processing sources:") source := "" target := "" for _, source = range sources { target = targetDir + "/" + rePath.ReplaceAllLiteralString(source, "") if sourceStat, err := os.Stat(source); err == nil && sourceStat.Mode().IsRegular() { // log.Println("source: ", source) // log.Println("target: ", target) targetStat, err := os.Stat(target) if err == nil { if os.SameFile(sourceStat, targetStat) { log.Println("Skipped mv", source, "->", target, "same actual file") continue } if targetStat.Mode().IsRegular() { // target exists and is a file if sameFileContent(source, target) { // delete source log.Println("Files are the same. Deleting", source) if !*noActionPtr { err := os.Remove(source) if err != nil { log.Fatal(err) } } } else { // Not the same, skip file log.Println("Skipped mv", source, "->", target, "files differ") } } else { // target exists and is not a regular file. Bail out? log.Println("Skipped mv", source, "->", target, "target exists, but not a file") continue } } else { // target does not exist: Move file if noActionPtr not set log.Println("mv", source, "->", target) if !*noActionPtr && !*removePtr { move(source, target) } } } else { log.Println("Skipped mv", source, "->", target, "source not a regular file") } } } func sameFileContent(source string, target string) bool { // 2 ways to go about this. Calculate a hash for both files // or do a blockwise compare of both files. // As we're not reusing things the blockwise compare is likely // slightly faster // Modern SSDs work with 8Kb blocks, so that's a good size as a start. // Could also do 8Mb blocks, because, meh. bufSize := 8096 // log.Println("reading file...") sourceFH, err := os.Open(source) if err != nil { log.Println("Can't open source file for reading: ", source) } sourceBuf := make([]byte, bufSize) targetFH, err := os.Open(target) if err != nil { log.Println("Can't open target file for reading: ", target) } targetBuf := make([]byte, bufSize) eofSource, eofTarget := false, false sourceBytesRead, targetBytesRead := 0, 0 for !eofSource && !eofTarget { if sourceBytesRead, err = sourceFH.Read(sourceBuf); err != nil { switch err { case io.EOF: eofSource = true case nil: default: log.Println(err) } } if targetBytesRead, err = targetFH.Read(targetBuf); err != nil { switch err { case io.EOF: eofTarget = true case nil: default: log.Println(err) } } if bytes.Compare(sourceBuf[:sourceBytesRead], targetBuf[:targetBytesRead]) != 0 { // fmt.Println("Files not same") return false } } // fmt.Println("files same") return true } func move(source string, target string) { err := os.Rename(source, target) if err != nil { // Renaming only works within same partition. // If different partitions, copy & delete source newerr := copy(source, target) if newerr != nil { log.Fatal("Copy failed ", source, " -> ", target, ":", newerr) } err := os.Remove(source) if err != nil { log.Fatal(err) } } } func copy(source string, target string) error { sourceFH, err := os.Open(source) if err != nil { log.Println("Can't open source file for reading: ", source) } targetFH, err := os.Create(target) if err != nil { log.Println("Can't open target file for reading: ", target) } _, copyerr := io.Copy(targetFH, sourceFH) return copyerr } func main() { cmdline() checkTargetDir() processSources() }