diff --git a/src/changes.go b/src/changes.go index f23934a4..c3b7beec 100644 --- a/src/changes.go +++ b/src/changes.go @@ -26,6 +26,13 @@ import ( "github.com/odeke-em/log" ) +type destination int + +const ( + SelectSrc = 1 << iota + SelectDest +) + type dirList struct { remote *File local *File @@ -293,20 +300,19 @@ func merge(remotes, locals chan *File) (merged []*dirList) { return } -func reduceToSize(changes []*Change, fromSrc bool) (totalSize int64) { - totalSize = 0 +func reduceToSize(changes []*Change, destMask destination) (srcSize, destSize int64) { + fromSrc := (destMask & SelectSrc) != 0 + fromDest := (destMask & SelectDest) != 0 + for _, c := range changes { - if fromSrc { - if c.Src != nil { - totalSize += c.Src.Size - } - } else { - if c.Dest != nil { - totalSize += c.Dest.Size - } + if fromSrc && c.Src != nil { + srcSize += c.Src.Size + } + if fromDest && c.Dest != nil { + destSize += c.Dest.Size } } - return totalSize + return } func conflict(src, dest *File, index *config.Index, push bool) bool { @@ -379,13 +385,13 @@ func warnConflictsPersist(logy *log.Logger, conflicts []*Change) { } } -func printChanges(logy *log.Logger, changes []*Change, reduce bool) { - opMap := map[int]sizeCounter{} +func opChangeCount(changes []*Change) map[Operation]sizeCounter { + opMap := map[Operation]sizeCounter{} for _, c := range changes { op := c.Op() - if op != OpNone { - logy.Logln(c.Symbol(), c.Path) + if op == OpNone { + continue } counter := opMap[op] counter.count += 1 @@ -398,25 +404,39 @@ func printChanges(logy *log.Logger, changes []*Change, reduce bool) { opMap[op] = counter } + return opMap +} + +func previewChanges(logy *log.Logger, cl []*Change, reduce bool, opMap map[Operation]sizeCounter) { + for _, c := range cl { + op := c.Op() + if op != OpNone { + logy.Logln(c.Symbol(), c.Path) + } + } + if reduce { for op, counter := range opMap { if counter.count < 1 { continue } - _, name := opToString(op) + _, name := op.description() logy.Logf("%s %s\n", name, counter.String()) } } } -func printChangeList(logy *log.Logger, changes []*Change, noPrompt bool, noClobber bool) bool { +func printChangeList(logy *log.Logger, changes []*Change, noPrompt bool, noClobber bool) (bool, *map[Operation]sizeCounter) { if len(changes) == 0 { logy.Logln("Everything is up-to-date.") - return false + return false, nil } if noPrompt { - return true + return true, nil } - printChanges(logy, changes, true) - return promptForChanges() + + opMap := opChangeCount(changes) + previewChanges(logy, changes, true, opMap) + + return promptForChanges(), &opMap } diff --git a/src/diff.go b/src/diff.go index 3776352a..d3600340 100644 --- a/src/diff.go +++ b/src/diff.go @@ -55,7 +55,6 @@ func (g *Commands) Diff() (err error) { if dErr != nil { g.log.LogErrln(dErr) } - g.log.Logf("\n%s\n", Ruler) } return } @@ -96,20 +95,28 @@ func (g *Commands) perDiff(change *Change, diffProgPath, cwd string) (err error) change.Path, l.Size) } + mask := fileDifferences(r, l, g.opts.IgnoreChecksum) + if mask == DifferNone { + // No output when "no changes found" + return nil + } + typeName := "File" if l.IsDir { typeName = "Directory" } g.log.Logf("%s: %s\n", typeName, change.Path) - mask := fileDifferences(r, l, g.opts.IgnoreChecksum) + if modTimeDiffers(mask) { g.log.Logf("* %-15s %-40s\n* %-15s %-40s\n", "local:", toUTCString(l.ModTime), "remote:", toUTCString(r.ModTime)) - } else if mask == DifferNone { - // No output when "no changes found" - return nil + + if mask == DifferModTime { // No further change + return + } } + defer g.log.Logf("\n%s\n", Ruler) var frTmp, fl *os.File var blob io.ReadCloser diff --git a/src/misc.go b/src/misc.go index b0e7d13b..bfea23a2 100644 --- a/src/misc.go +++ b/src/misc.go @@ -251,6 +251,29 @@ func readCommentedFile(p, comment string) (clauses []string, err error) { return } +func chunkInt64(v int64) chan int { + var maxInt int + maxInt = 1<<31 - 1 + maxIntCast := int64(maxInt) + + chunks := make(chan int) + + go func() { + q, r := v/maxIntCast, v%maxIntCast + for i := int64(0); i < q; i += 1 { + chunks <- maxInt + } + + if r > 0 { + chunks <- int(r) + } + + close(chunks) + }() + + return chunks +} + func NonEmptyStrings(v []string) (splits []string) { for _, elem := range v { if elem != "" { diff --git a/src/pull.go b/src/pull.go index 4b7577d5..2652b8a2 100644 --- a/src/pull.go +++ b/src/pull.go @@ -69,12 +69,12 @@ func (g *Commands) Pull() (err error) { nonConflicts := *nonConflictsPtr - ok := printChangeList(g.log, nonConflicts, !g.opts.canPrompt(), g.opts.NoClobber) + ok, opMap := printChangeList(g.log, nonConflicts, !g.opts.canPrompt(), g.opts.NoClobber) if !ok { return } - return g.playPullChangeList(nonConflicts, g.opts.Exports) + return g.playPullChanges(nonConflicts, g.opts.Exports, opMap) } func (g *Commands) PullMatches() (err error) { @@ -117,11 +117,12 @@ func (g *Commands) PullMatches() (err error) { nonConflicts := *nonConflictsPtr - ok := printChangeList(g.log, nonConflicts, !g.opts.canPrompt(), g.opts.NoClobber) - if ok { - return g.playPullChangeList(nonConflicts, g.opts.Exports) + ok, opMap := printChangeList(g.log, nonConflicts, !g.opts.canPrompt(), g.opts.NoClobber) + if !ok { + return nil } - return nil + + return g.playPullChanges(nonConflicts, g.opts.Exports, opMap) } func (g *Commands) PullPiped() (err error) { @@ -152,12 +153,24 @@ func (g *Commands) PullPiped() (err error) { return } -func (g *Commands) playPullChangeList(cl []*Change, exports []string) (err error) { +func (g *Commands) playPullChanges(cl []*Change, exports []string, opMap *map[Operation]sizeCounter) (err error) { var next []*Change - pullSize := reduceToSize(cl, true) + if opMap == nil { + result := opChangeCount(cl) + opMap = &result + } + + totalSize := int64(0) + ops := *opMap + + for _, counter := range ops { + totalSize += counter.src + } - g.taskStart(int64(len(cl)) + pullSize) + g.taskStart(int64(len(cl)) + totalSize) + + defer close(g.rem.progressChan) // TODO: Only provide precedence ordering if all the other options are allowed // Currently noop on sorting by precedence @@ -221,6 +234,8 @@ func (g *Commands) localMod(wg *sync.WaitGroup, change *Change, exports []string destAbsPath := g.context.AbsPathOf(change.Path) + downloadPerformed := false + // Simple heuristic to avoid downloading all the // content yet it could just be a modTime difference mask := fileDifferences(change.Src, change.Dest, change.IgnoreChecksum) @@ -229,8 +244,19 @@ func (g *Commands) localMod(wg *sync.WaitGroup, change *Change, exports []string if err = g.download(change, exports); err != nil { return } + downloadPerformed = true } err = os.Chtimes(destAbsPath, change.Src.ModTime, change.Src.ModTime) + + // Update progress for the case in which you are only Chtime-ing + // since progress for downloaded files is already handled separately + if !downloadPerformed { + chunks := chunkInt64(change.Src.Size) + for n := range chunks { + g.rem.progressChan <- n + } + } + return } @@ -276,9 +302,19 @@ func (g *Commands) localAdd(wg *sync.WaitGroup, change *Change, exports []string } func (g *Commands) localDelete(wg *sync.WaitGroup, change *Change) (err error) { - defer g.taskDone() - defer wg.Done() - return os.RemoveAll(change.Dest.BlobAt) + defer func() { + if err == nil { + chunks := chunkInt64(change.Dest.Size) + for n := range chunks { + g.rem.progressChan <- n + } + } + + g.taskDone() + wg.Done() + }() + err = os.RemoveAll(change.Dest.BlobAt) + return } func touchFile(path string) (err error) { diff --git a/src/push.go b/src/push.go index ce858fb1..a53a6335 100644 --- a/src/push.go +++ b/src/push.go @@ -85,17 +85,19 @@ func (g *Commands) Push() (err error) { nonConflicts := *nonConflictsPtr - ok := printChangeList(g.log, nonConflicts, !g.opts.canPrompt(), g.opts.NoClobber) - if !ok { - return - } + pushSize, modSize := reduceToSize(cl, SelectDest|SelectSrc) - pushSize := reduceToSize(cl, true) + // TODO: Handle compensation from deletions and modifications + if false { + pushSize -= modSize + } - quotaStatus, qErr := g.QuotaStatus(pushSize) - if qErr != nil { - return qErr + // Warn about (near) quota exhaustion + quotaStatus, quotaErr := g.QuotaStatus(pushSize) + if quotaErr != nil { + return quotaErr } + unSafe := false switch quotaStatus { case AlmostExceeded: @@ -105,12 +107,19 @@ func (g *Commands) Push() (err error) { unSafe = true } if unSafe { - g.log.LogErrf(" projected size: %d (%d)\n", pushSize, prettyBytes(pushSize)) + g.log.LogErrf(" projected size: (%d) %s\n", pushSize, prettyBytes(pushSize)) if !promptForChanges() { return } } - return g.playPushChangeList(nonConflicts) + + // func printChangeList(logy *log.Logger, changes []*Change, noPrompt bool, noClobber bool) bool { + ok, opMap := printChangeList(g.log, nonConflicts, !g.opts.canPrompt(), g.opts.NoClobber) + if !ok { + return + } + + return g.playPushChanges(nonConflicts, opMap) } func (g *Commands) resolveConflicts(cl []*Change, push bool) (*[]*Change, *[]*Change) { @@ -178,7 +187,7 @@ func (g *Commands) PushPiped() (err error) { ignoreChecksum: g.opts.IgnoreChecksum, } - rem, rErr := g.rem.upsertByComparison(os.Stdin, &args) + rem, _, rErr := g.rem.upsertByComparison(os.Stdin, &args) if rErr != nil { g.log.LogErrf("%s: %v\n", relToRootPath, rErr) return rErr @@ -207,10 +216,20 @@ func (g *Commands) deserializeIndex(identifier string) *config.Index { return index } -func (g *Commands) playPushChangeList(cl []*Change) (err error) { - pushSize := reduceToSize(cl, true) +func (g *Commands) playPushChanges(cl []*Change, opMap *map[Operation]sizeCounter) (err error) { + + if opMap == nil { + result := opChangeCount(cl) + opMap = &result + } + + totalSize := int64(0) + ops := *opMap + for _, counter := range ops { + totalSize += counter.src + } - g.taskStart(int64(len(cl)) + pushSize) + g.taskStart(int64(len(cl)) + totalSize) defer close(g.rem.progressChan) @@ -331,9 +350,25 @@ func (g *Commands) indexAbsPath(fileId string) string { } func (g *Commands) remoteUntrash(change *Change) (err error) { - defer g.taskDone() + target := change.Src + defer func() { + g.taskAdd(target.Size) + g.taskDone() + }() + + err = g.rem.Untrash(target.Id) + if err != nil { + return + } + + index := target.ToIndex() + wErr := g.context.SerializeIndex(index, g.context.AbsPathOf("")) - return g.rem.Untrash(change.Src.Id) + // TODO: Should indexing errors be reported? + if wErr != nil { + g.log.LogErrf("serializeIndex %s: %v\n", target.Name, wErr) + } + return } func (g *Commands) remoteDelete(change *Change) (err error) { diff --git a/src/remote.go b/src/remote.go index 176ff83e..b4145adf 100644 --- a/src/remote.go +++ b/src/remote.go @@ -411,7 +411,7 @@ type upsertOpt struct { nonStatable bool } -func (r *Remote) upsertByComparison(body io.Reader, args *upsertOpt) (f *File, err error) { +func (r *Remote) upsertByComparison(body io.Reader, args *upsertOpt) (f *File, mediaInserted bool, err error) { uploaded := &drive.File{ // Must ensure that the path is prepared for a URL upload Title: urlToPath(args.src.Name, false), @@ -428,11 +428,13 @@ func (r *Remote) upsertByComparison(body io.Reader, args *upsertOpt) (f *File, e req := r.service.Files.Insert(uploaded) if !args.src.IsDir && body != nil { req = req.Media(body) + mediaInserted = true } if uploaded, err = req.Do(); err != nil { return } - return NewRemoteFile(uploaded), nil + f = NewRemoteFile(uploaded) + return } // update the existing @@ -459,14 +461,17 @@ func (r *Remote) upsertByComparison(body io.Reader, args *upsertOpt) (f *File, e if !args.src.IsDir { if args.dest == nil || args.nonStatable { req = req.Media(body) + mediaInserted = true } else if mask := fileDifferences(args.src, args.dest, args.ignoreChecksum); checksumDiffers(mask) { + mediaInserted = true req = req.Media(body) } } if uploaded, err = req.Do(); err != nil { return } - return NewRemoteFile(uploaded), nil + f = NewRemoteFile(uploaded) + return } func (r *Remote) rename(fileId, newTitle string) (*File, error) { @@ -527,7 +532,19 @@ func (r *Remote) UpsertByComparison(args *upsertOpt) (f *File, err error) { } }() - return r.upsertByComparison(bd, args) + mediaInserted := false + + f, mediaInserted, err = r.upsertByComparison(bd, args) + + // Case in which for example just Chtime-ing + if !mediaInserted && args.dest != nil { + chunks := chunkInt64(args.dest.Size) + for n := range chunks { + r.progressChan <- n + } + } + + return } func (r *Remote) findShared(p []string) (chan *File, error) { diff --git a/src/trash.go b/src/trash.go index edabc563..d3f36be6 100644 --- a/src/trash.go +++ b/src/trash.go @@ -121,11 +121,12 @@ func (g *Commands) trashByMatch(inTrash bool) error { } toTrash := !inTrash - ok := printChangeList(g.log, cl, !g.opts.canPrompt(), false) - if ok { - return g.playTrashChangeList(cl, toTrash) + ok, _ := printChangeList(g.log, cl, !g.opts.canPrompt(), false) + if !ok { + return nil } - return nil + + return g.playTrashChangeList(cl, toTrash) } func (g *Commands) TrashByMatch() error { @@ -147,16 +148,16 @@ func (g *Commands) reduce(args []string, toTrash bool) error { } } - ok := printChangeList(g.log, cl, !g.opts.canPrompt(), false) - if ok { - return g.playTrashChangeList(cl, toTrash) + ok, _ := printChangeList(g.log, cl, !g.opts.canPrompt(), false) + if !ok { + return nil } - return nil + return g.playTrashChangeList(cl, toTrash) } func (g *Commands) playTrashChangeList(cl []*Change, toTrash bool) (err error) { - trashSize := reduceToSize(cl, !toTrash) - g.taskStart(int64(len(cl)) + trashSize) + trashSize, unTrashSize := reduceToSize(cl, SelectDest|SelectSrc) + g.taskStart(int64(len(cl)) + trashSize + unTrashSize) var f = g.remoteUntrash if toTrash { diff --git a/src/types.go b/src/types.go index c15716d0..9592b7cb 100644 --- a/src/types.go +++ b/src/types.go @@ -25,6 +25,8 @@ import ( drive "github.com/odeke-em/google-api-go-client/drive/v2" ) +type Operation int + const ( OpNone = iota OpAdd @@ -34,8 +36,8 @@ const ( ) const ( - DifferNone = 1 << iota - DifferDirType + DifferNone = 0 + DifferDirType = 1 << iota DifferMd5Checksum DifferModTime DifferSize @@ -48,7 +50,7 @@ const ( // Arbitrary value. TODO: Get better definition of BigFileSize. var BigFileSize = int64(1024 * 1024 * 400) -var opPrecedence = map[int]int{ +var opPrecedence = map[Operation]int{ OpNone: 0, OpDelete: 1, OpAdd: 2, @@ -192,8 +194,8 @@ func (self *File) sameDirType(other *File) bool { return other != nil && self.IsDir == other.IsDir } -func opToString(op int) (string, string) { - switch op { +func (op *Operation) description() (symbol, info string) { + switch *op { case OpAdd: return "\033[32m+\033[0m", "Addition" case OpDelete: @@ -212,7 +214,8 @@ func (f *File) largeFile() bool { } func (c *Change) Symbol() string { - symbol, _ := opToString(c.Op()) + op := c.Op() + symbol, _ := op.description() return symbol } @@ -280,7 +283,9 @@ func fileDifferences(src, dest *File, ignoreChecksum bool) int { difference |= DifferDirType } - if !ignoreChecksum { + if ignoreChecksum && sizeDiffers(difference) { + difference |= DifferMd5Checksum + } else { // Only compute the checksum if the size differs if sizeDiffers(difference) || md5Checksum(src) != md5Checksum(dest) { difference |= DifferMd5Checksum @@ -293,7 +298,7 @@ func sameFileTillChecksum(src, dest *File, ignoreChecksum bool) bool { return fileDifferences(src, dest, ignoreChecksum) == DifferNone } -func (c *Change) op() int { +func (c *Change) op() Operation { if c.Src == nil && c.Dest == nil { return OpNone } @@ -325,7 +330,7 @@ func (c *Change) op() int { return OpNone } -func (c *Change) Op() int { +func (c *Change) Op() Operation { op := c.op() if c.Force { if op == OpModConflict {