From 386cb274bdd3bbf53662420c0eebdd16a5ee9abc Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Thu, 9 Nov 2017 21:16:29 +0000 Subject: [PATCH] lib/model: Incremental block stats usage reporting GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4498 LGTM: calmh --- cmd/syncthing/gui.go | 4 +-- cmd/syncthing/mocked_model_test.go | 2 +- cmd/syncthing/usage_report.go | 6 ++-- lib/model/folder.go | 4 --- lib/model/model.go | 18 ++++++------ lib/model/rwfolder.go | 47 ++++++++++++------------------ 6 files changed, 34 insertions(+), 47 deletions(-) diff --git a/cmd/syncthing/gui.go b/cmd/syncthing/gui.go index c486c5b8..f9bbe0cb 100644 --- a/cmd/syncthing/gui.go +++ b/cmd/syncthing/gui.go @@ -104,7 +104,7 @@ type modelIntf interface { CurrentSequence(folder string) (int64, bool) RemoteSequence(folder string) (int64, bool) State(folder string) (string, time.Time, error) - UsageReportingStats(version int) map[string]interface{} + UsageReportingStats(version int, preview bool) map[string]interface{} } type configIntf interface { @@ -980,7 +980,7 @@ func (s *apiService) getReport(w http.ResponseWriter, r *http.Request) { if val, _ := strconv.Atoi(r.URL.Query().Get("version")); val > 0 { version = val } - sendJSON(w, reportData(s.cfg, s.model, s.connectionsService, version)) + sendJSON(w, reportData(s.cfg, s.model, s.connectionsService, version, true)) } func (s *apiService) getRandomString(w http.ResponseWriter, r *http.Request) { diff --git a/cmd/syncthing/mocked_model_test.go b/cmd/syncthing/mocked_model_test.go index 8f9acd08..6aff58cb 100644 --- a/cmd/syncthing/mocked_model_test.go +++ b/cmd/syncthing/mocked_model_test.go @@ -115,6 +115,6 @@ func (m *mockedModel) State(folder string) (string, time.Time, error) { return "", time.Time{}, nil } -func (m *mockedModel) UsageReportingStats(version int) map[string]interface{} { +func (m *mockedModel) UsageReportingStats(version int, preview bool) map[string]interface{} { return nil } diff --git a/cmd/syncthing/usage_report.go b/cmd/syncthing/usage_report.go index 5dea04d5..9a83932d 100644 --- a/cmd/syncthing/usage_report.go +++ b/cmd/syncthing/usage_report.go @@ -36,7 +36,7 @@ const usageReportVersion = 3 // reportData returns the data to be sent in a usage report. It's used in // various places, so not part of the usageReportingManager object. -func reportData(cfg configIntf, m modelIntf, connectionsService connectionsIntf, version int) map[string]interface{} { +func reportData(cfg configIntf, m modelIntf, connectionsService connectionsIntf, version int, preview bool) map[string]interface{} { opts := cfg.Options() res := make(map[string]interface{}) res["urVersion"] = version @@ -310,7 +310,7 @@ func reportData(cfg configIntf, m modelIntf, connectionsService connectionsIntf, res["guiStats"] = guiStatsInterface } - for key, value := range m.UsageReportingStats(version) { + for key, value := range m.UsageReportingStats(version, preview) { res[key] = value } @@ -338,7 +338,7 @@ func newUsageReportingService(cfg *config.Wrapper, model *model.Model, connectio } func (s *usageReportingService) sendUsageReport() error { - d := reportData(s.cfg, s.model, s.connectionsService, s.cfg.Options().URAccepted) + d := reportData(s.cfg, s.model, s.connectionsService, s.cfg.Options().URAccepted, false) var b bytes.Buffer json.NewEncoder(&b).Encode(d) diff --git a/lib/model/folder.go b/lib/model/folder.go index 6dc4c291..38da32fa 100644 --- a/lib/model/folder.go +++ b/lib/model/folder.go @@ -73,10 +73,6 @@ func (f *folder) Stop() { f.cancel() } -func (f *folder) BlockStats() map[string]int { - return nil -} - // CheckHealth checks the folder for common errors, updates the folder state // and returns the current folder error, or nil if the folder is healthy. func (f *folder) CheckHealth() error { diff --git a/lib/model/model.go b/lib/model/model.go index 2678a150..259971cc 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -54,7 +54,6 @@ type service interface { Scan(subs []string) error Serve() Stop() - BlockStats() map[string]int CheckHealth() error getState() (folderState, time.Time, error) @@ -397,19 +396,20 @@ func (m *Model) RestartFolder(cfg config.FolderConfiguration) { m.fmut.Unlock() } -func (m *Model) UsageReportingStats(version int) map[string]interface{} { +func (m *Model) UsageReportingStats(version int, preview bool) map[string]interface{} { stats := make(map[string]interface{}) if version >= 3 { // Block stats - m.fmut.Lock() - blockStats := make(map[string]int) - for _, folder := range m.folderRunners { - for k, v := range folder.BlockStats() { - blockStats[k] += v + blockStatsMut.Lock() + copyBlockStats := make(map[string]int) + for k, v := range blockStats { + copyBlockStats[k] = v + if !preview { + blockStats[k] = 0 } } - m.fmut.Unlock() - stats["blockStats"] = blockStats + blockStatsMut.Unlock() + stats["blockStats"] = copyBlockStats // Transport stats m.pmut.Lock() diff --git a/lib/model/rwfolder.go b/lib/model/rwfolder.go index 5ff00bc7..1640a34b 100644 --- a/lib/model/rwfolder.go +++ b/lib/model/rwfolder.go @@ -29,6 +29,11 @@ import ( "github.com/syncthing/syncthing/lib/weakhash" ) +var ( + blockStats = make(map[string]int) + blockStatsMut = sync.NewMutex() +) + func init() { folderFactories[config.FolderTypeSendReceive] = newSendReceiveFolder } @@ -91,9 +96,6 @@ type sendReceiveFolder struct { errors map[string]string // path -> error string errorsMut sync.Mutex - - blockStats map[string]int - blockStatsMut sync.Mutex } func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem) service { @@ -107,9 +109,6 @@ func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver vers pullScheduled: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a pull if we're busy when it comes. errorsMut: sync.NewMutex(), - - blockStats: make(map[string]int), - blockStatsMut: sync.NewMutex(), } f.configureCopiersAndPullers() @@ -886,10 +885,10 @@ func (f *sendReceiveFolder) renameFile(source, target protocol.FileInfo) { } if err == nil { - f.blockStatsMut.Lock() - f.blockStats["total"] += len(target.Blocks) - f.blockStats["renamed"] += len(target.Blocks) - f.blockStatsMut.Unlock() + blockStatsMut.Lock() + blockStats["total"] += len(target.Blocks) + blockStats["renamed"] += len(target.Blocks) + blockStatsMut.Unlock() // The file was renamed, so we have handled both the necessary delete // of the source and the creation of the target. Fix-up the metadata, @@ -1457,14 +1456,16 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState) { if err != nil { f.newError("finisher", state.file.Name, err) } else { - f.blockStatsMut.Lock() - f.blockStats["total"] += state.reused + state.copyTotal + state.pullTotal - f.blockStats["reused"] += state.reused - f.blockStats["pulled"] += state.pullTotal - f.blockStats["copyOrigin"] += state.copyOrigin - f.blockStats["copyOriginShifted"] += state.copyOriginShifted - f.blockStats["copyElsewhere"] += state.copyTotal - state.copyOrigin - f.blockStatsMut.Unlock() + blockStatsMut.Lock() + blockStats["total"] += state.reused + state.copyTotal + state.pullTotal + blockStats["reused"] += state.reused + blockStats["pulled"] += state.pullTotal + // copyOriginShifted is counted towards copyOrigin due to progress bar reasons + // for reporting reasons we want to separate these. + blockStats["copyOrigin"] += state.copyOrigin - state.copyOriginShifted + blockStats["copyOriginShifted"] += state.copyOriginShifted + blockStats["copyElsewhere"] += state.copyTotal - state.copyOrigin + blockStatsMut.Unlock() } events.Default.Log(events.ItemFinished, map[string]interface{}{ @@ -1482,16 +1483,6 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState) { } } -func (f *sendReceiveFolder) BlockStats() map[string]int { - f.blockStatsMut.Lock() - stats := make(map[string]int) - for k, v := range f.blockStats { - stats[k] = v - } - f.blockStatsMut.Unlock() - return stats -} - // Moves the given filename to the front of the job queue func (f *sendReceiveFolder) BringToFront(filename string) { f.queue.BringToFront(filename)