cmd/syncthing: Add more stats to usage reports (ref #3628)

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4347

lib/config/config.go

@@ -32,7 +32,7 @@ import (
 
 const (
 	OldestHandledVersion = 10
-	CurrentVersion       = 23
+	CurrentVersion       = 24
 	MaxRescanIntervalS   = 365 * 24 * 60 * 60
 )
 
@@ -326,6 +326,9 @@ func (cfg *Configuration) clean() error {
 	if cfg.Version == 22 {
 		convertV22V23(cfg)
 	}
+	if cfg.Version == 23 {
+		convertV23V24(cfg)
+	}
 
 	// Build a list of available devices
 	existingDevices := make(map[protocol.DeviceID]bool)
@@ -375,6 +378,12 @@ func (cfg *Configuration) clean() error {
 	return nil
 }
 
+func convertV23V24(cfg *Configuration) {
+	cfg.Options.URSeen = 2
+
+	cfg.Version = 24
+}
+
 func convertV22V23(cfg *Configuration) {
 	permBits := fs.FileMode(0777)
 	if runtime.GOOS == "windows" {

lib/config/config_test.go

@@ -200,6 +200,7 @@ func TestOverriddenValues(t *testing.T) {
 		ProgressUpdateIntervalS: 10,
 		LimitBandwidthInLan:     true,
 		MinHomeDiskFree:         Size{5.2, "%"},
+		URSeen:                  2,
 		URURL:                   "https://localhost/newdata",
 		URInitialDelayS:         800,
 		URPostInsecurely:        true,

lib/config/optionsconfiguration.go

@@ -112,6 +112,7 @@ type OptionsConfiguration struct {
 	NATRenewalM             int                     `xml:"natRenewalMinutes" json:"natRenewalMinutes" default:"30"`
 	NATTimeoutS             int                     `xml:"natTimeoutSeconds" json:"natTimeoutSeconds" default:"10"`
 	URAccepted              int                     `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
+	URSeen                  int                     `xml:"urSeen" json:"urSeen"`         // Report which the user has been prompted for.
 	URUniqueID              string                  `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on.
 	URURL                   string                  `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"`
 	URPostInsecurely        bool                    `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"` // For testing

lib/config/testdata/v24.xml

@@ -0,0 +1,16 @@
+<configuration version="22">
+    <folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
+        <filesystemType>basic</filesystemType>
+        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
+        <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
+        <minDiskFree unit="%">1</minDiskFree>
+        <maxConflicts>-1</maxConflicts>
+        <fsync>true</fsync>
+    </folder>
+    <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
+        <address>tcp://a</address>
+    </device>
+    <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
+        <address>tcp://b</address>
+    </device>
+</configuration>

lib/connections/kcp_listen.go

@@ -12,14 +12,16 @@ import (
 	"net/url"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/AudriusButkevicius/kcp-go"
 	"github.com/AudriusButkevicius/pfilter"
 	"github.com/ccding/go-stun/stun"
+	"github.com/xtaci/smux"
+
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/nat"
-	"github.com/xtaci/smux"
 )
 
 func init() {
@@ -38,6 +40,7 @@ type kcpListener struct {
 	stop    chan struct{}
 	conns   chan internalConn
 	factory listenerFactory
+	nat     atomic.Value
 
 	address *url.URL
 	err     error
@@ -183,6 +186,14 @@ func (t *kcpListener) Factory() listenerFactory {
 	return t.factory
 }
 
+func (t *kcpListener) NATType() string {
+	v := t.nat.Load().(stun.NATType)
+	if v == stun.NATUnknown || v == stun.NATError {
+		return "unknown"
+	}
+	return v.String()
+}
+
 func (t *kcpListener) stunRenewal(listener net.PacketConn) {
 	client := stun.NewClientWithConnection(listener)
 	client.SetSoftwareName("syncthing")
@@ -199,6 +210,7 @@ func (t *kcpListener) stunRenewal(listener net.PacketConn) {
 		if t.cfg.Options().StunKeepaliveS < 1 {
 			time.Sleep(time.Second)
 			oldType = stun.NATUnknown
+			t.nat.Store(stun.NATUnknown)
 			t.mut.Lock()
 			t.address = nil
 			t.mut.Unlock()
@@ -222,6 +234,7 @@ func (t *kcpListener) stunRenewal(listener net.PacketConn) {
 
 			if oldType != natType {
 				l.Infof("%s detected NAT type: %s", t.uri, natType)
+				t.nat.Store(natType)
 			}
 
 			for {
@@ -273,7 +286,7 @@ func (t *kcpListener) stunRenewal(listener net.PacketConn) {
 type kcpListenerFactory struct{}
 
 func (f *kcpListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener {
-	return &kcpListener{
+	l := &kcpListener{
 		uri:     fixupPort(uri, config.DefaultKCPPort),
 		cfg:     cfg,
 		tlsCfg:  tlsCfg,
@@ -281,6 +294,8 @@ func (f *kcpListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.
 		stop:    make(chan struct{}),
 		factory: f,
 	}
+	l.nat.Store(stun.NATUnknown)
+	return l
 }
 
 func (kcpListenerFactory) Enabled(cfg config.Configuration) bool {

lib/connections/relay_listen.go

@@ -171,6 +171,10 @@ func (t *relayListener) String() string {
 	return t.uri.String()
 }
 
+func (t *relayListener) NATType() string {
+	return "unknown"
+}
+
 type relayListenerFactory struct{}
 
 func (f *relayListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener {

lib/connections/service.go

@@ -574,6 +574,18 @@ func (s *Service) Status() map[string]interface{} {
 	return result
 }
 
+func (s *Service) NATType() string {
+	s.listenersMut.RLock()
+	defer s.listenersMut.RUnlock()
+	for _, listener := range s.listeners {
+		natType := listener.NATType()
+		if natType != "unknown" {
+			return natType
+		}
+	}
+	return "unknown"
+}
+
 func (s *Service) getDialerFactory(cfg config.Configuration, uri *url.URL) (dialerFactory, error) {
 	dialerFactory, ok := dialers[uri.Scheme]
 	if !ok {

lib/connections/structs.go

@@ -25,6 +25,7 @@ type Connection interface {
 	protocol.Connection
 	io.Closer
 	Type() string
+	Transport() string
 	RemoteAddr() net.Addr
 }
 
@@ -74,10 +75,27 @@ func (t connType) String() string {
 	}
 }
 
+func (t connType) Transport() string {
+	switch t {
+	case connTypeRelayClient, connTypeRelayServer:
+		return "relay"
+	case connTypeTCPClient, connTypeTCPServer:
+		return "tcp"
+	case connTypeKCPClient, connTypeKCPServer:
+		return "kcp"
+	default:
+		return "unknown"
+	}
+}
+
 func (c internalConn) Type() string {
 	return c.connType.String()
 }
 
+func (c internalConn) Transport() string {
+	return c.connType.Transport()
+}
+
 func (c internalConn) String() string {
 	return fmt.Sprintf("%s-%s/%s", c.LocalAddr(), c.RemoteAddr(), c.connType.String())
 }
@@ -116,6 +134,7 @@ type genericListener interface {
 	OnAddressesChanged(func(genericListener))
 	String() string
 	Factory() listenerFactory
+	NATType() string
 }
 
 type Model interface {

lib/connections/tcp_listen.go

@@ -176,6 +176,10 @@ func (t *tcpListener) Factory() listenerFactory {
 	return t.factory
 }
 
+func (t *tcpListener) NATType() string {
+	return "unknown"
+}
+
 type tcpListenerFactory struct{}
 
 func (f *tcpListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener {

lib/model/folder.go

@@ -60,6 +60,10 @@ func (f *folder) Jobs() ([]string, []string) {
 
 func (f *folder) BringToFront(string) {}
 
+func (f *folder) BlockStats() map[string]int {
+	return nil
+}
+
 func (f *folder) scanSubdirs(subDirs []string) error {
 	if err := f.model.internalScanFolderSubdirs(f.ctx, f.folderID, subDirs); err != nil {
 		// Potentially sets the error twice, once in the scanner just

lib/model/model.go

@@ -52,6 +52,7 @@ type service interface {
 	Scan(subs []string) error
 	Serve()
 	Stop()
+	BlockStats() map[string]int
 
 	getState() (folderState, time.Time, error)
 	setState(state folderState)
@@ -405,6 +406,105 @@ func (m *Model) RestartFolder(cfg config.FolderConfiguration) {
 	m.fmut.Unlock()
 }
 
+func (m *Model) UsageReportingStats(version int) map[string]interface{} {
+	stats := make(map[string]interface{})
+	if version >= 3 {
+		// Block stats
+		m.fmut.Lock()
+		blockStats := make(map[string]int)
+		for _, folder := range m.folderRunners {
+			for k, v := range folder.BlockStats() {
+				blockStats[k] += v
+			}
+		}
+		m.fmut.Unlock()
+		stats["blockStats"] = blockStats
+
+		// Transport stats
+		m.pmut.Lock()
+		transportStats := make(map[string]int)
+		for _, conn := range m.conn {
+			transportStats[conn.Transport()]++
+		}
+		m.pmut.Unlock()
+		stats["transportStats"] = transportStats
+
+		// Ignore stats
+		ignoreStats := map[string]int{
+			"lines":           0,
+			"inverts":         0,
+			"folded":          0,
+			"deletable":       0,
+			"rooted":          0,
+			"includes":        0,
+			"escapedIncludes": 0,
+			"doubleStars":     0,
+			"stars":           0,
+		}
+		var seenPrefix [3]bool
+		for folder := range m.cfg.Folders() {
+			lines, _, err := m.GetIgnores(folder)
+			if err != nil {
+				continue
+			}
+			ignoreStats["lines"] += len(lines)
+
+			for _, line := range lines {
+				// Allow prefixes to be specified in any order, but only once.
+				for {
+					if strings.HasPrefix(line, "!") && !seenPrefix[0] {
+						seenPrefix[0] = true
+						line = line[1:]
+						ignoreStats["inverts"] += 1
+					} else if strings.HasPrefix(line, "(?i)") && !seenPrefix[1] {
+						seenPrefix[1] = true
+						line = line[4:]
+						ignoreStats["folded"] += 1
+					} else if strings.HasPrefix(line, "(?d)") && !seenPrefix[2] {
+						seenPrefix[2] = true
+						line = line[4:]
+						ignoreStats["deletable"] += 1
+					} else {
+						seenPrefix[0] = false
+						seenPrefix[1] = false
+						seenPrefix[2] = false
+						break
+					}
+				}
+
+				// Noops, remove
+				if strings.HasSuffix(line, "**") {
+					line = line[:len(line)-2]
+				}
+				if strings.HasPrefix(line, "**/") {
+					line = line[3:]
+				}
+
+				if strings.HasPrefix(line, "/") {
+					ignoreStats["rooted"] += 1
+				} else if strings.HasPrefix(line, "#include ") {
+					ignoreStats["includes"] += 1
+					if strings.Contains(line, "..") {
+						ignoreStats["escapedIncludes"] += 1
+					}
+				}
+
+				if strings.Contains(line, "**") {
+					ignoreStats["doubleStars"] += 1
+					// Remove not to trip up star checks.
+					strings.Replace(line, "**", "", -1)
+				}
+
+				if strings.Contains(line, "*") {
+					ignoreStats["stars"] += 1
+				}
+			}
+		}
+		stats["ignoreStats"] = ignoreStats
+	}
+	return stats
+}
+
 type ConnectionInfo struct {
 	protocol.Statistics
 	Connected     bool
@@ -2449,6 +2549,7 @@ func (m *Model) CommitConfiguration(from, to config.Configuration) bool {
 	// Some options don't require restart as those components handle it fine
 	// by themselves.
 	from.Options.URAccepted = to.Options.URAccepted
+	from.Options.URSeen = to.Options.URSeen
 	from.Options.URUniqueID = to.Options.URUniqueID
 	from.Options.ListenAddresses = to.Options.ListenAddresses
 	from.Options.RelaysEnabled = to.Options.RelaysEnabled

lib/model/model_test.go

@@ -308,6 +308,9 @@ func (f *fakeConnection) RemoteAddr() net.Addr {
 func (f *fakeConnection) Type() string {
 	return "fake"
 }
+func (f *fakeConnection) Transport() string {
+	return "fake"
+}
 
 func (f *fakeConnection) DownloadProgress(folder string, updates []protocol.FileDownloadProgressUpdate) {
 	f.downloadProgressMessages = append(f.downloadProgressMessages, downloadProgressMessage{

lib/model/rwfolder.go

@@ -93,6 +93,9 @@ type sendReceiveFolder struct {
 
 	errors    map[string]string // path -> error string
 	errorsMut sync.Mutex
+
+	blockStats    map[string]int
+	blockStatsMut sync.Mutex
 }
 
 func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem) service {
@@ -107,6 +110,9 @@ func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver vers
 		remoteIndex: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a notification if we're busy doing a pull when it comes.
 
 		errorsMut: sync.NewMutex(),
+
+		blockStats:    make(map[string]int),
+		blockStatsMut: sync.NewMutex(),
 	}
 
 	f.configureCopiersAndPullers()
@@ -875,6 +881,11 @@ func (f *sendReceiveFolder) renameFile(source, target protocol.FileInfo) {
 	}
 
 	if err == nil {
+		f.blockStatsMut.Lock()
+		f.blockStats["total"] += len(target.Blocks)
+		f.blockStats["renamed"] += len(target.Blocks)
+		f.blockStatsMut.Unlock()
+
 		// The file was renamed, so we have handled both the necessary delete
 		// of the source and the creation of the target. Fix-up the metadata,
 		// and update the local index of the target file.
@@ -1443,6 +1454,15 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState) {
 			if err != nil {
 				l.Debugln("Puller: final:", err)
 				f.newError(state.file.Name, err)
+			} else {
+				f.blockStatsMut.Lock()
+				f.blockStats["total"] += state.reused + state.copyTotal + state.pullTotal
+				f.blockStats["reused"] += state.reused
+				f.blockStats["pulled"] += state.pullTotal
+				f.blockStats["copyOrigin"] += state.copyOrigin
+				f.blockStats["copyOriginShifted"] += state.copyOriginShifted
+				f.blockStats["copyElsewhere"] += state.copyTotal - state.copyOrigin
+				f.blockStatsMut.Unlock()
 			}
 			events.Default.Log(events.ItemFinished, map[string]interface{}{
 				"folder": f.folderID,
@@ -1459,6 +1479,16 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState) {
 	}
 }
 
+func (f *sendReceiveFolder) BlockStats() map[string]int {
+	f.blockStatsMut.Lock()
+	stats := make(map[string]int)
+	for k, v := range f.blockStats {
+		stats[k] = v
+	}
+	f.blockStatsMut.Unlock()
+	return stats
+}
+
 // Moves the given filename to the front of the job queue
 func (f *sendReceiveFolder) BringToFront(filename string) {
 	f.queue.BringToFront(filename)