perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

cmd/syncthing, lib/api: Separate api/gui into own package (ref #4085) (#5529)

Browse Source 
* cmd/syncthing, lib/gui: Separate gui into own package (ref #4085)

* fix tests

* Don't use main as interface name (make old go happy)

* gui->api

* don't leak state via locations and use in-tree config

* let api (un-)subscribe to config

* interface naming and exporting

* lib/ur

* fix tests and lib/foldersummary

* shorter URVersion and ur debug fix

* review

* model.JsonCompletion(FolderCompletion) -> FolderCompletion.Map()

* rename debug facility https -> api

* folder summaries in model

* disassociate unrelated constants

* fix merge fail

* missing id assignement
This commit is contained in:
Simon Frei
2019-03-26 20:53:58 +01:00
committed by Audrius Butkevicius
parent d4e81fff8a
commit b50039a920
38 changed files with 728 additions and 360 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/syncthing/debug.go
View File

@@ -14,15 +14,9 @@ import (
)
var (
l = logger.DefaultLogger.NewFacility("main", "Main package")
httpl = logger.DefaultLogger.NewFacility("http", "REST API")
l = logger.DefaultLogger.NewFacility("main", "Main package")
)
func shouldDebugHTTP() bool {
return l.ShouldDebug("http")
}
func init() {
l.SetDebug("main", strings.Contains(os.Getenv("STTRACE"), "main") || os.Getenv("STTRACE") == "all")
l.SetDebug("http", strings.Contains(os.Getenv("STTRACE"), "http") || os.Getenv("STTRACE") == "all")
}

1730
cmd/syncthing/gui.go
View File

File diff suppressed because it is too large Load Diff

175
cmd/syncthing/gui_auth.go
View File

@@ -1,175 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bytes"
"crypto/tls"
"encoding/base64"
"fmt"
"net/http"
"strings"
"time"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sync"
"golang.org/x/crypto/bcrypt"
ldap "gopkg.in/ldap.v2"
)
var (
sessions = make(map[string]bool)
sessionsMut = sync.NewMutex()
)
func emitLoginAttempt(success bool, username string) {
events.Default.Log(events.LoginAttempt, map[string]interface{}{
"success": success,
"username": username,
})
}
func basicAuthAndSessionMiddleware(cookieName string, guiCfg config.GUIConfiguration, ldapCfg config.LDAPConfiguration, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if guiCfg.IsValidAPIKey(r.Header.Get("X-API-Key")) {
next.ServeHTTP(w, r)
return
}
cookie, err := r.Cookie(cookieName)
if err == nil && cookie != nil {
sessionsMut.Lock()
_, ok := sessions[cookie.Value]
sessionsMut.Unlock()
if ok {
next.ServeHTTP(w, r)
return
}
}
httpl.Debugln("Sessionless HTTP request with authentication; this is expensive.")
error := func() {
time.Sleep(time.Duration(rand.Intn(100)+100) * time.Millisecond)
w.Header().Set("WWW-Authenticate", "Basic realm=\"Authorization Required\"")
http.Error(w, "Not Authorized", http.StatusUnauthorized)
}
hdr := r.Header.Get("Authorization")
if !strings.HasPrefix(hdr, "Basic ") {
error()
return
}
hdr = hdr[6:]
bs, err := base64.StdEncoding.DecodeString(hdr)
if err != nil {
error()
return
}
fields := bytes.SplitN(bs, []byte(":"), 2)
if len(fields) != 2 {
error()
return
}
username := string(fields[0])
password := string(fields[1])
authOk := auth(username, password, guiCfg, ldapCfg)
if !authOk {
usernameIso := string(iso88591ToUTF8([]byte(username)))
passwordIso := string(iso88591ToUTF8([]byte(password)))
authOk = auth(usernameIso, passwordIso, guiCfg, ldapCfg)
if authOk {
username = usernameIso
}
}
if !authOk {
emitLoginAttempt(false, username)
error()
return
}
sessionid := rand.String(32)
sessionsMut.Lock()
sessions[sessionid] = true
sessionsMut.Unlock()
http.SetCookie(w, &http.Cookie{
Name: cookieName,
Value: sessionid,
MaxAge: 0,
})
emitLoginAttempt(true, username)
next.ServeHTTP(w, r)
})
}
func auth(username string, password string, guiCfg config.GUIConfiguration, ldapCfg config.LDAPConfiguration) bool {
if guiCfg.AuthMode == config.AuthModeLDAP {
return authLDAP(username, password, ldapCfg)
} else {
return authStatic(username, password, guiCfg.User, guiCfg.Password)
}
}
func authStatic(username string, password string, configUser string, configPassword string) bool {
configPasswordBytes := []byte(configPassword)
passwordBytes := []byte(password)
return bcrypt.CompareHashAndPassword(configPasswordBytes, passwordBytes) == nil && username == configUser
}
func authLDAP(username string, password string, cfg config.LDAPConfiguration) bool {
address := cfg.Address
var connection *ldap.Conn
var err error
if cfg.Transport == config.LDAPTransportTLS {
connection, err = ldap.DialTLS("tcp", address, &tls.Config{InsecureSkipVerify: cfg.InsecureSkipVerify})
} else {
connection, err = ldap.Dial("tcp", address)
}
if err != nil {
l.Warnln("LDAP Dial:", err)
return false
}
if cfg.Transport == config.LDAPTransportStartTLS {
err = connection.StartTLS(&tls.Config{InsecureSkipVerify: cfg.InsecureSkipVerify})
if err != nil {
l.Warnln("LDAP Start TLS:", err)
return false
}
}
defer connection.Close()
err = connection.Bind(fmt.Sprintf(cfg.BindDN, username), password)
if err != nil {
l.Warnln("LDAP Bind:", err)
return false
}
return true
}
// Convert an ISO-8859-1 encoded byte string to UTF-8. Works by the
// principle that ISO-8859-1 bytes are equivalent to unicode code points,
// that a rune slice is a list of code points, and that stringifying a slice
// of runes generates UTF-8 in Go.
func iso88591ToUTF8(s []byte) []byte {
runes := make([]rune, len(s))
for i := range s {
runes[i] = rune(s[i])
}
return []byte(string(runes))
}

40
cmd/syncthing/gui_auth_test.go
View File

@@ -1,40 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"testing"
"golang.org/x/crypto/bcrypt"
)
var passwordHashBytes []byte
func init() {
passwordHashBytes, _ = bcrypt.GenerateFromPassword([]byte("pass"), 0)
}
func TestStaticAuthOK(t *testing.T) {
ok := authStatic("user", "pass", "user", string(passwordHashBytes))
if !ok {
t.Fatalf("should pass auth")
}
}
func TestSimpleAuthUsernameFail(t *testing.T) {
ok := authStatic("userWRONG", "pass", "user", string(passwordHashBytes))
if ok {
t.Fatalf("should fail auth")
}
}
func TestStaticAuthPasswordFail(t *testing.T) {
ok := authStatic("user", "passWRONG", "user", string(passwordHashBytes))
if ok {
t.Fatalf("should fail auth")
}
}

143
cmd/syncthing/gui_csrf.go
View File

@@ -1,143 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bufio"
"fmt"
"net/http"
"os"
"strings"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/locations"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sync"
)
// csrfTokens is a list of valid tokens. It is sorted so that the most
// recently used token is first in the list. New tokens are added to the front
// of the list (as it is the most recently used at that time). The list is
// pruned to a maximum of maxCsrfTokens, throwing away the least recently used
// tokens.
var csrfTokens []string
var csrfMut = sync.NewMutex()
const maxCsrfTokens = 25
// Check for CSRF token on /rest/ URLs. If a correct one is not given, reject
// the request with 403. For / and /index.html, set a new CSRF cookie if none
// is currently set.
func csrfMiddleware(unique string, prefix string, cfg config.GUIConfiguration, next http.Handler) http.Handler {
loadCsrfTokens()
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Allow requests carrying a valid API key
if cfg.IsValidAPIKey(r.Header.Get("X-API-Key")) {
// Set the access-control-allow-origin header for CORS requests
// since a valid API key has been provided
w.Header().Add("Access-Control-Allow-Origin", "*")
next.ServeHTTP(w, r)
return
}
if strings.HasPrefix(r.URL.Path, "/rest/debug") {
// Debugging functions are only available when explicitly
// enabled, and can be accessed without a CSRF token
next.ServeHTTP(w, r)
return
}
// Allow requests for anything not under the protected path prefix,
// and set a CSRF cookie if there isn't already a valid one.
if !strings.HasPrefix(r.URL.Path, prefix) {
cookie, err := r.Cookie("CSRF-Token-" + unique)
if err != nil || !validCsrfToken(cookie.Value) {
httpl.Debugln("new CSRF cookie in response to request for", r.URL)
cookie = &http.Cookie{
Name: "CSRF-Token-" + unique,
Value: newCsrfToken(),
}
http.SetCookie(w, cookie)
}
next.ServeHTTP(w, r)
return
}
// Verify the CSRF token
token := r.Header.Get("X-CSRF-Token-" + unique)
if !validCsrfToken(token) {
http.Error(w, "CSRF Error", 403)
return
}
next.ServeHTTP(w, r)
})
}
func validCsrfToken(token string) bool {
csrfMut.Lock()
defer csrfMut.Unlock()
for i, t := range csrfTokens {
if t == token {
if i > 0 {
// Move this token to the head of the list. Copy the tokens at
// the front one step to the right and then replace the token
// at the head.
copy(csrfTokens[1:], csrfTokens[:i+1])
csrfTokens[0] = token
}
return true
}
}
return false
}
func newCsrfToken() string {
token := rand.String(32)
csrfMut.Lock()
csrfTokens = append([]string{token}, csrfTokens...)
if len(csrfTokens) > maxCsrfTokens {
csrfTokens = csrfTokens[:maxCsrfTokens]
}
defer csrfMut.Unlock()
saveCsrfTokens()
return token
}
func saveCsrfTokens() {
// We're ignoring errors in here. It's not super critical and there's
// nothing relevant we can do about them anyway...
name := locations.Get(locations.CsrfTokens)
f, err := osutil.CreateAtomic(name)
if err != nil {
return
}
for _, t := range csrfTokens {
fmt.Fprintln(f, t)
}
f.Close()
}
func loadCsrfTokens() {
f, err := os.Open(locations.Get(locations.CsrfTokens))
if err != nil {
return
}
defer f.Close()
s := bufio.NewScanner(f)
for s.Scan() {
csrfTokens = append(csrfTokens, s.Text())
}
}

207
cmd/syncthing/gui_statics.go
View File

@@ -1,207 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bytes"
"compress/gzip"
"fmt"
"io/ioutil"
"mime"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/syncthing/syncthing/lib/auto"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/sync"
)
type staticsServer struct {
assetDir string
assets map[string][]byte
availableThemes []string
mut sync.RWMutex
theme string
}
func newStaticsServer(theme, assetDir string) *staticsServer {
s := &staticsServer{
assetDir: assetDir,
assets: auto.Assets(),
mut: sync.NewRWMutex(),
theme: theme,
}
seen := make(map[string]struct{})
// Load themes from compiled in assets.
for file := range auto.Assets() {
theme := strings.Split(file, "/")[0]
if _, ok := seen[theme]; !ok {
seen[theme] = struct{}{}
s.availableThemes = append(s.availableThemes, theme)
}
}
if assetDir != "" {
// Load any extra themes from the asset override dir.
for _, dir := range dirNames(assetDir) {
if _, ok := seen[dir]; !ok {
seen[dir] = struct{}{}
s.availableThemes = append(s.availableThemes, dir)
}
}
}
return s
}
func (s *staticsServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/themes.json":
s.serveThemes(w, r)
default:
s.serveAsset(w, r)
}
}
func (s *staticsServer) serveAsset(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "no-cache, must-revalidate")
file := r.URL.Path
if file[0] == '/' {
file = file[1:]
}
if len(file) == 0 {
file = "index.html"
}
s.mut.RLock()
theme := s.theme
s.mut.RUnlock()
// Check for an override for the current theme.
if s.assetDir != "" {
p := filepath.Join(s.assetDir, theme, filepath.FromSlash(file))
if _, err := os.Stat(p); err == nil {
mtype := s.mimeTypeForFile(file)
if len(mtype) != 0 {
w.Header().Set("Content-Type", mtype)
}
http.ServeFile(w, r, p)
return
}
}
// Check for a compiled in asset for the current theme.
bs, ok := s.assets[theme+"/"+file]
if !ok {
// Check for an overridden default asset.
if s.assetDir != "" {
p := filepath.Join(s.assetDir, config.DefaultTheme, filepath.FromSlash(file))
if _, err := os.Stat(p); err == nil {
mtype := s.mimeTypeForFile(file)
if len(mtype) != 0 {
w.Header().Set("Content-Type", mtype)
}
http.ServeFile(w, r, p)
return
}
}
// Check for a compiled in default asset.
bs, ok = s.assets[config.DefaultTheme+"/"+file]
if !ok {
http.NotFound(w, r)
return
}
}
etag := fmt.Sprintf("%d", auto.Generated)
modified := time.Unix(auto.Generated, 0).UTC()
w.Header().Set("Last-Modified", modified.Format(http.TimeFormat))
w.Header().Set("Etag", etag)
if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil {
if modified.Equal(t) || modified.Before(t) {
w.WriteHeader(http.StatusNotModified)
return
}
}
if match := r.Header.Get("If-None-Match"); match != "" {
if strings.Contains(match, etag) {
w.WriteHeader(http.StatusNotModified)
return
}
}
mtype := s.mimeTypeForFile(file)
if len(mtype) != 0 {
w.Header().Set("Content-Type", mtype)
}
if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
w.Header().Set("Content-Encoding", "gzip")
} else {
// ungzip if browser not send gzip accepted header
var gr *gzip.Reader
gr, _ = gzip.NewReader(bytes.NewReader(bs))
bs, _ = ioutil.ReadAll(gr)
gr.Close()
}
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(bs)))
w.Write(bs)
}
func (s *staticsServer) serveThemes(w http.ResponseWriter, r *http.Request) {
sendJSON(w, map[string][]string{
"themes": s.availableThemes,
})
}
func (s *staticsServer) mimeTypeForFile(file string) string {
// We use a built in table of the common types since the system
// TypeByExtension might be unreliable. But if we don't know, we delegate
// to the system. All our files are UTF-8.
ext := filepath.Ext(file)
switch ext {
case ".htm", ".html":
return "text/html; charset=utf-8"
case ".css":
return "text/css; charset=utf-8"
case ".js":
return "application/javascript; charset=utf-8"
case ".json":
return "application/json; charset=utf-8"
case ".png":
return "image/png"
case ".ttf":
return "application/x-font-ttf"
case ".woff":
return "application/x-font-woff"
case ".svg":
return "image/svg+xml; charset=utf-8"
default:
return mime.TypeByExtension(ext)
}
}
func (s *staticsServer) setTheme(theme string) {
s.mut.Lock()
s.theme = theme
s.mut.Unlock()
}
func (s *staticsServer) String() string {
return fmt.Sprintf("staticsServer@%p", s)
}

1058
cmd/syncthing/gui_test.go
View File

File diff suppressed because it is too large Load Diff

54
cmd/syncthing/main.go
View File

@@ -29,6 +29,7 @@ import (
"syscall"
"time"
"github.com/syncthing/syncthing/lib/api"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/connections"
@@ -46,6 +47,7 @@ import (
"github.com/syncthing/syncthing/lib/sha256"
"github.com/syncthing/syncthing/lib/tlsutil"
"github.com/syncthing/syncthing/lib/upgrade"
"github.com/syncthing/syncthing/lib/ur"
"github.com/pkg/errors"
"github.com/thejerf/suture"
@@ -62,7 +64,6 @@ const (
const (
bepProtocolName = "bep/1.0"
tlsDefaultCommonName = "syncthing"
defaultEventTimeout = time.Minute
maxSystemErrors = 5
initialSystemLog = 10
maxSystemLog = 250
@@ -263,6 +264,7 @@ func parseCommandLineOptions() RuntimeOptions {
return options
}
// exiter implements api.Controller
type exiter struct {
stop chan int
}
@@ -287,7 +289,7 @@ func (e *exiter) waitForExit() int {
return <-e.stop
}
var exit = exiter{make(chan int)}
var exit = &exiter{make(chan int)}
func main() {
options := parseCommandLineOptions()
@@ -621,8 +623,8 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
// Event subscription for the API; must start early to catch the early
// events. The LocalChangeDetected event might overwhelm the event
// receiver in some situations so we will not subscribe to it here.
defaultSub := events.NewBufferedSubscription(events.Default.Subscribe(defaultEventMask), eventSubBufferSize)
diskSub := events.NewBufferedSubscription(events.Default.Subscribe(diskEventMask), eventSubBufferSize)
defaultSub := events.NewBufferedSubscription(events.Default.Subscribe(api.DefaultEventMask), api.EventSubBufferSize)
diskSub := events.NewBufferedSubscription(events.Default.Subscribe(api.DiskEventMask), api.EventSubBufferSize)
if len(os.Getenv("GOMAXPROCS")) == 0 {
runtime.GOMAXPROCS(runtime.NumCPU())
@@ -692,7 +694,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
}()
}
perf := cpuBench(3, 150*time.Millisecond, true)
perf := ur.CpuBench(3, 150*time.Millisecond, true)
l.Infof("Hashing performance is %.02f MB/s", perf)
dbFile := locations.Get(locations.Database)
@@ -832,10 +834,6 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
}
}
// GUI
setupGUI(mainService, cfg, m, defaultSub, diskSub, cachedDiscovery, connectionsService, errors, systemLog, runtimeOptions)
if runtimeOptions.cpuProfile {
f, err := os.Create(fmt.Sprintf("cpu-%d.pprof", os.Getpid()))
if err != nil {
@@ -848,20 +846,12 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
}
}
myDev, _ := cfg.Device(myID)
l.Infof(`My name is "%v"`, myDev.Name)
for _, device := range cfg.Devices() {
if device.DeviceID != myID {
l.Infof(`Device %s is "%v" at %v`, device.DeviceID, device.Name, device.Addresses)
}
}
// Candidate builds always run with usage reporting.
if opts := cfg.Options(); build.IsCandidate {
l.Infoln("Anonymous usage reporting is always enabled for candidate releases.")
if opts.URAccepted != usageReportVersion {
opts.URAccepted = usageReportVersion
if opts.URAccepted != ur.Version {
opts.URAccepted = ur.Version
cfg.SetOptions(opts)
cfg.Save()
// Unique ID will be set and config saved below if necessary.
@@ -875,9 +865,21 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
cfg.Save()
}
usageReportingSvc := newUsageReportingService(cfg, m, connectionsService)
usageReportingSvc := ur.New(cfg, m, connectionsService, noUpgradeFromEnv)
mainService.Add(usageReportingSvc)
// GUI
setupGUI(mainService, cfg, m, defaultSub, diskSub, cachedDiscovery, connectionsService, usageReportingSvc, errors, systemLog, runtimeOptions)
myDev, _ := cfg.Device(myID)
l.Infof(`My name is "%v"`, myDev.Name)
for _, device := range cfg.Devices() {
if device.DeviceID != myID {
l.Infof(`Device %s is "%v" at %v`, device.DeviceID, device.Name, device.Addresses)
}
}
if opts := cfg.Options(); opts.RestartOnWakeup {
go standbyMonitor()
}
@@ -1069,7 +1071,7 @@ func startAuditing(mainService *suture.Supervisor, auditFile string) {
l.Infoln("Audit log in", auditDest)
}
func setupGUI(mainService *suture.Supervisor, cfg config.Wrapper, m model.Model, defaultSub, diskSub events.BufferedSubscription, discoverer discover.CachingMux, connectionsService connections.Service, errors, systemLog logger.Recorder, runtimeOptions RuntimeOptions) {
func setupGUI(mainService *suture.Supervisor, cfg config.Wrapper, m model.Model, defaultSub, diskSub events.BufferedSubscription, discoverer discover.CachingMux, connectionsService connections.Service, urService *ur.Service, errors, systemLog logger.Recorder, runtimeOptions RuntimeOptions) {
guiCfg := cfg.GUI()
if !guiCfg.Enabled {
@@ -1083,11 +1085,13 @@ func setupGUI(mainService *suture.Supervisor, cfg config.Wrapper, m model.Model,
cpu := newCPUService()
mainService.Add(cpu)
api := newAPIService(myID, cfg, locations.Get(locations.HTTPSCertFile), locations.Get(locations.HTTPSKeyFile), runtimeOptions.assetDir, m, defaultSub, diskSub, discoverer, connectionsService, errors, systemLog, cpu)
cfg.Subscribe(api)
mainService.Add(api)
summaryService := model.NewFolderSummaryService(cfg, m, myID)
mainService.Add(summaryService)
if err := api.WaitForStart(); err != nil {
apiSvc := api.New(myID, cfg, runtimeOptions.assetDir, tlsDefaultCommonName, m, defaultSub, diskSub, discoverer, connectionsService, urService, summaryService, errors, systemLog, cpu, exit, noUpgradeFromEnv)
mainService.Add(apiSvc)
if err := apiSvc.WaitForStart(); err != nil {
l.Warnln("Failed starting API:", err)
os.Exit(exitError)
}

31
cmd/syncthing/memsize_darwin.go
View File

@@ -1,31 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"errors"
"os/exec"
"strconv"
"strings"
)
func memorySize() (int64, error) {
cmd := exec.Command("sysctl", "hw.memsize")
out, err := cmd.Output()
if err != nil {
return 0, err
}
fs := strings.Fields(string(out))
if len(fs) != 2 {
return 0, errors.New("sysctl parse error")
}
bytes, err := strconv.ParseInt(fs[1], 10, 64)
if err != nil {
return 0, err
}
return bytes, nil
}

39
cmd/syncthing/memsize_linux.go
View File

@@ -1,39 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bufio"
"errors"
"os"
"strconv"
"strings"
)
func memorySize() (int64, error) {
f, err := os.Open("/proc/meminfo")
if err != nil {
return 0, err
}
s := bufio.NewScanner(f)
if !s.Scan() {
return 0, errors.New("/proc/meminfo parse error 1")
}
l := s.Text()
fs := strings.Fields(l)
if len(fs) != 3 || fs[2] != "kB" {
return 0, errors.New("/proc/meminfo parse error 2")
}
kb, err := strconv.ParseInt(fs[1], 10, 64)
if err != nil {
return 0, err
}
return kb * 1024, nil
}

31
cmd/syncthing/memsize_netbsd.go
View File

@@ -1,31 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"errors"
"os/exec"
"strconv"
"strings"
)
func memorySize() (int64, error) {
cmd := exec.Command("/sbin/sysctl", "hw.physmem64")
out, err := cmd.Output()
if err != nil {
return 0, err
}
fs := strings.Fields(string(out))
if len(fs) != 3 {
return 0, errors.New("sysctl parse error")
}
bytes, err := strconv.ParseInt(fs[2], 10, 64)
if err != nil {
return 0, err
}
return bytes, nil
}

28
cmd/syncthing/memsize_solaris.go
View File

@@ -1,28 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build solaris
package main
import (
"os/exec"
"strconv"
)
func memorySize() (int64, error) {
cmd := exec.Command("prtconf", "-m")
out, err := cmd.CombinedOutput()
if err != nil {
return 0, err
}
mb, err := strconv.ParseInt(string(out), 10, 64)
if err != nil {
return 0, err
}
return mb * 1024 * 1024, nil
}

15
cmd/syncthing/memsize_unimpl.go
View File

@@ -1,15 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build freebsd openbsd dragonfly
package main
import "errors"
func memorySize() (int64, error) {
return 0, errors.New("not implemented")
}

30
cmd/syncthing/memsize_windows.go
View File

@@ -1,30 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"encoding/binary"
"syscall"
"unsafe"
)
var (
kernel32, _ = syscall.LoadLibrary("kernel32.dll")
globalMemoryStatusEx, _ = syscall.GetProcAddress(kernel32, "GlobalMemoryStatusEx")
)
func memorySize() (int64, error) {
var memoryStatusEx [64]byte
binary.LittleEndian.PutUint32(memoryStatusEx[:], 64)
ret, _, callErr := syscall.Syscall(uintptr(globalMemoryStatusEx), 1, uintptr(unsafe.Pointer(&memoryStatusEx[0])), 0, 0)
if ret == 0 {
return 0, callErr
}
return int64(binary.LittleEndian.Uint64(memoryStatusEx[8:])), nil
}

127
cmd/syncthing/mocked_config_test.go
View File

@@ -1,127 +0,0 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/util"
)
type mockedConfig struct {
gui config.GUIConfiguration
}
func (c *mockedConfig) GUI() config.GUIConfiguration {
return c.gui
}
func (c *mockedConfig) ListenAddresses() []string {
return nil
}
func (c *mockedConfig) LDAP() config.LDAPConfiguration {
return config.LDAPConfiguration{}
}
func (c *mockedConfig) RawCopy() config.Configuration {
cfg := config.Configuration{}
util.SetDefaults(&cfg.Options)
return cfg
}
func (c *mockedConfig) Options() config.OptionsConfiguration {
return config.OptionsConfiguration{}
}
func (c *mockedConfig) Replace(cfg config.Configuration) (config.Waiter, error) {
return noopWaiter{}, nil
}
func (c *mockedConfig) Subscribe(cm config.Committer) {}
func (c *mockedConfig) Unsubscribe(cm config.Committer) {}
func (c *mockedConfig) Folders() map[string]config.FolderConfiguration {
return nil
}
func (c *mockedConfig) Devices() map[protocol.DeviceID]config.DeviceConfiguration {
return nil
}
func (c *mockedConfig) SetDevice(config.DeviceConfiguration) (config.Waiter, error) {
return noopWaiter{}, nil
}
func (c *mockedConfig) SetDevices([]config.DeviceConfiguration) (config.Waiter, error) {
return noopWaiter{}, nil
}
func (c *mockedConfig) Save() error {
return nil
}
func (c *mockedConfig) RequiresRestart() bool {
return false
}
func (c *mockedConfig) AddOrUpdatePendingDevice(device protocol.DeviceID, name, address string) {}
func (c *mockedConfig) AddOrUpdatePendingFolder(id, label string, device protocol.DeviceID) {}
func (m *mockedConfig) MyName() string {
return ""
}
func (m *mockedConfig) ConfigPath() string {
return ""
}
func (m *mockedConfig) SetGUI(gui config.GUIConfiguration) (config.Waiter, error) {
return noopWaiter{}, nil
}
func (m *mockedConfig) SetOptions(opts config.OptionsConfiguration) (config.Waiter, error) {
return noopWaiter{}, nil
}
func (m *mockedConfig) Folder(id string) (config.FolderConfiguration, bool) {
return config.FolderConfiguration{}, false
}
func (m *mockedConfig) FolderList() []config.FolderConfiguration {
return nil
}
func (m *mockedConfig) SetFolder(fld config.FolderConfiguration) (config.Waiter, error) {
return noopWaiter{}, nil
}
func (m *mockedConfig) Device(id protocol.DeviceID) (config.DeviceConfiguration, bool) {
return config.DeviceConfiguration{}, false
}
func (m *mockedConfig) RemoveDevice(id protocol.DeviceID) (config.Waiter, error) {
return noopWaiter{}, nil
}
func (m *mockedConfig) IgnoredDevice(id protocol.DeviceID) bool {
return false
}
func (m *mockedConfig) IgnoredFolder(device protocol.DeviceID, folder string) bool {
return false
}
func (m *mockedConfig) GlobalDiscoveryServers() []string {
return nil
}
type noopWaiter struct{}
func (noopWaiter) Wait() {}

21
cmd/syncthing/mocked_connections_test.go
View File

@@ -1,21 +0,0 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
type mockedConnections struct{}
func (m *mockedConnections) Status() map[string]interface{} {
return nil
}
func (m *mockedConnections) NATType() string {
return ""
}
func (m *mockedConnections) Serve() {}
func (m *mockedConnections) Stop() {}

13
cmd/syncthing/mocked_cpuusage_test.go
View File

@@ -1,13 +0,0 @@
// Copyright (C) 2017 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
type mockedCPUService struct{}
func (*mockedCPUService) Rate() float64 {
return 42
}

52
cmd/syncthing/mocked_discovery_test.go
View File

@@ -1,52 +0,0 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"time"
"github.com/syncthing/syncthing/lib/discover"
"github.com/syncthing/syncthing/lib/protocol"
)
type mockedCachingMux struct{}
// from suture.Service
func (m *mockedCachingMux) Serve() {
select {}
}
func (m *mockedCachingMux) Stop() {
}
// from events.Finder
func (m *mockedCachingMux) Lookup(deviceID protocol.DeviceID) (direct []string, err error) {
return nil, nil
}
func (m *mockedCachingMux) Error() error {
return nil
}
func (m *mockedCachingMux) String() string {
return "mockedCachingMux"
}
func (m *mockedCachingMux) Cache() map[protocol.DeviceID]discover.CacheEntry {
return nil
}
// from events.CachingMux
func (m *mockedCachingMux) Add(finder discover.Finder, cacheTime, negCacheTime time.Duration) {
}
func (m *mockedCachingMux) ChildErrors() map[string]error {
return nil
}

19
cmd/syncthing/mocked_events_test.go
View File

@@ -1,19 +0,0 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"time"
"github.com/syncthing/syncthing/lib/events"
)
type mockedEventSub struct{}
func (s *mockedEventSub) Since(id int, into []events.Event, timeout time.Duration) []events.Event {
select {}
}

26
cmd/syncthing/mocked_logger_test.go
View File

@@ -1,26 +0,0 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"time"
"github.com/syncthing/syncthing/lib/logger"
)
type mockedLoggerRecorder struct{}
func (r *mockedLoggerRecorder) Since(t time.Time) []logger.Line {
return []logger.Line{
{
When: time.Now(),
Message: "Test message",
},
}
}
func (r *mockedLoggerRecorder) Clear() {}

189
cmd/syncthing/mocked_model_test.go
View File

@@ -1,189 +0,0 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"net"
"time"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/connections"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/model"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/stats"
"github.com/syncthing/syncthing/lib/versioner"
)
type mockedModel struct{}
func (m *mockedModel) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly bool) map[string]interface{} {
return nil
}
func (m *mockedModel) Completion(device protocol.DeviceID, folder string) model.FolderCompletion {
return model.FolderCompletion{}
}
func (m *mockedModel) Override(folder string) {}
func (m *mockedModel) Revert(folder string) {}
func (m *mockedModel) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated) {
return nil, nil, nil
}
func (m *mockedModel) RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error) {
return nil, nil
}
func (m *mockedModel) NeedSize(folder string) db.Counts {
return db.Counts{}
}
func (m *mockedModel) ConnectionStats() map[string]interface{} {
return nil
}
func (m *mockedModel) DeviceStatistics() map[string]stats.DeviceStatistics {
return nil
}
func (m *mockedModel) FolderStatistics() map[string]stats.FolderStatistics {
return nil
}
func (m *mockedModel) CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool) {
return protocol.FileInfo{}, false
}
func (m *mockedModel) CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool) {
return protocol.FileInfo{}, false
}
func (m *mockedModel) ResetFolder(folder string) {
}
func (m *mockedModel) Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []model.Availability {
return nil
}
func (m *mockedModel) GetIgnores(folder string) ([]string, []string, error) {
return nil, nil, nil
}
func (m *mockedModel) SetIgnores(folder string, content []string) error {
return nil
}
func (m *mockedModel) GetFolderVersions(folder string) (map[string][]versioner.FileVersion, error) {
return nil, nil
}
func (m *mockedModel) RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]string, error) {
return nil, nil
}
func (m *mockedModel) PauseDevice(device protocol.DeviceID) {
}
func (m *mockedModel) ResumeDevice(device protocol.DeviceID) {}
func (m *mockedModel) DelayScan(folder string, next time.Duration) {}
func (m *mockedModel) ScanFolder(folder string) error {
return nil
}
func (m *mockedModel) ScanFolders() map[string]error {
return nil
}
func (m *mockedModel) ScanFolderSubdirs(folder string, subs []string) error {
return nil
}
func (m *mockedModel) BringToFront(folder, file string) {}
func (m *mockedModel) Connection(deviceID protocol.DeviceID) (connections.Connection, bool) {
return nil, false
}
func (m *mockedModel) GlobalSize(folder string) db.Counts {
return db.Counts{}
}
func (m *mockedModel) LocalSize(folder string) db.Counts {
return db.Counts{}
}
func (m *mockedModel) ReceiveOnlyChangedSize(folder string) db.Counts {
return db.Counts{}
}
func (m *mockedModel) CurrentSequence(folder string) (int64, bool) {
return 0, false
}
func (m *mockedModel) RemoteSequence(folder string) (int64, bool) {
return 0, false
}
func (m *mockedModel) State(folder string) (string, time.Time, error) {
return "", time.Time{}, nil
}
func (m *mockedModel) UsageReportingStats(version int, preview bool) map[string]interface{} {
return nil
}
func (m *mockedModel) FolderErrors(folder string) ([]model.FileError, error) {
return nil, nil
}
func (m *mockedModel) WatchError(folder string) error {
return nil
}
func (m *mockedModel) LocalChangedFiles(folder string, page, perpage int) []db.FileInfoTruncated {
return nil
}
func (m *mockedModel) Serve() {}
func (m *mockedModel) Stop() {}
func (m *mockedModel) Index(deviceID protocol.DeviceID, folder string, files []protocol.FileInfo) {}
func (m *mockedModel) IndexUpdate(deviceID protocol.DeviceID, folder string, files []protocol.FileInfo) {
}
func (m *mockedModel) Request(deviceID protocol.DeviceID, folder, name string, size int32, offset int64, hash []byte, weakHash uint32, fromTemporary bool) (protocol.RequestResponse, error) {
return nil, nil
}
func (m *mockedModel) ClusterConfig(deviceID protocol.DeviceID, config protocol.ClusterConfig) {}
func (m *mockedModel) Closed(conn protocol.Connection, err error) {}
func (m *mockedModel) DownloadProgress(deviceID protocol.DeviceID, folder string, updates []protocol.FileDownloadProgressUpdate) {
}
func (m *mockedModel) AddConnection(conn connections.Connection, hello protocol.HelloResult) {}
func (m *mockedModel) OnHello(protocol.DeviceID, net.Addr, protocol.HelloResult) error {
return nil
}
func (m *mockedModel) GetHello(protocol.DeviceID) protocol.HelloIntf {
return nil
}
func (m *mockedModel) AddFolder(cfg config.FolderConfiguration) {}
func (m *mockedModel) RestartFolder(from, to config.FolderConfiguration) {}
func (m *mockedModel) StartFolder(folder string) {}
func (m *mockedModel) StartDeadlockDetector(timeout time.Duration) {}

233
cmd/syncthing/summaryservice.go
View File

@@ -1,233 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"time"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/model"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sync"
"github.com/thejerf/suture"
)
// The folderSummaryService adds summary information events (FolderSummary and
// FolderCompletion) into the event stream at certain intervals.
type folderSummaryService struct {
*suture.Supervisor
cfg config.Wrapper
model model.Model
stop chan struct{}
immediate chan string
// For keeping track of folders to recalculate for
foldersMut sync.Mutex
folders map[string]struct{}
// For keeping track of when the last event request on the API was
lastEventReq time.Time
lastEventReqMut sync.Mutex
}
func newFolderSummaryService(cfg config.Wrapper, m model.Model) *folderSummaryService {
service := &folderSummaryService{
Supervisor: suture.New("folderSummaryService", suture.Spec{
PassThroughPanics: true,
}),
cfg: cfg,
model: m,
stop: make(chan struct{}),
immediate: make(chan string),
folders: make(map[string]struct{}),
foldersMut: sync.NewMutex(),
lastEventReqMut: sync.NewMutex(),
}
service.Add(serviceFunc(service.listenForUpdates))
service.Add(serviceFunc(service.calculateSummaries))
return service
}
func (c *folderSummaryService) Stop() {
c.Supervisor.Stop()
close(c.stop)
}
// listenForUpdates subscribes to the event bus and makes note of folders that
// need their data recalculated.
func (c *folderSummaryService) listenForUpdates() {
sub := events.Default.Subscribe(events.LocalIndexUpdated | events.RemoteIndexUpdated | events.StateChanged | events.RemoteDownloadProgress | events.DeviceConnected | events.FolderWatchStateChanged)
defer events.Default.Unsubscribe(sub)
for {
// This loop needs to be fast so we don't miss too many events.
select {
case ev := <-sub.C():
if ev.Type == events.DeviceConnected {
// When a device connects we schedule a refresh of all
// folders shared with that device.
data := ev.Data.(map[string]string)
deviceID, _ := protocol.DeviceIDFromString(data["id"])
c.foldersMut.Lock()
nextFolder:
for _, folder := range c.cfg.Folders() {
for _, dev := range folder.Devices {
if dev.DeviceID == deviceID {
c.folders[folder.ID] = struct{}{}
continue nextFolder
}
}
}
c.foldersMut.Unlock()
continue
}
// The other events all have a "folder" attribute that they
// affect. Whenever the local or remote index is updated for a
// given folder we make a note of it.
data := ev.Data.(map[string]interface{})
folder := data["folder"].(string)
switch ev.Type {
case events.StateChanged:
if data["to"].(string) == "idle" && data["from"].(string) == "syncing" {
// The folder changed to idle from syncing. We should do an
// immediate refresh to update the GUI. The send to
// c.immediate must be nonblocking so that we can continue
// handling events.
c.foldersMut.Lock()
select {
case c.immediate <- folder:
delete(c.folders, folder)
default:
c.folders[folder] = struct{}{}
}
c.foldersMut.Unlock()
}
default:
// This folder needs to be refreshed whenever we do the next
// refresh.
c.foldersMut.Lock()
c.folders[folder] = struct{}{}
c.foldersMut.Unlock()
}
case <-c.stop:
return
}
}
}
// calculateSummaries periodically recalculates folder summaries and
// completion percentage, and sends the results on the event bus.
func (c *folderSummaryService) calculateSummaries() {
const pumpInterval = 2 * time.Second
pump := time.NewTimer(pumpInterval)
for {
select {
case <-pump.C:
t0 := time.Now()
for _, folder := range c.foldersToHandle() {
c.sendSummary(folder)
}
// We don't want to spend all our time calculating summaries. Lets
// set an arbitrary limit at not spending more than about 30% of
// our time here...
wait := 2*time.Since(t0) + pumpInterval
pump.Reset(wait)
case folder := <-c.immediate:
c.sendSummary(folder)
case <-c.stop:
return
}
}
}
// foldersToHandle returns the list of folders needing a summary update, and
// clears the list.
func (c *folderSummaryService) foldersToHandle() []string {
// We only recalculate summaries if someone is listening to events
// (a request to /rest/events has been made within the last
// pingEventInterval).
c.lastEventReqMut.Lock()
last := c.lastEventReq
c.lastEventReqMut.Unlock()
if time.Since(last) > defaultEventTimeout {
return nil
}
c.foldersMut.Lock()
res := make([]string, 0, len(c.folders))
for folder := range c.folders {
res = append(res, folder)
delete(c.folders, folder)
}
c.foldersMut.Unlock()
return res
}
// sendSummary send the summary events for a single folder
func (c *folderSummaryService) sendSummary(folder string) {
// The folder summary contains how many bytes, files etc
// are in the folder and how in sync we are.
data, err := folderSummary(c.cfg, c.model, folder)
if err != nil {
return
}
events.Default.Log(events.FolderSummary, map[string]interface{}{
"folder": folder,
"summary": data,
})
for _, devCfg := range c.cfg.Folders()[folder].Devices {
if devCfg.DeviceID.Equals(myID) {
// We already know about ourselves.
continue
}
if _, ok := c.model.Connection(devCfg.DeviceID); !ok {
// We're not interested in disconnected devices.
continue
}
// Get completion percentage of this folder for the
// remote device.
comp := jsonCompletion(c.model.Completion(devCfg.DeviceID, folder))
comp["folder"] = folder
comp["device"] = devCfg.DeviceID.String()
events.Default.Log(events.FolderCompletion, comp)
}
}
func (c *folderSummaryService) gotEventRequest() {
c.lastEventReqMut.Lock()
c.lastEventReq = time.Now()
c.lastEventReqMut.Unlock()
}
// serviceFunc wraps a function to create a suture.Service without stop
// functionality.
type serviceFunc func()
func (f serviceFunc) Serve() { f() }
func (f serviceFunc) Stop() {}

47
cmd/syncthing/support_bundle.go
View File

@@ -1,47 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"archive/zip"
"io"
"github.com/syncthing/syncthing/lib/config"
)
// getRedactedConfig redacting some parts of config
func getRedactedConfig(s *apiService) config.Configuration {
rawConf := s.cfg.RawCopy()
rawConf.GUI.APIKey = "REDACTED"
if rawConf.GUI.Password != "" {
rawConf.GUI.Password = "REDACTED"
}
if rawConf.GUI.User != "" {
rawConf.GUI.User = "REDACTED"
}
return rawConf
}
// writeZip writes a zip file containing the given entries
func writeZip(writer io.Writer, files []fileEntry) error {
zipWriter := zip.NewWriter(writer)
defer zipWriter.Close()
for _, file := range files {
zipFile, err := zipWriter.Create(file.name)
if err != nil {
return err
}
_, err = zipFile.Write(file.data)
if err != nil {
return err
}
}
return zipWriter.Close()
}

0
cmd/syncthing/testdata/.stfolder vendored
View File

1
cmd/syncthing/testdata/default/a vendored
View File

@@ -1 +0,0 @@
overridden-default

1
cmd/syncthing/testdata/default/b vendored
View File

@@ -1 +0,0 @@
overridden-default

1
cmd/syncthing/testdata/default/d vendored
View File

@@ -1 +0,0 @@
overridden-default

1
cmd/syncthing/testdata/foo/a vendored
View File

@@ -1 +0,0 @@
overridden-foo

0
cmd/syncthing/testdata/testfolder/.stfolder vendored
View File

455
cmd/syncthing/usage_report.go
View File

@@ -1,455 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bytes"
"context"
"crypto/rand"
"crypto/tls"
"encoding/json"
"net"
"net/http"
"runtime"
"sort"
"strings"
"sync"
"time"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/connections"
"github.com/syncthing/syncthing/lib/dialer"
"github.com/syncthing/syncthing/lib/model"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/upgrade"
)
// Current version number of the usage report, for acceptance purposes. If
// fields are added or changed this integer must be incremented so that users
// are prompted for acceptance of the new report.
const usageReportVersion = 3
// reportData returns the data to be sent in a usage report. It's used in
// various places, so not part of the usageReportingManager object.
func reportData(cfg config.Wrapper, m model.Model, connectionsService connections.Service, version int, preview bool) map[string]interface{} {
opts := cfg.Options()
res := make(map[string]interface{})
res["urVersion"] = version
res["uniqueID"] = opts.URUniqueID
res["version"] = build.Version
res["longVersion"] = build.LongVersion
res["platform"] = runtime.GOOS + "-" + runtime.GOARCH
res["numFolders"] = len(cfg.Folders())
res["numDevices"] = len(cfg.Devices())
var totFiles, maxFiles int
var totBytes, maxBytes int64
for folderID := range cfg.Folders() {
global := m.GlobalSize(folderID)
totFiles += int(global.Files)
totBytes += global.Bytes
if int(global.Files) > maxFiles {
maxFiles = int(global.Files)
}
if global.Bytes > maxBytes {
maxBytes = global.Bytes
}
}
res["totFiles"] = totFiles
res["folderMaxFiles"] = maxFiles
res["totMiB"] = totBytes / 1024 / 1024
res["folderMaxMiB"] = maxBytes / 1024 / 1024
var mem runtime.MemStats
runtime.ReadMemStats(&mem)
res["memoryUsageMiB"] = (mem.Sys - mem.HeapReleased) / 1024 / 1024
res["sha256Perf"] = cpuBench(5, 125*time.Millisecond, false)
res["hashPerf"] = cpuBench(5, 125*time.Millisecond, true)
bytes, err := memorySize()
if err == nil {
res["memorySize"] = bytes / 1024 / 1024
}
res["numCPU"] = runtime.NumCPU()
var rescanIntvs []int
folderUses := map[string]int{
"sendonly": 0,
"sendreceive": 0,
"receiveonly": 0,
"ignorePerms": 0,
"ignoreDelete": 0,
"autoNormalize": 0,
"simpleVersioning": 0,
"externalVersioning": 0,
"staggeredVersioning": 0,
"trashcanVersioning": 0,
}
for _, cfg := range cfg.Folders() {
rescanIntvs = append(rescanIntvs, cfg.RescanIntervalS)
switch cfg.Type {
case config.FolderTypeSendOnly:
folderUses["sendonly"]++
case config.FolderTypeSendReceive:
folderUses["sendreceive"]++
case config.FolderTypeReceiveOnly:
folderUses["receiveonly"]++
}
if cfg.IgnorePerms {
folderUses["ignorePerms"]++
}
if cfg.IgnoreDelete {
folderUses["ignoreDelete"]++
}
if cfg.AutoNormalize {
folderUses["autoNormalize"]++
}
if cfg.Versioning.Type != "" {
folderUses[cfg.Versioning.Type+"Versioning"]++
}
}
sort.Ints(rescanIntvs)
res["rescanIntvs"] = rescanIntvs
res["folderUses"] = folderUses
deviceUses := map[string]int{
"introducer": 0,
"customCertName": 0,
"compressAlways": 0,
"compressMetadata": 0,
"compressNever": 0,
"dynamicAddr": 0,
"staticAddr": 0,
}
for _, cfg := range cfg.Devices() {
if cfg.Introducer {
deviceUses["introducer"]++
}
if cfg.CertName != "" && cfg.CertName != "syncthing" {
deviceUses["customCertName"]++
}
if cfg.Compression == protocol.CompressAlways {
deviceUses["compressAlways"]++
} else if cfg.Compression == protocol.CompressMetadata {
deviceUses["compressMetadata"]++
} else if cfg.Compression == protocol.CompressNever {
deviceUses["compressNever"]++
}
for _, addr := range cfg.Addresses {
if addr == "dynamic" {
deviceUses["dynamicAddr"]++
} else {
deviceUses["staticAddr"]++
}
}
}
res["deviceUses"] = deviceUses
defaultAnnounceServersDNS, defaultAnnounceServersIP, otherAnnounceServers := 0, 0, 0
for _, addr := range opts.GlobalAnnServers {
if addr == "default" || addr == "default-v4" || addr == "default-v6" {
defaultAnnounceServersDNS++
} else {
otherAnnounceServers++
}
}
res["announce"] = map[string]interface{}{
"globalEnabled": opts.GlobalAnnEnabled,
"localEnabled": opts.LocalAnnEnabled,
"defaultServersDNS": defaultAnnounceServersDNS,
"defaultServersIP": defaultAnnounceServersIP,
"otherServers": otherAnnounceServers,
}
defaultRelayServers, otherRelayServers := 0, 0
for _, addr := range cfg.ListenAddresses() {
switch {
case addr == "dynamic+https://relays.syncthing.net/endpoint":
defaultRelayServers++
case strings.HasPrefix(addr, "relay://") || strings.HasPrefix(addr, "dynamic+http"):
otherRelayServers++
}
}
res["relays"] = map[string]interface{}{
"enabled": defaultRelayServers+otherAnnounceServers > 0,
"defaultServers": defaultRelayServers,
"otherServers": otherRelayServers,
}
res["usesRateLimit"] = opts.MaxRecvKbps > 0 || opts.MaxSendKbps > 0
res["upgradeAllowedManual"] = !(upgrade.DisabledByCompilation || noUpgradeFromEnv)
res["upgradeAllowedAuto"] = !(upgrade.DisabledByCompilation || noUpgradeFromEnv) && opts.AutoUpgradeIntervalH > 0
res["upgradeAllowedPre"] = !(upgrade.DisabledByCompilation || noUpgradeFromEnv) && opts.AutoUpgradeIntervalH > 0 && opts.UpgradeToPreReleases
if version >= 3 {
res["uptime"] = int(time.Since(startTime).Seconds())
res["natType"] = connectionsService.NATType()
res["alwaysLocalNets"] = len(opts.AlwaysLocalNets) > 0
res["cacheIgnoredFiles"] = opts.CacheIgnoredFiles
res["overwriteRemoteDeviceNames"] = opts.OverwriteRemoteDevNames
res["progressEmitterEnabled"] = opts.ProgressUpdateIntervalS > -1
res["customDefaultFolderPath"] = opts.DefaultFolderPath != "~"
res["customTrafficClass"] = opts.TrafficClass != 0
res["customTempIndexMinBlocks"] = opts.TempIndexMinBlocks != 10
res["temporariesDisabled"] = opts.KeepTemporariesH == 0
res["temporariesCustom"] = opts.KeepTemporariesH != 24
res["limitBandwidthInLan"] = opts.LimitBandwidthInLan
res["customReleaseURL"] = opts.ReleasesURL != "https://upgrades.syncthing.net/meta.json"
res["restartOnWakeup"] = opts.RestartOnWakeup
folderUsesV3 := map[string]int{
"scanProgressDisabled": 0,
"conflictsDisabled": 0,
"conflictsUnlimited": 0,
"conflictsOther": 0,
"disableSparseFiles": 0,
"disableTempIndexes": 0,
"alwaysWeakHash": 0,
"customWeakHashThreshold": 0,
"fsWatcherEnabled": 0,
}
pullOrder := make(map[string]int)
filesystemType := make(map[string]int)
var fsWatcherDelays []int
for _, cfg := range cfg.Folders() {
if cfg.ScanProgressIntervalS < 0 {
folderUsesV3["scanProgressDisabled"]++
}
if cfg.MaxConflicts == 0 {
folderUsesV3["conflictsDisabled"]++
} else if cfg.MaxConflicts < 0 {
folderUsesV3["conflictsUnlimited"]++
} else {
folderUsesV3["conflictsOther"]++
}
if cfg.DisableSparseFiles {
folderUsesV3["disableSparseFiles"]++
}
if cfg.DisableTempIndexes {
folderUsesV3["disableTempIndexes"]++
}
if cfg.WeakHashThresholdPct < 0 {
folderUsesV3["alwaysWeakHash"]++
} else if cfg.WeakHashThresholdPct != 25 {
folderUsesV3["customWeakHashThreshold"]++
}
if cfg.FSWatcherEnabled {
folderUsesV3["fsWatcherEnabled"]++
}
pullOrder[cfg.Order.String()]++
filesystemType[cfg.FilesystemType.String()]++
fsWatcherDelays = append(fsWatcherDelays, cfg.FSWatcherDelayS)
}
sort.Ints(fsWatcherDelays)
folderUsesV3Interface := map[string]interface{}{
"pullOrder": pullOrder,
"filesystemType": filesystemType,
"fsWatcherDelays": fsWatcherDelays,
}
for key, value := range folderUsesV3 {
folderUsesV3Interface[key] = value
}
res["folderUsesV3"] = folderUsesV3Interface
guiCfg := cfg.GUI()
// Anticipate multiple GUI configs in the future, hence store counts.
guiStats := map[string]int{
"enabled": 0,
"useTLS": 0,
"useAuth": 0,
"insecureAdminAccess": 0,
"debugging": 0,
"insecureSkipHostCheck": 0,
"insecureAllowFrameLoading": 0,
"listenLocal": 0,
"listenUnspecified": 0,
}
theme := make(map[string]int)
if guiCfg.Enabled {
guiStats["enabled"]++
if guiCfg.UseTLS() {
guiStats["useTLS"]++
}
if len(guiCfg.User) > 0 && len(guiCfg.Password) > 0 {
guiStats["useAuth"]++
}
if guiCfg.InsecureAdminAccess {
guiStats["insecureAdminAccess"]++
}
if guiCfg.Debugging {
guiStats["debugging"]++
}
if guiCfg.InsecureSkipHostCheck {
guiStats["insecureSkipHostCheck"]++
}
if guiCfg.InsecureAllowFrameLoading {
guiStats["insecureAllowFrameLoading"]++
}
addr, err := net.ResolveTCPAddr("tcp", guiCfg.Address())
if err == nil {
if addr.IP.IsLoopback() {
guiStats["listenLocal"]++
} else if addr.IP.IsUnspecified() {
guiStats["listenUnspecified"]++
}
}
theme[guiCfg.Theme]++
}
guiStatsInterface := map[string]interface{}{
"theme": theme,
}
for key, value := range guiStats {
guiStatsInterface[key] = value
}
res["guiStats"] = guiStatsInterface
}
for key, value := range m.UsageReportingStats(version, preview) {
res[key] = value
}
return res
}
type usageReportingService struct {
cfg config.Wrapper
model model.Model
connectionsService connections.Service
forceRun chan struct{}
stop chan struct{}
stopped chan struct{}
stopMut sync.RWMutex
}
func newUsageReportingService(cfg config.Wrapper, model model.Model, connectionsService connections.Service) *usageReportingService {
svc := &usageReportingService{
cfg: cfg,
model: model,
connectionsService: connectionsService,
forceRun: make(chan struct{}),
stop: make(chan struct{}),
stopped: make(chan struct{}),
}
close(svc.stopped) // Not yet running, dont block on Stop()
cfg.Subscribe(svc)
return svc
}
func (s *usageReportingService) sendUsageReport() error {
d := reportData(s.cfg, s.model, s.connectionsService, s.cfg.Options().URAccepted, false)
var b bytes.Buffer
if err := json.NewEncoder(&b).Encode(d); err != nil {
return err
}
client := &http.Client{
Transport: &http.Transport{
Dial: dialer.Dial,
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: s.cfg.Options().URPostInsecurely,
},
},
}
_, err := client.Post(s.cfg.Options().URURL, "application/json", &b)
return err
}
func (s *usageReportingService) Serve() {
s.stopMut.Lock()
s.stop = make(chan struct{})
s.stopped = make(chan struct{})
s.stopMut.Unlock()
t := time.NewTimer(time.Duration(s.cfg.Options().URInitialDelayS) * time.Second)
s.stopMut.RLock()
defer func() {
close(s.stopped)
s.stopMut.RUnlock()
}()
for {
select {
case <-s.stop:
return
case <-s.forceRun:
t.Reset(0)
case <-t.C:
if s.cfg.Options().URAccepted >= 2 {
err := s.sendUsageReport()
if err != nil {
l.Infoln("Usage report:", err)
} else {
l.Infof("Sent usage report (version %d)", s.cfg.Options().URAccepted)
}
}
t.Reset(24 * time.Hour) // next report tomorrow
}
}
}
func (s *usageReportingService) VerifyConfiguration(from, to config.Configuration) error {
return nil
}
func (s *usageReportingService) CommitConfiguration(from, to config.Configuration) bool {
if from.Options.URAccepted != to.Options.URAccepted || from.Options.URUniqueID != to.Options.URUniqueID || from.Options.URURL != to.Options.URURL {
s.stopMut.RLock()
select {
case s.forceRun <- struct{}{}:
case <-s.stop:
}
s.stopMut.RUnlock()
}
return true
}
func (s *usageReportingService) Stop() {
s.stopMut.RLock()
close(s.stop)
<-s.stopped
s.stopMut.RUnlock()
}
func (*usageReportingService) String() string {
return "usageReportingService"
}
// cpuBench returns CPU performance as a measure of single threaded SHA-256 MiB/s
func cpuBench(iterations int, duration time.Duration, useWeakHash bool) float64 {
dataSize := 16 * protocol.MinBlockSize
bs := make([]byte, dataSize)
rand.Reader.Read(bs)
var perf float64
for i := 0; i < iterations; i++ {
if v := cpuBenchOnce(duration, useWeakHash, bs); v > perf {
perf = v
}
}
blocksResult = nil
return perf
}
var blocksResult []protocol.BlockInfo // so the result is not optimized away
func cpuBenchOnce(duration time.Duration, useWeakHash bool, bs []byte) float64 {
t0 := time.Now()
b := 0
for time.Since(t0) < duration {
r := bytes.NewReader(bs)
blocksResult, _ = scanner.Blocks(context.TODO(), r, protocol.MinBlockSize, int64(len(bs)), nil, useWeakHash)
b += len(bs)
}
d := time.Since(t0)
return float64(int(float64(b)/d.Seconds()/(1<<20)*100)) / 100
}