perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/connections: Add QUIC protocol support (fixes #5377) (#5737)

Browse Source
This commit is contained in:
Audrius Butkevicius
2019-05-29 08:56:40 +01:00
committed by Jakob Borg
parent d8fa61e27c
commit e714df013f
32 changed files with 1290 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
lib/api/mocked_config_test.go
View File

@@ -74,51 +74,55 @@ func (c *mockedConfig) AddOrUpdatePendingDevice(device protocol.DeviceID, name,
func (c *mockedConfig) AddOrUpdatePendingFolder(id, label string, device protocol.DeviceID) {}
func (m *mockedConfig) MyName() string {
func (c *mockedConfig) MyName() string {
return ""
}
func (m *mockedConfig) ConfigPath() string {
func (c *mockedConfig) ConfigPath() string {
return ""
}
func (m *mockedConfig) SetGUI(gui config.GUIConfiguration) (config.Waiter, error) {
func (c *mockedConfig) SetGUI(gui config.GUIConfiguration) (config.Waiter, error) {
return noopWaiter{}, nil
}
func (m *mockedConfig) SetOptions(opts config.OptionsConfiguration) (config.Waiter, error) {
func (c *mockedConfig) SetOptions(opts config.OptionsConfiguration) (config.Waiter, error) {
return noopWaiter{}, nil
}
func (m *mockedConfig) Folder(id string) (config.FolderConfiguration, bool) {
func (c *mockedConfig) Folder(id string) (config.FolderConfiguration, bool) {
return config.FolderConfiguration{}, false
}
func (m *mockedConfig) FolderList() []config.FolderConfiguration {
func (c *mockedConfig) FolderList() []config.FolderConfiguration {
return nil
}
func (m *mockedConfig) SetFolder(fld config.FolderConfiguration) (config.Waiter, error) {
func (c *mockedConfig) SetFolder(fld config.FolderConfiguration) (config.Waiter, error) {
return noopWaiter{}, nil
}
func (m *mockedConfig) Device(id protocol.DeviceID) (config.DeviceConfiguration, bool) {
func (c *mockedConfig) Device(id protocol.DeviceID) (config.DeviceConfiguration, bool) {
return config.DeviceConfiguration{}, false
}
func (m *mockedConfig) RemoveDevice(id protocol.DeviceID) (config.Waiter, error) {
func (c *mockedConfig) RemoveDevice(id protocol.DeviceID) (config.Waiter, error) {
return noopWaiter{}, nil
}
func (m *mockedConfig) IgnoredDevice(id protocol.DeviceID) bool {
func (c *mockedConfig) IgnoredDevice(id protocol.DeviceID) bool {
return false
}
func (m *mockedConfig) IgnoredFolder(device protocol.DeviceID, folder string) bool {
func (c *mockedConfig) IgnoredFolder(device protocol.DeviceID, folder string) bool {
return false
}
func (m *mockedConfig) GlobalDiscoveryServers() []string {
func (c *mockedConfig) GlobalDiscoveryServers() []string {
return nil
}
func (c *mockedConfig) StunServers() []string {
return nil
}

31
lib/config/config.go
View File

@@ -39,6 +39,8 @@ const (
var (
// DefaultTCPPort defines default TCP port used if the URI does not specify one, for example tcp://0.0.0.0
DefaultTCPPort = 22000
// DefaultQUICPort defines default QUIC port used if the URI does not specify one, for example quic://0.0.0.0
DefaultQUICPort = 22000
// DefaultListenAddresses should be substituted when the configuration
// contains <listenAddress>default</listenAddress>. This is done by the
// "consumer" of the configuration as we don't want these saved to the
@@ -46,6 +48,7 @@ var (
DefaultListenAddresses = []string{
util.Address("tcp", net.JoinHostPort("0.0.0.0", strconv.Itoa(DefaultTCPPort))),
"dynamic+https://relays.syncthing.net/endpoint",
util.Address("quic", net.JoinHostPort("0.0.0.0", strconv.Itoa(DefaultQUICPort))),
}
DefaultGUIPort = 8384
// DefaultDiscoveryServersV4 should be substituted when the configuration
@@ -65,6 +68,30 @@ var (
DefaultDiscoveryServers = append(DefaultDiscoveryServersV4, DefaultDiscoveryServersV6...)
// DefaultTheme is the default and fallback theme for the web UI.
DefaultTheme = "default"
// Default stun servers should be substituted when the configuration
// contains <stunServer>default</stunServer>.
// DefaultPrimaryStunServers are servers provided by us (to avoid causing the public servers burden)
DefaultPrimaryStunServers = []string{
"stun.syncthing.net:3478",
}
DefaultSecondaryStunServers = []string{
"stun.callwithus.com:3478",
"stun.counterpath.com:3478",
"stun.counterpath.net:3478",
"stun.ekiga.net:3478",
"stun.ideasip.com:3478",
"stun.internetcalls.com:3478",
"stun.schlund.de:3478",
"stun.sipgate.net:10000",
"stun.sipgate.net:3478",
"stun.voip.aebc.com:3478",
"stun.voiparound.com:3478",
"stun.voipbuster.com:3478",
"stun.voipstunt.com:3478",
"stun.voxgratia.org:3478",
"stun.xten.com:3478",
}
)
func New(myID protocol.DeviceID) Configuration {
@@ -258,8 +285,8 @@ func (cfg *Configuration) clean() error {
existingFolders[folder.ID] = folder
}
cfg.Options.ListenAddresses = util.UniqueStrings(cfg.Options.ListenAddresses)
cfg.Options.GlobalAnnServers = util.UniqueStrings(cfg.Options.GlobalAnnServers)
cfg.Options.ListenAddresses = util.UniqueTrimmedStrings(cfg.Options.ListenAddresses)
cfg.Options.GlobalAnnServers = util.UniqueTrimmedStrings(cfg.Options.GlobalAnnServers)
if cfg.Version > 0 && cfg.Version < OldestHandledVersion {
l.Warnf("Configuration version %d is deprecated. Attempting best effort conversion, but please verify manually.", cfg.Version)

10
lib/config/config_test.go
View File

@@ -69,6 +69,9 @@ func TestDefaultValues(t *testing.T) {
UnackedNotificationIDs: []string{},
DefaultFolderPath: "~",
SetLowPriority: true,
StunKeepaliveStartS: 180,
StunKeepaliveMinS: 20,
StunServers: []string{"default"},
}
cfg := New(device1)
@@ -212,8 +215,11 @@ func TestOverriddenValues(t *testing.T) {
"channelNotification", // added in 17->18 migration
"fsWatcherNotification", // added in 27->28 migration
},
DefaultFolderPath: "/media/syncthing",
SetLowPriority: false,
DefaultFolderPath: "/media/syncthing",
SetLowPriority: false,
StunKeepaliveStartS: 9000,
StunKeepaliveMinS: 900,
StunServers: []string{"foo"},
}
os.Unsetenv("STNOUPGRADE")

39
lib/config/optionsconfiguration.go
View File

@@ -52,6 +52,9 @@ type OptionsConfiguration struct {
DefaultFolderPath string `xml:"defaultFolderPath" json:"defaultFolderPath" default:"~"`
SetLowPriority bool `xml:"setLowPriority" json:"setLowPriority" default:"true"`
MaxConcurrentScans int `xml:"maxConcurrentScans" json:"maxConcurrentScans"`
StunKeepaliveStartS int `xml:"stunKeepaliveStartS" json:"stunKeepaliveStartS" default:"180"` // 0 for off
StunKeepaliveMinS int `xml:"stunKeepaliveMinS" json:"stunKeepaliveMinS" default:"20"` // 0 for off
StunServers []string `xml:"stunServer" json:"stunServers" default:"default"`
DeprecatedUPnPEnabled bool `xml:"upnpEnabled,omitempty" json:"-"`
DeprecatedUPnPLeaseM int `xml:"upnpLeaseMinutes,omitempty" json:"-"`
@@ -61,29 +64,33 @@ type OptionsConfiguration struct {
DeprecatedMinHomeDiskFreePct float64 `xml:"minHomeDiskFreePct,omitempty" json:"-"`
}
func (orig OptionsConfiguration) Copy() OptionsConfiguration {
c := orig
c.ListenAddresses = make([]string, len(orig.ListenAddresses))
copy(c.ListenAddresses, orig.ListenAddresses)
c.GlobalAnnServers = make([]string, len(orig.GlobalAnnServers))
copy(c.GlobalAnnServers, orig.GlobalAnnServers)
c.AlwaysLocalNets = make([]string, len(orig.AlwaysLocalNets))
copy(c.AlwaysLocalNets, orig.AlwaysLocalNets)
c.UnackedNotificationIDs = make([]string, len(orig.UnackedNotificationIDs))
copy(c.UnackedNotificationIDs, orig.UnackedNotificationIDs)
return c
func (opts OptionsConfiguration) Copy() OptionsConfiguration {
optsCopy := opts
optsCopy.ListenAddresses = make([]string, len(opts.ListenAddresses))
copy(optsCopy.ListenAddresses, opts.ListenAddresses)
optsCopy.GlobalAnnServers = make([]string, len(opts.GlobalAnnServers))
copy(optsCopy.GlobalAnnServers, opts.GlobalAnnServers)
optsCopy.AlwaysLocalNets = make([]string, len(opts.AlwaysLocalNets))
copy(optsCopy.AlwaysLocalNets, opts.AlwaysLocalNets)
optsCopy.UnackedNotificationIDs = make([]string, len(opts.UnackedNotificationIDs))
copy(optsCopy.UnackedNotificationIDs, opts.UnackedNotificationIDs)
return optsCopy
}
// RequiresRestartOnly returns a copy with only the attributes that require
// restart on change.
func (orig OptionsConfiguration) RequiresRestartOnly() OptionsConfiguration {
copy := orig
func (opts OptionsConfiguration) RequiresRestartOnly() OptionsConfiguration {
optsCopy := opts
blank := OptionsConfiguration{}
util.CopyMatchingTag(&blank, &copy, "restart", func(v string) bool {
util.CopyMatchingTag(&blank, &optsCopy, "restart", func(v string) bool {
if len(v) > 0 && v != "true" {
panic(fmt.Sprintf(`unexpected tag value: %s. expected untagged or "true"`, v))
panic(fmt.Sprintf(`unexpected tag value: %s. Expected untagged or "true"`, v))
}
return v != "true"
})
return copy
return optsCopy
}
func (opts OptionsConfiguration) IsStunDisabled() bool {
return opts.StunKeepaliveMinS < 1 || opts.StunKeepaliveStartS < 1 || !opts.NATEnabled
}

3
lib/config/testdata/overridenvalues.xml vendored
View File

@@ -36,5 +36,8 @@
<tempIndexMinBlocks>100</tempIndexMinBlocks>
<defaultFolderPath>/media/syncthing</defaultFolderPath>
<setLowPriority>false</setLowPriority>
<stunKeepaliveStartS>9000</stunKeepaliveStartS>
<stunKeepaliveMinS>900</stunKeepaliveMinS>
<stunServer>foo</stunServer>
</options>
</configuration>

30
lib/config/wrapper.go
View File

@@ -14,6 +14,7 @@ import (
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/util"
)
@@ -88,6 +89,7 @@ type Wrapper interface {
ListenAddresses() []string
GlobalDiscoveryServers() []string
StunServers() []string
Subscribe(c Committer)
Unsubscribe(c Committer)
@@ -105,6 +107,30 @@ type wrapper struct {
requiresRestart uint32 // an atomic bool
}
func (w *wrapper) StunServers() []string {
var addresses []string
for _, addr := range w.cfg.Options.StunServers {
switch addr {
case "default":
defaultPrimaryAddresses := make([]string, len(DefaultPrimaryStunServers))
copy(defaultPrimaryAddresses, DefaultPrimaryStunServers)
rand.Shuffle(defaultPrimaryAddresses)
addresses = append(addresses, defaultPrimaryAddresses...)
defaultSecondaryAddresses := make([]string, len(DefaultSecondaryStunServers))
copy(defaultSecondaryAddresses, DefaultSecondaryStunServers)
rand.Shuffle(defaultSecondaryAddresses)
addresses = append(addresses, defaultSecondaryAddresses...)
default:
addresses = append(addresses, addr)
}
}
addresses = util.UniqueTrimmedStrings(addresses)
return addresses
}
// Wrap wraps an existing Configuration structure and ties it to a file on
// disk.
func Wrap(path string, cfg Configuration) Wrapper {
@@ -442,7 +468,7 @@ func (w *wrapper) GlobalDiscoveryServers() []string {
servers = append(servers, srv)
}
}
return util.UniqueStrings(servers)
return util.UniqueTrimmedStrings(servers)
}
func (w *wrapper) ListenAddresses() []string {
@@ -455,7 +481,7 @@ func (w *wrapper) ListenAddresses() []string {
addresses = append(addresses, addr)
}
}
return util.UniqueStrings(addresses)
return util.UniqueTrimmedStrings(addresses)
}
func (w *wrapper) RequiresRestart() bool {

12
lib/connections/config.go
View File

@@ -1,12 +0,0 @@
// Copyright (C) 2017 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package connections
const (
tcpPriority = 10
relayPriority = 200
)

128
lib/connections/quic_dial.go Normal file
View File

@@ -0,0 +1,128 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build go1.12
package connections
import (
"context"
"crypto/tls"
"net"
"net/url"
"time"
"github.com/lucas-clemente/quic-go"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/connections/registry"
"github.com/syncthing/syncthing/lib/protocol"
)
const quicPriority = 100
func init() {
factory := &quicDialerFactory{}
for _, scheme := range []string{"quic", "quic4", "quic6"} {
dialers[scheme] = factory
}
}
type quicDialer struct {
cfg config.Wrapper
tlsCfg *tls.Config
}
func (d *quicDialer) Dial(id protocol.DeviceID, uri *url.URL) (internalConn, error) {
uri = fixupPort(uri, config.DefaultQUICPort)
addr, err := net.ResolveUDPAddr("udp", uri.Host)
if err != nil {
return internalConn{}, err
}
var conn net.PacketConn
closeConn := false
if listenConn := registry.Get(uri.Scheme, packetConnLess); listenConn != nil {
conn = listenConn.(net.PacketConn)
} else {
if packetConn, err := net.ListenPacket("udp", ":0"); err != nil {
return internalConn{}, err
} else {
closeConn = true
conn = packetConn
}
}
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
session, err := quic.DialContext(ctx, conn, addr, uri.Host, d.tlsCfg, quicConfig)
if err != nil {
if closeConn {
_ = conn.Close()
}
return internalConn{}, err
}
// OpenStreamSync is blocks, but we want to make sure the connection is usable
// before we start killing off other connections, so do the dance.
ok := make(chan struct{})
go func() {
select {
case <-ok:
return
case <-time.After(10 * time.Second):
l.Debugln("timed out waiting for OpenStream on", session.RemoteAddr())
// This will unblock OpenStreamSync
_ = session.Close()
}
}()
stream, err := session.OpenStreamSync()
close(ok)
if err != nil {
// It's ok to close these, this does not close the underlying packetConn.
_ = session.Close()
if closeConn {
_ = conn.Close()
}
return internalConn{}, err
}
return internalConn{&quicTlsConn{session, stream}, connTypeQUICClient, quicPriority}, nil
}
func (d *quicDialer) RedialFrequency() time.Duration {
return time.Duration(d.cfg.Options().ReconnectIntervalS) * time.Second
}
type quicDialerFactory struct {
cfg config.Wrapper
tlsCfg *tls.Config
}
func (quicDialerFactory) New(cfg config.Wrapper, tlsCfg *tls.Config) genericDialer {
return &quicDialer{
cfg: cfg,
tlsCfg: tlsCfg,
}
}
func (quicDialerFactory) Priority() int {
return quicPriority
}
func (quicDialerFactory) AlwaysWAN() bool {
return false
}
func (quicDialerFactory) Valid(_ config.Configuration) error {
// Always valid
return nil
}
func (quicDialerFactory) String() string {
return "QUIC Dialer"
}

238
lib/connections/quic_listen.go Normal file
View File

@@ -0,0 +1,238 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build go1.12
package connections
import (
"crypto/tls"
"net"
"net/url"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/lucas-clemente/quic-go"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/connections/registry"
"github.com/syncthing/syncthing/lib/nat"
"github.com/syncthing/syncthing/lib/stun"
)
func init() {
factory := &quicListenerFactory{}
for _, scheme := range []string{"quic", "quic4", "quic6"} {
listeners[scheme] = factory
}
}
type quicListener struct {
nat atomic.Value
onAddressesChangedNotifier
uri *url.URL
cfg config.Wrapper
tlsCfg *tls.Config
stop chan struct{}
conns chan internalConn
factory listenerFactory
address *url.URL
err error
mut sync.Mutex
}
func (t *quicListener) OnNATTypeChanged(natType stun.NATType) {
if natType != stun.NATUnknown {
l.Infof("%s detected NAT type: %s", t.uri, natType)
}
t.nat.Store(natType)
}
func (t *quicListener) OnExternalAddressChanged(address *stun.Host, via string) {
var uri *url.URL
if address != nil {
uri = &(*t.uri)
uri.Host = address.TransportAddr()
}
t.mut.Lock()
existingAddress := t.address
t.address = uri
t.mut.Unlock()
if uri != nil && (existingAddress == nil || existingAddress.String() != uri.String()) {
l.Infof("%s resolved external address %s (via %s)", t.uri, uri.String(), via)
t.notifyAddressesChanged(t)
} else if uri == nil && existingAddress != nil {
t.notifyAddressesChanged(t)
}
}
func (t *quicListener) Serve() {
t.mut.Lock()
t.err = nil
t.mut.Unlock()
network := strings.Replace(t.uri.Scheme, "quic", "udp", -1)
packetConn, err := net.ListenPacket(network, t.uri.Host)
if err != nil {
t.mut.Lock()
t.err = err
t.mut.Unlock()
l.Infoln("Listen (BEP/quic):", err)
return
}
defer func() { _ = packetConn.Close() }()
svc, conn := stun.New(t.cfg, t, packetConn)
defer func() { _ = conn.Close() }()
go svc.Serve()
defer svc.Stop()
registry.Register(t.uri.Scheme, conn)
defer registry.Unregister(t.uri.Scheme, conn)
listener, err := quic.Listen(conn, t.tlsCfg, quicConfig)
if err != nil {
t.mut.Lock()
t.err = err
t.mut.Unlock()
l.Infoln("Listen (BEP/quic):", err)
return
}
l.Infof("QUIC listener (%v) starting", packetConn.LocalAddr())
defer l.Infof("QUIC listener (%v) shutting down", packetConn.LocalAddr())
// Accept is forever, so handle stops externally.
go func() {
select {
case <-t.stop:
_ = listener.Close()
}
}()
for {
// Blocks forever, see https://github.com/lucas-clemente/quic-go/issues/1915
session, err := listener.Accept()
select {
case <-t.stop:
if err == nil {
_ = session.Close()
}
return
default:
}
if err != nil {
if err, ok := err.(net.Error); !ok || !err.Timeout() {
l.Warnln("Listen (BEP/quic): Accepting connection:", err)
}
continue
}
l.Debugln("connect from", session.RemoteAddr())
// Accept blocks forever, give it 10s to do it's thing.
ok := make(chan struct{})
go func() {
select {
case <-ok:
return
case <-t.stop:
_ = session.Close()
case <-time.After(10 * time.Second):
l.Debugln("timed out waiting for AcceptStream on", session.RemoteAddr())
_ = session.Close()
}
}()
stream, err := session.AcceptStream()
close(ok)
if err != nil {
l.Debugln("failed to accept stream from", session.RemoteAddr(), err.Error())
_ = session.Close()
continue
}
t.conns <- internalConn{&quicTlsConn{session, stream}, connTypeQUICServer, quicPriority}
}
}
func (t *quicListener) Stop() {
close(t.stop)
}
func (t *quicListener) URI() *url.URL {
return t.uri
}
func (t *quicListener) WANAddresses() []*url.URL {
uris := t.LANAddresses()
t.mut.Lock()
if t.address != nil {
uris = append(uris, t.address)
}
t.mut.Unlock()
return uris
}
func (t *quicListener) LANAddresses() []*url.URL {
return []*url.URL{t.uri}
}
func (t *quicListener) Error() error {
t.mut.Lock()
err := t.err
t.mut.Unlock()
return err
}
func (t *quicListener) String() string {
return t.uri.String()
}
func (t *quicListener) Factory() listenerFactory {
return t.factory
}
func (t *quicListener) NATType() string {
v := t.nat.Load().(stun.NATType)
if v == stun.NATUnknown || v == stun.NATError {
return "unknown"
}
return v.String()
}
type quicListenerFactory struct{}
func (f *quicListenerFactory) Valid(config.Configuration) error {
return nil
}
func (f *quicListenerFactory) New(uri *url.URL, cfg config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener {
l := &quicListener{
uri: fixupPort(uri, config.DefaultQUICPort),
cfg: cfg,
tlsCfg: tlsCfg,
conns: conns,
stop: make(chan struct{}),
factory: f,
}
l.nat.Store(stun.NATUnknown)
return l
}
func (quicListenerFactory) Enabled(cfg config.Configuration) bool {
return true
}

57
lib/connections/quic_misc.go Normal file
View File

@@ -0,0 +1,57 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build go1.12
package connections
import (
"net"
"github.com/lucas-clemente/quic-go"
)
var (
quicConfig = &quic.Config{
ConnectionIDLength: 4,
KeepAlive: true,
}
)
type quicTlsConn struct {
quic.Session
quic.Stream
}
func (q *quicTlsConn) Close() error {
sterr := q.Stream.Close()
seerr := q.Session.Close()
if sterr != nil {
return sterr
}
return seerr
}
// Sort available packet connections by ip address, preferring unspecified local address.
func packetConnLess(i interface{}, j interface{}) bool {
iIsUnspecified := false
jIsUnspecified := false
iLocalAddr := i.(net.PacketConn).LocalAddr()
jLocalAddr := j.(net.PacketConn).LocalAddr()
if host, _, err := net.SplitHostPort(iLocalAddr.String()); err == nil {
iIsUnspecified = host == "" || net.ParseIP(host).IsUnspecified()
}
if host, _, err := net.SplitHostPort(jLocalAddr.String()); err == nil {
jIsUnspecified = host == "" || net.ParseIP(host).IsUnspecified()
}
if jIsUnspecified == iIsUnspecified {
return len(iLocalAddr.Network()) < len(jLocalAddr.Network())
}
return iIsUnspecified
}

92
lib/connections/quic_misc_test.go Normal file
View File

@@ -0,0 +1,92 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build go1.12
package connections
import (
"net"
"testing"
"time"
)
type mockPacketConn struct {
addr mockedAddr
}
func (mockPacketConn) ReadFrom(p []byte) (n int, addr net.Addr, err error) {
panic("implement me")
}
func (mockPacketConn) WriteTo(p []byte, addr net.Addr) (n int, err error) {
panic("implement me")
}
func (mockPacketConn) Close() error {
panic("implement me")
}
func (c *mockPacketConn) LocalAddr() net.Addr {
return c.addr
}
func (mockPacketConn) SetDeadline(t time.Time) error {
panic("implement me")
}
func (mockPacketConn) SetReadDeadline(t time.Time) error {
panic("implement me")
}
func (mockPacketConn) SetWriteDeadline(t time.Time) error {
panic("implement me")
}
type mockedAddr struct {
network string
addr string
}
func (a mockedAddr) Network() string {
return a.network
}
func (a mockedAddr) String() string {
return a.addr
}
func TestPacketConnLess(t *testing.T) {
cases := []struct {
netA string
addrA string
netB string
addrB string
}{
// B is assumed the winner.
{"tcp", "127.0.0.1:1234", "tcp", ":1235"},
{"tcp", "127.0.0.1:1234", "tcp", "0.0.0.0:1235"},
{"tcp4", "0.0.0.0:1234", "tcp", "0.0.0.0:1235"}, // tcp4 on the first one
}
for i, testCase := range cases {
conns := []*mockPacketConn{
{mockedAddr{testCase.netA, testCase.addrA}},
{mockedAddr{testCase.netB, testCase.addrB}},
}
if packetConnLess(conns[0], conns[1]) {
t.Error(i, "unexpected")
}
if !packetConnLess(conns[1], conns[0]) {
t.Error(i, "unexpected")
}
if packetConnLess(conns[0], conns[0]) || packetConnLess(conns[1], conns[1]) {
t.Error(i, "unexpected")
}
}
}

89
lib/connections/registry/registry.go Normal file
View File

@@ -0,0 +1,89 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// Registry tracks connections/addresses on which we are listening on, to allow us to pick a connection/address that
// has a NAT port mapping. This also makes our outgoing port stable and same as incoming port which should allow
// better probability of punching through.
package registry
import (
"sort"
"strings"
"github.com/syncthing/syncthing/lib/sync"
)
var (
Default = New()
)
type Registry struct {
mut sync.Mutex
available map[string][]interface{}
}
func New() *Registry {
return &Registry{
mut: sync.NewMutex(),
available: make(map[string][]interface{}),
}
}
func (r *Registry) Register(scheme string, item interface{}) {
r.mut.Lock()
defer r.mut.Unlock()
r.available[scheme] = append(r.available[scheme], item)
}
func (r *Registry) Unregister(scheme string, item interface{}) {
r.mut.Lock()
defer r.mut.Unlock()
candidates := r.available[scheme]
for i, existingItem := range candidates {
if existingItem == item {
copy(candidates[i:], candidates[i+1:])
candidates[len(candidates)-1] = nil
r.available[scheme] = candidates[:len(candidates)-1]
break
}
}
}
func (r *Registry) Get(scheme string, less func(i, j interface{}) bool) interface{} {
r.mut.Lock()
defer r.mut.Unlock()
candidates := make([]interface{}, 0)
for availableScheme, items := range r.available {
// quic:// should be considered ok for both quic4:// and quic6://
if strings.HasPrefix(scheme, availableScheme) {
candidates = append(candidates, items...)
}
}
if len(candidates) == 0 {
return nil
}
sort.Slice(candidates, func(i, j int) bool {
return less(candidates[i], candidates[j])
})
return candidates[0]
}
func Register(scheme string, item interface{}) {
Default.Register(scheme, item)
}
func Unregister(scheme string, item interface{}) {
Default.Unregister(scheme, item)
}
func Get(scheme string, less func(i, j interface{}) bool) interface{} {
return Default.Get(scheme, less)
}

71
lib/connections/registry/registry_test.go Normal file
View File

@@ -0,0 +1,71 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package registry
import (
"testing"
)
func TestRegistry(t *testing.T) {
r := New()
if res := r.Get("int", intLess); res != nil {
t.Error("unexpected")
}
r.Register("int", 1)
r.Register("int", 11)
r.Register("int4", 4)
r.Register("int4", 44)
r.Register("int6", 6)
r.Register("int6", 66)
if res := r.Get("int", intLess).(int); res != 1 {
t.Error("unexpected", res)
}
// int is prefix of int4, so returns 1
if res := r.Get("int4", intLess).(int); res != 1 {
t.Error("unexpected", res)
}
r.Unregister("int", 1)
// Check that falls through to 11
if res := r.Get("int", intLess).(int); res != 11 {
t.Error("unexpected", res)
}
// 6 is smaller than 11 available in int.
if res := r.Get("int6", intLess).(int); res != 6 {
t.Error("unexpected", res)
}
// Unregister 11, int should be impossible to find
r.Unregister("int", 11)
if res := r.Get("int", intLess); res != nil {
t.Error("unexpected")
}
// Unregister a second time does nothing.
r.Unregister("int", 1)
// Can have multiple of the same
r.Register("int", 1)
r.Register("int", 1)
r.Unregister("int", 1)
if res := r.Get("int4", intLess).(int); res != 1 {
t.Error("unexpected", res)
}
}
func intLess(i, j interface{}) bool {
iInt := i.(int)
jInt := j.(int)
return iInt < jInt
}

2
lib/connections/relay_dial.go
View File

@@ -17,6 +17,8 @@ import (
"github.com/syncthing/syncthing/lib/relay/client"
)
const relayPriority = 200
func init() {
dialers["relay"] = relayDialerFactory{}
}

6
lib/connections/service.go
View File

@@ -375,7 +375,7 @@ func (s *service) connect() {
}
}
addrs = util.UniqueStrings(addrs)
addrs = util.UniqueTrimmedStrings(addrs)
l.Debugln("Reconnect loop for", deviceID, addrs)
@@ -642,7 +642,7 @@ func (s *service) AllAddresses() []string {
}
}
s.listenersMut.RUnlock()
return util.UniqueStrings(addrs)
return util.UniqueTrimmedStrings(addrs)
}
func (s *service) ExternalAddresses() []string {
@@ -654,7 +654,7 @@ func (s *service) ExternalAddresses() []string {
}
}
s.listenersMut.RUnlock()
return util.UniqueStrings(addrs)
return util.UniqueTrimmedStrings(addrs)
}
func (s *service) ListenerStatus() map[string]ListenerStatusEntry {

24
lib/connections/structs.go
View File

@@ -9,6 +9,7 @@ package connections
import (
"crypto/tls"
"fmt"
"io"
"net"
"net/url"
"time"
@@ -43,10 +44,19 @@ func (c completeConn) Close(err error) {
c.internalConn.Close()
}
type tlsConn interface {
io.ReadWriteCloser
ConnectionState() tls.ConnectionState
RemoteAddr() net.Addr
SetDeadline(time.Time) error
SetWriteDeadline(time.Time) error
LocalAddr() net.Addr
}
// internalConn is the raw TLS connection plus some metadata on where it
// came from (type, priority).
type internalConn struct {
*tls.Conn
tlsConn
connType connType
priority int
}
@@ -58,6 +68,8 @@ const (
connTypeRelayServer
connTypeTCPClient
connTypeTCPServer
connTypeQUICClient
connTypeQUICServer
)
func (t connType) String() string {
@@ -70,6 +82,10 @@ func (t connType) String() string {
return "tcp-client"
case connTypeTCPServer:
return "tcp-server"
case connTypeQUICClient:
return "quic-client"
case connTypeQUICServer:
return "quic-server"
default:
return "unknown-type"
}
@@ -81,6 +97,8 @@ func (t connType) Transport() string {
return "relay"
case connTypeTCPClient, connTypeTCPServer:
return "tcp"
case connTypeQUICClient, connTypeQUICServer:
return "quic"
default:
return "unknown"
}
@@ -90,8 +108,8 @@ func (c internalConn) Close() {
// *tls.Conn.Close() does more than it says on the tin. Specifically, it
// sends a TLS alert message, which might block forever if the
// connection is dead and we don't have a deadline set.
c.SetWriteDeadline(time.Now().Add(250 * time.Millisecond))
c.Conn.Close()
_ = c.SetWriteDeadline(time.Now().Add(250 * time.Millisecond))
_ = c.tlsConn.Close()
}
func (c internalConn) Type() string {

2
lib/connections/tcp_dial.go
View File

@@ -16,6 +16,8 @@ import (
"github.com/syncthing/syncthing/lib/protocol"
)
const tcpPriority = 10
func init() {
factory := &tcpDialerFactory{}
for _, scheme := range []string{"tcp", "tcp4", "tcp6"} {

8
lib/discover/cache.go
View File

@@ -7,6 +7,7 @@
package discover
import (
"sort"
stdsync "sync"
"time"
@@ -121,11 +122,12 @@ func (m *cachingMux) Lookup(deviceID protocol.DeviceID) (addresses []string, err
}
m.mut.RUnlock()
addresses = util.UniqueTrimmedStrings(addresses)
sort.Strings(addresses)
l.Debugln("lookup results for", deviceID)
l.Debugln(" addresses: ", addresses)
addresses = util.UniqueStrings(addresses)
return addresses, nil
}
@@ -185,7 +187,7 @@ func (m *cachingMux) Cache() map[protocol.DeviceID]CacheEntry {
m.mut.RUnlock()
for k, v := range res {
v.Addresses = util.UniqueStrings(v.Addresses)
v.Addresses = util.UniqueTrimmedStrings(v.Addresses)
res[k] = v
}

7
lib/model/folder_sendrecv.go
View File

@@ -9,7 +9,6 @@ package model
import (
"bytes"
"fmt"
"math/rand"
"path/filepath"
"runtime"
"sort"
@@ -25,6 +24,7 @@ import (
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/sha256"
"github.com/syncthing/syncthing/lib/sync"
@@ -1089,10 +1089,7 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
}
// Shuffle the blocks
for i := range blocks {
j := rand.Intn(i + 1)
blocks[i], blocks[j] = blocks[j], blocks[i]
}
rand.Shuffle(blocks)
events.Default.Log(events.ItemStarted, map[string]string{
"folder": f.folderID,

8
lib/model/queue.go
View File

@@ -7,10 +7,10 @@
package model
import (
"math/rand"
"sort"
"time"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sync"
)
@@ -103,11 +103,7 @@ func (q *jobQueue) Shuffle() {
q.mut.Lock()
defer q.mut.Unlock()
l := len(q.queued)
for i := range q.queued {
r := rand.Intn(l)
q.queued[i], q.queued[r] = q.queued[r], q.queued[i]
}
rand.Shuffle(q.queued)
}
func (q *jobQueue) Reset() {

12
lib/rand/random.go
View File

@@ -14,6 +14,7 @@ import (
"encoding/binary"
"io"
mathRand "math/rand"
"reflect"
)
// Reader is the standard crypto/rand.Reader, re-exported for convenience
@@ -73,3 +74,14 @@ func SeedFromBytes(bs []byte) int64 {
// uint64s and XOR them together.
return int64(binary.BigEndian.Uint64(s[0:]) ^ binary.BigEndian.Uint64(s[8:]))
}
// Shuffle the order of elements
func Shuffle(slice interface{}) {
rv := reflect.ValueOf(slice)
swap := reflect.Swapper(slice)
length := rv.Len()
if length < 2 {
return
}
defaultSecureRand.Shuffle(length, swap)
}

11
lib/relay/client/dynamic.go
View File

@@ -6,13 +6,13 @@ import (
"crypto/tls"
"encoding/json"
"fmt"
"math/rand"
"net/http"
"net/url"
"sort"
"time"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/relay/protocol"
"github.com/syncthing/syncthing/lib/sync"
)
@@ -209,7 +209,7 @@ func relayAddressesOrder(input []string) []string {
var ids []int
for id, bucket := range buckets {
shuffle(bucket)
rand.Shuffle(bucket)
ids = append(ids, id)
}
@@ -223,10 +223,3 @@ func relayAddressesOrder(input []string) []string {
return addresses
}
func shuffle(slice []string) {
for i := len(slice) - 1; i > 0; i-- {
j := rand.Intn(i + 1)
slice[i], slice[j] = slice[j], slice[i]
}
}

22
lib/stun/debug.go Normal file
View File

@@ -0,0 +1,22 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package stun
import (
"os"
"strings"
"github.com/syncthing/syncthing/lib/logger"
)
var (
l = logger.DefaultLogger.NewFacility("stun", "STUN functionality")
)
func init() {
l.SetDebug("stun", strings.Contains(os.Getenv("STTRACE"), "stun") || os.Getenv("STTRACE") == "all")
}

64
lib/stun/filter.go Normal file
View File

@@ -0,0 +1,64 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package stun
import (
"bytes"
"net"
"sync"
"time"
)
const (
stunFilterPriority = 10
otherDataPriority = 100
)
type stunFilter struct {
ids map[string]time.Time
mut sync.Mutex
}
func (f *stunFilter) Outgoing(out []byte, addr net.Addr) {
if !f.isStunPayload(out) {
panic("not a stun payload")
}
f.mut.Lock()
f.ids[string(out[8:20])] = time.Now().Add(time.Minute)
f.reap()
f.mut.Unlock()
}
func (f *stunFilter) ClaimIncoming(in []byte, addr net.Addr) bool {
if f.isStunPayload(in) {
f.mut.Lock()
_, ok := f.ids[string(in[8:20])]
f.reap()
f.mut.Unlock()
return ok
}
return false
}
func (f *stunFilter) isStunPayload(data []byte) bool {
// Need at least 20 bytes
if len(data) < 20 {
return false
}
// First two bits always unset, and should always send magic cookie.
return data[0]&0xc0 == 0 && bytes.Equal(data[4:8], []byte{0x21, 0x12, 0xA4, 0x42})
}
func (f *stunFilter) reap() {
now := time.Now()
for id, timeout := range f.ids {
if timeout.Before(now) {
delete(f.ids, id)
}
}
}

310
lib/stun/stun.go Normal file
View File

@@ -0,0 +1,310 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package stun
import (
"net"
"sync/atomic"
"time"
"github.com/AudriusButkevicius/pfilter"
"github.com/ccding/go-stun/stun"
"github.com/syncthing/syncthing/lib/config"
)
const stunRetryInterval = 5 * time.Minute
type Host = stun.Host
type NATType = stun.NATType
// NAT types.
const (
NATError = stun.NATError
NATUnknown = stun.NATUnknown
NATNone = stun.NATNone
NATBlocked = stun.NATBlocked
NATFull = stun.NATFull
NATSymmetric = stun.NATSymmetric
NATRestricted = stun.NATRestricted
NATPortRestricted = stun.NATPortRestricted
NATSymmetricUDPFirewall = stun.NATSymmetricUDPFirewall
)
type writeTrackingPacketConn struct {
lastWrite int64
net.PacketConn
}
func (c *writeTrackingPacketConn) WriteTo(p []byte, addr net.Addr) (n int, err error) {
atomic.StoreInt64(&c.lastWrite, time.Now().Unix())
return c.PacketConn.WriteTo(p, addr)
}
func (c *writeTrackingPacketConn) getLastWrite() time.Time {
unix := atomic.LoadInt64(&c.lastWrite)
return time.Unix(unix, 0)
}
type Subscriber interface {
OnNATTypeChanged(natType NATType)
OnExternalAddressChanged(address *Host, via string)
}
type Service struct {
name string
cfg config.Wrapper
subscriber Subscriber
stunConn net.PacketConn
client *stun.Client
writeTrackingPacketConn *writeTrackingPacketConn
natType NATType
addr *Host
stop chan struct{}
}
func New(cfg config.Wrapper, subscriber Subscriber, conn net.PacketConn) (*Service, net.PacketConn) {
// Wrap the original connection to track writes on it
writeTrackingPacketConn := &writeTrackingPacketConn{lastWrite: 0, PacketConn: conn}
// Wrap it in a filter and split it up, so that stun packets arrive on stun conn, others arrive on the data conn
filterConn := pfilter.NewPacketFilter(writeTrackingPacketConn)
otherDataConn := filterConn.NewConn(otherDataPriority, nil)
stunConn := filterConn.NewConn(stunFilterPriority, &stunFilter{
ids: make(map[string]time.Time),
})
filterConn.Start()
// Construct the client to use the stun conn
client := stun.NewClientWithConnection(stunConn)
client.SetSoftwareName("") // Explicitly unset this, seems to freak some servers out.
// Return the service and the other conn to the client
return &Service{
name: "Stun@" + conn.LocalAddr().Network() + "://" + conn.LocalAddr().String(),
cfg: cfg,
subscriber: subscriber,
stunConn: stunConn,
client: client,
writeTrackingPacketConn: writeTrackingPacketConn,
natType: NATUnknown,
addr: nil,
stop: make(chan struct{}),
}, otherDataConn
}
func (s *Service) Stop() {
close(s.stop)
_ = s.stunConn.Close()
}
func (s *Service) Serve() {
for {
disabled:
s.setNATType(NATUnknown)
s.setExternalAddress(nil, "")
if s.cfg.Options().IsStunDisabled() {
select {
case <-s.stop:
return
case <-time.After(time.Second):
continue
}
}
l.Debugf("Starting stun for %s", s)
for _, addr := range s.cfg.StunServers() {
// This blocks until we hit an exit condition or there are issues with the STUN server.
// This returns a boolean signifying if a different STUN server should be tried (oppose to the whole thing
// shutting down and this winding itself down.
if !s.runStunForServer(addr) {
// Check exit conditions.
// Have we been asked to stop?
select {
case <-s.stop:
return
default:
}
// Are we disabled?
if s.cfg.Options().IsStunDisabled() {
l.Infoln("STUN disabled")
goto disabled
}
// Unpunchable NAT? Chillout for some time.
if !s.isCurrentNATTypePunchable() {
break
}
}
}
// Failed all servers, sad.
s.setNATType(NATUnknown)
s.setExternalAddress(nil, "")
// We failed to contact all provided stun servers or the nat is not punchable.
// Chillout for a while.
time.Sleep(stunRetryInterval)
}
}
func (s *Service) runStunForServer(addr string) (tryNext bool) {
l.Debugf("Running stun for %s via %s", s, addr)
// Resolve the address, so that in case the server advertises two
// IPs, we always hit the same one, as otherwise, the mapping might
// expire as we hit the other address, and cause us to flip flop
// between servers/external addresses, as a result flooding discovery
// servers.
udpAddr, err := net.ResolveUDPAddr("udp", addr)
if err != nil {
l.Debugf("%s stun addr resolution on %s: %s", s, addr, err)
return true
}
s.client.SetServerAddr(udpAddr.String())
natType, extAddr, err := s.client.Discover()
if err != nil || extAddr == nil {
l.Debugf("%s stun discovery on %s: %s", s, addr, err)
return true
}
// The stun server is most likely borked, try another one.
if natType == NATError || natType == NATUnknown || natType == NATBlocked {
l.Debugf("%s stun discovery on %s resolved to %s", s, addr, natType)
return true
}
s.setNATType(natType)
s.setExternalAddress(extAddr, addr)
l.Debugf("%s detected NAT type: %s via %s", s, natType, addr)
// We can't punch through this one, so no point doing keepalives
// and such, just let the caller check the nat type and work it out themselves.
if !s.isCurrentNATTypePunchable() {
l.Debugf("%s cannot punch %s, skipping", s, natType)
return false
}
return s.stunKeepAlive(addr, extAddr)
}
func (s *Service) stunKeepAlive(addr string, extAddr *Host) (tryNext bool) {
var err error
nextSleep := time.Duration(s.cfg.Options().StunKeepaliveStartS) * time.Second
l.Debugf("%s starting stun keepalive via %s, next sleep %s", s, addr, nextSleep)
for {
if areDifferent(s.addr, extAddr) {
// If the port has changed (addresses are not equal but the hosts are equal),
// we're probably spending too much time between keepalives, reduce the sleep.
if s.addr != nil && extAddr != nil && s.addr.IP() == extAddr.IP() {
nextSleep /= 2
l.Debugf("%s stun port change (%s to %s), next sleep %s", s, s.addr.TransportAddr(), extAddr.TransportAddr(), nextSleep)
}
s.setExternalAddress(extAddr, addr)
// The stun server is probably stuffed, we've gone beyond min timeout, yet the address keeps changing.
minSleep := time.Duration(s.cfg.Options().StunKeepaliveMinS) * time.Second
if nextSleep < minSleep {
l.Debugf("%s keepalive aborting, sleep below min: %s < %s", s, nextSleep, minSleep)
return true
}
}
// Adjust the keepalives to fire only nextSleep after last write.
lastWrite := s.writeTrackingPacketConn.getLastWrite()
minSleep := time.Duration(s.cfg.Options().StunKeepaliveMinS) * time.Second
if nextSleep < minSleep {
nextSleep = minSleep
}
tryLater:
sleepFor := nextSleep
timeUntilNextKeepalive := time.Until(lastWrite.Add(sleepFor))
if timeUntilNextKeepalive > 0 {
sleepFor = timeUntilNextKeepalive
}
l.Debugf("%s stun sleeping for %s", s, sleepFor)
select {
case <-time.After(sleepFor):
case <-s.stop:
l.Debugf("%s stopping, aborting stun", s)
return false
}
if s.cfg.Options().IsStunDisabled() {
// Disabled, give up
l.Debugf("%s disabled, aborting stun ", s)
return false
}
// Check if any writes happened while we were sleeping, if they did, sleep again
lastWrite = s.writeTrackingPacketConn.getLastWrite()
if gap := time.Since(lastWrite); gap < nextSleep {
l.Debugf("%s stun last write gap less than next sleep: %s < %s. Will try later", s, gap, nextSleep)
goto tryLater
}
l.Debugf("%s stun keepalive", s)
extAddr, err = s.client.Keepalive()
if err != nil {
l.Debugf("%s stun keepalive on %s: %s (%v)", s, addr, err, extAddr)
return true
}
}
}
func (s *Service) setNATType(natType NATType) {
if natType != s.natType {
l.Debugf("Notifying %s of NAT type change: %s", s.subscriber, natType)
s.subscriber.OnNATTypeChanged(natType)
}
s.natType = natType
}
func (s *Service) setExternalAddress(addr *Host, via string) {
if areDifferent(s.addr, addr) {
l.Debugf("Notifying %s of address change: %s via %s", s.subscriber, addr, via)
s.subscriber.OnExternalAddressChanged(addr, via)
}
s.addr = addr
}
func (s *Service) String() string {
return s.name
}
func (s *Service) isCurrentNATTypePunchable() bool {
return s.natType == NATNone || s.natType == NATPortRestricted || s.natType == NATRestricted || s.natType == NATFull
}
func areDifferent(first, second *Host) bool {
if (first == nil) != (second == nil) {
return true
}
if first != nil {
return first.TransportAddr() != second.TransportAddr()
}
return false
}

25
lib/util/utils.go
View File

@@ -10,7 +10,6 @@ import (
"fmt"
"net/url"
"reflect"
"sort"
"strconv"
"strings"
)
@@ -111,21 +110,23 @@ func CopyMatchingTag(from interface{}, to interface{}, tag string, shouldCopy fu
}
}
// UniqueStrings returns a list on unique strings, trimming and sorting them
// at the same time.
func UniqueStrings(ss []string) []string {
var m = make(map[string]bool, len(ss))
for _, s := range ss {
m[strings.Trim(s, " ")] = true
// UniqueTrimmedStrings returns a list on unique strings, trimming at the same time.
func UniqueTrimmedStrings(ss []string) []string {
// Trim all first
for i, v := range ss {
ss[i] = strings.Trim(v, " ")
}
var us = make([]string, 0, len(m))
for k := range m {
us = append(us, k)
var m = make(map[string]struct{}, len(ss))
var us = make([]string, 0, len(ss))
for _, v := range ss {
if _, ok := m[v]; ok {
continue
}
m[v] = struct{}{}
us = append(us, v)
}
sort.Strings(us)
return us
}

6
lib/util/utils_test.go
View File

@@ -74,10 +74,6 @@ func TestUniqueStrings(t *testing.T) {
nil,
nil,
},
{
[]string{"b", "a"},
[]string{"a", "b"},
},
{
[]string{" a ", " a ", "b ", " b"},
[]string{"a", "b"},
@@ -85,7 +81,7 @@ func TestUniqueStrings(t *testing.T) {
}
for _, test := range tests {
result := UniqueStrings(test.input)
result := UniqueTrimmedStrings(test.input)
if len(result) != len(test.expected) {
t.Errorf("%s != %s", result, test.expected)
}

4
lib/versioner/simple.go
View File

@@ -8,6 +8,7 @@ package versioner
import (
"path/filepath"
"sort"
"strconv"
"time"
@@ -71,7 +72,8 @@ func (v Simple) Archive(filePath string) error {
// Use all the found filenames. "~" sorts after "." so all old pattern
// files will be deleted before any new, which is as it should be.
versions := util.UniqueStrings(append(oldVersions, newVersions...))
versions := util.UniqueTrimmedStrings(append(oldVersions, newVersions...))
sort.Strings(versions)
if len(versions) > v.keep {
for _, toRemove := range versions[:len(versions)-v.keep] {

5
lib/versioner/staggered.go
View File

@@ -8,6 +8,7 @@ package versioner
import (
"path/filepath"
"sort"
"strconv"
"time"
@@ -242,7 +243,9 @@ func (v *Staggered) Archive(filePath string) error {
// Use all the found filenames.
versions := append(oldVersions, newVersions...)
v.expire(util.UniqueStrings(versions))
versions = util.UniqueTrimmedStrings(versions)
sort.Strings(versions)
v.expire(versions)
return nil
}