lib/protocol, lib/discover, lib/db: Use protocol buffer serialization (fixes #3080)

This changes the BEP protocol to use protocol buffer serialization
instead of XDR, and therefore also the database format. The local
discovery protocol is also updated to be protocol buffer format.

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3276
LGTM: AudriusButkevicius

lib/model/model.go

@@ -40,10 +40,8 @@ import (
 
 // How many files to send in each Index/IndexUpdate message.
 const (
-	indexTargetSize   = 250 * 1024 // Aim for making index messages no larger than 250 KiB (uncompressed)
-	indexPerFileSize  = 250        // Each FileInfo is approximately this big, in bytes, excluding BlockInfos
-	indexPerBlockSize = 40         // Each BlockInfo is approximately this big
-	indexBatchSize    = 1000       // Either way, don't include more files than this
+	indexTargetSize = 250 * 1024 // Aim for making index messages no larger than 250 KiB (uncompressed)
+	indexBatchSize  = 1000       // Either way, don't include more files than this
 )
 
 type service interface {
@@ -95,7 +93,7 @@ type Model struct {
 
 	conn              map[protocol.DeviceID]connections.Connection
 	helloMessages     map[protocol.DeviceID]protocol.HelloResult
-	deviceClusterConf map[protocol.DeviceID]protocol.ClusterConfigMessage
+	deviceClusterConf map[protocol.DeviceID]protocol.ClusterConfig
 	devicePaused      map[protocol.DeviceID]bool
 	deviceDownloads   map[protocol.DeviceID]*deviceDownloadState
 	pmut              sync.RWMutex // protects the above
@@ -149,7 +147,7 @@ func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName,
 		folderStatRefs:     make(map[string]*stats.FolderStatisticsReference),
 		conn:               make(map[protocol.DeviceID]connections.Connection),
 		helloMessages:      make(map[protocol.DeviceID]protocol.HelloResult),
-		deviceClusterConf:  make(map[protocol.DeviceID]protocol.ClusterConfigMessage),
+		deviceClusterConf:  make(map[protocol.DeviceID]protocol.ClusterConfig),
 		devicePaused:       make(map[protocol.DeviceID]bool),
 		deviceDownloads:    make(map[protocol.DeviceID]*deviceDownloadState),
 		fmut:               sync.NewRWMutex(),
@@ -414,7 +412,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
 		// This might might be more than it really is, because some blocks can be of a smaller size.
 		downloaded = int64(counts[ft.Name] * protocol.BlockSize)
 
-		fileNeed = ft.Size() - downloaded
+		fileNeed = ft.Size - downloaded
 		if fileNeed < 0 {
 			fileNeed = 0
 		}
@@ -436,7 +434,7 @@ func sizeOfFile(f db.FileIntf) (files, deleted int, bytes int64) {
 	} else {
 		deleted++
 	}
-	bytes += f.Size()
+	bytes += f.FileSize()
 	return
 }
 
@@ -548,12 +546,7 @@ func (m *Model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
 
 // Index is called when a new device is connected and we receive their full index.
 // Implements the protocol.Model interface.
-func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo, flags uint32, options []protocol.Option) {
-	if flags != 0 {
-		l.Warnln("protocol error: unknown flags 0x%x in Index message", flags)
-		return
-	}
-
+func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) {
 	l.Debugf("IDX(in): %s %q: %d files", deviceID, folder, len(fs))
 
 	if !m.folderSharedWith(folder, deviceID) {
@@ -595,12 +588,7 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F
 
 // IndexUpdate is called for incremental updates to connected devices' indexes.
 // Implements the protocol.Model interface.
-func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo, flags uint32, options []protocol.Option) {
-	if flags != 0 {
-		l.Warnln("protocol error: unknown flags 0x%x in IndexUpdate message", flags)
-		return
-	}
-
+func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) {
 	l.Debugf("%v IDXUP(in): %s / %q: %d files", m, deviceID, folder, len(fs))
 
 	if !m.folderSharedWith(folder, deviceID) {
@@ -651,7 +639,7 @@ func (m *Model) folderSharedWithUnlocked(folder string, deviceID protocol.Device
 	return false
 }
 
-func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterConfigMessage) {
+func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterConfig) {
 	// Check the peer device's announced folders against our own. Emits events
 	// for folders that we don't expect (unknown or not shared).
 	// Also, collect a list of folders we do share, and if he's interested in
@@ -659,19 +647,9 @@ func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterCon
 
 	tempIndexFolders := make([]string, 0, len(cm.Folders))
 
+	m.fmut.Lock()
 	for _, folder := range cm.Folders {
-		if folder.Flags&^protocol.FlagFolderAll != 0 {
-			// There are flags set that we don't know what they mean. Fatal!
-			l.Warnf("Device %v: unknown flags for folder %s", deviceID, folder.ID)
-			m.fmut.Unlock()
-			m.Close(deviceID, fmt.Errorf("Unknown folder flags from device %v", deviceID))
-			return
-		}
-
-		m.fmut.Lock()
-		shared := m.folderSharedWithUnlocked(folder.ID, deviceID)
-		m.fmut.Unlock()
-		if !shared {
+		if !m.folderSharedWithUnlocked(folder.ID, deviceID) {
 			events.Default.Log(events.FolderRejected, map[string]string{
 				"folder":      folder.ID,
 				"folderLabel": folder.Label,
@@ -680,10 +658,11 @@ func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterCon
 			l.Infof("Unexpected folder ID %q sent from device %q; ensure that the folder exists and that this device is selected under \"Share With\" in the folder configuration.", folder.ID, deviceID)
 			continue
 		}
-		if folder.Flags&protocol.FlagFolderDisabledTempIndexes == 0 {
+		if !folder.DisableTempIndexes {
 			tempIndexFolders = append(tempIndexFolders, folder.ID)
 		}
 	}
+	m.fmut.Unlock()
 
 	// This breaks if we send multiple CM messages during the same connection.
 	if len(tempIndexFolders) > 0 {
@@ -733,7 +712,7 @@ func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterCon
 					}
 
 					// The introducers' introducers are also our introducers.
-					if device.Flags&protocol.FlagIntroducer != 0 {
+					if device.Introducer {
 						l.Infof("Device %v is now also an introducer", id)
 						newDeviceCfg.Introducer = true
 					}
@@ -804,7 +783,7 @@ func (m *Model) Close(device protocol.DeviceID, err error) {
 
 // Request returns the specified data segment by reading it from local disk.
 // Implements the protocol.Model interface.
-func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, flags uint32, options []protocol.Option, buf []byte) error {
+func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
 	if offset < 0 {
 		return protocol.ErrInvalid
 	}
@@ -813,14 +792,8 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 		l.Warnf("Request from %s for file %s in unshared folder %q", deviceID, name, folder)
 		return protocol.ErrNoSuchFile
 	}
-
-	if flags != 0 && flags != protocol.FlagFromTemporary {
-		// We currently support only no flags, or FromTemporary flag.
-		return fmt.Errorf("protocol error: unknown flags 0x%x in Request message", flags)
-	}
-
 	if deviceID != protocol.LocalDeviceID {
-		l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d f=%d", m, deviceID, folder, name, offset, len(buf), flags)
+		l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d t=%v", m, deviceID, folder, name, offset, len(buf), fromTemporary)
 	}
 	m.fmut.RLock()
 	folderCfg := m.folderCfgs[folder]
@@ -880,7 +853,7 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 
 	// Only check temp files if the flag is set, and if we are set to advertise
 	// the temp indexes.
-	if flags&protocol.FlagFromTemporary != 0 && !folderCfg.DisableTempIndexes {
+	if fromTemporary && !folderCfg.DisableTempIndexes {
 		tempFn := filepath.Join(folderPath, defTempNamer.TempName(name))
 		if err := readOffsetIntoBuf(tempFn, offset, buf); err == nil {
 			return nil
@@ -1027,8 +1000,8 @@ func (m *Model) OnHello(remoteID protocol.DeviceID, addr net.Addr, hello protoco
 }
 
 // GetHello is called when we are about to connect to some remote device.
-func (m *Model) GetHello(protocol.DeviceID) protocol.Version13HelloMessage {
-	return protocol.Version13HelloMessage{
+func (m *Model) GetHello(protocol.DeviceID) protocol.HelloIntf {
+	return &protocol.Hello{
 		DeviceName:    m.deviceName,
 		ClientName:    m.clientName,
 		ClientVersion: m.clientVersion,
@@ -1101,7 +1074,7 @@ func (m *Model) PauseDevice(device protocol.DeviceID) {
 	events.Default.Log(events.DevicePaused, map[string]string{"device": device.String()})
 }
 
-func (m *Model) DownloadProgress(device protocol.DeviceID, folder string, updates []protocol.FileDownloadProgressUpdate, flags uint32, options []protocol.Option) {
+func (m *Model) DownloadProgress(device protocol.DeviceID, folder string, updates []protocol.FileDownloadProgressUpdate) {
 	if !m.folderSharedWith(folder, device) {
 		return
 	}
@@ -1238,13 +1211,13 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold
 
 		if len(batch) == indexBatchSize || currentBatchSize > indexTargetSize {
 			if initial {
-				if err = conn.Index(folder, batch, 0, nil); err != nil {
+				if err = conn.Index(folder, batch); err != nil {
 					return false
 				}
 				l.Debugf("sendIndexes for %s-%s/%q: %d files (<%d bytes) (initial index)", deviceID, name, folder, len(batch), currentBatchSize)
 				initial = false
 			} else {
-				if err = conn.IndexUpdate(folder, batch, 0, nil); err != nil {
+				if err = conn.IndexUpdate(folder, batch); err != nil {
 					return false
 				}
 				l.Debugf("sendIndexes for %s-%s/%q: %d files (<%d bytes) (batched update)", deviceID, name, folder, len(batch), currentBatchSize)
@@ -1255,17 +1228,17 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold
 		}
 
 		batch = append(batch, f)
-		currentBatchSize += indexPerFileSize + len(f.Blocks)*indexPerBlockSize
+		currentBatchSize += f.ProtoSize()
 		return true
 	})
 
 	if initial && err == nil {
-		err = conn.Index(folder, batch, 0, nil)
+		err = conn.Index(folder, batch)
 		if err == nil {
 			l.Debugf("sendIndexes for %s-%s/%q: %d files (small initial index)", deviceID, name, folder, len(batch))
 		}
 	} else if len(batch) > 0 && err == nil {
-		err = conn.IndexUpdate(folder, batch, 0, nil)
+		err = conn.IndexUpdate(folder, batch)
 		if err == nil {
 			l.Debugf("sendIndexes for %s-%s/%q: %d files (last batch)", deviceID, name, folder, len(batch))
 		}
@@ -1320,11 +1293,14 @@ func (m *Model) localChangeDetected(folder, path string, files []protocol.FileIn
 		objType := "file"
 		action := "modified"
 
-		// If our local vector is verison 1 AND it is the only version vector so far seen for this file then
-		// it is a new file.  Else if it is > 1 it's not new, and if it is 1 but another shortId version vector
-		// exists then it is new for us but created elsewhere so the file is still not new but modified by us.
-		// Only if it is truly new do we change this to 'added', else we leave it as 'modified'.
-		if len(file.Version) == 1 && file.Version[0].Value == 1 {
+		// If our local vector is verison 1 AND it is the only version
+		// vector so far seen for this file then it is a new file.  Else if
+		// it is > 1 it's not new, and if it is 1 but another shortId
+		// version vector exists then it is new for us but created elsewhere
+		// so the file is still not new but modified by us. Only if it is
+		// truly new do we change this to 'added', else we leave it as
+		// 'modified'.
+		if len(file.Version.Counters) == 1 && file.Version.Counters[0].Value == 1 {
 			action = "added"
 		}
 
@@ -1579,10 +1555,14 @@ func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error
 					// File has been ignored or an unsupported symlink. Set invalid bit.
 					l.Debugln("setting invalid bit on ignored", f)
 					nf := protocol.FileInfo{
-						Name:     f.Name,
-						Flags:    f.Flags | protocol.FlagInvalid,
-						Modified: f.Modified,
-						Version:  f.Version, // The file is still the same, so don't bump version
+						Name:          f.Name,
+						Type:          f.Type,
+						Size:          f.Size,
+						Modified:      f.Modified,
+						Permissions:   f.Permissions,
+						NoPermissions: f.NoPermissions,
+						Invalid:       true,
+						Version:       f.Version, // The file is still the same, so don't bump version
 					}
 					batch = append(batch, nf)
 				} else if _, err := osutil.Lstat(filepath.Join(folderCfg.Path(), f.Name)); err != nil {
@@ -1597,16 +1577,13 @@ func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error
 
 					nf := protocol.FileInfo{
 						Name:     f.Name,
-						Flags:    f.Flags | protocol.FlagDeleted,
+						Type:     f.Type,
+						Size:     f.Size,
 						Modified: f.Modified,
+						Deleted:  true,
 						Version:  f.Version.Update(m.shortID),
 					}
 
-					// The deleted file might have been ignored at some
-					// point, but it currently isn't so we make sure to
-					// clear the invalid bit.
-					nf.Flags &^= protocol.FlagInvalid
-
 					batch = append(batch, nf)
 				}
 			}
@@ -1672,30 +1649,21 @@ func (m *Model) numHashers(folder string) int {
 
 // generateClusterConfig returns a ClusterConfigMessage that is correct for
 // the given peer device
-func (m *Model) generateClusterConfig(device protocol.DeviceID) protocol.ClusterConfigMessage {
-	var message protocol.ClusterConfigMessage
+func (m *Model) generateClusterConfig(device protocol.DeviceID) protocol.ClusterConfig {
+	var message protocol.ClusterConfig
 
 	m.fmut.RLock()
 	for _, folder := range m.deviceFolders[device] {
 		folderCfg := m.cfg.Folders()[folder]
 		protocolFolder := protocol.Folder{
-			ID:    folder,
-			Label: folderCfg.Label,
+			ID:                 folder,
+			Label:              folderCfg.Label,
+			ReadOnly:           folderCfg.Type == config.FolderTypeReadOnly,
+			IgnorePermissions:  folderCfg.IgnorePerms,
+			IgnoreDelete:       folderCfg.IgnoreDelete,
+			DisableTempIndexes: folderCfg.DisableTempIndexes,
 		}
-		var flags uint32
-		if folderCfg.Type == config.FolderTypeReadOnly {
-			flags |= protocol.FlagFolderReadOnly
-		}
-		if folderCfg.IgnorePerms {
-			flags |= protocol.FlagFolderIgnorePerms
-		}
-		if folderCfg.IgnoreDelete {
-			flags |= protocol.FlagFolderIgnoreDelete
-		}
-		if folderCfg.DisableTempIndexes {
-			flags |= protocol.FlagFolderDisabledTempIndexes
-		}
-		protocolFolder.Flags = flags
+
 		for _, device := range m.folderDevices[folder] {
 			// DeviceID is a value type, but with an underlying array. Copy it
 			// so we don't grab aliases to the same array later on in device[:]
@@ -1707,14 +1675,11 @@ func (m *Model) generateClusterConfig(device protocol.DeviceID) protocol.Cluster
 				ID:          device[:],
 				Name:        deviceCfg.Name,
 				Addresses:   deviceCfg.Addresses,
-				Compression: uint32(deviceCfg.Compression),
+				Compression: deviceCfg.Compression,
 				CertName:    deviceCfg.CertName,
-				Flags:       protocol.FlagShareTrusted,
+				Introducer:  deviceCfg.Introducer,
 			}
 
-			if deviceCfg.Introducer {
-				protocolDevice.Flags |= protocol.FlagIntroducer
-			}
 			protocolFolder.Devices = append(protocolFolder.Devices, protocolDevice)
 		}
 		message.Folders = append(message.Folders, protocolFolder)
@@ -1759,7 +1724,7 @@ func (m *Model) Override(folder string) {
 		have, ok := fs.Get(protocol.LocalDeviceID, need.Name)
 		if !ok || have.Name != need.Name {
 			// We are missing the file
-			need.Flags |= protocol.FlagDeleted
+			need.Deleted = true
 			need.Blocks = nil
 			need.Version = need.Version.Update(m.shortID)
 		} else {
@@ -1868,7 +1833,7 @@ func (m *Model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly
 
 		if !dirsonly && base != "" {
 			last[base] = []interface{}{
-				time.Unix(f.Modified, 0), f.Size(),
+				time.Unix(f.Modified, 0), f.FileSize(),
 			}
 		}
 
@@ -2181,11 +2146,7 @@ func mapDeviceCfgs(devices []config.DeviceConfiguration) map[protocol.DeviceID]s
 
 func filterIndex(folder string, fs []protocol.FileInfo, dropDeletes bool, ignores *ignore.Matcher) []protocol.FileInfo {
 	for i := 0; i < len(fs); {
-		if fs[i].Flags&^protocol.FlagsAll != 0 {
-			l.Debugln("dropping update for file with unknown bits set", fs[i])
-			fs[i] = fs[len(fs)-1]
-			fs = fs[:len(fs)-1]
-		} else if fs[i].IsDeleted() && dropDeletes {
+		if fs[i].IsDeleted() && dropDeletes {
 			l.Debugln("dropping update for undesired delete", fs[i])
 			fs[i] = fs[len(fs)-1]
 			fs = fs[:len(fs)-1]

lib/model/model_test.go

@@ -55,19 +55,19 @@ func init() {
 var testDataExpected = map[string]protocol.FileInfo{
 	"foo": {
 		Name:     "foo",
-		Flags:    0,
+		Type:     protocol.FileInfoTypeFile,
 		Modified: 0,
 		Blocks:   []protocol.BlockInfo{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
 	},
 	"empty": {
 		Name:     "empty",
-		Flags:    0,
+		Type:     protocol.FileInfoTypeFile,
 		Modified: 0,
 		Blocks:   []protocol.BlockInfo{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
 	},
 	"bar": {
 		Name:     "bar",
-		Flags:    0,
+		Type:     protocol.FileInfoTypeFile,
 		Modified: 0,
 		Blocks:   []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
 	},
@@ -77,8 +77,9 @@ func init() {
 	// Fix expected test data to match reality
 	for n, f := range testDataExpected {
 		fi, _ := os.Stat("testdata/" + n)
-		f.Flags = uint32(fi.Mode())
+		f.Permissions = uint32(fi.Mode())
 		f.Modified = fi.ModTime().Unix()
+		f.Size = fi.Size()
 		testDataExpected[n] = f
 	}
 }
@@ -98,7 +99,7 @@ func TestRequest(t *testing.T) {
 
 	// Existing, shared file
 	bs = bs[:6]
-	err := m.Request(device1, "default", "foo", 0, nil, 0, nil, bs)
+	err := m.Request(device1, "default", "foo", 0, nil, false, bs)
 	if err != nil {
 		t.Error(err)
 	}
@@ -107,32 +108,32 @@ func TestRequest(t *testing.T) {
 	}
 
 	// Existing, nonshared file
-	err = m.Request(device2, "default", "foo", 0, nil, 0, nil, bs)
+	err = m.Request(device2, "default", "foo", 0, nil, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Nonexistent file
-	err = m.Request(device1, "default", "nonexistent", 0, nil, 0, nil, bs)
+	err = m.Request(device1, "default", "nonexistent", 0, nil, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Shared folder, but disallowed file name
-	err = m.Request(device1, "default", "../walk.go", 0, nil, 0, nil, bs)
+	err = m.Request(device1, "default", "../walk.go", 0, nil, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Negative offset
-	err = m.Request(device1, "default", "foo", -4, nil, 0, nil, bs[:0])
+	err = m.Request(device1, "default", "foo", -4, nil, false, bs[:0])
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Larger block than available
 	bs = bs[:42]
-	err = m.Request(device1, "default", "foo", 0, nil, 0, nil, bs)
+	err = m.Request(device1, "default", "foo", 0, nil, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
@@ -168,11 +169,11 @@ func benchmarkIndex(b *testing.B, nfiles int) {
 	m.ServeBackground()
 
 	files := genFiles(nfiles)
-	m.Index(device1, "default", files, 0, nil)
+	m.Index(device1, "default", files)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		m.Index(device1, "default", files, 0, nil)
+		m.Index(device1, "default", files)
 	}
 	b.ReportAllocs()
 }
@@ -199,11 +200,11 @@ func benchmarkIndexUpdate(b *testing.B, nfiles, nufiles int) {
 	files := genFiles(nfiles)
 	ufiles := genFiles(nufiles)
 
-	m.Index(device1, "default", files, 0, nil)
+	m.Index(device1, "default", files)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		m.IndexUpdate(device1, "default", ufiles, 0, nil)
+		m.IndexUpdate(device1, "default", ufiles)
 	}
 	b.ReportAllocs()
 }
@@ -211,8 +212,6 @@ func benchmarkIndexUpdate(b *testing.B, nfiles, nufiles int) {
 type downloadProgressMessage struct {
 	folder  string
 	updates []protocol.FileDownloadProgressUpdate
-	flags   uint32
-	options []protocol.Option
 }
 
 type FakeConnection struct {
@@ -240,11 +239,11 @@ func (f FakeConnection) Option(string) string {
 	return ""
 }
 
-func (FakeConnection) Index(string, []protocol.FileInfo, uint32, []protocol.Option) error {
+func (FakeConnection) Index(string, []protocol.FileInfo) error {
 	return nil
 }
 
-func (FakeConnection) IndexUpdate(string, []protocol.FileInfo, uint32, []protocol.Option) error {
+func (FakeConnection) IndexUpdate(string, []protocol.FileInfo) error {
 	return nil
 }
 
@@ -252,7 +251,7 @@ func (f FakeConnection) Request(folder, name string, offset int64, size int, has
 	return f.requestData, nil
 }
 
-func (FakeConnection) ClusterConfig(protocol.ClusterConfigMessage) {}
+func (FakeConnection) ClusterConfig(protocol.ClusterConfig) {}
 
 func (FakeConnection) Ping() bool {
 	return true
@@ -266,12 +265,10 @@ func (FakeConnection) Statistics() protocol.Statistics {
 	return protocol.Statistics{}
 }
 
-func (f *FakeConnection) DownloadProgress(folder string, updates []protocol.FileDownloadProgressUpdate, flags uint32, options []protocol.Option) {
+func (f *FakeConnection) DownloadProgress(folder string, updates []protocol.FileDownloadProgressUpdate) {
 	f.downloadProgressMessages = append(f.downloadProgressMessages, downloadProgressMessage{
 		folder:  folder,
 		updates: updates,
-		flags:   flags,
-		options: options,
 	})
 }
 
@@ -305,7 +302,7 @@ func BenchmarkRequest(b *testing.B) {
 		},
 		Connection: fc,
 	}, protocol.HelloResult{})
-	m.Index(device1, "default", files, 0, nil)
+	m.Index(device1, "default", files)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -451,13 +448,13 @@ func TestClusterConfig(t *testing.T) {
 	if id := r.Devices[0].ID; !bytes.Equal(id, device1[:]) {
 		t.Errorf("Incorrect device ID %x != %x", id, device1)
 	}
-	if r.Devices[0].Flags&protocol.FlagIntroducer == 0 {
+	if !r.Devices[0].Introducer {
 		t.Error("Device1 should be flagged as Introducer")
 	}
 	if id := r.Devices[1].ID; !bytes.Equal(id, device2[:]) {
 		t.Errorf("Incorrect device ID %x != %x", id, device2)
 	}
-	if r.Devices[1].Flags&protocol.FlagIntroducer != 0 {
+	if r.Devices[1].Introducer {
 		t.Error("Device2 should not be flagged as Introducer")
 	}
 
@@ -471,13 +468,13 @@ func TestClusterConfig(t *testing.T) {
 	if id := r.Devices[0].ID; !bytes.Equal(id, device1[:]) {
 		t.Errorf("Incorrect device ID %x != %x", id, device1)
 	}
-	if r.Devices[0].Flags&protocol.FlagIntroducer == 0 {
+	if !r.Devices[0].Introducer {
 		t.Error("Device1 should be flagged as Introducer")
 	}
 	if id := r.Devices[1].ID; !bytes.Equal(id, device2[:]) {
 		t.Errorf("Incorrect device ID %x != %x", id, device2)
 	}
-	if r.Devices[1].Flags&protocol.FlagIntroducer != 0 {
+	if r.Devices[1].Introducer {
 		t.Error("Device2 should not be flagged as Introducer")
 	}
 }
@@ -572,44 +569,6 @@ func TestIgnores(t *testing.T) {
 	}
 }
 
-func TestRefuseUnknownBits(t *testing.T) {
-	db := db.OpenMemory()
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
-	m.AddFolder(defaultFolderConfig)
-	m.ServeBackground()
-
-	m.ScanFolder("default")
-	m.Index(device1, "default", []protocol.FileInfo{
-		{
-			Name:  "invalid1",
-			Flags: (protocol.FlagsAll + 1) &^ protocol.FlagInvalid,
-		},
-		{
-			Name:  "invalid2",
-			Flags: (protocol.FlagsAll + 2) &^ protocol.FlagInvalid,
-		},
-		{
-			Name:  "invalid3",
-			Flags: (1 << 31) &^ protocol.FlagInvalid,
-		},
-		{
-			Name:  "valid",
-			Flags: protocol.FlagsAll &^ (protocol.FlagInvalid | protocol.FlagSymlink),
-		},
-	}, 0, nil)
-
-	for _, name := range []string{"invalid1", "invalid2", "invalid3"} {
-		f, ok := m.CurrentGlobalFile("default", name)
-		if ok || f.Name == name {
-			t.Error("Invalid file found or name match")
-		}
-	}
-	f, ok := m.CurrentGlobalFile("default", "valid")
-	if !ok || f.Name != "valid" {
-		t.Error("Valid file not found or name mismatch", ok, f)
-	}
-}
-
 func TestROScanRecovery(t *testing.T) {
 	ldb := db.OpenMemory()
 	set := db.NewFileSet("default", ldb)
@@ -789,17 +748,18 @@ func TestGlobalDirectoryTree(t *testing.T) {
 	m.ServeBackground()
 
 	b := func(isfile bool, path ...string) protocol.FileInfo {
-		flags := uint32(protocol.FlagDirectory)
+		typ := protocol.FileInfoTypeDirectory
 		blocks := []protocol.BlockInfo{}
 		if isfile {
-			flags = 0
+			typ = protocol.FileInfoTypeFile
 			blocks = []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}}
 		}
 		return protocol.FileInfo{
 			Name:     filepath.Join(path...),
-			Flags:    flags,
+			Type:     typ,
 			Modified: 0x666,
 			Blocks:   blocks,
+			Size:     0xa,
 		}
 	}
 
@@ -871,7 +831,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
 		return string(bytes)
 	}
 
-	m.Index(device1, "default", testdata, 0, nil)
+	m.Index(device1, "default", testdata)
 
 	result := m.GlobalDirectoryTree("default", "", -1, false)
 
@@ -1039,17 +999,18 @@ func TestGlobalDirectorySelfFixing(t *testing.T) {
 	m.ServeBackground()
 
 	b := func(isfile bool, path ...string) protocol.FileInfo {
-		flags := uint32(protocol.FlagDirectory)
+		typ := protocol.FileInfoTypeDirectory
 		blocks := []protocol.BlockInfo{}
 		if isfile {
-			flags = 0
+			typ = protocol.FileInfoTypeFile
 			blocks = []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}}
 		}
 		return protocol.FileInfo{
 			Name:     filepath.Join(path...),
-			Flags:    flags,
+			Type:     typ,
 			Modified: 0x666,
 			Blocks:   blocks,
+			Size:     0xa,
 		}
 	}
 
@@ -1130,7 +1091,7 @@ func TestGlobalDirectorySelfFixing(t *testing.T) {
 		return string(bytes)
 	}
 
-	m.Index(device1, "default", testdata, 0, nil)
+	m.Index(device1, "default", testdata)
 
 	result := m.GlobalDirectoryTree("default", "", -1, false)
 
@@ -1215,7 +1176,7 @@ func benchmarkTree(b *testing.B, n1, n2 int) {
 	m.ScanFolder("default")
 	files := genDeepFiles(n1, n2)
 
-	m.Index(device1, "default", files, 0, nil)
+	m.Index(device1, "default", files)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -1244,12 +1205,12 @@ func TestIgnoreDelete(t *testing.T) {
 	}
 
 	// Mark it for deletion
-	f.Flags = protocol.FlagDeleted
+	f.Deleted = true
 	f.Version = f.Version.Update(142) // arbitrary short remote ID
 	f.Blocks = nil
 
 	// Send the index
-	m.Index(device1, "default", []protocol.FileInfo{f}, 0, nil)
+	m.Index(device1, "default", []protocol.FileInfo{f})
 
 	// Make sure we ignored it
 	f, ok = m.CurrentGlobalFile("default", "foo")

lib/model/progressemitter.go

@@ -137,7 +137,7 @@ func (t *ProgressEmitter) sendDownloadProgressMessages() {
 			updates := state.update(folder, activePullers)
 
 			if len(updates) > 0 {
-				conn.DownloadProgress(folder, updates, 0, nil)
+				conn.DownloadProgress(folder, updates)
 			}
 		}
 	}

lib/model/progressemitter_test.go

@@ -100,7 +100,6 @@ func TestProgressEmitter(t *testing.T) {
 }
 
 func TestSendDownloadProgressMessages(t *testing.T) {
-
 	c := config.Wrap("/tmp/test", config.Configuration{})
 	c.SetOptions(config.OptionsConfiguration{
 		ProgressUpdateIntervalS: 0,
@@ -112,7 +111,7 @@ func TestSendDownloadProgressMessages(t *testing.T) {
 	p := NewProgressEmitter(c)
 	p.temporaryIndexSubscribe(fc, []string{"folder", "folder2"})
 
-	expect := func(updateIdx int, state *sharedPullerState, updateType uint32, version protocol.Vector, blocks []int32, remove bool) {
+	expect := func(updateIdx int, state *sharedPullerState, updateType protocol.FileDownloadProgressUpdateType, version protocol.Vector, blocks []int32, remove bool) {
 		messageIdx := -1
 		for i, msg := range fc.downloadProgressMessages {
 			if msg.folder == state.folder {
@@ -346,7 +345,7 @@ func TestSendDownloadProgressMessages(t *testing.T) {
 		file: protocol.FileInfo{
 			Name:    "state5",
 			Version: v1,
-			Flags:   protocol.FlagDirectory,
+			Type:    protocol.FileInfoTypeDirectory,
 			Blocks:  blocks,
 		},
 		mut:              sync.NewRWMutex(),
@@ -359,7 +358,7 @@ func TestSendDownloadProgressMessages(t *testing.T) {
 		file: protocol.FileInfo{
 			Name:    "state6",
 			Version: v1,
-			Flags:   protocol.FlagSymlink,
+			Type:    protocol.FileInfoTypeSymlinkUnknown,
 		},
 		mut:              sync.NewRWMutex(),
 		available:        []int32{1, 2, 3},

lib/model/rwfolder.go

@@ -163,7 +163,7 @@ func (f *rwFolder) configureCopiersAndPullers(config config.FolderConfiguration)
 // set on the local host or the FlagNoPermBits has been set on the file/dir
 // which is being pulled.
 func (f *rwFolder) ignorePermissions(file protocol.FileInfo) bool {
-	return f.ignorePerms || file.Flags&protocol.FlagNoPermBits != 0
+	return f.ignorePerms || file.NoPermissions
 }
 
 // Serve will run scans and pulls. It will return when Stop()ed or on a
@@ -435,7 +435,7 @@ func (f *rwFolder) pullerIteration(ignores *ignore.Matcher) int {
 		if !handleFile(file) {
 			// A new or changed file or symlink. This is the only case where we
 			// do stuff concurrently in the background
-			f.queue.Push(file.Name, file.Size(), file.Modified)
+			f.queue.Push(file.Name, file.Size, file.Modified)
 		}
 
 		changed++
@@ -569,7 +569,7 @@ func (f *rwFolder) handleDir(file protocol.FileInfo) {
 	}()
 
 	realName := filepath.Join(f.dir, file.Name)
-	mode := os.FileMode(file.Flags & 0777)
+	mode := os.FileMode(file.Permissions & 0777)
 	if f.ignorePermissions(file) {
 		mode = 0777
 	}
@@ -909,7 +909,7 @@ func (f *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
 		// touching the file. If we can't stat the file we'll just pull it.
 		if info, err := osutil.Lstat(realName); err == nil {
 			mtime := f.virtualMtimeRepo.GetMtime(file.Name, info.ModTime())
-			if mtime.Unix() != curFile.Modified || info.Size() != curFile.Size() {
+			if mtime.Unix() != curFile.Modified || info.Size() != curFile.Size {
 				l.Debugln("file modified but not rescanned; not pulling:", realName)
 				// Scan() is synchronous (i.e. blocks until the scan is
 				// completed and returns an error), but a scan can't happen
@@ -965,7 +965,7 @@ func (f *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
 	} else {
 		// Copy the blocks, as we don't want to shuffle them on the FileInfo
 		blocks = append(blocks, file.Blocks...)
-		blocksSize = file.Size()
+		blocksSize = file.Size
 	}
 
 	if f.checkFreeSpace {
@@ -1021,7 +1021,7 @@ func (f *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
 func (f *rwFolder) shortcutFile(file protocol.FileInfo) error {
 	realName := filepath.Join(f.dir, file.Name)
 	if !f.ignorePermissions(file) {
-		if err := os.Chmod(realName, os.FileMode(file.Flags&0777)); err != nil {
+		if err := os.Chmod(realName, os.FileMode(file.Permissions&0777)); err != nil {
 			l.Infof("Puller (folder %q, file %q): shortcut: chmod: %v", f.folderID, file.Name, err)
 			f.newError(file.Name, err)
 			return err
@@ -1235,7 +1235,7 @@ func (f *rwFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *sharedPul
 func (f *rwFolder) performFinish(state *sharedPullerState) error {
 	// Set the correct permission bits on the new file
 	if !f.ignorePermissions(state.file) {
-		if err := os.Chmod(state.tempName, os.FileMode(state.file.Flags&0777)); err != nil {
+		if err := os.Chmod(state.tempName, os.FileMode(state.file.Permissions&0777)); err != nil {
 			return err
 		}
 	}

lib/model/rwfolder_test.go

@@ -50,10 +50,8 @@ func setUpFile(filename string, blockNumbers []int) protocol.FileInfo {
 	}
 
 	return protocol.FileInfo{
-		Name:     filename,
-		Flags:    0,
-		Modified: 0,
-		Blocks:   existingBlocks,
+		Name:   filename,
+		Blocks: existingBlocks,
 	}
 }
 

lib/model/sharedpullerstate.go

@@ -12,7 +12,6 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
 )
@@ -150,7 +149,7 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
 	if s.sparse && !s.file.IsSymlink() {
 		// Truncate sets the size of the file. This creates a sparse file or a
 		// space reservation, depending on the underlying filesystem.
-		if err := fd.Truncate(s.file.Size()); err != nil {
+		if err := fd.Truncate(s.file.Size); err != nil {
 			s.failLocked("dst truncate", err)
 			return nil, err
 		}
@@ -293,8 +292,8 @@ func (s *sharedPullerState) Progress() *pullerProgress {
 		CopiedFromElsewhere: s.copyTotal - s.copyNeeded - s.copyOrigin,
 		Pulled:              s.pullTotal - s.pullNeeded,
 		Pulling:             s.pullNeeded,
-		BytesTotal:          db.BlocksToSize(total),
-		BytesDone:           db.BlocksToSize(done),
+		BytesTotal:          blocksToSize(total),
+		BytesDone:           blocksToSize(done),
 	}
 }
 
@@ -321,3 +320,10 @@ func (s *sharedPullerState) Available() []int32 {
 	s.mut.RUnlock()
 	return blocks
 }
+
+func blocksToSize(num int) int64 {
+	if num < 2 {
+		return protocol.BlockSize / 2
+	}
+	return int64(num-1)*protocol.BlockSize + protocol.BlockSize/2
+}