Add LocalFlags to FileInfo (#4952)

We have the invalid bit to indicate that a file isn't good. That's enough for remote devices. For ourselves, it would be good to know sometimes why the file isn't good - because it's an unsupported type, because it matches an ignore pattern, or because we detected the data is bad and we need to rescan it.

Or, and this is the main future reason for the PR, because it's a change detected on a receive only device. We will want something like the invalid flag for those changes, but marking them as invalid today means the scanner will rehash them. Hence something more fine grained is required.

This introduces a LocalFlags fields to the FileInfo where we can stash things that we care about locally. For example,

    FlagLocalUnsupported = 1 << 0 // The kind is unsupported, e.g. symlinks on Windows
    FlagLocalIgnored     = 1 << 1 // Matches local ignore patterns
    FlagLocalMustRescan  = 1 << 2 // Doesn't match content on disk, must be rechecked fully

The LocalFlags fields isn't sent over the wire; instead the Invalid attribute is calculated based on the flags at index sending time. It's on the FileInfo anyway because that's what we serialize to database etc.

The actual Invalid flag should after this just be considered when building the global state and figuring out availability for remote devices. It is not used for local file index entries.

lib/model/model.go

@@ -862,6 +862,11 @@ func (m *Model) handleIndex(deviceID protocol.DeviceID, folder string, fs []prot
 	if !update {
 		files.Drop(deviceID)
 	}
+	for i := range fs {
+		// The local flags should never be transmitted over the wire. Make
+		// sure they look like they weren't.
+		fs[i].LocalFlags = 0
+	}
 	files.Update(deviceID, fs)
 
 	events.Default.Log(events.RemoteIndexUpdated, map[string]interface{}{
@@ -1389,7 +1394,7 @@ func (m *Model) recheckFile(deviceID protocol.DeviceID, folderFs fs.Filesystem,
 	// The hashes provided part of the request match what we expect to find according
 	// to what we have in the database, yet the content we've read off the filesystem doesn't
 	// Something is fishy, invalidate the file and rescan it.
-	cf.Invalidate(m.shortID)
+	cf.SetMustRescan(m.shortID)
 
 	// Update the index and tell others
 	// The file will temporarily become invalid, which is ok as the content is messed up.
@@ -1736,6 +1741,10 @@ func sendIndexTo(prevSequence int64, conn protocol.Connection, folder string, fs
 
 		f = fi.(protocol.FileInfo)
 
+		// Mark the file as invalid if any of the local bad stuff flags are set.
+		f.RawInvalid = f.IsInvalid()
+		f.LocalFlags = 0 // never sent externally
+
 		if dropSymlinks && f.IsSymlink() {
 			// Do not send index entries with symlinks to clients that can't
 			// handle it. Fixes issue #3802. Once both sides are upgraded, a
@@ -2064,23 +2073,23 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
 			}
 
 			switch {
-			case !f.IsInvalid() && ignores.Match(f.Name).IsIgnored():
-				// File was valid at last pass but has been ignored. Set invalid bit.
-				l.Debugln("setting invalid bit on ignored", f)
-				nf := f.ConvertToInvalidFileInfo(m.id.Short())
+			case !f.IsIgnored() && ignores.Match(f.Name).IsIgnored():
+				// File was not ignored at last pass but has been ignored.
+				l.Debugln("marking file as ignored", f)
+				nf := f.ConvertToIgnoredFileInfo(m.id.Short())
 				batch = append(batch, nf)
 				batchSizeBytes += nf.ProtoSize()
 				changes++
 
-			case f.IsInvalid() && !ignores.Match(f.Name).IsIgnored():
+			case f.IsIgnored() && !ignores.Match(f.Name).IsIgnored():
 				// Successfully scanned items are already un-ignored during
 				// the scan, so check whether it is deleted.
 				fallthrough
-			case !f.IsInvalid() && !f.IsDeleted():
-				// The file is valid and not deleted. Lets check if it's
-				// still here.
-				// Simply stating it wont do as there are tons of corner
-				// cases (e.g. parent dir->simlink, missing permissions)
+			case !f.IsIgnored() && !f.IsDeleted():
+				// The file is not ignored and not deleted. Lets check if
+				// it's still here. Simply stat:ing it wont do as there are
+				// tons of corner cases (e.g. parent dir->symlink, missing
+				// permissions)
 				if !osutil.IsDeleted(mtimefs, f.Name) {
 					return true
 				}
@@ -2099,7 +2108,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
 				// counter makes sure we are in conflict with any
 				// other existing versions, which will be resolved
 				// by the normal pulling mechanisms.
-				if f.IsInvalid() {
+				if f.IsIgnored() {
 					nf.Version = nf.Version.DropOthers(m.shortID)
 				}