From c08024ebb80ad6fd5f91291d2140c7c1a8ac4412 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Wed, 16 May 2018 08:44:08 +0200 Subject: [PATCH] lib/db: Add db and test with invalid files (#4954) This adds a couple of utilities for transporting databases in JSON and a test to load a database and verify a couple of invalid bits. The test itself is quite pointless at the moment, but it lays the groundwork for testing the migration of this data in the next step (after the invalid bit should be changed to local flags for local files). When that happens we need to have a database in the old format already there in order to be able to test the migration. --- lib/db/leveldb_test.go | 72 ++++++++++++++ .../testdata/v0.14.48-ignoredfiles.db.jsons | 15 +++ lib/db/util_test.go | 94 +++++++++++++++++++ 3 files changed, 181 insertions(+) create mode 100644 lib/db/testdata/v0.14.48-ignoredfiles.db.jsons create mode 100644 lib/db/util_test.go diff --git a/lib/db/leveldb_test.go b/lib/db/leveldb_test.go index 5dd58e0a..3b953b03 100644 --- a/lib/db/leveldb_test.go +++ b/lib/db/leveldb_test.go @@ -10,6 +10,7 @@ import ( "bytes" "testing" + "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/protocol" ) @@ -152,3 +153,74 @@ func TestDropIndexIDs(t *testing.T) { t.Fatal("fail remote 4") } } + +func TestInvalidFiles(t *testing.T) { + ldb, err := openJSONS("testdata/v0.14.48-ignoredfiles.db.jsons") + if err != nil { + t.Fatal(err) + } + db := newDBInstance(ldb, "") + fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db) + + // The contents of the database are like this: + // + // fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db) + // fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{ + // { // invalid (ignored) file + // Name: "foo", + // Type: protocol.FileInfoTypeFile, + // Invalid: true, + // Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1000}}}, + // }, + // { // regular file + // Name: "bar", + // Type: protocol.FileInfoTypeFile, + // Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1001}}}, + // }, + // }) + // fs.Update(protocol.DeviceID{42}, []protocol.FileInfo{ + // { // invalid file + // Name: "baz", + // Type: protocol.FileInfoTypeFile, + // Invalid: true, + // Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1000}}}, + // }, + // { // regular file + // Name: "quux", + // Type: protocol.FileInfoTypeFile, + // Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1002}}}, + // }, + // }) + + fi, ok := fs.Get(protocol.LocalDeviceID, "foo") + if !ok { + t.Fatal("foo should exist") + } + if !fi.Invalid { + t.Error("foo should be invalid") + } + + fi, ok = fs.Get(protocol.LocalDeviceID, "bar") + if !ok { + t.Fatal("bar should exist") + } + if fi.Invalid { + t.Error("bar should not be invalid") + } + + fi, ok = fs.Get(protocol.DeviceID{42}, "baz") + if !ok { + t.Fatal("baz should exist") + } + if !fi.Invalid { + t.Error("baz should be invalid") + } + + fi, ok = fs.Get(protocol.DeviceID{42}, "quux") + if !ok { + t.Fatal("quux should exist") + } + if fi.Invalid { + t.Error("quux should not be invalid") + } +} diff --git a/lib/db/testdata/v0.14.48-ignoredfiles.db.jsons b/lib/db/testdata/v0.14.48-ignoredfiles.db.jsons new file mode 100644 index 00000000..0b335e34 --- /dev/null +++ b/lib/db/testdata/v0.14.48-ignoredfiles.db.jsons @@ -0,0 +1,15 @@ +{"k":"AAAAAAAAAAABYmFy","v":"CgNiYXJKBwoFCAEQ6QdQAg=="} +{"k":"AAAAAAAAAAABZm9v","v":"CgNmb284AUoHCgUIARDoB1AB"} +{"k":"AAAAAAAAAAACYmF6","v":"CgNiYXo4AUoHCgUIKhDoBw=="} +{"k":"AAAAAAAAAAACcXV1eA==","v":"CgRxdXV4SgcKBQgqEOoH"} +{"k":"AQAAAABiYXI=","v":"CisKBwoFCAEQ6QcSIP//////////////////////////////////////////"} +{"k":"AQAAAABiYXo=","v":"Ci0KBwoFCCoQ6AcSICoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGAE="} +{"k":"AQAAAABmb28=","v":"Ci0KBwoFCAEQ6AcSIP//////////////////////////////////////////GAE="} +{"k":"AQAAAABxdXV4","v":"CisKBwoFCCoQ6gcSICoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} +{"k":"BgAAAAAAAAAA","v":"dGVzdA=="} +{"k":"BwAAAAAAAAAA","v":""} +{"k":"BwAAAAEAAAAA","v":"//////////////////////////////////////////8="} +{"k":"BwAAAAIAAAAA","v":"KgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="} +{"k":"CQAAAAA=","v":"CicIATACigEg//////////////////////////////////////////8KJwgCMAKKASD4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+AolCAGKASAqAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABCoudnCuM+vlxU="} +{"k":"CwAAAAAAAAAAAAAAAQ==","v":"AAAAAAAAAAABZm9v"} +{"k":"CwAAAAAAAAAAAAAAAg==","v":"AAAAAAAAAAABYmFy"} diff --git a/lib/db/util_test.go b/lib/db/util_test.go new file mode 100644 index 00000000..8eb525ac --- /dev/null +++ b/lib/db/util_test.go @@ -0,0 +1,94 @@ +// Copyright (C) 2018 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package db + +import ( + "encoding/json" + "io" + "os" + + "github.com/syncthing/syncthing/lib/fs" + "github.com/syncthing/syncthing/lib/protocol" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/storage" + "github.com/syndtr/goleveldb/leveldb/util" +) + +// writeJSONS serializes the database to a JSON stream that can be checked +// in to the repo and used for tests. +func writeJSONS(w io.Writer, db *leveldb.DB) { + it := db.NewIterator(&util.Range{}, nil) + defer it.Release() + enc := json.NewEncoder(w) + for it.Next() { + enc.Encode(map[string][]byte{ + "k": it.Key(), + "v": it.Value(), + }) + } +} + +// openJSONS reads a JSON stream file into a leveldb.DB +func openJSONS(file string) (*leveldb.DB, error) { + fd, err := os.Open(file) + if err != nil { + return nil, err + } + dec := json.NewDecoder(fd) + + db, _ := leveldb.Open(storage.NewMemStorage(), nil) + + for { + var row map[string][]byte + + err := dec.Decode(&row) + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + + db.Put(row["k"], row["v"], nil) + } + + return db, nil +} + +func generateIgnoredFilesDB() { + // This generates a database with files with invalid flags, local and + // remote, in the format used in 0.14.48. + + db := OpenMemory() + fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db) + fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{ + { // invalid (ignored) file + Name: "foo", + Type: protocol.FileInfoTypeFile, + Invalid: true, + Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1000}}}, + }, + { // regular file + Name: "bar", + Type: protocol.FileInfoTypeFile, + Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1001}}}, + }, + }) + fs.Update(protocol.DeviceID{42}, []protocol.FileInfo{ + { // invalid file + Name: "baz", + Type: protocol.FileInfoTypeFile, + Invalid: true, + Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1000}}}, + }, + { // regular file + Name: "quux", + Type: protocol.FileInfoTypeFile, + Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1002}}}, + }, + }) + writeJSONS(os.Stdout, db.DB) +}