add command to regenerate the search index

cmd/liber/liber.go

@@ -24,6 +24,7 @@ var (
 	update         = flag.Bool("update", false, "update the db")
 	search         = flag.Bool("search", false, "search something")
 	remotesync     = flag.String("sync", "", "push data to remote server")
+	reindex        = flag.Bool("reindex", false, "re-create the search index")
 	httpserver     = flag.String("http-server", "", "start the HTTP server on the specified address")
 	noninteractive = flag.Bool("noninteractive", false, "disable interactive metadata prompts on update")
 )
@@ -185,6 +186,14 @@ func doHttpServer(db *liber.Database, addr string) {
 	log.Fatal(server.ListenAndServe())
 }
 
+func doReindex(db *liber.Database) {
+	log.Println("starting database indexing")
+	if err := db.Reindex(); err != nil {
+		log.Fatal(err)
+	}
+	log.Println("database indexing complete")
+}
+
 func b2i(b bool) int {
 	if b {
 		return 1
@@ -203,9 +212,9 @@ func expandTilde(path string) string {
 func main() {
 	log.SetFlags(0)
 	flag.Parse()
-	nset := b2i(*update) + b2i(*search) + b2i(*httpserver != "") + b2i(*remotesync != "")
+	nset := b2i(*update) + b2i(*search) + b2i(*httpserver != "") + b2i(*remotesync != "") + b2i(*reindex)
 	if nset != 1 {
-		log.Fatal("Must specify one of --update, --sync, --search or --http-server")
+		log.Fatal("Must specify one of --update, --sync, --search, --reindex or --http-server")
 	}
 	if *bookDir == "" {
 		log.Fatal("Must specify --book-dir")
@@ -236,6 +245,8 @@ func main() {
 			log.Fatal("No query specified")
 		}
 		doSearch(db, query)
+	} else if *reindex {
+		doReindex(db)
 	} else if *httpserver != "" {
 		doHttpServer(db, *httpserver)
 	}

database.go

@@ -175,6 +175,8 @@ type Database struct {
 	leveldbFilter *levigo.FilterPolicy
 
 	index bleve.Index
+
+	path string
 }
 
 func NewDb(path string) (*Database, error) {
@@ -186,7 +188,7 @@ func NewDb(path string) (*Database, error) {
 	}
 
 	// Initialize our database and the index.
-	d := &Database{}
+	d := &Database{path: path}
 	if err := d.setupLevelDb(filepath.Join(path, "db")); err != nil {
 		return nil, err
 	}
@@ -385,6 +387,32 @@ func (db *Database) Scan(bucket []byte) *DatabaseIterator {
 	}
 }
 
+// Reindex the entire database. This is an administrative operation,
+// to be performed after an incompatible index schema change. It will
+// delete the existing index and re-create it from scratch.
+func (db *Database) Reindex() error {
+	// Close the index, delete it, and re-open it.
+	db.index.Close()
+
+	indexPath := filepath.Join(db.path, "index")
+	if err := os.RemoveAll(indexPath); err != nil {
+		return err
+	}
+	if err := db.setupIndex(indexPath); err != nil {
+		return err
+	}
+
+	// Scan the database and re-index everything.
+	for i := db.Scan(BookBucket); i.Valid(); i.Next() {
+		var book Book
+		if err := i.Value(&book); err != nil {
+			continue
+		}
+		db.index.Index(i.Id().String(), flatten(&book))
+	}
+	return nil
+}
+
 type SearchResult struct {
 	Results    []*Book
 	NumResults int

database_test.go

@@ -229,6 +229,46 @@ func TestDatabase_Find(t *testing.T) {
 	}
 }
 
+func TestDatabase_Reindex(t *testing.T) {
+	td, db := newTestDatabase2(t)
+	defer td.Close()
+	book, _ := db.GetBook(td.refbookid)
+	m := book.Metadata
+
+	doTest := func() {
+		// Find the first time.
+		if result, err := db.Find(m.Uniques()); err != nil {
+			t.Errorf("Not found: %v", err)
+		} else if result.Id != book.Id {
+			t.Errorf("Bad match with ISBN: got=%d, expected=%d", result.Id, book.Id)
+		}
+	}
+
+	doTest()
+	if err := db.Reindex(); err != nil {
+		t.Fatalf("Reindex: %v", err)
+	}
+	doTest()
+}
+
+// func TestDatabase_Find2(t *testing.T) {
+// 	td, db := newTestDatabase(t)
+// 	defer td.Close()
+
+// 	testdata := []*Metadata{
+// 		&Metadata{Title: "PHILIP K", Creator: []string{"Antonio Marigonda"}, Language: []string{"it"}, Sources: []MetadataSource{MetadataSource{Name: "opf", ID: "6809f8b6-3151-41ef-b05a-0eb113746674"}}},
+// 	}
+
+// 	for _, m := range testdata {
+// 		t.Logf("uniques = %v", m.Uniques())
+// 		id := NewID()
+// 		db.PutBook(&Book{Id: id, Metadata: m})
+// 		if _, err := db.Find(m.Uniques()); err != nil {
+// 			t.Errorf("Could not find book: %#v", m)
+// 		}
+// 	}
+// }
+
 func TestDatabase_Suggest(t *testing.T) {
 	td, db := newTestDatabase(t)
 	defer td.Close()

sync_test.go

@@ -50,8 +50,7 @@ func TestSync_Sync(t *testing.T) {
 	defer srv.Close()
 
 	cl := NewRemoteServer(srv.URL)
-	err := db.Sync(clientStorage, cl)
-	if err != nil {
+	if err := db.Sync(clientStorage, cl); err != nil {
 		t.Fatalf("Sync(): %v", err)
 	}
 
@@ -64,4 +63,9 @@ func TestSync_Sync(t *testing.T) {
 			t.Errorf("Book %d missing from db2: %v", i+1, err)
 		}
 	}
+
+	// Calling Sync again should not result in errors.
+	if err := db.Sync(clientStorage, cl); err != nil {
+		t.Fatalf("Sync(): %v", err)
+	}
 }