From 034e11ae50fe61d0d14556fc7c317b18c93676e9 Mon Sep 17 00:00:00 2001
From: ale <ale@incal.net>
Date: Tue, 25 Nov 2014 23:57:47 +0000
Subject: [PATCH] add command to regenerate the search index
---
cmd/liber/liber.go | 15 +++++++++++++--
database.go | 30 +++++++++++++++++++++++++++++-
database_test.go | 40 ++++++++++++++++++++++++++++++++++++++++
sync_test.go | 8 ++++++--
4 files changed, 88 insertions(+), 5 deletions(-)
diff --git a/cmd/liber/liber.go b/cmd/liber/liber.go
index f423492..728a430 100644
--- a/cmd/liber/liber.go
+++ b/cmd/liber/liber.go
@@ -24,6 +24,7 @@ var (
update = flag.Bool("update", false, "update the db")
search = flag.Bool("search", false, "search something")
remotesync = flag.String("sync", "", "push data to remote server")
+ reindex = flag.Bool("reindex", false, "re-create the search index")
httpserver = flag.String("http-server", "", "start the HTTP server on the specified address")
noninteractive = flag.Bool("noninteractive", false, "disable interactive metadata prompts on update")
)
@@ -185,6 +186,14 @@ func doHttpServer(db *liber.Database, addr string) {
log.Fatal(server.ListenAndServe())
}
+func doReindex(db *liber.Database) {
+ log.Println("starting database indexing")
+ if err := db.Reindex(); err != nil {
+ log.Fatal(err)
+ }
+ log.Println("database indexing complete")
+}
+
func b2i(b bool) int {
if b {
return 1
@@ -203,9 +212,9 @@ func expandTilde(path string) string {
func main() {
log.SetFlags(0)
flag.Parse()
- nset := b2i(*update) + b2i(*search) + b2i(*httpserver != "") + b2i(*remotesync != "")
+ nset := b2i(*update) + b2i(*search) + b2i(*httpserver != "") + b2i(*remotesync != "") + b2i(*reindex)
if nset != 1 {
- log.Fatal("Must specify one of --update, --sync, --search or --http-server")
+ log.Fatal("Must specify one of --update, --sync, --search, --reindex or --http-server")
}
if *bookDir == "" {
log.Fatal("Must specify --book-dir")
@@ -236,6 +245,8 @@ func main() {
log.Fatal("No query specified")
}
doSearch(db, query)
+ } else if *reindex {
+ doReindex(db)
} else if *httpserver != "" {
doHttpServer(db, *httpserver)
}
diff --git a/database.go b/database.go
index 101e413..6db4027 100644
--- a/database.go
+++ b/database.go
@@ -175,6 +175,8 @@ type Database struct {
leveldbFilter *levigo.FilterPolicy
index bleve.Index
+
+ path string
}
func NewDb(path string) (*Database, error) {
@@ -186,7 +188,7 @@ func NewDb(path string) (*Database, error) {
}
// Initialize our database and the index.
- d := &Database{}
+ d := &Database{path: path}
if err := d.setupLevelDb(filepath.Join(path, "db")); err != nil {
return nil, err
}
@@ -385,6 +387,32 @@ func (db *Database) Scan(bucket []byte) *DatabaseIterator {
}
}
+// Reindex the entire database. This is an administrative operation,
+// to be performed after an incompatible index schema change. It will
+// delete the existing index and re-create it from scratch.
+func (db *Database) Reindex() error {
+ // Close the index, delete it, and re-open it.
+ db.index.Close()
+
+ indexPath := filepath.Join(db.path, "index")
+ if err := os.RemoveAll(indexPath); err != nil {
+ return err
+ }
+ if err := db.setupIndex(indexPath); err != nil {
+ return err
+ }
+
+ // Scan the database and re-index everything.
+ for i := db.Scan(BookBucket); i.Valid(); i.Next() {
+ var book Book
+ if err := i.Value(&book); err != nil {
+ continue
+ }
+ db.index.Index(i.Id().String(), flatten(&book))
+ }
+ return nil
+}
+
type SearchResult struct {
Results []*Book
NumResults int
diff --git a/database_test.go b/database_test.go
index 72665f8..d6453d4 100644
--- a/database_test.go
+++ b/database_test.go
@@ -229,6 +229,46 @@ func TestDatabase_Find(t *testing.T) {
}
}
+func TestDatabase_Reindex(t *testing.T) {
+ td, db := newTestDatabase2(t)
+ defer td.Close()
+ book, _ := db.GetBook(td.refbookid)
+ m := book.Metadata
+
+ doTest := func() {
+ // Find the first time.
+ if result, err := db.Find(m.Uniques()); err != nil {
+ t.Errorf("Not found: %v", err)
+ } else if result.Id != book.Id {
+ t.Errorf("Bad match with ISBN: got=%d, expected=%d", result.Id, book.Id)
+ }
+ }
+
+ doTest()
+ if err := db.Reindex(); err != nil {
+ t.Fatalf("Reindex: %v", err)
+ }
+ doTest()
+}
+
+// func TestDatabase_Find2(t *testing.T) {
+// td, db := newTestDatabase(t)
+// defer td.Close()
+
+// testdata := []*Metadata{
+// &Metadata{Title: "PHILIP K", Creator: []string{"Antonio Marigonda"}, Language: []string{"it"}, Sources: []MetadataSource{MetadataSource{Name: "opf", ID: "6809f8b6-3151-41ef-b05a-0eb113746674"}}},
+// }
+
+// for _, m := range testdata {
+// t.Logf("uniques = %v", m.Uniques())
+// id := NewID()
+// db.PutBook(&Book{Id: id, Metadata: m})
+// if _, err := db.Find(m.Uniques()); err != nil {
+// t.Errorf("Could not find book: %#v", m)
+// }
+// }
+// }
+
func TestDatabase_Suggest(t *testing.T) {
td, db := newTestDatabase(t)
defer td.Close()
diff --git a/sync_test.go b/sync_test.go
index acb7b7e..ec87c35 100644
--- a/sync_test.go
+++ b/sync_test.go
@@ -50,8 +50,7 @@ func TestSync_Sync(t *testing.T) {
defer srv.Close()
cl := NewRemoteServer(srv.URL)
- err := db.Sync(clientStorage, cl)
- if err != nil {
+ if err := db.Sync(clientStorage, cl); err != nil {
t.Fatalf("Sync(): %v", err)
}
@@ -64,4 +63,9 @@ func TestSync_Sync(t *testing.T) {
t.Errorf("Book %d missing from db2: %v", i+1, err)
}
}
+
+ // Calling Sync again should not result in errors.
+ if err := db.Sync(clientStorage, cl); err != nil {
+ t.Fatalf("Sync(): %v", err)
+ }
}
--
GitLab