Use batching to make reindexing faster

database.go

@@ -225,7 +225,7 @@ func (db *Database) setupLevelDb(path string) error {
 	// Use 128MB of cache and a small Bloom filter.
 	opts := &ldbopt.Options{
 		Filter:             ldbfilter.NewBloomFilter(10),
-		BlockCacheCapacity: 2 << 27,
+		BlockCacheCapacity: 1 << 28, // 256MB
 	}
 
 	ldb, err := leveldb.OpenFile(path, opts)
@@ -495,10 +495,14 @@ func (db *Database) Restore(r io.Reader) error {
 		if err != nil {
 			return err
 		}
+
 		db.RawPut(key, value)
 		count++
+		if count%1000 == 0 {
+			log.Printf("restored %d entries", count)
+		}
 	}
-	log.Printf("restored %d entries to the database", count)
+	log.Printf("restore complete (%d entries)", count)
 	return db.Reindex()
 }
 
@@ -517,11 +521,32 @@ func (db *Database) Reindex() error {
 		return err
 	}
 
-	// Scan the database and re-index everything.
-	return db.onAllBooks(func(book *Book) error {
-		db.index.Index(book.Id.String(), flatten(book))
+	// Scan the database and re-index everything. Use batch
+	// indexing to achieve decent performance.
+	i := 0
+	start := time.Now()
+	batch := db.index.NewBatch()
+	if err := db.onAllBooks(func(book *Book) error {
+		batch.Index(book.Id.String(), flatten(book))
+		i++
+		if i%100 == 0 {
+			if err := db.index.Batch(batch); err != nil {
+				return err
+			}
+			batch = db.index.NewBatch()
+		}
 		return nil
-	})
+	}); err != nil {
+		return err
+	}
+
+	// Flush the batch indexer if not empty.
+	if err := db.index.Batch(batch); err != nil {
+		return err
+	}
+
+	log.Printf("re-indexing complete, %d documents in %g seconds", i, time.Since(start).Seconds())
+	return nil
 }
 
 // Call a function for all books in the database.