Commit ce2898f6 authored by ale's avatar ale

Use batching to make reindexing faster

parent 1555c0d9
......@@ -225,7 +225,7 @@ func (db *Database) setupLevelDb(path string) error {
// Use 128MB of cache and a small Bloom filter.
opts := &ldbopt.Options{
Filter: ldbfilter.NewBloomFilter(10),
BlockCacheCapacity: 2 << 27,
BlockCacheCapacity: 1 << 28, // 256MB
}
ldb, err := leveldb.OpenFile(path, opts)
......@@ -495,10 +495,14 @@ func (db *Database) Restore(r io.Reader) error {
if err != nil {
return err
}
db.RawPut(key, value)
count++
if count%1000 == 0 {
log.Printf("restored %d entries", count)
}
}
log.Printf("restored %d entries to the database", count)
log.Printf("restore complete (%d entries)", count)
return db.Reindex()
}
......@@ -517,11 +521,32 @@ func (db *Database) Reindex() error {
return err
}
// Scan the database and re-index everything.
return db.onAllBooks(func(book *Book) error {
db.index.Index(book.Id.String(), flatten(book))
// Scan the database and re-index everything. Use batch
// indexing to achieve decent performance.
i := 0
start := time.Now()
batch := db.index.NewBatch()
if err := db.onAllBooks(func(book *Book) error {
batch.Index(book.Id.String(), flatten(book))
i++
if i%100 == 0 {
if err := db.index.Batch(batch); err != nil {
return err
}
batch = db.index.NewBatch()
}
return nil
})
}); err != nil {
return err
}
// Flush the batch indexer if not empty.
if err := db.index.Batch(batch); err != nil {
return err
}
log.Printf("re-indexing complete, %d documents in %g seconds", i, time.Since(start).Seconds())
return nil
}
// Call a function for all books in the database.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment