add a low-level db backup/restore tool

cmd/liberdbtool/liberdbtool.go

+// Raw backup/restore tool for the 'liber' database.  Mostly useful
+// when upgrading the backend to an on-disk incompatible version.
+//
+package main
+
+import (
+	"encoding/binary"
+	"flag"
+	"io"
+	"log"
+	"os"
+
+	"git.autistici.org/ale/liber"
+	"git.autistici.org/ale/liber/util"
+)
+
+var (
+	databaseDir = flag.String("db-dir", "~/.liber", "database directory")
+	doDump      = flag.Bool("dump", false, "dump the database")
+	doRestore   = flag.Bool("restore", false, "restore the database")
+)
+
+var buckets = [][]byte{
+	liber.BookBucket,
+	liber.FileBucket,
+	liber.BookFileBucket,
+}
+
+func writeBytes(w io.Writer, b []byte) error {
+	sz := uint64(len(b))
+	if err := binary.Write(w, binary.BigEndian, sz); err != nil {
+		return err
+	}
+	_, err := w.Write(b)
+	return err
+}
+
+func readBytes(r io.Reader) ([]byte, error) {
+	var sz uint64
+	if err := binary.Read(r, binary.BigEndian, &sz); err != nil {
+		return nil, err
+	}
+	b := make([]byte, sz)
+	_, err := r.Read(b)
+	return b, err
+}
+
+func writeRecord(w io.Writer, key, value []byte) error {
+	if err := writeBytes(w, key); err != nil {
+		return err
+	}
+	return writeBytes(w, value)
+}
+
+func readRecord(r io.Reader) ([]byte, []byte, error) {
+	key, err := readBytes(r)
+	if err != nil {
+		return nil, nil, err
+	}
+	value, err := readBytes(r)
+	if err != nil {
+		return nil, nil, err
+	}
+	return key, value, nil
+}
+
+func backup(db *liber.Database, w io.Writer) error {
+	for _, bkt := range buckets {
+		i := db.Scan(bkt)
+		for ; i.Valid(); i.Next() {
+			if err := writeRecord(w, i.RawKey(), i.RawValue()); err != nil {
+				return err
+			}
+		}
+		i.Close()
+	}
+	return nil
+}
+
+func restore(db *liber.Database, r io.Reader) error {
+	nrec := 0
+	for {
+		key, value, err := readRecord(r)
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return err
+		}
+		if err := db.RawPut(key, value); err != nil {
+			return err
+		}
+		nrec++
+	}
+	log.Printf("restored %d records, reindexing...", nrec)
+	if err := db.Reindex(); err != nil {
+		return err
+	}
+	log.Printf("done")
+	return nil
+}
+
+func main() {
+	log.SetFlags(0)
+	flag.Parse()
+
+	dbdir := util.ExpandTilde(*databaseDir)
+	db, err := liber.NewDb(dbdir)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer db.Close()
+
+	switch {
+	case *doDump:
+		err = backup(db, os.Stdout)
+	case *doRestore:
+		err = restore(db, os.Stdin)
+	}
+	if err != nil {
+		log.Fatal(err)
+	}
+}

database.go

@@ -309,16 +309,18 @@ func (db *Database) PutFile(f *File) error {
 	return nil
 }
 
+func (db *Database) RawPut(key, value []byte) error {
+	wo := levigo.NewWriteOptions()
+	defer wo.Close()
+	return db.leveldb.Put(wo, key, value)
+}
+
 func (db *Database) Put(bucket, key []byte, obj interface{}) error {
 	var buf bytes.Buffer
 	if err := gob.NewEncoder(&buf).Encode(obj); err != nil {
 		return err
 	}
-
-	wo := levigo.NewWriteOptions()
-	defer wo.Close()
-
-	return db.leveldb.Put(wo, bktToKey(bucket, key), buf.Bytes())
+	return db.RawPut(bktToKey(bucket, key), buf.Bytes())
 }
 
 func (db *Database) DeleteBook(bookid BookId) error {
@@ -375,6 +377,14 @@ func (i *DatabaseIterator) Id() BookId {
 	return keyToId(i.iter.Key())
 }
 
+func (i *DatabaseIterator) RawKey() []byte {
+	return i.iter.Key()
+}
+
+func (i *DatabaseIterator) RawValue() []byte {
+	return i.iter.Value()
+}
+
 func (i *DatabaseIterator) Value(obj interface{}) error {
 	return gob.NewDecoder(bytes.NewReader(i.iter.Value())).Decode(obj)
 }
@@ -413,7 +423,9 @@ func (db *Database) Reindex() error {
 	}
 
 	// Scan the database and re-index everything.
-	for i := db.Scan(BookBucket); i.Valid(); i.Next() {
+	i := db.Scan(BookBucket)
+	defer i.Close()
+	for ; i.Valid(); i.Next() {
 		var book Book
 		if err := i.Value(&book); err != nil {
 			continue