Commit 825467fc authored by ale's avatar ale

Add methods to dump the entire database

There aren't any external tools to backup a LevelDB database, so it's
nice to have a way to do so safely: the dump method is exposed to
localhost on the web interface.
parent 7f2d5345
Pipeline #454 passed with stages
in 1 minute and 26 seconds
......@@ -7,6 +7,7 @@ import (
"encoding/gob"
"errors"
"fmt"
"io"
"log"
"math/rand"
"os"
......@@ -400,6 +401,61 @@ func (db *Database) Scan(bucket []byte) *DatabaseIterator {
return &DatabaseIterator{iter: it}
}
func writeBytes(w io.Writer, b []byte) error {
binary.Write(w, binary.LittleEndian, uint32(len(b)))
_, err := w.Write(b)
return err
}
func readBytes(r io.Reader) ([]byte, error) {
var sz uint32
if err := binary.Read(r, binary.LittleEndian, &sz); err != nil {
return nil, err
}
b := make([]byte, sz)
_, err := r.Read(b)
return b, err
}
// Dump the contents of the database to a Writer.
func (db *Database) Dump(w io.Writer) error {
it := db.ldb.NewIterator(nil, &ldbopt.ReadOptions{DontFillCache: true})
defer it.Release()
count := 0
for it.Next() {
writeBytes(w, it.Key())
writeBytes(w, it.Value())
count++
}
log.Printf("dumped %d entries from the database", count)
return nil
}
// Restore a backup to the current database (assuming it is empty).
func (db *Database) Restore(r io.Reader) error {
count := 0
for {
key, err := readBytes(r)
if err == io.EOF {
break
}
if err != nil {
return err
}
value, err := readBytes(r)
if err == io.EOF {
return errors.New("unexpected eof")
}
if err != nil {
return err
}
db.RawPut(key, value)
count++
}
log.Printf("restored %d entries to the database", count)
return db.Reindex()
}
// Reindex the entire database. This is an administrative operation,
// to be performed after an incompatible index schema change. It will
// delete the existing index and re-create it from scratch.
......
package liber
import (
"bytes"
"io"
"io/ioutil"
"os"
......@@ -20,6 +21,15 @@ func (td *testDatabase) Close() {
os.RemoveAll(td.path)
}
func newEmptyTestDatabase(t *testing.T) (*testDatabase, *Database) {
path, _ := ioutil.TempDir("", "testdb-")
db, err := NewDb(path)
if err != nil {
t.Fatalf("NewDb(): %v", err)
}
return &testDatabase{db: db, path: path}, db
}
func newTestDatabase(t *testing.T) (*testDatabase, *Database) {
path, _ := ioutil.TempDir("", "testdb-")
db, err := NewDb(path)
......@@ -251,6 +261,32 @@ func TestDatabase_Reindex(t *testing.T) {
doTest()
}
func TestDatabase_DumpAndRestore(t *testing.T) {
td, db := newTestDatabase2(t)
defer td.Close()
book, _ := db.GetBook(td.refbookid)
m := book.Metadata
var buf bytes.Buffer
if err := db.Dump(&buf); err != nil {
t.Fatalf("Dump: %v", err)
}
td2, db2 := newEmptyTestDatabase(t)
defer td2.Close()
if err := db2.Restore(&buf); err != nil {
t.Fatalf("Restore: %v", err)
}
// Find the sample book.
if result, err := db2.Find(m.Uniques()); err != nil {
t.Errorf("Not found: %v", err)
} else if result.Id != book.Id {
t.Errorf("Bad match with ISBN: got=%d, expected=%d", result.Id, book.Id)
}
}
// func TestDatabase_Find2(t *testing.T) {
// td, db := newTestDatabase(t)
// defer td.Close()
......
......@@ -12,6 +12,7 @@ import (
_ "image/png"
"io"
"log"
"net"
"net/http"
"net/url"
"os"
......@@ -324,6 +325,19 @@ func (s *uiServer) handleShowThumbnail(book *Book, w http.ResponseWriter, req *h
w.Write(data)
}
func (s *uiServer) handleDumpDatabase(w http.ResponseWriter, req *http.Request) {
// Only allow requests from localhost.
host, _, _ := net.SplitHostPort(req.RemoteAddr)
ip := net.ParseIP(host)
if !ip.IsLoopback() {
http.Error(w, "Unauthorized", http.StatusUnauthorized)
return
}
w.Header().Set("Content-Type", "application/octet-stream")
s.db.Dump(w)
}
func (s *uiServer) handleHome(w http.ResponseWriter, req *http.Request) {
render("index.html", w, nil)
}
......@@ -376,6 +390,7 @@ func NewHttpServer(db *Database, storage, cache *RWFileStorage, addr string) *ht
http.FileServer(http.Dir(filepath.Join(*htdocsDir, "static")))))
r.HandleFunc("/api/sync/upload", syncsrv.handleSyncUpload).Methods("POST")
r.HandleFunc("/api/sync/diff", syncsrv.handleDiffRequest).Methods("POST")
r.HandleFunc("/internal/dump_db", uisrv.handleDumpDatabase)
r.Handle("/book/cover/{id:[0-9]+}", uisrv.withBook(uisrv.handleShowCover))
r.Handle("/book/thumb/{id:[0-9]+}", uisrv.withBook(uisrv.handleShowThumbnail))
r.Handle("/book/{id:[0-9]+}", uisrv.withBook(uisrv.handleShowBook))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment