Commit 825467fc authored by ale's avatar ale

Add methods to dump the entire database

There aren't any external tools to backup a LevelDB database, so it's
nice to have a way to do so safely: the dump method is exposed to
localhost on the web interface.
parent 7f2d5345
Pipeline #454 passed with stages
in 1 minute and 26 seconds
...@@ -7,6 +7,7 @@ import ( ...@@ -7,6 +7,7 @@ import (
"encoding/gob" "encoding/gob"
"errors" "errors"
"fmt" "fmt"
"io"
"log" "log"
"math/rand" "math/rand"
"os" "os"
...@@ -400,6 +401,61 @@ func (db *Database) Scan(bucket []byte) *DatabaseIterator { ...@@ -400,6 +401,61 @@ func (db *Database) Scan(bucket []byte) *DatabaseIterator {
return &DatabaseIterator{iter: it} return &DatabaseIterator{iter: it}
} }
func writeBytes(w io.Writer, b []byte) error {
binary.Write(w, binary.LittleEndian, uint32(len(b)))
_, err := w.Write(b)
return err
}
func readBytes(r io.Reader) ([]byte, error) {
var sz uint32
if err := binary.Read(r, binary.LittleEndian, &sz); err != nil {
return nil, err
}
b := make([]byte, sz)
_, err := r.Read(b)
return b, err
}
// Dump the contents of the database to a Writer.
func (db *Database) Dump(w io.Writer) error {
it := db.ldb.NewIterator(nil, &ldbopt.ReadOptions{DontFillCache: true})
defer it.Release()
count := 0
for it.Next() {
writeBytes(w, it.Key())
writeBytes(w, it.Value())
count++
}
log.Printf("dumped %d entries from the database", count)
return nil
}
// Restore a backup to the current database (assuming it is empty).
func (db *Database) Restore(r io.Reader) error {
count := 0
for {
key, err := readBytes(r)
if err == io.EOF {
break
}
if err != nil {
return err
}
value, err := readBytes(r)
if err == io.EOF {
return errors.New("unexpected eof")
}
if err != nil {
return err
}
db.RawPut(key, value)
count++
}
log.Printf("restored %d entries to the database", count)
return db.Reindex()
}
// Reindex the entire database. This is an administrative operation, // Reindex the entire database. This is an administrative operation,
// to be performed after an incompatible index schema change. It will // to be performed after an incompatible index schema change. It will
// delete the existing index and re-create it from scratch. // delete the existing index and re-create it from scratch.
......
package liber package liber
import ( import (
"bytes"
"io" "io"
"io/ioutil" "io/ioutil"
"os" "os"
...@@ -20,6 +21,15 @@ func (td *testDatabase) Close() { ...@@ -20,6 +21,15 @@ func (td *testDatabase) Close() {
os.RemoveAll(td.path) os.RemoveAll(td.path)
} }
func newEmptyTestDatabase(t *testing.T) (*testDatabase, *Database) {
path, _ := ioutil.TempDir("", "testdb-")
db, err := NewDb(path)
if err != nil {
t.Fatalf("NewDb(): %v", err)
}
return &testDatabase{db: db, path: path}, db
}
func newTestDatabase(t *testing.T) (*testDatabase, *Database) { func newTestDatabase(t *testing.T) (*testDatabase, *Database) {
path, _ := ioutil.TempDir("", "testdb-") path, _ := ioutil.TempDir("", "testdb-")
db, err := NewDb(path) db, err := NewDb(path)
...@@ -251,6 +261,32 @@ func TestDatabase_Reindex(t *testing.T) { ...@@ -251,6 +261,32 @@ func TestDatabase_Reindex(t *testing.T) {
doTest() doTest()
} }
func TestDatabase_DumpAndRestore(t *testing.T) {
td, db := newTestDatabase2(t)
defer td.Close()
book, _ := db.GetBook(td.refbookid)
m := book.Metadata
var buf bytes.Buffer
if err := db.Dump(&buf); err != nil {
t.Fatalf("Dump: %v", err)
}
td2, db2 := newEmptyTestDatabase(t)
defer td2.Close()
if err := db2.Restore(&buf); err != nil {
t.Fatalf("Restore: %v", err)
}
// Find the sample book.
if result, err := db2.Find(m.Uniques()); err != nil {
t.Errorf("Not found: %v", err)
} else if result.Id != book.Id {
t.Errorf("Bad match with ISBN: got=%d, expected=%d", result.Id, book.Id)
}
}
// func TestDatabase_Find2(t *testing.T) { // func TestDatabase_Find2(t *testing.T) {
// td, db := newTestDatabase(t) // td, db := newTestDatabase(t)
// defer td.Close() // defer td.Close()
......
...@@ -12,6 +12,7 @@ import ( ...@@ -12,6 +12,7 @@ import (
_ "image/png" _ "image/png"
"io" "io"
"log" "log"
"net"
"net/http" "net/http"
"net/url" "net/url"
"os" "os"
...@@ -324,6 +325,19 @@ func (s *uiServer) handleShowThumbnail(book *Book, w http.ResponseWriter, req *h ...@@ -324,6 +325,19 @@ func (s *uiServer) handleShowThumbnail(book *Book, w http.ResponseWriter, req *h
w.Write(data) w.Write(data)
} }
func (s *uiServer) handleDumpDatabase(w http.ResponseWriter, req *http.Request) {
// Only allow requests from localhost.
host, _, _ := net.SplitHostPort(req.RemoteAddr)
ip := net.ParseIP(host)
if !ip.IsLoopback() {
http.Error(w, "Unauthorized", http.StatusUnauthorized)
return
}
w.Header().Set("Content-Type", "application/octet-stream")
s.db.Dump(w)
}
func (s *uiServer) handleHome(w http.ResponseWriter, req *http.Request) { func (s *uiServer) handleHome(w http.ResponseWriter, req *http.Request) {
render("index.html", w, nil) render("index.html", w, nil)
} }
...@@ -376,6 +390,7 @@ func NewHttpServer(db *Database, storage, cache *RWFileStorage, addr string) *ht ...@@ -376,6 +390,7 @@ func NewHttpServer(db *Database, storage, cache *RWFileStorage, addr string) *ht
http.FileServer(http.Dir(filepath.Join(*htdocsDir, "static"))))) http.FileServer(http.Dir(filepath.Join(*htdocsDir, "static")))))
r.HandleFunc("/api/sync/upload", syncsrv.handleSyncUpload).Methods("POST") r.HandleFunc("/api/sync/upload", syncsrv.handleSyncUpload).Methods("POST")
r.HandleFunc("/api/sync/diff", syncsrv.handleDiffRequest).Methods("POST") r.HandleFunc("/api/sync/diff", syncsrv.handleDiffRequest).Methods("POST")
r.HandleFunc("/internal/dump_db", uisrv.handleDumpDatabase)
r.Handle("/book/cover/{id:[0-9]+}", uisrv.withBook(uisrv.handleShowCover)) r.Handle("/book/cover/{id:[0-9]+}", uisrv.withBook(uisrv.handleShowCover))
r.Handle("/book/thumb/{id:[0-9]+}", uisrv.withBook(uisrv.handleShowThumbnail)) r.Handle("/book/thumb/{id:[0-9]+}", uisrv.withBook(uisrv.handleShowThumbnail))
r.Handle("/book/{id:[0-9]+}", uisrv.withBook(uisrv.handleShowBook)) r.Handle("/book/{id:[0-9]+}", uisrv.withBook(uisrv.handleShowBook))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment