Commit c9254c36 authored by ale's avatar ale

Merge branch 'master' of git.autistici.org:ale/liber

parents a19b42bc bff49bef
Pipeline #461 passed with stages
in 1 minute and 23 seconds
......@@ -40,11 +40,11 @@ func (c *dumpCommand) Execute(ctx context.Context, f *flag.FlagSet, _ ...interfa
// Restore the database.
type restoreCommand struct{}
func (c *restoreCommand) Name() string { return "dump-db" }
func (c *restoreCommand) Synopsis() string { return "Dump the database" }
func (c *restoreCommand) Name() string { return "restore-db" }
func (c *restoreCommand) Synopsis() string { return "Restore the database" }
func (c *restoreCommand) Usage() string {
return `dump-db
Dump the database.
return `restore-db
Restore the database.
Requires exclusive access to the db, so the HTTP server must not be
running at the same time.
......
......@@ -37,6 +37,7 @@ func main() {
subcommands.Register(&dumpCommand{}, "Maintenance")
subcommands.Register(&restoreCommand{}, "Maintenance")
subcommands.Register(&reindexCommand{}, "Maintenance")
subcommands.Register(&listCommand{}, "Maintenance")
log.SetFlags(0)
flag.Parse()
......
package main
import (
"context"
"flag"
"fmt"
"log"
"os"
"strings"
"git.autistici.org/ale/liber"
"github.com/google/subcommands"
)
type listCommand struct{}
func (c *listCommand) Name() string { return "list-books" }
func (c *listCommand) Synopsis() string { return "List books that match certain criteria" }
func (c *listCommand) SetFlags(f *flag.FlagSet) {}
func (c *listCommand) Usage() string {
return `list-books <CRITERIA>
List books that match certain criteria. Known tags you can use:
+all
+incomplete
+invalid
+source:<SOURCE>
+nofiles
+missingcover
`
}
func matchSource(source string) func(*liber.Book) bool {
return func(book *liber.Book) bool {
if book.Metadata == nil {
return false
}
for _, msrc := range book.Metadata.Sources {
if msrc.Name == source {
return true
}
}
return false
}
}
func matchNoFiles(db *liber.Database) func(*liber.Book) bool {
return func(book *liber.Book) bool {
f, _ := db.GetBookFiles(book.Id)
return len(f) == 0
}
}
func matchMissingCover(book *liber.Book) bool {
if book.CoverPath == "" {
return true
}
_, err := os.Stat(book.CoverPath)
return err != nil && os.IsNotExist(err)
}
func matchIncomplete(book *liber.Book) bool {
return (book.Metadata != nil && book.Metadata.Complete())
}
func matchInvalid(_ *liber.Book) bool {
return false
}
func matchAll(_ *liber.Book) bool {
return true
}
func (c *listCommand) parseTag(db *liber.Database, tag string) (func(*liber.Book) bool, error) {
if strings.HasPrefix(tag, "+source:") {
return matchSource(tag[7:]), nil
}
switch tag {
case "+all":
return matchAll, nil
case "+incomplete":
return matchIncomplete, nil
case "+invalid":
return matchInvalid, nil
case "+nofiles":
return matchNoFiles(db), nil
case "+missingcover":
return matchMissingCover, nil
default:
return nil, fmt.Errorf("unknown tag '%s'", tag)
}
}
func (c *listCommand) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
if f.NArg() == 0 {
log.Printf("Not enough arguments")
return subcommands.ExitUsageError
}
db := openDB()
defer db.Close()
var matchFuncs []func(*liber.Book) bool
for _, arg := range f.Args() {
f, err := c.parseTag(db, arg)
if err != nil {
log.Printf("error: %v", err)
return subcommands.ExitUsageError
}
matchFuncs = append(matchFuncs, f)
}
db.ListBooks(os.Stdout, matchFuncs...)
return subcommands.ExitSuccess
}
package main
import (
"context"
"flag"
"log"
"os"
"path/filepath"
"github.com/google/subcommands"
"git.autistici.org/ale/liber/util"
)
type refineCommand struct{}
func (c *refineCommand) Name() string { return "update" }
func (c *refineCommand) Synopsis() string { return "Add books to the local db" }
func (c *refineCommand) SetFlags(f *flag.FlagSet) {}
func (c *refineCommand) Usage() string {
return `update [<OPTIONS>]
Add books to the local database.
`
}
func (c *refineCommand) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
if f.NArg() > 0 {
log.Printf("Too many arguments")
return subcommands.ExitUsageError
}
db := openDB()
defer db.Close()
// Redirect logging to dbdir/refine.log.
logf, err := os.OpenFile(filepath.Join(*databaseDir, "refine.log"), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
if err == nil {
defer logf.Close()
log.SetOutput(logf)
log.SetFlags(log.Ldate | log.Ltime)
}
db.Refine(util.ExpandTilde(*bookDir))
return subcommands.ExitSuccess
}
......@@ -50,9 +50,22 @@ func (id BookId) String() string {
}
func (id BookId) Key() []byte {
var buf bytes.Buffer
binary.Write(&buf, binary.LittleEndian, id)
return buf.Bytes()
var b [8]byte
binary.LittleEndian.PutUint64(b[:], uint64(id))
return b[:]
}
func NewID() BookId {
return BookId(rand.Int63())
}
func ParseID(s string) BookId {
id, _ := strconv.ParseUint(s, 10, 64)
return BookId(id)
}
func ParseBinaryID(b []byte) BookId {
return BookId(binary.LittleEndian.Uint64(b))
}
type Book struct {
......@@ -85,15 +98,6 @@ func init() {
rand.Seed(seed)
}
func NewID() BookId {
return BookId(rand.Int63())
}
func ParseID(s string) BookId {
id, _ := strconv.ParseUint(s, 10, 64)
return BookId(id)
}
// The structure that gets actually indexed.
type flatBook struct {
Title string `json:"title"`
......@@ -251,6 +255,42 @@ func (db *Database) setupIndex(path string) error {
return nil
}
var schemaVersionKey = []byte("_liber_schema_version")
func (db *Database) getSchemaVersion() uint64 {
data, err := db.ldb.Get(schemaVersionKey, nil)
if err != nil {
return 0
}
return binary.LittleEndian.Uint64(data)
}
func (db *Database) setSchemaVersion(v uint64) {
var b [8]byte
binary.LittleEndian.PutUint64(b[:], v)
db.ldb.Put(schemaVersionKey, b[:], nil)
}
type databaseMigration struct {
version uint64
run func(db *Database) error
}
func (db *Database) runMigrations(migrations []databaseMigration) error {
version := db.getSchemaVersion()
for _, m := range migrations {
if m.version < version {
continue
}
if err := m.run(db); err != nil {
return err
}
version = m.version
db.setSchemaVersion(version)
}
return nil
}
func (db *Database) Close() {
db.index.Close()
db.ldb.Close()
......@@ -478,15 +518,25 @@ func (db *Database) Reindex() error {
}
// Scan the database and re-index everything.
i := db.Scan(BookBucket)
for i.Next() {
return db.onAllBooks(func(book *Book) error {
db.index.Index(book.Id.String(), flatten(book))
return nil
})
}
// Call a function for all books in the database.
func (db *Database) onAllBooks(f func(*Book) error) error {
it := db.Scan(BookBucket)
for it.Next() {
var book Book
if err := i.Value(&book); err != nil {
if err := it.Value(&book); err != nil {
continue
}
db.index.Index(i.Id().String(), flatten(&book))
if err := f(&book); err != nil {
return err
}
}
return i.Close()
return it.Close()
}
type SearchResult struct {
......@@ -560,10 +610,7 @@ func keyToId(key []byte) BookId {
if n < 0 {
return 0
}
var id uint64
binary.Read(bytes.NewReader(key[n+1:]), binary.LittleEndian, &id)
return BookId(id)
return ParseBinaryID(key[n+1:])
}
func keyRange(prefix []byte) ([]byte, []byte) {
......
package liber
import (
"bufio"
"bytes"
"fmt"
"io"
"log"
)
func (db *Database) Refine(dir string) error {
//storage := NewFileStorage(dir)
return db.onAllBooks(func(book *Book) error {
return nil
})
}
// ListBooks writes IDs of books that match any of a series of
// functions to an io.Writer.
func (db *Database) ListBooks(w io.Writer, matchFuncs ...func(book *Book) bool) error {
return db.onAllBooks(func(book *Book) error {
ok := false
for _, f := range matchFuncs {
if f(book) {
ok = true
break
}
}
if ok {
fmt.Fprintf(w, "%s\n", book.Id)
}
return nil
})
}
// WithBookIDs calls a function on books whose IDs are read from the
// specified io.Reader.
func (db *Database) WithBookIDs(r io.Reader, f func(book *Book) error) error {
scanner := bufio.NewScanner(r)
for scanner.Scan() {
id := ParseID(string(bytes.TrimSpace(scanner.Bytes())))
book, err := db.GetBook(id)
if err != nil {
log.Printf("error: no such book %s", id)
continue
}
if err := f(book); err != nil {
log.Printf("error: %s: %v", id, err)
// Stop?
}
}
return scanner.Err()
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment