Add a list-books command to scan the book database

The idea is to support a set of metadata checks of various
kinds, to select books in large databases for further processing.

cmd/liber/liber.go

@@ -37,6 +37,7 @@ func main() {
 	subcommands.Register(&dumpCommand{}, "Maintenance")
 	subcommands.Register(&restoreCommand{}, "Maintenance")
 	subcommands.Register(&reindexCommand{}, "Maintenance")
+	subcommands.Register(&listCommand{}, "Maintenance")
 
 	log.SetFlags(0)
 	flag.Parse()

cmd/liber/list.go

+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"git.autistici.org/ale/liber"
+
+	"github.com/google/subcommands"
+)
+
+type listCommand struct{}
+
+func (c *listCommand) Name() string             { return "list-books" }
+func (c *listCommand) Synopsis() string         { return "List books that match certain criteria" }
+func (c *listCommand) SetFlags(f *flag.FlagSet) {}
+func (c *listCommand) Usage() string {
+	return `list-books <CRITERIA>
+  List books that match certain criteria. Known tags you can use:
+
+    +all
+    +incomplete
+    +invalid
+    +source:<SOURCE>
+
+`
+}
+
+func matchSource(source string) func(*liber.Book) bool {
+	return func(book *liber.Book) bool {
+		if book.Metadata == nil {
+			return false
+		}
+		for _, msrc := range book.Metadata.Sources {
+			if msrc.Name == source {
+				return true
+			}
+		}
+		return false
+	}
+}
+
+func matchIncomplete(book *liber.Book) bool {
+	return (book.Metadata != nil && book.Metadata.Complete())
+}
+
+func matchInvalid(_ *liber.Book) bool {
+	return false
+}
+
+func matchAll(_ *liber.Book) bool {
+	return true
+}
+
+func (c *listCommand) parseTag(tag string) (func(*liber.Book) bool, error) {
+	if strings.HasPrefix(tag, "+source:") {
+		return matchSource(tag[7:]), nil
+	}
+
+	switch tag {
+	case "+all":
+		return matchAll, nil
+	case "+incomplete":
+		return matchIncomplete, nil
+	case "+invalid":
+		return matchInvalid, nil
+	default:
+		return nil, fmt.Errorf("unknown tag '%s'", tag)
+	}
+}
+
+func (c *listCommand) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
+	if f.NArg() == 0 {
+		log.Printf("Not enough arguments")
+		return subcommands.ExitUsageError
+	}
+
+	db := openDB()
+	defer db.Close()
+
+	var matchFuncs []func(*liber.Book) bool
+	for _, arg := range f.Args() {
+		f, err := c.parseTag(arg)
+		if err != nil {
+			log.Printf("error: %v", err)
+			return subcommands.ExitUsageError
+		}
+		matchFuncs = append(matchFuncs, f)
+	}
+
+	db.ListBooks(os.Stdout, matchFuncs...)
+
+	return subcommands.ExitSuccess
+}

cmd/liber/refine.go

+package main
+
+import (
+	"context"
+	"flag"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/google/subcommands"
+
+	"git.autistici.org/ale/liber/util"
+)
+
+type refineCommand struct{}
+
+func (c *refineCommand) Name() string             { return "update" }
+func (c *refineCommand) Synopsis() string         { return "Add books to the local db" }
+func (c *refineCommand) SetFlags(f *flag.FlagSet) {}
+func (c *refineCommand) Usage() string {
+	return `update [<OPTIONS>]
+  Add books to the local database.
+
+`
+}
+
+func (c *refineCommand) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
+	if f.NArg() > 0 {
+		log.Printf("Too many arguments")
+		return subcommands.ExitUsageError
+	}
+
+	db := openDB()
+	defer db.Close()
+
+	// Redirect logging to dbdir/refine.log.
+	logf, err := os.OpenFile(filepath.Join(*databaseDir, "refine.log"), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
+	if err == nil {
+		defer logf.Close()
+		log.SetOutput(logf)
+		log.SetFlags(log.Ldate | log.Ltime)
+	}
+
+	db.Refine(util.ExpandTilde(*bookDir))
+
+	return subcommands.ExitSuccess
+}

database.go

@@ -478,15 +478,25 @@ func (db *Database) Reindex() error {
 	}
 
 	// Scan the database and re-index everything.
-	i := db.Scan(BookBucket)
-	for i.Next() {
+	return db.onAllBooks(func(book *Book) error {
+		db.index.Index(book.Id.String(), flatten(book))
+		return nil
+	})
+}
+
+// Call a function for all books in the database.
+func (db *Database) onAllBooks(f func(*Book) error) error {
+	it := db.Scan(BookBucket)
+	for it.Next() {
 		var book Book
-		if err := i.Value(&book); err != nil {
+		if err := it.Value(&book); err != nil {
 			continue
 		}
-		db.index.Index(i.Id().String(), flatten(&book))
+		if err := f(&book); err != nil {
+			return err
+		}
 	}
-	return i.Close()
+	return it.Close()
 }
 
 type SearchResult struct {

refine.go

+package liber
+
+import (
+	"fmt"
+	"io"
+)
+
+func (db *Database) Refine(dir string) error {
+	//storage := NewFileStorage(dir)
+	return db.onAllBooks(func(book *Book) error {
+		return nil
+	})
+}
+
+// ListBooks writes IDs of books that match any of a series of
+// functions to an io.Writer.
+func (db *Database) ListBooks(w io.Writer, matchFuncs ...func(book *Book) bool) error {
+	return db.onAllBooks(func(book *Book) error {
+		ok := false
+		for _, f := range matchFuncs {
+			if f(book) {
+				ok = true
+				break
+			}
+		}
+		if ok {
+			fmt.Fprintf(w, "%s\n", book.Id)
+		}
+		return nil
+	})
+}