diff --git a/cmd/liber/liber.go b/cmd/liber/liber.go
index 59819b7092afbd4e87d7559679b8f46710f3ae14..10c55409b680a5b29d691312a9e2e302230f4c7d 100644
--- a/cmd/liber/liber.go
+++ b/cmd/liber/liber.go
@@ -13,13 +13,13 @@ import (
)
var (
- databaseDir = flag.String("db-dir", "~/.liber", "database directory")
- bookDir = flag.String("book-dir", "", "books directory")
+ databaseDir = flag.String("db", "~/.liber", "database directory")
+ bookDir = flag.String("books", "", "books directory")
)
func openDB() *liber.Database {
dbdir := util.ExpandTilde(*databaseDir)
- db, err := liber.NewDb(dbdir)
+ db, err := liber.NewDb("leveldb", dbdir)
if err != nil {
log.Fatal("error opening database: ", err)
}
diff --git a/database.go b/database.go
index b5fc26570a6c6a41ae82fd0942e3d202dc35cd64..508cce4c9758670012b2b70c696b4775e92ff2d2 100644
--- a/database.go
+++ b/database.go
@@ -29,31 +29,38 @@ import (
blevequery "github.com/blevesearch/bleve/search/query"
)
+// BookId is a unique identifier assigned to each book.
type BookId uint64
func (id BookId) String() string {
return strconv.FormatUint(uint64(id), 10)
}
+// Key returns a binary-encoded version of the identifier.
func (id BookId) Key() []byte {
var b [8]byte
binary.LittleEndian.PutUint64(b[:], uint64(id))
return b[:]
}
+// NewID creates a new random unique book identifier.
func NewID() BookId {
return BookId(rand.Int63())
}
+// ParseID parses a string with a hex-encoded representation of a
+// book identifier.
func ParseID(s string) BookId {
id, _ := strconv.ParseUint(s, 10, 64)
return BookId(id)
}
+// ParseBinaryID parses a binary-encoded book identifier.
func ParseBinaryID(b []byte) BookId {
return BookId(binary.LittleEndian.Uint64(b))
}
+// Book holds information about a book.
type Book struct {
Id BookId
CoverPath string
@@ -64,6 +71,7 @@ func (b *Book) String() string {
return fmt.Sprintf("%s (%s)", b.Metadata.String(), b.Id.String())
}
+// File holds information about a file that is archived in our storage.
type File struct {
Path string
FileType string
@@ -73,6 +81,8 @@ type File struct {
Id BookId
}
+// HasChanged returns true if the file metadata has changed compared
+// to what is passed as argument.
func (f *File) HasChanged(info os.FileInfo) bool {
return !info.ModTime().Equal(f.Mtime) || info.Size() != f.Size
}
@@ -197,6 +207,8 @@ type backend interface {
ScanFiles() <-chan *File
}
+// Database binds together a database of books/files with a searchable
+// full-text index, making sure that updates are synchronized.
type Database struct {
backend
@@ -204,6 +216,8 @@ type Database struct {
index bleve.Index
}
+// NewDb creates a new Database at the specified location. The only
+// supported backendType at the moment is 'leveldb'.
func NewDb(backendType, path string) (*Database, error) {
// Make sure that path exists.
if _, err := os.Stat(path); err != nil {
@@ -227,6 +241,7 @@ func NewDb(backendType, path string) (*Database, error) {
d := &Database{backend: b, path: path}
// Initialize the index.
if err := d.setupIndex(filepath.Join(path, "index")); err != nil {
+ b.Close()
return nil, err
}
return d, nil
@@ -251,11 +266,14 @@ func (db *Database) setupIndex(path string) error {
return nil
}
+// Close the database, the index, and all associated resources.
func (db *Database) Close() {
db.index.Close()
db.backend.Close()
}
+// PutBook stores a new book in the database (or updates an existing
+// one).
func (db *Database) PutBook(b *Book) error {
if err := db.backend.PutBook(b); err != nil {
return err
@@ -263,6 +281,7 @@ func (db *Database) PutBook(b *Book) error {
return db.index.Index(b.Id.String(), flatten(b))
}
+// DeleteBook removes a book from the database.
func (db *Database) DeleteBook(bookid BookId) error {
db.backend.DeleteBook(bookid)
return db.index.Delete(bookid.String())
@@ -438,6 +457,8 @@ func (db *Database) onAllBooks(f func(*Book) error) error {
return nil
}
+// SearchResult holds the result of a search query. NumResults refers
+// to the total number of results, regardless of pagination.
type SearchResult struct {
Results []*Book
NumResults int
@@ -464,7 +485,7 @@ func (db *Database) Search(queryStr string, offset, limit int) (*SearchResult, e
return db.doSearch(bleve.NewQueryStringQuery(queryStr), offset, limit)
}
-// Autocomplete runs a fuzzy search for a term.
+// Suggest runs a fuzzy search for a term, for autocompletion.
func (db *Database) Suggest(term string) (*SearchResult, error) {
query := bleve.NewTermQuery(term)
query.SetField("_suggest")
diff --git a/files.go b/files.go
index 6cd3f537563bb66f0ea5ab73b4cacbb07042a444..dbe60475314d3488091a68ca7d408f9c6e0387c2 100644
--- a/files.go
+++ b/files.go
@@ -17,13 +17,14 @@ type FileStorage struct {
Root string
}
+// NewFileStorage creates a new FileStorage with the specified root path.
func NewFileStorage(root string) *FileStorage {
return &FileStorage{
Root: root,
}
}
-// Return the absolute path of a file, given its relative path.
+// Abs returns the absolute path of a file, given its relative path.
func (s *FileStorage) Abs(path string) string {
if strings.HasPrefix(path, "/") {
return path
@@ -31,7 +32,7 @@ func (s *FileStorage) Abs(path string) string {
return filepath.Join(s.Root, path)
}
-// Return the relative path of a file with respect to the storage
+// Rel returns the relative path of a file with respect to the storage
// root.
func (s *FileStorage) Rel(abspath string) (string, error) {
return filepath.Rel(s.Root, abspath)
@@ -63,8 +64,9 @@ func (s *FileStorage) Rename(oldpath, newpath string) error {
return os.Rename(s.Abs(oldpath), s.Abs(newpath))
}
-func (s *FileStorage) Walk(w *util.Walker, fn filepath.WalkFunc) {
- w.Walk(s.Root, func(path string, info os.FileInfo, ferr error) error {
+// Walk the file hierarchy.
+func (s *FileStorage) Walk(w *util.Walker, fn filepath.WalkFunc) error {
+ return w.Walk(s.Root, func(path string, info os.FileInfo, ferr error) error {
if ferr != nil {
return nil
}
@@ -83,6 +85,8 @@ type RWFileStorage struct {
Nesting int
}
+// NewRWFileStorage creates a new RWFileStorage with the given
+// directory nesting factor.
func NewRWFileStorage(root string, nesting int) *RWFileStorage {
return &RWFileStorage{
FileStorage: NewFileStorage(root),
diff --git a/isbn_detect.go b/isbn_detect.go
index 57017092f3e0e8b680d2265629b12c728ba2faa8..ef2adf59b4135ff770c85bb3aef72f8b208e73b3 100644
--- a/isbn_detect.go
+++ b/isbn_detect.go
@@ -53,7 +53,7 @@ func findISBNInPage(r io.Reader) []string {
func validateIsbn10(isbn string) bool {
var sum int
- var multiply int = 10
+ multiply := 10
for i, v := range isbn {
if v == '-' {
continue
diff --git a/metadata.go b/metadata.go
index c54d4ef2e9b25afc99af22d17076f1d8cd0f41b7..c51da867d8a1e0e7225f38fd9585107a593149d1 100644
--- a/metadata.go
+++ b/metadata.go
@@ -8,18 +8,18 @@ import (
"regexp"
"strings"
- "github.com/meskio/epubgo"
"git.autistici.org/ale/liber/third_party/gobipocket"
+ "github.com/meskio/epubgo"
)
-// A metadata provider generates metadata from the local filesystem.
+// A MetadataProvider generates metadata from the local filesystem.
type MetadataProvider interface {
Name() string
Lookup(*FileStorage, string, string) (*Metadata, error)
GetBookCover(*FileStorage, string) (string, error)
}
-// A metadata refiner improves on existing metadata and may provide
+// A MetadataRefiner improves on existing metadata and may provide
// more than one result to choose from. It usually involves talking to
// a remote service.
type MetadataRefiner interface {
@@ -28,11 +28,13 @@ type MetadataRefiner interface {
GetBookCover(*Metadata) ([]byte, error)
}
+// MetadataSource identifies a document within a specific source.
type MetadataSource struct {
Name string
ID string
}
+// Metadata for a book.
type Metadata struct {
Title string
Date string
@@ -290,6 +292,8 @@ func (p *fileProvider) Name() string {
return "file"
}
+// GetFileType returns a file's type (i.e. the extension), if it is
+// one of the supported ones.
func GetFileType(path string) (string, error) {
filetype := strings.ToLower(filepath.Ext(path))
if filetype != ".epub" && filetype != ".mobi" && filetype != ".pdf" {
diff --git a/sync.go b/sync.go
index b16ced567eec76fbcea559b993923f052b084901..315a3eebb814fc83a4a3df9e4f1b0e3c6f45097e 100644
--- a/sync.go
+++ b/sync.go
@@ -120,27 +120,27 @@ func (r *remoteServer) SendBook(book *Book, storage *FileStorage, files []*File)
if err != nil {
return err
}
- if err := json.NewEncoder(part).Encode(book.Metadata); err != nil {
+ if err = json.NewEncoder(part).Encode(book.Metadata); err != nil {
return err
}
for i, f := range files {
varname := fmt.Sprintf("book%d", i)
filename := fmt.Sprintf("%d%s", book.Id, f.FileType)
- if err := addFilePart(w, varname, storage, f.Path, filename); err != nil {
+ if err = addFilePart(w, varname, storage, f.Path, filename); err != nil {
w.Close()
return err
}
}
if book.CoverPath != "" {
- if err := addFilePart(w, "cover", storage, book.CoverPath, "cover.jpg"); err != nil {
+ if err = addFilePart(w, "cover", storage, book.CoverPath, "cover.jpg"); err != nil {
w.Close()
return err
}
}
- if err := w.Close(); err != nil {
+ if err = w.Close(); err != nil {
return err
}
diff --git a/update.go b/update.go
index fc28c2f3c1be1dbe6324a1c22b0906de5f8e9a4c..6c0bbd6694b051efca489ccff2a4d18a2090465d 100644
--- a/update.go
+++ b/update.go
@@ -10,10 +10,13 @@ import (
)
const (
- SourceDB = 1 << iota
- SourceFS
+ sourceDB = 1 << iota
+ sourceFS
)
+// MetadataChooserFunc is a function that should allow the user to
+// select one of multiple candidates for metadata resolution. The
+// function can be interactive.
type MetadataChooserFunc func(string, []*Metadata) *Metadata
type fileData struct {
@@ -51,7 +54,7 @@ type updateContext struct {
func (uc *updateContext) dbFileScanner(fileCh chan fileData) {
for f := range uc.db.ScanFiles() {
fileCh <- fileData{
- source: SourceDB,
+ source: sourceDB,
path: f.Path,
id: f.Id,
}
@@ -59,14 +62,20 @@ func (uc *updateContext) dbFileScanner(fileCh chan fileData) {
}
func (uc *updateContext) localFileScanner(basedir string, fileCh chan fileData) {
- uc.storage.Walk(util.NewDefaultWalker(), func(path string, info os.FileInfo, err error) error {
+ err := uc.storage.Walk(util.NewDefaultWalker(), func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return nil
+ }
fileCh <- fileData{
- source: SourceFS,
+ source: sourceFS,
path: path,
info: info,
}
return nil
})
+ if err != nil {
+ log.Printf("walk error: %v", err)
+ }
}
func (uc *updateContext) differ(basedir string) chan fileData {
@@ -93,9 +102,9 @@ func (uc *updateContext) differ(basedir string) chan fileData {
// Merge the two sources and keep track of files that
// only appear in the database but not on the
// filesystem, so we can remove them at the end.
- // All entries with source == SourceFS will be sent to
+ // All entries with source == sourceFS will be sent to
// the output channel in any case.
- allSources := SourceDB | SourceFS
+ allSources := sourceDB | sourceFS
tmp := make(map[string]int)
for f := range fileCh {
tmp[f.path] |= f.source
@@ -104,12 +113,12 @@ func (uc *updateContext) differ(basedir string) chan fileData {
if tmp[f.path] == allSources {
delete(tmp, f.path)
}
- if f.source == SourceFS {
+ if f.source == sourceFS {
outCh <- f
}
}
for path, value := range tmp {
- if value == SourceDB {
+ if value == sourceDB {
log.Printf("file %s has been removed", path)
uc.db.DeleteFile(path)
}
@@ -283,6 +292,12 @@ var (
const numUpdateMetadataWorkers = 10
+// Update the database with the contents of the storage at dir. If
+// there are multiple options for metadata resolution, invoke the
+// provided MetadataChooserFunc.
+//
+// This will walk the source filesystem, detect changed files, and
+// trigger the metadata resolution process.
func (db *Database) Update(dir string, chooser MetadataChooserFunc) {
// Parallelize metadata extraction, serialize database updates
// (so that index-based de-duplication works).