Commit 30145e43 authored by ale's avatar ale

use relative paths to a FileStorage wherever possible

parent 9927da5b
package liber
import (
"fmt"
"os"
"path/filepath"
"strings"
"git.autistici.org/ale/liber/util"
)
// FileStorage exposes a read-only filesystem hierarchy as a root for
// relative paths (so that you can move archives around while the
// database is still valid). Calls will still accept absolute paths
// for backwards compatibility.
type FileStorage struct {
Root string
Nesting int
Root string
}
func NewFileStorage(root string, nesting int) *FileStorage {
func NewFileStorage(root string) *FileStorage {
return &FileStorage{
Root: root,
Nesting: nesting,
Root: root,
}
}
// Path of the file corresponding to the given key, relative to the
// root directory.
func (s *FileStorage) Path(key string) string {
var parts []string
for i := 0; i < s.Nesting; i++ {
if i >= len(key) {
break
}
parts = append(parts, key[i:i+1])
// Return the absolute path of a file, given its relative path.
func (s *FileStorage) Abs(path string) string {
if strings.HasPrefix(path, "/") {
return path
}
parts = append(parts, key)
return filepath.Join(parts...)
return filepath.Join(s.Root, path)
}
// Return the relative path of a file with respect to the storage
// root.
func (s *FileStorage) Rel(abspath string) (string, error) {
return filepath.Rel(s.Root, abspath)
}
// Create a new file for the given key.
// Create a new file for the given key. Directories containing the
// output file will be automatically created.
func (s *FileStorage) Create(path string) (*os.File, error) {
p := filepath.Join(s.Root, path)
if err := os.MkdirAll(filepath.Dir(p), 0700); err != nil {
path = s.Abs(path)
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
return nil, err
}
return os.Create(p)
return os.Create(path)
}
// Exists returns true if the specified file exists.
func (s *FileStorage) Exists(path string) bool {
_, err := os.Stat(s.Abs(path))
return err == nil
}
// Open a file.
func (s *FileStorage) Open(path string) (*os.File, error) {
if strings.HasPrefix(path, "/") {
return os.Open(path)
}
return os.Open(filepath.Join(s.Root, path))
return os.Open(s.Abs(path))
}
// Rename oldpath to newpath.
func (s *FileStorage) Rename(oldpath, newpath string) error {
if !strings.HasPrefix(oldpath, "/") {
oldpath = filepath.Join(s.Root, oldpath)
return os.Rename(s.Abs(oldpath), s.Abs(newpath))
}
func (s *FileStorage) Walk(w *util.Walker, fn filepath.WalkFunc) {
w.Walk(s.Root, func(path string, info os.FileInfo, ferr error) error {
if ferr != nil {
return nil
}
relpath, err := s.Rel(path)
if err != nil {
return fmt.Errorf("%s is outside %s (?)", path, s.Root)
}
return fn(relpath, info, nil)
})
}
// RWFileStorage adds a read-write API on top of a FileStorage, based
// on unique keys and directory sharding.
type RWFileStorage struct {
*FileStorage
Nesting int
}
func NewRWFileStorage(root string, nesting int) *RWFileStorage {
return &RWFileStorage{
FileStorage: NewFileStorage(root),
Nesting: nesting,
}
if !strings.HasPrefix(newpath, "/") {
newpath = filepath.Join(s.Root, newpath)
}
// Path of the file corresponding to the given key, relative to the
// root directory.
func (s *RWFileStorage) Path(key string) string {
var parts []string
for i := 0; i < s.Nesting; i++ {
if i >= len(key) {
break
}
parts = append(parts, key[i:i+1])
}
return os.Rename(oldpath, newpath)
parts = append(parts, key)
return filepath.Join(parts...)
}
......@@ -247,7 +247,7 @@ func (db *Database) Sync(remote SyncClient) error {
type syncServer struct {
db *Database
storage *FileStorage
storage *RWFileStorage
}
func (l *syncServer) handleDiffRequest(w http.ResponseWriter, req *http.Request) {
......@@ -355,7 +355,7 @@ func (l *syncServer) handleSyncUpload(w http.ResponseWriter, req *http.Request)
w.WriteHeader(200)
}
func savePart(req *http.Request, fieldname string, storage *FileStorage, outname string) (int64, *multipart.FileHeader, error) {
func savePart(req *http.Request, fieldname string, storage *RWFileStorage, outname string) (int64, *multipart.FileHeader, error) {
f, hdr, err := req.FormFile(fieldname)
if err != nil {
return 0, nil, err
......
......@@ -11,7 +11,7 @@ import (
)
func newTestSyncHttpServer(db *Database, updir string) *httptest.Server {
localsrv := &syncServer{db, &FileStorage{Root: updir, Nesting: 2}}
localsrv := &syncServer{db, NewRWFileStorage(updir, 2)}
mux := http.NewServeMux()
mux.HandleFunc("/api/sync/upload", localsrv.handleSyncUpload)
......
......@@ -19,12 +19,13 @@ type MetadataChooserFunc func(string, []*Metadata) *Metadata
type fileData struct {
source int
path string
relpath string
filetype string
id BookId
info os.FileInfo
}
func (f fileData) toLiberFile(haserr bool) *File {
func (f fileData) toLiberFile(storage *FileStorage, haserr bool) (*File, error) {
return &File{
Path: f.path,
FileType: f.filetype,
......@@ -32,7 +33,7 @@ func (f fileData) toLiberFile(haserr bool) *File {
Size: f.info.Size(),
Id: f.id,
Error: haserr,
}
}, nil
}
type fileAndBook struct {
......@@ -40,8 +41,14 @@ type fileAndBook struct {
b *Book
}
func dbFileScanner(db *Database, fileCh chan fileData) {
for iter := db.Scan(FileBucket); iter.Valid(); iter.Next() {
type updateContext struct {
db *Database
storage *FileStorage
chooser MetadataChooserFunc
}
func (uc *updateContext) dbFileScanner(fileCh chan fileData) {
for iter := uc.db.Scan(FileBucket); iter.Valid(); iter.Next() {
var f File
if err := iter.Value(&f); err != nil {
continue
......@@ -54,12 +61,8 @@ func dbFileScanner(db *Database, fileCh chan fileData) {
}
}
func localFileScanner(db *Database, basedir string, fileCh chan fileData) {
util.NewDefaultWalker().Walk(basedir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return nil
}
func (uc *updateContext) localFileScanner(basedir string, fileCh chan fileData) {
uc.storage.Walk(util.NewDefaultWalker(), func(path string, info os.FileInfo, err error) error {
fileCh <- fileData{
source: SourceFS,
path: path,
......@@ -69,7 +72,7 @@ func localFileScanner(db *Database, basedir string, fileCh chan fileData) {
})
}
func differ(db *Database, basedir string) chan fileData {
func (uc *updateContext) differ(basedir string) chan fileData {
fileCh := make(chan fileData, 100)
outCh := make(chan fileData, 100)
var wg sync.WaitGroup
......@@ -77,11 +80,11 @@ func differ(db *Database, basedir string) chan fileData {
// Start two sources in parallel and send their output to fileCh.
go func() {
localFileScanner(db, basedir, fileCh)
uc.localFileScanner(basedir, fileCh)
wg.Done()
}()
go func() {
dbFileScanner(db, fileCh)
uc.dbFileScanner(fileCh)
wg.Done()
}()
// Once they are done, close the channel.
......@@ -110,8 +113,8 @@ func differ(db *Database, basedir string) chan fileData {
}
for path, value := range tmp {
if value == SourceDB {
log.Printf("removing file %s", path)
db.DeleteFile(path)
log.Printf("file %s has been removed", path)
uc.db.DeleteFile(path)
}
}
close(outCh)
......@@ -119,15 +122,15 @@ func differ(db *Database, basedir string) chan fileData {
return outCh
}
func extractor(db *Database, chooser MetadataChooserFunc, fileCh chan fileData, outCh chan fileAndBook) {
func (uc *updateContext) extractor(fileCh chan fileData, outCh chan fileAndBook) {
for f := range fileCh {
if oldfile, err := db.GetFile(f.path); err == nil {
if oldfile, err := uc.db.GetFile(f.path); err == nil {
if !oldfile.HasChanged(f.info) {
continue
}
f.id = oldfile.Id
}
book, filetype, err := parseMeta(f, chooser)
book, filetype, err := uc.parseMeta(f)
if err == nil {
f.filetype = filetype
outCh <- fileAndBook{f: f, b: book}
......@@ -136,22 +139,26 @@ func extractor(db *Database, chooser MetadataChooserFunc, fileCh chan fileData,
// Parse errors are permanent.
log.Printf("Could not parse %s: %v", f.path, err)
file := f.toLiberFile(true)
if err := db.PutFile(file); err != nil {
file, err := f.toLiberFile(uc.storage, true)
if err != nil {
log.Printf("Error saving file %s: %v", file.Path, err)
continue
}
if err := uc.db.PutFile(file); err != nil {
log.Printf("Error saving file %s to db: %v", file.Path, err)
}
}
}
func parseMeta(f fileData, chooser MetadataChooserFunc) (*Book, string, error) {
func (uc *updateContext) parseMeta(f fileData) (*Book, string, error) {
// Attempt direct metadata extraction.
book, filetype, err := Parse(f.path)
book, filetype, err := Parse(uc.storage.Abs(f.path))
if err != nil {
return nil, "", err
}
// Check if a Calibre OPF file exists.
if opfmeta, err := opfOpen(opfMetadataPath(f.path)); err == nil {
if opfmeta, err := opfOpen(opfMetadataPath(uc.storage.Abs(f.path))); err == nil {
book.Metadata.Merge(opfmeta)
} else {
// No local metadata, use Google Books to retrieve
......@@ -162,8 +169,8 @@ func parseMeta(f fileData, chooser MetadataChooserFunc) (*Book, string, error) {
if len(candidates) == 1 {
log.Printf("found Google Books match: %s", candidates[0].String())
book.Metadata.Merge(candidates[0])
} else {
if userchoice := chooser(f.path, candidates); userchoice != nil {
} else if uc.chooser != nil {
if userchoice := uc.chooser(f.path, candidates); userchoice != nil {
book.Metadata.Merge(userchoice)
}
}
......@@ -179,11 +186,11 @@ func parseMeta(f fileData, chooser MetadataChooserFunc) (*Book, string, error) {
// Try to find a cover image. Look on the local filesystem
// first, otherwise check Google Books.
localCoverPath := opfCoverPath(f.path)
if _, err := os.Stat(localCoverPath); err == nil {
if uc.storage.Exists(localCoverPath) {
book.CoverPath = localCoverPath
} else if imageData, err := GetGoogleBooksCover(book.Metadata); err == nil {
imageFileName := f.path + ".cover.png"
if imgf, err := os.Create(imageFileName); err != nil {
if imgf, err := os.Create(uc.storage.Abs(imageFileName)); err != nil {
log.Printf("Could not save cover image for %d: %v", book.Id, err)
} else {
imgf.Write(imageData)
......@@ -195,7 +202,7 @@ func parseMeta(f fileData, chooser MetadataChooserFunc) (*Book, string, error) {
return book, filetype, nil
}
func dbwriter(db *Database, ch chan fileAndBook) {
func (uc *updateContext) dbwriter(ch chan fileAndBook) {
for pair := range ch {
saveBook := true
......@@ -203,7 +210,7 @@ func dbwriter(db *Database, ch chan fileAndBook) {
// existing book.
if pair.f.id == 0 {
log.Printf("potential new book: %#v", pair.b.Metadata)
if match, err := db.Find(pair.b.Metadata.Uniques()); err == nil {
if match, err := uc.db.Find(pair.b.Metadata.Uniques()); err == nil {
log.Printf("%s matches existing book %d", pair.f.path, match.Id)
// Ignore new metadata.
pair.b = match
......@@ -219,15 +226,19 @@ func dbwriter(db *Database, ch chan fileAndBook) {
}
if saveBook {
if err := db.PutBook(pair.b); err != nil {
if err := uc.db.PutBook(pair.b); err != nil {
log.Printf("Error saving book %d to db: %v", pair.b.Id, err)
continue
}
log.Printf("%s -> %d", pair.f.path, pair.b.Id)
}
file := pair.f.toLiberFile(false)
if err := db.PutFile(file); err != nil {
file, err := pair.f.toLiberFile(uc.storage, false)
if err != nil {
log.Printf("Error saving file %s: %v", pair.f.path, err)
continue
}
if err := uc.db.PutFile(file); err != nil {
log.Printf("Error saving file %s to db: %v", file.Path, err)
}
}
......@@ -236,13 +247,19 @@ func dbwriter(db *Database, ch chan fileAndBook) {
func (db *Database) Update(dir string, chooser MetadataChooserFunc) {
// Parallelize metadata extraction, serialize database updates
// (so that index-based de-duplication works).
uc := &updateContext{
db: db,
chooser: chooser,
storage: NewFileStorage(dir),
}
var wg sync.WaitGroup
ch := differ(db, dir)
ch := uc.differ(dir)
pch := make(chan fileAndBook)
for i := 0; i < 10; i++ {
wg.Add(1)
go func() {
extractor(db, chooser, ch, pch)
uc.extractor(ch, pch)
wg.Done()
}()
}
......@@ -250,5 +267,5 @@ func (db *Database) Update(dir string, chooser MetadataChooserFunc) {
wg.Wait()
close(pch)
}()
dbwriter(db, pch)
uc.dbwriter(pch)
}
......@@ -4,6 +4,7 @@ import (
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"git.autistici.org/ale/liber/util"
......@@ -45,14 +46,43 @@ func TestDatabase_Update(t *testing.T) {
if _, err := db.GetBook(td.refbookid); err == nil {
t.Errorf("%s: test book still in database", tag)
}
// Test OPF ebook should have been found by Update.
if result, err := db.Search("isbn:9781939293015", 0, 1); err != nil || result.NumResults != 1 {
t.Errorf("%s: new book not found in database", tag)
}
}
// The second update should do nothing.
db.Update(tmpdir, chooser)
testDb("first update")
db.Update(tmpdir, chooser)
testDb("second update")
// Check that the test file is there.
if _, err := db.GetFile("book/Test Ebook.pdf"); err != nil {
t.Errorf("test file is not in the database")
}
// Files should have relative paths.
for i := db.Scan(FileBucket); i.Valid(); i.Next() {
var f File
if err := i.Value(&f); err != nil {
t.Fatal(err)
}
if strings.HasPrefix(f.Path, "/") {
t.Errorf("file has absolute path: %v", f.Path)
}
}
// Book cover images should have relative paths.
for i := db.Scan(BookBucket); i.Valid(); i.Next() {
var b Book
if err := i.Value(&b); err != nil {
t.Fatal(err)
}
if b.CoverPath != "" && strings.HasPrefix(b.CoverPath, "/") {
t.Errorf("file has absolute path: %v", b.CoverPath)
}
}
}
......@@ -27,8 +27,8 @@ var (
type uiServer struct {
db *Database
storage *FileStorage
cache *FileStorage
storage *RWFileStorage
cache *RWFileStorage
}
type pagination struct {
......@@ -336,7 +336,7 @@ func handleOpenSearchXml(w http.ResponseWriter, req *http.Request) {
render("opensearch_xml.html", w, &ctx)
}
func NewHttpServer(db *Database, storage, cache *FileStorage, addr string) *http.Server {
func NewHttpServer(db *Database, storage, cache *RWFileStorage, addr string) *http.Server {
var err error
tpl, err = template.New("liber").Funcs(template.FuncMap{
"join": strings.Join,
......
......@@ -29,7 +29,7 @@ func newTestHttpServer(t *testing.T) (*testHttpServer, *httptest.Server) {
ts.tmpdir, _ = ioutil.TempDir("", "tmp-storage-")
ts.td, ts.db = newTestDatabase(t)
tempStorage := NewFileStorage(ts.tmpdir, 2)
tempStorage := NewRWFileStorage(ts.tmpdir, 2)
server := NewHttpServer(ts.db, tempStorage, tempStorage, ":1234")
return &ts, httptest.NewServer(server.Handler)
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment