From f570971f6ea503fce036b71f03fd466d1afaf268 Mon Sep 17 00:00:00 2001
From: ale <ale@incal.net>
Date: Mon, 10 Nov 2014 11:02:56 +0000
Subject: [PATCH] support multiple files per ebook
---
cmd/liber/liber.go | 29 +++++---
database.go | 69 +++++++++++++++---
database_test.go | 78 ++++++++++++++++++---
files.go | 11 +++
htdocs/templates/book.html | 6 +-
metadata.go | 13 ++--
sync.go | 99 ++++++++++++++++----------
sync_test.go | 16 ++---
update.go | 140 +++++++++++++++++++++++--------------
web.go | 31 ++++++--
web_test.go | 94 +++++++++++++++++++++++++
11 files changed, 443 insertions(+), 143 deletions(-)
create mode 100644 web_test.go
diff --git a/cmd/liber/liber.go b/cmd/liber/liber.go
index c25ef76..e578ae8 100644
--- a/cmd/liber/liber.go
+++ b/cmd/liber/liber.go
@@ -19,12 +19,13 @@ import (
)
var (
- databaseDir = flag.String("db-dir", "~/.liber", "database directory")
- bookDir = flag.String("book-dir", "", "books directory")
- update = flag.Bool("update", false, "update the db")
- search = flag.Bool("search", false, "search something")
- remotesync = flag.String("sync", "", "push data to remote server")
- httpserver = flag.String("http-server", "", "start the HTTP server on the specified address")
+ databaseDir = flag.String("db-dir", "~/.liber", "database directory")
+ bookDir = flag.String("book-dir", "", "books directory")
+ update = flag.Bool("update", false, "update the db")
+ search = flag.Bool("search", false, "search something")
+ remotesync = flag.String("sync", "", "push data to remote server")
+ httpserver = flag.String("http-server", "", "start the HTTP server on the specified address")
+ noninteractive = flag.Bool("noninteractive", false, "disable interactive metadata prompts on update")
)
// Various ways to ask a user to choose something.
@@ -127,6 +128,10 @@ func promptUserDialog(path string, choices []*liber.Metadata) (*liber.Metadata,
var promptMutex sync.Mutex
func promptUser(path string, choices []*liber.Metadata) *liber.Metadata {
+ if *noninteractive {
+ return nil
+ }
+
promptMutex.Lock()
defer promptMutex.Unlock()
@@ -160,9 +165,14 @@ func doSearch(db *liber.Database, query string) {
fmt.Printf("No results.\n")
} else {
fmt.Printf("%d results found:\n\n", results.NumResults)
- for i, r := range results.Results {
- fmt.Printf("%d) %s\n", i+1, r.Metadata.String())
- fmt.Printf(" %s\n", r.Path)
+ for i, book := range results.Results {
+ fmt.Printf("%d) %s\n", i+1, book.Metadata.String())
+ if files, err := db.GetBookFiles(book.Id); err == nil {
+ for _, f := range files {
+ fmt.Printf(" %s: %s\n", strings.TrimPrefix(f.FileType, "."), f.Path)
+ }
+ }
+ fmt.Printf("\n")
}
}
}
@@ -213,6 +223,7 @@ func main() {
if err == nil {
defer logf.Close()
log.SetOutput(logf)
+ log.SetFlags(log.Ldate | log.Ltime)
}
doUpdate(db, expandTilde(*bookDir))
diff --git a/database.go b/database.go
index fb88395..070200b 100644
--- a/database.go
+++ b/database.go
@@ -17,8 +17,9 @@ import (
)
var (
- BookBucket = []byte("ebook")
- FileBucket = []byte("file")
+ BookBucket = []byte("ebook")
+ FileBucket = []byte("file")
+ BookFileBucket = []byte("ebook_file")
keySeparator = byte('/')
)
@@ -84,10 +85,9 @@ func defaultIndexMapping() *bleve.IndexMapping {
}
type Book struct {
- Id BookId
- Path string
+ Id BookId
+ // Path string
CoverPath string
- FileType string
Metadata *Metadata
}
@@ -96,11 +96,12 @@ func (b *Book) Type() string {
}
type File struct {
- Path string
- Mtime time.Time
- Size int64
- Error bool
- Id BookId
+ Path string
+ FileType string
+ Mtime time.Time
+ Size int64
+ Error bool
+ Id BookId
}
func (f *File) HasChanged(info os.FileInfo) bool {
@@ -135,6 +136,7 @@ func NewDb(path string) (*Database, error) {
}
func (db *Database) setupLevelDb(path string) error {
+ // Use 256MB of cache and a small Bloom filter.
opts := levigo.NewOptions()
db.leveldbCache = levigo.NewLRUCache(2 << 28)
opts.SetCache(db.leveldbCache)
@@ -185,6 +187,24 @@ func (db *Database) GetFile(path string) (*File, error) {
return &f, nil
}
+func (db *Database) GetBookFiles(bookid BookId) ([]*File, error) {
+ ro := levigo.NewReadOptions()
+ defer ro.Close()
+ it := db.leveldb.NewIterator(ro)
+ defer it.Close()
+ start, end := keyRange(bktToKey(BookFileBucket, bookid.Key()))
+ var out []*File
+ for it.Seek(start); it.Valid() && bytes.Compare(it.Key(), end) < 0; it.Next() {
+ var filepath string
+ if json.Unmarshal(it.Value(), &filepath) == nil {
+ if file, err := db.GetFile(filepath); err == nil {
+ out = append(out, file)
+ }
+ }
+ }
+ return out, nil
+}
+
func (db *Database) Get(bucket, key []byte, obj interface{}) error {
ro := levigo.NewReadOptions()
defer ro.Close()
@@ -202,8 +222,18 @@ func (db *Database) PutBook(b *Book) error {
return db.index.Index(b.Id.String(), b.Metadata)
}
+func fileBookKey(path string, bookid BookId) []byte {
+ return bytes.Join([][]byte{bookid.Key(), []byte(path)}, []byte{keySeparator})
+}
+
func (db *Database) PutFile(f *File) error {
- return db.Put(FileBucket, []byte(f.Path), f)
+ if err := db.Put(FileBucket, []byte(f.Path), f); err != nil {
+ return err
+ }
+ if !f.Error {
+ return db.Put(BookFileBucket, fileBookKey(f.Path, f.Id), f.Path)
+ }
+ return nil
}
func (db *Database) Put(bucket, key []byte, obj interface{}) error {
@@ -223,6 +253,23 @@ func (db *Database) DeleteBook(bookid BookId) error {
return db.index.Delete(bookid.String())
}
+func (db *Database) DeleteFile(path string) error {
+ f, err := db.GetFile(path)
+ if err != nil {
+ return nil
+ }
+
+ db.Delete(FileBucket, []byte(path))
+ db.Delete(BookFileBucket, fileBookKey(path, f.Id))
+
+ // Delete the book if there are no files left.
+ if files, err := db.GetBookFiles(f.Id); err == nil && len(files) == 0 {
+ db.DeleteBook(f.Id)
+ }
+
+ return nil
+}
+
func (db *Database) Delete(bucket, key []byte) error {
wo := levigo.NewWriteOptions()
defer wo.Close()
diff --git a/database_test.go b/database_test.go
index 6d6a2c4..660ba83 100644
--- a/database_test.go
+++ b/database_test.go
@@ -1,16 +1,16 @@
package liber
import (
+ "io"
"io/ioutil"
"os"
"testing"
)
-var testdbdir = "/tmp/liber-test-database"
-
type testDatabase struct {
- db *Database
- path string
+ db *Database
+ path string
+ refbookid BookId
}
func (td *testDatabase) Close() {
@@ -29,15 +29,28 @@ func newTestDatabase(t *testing.T) (*testDatabase, *Database) {
if err = db.PutBook(book); err != nil {
t.Fatalf("PutBook(): %v", err)
}
+ if err = db.PutFile(testEpubFile(path, book.Id)); err != nil {
+ t.Fatalf("PutFile(): %v", err)
+ }
- return &testDatabase{db: db, path: path}, db
+ return &testDatabase{db: db, path: path, refbookid: book.Id}, db
+}
+
+func testEpubFile(dir string, bookid BookId) *File {
+ f, _ := ioutil.TempFile(dir, "ebook-")
+ io.WriteString(f, "epub\n")
+ f.Close()
+ return &File{
+ Id: bookid,
+ Path: f.Name(),
+ FileType: ".epub",
+ Size: 4,
+ }
}
func testEbook() *Book {
return &Book{
- Id: NewID(),
- Path: "/path/to/ebook",
- FileType: ".epub",
+ Id: NewID(),
Metadata: &Metadata{
Title: "20,000 Leagues under the sea",
Creator: []string{"Jules Verne"},
@@ -46,6 +59,55 @@ func testEbook() *Book {
}
}
+func TestDatabase_Get(t *testing.T) {
+ td, db := newTestDatabase(t)
+ defer td.Close()
+
+ _, err := db.GetBook(td.refbookid)
+ if err != nil {
+ t.Fatalf("GetBook(%d): %v", td.refbookid, err)
+ }
+ files, err := db.GetBookFiles(td.refbookid)
+ if err != nil {
+ t.Fatalf("GetBookFiles(%d): %v", td.refbookid, err)
+ }
+ if len(files) != 1 {
+ t.Fatalf("GetBookFiles(%d) bad result: %v", td.refbookid, files)
+ }
+}
+
+func TestDatabase_BookFileRelation(t *testing.T) {
+ td, db := newTestDatabase(t)
+ defer td.Close()
+
+ checkFiles := func(tag string, n int) []*File {
+ files, err := db.GetBookFiles(td.refbookid)
+ if err != nil {
+ t.Fatalf("GetBookFiles@%s(%d): %v", tag, td.refbookid, err)
+ }
+ if len(files) != n {
+ t.Fatalf("GetBookFiles@%s(%d) bad result (exp. len=%d): %v", tag, td.refbookid, n, files)
+ }
+ return files
+ }
+
+ files := checkFiles("init", 1)
+ file0 := files[0]
+ file1 := testEpubFile(td.path, td.refbookid)
+ db.PutFile(file1)
+ checkFiles("post_add", 2)
+
+ db.DeleteFile(file1.Path)
+ checkFiles("post_delete", 1)
+
+ db.DeleteFile(file0.Path)
+ checkFiles("post_delete_2", 0)
+
+ if _, err := db.GetBook(td.refbookid); err == nil {
+ t.Fatal("Book was not removed when n.files==0")
+ }
+}
+
func TestDatabase_Search(t *testing.T) {
td, db := newTestDatabase(t)
defer td.Close()
diff --git a/files.go b/files.go
index 2f399f2..4cfae94 100644
--- a/files.go
+++ b/files.go
@@ -48,3 +48,14 @@ func (s *FileStorage) Open(path string) (*os.File, error) {
}
return os.Open(filepath.Join(s.Root, path))
}
+
+// Rename oldpath to newpath.
+func (s *FileStorage) Rename(oldpath, newpath string) error {
+ if !strings.HasPrefix(oldpath, "/") {
+ oldpath = filepath.Join(s.Root, oldpath)
+ }
+ if !strings.HasPrefix(newpath, "/") {
+ newpath = filepath.Join(s.Root, newpath)
+ }
+ return os.Rename(oldpath, newpath)
+}
diff --git a/htdocs/templates/book.html b/htdocs/templates/book.html
index 0d8ff7c..505b104 100644
--- a/htdocs/templates/book.html
+++ b/htdocs/templates/book.html
@@ -32,12 +32,14 @@
<p>{{.Book.Metadata.Description}}</p>
{{end}}
+ {{range $i, $f := .Files}}
<p>
- <a class="btn btn-large btn-primary" href="/dl/{{.Book.Id}}">
+ <a class="btn btn-large btn-primary" href="/dl/{{$f.Id}}/{{$i}}">
<span class="glyphicon glyphicon-download-alt"></span>
- Download
+ Download ({{$f.FileType}})
</a>
</p>
+ {{end}}
</div>
diff --git a/metadata.go b/metadata.go
index d762b6e..e6456d9 100644
--- a/metadata.go
+++ b/metadata.go
@@ -244,7 +244,7 @@ func parseAnything(filename string) (*Metadata, error) {
}, nil
}
-func Parse(filename string) (*Book, error) {
+func Parse(filename string) (*Book, string, error) {
var m *Metadata
var err error
ext := strings.ToLower(filepath.Ext(filename))
@@ -256,16 +256,11 @@ func Parse(filename string) (*Book, error) {
case ".pdf":
m, err = parseAnything(filename)
default:
- return nil, errors.New("unsupported file format")
+ return nil, "", errors.New("unsupported file format")
}
if err != nil {
- return nil, err
+ return nil, "", err
}
- b := &Book{
- Path: filename,
- FileType: ext,
- Metadata: m,
- }
- return b, nil
+ return &Book{Metadata: m}, ext, nil
}
diff --git a/sync.go b/sync.go
index e59eae6..7b4ce84 100644
--- a/sync.go
+++ b/sync.go
@@ -11,6 +11,7 @@ import (
"os"
"path/filepath"
"sync"
+ "time"
)
const (
@@ -20,7 +21,7 @@ const (
type SyncClient interface {
DiffRequest(*diffRequest) (*diffResponse, error)
- SendBook(*Book) error
+ SendBook(*Book, []*File) error
}
type remoteServer struct {
@@ -64,14 +65,14 @@ func (r *remoteServer) DiffRequest(diffreq *diffRequest) (*diffResponse, error)
return &diffresp, nil
}
-func addFilePart(w *multipart.Writer, varname, filename string) error {
+func addFilePart(w *multipart.Writer, varname, filename, mimeFilename string) error {
file, err := os.Open(filename)
if err != nil {
return err
}
defer file.Close()
- part, err := w.CreateFormFile(varname, filepath.Base(filename))
+ part, err := w.CreateFormFile(varname, mimeFilename)
if err != nil {
return err
}
@@ -83,25 +84,29 @@ func addFilePart(w *multipart.Writer, varname, filename string) error {
}
// SendBook uploads a book to the remote server.
-func (r *remoteServer) SendBook(book *Book) error {
+func (r *remoteServer) SendBook(book *Book, files []*File) error {
// Create a multipart request with the JSON-encoded metadata
// and the actual file contents as two separate mime/multipart
// sections.
var body bytes.Buffer
w := multipart.NewWriter(&body)
- w.WriteField("type", book.FileType)
+
part, err := w.CreateFormFile("meta", "meta.json")
if err := json.NewEncoder(part).Encode(book.Metadata); err != nil {
return err
}
- if err := addFilePart(w, "book", book.Path); err != nil {
- w.Close()
- return err
+ for i, f := range files {
+ varname := fmt.Sprintf("book%d", i)
+ filename := fmt.Sprintf("%d%s", book.Id, f.FileType)
+ if err := addFilePart(w, varname, f.Path, filename); err != nil {
+ w.Close()
+ return err
+ }
}
if book.CoverPath != "" {
- if err := addFilePart(w, "cover", book.CoverPath); err != nil {
+ if err := addFilePart(w, "cover", book.CoverPath, "cover.jpg"); err != nil {
w.Close()
return err
}
@@ -195,9 +200,12 @@ func (db *Database) Sync(remote SyncClient) error {
wg.Add(1)
go func() {
for id := range ch {
- if book, err := db.GetBook(ParseID(id)); err == nil {
- if err := remote.SendBook(book); err != nil {
- log.Printf("SendBook(%d): %v", id, err)
+ bookid := ParseID(id)
+ if book, err := db.GetBook(bookid); err == nil {
+ if files, err := db.GetBookFiles(bookid); err == nil {
+ if err := remote.SendBook(book, files); err != nil {
+ log.Printf("SendBook(%d): %v", id, err)
+ }
}
}
}
@@ -251,13 +259,6 @@ func (l *syncServer) handleDiffRequest(w http.ResponseWriter, req *http.Request)
}
func (l *syncServer) handleSyncUpload(w http.ResponseWriter, req *http.Request) {
- filetype := req.FormValue("type")
- if filetype == "" {
- log.Printf("request with no 'type' field")
- http.Error(w, "Bad request", http.StatusBadRequest)
- return
- }
-
mf, _, err := req.FormFile("meta")
if err != nil {
log.Printf("request with no 'meta' field")
@@ -284,21 +285,46 @@ func (l *syncServer) handleSyncUpload(w http.ResponseWriter, req *http.Request)
book := &Book{
Id: bookid,
Metadata: &md,
- FileType: filetype,
- Path: l.storage.Path(fmt.Sprintf("%d%s", bookid, filetype)),
}
- // Store the data into a custom path and save the book into
- // the local database.
- if err := savePart(req, "book", l.storage, book.Path); err != nil {
- log.Printf("error saving local file: %v", err)
- http.Error(w, err.Error(), http.StatusInternalServerError)
- return
+ // Save the file data to our local storage.
+ for i := 0; i < 10; i++ {
+ // Use a temporary file, we'll know the right
+ // extension to use only after having parsed the
+ // file's MIME header.
+ tmppath := l.storage.Path(fmt.Sprintf("%d.%d.tmp", bookid, i))
+ varname := fmt.Sprintf("book%d", i)
+ size, hdr, err := savePart(req, varname, l.storage, tmppath)
+ if err == http.ErrMissingFile {
+ break
+ } else if err != nil {
+ log.Printf("error saving local file: %v", err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ filetype := filepath.Ext(hdr.Filename)
+ path := l.storage.Path(fmt.Sprintf("%d.%d%s", bookid, i, filetype))
+ if err := l.storage.Rename(tmppath, path); err != nil {
+ log.Printf("error moving local file: %v", err)
+ }
+
+ file := &File{
+ Path: path,
+ FileType: filetype,
+ Mtime: time.Now(),
+ Size: size,
+ Id: bookid,
+ }
+ if err := l.db.PutFile(file); err != nil {
+ log.Printf("error saving file to the database: %v", err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
}
// If the request contains a cover image, save that as well.
- coverPath := l.storage.Path(fmt.Sprintf("%s%s.cover.png", book.Id, filetype))
- if err := savePart(req, "cover", l.storage, coverPath); err == nil {
+ coverPath := l.storage.Path(fmt.Sprintf("%s.cover.png", book.Id))
+ if _, _, err := savePart(req, "cover", l.storage, coverPath); err == nil {
book.CoverPath = coverPath
}
@@ -312,21 +338,22 @@ func (l *syncServer) handleSyncUpload(w http.ResponseWriter, req *http.Request)
w.WriteHeader(200)
}
-func savePart(req *http.Request, fieldname string, storage *FileStorage, outname string) error {
- f, _, err := req.FormFile(fieldname)
+func savePart(req *http.Request, fieldname string, storage *FileStorage, outname string) (int64, *multipart.FileHeader, error) {
+ f, hdr, err := req.FormFile(fieldname)
if err != nil {
- return err
+ return 0, nil, err
}
outf, err := storage.Create(outname)
if err != nil {
- return err
+ return 0, nil, err
}
defer outf.Close()
- if _, err := io.Copy(outf, f); err != nil {
- return err
+ n, err := io.Copy(outf, f)
+ if err != nil {
+ return 0, nil, err
}
- return nil
+ return n, hdr, nil
}
diff --git a/sync_test.go b/sync_test.go
index f0efe27..82e66ba 100644
--- a/sync_test.go
+++ b/sync_test.go
@@ -2,7 +2,6 @@ package liber
import (
"fmt"
- "io"
"io/ioutil"
"net/http"
"net/http/httptest"
@@ -11,7 +10,7 @@ import (
"testing"
)
-func newTestHttpServer(db *Database, updir string) *httptest.Server {
+func newTestSyncHttpServer(db *Database, updir string) *httptest.Server {
localsrv := &syncServer{db, &FileStorage{Root: updir, Nesting: 2}}
mux := http.NewServeMux()
@@ -22,11 +21,6 @@ func newTestHttpServer(db *Database, updir string) *httptest.Server {
}
func TestSync_Sync(t *testing.T) {
- // Actually create a file to upload, or the sync will fail.
- f, _ := ioutil.TempFile("", "ebook-")
- io.WriteString(f, "foo\n")
- f.Close()
-
// Create a temporary directory to store uploads.
updir, _ := ioutil.TempDir("", "ebook-upload-")
defer os.RemoveAll(updir)
@@ -38,20 +32,20 @@ func TestSync_Sync(t *testing.T) {
defer td2.Close()
for i := 0; i < 10; i++ {
+ bookid := NewID()
db.PutBook(&Book{
- Id: NewID(),
- Path: f.Name(),
- FileType: ".epub",
+ Id: bookid,
Metadata: &Metadata{
Title: fmt.Sprintf("Book #%d", i+1),
Creator: []string{"Random Author"},
ISBN: []string{strconv.Itoa(i + 1)},
},
})
+ db.PutFile(testEpubFile(updir, bookid))
}
// Run a sync from db to db2.
- srv := newTestHttpServer(db2, updir)
+ srv := newTestSyncHttpServer(db2, updir)
defer srv.Close()
cl := NewRemoteServer(srv.URL)
diff --git a/update.go b/update.go
index b97c2bd..8068809 100644
--- a/update.go
+++ b/update.go
@@ -17,22 +17,29 @@ const (
type MetadataChooserFunc func(string, []*Metadata) *Metadata
type fileData struct {
- source int
- path string
- id BookId
- info os.FileInfo
+ source int
+ path string
+ filetype string
+ id BookId
+ info os.FileInfo
}
func (f fileData) toLiberFile(haserr bool) *File {
return &File{
- Path: f.path,
- Mtime: f.info.ModTime(),
- Size: f.info.Size(),
- Id: f.id,
- Error: haserr,
+ Path: f.path,
+ FileType: f.filetype,
+ Mtime: f.info.ModTime(),
+ Size: f.info.Size(),
+ Id: f.id,
+ Error: haserr,
}
}
+type fileAndBook struct {
+ f fileData
+ b *Book
+}
+
func dbFileScanner(db *Database, fileCh chan fileData) {
for iter := db.Scan(FileBucket); iter.Valid(); iter.Next() {
var f File
@@ -83,34 +90,28 @@ func differ(db *Database, basedir string) chan fileData {
close(fileCh)
}()
go func() {
- // Detect files that have not changed, i.e. appear in
- // the database and the filesystem. Keep track of book
- // IDs so that once all entries have been processed we
- // can delete those books from the database where the
- // original file has been removed.
+ // Merge the two sources and keep track of files that
+ // only appear in the database but not on the
+ // filesystem, so we can remove them at the end.
+ // All entries with source == SourceFS will be sent to
+ // the output channel in any case.
allSources := SourceDB | SourceFS
tmp := make(map[string]int)
- ids := make(map[string]BookId)
for f := range fileCh {
- // log.Printf("differ: %#v", f)
tmp[f.path] |= f.source
// Delete entries as soon as we've seen them
- // from both sources.
+ // originate from both sources.
if tmp[f.path] == allSources {
- // log.Printf("differ: dropping %s", f.path)
delete(tmp, f.path)
- delete(ids, f.path)
}
if f.source == SourceFS {
outCh <- f
- } else {
- ids[f.path] = f.id
}
}
for path, value := range tmp {
if value == SourceDB {
- log.Printf("removing book %s", path)
- db.DeleteBook(ids[path])
+ log.Printf("removing file %s", path)
+ db.DeleteFile(path)
}
}
close(outCh)
@@ -118,41 +119,35 @@ func differ(db *Database, basedir string) chan fileData {
return outCh
}
-func adder(db *Database, chooser MetadataChooserFunc, fileCh chan fileData) {
+func extractor(db *Database, chooser MetadataChooserFunc, fileCh chan fileData, outCh chan fileAndBook) {
for f := range fileCh {
- var oldid BookId
if oldfile, err := db.GetFile(f.path); err == nil {
if !oldfile.HasChanged(f.info) {
continue
}
- oldid = oldfile.Id
+ f.id = oldfile.Id
}
-
- var err error
- f.id, err = importBook(db, f, oldid, chooser)
- if err != nil {
- log.Printf("Could not add %s: %v", f.path, err)
+ book, filetype, err := parseMeta(f, chooser)
+ if err == nil {
+ f.filetype = filetype
+ outCh <- fileAndBook{f: f, b: book}
continue
}
- file := f.toLiberFile(err != nil)
+ // Parse errors are permanent.
+ log.Printf("Could not parse %s: %v", f.path, err)
+ file := f.toLiberFile(true)
if err := db.PutFile(file); err != nil {
- log.Println(err)
+ log.Printf("Error saving file %s to db: %v", file.Path, err)
}
}
}
-func importBook(db *Database, f fileData, oldid BookId, chooser MetadataChooserFunc) (BookId, error) {
+func parseMeta(f fileData, chooser MetadataChooserFunc) (*Book, string, error) {
// Attempt direct metadata extraction.
- book, err := Parse(f.path)
+ book, filetype, err := Parse(f.path)
if err != nil {
- return 0, err
- }
-
- if oldid != 0 {
- book.Id = oldid
- } else {
- book.Id = NewID()
+ return nil, "", err
}
// Check if a Calibre OPF file exists.
@@ -175,9 +170,10 @@ func importBook(db *Database, f fileData, oldid BookId, chooser MetadataChooserF
}
}
- // Check if the book metadata looks ok.
+ // Check if the book metadata looks ok. If not, don't even
+ // bother looking for a cover image.
if !book.Metadata.Sufficient() {
- return 0, errors.New("insufficient metadata")
+ return nil, "", errors.New("insufficient metadata")
}
// Try to find a cover image. Look on the local filesystem
@@ -186,7 +182,7 @@ func importBook(db *Database, f fileData, oldid BookId, chooser MetadataChooserF
if _, err := os.Stat(localCoverPath); err == nil {
book.CoverPath = localCoverPath
} else if imageData, err := GetGoogleBooksCover(book.Metadata); err == nil {
- imageFileName := book.Path + ".cover.png"
+ imageFileName := f.path + ".cover.png"
if imgf, err := os.Create(imageFileName); err != nil {
log.Printf("Could not save cover image for %d: %v", book.Id, err)
} else {
@@ -196,23 +192,63 @@ func importBook(db *Database, f fileData, oldid BookId, chooser MetadataChooserF
}
}
- // Save the book in our database.
- if err := db.PutBook(book); err != nil {
- return book.Id, err
+ return book, filetype, nil
+}
+
+func dbwriter(db *Database, ch chan fileAndBook) {
+ for pair := range ch {
+ saveBook := true
+
+ // If this is a new file, see if it matches an already
+ // existing book.
+ if pair.f.id == 0 {
+ log.Printf("potential new book: %#v", pair.b.Metadata)
+ if match, err := db.Find(pair.b.Metadata); err == nil {
+ log.Printf("%s matches existing book %d", pair.f.path, match.Id)
+ // Ignore new metadata.
+ pair.b = match
+ saveBook = false
+ } else {
+ // Assign a new ID to the book.
+ pair.b.Id = NewID()
+ }
+ pair.f.id = pair.b.Id
+ } else {
+ // Overwrite the old book metadata.
+ pair.b.Id = pair.f.id
+ }
+
+ if saveBook {
+ if err := db.PutBook(pair.b); err != nil {
+ log.Printf("Error saving book %d to db: %v", pair.b.Id, err)
+ continue
+ }
+ log.Printf("%s -> %d", pair.f.path, pair.b.Id)
+ }
+
+ file := pair.f.toLiberFile(false)
+ if err := db.PutFile(file); err != nil {
+ log.Printf("Error saving file %s to db: %v", file.Path, err)
+ }
}
- log.Printf("%s -> %d", f.path, book.Id)
- return book.Id, nil
}
func (db *Database) Update(dir string, chooser MetadataChooserFunc) {
+ // Parallelize metadata extraction, serialize database updates
+ // (so that index-based de-duplication works).
var wg sync.WaitGroup
ch := differ(db, dir)
+ pch := make(chan fileAndBook)
for i := 0; i < 10; i++ {
wg.Add(1)
go func() {
- adder(db, chooser, ch)
+ extractor(db, chooser, ch, pch)
wg.Done()
}()
}
- wg.Wait()
+ go func() {
+ wg.Wait()
+ close(pch)
+ }()
+ dbwriter(db, pch)
}
diff --git a/web.go b/web.go
index 3a173c9..0e6bdd0 100644
--- a/web.go
+++ b/web.go
@@ -142,10 +142,18 @@ func (s *uiServer) withBook(f func(*Book, http.ResponseWriter, *http.Request)) h
}
func (s *uiServer) handleShowBook(book *Book, w http.ResponseWriter, req *http.Request) {
+ files, err := s.db.GetBookFiles(book.Id)
+ if err != nil {
+ log.Printf("ShowBook(%d): %v", book.Id, err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
ctx := struct {
Query string
Book *Book
- }{Book: book}
+ Files []*File
+ }{Book: book, Files: files}
render("book.html", w, &ctx)
}
@@ -156,7 +164,20 @@ var contentTypeMap = map[string]string{
}
func (s *uiServer) handleDownloadBook(book *Book, w http.ResponseWriter, req *http.Request) {
- f, err := s.storage.Open(book.Path)
+ idx, _ := strconv.Atoi(mux.Vars(req)["n"])
+
+ files, err := s.db.GetBookFiles(book.Id)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ if idx < 0 || idx >= len(files) {
+ http.NotFound(w, req)
+ return
+ }
+ file := files[idx]
+
+ f, err := s.storage.Open(file.Path)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
@@ -164,8 +185,8 @@ func (s *uiServer) handleDownloadBook(book *Book, w http.ResponseWriter, req *ht
defer f.Close()
w.Header().Set("ETag", book.Id.String())
- w.Header().Set("Content-Type", contentTypeMap[book.FileType])
- w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%d%s\"", book.Id, book.FileType))
+ w.Header().Set("Content-Type", contentTypeMap[file.FileType])
+ w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%d%s\"", book.Id, file.FileType))
http.ServeContent(w, req, "", time.Time{}, f)
}
@@ -261,7 +282,7 @@ func NewHttpServer(db *Database, storage, cache *FileStorage, addr string) *http
r.Handle("/book/cover/{id:[0-9]+}", uisrv.withBook(uisrv.handleShowCover))
r.Handle("/book/thumb/{id:[0-9]+}", uisrv.withBook(uisrv.handleShowThumbnail))
r.Handle("/book/{id:[0-9]+}", uisrv.withBook(uisrv.handleShowBook))
- r.Handle("/dl/{id:[0-9]+}", uisrv.withBook(uisrv.handleDownloadBook))
+ r.Handle("/dl/{id:[0-9]+}/{n:[0-9]+}", uisrv.withBook(uisrv.handleDownloadBook))
r.HandleFunc("/suggest", uisrv.handleAutocomplete)
r.HandleFunc("/search", uisrv.handleSearch)
r.HandleFunc("/", uisrv.handleHome)
diff --git a/web_test.go b/web_test.go
new file mode 100644
index 0000000..1562f98
--- /dev/null
+++ b/web_test.go
@@ -0,0 +1,94 @@
+package liber
+
+import (
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "strings"
+ "testing"
+)
+
+type testHttpServer struct {
+ tmpdir string
+ td *testDatabase
+ db *Database
+}
+
+func (ts *testHttpServer) Close() {
+ ts.td.Close()
+ os.RemoveAll(ts.tmpdir)
+}
+
+func newTestHttpServer(t *testing.T) (*testHttpServer, *httptest.Server) {
+ var ts testHttpServer
+
+ *htdocsDir = "./htdocs"
+
+ ts.tmpdir, _ = ioutil.TempDir("", "tmp-storage-")
+ ts.td, ts.db = newTestDatabase(t)
+
+ tempStorage := NewFileStorage(ts.tmpdir, 2)
+ server := NewHttpServer(ts.db, tempStorage, tempStorage, ":1234")
+ return &ts, httptest.NewServer(server.Handler)
+}
+
+func readTestResponseData(resp *http.Response, t *testing.T) string {
+ data, err := ioutil.ReadAll(resp.Body)
+ resp.Body.Close()
+ if err != nil {
+ t.Fatalf("Read response: %v", err)
+ }
+ return string(data)
+}
+
+func TestWeb_Home(t *testing.T) {
+ ts, srv := newTestHttpServer(t)
+ defer srv.Close()
+ defer ts.Close()
+
+ resp, err := http.Get(srv.URL)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if resp.StatusCode != 200 {
+ t.Fatalf("Bad HTTP response: %s\n%s", resp.Status, readTestResponseData(resp, t))
+ }
+}
+
+func TestWeb_Search(t *testing.T) {
+ ts, srv := newTestHttpServer(t)
+ defer srv.Close()
+ defer ts.Close()
+
+ resp, err := http.Get(srv.URL + "/search?q=jules")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if resp.StatusCode != 200 {
+ t.Fatalf("Bad HTTP response: %s\n%s", resp.Status, readTestResponseData(resp, t))
+ }
+ data := readTestResponseData(resp, t)
+ if !strings.Contains(data, "20,000 Leagues") {
+ t.Fatalf("Response does not contain book title:\n%s", data)
+ }
+}
+
+func TestWeb_ShowBook(t *testing.T) {
+ ts, srv := newTestHttpServer(t)
+ defer srv.Close()
+ defer ts.Close()
+
+ resp, err := http.Get(fmt.Sprintf("%s/book/%d", srv.URL, ts.td.refbookid))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if resp.StatusCode != 200 {
+ t.Fatalf("Bad HTTP response: %s\n%s", resp.Status, readTestResponseData(resp, t))
+ }
+ data := readTestResponseData(resp, t)
+ if !strings.Contains(data, "20,000 Leagues") {
+ t.Fatalf("Response does not contain book title:\n%s", data)
+ }
+}
--
GitLab