Skip to content

liber
liber
Commit f570971f authored by ale's avatar ale
Browse files
Options

support multiple files per ebook

parent d4b72fa0
Hide whitespace changes
Inline Side-by-side
Showing with 443 additions and 143 deletions
cmd/liber/liber.go
View file @ f570971f
......@@ -19,12 +19,13 @@ import (
)
var (
databaseDir = flag.String("db-dir", "~/.liber", "database directory")
bookDir = flag.String("book-dir", "", "books directory")
update = flag.Bool("update", false, "update the db")
search = flag.Bool("search", false, "search something")
remotesync = flag.String("sync", "", "push data to remote server")
httpserver = flag.String("http-server", "", "start the HTTP server on the specified address")
databaseDir = flag.String("db-dir", "~/.liber", "database directory")
bookDir = flag.String("book-dir", "", "books directory")
update = flag.Bool("update", false, "update the db")
search = flag.Bool("search", false, "search something")
remotesync = flag.String("sync", "", "push data to remote server")
httpserver = flag.String("http-server", "", "start the HTTP server on the specified address")
noninteractive = flag.Bool("noninteractive", false, "disable interactive metadata prompts on update")
)
// Various ways to ask a user to choose something.
......@@ -127,6 +128,10 @@ func promptUserDialog(path string, choices []*liber.Metadata) (*liber.Metadata,
var promptMutex sync.Mutex
func promptUser(path string, choices []*liber.Metadata) *liber.Metadata {
if *noninteractive {
return nil
}
promptMutex.Lock()
defer promptMutex.Unlock()
......@@ -160,9 +165,14 @@ func doSearch(db *liber.Database, query string) {
fmt.Printf("No results.\n")
} else {
fmt.Printf("%d results found:\n\n", results.NumResults)
for i, r := range results.Results {
fmt.Printf("%d) %s\n", i+1, r.Metadata.String())
fmt.Printf(" %s\n", r.Path)
for i, book := range results.Results {
fmt.Printf("%d) %s\n", i+1, book.Metadata.String())
if files, err := db.GetBookFiles(book.Id); err == nil {
for _, f := range files {
fmt.Printf(" %s: %s\n", strings.TrimPrefix(f.FileType, "."), f.Path)
}
}
fmt.Printf("\n")
}
}
}
......@@ -213,6 +223,7 @@ func main() {
if err == nil {
defer logf.Close()
log.SetOutput(logf)
log.SetFlags(log.Ldate | log.Ltime)
}
doUpdate(db, expandTilde(*bookDir))
......
database.go
View file @ f570971f
......@@ -17,8 +17,9 @@ import (
)
var (
BookBucket = []byte("ebook")
FileBucket = []byte("file")
BookBucket = []byte("ebook")
FileBucket = []byte("file")
BookFileBucket = []byte("ebook_file")
keySeparator = byte('/')
)
......@@ -84,10 +85,9 @@ func defaultIndexMapping() *bleve.IndexMapping {
}
type Book struct {
Id BookId
Path string
Id BookId
// Path string
CoverPath string
FileType string
Metadata *Metadata
}
......@@ -96,11 +96,12 @@ func (b *Book) Type() string {
}
type File struct {
Path string
Mtime time.Time
Size int64
Error bool
Id BookId
Path string
FileType string
Mtime time.Time
Size int64
Error bool
Id BookId
}
func (f *File) HasChanged(info os.FileInfo) bool {
......@@ -135,6 +136,7 @@ func NewDb(path string) (*Database, error) {
}
func (db *Database) setupLevelDb(path string) error {
// Use 256MB of cache and a small Bloom filter.
opts := levigo.NewOptions()
db.leveldbCache = levigo.NewLRUCache(2 << 28)
opts.SetCache(db.leveldbCache)
......@@ -185,6 +187,24 @@ func (db *Database) GetFile(path string) (*File, error) {
return &f, nil
}
func (db *Database) GetBookFiles(bookid BookId) ([]*File, error) {
ro := levigo.NewReadOptions()
defer ro.Close()
it := db.leveldb.NewIterator(ro)
defer it.Close()
start, end := keyRange(bktToKey(BookFileBucket, bookid.Key()))
var out []*File
for it.Seek(start); it.Valid() && bytes.Compare(it.Key(), end) < 0; it.Next() {
var filepath string
if json.Unmarshal(it.Value(), &filepath) == nil {
if file, err := db.GetFile(filepath); err == nil {
out = append(out, file)
}
}
}
return out, nil
}
func (db *Database) Get(bucket, key []byte, obj interface{}) error {
ro := levigo.NewReadOptions()
defer ro.Close()
......@@ -202,8 +222,18 @@ func (db *Database) PutBook(b *Book) error {
return db.index.Index(b.Id.String(), b.Metadata)
}
func fileBookKey(path string, bookid BookId) []byte {
return bytes.Join([][]byte{bookid.Key(), []byte(path)}, []byte{keySeparator})
}
func (db *Database) PutFile(f *File) error {
return db.Put(FileBucket, []byte(f.Path), f)
if err := db.Put(FileBucket, []byte(f.Path), f); err != nil {
return err
}
if !f.Error {
return db.Put(BookFileBucket, fileBookKey(f.Path, f.Id), f.Path)
}
return nil
}
func (db *Database) Put(bucket, key []byte, obj interface{}) error {
......@@ -223,6 +253,23 @@ func (db *Database) DeleteBook(bookid BookId) error {
return db.index.Delete(bookid.String())
}
func (db *Database) DeleteFile(path string) error {
f, err := db.GetFile(path)
if err != nil {
return nil
}
db.Delete(FileBucket, []byte(path))
db.Delete(BookFileBucket, fileBookKey(path, f.Id))
// Delete the book if there are no files left.
if files, err := db.GetBookFiles(f.Id); err == nil && len(files) == 0 {
db.DeleteBook(f.Id)
}
return nil
}
func (db *Database) Delete(bucket, key []byte) error {
wo := levigo.NewWriteOptions()
defer wo.Close()
......
database_test.go
View file @ f570971f
package liber
import (
"io"
"io/ioutil"
"os"
"testing"
)
var testdbdir = "/tmp/liber-test-database"
type testDatabase struct {
db *Database
path string
db *Database
path string
refbookid BookId
}
func (td *testDatabase) Close() {
......@@ -29,15 +29,28 @@ func newTestDatabase(t *testing.T) (*testDatabase, *Database) {
if err = db.PutBook(book); err != nil {
t.Fatalf("PutBook(): %v", err)
}
if err = db.PutFile(testEpubFile(path, book.Id)); err != nil {
t.Fatalf("PutFile(): %v", err)
}
return &testDatabase{db: db, path: path}, db
return &testDatabase{db: db, path: path, refbookid: book.Id}, db
}
func testEpubFile(dir string, bookid BookId) *File {
f, _ := ioutil.TempFile(dir, "ebook-")
io.WriteString(f, "epub\n")
f.Close()
return &File{
Id: bookid,
Path: f.Name(),
FileType: ".epub",
Size: 4,
}
}
func testEbook() *Book {
return &Book{
Id: NewID(),
Path: "/path/to/ebook",
FileType: ".epub",
Id: NewID(),
Metadata: &Metadata{
Title: "20,000 Leagues under the sea",
Creator: []string{"Jules Verne"},
......@@ -46,6 +59,55 @@ func testEbook() *Book {
}
}
func TestDatabase_Get(t *testing.T) {
td, db := newTestDatabase(t)
defer td.Close()
_, err := db.GetBook(td.refbookid)
if err != nil {
t.Fatalf("GetBook(%d): %v", td.refbookid, err)
}
files, err := db.GetBookFiles(td.refbookid)
if err != nil {
t.Fatalf("GetBookFiles(%d): %v", td.refbookid, err)
}
if len(files) != 1 {
t.Fatalf("GetBookFiles(%d) bad result: %v", td.refbookid, files)
}
}
func TestDatabase_BookFileRelation(t *testing.T) {
td, db := newTestDatabase(t)
defer td.Close()
checkFiles := func(tag string, n int) []*File {
files, err := db.GetBookFiles(td.refbookid)
if err != nil {
t.Fatalf("GetBookFiles@%s(%d): %v", tag, td.refbookid, err)
}
if len(files) != n {
t.Fatalf("GetBookFiles@%s(%d) bad result (exp. len=%d): %v", tag, td.refbookid, n, files)
}
return files
}
files := checkFiles("init", 1)
file0 := files[0]
file1 := testEpubFile(td.path, td.refbookid)
db.PutFile(file1)
checkFiles("post_add", 2)
db.DeleteFile(file1.Path)
checkFiles("post_delete", 1)
db.DeleteFile(file0.Path)
checkFiles("post_delete_2", 0)
if _, err := db.GetBook(td.refbookid); err == nil {
t.Fatal("Book was not removed when n.files==0")
}
}
func TestDatabase_Search(t *testing.T) {
td, db := newTestDatabase(t)
defer td.Close()
......
files.go
View file @ f570971f
......@@ -48,3 +48,14 @@ func (s *FileStorage) Open(path string) (*os.File, error) {
}
return os.Open(filepath.Join(s.Root, path))
}
// Rename oldpath to newpath.
func (s *FileStorage) Rename(oldpath, newpath string) error {
if !strings.HasPrefix(oldpath, "/") {
oldpath = filepath.Join(s.Root, oldpath)
}
if !strings.HasPrefix(newpath, "/") {
newpath = filepath.Join(s.Root, newpath)
}
return os.Rename(oldpath, newpath)
}
htdocs/templates/book.html
View file @ f570971f
......@@ -32,12 +32,14 @@
<p>{{.Book.Metadata.Description}}</p>
{{end}}
{{range $i, $f := .Files}}
<p>
<a class="btn btn-large btn-primary" href="/dl/{{.Book.Id}}">
<a class="btn btn-large btn-primary" href="/dl/{{$f.Id}}/{{$i}}">
<span class="glyphicon glyphicon-download-alt"></span>
Download
Download ({{$f.FileType}})
</a>
</p>
{{end}}
</div>
......
metadata.go
View file @ f570971f
......@@ -244,7 +244,7 @@ func parseAnything(filename string) (*Metadata, error) {
}, nil
}
func Parse(filename string) (*Book, error) {
func Parse(filename string) (*Book, string, error) {
var m *Metadata
var err error
ext := strings.ToLower(filepath.Ext(filename))
......@@ -256,16 +256,11 @@ func Parse(filename string) (*Book, error) {
case ".pdf":
m, err = parseAnything(filename)
default:
return nil, errors.New("unsupported file format")
return nil, "", errors.New("unsupported file format")
}
if err != nil {
return nil, err
return nil, "", err
}
b := &Book{
Path: filename,
FileType: ext,
Metadata: m,
}
return b, nil
return &Book{Metadata: m}, ext, nil
}
sync.go
View file @ f570971f
......@@ -11,6 +11,7 @@ import (
"os"
"path/filepath"
"sync"
"time"
)
const (
......@@ -20,7 +21,7 @@ const (
type SyncClient interface {
DiffRequest(*diffRequest) (*diffResponse, error)
SendBook(*Book) error
SendBook(*Book, []*File) error
}
type remoteServer struct {
......@@ -64,14 +65,14 @@ func (r *remoteServer) DiffRequest(diffreq *diffRequest) (*diffResponse, error)
return &diffresp, nil
}
func addFilePart(w *multipart.Writer, varname, filename string) error {
func addFilePart(w *multipart.Writer, varname, filename, mimeFilename string) error {
file, err := os.Open(filename)
if err != nil {
return err
}
defer file.Close()
part, err := w.CreateFormFile(varname, filepath.Base(filename))
part, err := w.CreateFormFile(varname, mimeFilename)
if err != nil {
return err
}
......@@ -83,25 +84,29 @@ func addFilePart(w *multipart.Writer, varname, filename string) error {
}
// SendBook uploads a book to the remote server.
func (r *remoteServer) SendBook(book *Book) error {
func (r *remoteServer) SendBook(book *Book, files []*File) error {
// Create a multipart request with the JSON-encoded metadata
// and the actual file contents as two separate mime/multipart
// sections.
var body bytes.Buffer
w := multipart.NewWriter(&body)
w.WriteField("type", book.FileType)
part, err := w.CreateFormFile("meta", "meta.json")
if err := json.NewEncoder(part).Encode(book.Metadata); err != nil {
return err
}
if err := addFilePart(w, "book", book.Path); err != nil {
w.Close()
return err
for i, f := range files {
varname := fmt.Sprintf("book%d", i)
filename := fmt.Sprintf("%d%s", book.Id, f.FileType)
if err := addFilePart(w, varname, f.Path, filename); err != nil {
w.Close()
return err
}
}
if book.CoverPath != "" {
if err := addFilePart(w, "cover", book.CoverPath); err != nil {
if err := addFilePart(w, "cover", book.CoverPath, "cover.jpg"); err != nil {
w.Close()
return err
}
......@@ -195,9 +200,12 @@ func (db *Database) Sync(remote SyncClient) error {
wg.Add(1)
go func() {
for id := range ch {
if book, err := db.GetBook(ParseID(id)); err == nil {
if err := remote.SendBook(book); err != nil {
log.Printf("SendBook(%d): %v", id, err)
bookid := ParseID(id)
if book, err := db.GetBook(bookid); err == nil {
if files, err := db.GetBookFiles(bookid); err == nil {
if err := remote.SendBook(book, files); err != nil {
log.Printf("SendBook(%d): %v", id, err)
}
}
}
}
......@@ -251,13 +259,6 @@ func (l *syncServer) handleDiffRequest(w http.ResponseWriter, req *http.Request)
}
func (l *syncServer) handleSyncUpload(w http.ResponseWriter, req *http.Request) {
filetype := req.FormValue("type")
if filetype == "" {
log.Printf("request with no 'type' field")
http.Error(w, "Bad request", http.StatusBadRequest)
return
}
mf, _, err := req.FormFile("meta")
if err != nil {
log.Printf("request with no 'meta' field")
......@@ -284,21 +285,46 @@ func (l *syncServer) handleSyncUpload(w http.ResponseWriter, req *http.Request)
book := &Book{
Id: bookid,
Metadata: &md,
FileType: filetype,
Path: l.storage.Path(fmt.Sprintf("%d%s", bookid, filetype)),
}
// Store the data into a custom path and save the book into
// the local database.
if err := savePart(req, "book", l.storage, book.Path); err != nil {
log.Printf("error saving local file: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
// Save the file data to our local storage.
for i := 0; i < 10; i++ {
// Use a temporary file, we'll know the right
// extension to use only after having parsed the
// file's MIME header.
tmppath := l.storage.Path(fmt.Sprintf("%d.%d.tmp", bookid, i))
varname := fmt.Sprintf("book%d", i)
size, hdr, err := savePart(req, varname, l.storage, tmppath)
if err == http.ErrMissingFile {
break
} else if err != nil {
log.Printf("error saving local file: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
filetype := filepath.Ext(hdr.Filename)
path := l.storage.Path(fmt.Sprintf("%d.%d%s", bookid, i, filetype))
if err := l.storage.Rename(tmppath, path); err != nil {
log.Printf("error moving local file: %v", err)
}
file := &File{
Path: path,
FileType: filetype,
Mtime: time.Now(),
Size: size,
Id: bookid,
}
if err := l.db.PutFile(file); err != nil {
log.Printf("error saving file to the database: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
// If the request contains a cover image, save that as well.
coverPath := l.storage.Path(fmt.Sprintf("%s%s.cover.png", book.Id, filetype))
if err := savePart(req, "cover", l.storage, coverPath); err == nil {
coverPath := l.storage.Path(fmt.Sprintf("%s.cover.png", book.Id))
if _, _, err := savePart(req, "cover", l.storage, coverPath); err == nil {
book.CoverPath = coverPath
}
......@@ -312,21 +338,22 @@ func (l *syncServer) handleSyncUpload(w http.ResponseWriter, req *http.Request)
w.WriteHeader(200)
}
func savePart(req *http.Request, fieldname string, storage *FileStorage, outname string) error {
f, _, err := req.FormFile(fieldname)
func savePart(req *http.Request, fieldname string, storage *FileStorage, outname string) (int64, *multipart.FileHeader, error) {
f, hdr, err := req.FormFile(fieldname)
if err != nil {
return err
return 0, nil, err
}
outf, err := storage.Create(outname)
if err != nil {
return err
return 0, nil, err
}
defer outf.Close()
if _, err := io.Copy(outf, f); err != nil {
return err
n, err := io.Copy(outf, f)
if err != nil {
return 0, nil, err
}
return nil
return n, hdr, nil
}
sync_test.go
View file @ f570971f
......@@ -2,7 +2,6 @@ package liber
import (
"fmt"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
......@@ -11,7 +10,7 @@ import (
"testing"
)
func newTestHttpServer(db *Database, updir string) *httptest.Server {
func newTestSyncHttpServer(db *Database, updir string) *httptest.Server {
localsrv := &syncServer{db, &FileStorage{Root: updir, Nesting: 2}}
mux := http.NewServeMux()
......@@ -22,11 +21,6 @@ func newTestHttpServer(db *Database, updir string) *httptest.Server {
}
func TestSync_Sync(t *testing.T) {
// Actually create a file to upload, or the sync will fail.
f, _ := ioutil.TempFile("", "ebook-")
io.WriteString(f, "foo\n")
f.Close()
// Create a temporary directory to store uploads.
updir, _ := ioutil.TempDir("", "ebook-upload-")
defer os.RemoveAll(updir)
......@@ -38,20 +32,20 @@ func TestSync_Sync(t *testing.T) {
defer td2.Close()
for i := 0; i < 10; i++ {
bookid := NewID()
db.PutBook(&Book{
Id: NewID(),
Path: f.Name(),
FileType: ".epub",
Id: bookid,
Metadata: &Metadata{
Title: fmt.Sprintf("Book #%d", i+1),
Creator: []string{"Random Author"},
ISBN: []string{strconv.Itoa(i + 1)},
},
})
db.PutFile(testEpubFile(updir, bookid))
}
// Run a sync from db to db2.
srv := newTestHttpServer(db2, updir)
srv := newTestSyncHttpServer(db2, updir)
defer srv.Close()
cl := NewRemoteServer(srv.URL)
......
update.go
View file @ f570971f
......@@ -17,22 +17,29 @@ const (
type MetadataChooserFunc func(string, []*Metadata) *Metadata
type fileData struct {
source int
path string
id BookId
info os.FileInfo
source int
path string
filetype string
id BookId
info os.FileInfo
}
func (f fileData) toLiberFile(haserr bool) *File {
return &File{
Path: f.path,
Mtime: f.info.ModTime(),
Size: f.info.Size(),
Id: f.id,
Error: haserr,
Path: f.path,
FileType: f.filetype,
Mtime: f.info.ModTime(),
Size: f.info.Size(),
Id: f.id,
Error: haserr,
}
}
type fileAndBook struct {
f fileData
b *Book
}
func dbFileScanner(db *Database, fileCh chan fileData) {
for iter := db.Scan(FileBucket); iter.Valid(); iter.Next() {
var f File
......@@ -83,34 +90,28 @@ func differ(db *Database, basedir string) chan fileData {
close(fileCh)
}()
go func() {
// Detect files that have not changed, i.e. appear in
// the database and the filesystem. Keep track of book
// IDs so that once all entries have been processed we
// can delete those books from the database where the
// original file has been removed.
// Merge the two sources and keep track of files that
// only appear in the database but not on the
// filesystem, so we can remove them at the end.
// All entries with source == SourceFS will be sent to
// the output channel in any case.
allSources := SourceDB | SourceFS
tmp := make(map[string]int)
ids := make(map[string]BookId)
for f := range fileCh {
// log.Printf("differ: %#v", f)
tmp[f.path] |= f.source
// Delete entries as soon as we've seen them
// from both sources.
// originate from both sources.
if tmp[f.path] == allSources {
// log.Printf("differ: dropping %s", f.path)
delete(tmp, f.path)
delete(ids, f.path)
}
if f.source == SourceFS {
outCh <- f
} else {
ids[f.path] = f.id
}
}
for path, value := range tmp {
if value == SourceDB {
log.Printf("removing book %s", path)
db.DeleteBook(ids[path])
log.Printf("removing file %s", path)
db.DeleteFile(path)
}
}
close(outCh)
......@@ -118,41 +119,35 @@ func differ(db *Database, basedir string) chan fileData {
return outCh
}
func adder(db *Database, chooser MetadataChooserFunc, fileCh chan fileData) {
func extractor(db *Database, chooser MetadataChooserFunc, fileCh chan fileData, outCh chan fileAndBook) {
for f := range fileCh {
var oldid BookId
if oldfile, err := db.GetFile(f.path); err == nil {
if !oldfile.HasChanged(f.info) {
continue
}
oldid = oldfile.Id
f.id = oldfile.Id
}
var err error
f.id, err = importBook(db, f, oldid, chooser)
if err != nil {
log.Printf("Could not add %s: %v", f.path, err)
book, filetype, err := parseMeta(f, chooser)
if err == nil {
f.filetype = filetype
outCh <- fileAndBook{f: f, b: book}
continue
}
file := f.toLiberFile(err != nil)
// Parse errors are permanent.
log.Printf("Could not parse %s: %v", f.path, err)
file := f.toLiberFile(true)
if err := db.PutFile(file); err != nil {
log.Println(err)
log.Printf("Error saving file %s to db: %v", file.Path, err)
}
}
}
func importBook(db *Database, f fileData, oldid BookId, chooser MetadataChooserFunc) (BookId, error) {
func parseMeta(f fileData, chooser MetadataChooserFunc) (*Book, string, error) {
// Attempt direct metadata extraction.
book, err := Parse(f.path)
book, filetype, err := Parse(f.path)
if err != nil {
return 0, err
}
if oldid != 0 {
book.Id = oldid
} else {
book.Id = NewID()
return nil, "", err
}
// Check if a Calibre OPF file exists.
......@@ -175,9 +170,10 @@ func importBook(db *Database, f fileData, oldid BookId, chooser MetadataChooserF
}
}
// Check if the book metadata looks ok.
// Check if the book metadata looks ok. If not, don't even
// bother looking for a cover image.
if !book.Metadata.Sufficient() {
return 0, errors.New("insufficient metadata")
return nil, "", errors.New("insufficient metadata")
}
// Try to find a cover image. Look on the local filesystem
......@@ -186,7 +182,7 @@ func importBook(db *Database, f fileData, oldid BookId, chooser MetadataChooserF
if _, err := os.Stat(localCoverPath); err == nil {
book.CoverPath = localCoverPath
} else if imageData, err := GetGoogleBooksCover(book.Metadata); err == nil {
imageFileName := book.Path + ".cover.png"
imageFileName := f.path + ".cover.png"
if imgf, err := os.Create(imageFileName); err != nil {
log.Printf("Could not save cover image for %d: %v", book.Id, err)
} else {
......@@ -196,23 +192,63 @@ func importBook(db *Database, f fileData, oldid BookId, chooser MetadataChooserF
}
}