support multiple files per ebook

cmd/liber/liber.go

@@ -19,12 +19,13 @@ import (
 )
 
 var (
-	databaseDir = flag.String("db-dir", "~/.liber", "database directory")
-	bookDir     = flag.String("book-dir", "", "books directory")
-	update      = flag.Bool("update", false, "update the db")
-	search      = flag.Bool("search", false, "search something")
-	remotesync  = flag.String("sync", "", "push data to remote server")
-	httpserver  = flag.String("http-server", "", "start the HTTP server on the specified address")
+	databaseDir    = flag.String("db-dir", "~/.liber", "database directory")
+	bookDir        = flag.String("book-dir", "", "books directory")
+	update         = flag.Bool("update", false, "update the db")
+	search         = flag.Bool("search", false, "search something")
+	remotesync     = flag.String("sync", "", "push data to remote server")
+	httpserver     = flag.String("http-server", "", "start the HTTP server on the specified address")
+	noninteractive = flag.Bool("noninteractive", false, "disable interactive metadata prompts on update")
 )
 
 // Various ways to ask a user to choose something.
@@ -127,6 +128,10 @@ func promptUserDialog(path string, choices []*liber.Metadata) (*liber.Metadata,
 var promptMutex sync.Mutex
 
 func promptUser(path string, choices []*liber.Metadata) *liber.Metadata {
+	if *noninteractive {
+		return nil
+	}
+
 	promptMutex.Lock()
 	defer promptMutex.Unlock()
 
@@ -160,9 +165,14 @@ func doSearch(db *liber.Database, query string) {
 		fmt.Printf("No results.\n")
 	} else {
 		fmt.Printf("%d results found:\n\n", results.NumResults)
-		for i, r := range results.Results {
-			fmt.Printf("%d) %s\n", i+1, r.Metadata.String())
-			fmt.Printf("      %s\n", r.Path)
+		for i, book := range results.Results {
+			fmt.Printf("%d) %s\n", i+1, book.Metadata.String())
+			if files, err := db.GetBookFiles(book.Id); err == nil {
+				for _, f := range files {
+					fmt.Printf("      %s: %s\n", strings.TrimPrefix(f.FileType, "."), f.Path)
+				}
+			}
+			fmt.Printf("\n")
 		}
 	}
 }
@@ -213,6 +223,7 @@ func main() {
 		if err == nil {
 			defer logf.Close()
 			log.SetOutput(logf)
+			log.SetFlags(log.Ldate | log.Ltime)
 		}
 
 		doUpdate(db, expandTilde(*bookDir))

database.go

@@ -17,8 +17,9 @@ import (
 )
 
 var (
-	BookBucket = []byte("ebook")
-	FileBucket = []byte("file")
+	BookBucket     = []byte("ebook")
+	FileBucket     = []byte("file")
+	BookFileBucket = []byte("ebook_file")
 
 	keySeparator = byte('/')
 )
@@ -84,10 +85,9 @@ func defaultIndexMapping() *bleve.IndexMapping {
 }
 
 type Book struct {
-	Id        BookId
-	Path      string
+	Id BookId
+	// Path      string
 	CoverPath string
-	FileType  string
 	Metadata  *Metadata
 }
 
@@ -96,11 +96,12 @@ func (b *Book) Type() string {
 }
 
 type File struct {
-	Path  string
-	Mtime time.Time
-	Size  int64
-	Error bool
-	Id    BookId
+	Path     string
+	FileType string
+	Mtime    time.Time
+	Size     int64
+	Error    bool
+	Id       BookId
 }
 
 func (f *File) HasChanged(info os.FileInfo) bool {
@@ -135,6 +136,7 @@ func NewDb(path string) (*Database, error) {
 }
 
 func (db *Database) setupLevelDb(path string) error {
+	// Use 256MB of cache and a small Bloom filter.
 	opts := levigo.NewOptions()
 	db.leveldbCache = levigo.NewLRUCache(2 << 28)
 	opts.SetCache(db.leveldbCache)
@@ -185,6 +187,24 @@ func (db *Database) GetFile(path string) (*File, error) {
 	return &f, nil
 }
 
+func (db *Database) GetBookFiles(bookid BookId) ([]*File, error) {
+	ro := levigo.NewReadOptions()
+	defer ro.Close()
+	it := db.leveldb.NewIterator(ro)
+	defer it.Close()
+	start, end := keyRange(bktToKey(BookFileBucket, bookid.Key()))
+	var out []*File
+	for it.Seek(start); it.Valid() && bytes.Compare(it.Key(), end) < 0; it.Next() {
+		var filepath string
+		if json.Unmarshal(it.Value(), &filepath) == nil {
+			if file, err := db.GetFile(filepath); err == nil {
+				out = append(out, file)
+			}
+		}
+	}
+	return out, nil
+}
+
 func (db *Database) Get(bucket, key []byte, obj interface{}) error {
 	ro := levigo.NewReadOptions()
 	defer ro.Close()
@@ -202,8 +222,18 @@ func (db *Database) PutBook(b *Book) error {
 	return db.index.Index(b.Id.String(), b.Metadata)
 }
 
+func fileBookKey(path string, bookid BookId) []byte {
+	return bytes.Join([][]byte{bookid.Key(), []byte(path)}, []byte{keySeparator})
+}
+
 func (db *Database) PutFile(f *File) error {
-	return db.Put(FileBucket, []byte(f.Path), f)
+	if err := db.Put(FileBucket, []byte(f.Path), f); err != nil {
+		return err
+	}
+	if !f.Error {
+		return db.Put(BookFileBucket, fileBookKey(f.Path, f.Id), f.Path)
+	}
+	return nil
 }
 
 func (db *Database) Put(bucket, key []byte, obj interface{}) error {
@@ -223,6 +253,23 @@ func (db *Database) DeleteBook(bookid BookId) error {
 	return db.index.Delete(bookid.String())
 }
 
+func (db *Database) DeleteFile(path string) error {
+	f, err := db.GetFile(path)
+	if err != nil {
+		return nil
+	}
+
+	db.Delete(FileBucket, []byte(path))
+	db.Delete(BookFileBucket, fileBookKey(path, f.Id))
+
+	// Delete the book if there are no files left.
+	if files, err := db.GetBookFiles(f.Id); err == nil && len(files) == 0 {
+		db.DeleteBook(f.Id)
+	}
+
+	return nil
+}
+
 func (db *Database) Delete(bucket, key []byte) error {
 	wo := levigo.NewWriteOptions()
 	defer wo.Close()

database_test.go

 package liber
 
 import (
+	"io"
 	"io/ioutil"
 	"os"
 	"testing"
 )
 
-var testdbdir = "/tmp/liber-test-database"
-
 type testDatabase struct {
-	db   *Database
-	path string
+	db        *Database
+	path      string
+	refbookid BookId
 }
 
 func (td *testDatabase) Close() {
@@ -29,15 +29,28 @@ func newTestDatabase(t *testing.T) (*testDatabase, *Database) {
 	if err = db.PutBook(book); err != nil {
 		t.Fatalf("PutBook(): %v", err)
 	}
+	if err = db.PutFile(testEpubFile(path, book.Id)); err != nil {
+		t.Fatalf("PutFile(): %v", err)
+	}
 
-	return &testDatabase{db: db, path: path}, db
+	return &testDatabase{db: db, path: path, refbookid: book.Id}, db
+}
+
+func testEpubFile(dir string, bookid BookId) *File {
+	f, _ := ioutil.TempFile(dir, "ebook-")
+	io.WriteString(f, "epub\n")
+	f.Close()
+	return &File{
+		Id:       bookid,
+		Path:     f.Name(),
+		FileType: ".epub",
+		Size:     4,
+	}
 }
 
 func testEbook() *Book {
 	return &Book{
-		Id:       NewID(),
-		Path:     "/path/to/ebook",
-		FileType: ".epub",
+		Id: NewID(),
 		Metadata: &Metadata{
 			Title:   "20,000 Leagues under the sea",
 			Creator: []string{"Jules Verne"},
@@ -46,6 +59,55 @@ func testEbook() *Book {
 	}
 }
 
+func TestDatabase_Get(t *testing.T) {
+	td, db := newTestDatabase(t)
+	defer td.Close()
+
+	_, err := db.GetBook(td.refbookid)
+	if err != nil {
+		t.Fatalf("GetBook(%d): %v", td.refbookid, err)
+	}
+	files, err := db.GetBookFiles(td.refbookid)
+	if err != nil {
+		t.Fatalf("GetBookFiles(%d): %v", td.refbookid, err)
+	}
+	if len(files) != 1 {
+		t.Fatalf("GetBookFiles(%d) bad result: %v", td.refbookid, files)
+	}
+}
+
+func TestDatabase_BookFileRelation(t *testing.T) {
+	td, db := newTestDatabase(t)
+	defer td.Close()
+
+	checkFiles := func(tag string, n int) []*File {
+		files, err := db.GetBookFiles(td.refbookid)
+		if err != nil {
+			t.Fatalf("GetBookFiles@%s(%d): %v", tag, td.refbookid, err)
+		}
+		if len(files) != n {
+			t.Fatalf("GetBookFiles@%s(%d) bad result (exp. len=%d): %v", tag, td.refbookid, n, files)
+		}
+		return files
+	}
+
+	files := checkFiles("init", 1)
+	file0 := files[0]
+	file1 := testEpubFile(td.path, td.refbookid)
+	db.PutFile(file1)
+	checkFiles("post_add", 2)
+
+	db.DeleteFile(file1.Path)
+	checkFiles("post_delete", 1)
+
+	db.DeleteFile(file0.Path)
+	checkFiles("post_delete_2", 0)
+
+	if _, err := db.GetBook(td.refbookid); err == nil {
+		t.Fatal("Book was not removed when n.files==0")
+	}
+}
+
 func TestDatabase_Search(t *testing.T) {
 	td, db := newTestDatabase(t)
 	defer td.Close()

files.go

@@ -48,3 +48,14 @@ func (s *FileStorage) Open(path string) (*os.File, error) {
 	}
 	return os.Open(filepath.Join(s.Root, path))
 }
+
+// Rename oldpath to newpath.
+func (s *FileStorage) Rename(oldpath, newpath string) error {
+	if !strings.HasPrefix(oldpath, "/") {
+		oldpath = filepath.Join(s.Root, oldpath)
+	}
+	if !strings.HasPrefix(newpath, "/") {
+		newpath = filepath.Join(s.Root, newpath)
+	}
+	return os.Rename(oldpath, newpath)
+}

htdocs/templates/book.html

@@ -32,12 +32,14 @@
     <p>{{.Book.Metadata.Description}}</p>
     {{end}}
 
+    {{range $i, $f := .Files}}
     <p>
-      <a class="btn btn-large btn-primary" href="/dl/{{.Book.Id}}">
+      <a class="btn btn-large btn-primary" href="/dl/{{$f.Id}}/{{$i}}">
         <span class="glyphicon glyphicon-download-alt"></span>
-        Download
+        Download ({{$f.FileType}})
       </a>
     </p>
+    {{end}}
 
   </div>
 

metadata.go

@@ -244,7 +244,7 @@ func parseAnything(filename string) (*Metadata, error) {
 	}, nil
 }
 
-func Parse(filename string) (*Book, error) {
+func Parse(filename string) (*Book, string, error) {
 	var m *Metadata
 	var err error
 	ext := strings.ToLower(filepath.Ext(filename))
@@ -256,16 +256,11 @@ func Parse(filename string) (*Book, error) {
 	case ".pdf":
 		m, err = parseAnything(filename)
 	default:
-		return nil, errors.New("unsupported file format")
+		return nil, "", errors.New("unsupported file format")
 	}
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
 
-	b := &Book{
-		Path:     filename,
-		FileType: ext,
-		Metadata: m,
-	}
-	return b, nil
+	return &Book{Metadata: m}, ext, nil
 }

sync.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
+	"time"
 )
 
 const (
@@ -20,7 +21,7 @@ const (
 
 type SyncClient interface {
 	DiffRequest(*diffRequest) (*diffResponse, error)
-	SendBook(*Book) error
+	SendBook(*Book, []*File) error
 }
 
 type remoteServer struct {
@@ -64,14 +65,14 @@ func (r *remoteServer) DiffRequest(diffreq *diffRequest) (*diffResponse, error)
 	return &diffresp, nil
 }
 
-func addFilePart(w *multipart.Writer, varname, filename string) error {
+func addFilePart(w *multipart.Writer, varname, filename, mimeFilename string) error {
 	file, err := os.Open(filename)
 	if err != nil {
 		return err
 	}
 	defer file.Close()
 
-	part, err := w.CreateFormFile(varname, filepath.Base(filename))
+	part, err := w.CreateFormFile(varname, mimeFilename)
 	if err != nil {
 		return err
 	}
@@ -83,25 +84,29 @@ func addFilePart(w *multipart.Writer, varname, filename string) error {
 }
 
 // SendBook uploads a book to the remote server.
-func (r *remoteServer) SendBook(book *Book) error {
+func (r *remoteServer) SendBook(book *Book, files []*File) error {
 	// Create a multipart request with the JSON-encoded metadata
 	// and the actual file contents as two separate mime/multipart
 	// sections.
 	var body bytes.Buffer
 	w := multipart.NewWriter(&body)
-	w.WriteField("type", book.FileType)
+
 	part, err := w.CreateFormFile("meta", "meta.json")
 	if err := json.NewEncoder(part).Encode(book.Metadata); err != nil {
 		return err
 	}
 
-	if err := addFilePart(w, "book", book.Path); err != nil {
-		w.Close()
-		return err
+	for i, f := range files {
+		varname := fmt.Sprintf("book%d", i)
+		filename := fmt.Sprintf("%d%s", book.Id, f.FileType)
+		if err := addFilePart(w, varname, f.Path, filename); err != nil {
+			w.Close()
+			return err
+		}
 	}
 
 	if book.CoverPath != "" {
-		if err := addFilePart(w, "cover", book.CoverPath); err != nil {
+		if err := addFilePart(w, "cover", book.CoverPath, "cover.jpg"); err != nil {
 			w.Close()
 			return err
 		}
@@ -195,9 +200,12 @@ func (db *Database) Sync(remote SyncClient) error {
 		wg.Add(1)
 		go func() {
 			for id := range ch {
-				if book, err := db.GetBook(ParseID(id)); err == nil {
-					if err := remote.SendBook(book); err != nil {
-						log.Printf("SendBook(%d): %v", id, err)
+				bookid := ParseID(id)
+				if book, err := db.GetBook(bookid); err == nil {
+					if files, err := db.GetBookFiles(bookid); err == nil {
+						if err := remote.SendBook(book, files); err != nil {
+							log.Printf("SendBook(%d): %v", id, err)
+						}
 					}
 				}
 			}
@@ -251,13 +259,6 @@ func (l *syncServer) handleDiffRequest(w http.ResponseWriter, req *http.Request)
 }
 
 func (l *syncServer) handleSyncUpload(w http.ResponseWriter, req *http.Request) {
-	filetype := req.FormValue("type")
-	if filetype == "" {
-		log.Printf("request with no 'type' field")
-		http.Error(w, "Bad request", http.StatusBadRequest)
-		return
-	}
-
 	mf, _, err := req.FormFile("meta")
 	if err != nil {
 		log.Printf("request with no 'meta' field")
@@ -284,21 +285,46 @@ func (l *syncServer) handleSyncUpload(w http.ResponseWriter, req *http.Request)
 	book := &Book{
 		Id:       bookid,
 		Metadata: &md,
-		FileType: filetype,
-		Path:     l.storage.Path(fmt.Sprintf("%d%s", bookid, filetype)),
 	}
 
-	// Store the data into a custom path and save the book into
-	// the local database.
-	if err := savePart(req, "book", l.storage, book.Path); err != nil {
-		log.Printf("error saving local file: %v", err)
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
+	// Save the file data to our local storage.
+	for i := 0; i < 10; i++ {
+		// Use a temporary file, we'll know the right
+		// extension to use only after having parsed the
+		// file's MIME header.
+		tmppath := l.storage.Path(fmt.Sprintf("%d.%d.tmp", bookid, i))
+		varname := fmt.Sprintf("book%d", i)
+		size, hdr, err := savePart(req, varname, l.storage, tmppath)
+		if err == http.ErrMissingFile {
+			break
+		} else if err != nil {
+			log.Printf("error saving local file: %v", err)
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		filetype := filepath.Ext(hdr.Filename)
+		path := l.storage.Path(fmt.Sprintf("%d.%d%s", bookid, i, filetype))
+		if err := l.storage.Rename(tmppath, path); err != nil {
+			log.Printf("error moving local file: %v", err)
+		}
+
+		file := &File{
+			Path:     path,
+			FileType: filetype,
+			Mtime:    time.Now(),
+			Size:     size,
+			Id:       bookid,
+		}
+		if err := l.db.PutFile(file); err != nil {
+			log.Printf("error saving file to the database: %v", err)
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
 	}
 
 	// If the request contains a cover image, save that as well.
-	coverPath := l.storage.Path(fmt.Sprintf("%s%s.cover.png", book.Id, filetype))
-	if err := savePart(req, "cover", l.storage, coverPath); err == nil {
+	coverPath := l.storage.Path(fmt.Sprintf("%s.cover.png", book.Id))
+	if _, _, err := savePart(req, "cover", l.storage, coverPath); err == nil {
 		book.CoverPath = coverPath
 	}
 
@@ -312,21 +338,22 @@ func (l *syncServer) handleSyncUpload(w http.ResponseWriter, req *http.Request)
 	w.WriteHeader(200)
 }
 
-func savePart(req *http.Request, fieldname string, storage *FileStorage, outname string) error {
-	f, _, err := req.FormFile(fieldname)
+func savePart(req *http.Request, fieldname string, storage *FileStorage, outname string) (int64, *multipart.FileHeader, error) {
+	f, hdr, err := req.FormFile(fieldname)
 	if err != nil {
-		return err
+		return 0, nil, err
 	}
 
 	outf, err := storage.Create(outname)
 	if err != nil {
-		return err
+		return 0, nil, err
 	}
 	defer outf.Close()
 
-	if _, err := io.Copy(outf, f); err != nil {
-		return err
+	n, err := io.Copy(outf, f)
+	if err != nil {
+		return 0, nil, err
 	}
 
-	return nil
+	return n, hdr, nil
 }

sync_test.go

@@ -2,7 +2,6 @@ package liber
 
 import (
 	"fmt"
-	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -11,7 +10,7 @@ import (
 	"testing"
 )
 
-func newTestHttpServer(db *Database, updir string) *httptest.Server {
+func newTestSyncHttpServer(db *Database, updir string) *httptest.Server {
 	localsrv := &syncServer{db, &FileStorage{Root: updir, Nesting: 2}}
 
 	mux := http.NewServeMux()
@@ -22,11 +21,6 @@ func newTestHttpServer(db *Database, updir string) *httptest.Server {
 }
 
 func TestSync_Sync(t *testing.T) {
-	// Actually create a file to upload, or the sync will fail.
-	f, _ := ioutil.TempFile("", "ebook-")
-	io.WriteString(f, "foo\n")
-	f.Close()
-
 	// Create a temporary directory to store uploads.
 	updir, _ := ioutil.TempDir("", "ebook-upload-")
 	defer os.RemoveAll(updir)
@@ -38,20 +32,20 @@ func TestSync_Sync(t *testing.T) {
 	defer td2.Close()
 
 	for i := 0; i < 10; i++ {
+		bookid := NewID()
 		db.PutBook(&Book{
-			Id:       NewID(),
-			Path:     f.Name(),
-			FileType: ".epub",
+			Id: bookid,
 			Metadata: &Metadata{
 				Title:   fmt.Sprintf("Book #%d", i+1),
 				Creator: []string{"Random Author"},
 				ISBN:    []string{strconv.Itoa(i + 1)},
 			},
 		})
+		db.PutFile(testEpubFile(updir, bookid))
 	}
 
 	// Run a sync from db to db2.
-	srv := newTestHttpServer(db2, updir)
+	srv := newTestSyncHttpServer(db2, updir)
 	defer srv.Close()
 
 	cl := NewRemoteServer(srv.URL)

update.go

@@ -17,22 +17,29 @@ const (
 type MetadataChooserFunc func(string, []*Metadata) *Metadata
 
 type fileData struct {
-	source int
-	path   string
-	id     BookId
-	info   os.FileInfo
+	source   int
+	path     string
+	filetype string
+	id       BookId
+	info     os.FileInfo
 }
 
 func (f fileData) toLiberFile(haserr bool) *File {
 	return &File{
-		Path:  f.path,
-		Mtime: f.info.ModTime(),
-		Size:  f.info.Size(),
-		Id:    f.id,
-		Error: haserr,
+		Path:     f.path,
+		FileType: f.filetype,
+		Mtime:    f.info.ModTime(),
+		Size:     f.info.Size(),
+		Id:       f.id,
+		Error:    haserr,
 	}
 }
 
+type fileAndBook struct {
+	f fileData
+	b *Book
+}
+
 func dbFileScanner(db *Database, fileCh chan fileData) {
 	for iter := db.Scan(FileBucket); iter.Valid(); iter.Next() {
 		var f File
@@ -83,34 +90,28 @@ func differ(db *Database, basedir string) chan fileData {
 		close(fileCh)
 	}()
 	go func() {
-		// Detect files that have not changed, i.e. appear in
-		// the database and the filesystem. Keep track of book
-		// IDs so that once all entries have been processed we
-		// can delete those books from the database where the
-		// original file has been removed.
+		// Merge the two sources and keep track of files that
+		// only appear in the database but not on the
+		// filesystem, so we can remove them at the end.
+		// All entries with source == SourceFS will be sent to
+		// the output channel in any case.
 		allSources := SourceDB | SourceFS
 		tmp := make(map[string]int)
-		ids := make(map[string]BookId)
 		for f := range fileCh {
-			// log.Printf("differ: %#v", f)
 			tmp[f.path] |= f.source
 			// Delete entries as soon as we've seen them
-			// from both sources.
+			// originate from both sources.
 			if tmp[f.path] == allSources {
-				// log.Printf("differ: dropping %s", f.path)
 				delete(tmp, f.path)
-				delete(ids, f.path)
 			}
 			if f.source == SourceFS {