Commit f570971f authored by ale's avatar ale

support multiple files per ebook

parent d4b72fa0
...@@ -19,12 +19,13 @@ import ( ...@@ -19,12 +19,13 @@ import (
) )
var ( var (
databaseDir = flag.String("db-dir", "~/.liber", "database directory") databaseDir = flag.String("db-dir", "~/.liber", "database directory")
bookDir = flag.String("book-dir", "", "books directory") bookDir = flag.String("book-dir", "", "books directory")
update = flag.Bool("update", false, "update the db") update = flag.Bool("update", false, "update the db")
search = flag.Bool("search", false, "search something") search = flag.Bool("search", false, "search something")
remotesync = flag.String("sync", "", "push data to remote server") remotesync = flag.String("sync", "", "push data to remote server")
httpserver = flag.String("http-server", "", "start the HTTP server on the specified address") httpserver = flag.String("http-server", "", "start the HTTP server on the specified address")
noninteractive = flag.Bool("noninteractive", false, "disable interactive metadata prompts on update")
) )
// Various ways to ask a user to choose something. // Various ways to ask a user to choose something.
...@@ -127,6 +128,10 @@ func promptUserDialog(path string, choices []*liber.Metadata) (*liber.Metadata, ...@@ -127,6 +128,10 @@ func promptUserDialog(path string, choices []*liber.Metadata) (*liber.Metadata,
var promptMutex sync.Mutex var promptMutex sync.Mutex
func promptUser(path string, choices []*liber.Metadata) *liber.Metadata { func promptUser(path string, choices []*liber.Metadata) *liber.Metadata {
if *noninteractive {
return nil
}
promptMutex.Lock() promptMutex.Lock()
defer promptMutex.Unlock() defer promptMutex.Unlock()
...@@ -160,9 +165,14 @@ func doSearch(db *liber.Database, query string) { ...@@ -160,9 +165,14 @@ func doSearch(db *liber.Database, query string) {
fmt.Printf("No results.\n") fmt.Printf("No results.\n")
} else { } else {
fmt.Printf("%d results found:\n\n", results.NumResults) fmt.Printf("%d results found:\n\n", results.NumResults)
for i, r := range results.Results { for i, book := range results.Results {
fmt.Printf("%d) %s\n", i+1, r.Metadata.String()) fmt.Printf("%d) %s\n", i+1, book.Metadata.String())
fmt.Printf(" %s\n", r.Path) if files, err := db.GetBookFiles(book.Id); err == nil {
for _, f := range files {
fmt.Printf(" %s: %s\n", strings.TrimPrefix(f.FileType, "."), f.Path)
}
}
fmt.Printf("\n")
} }
} }
} }
...@@ -213,6 +223,7 @@ func main() { ...@@ -213,6 +223,7 @@ func main() {
if err == nil { if err == nil {
defer logf.Close() defer logf.Close()
log.SetOutput(logf) log.SetOutput(logf)
log.SetFlags(log.Ldate | log.Ltime)
} }
doUpdate(db, expandTilde(*bookDir)) doUpdate(db, expandTilde(*bookDir))
......
...@@ -17,8 +17,9 @@ import ( ...@@ -17,8 +17,9 @@ import (
) )
var ( var (
BookBucket = []byte("ebook") BookBucket = []byte("ebook")
FileBucket = []byte("file") FileBucket = []byte("file")
BookFileBucket = []byte("ebook_file")
keySeparator = byte('/') keySeparator = byte('/')
) )
...@@ -84,10 +85,9 @@ func defaultIndexMapping() *bleve.IndexMapping { ...@@ -84,10 +85,9 @@ func defaultIndexMapping() *bleve.IndexMapping {
} }
type Book struct { type Book struct {
Id BookId Id BookId
Path string // Path string
CoverPath string CoverPath string
FileType string
Metadata *Metadata Metadata *Metadata
} }
...@@ -96,11 +96,12 @@ func (b *Book) Type() string { ...@@ -96,11 +96,12 @@ func (b *Book) Type() string {
} }
type File struct { type File struct {
Path string Path string
Mtime time.Time FileType string
Size int64 Mtime time.Time
Error bool Size int64
Id BookId Error bool
Id BookId
} }
func (f *File) HasChanged(info os.FileInfo) bool { func (f *File) HasChanged(info os.FileInfo) bool {
...@@ -135,6 +136,7 @@ func NewDb(path string) (*Database, error) { ...@@ -135,6 +136,7 @@ func NewDb(path string) (*Database, error) {
} }
func (db *Database) setupLevelDb(path string) error { func (db *Database) setupLevelDb(path string) error {
// Use 256MB of cache and a small Bloom filter.
opts := levigo.NewOptions() opts := levigo.NewOptions()
db.leveldbCache = levigo.NewLRUCache(2 << 28) db.leveldbCache = levigo.NewLRUCache(2 << 28)
opts.SetCache(db.leveldbCache) opts.SetCache(db.leveldbCache)
...@@ -185,6 +187,24 @@ func (db *Database) GetFile(path string) (*File, error) { ...@@ -185,6 +187,24 @@ func (db *Database) GetFile(path string) (*File, error) {
return &f, nil return &f, nil
} }
func (db *Database) GetBookFiles(bookid BookId) ([]*File, error) {
ro := levigo.NewReadOptions()
defer ro.Close()
it := db.leveldb.NewIterator(ro)
defer it.Close()
start, end := keyRange(bktToKey(BookFileBucket, bookid.Key()))
var out []*File
for it.Seek(start); it.Valid() && bytes.Compare(it.Key(), end) < 0; it.Next() {
var filepath string
if json.Unmarshal(it.Value(), &filepath) == nil {
if file, err := db.GetFile(filepath); err == nil {
out = append(out, file)
}
}
}
return out, nil
}
func (db *Database) Get(bucket, key []byte, obj interface{}) error { func (db *Database) Get(bucket, key []byte, obj interface{}) error {
ro := levigo.NewReadOptions() ro := levigo.NewReadOptions()
defer ro.Close() defer ro.Close()
...@@ -202,8 +222,18 @@ func (db *Database) PutBook(b *Book) error { ...@@ -202,8 +222,18 @@ func (db *Database) PutBook(b *Book) error {
return db.index.Index(b.Id.String(), b.Metadata) return db.index.Index(b.Id.String(), b.Metadata)
} }
func fileBookKey(path string, bookid BookId) []byte {
return bytes.Join([][]byte{bookid.Key(), []byte(path)}, []byte{keySeparator})
}
func (db *Database) PutFile(f *File) error { func (db *Database) PutFile(f *File) error {
return db.Put(FileBucket, []byte(f.Path), f) if err := db.Put(FileBucket, []byte(f.Path), f); err != nil {
return err
}
if !f.Error {
return db.Put(BookFileBucket, fileBookKey(f.Path, f.Id), f.Path)
}
return nil
} }
func (db *Database) Put(bucket, key []byte, obj interface{}) error { func (db *Database) Put(bucket, key []byte, obj interface{}) error {
...@@ -223,6 +253,23 @@ func (db *Database) DeleteBook(bookid BookId) error { ...@@ -223,6 +253,23 @@ func (db *Database) DeleteBook(bookid BookId) error {
return db.index.Delete(bookid.String()) return db.index.Delete(bookid.String())
} }
func (db *Database) DeleteFile(path string) error {
f, err := db.GetFile(path)
if err != nil {
return nil
}
db.Delete(FileBucket, []byte(path))
db.Delete(BookFileBucket, fileBookKey(path, f.Id))
// Delete the book if there are no files left.
if files, err := db.GetBookFiles(f.Id); err == nil && len(files) == 0 {
db.DeleteBook(f.Id)
}
return nil
}
func (db *Database) Delete(bucket, key []byte) error { func (db *Database) Delete(bucket, key []byte) error {
wo := levigo.NewWriteOptions() wo := levigo.NewWriteOptions()
defer wo.Close() defer wo.Close()
......
package liber package liber
import ( import (
"io"
"io/ioutil" "io/ioutil"
"os" "os"
"testing" "testing"
) )
var testdbdir = "/tmp/liber-test-database"
type testDatabase struct { type testDatabase struct {
db *Database db *Database
path string path string
refbookid BookId
} }
func (td *testDatabase) Close() { func (td *testDatabase) Close() {
...@@ -29,15 +29,28 @@ func newTestDatabase(t *testing.T) (*testDatabase, *Database) { ...@@ -29,15 +29,28 @@ func newTestDatabase(t *testing.T) (*testDatabase, *Database) {
if err = db.PutBook(book); err != nil { if err = db.PutBook(book); err != nil {
t.Fatalf("PutBook(): %v", err) t.Fatalf("PutBook(): %v", err)
} }
if err = db.PutFile(testEpubFile(path, book.Id)); err != nil {
t.Fatalf("PutFile(): %v", err)
}
return &testDatabase{db: db, path: path}, db return &testDatabase{db: db, path: path, refbookid: book.Id}, db
}
func testEpubFile(dir string, bookid BookId) *File {
f, _ := ioutil.TempFile(dir, "ebook-")
io.WriteString(f, "epub\n")
f.Close()
return &File{
Id: bookid,
Path: f.Name(),
FileType: ".epub",
Size: 4,
}
} }
func testEbook() *Book { func testEbook() *Book {
return &Book{ return &Book{
Id: NewID(), Id: NewID(),
Path: "/path/to/ebook",
FileType: ".epub",
Metadata: &Metadata{ Metadata: &Metadata{
Title: "20,000 Leagues under the sea", Title: "20,000 Leagues under the sea",
Creator: []string{"Jules Verne"}, Creator: []string{"Jules Verne"},
...@@ -46,6 +59,55 @@ func testEbook() *Book { ...@@ -46,6 +59,55 @@ func testEbook() *Book {
} }
} }
func TestDatabase_Get(t *testing.T) {
td, db := newTestDatabase(t)
defer td.Close()
_, err := db.GetBook(td.refbookid)
if err != nil {
t.Fatalf("GetBook(%d): %v", td.refbookid, err)
}
files, err := db.GetBookFiles(td.refbookid)
if err != nil {
t.Fatalf("GetBookFiles(%d): %v", td.refbookid, err)
}
if len(files) != 1 {
t.Fatalf("GetBookFiles(%d) bad result: %v", td.refbookid, files)
}
}
func TestDatabase_BookFileRelation(t *testing.T) {
td, db := newTestDatabase(t)
defer td.Close()
checkFiles := func(tag string, n int) []*File {
files, err := db.GetBookFiles(td.refbookid)
if err != nil {
t.Fatalf("GetBookFiles@%s(%d): %v", tag, td.refbookid, err)
}
if len(files) != n {
t.Fatalf("GetBookFiles@%s(%d) bad result (exp. len=%d): %v", tag, td.refbookid, n, files)
}
return files
}
files := checkFiles("init", 1)
file0 := files[0]
file1 := testEpubFile(td.path, td.refbookid)
db.PutFile(file1)
checkFiles("post_add", 2)
db.DeleteFile(file1.Path)
checkFiles("post_delete", 1)
db.DeleteFile(file0.Path)
checkFiles("post_delete_2", 0)
if _, err := db.GetBook(td.refbookid); err == nil {
t.Fatal("Book was not removed when n.files==0")
}
}
func TestDatabase_Search(t *testing.T) { func TestDatabase_Search(t *testing.T) {
td, db := newTestDatabase(t) td, db := newTestDatabase(t)
defer td.Close() defer td.Close()
......
...@@ -48,3 +48,14 @@ func (s *FileStorage) Open(path string) (*os.File, error) { ...@@ -48,3 +48,14 @@ func (s *FileStorage) Open(path string) (*os.File, error) {
} }
return os.Open(filepath.Join(s.Root, path)) return os.Open(filepath.Join(s.Root, path))
} }
// Rename oldpath to newpath.
func (s *FileStorage) Rename(oldpath, newpath string) error {
if !strings.HasPrefix(oldpath, "/") {
oldpath = filepath.Join(s.Root, oldpath)
}
if !strings.HasPrefix(newpath, "/") {
newpath = filepath.Join(s.Root, newpath)
}
return os.Rename(oldpath, newpath)
}
...@@ -32,12 +32,14 @@ ...@@ -32,12 +32,14 @@
<p>{{.Book.Metadata.Description}}</p> <p>{{.Book.Metadata.Description}}</p>
{{end}} {{end}}
{{range $i, $f := .Files}}
<p> <p>
<a class="btn btn-large btn-primary" href="/dl/{{.Book.Id}}"> <a class="btn btn-large btn-primary" href="/dl/{{$f.Id}}/{{$i}}">
<span class="glyphicon glyphicon-download-alt"></span> <span class="glyphicon glyphicon-download-alt"></span>
Download Download ({{$f.FileType}})
</a> </a>
</p> </p>
{{end}}
</div> </div>
......
...@@ -244,7 +244,7 @@ func parseAnything(filename string) (*Metadata, error) { ...@@ -244,7 +244,7 @@ func parseAnything(filename string) (*Metadata, error) {
}, nil }, nil
} }
func Parse(filename string) (*Book, error) { func Parse(filename string) (*Book, string, error) {
var m *Metadata var m *Metadata
var err error var err error
ext := strings.ToLower(filepath.Ext(filename)) ext := strings.ToLower(filepath.Ext(filename))
...@@ -256,16 +256,11 @@ func Parse(filename string) (*Book, error) { ...@@ -256,16 +256,11 @@ func Parse(filename string) (*Book, error) {
case ".pdf": case ".pdf":
m, err = parseAnything(filename) m, err = parseAnything(filename)
default: default:
return nil, errors.New("unsupported file format") return nil, "", errors.New("unsupported file format")
} }
if err != nil { if err != nil {
return nil, err return nil, "", err
} }
b := &Book{ return &Book{Metadata: m}, ext, nil
Path: filename,
FileType: ext,
Metadata: m,
}
return b, nil
} }
...@@ -11,6 +11,7 @@ import ( ...@@ -11,6 +11,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"sync" "sync"
"time"
) )
const ( const (
...@@ -20,7 +21,7 @@ const ( ...@@ -20,7 +21,7 @@ const (
type SyncClient interface { type SyncClient interface {
DiffRequest(*diffRequest) (*diffResponse, error) DiffRequest(*diffRequest) (*diffResponse, error)
SendBook(*Book) error SendBook(*Book, []*File) error
} }
type remoteServer struct { type remoteServer struct {
...@@ -64,14 +65,14 @@ func (r *remoteServer) DiffRequest(diffreq *diffRequest) (*diffResponse, error) ...@@ -64,14 +65,14 @@ func (r *remoteServer) DiffRequest(diffreq *diffRequest) (*diffResponse, error)
return &diffresp, nil return &diffresp, nil
} }
func addFilePart(w *multipart.Writer, varname, filename string) error { func addFilePart(w *multipart.Writer, varname, filename, mimeFilename string) error {
file, err := os.Open(filename) file, err := os.Open(filename)
if err != nil { if err != nil {
return err return err
} }
defer file.Close() defer file.Close()
part, err := w.CreateFormFile(varname, filepath.Base(filename)) part, err := w.CreateFormFile(varname, mimeFilename)
if err != nil { if err != nil {
return err return err
} }
...@@ -83,25 +84,29 @@ func addFilePart(w *multipart.Writer, varname, filename string) error { ...@@ -83,25 +84,29 @@ func addFilePart(w *multipart.Writer, varname, filename string) error {
} }
// SendBook uploads a book to the remote server. // SendBook uploads a book to the remote server.
func (r *remoteServer) SendBook(book *Book) error { func (r *remoteServer) SendBook(book *Book, files []*File) error {
// Create a multipart request with the JSON-encoded metadata // Create a multipart request with the JSON-encoded metadata
// and the actual file contents as two separate mime/multipart // and the actual file contents as two separate mime/multipart
// sections. // sections.
var body bytes.Buffer var body bytes.Buffer
w := multipart.NewWriter(&body) w := multipart.NewWriter(&body)
w.WriteField("type", book.FileType)
part, err := w.CreateFormFile("meta", "meta.json") part, err := w.CreateFormFile("meta", "meta.json")
if err := json.NewEncoder(part).Encode(book.Metadata); err != nil { if err := json.NewEncoder(part).Encode(book.Metadata); err != nil {
return err return err
} }
if err := addFilePart(w, "book", book.Path); err != nil { for i, f := range files {
w.Close() varname := fmt.Sprintf("book%d", i)
return err filename := fmt.Sprintf("%d%s", book.Id, f.FileType)
if err := addFilePart(w, varname, f.Path, filename); err != nil {
w.Close()
return err
}
} }
if book.CoverPath != "" { if book.CoverPath != "" {
if err := addFilePart(w, "cover", book.CoverPath); err != nil { if err := addFilePart(w, "cover", book.CoverPath, "cover.jpg"); err != nil {
w.Close() w.Close()
return err return err
} }
...@@ -195,9 +200,12 @@ func (db *Database) Sync(remote SyncClient) error { ...@@ -195,9 +200,12 @@ func (db *Database) Sync(remote SyncClient) error {
wg.Add(1) wg.Add(1)
go func() { go func() {
for id := range ch { for id := range ch {
if book, err := db.GetBook(ParseID(id)); err == nil { bookid := ParseID(id)
if err := remote.SendBook(book); err != nil { if book, err := db.GetBook(bookid); err == nil {
log.Printf("SendBook(%d): %v", id, err) if files, err := db.GetBookFiles(bookid); err == nil {
if err := remote.SendBook(book, files); err != nil {
log.Printf("SendBook(%d): %v", id, err)
}
} }
} }
} }
...@@ -251,13 +259,6 @@ func (l *syncServer) handleDiffRequest(w http.ResponseWriter, req *http.Request) ...@@ -251,13 +259,6 @@ func (l *syncServer) handleDiffRequest(w http.ResponseWriter, req *http.Request)
} }
func (l *syncServer) handleSyncUpload(w http.ResponseWriter, req *http.Request) { func (l *syncServer) handleSyncUpload(w http.ResponseWriter, req *http.Request) {
filetype := req.FormValue("type")
if filetype == "" {
log.Printf("request with no 'type' field")
http.Error(w, "Bad request", http.StatusBadRequest)
return
}
mf, _, err := req.FormFile("meta") mf, _, err := req.FormFile("meta")
if err != nil { if err != nil {
log.Printf("request with no 'meta' field") log.Printf("request with no 'meta' field")
...@@ -284,21 +285,46 @@ func (l *syncServer) handleSyncUpload(w http.ResponseWriter, req *http.Request) ...@@ -284,21 +285,46 @@ func (l *syncServer) handleSyncUpload(w http.ResponseWriter, req *http.Request)
book := &Book{ book := &Book{
Id: bookid, Id: bookid,
Metadata: &md, Metadata: &md,
FileType: filetype,
Path: l.storage.Path(fmt.Sprintf("%d%s", bookid, filetype)),
} }
// Store the data into a custom path and save the book into // Save the file data to our local storage.
// the local database. for i := 0; i < 10; i++ {
if err := savePart(req, "book", l.storage, book.Path); err != nil { // Use a temporary file, we'll know the right
log.Printf("error saving local file: %v", err) // extension to use only after having parsed the
http.Error(w, err.Error(), http.StatusInternalServerError) // file's MIME header.
return tmppath := l.storage.Path(fmt.Sprintf("%d.%d.tmp", bookid, i))
varname := fmt.Sprintf("book%d", i)
size, hdr, err := savePart(req, varname, l.storage, tmppath)
if err == http.ErrMissingFile {
break
} else if err != nil {
log.Printf("error saving local file: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
filetype := filepath.Ext(hdr.Filename)
path := l.storage.Path(fmt.Sprintf("%d.%d%s", bookid, i, filetype))
if err := l.storage.Rename(tmppath, path); err != nil {
log.Printf("error moving local file: %v", err)
}
file := &File{
Path: path,
FileType: filetype,
Mtime: time.Now(),
Size: size,
Id: bookid,
}
if err := l.db.PutFile(file); err != nil {
log.Printf("error saving file to the database: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
} }
// If the request contains a cover image, save that as well. // If the request contains a cover image, save that as well.
coverPath := l.storage.Path(fmt.Sprintf("%s%s.cover.png", book.Id, filetype)) coverPath := l.storage.Path(fmt.Sprintf("%s.cover.png", book.Id))
if err := savePart(req, "cover", l.storage, coverPath); err == nil { if _, _, err := savePart(req, "cover", l.storage, coverPath); err == nil {
book.CoverPath = coverPath book.CoverPath = coverPath
} }
...@@ -312,21 +338,22 @@ func (l *syncServer) handleSyncUpload(w http.ResponseWriter, req *http.Request) ...@@ -312,21 +338,22 @@ func (l *syncServer) handleSyncUpload(w http.ResponseWriter, req *http.Request)
w.WriteHeader(200) w.WriteHeader(200)
} }
func savePart(req *http.Request, fieldname string, storage *FileStorage, outname string) error { func savePart(req *http.Request, fieldname string, storage *FileStorage, outname string) (int64, *multipart.FileHeader, error) {
f, _, err := req.FormFile(fieldname) f, hdr, err := req.FormFile(fieldname)
if err != nil { if err != nil {
return err return 0, nil, err
} }
outf, err := storage.Create(outname) outf, err := storage.Create(outname)
if err != nil { if err != nil {
return err return 0, nil, err
} }
defer outf.Close() defer outf.Close()
if _, err := io.Copy(outf, f); err != nil { n, err := io.Copy(outf, f)
return err if err != nil {
return 0, nil, err
} }
return nil return n, hdr, nil
} }
...@@ -2,7 +2,6 @@ package liber ...@@ -2,7 +2,6 @@ package liber
import ( import (
"fmt" "fmt"
"io"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
...@@ -11,7 +10,7 @@ import ( ...@@ -11,7 +10,7 @@ import (
"testing" "testing"
) )
func newTestHttpServer(db *Database, updir string) *httptest.Server { func newTestSyncHttpServer(db *Database, updir string) *httptest.Server {
localsrv := &syncServer{db, &FileStorage{Root: updir, Nesting: 2}} localsrv := &syncServer{db, &FileStorage{Root: updir, Nesting: 2}}
mux := http.NewServeMux() mux := http.NewServeMux()
...@@ -22,11 +21,6 @@ func newTestHttpServer(db *Database, updir string) *httptest.Server { ...@@ -22,11 +21,6 @@ func newTestHttpServer(db *Database, updir string) *httptest.Server {
} }
func TestSync_Sync(t *testing.T) { func TestSync_Sync(t *testing.T) {
// Actually create a file to upload, or the sync will fail.
f, _ := ioutil.TempFile("", "ebook-")
io.WriteString(f, "foo\n")
f.Close()
// Create a temporary directory to store uploads. // Create a temporary directory to store uploads.
updir, _ := ioutil.TempDir("", "ebook-upload-") updir, _ := ioutil.TempDir("", "ebook-upload-")
defer os.RemoveAll(updir) defer os.RemoveAll(updir)
...@@ -38,20 +32,20 @@ func TestSync_Sync(t *testing.T) { ...@@ -38,20 +32,20 @@ func TestSync_Sync(t *testing.T) {
defer td2.Close() defer td2.Close()
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
bookid := NewID()
db.PutBook(&Book{ db.PutBook(&Book{
Id: NewID(), Id: bookid,
Path: f.Name(),
FileType: ".epub",
Metadata: &Metadata{ Metadata: &Metadata{
Title: fmt.Sprintf("Book #%d", i+1), Title: fmt.Sprintf("Book #%d", i+1),
Creator: []string{"Random Author"}, Creator: []string{"Random Author"},
ISBN: []string{strconv.Itoa(i + 1)}, ISBN: []string{strconv.Itoa(i + 1)},
}, },
}) })
db.PutFile(testEpubFile(updir, bookid))
} }