Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
liber
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
2
Issues
2
List
Boards
Labels
Service Desk
Milestones
Merge Requests
1
Merge Requests
1
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Operations
Operations
Incidents
Environments
Analytics
Analytics
CI / CD
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
ale
liber
Commits
f570971f
Commit
f570971f
authored
Nov 10, 2014
by
ale
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
support multiple files per ebook
parent
d4b72fa0
Changes
11
Hide whitespace changes
Inline
Side-by-side
Showing
11 changed files
with
443 additions
and
143 deletions
+443
-143
cmd/liber/liber.go
cmd/liber/liber.go
+20
-9
database.go
database.go
+58
-11
database_test.go
database_test.go
+70
-8
files.go
files.go
+11
-0
htdocs/templates/book.html
htdocs/templates/book.html
+4
-2
metadata.go
metadata.go
+4
-9
sync.go
sync.go
+63
-36
sync_test.go
sync_test.go
+5
-11
update.go
update.go
+88
-52
web.go
web.go
+26
-5
web_test.go
web_test.go
+94
-0
No files found.
cmd/liber/liber.go
View file @
f570971f
...
...
@@ -19,12 +19,13 @@ import (
)
var
(
databaseDir
=
flag
.
String
(
"db-dir"
,
"~/.liber"
,
"database directory"
)
bookDir
=
flag
.
String
(
"book-dir"
,
""
,
"books directory"
)
update
=
flag
.
Bool
(
"update"
,
false
,
"update the db"
)
search
=
flag
.
Bool
(
"search"
,
false
,
"search something"
)
remotesync
=
flag
.
String
(
"sync"
,
""
,
"push data to remote server"
)
httpserver
=
flag
.
String
(
"http-server"
,
""
,
"start the HTTP server on the specified address"
)
databaseDir
=
flag
.
String
(
"db-dir"
,
"~/.liber"
,
"database directory"
)
bookDir
=
flag
.
String
(
"book-dir"
,
""
,
"books directory"
)
update
=
flag
.
Bool
(
"update"
,
false
,
"update the db"
)
search
=
flag
.
Bool
(
"search"
,
false
,
"search something"
)
remotesync
=
flag
.
String
(
"sync"
,
""
,
"push data to remote server"
)
httpserver
=
flag
.
String
(
"http-server"
,
""
,
"start the HTTP server on the specified address"
)
noninteractive
=
flag
.
Bool
(
"noninteractive"
,
false
,
"disable interactive metadata prompts on update"
)
)
// Various ways to ask a user to choose something.
...
...
@@ -127,6 +128,10 @@ func promptUserDialog(path string, choices []*liber.Metadata) (*liber.Metadata,
var
promptMutex
sync
.
Mutex
func
promptUser
(
path
string
,
choices
[]
*
liber
.
Metadata
)
*
liber
.
Metadata
{
if
*
noninteractive
{
return
nil
}
promptMutex
.
Lock
()
defer
promptMutex
.
Unlock
()
...
...
@@ -160,9 +165,14 @@ func doSearch(db *liber.Database, query string) {
fmt
.
Printf
(
"No results.
\n
"
)
}
else
{
fmt
.
Printf
(
"%d results found:
\n\n
"
,
results
.
NumResults
)
for
i
,
r
:=
range
results
.
Results
{
fmt
.
Printf
(
"%d) %s
\n
"
,
i
+
1
,
r
.
Metadata
.
String
())
fmt
.
Printf
(
" %s
\n
"
,
r
.
Path
)
for
i
,
book
:=
range
results
.
Results
{
fmt
.
Printf
(
"%d) %s
\n
"
,
i
+
1
,
book
.
Metadata
.
String
())
if
files
,
err
:=
db
.
GetBookFiles
(
book
.
Id
);
err
==
nil
{
for
_
,
f
:=
range
files
{
fmt
.
Printf
(
" %s: %s
\n
"
,
strings
.
TrimPrefix
(
f
.
FileType
,
"."
),
f
.
Path
)
}
}
fmt
.
Printf
(
"
\n
"
)
}
}
}
...
...
@@ -213,6 +223,7 @@ func main() {
if
err
==
nil
{
defer
logf
.
Close
()
log
.
SetOutput
(
logf
)
log
.
SetFlags
(
log
.
Ldate
|
log
.
Ltime
)
}
doUpdate
(
db
,
expandTilde
(
*
bookDir
))
...
...
database.go
View file @
f570971f
...
...
@@ -17,8 +17,9 @@ import (
)
var
(
BookBucket
=
[]
byte
(
"ebook"
)
FileBucket
=
[]
byte
(
"file"
)
BookBucket
=
[]
byte
(
"ebook"
)
FileBucket
=
[]
byte
(
"file"
)
BookFileBucket
=
[]
byte
(
"ebook_file"
)
keySeparator
=
byte
(
'/'
)
)
...
...
@@ -84,10 +85,9 @@ func defaultIndexMapping() *bleve.IndexMapping {
}
type
Book
struct
{
Id
BookId
Path
string
Id
BookId
//
Path string
CoverPath
string
FileType
string
Metadata
*
Metadata
}
...
...
@@ -96,11 +96,12 @@ func (b *Book) Type() string {
}
type
File
struct
{
Path
string
Mtime
time
.
Time
Size
int64
Error
bool
Id
BookId
Path
string
FileType
string
Mtime
time
.
Time
Size
int64
Error
bool
Id
BookId
}
func
(
f
*
File
)
HasChanged
(
info
os
.
FileInfo
)
bool
{
...
...
@@ -135,6 +136,7 @@ func NewDb(path string) (*Database, error) {
}
func
(
db
*
Database
)
setupLevelDb
(
path
string
)
error
{
// Use 256MB of cache and a small Bloom filter.
opts
:=
levigo
.
NewOptions
()
db
.
leveldbCache
=
levigo
.
NewLRUCache
(
2
<<
28
)
opts
.
SetCache
(
db
.
leveldbCache
)
...
...
@@ -185,6 +187,24 @@ func (db *Database) GetFile(path string) (*File, error) {
return
&
f
,
nil
}
func
(
db
*
Database
)
GetBookFiles
(
bookid
BookId
)
([]
*
File
,
error
)
{
ro
:=
levigo
.
NewReadOptions
()
defer
ro
.
Close
()
it
:=
db
.
leveldb
.
NewIterator
(
ro
)
defer
it
.
Close
()
start
,
end
:=
keyRange
(
bktToKey
(
BookFileBucket
,
bookid
.
Key
()))
var
out
[]
*
File
for
it
.
Seek
(
start
);
it
.
Valid
()
&&
bytes
.
Compare
(
it
.
Key
(),
end
)
<
0
;
it
.
Next
()
{
var
filepath
string
if
json
.
Unmarshal
(
it
.
Value
(),
&
filepath
)
==
nil
{
if
file
,
err
:=
db
.
GetFile
(
filepath
);
err
==
nil
{
out
=
append
(
out
,
file
)
}
}
}
return
out
,
nil
}
func
(
db
*
Database
)
Get
(
bucket
,
key
[]
byte
,
obj
interface
{})
error
{
ro
:=
levigo
.
NewReadOptions
()
defer
ro
.
Close
()
...
...
@@ -202,8 +222,18 @@ func (db *Database) PutBook(b *Book) error {
return
db
.
index
.
Index
(
b
.
Id
.
String
(),
b
.
Metadata
)
}
func
fileBookKey
(
path
string
,
bookid
BookId
)
[]
byte
{
return
bytes
.
Join
([][]
byte
{
bookid
.
Key
(),
[]
byte
(
path
)},
[]
byte
{
keySeparator
})
}
func
(
db
*
Database
)
PutFile
(
f
*
File
)
error
{
return
db
.
Put
(
FileBucket
,
[]
byte
(
f
.
Path
),
f
)
if
err
:=
db
.
Put
(
FileBucket
,
[]
byte
(
f
.
Path
),
f
);
err
!=
nil
{
return
err
}
if
!
f
.
Error
{
return
db
.
Put
(
BookFileBucket
,
fileBookKey
(
f
.
Path
,
f
.
Id
),
f
.
Path
)
}
return
nil
}
func
(
db
*
Database
)
Put
(
bucket
,
key
[]
byte
,
obj
interface
{})
error
{
...
...
@@ -223,6 +253,23 @@ func (db *Database) DeleteBook(bookid BookId) error {
return
db
.
index
.
Delete
(
bookid
.
String
())
}
func
(
db
*
Database
)
DeleteFile
(
path
string
)
error
{
f
,
err
:=
db
.
GetFile
(
path
)
if
err
!=
nil
{
return
nil
}
db
.
Delete
(
FileBucket
,
[]
byte
(
path
))
db
.
Delete
(
BookFileBucket
,
fileBookKey
(
path
,
f
.
Id
))
// Delete the book if there are no files left.
if
files
,
err
:=
db
.
GetBookFiles
(
f
.
Id
);
err
==
nil
&&
len
(
files
)
==
0
{
db
.
DeleteBook
(
f
.
Id
)
}
return
nil
}
func
(
db
*
Database
)
Delete
(
bucket
,
key
[]
byte
)
error
{
wo
:=
levigo
.
NewWriteOptions
()
defer
wo
.
Close
()
...
...
database_test.go
View file @
f570971f
package
liber
import
(
"io"
"io/ioutil"
"os"
"testing"
)
var
testdbdir
=
"/tmp/liber-test-database"
type
testDatabase
struct
{
db
*
Database
path
string
db
*
Database
path
string
refbookid
BookId
}
func
(
td
*
testDatabase
)
Close
()
{
...
...
@@ -29,15 +29,28 @@ func newTestDatabase(t *testing.T) (*testDatabase, *Database) {
if
err
=
db
.
PutBook
(
book
);
err
!=
nil
{
t
.
Fatalf
(
"PutBook(): %v"
,
err
)
}
if
err
=
db
.
PutFile
(
testEpubFile
(
path
,
book
.
Id
));
err
!=
nil
{
t
.
Fatalf
(
"PutFile(): %v"
,
err
)
}
return
&
testDatabase
{
db
:
db
,
path
:
path
},
db
return
&
testDatabase
{
db
:
db
,
path
:
path
,
refbookid
:
book
.
Id
},
db
}
func
testEpubFile
(
dir
string
,
bookid
BookId
)
*
File
{
f
,
_
:=
ioutil
.
TempFile
(
dir
,
"ebook-"
)
io
.
WriteString
(
f
,
"epub
\n
"
)
f
.
Close
()
return
&
File
{
Id
:
bookid
,
Path
:
f
.
Name
(),
FileType
:
".epub"
,
Size
:
4
,
}
}
func
testEbook
()
*
Book
{
return
&
Book
{
Id
:
NewID
(),
Path
:
"/path/to/ebook"
,
FileType
:
".epub"
,
Id
:
NewID
(),
Metadata
:
&
Metadata
{
Title
:
"20,000 Leagues under the sea"
,
Creator
:
[]
string
{
"Jules Verne"
},
...
...
@@ -46,6 +59,55 @@ func testEbook() *Book {
}
}
func
TestDatabase_Get
(
t
*
testing
.
T
)
{
td
,
db
:=
newTestDatabase
(
t
)
defer
td
.
Close
()
_
,
err
:=
db
.
GetBook
(
td
.
refbookid
)
if
err
!=
nil
{
t
.
Fatalf
(
"GetBook(%d): %v"
,
td
.
refbookid
,
err
)
}
files
,
err
:=
db
.
GetBookFiles
(
td
.
refbookid
)
if
err
!=
nil
{
t
.
Fatalf
(
"GetBookFiles(%d): %v"
,
td
.
refbookid
,
err
)
}
if
len
(
files
)
!=
1
{
t
.
Fatalf
(
"GetBookFiles(%d) bad result: %v"
,
td
.
refbookid
,
files
)
}
}
func
TestDatabase_BookFileRelation
(
t
*
testing
.
T
)
{
td
,
db
:=
newTestDatabase
(
t
)
defer
td
.
Close
()
checkFiles
:=
func
(
tag
string
,
n
int
)
[]
*
File
{
files
,
err
:=
db
.
GetBookFiles
(
td
.
refbookid
)
if
err
!=
nil
{
t
.
Fatalf
(
"GetBookFiles@%s(%d): %v"
,
tag
,
td
.
refbookid
,
err
)
}
if
len
(
files
)
!=
n
{
t
.
Fatalf
(
"GetBookFiles@%s(%d) bad result (exp. len=%d): %v"
,
tag
,
td
.
refbookid
,
n
,
files
)
}
return
files
}
files
:=
checkFiles
(
"init"
,
1
)
file0
:=
files
[
0
]
file1
:=
testEpubFile
(
td
.
path
,
td
.
refbookid
)
db
.
PutFile
(
file1
)
checkFiles
(
"post_add"
,
2
)
db
.
DeleteFile
(
file1
.
Path
)
checkFiles
(
"post_delete"
,
1
)
db
.
DeleteFile
(
file0
.
Path
)
checkFiles
(
"post_delete_2"
,
0
)
if
_
,
err
:=
db
.
GetBook
(
td
.
refbookid
);
err
==
nil
{
t
.
Fatal
(
"Book was not removed when n.files==0"
)
}
}
func
TestDatabase_Search
(
t
*
testing
.
T
)
{
td
,
db
:=
newTestDatabase
(
t
)
defer
td
.
Close
()
...
...
files.go
View file @
f570971f
...
...
@@ -48,3 +48,14 @@ func (s *FileStorage) Open(path string) (*os.File, error) {
}
return
os
.
Open
(
filepath
.
Join
(
s
.
Root
,
path
))
}
// Rename oldpath to newpath.
func
(
s
*
FileStorage
)
Rename
(
oldpath
,
newpath
string
)
error
{
if
!
strings
.
HasPrefix
(
oldpath
,
"/"
)
{
oldpath
=
filepath
.
Join
(
s
.
Root
,
oldpath
)
}
if
!
strings
.
HasPrefix
(
newpath
,
"/"
)
{
newpath
=
filepath
.
Join
(
s
.
Root
,
newpath
)
}
return
os
.
Rename
(
oldpath
,
newpath
)
}
htdocs/templates/book.html
View file @
f570971f
...
...
@@ -32,12 +32,14 @@
<p>
{{.Book.Metadata.Description}}
</p>
{{end}}
{{range $i, $f := .Files}}
<p>
<a
class=
"btn btn-large btn-primary"
href=
"/dl/{{
.Book.Id
}}"
>
<a
class=
"btn btn-large btn-primary"
href=
"/dl/{{
$f.Id}}/{{$i
}}"
>
<span
class=
"glyphicon glyphicon-download-alt"
></span>
Download
Download
({{$f.FileType}})
</a>
</p>
{{end}}
</div>
...
...
metadata.go
View file @
f570971f
...
...
@@ -244,7 +244,7 @@ func parseAnything(filename string) (*Metadata, error) {
},
nil
}
func
Parse
(
filename
string
)
(
*
Book
,
error
)
{
func
Parse
(
filename
string
)
(
*
Book
,
string
,
error
)
{
var
m
*
Metadata
var
err
error
ext
:=
strings
.
ToLower
(
filepath
.
Ext
(
filename
))
...
...
@@ -256,16 +256,11 @@ func Parse(filename string) (*Book, error) {
case
".pdf"
:
m
,
err
=
parseAnything
(
filename
)
default
:
return
nil
,
errors
.
New
(
"unsupported file format"
)
return
nil
,
""
,
errors
.
New
(
"unsupported file format"
)
}
if
err
!=
nil
{
return
nil
,
err
return
nil
,
""
,
err
}
b
:=
&
Book
{
Path
:
filename
,
FileType
:
ext
,
Metadata
:
m
,
}
return
b
,
nil
return
&
Book
{
Metadata
:
m
},
ext
,
nil
}
sync.go
View file @
f570971f
...
...
@@ -11,6 +11,7 @@ import (
"os"
"path/filepath"
"sync"
"time"
)
const
(
...
...
@@ -20,7 +21,7 @@ const (
type
SyncClient
interface
{
DiffRequest
(
*
diffRequest
)
(
*
diffResponse
,
error
)
SendBook
(
*
Book
)
error
SendBook
(
*
Book
,
[]
*
File
)
error
}
type
remoteServer
struct
{
...
...
@@ -64,14 +65,14 @@ func (r *remoteServer) DiffRequest(diffreq *diffRequest) (*diffResponse, error)
return
&
diffresp
,
nil
}
func
addFilePart
(
w
*
multipart
.
Writer
,
varname
,
filename
string
)
error
{
func
addFilePart
(
w
*
multipart
.
Writer
,
varname
,
filename
,
mimeFilename
string
)
error
{
file
,
err
:=
os
.
Open
(
filename
)
if
err
!=
nil
{
return
err
}
defer
file
.
Close
()
part
,
err
:=
w
.
CreateFormFile
(
varname
,
filepath
.
Base
(
filename
)
)
part
,
err
:=
w
.
CreateFormFile
(
varname
,
mimeFilename
)
if
err
!=
nil
{
return
err
}
...
...
@@ -83,25 +84,29 @@ func addFilePart(w *multipart.Writer, varname, filename string) error {
}
// SendBook uploads a book to the remote server.
func
(
r
*
remoteServer
)
SendBook
(
book
*
Book
)
error
{
func
(
r
*
remoteServer
)
SendBook
(
book
*
Book
,
files
[]
*
File
)
error
{
// Create a multipart request with the JSON-encoded metadata
// and the actual file contents as two separate mime/multipart
// sections.
var
body
bytes
.
Buffer
w
:=
multipart
.
NewWriter
(
&
body
)
w
.
WriteField
(
"type"
,
book
.
FileType
)
part
,
err
:=
w
.
CreateFormFile
(
"meta"
,
"meta.json"
)
if
err
:=
json
.
NewEncoder
(
part
)
.
Encode
(
book
.
Metadata
);
err
!=
nil
{
return
err
}
if
err
:=
addFilePart
(
w
,
"book"
,
book
.
Path
);
err
!=
nil
{
w
.
Close
()
return
err
for
i
,
f
:=
range
files
{
varname
:=
fmt
.
Sprintf
(
"book%d"
,
i
)
filename
:=
fmt
.
Sprintf
(
"%d%s"
,
book
.
Id
,
f
.
FileType
)
if
err
:=
addFilePart
(
w
,
varname
,
f
.
Path
,
filename
);
err
!=
nil
{
w
.
Close
()
return
err
}
}
if
book
.
CoverPath
!=
""
{
if
err
:=
addFilePart
(
w
,
"cover"
,
book
.
CoverPath
);
err
!=
nil
{
if
err
:=
addFilePart
(
w
,
"cover"
,
book
.
CoverPath
,
"cover.jpg"
);
err
!=
nil
{
w
.
Close
()
return
err
}
...
...
@@ -195,9 +200,12 @@ func (db *Database) Sync(remote SyncClient) error {
wg
.
Add
(
1
)
go
func
()
{
for
id
:=
range
ch
{
if
book
,
err
:=
db
.
GetBook
(
ParseID
(
id
));
err
==
nil
{
if
err
:=
remote
.
SendBook
(
book
);
err
!=
nil
{
log
.
Printf
(
"SendBook(%d): %v"
,
id
,
err
)
bookid
:=
ParseID
(
id
)
if
book
,
err
:=
db
.
GetBook
(
bookid
);
err
==
nil
{
if
files
,
err
:=
db
.
GetBookFiles
(
bookid
);
err
==
nil
{
if
err
:=
remote
.
SendBook
(
book
,
files
);
err
!=
nil
{
log
.
Printf
(
"SendBook(%d): %v"
,
id
,
err
)
}
}
}
}
...
...
@@ -251,13 +259,6 @@ func (l *syncServer) handleDiffRequest(w http.ResponseWriter, req *http.Request)
}
func
(
l
*
syncServer
)
handleSyncUpload
(
w
http
.
ResponseWriter
,
req
*
http
.
Request
)
{
filetype
:=
req
.
FormValue
(
"type"
)
if
filetype
==
""
{
log
.
Printf
(
"request with no 'type' field"
)
http
.
Error
(
w
,
"Bad request"
,
http
.
StatusBadRequest
)
return
}
mf
,
_
,
err
:=
req
.
FormFile
(
"meta"
)
if
err
!=
nil
{
log
.
Printf
(
"request with no 'meta' field"
)
...
...
@@ -284,21 +285,46 @@ func (l *syncServer) handleSyncUpload(w http.ResponseWriter, req *http.Request)
book
:=
&
Book
{
Id
:
bookid
,
Metadata
:
&
md
,
FileType
:
filetype
,
Path
:
l
.
storage
.
Path
(
fmt
.
Sprintf
(
"%d%s"
,
bookid
,
filetype
)),
}
// Store the data into a custom path and save the book into
// the local database.
if
err
:=
savePart
(
req
,
"book"
,
l
.
storage
,
book
.
Path
);
err
!=
nil
{
log
.
Printf
(
"error saving local file: %v"
,
err
)
http
.
Error
(
w
,
err
.
Error
(),
http
.
StatusInternalServerError
)
return
// Save the file data to our local storage.
for
i
:=
0
;
i
<
10
;
i
++
{
// Use a temporary file, we'll know the right
// extension to use only after having parsed the
// file's MIME header.
tmppath
:=
l
.
storage
.
Path
(
fmt
.
Sprintf
(
"%d.%d.tmp"
,
bookid
,
i
))
varname
:=
fmt
.
Sprintf
(
"book%d"
,
i
)
size
,
hdr
,
err
:=
savePart
(
req
,
varname
,
l
.
storage
,
tmppath
)
if
err
==
http
.
ErrMissingFile
{
break
}
else
if
err
!=
nil
{
log
.
Printf
(
"error saving local file: %v"
,
err
)
http
.
Error
(
w
,
err
.
Error
(),
http
.
StatusInternalServerError
)
return
}
filetype
:=
filepath
.
Ext
(
hdr
.
Filename
)
path
:=
l
.
storage
.
Path
(
fmt
.
Sprintf
(
"%d.%d%s"
,
bookid
,
i
,
filetype
))
if
err
:=
l
.
storage
.
Rename
(
tmppath
,
path
);
err
!=
nil
{
log
.
Printf
(
"error moving local file: %v"
,
err
)
}
file
:=
&
File
{
Path
:
path
,
FileType
:
filetype
,
Mtime
:
time
.
Now
(),
Size
:
size
,
Id
:
bookid
,
}
if
err
:=
l
.
db
.
PutFile
(
file
);
err
!=
nil
{
log
.
Printf
(
"error saving file to the database: %v"
,
err
)
http
.
Error
(
w
,
err
.
Error
(),
http
.
StatusInternalServerError
)
return
}
}
// If the request contains a cover image, save that as well.
coverPath
:=
l
.
storage
.
Path
(
fmt
.
Sprintf
(
"%s
%s.cover.png"
,
book
.
Id
,
filetype
))
if
err
:=
savePart
(
req
,
"cover"
,
l
.
storage
,
coverPath
);
err
==
nil
{
coverPath
:=
l
.
storage
.
Path
(
fmt
.
Sprintf
(
"%s
.cover.png"
,
book
.
Id
))
if
_
,
_
,
err
:=
savePart
(
req
,
"cover"
,
l
.
storage
,
coverPath
);
err
==
nil
{
book
.
CoverPath
=
coverPath
}
...
...
@@ -312,21 +338,22 @@ func (l *syncServer) handleSyncUpload(w http.ResponseWriter, req *http.Request)
w
.
WriteHeader
(
200
)
}
func
savePart
(
req
*
http
.
Request
,
fieldname
string
,
storage
*
FileStorage
,
outname
string
)
error
{
f
,
_
,
err
:=
req
.
FormFile
(
fieldname
)
func
savePart
(
req
*
http
.
Request
,
fieldname
string
,
storage
*
FileStorage
,
outname
string
)
(
int64
,
*
multipart
.
FileHeader
,
error
)
{
f
,
hdr
,
err
:=
req
.
FormFile
(
fieldname
)
if
err
!=
nil
{
return
err
return
0
,
nil
,
err
}
outf
,
err
:=
storage
.
Create
(
outname
)
if
err
!=
nil
{
return
err
return
0
,
nil
,
err
}
defer
outf
.
Close
()
if
_
,
err
:=
io
.
Copy
(
outf
,
f
);
err
!=
nil
{
return
err
n
,
err
:=
io
.
Copy
(
outf
,
f
)
if
err
!=
nil
{
return
0
,
nil
,
err
}
return
nil
return
n
,
hdr
,
n
il
}
sync_test.go
View file @
f570971f
...
...
@@ -2,7 +2,6 @@ package liber
import
(
"fmt"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
...
...
@@ -11,7 +10,7 @@ import (
"testing"
)
func
newTestHttpServer
(
db
*
Database
,
updir
string
)
*
httptest
.
Server
{
func
newTest
Sync
HttpServer
(
db
*
Database
,
updir
string
)
*
httptest
.
Server
{
localsrv
:=
&
syncServer
{
db
,
&
FileStorage
{
Root
:
updir
,
Nesting
:
2
}}
mux
:=
http
.
NewServeMux
()
...
...
@@ -22,11 +21,6 @@ func newTestHttpServer(db *Database, updir string) *httptest.Server {
}
func
TestSync_Sync
(
t
*
testing
.
T
)
{
// Actually create a file to upload, or the sync will fail.
f
,
_
:=
ioutil
.
TempFile
(
""
,
"ebook-"
)
io
.
WriteString
(
f
,
"foo
\n
"
)
f
.
Close
()
// Create a temporary directory to store uploads.
updir
,
_
:=
ioutil
.
TempDir
(
""
,
"ebook-upload-"
)
defer
os
.
RemoveAll
(
updir
)
...
...
@@ -38,20 +32,20 @@ func TestSync_Sync(t *testing.T) {
defer
td2
.
Close
()
for
i
:=
0
;
i
<
10
;
i
++
{
bookid
:=
NewID
()
db
.
PutBook
(
&
Book
{
Id
:
NewID
(),
Path
:
f
.
Name
(),
FileType
:
".epub"
,
Id
:
bookid
,
Metadata
:
&
Metadata
{
Title
:
fmt
.
Sprintf
(
"Book #%d"
,
i
+
1
),
Creator
:
[]
string
{
"Random Author"
},
ISBN
:
[]
string
{
strconv
.
Itoa
(
i
+
1
)},
},
})
db
.
PutFile
(
testEpubFile
(
updir
,
bookid
))
}
// Run a sync from db to db2.
srv
:=
newTestHttpServer
(
db2
,
updir
)
srv
:=
newTest
Sync
HttpServer
(
db2
,
updir
)
defer
srv
.
Close
()
cl
:=
NewRemoteServer
(
srv
.
URL
)
...
...
update.go
View file @
f570971f
...
...
@@ -17,22 +17,29 @@ const (
type
MetadataChooserFunc
func
(
string
,
[]
*
Metadata
)
*
Metadata
type
fileData
struct
{
source
int
path
string
id
BookId
info
os
.
FileInfo
source
int
path
string
filetype
string
id
BookId
info
os
.
FileInfo
}
func
(
f
fileData
)
toLiberFile
(
haserr
bool
)
*
File
{
return
&
File
{
Path
:
f
.
path
,
Mtime
:
f
.
info
.
ModTime
(),
Size
:
f
.
info
.
Size
(),
Id
:
f
.
id
,
Error
:
haserr
,
Path
:
f
.
path
,
FileType
:
f
.
filetype
,
Mtime
:
f
.
info
.
ModTime
(),
Size
:
f
.
info
.
Size
(),
Id
:
f
.
id
,
Error
:
haserr
,
}
}
type
fileAndBook
struct
{
f
fileData
b
*
Book
}
func
dbFileScanner
(
db
*
Database
,
fileCh
chan
fileData
)
{
for
iter
:=
db
.
Scan
(
FileBucket
);
iter
.
Valid
();
iter
.
Next
()
{
var
f
File
...
...
@@ -83,34 +90,28 @@ func differ(db *Database, basedir string) chan fileData {
close
(
fileCh
)
}()
go
func
()
{
//
Detect files that have not changed, i.e. appear in
//
the database and the filesystem. Keep track of book
//
IDs so that once all entries have been processed we
//
can delete those books from the database where the
//
original file has been removed
.
//
Merge the two sources and keep track of files that
//
only appear in the database but not on the
//
filesystem, so we can remove them at the end.
//
All entries with source == SourceFS will be sent to
//
the output channel in any case
.
allSources
:=
SourceDB
|
SourceFS
tmp
:=
make
(
map
[
string
]
int
)
ids
:=
make
(
map
[
string
]
BookId
)
for
f
:=
range
fileCh
{
// log.Printf("differ: %#v", f)
tmp
[
f
.
path
]
|=
f
.
source
// Delete entries as soon as we've seen them
// from both sources.
//
originate
from both sources.
if
tmp
[
f
.
path
]
==
allSources
{
// log.Printf("differ: dropping %s", f.path)
delete
(
tmp
,
f
.
path
)
delete
(
ids
,
f
.
path
)
}
if
f
.
source
==
SourceFS
{