Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
7
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
ale
liber
Commits
30145e43
Commit
30145e43
authored
Nov 16, 2014
by
ale
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
use relative paths to a FileStorage wherever possible
parent
9927da5b
Changes
7
Hide whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
162 additions
and
71 deletions
+162
-71
files.go
files.go
+73
-29
sync.go
sync.go
+2
-2
sync_test.go
sync_test.go
+1
-1
update.go
update.go
+52
-35
update_test.go
update_test.go
+30
-0
web.go
web.go
+3
-3
web_test.go
web_test.go
+1
-1
No files found.
files.go
View file @
30145e43
package
liber
import
(
"fmt"
"os"
"path/filepath"
"strings"
"git.autistici.org/ale/liber/util"
)
// FileStorage exposes a read-only filesystem hierarchy as a root for
// relative paths (so that you can move archives around while the
// database is still valid). Calls will still accept absolute paths
// for backwards compatibility.
type
FileStorage
struct
{
Root
string
Nesting
int
Root
string
}
func
NewFileStorage
(
root
string
,
nesting
int
)
*
FileStorage
{
func
NewFileStorage
(
root
string
)
*
FileStorage
{
return
&
FileStorage
{
Root
:
root
,
Nesting
:
nesting
,
Root
:
root
,
}
}
// Path of the file corresponding to the given key, relative to the
// root directory.
func
(
s
*
FileStorage
)
Path
(
key
string
)
string
{
var
parts
[]
string
for
i
:=
0
;
i
<
s
.
Nesting
;
i
++
{
if
i
>=
len
(
key
)
{
break
}
parts
=
append
(
parts
,
key
[
i
:
i
+
1
])
// Return the absolute path of a file, given its relative path.
func
(
s
*
FileStorage
)
Abs
(
path
string
)
string
{
if
strings
.
HasPrefix
(
path
,
"/"
)
{
return
path
}
parts
=
append
(
parts
,
key
)
return
filepath
.
Join
(
parts
...
)
return
filepath
.
Join
(
s
.
Root
,
path
)
}
// Return the relative path of a file with respect to the storage
// root.
func
(
s
*
FileStorage
)
Rel
(
abspath
string
)
(
string
,
error
)
{
return
filepath
.
Rel
(
s
.
Root
,
abspath
)
}
// Create a new file for the given key.
// Create a new file for the given key. Directories containing the
// output file will be automatically created.
func
(
s
*
FileStorage
)
Create
(
path
string
)
(
*
os
.
File
,
error
)
{
p
:=
filepath
.
Join
(
s
.
Root
,
path
)
if
err
:=
os
.
MkdirAll
(
filepath
.
Dir
(
p
),
0700
);
err
!=
nil
{
p
ath
=
s
.
Abs
(
path
)
if
err
:=
os
.
MkdirAll
(
filepath
.
Dir
(
p
ath
),
0700
);
err
!=
nil
{
return
nil
,
err
}
return
os
.
Create
(
p
)
return
os
.
Create
(
path
)
}
// Exists returns true if the specified file exists.
func
(
s
*
FileStorage
)
Exists
(
path
string
)
bool
{
_
,
err
:=
os
.
Stat
(
s
.
Abs
(
path
))
return
err
==
nil
}
// Open a file.
func
(
s
*
FileStorage
)
Open
(
path
string
)
(
*
os
.
File
,
error
)
{
if
strings
.
HasPrefix
(
path
,
"/"
)
{
return
os
.
Open
(
path
)
}
return
os
.
Open
(
filepath
.
Join
(
s
.
Root
,
path
))
return
os
.
Open
(
s
.
Abs
(
path
))
}
// Rename oldpath to newpath.
func
(
s
*
FileStorage
)
Rename
(
oldpath
,
newpath
string
)
error
{
if
!
strings
.
HasPrefix
(
oldpath
,
"/"
)
{
oldpath
=
filepath
.
Join
(
s
.
Root
,
oldpath
)
return
os
.
Rename
(
s
.
Abs
(
oldpath
),
s
.
Abs
(
newpath
))
}
func
(
s
*
FileStorage
)
Walk
(
w
*
util
.
Walker
,
fn
filepath
.
WalkFunc
)
{
w
.
Walk
(
s
.
Root
,
func
(
path
string
,
info
os
.
FileInfo
,
ferr
error
)
error
{
if
ferr
!=
nil
{
return
nil
}
relpath
,
err
:=
s
.
Rel
(
path
)
if
err
!=
nil
{
return
fmt
.
Errorf
(
"%s is outside %s (?)"
,
path
,
s
.
Root
)
}
return
fn
(
relpath
,
info
,
nil
)
})
}
// RWFileStorage adds a read-write API on top of a FileStorage, based
// on unique keys and directory sharding.
type
RWFileStorage
struct
{
*
FileStorage
Nesting
int
}
func
NewRWFileStorage
(
root
string
,
nesting
int
)
*
RWFileStorage
{
return
&
RWFileStorage
{
FileStorage
:
NewFileStorage
(
root
),
Nesting
:
nesting
,
}
if
!
strings
.
HasPrefix
(
newpath
,
"/"
)
{
newpath
=
filepath
.
Join
(
s
.
Root
,
newpath
)
}
// Path of the file corresponding to the given key, relative to the
// root directory.
func
(
s
*
RWFileStorage
)
Path
(
key
string
)
string
{
var
parts
[]
string
for
i
:=
0
;
i
<
s
.
Nesting
;
i
++
{
if
i
>=
len
(
key
)
{
break
}
parts
=
append
(
parts
,
key
[
i
:
i
+
1
])
}
return
os
.
Rename
(
oldpath
,
newpath
)
parts
=
append
(
parts
,
key
)
return
filepath
.
Join
(
parts
...
)
}
sync.go
View file @
30145e43
...
...
@@ -247,7 +247,7 @@ func (db *Database) Sync(remote SyncClient) error {
type
syncServer
struct
{
db
*
Database
storage
*
FileStorage
storage
*
RW
FileStorage
}
func
(
l
*
syncServer
)
handleDiffRequest
(
w
http
.
ResponseWriter
,
req
*
http
.
Request
)
{
...
...
@@ -355,7 +355,7 @@ func (l *syncServer) handleSyncUpload(w http.ResponseWriter, req *http.Request)
w
.
WriteHeader
(
200
)
}
func
savePart
(
req
*
http
.
Request
,
fieldname
string
,
storage
*
FileStorage
,
outname
string
)
(
int64
,
*
multipart
.
FileHeader
,
error
)
{
func
savePart
(
req
*
http
.
Request
,
fieldname
string
,
storage
*
RW
FileStorage
,
outname
string
)
(
int64
,
*
multipart
.
FileHeader
,
error
)
{
f
,
hdr
,
err
:=
req
.
FormFile
(
fieldname
)
if
err
!=
nil
{
return
0
,
nil
,
err
...
...
sync_test.go
View file @
30145e43
...
...
@@ -11,7 +11,7 @@ import (
)
func
newTestSyncHttpServer
(
db
*
Database
,
updir
string
)
*
httptest
.
Server
{
localsrv
:=
&
syncServer
{
db
,
&
FileStorage
{
Root
:
updir
,
Nesting
:
2
}
}
localsrv
:=
&
syncServer
{
db
,
NewRW
FileStorage
(
updir
,
2
)
}
mux
:=
http
.
NewServeMux
()
mux
.
HandleFunc
(
"/api/sync/upload"
,
localsrv
.
handleSyncUpload
)
...
...
update.go
View file @
30145e43
...
...
@@ -19,12 +19,13 @@ type MetadataChooserFunc func(string, []*Metadata) *Metadata
type
fileData
struct
{
source
int
path
string
relpath
string
filetype
string
id
BookId
info
os
.
FileInfo
}
func
(
f
fileData
)
toLiberFile
(
haserr
bool
)
*
File
{
func
(
f
fileData
)
toLiberFile
(
storage
*
FileStorage
,
haserr
bool
)
(
*
File
,
error
)
{
return
&
File
{
Path
:
f
.
path
,
FileType
:
f
.
filetype
,
...
...
@@ -32,7 +33,7 @@ func (f fileData) toLiberFile(haserr bool) *File {
Size
:
f
.
info
.
Size
(),
Id
:
f
.
id
,
Error
:
haserr
,
}
}
,
nil
}
type
fileAndBook
struct
{
...
...
@@ -40,8 +41,14 @@ type fileAndBook struct {
b
*
Book
}
func
dbFileScanner
(
db
*
Database
,
fileCh
chan
fileData
)
{
for
iter
:=
db
.
Scan
(
FileBucket
);
iter
.
Valid
();
iter
.
Next
()
{
type
updateContext
struct
{
db
*
Database
storage
*
FileStorage
chooser
MetadataChooserFunc
}
func
(
uc
*
updateContext
)
dbFileScanner
(
fileCh
chan
fileData
)
{
for
iter
:=
uc
.
db
.
Scan
(
FileBucket
);
iter
.
Valid
();
iter
.
Next
()
{
var
f
File
if
err
:=
iter
.
Value
(
&
f
);
err
!=
nil
{
continue
...
...
@@ -54,12 +61,8 @@ func dbFileScanner(db *Database, fileCh chan fileData) {
}
}
func
localFileScanner
(
db
*
Database
,
basedir
string
,
fileCh
chan
fileData
)
{
util
.
NewDefaultWalker
()
.
Walk
(
basedir
,
func
(
path
string
,
info
os
.
FileInfo
,
err
error
)
error
{
if
err
!=
nil
{
return
nil
}
func
(
uc
*
updateContext
)
localFileScanner
(
basedir
string
,
fileCh
chan
fileData
)
{
uc
.
storage
.
Walk
(
util
.
NewDefaultWalker
(),
func
(
path
string
,
info
os
.
FileInfo
,
err
error
)
error
{
fileCh
<-
fileData
{
source
:
SourceFS
,
path
:
path
,
...
...
@@ -69,7 +72,7 @@ func localFileScanner(db *Database, basedir string, fileCh chan fileData) {
})
}
func
differ
(
db
*
Database
,
basedir
string
)
chan
fileData
{
func
(
uc
*
updateContext
)
differ
(
basedir
string
)
chan
fileData
{
fileCh
:=
make
(
chan
fileData
,
100
)
outCh
:=
make
(
chan
fileData
,
100
)
var
wg
sync
.
WaitGroup
...
...
@@ -77,11 +80,11 @@ func differ(db *Database, basedir string) chan fileData {
// Start two sources in parallel and send their output to fileCh.
go
func
()
{
localFileScanner
(
db
,
basedir
,
fileCh
)
uc
.
localFileScanner
(
basedir
,
fileCh
)
wg
.
Done
()
}()
go
func
()
{
dbFileScanner
(
db
,
fileCh
)
uc
.
dbFileScanner
(
fileCh
)
wg
.
Done
()
}()
// Once they are done, close the channel.
...
...
@@ -110,8 +113,8 @@ func differ(db *Database, basedir string) chan fileData {
}
for
path
,
value
:=
range
tmp
{
if
value
==
SourceDB
{
log
.
Printf
(
"
removing file %s
"
,
path
)
db
.
DeleteFile
(
path
)
log
.
Printf
(
"
file %s has been removed
"
,
path
)
uc
.
db
.
DeleteFile
(
path
)
}
}
close
(
outCh
)
...
...
@@ -119,15 +122,15 @@ func differ(db *Database, basedir string) chan fileData {
return
outCh
}
func
extractor
(
db
*
Database
,
chooser
MetadataChooserFunc
,
fileCh
chan
fileData
,
outCh
chan
fileAndBook
)
{
func
(
uc
*
updateContext
)
extractor
(
fileCh
chan
fileData
,
outCh
chan
fileAndBook
)
{
for
f
:=
range
fileCh
{
if
oldfile
,
err
:=
db
.
GetFile
(
f
.
path
);
err
==
nil
{
if
oldfile
,
err
:=
uc
.
db
.
GetFile
(
f
.
path
);
err
==
nil
{
if
!
oldfile
.
HasChanged
(
f
.
info
)
{
continue
}
f
.
id
=
oldfile
.
Id
}
book
,
filetype
,
err
:=
parseMeta
(
f
,
chooser
)
book
,
filetype
,
err
:=
uc
.
parseMeta
(
f
)
if
err
==
nil
{
f
.
filetype
=
filetype
outCh
<-
fileAndBook
{
f
:
f
,
b
:
book
}
...
...
@@ -136,22 +139,26 @@ func extractor(db *Database, chooser MetadataChooserFunc, fileCh chan fileData,
// Parse errors are permanent.
log
.
Printf
(
"Could not parse %s: %v"
,
f
.
path
,
err
)
file
:=
f
.
toLiberFile
(
true
)
if
err
:=
db
.
PutFile
(
file
);
err
!=
nil
{
file
,
err
:=
f
.
toLiberFile
(
uc
.
storage
,
true
)
if
err
!=
nil
{
log
.
Printf
(
"Error saving file %s: %v"
,
file
.
Path
,
err
)
continue
}
if
err
:=
uc
.
db
.
PutFile
(
file
);
err
!=
nil
{
log
.
Printf
(
"Error saving file %s to db: %v"
,
file
.
Path
,
err
)
}
}
}
func
parseMeta
(
f
fileData
,
chooser
MetadataChooserFunc
)
(
*
Book
,
string
,
error
)
{
func
(
uc
*
updateContext
)
parseMeta
(
f
fileData
)
(
*
Book
,
string
,
error
)
{
// Attempt direct metadata extraction.
book
,
filetype
,
err
:=
Parse
(
f
.
path
)
book
,
filetype
,
err
:=
Parse
(
uc
.
storage
.
Abs
(
f
.
path
)
)
if
err
!=
nil
{
return
nil
,
""
,
err
}
// Check if a Calibre OPF file exists.
if
opfmeta
,
err
:=
opfOpen
(
opfMetadataPath
(
f
.
path
));
err
==
nil
{
if
opfmeta
,
err
:=
opfOpen
(
opfMetadataPath
(
uc
.
storage
.
Abs
(
f
.
path
))
)
;
err
==
nil
{
book
.
Metadata
.
Merge
(
opfmeta
)
}
else
{
// No local metadata, use Google Books to retrieve
...
...
@@ -162,8 +169,8 @@ func parseMeta(f fileData, chooser MetadataChooserFunc) (*Book, string, error) {
if
len
(
candidates
)
==
1
{
log
.
Printf
(
"found Google Books match: %s"
,
candidates
[
0
]
.
String
())
book
.
Metadata
.
Merge
(
candidates
[
0
])
}
else
{
if
userchoice
:=
chooser
(
f
.
path
,
candidates
);
userchoice
!=
nil
{
}
else
if
uc
.
chooser
!=
nil
{
if
userchoice
:=
uc
.
chooser
(
f
.
path
,
candidates
);
userchoice
!=
nil
{
book
.
Metadata
.
Merge
(
userchoice
)
}
}
...
...
@@ -179,11 +186,11 @@ func parseMeta(f fileData, chooser MetadataChooserFunc) (*Book, string, error) {
// Try to find a cover image. Look on the local filesystem
// first, otherwise check Google Books.
localCoverPath
:=
opfCoverPath
(
f
.
path
)
if
_
,
err
:=
os
.
Stat
(
localCoverPath
)
;
err
==
nil
{
if
uc
.
storage
.
Exists
(
localCoverPath
)
{
book
.
CoverPath
=
localCoverPath
}
else
if
imageData
,
err
:=
GetGoogleBooksCover
(
book
.
Metadata
);
err
==
nil
{
imageFileName
:=
f
.
path
+
".cover.png"
if
imgf
,
err
:=
os
.
Create
(
imageFileName
);
err
!=
nil
{
if
imgf
,
err
:=
os
.
Create
(
uc
.
storage
.
Abs
(
imageFileName
)
)
;
err
!=
nil
{
log
.
Printf
(
"Could not save cover image for %d: %v"
,
book
.
Id
,
err
)
}
else
{
imgf
.
Write
(
imageData
)
...
...
@@ -195,7 +202,7 @@ func parseMeta(f fileData, chooser MetadataChooserFunc) (*Book, string, error) {
return
book
,
filetype
,
nil
}
func
dbwriter
(
db
*
Database
,
ch
chan
fileAndBook
)
{
func
(
uc
*
updateContext
)
dbwriter
(
ch
chan
fileAndBook
)
{
for
pair
:=
range
ch
{
saveBook
:=
true
...
...
@@ -203,7 +210,7 @@ func dbwriter(db *Database, ch chan fileAndBook) {
// existing book.
if
pair
.
f
.
id
==
0
{
log
.
Printf
(
"potential new book: %#v"
,
pair
.
b
.
Metadata
)
if
match
,
err
:=
db
.
Find
(
pair
.
b
.
Metadata
.
Uniques
());
err
==
nil
{
if
match
,
err
:=
uc
.
db
.
Find
(
pair
.
b
.
Metadata
.
Uniques
());
err
==
nil
{
log
.
Printf
(
"%s matches existing book %d"
,
pair
.
f
.
path
,
match
.
Id
)
// Ignore new metadata.
pair
.
b
=
match
...
...
@@ -219,15 +226,19 @@ func dbwriter(db *Database, ch chan fileAndBook) {
}
if
saveBook
{
if
err
:=
db
.
PutBook
(
pair
.
b
);
err
!=
nil
{
if
err
:=
uc
.
db
.
PutBook
(
pair
.
b
);
err
!=
nil
{
log
.
Printf
(
"Error saving book %d to db: %v"
,
pair
.
b
.
Id
,
err
)
continue
}
log
.
Printf
(
"%s -> %d"
,
pair
.
f
.
path
,
pair
.
b
.
Id
)
}
file
:=
pair
.
f
.
toLiberFile
(
false
)
if
err
:=
db
.
PutFile
(
file
);
err
!=
nil
{
file
,
err
:=
pair
.
f
.
toLiberFile
(
uc
.
storage
,
false
)
if
err
!=
nil
{
log
.
Printf
(
"Error saving file %s: %v"
,
pair
.
f
.
path
,
err
)
continue
}
if
err
:=
uc
.
db
.
PutFile
(
file
);
err
!=
nil
{
log
.
Printf
(
"Error saving file %s to db: %v"
,
file
.
Path
,
err
)
}
}
...
...
@@ -236,13 +247,19 @@ func dbwriter(db *Database, ch chan fileAndBook) {
func
(
db
*
Database
)
Update
(
dir
string
,
chooser
MetadataChooserFunc
)
{
// Parallelize metadata extraction, serialize database updates
// (so that index-based de-duplication works).
uc
:=
&
updateContext
{
db
:
db
,
chooser
:
chooser
,
storage
:
NewFileStorage
(
dir
),
}
var
wg
sync
.
WaitGroup
ch
:=
differ
(
db
,
dir
)
ch
:=
uc
.
differ
(
dir
)
pch
:=
make
(
chan
fileAndBook
)
for
i
:=
0
;
i
<
10
;
i
++
{
wg
.
Add
(
1
)
go
func
()
{
extractor
(
db
,
chooser
,
ch
,
pch
)
uc
.
extractor
(
ch
,
pch
)
wg
.
Done
()
}()
}
...
...
@@ -250,5 +267,5 @@ func (db *Database) Update(dir string, chooser MetadataChooserFunc) {
wg
.
Wait
()
close
(
pch
)
}()
dbwriter
(
db
,
pch
)
uc
.
dbwriter
(
pch
)
}
update_test.go
View file @
30145e43
...
...
@@ -4,6 +4,7 @@ import (
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"git.autistici.org/ale/liber/util"
...
...
@@ -45,14 +46,43 @@ func TestDatabase_Update(t *testing.T) {
if
_
,
err
:=
db
.
GetBook
(
td
.
refbookid
);
err
==
nil
{
t
.
Errorf
(
"%s: test book still in database"
,
tag
)
}
// Test OPF ebook should have been found by Update.
if
result
,
err
:=
db
.
Search
(
"isbn:9781939293015"
,
0
,
1
);
err
!=
nil
||
result
.
NumResults
!=
1
{
t
.
Errorf
(
"%s: new book not found in database"
,
tag
)
}
}
// The second update should do nothing.
db
.
Update
(
tmpdir
,
chooser
)
testDb
(
"first update"
)
db
.
Update
(
tmpdir
,
chooser
)
testDb
(
"second update"
)
// Check that the test file is there.
if
_
,
err
:=
db
.
GetFile
(
"book/Test Ebook.pdf"
);
err
!=
nil
{
t
.
Errorf
(
"test file is not in the database"
)
}
// Files should have relative paths.
for
i
:=
db
.
Scan
(
FileBucket
);
i
.
Valid
();
i
.
Next
()
{
var
f
File
if
err
:=
i
.
Value
(
&
f
);
err
!=
nil
{
t
.
Fatal
(
err
)
}
if
strings
.
HasPrefix
(
f
.
Path
,
"/"
)
{
t
.
Errorf
(
"file has absolute path: %v"
,
f
.
Path
)
}
}
// Book cover images should have relative paths.
for
i
:=
db
.
Scan
(
BookBucket
);
i
.
Valid
();
i
.
Next
()
{
var
b
Book
if
err
:=
i
.
Value
(
&
b
);
err
!=
nil
{
t
.
Fatal
(
err
)
}
if
b
.
CoverPath
!=
""
&&
strings
.
HasPrefix
(
b
.
CoverPath
,
"/"
)
{
t
.
Errorf
(
"file has absolute path: %v"
,
b
.
CoverPath
)
}
}
}
web.go
View file @
30145e43
...
...
@@ -27,8 +27,8 @@ var (
type
uiServer
struct
{
db
*
Database
storage
*
FileStorage
cache
*
FileStorage
storage
*
RW
FileStorage
cache
*
RW
FileStorage
}
type
pagination
struct
{
...
...
@@ -336,7 +336,7 @@ func handleOpenSearchXml(w http.ResponseWriter, req *http.Request) {
render
(
"opensearch_xml.html"
,
w
,
&
ctx
)
}
func
NewHttpServer
(
db
*
Database
,
storage
,
cache
*
FileStorage
,
addr
string
)
*
http
.
Server
{
func
NewHttpServer
(
db
*
Database
,
storage
,
cache
*
RW
FileStorage
,
addr
string
)
*
http
.
Server
{
var
err
error
tpl
,
err
=
template
.
New
(
"liber"
)
.
Funcs
(
template
.
FuncMap
{
"join"
:
strings
.
Join
,
...
...
web_test.go
View file @
30145e43
...
...
@@ -29,7 +29,7 @@ func newTestHttpServer(t *testing.T) (*testHttpServer, *httptest.Server) {
ts
.
tmpdir
,
_
=
ioutil
.
TempDir
(
""
,
"tmp-storage-"
)
ts
.
td
,
ts
.
db
=
newTestDatabase
(
t
)
tempStorage
:=
NewFileStorage
(
ts
.
tmpdir
,
2
)
tempStorage
:=
New
RW
FileStorage
(
ts
.
tmpdir
,
2
)
server
:=
NewHttpServer
(
ts
.
db
,
tempStorage
,
tempStorage
,
":1234"
)
return
&
ts
,
httptest
.
NewServer
(
server
.
Handler
)
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment