Commit 81da84af authored by ale's avatar ale

implement autocompletion using a separate n-gram index

parent a86cd6dd
......@@ -6,6 +6,7 @@ import (
"encoding/binary"
"encoding/gob"
"errors"
"log"
"math/rand"
"os"
"path/filepath"
......@@ -78,6 +79,7 @@ type flatBook struct {
Description string `json:"description"`
ISBN []string `json:"isbn"`
Unique []string `json:"_unique"`
Suggest []string `json:"_suggest"`
}
func (f *flatBook) Type() string {
......@@ -85,12 +87,17 @@ func (f *flatBook) Type() string {
}
func flatten(book *Book) *flatBook {
suggest := []string{book.Metadata.Title}
if len(book.Metadata.Creator) > 0 {
suggest = append(suggest, book.Metadata.Creator...)
}
return &flatBook{
Title: book.Metadata.Title,
Author: book.Metadata.Creator,
Description: book.Metadata.Description,
ISBN: book.Metadata.ISBN,
Unique: book.Metadata.Uniques(),
Suggest: suggest,
}
}
......@@ -112,18 +119,51 @@ func metadataDocumentMapping() *bleve.DocumentMapping {
keywordFieldMapping.Analyzer = "keyword"
keywordFieldMapping.IncludeInAll = false
suggestFieldMapping := bleve.NewTextFieldMapping()
suggestFieldMapping.Store = false
suggestFieldMapping.Analyzer = "edgeNgram"
suggestFieldMapping.IncludeTermVectors = false
suggestFieldMapping.IncludeInAll = false
md.AddFieldMappingsAt("title", textFieldMapping)
md.AddFieldMappingsAt("author", authorFieldMapping)
md.AddFieldMappingsAt("description", textFieldMapping)
md.AddFieldMappingsAt("isbn", keywordFieldMapping)
md.AddFieldMappingsAt("_unique", keywordFieldMapping)
md.AddFieldMappingsAt("_suggest", suggestFieldMapping)
return md
}
func defaultIndexMapping() *bleve.IndexMapping {
i := bleve.NewIndexMapping()
err := i.AddCustomTokenFilter("edgeNgram325",
map[string]interface{}{
"type": "edge_ngram",
"min": 3.0,
"max": 25.0,
})
if err != nil {
log.Fatal(err)
}
err = i.AddCustomAnalyzer("edgeNgram",
map[string]interface{}{
"type": "custom",
"tokenizer": "unicode",
"token_filters": []string{
"to_lower",
"stop_en",
"edgeNgram325",
},
})
if err != nil {
log.Fatal(err)
}
i.AddDocumentMapping("ebook", metadataDocumentMapping())
i.DefaultAnalyzer = defaultTextAnalyzer
i.DefaultType = "ebook"
return i
......@@ -372,8 +412,9 @@ func (db *Database) Search(queryStr string, offset, limit int) (*SearchResult, e
}
// Autocomplete runs a fuzzy search for a term.
func (db *Database) Autocomplete(term string) (*SearchResult, error) {
return db.doSearch(bleve.NewFuzzyQuery(term), 0, 20)
func (db *Database) Suggest(term string) (*SearchResult, error) {
query := bleve.NewTermQuery(term).SetField("_suggest")
return db.doSearch(query, 0, 20)
}
// Find a book matching the given metadata, if possible.
......
......@@ -229,19 +229,21 @@ func TestDatabase_Find(t *testing.T) {
}
}
func TestDatabase_Autocomplete(t *testing.T) {
func TestDatabase_Suggest(t *testing.T) {
td, db := newTestDatabase(t)
defer td.Close()
r, err := db.Autocomplete("jules")
if err != nil {
t.Fatal(err)
}
if r.NumResults == 0 {
t.Error("No results")
for _, s := range []string{"jul", "jule", "jules", "ver", "vern", "verne", "twent", "thous"} {
r, err := db.Suggest(s)
if err != nil {
t.Fatal(err)
}
if r.NumResults == 0 {
t.Errorf("No results for '%s'", s)
}
}
r, err = db.Autocomplete("foo")
r, err := db.Suggest("foo")
if err != nil {
t.Fatal(err)
}
......
......@@ -100,14 +100,14 @@ type autocompleteResult struct {
Label string `json:"label"`
}
func (s *uiServer) handleAutocomplete(w http.ResponseWriter, req *http.Request) {
func (s *uiServer) handleSuggest(w http.ResponseWriter, req *http.Request) {
term := req.FormValue("term")
if term == "" {
http.Error(w, "No Query", http.StatusBadRequest)
return
}
result, err := s.db.Autocomplete(term)
result, err := s.db.Suggest(term)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
......@@ -359,7 +359,7 @@ func NewHttpServer(db *Database, storage, cache *FileStorage, addr string) *http
r.Handle("/read/{id:[0-9]+}/{fid:[0-9]+}", uisrv.withFile(uisrv.handleReadBook))
r.Handle("/dl/{id:[0-9]+}/{fid:[0-9]+}", uisrv.withFile(uisrv.handleDownloadBook))
r.HandleFunc("/opensearch.xml", handleOpenSearchXml)
r.HandleFunc("/suggest", uisrv.handleAutocomplete)
r.HandleFunc("/suggest", uisrv.handleSuggest)
r.HandleFunc("/search", uisrv.handleSearch)
r.HandleFunc("/", uisrv.handleHome)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment