implement autocompletion using a separate n-gram index

database.go

@@ -6,6 +6,7 @@ import (
 	"encoding/binary"
 	"encoding/gob"
 	"errors"
+	"log"
 	"math/rand"
 	"os"
 	"path/filepath"
@@ -78,6 +79,7 @@ type flatBook struct {
 	Description string   `json:"description"`
 	ISBN        []string `json:"isbn"`
 	Unique      []string `json:"_unique"`
+	Suggest     []string `json:"_suggest"`
 }
 
 func (f *flatBook) Type() string {
@@ -85,12 +87,17 @@ func (f *flatBook) Type() string {
 }
 
 func flatten(book *Book) *flatBook {
+	suggest := []string{book.Metadata.Title}
+	if len(book.Metadata.Creator) > 0 {
+		suggest = append(suggest, book.Metadata.Creator...)
+	}
 	return &flatBook{
 		Title:       book.Metadata.Title,
 		Author:      book.Metadata.Creator,
 		Description: book.Metadata.Description,
 		ISBN:        book.Metadata.ISBN,
 		Unique:      book.Metadata.Uniques(),
+		Suggest:     suggest,
 	}
 }
 
@@ -112,18 +119,51 @@ func metadataDocumentMapping() *bleve.DocumentMapping {
 	keywordFieldMapping.Analyzer = "keyword"
 	keywordFieldMapping.IncludeInAll = false
 
+	suggestFieldMapping := bleve.NewTextFieldMapping()
+	suggestFieldMapping.Store = false
+	suggestFieldMapping.Analyzer = "edgeNgram"
+	suggestFieldMapping.IncludeTermVectors = false
+	suggestFieldMapping.IncludeInAll = false
+
 	md.AddFieldMappingsAt("title", textFieldMapping)
 	md.AddFieldMappingsAt("author", authorFieldMapping)
 	md.AddFieldMappingsAt("description", textFieldMapping)
 	md.AddFieldMappingsAt("isbn", keywordFieldMapping)
 	md.AddFieldMappingsAt("_unique", keywordFieldMapping)
+	md.AddFieldMappingsAt("_suggest", suggestFieldMapping)
 
 	return md
 }
 
 func defaultIndexMapping() *bleve.IndexMapping {
 	i := bleve.NewIndexMapping()
+
+	err := i.AddCustomTokenFilter("edgeNgram325",
+		map[string]interface{}{
+			"type": "edge_ngram",
+			"min":  3.0,
+			"max":  25.0,
+		})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = i.AddCustomAnalyzer("edgeNgram",
+		map[string]interface{}{
+			"type":      "custom",
+			"tokenizer": "unicode",
+			"token_filters": []string{
+				"to_lower",
+				"stop_en",
+				"edgeNgram325",
+			},
+		})
+	if err != nil {
+		log.Fatal(err)
+	}
+
 	i.AddDocumentMapping("ebook", metadataDocumentMapping())
+
 	i.DefaultAnalyzer = defaultTextAnalyzer
 	i.DefaultType = "ebook"
 	return i
@@ -372,8 +412,9 @@ func (db *Database) Search(queryStr string, offset, limit int) (*SearchResult, e
 }
 
 // Autocomplete runs a fuzzy search for a term.
-func (db *Database) Autocomplete(term string) (*SearchResult, error) {
-	return db.doSearch(bleve.NewFuzzyQuery(term), 0, 20)
+func (db *Database) Suggest(term string) (*SearchResult, error) {
+	query := bleve.NewTermQuery(term).SetField("_suggest")
+	return db.doSearch(query, 0, 20)
 }
 
 // Find a book matching the given metadata, if possible.

database_test.go

@@ -229,19 +229,21 @@ func TestDatabase_Find(t *testing.T) {
 	}
 }
 
-func TestDatabase_Autocomplete(t *testing.T) {
+func TestDatabase_Suggest(t *testing.T) {
 	td, db := newTestDatabase(t)
 	defer td.Close()
 
-	r, err := db.Autocomplete("jules")
-	if err != nil {
-		t.Fatal(err)
-	}
-	if r.NumResults == 0 {
-		t.Error("No results")
+	for _, s := range []string{"jul", "jule", "jules", "ver", "vern", "verne", "twent", "thous"} {
+		r, err := db.Suggest(s)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if r.NumResults == 0 {
+			t.Errorf("No results for '%s'", s)
+		}
 	}
 
-	r, err = db.Autocomplete("foo")
+	r, err := db.Suggest("foo")
 	if err != nil {
 		t.Fatal(err)
 	}

web.go

@@ -100,14 +100,14 @@ type autocompleteResult struct {
 	Label string `json:"label"`
 }
 
-func (s *uiServer) handleAutocomplete(w http.ResponseWriter, req *http.Request) {
+func (s *uiServer) handleSuggest(w http.ResponseWriter, req *http.Request) {
 	term := req.FormValue("term")
 	if term == "" {
 		http.Error(w, "No Query", http.StatusBadRequest)
 		return
 	}
 
-	result, err := s.db.Autocomplete(term)
+	result, err := s.db.Suggest(term)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
@@ -359,7 +359,7 @@ func NewHttpServer(db *Database, storage, cache *FileStorage, addr string) *http
 	r.Handle("/read/{id:[0-9]+}/{fid:[0-9]+}", uisrv.withFile(uisrv.handleReadBook))
 	r.Handle("/dl/{id:[0-9]+}/{fid:[0-9]+}", uisrv.withFile(uisrv.handleDownloadBook))
 	r.HandleFunc("/opensearch.xml", handleOpenSearchXml)
-	r.HandleFunc("/suggest", uisrv.handleAutocomplete)
+	r.HandleFunc("/suggest", uisrv.handleSuggest)
 	r.HandleFunc("/search", uisrv.handleSearch)
 	r.HandleFunc("/", uisrv.handleHome)