Commit 37aff84a authored by ale's avatar ale

updated dependencies

parent 0b69db2f
...@@ -7,7 +7,7 @@ ...@@ -7,7 +7,7 @@
"Deps": [ "Deps": [
{ {
"ImportPath": "github.com/blevesearch/bleve", "ImportPath": "github.com/blevesearch/bleve",
"Rev": "1006bf54b4782b0644e66a600c27e80652937d91" "Rev": "0b171c85da0922a0baf5fd559ec8515dd35cadb8"
}, },
{ {
"ImportPath": "github.com/blevesearch/go-porterstemmer", "ImportPath": "github.com/blevesearch/go-porterstemmer",
...@@ -20,8 +20,8 @@ ...@@ -20,8 +20,8 @@
}, },
{ {
"ImportPath": "github.com/boltdb/bolt", "ImportPath": "github.com/boltdb/bolt",
"Comment": "v1.1.0-67-g2f846c3", "Comment": "v1.2.0-11-g831b652",
"Rev": "2f846c3551b76d7710f159be840d66c3d064abbe" "Rev": "831b652a7f8dbefaf94da0eb66abd46c0c4bcf23"
}, },
{ {
"ImportPath": "github.com/golang/protobuf/proto", "ImportPath": "github.com/golang/protobuf/proto",
...@@ -29,7 +29,7 @@ ...@@ -29,7 +29,7 @@
}, },
{ {
"ImportPath": "github.com/golang/snappy", "ImportPath": "github.com/golang/snappy",
"Rev": "c2359a1bd0bd4a2de4f1bd92ccd045fb60d0a994" "Rev": "5f1c01d9f64b941dd9582c638279d046eda6ca31"
}, },
{ {
"ImportPath": "github.com/gorilla/context", "ImportPath": "github.com/gorilla/context",
...@@ -53,7 +53,7 @@ ...@@ -53,7 +53,7 @@
}, },
{ {
"ImportPath": "github.com/syndtr/goleveldb/leveldb", "ImportPath": "github.com/syndtr/goleveldb/leveldb",
"Rev": "e7e6f5b5ef25adb580feac515f9ccec514d0bda8" "Rev": "93fc893f2dadb96ffde441c7546cc67ea290a3a8"
}, },
{ {
"ImportPath": "github.com/willf/bitset", "ImportPath": "github.com/willf/bitset",
...@@ -66,19 +66,19 @@ ...@@ -66,19 +66,19 @@
}, },
{ {
"ImportPath": "golang.org/x/text/encoding", "ImportPath": "golang.org/x/text/encoding",
"Rev": "07b9a78963006a15c538ec5175243979025fa7a8" "Rev": "1b466db55e0ba5d56ef5315c728216b42f796491"
}, },
{ {
"ImportPath": "golang.org/x/text/internal/utf8internal", "ImportPath": "golang.org/x/text/internal/utf8internal",
"Rev": "07b9a78963006a15c538ec5175243979025fa7a8" "Rev": "1b466db55e0ba5d56ef5315c728216b42f796491"
}, },
{ {
"ImportPath": "golang.org/x/text/runes", "ImportPath": "golang.org/x/text/runes",
"Rev": "07b9a78963006a15c538ec5175243979025fa7a8" "Rev": "1b466db55e0ba5d56ef5315c728216b42f796491"
}, },
{ {
"ImportPath": "golang.org/x/text/transform", "ImportPath": "golang.org/x/text/transform",
"Rev": "07b9a78963006a15c538ec5175243979025fa7a8" "Rev": "1b466db55e0ba5d56ef5315c728216b42f796491"
} }
] ]
} }
...@@ -4,6 +4,8 @@ ...@@ -4,6 +4,8 @@
.#* .#*
.project .project
.settings .settings
**/.idea/
**/*.iml
.DS_Store .DS_Store
/analysis/token_filters/cld2/cld2-read-only /analysis/token_filters/cld2/cld2-read-only
/analysis/token_filters/cld2/libcld2_full.a /analysis/token_filters/cld2/libcld2_full.a
......
# ![bleve](docs/bleve.png) bleve # ![bleve](docs/bleve.png) bleve
[![Build Status](https://travis-ci.org/blevesearch/bleve.svg?branch=master)](https://travis-ci.org/blevesearch/bleve) [![Coverage Status](https://coveralls.io/repos/blevesearch/bleve/badge.png?branch=master)](https://coveralls.io/r/blevesearch/bleve?branch=master) [![GoDoc](https://godoc.org/github.com/blevesearch/bleve?status.svg)](https://godoc.org/github.com/blevesearch/bleve) [![Join the chat at https://gitter.im/blevesearch/bleve](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/blevesearch/bleve?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Build Status](https://travis-ci.org/blevesearch/bleve.svg?branch=master)](https://travis-ci.org/blevesearch/bleve) [![Coverage Status](https://coveralls.io/repos/blevesearch/bleve/badge.png?branch=master)](https://coveralls.io/r/blevesearch/bleve?branch=master) [![GoDoc](https://godoc.org/github.com/blevesearch/bleve?status.svg)](https://godoc.org/github.com/blevesearch/bleve) [![Join the chat at https://gitter.im/blevesearch/bleve](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/blevesearch/bleve?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)[![codebeat](https://codebeat.co/badges/38a7cbc9-9cf5-41c0-a315-0746178230f4)](https://codebeat.co/projects/github-com-blevesearch-bleve)
modern text indexing in go - [blevesearch.com](http://www.blevesearch.com/) modern text indexing in go - [blevesearch.com](http://www.blevesearch.com/)
......
...@@ -12,14 +12,14 @@ package simple_analyzer ...@@ -12,14 +12,14 @@ package simple_analyzer
import ( import (
"git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/analysis" "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/analysis"
"git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter"
"git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/analysis/tokenizers/unicode" "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/analysis/tokenizers/letter"
"git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/registry" "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/registry"
) )
const Name = "simple" const Name = "simple"
func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) {
tokenizer, err := cache.TokenizerNamed(unicode.Name) tokenizer, err := cache.TokenizerNamed(letter.Name)
if err != nil { if err != nil {
return nil, err return nil, err
} }
......
...@@ -7,6 +7,13 @@ ...@@ -7,6 +7,13 @@
// either express or implied. See the License for the specific language governing permissions // either express or implied. See the License for the specific language governing permissions
// and limitations under the License. // and limitations under the License.
// Package en implements an analyzer with reasonable defaults for processing
// English text.
//
// It strips possessive suffixes ('s), transforms tokens to lower case,
// removes stopwords from a built-in list, and applies porter stemming.
//
// The built-in stopwords list is defined in EnglishStopWords.
package en package en
import ( import (
......
...@@ -16,6 +16,8 @@ import ( ...@@ -16,6 +16,8 @@ import (
"git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/registry" "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/registry"
) )
// PossessiveName is the name PossessiveFilter is registered as
// in the bleve registry.
const PossessiveName = "possessive_en" const PossessiveName = "possessive_en"
const rightSingleQuotationMark = '’' const rightSingleQuotationMark = '’'
...@@ -24,6 +26,11 @@ const fullWidthApostrophe = ''' ...@@ -24,6 +26,11 @@ const fullWidthApostrophe = '''
const apostropheChars = rightSingleQuotationMark + apostrophe + fullWidthApostrophe const apostropheChars = rightSingleQuotationMark + apostrophe + fullWidthApostrophe
// PossessiveFilter implements a TokenFilter which
// strips the English possessive suffix ('s) from tokens.
// It handle a variety of apostrophe types, is case-insensitive
// and doesn't distinguish between possessive and contraction.
// (ie "She's So Rad" becomes "She So Rad")
type PossessiveFilter struct { type PossessiveFilter struct {
} }
......
...@@ -7,10 +7,11 @@ import ( ...@@ -7,10 +7,11 @@ import (
const StopName = "stop_en" const StopName = "stop_en"
// EnglishStopWords is the built-in list of stopwords used by the "stop_en" TokenFilter.
//
// this content was obtained from: // this content was obtained from:
// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ // lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/
// ` was changed to ' to allow for literal string // ` was changed to ' to allow for literal string
var EnglishStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/english/stop.txt var EnglishStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/english/stop.txt
| This file is distributed under the BSD License. | This file is distributed under the BSD License.
| See http://snowball.tartarus.org/license.php | See http://snowball.tartarus.org/license.php
......
...@@ -7,6 +7,8 @@ ...@@ -7,6 +7,8 @@
// either express or implied. See the License for the specific language governing permissions // either express or implied. See the License for the specific language governing permissions
// and limitations under the License. // and limitations under the License.
// Package lower_case_filter implements a TokenFilter which converts
// tokens to lower case according to unicode rules.
package lower_case_filter package lower_case_filter
import ( import (
...@@ -18,6 +20,7 @@ import ( ...@@ -18,6 +20,7 @@ import (
"git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/registry" "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/registry"
) )
// Name is the name used to register LowerCaseFilter in the bleve registry
const Name = "to_lower" const Name = "to_lower"
type LowerCaseFilter struct { type LowerCaseFilter struct {
......
...@@ -23,6 +23,9 @@ func NewTokenMap() TokenMap { ...@@ -23,6 +23,9 @@ func NewTokenMap() TokenMap {
return make(TokenMap, 0) return make(TokenMap, 0)
} }
// LoadFile reads in a list of tokens from a text file,
// one per line.
// Comments are supported using `#` or `|`
func (t TokenMap) LoadFile(filename string) error { func (t TokenMap) LoadFile(filename string) error {
data, err := ioutil.ReadFile(filename) data, err := ioutil.ReadFile(filename)
if err != nil { if err != nil {
...@@ -31,6 +34,9 @@ func (t TokenMap) LoadFile(filename string) error { ...@@ -31,6 +34,9 @@ func (t TokenMap) LoadFile(filename string) error {
return t.LoadBytes(data) return t.LoadBytes(data)
} }
// LoadBytes reads in a list of tokens from memory,
// one per line.
// Comments are supported using `#` or `|`
func (t TokenMap) LoadBytes(data []byte) error { func (t TokenMap) LoadBytes(data []byte) error {
bytesReader := bytes.NewReader(data) bytesReader := bytes.NewReader(data)
bufioReader := bufio.NewReader(bytesReader) bufioReader := bufio.NewReader(bytesReader)
......
// Copyright (c) 2014 Couchbase, Inc. // Copyright (c) 2016 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at // except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0 // http://www.apache.org/licenses/LICENSE-2.0
...@@ -7,77 +7,65 @@ ...@@ -7,77 +7,65 @@
// either express or implied. See the License for the specific language governing permissions // either express or implied. See the License for the specific language governing permissions
// and limitations under the License. // and limitations under the License.
package ja package character
import ( import (
"git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/analysis" "unicode/utf8"
"git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/registry"
"github.com/ikawaha/kagome/tokenizer" "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/analysis"
) )
const TokenizerName = "kagome" type IsTokenRune func(r rune) bool
type KagomeMorphTokenizer struct {
tok tokenizer.Tokenizer
}
func init() {
_ = tokenizer.SysDic() // prepare system dictionary
}
func NewKagomeMorphTokenizer() *KagomeMorphTokenizer { type CharacterTokenizer struct {
return &KagomeMorphTokenizer{ isTokenRun IsTokenRune
tok: tokenizer.New(),
}
} }
func NewKagomeMorphTokenizerWithUserDic(userdic tokenizer.UserDic) *KagomeMorphTokenizer { func NewCharacterTokenizer(f IsTokenRune) *CharacterTokenizer {
k := tokenizer.New() return &CharacterTokenizer{
k.SetUserDic(userdic) isTokenRun: f,
return &KagomeMorphTokenizer{
tok: k,
} }
} }
func (t *KagomeMorphTokenizer) Tokenize(input []byte) analysis.TokenStream { func (c *CharacterTokenizer) Tokenize(input []byte) analysis.TokenStream {
var (
morphs []tokenizer.Token rv := make(analysis.TokenStream, 0, 1024)
prevstart int
) offset := 0
start := 0
rv := make(analysis.TokenStream, 0, len(input)) end := 0
if len(input) < 1 { count := 0
return rv for currRune, size := utf8.DecodeRune(input[offset:]); currRune != utf8.RuneError; currRune, size = utf8.DecodeRune(input[offset:]) {
} isToken := c.isTokenRun(currRune)
if isToken {
morphs = t.tok.Analyze(string(input), tokenizer.Search) end = offset + size
} else {
for i, m := range morphs { if end-start > 0 {
if m.Surface == "EOS" || m.Surface == "BOS" { // build token
continue rv = append(rv, &analysis.Token{
} Term: input[start:end],
Start: start,
surfacelen := len(m.Surface) End: end,
token := &analysis.Token{ Position: count + 1,
Term: []byte(m.Surface), Type: analysis.AlphaNumeric,
Position: i, })
Start: prevstart, count++
End: prevstart + surfacelen, }
Type: analysis.Ideographic, start = offset + size
end = start
} }
offset += size
prevstart = prevstart + surfacelen }
rv = append(rv, token) // if we ended in the middle of a token, finish it
if end-start > 0 {
// build token
rv = append(rv, &analysis.Token{
Term: input[start:end],
Start: start,
End: end,
Position: count + 1,
Type: analysis.AlphaNumeric,
})
} }
return rv return rv
} }
func KagomeMorphTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {
return NewKagomeMorphTokenizer(), nil
}
func init() {
registry.RegisterTokenizer(TokenizerName, KagomeMorphTokenizerConstructor)
}
// Copyright (c) 2014 Couchbase, Inc. // Copyright (c) 2016 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at // except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0 // http://www.apache.org/licenses/LICENSE-2.0
...@@ -7,31 +7,22 @@ ...@@ -7,31 +7,22 @@
// either express or implied. See the License for the specific language governing permissions // either express or implied. See the License for the specific language governing permissions
// and limitations under the License. // and limitations under the License.
package ja package letter
import ( import (
"unicode"
"git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/analysis" "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/analysis"
"git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/analysis/token_filters/unicode_normalize" "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/analysis/tokenizers/character"
"git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/registry" "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/registry"
) )
const AnalyzerName = "ja" const Name = "letter"
func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { func TokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {
kagomeTokenizer, err := cache.TokenizerNamed(TokenizerName) return character.NewCharacterTokenizer(unicode.IsLetter), nil
if err != nil {
return nil, err
}
normalizeFilter := unicode_normalize.MustNewUnicodeNormalizeFilter(unicode_normalize.NFKD)
rv := analysis.Analyzer{
Tokenizer: kagomeTokenizer,
TokenFilters: []analysis.TokenFilter{
normalizeFilter,
},
}
return &rv, nil
} }
func init() { func init() {
registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) registry.RegisterTokenizer(Name, TokenizerConstructor)
} }
...@@ -88,6 +88,7 @@ import ( ...@@ -88,6 +88,7 @@ import (
_ "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/index/store/boltdb" _ "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/index/store/boltdb"
_ "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/index/store/goleveldb" _ "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/index/store/goleveldb"
_ "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/index/store/gtreap" _ "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/index/store/gtreap"
_ "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/index/store/moss"
// index types // index types
_ "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/index/firestorm" _ "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/index/firestorm"
......
...@@ -12,5 +12,5 @@ ...@@ -12,5 +12,5 @@
package config package config
import ( import (
_ "git.autistici.org/ale/liber/Godeps/_workspace/src/github.com/blevesearch/bleve/analysis/language/ja" _ "github.com/blevesearch/blevex/lang/ja"
) )
...@@ -55,3 +55,18 @@ func (d *Document) GoString() string { ...@@ -55,3 +55,18 @@ func (d *Document) GoString() string {
} }
return fmt.Sprintf("&document.Document{ID:%s, Fields: %s, CompositeFields: %s}", d.ID, fields, compositeFields) return fmt.Sprintf("&document.Document{ID:%s, Fields: %s, CompositeFields: %s}", d.ID, fields, compositeFields)
} }
func (d *Document) NumPlainTextBytes() uint64 {
rv := uint64(0)
for _, field := range d.Fields {
rv += field.NumPlainTextBytes()
}
for _, compositeField := range d.CompositeFields {
for _, field := range d.Fields {
if compositeField.includesField(field.Name()) {
rv += field.NumPlainTextBytes()
}
}
}
return rv
}
...@@ -26,4 +26,9 @@ type Field interface { ...@@ -26,4 +26,9 @@ type Field interface {
Options() IndexingOptions Options() IndexingOptions
Analyze() (int, analysis.TokenFrequencies) Analyze() (int, analysis.TokenFrequencies)
Value() []byte Value() []byte
// NumPlainTextBytes should return the number of plain text bytes
// that this field represents - this is a common metric for tracking
// the rate of indexing
NumPlainTextBytes() uint64
} }
...@@ -18,10 +18,11 @@ import ( ...@@ -18,10 +18,11 @@ import (
const DefaultBooleanIndexingOptions = StoreField | IndexField const DefaultBooleanIndexingOptions = StoreField | IndexField
type BooleanField struct { type BooleanField struct {
name string name string
arrayPositions []uint64 arrayPositions []uint64
options IndexingOptions options IndexingOptions
value []byte value []byte
numPlainTextBytes uint64
} }
func (b *BooleanField) Name() string { func (b *BooleanField) Name() string {
...@@ -66,12 +67,17 @@ func (b *BooleanField) GoString() string { ...@@ -66,12 +67,17 @@ func (b *BooleanField) GoString() string {
return fmt.Sprintf("&document.BooleanField{Name:%s, Options: %s, Value: %s}", b.name, b.options, b.value) return fmt.Sprintf("&document.BooleanField{Name:%s, Options: %s, Value: %s}", b.name, b.options, b.value)
} }
func (b *BooleanField) NumPlainTextBytes() uint64 {
return b.numPlainTextBytes
}
func NewBooleanFieldFromBytes(name string, arrayPositions []uint64, value []byte) *BooleanField { func NewBooleanFieldFromBytes(name string, arrayPositions []uint64, value []byte) *BooleanField {
return &BooleanField{ return &BooleanField{
name: name, name: name,
arrayPositions: arrayPositions, arrayPositions: arrayPositions,
value: value, value: value,
options: DefaultNumericIndexingOptions, options: DefaultNumericIndexingOptions,
numPlainTextBytes: uint64(len(value)),
} }
} }
...@@ -80,14 +86,17 @@ func NewBooleanField(name string, arrayPositions []uint64, b bool) *BooleanField ...@@ -80,14 +86,17 @@ func NewBooleanField(name string, arrayPositions []uint64, b bool) *BooleanField
} }
func NewBooleanFieldWithIndexingOptions(name string, arrayPositions []uint64, b bool, options IndexingOptions) *BooleanField { func NewBooleanFieldWithIndexingOptions(name string, arrayPositions []uint64, b bool, options IndexingOptions) *BooleanField {
numPlainTextBytes := 5
v := []byte("F") v := []byte("F")
if b { if b {
numPlainTextBytes = 4
v = []byte("T") v = []byte("T")
} }
return &BooleanField{ return &BooleanField{
name: name, name: name,
arrayPositions: arrayPositions, arrayPositions: arrayPositions,
value: v, value: v,
options: options, options: options,
numPlainTextBytes: uint64(numPlainTextBytes),
} }
} }
...@@ -69,7 +69,11 @@ func (c *CompositeField) Value() []byte { ...@@ -69,7 +69,11 @@ func (c *CompositeField) Value() []byte {
return []byte{} return []byte{}
} }
func (c *CompositeField) Compose(field string, length int, freq analysis.TokenFrequencies) { func (c *CompositeField) NumPlainTextBytes() uint64 {
return 0
}
func (c *CompositeField) includesField(field string) bool {
shouldInclude := c.defaultInclude shouldInclude := c.defaultInclude
_, fieldShouldBeIncluded := c.includedFields[field] _, fieldShouldBeIncluded := c.includedFields[field]
if fieldShouldBeIncluded { if fieldShouldBeIncluded {
...@@ -79,8 +83,11 @@ func (c *CompositeField) Compose(field string, length int, freq analysis.TokenFr ...@@ -79,8 +83,11 @@ func (c *CompositeField) Compose(field string, length int, freq analysis.TokenFr
if fieldShouldBeExcluded { if fieldShouldBeExcluded {
shouldInclude = false shouldInclude = false
} }
return shouldInclude
}
if shouldInclude { func (c *CompositeField) Compose(field string, length int, freq analysis.TokenFrequencies) {
if c.includesField(field) {
c.totalLength += length c.totalLength += length