Commit 4fc0b1d2 authored by ale's avatar ale

minor golint fixes

parent b06e5a29
...@@ -99,7 +99,7 @@ func (h *warcSaveHandler) Handle(c *crawl.Crawler, u string, depth int, resp *ht ...@@ -99,7 +99,7 @@ func (h *warcSaveHandler) Handle(c *crawl.Crawler, u string, depth int, resp *ht
return extractLinks(c, u, depth, resp, err) return extractLinks(c, u, depth, resp, err)
} }
func NewSaveHandler(w *warc.Writer) crawl.Handler { func newWarcSaveHandler(w *warc.Writer) crawl.Handler {
info := strings.Join([]string{ info := strings.Join([]string{
"Software: crawl/1.0\r\n", "Software: crawl/1.0\r\n",
"Format: WARC File Format 1.0\r\n", "Format: WARC File Format 1.0\r\n",
...@@ -206,7 +206,7 @@ func main() { ...@@ -206,7 +206,7 @@ func main() {
w := warc.NewWriter(outf) w := warc.NewWriter(outf)
defer w.Close() defer w.Close()
saver := NewSaveHandler(w) saver := newWarcSaveHandler(w)
crawler, err := crawl.NewCrawler(*dbPath, seeds, scope, crawl.FetcherFunc(fetch), crawl.NewRedirectHandler(saver)) crawler, err := crawl.NewCrawler(*dbPath, seeds, scope, crawl.FetcherFunc(fetch), crawl.NewRedirectHandler(saver))
if err != nil { if err != nil {
......
...@@ -319,6 +319,7 @@ func (c *Crawler) Run(concurrency int) { ...@@ -319,6 +319,7 @@ func (c *Crawler) Run(concurrency int) {
wg.Wait() wg.Wait()
} }
// Close the database and release resources associated with the crawler state.
func (c *Crawler) Close() { func (c *Crawler) Close() {
c.db.Close() c.db.Close()
} }
......
...@@ -23,7 +23,7 @@ var ( ...@@ -23,7 +23,7 @@ var (
} }
) )
// A WARC header. Header field names are case-sensitive. // Header for a WARC record. Header field names are case-sensitive.
type Header map[string]string type Header map[string]string
// Set a header to the specified value. Multiple values are not // Set a header to the specified value. Multiple values are not
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment