diff --git a/crawler.go b/crawler.go index b2ad3d9208c8d6435a35f8bcc01e9adcffdb29a8..80f9dd7a873e00574ef3d91f48f5183b97444cc2 100644 --- a/crawler.go +++ b/crawler.go @@ -20,7 +20,7 @@ import ( lutil "github.com/syndtr/goleveldb/leveldb/util" ) -var errorRetryDelay = 180 * time.Second +var errorRetryDelay = 12 * time.Hour type gobDB struct { *leveldb.DB @@ -188,6 +188,7 @@ func (c *Crawler) Enqueue(link Outlink, depth int) error { // See if it's in scope. if !c.scope.Check(link, depth) { + log.Printf("%s is not in scope", link.URL) return nil }