Increase error retry delay

crawler.go

@@ -20,7 +20,7 @@ import (
 	lutil "github.com/syndtr/goleveldb/leveldb/util"
 )
 
-var errorRetryDelay = 180 * time.Second
+var errorRetryDelay = 12 * time.Hour
 
 type gobDB struct {
 	*leveldb.DB
@@ -188,6 +188,7 @@ func (c *Crawler) Enqueue(link Outlink, depth int) error {
 
 	// See if it's in scope.
 	if !c.scope.Check(link, depth) {
+		log.Printf("%s is not in scope", link.URL)
 		return nil
 	}