From e539096888e22e811be9a203f1a429921f2b4015 Mon Sep 17 00:00:00 2001
From: ale <ale@incal.net>
Date: Sun, 13 Nov 2022 12:52:22 +0000
Subject: [PATCH] Increase error retry delay
---
crawler.go | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/crawler.go b/crawler.go
index b2ad3d9..80f9dd7 100644
--- a/crawler.go
+++ b/crawler.go
@@ -20,7 +20,7 @@ import (
lutil "github.com/syndtr/goleveldb/leveldb/util"
)
-var errorRetryDelay = 180 * time.Second
+var errorRetryDelay = 12 * time.Hour
type gobDB struct {
*leveldb.DB
@@ -188,6 +188,7 @@ func (c *Crawler) Enqueue(link Outlink, depth int) error {
// See if it's in scope.
if !c.scope.Check(link, depth) {
+ log.Printf("%s is not in scope", link.URL)
return nil
}
--
GitLab