.gitlab-ci.yml

 include: "https://git.autistici.org/ai3/build-deb/raw/master/ci-common.yml"
 
-stages:
-  - test
-  - build_pkgsrc
-  - build_pkg
-  - upload_pkg
-
 run_tests:
   stage: test
-  image: "golang:latest"
+  image: "golang:1.19"
   script: "go test -v ./..."
 

crawler.go

@@ -20,7 +20,7 @@ import (
 	lutil "github.com/syndtr/goleveldb/leveldb/util"
 )
 
-var errorRetryDelay = 180 * time.Second
+var errorRetryDelay = 12 * time.Hour
 
 type gobDB struct {
 	*leveldb.DB
@@ -188,6 +188,7 @@ func (c *Crawler) Enqueue(link Outlink, depth int) error {
 
 	// See if it's in scope.
 	if !c.scope.Check(link, depth) {
+		log.Printf("%s is not in scope", link.URL)
 		return nil
 	}
 

gen-ignores.py

@@ -13,19 +13,19 @@ import os
 import sys
 
 archivebot_ignore_path = sys.argv[1]
-print 'package crawl\n\nvar defaultIgnorePatterns = []string{'
+print('package crawl\n\nvar defaultIgnorePatterns = []string{')
 for fn in glob.glob(os.path.join(archivebot_ignore_path, '*.json')):
     try:
         with open(fn) as fd:
-            print '\n\t// %s' % os.path.basename(fn)
+            print('\n\t// %s' % os.path.basename(fn))
             for p in json.load(fd)['patterns']:
                 if '\\\\1' in p or '(?!' in p:
                     # RE2 does not support backreferences or other
                     # fancy PCRE constructs. This excludes <10
                     # patterns from the ignore list.
                     continue
-                print '\t%s,' % json.dumps(p)
-    except Exception, e:
-        print >>sys.stderr, 'error in %s: %s' % (fn, e)
-print '}'
-
+                p = p.replace('{primary_netloc}', '.*')
+                print('\t%s,' % json.dumps(p))
+    except Exception as e:
+        print('error in %s: %s' % (fn, e), file=sys.stderr)
+print('}')