.gitlab-ci.yml

 include: "https://git.autistici.org/ai3/build-deb/raw/master/ci-common.yml"
 
-stages:
-  - test
-  - build_pkgsrc
-  - build_pkg
-  - upload_pkg
-
 run_tests:
   stage: test
-  image: "golang:latest"
+  image: "golang:1.19"
   script: "go test -v ./..."
 

crawler.go

@@ -20,7 +20,7 @@ import (
 	lutil "github.com/syndtr/goleveldb/leveldb/util"
 )
 
-var errorRetryDelay = 180 * time.Second
+var errorRetryDelay = 12 * time.Hour
 
 type gobDB struct {
 	*leveldb.DB
@@ -188,6 +188,7 @@ func (c *Crawler) Enqueue(link Outlink, depth int) error {
 
 	// See if it's in scope.
 	if !c.scope.Check(link, depth) {
+		log.Printf("%s is not in scope", link.URL)
 		return nil
 	}
 

gen-ignores.py

@@ -10,22 +10,25 @@
 import glob
 import json
 import os
+import re
 import sys
 
 archivebot_ignore_path = sys.argv[1]
-print 'package crawl\n\nvar defaultIgnorePatterns = []string{'
+print('package crawl\n\nvar defaultIgnorePatterns = []string{')
 for fn in glob.glob(os.path.join(archivebot_ignore_path, '*.json')):
     try:
         with open(fn) as fd:
-            print '\n\t// %s' % os.path.basename(fn)
+            print('\n\t// %s' % os.path.basename(fn))
             for p in json.load(fd)['patterns']:
-                if '\\\\1' in p or '(?!' in p:
+                if re.search(r'\\[0-9]', p) or ('(?!' in p) or ('(?=' in p):
                     # RE2 does not support backreferences or other
                     # fancy PCRE constructs. This excludes <10
                     # patterns from the ignore list.
                     continue
-                print '\t%s,' % json.dumps(p)
-    except Exception, e:
-        print >>sys.stderr, 'error in %s: %s' % (fn, e)
-print '}'
-
+                if re.search(r'(?:%[0-9A-F]{2}){3,}', p):
+                    continue
+                p = p.replace('{primary_netloc}', '.*')
+                print('\t%s,' % json.dumps(p))
+    except Exception as e:
+        print('error in %s: %s' % (fn, e), file=sys.stderr)
+print('}')

go.mod

@@ -3,12 +3,12 @@ module git.autistici.org/ale/crawl
 go 1.15
 
 require (
-	github.com/PuerkitoBio/goquery v1.7.1
-	github.com/PuerkitoBio/purell v0.1.0
+	github.com/PuerkitoBio/goquery v1.8.0
+	github.com/PuerkitoBio/purell v1.2.0
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/google/go-cmp v0.5.6
 	github.com/google/uuid v1.1.1 // indirect
 	github.com/pborman/uuid v1.2.1
-	github.com/syndtr/goleveldb v0.0.0-20190923125748-758128399b1d
+	github.com/syndtr/goleveldb v1.0.0
 	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 // indirect
 )

go.sum

@@ -2,20 +2,28 @@ github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP
 github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
 github.com/PuerkitoBio/goquery v1.7.1 h1:oE+T06D+1T7LNrn91B4aERsRIeCLJ/oPSa6xB9FPnz4=
 github.com/PuerkitoBio/goquery v1.7.1/go.mod h1:XY0pP4kfraEmmV1O7Uf6XyjoslwsneBbgeDjLYuN8xY=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
 github.com/PuerkitoBio/purell v0.0.0-20180310210909-975f53781597 h1:1H3FyRw7YsqIty9WHPOVEGJaFJ1sfGVZ3PPDUw3ob2w=
 github.com/PuerkitoBio/purell v0.0.0-20180310210909-975f53781597/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v0.1.0 h1:N8Bcc53nei5frgNYgAKo93qMUVdU5LUGHCBv8efdVcM=
 github.com/PuerkitoBio/purell v0.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/purell v1.2.0 h1:/Jdm5QfyM8zdlqT6WVZU4cfP23sot6CEHA4CS49Ezig=
+github.com/PuerkitoBio/purell v1.2.0/go.mod h1:OhLRTaaIzhvIyofkJfB24gokC7tM42Px5UhoT32THBk=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
 github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE=
 github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
@@ -40,6 +48,8 @@ github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=
 github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/syndtr/goleveldb v0.0.0-20190923125748-758128399b1d h1:OgkXbz/O0zsJoaB+z6n/a3bNGCbCWhBPLfGr6qaBprM=
 github.com/syndtr/goleveldb v0.0.0-20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
+github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
+github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -48,6 +58,10 @@ golang.org/x/net v0.0.0-20190926025831-c00fd9afed17 h1:qPnAdmjNA41t3QBTx2mFGf/SD
 golang.org/x/net v0.0.0-20190926025831-c00fd9afed17/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
 golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 h1:Yqz/iviulwKwAREEeUd3nbBFn0XuyJqkoft2IlrvOhc=
+golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -55,10 +69,15 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

vendor/github.com/PuerkitoBio/goquery/.travis.yml

-arch:
-    - amd64
-    - ppc64le
-language: go
-
-go:
-    - 1.7.x
-    - 1.8.x
-    - 1.9.x
-    - 1.10.x
-    - 1.11.x
-    - 1.12.x
-    - 1.13.x
-    - 1.14.x
-    - 1.15.x
-    - tip
-
-jobs:
- exclude:
-    - arch: ppc64le
-      go: 1.7.x
-    - arch: ppc64le
-      go: 1.8.x
-    - arch: ppc64le
-      go: 1.9.x
-    - arch: ppc64le
-      go: 1.10.x
-    - arch: ppc64le
-      go: 1.11.x
-    - arch: ppc64le
-      go: 1.12.x

vendor/github.com/PuerkitoBio/goquery/README.md

 # goquery - a little like that j-thing, only in Go
 
-[![builds.sr.ht status](https://builds.sr.ht/~mna/goquery/commits/fedora.yml.svg)](https://builds.sr.ht/~mna/goquery/commits/fedora.yml?)
-[![build status](https://secure.travis-ci.org/PuerkitoBio/goquery.svg?branch=master)](http://travis-ci.org/PuerkitoBio/goquery)
+[![Build Status](https://github.com/PuerkitoBio/goquery/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/PuerkitoBio/goquery/actions)
 [![Go Reference](https://pkg.go.dev/badge/github.com/PuerkitoBio/goquery.svg)](https://pkg.go.dev/github.com/PuerkitoBio/goquery)
 [![Sourcegraph Badge](https://sourcegraph.com/github.com/PuerkitoBio/goquery/-/badge.svg)](https://sourcegraph.com/github.com/PuerkitoBio/goquery?badge)
 
@@ -41,6 +40,7 @@ Please note that because of the net/html dependency, goquery requires Go1.1+ and
 
 **Note that goquery's API is now stable, and will not break.**
 
+*    **2021-10-25 (v1.8.0)** : Add `Render` function to render a `Selection` to an `io.Writer` (thanks [@anthonygedeon](https://github.com/anthonygedeon)).
 *    **2021-07-11 (v1.7.1)** : Update go.mod dependencies and add dependabot config (thanks [@jauderho](https://github.com/jauderho)).
 *    **2021-06-14 (v1.7.0)** : Add `Single` and `SingleMatcher` functions to optimize first-match selection (thanks [@gdollardollar](https://github.com/gdollardollar)).
 *    **2021-01-11 (v1.6.1)** : Fix panic when calling `{Prepend,Append,Set}Html` on a `Selection` that contains non-Element nodes.

vendor/github.com/PuerkitoBio/goquery/go.mod

 module github.com/PuerkitoBio/goquery
 
 require (
-	github.com/andybalholm/cascadia v1.2.0
-	golang.org/x/net v0.0.0-20210614182718-04defd469f4e
+	github.com/andybalholm/cascadia v1.3.1
+	golang.org/x/net v0.0.0-20210916014120-12bc252f5db8
 )
 
 go 1.13

vendor/github.com/PuerkitoBio/goquery/go.sum

-github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE=
-github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
-golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
-golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

vendor/github.com/PuerkitoBio/goquery/utilities.go

@@ -2,6 +2,7 @@ package goquery
 
 import (
 	"bytes"
+	"io"
 
 	"golang.org/x/net/html"
 )
@@ -50,13 +51,24 @@ func nodeName(node *html.Node) string {
 	case html.ElementNode, html.DoctypeNode:
 		return node.Data
 	default:
-		if node.Type >= 0 && int(node.Type) < len(nodeNames) {
+		if int(node.Type) < len(nodeNames) {
 			return nodeNames[node.Type]
 		}
 		return ""
 	}
 }
 
+// Render renders the html of the first element from selector and writes it to
+// the writer.  It behaves the same as OuterHtml but writes to w instead of
+// returning the string.
+func Render(w io.Writer, s *Selection) error {
+	if s.Length() == 0 {
+		return nil
+	}
+	n := s.Get(0)
+	return html.Render(w, n)
+}
+
 // OuterHtml returns the outer HTML rendering of the first item in
 // the selection - that is, the HTML including the first element's
 // tag and attributes.
@@ -66,12 +78,7 @@ func nodeName(node *html.Node) string {
 // a property provided by the DOM).
 func OuterHtml(s *Selection) (string, error) {
 	var buf bytes.Buffer
-
-	if s.Length() == 0 {
-		return "", nil
-	}
-	n := s.Get(0)
-	if err := html.Render(&buf, n); err != nil {
+	if err := Render(&buf, s); err != nil {
 		return "", err
 	}
 	return buf.String(), nil

vendor/github.com/PuerkitoBio/purell/.gitignore

 *.sublime-*
 .DS_Store
+*.swp
+*.swo
+tags

vendor/github.com/PuerkitoBio/purell/.travis.yml

-language: go
-

vendor/github.com/PuerkitoBio/purell/LICENSE

-Copyright (c) 2012, Martin Angers
+Copyright (c) 2012-2022, The Go Authors
+Copyright (c) 2012-2022, Martin Angers, Yuki Okushi & Contributors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

vendor/github.com/PuerkitoBio/purell/README.md

@@ -4,12 +4,21 @@ Purell is a tiny Go library to normalize URLs. It returns a pure URL. Pure-ell.
 
 Based on the [wikipedia paper][wiki] and the [RFC 3986 document][rfc].
 
-[![build status](https://secure.travis-ci.org/PuerkitoBio/purell.png)](http://travis-ci.org/PuerkitoBio/purell)
+[![CI](https://github.com/PuerkitoBio/purell/actions/workflows/ci.yml/badge.svg)](https://github.com/PuerkitoBio/purell/actions/workflows/ci.yml)
 
 ## Install
 
 `go get github.com/PuerkitoBio/purell`
 
+## Changelog
+
+*    **v1.1.1** : Fix failing test due to Go1.12 changes (thanks to @ianlancetaylor).
+*    **2016-11-14 (v1.1.0)** : IDN: Conform to RFC 5895: Fold character width (thanks to @beeker1121).
+*    **2016-07-27 (v1.0.0)** : Normalize IDN to ASCII (thanks to @zenovich).
+*    **2015-02-08** : Add fix for relative paths issue ([PR #5][pr5]) and add fix for unnecessary encoding of reserved characters ([see issue #7][iss7]).
+*    **v0.2.0** : Add benchmarks, Attempt IDN support.
+*    **v0.1.0** : Initial release.
+
 ## Examples
 
 From `example_test.go` (note that in your code, you would import "github.com/PuerkitoBio/purell", and would prefix references to its methods and constants with "purell."):
@@ -59,10 +68,11 @@ As seen in the examples above, purell offers three methods, `NormalizeURLString(
 ```go
 const (
 	// Safe normalizations
-  FlagLowercaseScheme           NormalizationFlags = 1 << iota // HTTP://host -> http://host
+	FlagLowercaseScheme           NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1
 	FlagLowercaseHost                                            // http://HOST -> http://host
 	FlagUppercaseEscapes                                         // http://host/t%ef -> http://host/t%EF
 	FlagDecodeUnnecessaryEscapes                                 // http://host/t%41 -> http://host/tA
+	FlagEncodeNecessaryEscapes                                   // http://host/!"#$ -> http://host/%21%22#$
 	FlagRemoveDefaultPort                                        // http://host:80 -> http://host
 	FlagRemoveEmptyQuerySeparator                                // http://host/path? -> http://host/path
 
@@ -89,7 +99,7 @@ const (
 	FlagRemoveEmptyPortSeparator  // http://host:/path -> http://host/path
 
 	// Convenience set of safe normalizations
-  FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator
+	FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagEncodeNecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator
 
 	// For convenience sets, "greedy" uses the "remove trailing slash" and "remove www. prefix" flags,
 	// while "non-greedy" uses the "add (or keep) the trailing slash" and "add www. prefix".
@@ -114,7 +124,7 @@ The [full godoc reference is available on gopkgdoc][godoc].
 
 Some things to note:
 
-*    `FlagDecodeUnnecessaryEscapes`, `FlagEncodeNecessaryEscapes`, `FlagUppercaseEscapes` and `FlagRemoveEmptyQuerySeparator` are always implicitly set, because internally, the URL string is parsed as an URL object, which automatically decodes unnecessary escapes, uppercases and encodes necessary ones, and removes empty query separators (an unnecessary `?` at the end of the url). So this operation cannot **not** be done. For this reason, `FlagRemoveEmptyQuerySeparator` (as well as the other three) has been included in the `FlagsSafe` convenience set, instead of `FlagsUnsafe`, where Wikipedia puts it (strangely?).
+*    `FlagDecodeUnnecessaryEscapes`, `FlagEncodeNecessaryEscapes`, `FlagUppercaseEscapes` and `FlagRemoveEmptyQuerySeparator` are always implicitly set, because internally, the URL string is parsed as an URL object, which automatically decodes unnecessary escapes, uppercases and encodes necessary ones, and removes empty query separators (an unnecessary `?` at the end of the url). So this operation cannot **not** be done. For this reason, `FlagRemoveEmptyQuerySeparator` (as well as the other three) has been included in the `FlagsSafe` convenience set, instead of `FlagsUnsafe`, where Wikipedia puts it.
 
 *    The `FlagDecodeUnnecessaryEscapes` decodes the following escapes (*from -> to*):
     -    %24 -> $
@@ -125,7 +135,6 @@ Some things to note:
     -    %5F -> _
     -    %61-%7A -> abcdefghijklmnopqrstuvwxyz
     -    %7E -> ~
-    -    When the test runs on Travis-ci, it fails because more escapes are decoded than on my machine, so it is either machine/OS-dependent or it is because they run a different Go version (**which they do, they run go1 while I run go1.0.2** so this is very likely the cause). To avoid a failing build banner on GitHub, I commented-out these specific tests (`TestDecodeUnnecessaryEscapesAll()`, `TestEncodeNecessaryEscapesAll()`, `TestGoVersion()`).
 
 
 *    When the `NormalizeURL` function is used (passing an URL object), this source URL object is modified (that is, after the call, the URL object will be modified to reflect the normalization).
@@ -156,13 +165,16 @@ And with `FlagsUnsafeGreedy`:
 
 ## TODOs
 
-*    Try to make jehiah's tests pass (more exactly, support IDNA domains/utf8/unicode encoding/escaping). For now theses tests are commented out.
 *    Add a class/default instance to allow specifying custom directory index names? At the moment, removing directory index removes `(^|/)((?:default|index)\.\w{1,4})$`.
 
 ## Thanks / Contributions
 
 @rogpeppe
 @jehiah
+@opennota
+@pchristopher1275
+@zenovich
+@beeker1121
 
 ## License
 
@@ -171,4 +183,6 @@ The [BSD 3-Clause license][bsd].
 [bsd]: http://opensource.org/licenses/BSD-3-Clause
 [wiki]: http://en.wikipedia.org/wiki/URL_normalization
 [rfc]: http://tools.ietf.org/html/rfc3986#section-6
-[godoc]: http://go.pkgdoc.org/github.com/puerkitobio/purell
+[godoc]: http://go.pkgdoc.org/github.com/PuerkitoBio/purell
+[pr5]: https://github.com/PuerkitoBio/purell/pull/5
+[iss7]: https://github.com/PuerkitoBio/purell/issues/7

vendor/github.com/PuerkitoBio/purell/go.mod

+module github.com/PuerkitoBio/purell
+
+go 1.19
+
+require (
+	golang.org/x/net v0.0.0-20220617184016-355a448f1bc9
+	golang.org/x/text v0.3.7
+)

vendor/github.com/PuerkitoBio/purell/go.sum

+golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 h1:Yqz/iviulwKwAREEeUd3nbBFn0XuyJqkoft2IlrvOhc=
+golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=

vendor/github.com/PuerkitoBio/purell/purell.go

@@ -12,6 +12,10 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+
+	"golang.org/x/net/idna"
+	"golang.org/x/text/unicode/norm"
+	"golang.org/x/text/width"
 )
 
 // A set of normalization flags determines how a URL will
@@ -20,7 +24,7 @@ type NormalizationFlags uint
 
 const (
 	// Safe normalizations
-	FlagLowercaseScheme           NormalizationFlags = 1 << iota // HTTP://host -> http://host
+	FlagLowercaseScheme           NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1
 	FlagLowercaseHost                                            // http://HOST -> http://host
 	FlagUppercaseEscapes                                         // http://host/t%ef -> http://host/t%EF
 	FlagDecodeUnnecessaryEscapes                                 // http://host/t%41 -> http://host/tA
@@ -135,23 +139,36 @@ var flags = map[NormalizationFlags]func(*url.URL){
 // MustNormalizeURLString returns the normalized string, and panics if an error occurs.
 // It takes an URL string as input, as well as the normalization flags.
 func MustNormalizeURLString(u string, f NormalizationFlags) string {
-	if parsed, e := url.Parse(u); e != nil {
+	result, e := NormalizeURLString(u, f)
+	if e != nil {
 		panic(e)
-	} else {
-		return NormalizeURL(parsed, f)
 	}
-	panic("Unreachable code.")
+	return result
 }
 
 // NormalizeURLString returns the normalized string, or an error if it can't be parsed into an URL object.
 // It takes an URL string as input, as well as the normalization flags.
 func NormalizeURLString(u string, f NormalizationFlags) (string, error) {
-	if parsed, e := url.Parse(u); e != nil {
-		return "", e
-	} else {
-		return NormalizeURL(parsed, f), nil
+	parsed, err := url.Parse(u)
+	if err != nil {
+		return "", err
+	}
+
+	if f&FlagLowercaseHost == FlagLowercaseHost {
+		parsed.Host = strings.ToLower(parsed.Host)
+	}
+
+	// The idna package doesn't fully conform to RFC 5895
+	// (https://tools.ietf.org/html/rfc5895), so we do it here.
+	// Taken from Go 1.8 cycle source, courtesy of bradfitz.
+	// TODO: Remove when (if?) idna package conforms to RFC 5895.
+	parsed.Host = width.Fold.String(parsed.Host)
+	parsed.Host = norm.NFC.String(parsed.Host)
+	if parsed.Host, err = idna.ToASCII(parsed.Host); err != nil {
+		return "", err
 	}
-	panic("Unreachable code.")
+
+	return NormalizeURL(parsed, f), nil
 }
 
 // NormalizeURL returns the normalized string.
@@ -162,7 +179,7 @@ func NormalizeURL(u *url.URL, f NormalizationFlags) string {
 			flags[k](u)
 		}
 	}
-	return u.String()
+	return escapeURL(u)
 }
 
 func lowercaseScheme(u *url.URL) {
@@ -190,20 +207,28 @@ func removeDefaultPort(u *url.URL) {
 }
 
 func removeTrailingSlash(u *url.URL) {
-	if l := len(u.Path); l > 0 && strings.HasSuffix(u.Path, "/") {
+	if l := len(u.Path); l > 0 {
+		if strings.HasSuffix(u.Path, "/") {
 			u.Path = u.Path[:l-1]
-	} else if l = len(u.Host); l > 0 && strings.HasSuffix(u.Host, "/") {
+		}
+	} else if l = len(u.Host); l > 0 {
+		if strings.HasSuffix(u.Host, "/") {
 			u.Host = u.Host[:l-1]
 		}
 	}
+}
 
 func addTrailingSlash(u *url.URL) {
-	if l := len(u.Path); l > 0 && !strings.HasSuffix(u.Path, "/") {
+	if l := len(u.Path); l > 0 {
+		if !strings.HasSuffix(u.Path, "/") {
 			u.Path += "/"
-	} else if l = len(u.Host); l > 0 && !strings.HasSuffix(u.Host, "/") {
+		}
+	} else if l = len(u.Host); l > 0 {
+		if !strings.HasSuffix(u.Host, "/") {
 			u.Host += "/"
 		}
 	}
+}
 
 func removeDotSegments(u *url.URL) {
 	if len(u.Path) > 0 {
@@ -223,7 +248,7 @@ func removeDotSegments(u *url.URL) {
 		}
 		// Special case if host does not end with / and new path does not begin with /
 		u.Path = strings.Join(dotFree, "/")
-		if !strings.HasSuffix(u.Host, "/") && !strings.HasPrefix(u.Path, "/") {
+		if u.Host != "" && !strings.HasSuffix(u.Host, "/") && !strings.HasPrefix(u.Path, "/") {
 			u.Path = "/" + u.Path
 		}
 		// Special case if the last segment was a dot, make sure the path ends with a slash
@@ -273,7 +298,7 @@ func sortQuery(u *url.URL) {
 	if len(q) > 0 {
 		arKeys := make([]string, len(q))
 		i := 0
-		for k, _ := range q {
+		for k := range q {
 			arKeys[i] = k
 			i++
 		}

vendor/github.com/PuerkitoBio/purell/urlesc.go

+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements query escaping as per RFC 3986.
+// It contains some parts of the net/url package, modified so as to allow
+// some reserved characters incorrectly escaped by net/url.
+// See https://github.com/golang/go/issues/5684
+package purell
+
+import (
+	"bytes"
+	"net/url"
+	"strings"
+)
+
+type encoding int
+
+const (
+	encodePath encoding = 1 + iota
+	encodeUserPassword
+	encodeQueryComponent
+	encodeFragment
+)
+
+// Return true if the specified character should be escaped when
+// appearing in a URL string, according to RFC 3986.
+func shouldEscape(c byte, mode encoding) bool {
+	// §2.3 Unreserved characters (alphanum)
+	if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' {
+		return false
+	}
+
+	switch c {
+	case '-', '.', '_', '~': // §2.3 Unreserved characters (mark)
+		return false
+
+	// §2.2 Reserved characters (reserved)
+	case ':', '/', '?', '#', '[', ']', '@', // gen-delims
+		'!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=': // sub-delims
+		// Different sections of the URL allow a few of
+		// the reserved characters to appear unescaped.
+		switch mode {
+		case encodePath: // §3.3
+			// The RFC allows sub-delims and : @.
+			// '/', '[' and ']' can be used to assign meaning to individual path
+			// segments.  This package only manipulates the path as a whole,
+			// so we allow those as well.  That leaves only ? and # to escape.
+			return c == '?' || c == '#'
+
+		case encodeUserPassword: // §3.2.1
+			// The RFC allows : and sub-delims in
+			// userinfo.  The parsing of userinfo treats ':' as special so we must escape
+			// all the gen-delims.
+			return c == ':' || c == '/' || c == '?' || c == '#' || c == '[' || c == ']' || c == '@'
+
+		case encodeQueryComponent: // §3.4
+			// The RFC allows / and ?.
+			return c != '/' && c != '?'
+
+		case encodeFragment: // §4.1
+			// The RFC text is silent but the grammar allows
+			// everything, so escape nothing but #
+			return c == '#'
+		}
+	}
+
+	// Everything else must be escaped.
+	return true
+}
+
+func escape(s string, mode encoding) string {
+	spaceCount, hexCount := 0, 0
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if shouldEscape(c, mode) {
+			if c == ' ' && mode == encodeQueryComponent {
+				spaceCount++
+			} else {
+				hexCount++
+			}
+		}
+	}
+
+	if spaceCount == 0 && hexCount == 0 {
+		return s
+	}
+
+	t := make([]byte, len(s)+2*hexCount)
+	j := 0
+	for i := 0; i < len(s); i++ {
+		switch c := s[i]; {
+		case c == ' ' && mode == encodeQueryComponent:
+			t[j] = '+'
+			j++
+		case shouldEscape(c, mode):
+			t[j] = '%'
+			t[j+1] = "0123456789ABCDEF"[c>>4]
+			t[j+2] = "0123456789ABCDEF"[c&15]
+			j += 3
+		default:
+			t[j] = s[i]
+			j++
+		}
+	}
+	return string(t)
+}
+
+var uiReplacer = strings.NewReplacer(
+	"%21", "!",
+	"%27", "'",
+	"%28", "(",
+	"%29", ")",
+	"%2A", "*",
+)
+
+// unescapeUserinfo unescapes some characters that need not to be escaped as per RFC3986.
+func unescapeUserinfo(s string) string {
+	return uiReplacer.Replace(s)
+}
+
+// Escape reassembles the URL into a valid URL string.
+// The general form of the result is one of:
+//
+//	scheme:opaque
+//	scheme://userinfo@host/path?query#fragment
+//
+// If u.Opaque is non-empty, String uses the first form;
+// otherwise it uses the second form.
+//
+// In the second form, the following rules apply:
+//	- if u.Scheme is empty, scheme: is omitted.
+//	- if u.User is nil, userinfo@ is omitted.
+//	- if u.Host is empty, host/ is omitted.
+//	- if u.Scheme and u.Host are empty and u.User is nil,
+//	   the entire scheme://userinfo@host/ is omitted.
+//	- if u.Host is non-empty and u.Path begins with a /,
+//	   the form host/path does not add its own /.
+//	- if u.RawQuery is empty, ?query is omitted.
+//	- if u.Fragment is empty, #fragment is omitted.
+func escapeURL(u *url.URL) string {
+	var buf bytes.Buffer
+	if u.Scheme != "" {
+		buf.WriteString(u.Scheme)
+		buf.WriteByte(':')
+	}
+	if u.Opaque != "" {
+		buf.WriteString(u.Opaque)
+	} else {
+		if u.Scheme != "" || u.Host != "" || u.User != nil {
+			buf.WriteString("//")
+			if ui := u.User; ui != nil {
+				buf.WriteString(unescapeUserinfo(ui.String()))
+				buf.WriteByte('@')
+			}
+			if h := u.Host; h != "" {
+				buf.WriteString(h)
+			}
+		}
+		if u.Path != "" && u.Path[0] != '/' && u.Host != "" {
+			buf.WriteByte('/')
+		}
+		buf.WriteString(escape(u.Path, encodePath))
+	}
+	if u.RawQuery != "" {
+		buf.WriteByte('?')
+		buf.WriteString(u.RawQuery)
+	}
+	if u.Fragment != "" {
+		buf.WriteByte('#')
+		buf.WriteString(escape(u.Fragment, encodeFragment))
+	}
+	return buf.String()
+}

vendor/github.com/andybalholm/cascadia/go.mod

 module github.com/andybalholm/cascadia
 
-require golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01
+go 1.16
 
-go 1.13
+require golang.org/x/net v0.0.0-20210916014120-12bc252f5db8