vendor/git.autistici.org/ai3/go-common/clientutil/watcher.go

+package clientutil
+
+import (
+	"fmt"
+	"sync"
+	"time"
+)
+
+var dnsWatcherInterval = 1 * time.Minute
+
+// A DNSWatcher monitors a DNS name for changes, constantly attempting
+// to resolve it every minute and notifying a channel when the list of
+// returned IP addresses changes. All addresses must be in host:port
+// format.
+type DNSWatcher struct {
+	hostport string
+	resolver resolver
+	addrs    []string
+	updateCh chan []string
+	stopCh   chan struct{}
+}
+
+// NewDNSWatcher creates a new DNSWatcher.
+func NewDNSWatcher(hostport string) (*DNSWatcher, error) {
+	return newDNSWatcherWithResolver(hostport, defaultResolver)
+}
+
+func newDNSWatcherWithResolver(hostport string, resolver resolver) (*DNSWatcher, error) {
+	// Resolve names once before returning. Return a fatal error
+	// when there are no results, as it may indicate a syntax
+	// error in hostport.
+	addrs := resolver.ResolveIP(hostport)
+	if len(addrs) == 0 {
+		return nil, fmt.Errorf("can't resolve %s", hostport)
+	}
+	w := &DNSWatcher{
+		hostport: hostport,
+		resolver: resolver,
+		addrs:    addrs,
+		updateCh: make(chan []string, 10),
+		stopCh:   make(chan struct{}),
+	}
+	w.updateCh <- addrs
+	go w.loop()
+	return w, nil
+}
+
+// Stop the watcher.
+func (w *DNSWatcher) Stop() {
+	close(w.stopCh)
+}
+
+// Changes returns a channel where the resolved addresses are sent
+// whenever they change.
+func (w *DNSWatcher) Changes() <-chan []string {
+	return w.updateCh
+}
+
+func (w *DNSWatcher) check() {
+	addrs := w.resolver.ResolveIP(w.hostport)
+	if len(addrs) > 0 && !addrListEqual(addrs, w.addrs) {
+		w.addrs = addrs
+		w.updateCh <- addrs
+	}
+}
+
+func (w *DNSWatcher) loop() {
+	defer close(w.updateCh)
+
+	tick := time.NewTicker(dnsWatcherInterval)
+	defer tick.Stop()
+
+	for {
+		select {
+		case <-tick.C:
+			w.check()
+		case <-w.stopCh:
+			return
+		}
+	}
+}
+
+type multiDNSUpdate struct {
+	hostport string
+	addrs    []string
+}
+
+// A MultiDNSWatcher watches multiple addresses for DNS changes. The
+// results are merged and returned as a list of addresses.
+type MultiDNSWatcher struct {
+	watchers []*DNSWatcher
+	addrmap  map[string][]string
+	faninCh  chan multiDNSUpdate
+	updateCh chan []string
+}
+
+// NewMultiDNSWatcher creates a new MultiDNSWatcher.
+func NewMultiDNSWatcher(hostports []string) (*MultiDNSWatcher, error) {
+	return newMultiDNSWatcherWithResolver(hostports, defaultResolver)
+}
+
+func newMultiDNSWatcherWithResolver(hostports []string, resolver resolver) (*MultiDNSWatcher, error) {
+	mw := &MultiDNSWatcher{
+		addrmap:  make(map[string][]string),
+		faninCh:  make(chan multiDNSUpdate, 10),
+		updateCh: make(chan []string, 10),
+	}
+
+	// All the MultiDNSWatcher does is multiplex updates from the
+	// individual DNSWatchers onto faninCh, then merging those
+	// updates with all the others and sending the result to
+	// updateCh.
+	go func() {
+		defer close(mw.updateCh)
+		for up := range mw.faninCh {
+			mw.addrmap[up.hostport] = up.addrs
+			mw.updateCh <- mw.allAddrs()
+		}
+	}()
+
+	var wg sync.WaitGroup
+	for _, hostport := range hostports {
+		w, err := newDNSWatcherWithResolver(hostport, resolver)
+		if err != nil {
+			return nil, err
+		}
+		mw.watchers = append(mw.watchers, w)
+
+		wg.Add(1)
+		go func(hostport string) {
+			for addrs := range w.Changes() {
+				mw.faninCh <- multiDNSUpdate{
+					hostport: hostport,
+					addrs:    addrs,
+				}
+			}
+			wg.Done()
+		}(hostport)
+	}
+
+	go func() {
+		wg.Wait()
+		close(mw.faninCh)
+	}()
+
+	return mw, nil
+}
+
+func (mw *MultiDNSWatcher) allAddrs() []string {
+	var out []string
+	for _, addrs := range mw.addrmap {
+		out = append(out, addrs...)
+	}
+	return out
+}
+
+// Stop the watcher.
+func (mw *MultiDNSWatcher) Stop() {
+	for _, w := range mw.watchers {
+		w.Stop()
+	}
+}
+
+// Changes returns a channel where the aggregate resolved addresses
+// are sent whenever they change.
+func (mw *MultiDNSWatcher) Changes() <-chan []string {
+	return mw.updateCh
+}
+
+func addrListEqual(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+
+	tmp := make(map[string]struct{})
+	for _, aa := range a {
+		tmp[aa] = struct{}{}
+	}
+	for _, bb := range b {
+		if _, ok := tmp[bb]; !ok {
+			return false
+		}
+		delete(tmp, bb)
+	}
+	return len(tmp) == 0
+}

vendor/git.autistici.org/ai3/go-common/misc.go

@@ -2,6 +2,7 @@ package common
 
 import (
 	"crypto/x509"
+	"fmt"
 	"io/ioutil"
 )
 
@@ -12,6 +13,8 @@ func LoadCA(path string) (*x509.CertPool, error) {
 		return nil, err
 	}
 	cas := x509.NewCertPool()
-	cas.AppendCertsFromPEM(data)
+	if !cas.AppendCertsFromPEM(data) {
+		return nil, fmt.Errorf("no certificates could be parsed in %s", path)
+	}
 	return cas, nil
 }

vendor/git.autistici.org/ai3/go-common/renovate.json

+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base"
+  ]
+}

vendor/git.autistici.org/ai3/go-common/serverutil/http.go

 package serverutil
 
 import (
+	"compress/gzip"
 	"context"
 	"crypto/tls"
+	"fmt"
 	"io"
 	"log"
 	"net"
 	"net/http"
-	"net/http/pprof"
+	_ "net/http/pprof"
 	"os"
 	"os/signal"
+	"strconv"
 	"syscall"
 	"time"
 
-	"github.com/coreos/go-systemd/daemon"
+	"git.autistici.org/ai3/go-common/tracing"
+	"github.com/NYTimes/gziphandler"
+	"github.com/coreos/go-systemd/v22/daemon"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
-var gracefulShutdownTimeout = 3 * time.Second
+var (
+	gracefulShutdownTimeout = 3 * time.Second
+
+	gzipLevel        = gzip.BestSpeed
+	gzipMinSize      = 1300
+	gzipContentTypes = []string{
+		"application/json",
+		"application/javascript",
+		"text/html",
+		"text/plain",
+		"text/css",
+	}
+)
 
 // ServerConfig stores common HTTP/HTTPS server configuration parameters.
 type ServerConfig struct {
 	TLS                 *TLSServerConfig `yaml:"tls"`
 	MaxInflightRequests int              `yaml:"max_inflight_requests"`
+	RequestTimeoutSecs  int              `yaml:"request_timeout"`
 	TrustedForwarders   []string         `yaml:"trusted_forwarders"`
+
+	// TODO: switch do disable_compression (flip default) later.
+	EnableCompression bool `yaml:"enable_compression"`
 }
 
-func (config *ServerConfig) buildHTTPServer(h http.Handler) (*http.Server, error) {
+func (config *ServerConfig) buildHTTPHandler(h http.Handler) (http.Handler, *tls.Config, error) {
 	var tlsConfig *tls.Config
 	var err error
 	if config != nil {
 		if config.TLS != nil {
 			tlsConfig, err = config.TLS.TLSConfig()
 			if err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 			h, err = config.TLS.TLSAuthWrapper(h)
 			if err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 		}
 
@@ -47,7 +68,7 @@ func (config *ServerConfig) buildHTTPServer(h http.Handler) (*http.Server, error
 		if len(config.TrustedForwarders) > 0 {
 			h, err = newProxyHeaders(h, config.TrustedForwarders)
 			if err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 		}
 
@@ -56,18 +77,68 @@ func (config *ServerConfig) buildHTTPServer(h http.Handler) (*http.Server, error
 		if config.MaxInflightRequests > 0 {
 			h = newLoadSheddingWrapper(config.MaxInflightRequests, h)
 		}
+
+		// Wrap the handler with a TimeoutHandler if 'request_timeout'
+		// is set.
+		if config.RequestTimeoutSecs > 0 {
+			h = http.TimeoutHandler(h, time.Duration(config.RequestTimeoutSecs)*time.Second, "")
+		}
+	}
+
+	// Add all the default handlers (health, monitoring, etc).
+	h = addDefaultHandlers(h)
+
+	// Optionally enable compression.
+	if config != nil && config.EnableCompression {
+		gzwrap, err := gziphandler.GzipHandlerWithOpts(
+			gziphandler.CompressionLevel(gzipLevel),
+			gziphandler.MinSize(gzipMinSize),
+			gziphandler.ContentTypes(gzipContentTypes),
+		)
+		if err != nil {
+			return nil, nil, err
+		}
+		h = gzwrap(h)
+	}
+
+	return h, tlsConfig, nil
+}
+
+func buildListener(addr string, tlsConfig *tls.Config) (net.Listener, error) {
+	// Create the net.Listener first, so we can detect
+	// initialization-time errors safely.
+	l, err := net.Listen("tcp", addr)
+	if err != nil {
+		return nil, err
+	}
+	if tlsConfig != nil {
+		l = tls.NewListener(l, tlsConfig)
+	}
+	return l, nil
+}
+
+func buildServer(h http.Handler, config *ServerConfig, addr string) (*http.Server, error) {
+	// Wrap with tracing handler (exclude metrics and other
+	// debugging endpoints).
+	h = tracing.WrapHandler(h, guessEndpointName(addr))
+
+	// Create the top-level HTTP handler with all our additions.
+	hh, tlsConfig, err := config.buildHTTPHandler(h)
+	if err != nil {
+		return nil, err
 	}
 
 	// These are not meant to be external-facing servers, so we
 	// can be generous with the timeouts to keep the number of
 	// reconnections low.
-	return &http.Server{
-		Handler:      defaultHandler(h),
-		ReadTimeout:  30 * time.Second,
-		WriteTimeout: 30 * time.Second,
-		IdleTimeout:  600 * time.Second,
-		TLSConfig:    tlsConfig,
-	}, nil
+	srv := &http.Server{
+		Handler:           hh,
+		ReadHeaderTimeout: 30 * time.Second,
+		IdleTimeout:       600 * time.Second,
+		TLSConfig:         tlsConfig,
+	}
+
+	return srv, nil
 }
 
 // Serve HTTP(S) content on the specified address. If config.TLS is
@@ -77,21 +148,15 @@ func (config *ServerConfig) buildHTTPServer(h http.Handler) (*http.Server, error
 // the listener, otherwise it will handle graceful termination on
 // SIGINT or SIGTERM and return nil.
 func Serve(h http.Handler, config *ServerConfig, addr string) error {
-	// Create the HTTP server.
-	srv, err := config.buildHTTPServer(h)
+	srv, err := buildServer(h, config, addr)
 	if err != nil {
 		return err
 	}
 
-	// Create the net.Listener first, so we can detect
-	// initialization-time errors safely.
-	l, err := net.Listen("tcp", addr)
+	l, err := buildListener(addr, srv.TLSConfig)
 	if err != nil {
 		return err
 	}
-	if srv.TLSConfig != nil {
-		l = tls.NewListener(l, srv.TLSConfig)
-	}
 
 	// Install a signal handler for gentle process termination.
 	done := make(chan struct{})
@@ -115,8 +180,9 @@ func Serve(h http.Handler, config *ServerConfig, addr string) error {
 
 	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
 
-	// Notify systemd that we are ready to serve.
-	daemon.SdNotify(false, "READY=1")
+	// Notify systemd that we are ready to serve. This call is
+	// allowed to fail (in case there is no systemd).
+	daemon.SdNotify(false, "READY=1") // nolint
 
 	err = srv.Serve(l)
 	if err != http.ErrServerClosed {
@@ -127,29 +193,94 @@ func Serve(h http.Handler, config *ServerConfig, addr string) error {
 	return nil
 }
 
-func defaultHandler(h http.Handler) http.Handler {
+// ServeWithContext operates like Serve but with a controlling Context
+// that can be used to stop the HTTP server.
+func ServeWithContext(ctx context.Context, h http.Handler, config *ServerConfig, addr string) error {
+	srv, err := buildServer(h, config, addr)
+	if err != nil {
+		return err
+	}
+
+	l, err := buildListener(addr, srv.TLSConfig)
+	if err != nil {
+		return err
+	}
+
+	go func() {
+		<-ctx.Done()
+
+		sctx, cancel := context.WithTimeout(context.Background(), gracefulShutdownTimeout)
+		srv.Shutdown(sctx) // nolint: errcheck
+		srv.Close()
+		cancel()
+	}()
+
+	daemon.SdNotify(false, "READY=1") // nolint
+
+	err = srv.Serve(l)
+	if err == http.ErrServerClosed {
+		err = nil
+	}
+
+	return err
+}
+
+func addDefaultHandlers(h http.Handler) http.Handler {
 	root := http.NewServeMux()
 
 	// Add an endpoint for HTTP health checking probes.
 	root.Handle("/health", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		io.WriteString(w, "OK")
+		io.WriteString(w, "OK") // nolint
 	}))
 
 	// Add an endpoint to serve Prometheus metrics.
 	root.Handle("/metrics", promhttp.Handler())
 
-	// Add the net/http/pprof debug handlers.
-	root.Handle("/debug/pprof/", pprof.Handler(""))
+	// Let the default net/http handler deal with /debug/
+	// URLs. Packages such as net/http/pprof register their
+	// handlers there in ways that aren't reproducible.
+	root.Handle("/debug/", http.DefaultServeMux)
 
 	// Forward everything else to the main handler, adding
 	// Prometheus instrumentation (requests to /metrics and
 	// /health are not included).
 	root.Handle("/", promhttp.InstrumentHandlerInFlight(inFlightRequests,
-		promhttp.InstrumentHandlerCounter(totalRequests, h)))
+		promhttp.InstrumentHandlerCounter(totalRequests,
+			propagateDeadline(h))))
 
 	return root
 }
 
+const deadlineHeader = "X-RPC-Deadline"
+
+// Read an eventual deadline from the HTTP request, and set it as the
+// deadline of the request context.
+func propagateDeadline(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		if hdr := req.Header.Get(deadlineHeader); hdr != "" {
+			if deadlineNano, err := strconv.ParseInt(hdr, 10, 64); err == nil {
+				deadline := time.Unix(0, deadlineNano)
+				ctx, cancel := context.WithDeadline(req.Context(), deadline)
+				defer cancel()
+				req = req.WithContext(ctx)
+			}
+		}
+		h.ServeHTTP(w, req)
+	})
+}
+
+func guessEndpointName(addr string) string {
+	_, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return addr
+	}
+	host, err := os.Hostname()
+	if err != nil {
+		return addr
+	}
+	return fmt.Sprintf("%s:%s", host, port)
+}
+
 // HTTP-related metrics.
 var (
 	// Since we instrument the root HTTP handler, we don't really

vendor/git.autistici.org/ai3/go-common/serverutil/json.go

@@ -3,6 +3,7 @@ package serverutil
 import (
 	"encoding/json"
 	"log"
+	"mime"
 	"net/http"
 )
 
@@ -14,7 +15,7 @@ func DecodeJSONRequest(w http.ResponseWriter, r *http.Request, obj interface{})
 		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
 		return false
 	}
-	if r.Header.Get("Content-Type") != "application/json" {
+	if ct, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type")); ct != "application/json" {
 		http.Error(w, "Need JSON request", http.StatusBadRequest)
 		return false
 	}
@@ -29,19 +30,15 @@ func DecodeJSONRequest(w http.ResponseWriter, r *http.Request, obj interface{})
 
 // EncodeJSONResponse writes an application/json response to w.
 func EncodeJSONResponse(w http.ResponseWriter, obj interface{}) {
-	data, err := json.Marshal(obj)
-	if err != nil {
-		log.Printf("JSON serialization error: %v", err)
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-
 	w.Header().Set("Content-Type", "application/json")
 	w.Header().Set("Pragma", "no-cache")
 	w.Header().Set("Cache-Control", "no-store")
 	w.Header().Set("Expires", "-1")
 	w.Header().Set("X-Content-Type-Options", "nosniff")
-	if _, err = w.Write(data); err != nil {
-		log.Printf("error writing response: %v", err)
+
+	err := json.NewEncoder(w).Encode(obj)
+	if err != nil {
+		log.Printf("error writing JSON response: %v", err)
+		// Too late to return an error to the client now.
 	}
 }

vendor/git.autistici.org/ai3/go-common/serverutil/proxy_headers.go

@@ -4,13 +4,12 @@ import (
 	"fmt"
 	"net"
 	"net/http"
-
-	"github.com/gorilla/handlers"
+	"strings"
 )
 
 type proxyHeaders struct {
-	wrap, phWrap http.Handler
-	forwarders   []net.IPNet
+	wrap       http.Handler
+	forwarders []net.IPNet
 }
 
 func newProxyHeaders(h http.Handler, trustedForwarders []string) (http.Handler, error) {
@@ -20,7 +19,6 @@ func newProxyHeaders(h http.Handler, trustedForwarders []string) (http.Handler,
 	}
 	return &proxyHeaders{
 		wrap:       h,
-		phWrap:     handlers.ProxyHeaders(h),
 		forwarders: f,
 	}, nil
 }
@@ -32,12 +30,28 @@ func (p *proxyHeaders) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 	ip := net.ParseIP(host)
 	if ip != nil && matchIPNetList(ip, p.forwarders) {
-		p.phWrap.ServeHTTP(w, r)
-		return
+		if fwd := getForwardedIP(r); fwd != "" {
+			r.RemoteAddr = fwd
+		}
 	}
 	p.wrap.ServeHTTP(w, r)
 }
 
+// Parse the X-Real-IP or X-Forwarded-For headers, if present, to get
+// the original client IP.
+func getForwardedIP(r *http.Request) string {
+	if s := r.Header.Get("X-Real-IP"); s != "" {
+		return s
+	}
+	if s := r.Header.Get("X-Forwarded-For"); s != "" {
+		if n := strings.IndexByte(s, ','); n > 0 {
+			s = s[:n]
+		}
+		return s
+	}
+	return ""
+}
+
 func fullMask(ip net.IP) net.IPMask {
 	if ip.To4() == nil {
 		return net.CIDRMask(128, 128)

vendor/git.autistici.org/ai3/go-common/serverutil/tls.go

@@ -2,15 +2,20 @@ package serverutil
 
 import (
 	"crypto/tls"
+	"errors"
+	"fmt"
+	"log"
 	"net/http"
 	"regexp"
+	"strings"
 
 	common "git.autistici.org/ai3/go-common"
 )
 
 // TLSAuthACL describes a single access control entry. Path and
 // CommonName are anchored regular expressions (they must match the
-// entire string).
+// entire string). The first path to match in a list of ACLs will
+// identify the ACL to be applied.
 type TLSAuthACL struct {
 	Path       string `yaml:"path"`
 	CommonName string `yaml:"cn"`
@@ -28,10 +33,11 @@ func (p *TLSAuthACL) compile() error {
 	return err
 }
 
-func (p *TLSAuthACL) match(req *http.Request) bool {
-	if !p.pathRx.MatchString(req.URL.Path) {
-		return false
-	}
+func (p *TLSAuthACL) matchPath(req *http.Request) bool {
+	return p.pathRx.MatchString(req.URL.Path)
+}
+
+func (p *TLSAuthACL) matchCN(req *http.Request) bool {
 	for _, cert := range req.TLS.PeerCertificates {
 		if p.cnRx.MatchString(cert.Subject.CommonName) {
 			return true
@@ -40,6 +46,31 @@ func (p *TLSAuthACL) match(req *http.Request) bool {
 	return false
 }
 
+// TLSAuthACLListFlag is a convenience type that allows callers to use
+// the 'flag' package to specify a list of TLSAuthACL objects. It
+// implements the flag.Value interface.
+type TLSAuthACLListFlag []*TLSAuthACL
+
+func (l TLSAuthACLListFlag) String() string {
+	var out []string
+	for _, acl := range l {
+		out = append(out, fmt.Sprintf("%s:%s", acl.Path, acl.CommonName))
+	}
+	return strings.Join(out, ",")
+}
+
+func (l *TLSAuthACLListFlag) Set(value string) error {
+	parts := strings.SplitN(value, ":", 2)
+	if len(parts) != 2 {
+		return errors.New("bad acl format")
+	}
+	*l = append(*l, &TLSAuthACL{
+		Path:       parts[0],
+		CommonName: parts[1],
+	})
+	return nil
+}
+
 // TLSAuthConfig stores access control lists for TLS authentication. Access
 // control lists are matched against the request path and the
 // CommonName component of the peer certificate subject.
@@ -52,9 +83,13 @@ func (c *TLSAuthConfig) match(req *http.Request) bool {
 	if c == nil || len(c.Allow) == 0 {
 		return true
 	}
+
 	for _, acl := range c.Allow {
-		if acl.match(req) {
-			return true
+		if acl.matchPath(req) {
+			if acl.matchCN(req) {
+				return true
+			}
+			break
 		}
 	}
 	return false
@@ -81,21 +116,27 @@ func (c *TLSServerConfig) TLSConfig() (*tls.Config, error) {
 		return nil, err
 	}
 
-	cas, err := common.LoadCA(c.CA)
-	if err != nil {
-		return nil, err
-	}
-
 	// Set some TLS-level parameters (cipher-related), assuming
 	// we're using EC keys.
 	tlsConf := &tls.Config{
 		Certificates:             []tls.Certificate{cert},
-		ClientAuth:               tls.RequireAndVerifyClientCert,
-		ClientCAs:                cas,
 		CipherSuites:             serverCiphers,
 		MinVersion:               tls.VersionTLS12,
 		PreferServerCipherSuites: true,
+		NextProtos:               []string{"h2", "http/1.1"},
+	}
+
+	// Require client certificates if a CA is specified.
+	if c.CA != "" {
+		cas, err := common.LoadCA(c.CA)
+		if err != nil {
+			return nil, err
+		}
+
+		tlsConf.ClientAuth = tls.RequireAndVerifyClientCert
+		tlsConf.ClientCAs = cas
 	}
+
 	tlsConf.BuildNameToCertificate()
 
 	return tlsConf, nil
@@ -119,6 +160,13 @@ func (c *TLSServerConfig) TLSAuthWrapper(h http.Handler) (http.Handler, error) {
 			h.ServeHTTP(w, r)
 			return
 		}
-		http.Error(w, "Unauthorized", http.StatusUnauthorized)
+
+		// Log the failed access, useful for debugging.
+		var tlsmsg string
+		if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 {
+			tlsmsg = fmt.Sprintf("TLS client '%s' at", r.TLS.PeerCertificates[0].Subject.CommonName)
+		}
+		log.Printf("unauthorized access to %s from %s%s", r.URL.Path, tlsmsg, r.RemoteAddr)
+		http.Error(w, "Forbidden", http.StatusForbidden)
 	}), nil
 }

vendor/git.autistici.org/ai3/go-common/tracing/tracing.go

+package tracing
+
+import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+
+	othttp "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	b3 "go.opentelemetry.io/contrib/propagators/b3"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/zipkin"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	"go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.12.0"
+)
+
+var (
+	// Enabled reports whether tracing is globally enabled or not.
+	Enabled bool
+
+	initOnce sync.Once
+)
+
+const globalTracingConfigPath = "/etc/tracing/client.conf"
+
+type tracingConfig struct {
+	ReportURL string `json:"report_url"`
+	Sample    string `json:"sample"`
+}
+
+// Read the global tracing configuration file. Its location is
+// hardcoded, but it can be overriden using the TRACING_CONFIG
+// environment variable.
+func readTracingConfig() (*tracingConfig, error) {
+	// Read and decode configuration.
+	cfgPath := globalTracingConfigPath
+	if s := os.Getenv("TRACING_CONFIG"); s != "" {
+		cfgPath = s
+	}
+	data, err := ioutil.ReadFile(cfgPath)
+	if err != nil {
+		return nil, err
+	}
+
+	var config tracingConfig
+	if err := json.Unmarshal(data, &config); err != nil {
+		log.Printf("warning: error in tracing configuration: %v, tracing disabled", err)
+		return nil, err
+	}
+
+	if config.ReportURL == "" {
+		log.Printf("warning: tracing configuration contains no report_url, tracing disabled")
+		return nil, errors.New("no report_url")
+	}
+
+	return &config, nil
+}
+
+// Compute the service name for Zipkin: this is usually the program
+// name (without path), but it can be overriden by the TRACING_SERVICE
+// environment variable.
+func getServiceName() string {
+	if s := os.Getenv("TRACING_SERVICE"); s != "" {
+		return s
+	}
+	if s, err := os.Executable(); err == nil {
+		return filepath.Base(s)
+	}
+	return "unknown_service"
+}
+
+func defaultResource(serviceName string) *resource.Resource {
+	attrs := []attribute.KeyValue{
+		semconv.ServiceNameKey.String(serviceName),
+	}
+
+	if s, err := os.Hostname(); err == nil {
+		attrs = append(attrs, semconv.HostNameKey.String(s))
+	}
+	if s := os.Getenv("FLOAT_SERVICE"); s != "" {
+		attrs = append(attrs, attribute.Key("float.service").String(s))
+	}
+	if s := os.Getenv("FLOAT_CONTAINER_NAME"); s != "" {
+		attrs = append(attrs, semconv.ContainerNameKey.String(s))
+	}
+	if s := os.Getenv("FLOAT_CONTAINER_IMAGE"); s != "" {
+		imageName := s
+		tag := "latest"
+		if strings.Contains(imageName, ":") {
+			parts := strings.SplitN(imageName, ":", 2)
+			imageName = parts[0]
+			tag = parts[1]
+		}
+
+		attrs = append(attrs, semconv.ContainerImageNameKey.String(imageName))
+		attrs = append(attrs, semconv.ContainerImageTagKey.String(tag))
+	}
+
+	r, _ := resource.Merge(
+		resource.Default(),
+		resource.NewWithAttributes(semconv.SchemaURL, attrs...),
+	)
+	return r
+}
+
+// Initialize tracing. Tracing will be enabled if the system-wide
+// tracing configuration file is present and valid. Explicitly set
+// TRACING_ENABLE=0 in the environment to disable tracing.
+//
+// We need to check the configuration as soon as possible, because
+// it's likely that client transports are created before HTTP servers,
+// and we need to wrap them with opencensus at creation time.
+func initTracing(serviceName string) {
+	initOnce.Do(func() {
+		// Kill switch from environment.
+		if s := os.Getenv("TRACING_ENABLE"); s == "0" {
+			return
+		}
+
+		config, err := readTracingConfig()
+		if err != nil {
+			return
+		}
+
+		ze, err := zipkin.New(config.ReportURL)
+		if err != nil {
+			log.Printf("error creating Zipkin exporter: %v", err)
+			return
+		}
+
+		// The sampling policy only applies to incoming requests for
+		// which tracing is not already enabled: in this case, we
+		// always pass-through.
+		var sampler trace.Sampler
+		switch config.Sample {
+		case "", "always":
+			sampler = trace.AlwaysSample()
+		case "never":
+			sampler = trace.NeverSample()
+		default:
+			frac, err := strconv.ParseFloat(config.Sample, 64)
+			if err != nil {
+				log.Printf("warning: error in tracing configuration: sample: %v, tracing disabled", err)
+				return
+			}
+			sampler = trace.TraceIDRatioBased(frac)
+		}
+
+		tp := trace.NewTracerProvider(
+			trace.WithSampler(trace.ParentBased(sampler)),
+			trace.WithBatcher(ze),
+			trace.WithResource(defaultResource(serviceName)),
+		)
+
+		otel.SetTracerProvider(tp)
+
+		otel.SetTextMapPropagator(
+			propagation.NewCompositeTextMapPropagator(
+				propagation.TraceContext{},
+				propagation.Baggage{},
+				b3.New()))
+
+		log.Printf("tracing enabled (report_url %s)", config.ReportURL)
+
+		Enabled = true
+	})
+}
+
+// Init tracing support, if not using WrapHandler.
+func Init() {
+	initTracing(getServiceName())
+}
+
+// WrapTransport optionally wraps a http.RoundTripper with OpenCensus
+// tracing functionality, if it is globally enabled.
+//
+// Must call Init() first.
+func WrapTransport(t http.RoundTripper) http.RoundTripper {
+	if Enabled {
+		t = othttp.NewTransport(t)
+	}
+	return t
+}
+
+// WrapHandler wraps a http.Handler with OpenCensus tracing
+// functionality, if globally enabled. Automatically calls Init().
+func WrapHandler(h http.Handler, endpointAddr string) http.Handler {
+	serviceName := getServiceName()
+	initTracing(serviceName)
+	if !Enabled {
+		return h
+	}
+
+	// Format span names with the request URL path.
+	return othttp.NewHandler(
+		h, serviceName,
+		othttp.WithSpanNameFormatter(func(op string, r *http.Request) string {
+			return r.URL.Path
+		}),
+	)
+}

vendor/git.autistici.org/ai3/tools/replds/.gitlab-ci.yml

+include:
+  - "https://git.autistici.org/pipelines/debian/raw/master/common.yml"
+  - "https://git.autistici.org/pipelines/images/test/golang/raw/master/ci.yml"

vendor/git.autistici.org/ai3/tools/replds/LICENSE

+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

vendor/git.autistici.org/ai3/replds/README.md → vendor/git.autistici.org/ai3/tools/replds/README.md

@@ -122,3 +122,15 @@ directory* to run the repository tools on, and the final workflow is:
 * run the metadata-generation tools on the staging dir;
 * synchronize the data back to replds using the *sync* command.
 
+## Usage
+
+The Debian package comes with a
+[replds-instance-create](debian/replds-instance-create) script that
+can be used to set up multiple replds instances. For an instance named
+*foo*, the script will setup the *replds@foo* systemd service, and it
+will create the *replds-foo* user and group. Add users that need to
+read the repository files to that group. The configuration will be
+read from */etc/replds/foo.yml*.
+
+Note that files created by the daemon will be world-readable by
+default. Set the process umask if you wish to restrict this further.

vendor/git.autistici.org/ai3/replds/client.go → vendor/git.autistici.org/ai3/tools/replds/client.go

@@ -31,17 +31,17 @@ func newClient(config *clientutil.BackendConfig) (*client, error) {
 
 func (c *client) internalGetNodes(ctx context.Context, req *internalGetNodesRequest) (*internalGetNodesResponse, error) {
 	var resp internalGetNodesResponse
-	err := clientutil.DoJSONHTTPRequest(ctx, c.be.Client(""), c.be.URL("")+"/api/internal/get_nodes", req, &resp)
+	err := c.be.Call(ctx, "", "/api/internal/get_nodes", req, &resp)
 	return &resp, err
 }
 
 func (c *client) internalUpdateNodes(ctx context.Context, req *internalUpdateNodesRequest) error {
-	return clientutil.DoJSONHTTPRequest(ctx, c.be.Client(""), c.be.URL("")+"/api/internal/update_nodes", req, nil)
+	return c.be.Call(ctx, "", "/api/internal/update_nodes", req, nil)
 }
 
 func (c *client) SetNodes(ctx context.Context, req *SetNodesRequest) (*SetNodesResponse, error) {
 	var resp SetNodesResponse
-	err := clientutil.DoJSONHTTPRequest(ctx, c.be.Client(""), c.be.URL("")+"/api/set_nodes", req, &resp)
+	err := c.be.Call(ctx, "", "/api/set_nodes", req, &resp)
 	return &resp, err
 }
 

vendor/git.autistici.org/ai3/replds/fs.go → vendor/git.autistici.org/ai3/tools/replds/fs.go

@@ -20,9 +20,9 @@ func newFS(dir string) *FS {
 
 // getAllNodes scans the root directory and returns a list of
 // Nodes. All errors are ignored.
-func (f *FS) getAllNodes() []Node {
-	var nodes []Node
-	dirLen := len(f.dir)
+func (f *FS) getAllNodes() []*Node {
+	var nodes []*Node
+	dirOffset := len(f.dir) + 1
 	if err := filepath.Walk(f.dir, func(path string, info os.FileInfo, err error) error {
 		// Ignore errors.
 		if err != nil {
@@ -32,8 +32,8 @@ func (f *FS) getAllNodes() []Node {
 		if !info.Mode().IsRegular() {
 			return nil
 		}
-		nodes = append(nodes, Node{
-			Path:      path[dirLen+1:],
+		nodes = append(nodes, &Node{
+			Path:      path[dirOffset:],
 			Timestamp: info.ModTime(),
 		})
 		return nil
@@ -44,7 +44,7 @@ func (f *FS) getAllNodes() []Node {
 }
 
 // setNode applies a Node to the underlying filesystem.
-func (f *FS) setNode(node Node) error {
+func (f *FS) setNode(node *Node) error {
 	path := filepath.Join(f.dir, node.Path)
 
 	// Tombstone? Then delete the file (ignore errors).
@@ -55,7 +55,7 @@ func (f *FS) setNode(node Node) error {
 	// Create parent directory if it does not exist.
 	dir := filepath.Dir(path)
 	if _, err := os.Stat(dir); os.IsNotExist(err) {
-		if err = os.MkdirAll(dir, 0700); err != nil {
+		if err = os.MkdirAll(dir, 0755); err != nil {
 			return err
 		}
 	}
@@ -81,7 +81,7 @@ func writeFile(path string, data []byte) error {
 	err := os.Rename(tmp, path)
 	if err != nil {
 		// Clean up if something went wrong.
-		os.Remove(tmp)
+		os.Remove(tmp) // nolint
 	}
 	return err
 }

vendor/git.autistici.org/ai3/replds/http.go → vendor/git.autistici.org/ai3/tools/replds/http.go

@@ -84,7 +84,7 @@ func (s *HTTPServer) Handler() http.Handler {
 	return h
 }
 
-func nodes2str(nodes []Node) string {
+func nodes2str(nodes []*Node) string {
 	var tmp []string
 	for _, node := range nodes {
 		tmp = append(tmp, fmt.Sprintf("%s@%d", node.Path, node.Timestamp.Unix()))

vendor/git.autistici.org/ai3/tools/replds/renovate.json

+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base"
+  ]
+}

vendor/git.autistici.org/ai3/replds/server.go → vendor/git.autistici.org/ai3/tools/replds/server.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"log"
+	"math/rand"
 	"strings"
 	"sync"
 	"time"
@@ -21,7 +22,7 @@ var (
 	pollPeriod = 120 * time.Second
 
 	// Timeout for InternalGetNodes requests.
-	getNodesTimeout = 60 * time.Second
+	getNodesTimeout = 120 * time.Second
 )
 
 // Node is an annotated path/value entry.
@@ -34,6 +35,15 @@ type Node struct {
 	Deleted bool `json:"deleted,omitempty"`
 }
 
+func (n *Node) Copy() *Node {
+	return &Node{
+		Path:      n.Path,
+		Value:     n.Value,
+		Timestamp: n.Timestamp,
+		Deleted:   n.Deleted,
+	}
+}
+
 func (n *Node) withoutValue() *Node {
 	return &Node{
 		Path:      n.Path,
@@ -42,29 +52,29 @@ func (n *Node) withoutValue() *Node {
 	}
 }
 
-func (n *Node) metadataOnly() Node {
-	return Node{
+func (n *Node) metadataOnly() *Node {
+	return &Node{
 		Path:      n.Path,
 		Timestamp: n.Timestamp,
 	}
 }
 
 type internalGetNodesRequest struct {
-	Nodes []Node `json:"nodes"`
+	Nodes []*Node `json:"nodes"`
 }
 
 type internalGetNodesResponse struct {
-	Nodes   []Node `json:"nodes"`
-	Partial bool   `json:"partial,omitempty"`
+	Nodes   []*Node `json:"nodes"`
+	Partial bool    `json:"partial,omitempty"`
 }
 
 type internalUpdateNodesRequest struct {
-	Nodes []Node `json:"nodes"`
+	Nodes []*Node `json:"nodes"`
 }
 
 // SetNodesRequest is the request type for the SetNodes method.
 type SetNodesRequest struct {
-	Nodes []Node `json:"nodes"`
+	Nodes []*Node `json:"nodes"`
 }
 
 // SetNodesResponse is the response returned by the SetNodes method.
@@ -76,17 +86,18 @@ type SetNodesResponse struct {
 var errTooOld = errors.New("a more recent value exists for this key")
 
 type storage interface {
-	getAllNodes() []Node
+	getAllNodes() []*Node
 	getNodeValue(string) ([]byte, error)
-	setNode(Node) error
+	setNode(*Node) error
 }
 
 // Server for the replicated filesync.
 type Server struct {
-	peers   []string
-	network *network
-	storage storage
-	logger  *log.Logger
+	peers    []string
+	network  *network
+	storage  storage
+	logger   *log.Logger
+	readonly bool
 
 	wg    sync.WaitGroup
 	stop  chan bool
@@ -103,7 +114,7 @@ func stripTrailingSlash(peers []string) []string {
 }
 
 // NewServer creates a new Server with the given peers and backends.
-func NewServer(peers []string, dir string, tlsConfig *clientutil.TLSClientConfig) (*Server, error) {
+func NewServer(peers []string, dir string, tlsConfig *clientutil.TLSClientConfig, readonly bool) (*Server, error) {
 	peers = stripTrailingSlash(peers)
 
 	network, err := newNetwork(peers, tlsConfig)
@@ -112,26 +123,44 @@ func NewServer(peers []string, dir string, tlsConfig *clientutil.TLSClientConfig
 	}
 
 	s := &Server{
-		peers:   peers,
-		network: network,
-		storage: newFS(dir),
-		stop:    make(chan bool),
-		nodes:   make(map[string]*Node),
+		peers:    peers,
+		network:  network,
+		storage:  newFS(dir),
+		stop:     make(chan bool),
+		nodes:    make(map[string]*Node),
+		readonly: readonly,
 	}
 
-	for _, node := range s.storage.getAllNodes() {
-		s.nodes[node.Path] = &node
-	}
-	if len(s.nodes) > 0 {
-		log.Printf("found %d entries in %s", len(s.nodes), dir)
+	// Do not scan the local filesystem at startup on readonly
+	// instances, downloading everything is safer as we do not
+	// risk injecting stale data.
+	if !readonly {
+		for _, node := range s.storage.getAllNodes() {
+			s.nodes[node.Path] = node
+		}
+		if len(s.nodes) > 0 {
+			log.Printf("found %d entries in %s", len(s.nodes), dir)
+		}
 	}
 
-	for _, peer := range peers {
+	// The background workers are different if the instance is
+	// readonly: normally we run a synchronization worker for each
+	// peer, while in the readonly case we periodically poll a
+	// different random peer.
+	if readonly {
 		s.wg.Add(1)
-		go func(peer string) {
-			s.pollThread(peer)
+		go func() {
+			s.pollRandomPeerThread()
 			s.wg.Done()
-		}(peer)
+		}()
+	} else {
+		for _, peer := range peers {
+			s.wg.Add(1)
+			go func(peer string) {
+				s.pollThread(peer)
+				s.wg.Done()
+			}(peer)
+		}
 	}
 
 	return s, nil
@@ -156,7 +185,7 @@ func (s *Server) log(fmt string, args ...interface{}) {
 // Update one or more nodes as a "transaction": if one update fails,
 // the rest are aborted (but previous commits are not reverted, yet).
 // Will ping peers with the data when updatePeers is true.
-func (s *Server) doSetNodes(ctx context.Context, nodes []Node, updatePeers bool) (*SetNodesResponse, error) {
+func (s *Server) doSetNodes(ctx context.Context, nodes []*Node, updatePeers bool) (*SetNodesResponse, error) {
 	// Update local state.
 	for _, node := range nodes {
 		cur, ok := s.nodes[node.Path]
@@ -185,7 +214,9 @@ func (s *Server) doSetNodes(ctx context.Context, nodes []Node, updatePeers bool)
 
 	var res SetNodesResponse
 	if updatePeers {
-		// Ping remote nodes with updated state.
+		// Ping remote nodes with the updated state. This
+		// reduces the latency of update propagation (we don't
+		// have to wait until the peers poll us).
 		res.HostsOk = 1
 		for _, peer := range s.peers {
 			err := s.network.Client(peer).internalUpdateNodes(ctx, &internalUpdateNodesRequest{
@@ -216,13 +247,13 @@ func (s *Server) internalUpdateNodes(ctx context.Context, req *internalUpdateNod
 	return err
 }
 
-func (s *Server) getNodeWithValue(node *Node) (out Node, err error) {
-	out = *node
+func (s *Server) getNodeWithValue(node *Node) (out *Node, err error) {
+	out = node.Copy()
 	out.Value, err = s.storage.getNodeValue(node.Path)
 	return
 }
 
-func nodeDiff(reqNodes []Node, myNodes map[string]*Node, adder func(*Node) bool) ([]string, bool) {
+func nodeDiff(reqNodes []*Node, myNodes map[string]*Node, adder func(*Node) bool) ([]string, bool) {
 	var missing []string
 	tmp := make(map[string]struct{})
 	for _, reqNode := range reqNodes {
@@ -293,14 +324,8 @@ func (s *Server) internalGetNodes(ctx context.Context, req *internalGetNodesRequ
 	return &resp, nil
 }
 
-func (s *Server) pollThread(peer string) {
-	poll := func() {
-		if err := s.pollPeer(peer); err != nil {
-			s.log("error polling peer %s: %v", peer, err)
-		}
-	}
-
-	poll()
+func (s *Server) backgroundThread(fn func()) {
+	fn()
 
 	tick := time.NewTicker(pollPeriod)
 	defer tick.Stop()
@@ -309,11 +334,32 @@ func (s *Server) pollThread(peer string) {
 		case <-s.stop:
 			return
 		case <-tick.C:
-			poll()
+			fn()
 		}
 	}
 }
 
+func (s *Server) pollThread(peer string) {
+	s.backgroundThread(func() {
+		if err := s.pollPeer(peer); err != nil {
+			s.log("error polling peer %s: %v", peer, err)
+		}
+	})
+}
+
+func (s *Server) pollRandomPeerThread() {
+	s.backgroundThread(func() {
+		if len(s.peers) == 0 {
+			s.log("no peers to poll!")
+			return
+		}
+		peer := s.peers[rand.Intn(len(s.peers))]
+		if err := s.pollPeer(peer); err != nil {
+			s.log("error polling peer %s: %v", peer, err)
+		}
+	})
+}
+
 func (s *Server) pollPeer(peer string) error {
 	for {
 		partial, err := s.pollPeerRequest(peer)
@@ -353,7 +399,7 @@ var (
 	maxTSValue int64
 )
 
-func updateMaxTimestamp(nodes []Node) {
+func updateMaxTimestamp(nodes []*Node) {
 	var max int64
 	for _, node := range nodes {
 		t := node.Timestamp.Unix()
@@ -375,6 +421,9 @@ func (s *Server) pollPeerRequest(peer string) (bool, error) {
 	peerName := stripHTTP(peer)
 	peerRequests.With(prometheus.Labels{"peer": peerName}).Inc()
 
+	// Create a shallow copy of the list of the Nodes we know
+	// about, containing just path/timestamp, to send to the peer
+	// in our GetNodes request.
 	s.mx.Lock()
 	var req internalGetNodesRequest
 	for _, node := range s.nodes {
@@ -399,6 +448,7 @@ func (s *Server) pollPeerRequest(peer string) (bool, error) {
 		return false, nil
 	}
 
+	// Update the internal state with the new data.
 	return resp.Partial, s.internalUpdateNodes(ctx, &internalUpdateNodesRequest{Nodes: resp.Nodes})
 }
 

vendor/github.com/NYTimes/gziphandler/.gitignore

+*.swp

vendor/github.com/NYTimes/gziphandler/.travis.yml

+language: go
+go:
+  - 1.x
+  - tip
+env:
+  - GO111MODULE=on
+install:
+  - go mod download
+script:
+  - go test -race -v

vendor/github.com/NYTimes/gziphandler/CODE_OF_CONDUCT.md

+---
+layout: code-of-conduct
+version: v1.0
+---
+
+This code of conduct outlines our expectations for participants within the **NYTimes/gziphandler** community, as well as steps to reporting unacceptable behavior. We are committed to providing a welcoming and inspiring community for all and expect our code of conduct to be honored. Anyone who violates this code of conduct may be banned from the community.
+
+Our open source community strives to:
+
+* **Be friendly and patient.**
+* **Be welcoming**: We strive to be a community that welcomes and supports people of all backgrounds and identities. This includes, but is not limited to members of any race, ethnicity, culture, national origin, colour, immigration status, social and economic class, educational level, sex, sexual orientation, gender identity and expression, age, size, family status, political belief, religion, and mental and physical ability.
+* **Be considerate**: Your work will be used by other people, and you in turn will depend on the work of others. Any decision you take will affect users and colleagues, and you should take those consequences into account when making decisions. Remember that we're a world-wide community, so you might not be communicating in someone else's primary language.
+* **Be respectful**:  Not all of us will agree all the time, but disagreement is no excuse for poor behavior and poor manners. We might all experience some frustration now and then, but we cannot allow that frustration to turn into a personal attack. It’s important to remember that a community where people feel uncomfortable or threatened is not a productive one.
+* **Be careful in the words that we choose**: we are a community of professionals, and we conduct ourselves professionally. Be kind to others. Do not insult or put down other participants. Harassment and other exclusionary behavior aren't acceptable.
+* **Try to understand why we disagree**: Disagreements, both social and technical, happen all the time. It is important that we resolve disagreements and differing views constructively. Remember that we’re different. The strength of our community comes from its diversity, people from a wide range of backgrounds. Different people have different perspectives on issues. Being unable to understand why someone holds a viewpoint doesn’t mean that they’re wrong. Don’t forget that it is human to err and blaming each other doesn’t get us anywhere. Instead, focus on helping to resolve issues and learning from mistakes.
+
+## Definitions
+
+Harassment includes, but is not limited to:
+
+- Offensive comments related to gender, gender identity and expression, sexual orientation, disability, mental illness, neuro(a)typicality, physical appearance, body size, race, age, regional discrimination, political or religious affiliation
+- Unwelcome comments regarding a person’s lifestyle choices and practices, including those related to food, health, parenting, drugs, and employment
+- Deliberate misgendering. This includes deadnaming or persistently using a pronoun that does not correctly reflect a person's gender identity. You must address people by the name they give you when not addressing them by their username or handle
+- Physical contact and simulated physical contact (eg, textual descriptions like “*hug*” or “*backrub*”) without consent or after a request to stop
+- Threats of violence, both physical and psychological
+- Incitement of violence towards any individual, including encouraging a person to commit suicide or to engage in self-harm
+- Deliberate intimidation
+- Stalking or following
+- Harassing photography or recording, including logging online activity for harassment purposes
+- Sustained disruption of discussion
+- Unwelcome sexual attention, including gratuitous or off-topic sexual images or behaviour
+- Pattern of inappropriate social contact, such as requesting/assuming inappropriate levels of intimacy with others
+- Continued one-on-one communication after requests to cease
+- Deliberate “outing” of any aspect of a person’s identity without their consent except as necessary to protect others from intentional abuse
+- Publication of non-harassing private communication
+
+Our open source community prioritizes marginalized people’s safety over privileged people’s comfort. We will not act on complaints regarding:
+
+- ‘Reverse’ -isms, including ‘reverse racism,’ ‘reverse sexism,’ and ‘cisphobia’
+- Reasonable communication of boundaries, such as “leave me alone,” “go away,” or “I’m not discussing this with you”
+- Refusal to explain or debate social justice concepts
+- Communicating in a ‘tone’ you don’t find congenial
+- Criticizing racist, sexist, cissexist, or otherwise oppressive behavior or assumptions
+
+
+### Diversity Statement
+
+We encourage everyone to participate and are committed to building a community for all. Although we will fail at times, we seek to treat everyone both as fairly and equally as possible. Whenever a participant has made a mistake, we expect them to take responsibility for it. If someone has been harmed or offended, it is our responsibility to listen carefully and respectfully, and do our best to right the wrong.
+
+Although this list cannot be exhaustive, we explicitly honor diversity in age, gender, gender identity or expression, culture, ethnicity, language, national origin, political beliefs, profession, race, religion, sexual orientation, socioeconomic status, and technical ability. We will not tolerate discrimination based on any of the protected
+characteristics above, including participants with disabilities.
+
+### Reporting Issues
+
+If you experience or witness unacceptable behavior—or have any other concerns—please report it by contacting us via **code@nytimes.com**. All reports will be handled with discretion. In your report please include:
+
+- Your contact information.
+- Names (real, nicknames, or pseudonyms) of any individuals involved. If there are additional witnesses, please
+include them as well. Your account of what occurred, and if you believe the incident is ongoing. If there is a publicly available record (e.g. a mailing list archive or a public IRC logger), please include a link.
+- Any additional information that may be helpful.
+
+After filing a report, a representative will contact you personally, review the incident, follow up with any additional questions, and make a decision as to how to respond. If the person who is harassing you is part of the response team, they will recuse themselves from handling your incident. If the complaint originates from a member of the response team, it will be handled by a different member of the response team. We will respect confidentiality requests for the purpose of protecting victims of abuse.
+
+### Attribution & Acknowledgements
+
+We all stand on the shoulders of giants across many open source communities.  We'd like to thank the communities and projects that established code of conducts and diversity statements as our inspiration:
+
+* [Django](https://www.djangoproject.com/conduct/reporting/)
+* [Python](https://www.python.org/community/diversity/)
+* [Ubuntu](http://www.ubuntu.com/about/about-ubuntu/conduct)
+* [Contributor Covenant](http://contributor-covenant.org/)
+* [Geek Feminism](http://geekfeminism.org/about/code-of-conduct/)
+* [Citizen Code of Conduct](http://citizencodeofconduct.org/)
+
+This Code of Conduct was based on https://github.com/todogroup/opencodeofconduct