vendor/git.autistici.org/ai3/go-common/serverutil/tls.go

@@ -2,15 +2,20 @@ package serverutil
 
 import (
 	"crypto/tls"
+	"errors"
+	"fmt"
+	"log"
 	"net/http"
 	"regexp"
+	"strings"
 
 	common "git.autistici.org/ai3/go-common"
 )
 
 // TLSAuthACL describes a single access control entry. Path and
 // CommonName are anchored regular expressions (they must match the
-// entire string).
+// entire string). The first path to match in a list of ACLs will
+// identify the ACL to be applied.
 type TLSAuthACL struct {
 	Path       string `yaml:"path"`
 	CommonName string `yaml:"cn"`
@@ -28,10 +33,11 @@ func (p *TLSAuthACL) compile() error {
 	return err
 }
 
-func (p *TLSAuthACL) match(req *http.Request) bool {
-	if !p.pathRx.MatchString(req.URL.Path) {
-		return false
-	}
+func (p *TLSAuthACL) matchPath(req *http.Request) bool {
+	return p.pathRx.MatchString(req.URL.Path)
+}
+
+func (p *TLSAuthACL) matchCN(req *http.Request) bool {
 	for _, cert := range req.TLS.PeerCertificates {
 		if p.cnRx.MatchString(cert.Subject.CommonName) {
 			return true
@@ -40,6 +46,31 @@ func (p *TLSAuthACL) match(req *http.Request) bool {
 	return false
 }
 
+// TLSAuthACLListFlag is a convenience type that allows callers to use
+// the 'flag' package to specify a list of TLSAuthACL objects. It
+// implements the flag.Value interface.
+type TLSAuthACLListFlag []*TLSAuthACL
+
+func (l TLSAuthACLListFlag) String() string {
+	var out []string
+	for _, acl := range l {
+		out = append(out, fmt.Sprintf("%s:%s", acl.Path, acl.CommonName))
+	}
+	return strings.Join(out, ",")
+}
+
+func (l *TLSAuthACLListFlag) Set(value string) error {
+	parts := strings.SplitN(value, ":", 2)
+	if len(parts) != 2 {
+		return errors.New("bad acl format")
+	}
+	*l = append(*l, &TLSAuthACL{
+		Path:       parts[0],
+		CommonName: parts[1],
+	})
+	return nil
+}
+
 // TLSAuthConfig stores access control lists for TLS authentication. Access
 // control lists are matched against the request path and the
 // CommonName component of the peer certificate subject.
@@ -52,9 +83,13 @@ func (c *TLSAuthConfig) match(req *http.Request) bool {
 	if c == nil || len(c.Allow) == 0 {
 		return true
 	}
+
 	for _, acl := range c.Allow {
-		if acl.match(req) {
-			return true
+		if acl.matchPath(req) {
+			if acl.matchCN(req) {
+				return true
+			}
+			break
 		}
 	}
 	return false
@@ -81,21 +116,27 @@ func (c *TLSServerConfig) TLSConfig() (*tls.Config, error) {
 		return nil, err
 	}
 
-	cas, err := common.LoadCA(c.CA)
-	if err != nil {
-		return nil, err
-	}
-
 	// Set some TLS-level parameters (cipher-related), assuming
 	// we're using EC keys.
 	tlsConf := &tls.Config{
 		Certificates:             []tls.Certificate{cert},
-		ClientAuth:               tls.RequireAndVerifyClientCert,
-		ClientCAs:                cas,
 		CipherSuites:             serverCiphers,
 		MinVersion:               tls.VersionTLS12,
 		PreferServerCipherSuites: true,
+		NextProtos:               []string{"h2", "http/1.1"},
+	}
+
+	// Require client certificates if a CA is specified.
+	if c.CA != "" {
+		cas, err := common.LoadCA(c.CA)
+		if err != nil {
+			return nil, err
+		}
+
+		tlsConf.ClientAuth = tls.RequireAndVerifyClientCert
+		tlsConf.ClientCAs = cas
 	}
+
 	tlsConf.BuildNameToCertificate()
 
 	return tlsConf, nil
@@ -119,6 +160,13 @@ func (c *TLSServerConfig) TLSAuthWrapper(h http.Handler) (http.Handler, error) {
 			h.ServeHTTP(w, r)
 			return
 		}
-		http.Error(w, "Unauthorized", http.StatusUnauthorized)
+
+		// Log the failed access, useful for debugging.
+		var tlsmsg string
+		if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 {
+			tlsmsg = fmt.Sprintf("TLS client '%s' at", r.TLS.PeerCertificates[0].Subject.CommonName)
+		}
+		log.Printf("unauthorized access to %s from %s%s", r.URL.Path, tlsmsg, r.RemoteAddr)
+		http.Error(w, "Forbidden", http.StatusForbidden)
 	}), nil
 }

vendor/git.autistici.org/ai3/go-common/tracing/tracing.go

+package tracing
+
+import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+
+	othttp "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	b3 "go.opentelemetry.io/contrib/propagators/b3"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/zipkin"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	"go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.12.0"
+)
+
+var (
+	// Enabled reports whether tracing is globally enabled or not.
+	Enabled bool
+
+	initOnce sync.Once
+)
+
+const globalTracingConfigPath = "/etc/tracing/client.conf"
+
+type tracingConfig struct {
+	ReportURL string `json:"report_url"`
+	Sample    string `json:"sample"`
+}
+
+// Read the global tracing configuration file. Its location is
+// hardcoded, but it can be overriden using the TRACING_CONFIG
+// environment variable.
+func readTracingConfig() (*tracingConfig, error) {
+	// Read and decode configuration.
+	cfgPath := globalTracingConfigPath
+	if s := os.Getenv("TRACING_CONFIG"); s != "" {
+		cfgPath = s
+	}
+	data, err := ioutil.ReadFile(cfgPath)
+	if err != nil {
+		return nil, err
+	}
+
+	var config tracingConfig
+	if err := json.Unmarshal(data, &config); err != nil {
+		log.Printf("warning: error in tracing configuration: %v, tracing disabled", err)
+		return nil, err
+	}
+
+	if config.ReportURL == "" {
+		log.Printf("warning: tracing configuration contains no report_url, tracing disabled")
+		return nil, errors.New("no report_url")
+	}
+
+	return &config, nil
+}
+
+// Compute the service name for Zipkin: this is usually the program
+// name (without path), but it can be overriden by the TRACING_SERVICE
+// environment variable.
+func getServiceName() string {
+	if s := os.Getenv("TRACING_SERVICE"); s != "" {
+		return s
+	}
+	if s, err := os.Executable(); err == nil {
+		return filepath.Base(s)
+	}
+	return "unknown_service"
+}
+
+func defaultResource(serviceName string) *resource.Resource {
+	attrs := []attribute.KeyValue{
+		semconv.ServiceNameKey.String(serviceName),
+	}
+
+	if s, err := os.Hostname(); err == nil {
+		attrs = append(attrs, semconv.HostNameKey.String(s))
+	}
+	if s := os.Getenv("FLOAT_SERVICE"); s != "" {
+		attrs = append(attrs, attribute.Key("float.service").String(s))
+	}
+	if s := os.Getenv("FLOAT_CONTAINER_NAME"); s != "" {
+		attrs = append(attrs, semconv.ContainerNameKey.String(s))
+	}
+	if s := os.Getenv("FLOAT_CONTAINER_IMAGE"); s != "" {
+		imageName := s
+		tag := "latest"
+		if strings.Contains(imageName, ":") {
+			parts := strings.SplitN(imageName, ":", 2)
+			imageName = parts[0]
+			tag = parts[1]
+		}
+
+		attrs = append(attrs, semconv.ContainerImageNameKey.String(imageName))
+		attrs = append(attrs, semconv.ContainerImageTagKey.String(tag))
+	}
+
+	r, _ := resource.Merge(
+		resource.Default(),
+		resource.NewWithAttributes(semconv.SchemaURL, attrs...),
+	)
+	return r
+}
+
+// Initialize tracing. Tracing will be enabled if the system-wide
+// tracing configuration file is present and valid. Explicitly set
+// TRACING_ENABLE=0 in the environment to disable tracing.
+//
+// We need to check the configuration as soon as possible, because
+// it's likely that client transports are created before HTTP servers,
+// and we need to wrap them with opencensus at creation time.
+func initTracing(serviceName string) {
+	initOnce.Do(func() {
+		// Kill switch from environment.
+		if s := os.Getenv("TRACING_ENABLE"); s == "0" {
+			return
+		}
+
+		config, err := readTracingConfig()
+		if err != nil {
+			return
+		}
+
+		ze, err := zipkin.New(config.ReportURL)
+		if err != nil {
+			log.Printf("error creating Zipkin exporter: %v", err)
+			return
+		}
+
+		// The sampling policy only applies to incoming requests for
+		// which tracing is not already enabled: in this case, we
+		// always pass-through.
+		var sampler trace.Sampler
+		switch config.Sample {
+		case "", "always":
+			sampler = trace.AlwaysSample()
+		case "never":
+			sampler = trace.NeverSample()
+		default:
+			frac, err := strconv.ParseFloat(config.Sample, 64)
+			if err != nil {
+				log.Printf("warning: error in tracing configuration: sample: %v, tracing disabled", err)
+				return
+			}
+			sampler = trace.TraceIDRatioBased(frac)
+		}
+
+		tp := trace.NewTracerProvider(
+			trace.WithSampler(trace.ParentBased(sampler)),
+			trace.WithBatcher(ze),
+			trace.WithResource(defaultResource(serviceName)),
+		)
+
+		otel.SetTracerProvider(tp)
+
+		otel.SetTextMapPropagator(
+			propagation.NewCompositeTextMapPropagator(
+				propagation.TraceContext{},
+				propagation.Baggage{},
+				b3.New()))
+
+		log.Printf("tracing enabled (report_url %s)", config.ReportURL)
+
+		Enabled = true
+	})
+}
+
+// Init tracing support, if not using WrapHandler.
+func Init() {
+	initTracing(getServiceName())
+}
+
+// WrapTransport optionally wraps a http.RoundTripper with OpenCensus
+// tracing functionality, if it is globally enabled.
+//
+// Must call Init() first.
+func WrapTransport(t http.RoundTripper) http.RoundTripper {
+	if Enabled {
+		t = othttp.NewTransport(t)
+	}
+	return t
+}
+
+// WrapHandler wraps a http.Handler with OpenCensus tracing
+// functionality, if globally enabled. Automatically calls Init().
+func WrapHandler(h http.Handler, endpointAddr string) http.Handler {
+	serviceName := getServiceName()
+	initTracing(serviceName)
+	if !Enabled {
+		return h
+	}
+
+	// Format span names with the request URL path.
+	return othttp.NewHandler(
+		h, serviceName,
+		othttp.WithSpanNameFormatter(func(op string, r *http.Request) string {
+			return r.URL.Path
+		}),
+	)
+}

vendor/git.autistici.org/ai3/tools/replds/.gitlab-ci.yml

+include:
+  - "https://git.autistici.org/pipelines/debian/raw/master/common.yml"
+  - "https://git.autistici.org/pipelines/images/test/golang/raw/master/ci.yml"

vendor/git.autistici.org/ai3/tools/replds/LICENSE

+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

vendor/git.autistici.org/ai3/replds/README.md → vendor/git.autistici.org/ai3/tools/replds/README.md

@@ -122,3 +122,15 @@ directory* to run the repository tools on, and the final workflow is:
 * run the metadata-generation tools on the staging dir;
 * synchronize the data back to replds using the *sync* command.
 
+## Usage
+
+The Debian package comes with a
+[replds-instance-create](debian/replds-instance-create) script that
+can be used to set up multiple replds instances. For an instance named
+*foo*, the script will setup the *replds@foo* systemd service, and it
+will create the *replds-foo* user and group. Add users that need to
+read the repository files to that group. The configuration will be
+read from */etc/replds/foo.yml*.
+
+Note that files created by the daemon will be world-readable by
+default. Set the process umask if you wish to restrict this further.

vendor/git.autistici.org/ai3/replds/client.go → vendor/git.autistici.org/ai3/tools/replds/client.go

@@ -31,17 +31,17 @@ func newClient(config *clientutil.BackendConfig) (*client, error) {
 
 func (c *client) internalGetNodes(ctx context.Context, req *internalGetNodesRequest) (*internalGetNodesResponse, error) {
 	var resp internalGetNodesResponse
-	err := clientutil.DoJSONHTTPRequest(ctx, c.be.Client(""), c.be.URL("")+"/api/internal/get_nodes", req, &resp)
+	err := c.be.Call(ctx, "", "/api/internal/get_nodes", req, &resp)
 	return &resp, err
 }
 
 func (c *client) internalUpdateNodes(ctx context.Context, req *internalUpdateNodesRequest) error {
-	return clientutil.DoJSONHTTPRequest(ctx, c.be.Client(""), c.be.URL("")+"/api/internal/update_nodes", req, nil)
+	return c.be.Call(ctx, "", "/api/internal/update_nodes", req, nil)
 }
 
 func (c *client) SetNodes(ctx context.Context, req *SetNodesRequest) (*SetNodesResponse, error) {
 	var resp SetNodesResponse
-	err := clientutil.DoJSONHTTPRequest(ctx, c.be.Client(""), c.be.URL("")+"/api/set_nodes", req, &resp)
+	err := c.be.Call(ctx, "", "/api/set_nodes", req, &resp)
 	return &resp, err
 }
 

vendor/git.autistici.org/ai3/replds/fs.go → vendor/git.autistici.org/ai3/tools/replds/fs.go

@@ -20,9 +20,9 @@ func newFS(dir string) *FS {
 
 // getAllNodes scans the root directory and returns a list of
 // Nodes. All errors are ignored.
-func (f *FS) getAllNodes() []Node {
-	var nodes []Node
-	dirLen := len(f.dir)
+func (f *FS) getAllNodes() []*Node {
+	var nodes []*Node
+	dirOffset := len(f.dir) + 1
 	if err := filepath.Walk(f.dir, func(path string, info os.FileInfo, err error) error {
 		// Ignore errors.
 		if err != nil {
@@ -32,8 +32,8 @@ func (f *FS) getAllNodes() []Node {
 		if !info.Mode().IsRegular() {
 			return nil
 		}
-		nodes = append(nodes, Node{
-			Path:      path[dirLen+1:],
+		nodes = append(nodes, &Node{
+			Path:      path[dirOffset:],
 			Timestamp: info.ModTime(),
 		})
 		return nil
@@ -44,7 +44,7 @@ func (f *FS) getAllNodes() []Node {
 }
 
 // setNode applies a Node to the underlying filesystem.
-func (f *FS) setNode(node Node) error {
+func (f *FS) setNode(node *Node) error {
 	path := filepath.Join(f.dir, node.Path)
 
 	// Tombstone? Then delete the file (ignore errors).
@@ -55,7 +55,7 @@ func (f *FS) setNode(node Node) error {
 	// Create parent directory if it does not exist.
 	dir := filepath.Dir(path)
 	if _, err := os.Stat(dir); os.IsNotExist(err) {
-		if err = os.MkdirAll(dir, 0700); err != nil {
+		if err = os.MkdirAll(dir, 0755); err != nil {
 			return err
 		}
 	}
@@ -81,7 +81,7 @@ func writeFile(path string, data []byte) error {
 	err := os.Rename(tmp, path)
 	if err != nil {
 		// Clean up if something went wrong.
-		os.Remove(tmp)
+		os.Remove(tmp) // nolint
 	}
 	return err
 }

vendor/git.autistici.org/ai3/replds/http.go → vendor/git.autistici.org/ai3/tools/replds/http.go

@@ -84,7 +84,7 @@ func (s *HTTPServer) Handler() http.Handler {
 	return h
 }
 
-func nodes2str(nodes []Node) string {
+func nodes2str(nodes []*Node) string {
 	var tmp []string
 	for _, node := range nodes {
 		tmp = append(tmp, fmt.Sprintf("%s@%d", node.Path, node.Timestamp.Unix()))

vendor/git.autistici.org/ai3/tools/replds/renovate.json

+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base"
+  ]
+}

vendor/git.autistici.org/ai3/replds/server.go → vendor/git.autistici.org/ai3/tools/replds/server.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"log"
+	"math/rand"
 	"strings"
 	"sync"
 	"time"
@@ -21,7 +22,7 @@ var (
 	pollPeriod = 120 * time.Second
 
 	// Timeout for InternalGetNodes requests.
-	getNodesTimeout = 60 * time.Second
+	getNodesTimeout = 120 * time.Second
 )
 
 // Node is an annotated path/value entry.
@@ -34,6 +35,15 @@ type Node struct {
 	Deleted bool `json:"deleted,omitempty"`
 }
 
+func (n *Node) Copy() *Node {
+	return &Node{
+		Path:      n.Path,
+		Value:     n.Value,
+		Timestamp: n.Timestamp,
+		Deleted:   n.Deleted,
+	}
+}
+
 func (n *Node) withoutValue() *Node {
 	return &Node{
 		Path:      n.Path,
@@ -42,29 +52,29 @@ func (n *Node) withoutValue() *Node {
 	}
 }
 
-func (n *Node) metadataOnly() Node {
-	return Node{
+func (n *Node) metadataOnly() *Node {
+	return &Node{
 		Path:      n.Path,
 		Timestamp: n.Timestamp,
 	}
 }
 
 type internalGetNodesRequest struct {
-	Nodes []Node `json:"nodes"`
+	Nodes []*Node `json:"nodes"`
 }
 
 type internalGetNodesResponse struct {
-	Nodes   []Node `json:"nodes"`
-	Partial bool   `json:"partial,omitempty"`
+	Nodes   []*Node `json:"nodes"`
+	Partial bool    `json:"partial,omitempty"`
 }
 
 type internalUpdateNodesRequest struct {
-	Nodes []Node `json:"nodes"`
+	Nodes []*Node `json:"nodes"`
 }
 
 // SetNodesRequest is the request type for the SetNodes method.
 type SetNodesRequest struct {
-	Nodes []Node `json:"nodes"`
+	Nodes []*Node `json:"nodes"`
 }
 
 // SetNodesResponse is the response returned by the SetNodes method.
@@ -76,17 +86,18 @@ type SetNodesResponse struct {
 var errTooOld = errors.New("a more recent value exists for this key")
 
 type storage interface {
-	getAllNodes() []Node
+	getAllNodes() []*Node
 	getNodeValue(string) ([]byte, error)
-	setNode(Node) error
+	setNode(*Node) error
 }
 
 // Server for the replicated filesync.
 type Server struct {
-	peers   []string
-	network *network
-	storage storage
-	logger  *log.Logger
+	peers    []string
+	network  *network
+	storage  storage
+	logger   *log.Logger
+	readonly bool
 
 	wg    sync.WaitGroup
 	stop  chan bool
@@ -103,7 +114,7 @@ func stripTrailingSlash(peers []string) []string {
 }
 
 // NewServer creates a new Server with the given peers and backends.
-func NewServer(peers []string, dir string, tlsConfig *clientutil.TLSClientConfig) (*Server, error) {
+func NewServer(peers []string, dir string, tlsConfig *clientutil.TLSClientConfig, readonly bool) (*Server, error) {
 	peers = stripTrailingSlash(peers)
 
 	network, err := newNetwork(peers, tlsConfig)
@@ -112,26 +123,44 @@ func NewServer(peers []string, dir string, tlsConfig *clientutil.TLSClientConfig
 	}
 
 	s := &Server{
-		peers:   peers,
-		network: network,
-		storage: newFS(dir),
-		stop:    make(chan bool),
-		nodes:   make(map[string]*Node),
+		peers:    peers,
+		network:  network,
+		storage:  newFS(dir),
+		stop:     make(chan bool),
+		nodes:    make(map[string]*Node),
+		readonly: readonly,
 	}
 
-	for _, node := range s.storage.getAllNodes() {
-		s.nodes[node.Path] = &node
-	}
-	if len(s.nodes) > 0 {
-		log.Printf("found %d entries in %s", len(s.nodes), dir)
+	// Do not scan the local filesystem at startup on readonly
+	// instances, downloading everything is safer as we do not
+	// risk injecting stale data.
+	if !readonly {
+		for _, node := range s.storage.getAllNodes() {
+			s.nodes[node.Path] = node
+		}
+		if len(s.nodes) > 0 {
+			log.Printf("found %d entries in %s", len(s.nodes), dir)
+		}
 	}
 
-	for _, peer := range peers {
+	// The background workers are different if the instance is
+	// readonly: normally we run a synchronization worker for each
+	// peer, while in the readonly case we periodically poll a
+	// different random peer.
+	if readonly {
 		s.wg.Add(1)
-		go func(peer string) {
-			s.pollThread(peer)
+		go func() {
+			s.pollRandomPeerThread()
 			s.wg.Done()
-		}(peer)
+		}()
+	} else {
+		for _, peer := range peers {
+			s.wg.Add(1)
+			go func(peer string) {
+				s.pollThread(peer)
+				s.wg.Done()
+			}(peer)
+		}
 	}
 
 	return s, nil
@@ -156,7 +185,7 @@ func (s *Server) log(fmt string, args ...interface{}) {
 // Update one or more nodes as a "transaction": if one update fails,
 // the rest are aborted (but previous commits are not reverted, yet).
 // Will ping peers with the data when updatePeers is true.
-func (s *Server) doSetNodes(ctx context.Context, nodes []Node, updatePeers bool) (*SetNodesResponse, error) {
+func (s *Server) doSetNodes(ctx context.Context, nodes []*Node, updatePeers bool) (*SetNodesResponse, error) {
 	// Update local state.
 	for _, node := range nodes {
 		cur, ok := s.nodes[node.Path]
@@ -185,7 +214,9 @@ func (s *Server) doSetNodes(ctx context.Context, nodes []Node, updatePeers bool)
 
 	var res SetNodesResponse
 	if updatePeers {
-		// Ping remote nodes with updated state.
+		// Ping remote nodes with the updated state. This
+		// reduces the latency of update propagation (we don't
+		// have to wait until the peers poll us).
 		res.HostsOk = 1
 		for _, peer := range s.peers {
 			err := s.network.Client(peer).internalUpdateNodes(ctx, &internalUpdateNodesRequest{
@@ -216,13 +247,13 @@ func (s *Server) internalUpdateNodes(ctx context.Context, req *internalUpdateNod
 	return err
 }
 
-func (s *Server) getNodeWithValue(node *Node) (out Node, err error) {
-	out = *node
+func (s *Server) getNodeWithValue(node *Node) (out *Node, err error) {
+	out = node.Copy()
 	out.Value, err = s.storage.getNodeValue(node.Path)
 	return
 }
 
-func nodeDiff(reqNodes []Node, myNodes map[string]*Node, adder func(*Node) bool) ([]string, bool) {
+func nodeDiff(reqNodes []*Node, myNodes map[string]*Node, adder func(*Node) bool) ([]string, bool) {
 	var missing []string
 	tmp := make(map[string]struct{})
 	for _, reqNode := range reqNodes {
@@ -293,14 +324,8 @@ func (s *Server) internalGetNodes(ctx context.Context, req *internalGetNodesRequ
 	return &resp, nil
 }
 
-func (s *Server) pollThread(peer string) {
-	poll := func() {
-		if err := s.pollPeer(peer); err != nil {
-			s.log("error polling peer %s: %v", peer, err)
-		}
-	}
-
-	poll()
+func (s *Server) backgroundThread(fn func()) {
+	fn()
 
 	tick := time.NewTicker(pollPeriod)
 	defer tick.Stop()
@@ -309,11 +334,32 @@ func (s *Server) pollThread(peer string) {
 		case <-s.stop:
 			return
 		case <-tick.C:
-			poll()
+			fn()
 		}
 	}
 }
 
+func (s *Server) pollThread(peer string) {
+	s.backgroundThread(func() {
+		if err := s.pollPeer(peer); err != nil {
+			s.log("error polling peer %s: %v", peer, err)
+		}
+	})
+}
+
+func (s *Server) pollRandomPeerThread() {
+	s.backgroundThread(func() {
+		if len(s.peers) == 0 {
+			s.log("no peers to poll!")
+			return
+		}
+		peer := s.peers[rand.Intn(len(s.peers))]
+		if err := s.pollPeer(peer); err != nil {
+			s.log("error polling peer %s: %v", peer, err)
+		}
+	})
+}
+
 func (s *Server) pollPeer(peer string) error {
 	for {
 		partial, err := s.pollPeerRequest(peer)
@@ -353,7 +399,7 @@ var (
 	maxTSValue int64
 )
 
-func updateMaxTimestamp(nodes []Node) {
+func updateMaxTimestamp(nodes []*Node) {
 	var max int64
 	for _, node := range nodes {
 		t := node.Timestamp.Unix()
@@ -375,6 +421,9 @@ func (s *Server) pollPeerRequest(peer string) (bool, error) {
 	peerName := stripHTTP(peer)
 	peerRequests.With(prometheus.Labels{"peer": peerName}).Inc()
 
+	// Create a shallow copy of the list of the Nodes we know
+	// about, containing just path/timestamp, to send to the peer
+	// in our GetNodes request.
 	s.mx.Lock()
 	var req internalGetNodesRequest
 	for _, node := range s.nodes {
@@ -399,6 +448,7 @@ func (s *Server) pollPeerRequest(peer string) (bool, error) {
 		return false, nil
 	}
 
+	// Update the internal state with the new data.
 	return resp.Partial, s.internalUpdateNodes(ctx, &internalUpdateNodesRequest{Nodes: resp.Nodes})
 }
 

vendor/github.com/NYTimes/gziphandler/.gitignore

+*.swp