Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • ai3/tools/acmeserver
  • godog/acmeserver
  • svp-bot/acmeserver
3 results
Show changes
Showing
with 871 additions and 130 deletions
// +build !go1.9
package clientutil
import (
"context"
"net"
"time"
)
// Go < 1.9 does not have net.DialContext, reimplement it in terms of
// net.DialTimeout.
func netDialContext(addr string, connectTimeout time.Duration) func(context.Context, string, string) (net.Conn, error) {
return func(ctx context.Context, net string, _ string) (net.Conn, error) {
if deadline, ok := ctx.Deadline(); ok {
ctxTimeout := time.Until(deadline)
if ctxTimeout < connectTimeout {
connectTimeout = ctxTimeout
}
}
return net.DialTimeout(network, addr, connectTimeout)
}
}
package clientutil package clientutil
import ( import (
"context"
"crypto/tls" "crypto/tls"
"net"
"net/http" "net/http"
"sync" "sync"
"time" "time"
...@@ -11,31 +9,42 @@ import ( ...@@ -11,31 +9,42 @@ import (
"git.autistici.org/ai3/go-common/tracing" "git.autistici.org/ai3/go-common/tracing"
) )
var defaultConnectTimeout = 30 * time.Second
// The transportCache is just a cache of http transports, each // The transportCache is just a cache of http transports, each
// connecting to a specific address. // connecting to a specific address.
// //
// We use this to control the HTTP Host header and the TLS ServerName // We use this to control the HTTP Host header and the TLS ServerName
// independently of the target address. // independently of the target address.
type transportCache struct { type transportCache struct {
tlsConfig *tls.Config tlsConfig *tls.Config
connectTimeout time.Duration
mx sync.RWMutex mx sync.RWMutex
transports map[string]http.RoundTripper transports map[string]http.RoundTripper
} }
func newTransportCache(tlsConfig *tls.Config) *transportCache { func newTransportCache(tlsConfig *tls.Config, connectTimeout time.Duration) *transportCache {
if connectTimeout == 0 {
connectTimeout = defaultConnectTimeout
}
return &transportCache{ return &transportCache{
tlsConfig: tlsConfig, tlsConfig: tlsConfig,
transports: make(map[string]http.RoundTripper), connectTimeout: connectTimeout,
transports: make(map[string]http.RoundTripper),
} }
} }
func (m *transportCache) newTransport(addr string) http.RoundTripper { func (m *transportCache) newTransport(addr string) http.RoundTripper {
return tracing.WrapTransport(&http.Transport{ return tracing.WrapTransport(&http.Transport{
TLSClientConfig: m.tlsConfig, TLSClientConfig: m.tlsConfig,
DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { DialContext: netDialContext(addr, m.connectTimeout),
return netDialContext(ctx, network, addr)
}, // Parameters match those of net/http.DefaultTransport.
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}) })
} }
...@@ -55,13 +64,3 @@ func (m *transportCache) getTransport(addr string) http.RoundTripper { ...@@ -55,13 +64,3 @@ func (m *transportCache) getTransport(addr string) http.RoundTripper {
return t return t
} }
// Go < 1.9 does not have net.DialContext, reimplement it in terms of
// net.DialTimeout.
func netDialContext(ctx context.Context, network, addr string) (net.Conn, error) {
timeout := 60 * time.Second // some arbitrary max timeout
if deadline, ok := ctx.Deadline(); ok {
timeout = time.Until(deadline)
}
return net.DialTimeout(network, addr, timeout)
}
{
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
"extends": [
"config:base"
]
}
package serverutil package serverutil
import ( import (
"compress/gzip"
"context" "context"
"crypto/tls" "crypto/tls"
"fmt" "fmt"
...@@ -11,36 +12,54 @@ import ( ...@@ -11,36 +12,54 @@ import (
_ "net/http/pprof" _ "net/http/pprof"
"os" "os"
"os/signal" "os/signal"
"strconv"
"syscall" "syscall"
"time" "time"
"git.autistici.org/ai3/go-common/tracing" "git.autistici.org/ai3/go-common/tracing"
"github.com/coreos/go-systemd/daemon" "github.com/NYTimes/gziphandler"
"github.com/coreos/go-systemd/v22/daemon"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/client_golang/prometheus/promhttp"
) )
var gracefulShutdownTimeout = 3 * time.Second var (
gracefulShutdownTimeout = 3 * time.Second
gzipLevel = gzip.BestSpeed
gzipMinSize = 1300
gzipContentTypes = []string{
"application/json",
"application/javascript",
"text/html",
"text/plain",
"text/css",
}
)
// ServerConfig stores common HTTP/HTTPS server configuration parameters. // ServerConfig stores common HTTP/HTTPS server configuration parameters.
type ServerConfig struct { type ServerConfig struct {
TLS *TLSServerConfig `yaml:"tls"` TLS *TLSServerConfig `yaml:"tls"`
MaxInflightRequests int `yaml:"max_inflight_requests"` MaxInflightRequests int `yaml:"max_inflight_requests"`
RequestTimeoutSecs int `yaml:"request_timeout"`
TrustedForwarders []string `yaml:"trusted_forwarders"` TrustedForwarders []string `yaml:"trusted_forwarders"`
// TODO: switch do disable_compression (flip default) later.
EnableCompression bool `yaml:"enable_compression"`
} }
func (config *ServerConfig) buildHTTPServer(h http.Handler) (*http.Server, error) { func (config *ServerConfig) buildHTTPHandler(h http.Handler) (http.Handler, *tls.Config, error) {
var tlsConfig *tls.Config var tlsConfig *tls.Config
var err error var err error
if config != nil { if config != nil {
if config.TLS != nil { if config.TLS != nil {
tlsConfig, err = config.TLS.TLSConfig() tlsConfig, err = config.TLS.TLSConfig()
if err != nil { if err != nil {
return nil, err return nil, nil, err
} }
h, err = config.TLS.TLSAuthWrapper(h) h, err = config.TLS.TLSAuthWrapper(h)
if err != nil { if err != nil {
return nil, err return nil, nil, err
} }
} }
...@@ -49,7 +68,7 @@ func (config *ServerConfig) buildHTTPServer(h http.Handler) (*http.Server, error ...@@ -49,7 +68,7 @@ func (config *ServerConfig) buildHTTPServer(h http.Handler) (*http.Server, error
if len(config.TrustedForwarders) > 0 { if len(config.TrustedForwarders) > 0 {
h, err = newProxyHeaders(h, config.TrustedForwarders) h, err = newProxyHeaders(h, config.TrustedForwarders)
if err != nil { if err != nil {
return nil, err return nil, nil, err
} }
} }
...@@ -58,18 +77,68 @@ func (config *ServerConfig) buildHTTPServer(h http.Handler) (*http.Server, error ...@@ -58,18 +77,68 @@ func (config *ServerConfig) buildHTTPServer(h http.Handler) (*http.Server, error
if config.MaxInflightRequests > 0 { if config.MaxInflightRequests > 0 {
h = newLoadSheddingWrapper(config.MaxInflightRequests, h) h = newLoadSheddingWrapper(config.MaxInflightRequests, h)
} }
// Wrap the handler with a TimeoutHandler if 'request_timeout'
// is set.
if config.RequestTimeoutSecs > 0 {
h = http.TimeoutHandler(h, time.Duration(config.RequestTimeoutSecs)*time.Second, "")
}
}
// Add all the default handlers (health, monitoring, etc).
h = addDefaultHandlers(h)
// Optionally enable compression.
if config != nil && config.EnableCompression {
gzwrap, err := gziphandler.GzipHandlerWithOpts(
gziphandler.CompressionLevel(gzipLevel),
gziphandler.MinSize(gzipMinSize),
gziphandler.ContentTypes(gzipContentTypes),
)
if err != nil {
return nil, nil, err
}
h = gzwrap(h)
}
return h, tlsConfig, nil
}
func buildListener(addr string, tlsConfig *tls.Config) (net.Listener, error) {
// Create the net.Listener first, so we can detect
// initialization-time errors safely.
l, err := net.Listen("tcp", addr)
if err != nil {
return nil, err
}
if tlsConfig != nil {
l = tls.NewListener(l, tlsConfig)
}
return l, nil
}
func buildServer(h http.Handler, config *ServerConfig, addr string) (*http.Server, error) {
// Wrap with tracing handler (exclude metrics and other
// debugging endpoints).
h = tracing.WrapHandler(h, guessEndpointName(addr))
// Create the top-level HTTP handler with all our additions.
hh, tlsConfig, err := config.buildHTTPHandler(h)
if err != nil {
return nil, err
} }
// These are not meant to be external-facing servers, so we // These are not meant to be external-facing servers, so we
// can be generous with the timeouts to keep the number of // can be generous with the timeouts to keep the number of
// reconnections low. // reconnections low.
return &http.Server{ srv := &http.Server{
Handler: defaultHandler(h), Handler: hh,
ReadTimeout: 30 * time.Second, ReadHeaderTimeout: 30 * time.Second,
WriteTimeout: 30 * time.Second, IdleTimeout: 600 * time.Second,
IdleTimeout: 600 * time.Second, TLSConfig: tlsConfig,
TLSConfig: tlsConfig, }
}, nil
return srv, nil
} }
// Serve HTTP(S) content on the specified address. If config.TLS is // Serve HTTP(S) content on the specified address. If config.TLS is
...@@ -79,25 +148,15 @@ func (config *ServerConfig) buildHTTPServer(h http.Handler) (*http.Server, error ...@@ -79,25 +148,15 @@ func (config *ServerConfig) buildHTTPServer(h http.Handler) (*http.Server, error
// the listener, otherwise it will handle graceful termination on // the listener, otherwise it will handle graceful termination on
// SIGINT or SIGTERM and return nil. // SIGINT or SIGTERM and return nil.
func Serve(h http.Handler, config *ServerConfig, addr string) error { func Serve(h http.Handler, config *ServerConfig, addr string) error {
// Wrap with tracing handler (exclude metrics and other srv, err := buildServer(h, config, addr)
// debugging endpoints).
h = tracing.WrapHandler(h, guessEndpointName(addr))
// Create the HTTP server.
srv, err := config.buildHTTPServer(h)
if err != nil { if err != nil {
return err return err
} }
// Create the net.Listener first, so we can detect l, err := buildListener(addr, srv.TLSConfig)
// initialization-time errors safely.
l, err := net.Listen("tcp", addr)
if err != nil { if err != nil {
return err return err
} }
if srv.TLSConfig != nil {
l = tls.NewListener(l, srv.TLSConfig)
}
// Install a signal handler for gentle process termination. // Install a signal handler for gentle process termination.
done := make(chan struct{}) done := make(chan struct{})
...@@ -134,7 +193,39 @@ func Serve(h http.Handler, config *ServerConfig, addr string) error { ...@@ -134,7 +193,39 @@ func Serve(h http.Handler, config *ServerConfig, addr string) error {
return nil return nil
} }
func defaultHandler(h http.Handler) http.Handler { // ServeWithContext operates like Serve but with a controlling Context
// that can be used to stop the HTTP server.
func ServeWithContext(ctx context.Context, h http.Handler, config *ServerConfig, addr string) error {
srv, err := buildServer(h, config, addr)
if err != nil {
return err
}
l, err := buildListener(addr, srv.TLSConfig)
if err != nil {
return err
}
go func() {
<-ctx.Done()
sctx, cancel := context.WithTimeout(context.Background(), gracefulShutdownTimeout)
srv.Shutdown(sctx) // nolint: errcheck
srv.Close()
cancel()
}()
daemon.SdNotify(false, "READY=1") // nolint
err = srv.Serve(l)
if err == http.ErrServerClosed {
err = nil
}
return err
}
func addDefaultHandlers(h http.Handler) http.Handler {
root := http.NewServeMux() root := http.NewServeMux()
// Add an endpoint for HTTP health checking probes. // Add an endpoint for HTTP health checking probes.
...@@ -154,11 +245,30 @@ func defaultHandler(h http.Handler) http.Handler { ...@@ -154,11 +245,30 @@ func defaultHandler(h http.Handler) http.Handler {
// Prometheus instrumentation (requests to /metrics and // Prometheus instrumentation (requests to /metrics and
// /health are not included). // /health are not included).
root.Handle("/", promhttp.InstrumentHandlerInFlight(inFlightRequests, root.Handle("/", promhttp.InstrumentHandlerInFlight(inFlightRequests,
promhttp.InstrumentHandlerCounter(totalRequests, h))) promhttp.InstrumentHandlerCounter(totalRequests,
propagateDeadline(h))))
return root return root
} }
const deadlineHeader = "X-RPC-Deadline"
// Read an eventual deadline from the HTTP request, and set it as the
// deadline of the request context.
func propagateDeadline(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if hdr := req.Header.Get(deadlineHeader); hdr != "" {
if deadlineNano, err := strconv.ParseInt(hdr, 10, 64); err == nil {
deadline := time.Unix(0, deadlineNano)
ctx, cancel := context.WithDeadline(req.Context(), deadline)
defer cancel()
req = req.WithContext(ctx)
}
}
h.ServeHTTP(w, req)
})
}
func guessEndpointName(addr string) string { func guessEndpointName(addr string) string {
_, port, err := net.SplitHostPort(addr) _, port, err := net.SplitHostPort(addr)
if err != nil { if err != nil {
......
...@@ -3,6 +3,7 @@ package serverutil ...@@ -3,6 +3,7 @@ package serverutil
import ( import (
"encoding/json" "encoding/json"
"log" "log"
"mime"
"net/http" "net/http"
) )
...@@ -14,7 +15,7 @@ func DecodeJSONRequest(w http.ResponseWriter, r *http.Request, obj interface{}) ...@@ -14,7 +15,7 @@ func DecodeJSONRequest(w http.ResponseWriter, r *http.Request, obj interface{})
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return false return false
} }
if r.Header.Get("Content-Type") != "application/json" { if ct, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type")); ct != "application/json" {
http.Error(w, "Need JSON request", http.StatusBadRequest) http.Error(w, "Need JSON request", http.StatusBadRequest)
return false return false
} }
...@@ -29,19 +30,15 @@ func DecodeJSONRequest(w http.ResponseWriter, r *http.Request, obj interface{}) ...@@ -29,19 +30,15 @@ func DecodeJSONRequest(w http.ResponseWriter, r *http.Request, obj interface{})
// EncodeJSONResponse writes an application/json response to w. // EncodeJSONResponse writes an application/json response to w.
func EncodeJSONResponse(w http.ResponseWriter, obj interface{}) { func EncodeJSONResponse(w http.ResponseWriter, obj interface{}) {
data, err := json.Marshal(obj)
if err != nil {
log.Printf("JSON serialization error: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
w.Header().Set("Pragma", "no-cache") w.Header().Set("Pragma", "no-cache")
w.Header().Set("Cache-Control", "no-store") w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Expires", "-1") w.Header().Set("Expires", "-1")
w.Header().Set("X-Content-Type-Options", "nosniff") w.Header().Set("X-Content-Type-Options", "nosniff")
if _, err = w.Write(data); err != nil {
log.Printf("error writing response: %v", err) err := json.NewEncoder(w).Encode(obj)
if err != nil {
log.Printf("error writing JSON response: %v", err)
// Too late to return an error to the client now.
} }
} }
...@@ -4,13 +4,12 @@ import ( ...@@ -4,13 +4,12 @@ import (
"fmt" "fmt"
"net" "net"
"net/http" "net/http"
"strings"
"github.com/gorilla/handlers"
) )
type proxyHeaders struct { type proxyHeaders struct {
wrap, phWrap http.Handler wrap http.Handler
forwarders []net.IPNet forwarders []net.IPNet
} }
func newProxyHeaders(h http.Handler, trustedForwarders []string) (http.Handler, error) { func newProxyHeaders(h http.Handler, trustedForwarders []string) (http.Handler, error) {
...@@ -20,7 +19,6 @@ func newProxyHeaders(h http.Handler, trustedForwarders []string) (http.Handler, ...@@ -20,7 +19,6 @@ func newProxyHeaders(h http.Handler, trustedForwarders []string) (http.Handler,
} }
return &proxyHeaders{ return &proxyHeaders{
wrap: h, wrap: h,
phWrap: handlers.ProxyHeaders(h),
forwarders: f, forwarders: f,
}, nil }, nil
} }
...@@ -32,12 +30,28 @@ func (p *proxyHeaders) ServeHTTP(w http.ResponseWriter, r *http.Request) { ...@@ -32,12 +30,28 @@ func (p *proxyHeaders) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} }
ip := net.ParseIP(host) ip := net.ParseIP(host)
if ip != nil && matchIPNetList(ip, p.forwarders) { if ip != nil && matchIPNetList(ip, p.forwarders) {
p.phWrap.ServeHTTP(w, r) if fwd := getForwardedIP(r); fwd != "" {
return r.RemoteAddr = fwd
}
} }
p.wrap.ServeHTTP(w, r) p.wrap.ServeHTTP(w, r)
} }
// Parse the X-Real-IP or X-Forwarded-For headers, if present, to get
// the original client IP.
func getForwardedIP(r *http.Request) string {
if s := r.Header.Get("X-Real-IP"); s != "" {
return s
}
if s := r.Header.Get("X-Forwarded-For"); s != "" {
if n := strings.IndexByte(s, ','); n > 0 {
s = s[:n]
}
return s
}
return ""
}
func fullMask(ip net.IP) net.IPMask { func fullMask(ip net.IP) net.IPMask {
if ip.To4() == nil { if ip.To4() == nil {
return net.CIDRMask(128, 128) return net.CIDRMask(128, 128)
......
...@@ -2,17 +2,20 @@ package serverutil ...@@ -2,17 +2,20 @@ package serverutil
import ( import (
"crypto/tls" "crypto/tls"
"errors"
"fmt" "fmt"
"log" "log"
"net/http" "net/http"
"regexp" "regexp"
"strings"
common "git.autistici.org/ai3/go-common" common "git.autistici.org/ai3/go-common"
) )
// TLSAuthACL describes a single access control entry. Path and // TLSAuthACL describes a single access control entry. Path and
// CommonName are anchored regular expressions (they must match the // CommonName are anchored regular expressions (they must match the
// entire string). // entire string). The first path to match in a list of ACLs will
// identify the ACL to be applied.
type TLSAuthACL struct { type TLSAuthACL struct {
Path string `yaml:"path"` Path string `yaml:"path"`
CommonName string `yaml:"cn"` CommonName string `yaml:"cn"`
...@@ -30,10 +33,11 @@ func (p *TLSAuthACL) compile() error { ...@@ -30,10 +33,11 @@ func (p *TLSAuthACL) compile() error {
return err return err
} }
func (p *TLSAuthACL) match(req *http.Request) bool { func (p *TLSAuthACL) matchPath(req *http.Request) bool {
if !p.pathRx.MatchString(req.URL.Path) { return p.pathRx.MatchString(req.URL.Path)
return false }
}
func (p *TLSAuthACL) matchCN(req *http.Request) bool {
for _, cert := range req.TLS.PeerCertificates { for _, cert := range req.TLS.PeerCertificates {
if p.cnRx.MatchString(cert.Subject.CommonName) { if p.cnRx.MatchString(cert.Subject.CommonName) {
return true return true
...@@ -42,6 +46,31 @@ func (p *TLSAuthACL) match(req *http.Request) bool { ...@@ -42,6 +46,31 @@ func (p *TLSAuthACL) match(req *http.Request) bool {
return false return false
} }
// TLSAuthACLListFlag is a convenience type that allows callers to use
// the 'flag' package to specify a list of TLSAuthACL objects. It
// implements the flag.Value interface.
type TLSAuthACLListFlag []*TLSAuthACL
func (l TLSAuthACLListFlag) String() string {
var out []string
for _, acl := range l {
out = append(out, fmt.Sprintf("%s:%s", acl.Path, acl.CommonName))
}
return strings.Join(out, ",")
}
func (l *TLSAuthACLListFlag) Set(value string) error {
parts := strings.SplitN(value, ":", 2)
if len(parts) != 2 {
return errors.New("bad acl format")
}
*l = append(*l, &TLSAuthACL{
Path: parts[0],
CommonName: parts[1],
})
return nil
}
// TLSAuthConfig stores access control lists for TLS authentication. Access // TLSAuthConfig stores access control lists for TLS authentication. Access
// control lists are matched against the request path and the // control lists are matched against the request path and the
// CommonName component of the peer certificate subject. // CommonName component of the peer certificate subject.
...@@ -54,9 +83,13 @@ func (c *TLSAuthConfig) match(req *http.Request) bool { ...@@ -54,9 +83,13 @@ func (c *TLSAuthConfig) match(req *http.Request) bool {
if c == nil || len(c.Allow) == 0 { if c == nil || len(c.Allow) == 0 {
return true return true
} }
for _, acl := range c.Allow { for _, acl := range c.Allow {
if acl.match(req) { if acl.matchPath(req) {
return true if acl.matchCN(req) {
return true
}
break
} }
} }
return false return false
...@@ -83,21 +116,27 @@ func (c *TLSServerConfig) TLSConfig() (*tls.Config, error) { ...@@ -83,21 +116,27 @@ func (c *TLSServerConfig) TLSConfig() (*tls.Config, error) {
return nil, err return nil, err
} }
cas, err := common.LoadCA(c.CA)
if err != nil {
return nil, err
}
// Set some TLS-level parameters (cipher-related), assuming // Set some TLS-level parameters (cipher-related), assuming
// we're using EC keys. // we're using EC keys.
tlsConf := &tls.Config{ tlsConf := &tls.Config{
Certificates: []tls.Certificate{cert}, Certificates: []tls.Certificate{cert},
ClientAuth: tls.RequireAndVerifyClientCert,
ClientCAs: cas,
CipherSuites: serverCiphers, CipherSuites: serverCiphers,
MinVersion: tls.VersionTLS12, MinVersion: tls.VersionTLS12,
PreferServerCipherSuites: true, PreferServerCipherSuites: true,
NextProtos: []string{"h2", "http/1.1"},
} }
// Require client certificates if a CA is specified.
if c.CA != "" {
cas, err := common.LoadCA(c.CA)
if err != nil {
return nil, err
}
tlsConf.ClientAuth = tls.RequireAndVerifyClientCert
tlsConf.ClientCAs = cas
}
tlsConf.BuildNameToCertificate() tlsConf.BuildNameToCertificate()
return tlsConf, nil return tlsConf, nil
...@@ -125,7 +164,7 @@ func (c *TLSServerConfig) TLSAuthWrapper(h http.Handler) (http.Handler, error) { ...@@ -125,7 +164,7 @@ func (c *TLSServerConfig) TLSAuthWrapper(h http.Handler) (http.Handler, error) {
// Log the failed access, useful for debugging. // Log the failed access, useful for debugging.
var tlsmsg string var tlsmsg string
if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 { if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 {
tlsmsg = fmt.Sprintf(" TLS client '%s' at", r.TLS.PeerCertificates[0].Subject.CommonName) tlsmsg = fmt.Sprintf("TLS client '%s' at", r.TLS.PeerCertificates[0].Subject.CommonName)
} }
log.Printf("unauthorized access to %s from %s%s", r.URL.Path, tlsmsg, r.RemoteAddr) log.Printf("unauthorized access to %s from %s%s", r.URL.Path, tlsmsg, r.RemoteAddr)
http.Error(w, "Forbidden", http.StatusForbidden) http.Error(w, "Forbidden", http.StatusForbidden)
......
package tracing
import (
"encoding/json"
"errors"
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
othttp "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
b3 "go.opentelemetry.io/contrib/propagators/b3"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/zipkin"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
"go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.12.0"
)
var (
// Enabled reports whether tracing is globally enabled or not.
Enabled bool
initOnce sync.Once
)
const globalTracingConfigPath = "/etc/tracing/client.conf"
type tracingConfig struct {
ReportURL string `json:"report_url"`
Sample string `json:"sample"`
}
// Read the global tracing configuration file. Its location is
// hardcoded, but it can be overriden using the TRACING_CONFIG
// environment variable.
func readTracingConfig() (*tracingConfig, error) {
// Read and decode configuration.
cfgPath := globalTracingConfigPath
if s := os.Getenv("TRACING_CONFIG"); s != "" {
cfgPath = s
}
data, err := ioutil.ReadFile(cfgPath)
if err != nil {
return nil, err
}
var config tracingConfig
if err := json.Unmarshal(data, &config); err != nil {
log.Printf("warning: error in tracing configuration: %v, tracing disabled", err)
return nil, err
}
if config.ReportURL == "" {
log.Printf("warning: tracing configuration contains no report_url, tracing disabled")
return nil, errors.New("no report_url")
}
return &config, nil
}
// Compute the service name for Zipkin: this is usually the program
// name (without path), but it can be overriden by the TRACING_SERVICE
// environment variable.
func getServiceName() string {
if s := os.Getenv("TRACING_SERVICE"); s != "" {
return s
}
if s, err := os.Executable(); err == nil {
return filepath.Base(s)
}
return "unknown_service"
}
func defaultResource(serviceName string) *resource.Resource {
attrs := []attribute.KeyValue{
semconv.ServiceNameKey.String(serviceName),
}
if s, err := os.Hostname(); err == nil {
attrs = append(attrs, semconv.HostNameKey.String(s))
}
if s := os.Getenv("FLOAT_SERVICE"); s != "" {
attrs = append(attrs, attribute.Key("float.service").String(s))
}
if s := os.Getenv("FLOAT_CONTAINER_NAME"); s != "" {
attrs = append(attrs, semconv.ContainerNameKey.String(s))
}
if s := os.Getenv("FLOAT_CONTAINER_IMAGE"); s != "" {
imageName := s
tag := "latest"
if strings.Contains(imageName, ":") {
parts := strings.SplitN(imageName, ":", 2)
imageName = parts[0]
tag = parts[1]
}
attrs = append(attrs, semconv.ContainerImageNameKey.String(imageName))
attrs = append(attrs, semconv.ContainerImageTagKey.String(tag))
}
r, _ := resource.Merge(
resource.Default(),
resource.NewWithAttributes(semconv.SchemaURL, attrs...),
)
return r
}
// Initialize tracing. Tracing will be enabled if the system-wide
// tracing configuration file is present and valid. Explicitly set
// TRACING_ENABLE=0 in the environment to disable tracing.
//
// We need to check the configuration as soon as possible, because
// it's likely that client transports are created before HTTP servers,
// and we need to wrap them with opencensus at creation time.
func initTracing(serviceName string) {
initOnce.Do(func() {
// Kill switch from environment.
if s := os.Getenv("TRACING_ENABLE"); s == "0" {
return
}
config, err := readTracingConfig()
if err != nil {
return
}
ze, err := zipkin.New(config.ReportURL)
if err != nil {
log.Printf("error creating Zipkin exporter: %v", err)
return
}
// The sampling policy only applies to incoming requests for
// which tracing is not already enabled: in this case, we
// always pass-through.
var sampler trace.Sampler
switch config.Sample {
case "", "always":
sampler = trace.AlwaysSample()
case "never":
sampler = trace.NeverSample()
default:
frac, err := strconv.ParseFloat(config.Sample, 64)
if err != nil {
log.Printf("warning: error in tracing configuration: sample: %v, tracing disabled", err)
return
}
sampler = trace.TraceIDRatioBased(frac)
}
tp := trace.NewTracerProvider(
trace.WithSampler(trace.ParentBased(sampler)),
trace.WithBatcher(ze),
trace.WithResource(defaultResource(serviceName)),
)
otel.SetTracerProvider(tp)
otel.SetTextMapPropagator(
propagation.NewCompositeTextMapPropagator(
propagation.TraceContext{},
propagation.Baggage{},
b3.New()))
log.Printf("tracing enabled (report_url %s)", config.ReportURL)
Enabled = true
})
}
// Init tracing support, if not using WrapHandler.
func Init() {
initTracing(getServiceName())
}
// WrapTransport optionally wraps a http.RoundTripper with OpenCensus
// tracing functionality, if it is globally enabled.
//
// Must call Init() first.
func WrapTransport(t http.RoundTripper) http.RoundTripper {
if Enabled {
t = othttp.NewTransport(t)
}
return t
}
// WrapHandler wraps a http.Handler with OpenCensus tracing
// functionality, if globally enabled. Automatically calls Init().
func WrapHandler(h http.Handler, endpointAddr string) http.Handler {
serviceName := getServiceName()
initTracing(serviceName)
if !Enabled {
return h
}
// Format span names with the request URL path.
return othttp.NewHandler(
h, serviceName,
othttp.WithSpanNameFormatter(func(op string, r *http.Request) string {
return r.URL.Path
}),
)
}
include:
- "https://git.autistici.org/pipelines/debian/raw/master/common.yml"
- "https://git.autistici.org/pipelines/images/test/golang/raw/master/ci.yml"
GNU LESSER GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
This version of the GNU Lesser General Public License incorporates
the terms and conditions of version 3 of the GNU General Public
License, supplemented by the additional permissions listed below.
0. Additional Definitions.
As used herein, "this License" refers to version 3 of the GNU Lesser
General Public License, and the "GNU GPL" refers to version 3 of the GNU
General Public License.
"The Library" refers to a covered work governed by this License,
other than an Application or a Combined Work as defined below.
An "Application" is any work that makes use of an interface provided
by the Library, but which is not otherwise based on the Library.
Defining a subclass of a class defined by the Library is deemed a mode
of using an interface provided by the Library.
A "Combined Work" is a work produced by combining or linking an
Application with the Library. The particular version of the Library
with which the Combined Work was made is also called the "Linked
Version".
The "Minimal Corresponding Source" for a Combined Work means the
Corresponding Source for the Combined Work, excluding any source code
for portions of the Combined Work that, considered in isolation, are
based on the Application, and not on the Linked Version.
The "Corresponding Application Code" for a Combined Work means the
object code and/or source code for the Application, including any data
and utility programs needed for reproducing the Combined Work from the
Application, but excluding the System Libraries of the Combined Work.
1. Exception to Section 3 of the GNU GPL.
You may convey a covered work under sections 3 and 4 of this License
without being bound by section 3 of the GNU GPL.
2. Conveying Modified Versions.
If you modify a copy of the Library, and, in your modifications, a
facility refers to a function or data to be supplied by an Application
that uses the facility (other than as an argument passed when the
facility is invoked), then you may convey a copy of the modified
version:
a) under this License, provided that you make a good faith effort to
ensure that, in the event an Application does not supply the
function or data, the facility still operates, and performs
whatever part of its purpose remains meaningful, or
b) under the GNU GPL, with none of the additional permissions of
this License applicable to that copy.
3. Object Code Incorporating Material from Library Header Files.
The object code form of an Application may incorporate material from
a header file that is part of the Library. You may convey such object
code under terms of your choice, provided that, if the incorporated
material is not limited to numerical parameters, data structure
layouts and accessors, or small macros, inline functions and templates
(ten or fewer lines in length), you do both of the following:
a) Give prominent notice with each copy of the object code that the
Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the object code with a copy of the GNU GPL and this license
document.
4. Combined Works.
You may convey a Combined Work under terms of your choice that,
taken together, effectively do not restrict modification of the
portions of the Library contained in the Combined Work and reverse
engineering for debugging such modifications, if you also do each of
the following:
a) Give prominent notice with each copy of the Combined Work that
the Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the Combined Work with a copy of the GNU GPL and this license
document.
c) For a Combined Work that displays copyright notices during
execution, include the copyright notice for the Library among
these notices, as well as a reference directing the user to the
copies of the GNU GPL and this license document.
d) Do one of the following:
0) Convey the Minimal Corresponding Source under the terms of this
License, and the Corresponding Application Code in a form
suitable for, and under terms that permit, the user to
recombine or relink the Application with a modified version of
the Linked Version to produce a modified Combined Work, in the
manner specified by section 6 of the GNU GPL for conveying
Corresponding Source.
1) Use a suitable shared library mechanism for linking with the
Library. A suitable mechanism is one that (a) uses at run time
a copy of the Library already present on the user's computer
system, and (b) will operate properly with a modified version
of the Library that is interface-compatible with the Linked
Version.
e) Provide Installation Information, but only if you would otherwise
be required to provide such information under section 6 of the
GNU GPL, and only to the extent that such information is
necessary to install and execute a modified version of the
Combined Work produced by recombining or relinking the
Application with a modified version of the Linked Version. (If
you use option 4d0, the Installation Information must accompany
the Minimal Corresponding Source and Corresponding Application
Code. If you use option 4d1, you must provide the Installation
Information in the manner specified by section 6 of the GNU GPL
for conveying Corresponding Source.)
5. Combined Libraries.
You may place library facilities that are a work based on the
Library side by side in a single library together with other library
facilities that are not Applications and are not covered by this
License, and convey such a combined library under terms of your
choice, if you do both of the following:
a) Accompany the combined library with a copy of the same work based
on the Library, uncombined with any other library facilities,
conveyed under the terms of this License.
b) Give prominent notice with the combined library that part of it
is a work based on the Library, and explaining where to find the
accompanying uncombined form of the same work.
6. Revised Versions of the GNU Lesser General Public License.
The Free Software Foundation may publish revised and/or new versions
of the GNU Lesser General Public License from time to time. Such new
versions will be similar in spirit to the present version, but may
differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the
Library as you received it specifies that a certain numbered version
of the GNU Lesser General Public License "or any later version"
applies to it, you have the option of following the terms and
conditions either of that published version or of any later version
published by the Free Software Foundation. If the Library as you
received it does not specify a version number of the GNU Lesser
General Public License, you may choose any version of the GNU Lesser
General Public License ever published by the Free Software Foundation.
If the Library as you received it specifies that a proxy can decide
whether future versions of the GNU Lesser General Public License shall
apply, that proxy's public statement of acceptance of any version is
permanent authorization for you to choose that version for the
Library.
...@@ -131,3 +131,6 @@ can be used to set up multiple replds instances. For an instance named ...@@ -131,3 +131,6 @@ can be used to set up multiple replds instances. For an instance named
will create the *replds-foo* user and group. Add users that need to will create the *replds-foo* user and group. Add users that need to
read the repository files to that group. The configuration will be read the repository files to that group. The configuration will be
read from */etc/replds/foo.yml*. read from */etc/replds/foo.yml*.
Note that files created by the daemon will be world-readable by
default. Set the process umask if you wish to restrict this further.
...@@ -20,9 +20,9 @@ func newFS(dir string) *FS { ...@@ -20,9 +20,9 @@ func newFS(dir string) *FS {
// getAllNodes scans the root directory and returns a list of // getAllNodes scans the root directory and returns a list of
// Nodes. All errors are ignored. // Nodes. All errors are ignored.
func (f *FS) getAllNodes() []Node { func (f *FS) getAllNodes() []*Node {
var nodes []Node var nodes []*Node
dirLen := len(f.dir) dirOffset := len(f.dir) + 1
if err := filepath.Walk(f.dir, func(path string, info os.FileInfo, err error) error { if err := filepath.Walk(f.dir, func(path string, info os.FileInfo, err error) error {
// Ignore errors. // Ignore errors.
if err != nil { if err != nil {
...@@ -32,8 +32,8 @@ func (f *FS) getAllNodes() []Node { ...@@ -32,8 +32,8 @@ func (f *FS) getAllNodes() []Node {
if !info.Mode().IsRegular() { if !info.Mode().IsRegular() {
return nil return nil
} }
nodes = append(nodes, Node{ nodes = append(nodes, &Node{
Path: path[dirLen+1:], Path: path[dirOffset:],
Timestamp: info.ModTime(), Timestamp: info.ModTime(),
}) })
return nil return nil
...@@ -44,7 +44,7 @@ func (f *FS) getAllNodes() []Node { ...@@ -44,7 +44,7 @@ func (f *FS) getAllNodes() []Node {
} }
// setNode applies a Node to the underlying filesystem. // setNode applies a Node to the underlying filesystem.
func (f *FS) setNode(node Node) error { func (f *FS) setNode(node *Node) error {
path := filepath.Join(f.dir, node.Path) path := filepath.Join(f.dir, node.Path)
// Tombstone? Then delete the file (ignore errors). // Tombstone? Then delete the file (ignore errors).
...@@ -55,7 +55,7 @@ func (f *FS) setNode(node Node) error { ...@@ -55,7 +55,7 @@ func (f *FS) setNode(node Node) error {
// Create parent directory if it does not exist. // Create parent directory if it does not exist.
dir := filepath.Dir(path) dir := filepath.Dir(path)
if _, err := os.Stat(dir); os.IsNotExist(err) { if _, err := os.Stat(dir); os.IsNotExist(err) {
if err = os.MkdirAll(dir, 0700); err != nil { if err = os.MkdirAll(dir, 0755); err != nil {
return err return err
} }
} }
......
...@@ -84,7 +84,7 @@ func (s *HTTPServer) Handler() http.Handler { ...@@ -84,7 +84,7 @@ func (s *HTTPServer) Handler() http.Handler {
return h return h
} }
func nodes2str(nodes []Node) string { func nodes2str(nodes []*Node) string {
var tmp []string var tmp []string
for _, node := range nodes { for _, node := range nodes {
tmp = append(tmp, fmt.Sprintf("%s@%d", node.Path, node.Timestamp.Unix())) tmp = append(tmp, fmt.Sprintf("%s@%d", node.Path, node.Timestamp.Unix()))
......
{
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
"extends": [
"config:base"
]
}
...@@ -4,6 +4,7 @@ import ( ...@@ -4,6 +4,7 @@ import (
"context" "context"
"errors" "errors"
"log" "log"
"math/rand"
"strings" "strings"
"sync" "sync"
"time" "time"
...@@ -21,7 +22,7 @@ var ( ...@@ -21,7 +22,7 @@ var (
pollPeriod = 120 * time.Second pollPeriod = 120 * time.Second
// Timeout for InternalGetNodes requests. // Timeout for InternalGetNodes requests.
getNodesTimeout = 20 * time.Second getNodesTimeout = 120 * time.Second
) )
// Node is an annotated path/value entry. // Node is an annotated path/value entry.
...@@ -34,6 +35,15 @@ type Node struct { ...@@ -34,6 +35,15 @@ type Node struct {
Deleted bool `json:"deleted,omitempty"` Deleted bool `json:"deleted,omitempty"`
} }
func (n *Node) Copy() *Node {
return &Node{
Path: n.Path,
Value: n.Value,
Timestamp: n.Timestamp,
Deleted: n.Deleted,
}
}
func (n *Node) withoutValue() *Node { func (n *Node) withoutValue() *Node {
return &Node{ return &Node{
Path: n.Path, Path: n.Path,
...@@ -42,29 +52,29 @@ func (n *Node) withoutValue() *Node { ...@@ -42,29 +52,29 @@ func (n *Node) withoutValue() *Node {
} }
} }
func (n *Node) metadataOnly() Node { func (n *Node) metadataOnly() *Node {
return Node{ return &Node{
Path: n.Path, Path: n.Path,
Timestamp: n.Timestamp, Timestamp: n.Timestamp,
} }
} }
type internalGetNodesRequest struct { type internalGetNodesRequest struct {
Nodes []Node `json:"nodes"` Nodes []*Node `json:"nodes"`
} }
type internalGetNodesResponse struct { type internalGetNodesResponse struct {
Nodes []Node `json:"nodes"` Nodes []*Node `json:"nodes"`
Partial bool `json:"partial,omitempty"` Partial bool `json:"partial,omitempty"`
} }
type internalUpdateNodesRequest struct { type internalUpdateNodesRequest struct {
Nodes []Node `json:"nodes"` Nodes []*Node `json:"nodes"`
} }
// SetNodesRequest is the request type for the SetNodes method. // SetNodesRequest is the request type for the SetNodes method.
type SetNodesRequest struct { type SetNodesRequest struct {
Nodes []Node `json:"nodes"` Nodes []*Node `json:"nodes"`
} }
// SetNodesResponse is the response returned by the SetNodes method. // SetNodesResponse is the response returned by the SetNodes method.
...@@ -76,17 +86,18 @@ type SetNodesResponse struct { ...@@ -76,17 +86,18 @@ type SetNodesResponse struct {
var errTooOld = errors.New("a more recent value exists for this key") var errTooOld = errors.New("a more recent value exists for this key")
type storage interface { type storage interface {
getAllNodes() []Node getAllNodes() []*Node
getNodeValue(string) ([]byte, error) getNodeValue(string) ([]byte, error)
setNode(Node) error setNode(*Node) error
} }
// Server for the replicated filesync. // Server for the replicated filesync.
type Server struct { type Server struct {
peers []string peers []string
network *network network *network
storage storage storage storage
logger *log.Logger logger *log.Logger
readonly bool
wg sync.WaitGroup wg sync.WaitGroup
stop chan bool stop chan bool
...@@ -103,7 +114,7 @@ func stripTrailingSlash(peers []string) []string { ...@@ -103,7 +114,7 @@ func stripTrailingSlash(peers []string) []string {
} }
// NewServer creates a new Server with the given peers and backends. // NewServer creates a new Server with the given peers and backends.
func NewServer(peers []string, dir string, tlsConfig *clientutil.TLSClientConfig) (*Server, error) { func NewServer(peers []string, dir string, tlsConfig *clientutil.TLSClientConfig, readonly bool) (*Server, error) {
peers = stripTrailingSlash(peers) peers = stripTrailingSlash(peers)
network, err := newNetwork(peers, tlsConfig) network, err := newNetwork(peers, tlsConfig)
...@@ -112,26 +123,44 @@ func NewServer(peers []string, dir string, tlsConfig *clientutil.TLSClientConfig ...@@ -112,26 +123,44 @@ func NewServer(peers []string, dir string, tlsConfig *clientutil.TLSClientConfig
} }
s := &Server{ s := &Server{
peers: peers, peers: peers,
network: network, network: network,
storage: newFS(dir), storage: newFS(dir),
stop: make(chan bool), stop: make(chan bool),
nodes: make(map[string]*Node), nodes: make(map[string]*Node),
readonly: readonly,
} }
for _, node := range s.storage.getAllNodes() { // Do not scan the local filesystem at startup on readonly
s.nodes[node.Path] = &node // instances, downloading everything is safer as we do not
} // risk injecting stale data.
if len(s.nodes) > 0 { if !readonly {
log.Printf("found %d entries in %s", len(s.nodes), dir) for _, node := range s.storage.getAllNodes() {
s.nodes[node.Path] = node
}
if len(s.nodes) > 0 {
log.Printf("found %d entries in %s", len(s.nodes), dir)
}
} }
for _, peer := range peers { // The background workers are different if the instance is
// readonly: normally we run a synchronization worker for each
// peer, while in the readonly case we periodically poll a
// different random peer.
if readonly {
s.wg.Add(1) s.wg.Add(1)
go func(peer string) { go func() {
s.pollThread(peer) s.pollRandomPeerThread()
s.wg.Done() s.wg.Done()
}(peer) }()
} else {
for _, peer := range peers {
s.wg.Add(1)
go func(peer string) {
s.pollThread(peer)
s.wg.Done()
}(peer)
}
} }
return s, nil return s, nil
...@@ -156,7 +185,7 @@ func (s *Server) log(fmt string, args ...interface{}) { ...@@ -156,7 +185,7 @@ func (s *Server) log(fmt string, args ...interface{}) {
// Update one or more nodes as a "transaction": if one update fails, // Update one or more nodes as a "transaction": if one update fails,
// the rest are aborted (but previous commits are not reverted, yet). // the rest are aborted (but previous commits are not reverted, yet).
// Will ping peers with the data when updatePeers is true. // Will ping peers with the data when updatePeers is true.
func (s *Server) doSetNodes(ctx context.Context, nodes []Node, updatePeers bool) (*SetNodesResponse, error) { func (s *Server) doSetNodes(ctx context.Context, nodes []*Node, updatePeers bool) (*SetNodesResponse, error) {
// Update local state. // Update local state.
for _, node := range nodes { for _, node := range nodes {
cur, ok := s.nodes[node.Path] cur, ok := s.nodes[node.Path]
...@@ -185,7 +214,9 @@ func (s *Server) doSetNodes(ctx context.Context, nodes []Node, updatePeers bool) ...@@ -185,7 +214,9 @@ func (s *Server) doSetNodes(ctx context.Context, nodes []Node, updatePeers bool)
var res SetNodesResponse var res SetNodesResponse
if updatePeers { if updatePeers {
// Ping remote nodes with updated state. // Ping remote nodes with the updated state. This
// reduces the latency of update propagation (we don't
// have to wait until the peers poll us).
res.HostsOk = 1 res.HostsOk = 1
for _, peer := range s.peers { for _, peer := range s.peers {
err := s.network.Client(peer).internalUpdateNodes(ctx, &internalUpdateNodesRequest{ err := s.network.Client(peer).internalUpdateNodes(ctx, &internalUpdateNodesRequest{
...@@ -216,13 +247,13 @@ func (s *Server) internalUpdateNodes(ctx context.Context, req *internalUpdateNod ...@@ -216,13 +247,13 @@ func (s *Server) internalUpdateNodes(ctx context.Context, req *internalUpdateNod
return err return err
} }
func (s *Server) getNodeWithValue(node *Node) (out Node, err error) { func (s *Server) getNodeWithValue(node *Node) (out *Node, err error) {
out = *node out = node.Copy()
out.Value, err = s.storage.getNodeValue(node.Path) out.Value, err = s.storage.getNodeValue(node.Path)
return return
} }
func nodeDiff(reqNodes []Node, myNodes map[string]*Node, adder func(*Node) bool) ([]string, bool) { func nodeDiff(reqNodes []*Node, myNodes map[string]*Node, adder func(*Node) bool) ([]string, bool) {
var missing []string var missing []string
tmp := make(map[string]struct{}) tmp := make(map[string]struct{})
for _, reqNode := range reqNodes { for _, reqNode := range reqNodes {
...@@ -293,14 +324,8 @@ func (s *Server) internalGetNodes(ctx context.Context, req *internalGetNodesRequ ...@@ -293,14 +324,8 @@ func (s *Server) internalGetNodes(ctx context.Context, req *internalGetNodesRequ
return &resp, nil return &resp, nil
} }
func (s *Server) pollThread(peer string) { func (s *Server) backgroundThread(fn func()) {
poll := func() { fn()
if err := s.pollPeer(peer); err != nil {
s.log("error polling peer %s: %v", peer, err)
}
}
poll()
tick := time.NewTicker(pollPeriod) tick := time.NewTicker(pollPeriod)
defer tick.Stop() defer tick.Stop()
...@@ -309,11 +334,32 @@ func (s *Server) pollThread(peer string) { ...@@ -309,11 +334,32 @@ func (s *Server) pollThread(peer string) {
case <-s.stop: case <-s.stop:
return return
case <-tick.C: case <-tick.C:
poll() fn()
} }
} }
} }
func (s *Server) pollThread(peer string) {
s.backgroundThread(func() {
if err := s.pollPeer(peer); err != nil {
s.log("error polling peer %s: %v", peer, err)
}
})
}
func (s *Server) pollRandomPeerThread() {
s.backgroundThread(func() {
if len(s.peers) == 0 {
s.log("no peers to poll!")
return
}
peer := s.peers[rand.Intn(len(s.peers))]
if err := s.pollPeer(peer); err != nil {
s.log("error polling peer %s: %v", peer, err)
}
})
}
func (s *Server) pollPeer(peer string) error { func (s *Server) pollPeer(peer string) error {
for { for {
partial, err := s.pollPeerRequest(peer) partial, err := s.pollPeerRequest(peer)
...@@ -353,7 +399,7 @@ var ( ...@@ -353,7 +399,7 @@ var (
maxTSValue int64 maxTSValue int64
) )
func updateMaxTimestamp(nodes []Node) { func updateMaxTimestamp(nodes []*Node) {
var max int64 var max int64
for _, node := range nodes { for _, node := range nodes {
t := node.Timestamp.Unix() t := node.Timestamp.Unix()
...@@ -375,6 +421,9 @@ func (s *Server) pollPeerRequest(peer string) (bool, error) { ...@@ -375,6 +421,9 @@ func (s *Server) pollPeerRequest(peer string) (bool, error) {
peerName := stripHTTP(peer) peerName := stripHTTP(peer)
peerRequests.With(prometheus.Labels{"peer": peerName}).Inc() peerRequests.With(prometheus.Labels{"peer": peerName}).Inc()
// Create a shallow copy of the list of the Nodes we know
// about, containing just path/timestamp, to send to the peer
// in our GetNodes request.
s.mx.Lock() s.mx.Lock()
var req internalGetNodesRequest var req internalGetNodesRequest
for _, node := range s.nodes { for _, node := range s.nodes {
...@@ -399,6 +448,7 @@ func (s *Server) pollPeerRequest(peer string) (bool, error) { ...@@ -399,6 +448,7 @@ func (s *Server) pollPeerRequest(peer string) (bool, error) {
return false, nil return false, nil
} }
// Update the internal state with the new data.
return resp.Partial, s.internalUpdateNodes(ctx, &internalUpdateNodesRequest{Nodes: resp.Nodes}) return resp.Partial, s.internalUpdateNodes(ctx, &internalUpdateNodesRequest{Nodes: resp.Nodes})
} }
......
*.swp
language: go
go:
- 1.x
- tip
env:
- GO111MODULE=on
install:
- go mod download
script:
- go test -race -v
---
layout: code-of-conduct
version: v1.0
---
This code of conduct outlines our expectations for participants within the **NYTimes/gziphandler** community, as well as steps to reporting unacceptable behavior. We are committed to providing a welcoming and inspiring community for all and expect our code of conduct to be honored. Anyone who violates this code of conduct may be banned from the community.
Our open source community strives to:
* **Be friendly and patient.**
* **Be welcoming**: We strive to be a community that welcomes and supports people of all backgrounds and identities. This includes, but is not limited to members of any race, ethnicity, culture, national origin, colour, immigration status, social and economic class, educational level, sex, sexual orientation, gender identity and expression, age, size, family status, political belief, religion, and mental and physical ability.
* **Be considerate**: Your work will be used by other people, and you in turn will depend on the work of others. Any decision you take will affect users and colleagues, and you should take those consequences into account when making decisions. Remember that we're a world-wide community, so you might not be communicating in someone else's primary language.
* **Be respectful**: Not all of us will agree all the time, but disagreement is no excuse for poor behavior and poor manners. We might all experience some frustration now and then, but we cannot allow that frustration to turn into a personal attack. It’s important to remember that a community where people feel uncomfortable or threatened is not a productive one.
* **Be careful in the words that we choose**: we are a community of professionals, and we conduct ourselves professionally. Be kind to others. Do not insult or put down other participants. Harassment and other exclusionary behavior aren't acceptable.
* **Try to understand why we disagree**: Disagreements, both social and technical, happen all the time. It is important that we resolve disagreements and differing views constructively. Remember that we’re different. The strength of our community comes from its diversity, people from a wide range of backgrounds. Different people have different perspectives on issues. Being unable to understand why someone holds a viewpoint doesn’t mean that they’re wrong. Don’t forget that it is human to err and blaming each other doesn’t get us anywhere. Instead, focus on helping to resolve issues and learning from mistakes.
## Definitions
Harassment includes, but is not limited to:
- Offensive comments related to gender, gender identity and expression, sexual orientation, disability, mental illness, neuro(a)typicality, physical appearance, body size, race, age, regional discrimination, political or religious affiliation
- Unwelcome comments regarding a person’s lifestyle choices and practices, including those related to food, health, parenting, drugs, and employment
- Deliberate misgendering. This includes deadnaming or persistently using a pronoun that does not correctly reflect a person's gender identity. You must address people by the name they give you when not addressing them by their username or handle
- Physical contact and simulated physical contact (eg, textual descriptions like “*hug*” or “*backrub*”) without consent or after a request to stop
- Threats of violence, both physical and psychological
- Incitement of violence towards any individual, including encouraging a person to commit suicide or to engage in self-harm
- Deliberate intimidation
- Stalking or following
- Harassing photography or recording, including logging online activity for harassment purposes
- Sustained disruption of discussion
- Unwelcome sexual attention, including gratuitous or off-topic sexual images or behaviour
- Pattern of inappropriate social contact, such as requesting/assuming inappropriate levels of intimacy with others
- Continued one-on-one communication after requests to cease
- Deliberate “outing” of any aspect of a person’s identity without their consent except as necessary to protect others from intentional abuse
- Publication of non-harassing private communication
Our open source community prioritizes marginalized people’s safety over privileged people’s comfort. We will not act on complaints regarding:
- ‘Reverse’ -isms, including ‘reverse racism,’ ‘reverse sexism,’ and ‘cisphobia’
- Reasonable communication of boundaries, such as “leave me alone,” “go away,” or “I’m not discussing this with you”
- Refusal to explain or debate social justice concepts
- Communicating in a ‘tone’ you don’t find congenial
- Criticizing racist, sexist, cissexist, or otherwise oppressive behavior or assumptions
### Diversity Statement
We encourage everyone to participate and are committed to building a community for all. Although we will fail at times, we seek to treat everyone both as fairly and equally as possible. Whenever a participant has made a mistake, we expect them to take responsibility for it. If someone has been harmed or offended, it is our responsibility to listen carefully and respectfully, and do our best to right the wrong.
Although this list cannot be exhaustive, we explicitly honor diversity in age, gender, gender identity or expression, culture, ethnicity, language, national origin, political beliefs, profession, race, religion, sexual orientation, socioeconomic status, and technical ability. We will not tolerate discrimination based on any of the protected
characteristics above, including participants with disabilities.
### Reporting Issues
If you experience or witness unacceptable behavior—or have any other concerns—please report it by contacting us via **code@nytimes.com**. All reports will be handled with discretion. In your report please include:
- Your contact information.
- Names (real, nicknames, or pseudonyms) of any individuals involved. If there are additional witnesses, please
include them as well. Your account of what occurred, and if you believe the incident is ongoing. If there is a publicly available record (e.g. a mailing list archive or a public IRC logger), please include a link.
- Any additional information that may be helpful.
After filing a report, a representative will contact you personally, review the incident, follow up with any additional questions, and make a decision as to how to respond. If the person who is harassing you is part of the response team, they will recuse themselves from handling your incident. If the complaint originates from a member of the response team, it will be handled by a different member of the response team. We will respect confidentiality requests for the purpose of protecting victims of abuse.
### Attribution & Acknowledgements
We all stand on the shoulders of giants across many open source communities. We'd like to thank the communities and projects that established code of conducts and diversity statements as our inspiration:
* [Django](https://www.djangoproject.com/conduct/reporting/)
* [Python](https://www.python.org/community/diversity/)
* [Ubuntu](http://www.ubuntu.com/about/about-ubuntu/conduct)
* [Contributor Covenant](http://contributor-covenant.org/)
* [Geek Feminism](http://geekfeminism.org/about/code-of-conduct/)
* [Citizen Code of Conduct](http://citizencodeofconduct.org/)
This Code of Conduct was based on https://github.com/todogroup/opencodeofconduct
# Contributing to NYTimes/gziphandler
This is an open source project started by handful of developers at The New York Times and open to the entire Go community.
We really appreciate your help!
## Filing issues
When filing an issue, make sure to answer these five questions:
1. What version of Go are you using (`go version`)?
2. What operating system and processor architecture are you using?
3. What did you do?
4. What did you expect to see?
5. What did you see instead?
## Contributing code
Before submitting changes, please follow these guidelines:
1. Check the open issues and pull requests for existing discussions.
2. Open an issue to discuss a new feature.
3. Write tests.
4. Make sure code follows the ['Go Code Review Comments'](https://github.com/golang/go/wiki/CodeReviewComments).
5. Make sure your changes pass `go test`.
6. Make sure the entire test suite passes locally and on Travis CI.
7. Open a Pull Request.
8. [Squash your commits](http://gitready.com/advanced/2009/02/10/squashing-commits-with-rebase.html) after receiving feedback and add a [great commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
Unless otherwise noted, the gziphandler source files are distributed under the Apache 2.0-style license found in the LICENSE.md file.