Skip to content
Snippets Groups Projects
Commit 531700b8 authored by ale's avatar ale
Browse files

Merge branch 'renovate/git.autistici.org-ai3-go-common-digest' into 'master'

Update git.autistici.org/ai3/go-common digest to 1eb6de8

See merge request !9
parents 9186b608 68ee9649
Branches
No related tags found
1 merge request!9Update git.autistici.org/ai3/go-common digest to 1eb6de8
Showing
with 1936 additions and 329 deletions
......@@ -3,8 +3,8 @@ module git.autistici.org/ai3/tools/aux-db
go 1.14
require (
git.autistici.org/ai3/go-common v0.0.0-20211206113217-48c52c9f8160
git.autistici.org/ai3/go-common v0.0.0-20220813062958-1eb6de82ce83
github.com/golang-migrate/migrate/v4 v4.14.1
github.com/mattn/go-sqlite3 v1.14.7
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
gopkg.in/yaml.v3 v3.0.1
)
This diff is collapsed.
......@@ -9,6 +9,8 @@ run_tests:
artifacts:
when: always
reports:
cobertura: cover.xml
coverage_report:
coverage_format: cobertura
path: cover.xml
junit: report.xml
......@@ -5,25 +5,24 @@ go 1.11
require (
contrib.go.opencensus.io/exporter/zipkin v0.1.2
github.com/NYTimes/gziphandler v1.1.1
github.com/amoghe/go-crypt v0.0.0-20191109212615-b2ff80594b7f
github.com/amoghe/go-crypt v0.0.0-20220222110647-20eada5f5964
github.com/bbrks/wrap/v2 v2.5.0
github.com/cenkalti/backoff/v4 v4.1.2
github.com/cenkalti/backoff/v4 v4.1.3
github.com/coreos/go-systemd/v22 v22.3.2
github.com/duo-labs/webauthn v0.0.0-20200714211715-1daaee874e43
github.com/duo-labs/webauthn v0.0.0-20220330035159-03696f3d4499
github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594
github.com/fxamacker/cbor/v2 v2.2.0
github.com/go-asn1-ber/asn1-ber v1.5.3
github.com/go-ldap/ldap/v3 v3.4.1
github.com/fxamacker/cbor/v2 v2.4.0
github.com/go-asn1-ber/asn1-ber v1.5.4
github.com/go-ldap/ldap/v3 v3.4.4
github.com/gofrs/flock v0.8.0 // indirect
github.com/google/go-cmp v0.5.6
github.com/gorilla/handlers v1.5.1
github.com/lunixbochs/struc v0.0.0-20200707160740-784aaebc1d40
github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75
github.com/openzipkin/zipkin-go v0.2.5
github.com/prometheus/client_golang v1.11.0
github.com/openzipkin/zipkin-go v0.4.0
github.com/prometheus/client_golang v1.12.2
github.com/russross/blackfriday/v2 v2.1.0
github.com/theckman/go-flock v0.8.1
go.opencensus.io v0.23.0
golang.org/x/crypto v0.0.0-20211202192323-5770296d904e
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
)
This diff is collapsed.
......@@ -4,12 +4,11 @@ import (
"fmt"
"net"
"net/http"
"github.com/gorilla/handlers"
"strings"
)
type proxyHeaders struct {
wrap, phWrap http.Handler
wrap http.Handler
forwarders []net.IPNet
}
......@@ -20,7 +19,6 @@ func newProxyHeaders(h http.Handler, trustedForwarders []string) (http.Handler,
}
return &proxyHeaders{
wrap: h,
phWrap: handlers.ProxyHeaders(h),
forwarders: f,
}, nil
}
......@@ -32,12 +30,28 @@ func (p *proxyHeaders) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
ip := net.ParseIP(host)
if ip != nil && matchIPNetList(ip, p.forwarders) {
p.phWrap.ServeHTTP(w, r)
return
if fwd := getForwardedIP(r); fwd != "" {
r.RemoteAddr = fwd
}
}
p.wrap.ServeHTTP(w, r)
}
// Parse the X-Real-IP or X-Forwarded-For headers, if present, to get
// the original client IP.
func getForwardedIP(r *http.Request) string {
if s := r.Header.Get("X-Real-IP"); s != "" {
return s
}
if s := r.Header.Get("X-Forwarded-For"); s != "" {
if n := strings.IndexByte(s, ','); n > 0 {
s = s[:n]
}
return s
}
return ""
}
func fullMask(ip net.IP) net.IPMask {
if ip.To4() == nil {
return net.CIDRMask(128, 128)
......
......@@ -147,6 +147,9 @@ func (b *ExponentialBackOff) incrementCurrentInterval() {
// Returns a random value from the following interval:
// [currentInterval - randomizationFactor * currentInterval, currentInterval + randomizationFactor * currentInterval].
func getRandomValueFromInterval(randomizationFactor, random float64, currentInterval time.Duration) time.Duration {
if randomizationFactor == 0 {
return currentInterval // make sure no randomness is used when randomizationFactor is 0.
}
var delta = randomizationFactor * float64(currentInterval)
var minInterval = float64(currentInterval) - delta
var maxInterval = float64(currentInterval) + delta
......
language: go
go:
- "1.x"
- master
env:
- TAGS=""
- TAGS="-tags purego"
script: go test $TAGS -v ./...
# xxhash
[![GoDoc](https://godoc.org/github.com/cespare/xxhash?status.svg)](https://godoc.org/github.com/cespare/xxhash)
[![Build Status](https://travis-ci.org/cespare/xxhash.svg?branch=master)](https://travis-ci.org/cespare/xxhash)
[![Go Reference](https://pkg.go.dev/badge/github.com/cespare/xxhash/v2.svg)](https://pkg.go.dev/github.com/cespare/xxhash/v2)
[![Test](https://github.com/cespare/xxhash/actions/workflows/test.yml/badge.svg)](https://github.com/cespare/xxhash/actions/workflows/test.yml)
xxhash is a Go implementation of the 64-bit
[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a
......@@ -64,4 +64,6 @@ $ go test -benchtime 10s -bench '/xxhash,direct,bytes'
- [InfluxDB](https://github.com/influxdata/influxdb)
- [Prometheus](https://github.com/prometheus/prometheus)
- [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics)
- [FreeCache](https://github.com/coocood/freecache)
- [FastCache](https://github.com/VictoriaMetrics/fastcache)
......@@ -193,7 +193,6 @@ func (d *Digest) UnmarshalBinary(b []byte) error {
b, d.v4 = consumeUint64(b)
b, d.total = consumeUint64(b)
copy(d.mem[:], b)
b = b[len(d.mem):]
d.n = int(d.total % uint64(len(d.mem)))
return nil
}
......
......@@ -6,7 +6,7 @@
// Register allocation:
// AX h
// CX pointer to advance through b
// SI pointer to advance through b
// DX n
// BX loop end
// R8 v1, k1
......@@ -16,39 +16,39 @@
// R12 tmp
// R13 prime1v
// R14 prime2v
// R15 prime4v
// DI prime4v
// round reads from and advances the buffer pointer in CX.
// round reads from and advances the buffer pointer in SI.
// It assumes that R13 has prime1v and R14 has prime2v.
#define round(r) \
MOVQ (CX), R12 \
ADDQ $8, CX \
MOVQ (SI), R12 \
ADDQ $8, SI \
IMULQ R14, R12 \
ADDQ R12, r \
ROLQ $31, r \
IMULQ R13, r
// mergeRound applies a merge round on the two registers acc and val.
// It assumes that R13 has prime1v, R14 has prime2v, and R15 has prime4v.
// It assumes that R13 has prime1v, R14 has prime2v, and DI has prime4v.
#define mergeRound(acc, val) \
IMULQ R14, val \
ROLQ $31, val \
IMULQ R13, val \
XORQ val, acc \
IMULQ R13, acc \
ADDQ R15, acc
ADDQ DI, acc
// func Sum64(b []byte) uint64
TEXT ·Sum64(SB), NOSPLIT, $0-32
// Load fixed primes.
MOVQ ·prime1v(SB), R13
MOVQ ·prime2v(SB), R14
MOVQ ·prime4v(SB), R15
MOVQ ·prime4v(SB), DI
// Load slice.
MOVQ b_base+0(FP), CX
MOVQ b_base+0(FP), SI
MOVQ b_len+8(FP), DX
LEAQ (CX)(DX*1), BX
LEAQ (SI)(DX*1), BX
// The first loop limit will be len(b)-32.
SUBQ $32, BX
......@@ -65,14 +65,14 @@ TEXT ·Sum64(SB), NOSPLIT, $0-32
XORQ R11, R11
SUBQ R13, R11
// Loop until CX > BX.
// Loop until SI > BX.
blockLoop:
round(R8)
round(R9)
round(R10)
round(R11)
CMPQ CX, BX
CMPQ SI, BX
JLE blockLoop
MOVQ R8, AX
......@@ -100,16 +100,16 @@ noBlocks:
afterBlocks:
ADDQ DX, AX
// Right now BX has len(b)-32, and we want to loop until CX > len(b)-8.
// Right now BX has len(b)-32, and we want to loop until SI > len(b)-8.
ADDQ $24, BX
CMPQ CX, BX
CMPQ SI, BX
JG fourByte
wordLoop:
// Calculate k1.
MOVQ (CX), R8
ADDQ $8, CX
MOVQ (SI), R8
ADDQ $8, SI
IMULQ R14, R8
ROLQ $31, R8
IMULQ R13, R8
......@@ -117,18 +117,18 @@ wordLoop:
XORQ R8, AX
ROLQ $27, AX
IMULQ R13, AX
ADDQ R15, AX
ADDQ DI, AX
CMPQ CX, BX
CMPQ SI, BX
JLE wordLoop
fourByte:
ADDQ $4, BX
CMPQ CX, BX
CMPQ SI, BX
JG singles
MOVL (CX), R8
ADDQ $4, CX
MOVL (SI), R8
ADDQ $4, SI
IMULQ R13, R8
XORQ R8, AX
......@@ -138,19 +138,19 @@ fourByte:
singles:
ADDQ $4, BX
CMPQ CX, BX
CMPQ SI, BX
JGE finalize
singlesLoop:
MOVBQZX (CX), R12
ADDQ $1, CX
MOVBQZX (SI), R12
ADDQ $1, SI
IMULQ ·prime5v(SB), R12
XORQ R12, AX
ROLQ $11, AX
IMULQ R13, AX
CMPQ CX, BX
CMPQ SI, BX
JL singlesLoop
finalize:
......@@ -179,9 +179,9 @@ TEXT ·writeBlocks(SB), NOSPLIT, $0-40
MOVQ ·prime2v(SB), R14
// Load slice.
MOVQ b_base+8(FP), CX
MOVQ b_base+8(FP), SI
MOVQ b_len+16(FP), DX
LEAQ (CX)(DX*1), BX
LEAQ (SI)(DX*1), BX
SUBQ $32, BX
// Load vN from d.
......@@ -199,7 +199,7 @@ blockLoop:
round(R10)
round(R11)
CMPQ CX, BX
CMPQ SI, BX
JLE blockLoop
// Copy vN back to d.
......@@ -208,8 +208,8 @@ blockLoop:
MOVQ R10, 16(AX)
MOVQ R11, 24(AX)
// The number of bytes written is CX minus the old base pointer.
SUBQ b_base+8(FP), CX
MOVQ CX, ret+32(FP)
// The number of bytes written is SI minus the old base pointer.
SUBQ b_base+8(FP), SI
MOVQ SI, ret+32(FP)
RET
......@@ -6,41 +6,52 @@
package xxhash
import (
"reflect"
"unsafe"
)
// Notes:
// In the future it's possible that compiler optimizations will make these
// XxxString functions unnecessary by realizing that calls such as
// Sum64([]byte(s)) don't need to copy s. See https://golang.org/issue/2205.
// If that happens, even if we keep these functions they can be replaced with
// the trivial safe code.
// NOTE: The usual way of doing an unsafe string-to-[]byte conversion is:
//
// See https://groups.google.com/d/msg/golang-nuts/dcjzJy-bSpw/tcZYBzQqAQAJ
// for some discussion about these unsafe conversions.
// var b []byte
// bh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
// bh.Data = (*reflect.StringHeader)(unsafe.Pointer(&s)).Data
// bh.Len = len(s)
// bh.Cap = len(s)
//
// In the future it's possible that compiler optimizations will make these
// unsafe operations unnecessary: https://golang.org/issue/2205.
// Unfortunately, as of Go 1.15.3 the inliner's cost model assigns a high enough
// weight to this sequence of expressions that any function that uses it will
// not be inlined. Instead, the functions below use a different unsafe
// conversion designed to minimize the inliner weight and allow both to be
// inlined. There is also a test (TestInlining) which verifies that these are
// inlined.
//
// Both of these wrapper functions still incur function call overhead since they
// will not be inlined. We could write Go/asm copies of Sum64 and Digest.Write
// for strings to squeeze out a bit more speed. Mid-stack inlining should
// eventually fix this.
// See https://github.com/golang/go/issues/42739 for discussion.
// Sum64String computes the 64-bit xxHash digest of s.
// It may be faster than Sum64([]byte(s)) by avoiding a copy.
func Sum64String(s string) uint64 {
var b []byte
bh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
bh.Data = (*reflect.StringHeader)(unsafe.Pointer(&s)).Data
bh.Len = len(s)
bh.Cap = len(s)
b := *(*[]byte)(unsafe.Pointer(&sliceHeader{s, len(s)}))
return Sum64(b)
}
// WriteString adds more data to d. It always returns len(s), nil.
// It may be faster than Write([]byte(s)) by avoiding a copy.
func (d *Digest) WriteString(s string) (n int, err error) {
var b []byte
bh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
bh.Data = (*reflect.StringHeader)(unsafe.Pointer(&s)).Data
bh.Len = len(s)
bh.Cap = len(s)
return d.Write(b)
d.Write(*(*[]byte)(unsafe.Pointer(&sliceHeader{s, len(s)})))
// d.Write always returns len(s), nil.
// Ignoring the return output and returning these fixed values buys a
// savings of 6 in the inliner's cost model.
return len(s), nil
}
// sliceHeader is similar to reflect.SliceHeader, but it assumes that the layout
// of the first two words is the same as the layout of a string.
type sliceHeader struct {
s string
cap int
}
language: go
go:
- 1.6
- 1.7
- 1.8
Copyright (c) 2016 Felix Geisendörfer (felix@debuggable.com)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
.PHONY: ci generate clean
ci: clean generate
go test -v ./...
generate:
go generate .
clean:
rm -rf *_generated*.go
# httpsnoop
Package httpsnoop provides an easy way to capture http related metrics (i.e.
response time, bytes written, and http status code) from your application's
http.Handlers.
Doing this requires non-trivial wrapping of the http.ResponseWriter interface,
which is also exposed for users interested in a more low-level API.
[![GoDoc](https://godoc.org/github.com/felixge/httpsnoop?status.svg)](https://godoc.org/github.com/felixge/httpsnoop)
[![Build Status](https://travis-ci.org/felixge/httpsnoop.svg?branch=master)](https://travis-ci.org/felixge/httpsnoop)
## Usage Example
```go
// myH is your app's http handler, perhaps a http.ServeMux or similar.
var myH http.Handler
// wrappedH wraps myH in order to log every request.
wrappedH := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
m := httpsnoop.CaptureMetrics(myH, w, r)
log.Printf(
"%s %s (code=%d dt=%s written=%d)",
r.Method,
r.URL,
m.Code,
m.Duration,
m.Written,
)
})
http.ListenAndServe(":8080", wrappedH)
```
## Why this package exists
Instrumenting an application's http.Handler is surprisingly difficult.
However if you google for e.g. "capture ResponseWriter status code" you'll find
lots of advise and code examples that suggest it to be a fairly trivial
undertaking. Unfortunately everything I've seen so far has a high chance of
breaking your application.
The main problem is that a `http.ResponseWriter` often implements additional
interfaces such as `http.Flusher`, `http.CloseNotifier`, `http.Hijacker`, `http.Pusher`, and
`io.ReaderFrom`. So the naive approach of just wrapping `http.ResponseWriter`
in your own struct that also implements the `http.ResponseWriter` interface
will hide the additional interfaces mentioned above. This has a high change of
introducing subtle bugs into any non-trivial application.
Another approach I've seen people take is to return a struct that implements
all of the interfaces above. However, that's also problematic, because it's
difficult to fake some of these interfaces behaviors when the underlying
`http.ResponseWriter` doesn't have an implementation. It's also dangerous,
because an application may choose to operate differently, merely because it
detects the presence of these additional interfaces.
This package solves this problem by checking which additional interfaces a
`http.ResponseWriter` implements, returning a wrapped version implementing the
exact same set of interfaces.
Additionally this package properly handles edge cases such as `WriteHeader` not
being called, or called more than once, as well as concurrent calls to
`http.ResponseWriter` methods, and even calls happening after the wrapped
`ServeHTTP` has already returned.
Unfortunately this package is not perfect either. It's possible that it is
still missing some interfaces provided by the go core (let me know if you find
one), and it won't work for applications adding their own interfaces into the
mix.
However, hopefully the explanation above has sufficiently scared you of rolling
your own solution to this problem. httpsnoop may still break your application,
but at least it tries to avoid it as much as possible.
Anyway, the real problem here is that smuggling additional interfaces inside
`http.ResponseWriter` is a problematic design choice, but it probably goes as
deep as the Go language specification itself. But that's okay, I still prefer
Go over the alternatives ;).
## Performance
```
BenchmarkBaseline-8 20000 94912 ns/op
BenchmarkCaptureMetrics-8 20000 95461 ns/op
```
As you can see, using `CaptureMetrics` on a vanilla http.Handler introduces an
overhead of ~500 ns per http request on my machine. However, the margin of
error appears to be larger than that, therefor it should be reasonable to
assume that the overhead introduced by `CaptureMetrics` is absolutely
negligible.
## License
MIT
package httpsnoop
import (
"io"
"net/http"
"sync"
"time"
)
// Metrics holds metrics captured from CaptureMetrics.
type Metrics struct {
// Code is the first http response code passed to the WriteHeader func of
// the ResponseWriter. If no such call is made, a default code of 200 is
// assumed instead.
Code int
// Duration is the time it took to execute the handler.
Duration time.Duration
// Written is the number of bytes successfully written by the Write or
// ReadFrom function of the ResponseWriter. ResponseWriters may also write
// data to their underlaying connection directly (e.g. headers), but those
// are not tracked. Therefor the number of Written bytes will usually match
// the size of the response body.
Written int64
}
// CaptureMetrics wraps the given hnd, executes it with the given w and r, and
// returns the metrics it captured from it.
func CaptureMetrics(hnd http.Handler, w http.ResponseWriter, r *http.Request) Metrics {
return CaptureMetricsFn(w, func(ww http.ResponseWriter) {
hnd.ServeHTTP(ww, r)
})
}
// CaptureMetricsFn wraps w and calls fn with the wrapped w and returns the
// resulting metrics. This is very similar to CaptureMetrics (which is just
// sugar on top of this func), but is a more usable interface if your
// application doesn't use the Go http.Handler interface.
func CaptureMetricsFn(w http.ResponseWriter, fn func(http.ResponseWriter)) Metrics {
var (
start = time.Now()
m = Metrics{Code: http.StatusOK}
headerWritten bool
lock sync.Mutex
hooks = Hooks{
WriteHeader: func(next WriteHeaderFunc) WriteHeaderFunc {
return func(code int) {
next(code)
lock.Lock()
defer lock.Unlock()
if !headerWritten {
m.Code = code
headerWritten = true
}
}
},
Write: func(next WriteFunc) WriteFunc {
return func(p []byte) (int, error) {
n, err := next(p)
lock.Lock()
defer lock.Unlock()
m.Written += int64(n)
headerWritten = true
return n, err
}
},
ReadFrom: func(next ReadFromFunc) ReadFromFunc {
return func(src io.Reader) (int64, error) {
n, err := next(src)
lock.Lock()
defer lock.Unlock()
headerWritten = true
m.Written += n
return n, err
}
},
}
)
fn(Wrap(w, hooks))
m.Duration = time.Since(start)
return m
}
// Package httpsnoop provides an easy way to capture http related metrics (i.e.
// response time, bytes written, and http status code) from your application's
// http.Handlers.
//
// Doing this requires non-trivial wrapping of the http.ResponseWriter
// interface, which is also exposed for users interested in a more low-level
// API.
package httpsnoop
//go:generate go run codegen/main.go
module github.com/felixge/httpsnoop
go 1.13
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment