Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
ai3
accountserver
Commits
a4cafc31
Commit
a4cafc31
authored
Apr 01, 2018
by
ale
Browse files
Add vendor dependencies
parent
bcdecbe5
Changes
225
Expand all
Hide whitespace changes
Inline
Side-by-side
vendor/git.autistici.org/ai3/go-common/clientutil/backend.go
0 → 100644
View file @
a4cafc31
package
clientutil
import
(
"crypto/tls"
"fmt"
"net/http"
"net/url"
"sync"
"time"
)
// BackendConfig specifies the configuration to access a service.
//
// Services with multiple backends can be replicated or partitioned,
// depending on a configuration switch, making it a deployment-time
// decision. Clients are expected to compute their own sharding
// function (either by database lookup or other methods), and expose a
// 'shard' parameter on their APIs.
type
BackendConfig
struct
{
URL
string
`yaml:"url"`
Sharded
bool
`yaml:"sharded"`
TLSConfig
*
TLSClientConfig
`yaml:"tls_config"`
}
// Backend is a runtime class that provides http Clients for use with
// a specific service backend. If the service can't be partitioned,
// pass an empty string to the Client method.
type
Backend
interface
{
// URL for the service for a specific shard.
URL
(
string
)
string
// Client that can be used to make a request to the service.
Client
(
string
)
*
http
.
Client
}
// NewBackend returns a new Backend with the given config.
func
NewBackend
(
config
*
BackendConfig
)
(
Backend
,
error
)
{
u
,
err
:=
url
.
Parse
(
config
.
URL
)
if
err
!=
nil
{
return
nil
,
err
}
var
tlsConfig
*
tls
.
Config
if
config
.
TLSConfig
!=
nil
{
tlsConfig
,
err
=
config
.
TLSConfig
.
TLSConfig
()
if
err
!=
nil
{
return
nil
,
err
}
}
if
config
.
Sharded
{
return
&
replicatedClient
{
u
:
u
,
c
:
newHTTPClient
(
u
,
tlsConfig
),
},
nil
}
return
&
shardedClient
{
baseURL
:
u
,
tlsConfig
:
tlsConfig
,
urls
:
make
(
map
[
string
]
*
url
.
URL
),
shards
:
make
(
map
[
string
]
*
http
.
Client
),
},
nil
}
type
replicatedClient
struct
{
c
*
http
.
Client
u
*
url
.
URL
}
func
(
r
*
replicatedClient
)
Client
(
_
string
)
*
http
.
Client
{
return
r
.
c
}
func
(
r
*
replicatedClient
)
URL
(
_
string
)
string
{
return
r
.
u
.
String
()
}
type
shardedClient
struct
{
baseURL
*
url
.
URL
tlsConfig
*
tls
.
Config
mx
sync
.
Mutex
urls
map
[
string
]
*
url
.
URL
shards
map
[
string
]
*
http
.
Client
}
func
(
s
*
shardedClient
)
getShardURL
(
shard
string
)
*
url
.
URL
{
if
shard
==
""
{
return
s
.
baseURL
}
u
,
ok
:=
s
.
urls
[
shard
]
if
!
ok
{
var
tmp
=
*
s
.
baseURL
tmp
.
Host
=
fmt
.
Sprintf
(
"%s.%s"
,
shard
,
tmp
.
Host
)
u
=
&
tmp
s
.
urls
[
shard
]
=
u
}
return
u
}
func
(
s
*
shardedClient
)
URL
(
shard
string
)
string
{
s
.
mx
.
Lock
()
defer
s
.
mx
.
Unlock
()
return
s
.
getShardURL
(
shard
)
.
String
()
}
func
(
s
*
shardedClient
)
Client
(
shard
string
)
*
http
.
Client
{
s
.
mx
.
Lock
()
defer
s
.
mx
.
Unlock
()
client
,
ok
:=
s
.
shards
[
shard
]
if
!
ok
{
u
:=
s
.
getShardURL
(
shard
)
client
=
newHTTPClient
(
u
,
s
.
tlsConfig
)
s
.
shards
[
shard
]
=
client
}
return
client
}
func
newHTTPClient
(
u
*
url
.
URL
,
tlsConfig
*
tls
.
Config
)
*
http
.
Client
{
return
&
http
.
Client
{
Transport
:
NewTransport
([]
string
{
u
.
Host
},
tlsConfig
,
nil
),
Timeout
:
30
*
time
.
Second
,
}
}
vendor/git.autistici.org/ai3/go-common/clientutil/json.go
0 → 100644
View file @
a4cafc31
package
clientutil
import
(
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
)
// DoJSONHTTPRequest makes an HTTP POST request to the specified uri,
// with a JSON-encoded request body. It will attempt to decode the
// response body as JSON.
func
DoJSONHTTPRequest
(
ctx
context
.
Context
,
client
*
http
.
Client
,
uri
string
,
req
,
resp
interface
{})
error
{
data
,
err
:=
json
.
Marshal
(
req
)
if
err
!=
nil
{
return
err
}
httpReq
,
err
:=
http
.
NewRequest
(
"POST"
,
uri
,
bytes
.
NewReader
(
data
))
if
err
!=
nil
{
return
err
}
httpReq
.
Header
.
Set
(
"Content-Type"
,
"application/json"
)
httpReq
=
httpReq
.
WithContext
(
ctx
)
httpResp
,
err
:=
RetryHTTPDo
(
client
,
httpReq
,
NewExponentialBackOff
())
if
err
!=
nil
{
return
err
}
defer
httpResp
.
Body
.
Close
()
if
httpResp
.
StatusCode
!=
200
{
return
fmt
.
Errorf
(
"HTTP status %d"
,
httpResp
.
StatusCode
)
}
if
httpResp
.
Header
.
Get
(
"Content-Type"
)
!=
"application/json"
{
return
errors
.
New
(
"not a JSON response"
)
}
if
resp
==
nil
{
return
nil
}
return
json
.
NewDecoder
(
httpResp
.
Body
)
.
Decode
(
resp
)
}
vendor/git.autistici.org/ai3/go-common/clientutil/retry.go
0 → 100644
View file @
a4cafc31
package
clientutil
import
(
"errors"
"net/http"
"time"
"github.com/cenkalti/backoff"
)
// NewExponentialBackOff creates a backoff.ExponentialBackOff object
// with our own default values.
func
NewExponentialBackOff
()
*
backoff
.
ExponentialBackOff
{
b
:=
backoff
.
NewExponentialBackOff
()
b
.
InitialInterval
=
100
*
time
.
Millisecond
//b.Multiplier = 1.4142
return
b
}
// A temporary (retriable) error is something that has a Temporary method.
type
tempError
interface
{
Temporary
()
bool
}
type
tempErrorWrapper
struct
{
error
}
func
(
t
tempErrorWrapper
)
Temporary
()
bool
{
return
true
}
// TempError makes a temporary (retriable) error out of a normal error.
func
TempError
(
err
error
)
error
{
return
tempErrorWrapper
{
err
}
}
// Retry operation op until it succeeds according to the backoff
// policy b.
//
// Note that this function reverses the error semantics of
// backoff.Operation: all errors are permanent unless explicitly
// marked as temporary (i.e. they have a Temporary() method that
// returns true). This is to better align with the errors returned by
// the net package.
func
Retry
(
op
backoff
.
Operation
,
b
backoff
.
BackOff
)
error
{
innerOp
:=
func
()
error
{
err
:=
op
()
if
err
==
nil
{
return
err
}
if
tmpErr
,
ok
:=
err
.
(
tempError
);
ok
&&
tmpErr
.
Temporary
()
{
return
err
}
return
backoff
.
Permanent
(
err
)
}
return
backoff
.
Retry
(
innerOp
,
b
)
}
var
errHTTPBackOff
=
TempError
(
errors
.
New
(
"temporary http error"
))
func
isStatusTemporary
(
code
int
)
bool
{
switch
code
{
case
http
.
StatusTooManyRequests
,
http
.
StatusBadGateway
,
http
.
StatusServiceUnavailable
,
http
.
StatusGatewayTimeout
:
return
true
default
:
return
false
}
}
// RetryHTTPDo retries an HTTP request until it succeeds, according to
// the backoff policy b. It will retry on temporary network errors and
// upon receiving specific temporary HTTP errors. It will use the
// context associated with the HTTP request object.
func
RetryHTTPDo
(
client
*
http
.
Client
,
req
*
http
.
Request
,
b
backoff
.
BackOff
)
(
*
http
.
Response
,
error
)
{
var
resp
*
http
.
Response
op
:=
func
()
error
{
// Clear up previous response if set.
if
resp
!=
nil
{
resp
.
Body
.
Close
()
}
var
err
error
resp
,
err
=
client
.
Do
(
req
)
if
err
==
nil
&&
isStatusTemporary
(
resp
.
StatusCode
)
{
resp
.
Body
.
Close
()
return
errHTTPBackOff
}
return
err
}
err
:=
Retry
(
op
,
backoff
.
WithContext
(
b
,
req
.
Context
()))
return
resp
,
err
}
vendor/git.autistici.org/ai3/go-common/clientutil/tls.go
0 → 100644
View file @
a4cafc31
package
clientutil
import
(
"crypto/tls"
common
"git.autistici.org/ai3/go-common"
)
// TLSClientConfig defines the TLS parameters for a client connection
// that should use a client X509 certificate for authentication.
type
TLSClientConfig
struct
{
Cert
string
`yaml:"cert"`
Key
string
`yaml:"key"`
CA
string
`yaml:"ca"`
}
// TLSConfig returns a tls.Config object with the current configuration.
func
(
c
*
TLSClientConfig
)
TLSConfig
()
(
*
tls
.
Config
,
error
)
{
cert
,
err
:=
tls
.
LoadX509KeyPair
(
c
.
Cert
,
c
.
Key
)
if
err
!=
nil
{
return
nil
,
err
}
tlsConf
:=
&
tls
.
Config
{
Certificates
:
[]
tls
.
Certificate
{
cert
},
}
if
c
.
CA
!=
""
{
cas
,
err
:=
common
.
LoadCA
(
c
.
CA
)
if
err
!=
nil
{
return
nil
,
err
}
tlsConf
.
RootCAs
=
cas
}
tlsConf
.
BuildNameToCertificate
()
return
tlsConf
,
nil
}
vendor/git.autistici.org/ai3/go-common/clientutil/transport.go
0 → 100644
View file @
a4cafc31
package
clientutil
import
(
"context"
"crypto/tls"
"errors"
"log"
"net"
"net/http"
"sync"
"time"
)
var
errAllBackendsFailed
=
errors
.
New
(
"all backends failed"
)
type
dnsResolver
struct
{}
func
(
r
*
dnsResolver
)
ResolveIPs
(
hosts
[]
string
)
[]
string
{
var
resolved
[]
string
for
_
,
hostport
:=
range
hosts
{
host
,
port
,
err
:=
net
.
SplitHostPort
(
hostport
)
if
err
!=
nil
{
log
.
Printf
(
"error parsing %s: %v"
,
hostport
,
err
)
continue
}
hostIPs
,
err
:=
net
.
LookupIP
(
host
)
if
err
!=
nil
{
log
.
Printf
(
"error resolving %s: %v"
,
host
,
err
)
continue
}
for
_
,
ip
:=
range
hostIPs
{
resolved
=
append
(
resolved
,
net
.
JoinHostPort
(
ip
.
String
(),
port
))
}
}
return
resolved
}
var
defaultResolver
=
&
dnsResolver
{}
type
resolver
interface
{
ResolveIPs
([]
string
)
[]
string
}
// Balancer for HTTP connections. It will round-robin across available
// backends, trying to avoid ones that are erroring out, until one
// succeeds or they all fail.
//
// This object should not be used for load balancing of individual
// HTTP requests: once a new connection is established, requests will
// be sent over it until it errors out. It's meant to provide a
// *reliable* connection to a set of equivalent backends for HA
// purposes.
type
balancer
struct
{
hosts
[]
string
resolver
resolver
stop
chan
bool
// List of currently valid (or untested) backends, and ones
// that errored out at least once.
mx
sync
.
Mutex
addrs
[]
string
ok
map
[
string
]
bool
}
var
backendUpdateInterval
=
60
*
time
.
Second
// Periodically update the list of available backends.
func
(
b
*
balancer
)
updateProc
()
{
tick
:=
time
.
NewTicker
(
backendUpdateInterval
)
for
{
select
{
case
<-
b
.
stop
:
return
case
<-
tick
.
C
:
resolved
:=
b
.
resolver
.
ResolveIPs
(
b
.
hosts
)
if
len
(
resolved
)
>
0
{
b
.
mx
.
Lock
()
b
.
addrs
=
resolved
b
.
mx
.
Unlock
()
}
}
}
}
// Returns a list of all available backends, split into "good ones"
// (no errors seen since last successful connection) and "bad ones".
func
(
b
*
balancer
)
getBackends
()
([]
string
,
[]
string
)
{
b
.
mx
.
Lock
()
defer
b
.
mx
.
Unlock
()
var
good
,
bad
[]
string
for
_
,
addr
:=
range
b
.
addrs
{
if
ok
:=
b
.
ok
[
addr
];
ok
{
good
=
append
(
good
,
addr
)
}
else
{
bad
=
append
(
bad
,
addr
)
}
}
return
good
,
bad
}
func
(
b
*
balancer
)
notify
(
addr
string
,
ok
bool
)
{
b
.
mx
.
Lock
()
b
.
ok
[
addr
]
=
ok
b
.
mx
.
Unlock
()
}
func
netDialContext
(
ctx
context
.
Context
,
network
,
addr
string
)
(
net
.
Conn
,
error
)
{
timeout
:=
30
*
time
.
Second
// Go < 1.9 does not have net.DialContext, reimplement it in
// terms of net.DialTimeout.
if
deadline
,
ok
:=
ctx
.
Deadline
();
ok
{
timeout
=
time
.
Until
(
deadline
)
}
return
net
.
DialTimeout
(
network
,
addr
,
timeout
)
}
func
(
b
*
balancer
)
dial
(
ctx
context
.
Context
,
network
,
addr
string
)
(
net
.
Conn
,
error
)
{
// Start by attempting a connection on 'good' targets.
good
,
bad
:=
b
.
getBackends
()
for
_
,
addr
:=
range
good
{
// Go < 1.9 does not have DialContext, deal with it
conn
,
err
:=
netDialContext
(
ctx
,
network
,
addr
)
if
err
==
nil
{
return
conn
,
nil
}
else
if
err
==
context
.
Canceled
{
// A timeout might be bad, set the error bit
// on the connection.
b
.
notify
(
addr
,
false
)
return
nil
,
err
}
b
.
notify
(
addr
,
false
)
}
for
_
,
addr
:=
range
bad
{
conn
,
err
:=
netDialContext
(
ctx
,
network
,
addr
)
if
err
==
nil
{
b
.
notify
(
addr
,
true
)
return
conn
,
nil
}
else
if
err
==
context
.
Canceled
{
return
nil
,
err
}
}
return
nil
,
errAllBackendsFailed
}
// NewTransport returns a suitably configured http.RoundTripper that
// talks to a specific backend service. It performs discovery of
// available backends via DNS (using A or AAAA record lookups), tries
// to route traffic away from faulty backends.
//
// It will periodically attempt to rediscover new backends.
func
NewTransport
(
backends
[]
string
,
tlsConf
*
tls
.
Config
,
resolver
resolver
)
http
.
RoundTripper
{
if
resolver
==
nil
{
resolver
=
defaultResolver
}
addrs
:=
resolver
.
ResolveIPs
(
backends
)
b
:=
&
balancer
{
hosts
:
backends
,
resolver
:
resolver
,
addrs
:
addrs
,
ok
:
make
(
map
[
string
]
bool
),
}
go
b
.
updateProc
()
return
&
http
.
Transport
{
DialContext
:
b
.
dial
,
TLSClientConfig
:
tlsConf
,
}
}
vendor/git.autistici.org/ai3/go-common/ldap/parse.go
0 → 100644
View file @
a4cafc31
package
ldaputil
import
(
"fmt"
"gopkg.in/ldap.v2"
)
// ParseScope parses a string representation of an LDAP scope into the
// proper enum value.
func
ParseScope
(
s
string
)
(
int
,
error
)
{
switch
s
{
case
"base"
:
return
ldap
.
ScopeBaseObject
,
nil
case
"one"
:
return
ldap
.
ScopeSingleLevel
,
nil
case
"sub"
:
return
ldap
.
ScopeWholeSubtree
,
nil
default
:
return
0
,
fmt
.
Errorf
(
"unknown LDAP scope '%s'"
,
s
)
}
}
vendor/git.autistici.org/ai3/go-common/ldap/pool.go
0 → 100644
View file @
a4cafc31
package
ldaputil
import
(
"context"
"errors"
"net"
"net/url"
"time"
"git.autistici.org/ai3/go-common/clientutil"
"github.com/cenkalti/backoff"
"gopkg.in/ldap.v2"
)
// ConnectionPool provides a goroutine-safe pool of long-lived LDAP
// connections that will reconnect on errors.
type
ConnectionPool
struct
{
network
string
addr
string
bindDN
string
bindPw
string
c
chan
*
ldap
.
Conn
}
var
defaultConnectTimeout
=
5
*
time
.
Second
func
(
p
*
ConnectionPool
)
connect
(
ctx
context
.
Context
)
(
*
ldap
.
Conn
,
error
)
{
// Dial the connection with a timeout, if the context has a
// deadline (as it should). If the context does not have a
// deadline, we set a default timeout.
deadline
,
ok
:=
ctx
.
Deadline
()
if
!
ok
{
deadline
=
time
.
Now
()
.
Add
(
defaultConnectTimeout
)
}
c
,
err
:=
net
.
DialTimeout
(
p
.
network
,
p
.
addr
,
time
.
Until
(
deadline
))
if
err
!=
nil
{
return
nil
,
err
}
conn
:=
ldap
.
NewConn
(
c
,
false
)
conn
.
Start
()
if
p
.
bindDN
!=
""
{
conn
.
SetTimeout
(
time
.
Until
(
deadline
))
if
_
,
err
=
conn
.
SimpleBind
(
ldap
.
NewSimpleBindRequest
(
p
.
bindDN
,
p
.
bindPw
,
nil
));
err
!=
nil
{
conn
.
Close
()
return
nil
,
err
}
}
return
conn
,
err
}
// Get a fresh connection from the pool.
func
(
p
*
ConnectionPool
)
Get
(
ctx
context
.
Context
)
(
*
ldap
.
Conn
,
error
)
{
// Grab a connection from the cache, or create a new one if
// there are no available connections.
select
{
case
conn
:=
<-
p
.
c
:
return
conn
,
nil
default
:
return
p
.
connect
(
ctx
)
}
}
// Release a used connection onto the pool.
func
(
p
*
ConnectionPool
)
Release
(
conn
*
ldap
.
Conn
,
err
error
)
{
// Connections that failed should not be reused.
if
err
!=
nil
{
conn
.
Close
()
return
}
// Return the connection to the cache, or close it if it's
// full.
select
{
case
p
.
c
<-
conn
:
default
:
conn
.
Close
()
}
}
// Close all connections. Not implemented yet.
func
(
p
*
ConnectionPool
)
Close
()
{}
// Parse a LDAP URI into network and address strings suitable for
// ldap.Dial.
func
parseLDAPURI
(
uri
string
)
(
string
,
string
,
error
)
{
u
,
err
:=
url
.
Parse
(
uri
)
if
err
!=
nil
{
return
""
,
""
,
err
}
network
:=
"tcp"
addr
:=
"localhost:389"
switch
u
.
Scheme
{
case
"ldap"
: