Commit ef7d8d83 authored by ale's avatar ale

Update ai3/go-common

parent 3d998e63
Pipeline #3882 passed with stages
in 4 minutes and 26 seconds
ai3/go-common
===
Common code for ai3 services and tools.
A quick overview of the contents:
* [client](clientutil/) and [server](serverutil/) HTTP-based
"RPC" implementation, just JSON POST requests but with retries,
backoff, timeouts, tracing, etc.
* [server implementation of a generic line-based protocol over a UNIX
socket](unix/).
* a [LDAP connection pool](ldap/).
* utilities to [serialize composite data types](ldap/compositetypes/)
used in our LDAP database.
* a [password hashing library](pwhash/) that uses fancy advanced
crypto by default but is also backwards compatible with old
libc crypto.
* utilities to [manage encryption keys](userenckey/), themselves
encrypted with a password and a KDF.
......@@ -98,43 +98,69 @@ func newBalancedBackend(config *BackendConfig, resolver resolver) (*balancedBack
// with a JSON-encoded request body. It will attempt to decode the
// response body as JSON.
func (b *balancedBackend) Call(ctx context.Context, shard, path string, req, resp interface{}) error {
// Serialize the request body.
data, err := json.Marshal(req)
if err != nil {
return err
}
var tg targetGenerator = b.backendTracker
if b.sharded {
if shard == "" {
return fmt.Errorf("call without shard to sharded service %s", b.baseURI.String())
}
tg = newShardedGenerator(shard, b.baseURI.Host, b.resolver)
// Create the target sequence for this call. If there are multiple
// targets, reduce the timeout on each individual call accordingly to
// accomodate eventual failover.
seq, err := b.makeSequence(shard)
if err != nil {
return err
}
innerTimeout := 1 * time.Hour
if deadline, ok := ctx.Deadline(); ok {
innerTimeout = time.Until(deadline) / time.Duration(seq.Len())
}
seq := newSequence(tg)
b.log.Printf("%016x: initialized", seq.ID())
var httpResp *http.Response
err = backoff.Retry(func() error {
// Call the backends in the sequence until one succeeds, with an
// exponential backoff policy controlled by the outer Context.
return backoff.Retry(func() error {
req, rerr := b.newJSONRequest(path, shard, data)
if rerr != nil {
return rerr
}
httpResp, rerr = b.do(ctx, seq, req)
return rerr
innerCtx, cancel := context.WithTimeout(ctx, innerTimeout)
defer cancel()
// When do() returns successfully, we already know that the
// response had an HTTP status of 200.
httpResp, rerr := b.do(innerCtx, seq, req)
if rerr != nil {
return rerr
}
defer httpResp.Body.Close() // nolint
// Decode the response, unless the 'resp' output is nil.
if httpResp.Header.Get("Content-Type") != "application/json" {
return errors.New("not a JSON response")
}
if resp == nil {
return nil
}
return json.NewDecoder(httpResp.Body).Decode(resp)
}, backoff.WithContext(newExponentialBackOff(), ctx))
if err != nil {
return err
}
defer httpResp.Body.Close() // nolint
}
if httpResp.Header.Get("Content-Type") != "application/json" {
return errors.New("not a JSON response")
// Initialize a new target sequence.
func (b *balancedBackend) makeSequence(shard string) (*sequence, error) {
var tg targetGenerator = b.backendTracker
if b.sharded {
if shard == "" {
return nil, fmt.Errorf("call without shard to sharded service %s", b.baseURI.String())
}
tg = newShardedGenerator(shard, b.baseURI.Host, b.resolver)
}
if resp == nil {
return nil
seq := newSequence(tg)
if seq.Len() == 0 {
return nil, errNoTargets
}
return json.NewDecoder(httpResp.Body).Decode(resp)
b.log.Printf("%016x: initialized", seq.ID())
return seq, nil
}
// Return the URI to be used for the request. This is used both in the
......@@ -213,6 +239,8 @@ func newSequence(tg targetGenerator) *sequence {
func (s *sequence) ID() uint64 { return s.id }
func (s *sequence) Len() int { return len(s.targets) }
func (s *sequence) reloadTargets() {
targets := s.tg.getTargets()
if len(targets) > 0 {
......
......@@ -8,6 +8,7 @@ import (
"net/http"
"os"
"path/filepath"
"strconv"
"sync"
openzipkin "github.com/openzipkin/zipkin-go"
......@@ -31,6 +32,7 @@ const globalTracingConfigPath = "/etc/tracing/client.conf"
type tracingConfig struct {
ReportURL string `json:"report_url"`
Sample string `json:"sample"`
}
// Read the global tracing configuration file. Its location is
......@@ -91,6 +93,9 @@ func init() {
}
func initTracing(endpointAddr string) {
if !Enabled {
return
}
initOnce.Do(func() {
localEndpoint, err := openzipkin.NewEndpoint(getServiceName(), endpointAddr)
if err != nil {
......@@ -100,9 +105,23 @@ func initTracing(endpointAddr string) {
reporter := zipkinHTTP.NewReporter(config.ReportURL)
ze := zipkin.NewExporter(reporter, localEndpoint)
trace.RegisterExporter(ze)
trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
var tc trace.Config
switch config.Sample {
case "", "always":
tc.DefaultSampler = trace.AlwaysSample()
case "never":
tc.DefaultSampler = trace.NeverSample()
default:
frac, err := strconv.ParseFloat(config.Sample, 64)
if err != nil {
log.Printf("warning: error in tracing configuration: sample: %v, tracing disabled", err)
return
}
tc.DefaultSampler = trace.ProbabilitySampler(frac)
}
trace.ApplyConfig(tc)
log.Printf("tracing enabled (report_url %s)", config.ReportURL)
......@@ -110,6 +129,11 @@ func initTracing(endpointAddr string) {
})
}
// Init tracing support, if not using WrapHandler.
func Init() {
initTracing("")
}
// WrapTransport optionally wraps a http.RoundTripper with OpenCensus
// tracing functionality, if it is globally enabled.
func WrapTransport(t http.RoundTripper) http.RoundTripper {
......@@ -120,7 +144,7 @@ func WrapTransport(t http.RoundTripper) http.RoundTripper {
}
// WrapHandler wraps a http.Handler with OpenCensus tracing
// functionality, if globally enabled.
// functionality, if globally enabled. Automatically calls Init().
func WrapHandler(h http.Handler, endpointAddr string) http.Handler {
if Enabled {
initTracing(endpointAddr)
......
......@@ -116,6 +116,8 @@ Each service definition is a dictionary with the following attributes:
only for interactive services)
* `enforce_2fa` is a boolean flag that, when true, will disable
non-2FA logins for this service
* `enable_last_login_reporting` is a boolean flag that enables last login
reporting to usermetadb
* `enable_device_tracking` is a boolean flag that enables device
tracking for this service (assuming the client provides device
information)
......
......@@ -10,13 +10,13 @@ import (
// simple persistent cookie to track the same client device across
// multiple session.
type DeviceInfo struct {
ID string
RemoteAddr string
RemoteZone string
UserAgent string
Browser string
OS string
Mobile bool
ID string `json:"id"`
RemoteAddr string `json:"remote_addr"`
RemoteZone string `json:"remote_zone"`
UserAgent string `json:"user_agent"`
Browser string `json:"browser"`
OS string `json:"os"`
Mobile bool `json:"mobile"`
}
func (d *DeviceInfo) encodeToMap(m map[string]string, prefix string) {
......
......@@ -83,6 +83,10 @@ attributes:
client IP address
* `max_inflight_requests`: maximum number of in-flight requests to
allow before server-side throttling kicks in
* `site_name`: sting to be used as site `<title>`.
* `site_logo`: path to an image to be used as logo, placed above the
modal form.
* `site_favicon`: path to a favicon.
## Device tracking
......@@ -102,6 +106,60 @@ associated key will be cleared either on logout, or when the login
session expires.
# SSO Proxy
The *sso-proxy* server adds SSO authentication and access controls to
unauthenticated backends (legacy applications, or apps that do not
support authentication altogether).
It is a straightforward reverse proxy that handles the SSO-related
methods directly and forwards everything else unchanged to the
backend. While it is possible to specify multiple backends for each
endpoint, the load balancing algorithm is extremely unsophisticated:
the proxy will simply pick a random backend on every request, without
any tracking of whether backends are up or not (this is obviously
improvable). Also note that the authenticated identity is **not**
passed along to the backend: since the backends are unauthenticated,
it wouldn't be safe for them to trust this information anyway, unless
they have a way to ensure it comes only from the trusted sso-proxy
(perhaps using TLS or other forms of transport verification). Finally,
*sso-proxy* only handles incoming requests based on their Host
attribute, not the request path. And the only access control rules
currently supported are group-based.
The proxy server has its own configuration file, */etc/sso/proxy.yml*
by default, which has the following attributes:
* `session_auth_key` and `session_enc_key` are secrets to be used for
HTTP-based sessions. For details on their syntax see the description
for `session_secrets` above.
* `sso_server_url` is the URL for the login server
* `sso_public_key_file` should point at a file containing the SSO
public key
* `sso_domain` is the SSO domain
* `backends` is the list of configured endpoints and associated
backends, each entry has the following attributes:
* `host` the HTTP host to serve
* `allowed_groups` is a list of the groups whose users will be
allowed access to the service
* `upstream` is a list of *host:port* addresses for the upstream
backends
* `tls_server_name` allows you to explicitly set the value of the
ServerName TLS extension on the outbound request. This is done do
de-couple the transport layer between proxy and backend from the
details of the actual HTTP request.
* `client_tls` specifies the client TLS configuration. If set, the
upstream request will use HTTPS, otherwise plain HTTP. Known
attributes:
* `cert`: path to the client certificate
* `key`: path to the private key
* `ca`: path to the CA used to validate the server
Given its characteristics, the proxy is currently best suited for
relatively low-volume, administrative applications, rather than for
user-visible services.
# API
The *sso-server* binary serves different types of HTTP traffic:
......
......@@ -26,6 +26,11 @@ a specific device and an account if one is in possession of the
server-side log database (only partially mitigated by the fact that
the cookie is encrypted).
`usermetadb` also stores last-login information for internal infrastructure
maintenance. The idea is to retain the minimal amount of information to perform
tasks such as "disable accounts that have not been active for more than N
years".
# API
The server exports an API over HTTP/HTTPS, all requests should be made
......@@ -39,14 +44,14 @@ and the *analysis* API.
## Log API
`/api/add_log` (*AddLogRequest*)
Stores a new log entry for a user in the database. The request must be
a `LogEntry` object. The method returns an empty response. If the log
entry contains device information, the list of devices for the
specified user is updated with that information.
`/api/get_user_logs` (*GetUserLogsRequest*) -> *GetUserLogsResponse*
Returns recent logs for a specific user.
`/api/get_user_devices` (*GetUserDevicesRequest*) -> *GetUserDevicesResponse*
......@@ -60,6 +65,20 @@ Returns the list of known devices for a user.
Returns information about a device, whether we have seen it before, if
the localization information matches the historical trend, etc.
## Last-login API
`/api/set_last_login` (*SetLastLoginRequest*)
Stores the last login of a user in the database. The request must be a
`LastLoginEntry` object. The method returns an empty response. The service name
must be specified in the last login entry.
`/api/get_last_login` (*GetLastLoginRequest*) -> *GetLastLoginResponse*
Returns the last login of a given user. If the service name is specified it
returns the last login for that specific service, otherwise return last login
for all services.
# Configuration
......
......@@ -15,6 +15,8 @@ type Client interface {
AddLog(context.Context, string, *usermetadb.LogEntry) error
GetUserDevices(context.Context, string, string) ([]*usermetadb.MetaDeviceInfo, error)
GetUserLogs(context.Context, string, string, int, int) ([]*usermetadb.LogEntry, error)
SetLastLogin(context.Context, string, *usermetadb.LastLoginEntry) error
GetLastLogin(context.Context, string, string, string) ([]*usermetadb.LastLoginEntry, error)
}
type udbClient struct {
......@@ -63,3 +65,19 @@ func (c *udbClient) GetUserLogs(ctx context.Context, shard, username string, max
err := c.be.Call(ctx, shard, "/api/get_user_logs", &req, &resp)
return resp.Results, err
}
func (c *udbClient) SetLastLogin(ctx context.Context, shard string, entry *usermetadb.LastLoginEntry) error {
req := usermetadb.SetLastLoginRequest{LastLogin: entry}
return c.be.Call(ctx, shard, "/api/set_last_login", &req, nil)
}
func (c *udbClient) GetLastLogin(ctx context.Context, shard string, username string, service string) ([]*usermetadb.LastLoginEntry, error) {
req := usermetadb.GetLastLoginRequest{
Username: username,
Service: service,
}
var resp usermetadb.GetLastLoginResponse
err := c.be.Call(ctx, shard, "/api/get_last_login", &req, &resp)
return resp.Results, err
}
......@@ -92,3 +92,35 @@ type GetUserLogsRequest struct {
type GetUserLogsResponse struct {
Results []*LogEntry `json:"result"`
}
type LastLoginEntry struct {
Timestamp time.Time `json:"timestamp"`
Username string `json:"username"`
Service string `json:"service"`
}
func (e *LastLoginEntry) Validate() error {
if e.Username == "" {
return errors.New("invalid last login entry: missing username")
}
if e.Service == "" {
return errors.New("invalid last login entry: missing service")
}
return nil
}
type SetLastLoginRequest struct {
LastLogin *LastLoginEntry `json:"last_login"`
}
type SetLastLoginResponse struct{}
type GetLastLoginRequest struct {
Username string `json:"username"`
Service string `json:"service,omitempty"`
}
type GetLastLoginResponse struct {
Results []*LastLoginEntry `json:"result"`
}
......@@ -3,77 +3,77 @@
"ignore": "test",
"package": [
{
"checksumSHA1": "pLvPnUablirQucyALgrso9hLG4E=",
"checksumSHA1": "oUOxU+Tw1/jOzWVP05HuGvVSC/A=",
"path": "git.autistici.org/ai3/go-common",
"revision": "1f95fcdd58ebf63d338f05ceae29d2de811a2d2f",
"revisionTime": "2018-11-18T16:11:30Z"
"revision": "54f0ac4c46184ae44486a31ca2705076abcc5321",
"revisionTime": "2019-06-30T08:30:15Z"
},
{
"checksumSHA1": "1ChQcW9Biu/AgiKjsbJFg/+WhjQ=",
"checksumSHA1": "kJwm6y9JXhybelO2zUl7UbzIdP0=",
"path": "git.autistici.org/ai3/go-common/clientutil",
"revision": "1f95fcdd58ebf63d338f05ceae29d2de811a2d2f",
"revisionTime": "2018-11-18T16:11:30Z"
"revision": "54f0ac4c46184ae44486a31ca2705076abcc5321",
"revisionTime": "2019-06-30T08:30:15Z"
},
{
"checksumSHA1": "d8aQcSXveyjPfFJgfB8NnM+x8dg=",
"path": "git.autistici.org/ai3/go-common/ldap",
"revision": "2934fd63c275d37b0fe60afabb484a251662bd49",
"revisionTime": "2019-02-17T09:01:06Z"
"revision": "54f0ac4c46184ae44486a31ca2705076abcc5321",
"revisionTime": "2019-06-30T08:30:15Z"
},
{
"checksumSHA1": "ETt1H7ZXeT+mOGVuWDvgGBVx98k=",
"path": "git.autistici.org/ai3/go-common/ldap/compositetypes",
"revision": "95125bd587550f8f2ae1e7c412bb0ef94671f014",
"revisionTime": "2019-03-27T08:48:39Z"
"revision": "54f0ac4c46184ae44486a31ca2705076abcc5321",
"revisionTime": "2019-06-30T08:30:15Z"
},
{
"checksumSHA1": "JcfbQfBD7HTqjhFYT1gBjbThSI4=",
"checksumSHA1": "1TsCGuI5907zG+voDXDpZ9I+H8E=",
"path": "git.autistici.org/ai3/go-common/pwhash",
"revision": "b4364e842290fdecd412056674b471af77663757",
"revisionTime": "2019-04-03T06:59:52Z"
"revision": "54f0ac4c46184ae44486a31ca2705076abcc5321",
"revisionTime": "2019-06-30T08:30:15Z"
},
{
"checksumSHA1": "TKGUNmKxj7KH3qhwiCh/6quUnwc=",
"path": "git.autistici.org/ai3/go-common/serverutil",
"revision": "1f95fcdd58ebf63d338f05ceae29d2de811a2d2f",
"revisionTime": "2018-11-18T16:11:30Z"
"revision": "54f0ac4c46184ae44486a31ca2705076abcc5321",
"revisionTime": "2019-06-30T08:30:15Z"
},
{
"checksumSHA1": "WvuSF0pz3rk7bu+5g9lqTqq97Ow=",
"checksumSHA1": "y5pRYZ/NhfEOCFslPEuUZTYXcro=",
"path": "git.autistici.org/ai3/go-common/tracing",
"revision": "1f95fcdd58ebf63d338f05ceae29d2de811a2d2f",
"revisionTime": "2018-11-18T16:11:30Z"
"revision": "54f0ac4c46184ae44486a31ca2705076abcc5321",
"revisionTime": "2019-06-30T08:30:15Z"
},
{
"checksumSHA1": "witSYnNsDhNaoA85UYilt17H+ng=",
"path": "git.autistici.org/ai3/go-common/userenckey",
"revision": "1f95fcdd58ebf63d338f05ceae29d2de811a2d2f",
"revisionTime": "2018-11-18T16:11:30Z"
"revision": "54f0ac4c46184ae44486a31ca2705076abcc5321",
"revisionTime": "2019-06-30T08:30:15Z"
},
{
"checksumSHA1": "6D5Xt9WoGSeTJE3XFw6P2/nKYrQ=",
"checksumSHA1": "yRc5umgrV1NRzXHqjkHSUAXIvpg=",
"origin": "git.autistici.org/id/usermetadb/vendor/git.autistici.org/id/auth",
"path": "git.autistici.org/id/auth",
"revision": "45a125c8573dc76b222ec0f09cb329d4fd4f424d",
"revisionTime": "2018-11-18T17:45:09Z"
"revision": "ca5dcc5613ac6452405d1ce54abb9dc7db73c768",
"revisionTime": "2019-06-30T08:46:28Z"
},
{
"checksumSHA1": "MszadHmYMr3JQMX2gRg7TfsQWVc=",
"checksumSHA1": "4rmvXVqg6IWBF9bnf97k0LIx2OQ=",
"path": "git.autistici.org/id/go-sso",
"revision": "ad4e6235791228c6ffc229174b46fb4cdbe27dc2",
"revisionTime": "2018-11-18T17:45:41Z"
"revision": "a218499d322a355250890a0a26075b4fbb4273a9",
"revisionTime": "2019-06-30T08:48:05Z"
},
{
"checksumSHA1": "NtTOmajRf6xh0mkVSm18L70ncl0=",
"checksumSHA1": "J0QeD9LVccFOejgPKa0td8JD0rY=",
"path": "git.autistici.org/id/usermetadb",
"revision": "b0ce6ca3aba1819445d93d14d107b4651a239397",
"revisionTime": "2018-11-18T17:18:31Z"
"revision": "ca5dcc5613ac6452405d1ce54abb9dc7db73c768",
"revisionTime": "2019-06-30T08:46:28Z"
},
{
"checksumSHA1": "kwosXxbzygo9ZUZSYvD+WiAbM3Q=",
"checksumSHA1": "HnFofi1vg8of8d1uVSRUX7LIAxI=",
"path": "git.autistici.org/id/usermetadb/client",
"revision": "b0ce6ca3aba1819445d93d14d107b4651a239397",
"revisionTime": "2018-11-18T17:18:31Z"
"revision": "ca5dcc5613ac6452405d1ce54abb9dc7db73c768",
"revisionTime": "2019-06-30T08:46:28Z"
},
{
"checksumSHA1": "yReqUM4tQkY+1YEI+L2d0SOzFWs=",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment