Commit 438e07d2 authored by ale's avatar ale

Add support for sharding to client API

Using the clientutil.Backend abstraction.
parent 899573db
Pipeline #720 passed with stages
in 1 minute and 27 seconds
......@@ -2,10 +2,6 @@ package client
import (
"context"
"crypto/tls"
"net/http"
"net/url"
"time"
"git.autistici.org/ai3/go-common/clientutil"
"git.autistici.org/id/auth"
......@@ -14,73 +10,56 @@ import (
)
// Client for the user-meta-server API.
type Client struct {
*http.Client
backendURL string
type Client interface {
CheckDevice(context.Context, string, string, *auth.DeviceInfo) (bool, error)
AddLog(context.Context, string, *usermetadb.LogEntry) error
GetUserDevices(context.Context, string, string) ([]*usermetadb.MetaDeviceInfo, error)
GetUserLogs(context.Context, string, string, int, int) ([]*usermetadb.LogEntry, error)
}
// Config for a user-meta-server API client.
type Config struct {
BackendURL string `yaml:"backend_url"`
TLSConfig *clientutil.TLSClientConfig `yaml:"tls_config"`
type udbClient struct {
be clientutil.Backend
}
// New returns a new Client with the given Config.
func New(config *Config) (*Client, error) {
u, err := url.Parse(config.BackendURL)
func New(config *clientutil.BackendConfig) (Client, error) {
be, err := clientutil.NewBackend(config)
if err != nil {
return nil, err
}
var tlsConfig *tls.Config
if config.TLSConfig != nil {
tlsConfig, err = config.TLSConfig.TLSConfig()
if err != nil {
return nil, err
}
}
c := &http.Client{
Transport: clientutil.NewTransport([]string{u.Host}, tlsConfig, nil),
Timeout: 10 * time.Second,
}
return &Client{
Client: c,
backendURL: config.BackendURL,
}, nil
return &udbClient{be}, nil
}
func (c *Client) CheckDevice(ctx context.Context, username string, dev *auth.DeviceInfo) (bool, error) {
func (c *udbClient) CheckDevice(ctx context.Context, shard, username string, dev *auth.DeviceInfo) (bool, error) {
req := usermetadb.CheckDeviceRequest{
Username: username,
DeviceInfo: dev,
}
var resp usermetadb.CheckDeviceResponse
err := clientutil.DoJSONHTTPRequest(ctx, c.Client, c.backendURL+"/api/check_device", &req, &resp)
err := clientutil.DoJSONHTTPRequest(ctx, c.be.Client(shard), c.be.URL(shard)+"/api/check_device", &req, &resp)
return resp.Seen, err
}
func (c *Client) AddLog(ctx context.Context, entry *usermetadb.LogEntry) error {
func (c *udbClient) AddLog(ctx context.Context, shard string, entry *usermetadb.LogEntry) error {
req := usermetadb.AddLogRequest{Log: entry}
return clientutil.DoJSONHTTPRequest(ctx, c.Client, c.backendURL+"/api/add_log", &req, nil)
return clientutil.DoJSONHTTPRequest(ctx, c.be.Client(shard), c.be.URL(shard)+"/api/add_log", &req, nil)
}
func (c *Client) GetUserDevices(ctx context.Context, username string) ([]*usermetadb.MetaDeviceInfo, error) {
func (c *udbClient) GetUserDevices(ctx context.Context, shard, username string) ([]*usermetadb.MetaDeviceInfo, error) {
req := usermetadb.GetUserDevicesRequest{Username: username}
var resp usermetadb.GetUserDevicesResponse
err := clientutil.DoJSONHTTPRequest(ctx, c.Client, c.backendURL+"/api/get_user_devices", &req, &resp)
err := clientutil.DoJSONHTTPRequest(ctx, c.be.Client(shard), c.be.URL(shard)+"/api/get_user_devices", &req, &resp)
return resp.Devices, err
}
func (c *Client) GetUserLogs(ctx context.Context, username string, maxDays, limit int) ([]*usermetadb.LogEntry, error) {
func (c *udbClient) GetUserLogs(ctx context.Context, shard, username string, maxDays, limit int) ([]*usermetadb.LogEntry, error) {
req := usermetadb.GetUserLogsRequest{
Username: username,
MaxDays: maxDays,
Limit: limit,
}
var resp usermetadb.GetUserLogsResponse
err := clientutil.DoJSONHTTPRequest(ctx, c.Client, c.backendURL+"/api/get_user_logs", &req, &resp)
err := clientutil.DoJSONHTTPRequest(ctx, c.be.Client(shard), c.be.URL(shard)+"/api/get_user_logs", &req, &resp)
return resp.Results, err
}
......@@ -44,6 +44,7 @@ func main() {
}
defer db.Close()
log.Printf("starting user-meta-server on %s", *addr)
if err := serverutil.Serve(db.Handler(), config.ServerConfig, *addr); err != nil {
log.Fatal(err)
}
......
......@@ -6,6 +6,7 @@ import (
"os"
"testing"
"git.autistici.org/ai3/go-common/clientutil"
"git.autistici.org/id/usermetadb/client"
)
......@@ -19,10 +20,10 @@ func TestServer_AddLog(t *testing.T) {
httpSrv := httptest.NewServer(srv.Handler())
defer httpSrv.Close()
c, _ := client.New(&client.Config{BackendURL: httpSrv.URL})
c, _ := client.New(&clientutil.BackendConfig{URL: httpSrv.URL})
entries := generateTestLogs(100, generateAllRandomDevices())
for _, e := range entries {
if err := c.AddLog(context.Background(), e); err != nil {
if err := c.AddLog(context.Background(), "", e); err != nil {
t.Fatalf("AddLog(%+v): %v", e, err)
}
}
......@@ -40,10 +41,10 @@ func BenchmarkServer_AddLog(b *testing.B) {
b.ResetTimer()
c, _ := client.New(&client.Config{BackendURL: httpSrv.URL})
c, _ := client.New(&clientutil.BackendConfig{URL: httpSrv.URL})
entries := generateTestLogs(b.N, generateAllRandomDevices())
for _, e := range entries {
if err := c.AddLog(context.Background(), e); err != nil {
if err := c.AddLog(context.Background(), "", e); err != nil {
b.Fatalf("AddLog(%+v): %v", e, err)
}
}
......
package clientutil
import (
"crypto/tls"
"fmt"
"net/http"
"net/url"
"sync"
"time"
)
// BackendConfig specifies the configuration to access a service.
//
// Services with multiple backends can be replicated or partitioned,
// depending on a configuration switch, making it a deployment-time
// decision. Clients are expected to compute their own sharding
// function (either by database lookup or other methods), and expose a
// 'shard' parameter on their APIs.
type BackendConfig struct {
URL string `yaml:"url"`
Sharded bool `yaml:"sharded"`
TLSConfig *TLSClientConfig `yaml:"tls_config"`
}
// Backend is a runtime class that provides http Clients for use with
// a specific service backend. If the service can't be partitioned,
// pass an empty string to the Client method.
type Backend interface {
// URL for the service for a specific shard.
URL(string) string
// Client that can be used to make a request to the service.
Client(string) *http.Client
}
// NewBackend returns a new Backend with the given config.
func NewBackend(config *BackendConfig) (Backend, error) {
u, err := url.Parse(config.URL)
if err != nil {
return nil, err
}
var tlsConfig *tls.Config
if config.TLSConfig != nil {
tlsConfig, err = config.TLSConfig.TLSConfig()
if err != nil {
return nil, err
}
}
if config.Sharded {
return &replicatedClient{
u: u,
c: newHTTPClient(u, tlsConfig),
}, nil
}
return &shardedClient{
baseURL: u,
tlsConfig: tlsConfig,
urls: make(map[string]*url.URL),
shards: make(map[string]*http.Client),
}, nil
}
type replicatedClient struct {
c *http.Client
u *url.URL
}
func (r *replicatedClient) Client(_ string) *http.Client { return r.c }
func (r *replicatedClient) URL(_ string) string { return r.u.String() }
type shardedClient struct {
baseURL *url.URL
tlsConfig *tls.Config
mx sync.Mutex
urls map[string]*url.URL
shards map[string]*http.Client
}
func (s *shardedClient) getShardURL(shard string) *url.URL {
if shard == "" {
return s.baseURL
}
u, ok := s.urls[shard]
if !ok {
var tmp = *s.baseURL
tmp.Host = fmt.Sprintf("%s.%s", shard, tmp.Host)
u = &tmp
s.urls[shard] = u
}
return u
}
func (s *shardedClient) URL(shard string) string {
s.mx.Lock()
defer s.mx.Unlock()
return s.getShardURL(shard).String()
}
func (s *shardedClient) Client(shard string) *http.Client {
s.mx.Lock()
defer s.mx.Unlock()
client, ok := s.shards[shard]
if !ok {
u := s.getShardURL(shard)
client = newHTTPClient(u, s.tlsConfig)
s.shards[shard] = client
}
return client
}
func newHTTPClient(u *url.URL, tlsConfig *tls.Config) *http.Client {
return &http.Client{
Transport: NewTransport([]string{u.Host}, tlsConfig, nil),
Timeout: 30 * time.Second,
}
}
......@@ -9,10 +9,10 @@
"revisionTime": "2017-12-14T08:46:15Z"
},
{
"checksumSHA1": "2X2UMundICtpGTb8pTdBk7PCKss=",
"checksumSHA1": "o+rWKVQIDy79ZwrItwa5/whAL6g=",
"path": "git.autistici.org/ai3/go-common/clientutil",
"revision": "0cc062297e2c27f9a1abcb1a00172d1e0281f8cb",
"revisionTime": "2017-12-14T08:46:15Z"
"revision": "9b20acad90c411c48f7ddc837a35ef3d0d6f98d4",
"revisionTime": "2017-12-17T20:32:41Z"
},
{
"checksumSHA1": "wY0SM35qAhX3P2IZzDnYa068cPw=",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment