Load external source specs in 'iprep server'

cmd/iprep/server.go

@@ -3,12 +3,15 @@ package main
 import (
 	"context"
 	"flag"
+	"fmt"
+	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"syscall"
 	"time"
 
@@ -18,18 +21,21 @@ import (
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+	"gopkg.in/yaml.v2"
 
+	"git.autistici.org/ai3/tools/iprep/ext"
 	ippb "git.autistici.org/ai3/tools/iprep/proto"
 	"git.autistici.org/ai3/tools/iprep/server"
 )
 
 type serverCommand struct {
-	rpcAddr    string
-	httpAddr   string
-	dbURI      string
-	scriptPath string
-	tlsCert    string
-	tlsKey     string
+	rpcAddr        string
+	httpAddr       string
+	dbURI          string
+	scriptPath     string
+	externalSrcDir string
+	tlsCert        string
+	tlsKey         string
 }
 
 func (c *serverCommand) Name() string     { return "server" }
@@ -45,8 +51,9 @@ func (c *serverCommand) Usage() string {
 func (c *serverCommand) SetFlags(f *flag.FlagSet) {
 	f.StringVar(&c.rpcAddr, "rpc-addr", ":7170", "`address` of GRPC listener")
 	f.StringVar(&c.httpAddr, "http-addr", ":7180", "`address` of HTTP debug listener")
-	f.StringVar(&c.dbURI, "db", "/var/lib/iprep/data", "database `uri` (sqlite:// or leveldb://)")
+	f.StringVar(&c.dbURI, "db", "leveldb:///var/lib/iprep/data", "database `uri` (sqlite:// or leveldb://)")
 	f.StringVar(&c.scriptPath, "scoring-script", "/etc/iprep/score.td", "`path` to a custom scoring script")
+	f.StringVar(&c.externalSrcDir, "ext-sources-dir", "/etc/iprep/external", "`path` to a directory containing external source definitions")
 
 	f.StringVar(&c.tlsCert, "tls-cert", "", "TLS certificate `path` (grpc only)")
 	f.StringVar(&c.tlsKey, "tls-key", "", "TLS private key `path` (grpc only)")
@@ -66,8 +73,52 @@ func (c *serverCommand) Execute(ctx context.Context, f *flag.FlagSet, args ...in
 	return subcommands.ExitSuccess
 }
 
+// Definition of an ExternalSource, represented as YAML.
+type externalSourceSpec struct {
+	Name   string                 `yaml:"name"`
+	Type   string                 `yaml:"type"`
+	Params map[string]interface{} `yaml:"params"`
+}
+
+// Read all files in externalSrcDir and build a map of ExternalSources.
+func (c *serverCommand) loadExternalSources() (map[string]ext.ExternalSource, error) {
+	files, err := filepath.Glob(filepath.Join(c.externalSrcDir, "*.yml"))
+	if err != nil {
+		return nil, err
+	}
+
+	m := make(map[string]ext.ExternalSource)
+	for _, f := range files {
+		data, err := ioutil.ReadFile(f)
+		if err != nil {
+			return nil, err
+		}
+		var spec externalSourceSpec
+		if err := yaml.Unmarshal(data, &spec); err != nil {
+			return nil, err
+		}
+		if spec.Name == "" {
+			return nil, fmt.Errorf("error in external source spec %s: missing name", f)
+		}
+		if _, ok := m[spec.Name]; ok {
+			return nil, fmt.Errorf("duplicate external source '%s'", spec.Name)
+		}
+		src, err := ext.New(spec.Type, spec.Params)
+		if err != nil {
+			return nil, fmt.Errorf("error creating external source '%s': %v", spec.Name, err)
+		}
+		m[spec.Name] = src
+	}
+
+	return m, nil
+}
+
 func (c *serverCommand) run(ctx context.Context) error {
-	srv, err := server.New(c.dbURI, c.scriptPath)
+	srcs, err := c.loadExternalSources()
+	if err != nil {
+		return err
+	}
+	srv, err := server.New(c.dbURI, c.scriptPath, srcs)
 	if err != nil {
 		return err
 	}

db/db.go

 package db
 
 import (
+	"errors"
 	"fmt"
 	"net/url"
 	"time"
@@ -23,6 +24,9 @@ func Open(path string) (DB, error) {
 	if err != nil {
 		return nil, err
 	}
+	if u.Path == "" {
+		return nil, errors.New("empty path in DB URI")
+	}
 
 	switch u.Scheme {
 	case "", "leveldb":