Switch to vmime for test VMs

.gitlab-ci.yml

@@ -26,14 +26,16 @@ variables:
       ${APT_PROXY:+-e config.apt_proxy=${APT_PROXY}}
       $CREATE_ENV_VARS $BUILD_DIR
 
-    - bash -c 'sleep $(( $RANDOM % 5 ))'
-    - with-ssh-key ./test-driver init $BUILD_DIR
+    - with-ssh-key ./scripts/floatup.py ${LIBVIRT:+--ssh $LIBVIRT} --inventory $BUILD_DIR/hosts.yml --ram 3072 --image $VM_IMAGE up
+    - with-ssh-key ./test-driver init --no-vagrant $BUILD_DIR
     - with-ssh-key ./test-driver run $BUILD_DIR
   after_script:
-    - with-ssh-key ./test-driver cleanup $BUILD_DIR
+    - with-ssh-key ./scripts/floatup.py ${LIBVIRT:+--ssh $LIBVIRT} down
+    - with-ssh-key ./test-driver cleanup --no-vagrant $BUILD_DIR
   variables:
     CREATE_ENV_VARS: ""
     TEST_DIR: ""
+    VM_IMAGE: "buster"
   tags: [ai3]
   except:
     - schedules
@@ -56,6 +58,7 @@ base_test:
 base_bullseye_test:
   <<: *base_test
   variables:
+    VM_IMAGE: "bullseye"
     CREATE_ENV_VARS: "-e config.float_debian_dist=bullseye -e inventory.group_vars.vagrant.ansible_python_interpreter=/usr/bin/python3"
     TEST_DIR: "test/base.ref"
 
@@ -67,6 +70,7 @@ full_test:
 full_bullseye_test:
   <<: *base_test
   variables:
+    VM_IMAGE: "bullseye"
     CREATE_ENV_VARS: "-e config.float_debian_dist=bullseye -e inventory.group_vars.vagrant.ansible_python_interpreter=/usr/bin/python3"
     TEST_DIR: "test/full.ref"
 

scripts/floatup.py

+#!/usr/bin/env python3
+#
+# Read a hosts.yml float inventory, and manage a VM group derived by it.
+#
+
+import argparse
+import json
+import os
+import re
+import shlex
+import subprocess
+import yaml
+
+
+def parse_inventory(path, host_attrs):
+    with open(path) as fd:
+        inventory = yaml.safe_load(fd)
+
+    hosts = [{'ip': v['ansible_host'], 'name': k}
+             for k, v in inventory['hosts'].items()]
+    for h in hosts:
+        h.update(host_attrs)
+
+    # We know that the network is a /24.
+    net = re.sub(r'[0-9]+$', '.0/24', hosts[0]['ip'])
+    return {
+        'network': net,
+        'hosts': hosts,
+    }
+
+
+def do_request(url, ssh_gw, payload):
+    data = json.dumps(payload)
+    cmd = "curl -s -X POST -H 'Content-Type: application/json' -d %s %s" % (
+        shlex.quote(data), url)
+    if ssh_gw:
+        cmd = "ssh %s %s" % (ssh_gw, shlex.quote(cmd))
+    return json.loads(subprocess.check_output(cmd, shell=True))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--url', metavar='URL', default='http://127.0.0.1:4949',
+        help='URL of the vmine API server')
+    parser.add_argument(
+        '--ssh', metavar='USER@HOST',
+        help='proxy the vmine API request through SSH')
+    parser.add_argument(
+        '--state-file', metavar='FILE', default='.vmine_group',
+        help='state file to store the vmine group ID')
+    parser.add_argument(
+        '--inventory', metavar='FILE', default='hosts.yml',
+        help='float host inventory')
+    parser.add_argument(
+        '--image', metavar='NAME',
+        help='base image to use for the VMs')
+    parser.add_argument(
+        '--ram', type=int,
+        help='memory reservation for the VMs')
+    parser.add_argument(
+        '--ttl', metavar='DURATION', default='1h',
+        help='TTL for the virtual machines')
+    parser.add_argument(
+        'cmd',
+        choices=['up', 'down'])
+    args = parser.parse_args()
+
+    if args.cmd == 'up':
+        host_attrs = {}
+        if args.ram:
+            host_attrs['ram'] = args.ram
+        if args.image:
+            host_attrs['image'] = args.image
+        req = parse_inventory(args.inventory, host_attrs)
+        req['ttl'] = args.ttl
+
+        resp = do_request(args.url + '/api/create-group', args.ssh, req)
+        with open(args.state_file) as fd:
+            fd.write(resp['group_id'])
+
+    elif args.cmd == 'down':
+        with open(args.state_file) as fd:
+            group_id = fd.read().strip()
+        do_request(args.url + '/api/stop-group', args.ssh,
+                   {'group_id': group_id})
+        os.remove(args.state_file)
+
+
+if __name__ == '__main__':
+    main()

test-driver

@@ -6,6 +6,7 @@ bin_dir=$(dirname "$0")
 bin_dir=${bin_dir:-.}
 bin_dir=$(cd "${bin_dir}" && pwd)
 float_dir="${bin_dir}"
+use_vagrant=1
 
 log() {
     echo " ***" >&2
@@ -57,8 +58,10 @@ save_logs() {
 }
 
 run_init() {
+    if [ $use_vagrant -eq 1 ]; then
         start_vagrant \
             || die "could not start VMs"
+    fi
 
     wait_for_vms \
         || die "could not reach the VMs with Ansible"
@@ -73,7 +76,9 @@ run_init() {
 }
 
 run_cleanup() {
+    if [ $use_vagrant -eq 1 ]; then
         stop_vagrant
+    fi
 }
 
 usage() {
@@ -91,6 +96,11 @@ Commands:
 
 If DIR is specified, chdir there before running anything.
 
+The 'init' and 'cleanup' commands understand a --no-vagrant option
+(which must immediately follow the command) in which case Vagrant
+will not be invoked at all, and the virtual machines will be expected
+to be already up and running.
+
 EOF
     exit 2
 }
@@ -98,6 +108,10 @@ EOF
 cmd="$1"
 shift
 
+if [ "x$1" = "x--no-vagrant" ]; then
+    use_vagrant=0 ; shift
+fi
+
 if [ $# -gt 0 ]; then
     cd "$1"
     shift