float 10.5 KB
Newer Older
1 2 3 4 5 6
#!/usr/bin/env python

from __future__ import print_function

import argparse
import os
7 8
import random
import re
9 10
import subprocess
import sys
11
import yaml
12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51


# Find the root ai3/float source directory.
srcdir = os.path.dirname(__file__)


def support_ansible_vault_gpg_passphrase():
    """Support GPG-encrypted Ansible Vault passphrases.

    We do this by rewriting the ANSIBLE_VAULT_PASSWORD_FILE
    environment variable and pointing it at a wrapper script.

    The function does not return anything but as a side effect it will
    modify os.environ (so that the subprocess modulee will pick up the
    changes).

    """
    pwfile = os.getenv('ANSIBLE_VAULT_PASSWORD_FILE')
    if not pwfile:
        raise Exception(
            'The environment variable ANSIBLE_VAULT_PASSWORD_FILE is not '
            'defined. You should point it at the file containing your '
            'Ansible Vault passphrase (possibly encrypted with GPG).')
    if not os.path.exists(pwfile):
        raise Exception(
            'The ANSIBLE_VAULT_PASSWORD_FILE (%s) does not exist.' % (
                pwfile,))

    # Be friendly to the user and resolve tilde (~ and ~user) paths,
    # which the shell would not do otherwise in environment variables.
    # Then ensure it is an absolute path.
    pwfile = os.path.abspath(os.path.expanduser(pwfile))

    # If the file has a .gpg extension, wrap it with our decoding
    # script: Ansible will execute the script due to its +x
    # permissions, which in turn will use gpg to decrypt the
    # passphrase (using another environment variable to find the
    # original file).
    if pwfile.endswith('.gpg'):
        os.environ['FLOAT_VAULT_PASSWORD_FILE'] = pwfile
52 53
        pwfile = os.path.abspath(
            os.path.join(srcdir, 'scripts', 'get-vault-password'))
54 55 56 57

    os.environ['ANSIBLE_VAULT_PASSWORD_FILE'] = pwfile


58
def command_run(config, playbooks,
59
                ansible_verbosity=0,
60
                ansible_check=False,
61
                ansible_diff=False,
62 63
                ansible_stdout=None,
                ansible_extra_vars=[]):
64
    if not os.path.exists(config):
65
        raise Exception(
66
            'The configuration file %s does not exist!' % (config,))
67

68 69 70
    if ansible_stdout:
        os.environ['ANSIBLE_STDOUT_CALLBACK'] = ansible_stdout

71 72 73 74 75 76 77 78 79 80 81 82 83
    for arg in playbooks:
        if not os.path.exists(arg):
            # See if we have a stock playbook with that name.
            if not arg.endswith('.yml'):
                arg += '.yml'
            pbk = os.path.join(srcdir, 'playbooks', arg)
            if os.path.exists(pbk):
                arg = pbk

        print('Running playbook %s...' % (arg,))

        os.environ['LC_ALL'] = 'C'
        support_ansible_vault_gpg_passphrase()
84 85
        cmd = [os.getenv('ANSIBLE_PLAYBOOK', 'ansible-playbook'),
               '-i', config]
86 87
        if ansible_verbosity > 0:
            cmd.append('-' + ('v' * ansible_verbosity))
88 89 90 91
        if ansible_check:
            cmd.append('--check')
        if ansible_diff:
            cmd.append('--diff')
92 93
        for v in ansible_extra_vars:
            cmd.append('--extra-vars=' + v)
94 95
        cmd.append(arg)
        subprocess.check_call(cmd)
96 97 98 99 100 101


def command_init_credentials(config):
    command_run(config, ['init-credentials.yml'])


102 103 104 105 106
def _parse_network(s):
    # Given a network in the form a.b.c.0, return the first three octects.
    m = re.match(r'(\d+\.\d+\.\d+)\.0$', s)
    if not m:
        raise Exception('Syntax error: %s does not match format a.b.c.0' % s)
107
    return m.group(1)
108 109


110
def command_create_env(path, domain, vagrant, mitogen, num_hosts, net):
111 112 113
    if os.path.exists(path):
        raise Exception('Target %s already exists' % path)

114 115 116 117 118 119 120 121
    if net:
        net = _parse_network(net)
    else:
        # Avoid low-numbered 10.x networks that may be used by Vagrant.
        net = '10.%d.%d' % (
            random.randint(10, 254),
            random.randint(0, 254))

122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
    dirs = [
        'credentials',
        'group_vars/all',
        'playbooks',
    ]

    skel = {
        'ansible.cfg': '''
[defaults]
roles_path = %(srcdir)s/roles:roles
inventory_plugins = %(srcdir)s/plugins/inventory
action_plugins = %(srcdir)s/plugins/action
vars_plugins = %(srcdir)s/plugins/vars
display_skipped_hosts = False
nocows = 1
ale's avatar
ale committed
137
force_handlers = True
138 139 140 141 142 143
callback_whitelist = profile_tasks

[inventory]
enable_plugins = float

[ssh_connection]
144
ssh_args = -C -o ControlMaster=auto -o ControlPersist=120s
145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
control_path_dir = ~/.ansible/cp
control_path = %%(directory)s/%%%%h-%%%%r
pipelining = True
scp_if_ssh = True
''',
        'config.yml': '''---
plugin: float
services_file: services.yml
hosts_file: hosts.yml
passwords_file: passwords.yml
credentials_dir: credentials
vars_dir: group_vars/all
''',
        'services.yml': '''---
include:
  - %(srcdir)s/services.yml.default
''',
        'passwords.yml': '''---
- include: %(srcdir)s/passwords.yml.default
''',
        'site.yml': '''---
- import_playbook: %(srcdir)s/playbooks/all.yml
''',
        'group_vars/all/config.yml': '''---
domain: infra.%(domain)s
domain_public:
  - %(domain)s
net_overlays:
173
  - name: vpn0
ale's avatar
ale committed
174
    network: 172.16.1.0/24
175
enable_ssh: false
ale's avatar
ale committed
176
enable_osquery: false
177 178 179 180 181 182
admins:
  - name: admin
    email: "admin@%(domain)s"
    password: "$s$16384$8$1$c479e8eb722f1b071efea7826ccf9c20$96d63ebed0c64afb746026f56f71b2a1f8796c73141d2d6b1958d4ea26c60a0b"
''',
    }
183 184 185 186 187 188 189 190 191

    if mitogen:
        skel['ansible.cfg'] = skel['ansible.cfg'].replace(
            '[defaults]\n',
            '''[defaults]
strategy_plugins = %s/ansible_mitogen/plugins/strategy
strategy = mitogen_linear
''' % (mitogen,))

192 193 194 195 196 197
    skel_vagrant = {
        'Vagrantfile': '''
NUM_HOSTS = %(num_hosts)d

Vagrant.configure(2) do |config|
  config.vm.box = "debian/stretch64"
198 199

  # Use the old insecure Vagrant SSH key for access.
200
  config.ssh.insert_key = false
201 202

  # Disable synchronization of the /vagrant folder for faster startup.
203 204
  config.vm.synced_folder ".", "/vagrant", disabled: true

205 206
  # ELK can't run with the default 512M, so increase RAM to 1G.
  config.vm.provider :virtualbox do |vb|
207
    vb.customize ["modifyvm", :id, "--memory", "2048"]
208 209 210
  end

  # Create progressively numbered hosts 'hostN', with IP 9+N.
211 212 213
  (1..NUM_HOSTS).each do |i|
    config.vm.define "host#{i}" do |m|
      m.vm.hostname = "host#{i}"
214
      m.vm.network "private_network", ip: "%(net)s.#{9+i}"
215 216 217 218 219 220 221
    end
  end
end
''',
        'hosts.yml': yaml.dump({
            'hosts': dict(
                ('host%d' % (i+1), {
222 223
                    'ansible_host': '%s.%d' % (net, i+10),
                    'ip': '%s.%d' % (net, i+10),
224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
                    'ip_vpn0': '172.16.1.%d' % (i+1),
                    'shard_id': 'host%d' % (i+1),
                    'groups': ['vagrant', 'frontend' if i == 0 else 'backend'],
                }) for i in range(num_hosts)),
            'group_vars': {
                'vagrant': {
                    'ansible_user': 'vagrant',
                    'ansible_become': True,
                    'ansible_ssh_private_key_file': '~/.vagrant.d/insecure_private_key',
                },
            },
        }),
    }

    v = {
        'srcdir': os.path.relpath(srcdir, path),
        'domain': domain,
        'num_hosts': num_hosts,
242
        'net': net,
243 244 245 246 247 248 249 250 251 252 253 254 255 256
    }
    
    for d in dirs:
        print('creating directory %s' % os.path.join(path, d))
        os.makedirs(os.path.join(path, d))
    filedata = skel
    if vagrant:
        filedata.update(skel_vagrant)
    for name in filedata:
        print('creating file %s' % os.path.join(path, name))
        with open(os.path.join(path, name), 'w') as fd:
            fd.write(filedata[name] % v)


257 258 259 260 261 262 263 264 265
def main():
    parser = argparse.ArgumentParser(
        description='Container-based cluster management CLI.')

    subparsers = parser.add_subparsers(dest='subparser')

    help_parser = subparsers.add_parser(
        'help', help='print help')

266 267 268 269 270 271 272 273 274 275
    create_env_parser = subparsers.add_parser(
        'create-env',
        help='initialize a new environment',
        description='Initialize a new (test) environment.')
    create_env_parser.add_argument(
        'path', help='output directory')
    create_env_parser.add_argument(
        '--vagrant', action='store_true',
        help='set up a Vagrantfile')
    create_env_parser.add_argument(
276
        '--num-hosts', metavar='N', type=int, default=3,
ale's avatar
ale committed
277
        help='number of VMs to create when using --vagrant (default: 3)')
278 279 280 281
    create_env_parser.add_argument(
        '--mitogen', metavar='PATH',
        help='directory with the Mitogen source repository'
        ' (enables Mitogen in the generated ansible.cfg)')
282 283
    create_env_parser.add_argument(
        '--domain', default='example.com',
ale's avatar
ale committed
284
        help='public domain to use (default: example.com)')
285 286
    create_env_parser.add_argument(
        '--net',
ale's avatar
ale committed
287
        help='Vagrant private network (default: randomly selected)')
288

289 290 291 292 293 294 295
    run_parser = subparsers.add_parser(
        'run',
        help='run Ansible playbooks',
        description='Run Ansible playbooks.')
    run_parser.add_argument(
        'playbooks', metavar='playbook', nargs='*',
        default=['site.yml'], help='Playbooks to run')
296 297 298
    run_parser.add_argument(
        '--config', metavar='file', default='config.yml',
        help='Path to the configuration file')
299
    run_parser.add_argument(
300 301
        '-v', '--verbose', dest='ansible_verbosity', action='count',
        help='Increase ansible-playbook verbosity')
302 303 304
    run_parser.add_argument(
        '-C', '--check', dest='ansible_check', action='store_true',
        help='Run ansible-playbook with the --check flag')
305 306 307 308
    run_parser.add_argument(
        '-O', '--stdout', dest='ansible_stdout', type=str,
        help='Select ansible stdout callback (e.g. minimal, actionable)',
        default=None)
309 310 311
    run_parser.add_argument(
        '-D', '--diff', dest='ansible_diff', action='store_true',
        help='Run ansible-playbook with the --diff flag')
312 313 314 315
    run_parser.add_argument(
        '-e', '--extra-vars', dest='ansible_extra_vars',
        action='append', default=[],
        help='Extra variables for Ansible')
316 317 318 319 320

    init_credentials_parser = subparsers.add_parser(
        'init-credentials',
        help='initialize credentials',
        description='Initialize credentials.')
321 322 323
    init_credentials_parser.add_argument(
        '--config', metavar='file', default='config.yml',
        help='Path to the configuration file')
324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341

    kwargs = vars(parser.parse_args())
    cmd = kwargs.pop('subparser')

    if cmd == 'help' or not cmd:
        parser.print_help()
        return

    handler = 'command_' + cmd.replace('-', '_')
    globals()[handler](**kwargs)


if __name__ == '__main__':
    try:
        main()
    except Exception as e:
        print("ERROR: %s" % (str(e),), file=sys.stderr)
        sys.exit(1)