add support for cloud-init API configuation

This change makes the MGT API service fully configurable to either IPv4
or IPv6 address.

Implements blueprint: cloud-init-provisioning
Change-Id: Ibff39030c4e3fe04c3f8cc238508e33d450a4398

akanda/router/commands/management.py

@@ -1,6 +1,5 @@
 # Copyright 2014 DreamHost, LLC
-#
-# Author: DreamHost, LLC
+# Copyright 2015 Akanda, Inc
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -15,31 +14,31 @@
 # under the License.
 
 
+import argparse
 import re
 import sys
-import textwrap
+
+import netaddr
 
 from akanda.router import defaults
+from akanda.router import utils
 from akanda.router.drivers import ip
 
 
-def configure_ssh():
+def configure_ssh(listen_ip):
     """
     """
-    mgr = ip.IPManager()
-
-    listen_ip = mgr.get_management_address(ensure_configuration=True)
-
-    if not listen_ip:
-        sys.stderr.write('Unable to bring up first interface (ge0)!\n')
-        sys.exit(1)
-
     config = open('/etc/ssh/sshd_config', 'r').read()
-    config = re.sub('(^|\n)(#)?(ListenAddress|AddressFamily) .*', '', config)
+    config = re.sub(
+        '(^|\n)(#)?(ListenAddress|AddressFamily|UseDNS) .*',
+        '',
+        config
+    )
+
     config += '\n'.join([
         '',  # make sure we have a blank line at the end before adding more
-        'AddressFamily inet6',
-        'ListenAddress ' + listen_ip,
+        'AddressFamily inet%s' % ('6' if listen_ip.version == 6 else ''),
+        'ListenAddress ' + str(listen_ip),
         'UseDNS no'
     ])
     try:
@@ -49,36 +48,52 @@ def configure_ssh():
         sys.stderr.write('Unable to write sshd configuration file.')
 
 
-def configure_gunicorn():
+def configure_gunicorn(listen_ip):
     """
     """
-    mgr = ip.IPManager()
+    if listen_ip.version == 6:
+        bind = "'[%s]:%d'" % (listen_ip, defaults.API_SERVICE)
+    else:
+        bind = "'%s:%d'" % (listen_ip, defaults.API_SERVICE)
 
-    listen_ip = mgr.get_management_address(ensure_configuration=True)
-
-    if not listen_ip:
-        sys.stderr.write('Unable to bring up first interface (ge0)!\n')
-        sys.exit(1)
-
-    args = {'host': listen_ip,
-            'port': defaults.API_SERVICE}
-
-    config = """
-    import multiprocessing
-
-    bind = '[%(host)s]:%(port)d'
-    workers = workers = multiprocessing.cpu_count() * 2 + 1
-    backlog = 2048
-    worker_class ="sync"
-    debug = False
-    daemon = True
-    pidfile = "/var/run/gunicorn.pid"
-    logfile = "/tmp/gunicorn.log"
-    """
-    config = textwrap.dedent(config % args).lstrip()
+    config = open('/etc/akanda_gunicorn_config', 'r').read()
+    config = re.sub('\nbind(\s)?\=(\s)?.*', '\nbind = %s' % bind, config)
 
     try:
         open('/etc/akanda_gunicorn_config', 'w+').write(config)
         sys.stderr.write('http configured to listen on %s\n' % listen_ip)
     except:
         sys.stderr.write('Unable to write gunicorn configuration file.')
+
+
+def configure_management():
+    parser = argparse.ArgumentParser(
+        description='Configure Management Interface'
+    )
+    parser.add_argument('mac_address', metavar='lladdr', type=str)
+    parser.add_argument('ip_address', metavar='ipaddr', type=str)
+    args = parser.parse_args()
+
+    ip_addr = netaddr.IPNetwork(args.ip_address)
+
+    mgr = ip.IPManager()
+
+    for intf in mgr.get_interfaces():
+        if args.mac_address == intf.lladdr:
+            if not intf.is_up:
+                mgr.up(intf)
+
+            if ip_addr not in intf.addresses:
+                if ip_addr.version == 6:
+                    real_ifname = mgr.generic_to_host(intf.ifname)
+                    utils.execute([
+                        'sysctl',
+                        '-w',
+                        'net.ipv6.conf.%s.accept_dad=0' % real_ifname
+                    ])
+
+                intf.addresses.append(ip_addr)
+                mgr.update_interface(intf)
+                configure_ssh(ip_addr.ip)
+                configure_gunicorn(ip_addr.ip)
+        break

akanda/router/drivers/hostname.py

@@ -16,7 +16,7 @@
 
 import logging
 
-from akanda.router.drivers import (base, ip)
+from akanda.router.drivers import base
 from akanda.router import utils
 
 
@@ -38,12 +38,15 @@ class HostnameManager(base.Manager):
         )
 
     def update_hosts(self, config):
-        mgr = ip.IPManager()
-        listen_ip = mgr.get_management_address()
+        mgt_addr = config.management_address
+
+        if not mgt_addr:
+            return
+
         config_data = [
             '127.0.0.1  localhost',
             '::1  localhost ip6-localhost ip6-loopback',
-            '%s  %s' % (listen_ip, config.hostname)
+            '%s  %s' % (mgt_addr, config.hostname)
         ]
         utils.replace_file('/tmp/hosts', '\n'.join(config_data))
         utils.execute(['mv', '/tmp/hosts', '/etc/hosts'], self.root_helper)

akanda/router/drivers/ip.py

@@ -234,30 +234,6 @@ class IPManager(base.Manager):
             if ip.version == 4:
                 self._delete_conntrack_state(ip)
 
-    def get_management_address(self, ensure_configuration=False):
-        """
-        Get the network interface address that will be used for management
-        traffic.
-
-        :param ensure_configuration: when `True`, this method will ensure that
-                                     the management address if configured on
-                                     `ge0`.
-        :rtype: str
-        """
-        primary = self.get_interface(GENERIC_IFNAME + '0')
-        prefix, prefix_len = ULA_PREFIX.split('/', 1)
-        eui = netaddr.EUI(primary.lladdr)
-        ip_str = str(eui.ipv6_link_local()).replace('fe80::', prefix[:-1])
-
-        if not primary.is_up:
-            self.up(primary)
-
-        ip = netaddr.IPNetwork('%s/%s' % (ip_str, prefix_len))
-        if ensure_configuration and ip not in primary.addresses:
-            primary.addresses.append(ip)
-            self.update_interface(primary)
-        return ip_str
-
     def update_default_gateway(self, config):
         """
         Sets the default gateway for v4 and v6 via the use of `ip route add`.

akanda/router/models.py

@@ -395,6 +395,10 @@ class Network(ModelBase):
     def is_external_network(self):
         return self._network_type == self.TYPE_EXTERNAL
 
+    @property
+    def is_management_network(self):
+        return self._network_type == self.TYPE_MANAGEMENT
+
     @property
     def network_type(self):
         return self._network_type
@@ -560,3 +564,15 @@ class Configuration(ModelBase):
     @property
     def interfaces(self):
         return [n.interface for n in self.networks if n.interface]
+
+    @property
+    def management_address(self):
+        addrs = []
+        for net in self.networks:
+            if net.is_management_network:
+                addrs.extend((net.interface.first_v4, net.interface.first_v6))
+
+        addrs = sorted(a for a in addrs if a)
+
+        if addrs:
+            return addrs[0]

ansible/tasks/akanda.yml

@@ -18,6 +18,9 @@
 - name: install akanda-appliance
   command: python setup.py install chdir=/tmp/akanda-appliance
 
+- name: install gunicorn config file
+  template: src=gunicorn.j2 dest=/etc/akanda_gunicorn_config
+
 - name: install init.d files
   copy: src={{playbook_dir}}/../scripts/etc/init.d/{{item}} dest=/etc/init.d/{{item}} mode=0555
   with_items:

ansible/tasks/base.yml

@@ -30,3 +30,14 @@
 - name: disable fsck on boot via fastboot
   file: path=/fastboot state=touch
 
+- name: reset v4 persistent table rules
+  template: src=rules_v4.j2 dest=/etc/iptables/rules.v4
+
+- name: reset v6 persistent table rules
+  template: src=rules_v6.j2 dest=/etc/iptables/rules.v6
+
+- name: clear out network interfaces.d
+  shell: rm -f /etc/network/interfaces.d/*
+
+- name: reset network interfaces
+  file: content="auto lo\niface lo inet loopback" dest=/etc/network/interfaces

ansible/tasks/update_kernel.yml

@@ -7,7 +7,7 @@
   register: boot_dir
 
 - name: install kernel (Debian)
-  apt: name=linux-image-amd64 state=latest install_recommends=no
+  apt: name=linux-image-amd64 state=latest install_recommends=no default_release=wheezy-backports
 
 - name: update grub conf
   when: grub_dir.stat.exists == True
@@ -17,5 +17,5 @@
   register: boot_dir_after
 
 - name: update-grub
-  when: boot_dir_after.stat.mtime > boot_dir.stat.mtime
+  when: boot_dir_after.stat.mtime > boot_dir.stat.mtime and grub_dir.stat.exists == True
   command: update-grub

ansible/templates/gunicorn.j2

@@ -0,0 +1,10 @@
+import multiprocessing
+
+bind = '[::]:5000'
+workers = workers = multiprocessing.cpu_count() * 2 + 1
+backlog = 2048
+worker_class ="sync"
+debug = False
+daemon = True
+pidfile = "/var/run/gunicorn.pid"
+errorfile = "/tmp/gunicorn.log"

ansible/templates/rules_v4.j2

@@ -0,0 +1,25 @@
+# Generated by iptables-save v1.4.14 on Sun Apr 12 20:07:15 2015
+*filter
+:INPUT ACCEPT [0:0]
+:FORWARD ACCEPT [0:0]
+:OUTPUT ACCEPT [0:0]
+COMMIT
+# Completed on Sun Apr 12 20:07:15 2015
+# Generated by iptables-save v1.4.14 on Sun Apr 12 20:07:15 2015
+*mangle
+:PREROUTING ACCEPT [0:0]
+:INPUT ACCEPT [0:0]
+:FORWARD ACCEPT [0:0]
+:OUTPUT ACCEPT [0:0]
+:POSTROUTING ACCEPT [0:0]
+COMMIT
+# Completed on Sun Apr 12 20:07:15 2015
+# Generated by iptables-save v1.4.14 on Sun Apr 12 20:07:15 2015
+*nat
+:PREROUTING ACCEPT [0:0]
+:INPUT ACCEPT [0:0]
+:OUTPUT ACCEPT [0:0]
+:POSTROUTING ACCEPT [0:0]
+COMMIT
+# Completed on Sun Apr 12 20:07:15 2015
+

ansible/templates/rules_v6.j2

@@ -0,0 +1,7 @@
+# Generated by ip6tables-save v1.4.14 on Sun Apr 12 20:14:31 2015
+*filter
+:INPUT ACCEPT [0:0]
+:FORWARD ACCEPT [0:0]
+:OUTPUT ACCEPT [0:0]
+COMMIT
+# Completed on Sun Apr 12 20:14:31 2015

scripts/etc/init.d/akanda-router-api-server

@@ -20,13 +20,8 @@ test -x $DAEMON || exit 0
 
 . /lib/lsb/init-functions
 
-akanda_configure_gunicorn() {
-        /usr/local/bin/akanda-configure-gunicorn
-}
-
 case "$1" in
   start)
-        akanda_configure_gunicorn
 	log_daemon_msg "Starting akanda-router-api-server" $NAME
 	start_daemon -p $PIDFILE $DAEMON $OPTIONS
 	log_end_msg $?

setup.py

@@ -1,6 +1,5 @@
 # Copyright 2014 DreamHost, LLC
-#
-# Author: DreamHost, LLC
+# Copyright 2015 Akanda, Inc
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -20,11 +19,11 @@ from setuptools import setup, find_packages
 
 setup(
     name='akanda-router',
-    version='0.2.0',
+    version='0.3.0',
     description='A packet filter based router appliance',
-    author='DreamHost',
-    author_email='dev-community@dreamhost.com',
-    url='http://github.com/dreamhost/akanda',
+    author='Akanda',
+    author_email='dev-community@akanda.io',
+    url='http://github.com/akanda/akanda',
     license='Apache2',
     install_requires=[
         'flask>=0.9',
@@ -40,10 +39,8 @@ setup(
     zip_safe=False,
     entry_points={
         'console_scripts': [
-            'akanda-configure-ssh ='
-            'akanda.router.commands.management:configure_ssh',
-            'akanda-configure-gunicorn = '
-            'akanda.router.commands.management:configure_gunicorn',
+            'akanda-configure-management ='
+            'akanda.router.commands.management:configure_management',
             'akanda-api-dev-server = akanda.router.api.server:main',
             'akanda-metadata-proxy = akanda.router.metadata_proxy:main',
         ]

test/unit/drivers/test_hostname.py

@@ -23,6 +23,7 @@ from akanda.router.drivers import hostname, ip
 
 CONFIG = mock.Mock()
 CONFIG.hostname = 'akanda'
+CONFIG.management_address = 'fdca:3ba5:a17a:acda:f816:3eff:fe66:33b6'
 
 
 class HostnameTestCase(TestCase):
@@ -44,14 +45,12 @@ class HostnameTestCase(TestCase):
             mock.call(['mv', '/tmp/hostname', '/etc/hostname'], 'sudo')
         ])
 
-    @mock.patch.object(ip.IPManager, 'get_management_address')
-    def test_update_hosts(self, addr):
+    def test_update_hosts(self):
         expected = [
             '127.0.0.1  localhost',
             '::1  localhost ip6-localhost ip6-loopback',
             'fdca:3ba5:a17a:acda:f816:3eff:fe66:33b6  akanda'
         ]
-        addr.return_value = 'fdca:3ba5:a17a:acda:f816:3eff:fe66:33b6'
         self.mgr.update_hosts(CONFIG)
         self.mock_execute.assert_has_calls([
             mock.call(['mv', '/tmp/hosts', '/etc/hosts'], 'sudo')

test/unit/drivers/test_ip.py

@@ -335,42 +335,6 @@ class IPTestCase(TestCase):
         mgr._update_set('em0', iface, old_iface, 'all_addresses', add, delete)
         self.assertEqual(self.mock_execute.call_count, 0)
 
-    def test_get_management_address(self):
-        # Mark eth0 as DOWN
-        output = """2: eth0: <BROADCAST,MULTICAST> mtu 1500 qdisc pfifo_fast state DOWN qlen 1000
-    link/ether fa:16:3e:34:ba:28 brd ff:ff:ff:ff:ff:ff
-       valid_lft forever preferred_lft forever"""  # noqa
-
-        fake_output = lambda *x: output
-        with mock.patch.object(ip.IPManager, 'do', fake_output):
-            mgr = ip.IPManager()
-            addr = mgr.get_management_address()
-            assert addr == 'fdca:3ba5:a17a:acda:f816:3eff:fe34:ba28'
-            assert self.mock_execute.call_args_list == [
-                mock.call(['/sbin/ip', 'link', 'set', 'eth0', 'up'], 'sudo'),
-            ]
-
-    def test_get_management_address_with_autoconfiguration(self):
-        cmd = '/sbin/ip'
-        # Mark eth0 as DOWN
-        output = """2: eth0: <BROADCAST,MULTICAST> mtu 1500 qdisc pfifo_fast state DOWN qlen 1000
-    link/ether fa:16:3e:34:ba:28 brd ff:ff:ff:ff:ff:ff
-       valid_lft forever preferred_lft forever"""  # noqa
-
-        fake_output = lambda *x: output
-        with mock.patch.object(ip.IPManager, 'do', fake_output):
-            mgr = ip.IPManager()
-            addr = mgr.get_management_address(ensure_configuration=True)
-            assert addr == 'fdca:3ba5:a17a:acda:f816:3eff:fe34:ba28'
-            assert self.mock_execute.call_args_list == [
-                mock.call([cmd, 'link', 'set', 'eth0', 'up'], 'sudo'),
-                mock.call([
-                    cmd, '-6', 'addr', 'add',
-                    'fdca:3ba5:a17a:acda:f816:3eff:fe34:ba28/64', 'dev', 'eth0'
-                ], 'sudo'),
-                mock.call([cmd, 'link', 'set', 'eth0', 'up'], 'sudo')
-            ]
-
 
 class TestDisableDAD(TestCase):
     """