FreeBsd: fix initscripts and add working config file

This set of changes generally produces a functional cloud-init on FreeBsd.

ChangeLog

@@ -27,6 +27,7 @@
    (LP: #1340903) [Patrick Lucas]
  - no longer use pylint as a checker, fix pep8 [Jay Faulkner].
  - Openstack: do not load some urls twice.
+ - FreeBsd: fix initscripts and add working config file [Harm Weites]
 0.7.5:
  - open 0.7.5
  - Add a debug log message around import failures

cloudinit/distros/freebsd.py

@@ -26,6 +26,9 @@ from cloudinit import log as logging
 from cloudinit import ssh_util
 from cloudinit import util
 
+from cloudinit.distros import net_util
+from cloudinit.distros.parsers.resolv_conf import ResolvConf
+
 LOG = logging.getLogger(__name__)
 
 
@@ -33,6 +36,7 @@ class Distro(distros.Distro):
     rc_conf_fn = "/etc/rc.conf"
     login_conf_fn = '/etc/login.conf'
     login_conf_fn_bak = '/etc/login.conf.orig'
+    resolv_conf_fn = '/etc/resolv.conf'
 
     def __init__(self, name, cfg, paths):
         distros.Distro.__init__(self, name, cfg, paths)
@@ -44,30 +48,53 @@ class Distro(distros.Distro):
 
     # Updates a key in /etc/rc.conf.
     def updatercconf(self, key, value):
-        LOG.debug("updatercconf: %s => %s", key, value)
+        LOG.debug("Checking %s for: %s = %s", self.rc_conf_fn, key, value)
         conf = self.loadrcconf()
         config_changed = False
-        for item in conf:
+        if key not in conf:
-            if item == key and conf[item] != value:
+            LOG.debug("Adding key in %s: %s = %s", self.rc_conf_fn, key,
-                conf[item] = value
+                      value)
-                LOG.debug("[rc.conf]: Value %s for key %s needs to be changed",
+            conf[key] = value
-                          value, key)
+            config_changed = True
-                config_changed = True
+        else:
+            for item in conf.keys():
+                if item == key and conf[item] != value:
+                    conf[item] = value
+                    LOG.debug("Changing key in %s: %s = %s", self.rc_conf_fn,
+                              key, value)
+                    config_changed = True
 
         if config_changed:
-            LOG.debug("Writing new %s file", self.rc_conf_fn)
+            LOG.info("Writing %s", self.rc_conf_fn)
             buf = StringIO()
             for keyval in conf.items():
-                buf.write("%s=%s\n" % keyval)
+                buf.write('%s="%s"\n' % keyval)
             util.write_file(self.rc_conf_fn, buf.getvalue())
 
-    # Load the contents of /etc/rc.conf and store all keys in a dict.
+    # Load the contents of /etc/rc.conf and store all keys in a dict. Make sure
+    # quotes are ignored:
+    #  hostname="bla"
     def loadrcconf(self):
+        RE_MATCH = re.compile(r'^(\w+)\s*=\s*(.*)\s*')
         conf = {}
         lines = util.load_file(self.rc_conf_fn).splitlines()
         for line in lines:
-            tok = line.split('=')
+            m = RE_MATCH.match(line)
-            conf[tok[0]] = tok[1].rstrip()
+            if not m:
+                LOG.debug("Skipping line from /etc/rc.conf: %s", line)
+                continue
+            key = m.group(1).rstrip()
+            val = m.group(2).rstrip()
+            # Kill them quotes (not completely correct, aka won't handle
+            # quoted values, but should be ok ...)
+            if val[0] in ('"', "'"):
+                val = val[1:]
+            if val[-1] in ('"', "'"):
+                val = val[0:-1]
+            if len(val) == 0:
+                LOG.debug("Skipping empty value from /etc/rc.conf: %s", line)
+                continue
+            conf[key] = val
         return conf
 
     def readrcconf(self, key):
@@ -218,7 +245,60 @@ class Distro(distros.Distro):
             ssh_util.setup_user_keys(keys, name, options=None)
 
     def _write_network(self, settings):
-        return
+        entries = net_util.translate_network(settings)
+        nameservers = []
+        searchdomains = []
+        dev_names = entries.keys()
+        for (dev, info) in entries.iteritems():
+            # Skip the loopback interface.
+            if dev.startswith('lo'):
+                continue
+
+            LOG.info('Configuring interface %s', dev)
+
+            if info.get('bootproto') == 'static':
+                LOG.debug('Configuring dev %s with %s / %s', dev, info.get('address'), info.get('netmask'))
+                # Configure an ipv4 address.
+                ifconfig = info.get('address') + ' netmask ' + info.get('netmask')
+
+                # Configure the gateway.
+                self.updatercconf('defaultrouter', info.get('gateway'))
+
+                if 'dns-nameservers' in info:
+                    nameservers.extend(info['dns-nameservers'])
+                if 'dns-search' in info:
+                    searchservers.extend(info['dns-search'])
+            else:
+                ifconfig = 'DHCP'
+     
+            self.updatercconf('ifconfig_' + dev, ifconfig)
+
+        # Try to read the /etc/resolv.conf or just start from scratch if that
+        # fails.
+        try:
+            resolvconf = ResolvConf(util.load_file(self.resolv_conf_fn))
+            resolvconf.parse()
+        except IOError:
+            util.logexc(LOG, "Failed to parse %s, use new empty file", self.resolv_conf_fn)
+            resolvconf = ResolvConf('')
+            resolvconf.parse()
+
+        # Add some nameservers
+        for server in nameservers:
+            try:
+                resolvconf.add_nameserver(server)
+            except ValueError:
+                util.logexc(LOG, "Failed to add nameserver %s", server)
+
+        # And add any searchdomains.
+        for domain in searchdomains:
+            try:
+                resolvconf.add_search_domain(domain)
+            except ValueError:
+                util.logexc(LOG, "Failed to add search domain %s", domain)
+        util.write_file(self.resolv_conf_fn, str(resolvconf), 0644)
+
+        return dev_names
 
     def apply_locale(self, locale, out_fn=None):
         # Adjust the locals value to the new value

config/cloud.cfg-freebsd

@@ -0,0 +1,88 @@
+# The top level settings are used as module
+# and system configuration.
+
+syslog_fix_perms: root:wheel
+
+# This should not be required, but leave it in place until the real cause of
+# not beeing able to find -any- datasources is resolved.
+datasource_list: ['OpenStack']
+
+# A set of users which may be applied and/or used by various modules
+# when a 'default' entry is found it will reference the 'default_user'
+# from the distro configuration specified below
+users:
+   - default
+
+# If this is set, 'root' will not be able to ssh in and they 
+# will get a message to login instead as the above $user (ubuntu)
+disable_root: false
+
+# This will cause the set+update hostname module to not operate (if true)
+preserve_hostname: false
+
+# Example datasource config
+# datasource: 
+#    Ec2: 
+#      metadata_urls: [ 'blah.com' ]
+#      timeout: 5 # (defaults to 50 seconds)
+#      max_wait: 10 # (defaults to 120 seconds)
+
+# The modules that run in the 'init' stage
+cloud_init_modules:
+# - migrator
+ - seed_random
+ - bootcmd
+# - write-files
+ - growpart
+ - resizefs
+ - set_hostname
+ - update_hostname
+# - update_etc_hosts
+# - ca-certs
+# - rsyslog
+ - users-groups
+ - ssh
+
+# The modules that run in the 'config' stage
+cloud_config_modules:
+# - disk_setup
+# - mounts
+ - ssh-import-id
+ - locale
+# - set-passwords
+# - package-update-upgrade-install
+# - landscape
+# - timezone
+# - puppet
+# - chef
+# - salt-minion
+# - mcollective
+ - disable-ec2-metadata
+ - runcmd
+# - byobu
+
+# The modules that run in the 'final' stage
+cloud_final_modules:
+ - rightscale_userdata
+ - scripts-vendor
+ - scripts-per-once
+ - scripts-per-boot
+ - scripts-per-instance
+ - scripts-user
+ - ssh-authkey-fingerprints
+ - keys-to-console
+ - phone-home
+ - final-message
+ - power-state-change
+
+# System and/or distro specific settings
+# (not accessible to handlers/transforms)
+system_info:
+   distro: freebsd
+   default_user:
+     name: beastie
+     lock_passwd: True
+     gecos: FreeBSD
+     groups: [wheel]
+     sudo: ["ALL=(ALL) NOPASSWD:ALL"]
+     shell: /bin/sh

setup.py

@@ -63,18 +63,28 @@ def systemd_unitdir():
 
 INITSYS_FILES = {
     'sysvinit': [f for f in glob('sysvinit/redhat/*') if is_f(f)],
+    'sysvinit_freebsd': [f for f in glob('sysvinit/freebsd/*') if is_f(f)],
     'sysvinit_deb': [f for f in glob('sysvinit/debian/*') if is_f(f)],
     'systemd': [f for f in glob('systemd/*') if is_f(f)],
     'upstart': [f for f in glob('upstart/*') if is_f(f)],
 }
 INITSYS_ROOTS = {
     'sysvinit': '/etc/rc.d/init.d',
+    'sysvinit_freebsd': '/usr/local/etc/rc.d',
     'sysvinit_deb': '/etc/init.d',
     'systemd': systemd_unitdir(),
     'upstart': '/etc/init/',
 }
 INITSYS_TYPES = sorted(list(INITSYS_ROOTS.keys()))
 
+# Install everything in the right location and take care of Linux (default) and
+# FreeBSD systems.
+USR = "/usr"
+ETC = "/etc"
+if os.uname()[0] == 'FreeBSD':
+    USR = "/usr/local"
+    ETC = "/usr/local/etc"
+
 
 def get_version():
     cmd = ['tools/read-version']
@@ -136,18 +146,17 @@ setuptools.setup(name='cloud-init',
                'tools/cloud-init-per',
                ],
       license='GPLv3',
-      data_files=[('/etc/cloud', glob('config/*.cfg')),
+      data_files=[(ETC + '/cloud', glob('config/*.cfg')),
-                  ('/etc/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')),
+                  (ETC + '/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')),
-                  ('/etc/cloud/templates', glob('templates/*')),
+                  (ETC + '/cloud/templates', glob('templates/*')),
-                  ('/usr/share/cloud-init', []),
+                  (USR + '/lib/cloud-init',
-                  ('/usr/lib/cloud-init',
                     ['tools/uncloud-init',
                      'tools/write-ssh-key-fingerprints']),
-                  ('/usr/share/doc/cloud-init',
+                  (USR + '/share/doc/cloud-init',
                    [f for f in glob('doc/*') if is_f(f)]),
-                  ('/usr/share/doc/cloud-init/examples',
+                  (USR + '/share/doc/cloud-init/examples',
                    [f for f in glob('doc/examples/*') if is_f(f)]),
-                  ('/usr/share/doc/cloud-init/examples/seed',
+                  (USR + '/share/doc/cloud-init/examples/seed',
                    [f for f in glob('doc/examples/seed/*') if is_f(f)]),
                  ],
       install_requires=read_requires(),

sysvinit/freebsd/cloudconfig

@@ -6,28 +6,28 @@
 
 . /etc/rc.subr
 
+export CLOUD_CFG=/usr/local/etc/cloud/cloud.cfg
+
 name="cloudconfig"
-command="/usr/bin/cloud-init"
+command="/usr/local/bin/cloud-init"
 start_cmd="cloudconfig_start"
 stop_cmd=":"
 rcvar="cloudinit_enable"
 start_precmd="cloudinit_override"
 start_cmd="cloudconfig_start"
 
-: ${cloudinit_config:="/etc/cloud/cloud.cfg"}
-
 cloudinit_override()
 {
-	# If there exist sysconfig/default variable override files use it...
+	# If there exist sysconfig/defaults variable override files use it...
-	if [ -f /etc/default/cloud-init ]; then
+	if [ -f /etc/defaults/cloud-init ]; then
-		. /etc/default/cloud-init
+		. /etc/defaults/cloud-init
 	fi
 }
 
 cloudconfig_start()
 {
 	echo "${command} starting"
-	${command} ${cloudinit_config} modules --mode config
+	${command} modules --mode config
 }
 
 load_rc_config $name

sysvinit/freebsd/cloudfinal

@@ -6,28 +6,28 @@
 
 . /etc/rc.subr
 
+export CLOUD_CFG=/usr/local/etc/cloud/cloud.cfg
+
 name="cloudfinal"
-command="/usr/bin/cloud_init"
+command="/usr/local/bin/cloud-init"
 start_cmd="cloudfinal_start"
 stop_cmd=":"
 rcvar="cloudinit_enable"
 start_precmd="cloudinit_override"
 start_cmd="cloudfinal_start"
 
-: ${cloudinit_config:="/etc/cloud/cloud.cfg"}
-
 cloudinit_override()
 {
-	# If there exist sysconfig/default variable override files use it...
+	# If there exist sysconfig/defaults variable override files use it...
-	if [ -f /etc/default/cloud-init ]; then
+	if [ -f /etc/defaults/cloud-init ]; then
-		 . /etc/default/cloud-init
+		 . /etc/defaults/cloud-init
 	fi
 }
 
 cloudfinal_start()
 {
 	echo -n "${command} starting"
-	${command} ${cloudinit_config} modules --mode final
+	${command} modules --mode final
 }
 
 load_rc_config $name

sysvinit/freebsd/cloudinit

@@ -6,28 +6,28 @@
 
 . /etc/rc.subr
 
+export CLOUD_CFG=/usr/local/etc/cloud/cloud.cfg
+
 name="cloudinit"
-command="/usr/bin/cloud_init"
+command="/usr/local/bin/cloud-init"
 start_cmd="cloudinit_start"
 stop_cmd=":"
 rcvar="cloudinit_enable"
 start_precmd="cloudinit_override"
 start_cmd="cloudinit_start"
 
-: ${cloudinit_config:="/etc/cloud/cloud.cfg"}
-
 cloudinit_override()
 {
-	# If there exist sysconfig/default variable override files use it...
+	# If there exist sysconfig/defaults variable override files use it...
-	if [ -f /etc/default/cloud-init ]; then
+	if [ -f /etc/defaults/cloud-init ]; then
-		. /etc/default/cloud-init
+		. /etc/defaults/cloud-init
 	fi
 }
 
 cloudinit_start()
 {
 	echo -n "${command} starting"
-	${command} ${cloudinit_config} init
+	${command} init
 }
 
 load_rc_config $name

sysvinit/freebsd/cloudinitlocal

@@ -6,28 +6,28 @@
 
 . /etc/rc.subr
 
+export CLOUD_CFG=/usr/local/etc/cloud/cloud.cfg
+
 name="cloudinitlocal"
-command="/usr/bin/cloud-init"
+command="/usr/local/bin/cloud-init"
 start_cmd="cloudlocal_start"
 stop_cmd=":"
 rcvar="cloudinit_enable"
 start_precmd="cloudinit_override"
 start_cmd="cloudlocal_start"
 
-: ${cloudinit_config:="/etc/cloud/cloud.cfg"}
-
 cloudinit_override()
 {
-	# If there exist sysconfig/default variable override files use it...
+	# If there exist sysconfig/defaults variable override files use it...
-	if [ -f /etc/default/cloud-init ]; then
+	if [ -f /etc/defaults/cloud-init ]; then
-		. /etc/default/cloud-init
+		. /etc/defaults/cloud-init
 	fi
 }
 
 cloudlocal_start()
 {
 	echo -n "${command} starting"
-	${command} ${cloudinit_config} init --local
+	${command} init --local
 }
 
 load_rc_config $name

tests/unittests/test_distros/test_netconfig.py

@@ -173,3 +173,60 @@ NETWORKING=yes
 '''
         self.assertCfgEquals(expected_buf, str(write_buf))
         self.assertEquals(write_buf.mode, 0644)
+
+    def test_simple_write_freebsd(self):
+        fbsd_distro = self._get_distro('freebsd')
+        util_mock = self.mocker.replace(util.write_file,
+                                        spec=False, passthrough=False)
+        exists_mock = self.mocker.replace(os.path.isfile,
+                                          spec=False, passthrough=False)
+        load_mock = self.mocker.replace(util.load_file,
+                                        spec=False, passthrough=False)
+
+        exists_mock(mocker.ARGS)
+        self.mocker.count(0, None)
+        self.mocker.result(False)
+
+        write_bufs = {}
+        read_bufs = {
+            '/etc/rc.conf': '',
+        }
+
+        def replace_write(filename, content, mode=0644, omode="wb"):
+            buf = WriteBuffer()
+            buf.mode = mode
+            buf.omode = omode
+            buf.write(content)
+            write_bufs[filename] = buf
+
+        def replace_read(fname, read_cb=None, quiet=False):
+            if fname not in read_bufs:
+                if fname in write_bufs:
+                    return str(write_bufs[fname])
+                raise IOError("%s not found" % fname)
+            else:
+                if fname in write_bufs:
+                    return str(write_bufs[fname])
+                return read_bufs[fname]
+
+        util_mock(mocker.ARGS)
+        self.mocker.call(replace_write)
+        self.mocker.count(0, None)
+
+        load_mock(mocker.ARGS)
+        self.mocker.call(replace_read)
+        self.mocker.count(0, None)
+
+        self.mocker.replay()
+        fbsd_distro.apply_network(BASE_NET_CFG, False)
+
+        self.assertIn('/etc/rc.conf', write_bufs)
+        write_buf = write_bufs['/etc/rc.conf']
+        expected_buf = '''
+ifconfig_eth0="192.168.1.5 netmask 255.255.255.0"
+ifconfig_eth1="DHCP"
+defaultrouter="192.168.1.254"
+'''
+        self.assertCfgEquals(expected_buf, str(write_buf))
+        self.assertEquals(write_buf.mode, 0644)
+

tools/build-on-freebsd

@@ -0,0 +1,57 @@
+#!/bin/sh
+# Since there is no official FreeBSD port yet, we need some way of building and
+# installing cloud-init. This script takes care of building and installing. It
+# will optionally make a first run at the end.
+
+fail() { echo "FAILED:" "$@" 1>&2; exit 1; }
+
+# Check dependencies:
+depschecked=/tmp/c-i.dependencieschecked
+pkgs="
+   dmidecode
+   py27-argparse
+   py27-boto gpart sudo
+   py27-configobj py27-yaml
+   py27-Jinja2
+   py27-oauth py27-serial
+   py27-prettytable
+   py27-requests py27-six
+   python py27-cheetah
+"
+[ -f "$depschecked" ] || pkg install ${pkgs} || fail "install packages"
+touch $depschecked
+
+# Required but unavailable port/pkg: py27-jsonpatch py27-jsonpointer
+# Luckily, the install step will take care of this by installing it from pypi...
+
+# Build the code and install in /usr/local/:
+python setup.py build
+python setup.py install -O1 --skip-build --prefix /usr/local/ --init-system sysvinit_freebsd
+
+# Install the correct config file:
+cp config/cloud.cfg-freebsd /usr/local/etc/cloud/cloud.cfg
+
+# Enable cloud-init in /etc/rc.conf:
+sed -i.bak -e "/cloudinit_enable=.*/d" /etc/rc.conf
+echo 'cloudinit_enable="YES"' >> /etc/rc.conf
+
+echo "Installation completed."
+
+if [ "$1" = "run" ]; then
+	echo "Ok, now let's see if it works."
+
+	# Backup SSH keys
+	mv /etc/ssh/ssh_host_* /tmp/
+
+	# Remove old metadata
+	rm -rf /var/lib/cloud
+
+	# Just log everything, quick&dirty
+	rm /usr/local/etc/cloud/cloud.cfg.d/05_logging.cfg 
+
+	# Start:
+	/usr/local/etc/rc.d/cloudinit start
+
+	# Restore SSH keys
+	mv /tmp/ssh_host_* /etc/ssh/
+fi