Ensure that at needed stages the local variables

of the init class are reset so that when they are
regenerated that they will use the updated data
instead of using previous data (since they weren't reset).

cloudinit/stages.py

@@ -36,6 +36,8 @@ from cloudinit.handlers import cloud_config as cc_part
 from cloudinit.handlers import shell_script as ss_part
 from cloudinit.handlers import upstart_job as up_part
 
+from cloudinit.sources import DataSourceNone
+
 from cloudinit import cloud
 from cloudinit import config
 from cloudinit import distros
@@ -58,8 +60,16 @@ class Init(object):
         self._cfg = None
         self._paths = None
         self._distro = None
-        # Created only when a fetch occurs
-        self.datasource = None
+        # Changed only when a fetch occurs
+        self.datasource = DataSourceNone.DataSourceNone({}, None, None)
+
+    def _reset(self, ds=False):
+        # Recreated on access
+        self._cfg = None
+        self._paths = None
+        self._distro = None
+        if ds:
+            self.datasource = DataSourceNone.DataSourceNone({}, None, None)
 
     @property
     def distro(self):
@@ -236,7 +246,7 @@ class Init(object):
         self.datasource = ds
         # Ensure we adjust our path members datasource
         # now that we have one (thus allowing ipath to be used)
-        self.paths.datasource = ds
+        self._reset()
         return ds
 
     def _get_instance_subdirs(self):
@@ -296,6 +306,10 @@ class Init(object):
         util.write_file(iid_fn, "%s\n" % iid)
         util.write_file(os.path.join(dp, 'previous-instance-id'),
                         "%s\n" % (previous_iid))
+        # Ensure needed components are regenerated
+        # after change of instance which may cause
+        # change of configuration
+        self._reset()
         return iid
 
     def fetch(self):
@@ -409,6 +423,17 @@ class Init(object):
             handlers.call_end(mod, data, frequency)
             called.append(mod)
 
+        # Perform post-consumption adjustments so that
+        # modules that run during the init stage reflect
+        # this consumed set.
+        #
+        # They will be recreated on future access...
+        self._reset()
+        # Note(harlowja): the 'active' datasource will have
+        # references to the previous config, distro, paths
+        # objects before the load of the userdata happened,
+        # this is expected.
+
 
 class Modules(object):
     def __init__(self, init, cfg_files=None):

tests/data/user_data.1.txt

@@ -0,0 +1,15 @@
+#cloud-config
+write_files:
+-   content: blah
+    path: /etc/blah.ini
+    permissions: 493
+
+system_info:
+ package_mirrors:
+   - arches: [i386, amd64, blah]
+     failsafe:
+      primary: http://my.archive.mydomain.com/ubuntu
+      security: http://my.security.mydomain.com/ubuntu
+     search:
+      primary: []
+      security: []

tests/unittests/test_merging.py

@@ -0,0 +1,62 @@
+from mocker import MockerTestCase
+
+from cloudinit import util
+
+
+class TestMergeDict(MockerTestCase):
+    def test_simple_merge(self):
+        """Test simple non-conflict merge."""
+        source = {"key1": "value1"}
+        candidate = {"key2": "value2"}
+        result = util.mergedict(source, candidate)
+        self.assertEqual({"key1": "value1", "key2": "value2"}, result)
+
+    def test_nested_merge(self):
+        """Test nested merge."""
+        source = {"key1": {"key1.1": "value1.1"}}
+        candidate = {"key1": {"key1.2": "value1.2"}}
+        result = util.mergedict(source, candidate)
+        self.assertEqual(
+            {"key1": {"key1.1": "value1.1", "key1.2": "value1.2"}}, result)
+
+    def test_merge_does_not_override(self):
+        """Test that candidate doesn't override source."""
+        source = {"key1": "value1", "key2": "value2"}
+        candidate = {"key1": "value2", "key2": "NEW VALUE"}
+        result = util.mergedict(source, candidate)
+        self.assertEqual(source, result)
+
+    def test_empty_candidate(self):
+        """Test empty candidate doesn't change source."""
+        source = {"key": "value"}
+        candidate = {}
+        result = util.mergedict(source, candidate)
+        self.assertEqual(source, result)
+
+    def test_empty_source(self):
+        """Test empty source is replaced by candidate."""
+        source = {}
+        candidate = {"key": "value"}
+        result = util.mergedict(source, candidate)
+        self.assertEqual(candidate, result)
+
+    def test_non_dict_candidate(self):
+        """Test non-dict candidate is discarded."""
+        source = {"key": "value"}
+        candidate = "not a dict"
+        result = util.mergedict(source, candidate)
+        self.assertEqual(source, result)
+
+    def test_non_dict_source(self):
+        """Test non-dict source is not modified with a dict candidate."""
+        source = "not a dict"
+        candidate = {"key": "value"}
+        result = util.mergedict(source, candidate)
+        self.assertEqual(source, result)
+
+    def test_neither_dict(self):
+        """Test if neither candidate or source is dict source wins."""
+        source = "source"
+        candidate = "candidate"
+        result = util.mergedict(source, candidate)
+        self.assertEqual(source, result)

tests/unittests/test_runs/test_merge_run.py

@@ -0,0 +1,52 @@
+import os
+
+from tests.unittests import helpers
+
+from cloudinit.settings import (PER_INSTANCE)
+from cloudinit import stages
+from cloudinit import util
+
+
+class TestSimpleRun(helpers.FilesystemMockingTestCase):
+    def _patchIn(self, root):
+        self.restore()
+        self.patchOS(root)
+        self.patchUtils(root)
+
+    def test_none_ds(self):
+        new_root = self.makeDir()
+        self.replicateTestRoot('simple_ubuntu', new_root)
+        cfg = {
+            'datasource_list': ['None'],
+            'cloud_init_modules': ['write-files'],
+        }
+        ud = self.readResource('user_data.1.txt')
+        cloud_cfg = util.yaml_dumps(cfg)
+        util.ensure_dir(os.path.join(new_root, 'etc', 'cloud'))
+        util.write_file(os.path.join(new_root, 'etc',
+                                     'cloud', 'cloud.cfg'), cloud_cfg)
+        self._patchIn(new_root)
+
+        # Now start verifying whats created
+        initer = stages.Init()
+        initer.read_cfg()
+        initer.initialize()
+        initer.fetch()
+        initer.datasource.userdata_raw = ud
+        iid = initer.instancify()
+        initer.update()
+        initer.cloudify().run('consume_userdata',
+                              initer.consume_userdata,
+                              args=[PER_INSTANCE],
+                              freq=PER_INSTANCE)
+        mirrors = initer.distro.get_option('package_mirrors')
+        self.assertEquals(1, len(mirrors))
+        mirror = mirrors[0]
+        self.assertEquals(mirror['arches'], ['i386', 'amd64', 'blah'])
+        mods = stages.Modules(initer)
+        (which_ran, failures) = mods.run_section('cloud_init_modules')
+        self.assertTrue(len(failures) == 0)
+        self.assertTrue(os.path.exists('/etc/blah.ini'))
+        self.assertIn('write-files', which_ran)
+        contents = util.load_file('/etc/blah.ini')
+        self.assertEquals(contents, 'blah')

tests/unittests/test_util.py

@@ -28,65 +28,6 @@ class FakeSelinux(object):
         self.restored.append(path)
 
 
-class TestMergeDict(MockerTestCase):
-    def test_simple_merge(self):
-        """Test simple non-conflict merge."""
-        source = {"key1": "value1"}
-        candidate = {"key2": "value2"}
-        result = util.mergedict(source, candidate)
-        self.assertEqual({"key1": "value1", "key2": "value2"}, result)
-
-    def test_nested_merge(self):
-        """Test nested merge."""
-        source = {"key1": {"key1.1": "value1.1"}}
-        candidate = {"key1": {"key1.2": "value1.2"}}
-        result = util.mergedict(source, candidate)
-        self.assertEqual(
-            {"key1": {"key1.1": "value1.1", "key1.2": "value1.2"}}, result)
-
-    def test_merge_does_not_override(self):
-        """Test that candidate doesn't override source."""
-        source = {"key1": "value1", "key2": "value2"}
-        candidate = {"key1": "value2", "key2": "NEW VALUE"}
-        result = util.mergedict(source, candidate)
-        self.assertEqual(source, result)
-
-    def test_empty_candidate(self):
-        """Test empty candidate doesn't change source."""
-        source = {"key": "value"}
-        candidate = {}
-        result = util.mergedict(source, candidate)
-        self.assertEqual(source, result)
-
-    def test_empty_source(self):
-        """Test empty source is replaced by candidate."""
-        source = {}
-        candidate = {"key": "value"}
-        result = util.mergedict(source, candidate)
-        self.assertEqual(candidate, result)
-
-    def test_non_dict_candidate(self):
-        """Test non-dict candidate is discarded."""
-        source = {"key": "value"}
-        candidate = "not a dict"
-        result = util.mergedict(source, candidate)
-        self.assertEqual(source, result)
-
-    def test_non_dict_source(self):
-        """Test non-dict source is not modified with a dict candidate."""
-        source = "not a dict"
-        candidate = {"key": "value"}
-        result = util.mergedict(source, candidate)
-        self.assertEqual(source, result)
-
-    def test_neither_dict(self):
-        """Test if neither candidate or source is dict source wins."""
-        source = "source"
-        candidate = "candidate"
-        result = util.mergedict(source, candidate)
-        self.assertEqual(source, result)
-
-
 class TestGetCfgOptionListOrStr(TestCase):
     def test_not_found_no_default(self):
         """None is returned if key is not found and no default given."""

tools/run-pep8

@@ -21,10 +21,21 @@ else
     base=`pwd`/tools/
 fi
 
+IGNORE="E501" # Line too long (these are caught by pylint)
+
+# King Arthur: Be quiet! ... Be Quiet! I Order You to Be Quiet.
+IGNORE="$IGNORE,E121" # Continuation line indentation is not a multiple of four
+IGNORE="$IGNORE,E123" # Closing bracket does not match indentation of opening bracket's line
+IGNORE="$IGNORE,E124" # Closing bracket missing visual indentation
+IGNORE="$IGNORE,E125" # Continuation line does not distinguish itself from next logical line
+IGNORE="$IGNORE,E126" # Continuation line over-indented for hanging indent
+IGNORE="$IGNORE,E127" # Continuation line over-indented for visual indent
+IGNORE="$IGNORE,E128" # Continuation line under-indented for visual indent
+
 cmd=(
     ${base}/hacking.py
 
-    --ignore=E501 # Line too long (these are caught by pylint)
+    --ignore="$IGNORE"
 
     "${files[@]}"
 )