Continue working on merging.

cloudinit/handlers/cloud_config.py

@@ -85,17 +85,16 @@ class CloudConfigPartHandler(handlers.Handler):
         all_mergers.extend(mergers_header)
         if not all_mergers:
             all_mergers = DEF_MERGERS
-        return all_mergers
+        return (payload_yaml, all_mergers)
 
     def _merge_part(self, payload, headers):
-        my_mergers = self._extract_mergers(payload, headers)
+        (payload_yaml, my_mergers) = self._extract_mergers(payload, headers)
         LOG.debug("Merging by applying %s", my_mergers)
         merger = mergers.construct(my_mergers)
         if self.cloud_buf is None:
             # First time through, merge with an empty dict...
             self.cloud_buf = {}
-        self.cloud_buf = merger.merge(self.cloud_buf,
+        self.cloud_buf = merger.merge(self.cloud_buf, payload_yaml)
-                                      util.load_yaml(payload))
 
     def _reset(self):
         self.file_names = []

cloudinit/mergers/__init__.py

@@ -55,6 +55,9 @@ class UnknownMerger(object):
         if not meth:
             meth = self._handle_unknown
             args.insert(0, method_name)
+        LOG.debug("Merging '%s' into '%s' using method '%s' of '%s'",
+                  type_name, type_utils.obj_name(merge_with),
+                  meth.__name__, self)
         return meth(*args)
 
 
@@ -66,6 +69,9 @@ class LookupMerger(UnknownMerger):
         else:
             self._lookups = lookups
 
+    def __str__(self):
+        return 'LookupMerger: (%s)' % (len(self._lookups))
+
     # For items which can not be merged by the parent this object
     # will lookup in a internally maintained set of objects and
     # find which one of those objects can perform the merge. If
@@ -78,6 +84,8 @@ class LookupMerger(UnknownMerger):
                 # First one that has that method/attr gets to be
                 # the one that will be called
                 meth = getattr(merger, meth_wanted)
+                LOG.debug(("Merging using located merger '%s'"
+                            " since it had method '%s'"), merger, meth_wanted)
                 break
         if not meth:
             return UnknownMerger._handle_unknown(self, meth_wanted,
@@ -87,9 +95,9 @@ class LookupMerger(UnknownMerger):
 
 def dict_extract_mergers(config):
     parsed_mergers = []
-    raw_mergers = config.get('merge_how')
+    raw_mergers = config.pop('merge_how', None)
     if raw_mergers is None:
-        raw_mergers = config.get('merge_type')
+        raw_mergers = config.pop('merge_type', None)
     if raw_mergers is None:
         return parsed_mergers
     if isinstance(raw_mergers, (str, basestring)):

cloudinit/mergers/m_dict.py

@@ -20,30 +20,59 @@
 class Merger(object):
     def __init__(self, merger, opts):
         self._merger = merger
-        self._not_overwrite = 'not_overwrite' in opts
+        # Affects merging behavior...
+        self._method = 'replace'
+        for m in ['replace', 'no_replace']:
+            if m in opts:
+                self._method = m
+                break
+        # Affect how recursive merging is done on other primitives
+        self._recurse_str = 'recurse_str' in opts
+        self._recurse_dict = True
+        self._recurse_array = 'recurse_array' in opts
+        self._allow_delete = 'allow_delete' in opts
+
+    def __str__(self):
+        s = ('DictMerger: (method=%s,recurse_str=%s,'
+             'recurse_dict=%s,recurse_array=%s,allow_delete=%s)')
+        s  = s % (self._method,
+                  self._recurse_str,
+                  self._recurse_dict,
+                  self._recurse_array,
+                  self._allow_delete)
+        return s
+
+    def _do_dict_replace(self, value, merge_with, do_replace=True):
+
+        def merge_same_key(old_v, new_v):
+            if do_replace:
+                return new_v
+            if isinstance(new_v, (list, tuple)) and self._recurse_array:
+                return self._merger.merge(old_v, new_v)
+            if isinstance(new_v, (str, basestring)) and self._recurse_str:
+                return self._merger.merge(old_v, new_v)
+            if isinstance(new_v, (dict)) and self._recurse_dict:
+                return self._merger.merge(old_v, new_v)
+            # Otherwise leave it be...
+            return old_v
+
+        for (k, v) in merge_with.items():
+            if k in value:
+                if v is None and self._allow_delete:
+                    value.pop(k)
+                else:
+                    value[k] = merge_same_key(value[k], v)
+            else:
+                value[k] = v
+        return value
 
-    # This merging algorithm will attempt to merge with
-    # another dictionary, on encountering any other type of object
-    # it will not merge with said object, but will instead return
-    # the original value
-    #
-    # On encountering a dictionary, it will create a new dictionary
-    # composed of the original and the one to merge with, if 'overwrite'
-    # is enabled then keys that exist in the original will be overwritten
-    # by keys in the one to merge with (and associated values). Otherwise
-    # if not in overwrite mode the 2 conflicting keys themselves will
-    # be merged.
     def _on_dict(self, value, merge_with):
         if not isinstance(merge_with, (dict)):
             return value
-        merged = dict(value)
+        if self._method == 'replace':
-        for (k, v) in merge_with.items():
+            merged = self._do_dict_replace(dict(value), merge_with)
-            if k in merged:
+        elif self._method == 'no_replace':
-                if self._not_overwrite:
+            merged = self._do_dict_replace(dict(value), merge_with, False)
-                    # Attempt to merge them....
+        else:
-                    merged[k] = self._merger.merge(merged[k], v)
+            raise NotImplementedError("Unknown merge type %s" % (self._method))
-                else:
-                    merged[k] = v
-            else:
-                merged[k] = v
         return merged

cloudinit/mergers/m_list.py

@@ -20,18 +20,55 @@
 class Merger(object):
     def __init__(self, merger, opts):
         self._merger = merger
-        self._extend = 'extend' in opts
+        # Affects merging behavior...
+        self._method = 'replace'
+        for m in ['append', 'prepend', 'replace']:
+            if m in opts:
+                self._method = m
+                break
+        # Affect how recursive merging is done on other primitives
+        self._recurse_str = 'recurse_str' in opts
+        self._recurse_dict = 'recurse_dict' in opts
+        self._recurse_array = 'recurse_array' in opts
+
+    def __str__(self):
+        return 'ListMerger: (m=%s,rs=%s,rd=%s,ra=%s)' % (self._method,
+                                                         self._recurse_str,
+                                                         self._recurse_dict,
+                                                         self._recurse_array)
 
     def _on_tuple(self, value, merge_with):
-        return self._on_list(list(value), merge_with)
+        return tuple(self._on_list(list(value), merge_with))
 
-    # On encountering a list or tuple type this action will be applied
-    # a new list will be returned, if the value to merge with is itself
-    # a list and we have been told to 'extend', then the value here will
-    # be extended with the other list.
     def _on_list(self, value, merge_with):
-        if not self._extend or not isinstance(merge_with, (tuple, list)):
+        if (self._method == 'replace' and
-            return merge_with
+            not isinstance(merge_with, (tuple, list))):
-        # Leave the original list alone...
+                return merge_with
-        value = list(value)
+
-        return value.extend(merge_with)
+        # Ok we now know that what we are merging with is a list or tuple.
+        merged_list = []
+        if self._method == 'prepend':
+            merged_list.extend(merge_with)
+            merged_list.extend(value)
+            return merged_list
+        elif self._method == 'append':
+            merged_list.extend(value)
+            merged_list.extend(merge_with)
+            return merged_list
+
+        def merge_same_index(old_v, new_v):
+            if isinstance(new_v, (list, tuple)) and self._recurse_array:
+                return self._merger.merge(old_v, new_v)
+            if isinstance(new_v, (str, basestring)) and self._recurse_str:
+                return self._merger.merge(old_v, new_v)
+            if isinstance(new_v, (dict)) and self._recurse_dict:
+                return self._merger.merge(old_v, new_v)
+            # Otherwise leave it be...
+            return old_v
+
+        # Ok now we are replacing same indexes
+        merged_list.extend(value)
+        common_len = min(len(merged_list), len(merge_with))
+        for i in xrange(0, common_len):
+            merged_list[i] = merge_same_index(merged_list[i], merge_with[i])
+        return merged_list