Get rpm building working with a template.

packages/brpm

@@ -6,9 +6,19 @@ import subprocess
 import sys
 import tempfile
 import re
+import textwrap
+import shutil
+import zipfile
+
+import glob
 
 import tempita
 
+from datetime import datetime
+from datetime import date
+
+from distutils import version as ver
+
 # This is more just for running from the bin folder so that
 # cloud-init binary can find the cloudinit module
 possible_topdir = os.path.normpath(os.path.join(os.path.abspath(
@@ -16,28 +26,268 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(
 if os.path.exists(os.path.join(possible_topdir, "cloudinit", "__init__.py")):
     sys.path.insert(0, possible_topdir)
 
+from cloudinit import util
+from cloudinit import version
+
+import contextlib
+
+# Mapping of expected packages to there full name...
+PKG_MP = {
+    'boto': 'python-boto',
+    'tempita': 'python-tempita',
+    'prettytable': 'python-prettytable',
+    'oauth': 'python-oauth',
+    'configobj': 'python-configobj',
+    'yaml': 'PyYAML',
+    'argparse': 'python-argparse'
+}
+
+
+@contextlib.contextmanager
+def chdir(where_to):
+    cur_cwd = os.path.abspath(os.getcwd())
+    where_to = os.path.abspath(where_to)
+    os.chdir(where_to)
+    yield where_to
+    os.chdir(cur_cwd)
+
+
 PWD = os.getcwd()
 
 def info(msg):
-    sys.stderr.write("INFO: %s\n" % (msg))
+    print("INFO: %s" % (msg))
 
 
 def warn(msg):
-    sys.stderr.write("WARNING: %s\n" % (msg))
+    print("WARNING: %s" % (msg))
+
+
+def cut_up(entry, maxline=80):
+    if len(entry) < maxline:
+        return entry
+    else:
+        c = entry[0:maxline]
+        return "%s..." % (c)
+
+
+def extract_entry(collecting):
+    a_entry = {}
+    for t in ['tags', 'revno', 'author', 'timestamp', 'committer']:
+        look_for = "%s:" % (t)
+        for v in collecting:
+            if v.startswith(look_for):
+                a_entry[t] = v[len(look_for):].strip()
+                break
+    i = -1
+    for a, v in enumerate(collecting):
+        if v.startswith("message:"):
+            i = a
+            break
+    if i != -1:
+        msg_lines = collecting[i + 1:]
+        n_lines = []
+        for m in msg_lines:
+            m = m.strip()
+            if not m:
+                continue
+            m = m.replace("\n", " ")
+            n_lines.append("" + m.lstrip())
+        message = " ".join(n_lines).lstrip()
+        a_entry['message'] = message
+    return a_entry
+
+
+def build_changelog(history=-1):
+    cmd = ['bzr', 'log', '--timezone=utc']
+    (stdout, _stderr) = util.subp(cmd)
+    # Clean the format up
+    entries = stdout.splitlines()
+    all_entries = []
+    collecting = []
+    for e in entries:
+        if e.startswith("---"):
+            if collecting:
+                a_entry = extract_entry(collecting)
+                if a_entry:
+                    all_entries.append(a_entry)
+                collecting = []
+        else:
+            collecting.append(e)
+
+    a_entry = extract_entry(collecting)
+    if a_entry:
+        all_entries.append(a_entry)
+
+    if history > 0:
+        take_entries = list(all_entries[0:history])
+    else:
+        take_entries = list(all_entries)
+
+    # Merge those with same date
+    date_entries = {}
+    for e in take_entries:
+        author = e.get('author')
+        if not author:
+            author = e.get('committer')
+        if not author:
+            continue
+        timestamp = e.get('timestamp')
+        if not timestamp:
+            continue
+        msg = e.get('message')
+        if not msg:
+            continue
+        revno = e.get('revno')
+        if not revno:
+            continue
+        # http://bugs.python.org/issue6641
+        timestamp = timestamp.replace("+0000", '').strip()
+        ds = datetime.strptime(timestamp, '%a %Y-%m-%d %H:%M:%S')
+        c_ds = ds.date()
+        if c_ds not in date_entries:
+            ap_entry = {}
+            ap_entry['messages'] = []
+            ap_entry['authors'] = []
+            ap_entry['revnos'] = []
+            date_entries[c_ds] = ap_entry
+        ap_entry = date_entries[c_ds]
+        ap_entry['messages'].append(msg)
+        ap_entry['authors'].append(author)
+        ap_entry['revnos'].append(revno)
+
+    dates = sorted(date_entries.keys())
+    chglog = []
+    for ds in reversed(dates):
+        e = date_entries[ds]
+        authors = ", ".join(set(e['authors']))
+        revnos = ", ".join(list(sorted(e['revnos'])))
+        top_line = "%s %s - %s" % (ds.strftime("%a %b %d %Y"),
+                                   authors, revnos)
+        chglog.append("* %s" % (top_line))
+        for msg in e['messages']:
+            chglog.append("- %s" % (cut_up(msg)))
+    return "\n".join(chglog)
+
+
+def generate_spec_contents(tmpl_fn):
+    # Version junk
+    cmd = [os.path.join(os.pardir, 'tools', 'read-version')]
+    (stdout, _stderr) = util.subp(cmd)
+    i_version = stdout.strip()
+
+    # Ensure ok match!
+    if ver.StrictVersion(i_version) != version.version():
+        raise RuntimeError("Version found does not match the code version")
+
+    # Tmpl params
+    subs = {}
+    subs['version'] = i_version
+    (stdout, _stderr) = util.subp(['bzr', 'revno'])
+    subs['revno'] = "%s" % (stdout.strip())
+    subs['release'] = "%s" % (subs['revno'])
+    subs['archive_name'] = '%{name}-%{version}-' + subs['revno'] + '.tar.gz'
+    subs['bd_requires'] = ['python-devel', 'python-setuptools']
+
+    requires = []
+    cmd = [os.path.join(os.pardir, 'tools', 'read-dependencies')]
+    (stdout, _stderr) = util.subp(cmd)
+    pkgs = stdout.splitlines()
+    
+    # Map to known packages
+    for e in pkgs:
+        e = e.lower().strip()
+        tgt_pkg = None
+        for n in PKG_MP.keys():
+            if e.find(n) != -1:
+                tgt_pkg = PKG_MP.get(n)
+        if not tgt_pkg:
+            raise RuntimeError(("Do not know how to translate %s to "
+                                " a known package") % (e))
+        else:
+            requires.append(tgt_pkg)
+
+    base_name = 'cloud-init-%s-%s' % (i_version, subs['revno'])
+    subs['requires'] = requires
+    subs['changelog'] = build_changelog()
+
+    # See: http://www.zarb.org/~jasonc/macros.php
+    
+    # Pickup any special files
+    docs = [
+        'TODO',
+        'LICENSE',
+        'ChangeLog',
+        'Requires',
+        '%{_defaultdocdir}/cloud-init/*',
+    ]
+    subs['docs'] = docs
+    configs = [
+        'cloud/cloud.cfg',
+        'cloud/cloud.cfg.d/*.cfg',
+        'cloud/cloud.cfg.d/README',
+        'cloud/templates/*',
+    ]
+    subs['configs'] = configs
+    other_files = [
+        '%{_bindir}/*',
+        '/usr/lib/cloud-init/*',
+    ]
+    subs['files'] = other_files
+    
+    with open(tmpl_fn, 'r') as fh:
+        tmpl = tempita.Template(fh.read())
+        contents = tmpl.substitute(**subs)
+        return (base_name, '%s.tar.gz' % (base_name), contents)
 
 
 def main():
-    if not os.path.isfile(os.path.join(PWD, 'brpm.tmpl')):
+    root_dir = os.path.expanduser("~/rpmbuild")
-        warn("Can not find required template file 'brpm.tmpl'")
+    info("Cleaning %s" % (root_dir))
-        return 1
+    util.delete_dir_contents(root_dir)
-    if not os.path.isfile(os.path.join(os.pardir, 'setup.py')):
+    arc_dir = os.path.join(root_dir, 'SOURCES')
-        warn("Can not find required root 'setup.py' file")
+    util.ensure_dirs([root_dir, arc_dir])
-        return 1
+    tmpl_fn = os.path.join(os.getcwd(), 'brpm.tmpl')
-
+    info("Generated spec file from template %s" % (tmpl_fn))
-    cmd = [os.path.join(os.pardir, 'tools', 'read-version')]
+    (base_name, arc_name, contents) = generate_spec_contents(tmpl_fn)
-    version = subprocess.check_Call(cmd)
+    spec_fn = os.path.join(root_dir, 'cloud-init.spec')
-    
+    util.write_file(spec_fn, contents)
-    
+    info("Wrote spec file to %s" % (spec_fn))
+    with util.tempdir() as td:
+        src_dir = os.path.join(td, base_name)
+        os.makedirs(src_dir)
+        for fn in os.listdir(os.pardir):
+            if fn.startswith("."):
+                continue
+            full_fn = os.path.abspath(os.path.join(os.pardir, fn))
+            if os.path.isfile(full_fn):
+                shutil.copy(full_fn, os.path.join(src_dir, fn))
+            else:
+                shutil.copytree(full_fn, os.path.join(src_dir, fn),
+                                ignore=shutil.ignore_patterns('*.pyc',
+                                                              '.bzr',
+                                                              'tmp*',
+                                                              '*bzr*'))
+            arc_fn = os.path.join(arc_dir, arc_name)
+        cmd = ['tar', '-zcvf', arc_fn, '-C', td]
+        cmd.extend(os.listdir(td))
+        util.subp(cmd)
+        info("Archived code at %s" % (arc_fn))
+    cmd = ['rpmbuild', '-ba', spec_fn]
+    info("Running rpmbuild %s" % (cmd))
+    util.subp(cmd)
+    info("Rpmbuild completed!")
+    globs = [] 
+    globs.extend(glob.glob("%s/*.rpm" %
+                           (os.path.join(root_dir, 'RPMS', 'noarch'))))
+    globs.extend(glob.glob("%s/*.rpm" %
+                           (os.path.join(root_dir, 'RPMS'))))
+    globs.extend(glob.glob("%s/*.rpm" %
+                           (os.path.join(root_dir, 'SRPMS'))))
+    for fn in globs:
+        n = os.path.basename(fn)
+        tgt_fn = os.path.join(os.getcwd(), n)
+        util.copy(fn, tgt_fn)
+        info("Copied %s to %s" % (n, tgt_fn))
     return 0
 
 

packages/brpm.tmpl

@@ -9,28 +9,21 @@ Group:          System Environment/Base
 License:        GPLv3
 URL:            http://launchpad.net/cloud-init
 
-Source0:        %{name}-%{version}-bzr532.tar.gz
+Source0:        {{archive_name}}
 
 BuildArch:      noarch
-BuildRoot:      %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
 
-BuildRequires:  python-devel
+BuildRoot:      %{_tmppath}
-BuildRequires:  python-setuptools-devel
+
-Requires:       e2fsprogs
+
-Requires:       iproute
+{{for r in bd_requires}}
-Requires:       libselinux-python
+BuildRequires: {{r}}
-Requires:       net-tools
+{{endfor}}
-Requires:       procps
+
-Requires:       python-boto
+# Install requirements
-Requires:       python-cheetah
+{{for r in requires}}
-Requires:       python-configobj
+Requires: {{r}}
-Requires:       PyYAML
+{{endfor}}
-Requires:       rsyslog
-Requires:       shadow-utils
-Requires:       /usr/bin/run-parts
-Requires(post):   chkconfig
-Requires(preun):  chkconfig
-Requires(postun): initscripts
 
 %description
 Cloud-init is a set of init scripts for cloud instances.  Cloud instances
@@ -39,14 +32,7 @@ ssh keys and to let the user run various scripts.
 
 
 %prep
-%setup -q -n %{name}-%{version}-bzr532
+%setup -q -n %{name}-%{version}-{{revno}}
-%patch0 -p0
-%patch1 -p0
-%patch2 -p1
-%patch3 -p1
-
-cp -p %{SOURCE2} README.fedora
-
 
 %build
 %{__python} setup.py build
@@ -56,118 +42,29 @@ cp -p %{SOURCE2} README.fedora
 rm -rf $RPM_BUILD_ROOT
 %{__python} setup.py install -O1 --skip-build --root $RPM_BUILD_ROOT
 
-for x in $RPM_BUILD_ROOT/%{_bindir}/*.py; do mv "$x" "${x%.py}"; done
-chmod +x $RPM_BUILD_ROOT/%{python_sitelib}/cloudinit/SshUtil.py
-mkdir -p $RPM_BUILD_ROOT/%{_sharedstatedir}/cloud
-
-# We supply our own config file since our software differs from Ubuntu's.
-cp -p %{SOURCE1} $RPM_BUILD_ROOT/%{_sysconfdir}/cloud/cloud.cfg
-
-# Note that /etc/rsyslog.d didn't exist by default until F15.
-# el6 request: https://bugzilla.redhat.com/show_bug.cgi?id=740420
-mkdir -p $RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d
-cp -p tools/21-cloudinit.conf $RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d/21-cloudinit.conf
-
-# Install the init scripts
-install -p -D -m 755 %{SOURCE3} %{buildroot}%{_initrddir}/cloud-config
-install -p -D -m 755 %{SOURCE4} %{buildroot}%{_initrddir}/cloud-final
-install -p -D -m 755 %{SOURCE5} %{buildroot}%{_initrddir}/cloud-init
-install -p -D -m 755 %{SOURCE6} %{buildroot}%{_initrddir}/cloud-init-local
-
-
 %clean
 rm -rf $RPM_BUILD_ROOT
 
-
-%post
-if [ $1 -eq 1 ] ; then
-    # Initial installation
-    # Enabled by default per "runs once then goes away" exception
-    for svc in config final init init-local; do
-        chkconfig --add cloud-$svc
-        chkconfig cloud-$svc on
-    done
-fi
-
-%preun
-if [ $1 -eq 0 ] ; then
-    # Package removal, not upgrade
-    for svc in config final init init-local; do
-        chkconfig --del cloud-$svc
-        chkconfig cloud-$svc on
-    done
-    # One-shot services -> no need to stop
-fi
-
-%postun
-# One-shot services -> no need to restart
-
-
 %files
-%doc ChangeLog LICENSE TODO README.fedora
+
-%config(noreplace) %{_sysconfdir}/cloud/cloud.cfg
+# Docs
-%dir               %{_sysconfdir}/cloud/cloud.cfg.d
+{{for r in docs}}
-%config(noreplace) %{_sysconfdir}/cloud/cloud.cfg.d/*.cfg
+%doc {{r}}
-%doc               %{_sysconfdir}/cloud/cloud.cfg.d/README
+{{endfor}}
-%dir               %{_sysconfdir}/cloud/templates
+
-%config(noreplace) %{_sysconfdir}/cloud/templates/*
+# Configs
-%{_initrddir}/cloud-*
+{{for r in configs}}
+%config(noreplace) %{_sysconfdir}/{{r}}
+{{endfor}}
+
+# Other files
+{{for r in files}}
+{{r}}
+{{endfor}}
+
+# Python sitelib
 %{python_sitelib}/*
-%{_libexecdir}/%{name}
-%{_bindir}/cloud-init*
-%doc %{_datadir}/doc/%{name}
-%dir %{_sharedstatedir}/cloud
-
-%config(noreplace) %{_sysconfdir}/rsyslog.d/21-cloudinit.conf
-
 
 %changelog
-* Mon Jun 18 2012 Pádraig Brady <P@draigBrady.com> - 0.6.3-0.6.bzr532
-- Further adjustments to support EPEL 6
 
-* Fri Jun 15 2012 Tomas Karasek <tomas.karasek@cern.ch> - 0.6.3-0.5.bzr532
+{{changelog}}
-- Fix cloud-init-cfg invocation in init script
-
-* Tue May 22 2012 Pádraig Brady <P@draigBrady.com> - 0.6.3-0.4.bzr532
-- Support EPEL 6
-
-* Sat Mar 31 2012 Andy Grimm <agrimm@gmail.com> - 0.6.3-0.2.bzr532
-- Fixed incorrect interpretation of relative path for
-  AuthorizedKeysFile (BZ #735521)
-
-* Mon Mar  5 2012 Garrett Holmstrom <gholms@fedoraproject.org> - 0.6.3-0.1.bzr532
-- Rebased against upstream rev 532
-- Fixed runparts() incompatibility with Fedora
-
-* Thu Jan 12 2012 Fedora Release Engineering <rel-eng@lists.fedoraproject.org> - 0.6.2-0.8.bzr457
-- Rebuilt for https://fedoraproject.org/wiki/Fedora_17_Mass_Rebuild
-
-* Wed Oct  5 2011 Garrett Holmstrom <gholms@fedoraproject.org> - 0.6.2-0.7.bzr457
-- Disabled SSH key-deleting on startup
-
-* Wed Sep 28 2011 Garrett Holmstrom <gholms@fedoraproject.org> - 0.6.2-0.6.bzr457
-- Consolidated selinux file context patches
-- Fixed cloud-init.service dependencies
-- Updated sshkeytypes patch
-- Dealt with differences from Ubuntu's sshd
-
-* Sat Sep 24 2011 Garrett Holmstrom <gholms@fedoraproject.org> - 0.6.2-0.5.bzr457
-- Rebased against upstream rev 457
-- Added missing dependencies
-
-* Fri Sep 23 2011 Garrett Holmstrom <gholms@fedoraproject.org> - 0.6.2-0.4.bzr450
-- Added more macros to the spec file
-
-* Fri Sep 23 2011 Garrett Holmstrom <gholms@fedoraproject.org> - 0.6.2-0.3.bzr450
-- Fixed logfile permission checking
-- Fixed SSH key generation
-- Fixed a bad method call in FQDN-guessing [LP:857891]
-- Updated localefile patch
-- Disabled the grub_dpkg module
-- Fixed failures due to empty script dirs [LP:857926]
-
-* Fri Sep 23 2011 Garrett Holmstrom <gholms@fedoraproject.org> - 0.6.2-0.2.bzr450
-- Updated tzsysconfig patch
-
-* Wed Sep 21 2011 Garrett Holmstrom <gholms@fedoraproject.org> - 0.6.2-0.1.bzr450
-- Initial packaging