Merge "Move to a new way to fetch git repos."

turbo_hipster/lib/utils.py

@@ -93,7 +93,7 @@ def execute_to_log(cmd, logfile, timeout=-1,
                        ('[sqlslo]', '/var/log/mysql/slow-queries.log'),
                        ('[sqlerr]', '/var/log/mysql/error.log')
                    ],
-                   heartbeat=True
+                   heartbeat=True, env=None, cwd=None
                    ):
     """ Executes a command and logs the STDOUT/STDERR and output of any
     supplied watch_logs from logs into a new logfile
@@ -130,7 +130,8 @@ def execute_to_log(cmd, logfile, timeout=-1,
     cmd += ' 2>&1'
     start_time = time.time()
     p = subprocess.Popen(
-        cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+        env=env, cwd=cwd)
 
     descriptors[p.stdout.fileno()] = dict(
         name='[output]',

turbo_hipster/task_plugins/gate_real_db_upgrade/gerrit-git-prep.sh

@@ -0,0 +1,93 @@
+#!/bin/bash -e
+
+# Stolen from http://git.openstack.org/cgit/openstack-infra/config/plain/modules/jenkins/files/slave_scripts/gerrit-git-prep.sh
+
+GERRIT_SITE=$1
+ZUUL_SITE=$2
+GIT_ORIGIN=$3
+
+if [ -z "$GERRIT_SITE" ]
+then
+  echo "The gerrit site name (eg 'https://review.openstack.org') must be the first argument."
+  exit 1
+fi
+
+if [ -z "$ZUUL_SITE" ]
+then
+  echo "The zuul site name (eg 'http://zuul.openstack.org') must be the second argument."
+  exit 1
+fi
+
+if [ -z "$GIT_ORIGIN" ] || [ -n "$ZUUL_NEWREV" ]
+then
+    GIT_ORIGIN="$GERRIT_SITE/p"
+    # git://git.openstack.org/
+    # https://review.openstack.org/p
+fi
+
+if [ -z "$ZUUL_REF" ]
+then
+    echo "This job may only be triggered by Zuul."
+    exit 1
+fi
+
+if [ ! -z "$ZUUL_CHANGE" ]
+then
+    echo "Triggered by: $GERRIT_SITE/$ZUUL_CHANGE"
+fi
+
+set -x
+if [[ ! -e .git ]]
+then
+    ls -a
+    rm -fr .[^.]* *
+    if [ -d /opt/git/$ZUUL_PROJECT/.git ]
+    then
+        git clone file:///opt/git/$ZUUL_PROJECT .
+    else
+        git clone $GIT_ORIGIN/$ZUUL_PROJECT .
+    fi
+fi
+git checkout master
+git branch -D working || true
+git remote set-url origin $GIT_ORIGIN/$ZUUL_PROJECT
+
+# attempt to work around bugs 925790 and 1229352
+if ! git remote update
+then
+    echo "The remote update failed, so garbage collecting before trying again."
+    git gc
+    git remote update
+fi
+
+git reset --hard
+if ! git clean -x -f -d -q ; then
+    sleep 1
+    git clean -x -f -d -q
+fi
+
+if [ -z "$ZUUL_NEWREV" ]
+then
+    git fetch $ZUUL_SITE/p/$ZUUL_PROJECT $ZUUL_REF
+    git checkout FETCH_HEAD
+    git reset --hard FETCH_HEAD
+    if ! git clean -x -f -d -q ; then
+        sleep 1
+        git clean -x -f -d -q
+    fi
+else
+    git checkout $ZUUL_NEWREV
+    git reset --hard $ZUUL_NEWREV
+    if ! git clean -x -f -d -q ; then
+        sleep 1
+        git clean -x -f -d -q
+    fi
+fi
+git checkout -b working
+
+if [ -f .gitmodules ]
+then
+    git submodule init
+    git submodule sync
+    git submodule update --init
+fi`

turbo_hipster/task_plugins/gate_real_db_upgrade/task.py

@@ -85,9 +85,8 @@ class Runner(object):
                 # Step 2: Checkout updates from git!
                 self._do_next_step()
                 self.git_path = self._grab_patchset(
-                    self.job_arguments['ZUUL_PROJECT'],
-                    self.job_arguments['ZUUL_CHANGES'].split(':')[-1]
-                )
+                    self.job_arguments,
+                    self.job_datasets[0]['job_log_file_path'])
 
                 # Step 3: Run migrations on datasets
                 self._do_next_step()
@@ -298,28 +297,22 @@ class Runner(object):
             )
             return rc
 
-    def _grab_patchset(self, project_name, zuul_ref):
+    def _grab_patchset(self, job_args, job_log_file_path):
         """ Checkout the reference into config['git_working_dir'] """
 
         self.log.debug("Grab the patchset we want to test against")
+        local_path = os.path.join(self.global_config['git_working_dir'],
+                                  self.job_name, job_args['ZUUL_PROJECT'])
+        if not os.path.exists(local_path):
+            os.makedirs(local_path)
 
-        repo = utils.GitRepository(
-            'https://review.openstack.org/' + project_name,
-            os.path.join(
-                self.global_config['git_working_dir'],
-                self.job_name,
-                project_name
-            )
-        )
-
-        # reset to git's master
-        repo.reset()
-
-        # Fetch patchset and checkout
-        repo.fetch(zuul_ref)
-        repo.checkout('FETCH_HEAD')
-
-        return repo.local_path
+        cmd = os.path.join(os.path.join(os.path.dirname(__file__),
+                                        'gerrit-git-prep.sh'))
+        cmd += ' https://review.openstack.org'
+        cmd += ' http://zuul.rcbops.com'
+        utils.execute_to_log(cmd, job_log_file_path,
+                             env=job_args, cwd=local_path)
+        return local_path
 
     def _get_work_data(self):
         if self.work_data is None: