Better cost metrics.

This review attempts to provide better cost metrics for the upgrade
steps. I do a few things:

 - restart mysql before the upgrades to flush innodb buffers and
   caches
 - run each migration as a separate job
 - log the innodb counters from mysql after each run

This will provide some first order metrics which are better than
wall time -- for example the number of written rows is probably
a more interesting metric than the number of seconds on a loaded
instance with a noisy neighbour.

We also need to do git pulls on stable branches, as zuul doesn't
do that for us.

Change-Id: I39e964a9bbe43717b5a5b5dfd236ee2fa2c62a00

etc/sudoers

@@ -27,6 +27,7 @@ root	ALL=(ALL:ALL) ALL
 
 # Turbo Hipster
 th ALL=(root) NOPASSWD: /sbin/ip netns exec nonet *
+th ALL=(root) NOPASSWD: /usr/sbin/service mysql *
 
 # See sudoers(5) for more information on "#include" directives:
 

turbo_hipster/task_plugins/gate_real_db_upgrade/gerrit-git-prep.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -e
 
-# Stolen from http://git.openstack.org/cgit/openstack-infra/config/plain/modules/jenkins/files/slave_scripts/gerrit-git-prep.sh
+# Stolen from http://git.openstack.org/cgit/openstack-infra/config/plain/modules/jenkins/files/slave_scripts/gerrit-git-prep.sh but has been hacked to be AWESOME!
 
 GERRIT_SITE=$1
 ZUUL_SITE=$2
@@ -49,7 +49,7 @@ then
     fi
 fi
 git checkout master
-git branch -D working || true
+git pull
 git remote set-url origin $GIT_ORIGIN/$ZUUL_PROJECT
 
 # attempt to work around bugs 925790 and 1229352
@@ -83,6 +83,7 @@ else
         git clean -x -f -d -q
     fi
 fi
+git branch -D working || true
 git checkout -b working
 
 if [ -f .gitmodules ]

turbo_hipster/task_plugins/gate_real_db_upgrade/nova_mysql_migrations.sh

@@ -49,7 +49,7 @@ db_sync() {
   # $5 is the nova db password
   # $6 is the nova db name
   # $7 is the logging.conf for openstack
-  # $8 is any sync options
+  # $8 is an (optional) destination version number
 
   # Create a nova.conf file
   cat - > $2/nova-$1.conf <<EOF
@@ -58,24 +58,66 @@ sql_connection = mysql://$4:$5@172.16.0.1/$6?charset=utf8
 log_config = $7
 EOF
 
-  find $3 -type f -name "*.pyc" -exec rm -f {} \;
+  # Silently return git to a known good state (delete untracked files)
+  git clean -xfdq
+
   echo "***** Start DB upgrade to state of $1 *****"
+  echo "HEAD of branch under test is:"
+  git log -n 1
+
   echo "Setting up the nova-manage entry point"
   python setup.py -q clean
   python setup.py -q develop
   python setup.py -q install
-  set -x
-  sudo /sbin/ip netns exec nonet `dirname $0`/nova-manage-wrapper $VENV_PATH --config-file $2/nova-$1.conf --verbose db sync $8
-  manage_exit=$?
-  set +x
 
-  echo "nova-manage returned exit code $manage_exit"
+  # Log the migrations present
-  if [ $manage_exit -gt 0 ]
+  echo "Migrations present:"
+  ls $3/nova/db/sqlalchemy/migrate_repo/versions/*.py | sed 's/.*\///' | egrep "^[0-9]+_"
+
+  # Flush innodb's caches
+  echo "Restarting mysql"
+  sudo service mysql stop
+  sudo service mysql start
+
+  echo "MySQL counters before upgrade:"
+  mysql -u $4 --password=$5 $6 -e "show status like 'innodb%';"
+
+  start_version=`mysql -u $4 --password=$5 $6 -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
+
+  if [ "%$8%" == "%%" ]
   then
-    echo "Aborting early"
+    end_version=`ls $3/nova/db/sqlalchemy/migrate_repo/versions/*.py | sed 's/.*\///' | egrep "^[0-9]+_" | tail -1 | cut -f 1 -d "_"`
-    exit $manage_exit
+  else
+    end_version=$8
   fi
 
+  echo "Test will migrate from $start_version to $end_version"
+  if [ $end_version -lt $start_version ]
+  then
+    increment=-1
+    end_version=$(( $end_version + 1 ))
+  else
+    increment=1
+    start_version=$(( $start_version + 1))
+  fi
+
+  for i in `seq $start_version $increment $end_version`
+  do
+    set -x
+    sudo /sbin/ip netns exec nonet `dirname $0`/nova-manage-wrapper $VENV_PATH --config-file $2/nova-$1.conf --verbose db sync --version $i
+    manage_exit=$?
+    set +x
+
+    echo "MySQL counters after upgrade:"
+    mysql -u $4 --password=$5 $6 -e "show status like 'innodb%';"
+
+    echo "nova-manage returned exit code $manage_exit"
+    if [ $manage_exit -gt 0 ]
+    then
+      echo "Aborting early"
+      exit $manage_exit
+    fi
+  done
 
   echo "***** Finished DB upgrade to state of $1 *****"
 }
@@ -95,7 +137,10 @@ stable_release_db_sync() {
   if [ $version == "133" ] # I think this should be [ $version lt "133" ]
   then
     echo "Database is from Folsom! Upgrade via Grizzly"
-    git checkout stable/grizzly
+    git branch -D stable/grizzly || true
+    git remote update
+    git checkout -b stable/grizzly
+    git reset --hard remotes/origin/stable/grizzly
     pip_requires
     db_sync "grizzly" $1 $2 $3 $4 $5 $6
   fi
@@ -106,7 +151,10 @@ stable_release_db_sync() {
   if [ $version == "161" ] # I think this should be [ $version lt "161" ]
   then
     echo "Database is from Grizzly! Upgrade via Havana"
-    git checkout stable/havana
+    git branch -D stable/havana || true
+    git remote update
+    git checkout -b stable/havana
+    git reset --hard remotes/origin/stable/havana
     pip_requires
     db_sync "havana" $1 $2 $3 $4 $5 $6
   fi
@@ -149,10 +197,6 @@ then
   exit 1
 fi
 
-# zuul puts us in a headless mode, lets check it out into a working branch
-git branch -D working 2> /dev/null
-git checkout -b working
-
 stable_release_db_sync $2 $3 $4 $5 $6 $8
 
 last_stable_version=`mysql -u $4 --password=$5 $6 -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
@@ -179,16 +223,14 @@ db_sync "patchset" $2 $3 $4 $5 $6 $8
 version=`mysql -u $4 --password=$5 $6 -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
 echo "Schema version is $version"
 
-#target_version=`ls $3/nova/db/sqlalchemy/migrate_repo/versions | head -1 | cut -f 1 -d "_"`
 echo "Now downgrade all the way back to the last stable version (v$last_stable_version)"
-db_sync "patchset" $2 $3 $4 $5 $6 $8 "--version $last_stable_version"
+db_sync "downgrade" $2 $3 $4 $5 $6 $8 $last_stable_version
 
 # Determine the schema version
 version=`mysql -u $4 --password=$5 $6 -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
 echo "Schema version is $version"
 
 echo "And now back up to head from the start of trunk"
-git checkout working  # I think this line is redundant
 db_sync "patchset" $2 $3 $4 $5 $6 $8
 
 # Determine the final schema version

turbo_hipster/task_plugins/gate_real_db_upgrade/task.py

@@ -13,6 +13,7 @@
 # under the License.
 
 
+import copy
 import json
 import logging
 import os
@@ -306,12 +307,15 @@ class Runner(object):
         if not os.path.exists(local_path):
             os.makedirs(local_path)
 
+        git_args = copy.deepcopy(job_args)
+        git_args['GIT_ORIGIN'] = 'git://git.openstack.org/'
+
         cmd = os.path.join(os.path.join(os.path.dirname(__file__),
                                         'gerrit-git-prep.sh'))
         cmd += ' https://review.openstack.org'
         cmd += ' http://zuul.rcbops.com'
-        utils.execute_to_log(cmd, job_log_file_path,
+        utils.execute_to_log(cmd, job_log_file_path, env=git_args,
-                             env=job_args, cwd=local_path)
+                             cwd=local_path)
         return local_path
 
     def _get_work_data(self):