add extra error handling

tools/setup_software.sh

@@ -21,8 +21,6 @@ chown mysql.mysql /var/log/mysql/slow-queries.log
 chmod ugo+rx /var/log/mysql
 chmod ugo+r /var/log/syslog /var/log/mysql/slow-queries.log /var/log/mysql/error.log
 
-chown -R mysql.mysql /srv/mysql
-
 if [ -e /etc/logrotate.d/percona-server-server-5.5 ]
 then
   rm /etc/logrotate.d/percona-server-server-5.5
@@ -31,3 +29,9 @@ fi
 
 /etc/init.d/apparmor restart
 /etc/init.d/mysql restart
+
+mkdir -p /var/log/turbo-hipster
+chown turbo-hipster:turbo-hipster /var/log/turbo-hipster
+
+mkdir -p /var/lib/turbo-hipster
+chown turbo-hipster:turbo-hipster /var/log/turbo-hipster

turbo_hipster/task_plugins/gate_real_db_upgrade/handle_results.py

@@ -70,7 +70,9 @@ def generate_push_results(datasets, job_unique_number, publish_config):
 
 
 def check_log_for_errors(logfile):
-    """ Run regex over the given logfile to find errors """
+    """ Run regex over the given logfile to find errors
+
+        :returns:   success (boolean), message (string)"""
     MIGRATION_START_RE = re.compile('([0-9]+) -\> ([0-9]+)\.\.\. $')
     MIGRATION_END_RE = re.compile('done$')
     #MIGRATION_COMMAND_START = '***** Start DB upgrade to state of'
@@ -79,11 +81,18 @@ def check_log_for_errors(logfile):
     with open(logfile, 'r') as fd:
         migration_started = False
         for line in fd:
-            if MIGRATION_START_RE.search(line):
+            if 'ERROR 1045' in line:
+                return False, "FAILURE: Could not setup seed database."
+            elif 'ERROR 1049' in line:
+                return False, "FAILURE: Could not find seed database."
+            elif 'ImportError' in line:
+                return False, "FAILURE: Could not import required module."
+            elif MIGRATION_START_RE.search(line):
                 if migration_started:
                     # We didn't see the last one finish,
                     # something must have failed
-                    return False
+                    return False, ("FAILURE: Did not find the end of a "
+                                   "migration after a start")
 
                 migration_started = True
             elif MIGRATION_END_RE.search(line):
@@ -94,6 +103,7 @@ def check_log_for_errors(logfile):
         if migration_started:
             # We never saw the end of a migration,
             # something must have failed
-            return False
+            return False, ("FAILURE: Did not find the end of a migration "
+                            "after a start")
 
-    return True
+    return True, "SUCCESS"

turbo_hipster/task_plugins/gate_real_db_upgrade/task.py

@@ -165,19 +165,17 @@ class Runner(threading.Thread):
 
     def _check_all_dataset_logs_for_errors(self):
         self.log.debug("Check logs for errors")
-        failed = False
+        success = False
         for i, dataset in enumerate(self.job_datasets):
             # Look for the beginning of the migration start
-            result = \
+            success, message = \
                 handle_results.check_log_for_errors(dataset['log_file_path'])
-            self.job_datasets[i]['result'] = 'SUCCESS' if result else 'FAILURE'
-            if not result:
-                failed = True
+            self.job_datasets[i]['result'] = message
 
-        if failed:
-            self.work_data['result'] = "Failed: errors found in dataset log(s)"
-        else:
+        if success:
             self.work_data['result'] = "SUCCESS"
+        else:
+            self.work_data['result'] = "Failed: errors found in dataset log(s)"
 
     def _get_datasets(self):
         self.log.debug("Get configured datasets to run tests against")