From ca2ee69e60accf0292b9e3fdb4605b51be646525 Mon Sep 17 00:00:00 2001
From: Ian Wienand <iwienand@redhat.com>
Date: Mon, 30 Mar 2020 12:08:52 +1100
Subject: [PATCH] local-log-download : role with script to download all log
 files

This is an alternative to I98c80f657f38c5e1ed5f28e5d36988a3429ad1f8
which does not modify the upload script, but rather queries the API
and manifest for what to download.

The script is a hybrid of python and bash to not implement json API
parsing badly in bash, but not replicate curl badly in python either.
The script sanity checks for dependencies, which are considered pretty
standard for any developer who would be interested in downloading logs
like this.

The role writes out the script with the correct build values coded
into it, so a potential user just has to run it without any arguments
or modification.

Change-Id: Ic33732adbfd3210191bf4976c3ee316cfc50568e
---
 doc/source/log-roles.rst                      |   1 +
 roles/local-log-download/README.rst           |  11 ++
 roles/local-log-download/tasks/main.yaml      |  12 ++
 .../templates/download-logs.sh.j2             | 105 ++++++++++++++++++
 test-playbooks/local-log-download.yaml        |  21 ++++
 zuul-tests.d/logs-jobs.yaml                   |  18 +++
 6 files changed, 168 insertions(+)
 create mode 100644 roles/local-log-download/README.rst
 create mode 100644 roles/local-log-download/tasks/main.yaml
 create mode 100644 roles/local-log-download/templates/download-logs.sh.j2
 create mode 100644 test-playbooks/local-log-download.yaml
 create mode 100644 zuul-tests.d/logs-jobs.yaml

diff --git a/doc/source/log-roles.rst b/doc/source/log-roles.rst
index 1ebcd3e4d..6653b869f 100644
--- a/doc/source/log-roles.rst
+++ b/doc/source/log-roles.rst
@@ -8,6 +8,7 @@ Log Roles
 .. zuul:autorole:: fetch-output-openshift
 .. zuul:autorole:: generate-zuul-manifest
 .. zuul:autorole:: htmlify-logs
+.. zuul:autorole:: local-log-download
 .. zuul:autorole:: merge-output-to-logs
 .. zuul:autorole:: publish-artifacts-to-fileserver
 .. zuul:autorole:: set-zuul-log-path-fact
diff --git a/roles/local-log-download/README.rst b/roles/local-log-download/README.rst
new file mode 100644
index 000000000..7448fd3d4
--- /dev/null
+++ b/roles/local-log-download/README.rst
@@ -0,0 +1,11 @@
+Add a script for users to bulk download logs locally
+
+This adds a script for users to bulk download all logs to their local
+system.  It queries the Zuul API for the manifest and then copies all
+files locally from the log server.
+
+**Role Variables**
+
+.. zuul:rolevar:: local_log_download_api
+
+   The Zuul API endpoint to use.  Example: ``https://zuul.example.org/api/tenant/{{ zuul.tenant }}``
diff --git a/roles/local-log-download/tasks/main.yaml b/roles/local-log-download/tasks/main.yaml
new file mode 100644
index 000000000..0c3cb94b6
--- /dev/null
+++ b/roles/local-log-download/tasks/main.yaml
@@ -0,0 +1,12 @@
+- name: Check API endpoint is defined
+  assert:
+    that:
+      - local_log_download_api is defined
+    msg: 'local_log_download_api must be defined'
+
+- name: Create download script
+  delegate_to: localhost
+  template:
+    dest: '{{ zuul.executor.log_root }}/download-logs.sh'
+    src: 'download-logs.sh.j2'
+    mode: 0755
diff --git a/roles/local-log-download/templates/download-logs.sh.j2 b/roles/local-log-download/templates/download-logs.sh.j2
new file mode 100644
index 000000000..7047e1290
--- /dev/null
+++ b/roles/local-log-download/templates/download-logs.sh.j2
@@ -0,0 +1,105 @@
+#!/bin/bash
+
+set -e
+
+ZUUL_API=${ZUUL_API:-"{{ local_log_download_api }}"}
+ZUUL_BUILD_UUID=${ZUUL_BUILD_UUID:-"{{ zuul.build }}"}
+
+ZUUL_API_URL=${ZUUL_API}/build/${ZUUL_BUILD_UUID}
+
+(( ${BASH_VERSION%%.*} >= 4 )) || { echo >&2 "bash >=4 required to download."; exit 1; }
+command -v python3 >/dev/null 2>&1 || { echo >&2 "Python3 is required to download."; exit 1; }
+command -v curl >/dev/null 2>&1 || { echo >&2 "curl is required to download."; exit 1; }
+
+function log {
+    echo "$(date -Iseconds) | $@"
+}
+
+{#
+ # Parse the zuul build results to find the manifest, then parse
+ # the manifest and print files that shoud be downloaded.  The
+ # first line of output is the base url, then every line after is a
+ # file to download.
+#}
+function get_urls {
+    /usr/bin/env python3 - <<EOF
+import gzip
+import json
+import urllib.request
+
+base_url = urllib.request.urlopen("${ZUUL_API_URL}").read()
+base_json = json.loads(base_url)
+manifest_url = [x['url'] for x in base_json['artifacts'] if x.get('metadata', {}).get('type') == 'zuul_manifest'][0]
+manifest = urllib.request.urlopen(manifest_url)
+if manifest.info().get('Content-Encoding') == 'gzip':
+    manifest_json = json.loads(gzip.decompress(manifest.read()))
+else:
+    manifest_json = json.loads(manifest.read())
+
+def p(node, parent):
+    if node.get('mimetype') != 'application/directory':
+        print(parent+node['name'])
+    if node.get('children'):
+        for child in node.get('children'):
+                p(child, parent+node['name']+'/')
+
+print(base_json['log_url'])
+for i in manifest_json['tree']:
+    p(i, '')
+
+EOF
+}
+
+function save_file {
+    local base_url="$1"
+    local file="$2"
+
+    curl -s --compressed --create-dirs -o "${file}" "${base_url}/${file}"
+   {#
+    # Using --compressed we will send an Accept-Encoding: gzip header
+    # and the data will come to us across the network compressed.
+    # However, we see weird things like .gz files (as stored on disk)
+    # actually uncompressed, so we check if this really looks like an
+    # ASCII file and rename for clarity.
+   #}
+    if [[ "${file}" == *.gz ]]; then
+        local type=$(file "${file}")
+        if [[ "${type}" =~ "ASCII text" ]] || [[ "${type}" =~ "Unicode text" ]]; then
+            local new_name=${file%.gz}
+            log "Renaming to ${new_name}"
+            mv "${file}" "${new_name}"
+        fi
+    fi
+
+}
+
+{#
+ # - read in file _files so we exit if the lookup fails for some reason
+ # - jinja gets confused with ${#files[@]} as it looks like a comment,
+ #   wrap it in raw
+#}
+log "Querying ${ZUUL_API_URL} for manifest"
+_files="$(get_urls)"
+readarray -t files <<< "${_files}"
+{% raw %}
+len="${#files[@]}"
+{% endraw %}
+if [[ -z "${DOWNLOAD_DIR}" ]]; then
+    DOWNLOAD_DIR=$(mktemp -d --tmpdir zuul-logs.XXXXXX)
+fi
+log "Saving logs to ${DOWNLOAD_DIR}"
+
+pushd "${DOWNLOAD_DIR}" > /dev/null
+
+base_url="${files[0]}"
+log "Getting logs from ${base_url}"
+for (( i=1; i<$len; i++ )); do
+    file="${files[i]}"
+    printf -v _out "  %-80s [ %04d/%04d ]" "${file}" "${i}" $(( len -1 ))
+    log "$_out"
+    save_file $base_url $file
+done
+
+popd >/dev/null
+
+log "Download complete!"
diff --git a/test-playbooks/local-log-download.yaml b/test-playbooks/local-log-download.yaml
new file mode 100644
index 000000000..2bf84e7cc
--- /dev/null
+++ b/test-playbooks/local-log-download.yaml
@@ -0,0 +1,21 @@
+- hosts: all
+  tasks:
+
+    - name: Run local-log-download role
+      include_role:
+        name: local-log-download
+      vars:
+        local_log_download_api: 'https://zuul.opendev.org/api/tenant/{{ zuul.tenant }}'
+
+  post_tasks:
+    - name: Check for download script
+      delegate_to: localhost
+      file:
+        path: "{{ zuul.executor.log_root }}/download-logs.sh"
+        state: file
+      register: download_script
+
+    - name: Validate download script
+      assert:
+        that:
+          - download_script is not changed
diff --git a/zuul-tests.d/logs-jobs.yaml b/zuul-tests.d/logs-jobs.yaml
new file mode 100644
index 000000000..0160c8b78
--- /dev/null
+++ b/zuul-tests.d/logs-jobs.yaml
@@ -0,0 +1,18 @@
+- job:
+    name: zuul-jobs-test-local-log-download
+    description: Test the local-log-download role
+    files:
+      - roles/local-log-download/.*
+    run: test-playbooks/local-log-download.yaml
+
+# -* AUTOGENERATED *-
+#  The following project section is autogenerated by
+#    tox -e update-test-platforms
+#  Please re-run to generate new job lists
+
+- project:
+    check:
+      jobs: &id001
+        - zuul-jobs-test-local-log-download
+    gate:
+      jobs: *id001