From 6ee7a5a116569de8ce06493005a88e95ec3974ec Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Mon, 19 Oct 2020 13:59:37 +1100 Subject: [PATCH] mirror-update/reprepro : use common functions This converts the reprepro mirror script to use the common functions for timestamps and vos release. This function ssh's to the AFS server and runs vos release directly there, avoiding many issues with kerberos timeouts. This has been working successfully for the rsync mirrors. This will also send stats back so we can keep an eye on the timing. Change-Id: I1be29f2d9ecaad03b22c87819e5ae8d16c4f177e --- .../mirror-update/files/fedora-mirror-update | 4 --- .../roles/mirror-update/files/functions.sh | 4 +++ playbooks/roles/mirror-update/tasks/main.yaml | 22 ++++++++++++++++ .../roles/mirror-update/tasks/rsync.yaml | 22 ---------------- .../reprepro/files/reprepro-mirror-update | 25 ++++++++----------- 5 files changed, 36 insertions(+), 41 deletions(-) diff --git a/playbooks/roles/mirror-update/files/fedora-mirror-update b/playbooks/roles/mirror-update/files/fedora-mirror-update index ec84e2f95d..ea689ac098 100755 --- a/playbooks/roles/mirror-update/files/fedora-mirror-update +++ b/playbooks/roles/mirror-update/files/fedora-mirror-update @@ -17,10 +17,6 @@ source /usr/share/mirror-update/functions.sh MIRROR_VOLUME=$1 -function echo_ts { - printf "%(%Y-%m-%d %H:%M:%S)T | %s\n" -1 "$@" -} - if [[ ${NO_TIMEOUT:-0} -eq 1 ]]; then echo_ts "Running interactively" set -x diff --git a/playbooks/roles/mirror-update/files/functions.sh b/playbooks/roles/mirror-update/files/functions.sh index 4213089ee1..2b9d7514d1 100644 --- a/playbooks/roles/mirror-update/files/functions.sh +++ b/playbooks/roles/mirror-update/files/functions.sh @@ -5,6 +5,10 @@ AFS_SERVER=afs01.dfw.openstack.org VOS_RELEASE="ssh -i /root/.ssh/id_vos_release vos_release@${AFS_SERVER} vos release " +function echo_ts { + printf "%(%Y-%m-%d %H:%M:%S)T | %s\n" -1 "$@" +} + # Send a timer stat to statsd # send_timer metric [start_time] # * uses timer metric afs.release.<$1> normalised for stats diff --git a/playbooks/roles/mirror-update/tasks/main.yaml b/playbooks/roles/mirror-update/tasks/main.yaml index 234051b706..c0f743b6cb 100644 --- a/playbooks/roles/mirror-update/tasks/main.yaml +++ b/playbooks/roles/mirror-update/tasks/main.yaml @@ -27,6 +27,28 @@ env: yes value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +- name: Create common function dir + file: + path: /usr/share/mirror-update + state: directory + owner: root + group: root + mode: '0755' + +- name: Copy in common functions + copy: + src: 'functions.sh' + dest: '/usr/share/mirror-update' + owner: root + group: root + mode: '0644' + +- name: Set cron flag to enable reporting stats + cron: + name: UNDER_CRON + env: yes + job: '1' + - name: Setup log publisher script include_tasks: log_publish.yaml diff --git a/playbooks/roles/mirror-update/tasks/rsync.yaml b/playbooks/roles/mirror-update/tasks/rsync.yaml index a91fc6a419..7db8f5a3da 100644 --- a/playbooks/roles/mirror-update/tasks/rsync.yaml +++ b/playbooks/roles/mirror-update/tasks/rsync.yaml @@ -8,22 +8,6 @@ group: root mode: '0755' -- name: Create common function dir - file: - path: /usr/share/mirror-update - state: directory - owner: root - group: root - mode: '0755' - -- name: Copy in common functions - copy: - src: 'functions.sh' - dest: '/usr/share/mirror-update' - owner: root - group: root - mode: '0644' - - name: Set update script names set_fact: rsync_update_scripts: @@ -55,12 +39,6 @@ mode: '0755' loop: '{{ rsync_update_scripts }}' -- name: Set cron flag to enable reporting stats - cron: - name: UNDER_CRON - env: yes - job: '1' - - name: Install update cron jobs cron: name: '{{ item }} mirror sync' diff --git a/playbooks/roles/reprepro/files/reprepro-mirror-update b/playbooks/roles/reprepro/files/reprepro-mirror-update index 31c3ee58ac..38385cb485 100644 --- a/playbooks/roles/reprepro/files/reprepro-mirror-update +++ b/playbooks/roles/reprepro/files/reprepro-mirror-update @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e # Copyright 2016 IBM Corp. # @@ -14,12 +14,12 @@ # License for the specific language governing permissions and limitations # under the License. -set -e +source /usr/share/mirror-update/functions.sh # For initial clones, and debugging, set this for more verbose output # that doesn't time out. if [[ ${NO_TIMEOUT:-0} -eq 1 ]]; then - echo "Running interactively" + echo_ts "Running interactively" TIMEOUT="" set -x else @@ -35,28 +35,23 @@ UNREF_FILE=/var/run/reprepro/${MIRROR_VOLUME}.${CONFIG_KEY}.unreferenced-files K5START="k5start -t -f /etc/reprepro.keytab service/reprepro -- ${TIMEOUT} " REPREPRO="$K5START reprepro --confdir $REPREPRO_CONFIG" -date --iso-8601=ns -echo "Obtaining reprepro tokens and running reprepro update" +echo_ts "Obtaining reprepro tokens and running reprepro update" $REPREPRO update if [ -f $UNREF_FILE ] ; then - date --iso-8601=ns - echo "Cleaning up files made unreferenced on the last run" + echo_ts "Cleaning up files made unreferenced on the last run" $REPREPRO deleteifunreferenced < $UNREF_FILE fi -date --iso-8601=ns -echo "Saving list of newly unreferenced files for next time" +echo_ts "Saving list of newly unreferenced files for next time" k5start -t -f /etc/reprepro.keytab service/reprepro -- bash -c "reprepro --confdir $REPREPRO_CONFIG dumpunreferenced > $UNREF_FILE" -date --iso-8601=ns -echo "Checking state of mirror" +echo_ts "Checking state of mirror" $REPREPRO checkpool fast $REPREPRO check date --iso-8601=ns | $K5START tee $BASE/timestamp.txt -echo "reprepro completed successfully, running vos release." -k5start -t -f /etc/afsadmin.keytab service/afsadmin -- vos release -v $MIRROR_VOLUME +echo_ts "reprepro completed successfully, running vos release." +vos_release $MIRROR_VOLUME -date --iso-8601=ns -echo "Done." +echo_ts "Done."