openinfra/refstack
Code Issues Proposed changes

add token checking and creating into result update script

Browse Source 
Change-Id: I5dd8798bc778e9777658d5d0cff7bc156f9570f0
This commit is contained in:
Megan Guiney 2017-09-12 11:27:39 -07:00 committed by Megan
parent 45f00cc13f
commit c10ee24a96
2 changed files with 128 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

193
tools/update-rs-db.py
View File

@ -19,26 +19,59 @@
Test result update & verify script for the local refstack database. Test result update & verify script for the local refstack database.
""" """
import os
import argparse import argparse
import datetime import datetime
import requests
from collections import namedtuple
import json import json
import jwt
import os
import requests
import sys
from collections import namedtuple
def generate_token(keyfile, _id):
# get private key in string format
with open(keyfile) as pemfile:
secret = pemfile.read().rstrip()
exp = datetime.timedelta(seconds=100500)
payload =\
{
"user_openid": _id,
"exp": datetime.datetime.now() + exp
}
token = jwt.encode(payload, secret, algorithm="RS256")
with open("./token", "w+") as tokenfile:
tokenfile.write(str(token))
print("token stored in the current working directory.")
def testAuth(link, tokenfile):
auth_test_url = link.split("/v1")[0] + "/#/profile"
with open(tokenfile) as tokenfile:
token = tokenfile.read().strip()
headers = {"Authorization": "Bearer " + token}
response = requests.get(auth_test_url, headers)
if response.status_code == 200:
return True, token
else:
print("token auth failed. status code = %d" % (response.status_code))
print("auth failure detail: %s" % (response.text))
return False, None
def getData(entry): def getData(entry):
"""Extract and reformat data from a product data csv""" guidelines = ["2015.03", "2015.04", "2015.05", "2015.07", "2016.01",
guidelines = ['2015.03', '2015.04', '2015.05', '2015.07', '2016.01', "2016.08", "2017.01", "2017.09"]
'2016.08', '2017.01', '2017.09'] # NOTE: Storage is an alias of Object
components = ['platform', 'compute', 'storage', 'object'] components = ["platform", "compute", "storage", "object"]
if len(entry) < 10: if len(entry) < 10:
return None, None, None return None, None, None
refstackLink = entry[9].strip() refstackLink = entry[9].strip()
guideline = entry[4].strip() guideline = entry[4].strip()
target = entry[5].lower().strip() target = entry[5].lower().strip()
if refstackLink: if refstackLink:
testId = refstackLink.split('/')[-1] testId = refstackLink.split("/")[-1]
else: else:
refstackLink = None refstackLink = None
testId = None testId = None
@ -53,27 +86,28 @@ def getData(entry):
def linkChk(link, token): def linkChk(link, token):
"""Check existence of and access to api result link""" """Check existence of and access to api result link"""
print("now checking result: " + link) print("checking result with a test ID of: %s" % (link.split("/")[-1]))
if not link: if not link:
return False return False
try: try:
if " " in link: if " " in link:
return False return False
headers = {'Authorization': 'Bearer ' + token} headers = {"Authorization": "Bearer " + token}
response = requests.get(link, headers) response = requests.get(link, headers)
if response.status_code == 200: if response.status_code == 200:
return json.loads(response.text) return json.loads(response.text)
elif response.status_code == 401 or response.status_code == 403: elif response.status_code == 401 or response.status_code == 403:
print("Authentication Failed. link check response code: " + print("Authentication Failed. link check response code: %d" %
str(response.status_code)) (response.status_code))
return False return False
elif response.status_code == 400: elif response.status_code == 400:
print("Malformed Request. link response code: " + print("Malformed Request. link response code: %d" %
str(response.status_code)) (response.status_code))
return False return False
else: else:
print("Link check response_status_code=" + print("Link check response_status_code = %d" %
str(response.status_code)) (response.status_code))
print("Link check response detail: %s" % (response.text))
return False return False
except requests.exceptions as err: except requests.exceptions as err:
print(err) print(err)
@ -82,95 +116,96 @@ def linkChk(link, token):
def updateField(header, apiLink, raw_data): def updateField(header, apiLink, raw_data):
"""Update a given metadata field""" """Update a given metadata field"""
valid_keytype = ['shared', 'guideline', 'target'] valid_keytype = ["shared", "guideline", "target"]
keytype = raw_data.type keytype = raw_data.type
keyval = raw_data.value keyval = raw_data.value
if keytype not in valid_keytype or not keyval: if keytype not in valid_keytype or not keyval:
updresult = "%s keypair does not exist" % (keytype) updresult = "%s keypair does not exist" % (keytype)
return updresult, False return updresult, False
link = apiLink.strip() + '/meta/' + keytype link = apiLink.strip() + "/meta/" + keytype
response = requests.post(link, data=keyval, headers=header) response = requests.post(link, data=keyval, headers=header)
if response.status_code != 201: if response.status_code != 201:
print('update response status code=%d' % print("update response status code=%d" %
response.status_code) (response.status_code))
print('update response text=' + response.text) print("update response text=%s" % (response.text))
updresult = ("%s field update failed. reason: %s" % updresult = ("%s field update failed. reason: %s" %
(keytype, response.text.replace(',', ' '))) (keytype, response.text.replace(",", " ")))
return updresult, False return updresult, False
else: else:
updresult = "%s field update successful," % (keytype) updresult = ("%s field update successful," % (keytype))
return updresult, True return updresult, True
def updateResult(apiLink, target, guideline, token, record): def updateResult(apiLink, target, guideline, token, record):
"""Update metadata for result and verify if all updates are a success""" """Update metadata for result and verify if all updates are a success"""
MetadataField = namedtuple('MetadataField', ['type', 'value']) MetadataField = namedtuple("MetadataField", ["type", "value"])
success = [] success = []
header = {'Authorization': 'Bearer ' + token} header = {"Authorization": "Bearer " + token}
with open(record, 'a') as r: with open(record, "a") as r:
r.write(str(datetime.datetime.now()) + "," + apiLink + ",") r.write(str(datetime.datetime.now()) + "," + apiLink + ",")
# update the shared field # update the shared field
data = MetadataField('shared', 'true') data = MetadataField("shared", "true")
shared_result, shared_status = updateField(header, apiLink, data) shared_result, shared_status = updateField(header, apiLink, data)
r.write(shared_result) r.write(shared_result)
success.append(shared_status) success.append(shared_status)
# update the target field # update the target field
data = MetadataField('target', target) data = MetadataField("target", target)
target_result, target_status = updateField(header, apiLink, data) target_result, target_status = updateField(header, apiLink, data)
r.write(target_result) r.write(target_result)
success.append(target_status) success.append(target_status)
# update the guideline field # update the guideline field
data = MetadataField('guideline', guideline + '.json') data = MetadataField("guideline", guideline + ".json")
gl_result, gl_status = updateField(header, apiLink, data) gl_result, gl_status = updateField(header, apiLink, data)
r.write(gl_result) r.write(gl_result)
success.append(gl_status) success.append(gl_status)
if not all(success): if not all(success):
r.write('unable to verify.\n') r.write("unable to verify.\n")
return False return False
# if there were no update failures, we can verify the result # if there were no update failures, we can verify the result
# this is the operation most likely to fail, so extra checks are # this is the operation most likely to fail, so extra checks are
# in order # in order
print('Test Result updated successfully. Attempting verification.') print("Test Result updated successfully. Attempting verification.")
try: try:
response = requests.put(apiLink, response = requests.put(apiLink,
json={'verification_status': 1}, json={"verification_status": 1},
headers=header) headers=header)
except Exception as ex: except Exception as ex:
print('Exception raised while verifying test result: %s' % print("Exception raised while verifying test result: %s" %
(str(ex))) (str(ex)))
r.write('verification failed: %s\n' % (str(ex))) r.write("verification failed: %s\n" % (str(ex)))
return False return False
updated = verification_chk(apiLink, header) updated = verification_chk(apiLink, header)
if response.status_code not in (200, 201): if response.status_code not in (200, 201):
print('verification failure status code=%d' % print("verification failure status code=%d" %
response.status_code) (response.status_code))
print('verification failure detail=%s' % print("verification failure detail=%s" %
response.text) (response.text))
r.write('verification unsuccessful: detail: %s\n' % r.write("verification unsuccessful: detail: %s\n" %
(response.text)) (response.text))
return False return False
elif not updated: elif not updated:
print("verification_status field failed to update") print("verification_status field failed to update")
r.write('verification status update failed. detail: %s\n' % r.write("verification status update failed. detail: %s\n" %
(response.text)) (response.text))
return False return False
else: else:
print('Test result verified!\n') print("Test result verified!\n")
r.write('Test result successfully verified\n') r.write("Test result successfully verified\n")
return True return True
def verification_chk(link, header): def verification_chk(link, header):
try: try:
response = requests.get(link, header) response = requests.get(link, header)
status = int(response.json()['verification_status']) status = int(response.json()["verification_status"])
if status == 1: if status == 1:
return True return True
else: else:
return False return False
except Exception as ex: except Exception as ex:
print('Exception raised while ensuring update of ' + print(
'verification status: ' + str(ex)) "Exception raised while ensuring verification status update: %s" %
str(ex))
return False return False
@ -178,51 +213,75 @@ def main():
linect = 0 linect = 0
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
"Update the internal RefStack db using a csv file") "Update the internal RefStack db using a csv file")
parser.add_argument("--file", "-f", metavar='f', type=str, action="store", # token handling options- we either need the path of a working token,
# or the data to generate a new token
token_flags = parser.add_mutually_exclusive_group(required=True)
token_flags.add_argument("--tokenfile", type=str, action="store",
help=("Absolute path to a json web token to "
"use to auth to the RefStack API"))
token_flags.add_argument("--generate", nargs=2,
metavar=("ssh-key", "openstack-id"),
help=("data needed to create a new auth token "
"ssh - key should be an absolute path to "
"a rsa ssh key. openstack - id indicates "
"an openstackid url to use for auth. "
"example: "
"https://openstackid.org/<your id >"))
# non token-related flags
parser.add_argument("--file", "-f", metavar="f", type=str, action="store",
required=True, required=True,
help="csv source for the data to use in updates") help="csv source for the data to use in updates")
parser.add_argument( parser.add_argument(
"--endpoint", "-e", metavar='e', "--endpoint", "-e", metavar="e",
type=str, action="store", required=True, type=str, action="store", required=True,
help="the base URL of the endpoint. ex: http://examplerefstack.com/v1") help="the base URL of the endpoint. ex: http://examplerefstack.com/v1")
parser.add_argument("--token", "-t", metavar="t", type=str,
action="store", required=True, help="API auth token")
parser.add_argument("--record", "-r", metavar="r", type=str, parser.add_argument("--record", "-r", metavar="r", type=str,
action="store", default="verification_results.csv", action="store", default="verification_results.csv",
help="name of file to output update & verification " + help=("name of file to output update & verification "
" run record data into") "run record data into"))
result = parser.parse_args() args = parser.parse_args()
infile = result.file infile = args.file
record = result.record record = args.record
endpoint = result.endpoint endpoint = args.endpoint
token = result.token if args.generate:
keypath = args.generate[0]
_id = args.generate[1]
generate_token(keypath, _id)
tokenfile = "./token"
else:
tokenfile = args.tokenfile
auth_success, token = testAuth(endpoint, tokenfile)
if not auth_success:
print(("Please enter either a valid token or an openstackid and the "
"absolute path to an rsa ssh key."))
sys.exit(1)
with open(infile) as f: with open(infile) as f:
for line in f: for line in f:
linect = linect + 1 linect = linect + 1
entry = line.split(",") entry = line.split(",")
testId, guideline, target = getData(entry) testId, guideline, target = getData(entry)
if None in (testId, guideline, target): if None in (testId, guideline, target):
print( print(("entry found at line %d cannot be updated and "
"entry found at line " + str(linect) + "verified: entry incomplete.\n") % (linect))
" cannot be updated and verified: entry incomplete.\n")
else: else:
apiLink = os.path.join(endpoint, 'results', testId) apiLink = os.path.join(endpoint, "results", testId)
testResult = linkChk(apiLink, token) testResult = linkChk(apiLink, token)
if testResult: if testResult:
if testResult.get('verification_status'): if testResult.get("verification_status"):
print("Result has been verified.\n") print("Result has been verified.\n")
else: else:
print( print(
"Result link is valid. Updating...") "Result link is valid. Updating result with ID %s"
% (testId))
success = updateResult(apiLink, target, guideline, success = updateResult(apiLink, target, guideline,
token, record) token, record)
if not success: if not success:
print("update of the results with the ID " + print(("update of the results with the ID %s "
testId + " failed. please recheck your " + "failed. please recheck your spreadsheet "
"spreadsheet and try again\n") "and try again" % (testId)))
else: else:
print("the test result " + testId + " cannot be " + print(("the test result: % s cannot be updated or "
"verified due to a link verification failure\n") "verified due to a broken result link." % (testId)))
main() main()

4
tools/update-rs-db.rst
View File

@ -7,7 +7,7 @@ successful in the usage of the script update-rs-db.py.
The script can be run using the following formatting: The script can be run using the following formatting:
"./update-rs-db.py --file /tmp/datasource.csv --endpoint "./update-rs-db.py --file /tmp/datasource.csv --endpoint
http://example.com:8000/v1 --token <my-token>". In order to http://example.com:8000/v1 --tokenfile <token file path>". In order to
successfully update and verify results, you will need admin rights successfully update and verify results, you will need admin rights
for the refstack server in question. Instructions on how to get for the refstack server in question. Instructions on how to get
these for your local install can be found at https://github.com/openstack/refstack/blob/master/doc/source/refstack.rst#optional-configure-foundation-organization-and-group these for your local install can be found at https://github.com/openstack/refstack/blob/master/doc/source/refstack.rst#optional-configure-foundation-organization-and-group
@ -64,7 +64,7 @@ Because editing arbitrary test results requires administrative privileges,
an auth token must be used with the RefStack API. This token can be an auth token must be used with the RefStack API. This token can be
generated by entering the command "jwt --key="$( cat <path to private key> generated by entering the command "jwt --key="$( cat <path to private key>
)" --alg=RS256 user_openid=<openstackid> exp=+100500". This generates a )" --alg=RS256 user_openid=<openstackid> exp=+100500". This generates a
json web token, which we must link using the "-t" or "--token" flag. Because json web token, which we must link using the "--tokenfile" flag. Because
we cannot auth without this token, the token is a required flag. we cannot auth without this token, the token is a required flag.
The script will go through each line of the CSV, grabbing the refstack link, The script will go through each line of the CSV, grabbing the refstack link,