From 3c7bcc0c7a7e58da5efa7cbd59cde3f56fe143fd Mon Sep 17 00:00:00 2001
From: Phil Bridges <pgbridge@us.ibm.com>
Date: Wed, 8 Jun 2016 15:53:51 -0500
Subject: [PATCH] Fix Gerrit gates and other build issues
There are a few problems with the configuration in our source tree that
causes the Jenkins gates to always fail, and things that might cause
a problem later. This patch set will fix those and other problems, to bring
us into a known valid state for future commits.
Change-Id: Idf7a0ce5902c40985caa78390b01f9fc2dfbfcf4
---
.gitreview | 14 +++---
bin/swiftonhpss-nstool | 63 ++++++++++--------------
setup.py | 4 +-
swiftonhpss/swift/common/hpssfs_ioctl.py | 40 +++++++++++++++
swiftonhpss/swift/common/utils.py | 1 -
swiftonhpss/swift/obj/diskfile.py | 32 +++++++-----
swiftonhpss/swift/obj/server.py | 50 +++++++++----------
tox.ini | 10 ++--
8 files changed, 123 insertions(+), 91 deletions(-)
create mode 100644 swiftonhpss/swift/common/hpssfs_ioctl.py
diff --git a/.gitreview b/.gitreview
index 6e4fd6e..d89c6df 100644
--- a/.gitreview
+++ b/.gitreview
@@ -1,8 +1,6 @@
-# TODO: get ourselves a nice and shiny CI system like this
-
-#[gerrit]
-#host=review.openstack.org
-#port=29418
-#project=openstack/swiftonfile.git
-#defaultbranch=master
-#defaultremote=gerrit
+[gerrit]
+host=review.openstack.org
+port=29418
+project=openstack/swiftonhpss.git
+defaultbranch=master
+defaultremote=gerrit
diff --git a/bin/swiftonhpss-nstool b/bin/swiftonhpss-nstool
index c6525a5..3996f69 100755
--- a/bin/swiftonhpss-nstool
+++ b/bin/swiftonhpss-nstool
@@ -18,7 +18,6 @@
import sys
import stat
import os
-import multiprocessing
from pwd import getpwuid
import logging
import argparse
@@ -87,11 +86,11 @@ def main(program_args):
del password
# Figure out what we're doing.
- target_account, target_container = program_args.account,\
- program_args.container
+ target_account = program_args.account
+ target_container = program_args.container
# Start doing it.
- #pool = multiprocessing.Pool(processes=multiprocessing.cpu_count()-1)
+ # pool = multiprocessing.Pool(processes=multiprocessing.cpu_count()-1)
# Multiprocessing does not play nicely with the lazy loading that
# keystoneclient does, so let's not mess with it for now.
@@ -227,7 +226,7 @@ def check_usage():
return parser.parse_args()
-class Reconciler:
+class Reconciler(object):
def __init__(self, args):
self._args = args
@@ -436,7 +435,8 @@ class Reconciler:
logging.exception("Putting container %s went wrong" %
target_container)
raise e
- print "Reconciling container %s/%s" % (target_account, target_container)
+ print "Reconciling container %s/%s" % \
+ (target_account, target_container)
# Make sure those objects get added into the Swift metadata DBs
self.add_objects_from_hpss(swift_api, target_container, container_dir)
@@ -458,7 +458,8 @@ class Reconciler:
return objects
@trace_function
- def add_objects_from_hpss(self, swift_api, target_container, container_dir):
+ def add_objects_from_hpss(self, swift_api, target_container,
+ container_dir):
"""
Update object metadata on object creates, and returns a list of all the
objects existing in the container from Swift.
@@ -512,7 +513,8 @@ class Reconciler:
try:
hpss_containers = os.listdir(account_directory)
except OSError as err:
- print "Unable to list files under directory: %s" % account_directory
+ print "Unable to list files under directory: %s" % \
+ account_directory
raise err
# Delete containers that only exist in Swift, but not HPSS
@@ -552,8 +554,8 @@ class Reconciler:
known_good_objects = []
swift_only_objects = list(set(swift_objects) - set(hpss_objects))
- # If we have objects that only exist in the Swift metadata, delete those
- # objects.
+ # If we have objects that only exist in the Swift metadata,
+ # delete those objects.
for target_obj in swift_only_objects:
try:
swift_api.delete_object(target_container, target_obj)
@@ -614,7 +616,8 @@ class Reconciler:
if file_user not in keystone_users:
fail_reason = \
"Cannot configure proper permissions for this path %s\
- because user %s does not exist in keystone" % (path, file_user)
+ because user %s does not exist in keystone" % \
+ (path, file_user)
print fail_reason
logging.error(fail_reason)
raise IOError(fail_reason)
@@ -639,7 +642,7 @@ class Reconciler:
# This only exists because the keystoneclient library is so massive that it has
# to have a lazy-loading mechanism that ensures only one of it can be active,
# so we can't have handles to multiple different Keystone scopes simultaneously
-class LightweightKeystoneAPI:
+class LightweightKeystoneAPI(object):
MEMBER_ROLE_ID = '9fe2ff9ee4384b1894a90878d3e92bab'
@@ -677,33 +680,19 @@ class LightweightKeystoneAPI:
if self.version == 'v2':
url = '%s/tokens' % self.url
+ creds = {'username': self.username, 'password': self.password}
token_req = {'auth': {'tenantName': self.tenant_name,
- 'passwordCredentials': {
- 'username': self.username,
- 'password': self.password
- }}}
+ 'passwordCredentials': creds}}
else:
url = '%s/auth/tokens' % self.url
- token_req = {'auth': {'identity':
- {'methods': ['password'],
- 'password': {
- 'user': {
- 'name': self.username,
- 'password': self.password,
- 'domain': {'id': 'default'}
- }
- }
- },
- 'scope': {
- 'project': {
- 'name': self.tenant_name,
- 'domain': {
- 'id': 'default'
- }
- }
- }
- }
- }
+ domain = {'id': 'default'}
+ creds = {'user': {'name': self.username,
+ 'password': self.password,
+ 'domain': domain}}
+ scope = {'project': {'name': self.tenant_name, 'domain': domain}}
+ token_req = {'auth': {'identity': {'methods': ['password'],
+ 'password': creds},
+ 'scope': scope}}
try:
resp_headers, resp_json =\
self._get_keystone_response(requests.post,
@@ -852,4 +841,4 @@ if __name__ == "__main__":
log_level = logging.ERROR
logging.basicConfig(filename=_args.logfile,
level=log_level)
- main(_args)
\ No newline at end of file
+ main(_args)
diff --git a/setup.py b/setup.py
index ec139f9..0e9f132 100644
--- a/setup.py
+++ b/setup.py
@@ -66,7 +66,7 @@ if 'install' in sys.argv:
# Install man pages the crappy hacky way, because setuptools doesn't
# have any facility to do it.
man_path = '/usr/local/share/man/1'
- man_pages = filter(lambda x: os.path.isfile('./doc/troff/'+x),
+ man_pages = filter(lambda x: os.path.isfile('./doc/troff/%s' % x),
os.listdir('./doc/troff'))
for page in man_pages:
- shutil.copyfile('./doc/troff/'+page, man_path)
+ shutil.copyfile('./doc/troff/%s' % page, man_path)
diff --git a/swiftonhpss/swift/common/hpssfs_ioctl.py b/swiftonhpss/swift/common/hpssfs_ioctl.py
new file mode 100644
index 0000000..fbdeb1e
--- /dev/null
+++ b/swiftonhpss/swift/common/hpssfs_ioctl.py
@@ -0,0 +1,40 @@
+# Copyright (c) 2016 IBM Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import array
+import fcntl
+
+HPSSFS_GET_COS = 0x80046c01
+HPSSFS_SET_COS_HINT = 0x40046c02
+
+HPSSFS_SET_FSIZE_HINT = 0x40086c03
+HPSSFS_SET_MAXSEGSZ_HINT = 0x40046c04
+
+HPSSFS_PURGE_CACHE = 0x00006c05
+HPSSFS_PURGE_LOCK = 0x40046c06
+
+HPSSFS_UNDELETE = 0x40046c07
+HPSSFS_UNDELETE_NONE = 0x00000000
+HPSSFS_UNDELETE_RESTORE_TIME = 0x00000001
+HPSSFS_UNDELETE_OVERWRITE = 0x00000002
+HPSSFS_UNDELETE_OVERWRITE_AND_RESTORE = 0x00000003
+
+
+def ioctl(fd, cmd, val=None):
+ if val is not None:
+ valbuf = array.array("i", val)
+ fcntl.ioctl(fd, cmd, valbuf)
+ else:
+ fcntl.ioctl(fd, cmd)
diff --git a/swiftonhpss/swift/common/utils.py b/swiftonhpss/swift/common/utils.py
index 94660ec..236026b 100644
--- a/swiftonhpss/swift/common/utils.py
+++ b/swiftonhpss/swift/common/utils.py
@@ -24,7 +24,6 @@ from eventlet import sleep
import cPickle as pickle
from cStringIO import StringIO
import pickletools
-import xattr
from swiftonhpss.swift.common.exceptions import SwiftOnFileSystemIOError
from swift.common.exceptions import DiskFileNoSpace
from swift.common.db import utf8encodekeys
diff --git a/swiftonhpss/swift/obj/diskfile.py b/swiftonhpss/swift/obj/diskfile.py
index ff72c7a..76031bf 100644
--- a/swiftonhpss/swift/obj/diskfile.py
+++ b/swiftonhpss/swift/obj/diskfile.py
@@ -23,7 +23,10 @@ except ImportError:
import random
import logging
import time
-import hpssfs
+try:
+ import hpssfs
+except ImportError:
+ import swiftonhpss.swift.common.hpssfs_ioctl as hpssfs
import xattr
from uuid import uuid4
from hashlib import md5
@@ -55,7 +58,7 @@ from swift.obj.diskfile import get_async_dir
# FIXME: Hopefully we'll be able to move to Python 2.7+ where O_CLOEXEC will
# be back ported. See http://www.python.org/dev/peps/pep-0433/
-O_CLOEXEC = 0o20000000
+O_CLOEXEC = 0o02000000
MAX_RENAME_ATTEMPTS = 10
MAX_OPEN_ATTEMPTS = 10
@@ -313,11 +316,12 @@ class DiskFileWriter(object):
# (HPSS) Purge lock the file now if we're asked to.
if purgelock:
try:
- hpssfs.ioctl(self._fd, hpssfs.HPSSFS_PURGE_LOCK, int(purgelock))
+ hpssfs.ioctl(self._fd, hpssfs.HPSSFS_PURGE_LOCK,
+ int(purgelock))
except IOError as err:
- raise SwiftOnFileSystemIOError(err.errno,
- '%s, hpssfs.ioctl("%s", ...)' % (
- err.strerror, self._fd))
+ raise SwiftOnFileSystemIOError(
+ err.errno,
+ '%s, hpssfs.ioctl("%s", ...)' % (err.strerror, self._fd))
# From the Department of the Redundancy Department, make sure
# we call drop_cache() after fsync() to avoid redundant work
@@ -811,7 +815,7 @@ class DiskFile(object):
raise SwiftOnFileSystemIOError(
err.errno,
'%s, xattr.getxattr("system.hpss.level", ...)' % err.strerror
- )
+ )
try:
file_levels = raw_file_levels.split(";")
top_level = file_levels[0].split(':')
@@ -1051,17 +1055,19 @@ class DiskFile(object):
hpssfs.ioctl(fd, hpssfs.HPSSFS_SET_FSIZE_HINT,
long(size))
except IOError as err:
- raise SwiftOnFileSystemIOError(err.errno,
- '%s, hpssfs.ioctl("%s", SET_FSIZE)' % (
- err.strerror, fd))
+ message = '%s, hpssfs.ioctl("%s", SET_FSIZE)'
+ raise SwiftOnFileSystemIOError(
+ err.errno,
+ message % (err.strerror, fd))
if cos:
try:
hpssfs.ioctl(fd, hpssfs.HPSSFS_SET_COS_HINT, int(cos))
except IOError as err:
- raise SwiftOnFileSystemIOError(err.errno,
- '%s, hpssfs.ioctl("%s", SET_COS)' % (
- err.strerror, fd))
+ message = '%s, hpssfs.ioctl("%s", SET_COS)'
+ raise SwiftOnFileSystemIOError(
+ err.errno,
+ message % (err.strerror, fd))
except SwiftOnFileSystemOSError as gerr:
if gerr.errno in (errno.ENOSPC, errno.EDQUOT):
diff --git a/swiftonhpss/swift/obj/server.py b/swiftonhpss/swift/obj/server.py
index 1c73754..6e860c9 100644
--- a/swiftonhpss/swift/obj/server.py
+++ b/swiftonhpss/swift/obj/server.py
@@ -19,7 +19,10 @@ import math
import logging
import xattr
import os
-import hpssfs
+try:
+ import hpssfs
+except ImportError:
+ import swiftonhpss.swift.common.hpssfs_ioctl as hpssfs
import time
import eventlet
@@ -29,7 +32,7 @@ from swift.common.swob import HTTPConflict, HTTPBadRequest, HeaderKeyDict, \
HTTPInsufficientStorage, HTTPPreconditionFailed, HTTPRequestTimeout, \
HTTPClientDisconnect, HTTPUnprocessableEntity, HTTPNotImplemented, \
HTTPServiceUnavailable, HTTPCreated, HTTPNotFound, HTTPAccepted, \
- HTTPNoContent, Request, Response
+ HTTPNoContent, Response
from swift.common.utils import public, timing_stats, replication, \
config_true_value, Timestamp, csv_append
from swift.common.request_helpers import get_name_and_placement, \
@@ -40,7 +43,7 @@ from swiftonhpss.swift.common.exceptions import AlreadyExistsAsFile, \
from swift.common.exceptions import DiskFileDeviceUnavailable, \
DiskFileNotExist, DiskFileQuarantined, ChunkReadTimeout, DiskFileNoSpace, \
DiskFileXattrNotSupported, DiskFileExpired, DiskFileDeleted
-from swift.common.constraints import valid_timestamp, check_account_format
+from swift.common.constraints import valid_timestamp
from swift.obj import server
from swift.common.ring import Ring
@@ -64,8 +67,9 @@ class SwiftOnFileDiskFileRouter(object):
class ObjectController(server.ObjectController):
"""
- Subclass of the object server's ObjectController that supports HPSS-specific
- metadata headers and operations (such as COS assignment and purge locking).
+ Subclass of the object server's ObjectController that supports
+ HPSS-specific metadata headers and operations (such as COS assignment
+ and purge locking).
"""
def setup(self, conf):
@@ -91,7 +95,6 @@ class ObjectController(server.ObjectController):
self.container_ring = Ring(self.swift_dir, ring_name='container')
return self.container_ring
-
@public
@timing_stats()
def PUT(self, request):
@@ -195,8 +198,9 @@ class ObjectController(server.ObjectController):
return HTTPUnprocessableEntity(request=request)
# Update object metadata
+ content_type = request.headers['content-type']
metadata = {'X-Timestamp': request.timestamp.internal,
- 'Content-Type': request.headers['content-type'],
+ 'Content-Type': content_type,
'ETag': etag,
'Content-Length': str(upload_size),
}
@@ -206,7 +210,8 @@ class ObjectController(server.ObjectController):
metadata.update(meta_headers)
backend_headers = \
request.headers.get('X-Backend-Replication-Headers')
- for header_key in (backend_headers or self.allowed_headers):
+ for header_key in (backend_headers or
+ self.allowed_headers):
if header_key in request.headers:
header_caps = header_key.title()
metadata[header_caps] = request.headers[header_key]
@@ -259,16 +264,12 @@ class ObjectController(server.ObjectController):
self.delete_at_update('DELETE', orig_delete_at, account,
container, obj, request, device,
policy)
+ container_headers = {'x-size': metadata['Content-Length'],
+ 'x-content-type': metadata['Content-Type'],
+ 'x-timestamp': metadata['X-Timestamp'],
+ 'x-etag': metadata['ETag']}
self.container_update('PUT', account, container, obj, request,
- HeaderKeyDict(
- {'x-size':
- metadata['Content-Length'],
- 'x-content-type':
- metadata['Content-Type'],
- 'x-timestamp':
- metadata['X-Timestamp'],
- 'x-etag':
- metadata['ETag']}),
+ HeaderKeyDict(container_headers),
device, policy)
# Create convenience symlink
try:
@@ -346,8 +347,8 @@ class ObjectController(server.ObjectController):
# Get DiskFile
try:
- disk_file = self.get_diskfile(device, partition, account, container,
- obj, policy=policy)
+ disk_file = self.get_diskfile(device, partition, account,
+ container, obj, policy=policy)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
@@ -417,8 +418,8 @@ class ObjectController(server.ObjectController):
# Get Diskfile
try:
- disk_file = self.get_diskfile(device, partition, account, container,
- obj, policy)
+ disk_file = self.get_diskfile(device, partition, account,
+ container, obj, policy)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
@@ -446,7 +447,7 @@ class ObjectController(server.ObjectController):
)
for key, value in metadata.iteritems():
if is_sys_or_user_meta('object', key) or \
- key.lower() in self.allowed_headers:
+ key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
response.last_modified = math.ceil(float(file_x_ts))
@@ -524,10 +525,9 @@ class ObjectController(server.ObjectController):
return HTTPNotFound(request=request)
orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0))
if orig_timestamp >= req_timestamp:
+ backend_headers = {'X-Backend-Timestamp': orig_timestamp.internal}
return HTTPConflict(request=request,
- headers={
- 'X-Backend-Timestamp': orig_timestamp.internal
- })
+ headers=backend_headers)
metadata = {'X-Timestamp': req_timestamp.internal}
metadata.update(val for val in request.headers.iteritems()
if is_user_meta('object', val[0]))
diff --git a/tox.ini b/tox.ini
index 2deccdf..2afc14e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,6 @@
[tox]
-envlist = py27,pep8,functest
+#envlist = py27,pep8,functest
+envlist = py27,pep8
minversion = 1.6
skipsdist = True
@@ -10,7 +11,7 @@ whitelist_externals=bash
setenv = VIRTUAL_ENV={envdir}
NOSE_WITH_COVERAGE=1
NOSE_COVER_BRANCHES=1
- NOSE_COVER_PACKAGE=swiftonfile
+ NOSE_COVER_PACKAGE=swiftonhpss
deps =
# Note: pip supports installing from git repos.
# https://pip.pypa.io/en/latest/reference/pip_install.html#git
@@ -35,9 +36,8 @@ commands = bash ./.functests -q
[testenv:pep8]
changedir = {toxinidir}
-commands =
-flake8 swiftonhpss test setup.py
-flake8 --filename=swiftonhpss* bin
+commands = flake8 swiftonhpss test setup.py
+ flake8 --filename=swiftonhpss* bin
[testenv:venv]
changedir = {toxinidir}