etree: add support for xml validation

This patch adds support for XML validation with RelaxNG.  The original S3
schema is available at http://doc.s3.amazonaws.com/2006-03-01/AmazonS3.xsd, but
I write a schema for swift3 from scratch.  It is because:

 - The original schema does not support all of the latest S3 API.  Even if we
   use it, we have to update and maintain it in either way.

 - The original schema is written with XML Schema, but the language is not
   enough to describe some of the latest S3 API.  For example, the Multi-Object
   Delete operation sends an XML document, which can interleave a Quiet element
   with Object elements.  XML Schema cannot define such kinds of XML documents.

This patch includes both RelaxNG (rng) and RelaxNG compact syntax files (rnc)
for validation.  What I wrote are only rnc files, and rng files are
automatically generated from rnc with trang, the most well-known schema
converter.  If possible, I'd like to use only rnc schemas since rng files are
complicated and not so human-readable.  However, lxml doesn't support RelaxNG
compact syntax currently, unfortunately.  After lxml supports compact syntax,
let's remove all of the rng files.

This patch also fixes some XML errors detected by RelaxNG validator, and some
helper methods to convert strings between camel-case name and snake-case name.

Change-Id: I4513345a6b981efc5d5f5e8cf528aba84ac1bdad

doc/rnc/access_control_policy.rnc

@@ -0,0 +1,7 @@
+include "common.rnc"
+
+start =
+  element AccessControlPolicy {
+    element Owner { CanonicalUser } &
+    element AccessControlList { AccessControlList }
+  }

doc/rnc/bucket_logging_status.rnc

@@ -0,0 +1,10 @@
+include "common.rnc"
+
+start =
+  element BucketLoggingStatus {
+    element LoggingEnabled {
+      element TargetBucket { xsd:string } &
+      element TargetPrefix { xsd:string } &
+      element TargetGrants { AccessControlList }?
+    }?
+  }

doc/rnc/common.rnc

@@ -0,0 +1,26 @@
+namespace xsi = "http://www.w3.org/2001/XMLSchema-instance"
+
+CanonicalUser =
+  element ID { xsd:string } &
+  element DisplayName { xsd:string }?
+
+StorageClass = "STANDARD" | "REDUCED_REDUNDANCY" | "GLACIER" | "UNKNOWN"
+
+AccessControlList =
+  element Grant {
+    element Grantee {
+      (
+        attribute xsi:type { "AmazonCustomerByEmail" },
+        element EmailAddress { xsd:string }
+      ) | (
+        attribute xsi:type { "CanonicalUser" },
+        CanonicalUser
+      ) | (
+        attribute xsi:type { "Group" },
+        element URI { xsd:string }
+      )
+    } &
+    element Permission {
+      "READ" | "WRITE" | "READ_ACP" | "WRITE_ACP" | "FULL_CONTROL"
+    }
+  }*

doc/rnc/complete_multipart_upload.rnc

@@ -0,0 +1,7 @@
+start =
+  element CompleteMultipartUpload {
+    element Part {
+      element PartNumber { xsd:int } &
+      element ETag { xsd:string }
+    }+
+  }

doc/rnc/complete_multipart_upload_result.rnc

@@ -0,0 +1,7 @@
+start =
+  element CompleteMultipartUploadResult {
+    element Location { xsd:anyURI },
+    element Bucket { xsd:string },
+    element Key { xsd:string },
+    element ETag { xsd:string }
+  }

doc/rnc/copy_object_result.rnc

@@ -0,0 +1,5 @@
+start =
+  element CopyObjectResult {
+    element LastModified { xsd:dateTime },
+    element ETag { xsd:string }
+  }

doc/rnc/create_bucket_configuration.rnc

@@ -0,0 +1,4 @@
+start =
+  element CreateBucketConfiguration {
+    element LocationConstraint { xsd:string }
+  }

doc/rnc/delete.rnc

@@ -0,0 +1,8 @@
+start =
+  element Delete {
+    element Quiet { xsd:boolean }? &
+    element Object {
+      element Key { xsd:string } &
+      element VersionId { xsd:string }?
+    }+
+  }

doc/rnc/delete_result.rnc

@@ -0,0 +1,17 @@
+start =
+  element DeleteResult {
+    (
+      element Deleted {
+        element Key { xsd:string },
+        element VersionId { xsd:string }?,
+        element DeleteMarker { xsd:boolean }?,
+        element DeleteMarkerVersionId { xsd:string }?
+      } |
+      element Error {
+        element Key { xsd:string },
+        element VersionId { xsd:string }?,
+        element Code { xsd:string },
+        element Message { xsd:string }
+      }
+    )*
+  }

doc/rnc/error.rnc

@@ -0,0 +1,11 @@
+start =
+  element Error {
+    element Code { xsd:string },
+    element Message { xsd:string },
+    DebugInfo*
+  }
+
+DebugInfo =
+  element * {
+    (attribute * { text } | text | DebugInfo)*
+  }

doc/rnc/initiate_multipart_upload_result.rnc

@@ -0,0 +1,6 @@
+start =
+  element InitiateMultipartUploadResult {
+    element Bucket { xsd:string },
+    element Key { xsd:string },
+    element UploadId { xsd:string }
+  }

doc/rnc/lifecycle_configuration.rnc

@@ -0,0 +1,20 @@
+include "common.rnc"
+
+start =
+  element LifecycleConfiguration {
+    element Rule {
+      element ID { xsd:string }? &
+      element Prefix { xsd:string } &
+      element Status { "Enabled" | "Disabled" } &
+      element Transition { Transition }? &
+      element Expiration { Expiration }?
+    }+
+  }
+
+Expiration =
+  element Days { xsd:int } |
+  element Date { xsd:dateTime }
+
+Transition =
+  Expiration &
+  element StorageClass { StorageClass }

doc/rnc/list_all_my_buckets_result.rnc

@@ -0,0 +1,12 @@
+include "common.rnc"
+
+start =
+  element ListAllMyBucketsResult {
+    element Owner { CanonicalUser },
+    element Buckets {
+      element Bucket {
+        element Name { xsd:string },
+        element CreationDate { xsd:dateTime }
+      }*
+    }
+  }

doc/rnc/list_bucket_result.rnc

@@ -0,0 +1,24 @@
+include "common.rnc"
+
+start =
+  element ListBucketResult {
+    element Name { xsd:string },
+    element Prefix { xsd:string },
+    element Marker { xsd:string },
+    element NextMarker { xsd:string }?,
+    element MaxKeys { xsd:int },
+    element EncodingType { xsd:string }?,
+    element Delimiter { xsd:string }?,
+    element IsTruncated { xsd:boolean },
+    element Contents {
+      element Key { xsd:string },
+      element LastModified { xsd:dateTime },
+      element ETag { xsd:string },
+      element Size { xsd:long },
+      element Owner { CanonicalUser }?,
+      element StorageClass { StorageClass }
+    }*,
+    element CommonPrefixes {
+      element Prefix { xsd:string }
+    }*
+  }

doc/rnc/list_multipart_uploads_result.rnc

@@ -0,0 +1,26 @@
+include "common.rnc"
+
+start =
+  element ListMultipartUploadsResult {
+    element Bucket { xsd:string },
+    element KeyMarker { xsd:string },
+    element UploadIdMarker { xsd:string },
+    element NextKeyMarker { xsd:string },
+    element NextUploadIdMarker { xsd:string },
+    element Delimiter { xsd:string }?,
+    element Prefix { xsd:string }?,
+    element MaxUploads { xsd:int },
+    element EncodingType { xsd:string }?,
+    element IsTruncated { xsd:boolean },
+    element Upload {
+      element Key { xsd:string },
+      element UploadId { xsd:string },
+      element Initiator { CanonicalUser },
+      element Owner { CanonicalUser },
+      element StorageClass { StorageClass },
+      element Initiated { xsd:dateTime }
+    }*,
+    element CommonPrefixes {
+      element Prefix { xsd:string }
+    }*
+  }

doc/rnc/list_parts_result.rnc

@@ -0,0 +1,22 @@
+include "common.rnc"
+
+start =
+    element ListPartsResult {
+    element Bucket { xsd:string },
+    element Key { xsd:string },
+    element UploadId { xsd:string },
+    element Initiator { CanonicalUser },
+    element Owner { CanonicalUser },
+    element StorageClass { StorageClass },
+    element PartNumberMarker { xsd:int },
+    element NextPartNumberMarker { xsd:int },
+    element MaxParts { xsd:int },
+    element EncodingType { xsd:string }?,
+    element IsTruncated { xsd:boolean },
+    element Part {
+      element PartNumber { xsd:int },
+      element LastModified { xsd:dateTime },
+      element ETag { xsd:string },
+      element Size { xsd:long }
+    }*
+  }

doc/rnc/list_versions_result.rnc

@@ -0,0 +1,37 @@
+include "common.rnc"
+
+start =
+  element ListVersionsResult {
+    element Name { xsd:string },
+    element Prefix { xsd:string },
+    element KeyMarker { xsd:string },
+    element VersionIdMarker { xsd:string },
+    element NextKeyMarker { xsd:string }?,
+    element NextVersionIdMarker { xsd:string }?,
+    element MaxKeys { xsd:int },
+    element EncodingType { xsd:string }?,
+    element Delimiter { xsd:string }?,
+    element IsTruncated { xsd:boolean },
+    (
+      element Version {
+        element Key { xsd:string },
+        element VersionId { xsd:string },
+        element IsLatest { xsd:boolean },
+        element LastModified { xsd:dateTime },
+        element ETag { xsd:string },
+        element Size { xsd:long },
+        element Owner { CanonicalUser }?,
+        element StorageClass { StorageClass }
+      } |
+      element DeleteMarker {
+        element Key { xsd:string },
+        element VersionId { xsd:string },
+        element IsLatest { xsd:boolean },
+        element LastModified { xsd:dateTime },
+        element Owner { CanonicalUser }?
+      }
+    )*,
+    element CommonPrefixes {
+      element Prefix { xsd:string }
+    }*
+  }

doc/rnc/location_constraint.rnc

@@ -0,0 +1 @@
+start = element LocationConstraint { xsd:string }

doc/rnc/versioning_configuration.rnc

@@ -0,0 +1,5 @@
+start =
+  element VersioningConfiguration {
+    element Status { "Enabled" | "Suspended" }? &
+    element MfaDelete { "Enabled" | "Disabled" }?
+  }

swift3/controllers/acl.py

@@ -18,7 +18,8 @@ from swift.common.middleware.acl import parse_acl, referrer_allowed
 
 from swift3.controllers.base import Controller
 from swift3.response import HTTPOk, S3NotImplemented, MalformedACLError
-from swift3.etree import Element, SubElement, fromstring, tostring
+from swift3.etree import Element, SubElement, fromstring, tostring, \
+    DocumentInvalid
 
 XMLNS_XSI = 'http://www.w3.org/2001/XMLSchema-instance'
 
@@ -97,7 +98,10 @@ def swift_acl_translate(acl, group='', user='', xml=False):
                             ['HTTP_X_CONTAINER_READ', '.']]
     if xml:
         # We are working with XML and need to parse it
-        elem = fromstring(acl)
+        try:
+            elem = fromstring(acl, 'AccessControlPolicy')
+        except DocumentInvalid:
+            raise MalformedACLError()
         acl = 'unknown'
         for grant in elem.findall('./AccessControlList/Grant'):
             permission = grant.find('./Permission').text

swift3/controllers/bucket.py

@@ -62,16 +62,16 @@ class BucketController(Controller):
         objects = loads(resp.body)
 
         elem = Element('ListBucketResult')
+        SubElement(elem, 'Name').text = req.container_name
         SubElement(elem, 'Prefix').text = req.params.get('prefix')
         SubElement(elem, 'Marker').text = req.params.get('marker')
+        SubElement(elem, 'MaxKeys').text = str(max_keys)
         SubElement(elem, 'Delimiter').text = req.params.get('delimiter')
         if max_keys > 0 and len(objects) == max_keys + 1:
             is_truncated = 'true'
         else:
             is_truncated = 'false'
         SubElement(elem, 'IsTruncated').text = is_truncated
-        SubElement(elem, 'MaxKeys').text = str(max_keys)
-        SubElement(elem, 'Name').text = req.container_name
 
         for o in objects[:max_keys]:
             if 'subdir' not in o:
@@ -82,6 +82,7 @@ class BucketController(Controller):
                 SubElement(contents, 'ETag').text = o['hash']
                 SubElement(contents, 'Size').text = str(o['bytes'])
                 add_canonical_user(contents, 'Owner', req.user_id)
+                SubElement(contents, 'StorageClass').text = 'STANDARD'
 
         for o in objects[:max_keys]:
             if 'subdir' in o:

swift3/controllers/multi_delete.py

@@ -14,8 +14,10 @@
 # limitations under the License.
 
 from swift3.controllers.base import Controller
-from swift3.etree import Element, SubElement, fromstring, tostring
-from swift3.response import HTTPOk, S3NotImplemented, NoSuchKey, ErrorResponse
+from swift3.etree import Element, SubElement, fromstring, tostring, \
+    DocumentInvalid
+from swift3.response import HTTPOk, S3NotImplemented, NoSuchKey, \
+    ErrorResponse, MalformedXML
 
 
 class MultiObjectDeleteController(Controller):
@@ -28,7 +30,11 @@ class MultiObjectDeleteController(Controller):
         Handles Delete Multiple Objects.
         """
         def object_key_iter(xml):
-            elem = fromstring(xml)
+            try:
+                elem = fromstring(xml, 'Delete')
+            except DocumentInvalid:
+                raise MalformedXML()
+
             for obj in elem.iterchildren('Object'):
                 key = obj.find('./Key').text
                 version = obj.find('./VersionId')

swift3/controllers/service.py

@@ -34,6 +34,11 @@ class ServiceController(Controller):
         # we don't keep the creation time of a backet (s3cmd doesn't
         # work without that) so we use something bogus.
         elem = Element('ListAllMyBucketsResult')
+
+        owner = SubElement(elem, 'Owner')
+        SubElement(owner, 'ID').text = req.user_id
+        SubElement(owner, 'DisplayName').text = req.user_id
+
         buckets = SubElement(elem, 'Buckets')
         for c in containers:
             bucket = SubElement(buckets, 'Bucket')

swift3/etree.py

@@ -15,9 +15,21 @@
 
 import lxml.etree
 from copy import deepcopy
+from pkg_resources import resource_stream
+
+from swift.common.utils import get_logger
+
+from swift3.exception import S3Exception
+from swift3.utils import camel_to_snake
+from swift3.cfg import CONF
 
 XMLNS_S3 = 'http://s3.amazonaws.com/doc/2006-03-01/'
 
+LOGGER = get_logger(CONF, log_route='swift3')
+
+class DocumentInvalid(S3Exception, lxml.etree.DocumentInvalid):
+    pass
+
 
 def cleanup_namespaces(elem):
     def remove_ns(tag, ns):
@@ -36,10 +48,24 @@ def cleanup_namespaces(elem):
         cleanup_namespaces(e)
 
 
-def fromstring(text):
+def fromstring(text, root_tag=None):
     elem = lxml.etree.fromstring(text)
     cleanup_namespaces(elem)
 
+    if root_tag is not None:
+        # validate XML
+        try:
+            path = 'schema/%s.rng' % camel_to_snake(root_tag)
+            rng = resource_stream(__name__, path)
+            lxml.etree.RelaxNG(file=rng).assertValid(elem)
+        except IOError as e:
+            # Probably, the schema file doesn't exist.
+            LOGGER.error(e)
+            raise
+        except lxml.etree.DocumentInvalid as e:
+            LOGGER.debug(e)
+            raise DocumentInvalid(e)
+
     return elem
 
 

swift3/response.py

@@ -19,6 +19,7 @@ from functools import partial
 
 from swift.common import swob
 
+from swift3.utils import snake_to_camel
 from swift3.etree import Element, SubElement, tostring
 
 
@@ -157,8 +158,7 @@ class ErrorResponse(swob.HTTPException):
 
     def _dict_to_etree(self, parent, d):
         for key, value in d.items():
-            tag = key.title().replace('_', '')
-            tag = re.sub('\W', '', tag)
+            tag = re.sub('\W', '', snake_to_camel(key))
             elem = SubElement(parent, tag)
 
             if isinstance(value, (dict, DictMixin)):

swift3/schema/access_control_policy.rng

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0">
+  <include href="common.rng"/>
+  <start>
+    <element name="AccessControlPolicy">
+      <interleave>
+        <element name="Owner">
+          <ref name="CanonicalUser"/>
+        </element>
+        <element name="AccessControlList">
+          <ref name="AccessControlList"/>
+        </element>
+      </interleave>
+    </element>
+  </start>
+</grammar>

swift3/schema/bucket_logging_status.rng

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <include href="common.rng"/>
+  <start>
+    <element name="BucketLoggingStatus">
+      <optional>
+        <element name="LoggingEnabled">
+          <interleave>
+            <element name="TargetBucket">
+              <data type="string"/>
+            </element>
+            <element name="TargetPrefix">
+              <data type="string"/>
+            </element>
+            <optional>
+              <element name="TargetGrants">
+                <ref name="AccessControlList"/>
+              </element>
+            </optional>
+          </interleave>
+        </element>
+      </optional>
+    </element>
+  </start>
+</grammar>

swift3/schema/common.rng

@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <define name="CanonicalUser">
+    <interleave>
+      <element name="ID">
+        <data type="string"/>
+      </element>
+      <optional>
+        <element name="DisplayName">
+          <data type="string"/>
+        </element>
+      </optional>
+    </interleave>
+  </define>
+  <define name="StorageClass">
+    <choice>
+      <value>STANDARD</value>
+      <value>REDUCED_REDUNDANCY</value>
+      <value>GLACIER</value>
+      <value>UNKNOWN</value>
+    </choice>
+  </define>
+  <define name="AccessControlList">
+    <zeroOrMore>
+      <element name="Grant">
+        <interleave>
+          <element name="Grantee">
+            <choice>
+              <group>
+                <attribute name="xsi:type">
+                  <value>AmazonCustomerByEmail</value>
+                </attribute>
+                <element name="EmailAddress">
+                  <data type="string"/>
+                </element>
+              </group>
+              <group>
+                <attribute name="xsi:type">
+                  <value>CanonicalUser</value>
+                </attribute>
+                <ref name="CanonicalUser"/>
+              </group>
+              <group>
+                <attribute name="xsi:type">
+                  <value>Group</value>
+                </attribute>
+                <element name="URI">
+                  <data type="string"/>
+                </element>
+              </group>
+            </choice>
+          </element>
+          <element name="Permission">
+            <choice>
+              <value>READ</value>
+              <value>WRITE</value>
+              <value>READ_ACP</value>
+              <value>WRITE_ACP</value>
+              <value>FULL_CONTROL</value>
+            </choice>
+          </element>
+        </interleave>
+      </element>
+    </zeroOrMore>
+  </define>
+</grammar>

swift3/schema/complete_multipart_upload.rng

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <start>
+    <element name="CompleteMultipartUpload">
+      <oneOrMore>
+        <element name="Part">
+          <interleave>
+            <element name="PartNumber">
+              <data type="int"/>
+            </element>
+            <element name="ETag">
+              <data type="string"/>
+            </element>
+          </interleave>
+        </element>
+      </oneOrMore>
+    </element>
+  </start>
+</grammar>

swift3/schema/complete_multipart_upload_result.rng

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <start>
+    <element name="CompleteMultipartUploadResult">
+      <element name="Location">
+        <data type="anyURI"/>
+      </element>
+      <element name="Bucket">
+        <data type="string"/>
+      </element>
+      <element name="Key">
+        <data type="string"/>
+      </element>
+      <element name="ETag">
+        <data type="string"/>
+      </element>
+    </element>
+  </start>
+</grammar>

swift3/schema/copy_object_result.rng

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <start>
+    <element name="CopyObjectResult">
+      <element name="LastModified">
+        <data type="dateTime"/>
+      </element>
+      <element name="ETag">
+        <data type="string"/>
+      </element>
+    </element>
+  </start>
+</grammar>

swift3/schema/create_bucket_configuration.rng

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <start>
+    <element name="CreateBucketConfiguration">
+      <element name="LocationConstraint">
+        <data type="string"/>
+      </element>
+    </element>
+  </start>
+</grammar>

swift3/schema/delete.rng

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <start>
+    <element name="Delete">
+      <interleave>
+        <optional>
+          <element name="Quiet">
+            <data type="boolean"/>
+          </element>
+        </optional>
+        <oneOrMore>
+          <element name="Object">
+            <interleave>
+              <element name="Key">
+                <data type="string"/>
+              </element>
+              <optional>
+                <element name="VersionId">
+                  <data type="string"/>
+                </element>
+              </optional>
+            </interleave>
+          </element>
+        </oneOrMore>
+      </interleave>
+    </element>
+  </start>
+</grammar>

swift3/schema/delete_result.rng

@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <start>
+    <element name="DeleteResult">
+      <zeroOrMore>
+        <choice>
+          <element name="Deleted">
+            <element name="Key">
+              <data type="string"/>
+            </element>
+            <optional>
+              <element name="VersionId">
+                <data type="string"/>
+              </element>
+            </optional>
+            <optional>
+              <element name="DeleteMarker">
+                <data type="boolean"/>
+              </element>
+            </optional>
+            <optional>
+              <element name="DeleteMarkerVersionId">
+                <data type="string"/>
+              </element>
+            </optional>
+          </element>
+          <element name="Error">
+            <element name="Key">
+              <data type="string"/>
+            </element>
+            <optional>
+              <element name="VersionId">
+                <data type="string"/>
+              </element>
+            </optional>
+            <element name="Code">
+              <data type="string"/>
+            </element>
+            <element name="Message">
+              <data type="string"/>
+            </element>
+          </element>
+        </choice>
+      </zeroOrMore>
+    </element>
+  </start>
+</grammar>

swift3/schema/error.rng

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <start>
+    <element name="Error">
+      <element name="Code">
+        <data type="string"/>
+      </element>
+      <element name="Message">
+        <data type="string"/>
+      </element>
+      <zeroOrMore>
+        <ref name="DebugInfo"/>
+      </zeroOrMore>
+    </element>
+  </start>
+  <define name="DebugInfo">
+    <element>
+      <anyName/>
+      <zeroOrMore>
+        <choice>
+          <attribute>
+            <anyName/>
+          </attribute>
+          <text/>
+          <ref name="DebugInfo"/>
+        </choice>
+      </zeroOrMore>
+    </element>
+  </define>
+</grammar>

swift3/schema/initiate_multipart_upload_result.rng

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <start>
+    <element name="InitiateMultipartUploadResult">
+      <element name="Bucket">
+        <data type="string"/>
+      </element>
+      <element name="Key">
+        <data type="string"/>
+      </element>
+      <element name="UploadId">
+        <data type="string"/>
+      </element>
+    </element>
+  </start>
+</grammar>

swift3/schema/lifecycle_configuration.rng

@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <include href="common.rng"/>
+  <start>
+    <element name="LifecycleConfiguration">
+      <oneOrMore>
+        <element name="Rule">
+          <interleave>
+            <optional>
+              <element name="ID">
+                <data type="string"/>
+              </element>
+            </optional>
+            <element name="Prefix">
+              <data type="string"/>
+            </element>
+            <element name="Status">
+              <choice>
+                <value>Enabled</value>
+                <value>Disabled</value>
+              </choice>
+            </element>
+            <optional>
+              <element name="Transition">
+                <ref name="Transition"/>
+              </element>
+            </optional>
+            <optional>
+              <element name="Expiration">
+                <ref name="Expiration"/>
+              </element>
+            </optional>
+          </interleave>
+        </element>
+      </oneOrMore>
+    </element>
+  </start>
+  <define name="Expiration">
+    <choice>
+      <element name="Days">
+        <data type="int"/>
+      </element>
+      <element name="Date">
+        <data type="dateTime"/>
+      </element>
+    </choice>
+  </define>
+  <define name="Transition">
+    <interleave>
+      <ref name="Expiration"/>
+      <element name="StorageClass">
+        <ref name="StorageClass"/>
+      </element>
+    </interleave>
+  </define>
+</grammar>

swift3/schema/list_all_my_buckets_result.rng

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <include href="common.rng"/>
+  <start>
+    <element name="ListAllMyBucketsResult">
+      <element name="Owner">
+        <ref name="CanonicalUser"/>
+      </element>
+      <element name="Buckets">
+        <zeroOrMore>
+          <element name="Bucket">
+            <element name="Name">
+              <data type="string"/>
+            </element>
+            <element name="CreationDate">
+              <data type="dateTime"/>
+            </element>
+          </element>
+        </zeroOrMore>
+      </element>
+    </element>
+  </start>
+</grammar>

swift3/schema/list_bucket_result.rng

@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <include href="common.rng"/>
+  <start>
+    <element name="ListBucketResult">
+      <element name="Name">
+        <data type="string"/>
+      </element>
+      <element name="Prefix">
+        <data type="string"/>
+      </element>
+      <element name="Marker">
+        <data type="string"/>
+      </element>
+      <optional>
+        <element name="NextMarker">
+          <data type="string"/>
+        </element>
+      </optional>
+      <element name="MaxKeys">
+        <data type="int"/>
+      </element>
+      <optional>
+        <element name="EncodingType">
+          <data type="string"/>
+        </element>
+      </optional>
+      <optional>
+        <element name="Delimiter">
+          <data type="string"/>
+        </element>
+      </optional>
+      <element name="IsTruncated">
+        <data type="boolean"/>
+      </element>
+      <zeroOrMore>
+        <element name="Contents">
+          <element name="Key">
+            <data type="string"/>
+          </element>
+          <element name="LastModified">
+            <data type="dateTime"/>
+          </element>
+          <element name="ETag">
+            <data type="string"/>
+          </element>
+          <element name="Size">
+            <data type="long"/>
+          </element>
+          <optional>
+            <element name="Owner">
+              <ref name="CanonicalUser"/>
+            </element>
+          </optional>
+          <element name="StorageClass">
+            <ref name="StorageClass"/>
+          </element>
+        </element>
+      </zeroOrMore>
+      <zeroOrMore>
+        <element name="CommonPrefixes">
+          <element name="Prefix">
+            <data type="string"/>
+          </element>
+        </element>
+      </zeroOrMore>
+    </element>
+  </start>
+</grammar>

swift3/schema/list_multipart_uploads_result.rng

@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <include href="common.rng"/>
+  <start>
+    <element name="ListMultipartUploadsResult">
+      <element name="Bucket">
+        <data type="string"/>
+      </element>
+      <element name="KeyMarker">
+        <data type="string"/>
+      </element>
+      <element name="UploadIdMarker">
+        <data type="string"/>
+      </element>
+      <element name="NextKeyMarker">
+        <data type="string"/>
+      </element>
+      <element name="NextUploadIdMarker">
+        <data type="string"/>
+      </element>
+      <optional>
+        <element name="Delimiter">
+          <data type="string"/>
+        </element>
+      </optional>
+      <optional>
+        <element name="Prefix">
+          <data type="string"/>
+        </element>
+      </optional>
+      <element name="MaxUploads">
+        <data type="int"/>
+      </element>
+      <optional>
+        <element name="EncodingType">
+          <data type="string"/>
+        </element>
+      </optional>
+      <element name="IsTruncated">
+        <data type="boolean"/>
+      </element>
+      <zeroOrMore>
+        <element name="Upload">
+          <element name="Key">
+            <data type="string"/>
+          </element>
+          <element name="UploadId">
+            <data type="string"/>
+          </element>
+          <element name="Initiator">
+            <ref name="CanonicalUser"/>
+          </element>
+          <element name="Owner">
+            <ref name="CanonicalUser"/>
+          </element>
+          <element name="StorageClass">
+            <ref name="StorageClass"/>
+          </element>
+          <element name="Initiated">
+            <data type="dateTime"/>
+          </element>
+        </element>
+      </zeroOrMore>
+      <zeroOrMore>
+        <element name="CommonPrefixes">
+          <element name="Prefix">
+            <data type="string"/>
+          </element>
+        </element>
+      </zeroOrMore>
+    </element>
+  </start>
+</grammar>

swift3/schema/list_parts_result.rng

@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <include href="common.rng"/>
+  <start>
+    <element name="ListPartsResult">
+      <element name="Bucket">
+        <data type="string"/>
+      </element>
+      <element name="Key">
+        <data type="string"/>
+      </element>
+      <element name="UploadId">
+        <data type="string"/>
+      </element>
+      <element name="Initiator">
+        <ref name="CanonicalUser"/>
+      </element>
+      <element name="Owner">
+        <ref name="CanonicalUser"/>
+      </element>
+      <element name="StorageClass">
+        <ref name="StorageClass"/>
+      </element>
+      <element name="PartNumberMarker">
+        <data type="int"/>
+      </element>
+      <element name="NextPartNumberMarker">
+        <data type="int"/>
+      </element>
+      <element name="MaxParts">
+        <data type="int"/>
+      </element>
+      <optional>
+        <element name="EncodingType">
+          <data type="string"/>
+        </element>
+      </optional>
+      <element name="IsTruncated">
+        <data type="boolean"/>
+      </element>
+      <zeroOrMore>
+        <element name="Part">
+          <element name="PartNumber">
+            <data type="int"/>
+          </element>
+          <element name="LastModified">
+            <data type="dateTime"/>
+          </element>
+          <element name="ETag">
+            <data type="string"/>
+          </element>
+          <element name="Size">
+            <data type="long"/>
+          </element>
+        </element>
+      </zeroOrMore>
+    </element>
+  </start>
+</grammar>

swift3/schema/list_versions_result.rng

@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <include href="common.rng"/>
+  <start>
+    <element name="ListVersionsResult">
+      <element name="Name">
+        <data type="string"/>
+      </element>
+      <element name="Prefix">
+        <data type="string"/>
+      </element>
+      <element name="KeyMarker">
+        <data type="string"/>
+      </element>
+      <element name="VersionIdMarker">
+        <data type="string"/>
+      </element>
+      <optional>
+        <element name="NextKeyMarker">
+          <data type="string"/>
+        </element>
+      </optional>
+      <optional>
+        <element name="NextVersionIdMarker">
+          <data type="string"/>
+        </element>
+      </optional>
+      <element name="MaxKeys">
+        <data type="int"/>
+      </element>
+      <optional>
+        <element name="EncodingType">
+          <data type="string"/>
+        </element>
+      </optional>
+      <optional>
+        <element name="Delimiter">
+          <data type="string"/>
+        </element>
+      </optional>
+      <element name="IsTruncated">
+        <data type="boolean"/>
+      </element>
+      <zeroOrMore>
+        <choice>
+          <element name="Version">
+            <element name="Key">
+              <data type="string"/>
+            </element>
+            <element name="VersionId">
+              <data type="string"/>
+            </element>
+            <element name="IsLatest">
+              <data type="boolean"/>
+            </element>
+            <element name="LastModified">
+              <data type="dateTime"/>
+            </element>
+            <element name="ETag">
+              <data type="string"/>
+            </element>
+            <element name="Size">
+              <data type="long"/>
+            </element>
+            <optional>
+              <element name="Owner">
+                <ref name="CanonicalUser"/>
+              </element>
+            </optional>
+            <element name="StorageClass">
+              <ref name="StorageClass"/>
+            </element>
+          </element>
+          <element name="DeleteMarker">
+            <element name="Key">
+              <data type="string"/>
+            </element>
+            <element name="VersionId">
+              <data type="string"/>
+            </element>
+            <element name="IsLatest">
+              <data type="boolean"/>
+            </element>
+            <element name="LastModified">
+              <data type="dateTime"/>
+            </element>
+            <optional>
+              <element name="Owner">
+                <ref name="CanonicalUser"/>
+              </element>
+            </optional>
+          </element>
+        </choice>
+      </zeroOrMore>
+      <zeroOrMore>
+        <element name="CommonPrefixes">
+          <element name="Prefix">
+            <data type="string"/>
+          </element>
+        </element>
+      </zeroOrMore>
+    </element>
+  </start>
+</grammar>

swift3/schema/location_constraint.rng

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <start>
+    <element name="LocationConstraint">
+      <data type="string"/>
+    </element>
+  </start>
+</grammar>

swift3/schema/versioning_configuration.rng

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0">
+  <start>
+    <element name="VersioningConfiguration">
+      <interleave>
+        <optional>
+          <element name="Status">
+            <choice>
+              <value>Enabled</value>
+              <value>Suspended</value>
+            </choice>
+          </element>
+        </optional>
+        <optional>
+          <element name="MfaDelete">
+            <choice>
+              <value>Enabled</value>
+              <value>Disabled</value>
+            </choice>
+          </element>
+        </optional>
+      </interleave>
+    </element>
+  </start>
+</grammar>

swift3/test/functional/001.out

@@ -10,6 +10,10 @@ X-Trans-Id: TXID
 
 <?xml version='1.0' encoding='UTF-8'?>
 <ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Owner>
+    <ID>TESTER</ID>
+    <DisplayName>TESTER</DisplayName>
+  </Owner>
   <Buckets>
     <Bucket>
       <Name>5ABCDEFGHIJKLMNOPQRSTUVWXYZ.5-6789</Name>

swift3/test/functional/003.out

@@ -14,12 +14,12 @@ X-Trans-Id: TXID
 
 <?xml version='1.0' encoding='UTF-8'?>
 <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Name>bucket</Name>
   <Prefix>photos/2006/</Prefix>
   <Marker/>
+  <MaxKeys>1000</MaxKeys>
   <Delimiter>/</Delimiter>
   <IsTruncated>false</IsTruncated>
-  <MaxKeys>1000</MaxKeys>
-  <Name>bucket</Name>
   <CommonPrefixes>
     <Prefix>photos/2006/February/</Prefix>
   </CommonPrefixes>

swift3/test/unit/__init__.py

@@ -76,8 +76,7 @@ class Swift3TestCase(unittest.TestCase):
                             swob.HTTPNoContent, {}, None)
 
     def _get_error_code(self, body):
-        elem = fromstring(body)
-        self.assertEquals(elem.tag, 'Error')
+        elem = fromstring(body, 'Error')
         return elem.find('./Code').text
 
     def _test_method_error(self, method, path, response_class, headers={}):

swift3/test/unit/test_acl.py

@@ -29,8 +29,7 @@ class TestSwift3Acl(Swift3TestCase):
         super(TestSwift3Acl, self).setUp()
 
     def _check_acl(self, owner, body):
-        elem = fromstring(body)
-        self.assertEquals(elem.tag, 'AccessControlPolicy')
+        elem = fromstring(body, 'AccessControlPolicy')
         permission = elem.find('./AccessControlList/Grant/Permission').text
         self.assertEquals(permission, 'FULL_CONTROL')
         name = elem.find('./AccessControlList/Grant/Grantee/ID').text

swift3/test/unit/test_bucket.py

@@ -66,8 +66,7 @@ class TestSwift3Bucket(Swift3TestCase):
         status, headers, body = self.call_swift3(req)
         self.assertEquals(status.split()[0], '200')
 
-        elem = fromstring(body)
-        self.assertEquals(elem.tag, 'ListBucketResult')
+        elem = fromstring(body, 'ListBucketResult')
         name = elem.find('./Name').text
         self.assertEquals(name, bucket_name)
 
@@ -90,7 +89,7 @@ class TestSwift3Bucket(Swift3TestCase):
                                      'QUERY_STRING': 'max-keys=5'},
                             headers={'Authorization': 'AWS test:tester:hmac'})
         status, headers, body = self.call_swift3(req)
-        elem = fromstring(body)
+        elem = fromstring(body, 'ListBucketResult')
         self.assertEquals(elem.find('./IsTruncated').text, 'false')
 
         req = Request.blank('/%s' % bucket_name,
@@ -98,7 +97,7 @@ class TestSwift3Bucket(Swift3TestCase):
                                      'QUERY_STRING': 'max-keys=4'},
                             headers={'Authorization': 'AWS test:tester:hmac'})
         status, headers, body = self.call_swift3(req)
-        elem = fromstring(body)
+        elem = fromstring(body, 'ListBucketResult')
         self.assertEquals(elem.find('./IsTruncated').text, 'true')
 
     def test_bucket_GET_max_keys(self):
@@ -109,7 +108,7 @@ class TestSwift3Bucket(Swift3TestCase):
                                      'QUERY_STRING': 'max-keys=5'},
                             headers={'Authorization': 'AWS test:tester:hmac'})
         status, headers, body = self.call_swift3(req)
-        elem = fromstring(body)
+        elem = fromstring(body, 'ListBucketResult')
         self.assertEquals(elem.find('./MaxKeys').text, '5')
         _, path = self.swift.calls[-1]
         _, query_string = path.split('?')
@@ -121,7 +120,7 @@ class TestSwift3Bucket(Swift3TestCase):
                                      'QUERY_STRING': 'max-keys=5000'},
                             headers={'Authorization': 'AWS test:tester:hmac'})
         status, headers, body = self.call_swift3(req)
-        elem = fromstring(body)
+        elem = fromstring(body, 'ListBucketResult')
         self.assertEquals(elem.find('./MaxKeys').text, '1000')
         _, path = self.swift.calls[-1]
         _, query_string = path.split('?')
@@ -135,7 +134,7 @@ class TestSwift3Bucket(Swift3TestCase):
                                      'delimiter=a&marker=b&prefix=c'},
                             headers={'Authorization': 'AWS test:tester:hmac'})
         status, headers, body = self.call_swift3(req)
-        elem = fromstring(body)
+        elem = fromstring(body, 'ListBucketResult')
         self.assertEquals(elem.find('./Prefix').text, 'c')
         self.assertEquals(elem.find('./Marker').text, 'b')
         self.assertEquals(elem.find('./Delimiter').text, 'a')

swift3/test/unit/test_middleware.py

@@ -256,9 +256,8 @@ class TestSwift3Middleware(Swift3TestCase):
                             environ={'REQUEST_METHOD': 'PUT'},
                             headers={'Authorization': 'AWS test:tester:hmac'},
                             body=xml)
-        # FIXME: swift3 should handle invalid xml file
         status, headers, body = self.call_swift3(req)
-        self.assertEquals(self._get_error_code(body), 'InternalError')
+        self.assertEquals(self._get_error_code(body), 'MalformedACLError')
 
     def _test_unsupported_resource(self, resource):
         req = Request.blank('/error?' + resource,

swift3/test/unit/test_service.py

@@ -59,8 +59,7 @@ class TestSwift3Service(Swift3TestCase):
         status, headers, body = self.call_swift3(req)
         self.assertEquals(status.split()[0], '200')
 
-        elem = fromstring(body)
-        self.assertEquals(elem.tag, 'ListAllMyBucketsResult')
+        elem = fromstring(body, 'ListAllMyBucketsResult')
 
         all_buckets = elem.find('./Buckets')
         buckets = all_buckets.iterchildren('Bucket')

swift3/test/unit/test_utils.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2014 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from swift3 import utils
+
+strs = [
+    ('Owner', 'owner'),
+    ('DisplayName', 'display_name'),
+    ('AccessControlPolicy', 'access_control_policy'),
+]
+
+
+class TestSwift3Utils(unittest.TestCase):
+    def test_camel_to_snake(self):
+        for s1, s2 in strs:
+            self.assertEquals(utils.camel_to_snake(s1), s2)
+
+    def test_snake_to_camel(self):
+        for s1, s2 in strs:
+            self.assertEquals(s1, utils.snake_to_camel(s2))

swift3/test/unit/test_versioning.py

@@ -31,8 +31,7 @@ class TestSwift3Versioning(Swift3TestCase):
                             environ={'REQUEST_METHOD': 'GET'},
                             headers={'Authorization': 'AWS test:tester:hmac'})
         status, headers, body = self.call_swift3(req)
-        elem = fromstring(body)
-        self.assertEquals(elem.tag, 'VersioningConfiguration')
+        fromstring(body, 'VersioningConfiguration')
 
 if __name__ == '__main__':
     unittest.main()

swift3/utils.py

@@ -0,0 +1,23 @@
+# Copyright (c) 2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+def camel_to_snake(camel):
+    return re.sub('(.)([A-Z])', r'\1_\2', camel).lower()
+
+
+def snake_to_camel(snake):
+    return snake.title().replace('_', '')