Merge pull request #4 from basho/riakcs-2.1-honest-but-long-way-around

Update to upstream HEAD and changes for Riak CS 2.1 release

Author: kuenishi

s3tests/common.py

@@ -88,6 +88,8 @@ def connect(conf):
     mapping = dict(
         port='port',
         host='host',
+        proxy='proxy',
+        proxy_port='proxy_port',
         is_secure='is_secure',
         access_key='aws_access_key_id',
         secret_key='aws_secret_access_key',

s3tests/functional/__init__.py

@@ -295,6 +295,16 @@ def setup():
         regions.add(name, TargetConfig(cfg, section))
 
     for section in cfg.sections():
+        try:
+            proxy = cfg.get(section, 'proxy')
+        except ConfigParser.NoOptionError:
+            proxy = None
+
+        try:
+            proxy_port = cfg.getint(section, 'proxy_port')
+        except ConfigParser.NoOptionError:
+            proxy_port = None
+
         try:
             (type_, name) = section.split(None, 1)
         except ValueError:
@@ -323,6 +333,8 @@ def setup():
                 aws_access_key_id=cfg.get(section, 'access_key'),
                 aws_secret_access_key=cfg.get(section, 'secret_key'),
                 is_secure=conf.is_secure,
+                proxy=proxy,
+                proxy_port=proxy_port,
                 port=conf.port,
                 host=conf.host,
                 # TODO test vhost calling format

s3tests/functional/test_s3.py

@@ -213,6 +213,7 @@ def validate_bucket_list(bucket, prefix, delimiter, marker, max_keys,
 @attr(method='get')
 @attr(operation='list')
 @attr(assertion='prefixes in multi-component object names')
+@attr('fails_on_rcs')
 def test_bucket_list_delimiter_prefix():
     bucket = _create_keys(keys=['asdf', 'boo/bar', 'boo/baz/xyzzy', 'cquux/thud', 'cquux/bla'])
 
@@ -994,6 +995,7 @@ def test_object_set_get_unicode_metadata():
 @attr(operation='metadata write/re-write')
 @attr(assertion='non-UTF-8 values detected, but preserved')
 @attr('fails_strict_rfc2616')
+@attr('fails_on_rcs')
 def test_object_set_get_non_utf8_metadata():
     bucket = get_new_bucket()
     key = boto.s3.key.Key(bucket)
@@ -1022,6 +1024,7 @@ def _set_get_metadata_unreadable(metadata, bucket=None):
 @attr(operation='metadata write')
 @attr(assertion='non-priting prefixes noted and preserved')
 @attr('fails_strict_rfc2616')
+@attr('fails_on_rcs')
 def test_object_set_get_metadata_empty_to_unreadable_prefix():
     metadata = '\x04w'
     got = _set_get_metadata_unreadable(metadata)
@@ -1033,6 +1036,7 @@ def test_object_set_get_metadata_empty_to_unreadable_prefix():
 @attr(operation='metadata write')
 @attr(assertion='non-priting suffixes noted and preserved')
 @attr('fails_strict_rfc2616')
+@attr('fails_on_rcs')
 def test_object_set_get_metadata_empty_to_unreadable_suffix():
     metadata = 'h\x04'
     got = _set_get_metadata_unreadable(metadata)
@@ -1043,6 +1047,7 @@ def test_object_set_get_metadata_empty_to_unreadable_suffix():
 @attr(method='put')
 @attr(operation='metadata write')
 @attr(assertion='non-priting in-fixes noted and preserved')
+@attr('fails_on_rcs')
 def test_object_set_get_metadata_empty_to_unreadable_infix():
     metadata = 'h\x04w'
     got = _set_get_metadata_unreadable(metadata)
@@ -1054,6 +1059,7 @@ def test_object_set_get_metadata_empty_to_unreadable_infix():
 @attr(operation='metadata re-write')
 @attr(assertion='non-priting prefixes noted and preserved')
 @attr('fails_strict_rfc2616')
+@attr('fails_on_rcs')
 def test_object_set_get_metadata_overwrite_to_unreadable_prefix():
     metadata = '\x04w'
     got = _set_get_metadata_unreadable(metadata)
@@ -1068,6 +1074,7 @@ def test_object_set_get_metadata_overwrite_to_unreadable_prefix():
 @attr(operation='metadata re-write')
 @attr(assertion='non-priting suffixes noted and preserved')
 @attr('fails_strict_rfc2616')
+@attr('fails_on_rcs')
 def test_object_set_get_metadata_overwrite_to_unreadable_suffix():
     metadata = 'h\x04'
     got = _set_get_metadata_unreadable(metadata)
@@ -1081,6 +1088,7 @@ def test_object_set_get_metadata_overwrite_to_unreadable_suffix():
 @attr(method='put')
 @attr(operation='metadata re-write')
 @attr(assertion='non-priting in-fixes noted and preserved')
+@attr('fails_on_rcs')
 def test_object_set_get_metadata_overwrite_to_unreadable_infix():
     metadata = 'h\x04w'
     got = _set_get_metadata_unreadable(metadata)
@@ -2131,7 +2139,7 @@ def test_put_object_ifmatch_good():
     got_data = key.get_contents_as_string()
     eq(got_data, 'bar')
 
-    key.set_contents_from_string('zar', headers={'If-Match': key.etag.replace('"', '').strip()})
+    key.set_contents_from_string('zar', headers={'If-Match': key.etag.strip()})
     got_new_data = key.get_contents_as_string()
     eq(got_new_data, 'zar')
 
@@ -2223,7 +2231,7 @@ def test_put_object_ifnonmatch_failed():
     eq(got_data, 'bar')
 
     e = assert_raises(boto.exception.S3ResponseError, key.set_contents_from_string, 'zar',
-                      headers={'If-None-Match': key.etag.replace('"', '').strip()})
+                      headers={'If-None-Match': key.etag.strip()})
     eq(e.status, 412)
     eq(e.reason, 'Precondition Failed')
     eq(e.error_code, 'PreconditionFailed')
@@ -2310,7 +2318,10 @@ def _make_request(method, bucket, key, body=None, authenticated=False, response_
     else:
         class_ = HTTPConnection
 
-    c = class_(s3.main.host, s3.main.port, strict=True)
+    if s3.main.proxy is None:
+        c = class_(s3.main.host, s3.main.port, strict=True)
+    else:
+        c = class_(s3.main.proxy, s3.main.proxy_port, strict=True)
     c.request(method, path, body=body)
     res = c.getresponse()
 
@@ -2335,7 +2346,10 @@ def _make_bucket_request(method, bucket, body=None, authenticated=False, expires
     else:
         class_ = HTTPConnection
 
-    c = class_(s3.main.host, s3.main.port, strict=True)
+    if s3.main.proxy is None:
+        c = class_(s3.main.host, s3.main.port, strict=True)
+    else:
+        c = class_(s3.main.proxy, s3.main.proxy_port, strict=True)
     c.request(method, path, body=body)
     res = c.getresponse()
 
@@ -2487,6 +2501,7 @@ def test_object_raw_authenticated():
 @attr(operation='authenticated on private bucket/private object with modified response headers')
 @attr(assertion='succeeds')
 @attr('fails_on_rgw')
+@attr('fails_on_rcs')
 def test_object_raw_response_headers():
     (bucket, key) = _setup_request('private', 'private')
 
@@ -2819,6 +2834,7 @@ def test_bucket_create_naming_bad_punctuation():
 @attr(method='put')
 @attr(operation='create w/underscore in name')
 @attr(assertion='succeeds')
+@attr('fails_on_rcs')
 def test_bucket_create_naming_dns_underscore():
     check_good_bucket_name('foo_bar')
 
@@ -3377,6 +3393,7 @@ def test_object_acl_canned_bucketownerfullcontrol():
 @attr(method='put')
 @attr(operation='set write-acp')
 @attr(assertion='does not modify owner')
+@attr('fails_on_rcs')
 def test_object_acl_full_control_verify_owner():
     bucket = get_new_bucket(targets.main.default)
     bucket.set_acl('public-read-write')
@@ -3810,6 +3827,7 @@ def _get_acl_header(user=None, perms=None):
 @attr(operation='add all grants to user through headers')
 @attr(assertion='adds all grants individually to second user')
 @attr('fails_on_dho')
+@attr('fails_on_rcs')
 def test_object_header_acl_grants():
     bucket = get_new_bucket()
     headers = _get_acl_header()
@@ -3869,6 +3887,7 @@ def test_object_header_acl_grants():
 @attr(operation='add all grants to user through headers')
 @attr(assertion='adds all grants individually to second user')
 @attr('fails_on_dho')
+@attr('fails_on_rcs')
 def test_bucket_header_acl_grants():
     headers = _get_acl_header()
     bucket = get_new_bucket(targets.main.default, get_prefix(), headers)
@@ -4235,6 +4254,8 @@ def _create_connection_bad_auth(aws_access_key_id='badauth'):
         is_secure=main.is_secure,
         port=main.port,
         host=main.host,
+        proxy=main.proxy,
+        proxy_port=main.proxy_port,
         calling_format=main.calling_format,
         )
     return conn
@@ -4259,6 +4280,7 @@ def test_list_buckets_anonymous():
 @attr(method='get')
 @attr(operation='list all buckets (bad auth)')
 @attr(assertion='fails 403')
+@attr('fails_on_rcs')
 def test_list_buckets_invalid_auth():
     conn = _create_connection_bad_auth()
     e = assert_raises(boto.exception.S3ResponseError, conn.get_all_buckets)
@@ -4270,6 +4292,7 @@ def test_list_buckets_invalid_auth():
 @attr(method='get')
 @attr(operation='list all buckets (bad auth)')
 @attr(assertion='fails 403')
+@attr('fails_on_rcs')
 def test_list_buckets_bad_auth():
     conn = _create_connection_bad_auth(aws_access_key_id=s3.main.aws_access_key_id)
     e = assert_raises(boto.exception.S3ResponseError, conn.get_all_buckets)
@@ -4393,6 +4416,7 @@ def test_object_copy_same_bucket():
 @attr(method='put')
 @attr(operation='copy object to itself')
 @attr(assertion='fails')
+@attr('fails_on_rcs')
 def test_object_copy_to_itself():
     bucket = get_new_bucket()
     key = bucket.new_key('foo123bar')
@@ -4572,6 +4596,7 @@ def _multipart_upload(bucket, s3_key_name, size, part_size=5*1024*1024, do_list=
 @attr(resource='object')
 @attr(method='put')
 @attr(operation='check multipart upload without parts')
+@attr('fails_on_rcs')
 def test_multipart_upload_empty():
     bucket = get_new_bucket()
     key = "mymultipart"
@@ -4690,6 +4715,7 @@ def test_multipart_upload_multiple_sizes():
 @attr(method='put')
 @attr(operation='check failure on multiple multi-part upload with size too small')
 @attr(assertion='fails 400')
+@attr('fails_on_rcs')   # this is an artifact by enforce_multipart_part_size=false in r_t
 def test_multipart_upload_size_too_small():
     bucket = get_new_bucket()
     key="mymultipart"
@@ -4803,7 +4829,7 @@ def test_list_multipart_upload():
     upload2.cancel_upload()
     upload3.cancel_upload()
 
-def _simple_http_req_100_cont(host, port, is_secure, method, resource):
+def _simple_http_req_100_cont(host, port, proxy, proxy_port, is_secure, method, resource):
     """
     Send the specified request w/expect 100-continue
     and await confirmation.
@@ -4818,7 +4844,10 @@ def _simple_http_req_100_cont(host, port, is_secure, method, resource):
     if is_secure:
         s = ssl.wrap_socket(s);
     s.settimeout(5)
-    s.connect((host, port))
+    if proxy is None:
+        s.connect((host, port))
+    else:
+        s.connect((proxy, proxy_port))
     s.send(req)
 
     try:
@@ -4845,12 +4874,16 @@ def test_100_continue():
     objname = 'testobj'
     resource = '/{bucket}/{obj}'.format(bucket=bucket.name, obj=objname)
 
-    status = _simple_http_req_100_cont(s3.main.host, s3.main.port, s3.main.is_secure, 'PUT', resource)
+    status = _simple_http_req_100_cont(s3.main.host, s3.main.port,
+                                       s3.main.proxy, s3.main.proxy_port,
+                                       s3.main.is_secure, 'PUT', resource)
     eq(status, '403')
 
     bucket.set_acl('public-read-write')
 
-    status = _simple_http_req_100_cont(s3.main.host, s3.main.port, s3.main.is_secure, 'PUT', resource)
+    status = _simple_http_req_100_cont(s3.main.host, s3.main.port,
+                                       s3.main.proxy, s3.main.proxy_port,
+                                       s3.main.is_secure, 'PUT', resource)
     eq(status, '100')
 
 def _test_bucket_acls_changes_persistent(bucket):
@@ -4882,6 +4915,7 @@ def test_stress_bucket_acls_changes():
 @attr(method='put')
 @attr(operation='set cors')
 @attr(assertion='succeeds')
+@attr('cors')
 def test_set_cors():
     bucket = get_new_bucket()
     cfg = CORSConfiguration()
@@ -4925,6 +4959,7 @@ def _cors_request_and_check(func, url, headers, expect_status, expect_allow_orig
 @attr(method='get')
 @attr(operation='check cors response when origin header set')
 @attr(assertion='returning cors header')
+@attr('cors')
 def test_cors_origin_response():
     cfg = CORSConfiguration()
     bucket = get_new_bucket()
@@ -5096,6 +5131,8 @@ def _test_atomic_read(file_size):
         is_secure=s3['main'].is_secure,
         port=s3['main'].port,
         host=s3['main'].host,
+        proxy=s3['main'].proxy,
+        proxy_port=s3['main'].proxy_port,
         calling_format=s3['main'].calling_format,
         )