Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 02509ac

Browse files
Zuulopenstack-gerrit
Zuul
authored and
openstack-gerrit
committed
Merge "Add option to skip container PUT during upload"
2 parents 249201f + f1858d8 commit 02509ac

File tree

3 files changed

+102
-48
lines changed

3 files changed

+102
-48
lines changed

‎swiftclient/service.py‎

Lines changed: 50 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,7 @@ def _build_default_global_options():
202202
'leave_segments': False,
203203
'changed': None,
204204
'skip_identical': False,
205+
'skip_container_put': False,
205206
'version_id': None,
206207
'yes_all': False,
207208
'read_acl': None,
@@ -1465,6 +1466,7 @@ def upload(self, container, objects, options=None):
14651466
'leave_segments': False,
14661467
'changed': None,
14671468
'skip_identical': False,
1469+
'skip_container_put': False,
14681470
'fail_fast': False,
14691471
'dir_marker': False # Only for None sources
14701472
}
@@ -1490,54 +1492,57 @@ def upload(self, container, objects, options=None):
14901492
# the object name. (same as passing --object-name).
14911493
container, _sep, pseudo_folder = container.partition('/')
14921494

1493-
# Try to create the container, just in case it doesn't exist. If this
1494-
# fails, it might just be because the user doesn't have container PUT
1495-
# permissions, so we'll ignore any error. If there's really a problem,
1496-
# it'll surface on the first object PUT.
1497-
policy_header = {}
1498-
_header = split_headers(options["header"])
1499-
if POLICY in _header:
1500-
policy_header[POLICY] = \
1501-
_header[POLICY]
1502-
create_containers = [
1503-
self.thread_manager.container_pool.submit(
1504-
self._create_container_job, container, headers=policy_header)
1505-
]
1495+
if not options['skip_container_put']:
1496+
# Try to create the container, just in case it doesn't exist. If
1497+
# this fails, it might just be because the user doesn't have
1498+
# container PUT permissions, so we'll ignore any error. If there's
1499+
# really a problem, it'll surface on the first object PUT.
1500+
policy_header = {}
1501+
_header = split_headers(options["header"])
1502+
if POLICY in _header:
1503+
policy_header[POLICY] = \
1504+
_header[POLICY]
1505+
create_containers = [
1506+
self.thread_manager.container_pool.submit(
1507+
self._create_container_job, container,
1508+
headers=policy_header)
1509+
]
15061510

1507-
# wait for first container job to complete before possibly attempting
1508-
# segment container job because segment container job may attempt
1509-
# to HEAD the first container
1510-
for r in interruptable_as_completed(create_containers):
1511-
res = r.result()
1512-
yield res
1511+
# wait for first container job to complete before possibly
1512+
# attempting segment container job because segment container job
1513+
# may attempt to HEAD the first container
1514+
for r in interruptable_as_completed(create_containers):
1515+
res = r.result()
1516+
yield res
15131517

1514-
if segment_size:
1515-
seg_container = container + '_segments'
1516-
if options['segment_container']:
1517-
seg_container = options['segment_container']
1518-
if seg_container != container:
1519-
if not policy_header:
1520-
# Since no storage policy was specified on the command
1521-
# line, rather than just letting swift pick the default
1522-
# storage policy, we'll try to create the segments
1523-
# container with the same policy as the upload container
1524-
create_containers = [
1525-
self.thread_manager.container_pool.submit(
1526-
self._create_container_job, seg_container,
1527-
policy_source=container
1528-
)
1529-
]
1530-
else:
1531-
create_containers = [
1532-
self.thread_manager.container_pool.submit(
1533-
self._create_container_job, seg_container,
1534-
headers=policy_header
1535-
)
1536-
]
1518+
if segment_size:
1519+
seg_container = container + '_segments'
1520+
if options['segment_container']:
1521+
seg_container = options['segment_container']
1522+
if seg_container != container:
1523+
if not policy_header:
1524+
# Since no storage policy was specified on the command
1525+
# line, rather than just letting swift pick the default
1526+
# storage policy, we'll try to create the segments
1527+
# container with the same policy as the upload
1528+
# container
1529+
create_containers = [
1530+
self.thread_manager.container_pool.submit(
1531+
self._create_container_job, seg_container,
1532+
policy_source=container
1533+
)
1534+
]
1535+
else:
1536+
create_containers = [
1537+
self.thread_manager.container_pool.submit(
1538+
self._create_container_job, seg_container,
1539+
headers=policy_header
1540+
)
1541+
]
15371542

1538-
for r in interruptable_as_completed(create_containers):
1539-
res = r.result()
1540-
yield res
1543+
for r in interruptable_as_completed(create_containers):
1544+
res = r.result()
1545+
yield res
15411546

15421547
# We maintain a results queue here and a separate thread to monitor
15431548
# the futures because we want to get results back from potential

‎swiftclient/shell.py‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -978,8 +978,9 @@ def st_copy(parser, args, output_manager, return_parser=False):
978978
st_upload_options = '''[--changed] [--skip-identical] [--segment-size <size>]
979979
[--segment-container <container>] [--leave-segments]
980980
[--object-threads <thread>] [--segment-threads <threads>]
981-
[--meta <name:value>] [--header <header>] [--use-slo]
982-
[--ignore-checksum] [--object-name <object-name>]
981+
[--meta <name:value>] [--header <header>]
982+
[--use-slo] [--ignore-checksum] [--skip-container-put]
983+
[--object-name <object-name>]
983984
<container> <file_or_directory> [<file_or_directory>] [...]
984985
'''
985986

@@ -1025,11 +1026,13 @@ def st_copy(parser, args, output_manager, return_parser=False):
10251026
--use-slo When used in conjunction with --segment-size it will
10261027
create a Static Large Object instead of the default
10271028
Dynamic Large Object.
1029+
--ignore-checksum Turn off checksum validation for uploads.
1030+
--skip-container-put Assume all necessary containers already exist; don't
1031+
automatically try to create them.
10281032
--object-name <object-name>
10291033
Upload file and name object to <object-name> or upload
10301034
dir and use <object-name> as object prefix instead of
10311035
folder name.
1032-
--ignore-checksum Turn off checksum validation for uploads.
10331036
'''.strip('\n')
10341037

10351038

@@ -1044,6 +1047,10 @@ def st_upload(parser, args, output_manager, return_parser=False):
10441047
'--skip-identical', action='store_true', dest='skip_identical',
10451048
default=False, help='Skip uploading files that are identical on '
10461049
'both sides.')
1050+
parser.add_argument(
1051+
'--skip-container-put', action='store_true', dest='skip_container_put',
1052+
default=False, help='Assume all necessary containers already exist; '
1053+
"don't automatically try to create them.")
10471054
parser.add_argument(
10481055
'-S', '--segment-size', dest='segment_size', help='Upload files '
10491056
'in segments no larger than <size> (in Bytes) and then create a '

‎test/unit/test_shell.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -907,6 +907,48 @@ def test_upload(self, connection, walk):
907907
query_string='multipart-manifest=put',
908908
response_dict=mock.ANY)
909909

910+
@mock.patch('swiftclient.shell.walk')
911+
@mock.patch('swiftclient.service.Connection')
912+
def test_upload_skip_container_put(self, connection, walk):
913+
connection.return_value.head_object.return_value = {
914+
'content-length': '0'}
915+
connection.return_value.put_object.return_value = EMPTY_ETAG
916+
connection.return_value.attempts = 0
917+
argv = ["", "upload", "container", "--skip-container-put",
918+
self.tmpfile, "-H", "X-Storage-Policy:one",
919+
"--meta", "Color:Blue"]
920+
swiftclient.shell.main(argv)
921+
connection.return_value.put_container.assert_not_called()
922+
923+
connection.return_value.put_object.assert_called_with(
924+
'container',
925+
self.tmpfile.lstrip('/'),
926+
mock.ANY,
927+
content_length=0,
928+
headers={'x-object-meta-mtime': mock.ANY,
929+
'X-Storage-Policy': 'one',
930+
'X-Object-Meta-Color': 'Blue'},
931+
response_dict={})
932+
933+
# Upload in segments
934+
connection.return_value.head_container.return_value = {
935+
'x-storage-policy': 'one'}
936+
argv = ["", "upload", "container", "--skip-container-put",
937+
self.tmpfile, "-S", "10"]
938+
with open(self.tmpfile, "wb") as fh:
939+
fh.write(b'12345678901234567890')
940+
swiftclient.shell.main(argv)
941+
# Both base and segments container are assumed to exist already
942+
connection.return_value.put_container.assert_not_called()
943+
connection.return_value.put_object.assert_called_with(
944+
'container',
945+
self.tmpfile.lstrip('/'),
946+
'',
947+
content_length=0,
948+
headers={'x-object-manifest': mock.ANY,
949+
'x-object-meta-mtime': mock.ANY},
950+
response_dict={})
951+
910952
@mock.patch('swiftclient.service.SwiftService.upload')
911953
def test_upload_object_with_account_readonly(self, upload):
912954
argv = ["", "upload", "container", self.tmpfile]

0 commit comments

Comments
 (0)