Correctly aggregate totals for >10k items

tipabu · tipabu · commit 373fa26ce162 · 2021-09-21T16:06:56.000-07:00
Previously, we would write out totals for every page of listings, like

    $ swift list sync --prefix=09-21 --total -l
     80000000000
     80000000000
     80000000000
     58096000000

Now, roll those all into a single total:

    $ swift list sync --prefix=09-21 --total -l
    298096000000

Change-Id: Icc265636815220e33e8c9eec0a3ab80e9f899038
diff --git a/swiftclient/shell.py b/swiftclient/shell.py
@@ -531,8 +531,7 @@ def st_download(parser, args, output_manager, return_parser=False):
 
 def st_list(parser, args, output_manager, return_parser=False):
 
-    def _print_stats(options, stats, human):
-        total_count = total_bytes = 0
+    def _print_stats(options, stats, human, totals):
         container = stats.get("container", None)
         for item in stats["listing"]:
             item_name = item.get('name')
@@ -543,7 +542,7 @@ def _print_stats(options, stats, human):
                     item_bytes = item.get('bytes')
                     byte_str = prt_bytes(item_bytes, human)
                     count = item.get('count')
-                    total_count += count
+                    totals['count'] += count
                     try:
                         meta = item.get('meta')
                         utc = gmtime(float(meta.get('x-timestamp')))
@@ -578,17 +577,7 @@ def _print_stats(options, stats, human):
                             output_manager.print_msg(
                                 "%s %10s %8s %24s %s",
                                 byte_str, date, xtime, content_type, item_name)
-                total_bytes += item_bytes
-
-        # report totals
-        if options['long'] or human:
-            if not container:
-                output_manager.print_msg(
-                    "%12s %s", prt_bytes(total_count, True),
-                    prt_bytes(total_bytes, human))
-            else:
-                output_manager.print_msg(
-                    prt_bytes(total_bytes, human))
+                totals['bytes'] += item_bytes
 
     parser.add_argument(
         '-l', '--long', dest='long', action='store_true', default=False,
@@ -642,6 +631,7 @@ def _print_stats(options, stats, human):
         try:
             if not args:
                 stats_parts_gen = swift.list()
+                container = None
             else:
                 container = args[0]
                 args = args[1:]
@@ -667,12 +657,24 @@ def listing(stats_parts_gen=stats_parts_gen):
                     sort_keys=True, indent=2)
                 output_manager.print_msg('')
                 return
+
+            totals = {'count': 0, 'bytes': 0}
             for stats in stats_parts_gen:
                 if stats["success"]:
-                    _print_stats(options, stats, human)
+                    _print_stats(options, stats, human, totals)
                 else:
                     raise stats["error"]
 
+            # report totals
+            if options['long'] or human:
+                if container is None:
+                    output_manager.print_msg(
+                        "%12s %s", prt_bytes(totals['count'], True),
+                        prt_bytes(totals['bytes'], human))
+                else:
+                    output_manager.print_msg(
+                        prt_bytes(totals['bytes'], human))
+
         except SwiftError as e:
             output_manager.error(e.value)
 
diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py
@@ -547,9 +547,12 @@ def test_list_container(self, connection):
 
             self.assertEqual(output.out, 'object_a\n')
 
-        # Test container listing with --long
+        # Test container listing with --long and multiple pages
         connection.return_value.get_container.side_effect = [
-            [None, [{'name': 'object_a', 'bytes': 0,
+            [None, [{'name': 'object_a', 'bytes': 3,
+                     'content_type': 'type/content',
+                     'last_modified': '123T456'}]],
+            [None, [{'name': 'object_b', 'bytes': 5,
                      'content_type': 'type/content',
                      'last_modified': '123T456'}]],
             [None, []],
@@ -567,9 +570,11 @@ def test_list_container(self, connection):
             connection.return_value.get_container.assert_has_calls(calls)
 
             self.assertEqual(output.out,
-                             '           0        123      456'
+                             '           3        123      456'
                              '             type/content object_a\n'
-                             '           0\n')
+                             '           5        123      456'
+                             '             type/content object_b\n'
+                             '           8\n')
 
     @mock.patch('swiftclient.service.Connection')
     def test_list_container_with_headers(self, connection):
