Deprecate obsolete CombineFn.add_inputs. (#12802)

robertwb · web-flow · commit e76e599312e8 · 2020-09-11T15:52:36.000-07:00
diff --git a/sdks/python/apache_beam/transforms/combiners.py b/sdks/python/apache_beam/transforms/combiners.py
@@ -23,6 +23,7 @@
 from __future__ import division
 
 import copy
+import functools
 import heapq
 import operator
 import random
@@ -162,9 +163,6 @@ def create_accumulator(self):
   def add_input(self, accumulator, element):
     return accumulator + 1
 
-  def add_inputs(self, accumulator, elements):
-    return accumulator + len(list(elements))
-
   def merge_accumulators(self, accumulators):
     return sum(accumulators)
 
@@ -938,8 +936,10 @@ def full_combine(self, elements):
     return self.combine_fn.apply(elements)
 
   def add_only(self, elements):
-    return self.combine_fn.add_inputs(
-        self.combine_fn.create_accumulator(), elements)
+    return functools.reduce(
+        self.combine_fn.add_input,
+        elements,
+        self.combine_fn.create_accumulator())
 
   def merge_only(self, accumulators):
     return self.combine_fn.merge_accumulators(accumulators)
diff --git a/sdks/python/apache_beam/transforms/combiners_test.py b/sdks/python/apache_beam/transforms/combiners_test.py
@@ -21,6 +21,7 @@
 from __future__ import absolute_import
 from __future__ import division
 
+import functools
 import itertools
 import random
 import sys
@@ -252,7 +253,8 @@ def test_top_key_py2(self):
   def test_sharded_top_combine_fn(self):
     def test_combine_fn(combine_fn, shards, expected):
       accumulators = [
-          combine_fn.add_inputs(combine_fn.create_accumulator(), shard)
+          functools.reduce(
+              combine_fn.add_input, shard, combine_fn.create_accumulator())
           for shard in shards
       ]
       final_accumulator = combine_fn.merge_accumulators(accumulators)
diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py
@@ -22,6 +22,7 @@
 from __future__ import absolute_import
 
 import copy
+import functools
 import inspect
 import logging
 import random
@@ -918,7 +919,9 @@ def add_input(self, mutable_accumulator, element, *args, **kwargs):
     raise NotImplementedError(str(self))
 
   def add_inputs(self, mutable_accumulator, elements, *args, **kwargs):
-    """Returns the result of folding each element in elements into accumulator.
+    """DEPRECATED and unused.
+
+    Returns the result of folding each element in elements into accumulator.
 
     This is provided in case the implementation affords more efficient
     bulk addition of elements. The default implementation simply loops
@@ -988,10 +991,14 @@ def apply(self, elements, *args, **kwargs):
       *args: Additional arguments and side inputs.
       **kwargs: Additional arguments and side inputs.
     """
+    if args or kwargs:
+      add_input = lambda accumulator, element: self.add_input(
+          accumulator, element, *args, **kwargs)
+    else:
+      add_input = self.add_input
     return self.extract_output(
-        self.add_inputs(
-            self.create_accumulator(*args, **kwargs), elements, *args,
-            **kwargs),
+        functools.reduce(
+            add_input, elements, self.create_accumulator(*args, **kwargs)),
         *args,
         **kwargs)
 
@@ -1088,12 +1095,6 @@ def add_input(self, accumulator, element, *args, **kwargs):
       accumulator = [self._fn(accumulator, *args, **kwargs)]
     return accumulator
 
-  def add_inputs(self, accumulator, elements, *args, **kwargs):
-    accumulator.extend(elements)
-    if len(accumulator) > self._buffer_size:
-      accumulator = [self._fn(accumulator, *args, **kwargs)]
-    return accumulator
-
   def merge_accumulators(self, accumulators, *args, **kwargs):
     return [self._fn(_ReiterableChain(accumulators), *args, **kwargs)]
 
@@ -1165,12 +1166,6 @@ def add_input(self, accumulator, element):
       accumulator = [self._fn(accumulator)]
     return accumulator
 
-  def add_inputs(self, accumulator, elements):
-    accumulator.extend(elements)
-    if len(accumulator) > self._buffer_size:
-      accumulator = [self._fn(accumulator)]
-    return accumulator
-
   def merge_accumulators(self, accumulators):
     return [self._fn(_ReiterableChain(accumulators))]
 
@@ -2182,31 +2177,7 @@ def process(self, element, *args, **kwargs):
     # Expected elements input to this DoFn are 2-tuples of the form
     # (key, iter), with iter an iterable of all the values associated with key
     # in the input PCollection.
-    if self.runtime_type_check:
-      # Apply the combiner in a single operation rather than artificially
-      # breaking it up so that output type violations manifest as TypeCheck
-      # errors rather than type errors.
-      return [(element[0], self.combinefn.apply(element[1], *args, **kwargs))]
-
-    # Add the elements into three accumulators (for testing of merge).
-    elements = list(element[1])
-    accumulators = []
-    for k in range(3):
-      if len(elements) <= k:
-        break
-      accumulators.append(
-          self.combinefn.add_inputs(
-              self.combinefn.create_accumulator(*args, **kwargs),
-              elements[k::3],
-              *args,
-              **kwargs))
-    # Merge the accumulators.
-    accumulator = self.combinefn.merge_accumulators(
-        accumulators, *args, **kwargs)
-    # Convert accumulator to the final result.
-    return [(
-        element[0], self.combinefn.extract_output(accumulator, *args,
-                                                  **kwargs))]
+    yield element[0], self.combinefn.apply(element[1], *args, **kwargs)
 
   def default_type_hints(self):
     hints = self.combinefn.get_type_hints()
diff --git a/sdks/python/apache_beam/transforms/stats.py b/sdks/python/apache_beam/transforms/stats.py
@@ -724,27 +724,6 @@ def add_input(self, quantile_state, element):
     self._add_unbuffered(quantile_state, elements=[element])
     return quantile_state
 
-  def add_inputs(self, quantile_state, elements):
-    """Add new elements to the collection being summarized by quantile state.
-    """
-    if not elements:
-      return quantile_state
-
-    values = [
-        element[0] for element in elements
-    ] if self._weighted else elements
-    min_val = min(values)
-    max_val = max(values)
-    if quantile_state.is_empty():
-      quantile_state.min_val = min_val
-      quantile_state.max_val = max_val
-    elif self._comparator(min_val, quantile_state.min_val) < 0:
-      quantile_state.min_val = min_val
-    elif self._comparator(max_val, quantile_state.max_val) > 0:
-      quantile_state.max_val = max_val
-    self._add_unbuffered(quantile_state, elements=elements)
-    return quantile_state
-
   def merge_accumulators(self, accumulators):
     """Merges all the accumulators (quantile state) as one."""
     qs = self.create_accumulator()
