Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 031c4cc

Browse files
robertwbrobertwb
committed
Move names out of transform constructors.
sed -i -r 's/[|] (\S+)[(](["'"'"'][^"'"'"']+.)(, +|([)]))/| \2 >> \1(\4/g' Small number of tests will need to be fixed by hand.
1 parent 937cf69 commit 031c4cc

48 files changed

Lines changed: 537 additions & 537 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎sdks/python/apache_beam/dataflow_test.py‎

Lines changed: 48 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -54,33 +54,33 @@ class DataflowTest(unittest.TestCase):
5454
def Count(pcoll): # pylint: disable=invalid-name, no-self-argument
5555
"""A Count transform: v, ... => (v, n), ..."""
5656
return (pcoll
57-
| Map('AddCount', lambda x: (x, 1))
58-
| GroupByKey('GroupCounts')
59-
| Map('AddCounts', lambda (x, ones): (x, sum(ones))))
57+
| 'AddCount' >> Map(lambda x: (x, 1))
58+
| 'GroupCounts' >> GroupByKey()
59+
| 'AddCounts' >> Map(lambda (x, ones): (x, sum(ones))))
6060

6161
def test_word_count(self):
6262
pipeline = Pipeline('DirectPipelineRunner')
63-
lines = pipeline | Create('SomeWords', DataflowTest.SAMPLE_DATA)
63+
lines = pipeline | 'SomeWords' >> Create(DataflowTest.SAMPLE_DATA)
6464
result = (
65-
(lines | FlatMap('GetWords', lambda x: re.findall(r'\w+', x)))
65+
(lines | 'GetWords' >> FlatMap(lambda x: re.findall(r'\w+', x)))
6666
.apply('CountWords', DataflowTest.Count))
6767
assert_that(result, equal_to(DataflowTest.SAMPLE_RESULT))
6868
pipeline.run()
6969

7070
def test_map(self):
7171
pipeline = Pipeline('DirectPipelineRunner')
72-
lines = pipeline | Create('input', ['a', 'b', 'c'])
72+
lines = pipeline | 'input' >> Create(['a', 'b', 'c'])
7373
result = (lines
74-
| Map('upper', str.upper)
75-
| Map('prefix', lambda x, prefix: prefix + x, 'foo-'))
74+
| 'upper' >> Map(str.upper)
75+
| 'prefix' >> Map(lambda x, prefix: prefix + x, 'foo-'))
7676
assert_that(result, equal_to(['foo-A', 'foo-B', 'foo-C']))
7777
pipeline.run()
7878

7979
def test_par_do_with_side_input_as_arg(self):
8080
pipeline = Pipeline('DirectPipelineRunner')
8181
words_list = ['aa', 'bb', 'cc']
82-
words = pipeline | Create('SomeWords', words_list)
83-
prefix = pipeline | Create('SomeString', ['xyz']) # side in
82+
words = pipeline | 'SomeWords' >> Create(words_list)
83+
prefix = pipeline | 'SomeString' >> Create(['xyz']) # side in
8484
suffix = 'zyx'
8585
result = words | FlatMap(
8686
'DecorateWords',
@@ -92,9 +92,9 @@ def test_par_do_with_side_input_as_arg(self):
9292
def test_par_do_with_side_input_as_keyword_arg(self):
9393
pipeline = Pipeline('DirectPipelineRunner')
9494
words_list = ['aa', 'bb', 'cc']
95-
words = pipeline | Create('SomeWords', words_list)
95+
words = pipeline | 'SomeWords' >> Create(words_list)
9696
prefix = 'zyx'
97-
suffix = pipeline | Create('SomeString', ['xyz']) # side in
97+
suffix = pipeline | 'SomeString' >> Create(['xyz']) # side in
9898
result = words | FlatMap(
9999
'DecorateWords',
100100
lambda x, pfx, sfx: ['%s-%s-%s' % (pfx, x, sfx)],
@@ -111,10 +111,10 @@ def process(self, context, prefix, suffix):
111111

112112
pipeline = Pipeline('DirectPipelineRunner')
113113
words_list = ['aa', 'bb', 'cc']
114-
words = pipeline | Create('SomeWords', words_list)
114+
words = pipeline | 'SomeWords' >> Create(words_list)
115115
prefix = 'zyx'
116-
suffix = pipeline | Create('SomeString', ['xyz']) # side in
117-
result = words | ParDo('DecorateWordsDoFn', SomeDoFn(), prefix,
116+
suffix = pipeline | 'SomeString' >> Create(['xyz']) # side in
117+
result = words | 'DecorateWordsDoFn' >> ParDo(SomeDoFn(), prefix,
118118
suffix=AsSingleton(suffix))
119119
assert_that(result, equal_to(['zyx-%s-xyz' % x for x in words_list]))
120120
pipeline.run()
@@ -131,7 +131,7 @@ def process(self, context):
131131
yield SideOutputValue('odd', context.element)
132132

133133
pipeline = Pipeline('DirectPipelineRunner')
134-
nums = pipeline | Create('Some Numbers', [1, 2, 3, 4])
134+
nums = pipeline | 'Some Numbers' >> Create([1, 2, 3, 4])
135135
results = nums | ParDo(
136136
'ClassifyNumbers', SomeDoFn()).with_outputs('odd', 'even', main='main')
137137
assert_that(results.main, equal_to([1, 2, 3, 4]))
@@ -147,7 +147,7 @@ def some_fn(v):
147147
return [v, SideOutputValue('odd', v)]
148148

149149
pipeline = Pipeline('DirectPipelineRunner')
150-
nums = pipeline | Create('Some Numbers', [1, 2, 3, 4])
150+
nums = pipeline | 'Some Numbers' >> Create([1, 2, 3, 4])
151151
results = nums | FlatMap(
152152
'ClassifyNumbers', some_fn).with_outputs('odd', 'even', main='main')
153153
assert_that(results.main, equal_to([1, 2, 3, 4]))
@@ -157,45 +157,45 @@ def some_fn(v):
157157

158158
def test_empty_singleton_side_input(self):
159159
pipeline = Pipeline('DirectPipelineRunner')
160-
pcol = pipeline | Create('start', [1, 2])
161-
side = pipeline | Create('side', []) # Empty side input.
160+
pcol = pipeline | 'start' >> Create([1, 2])
161+
side = pipeline | 'side' >> Create([]) # Empty side input.
162162

163163
def my_fn(k, s):
164164
v = ('empty' if isinstance(s, EmptySideInput) else 'full')
165165
return [(k, v)]
166-
result = pcol | FlatMap('compute', my_fn, AsSingleton(side))
166+
result = pcol | 'compute' >> FlatMap(my_fn, AsSingleton(side))
167167
assert_that(result, equal_to([(1, 'empty'), (2, 'empty')]))
168168
pipeline.run()
169169

170170
def test_multi_valued_singleton_side_input(self):
171171
pipeline = Pipeline('DirectPipelineRunner')
172-
pcol = pipeline | Create('start', [1, 2])
173-
side = pipeline | Create('side', [3, 4]) # 2 values in side input.
174-
pcol | FlatMap('compute', lambda x, s: [x * s], AsSingleton(side)) # pylint: disable=expression-not-assigned
172+
pcol = pipeline | 'start' >> Create([1, 2])
173+
side = pipeline | 'side' >> Create([3, 4]) # 2 values in side input.
174+
pcol | 'compute' >> FlatMap(lambda x, s: [x * s], AsSingleton(side)) # pylint: disable=expression-not-assigned
175175
with self.assertRaises(ValueError):
176176
pipeline.run()
177177

178178
def test_default_value_singleton_side_input(self):
179179
pipeline = Pipeline('DirectPipelineRunner')
180-
pcol = pipeline | Create('start', [1, 2])
181-
side = pipeline | Create('side', []) # 0 values in side input.
180+
pcol = pipeline | 'start' >> Create([1, 2])
181+
side = pipeline | 'side' >> Create([]) # 0 values in side input.
182182
result = (
183-
pcol | FlatMap('compute', lambda x, s: [x * s], AsSingleton(side, 10)))
183+
pcol | 'compute' >> FlatMap(lambda x, s: [x * s], AsSingleton(side, 10)))
184184
assert_that(result, equal_to([10, 20]))
185185
pipeline.run()
186186

187187
def test_iterable_side_input(self):
188188
pipeline = Pipeline('DirectPipelineRunner')
189-
pcol = pipeline | Create('start', [1, 2])
190-
side = pipeline | Create('side', [3, 4]) # 2 values in side input.
189+
pcol = pipeline | 'start' >> Create([1, 2])
190+
side = pipeline | 'side' >> Create([3, 4]) # 2 values in side input.
191191
result = pcol | FlatMap('compute',
192192
lambda x, s: [x * y for y in s], AllOf(side))
193193
assert_that(result, equal_to([3, 4, 6, 8]))
194194
pipeline.run()
195195

196196
def test_undeclared_side_outputs(self):
197197
pipeline = Pipeline('DirectPipelineRunner')
198-
nums = pipeline | Create('Some Numbers', [1, 2, 3, 4])
198+
nums = pipeline | 'Some Numbers' >> Create([1, 2, 3, 4])
199199
results = nums | FlatMap(
200200
'ClassifyNumbers',
201201
lambda x: [x, SideOutputValue('even' if x % 2 == 0 else 'odd', x)]
@@ -210,7 +210,7 @@ def test_undeclared_side_outputs(self):
210210

211211
def test_empty_side_outputs(self):
212212
pipeline = Pipeline('DirectPipelineRunner')
213-
nums = pipeline | Create('Some Numbers', [1, 3, 5])
213+
nums = pipeline | 'Some Numbers' >> Create([1, 3, 5])
214214
results = nums | FlatMap(
215215
'ClassifyNumbers',
216216
lambda x: [x, SideOutputValue('even' if x % 2 == 0 else 'odd', x)]
@@ -224,9 +224,9 @@ def test_as_list_and_as_dict_side_inputs(self):
224224
a_list = [5, 1, 3, 2, 9]
225225
some_pairs = [('crouton', 17), ('supreme', None)]
226226
pipeline = Pipeline('DirectPipelineRunner')
227-
main_input = pipeline | Create('main input', [1])
228-
side_list = pipeline | Create('side list', a_list)
229-
side_pairs = pipeline | Create('side pairs', some_pairs)
227+
main_input = pipeline | 'main input' >> Create([1])
228+
side_list = pipeline | 'side list' >> Create(a_list)
229+
side_pairs = pipeline | 'side pairs' >> Create(some_pairs)
230230
results = main_input | FlatMap(
231231
'concatenate',
232232
lambda x, the_list, the_dict: [[x, the_list, the_dict]],
@@ -248,8 +248,8 @@ def test_as_singleton_without_unique_labels(self):
248248
# with the same defaults will return the same PCollectionView.
249249
a_list = [2]
250250
pipeline = Pipeline('DirectPipelineRunner')
251-
main_input = pipeline | Create('main input', [1])
252-
side_list = pipeline | Create('side list', a_list)
251+
main_input = pipeline | 'main input' >> Create([1])
252+
side_list = pipeline | 'side list' >> Create(a_list)
253253
results = main_input | FlatMap(
254254
'test',
255255
lambda x, s1, s2: [[x, s1, s2]],
@@ -271,8 +271,8 @@ def test_as_singleton_with_different_defaults_without_unique_labels(self):
271271
# distinct PCollectionViews with the same full_label.
272272
a_list = [2]
273273
pipeline = Pipeline('DirectPipelineRunner')
274-
main_input = pipeline | Create('main input', [1])
275-
side_list = pipeline | Create('side list', a_list)
274+
main_input = pipeline | 'main input' >> Create([1])
275+
side_list = pipeline | 'side list' >> Create(a_list)
276276

277277
with self.assertRaises(RuntimeError) as e:
278278
_ = main_input | FlatMap(
@@ -287,8 +287,8 @@ def test_as_singleton_with_different_defaults_without_unique_labels(self):
287287
def test_as_singleton_with_different_defaults_with_unique_labels(self):
288288
a_list = []
289289
pipeline = Pipeline('DirectPipelineRunner')
290-
main_input = pipeline | Create('main input', [1])
291-
side_list = pipeline | Create('side list', a_list)
290+
main_input = pipeline | 'main input' >> Create([1])
291+
side_list = pipeline | 'side list' >> Create(a_list)
292292
results = main_input | FlatMap(
293293
'test',
294294
lambda x, s1, s2: [[x, s1, s2]],
@@ -311,8 +311,8 @@ def test_as_list_without_unique_labels(self):
311311
# return the same PCollectionView.
312312
a_list = [1, 2, 3]
313313
pipeline = Pipeline('DirectPipelineRunner')
314-
main_input = pipeline | Create('main input', [1])
315-
side_list = pipeline | Create('side list', a_list)
314+
main_input = pipeline | 'main input' >> Create([1])
315+
side_list = pipeline | 'side list' >> Create(a_list)
316316
results = main_input | FlatMap(
317317
'test',
318318
lambda x, ls1, ls2: [[x, ls1, ls2]],
@@ -332,8 +332,8 @@ def match(actual):
332332
def test_as_list_with_unique_labels(self):
333333
a_list = [1, 2, 3]
334334
pipeline = Pipeline('DirectPipelineRunner')
335-
main_input = pipeline | Create('main input', [1])
336-
side_list = pipeline | Create('side list', a_list)
335+
main_input = pipeline | 'main input' >> Create([1])
336+
side_list = pipeline | 'side list' >> Create(a_list)
337337
results = main_input | FlatMap(
338338
'test',
339339
lambda x, ls1, ls2: [[x, ls1, ls2]],
@@ -353,8 +353,8 @@ def match(actual):
353353
def test_as_dict_with_unique_labels(self):
354354
some_kvs = [('a', 1), ('b', 2)]
355355
pipeline = Pipeline('DirectPipelineRunner')
356-
main_input = pipeline | Create('main input', [1])
357-
side_kvs = pipeline | Create('side kvs', some_kvs)
356+
main_input = pipeline | 'main input' >> Create([1])
357+
side_kvs = pipeline | 'side kvs' >> Create(some_kvs)
358358
results = main_input | FlatMap(
359359
'test',
360360
lambda x, dct1, dct2: [[x, dct1, dct2]],
@@ -383,10 +383,10 @@ def merge(self, existing_windows):
383383
return existing_windows
384384

385385
pipeline = Pipeline('DirectPipelineRunner')
386-
numbers = pipeline | Create('KVs', [(1, 10), (2, 20), (3, 30)])
386+
numbers = pipeline | 'KVs' >> Create([(1, 10), (2, 20), (3, 30)])
387387
result = (numbers
388-
| WindowInto('W', windowfn=TestWindowFn())
389-
| GroupByKey('G'))
388+
| 'W' >> WindowInto(windowfn=TestWindowFn())
389+
| 'G' >> GroupByKey())
390390
assert_that(
391391
result, equal_to([(1, [10]), (1, [10]), (2, [20]),
392392
(2, [20]), (3, [30]), (3, [30])]))

‎sdks/python/apache_beam/examples/complete/autocomplete.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,12 +45,12 @@ def run(argv=None):
4545
p = beam.Pipeline(options=pipeline_options)
4646

4747
(p # pylint: disable=expression-not-assigned
48-
| beam.io.Read('read', beam.io.TextFileSource(known_args.input))
49-
| beam.FlatMap('split', lambda x: re.findall(r'[A-Za-z\']+', x))
50-
| TopPerPrefix('TopPerPrefix', 5)
48+
| 'read' >> beam.io.Read(beam.io.TextFileSource(known_args.input))
49+
| 'split' >> beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
50+
| 'TopPerPrefix' >> TopPerPrefix(5)
5151
| beam.Map('format',
5252
lambda (prefix, candidates): '%s: %s' % (prefix, candidates))
53-
| beam.io.Write('write', beam.io.TextFileSink(known_args.output)))
53+
| 'write' >> beam.io.Write(beam.io.TextFileSink(known_args.output)))
5454
p.run()
5555

5656

‎sdks/python/apache_beam/examples/complete/autocomplete_test.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ class AutocompleteTest(unittest.TestCase):
3131

3232
def test_top_prefixes(self):
3333
p = beam.Pipeline('DirectPipelineRunner')
34-
words = p | beam.Create('create', self.WORDS)
35-
result = words | autocomplete.TopPerPrefix('test', 5)
34+
words = p | 'create' >> beam.Create(self.WORDS)
35+
result = words | 'test' >> autocomplete.TopPerPrefix(5)
3636
# values must be hashable for now
3737
result = result | beam.Map(lambda (k, vs): (k, tuple(vs)))
3838
assert_that(result, equal_to(

‎sdks/python/apache_beam/examples/complete/estimate_pi.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -96,9 +96,9 @@ def __init__(self, label):
9696
def apply(self, pcoll):
9797
# A hundred work items of a hundred thousand tries each.
9898
return (pcoll
99-
| beam.Create('Initialize', [100000] * 100).with_output_types(int)
100-
| beam.Map('Run trials', run_trials)
101-
| beam.CombineGlobally('Sum', combine_results).without_defaults())
99+
| 'Initialize' >> beam.Create([100000] * 100).with_output_types(int)
100+
| 'Run trials' >> beam.Map(run_trials)
101+
| 'Sum' >> beam.CombineGlobally(combine_results).without_defaults())
102102

103103

104104
def run(argv=None):
@@ -115,7 +115,7 @@ def run(argv=None):
115115
p = beam.Pipeline(options=pipeline_options)
116116

117117
(p # pylint: disable=expression-not-assigned
118-
| EstimatePiTransform('Estimate')
118+
| 'Estimate' >> EstimatePiTransform()
119119
| beam.io.Write('Write',
120120
beam.io.TextFileSink(known_args.output,
121121
coder=JsonCoder())))

‎sdks/python/apache_beam/examples/complete/estimate_pi_test.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ class EstimatePiTest(unittest.TestCase):
3939

4040
def test_basics(self):
4141
p = beam.Pipeline('DirectPipelineRunner')
42-
result = p | estimate_pi.EstimatePiTransform('Estimate')
42+
result = p | 'Estimate' >> estimate_pi.EstimatePiTransform()
4343

4444
# Note: Probabilistically speaking this test can fail with a probability
4545
# that is very small (VERY) given that we run at least 10 million trials.

‎sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ def point_set(n):
5050
yield (x, y)
5151

5252
julia_set_colors = (pipeline
53-
| beam.Create('add points', point_set(n))
53+
| 'add points' >> beam.Create(point_set(n))
5454
| beam.Map(
5555
get_julia_set_point_color, c, n, max_iterations))
5656

@@ -105,11 +105,11 @@ def run(argv=None): # pylint: disable=missing-docstring
105105
# Group each coordinate triplet by its x value, then write the coordinates to
106106
# the output file with an x-coordinate grouping per line.
107107
# pylint: disable=expression-not-assigned
108-
(coordinates | beam.Map('x coord key', lambda (x, y, i): (x, (x, y, i)))
109-
| beam.GroupByKey('x coord') | beam.Map(
108+
(coordinates | 'x coord key' >> beam.Map(lambda (x, y, i): (x, (x, y, i)))
109+
| 'x coord' >> beam.GroupByKey() | beam.Map(
110110
'format',
111111
lambda (k, coords): ' '.join('(%s, %s, %s)' % coord for coord in coords))
112-
| beam.io.Write('write', beam.io.TextFileSink(known_args.coordinate_output)))
112+
| 'write' >> beam.io.Write(beam.io.TextFileSink(known_args.coordinate_output)))
113113
# pylint: enable=expression-not-assigned
114114
p.run()
115115

0 commit comments

Comments
 (0)