From 95cb4f3221ad1078f49d98be9774792a0c319604 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 24 Dec 2023 18:02:20 -0600 Subject: [PATCH 1/9] Remove distracting comments --- Doc/library/itertools.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index c016fb76bfd0a0..3747d05cc28988 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1007,19 +1007,12 @@ which incur interpreter overhead. for element in filterfalse(seen.__contains__, iterable): seen.add(element) yield element - # For order preserving deduplication, - # a faster but non-lazy solution is: - # yield from dict.fromkeys(iterable) else: for element in iterable: k = key(element) if k not in seen: seen.add(k) yield element - # For use cases that allow the last matching element to be returned, - # a faster but non-lazy solution is: - # t1, t2 = tee(iterable) - # yield from dict(zip(map(key, t1), t2)).values() def unique_justseen(iterable, key=None): "List unique elements, preserving order. Remember only the element just seen." From 3d97e31b08436b4e710b38c21508a8871fd048d7 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 24 Dec 2023 18:06:05 -0600 Subject: [PATCH 2/9] Switch example to use casefold for cose insensitive compares --- Doc/library/itertools.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 3747d05cc28988..05484a6e5672f9 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1001,7 +1001,7 @@ which incur interpreter overhead. def unique_everseen(iterable, key=None): "List unique elements, preserving order. Remember all elements ever seen." # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBcCAD', str.lower) --> A B c D + # unique_everseen('ABBcCAD', str.casefold) --> A B c D seen = set() if key is None: for element in filterfalse(seen.__contains__, iterable): @@ -1017,7 +1017,7 @@ which incur interpreter overhead. def unique_justseen(iterable, key=None): "List unique elements, preserving order. Remember only the element just seen." # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B - # unique_justseen('ABBcCAD', str.lower) --> A B c A D + # unique_justseen('ABBcCAD', str.casefold) --> A B c A D if key is None: return map(operator.itemgetter(0), groupby(iterable)) return map(next, map(operator.itemgetter(1), groupby(iterable, key))) @@ -1555,16 +1555,16 @@ The following recipes have a more mathematical flavor: >>> list(unique_everseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBCcAD', str.lower)) + >>> list(unique_everseen('ABBCcAD', str.casefold)) ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBcCAD', str.lower)) + >>> list(unique_everseen('ABBcCAD', str.casefold)) ['A', 'B', 'c', 'D'] >>> list(unique_justseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D', 'A', 'B'] - >>> list(unique_justseen('ABBCcAD', str.lower)) + >>> list(unique_justseen('ABBCcAD', str.casefold)) ['A', 'B', 'C', 'A', 'D'] - >>> list(unique_justseen('ABBcCAD', str.lower)) + >>> list(unique_justseen('ABBcCAD', str.casefold)) ['A', 'B', 'c', 'A', 'D'] >>> d = dict(a=1, b=2, c=3) From 2cb8b0e8315141bb44c2747a315016cf30972d93 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 25 Dec 2023 14:53:49 -0600 Subject: [PATCH 3/9] Use chain() instead of a generator in before_and_after() --- Doc/library/itertools.rst | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 05484a6e5672f9..e0c00f663b5e11 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -980,9 +980,8 @@ which incur interpreter overhead. >>> ''.join(remainder) # takewhile() would lose the 'd' 'dEfGhI' - Note that the first iterator must be fully - consumed before the second iterator can - generate valid results. + Note that the true iterator must be fully consumed + before the remainder iterator can generate valid results. """ it = iter(it) transition = [] @@ -993,10 +992,7 @@ which incur interpreter overhead. else: transition.append(elem) return - def remainder_iterator(): - yield from transition - yield from it - return true_iterator(), remainder_iterator() + return true_iterator(), chain(transition, it) def unique_everseen(iterable, key=None): "List unique elements, preserving order. Remember all elements ever seen." From e4bc16fe2ef02de2ba2732e51fd02e71edc03382 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 25 Dec 2023 14:57:20 -0600 Subject: [PATCH 4/9] Shorten the max line length in the iter_except() docstring. --- Doc/library/itertools.rst | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index e0c00f663b5e11..2b774b9ded851c 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -900,12 +900,20 @@ which incur interpreter overhead. Like builtins.iter(func, sentinel) but uses an exception instead of a sentinel to end the loop. - Examples: - iter_except(functools.partial(heappop, h), IndexError) # priority queue iterator - iter_except(d.popitem, KeyError) # non-blocking dict iterator - iter_except(d.popleft, IndexError) # non-blocking deque iterator - iter_except(q.get_nowait, Queue.Empty) # loop over a producer Queue - iter_except(s.pop, KeyError) # non-blocking set iterator + Priority queue iterator: + iter_except(functools.partial(heappop, h), IndexError) + + Non-blocking dictionary iterator: + iter_except(d.popitem, KeyError) + + Non-blocking deque iterator: + iter_except(d.popleft, IndexError) + + Non-blocking iterator over a producer Queue: + iter_except(q.get_nowait, Queue.Empty) + + Non-blocking set iterator: + iter_except(s.pop, KeyError) """ try: From 7d9080eff99607c30d2eb541466b01520295a2ec Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 25 Dec 2023 15:01:14 -0600 Subject: [PATCH 5/9] Add docstring to sliding_window and move it before the grouper() recipe. --- Doc/library/itertools.rst | 65 ++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 2b774b9ded851c..2a4dffc723a62c 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -924,6 +924,15 @@ which incur interpreter overhead. except exception: pass + def sliding_window(iterable, n): + "Collect data into overlapping fixed-length chunks or blocks" + # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG + it = iter(iterable) + window = collections.deque(islice(it, n-1), maxlen=n) + for x in it: + window.append(x) + yield tuple(window) + def grouper(iterable, n, *, incomplete='fill', fillvalue=None): "Collect data into non-overlapping fixed-length chunks or blocks" # grouper('ABCDEFG', 3, fillvalue='x') --> ABC DEF Gxx @@ -940,14 +949,6 @@ which incur interpreter overhead. case _: raise ValueError('Expected fill, strict, or ignore') - def sliding_window(iterable, n): - # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG - it = iter(iterable) - window = collections.deque(islice(it, n-1), maxlen=n) - for x in it: - window.append(x) - yield tuple(window) - def roundrobin(*iterables): "roundrobin('ABC', 'D', 'EF') --> A D E B F C" # Recipe credited to George Sakkis @@ -977,6 +978,30 @@ which incur interpreter overhead. slices = starmap(slice, combinations(range(len(seq) + 1), 2)) return map(operator.getitem, repeat(seq), slices) + def unique_everseen(iterable, key=None): + "List unique elements, preserving order. Remember all elements ever seen." + # unique_everseen('AAAABBBCCDAABBB') --> A B C D + # unique_everseen('ABBcCAD', str.casefold) --> A B c D + seen = set() + if key is None: + for element in filterfalse(seen.__contains__, iterable): + seen.add(element) + yield element + else: + for element in iterable: + k = key(element) + if k not in seen: + seen.add(k) + yield element + + def unique_justseen(iterable, key=None): + "List unique elements, preserving order. Remember only the element just seen." + # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B + # unique_justseen('ABBcCAD', str.casefold) --> A B c A D + if key is None: + return map(operator.itemgetter(0), groupby(iterable)) + return map(next, map(operator.itemgetter(1), groupby(iterable, key))) + def before_and_after(predicate, it): """ Variant of takewhile() that allows complete access to the remainder of the iterator. @@ -1002,30 +1027,6 @@ which incur interpreter overhead. return return true_iterator(), chain(transition, it) - def unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBcCAD', str.casefold) --> A B c D - seen = set() - if key is None: - for element in filterfalse(seen.__contains__, iterable): - seen.add(element) - yield element - else: - for element in iterable: - k = key(element) - if k not in seen: - seen.add(k) - yield element - - def unique_justseen(iterable, key=None): - "List unique elements, preserving order. Remember only the element just seen." - # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B - # unique_justseen('ABBcCAD', str.casefold) --> A B c A D - if key is None: - return map(operator.itemgetter(0), groupby(iterable)) - return map(next, map(operator.itemgetter(1), groupby(iterable, key))) - The following recipes have a more mathematical flavor: From 8e5131ef1ca5592e4e4724ef1bc07c702b779086 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 25 Dec 2023 15:03:05 -0600 Subject: [PATCH 6/9] Move iter_except() recipe to near the bottom --- Doc/library/itertools.rst | 62 +++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 2a4dffc723a62c..64d072ccd8de7d 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -893,37 +893,6 @@ which incur interpreter overhead. except ValueError: pass - def iter_except(func, exception, first=None): - """ Call a function repeatedly until an exception is raised. - - Converts a call-until-exception interface to an iterator interface. - Like builtins.iter(func, sentinel) but uses an exception instead - of a sentinel to end the loop. - - Priority queue iterator: - iter_except(functools.partial(heappop, h), IndexError) - - Non-blocking dictionary iterator: - iter_except(d.popitem, KeyError) - - Non-blocking deque iterator: - iter_except(d.popleft, IndexError) - - Non-blocking iterator over a producer Queue: - iter_except(q.get_nowait, Queue.Empty) - - Non-blocking set iterator: - iter_except(s.pop, KeyError) - - """ - try: - if first is not None: - yield first() # For database APIs needing an initial cast to db.first() - while True: - yield func() - except exception: - pass - def sliding_window(iterable, n): "Collect data into overlapping fixed-length chunks or blocks" # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG @@ -1002,6 +971,37 @@ which incur interpreter overhead. return map(operator.itemgetter(0), groupby(iterable)) return map(next, map(operator.itemgetter(1), groupby(iterable, key))) + def iter_except(func, exception, first=None): + """ Call a function repeatedly until an exception is raised. + + Converts a call-until-exception interface to an iterator interface. + Like builtins.iter(func, sentinel) but uses an exception instead + of a sentinel to end the loop. + + Priority queue iterator: + iter_except(functools.partial(heappop, h), IndexError) + + Non-blocking dictionary iterator: + iter_except(d.popitem, KeyError) + + Non-blocking deque iterator: + iter_except(d.popleft, IndexError) + + Non-blocking iterator over a producer Queue: + iter_except(q.get_nowait, Queue.Empty) + + Non-blocking set iterator: + iter_except(s.pop, KeyError) + + """ + try: + if first is not None: + yield first() # For database APIs needing an initial cast to db.first() + while True: + yield func() + except exception: + pass + def before_and_after(predicate, it): """ Variant of takewhile() that allows complete access to the remainder of the iterator. From 5183085e34f82d57827b4316362d1213d2c802bb Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 25 Dec 2023 15:05:37 -0600 Subject: [PATCH 7/9] Move the uniquification recipes upwards --- Doc/library/itertools.rst | 48 +++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 64d072ccd8de7d..7aae37b2fb41ce 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -873,6 +873,30 @@ which incur interpreter overhead. # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x return next(filter(pred, iterable), default) + def unique_everseen(iterable, key=None): + "List unique elements, preserving order. Remember all elements ever seen." + # unique_everseen('AAAABBBCCDAABBB') --> A B C D + # unique_everseen('ABBcCAD', str.casefold) --> A B c D + seen = set() + if key is None: + for element in filterfalse(seen.__contains__, iterable): + seen.add(element) + yield element + else: + for element in iterable: + k = key(element) + if k not in seen: + seen.add(k) + yield element + + def unique_justseen(iterable, key=None): + "List unique elements, preserving order. Remember only the element just seen." + # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B + # unique_justseen('ABBcCAD', str.casefold) --> A B c A D + if key is None: + return map(operator.itemgetter(0), groupby(iterable)) + return map(next, map(operator.itemgetter(1), groupby(iterable, key))) + def iter_index(iterable, value, start=0, stop=None): "Return indices where a value occurs in a sequence or iterable." # iter_index('AABCADEAF', 'A') --> 0 1 4 7 @@ -947,30 +971,6 @@ which incur interpreter overhead. slices = starmap(slice, combinations(range(len(seq) + 1), 2)) return map(operator.getitem, repeat(seq), slices) - def unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBcCAD', str.casefold) --> A B c D - seen = set() - if key is None: - for element in filterfalse(seen.__contains__, iterable): - seen.add(element) - yield element - else: - for element in iterable: - k = key(element) - if k not in seen: - seen.add(k) - yield element - - def unique_justseen(iterable, key=None): - "List unique elements, preserving order. Remember only the element just seen." - # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B - # unique_justseen('ABBcCAD', str.casefold) --> A B c A D - if key is None: - return map(operator.itemgetter(0), groupby(iterable)) - return map(next, map(operator.itemgetter(1), groupby(iterable, key))) - def iter_except(func, exception, first=None): """ Call a function repeatedly until an exception is raised. From d370b6c2c3aae8609ef8dc631e02c09ea06c0b61 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 25 Dec 2023 15:10:16 -0600 Subject: [PATCH 8/9] Fix typo in comment --- Doc/library/itertools.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 7aae37b2fb41ce..85d243f6e4b2d6 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -996,7 +996,8 @@ which incur interpreter overhead. """ try: if first is not None: - yield first() # For database APIs needing an initial cast to db.first() + # For database APIs needing an initial call to db.first() + yield first() while True: yield func() except exception: From 82830cc63fc3afe5a62244cecd194a518bf165e4 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 25 Dec 2023 15:16:58 -0600 Subject: [PATCH 9/9] Consistent periods at the end of docstrings --- Doc/library/itertools.rst | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 85d243f6e4b2d6..5c8cc982a89a2c 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -803,11 +803,11 @@ which incur interpreter overhead. import random def take(n, iterable): - "Return first n items of the iterable as a list" + "Return first n items of the iterable as a list." return list(islice(iterable, n)) def prepend(value, iterable): - "Prepend a single value in front of an iterable" + "Prepend a single value in front of an iterable." # prepend(1, [2, 3, 4]) --> 1 2 3 4 return chain([value], iterable) @@ -825,15 +825,15 @@ which incur interpreter overhead. return starmap(func, repeat(args, times)) def flatten(list_of_lists): - "Flatten one level of nesting" + "Flatten one level of nesting." return chain.from_iterable(list_of_lists) def ncycles(iterable, n): - "Returns the sequence elements n times" + "Returns the sequence elements n times." return chain.from_iterable(repeat(tuple(iterable), n)) def tail(n, iterable): - "Return an iterator over the last n items" + "Return an iterator over the last n items." # tail(3, 'ABCDEFG') --> E F G return iter(collections.deque(iterable, maxlen=n)) @@ -848,7 +848,7 @@ which incur interpreter overhead. next(islice(iterator, n, n), None) def nth(iterable, n, default=None): - "Returns the nth item or a default value" + "Returns the nth item or a default value." return next(islice(iterable, n, None), default) def quantify(iterable, pred=bool): @@ -856,7 +856,7 @@ which incur interpreter overhead. return sum(map(pred, iterable)) def all_equal(iterable): - "Returns True if all the elements are equal to each other" + "Returns True if all the elements are equal to each other." g = groupby(iterable) return next(g, True) and not next(g, False) @@ -918,7 +918,7 @@ which incur interpreter overhead. pass def sliding_window(iterable, n): - "Collect data into overlapping fixed-length chunks or blocks" + "Collect data into overlapping fixed-length chunks or blocks." # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG it = iter(iterable) window = collections.deque(islice(it, n-1), maxlen=n) @@ -927,7 +927,7 @@ which incur interpreter overhead. yield tuple(window) def grouper(iterable, n, *, incomplete='fill', fillvalue=None): - "Collect data into non-overlapping fixed-length chunks or blocks" + "Collect data into non-overlapping fixed-length chunks or blocks." # grouper('ABCDEFG', 3, fillvalue='x') --> ABC DEF Gxx # grouper('ABCDEFG', 3, incomplete='strict') --> ABC DEF ValueError # grouper('ABCDEFG', 3, incomplete='ignore') --> ABC DEF @@ -943,7 +943,8 @@ which incur interpreter overhead. raise ValueError('Expected fill, strict, or ignore') def roundrobin(*iterables): - "roundrobin('ABC', 'D', 'EF') --> A D E B F C" + "Visit input iterables in a cycle until each is exhausted." + # roundrobin('ABC', 'D', 'EF') --> A D E B F C # Recipe credited to George Sakkis num_active = len(iterables) nexts = cycle(iter(it).__next__ for it in iterables) @@ -966,7 +967,7 @@ which incur interpreter overhead. return filterfalse(pred, t1), filter(pred, t2) def subslices(seq): - "Return all contiguous non-empty subslices of a sequence" + "Return all contiguous non-empty subslices of a sequence." # subslices('ABCD') --> A AB ABC ABCD B BC BCD C CD D slices = starmap(slice, combinations(range(len(seq) + 1), 2)) return map(operator.getitem, repeat(seq), slices) @@ -1019,6 +1020,7 @@ which incur interpreter overhead. """ it = iter(it) transition = [] + def true_iterator(): for elem in it: if predicate(elem): @@ -1026,6 +1028,7 @@ which incur interpreter overhead. else: transition.append(elem) return + return true_iterator(), chain(transition, it)