From 7faf39bf5339aace67caa6d93f4d7d26dc89f600 Mon Sep 17 00:00:00 2001 From: Takashi Menjo Date: Wed, 23 Jul 2025 02:27:02 +0900 Subject: [PATCH 01/11] chore: fix broken make proto (#235) Signed-off-by: Takashi Menjo --- Makefile | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index b403ac66..1ababadc 100644 --- a/Makefile +++ b/Makefile @@ -26,14 +26,13 @@ setup: poetry install --with dev --no-root proto: - python3 -m grpc_tools.protoc --pyi_out=pynumaflow/proto/sinker -I=pynumaflow/proto/sinker --python_out=pynumaflow/proto/sinker --grpc_python_out=pynumaflow/proto/sinker pynumaflow/proto/sinker/*.proto - python3 -m grpc_tools.protoc --pyi_out=pynumaflow/proto/mapper -I=pynumaflow/proto/mapper --python_out=pynumaflow/proto/mapper --grpc_python_out=pynumaflow/proto/mapper pynumaflow/proto/mapper/*.proto - python3 -m grpc_tools.protoc --pyi_out=pynumaflow/proto/mapstreamer -I=pynumaflow/proto/mapstreamer --python_out=pynumaflow/proto/mapstreamer --grpc_python_out=pynumaflow/proto/mapstreamer pynumaflow/proto/mapstreamer/*.proto - python3 -m grpc_tools.protoc --pyi_out=pynumaflow/proto/reducer -I=pynumaflow/proto/reducer --python_out=pynumaflow/proto/reducer --grpc_python_out=pynumaflow/proto/reducer pynumaflow/proto/reducer/*.proto - python3 -m grpc_tools.protoc --pyi_out=pynumaflow/proto/sourcetransformer -I=pynumaflow/proto/sourcetransformer --python_out=pynumaflow/proto/sourcetransformer --grpc_python_out=pynumaflow/proto/sourcetransformer pynumaflow/proto/sourcetransformer/*.proto - python3 -m grpc_tools.protoc --pyi_out=pynumaflow/proto/sideinput -I=pynumaflow/proto/sideinput --python_out=pynumaflow/proto/sideinput --grpc_python_out=pynumaflow/proto/sideinput pynumaflow/proto/sideinput/*.proto - python3 -m grpc_tools.protoc --pyi_out=pynumaflow/proto/sourcer -I=pynumaflow/proto/sourcer --python_out=pynumaflow/proto/sourcer --grpc_python_out=pynumaflow/proto/sourcer pynumaflow/proto/sourcer/*.proto - python3 -m grpc_tools.protoc --pyi_out=pynumaflow/proto/batchmapper -I=pynumaflow/proto/batchmapper --python_out=pynumaflow/proto/batchmapper --grpc_python_out=pynumaflow/proto/batchmapper pynumaflow/proto/batchmapper/*.proto + poetry run python3 -m grpc_tools.protoc --pyi_out=pynumaflow/proto/sinker -I=pynumaflow/proto/sinker --python_out=pynumaflow/proto/sinker --grpc_python_out=pynumaflow/proto/sinker pynumaflow/proto/sinker/*.proto + poetry run python3 -m grpc_tools.protoc --pyi_out=pynumaflow/proto/mapper -I=pynumaflow/proto/mapper --python_out=pynumaflow/proto/mapper --grpc_python_out=pynumaflow/proto/mapper pynumaflow/proto/mapper/*.proto + poetry run python3 -m grpc_tools.protoc --pyi_out=pynumaflow/proto/reducer -I=pynumaflow/proto/reducer --python_out=pynumaflow/proto/reducer --grpc_python_out=pynumaflow/proto/reducer pynumaflow/proto/reducer/*.proto + poetry run python3 -m grpc_tools.protoc --pyi_out=pynumaflow/proto/sourcetransformer -I=pynumaflow/proto/sourcetransformer --python_out=pynumaflow/proto/sourcetransformer --grpc_python_out=pynumaflow/proto/sourcetransformer pynumaflow/proto/sourcetransformer/*.proto + poetry run python3 -m grpc_tools.protoc --pyi_out=pynumaflow/proto/sideinput -I=pynumaflow/proto/sideinput --python_out=pynumaflow/proto/sideinput --grpc_python_out=pynumaflow/proto/sideinput pynumaflow/proto/sideinput/*.proto + poetry run python3 -m grpc_tools.protoc --pyi_out=pynumaflow/proto/sourcer -I=pynumaflow/proto/sourcer --python_out=pynumaflow/proto/sourcer --grpc_python_out=pynumaflow/proto/sourcer pynumaflow/proto/sourcer/*.proto - sed -i '' 's/^\(import.*_pb2\)/from . \1/' pynumaflow/proto/*/*.py + sed -i.bak -e 's/^\(import.*_pb2\)/from . \1/' pynumaflow/proto/*/*.py + rm pynumaflow/proto/*/*.py.bak From 49824144e6b1586f612bad601e47aeb2166963ca Mon Sep 17 00:00:00 2001 From: shrivardhan Date: Wed, 30 Jul 2025 16:21:27 -0700 Subject: [PATCH 02/11] feat: add Accumulator (#237) Signed-off-by: Sidhant Kohli Signed-off-by: srao12 Co-authored-by: Sidhant Kohli --- .github/workflows/build-push.yaml | 2 +- Makefile | 1 + examples/accumulator/streamsorter/Dockerfile | 55 ++ examples/accumulator/streamsorter/Makefile | 22 + .../streamsorter/Makefile.optimized | 52 ++ examples/accumulator/streamsorter/README.md | 43 ++ examples/accumulator/streamsorter/entry.sh | 4 + examples/accumulator/streamsorter/example.py | 72 +++ .../accumulator/streamsorter/pipeline.yaml | 49 ++ .../accumulator/streamsorter/pyproject.toml | 13 + examples/map/even_odd/Dockerfile | 1 - pynumaflow/_constants.py | 2 + pynumaflow/accumulator/__init__.py | 19 + pynumaflow/accumulator/_dtypes.py | 554 ++++++++++++++++++ pynumaflow/accumulator/async_server.py | 206 +++++++ pynumaflow/accumulator/servicer/__init__.py | 0 .../accumulator/servicer/async_servicer.py | 133 +++++ .../accumulator/servicer/task_manager.py | 349 +++++++++++ pynumaflow/info/types.py | 2 + pynumaflow/proto/accumulator/__init__.py | 0 .../proto/accumulator/accumulator.proto | 90 +++ .../proto/accumulator/accumulator_pb2.py | 52 ++ .../proto/accumulator/accumulator_pb2.pyi | 122 ++++ .../proto/accumulator/accumulator_pb2_grpc.py | 134 +++++ pynumaflow/proto/mapper/map_pb2.pyi | 3 - pynumaflow/proto/reducer/reduce_pb2.pyi | 3 - pynumaflow/proto/sinker/sink_pb2.pyi | 3 - pynumaflow/proto/sourcer/source_pb2.pyi | 7 - .../proto/sourcetransformer/transform_pb2.pyi | 3 - tests/accumulator/__init__.py | 0 tests/accumulator/test_async_accumulator.py | 476 +++++++++++++++ .../accumulator/test_async_accumulator_err.py | 175 ++++++ tests/accumulator/test_datatypes.py | 339 +++++++++++ tests/accumulator/utils.py | 23 + 34 files changed, 2988 insertions(+), 21 deletions(-) create mode 100644 examples/accumulator/streamsorter/Dockerfile create mode 100644 examples/accumulator/streamsorter/Makefile create mode 100644 examples/accumulator/streamsorter/Makefile.optimized create mode 100644 examples/accumulator/streamsorter/README.md create mode 100644 examples/accumulator/streamsorter/entry.sh create mode 100644 examples/accumulator/streamsorter/example.py create mode 100644 examples/accumulator/streamsorter/pipeline.yaml create mode 100644 examples/accumulator/streamsorter/pyproject.toml create mode 100644 pynumaflow/accumulator/__init__.py create mode 100644 pynumaflow/accumulator/_dtypes.py create mode 100644 pynumaflow/accumulator/async_server.py create mode 100644 pynumaflow/accumulator/servicer/__init__.py create mode 100644 pynumaflow/accumulator/servicer/async_servicer.py create mode 100644 pynumaflow/accumulator/servicer/task_manager.py create mode 100644 pynumaflow/proto/accumulator/__init__.py create mode 100644 pynumaflow/proto/accumulator/accumulator.proto create mode 100644 pynumaflow/proto/accumulator/accumulator_pb2.py create mode 100644 pynumaflow/proto/accumulator/accumulator_pb2.pyi create mode 100644 pynumaflow/proto/accumulator/accumulator_pb2_grpc.py create mode 100644 tests/accumulator/__init__.py create mode 100644 tests/accumulator/test_async_accumulator.py create mode 100644 tests/accumulator/test_async_accumulator_err.py create mode 100644 tests/accumulator/test_datatypes.py create mode 100644 tests/accumulator/utils.py diff --git a/.github/workflows/build-push.yaml b/.github/workflows/build-push.yaml index ab6a122f..a2c1ed8c 100644 --- a/.github/workflows/build-push.yaml +++ b/.github/workflows/build-push.yaml @@ -23,7 +23,7 @@ jobs: "examples/reducestream/counter", "examples/reducestream/sum", "examples/sideinput/simple_sideinput", "examples/sideinput/simple_sideinput/udf", "examples/sink/async_log", "examples/sink/log", "examples/source/simple_source", "examples/sourcetransform/event_time_filter", - "examples/batchmap/flatmap" + "examples/batchmap/flatmap", "examples/accumulator/streamsorter" ] steps: diff --git a/Makefile b/Makefile index 1ababadc..0e5334f2 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,7 @@ proto: poetry run python3 -m grpc_tools.protoc --pyi_out=pynumaflow/proto/sourcetransformer -I=pynumaflow/proto/sourcetransformer --python_out=pynumaflow/proto/sourcetransformer --grpc_python_out=pynumaflow/proto/sourcetransformer pynumaflow/proto/sourcetransformer/*.proto poetry run python3 -m grpc_tools.protoc --pyi_out=pynumaflow/proto/sideinput -I=pynumaflow/proto/sideinput --python_out=pynumaflow/proto/sideinput --grpc_python_out=pynumaflow/proto/sideinput pynumaflow/proto/sideinput/*.proto poetry run python3 -m grpc_tools.protoc --pyi_out=pynumaflow/proto/sourcer -I=pynumaflow/proto/sourcer --python_out=pynumaflow/proto/sourcer --grpc_python_out=pynumaflow/proto/sourcer pynumaflow/proto/sourcer/*.proto + poetry run python3 -m grpc_tools.protoc --pyi_out=pynumaflow/proto/accumulator -I=pynumaflow/proto/accumulator --python_out=pynumaflow/proto/accumulator --grpc_python_out=pynumaflow/proto/accumulator pynumaflow/proto/accumulator/*.proto sed -i.bak -e 's/^\(import.*_pb2\)/from . \1/' pynumaflow/proto/*/*.py diff --git a/examples/accumulator/streamsorter/Dockerfile b/examples/accumulator/streamsorter/Dockerfile new file mode 100644 index 00000000..dd2d605b --- /dev/null +++ b/examples/accumulator/streamsorter/Dockerfile @@ -0,0 +1,55 @@ +#################################################################################################### +# Stage 1: Base Builder - installs core dependencies using poetry +#################################################################################################### +FROM python:3.10-slim-bullseye AS base-builder + +ENV PYSETUP_PATH="/opt/pysetup" +WORKDIR $PYSETUP_PATH + +# Copy only core dependency files first for better caching +COPY pyproject.toml poetry.lock README.md ./ +COPY pynumaflow/ ./pynumaflow/ +RUN apt-get update && apt-get install --no-install-recommends -y \ + curl wget build-essential git \ + && apt-get clean && rm -rf /var/lib/apt/lists/* \ + && pip install poetry \ + && poetry install --no-root --no-interaction + +#################################################################################################### +# Stage 2: UDF Builder - adds UDF code and installs UDF-specific deps +#################################################################################################### +FROM base-builder AS udf-builder + +ENV EXAMPLE_PATH="/opt/pysetup/examples/accumulator/streamsorter" +ENV POETRY_VIRTUALENVS_IN_PROJECT=true + +WORKDIR $EXAMPLE_PATH +COPY examples/accumulator/streamsorter/ ./ +RUN poetry install --no-root --no-interaction + +#################################################################################################### +# Stage 3: UDF Runtime - clean container with only needed stuff +#################################################################################################### +FROM python:3.10-slim-bullseye AS udf + +ENV PYSETUP_PATH="/opt/pysetup" +ENV EXAMPLE_PATH="$PYSETUP_PATH/examples/accumulator/streamsorter" +ENV VENV_PATH="$EXAMPLE_PATH/.venv" +ENV PATH="$VENV_PATH/bin:$PATH" + +RUN apt-get update && apt-get install --no-install-recommends -y wget \ + && apt-get clean && rm -rf /var/lib/apt/lists/* \ + && wget -O /dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_x86_64 \ + && chmod +x /dumb-init + +WORKDIR $PYSETUP_PATH +COPY --from=udf-builder $VENV_PATH $VENV_PATH +COPY --from=udf-builder $EXAMPLE_PATH $EXAMPLE_PATH + +WORKDIR $EXAMPLE_PATH +RUN chmod +x entry.sh + +ENTRYPOINT ["/dumb-init", "--"] +CMD ["sh", "-c", "$EXAMPLE_PATH/entry.sh"] + +EXPOSE 5000 \ No newline at end of file diff --git a/examples/accumulator/streamsorter/Makefile b/examples/accumulator/streamsorter/Makefile new file mode 100644 index 00000000..5eb6a3e8 --- /dev/null +++ b/examples/accumulator/streamsorter/Makefile @@ -0,0 +1,22 @@ +TAG ?= stable +PUSH ?= false +IMAGE_REGISTRY = quay.io/numaio/numaflow-python/streamsorter:${TAG} +DOCKER_FILE_PATH = examples/accumulator/streamsorter/Dockerfile + +.PHONY: update +update: + poetry update -vv + +.PHONY: image-push +image-push: update + cd ../../../ && docker buildx build \ + -f ${DOCKER_FILE_PATH} \ + -t ${IMAGE_REGISTRY} \ + --platform linux/amd64,linux/arm64 . --push + +.PHONY: image +image: update + cd ../../../ && docker build \ + -f ${DOCKER_FILE_PATH} \ + -t ${IMAGE_REGISTRY} . + @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi diff --git a/examples/accumulator/streamsorter/Makefile.optimized b/examples/accumulator/streamsorter/Makefile.optimized new file mode 100644 index 00000000..136be046 --- /dev/null +++ b/examples/accumulator/streamsorter/Makefile.optimized @@ -0,0 +1,52 @@ +TAG ?= stable +PUSH ?= false +IMAGE_REGISTRY = quay.io/numaio/numaflow-python/streamsorter:${TAG} +DOCKER_FILE_PATH = examples/accumulator/streamsorter/Dockerfile +BASE_IMAGE_NAME = numaflow-python-base + +.PHONY: base-image +base-image: + @echo "Building shared base image..." + docker build -f Dockerfile.base -t ${BASE_IMAGE_NAME} . + +.PHONY: update +update: + poetry update -vv + +.PHONY: image-push +image-push: base-image update + cd ../../../ && docker buildx build \ + -f ${DOCKER_FILE_PATH} \ + -t ${IMAGE_REGISTRY} \ + --platform linux/amd64,linux/arm64 . --push + +.PHONY: image +image: base-image update + cd ../../../ && docker build \ + -f ${DOCKER_FILE_PATH} \ + -t ${IMAGE_REGISTRY} . + @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi + +.PHONY: image-fast +image-fast: update + @echo "Building with shared base image (fastest option)..." + cd ../../../ && docker build \ + -f examples/map/even_odd/Dockerfile.shared-base \ + -t ${IMAGE_REGISTRY} . + @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi + +.PHONY: clean +clean: + docker rmi ${BASE_IMAGE_NAME} 2>/dev/null || true + docker rmi ${IMAGE_REGISTRY} 2>/dev/null || true + +.PHONY: help +help: + @echo "Available targets:" + @echo " base-image - Build the shared base image with pynumaflow" + @echo " image - Build UDF image with optimized multi-stage build" + @echo " image-fast - Build UDF image using shared base (fastest)" + @echo " image-push - Build and push multi-platform image" + @echo " update - Update poetry dependencies" + @echo " clean - Remove built images" + @echo " help - Show this help message" \ No newline at end of file diff --git a/examples/accumulator/streamsorter/README.md b/examples/accumulator/streamsorter/README.md new file mode 100644 index 00000000..19b8da6e --- /dev/null +++ b/examples/accumulator/streamsorter/README.md @@ -0,0 +1,43 @@ +# Stream Sorter + +An example User Defined Function that sorts the incoming stream by event time. + +### Applying the Pipeline + +To apply the pipeline, use the following command: + +```shell + kubectl apply -f pipeline.yaml +``` + +### Publish messages + +Port-forward the HTTP endpoint, and make POST requests using curl. Remember to replace xxxx with the appropriate pod names. + +```shell + kubectl port-forward stream-sorter-http-one-0-xxxx 8444:8443 + + # Post data to the HTTP endpoint + curl -kq -X POST -d "101" https://localhost:8444/vertices/http-one -H "X-Numaflow-Event-Time: 60000" + curl -kq -X POST -d "102" https://localhost:8444/vertices/http-one -H "X-Numaflow-Event-Time: 61000" + curl -kq -X POST -d "103" https://localhost:8444/vertices/http-one -H "X-Numaflow-Event-Time: 62000" + curl -kq -X POST -d "104" https://localhost:8444/vertices/http-one -H "X-Numaflow-Event-Time: 63000" +``` + +```shell + kubectl port-forward stream-sorter-http-two-0-xxxx 8445:8443 + + # Post data to the HTTP endpoint + curl -kq -X POST -d "105" https://localhost:8445/vertices/http-two -H "X-Numaflow-Event-Time: 70000" + curl -kq -X POST -d "106" https://localhost:8445/vertices/http-two -H "X-Numaflow-Event-Time: 71000" + curl -kq -X POST -d "107" https://localhost:8445/vertices/http-two -H "X-Numaflow-Event-Time: 72000" + curl -kq -X POST -d "108" https://localhost:8445/vertices/http-two -H "X-Numaflow-Event-Time: 73000" +``` + +### Verify the output + +```shell + kubectl logs -f stream-sorter-log-sink-0-xxxx +``` + +The output should be sorted by event time. \ No newline at end of file diff --git a/examples/accumulator/streamsorter/entry.sh b/examples/accumulator/streamsorter/entry.sh new file mode 100644 index 00000000..073b05e3 --- /dev/null +++ b/examples/accumulator/streamsorter/entry.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eux + +python example.py diff --git a/examples/accumulator/streamsorter/example.py b/examples/accumulator/streamsorter/example.py new file mode 100644 index 00000000..8e0615ed --- /dev/null +++ b/examples/accumulator/streamsorter/example.py @@ -0,0 +1,72 @@ +import logging +import os +from collections.abc import AsyncIterable +from datetime import datetime + +from pynumaflow import setup_logging +from pynumaflow.accumulator import Accumulator, AccumulatorAsyncServer +from pynumaflow.accumulator import ( + Message, + Datum, +) +from pynumaflow.shared.asynciter import NonBlockingIterator + +_LOGGER = setup_logging(__name__) +if os.getenv("PYTHONDEBUG"): + _LOGGER.setLevel(logging.DEBUG) + + +class StreamSorter(Accumulator): + def __init__(self): + _LOGGER.info("StreamSorter initialized") + self.latest_wm = datetime.fromtimestamp(-1) + self.sorted_buffer: list[Datum] = [] + + async def handler( + self, + datums: AsyncIterable[Datum], + output: NonBlockingIterator, + ): + _LOGGER.info("StreamSorter handler started") + async for datum in datums: + _LOGGER.info( + f"Received datum with event time: {datum.event_time}, " + f"Current latest watermark: {self.latest_wm}, " + f"Datum watermark: {datum.watermark}" + ) + + # If watermark has moved forward + if datum.watermark and datum.watermark > self.latest_wm: + self.latest_wm = datum.watermark + await self.flush_buffer(output) + + self.insert_sorted(datum) + + def insert_sorted(self, datum: Datum): + # Binary insert to keep sorted buffer in order + left, right = 0, len(self.sorted_buffer) + while left < right: + mid = (left + right) // 2 + if self.sorted_buffer[mid].event_time > datum.event_time: + right = mid + else: + left = mid + 1 + self.sorted_buffer.insert(left, datum) + + async def flush_buffer(self, output: NonBlockingIterator): + _LOGGER.info(f"Watermark updated, flushing sortedBuffer: {self.latest_wm}") + i = 0 + for datum in self.sorted_buffer: + if datum.event_time > self.latest_wm: + break + await output.put(Message.from_datum(datum)) + _LOGGER.info(f"Sent datum with event time: {datum.event_time}") + i += 1 + # Remove flushed items + self.sorted_buffer = self.sorted_buffer[i:] + + +if __name__ == "__main__": + grpc_server = None + grpc_server = AccumulatorAsyncServer(StreamSorter) + grpc_server.start() diff --git a/examples/accumulator/streamsorter/pipeline.yaml b/examples/accumulator/streamsorter/pipeline.yaml new file mode 100644 index 00000000..d4ccab96 --- /dev/null +++ b/examples/accumulator/streamsorter/pipeline.yaml @@ -0,0 +1,49 @@ +apiVersion: numaflow.numaproj.io/v1alpha1 +kind: Pipeline +metadata: + name: stream-sorter +spec: + limits: + readBatchSize: 1 + vertices: + - name: http-one + scale: + min: 1 + max: 1 + source: + http: {} + - name: http-two + scale: + min: 1 + max: 1 + source: + http: {} + - name: py-accum + udf: + container: + image: quay.io/numaio/numaflow-python/streamsorter:stable + imagePullPolicy: Always + env: + - name: PYTHONDEBUG + value: "true" + groupBy: + window: + accumulator: + timeout: 10s + keyed: true + storage: + persistentVolumeClaim: + volumeSize: 1Gi + - name: py-sink + scale: + min: 1 + max: 1 + sink: + log: {} + edges: + - from: http-one + to: py-accum + - from: http-two + to: py-accum + - from: py-accum + to: py-sink diff --git a/examples/accumulator/streamsorter/pyproject.toml b/examples/accumulator/streamsorter/pyproject.toml new file mode 100644 index 00000000..9397268d --- /dev/null +++ b/examples/accumulator/streamsorter/pyproject.toml @@ -0,0 +1,13 @@ +[tool.poetry] +name = "stream-sorter" +version = "0.2.4" +description = "" +authors = ["Numaflow developers"] + +[tool.poetry.dependencies] +python = "~3.10" +pynumaflow = { path = "../../../"} + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/examples/map/even_odd/Dockerfile b/examples/map/even_odd/Dockerfile index 1bf155ca..0e9be000 100644 --- a/examples/map/even_odd/Dockerfile +++ b/examples/map/even_odd/Dockerfile @@ -9,7 +9,6 @@ WORKDIR $PYSETUP_PATH # Copy only core dependency files first for better caching COPY pyproject.toml poetry.lock README.md ./ COPY pynumaflow/ ./pynumaflow/ -RUN echo "Simulating long build step..." && sleep 20 RUN apt-get update && apt-get install --no-install-recommends -y \ curl wget build-essential git \ && apt-get clean && rm -rf /var/lib/apt/lists/* \ diff --git a/pynumaflow/_constants.py b/pynumaflow/_constants.py index ea5e2b9d..01ae44d5 100644 --- a/pynumaflow/_constants.py +++ b/pynumaflow/_constants.py @@ -26,6 +26,7 @@ MULTIPROC_MAP_SOCK_ADDR = "/var/run/numaflow/multiproc" FALLBACK_SINK_SOCK_PATH = "/var/run/numaflow/fb-sink.sock" BATCH_MAP_SOCK_PATH = "/var/run/numaflow/batchmap.sock" +ACCUMULATOR_SOCK_PATH = "/var/run/numaflow/accumulator.sock" # Server information file configs MAP_SERVER_INFO_FILE_PATH = "/var/run/numaflow/mapper-server-info" @@ -36,6 +37,7 @@ SIDE_INPUT_SERVER_INFO_FILE_PATH = "/var/run/numaflow/sideinput-server-info" SOURCE_SERVER_INFO_FILE_PATH = "/var/run/numaflow/sourcer-server-info" FALLBACK_SINK_SERVER_INFO_FILE_PATH = "/var/run/numaflow/fb-sinker-server-info" +ACCUMULATOR_SERVER_INFO_FILE_PATH = "/var/run/numaflow/accumulator-server-info" ENV_UD_CONTAINER_TYPE = "NUMAFLOW_UD_CONTAINER_TYPE" UD_CONTAINER_FALLBACK_SINK = "fb-udsink" diff --git a/pynumaflow/accumulator/__init__.py b/pynumaflow/accumulator/__init__.py new file mode 100644 index 00000000..0d1368d8 --- /dev/null +++ b/pynumaflow/accumulator/__init__.py @@ -0,0 +1,19 @@ +from pynumaflow.accumulator._dtypes import ( + Message, + Datum, + IntervalWindow, + DROP, + KeyedWindow, + Accumulator, +) +from pynumaflow.accumulator.async_server import AccumulatorAsyncServer + +__all__ = [ + "Message", + "Datum", + "IntervalWindow", + "DROP", + "AccumulatorAsyncServer", + "KeyedWindow", + "Accumulator", +] diff --git a/pynumaflow/accumulator/_dtypes.py b/pynumaflow/accumulator/_dtypes.py new file mode 100644 index 00000000..31a0d5fe --- /dev/null +++ b/pynumaflow/accumulator/_dtypes.py @@ -0,0 +1,554 @@ +from abc import ABCMeta, abstractmethod +from asyncio import Task +from dataclasses import dataclass +from datetime import datetime +from enum import IntEnum +from typing import TypeVar, Callable, Union, Optional +from collections.abc import AsyncIterable + +from pynumaflow.shared.asynciter import NonBlockingIterator +from pynumaflow._constants import DROP + +M = TypeVar("M", bound="Message") + + +class WindowOperation(IntEnum): + """ + Enumerate the type of Window operation received. + """ + + OPEN = (0,) + CLOSE = (1,) + APPEND = (2,) + + +@dataclass(init=False) +class Datum: + """ + Class to define the important information for the event. + Args: + keys: the keys of the event. + value: the payload of the event. + event_time: the event time of the event. + watermark: the watermark of the event. + >>> # Example usage + >>> from pynumaflow.accumulator import Datum + >>> from datetime import datetime, timezone + >>> payload = bytes("test_mock_message", encoding="utf-8") + >>> t1 = datetime.fromtimestamp(1662998400, timezone.utc) + >>> t2 = datetime.fromtimestamp(1662998460, timezone.utc) + >>> msg_headers = {"key1": "value1", "key2": "value2"} + >>> d = Datum( + ... keys=["test_key"], + ... value=payload, + ... event_time=t1, + ... watermark=t2, + ... headers=msg_headers + ... ) + """ + + __slots__ = ("_keys", "_value", "_event_time", "_watermark", "_headers", "_id") + + _keys: list[str] + _value: bytes + _event_time: datetime + _watermark: datetime + _headers: dict[str, str] + _id: str + + def __init__( + self, + keys: list[str], + value: bytes, + event_time: datetime, + watermark: datetime, + id_: str, + headers: Optional[dict[str, str]] = None, + ): + self._keys = keys or list() + self._value = value or b"" + if not isinstance(event_time, datetime): + raise TypeError(f"Wrong data type: {type(event_time)} for Datum.event_time") + self._event_time = event_time + if not isinstance(watermark, datetime): + raise TypeError(f"Wrong data type: {type(watermark)} for Datum.watermark") + self._watermark = watermark + self._headers = headers or {} + self._id = id_ + + def keys(self) -> list[str]: + """Returns the keys of the event. + + Returns: + list[str]: A list of string keys associated with this event. + """ + return self._keys + + @property + def value(self) -> bytes: + """Returns the value of the event. + + Returns: + bytes: The payload data of the event as bytes. + """ + return self._value + + @property + def event_time(self) -> datetime: + """Returns the event time of the event. + + Returns: + datetime: The timestamp when the event occurred. + """ + return self._event_time + + @property + def watermark(self) -> datetime: + """Returns the watermark of the event. + + Returns: + datetime: The watermark timestamp indicating the progress of event time. + """ + return self._watermark + + @property + def headers(self) -> dict[str, str]: + """Returns the headers of the event. + + Returns: + dict[str, str]: A dictionary containing header key-value pairs for this event. + """ + return self._headers.copy() + + @property + def id(self) -> str: + """Returns the id of the event. + + Returns: + str: The unique identifier for this event. + """ + return self._id + + +@dataclass(init=False) +class IntervalWindow: + """Defines the start and end of the interval window for the event.""" + + __slots__ = ("_start", "_end") + + _start: datetime + _end: datetime + + def __init__(self, start: datetime, end: datetime): + self._start = start + self._end = end + + @property + def start(self) -> datetime: + """Returns the start point of the interval window. + + Returns: + datetime: The start timestamp of the interval window. + """ + return self._start + + @property + def end(self) -> datetime: + """Returns the end point of the interval window. + + Returns: + datetime: The end timestamp of the interval window. + """ + return self._end + + +@dataclass(init=False) +class KeyedWindow: + """ + Defines the window for a accumulator operation which includes the + interval window along with the slot. + """ + + __slots__ = ("_window", "_slot", "_keys") + + _window: IntervalWindow + _slot: str + _keys: list[str] + + def __init__(self, start: datetime, end: datetime, slot: str = "", keys: list[str] = []): + self._window = IntervalWindow(start=start, end=end) + self._slot = slot + self._keys = keys + + @property + def start(self) -> datetime: + """Returns the start point of the interval window. + + Returns: + datetime: The start timestamp of the interval window. + """ + return self._window.start + + @property + def end(self) -> datetime: + """Returns the end point of the interval window. + + Returns: + datetime: The end timestamp of the interval window. + """ + return self._window.end + + @property + def slot(self) -> str: + """Returns the slot from the window. + + Returns: + str: The slot identifier for this window. + """ + return self._slot + + @property + def window(self) -> IntervalWindow: + """Returns the interval window. + + Returns: + IntervalWindow: The underlying interval window object. + """ + return self._window + + @property + def keys(self) -> list[str]: + """Returns the keys for window. + + Returns: + list[str]: A list of keys associated with this window. + """ + return self._keys + + +@dataclass +class AccumulatorResult: + """Defines the object to hold the result of accumulator computation.""" + + __slots__ = ( + "_future", + "_iterator", + "_key", + "_result_queue", + "_consumer_future", + "_latest_watermark", + ) + + _future: Task + _iterator: NonBlockingIterator + _key: list[str] + _result_queue: NonBlockingIterator + _consumer_future: Task + _latest_watermark: datetime + + @property + def future(self) -> Task: + """Returns the future result of computation. + + Returns: + Task: The asyncio Task representing the computation future. + """ + return self._future + + @property + def iterator(self) -> NonBlockingIterator: + """Returns the handle to the producer queue. + + Returns: + NonBlockingIterator: The iterator for producing data to the queue. + """ + return self._iterator + + @property + def keys(self) -> list[str]: + """Returns the keys of the partition. + + Returns: + list[str]: The keys associated with this partition. + """ + return self._key + + @property + def result_queue(self) -> NonBlockingIterator: + """Returns the async queue used to write the output for the tasks. + + Returns: + NonBlockingIterator: The queue for writing task output. + """ + return self._result_queue + + @property + def consumer_future(self) -> Task: + """Returns the async consumer task for the result queue. + + Returns: + Task: The asyncio Task for consuming from the result queue. + """ + return self._consumer_future + + @property + def latest_watermark(self) -> datetime: + """Returns the latest watermark for task. + + Returns: + datetime: The latest watermark timestamp for this task. + """ + return self._latest_watermark + + def update_watermark(self, new_watermark: datetime): + """Updates the latest watermark value. + + Args: + new_watermark (datetime): The new watermark timestamp to set. + + Raises: + TypeError: If new_watermark is not a datetime object. + """ + if not isinstance(new_watermark, datetime): + raise TypeError("new_watermark must be a datetime object") + self._latest_watermark = new_watermark + + +@dataclass +class AccumulatorRequest: + """Defines the object to hold a request for the accumulator operation.""" + + __slots__ = ("_operation", "_keyed_window", "_payload") + + _operation: WindowOperation + _keyed_window: KeyedWindow + _payload: Datum + + def __init__(self, operation: WindowOperation, keyed_window: KeyedWindow, payload: Datum): + self._operation = operation + self._keyed_window = keyed_window + self._payload = payload + + @property + def operation(self) -> WindowOperation: + """Returns the operation type. + + Returns: + WindowOperation: The type of window operation (OPEN, CLOSE, or APPEND). + """ + return self._operation + + @property + def keyed_window(self) -> KeyedWindow: + """Returns the keyed window. + + Returns: + KeyedWindow: The keyed window associated with this request. + """ + return self._keyed_window + + @property + def payload(self) -> Datum: + """Returns the payload of the window. + + Returns: + Datum: The data payload for this accumulator request. + """ + return self._payload + + +@dataclass(init=False) +class Message: + """ + Basic datatype for data passing to the next vertex/vertices. + + Args: + value: data in bytes + keys: []string keys for vertex (optional) + tags: []string tags for conditional forwarding (optional) + watermark: watermark for this message (optional) + event_time: event time for this message (optional) + headers: headers for this message (optional) + id: message id (optional) + """ + + __slots__ = ("_value", "_keys", "_tags", "_watermark", "_event_time", "_headers", "_id") + + _value: bytes + _keys: list[str] + _tags: list[str] + _watermark: datetime + _event_time: datetime + _headers: dict[str, str] + _id: str + + def __init__( + self, + value: bytes, + keys: list[str] = None, + tags: list[str] = None, + watermark: datetime = None, + event_time: datetime = None, + headers: dict[str, str] = None, + id: str = None, + ): + """ + Creates a Message object to send value to a vertex. + """ + self._keys = keys or [] + self._tags = tags or [] + self._value = value or b"" + self._watermark = watermark + self._event_time = event_time + self._headers = headers or {} + self._id = id or "" + + @classmethod + def to_drop(cls: type[M]) -> M: + """Creates a Message instance that indicates the message should be dropped. + + Returns: + M: A Message instance with empty value and DROP tag indicating + the message should be dropped. + """ + return cls(b"", None, [DROP]) + + @property + def value(self) -> bytes: + """Returns the message payload value. + + Returns: + bytes: The message payload data as bytes. + """ + return self._value + + @property + def keys(self) -> list[str]: + """Returns the message keys. + + Returns: + list[str]: A list of string keys associated with this message. + """ + return self._keys + + @property + def tags(self) -> list[str]: + """Returns the message tags for conditional forwarding. + + Returns: + list[str]: A list of string tags used for conditional forwarding. + """ + return self._tags + + @property + def watermark(self) -> datetime: + """Returns the watermark timestamp for this message. + + Returns: + datetime: The watermark timestamp, or None if not set. + """ + return self._watermark + + @property + def event_time(self) -> datetime: + """Returns the event time for this message. + + Returns: + datetime: The event time timestamp, or None if not set. + """ + return self._event_time + + @property + def headers(self) -> dict[str, str]: + """Returns the message headers. + + Returns: + dict[str, str]: A dictionary containing header key-value pairs for this message. + """ + return self._headers.copy() + + @property + def id(self) -> str: + """Returns the message ID. + + Returns: + str: The unique identifier for this message. + """ + return self._id + + @classmethod + def from_datum(cls, datum: Datum): + """Create a Message instance from a Datum object. + + Args: + datum: The Datum object to convert + + Returns: + Message: A new Message instance with data from the datum + """ + return cls( + value=datum.value, + keys=datum.keys(), + watermark=datum.watermark, + event_time=datum.event_time, + headers=datum.headers, + id=datum.id, + ) + + +AccumulatorAsyncCallable = Callable[[list[str], AsyncIterable[Datum], NonBlockingIterator], None] + + +class Accumulator(metaclass=ABCMeta): + """ + Accumulate can read unordered from the input stream and emit the ordered + data to the output stream. Once the watermark (WM) of the output stream progresses, + the data in WAL until that WM will be garbage collected. + NOTE: A message can be silently dropped if need be, + and it will be cleared from the WAL when the WM progresses. + """ + + def __call__(self, *args, **kwargs): + """ + Allow to call handler function directly if class instance is sent + as the accumulator_instance. + """ + return self.handler(*args, **kwargs) + + @abstractmethod + async def handler( + self, + datums: AsyncIterable[Datum], + output: NonBlockingIterator, + ): + """ + Implement this handler function which implements the AccumulatorStreamCallable interface. + """ + pass + + +class _AccumulatorBuilderClass: + """ + Class to build an Accumulator class instance. + Used Internally + + Args: + accumulator_class: the Accumulator class to be used for Accumulator UDF + args: the arguments to be passed to the accumulator class + kwargs: the keyword arguments to be passed to the accumulator class + """ + + def __init__(self, accumulator_class: type[Accumulator], args: tuple, kwargs: dict): + self._accumulator_class: type[Accumulator] = accumulator_class + self._args = args + self._kwargs = kwargs + + def create(self) -> Accumulator: + """ + Create a new Accumulator instance. + """ + return self._accumulator_class(*self._args, **self._kwargs) + + +# AccumulatorStreamCallable is a callable which can be used as a handler for the Accumulator UDF. +AccumulatorStreamCallable = Union[AccumulatorAsyncCallable, type[Accumulator]] diff --git a/pynumaflow/accumulator/async_server.py b/pynumaflow/accumulator/async_server.py new file mode 100644 index 00000000..042359ca --- /dev/null +++ b/pynumaflow/accumulator/async_server.py @@ -0,0 +1,206 @@ +import inspect +from typing import Optional + +import aiorun +import grpc + +from pynumaflow.accumulator.servicer.async_servicer import AsyncAccumulatorServicer +from pynumaflow.info.types import ServerInfo, ContainerType, MINIMUM_NUMAFLOW_VERSION +from pynumaflow.proto.accumulator import accumulator_pb2_grpc + + +from pynumaflow._constants import ( + MAX_MESSAGE_SIZE, + NUM_THREADS_DEFAULT, + _LOGGER, + MAX_NUM_THREADS, + ACCUMULATOR_SOCK_PATH, + ACCUMULATOR_SERVER_INFO_FILE_PATH, +) + +from pynumaflow.accumulator._dtypes import ( + AccumulatorStreamCallable, + _AccumulatorBuilderClass, + Accumulator, +) + +from pynumaflow.shared.server import NumaflowServer, check_instance, start_async_server + + +def get_handler( + accumulator_handler: AccumulatorStreamCallable, + init_args: tuple = (), + init_kwargs: Optional[dict] = None, +): + """ + Get the correct handler type based on the arguments passed + """ + if inspect.isfunction(accumulator_handler): + if init_args or init_kwargs: + # if the init_args or init_kwargs are passed, then the accumulator_instance + # can only be of class Accumulator type + raise TypeError("Cannot pass function handler with init args or kwargs") + # return the function handler + return accumulator_handler + elif not check_instance(accumulator_handler, Accumulator) and issubclass( + accumulator_handler, Accumulator + ): + # if handler is type of Class Accumulator, create a new instance of + # a AccumulatorBuilderClass + return _AccumulatorBuilderClass(accumulator_handler, init_args, init_kwargs) + else: + _LOGGER.error( + _error_msg := f"Invalid Class Type {accumulator_handler}: " + f"Please make sure the class type is passed, and it is a subclass of Accumulator" + ) + raise TypeError(_error_msg) + + +class AccumulatorAsyncServer(NumaflowServer): + """ + Class for a new Accumulator Server instance. + A new servicer instance is created and attached to the server. + The server instance is returned. + Args: + accumulator_instance: The accumulator instance to be used for + Accumulator UDF + init_args: The arguments to be passed to the accumulator_handler + init_kwargs: The keyword arguments to be passed to the + accumulator_handler + sock_path: The UNIX socket path to be used for the server + max_message_size: The max message size in bytes the server can receive and send + max_threads: The max number of threads to be spawned; + defaults to 4 and max capped at 16 + server_info_file: The path to the server info file + Example invocation: + import os + from collections.abc import AsyncIterable + from datetime import datetime + + from pynumaflow.accumulator import Accumulator, AccumulatorAsyncServer + from pynumaflow.accumulator import ( + Message, + Datum, + ) + from pynumaflow.shared.asynciter import NonBlockingIterator + + class StreamSorter(Accumulator): + def __init__(self, counter): + self.latest_wm = datetime.fromtimestamp(-1) + self.sorted_buffer: list[Datum] = [] + + async def handler( + self, + datums: AsyncIterable[Datum], + output: NonBlockingIterator, + ): + async for _ in datums: + # Process the datums and send output + if datum.watermark and datum.watermark > self.latest_wm: + self.latest_wm = datum.watermark + await self.flush_buffer(output) + + self.insert_sorted(datum) + + def insert_sorted(self, datum: Datum): + # Binary insert to keep sorted buffer in order + left, right = 0, len(self.sorted_buffer) + while left < right: + mid = (left + right) // 2 + if self.sorted_buffer[mid].event_time > datum.event_time: + right = mid + else: + left = mid + 1 + self.sorted_buffer.insert(left, datum) + + async def flush_buffer(self, output: NonBlockingIterator): + i = 0 + for datum in self.sorted_buffer: + if datum.event_time > self.latest_wm: + break + await output.put(Message.from_datum(datum)) + i += 1 + # Remove flushed items + self.sorted_buffer = self.sorted_buffer[i:] + + + if __name__ == "__main__": + grpc_server = AccumulatorAsyncServer(StreamSorter) + grpc_server.start() + + """ + + def __init__( + self, + accumulator_instance: AccumulatorStreamCallable, + init_args: tuple = (), + init_kwargs: dict = None, + sock_path=ACCUMULATOR_SOCK_PATH, + max_message_size=MAX_MESSAGE_SIZE, + max_threads=NUM_THREADS_DEFAULT, + server_info_file=ACCUMULATOR_SERVER_INFO_FILE_PATH, + ): + """ + Create a new grpc Accumulator Server instance. + A new servicer instance is created and attached to the server. + The server instance is returned. + Args: + accumulator_instance: The Accumulator instance to be used for + Accumulator UDF + init_args: The arguments to be passed to the accumulator_handler + init_kwargs: The keyword arguments to be passed to the + accumulator_handler + sock_path: The UNIX socket path to be used for the server + max_message_size: The max message size in bytes the server can receive and send + max_threads: The max number of threads to be spawned; + defaults to 4 and max capped at 16 + server_info_file: The path to the server info file + """ + if init_kwargs is None: + init_kwargs = {} + self.accumulator_handler = get_handler(accumulator_instance, init_args, init_kwargs) + self.sock_path = f"unix://{sock_path}" + self.max_message_size = max_message_size + self.max_threads = min(max_threads, MAX_NUM_THREADS) + self.server_info_file = server_info_file + + self._server_options = [ + ("grpc.max_send_message_length", self.max_message_size), + ("grpc.max_receive_message_length", self.max_message_size), + ] + # Get the servicer instance for the async server + self.servicer = AsyncAccumulatorServicer(self.accumulator_handler) + + def start(self): + """ + Starter function for the Async server class, need a separate caller + so that all the async coroutines can be started from a single context + """ + _LOGGER.info( + "Starting Async Accumulator Server", + ) + aiorun.run(self.aexec(), use_uvloop=True) + + async def aexec(self): + """ + Starts the Async gRPC server on the given UNIX socket with + given max threads. + """ + # As the server is async, we need to create a new server instance in the + # same thread as the event loop so that all the async calls are made in the + # same context + # Create a new async server instance and add the servicer to it + server = grpc.aio.server(options=self._server_options) + server.add_insecure_port(self.sock_path) + accumulator_pb2_grpc.add_AccumulatorServicer_to_server(self.servicer, server) + + serv_info = ServerInfo.get_default_server_info() + serv_info.minimum_numaflow_version = MINIMUM_NUMAFLOW_VERSION[ContainerType.Accumulator] + await start_async_server( + server_async=server, + sock_path=self.sock_path, + max_threads=self.max_threads, + cleanup_coroutines=list(), + server_info_file=self.server_info_file, + server_info=serv_info, + ) diff --git a/pynumaflow/accumulator/servicer/__init__.py b/pynumaflow/accumulator/servicer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pynumaflow/accumulator/servicer/async_servicer.py b/pynumaflow/accumulator/servicer/async_servicer.py new file mode 100644 index 00000000..6eebdbeb --- /dev/null +++ b/pynumaflow/accumulator/servicer/async_servicer.py @@ -0,0 +1,133 @@ +import asyncio +from collections.abc import AsyncIterable +from typing import Union + +from google.protobuf import empty_pb2 as _empty_pb2 + +from pynumaflow._constants import ERR_UDF_EXCEPTION_STRING +from pynumaflow.proto.accumulator import accumulator_pb2, accumulator_pb2_grpc +from pynumaflow.accumulator._dtypes import ( + Datum, + AccumulatorAsyncCallable, + _AccumulatorBuilderClass, + AccumulatorRequest, + KeyedWindow, +) +from pynumaflow.accumulator.servicer.task_manager import TaskManager +from pynumaflow.shared.server import handle_async_error +from pynumaflow.types import NumaflowServicerContext + + +async def datum_generator( + request_iterator: AsyncIterable[accumulator_pb2.AccumulatorRequest], +) -> AsyncIterable[AccumulatorRequest]: + """Generate a AccumulatorRequest from a AccumulatorRequest proto message.""" + async for d in request_iterator: + # Convert protobuf KeyedWindow to our KeyedWindow dataclass + keyed_window = KeyedWindow( + start=d.operation.keyedWindow.start.ToDatetime(), + end=d.operation.keyedWindow.end.ToDatetime(), + slot=d.operation.keyedWindow.slot, + keys=list(d.operation.keyedWindow.keys), + ) + + accumulator_request = AccumulatorRequest( + operation=d.operation.event, + keyed_window=keyed_window, # Use the new parameter name + payload=Datum( + keys=list(d.payload.keys), + value=d.payload.value, + event_time=d.payload.event_time.ToDatetime(), + watermark=d.payload.watermark.ToDatetime(), + id_=d.payload.id, + headers=dict(d.payload.headers), + ), + ) + yield accumulator_request + + +class AsyncAccumulatorServicer(accumulator_pb2_grpc.AccumulatorServicer): + """ + This class is used to create a new grpc Accumulator servicer instance. + Provides the functionality for the required rpc methods. + """ + + def __init__( + self, + handler: Union[AccumulatorAsyncCallable, _AccumulatorBuilderClass], + ): + # The accumulator handler can be a function or a builder class instance. + self.__accumulator_handler: Union[ + AccumulatorAsyncCallable, _AccumulatorBuilderClass + ] = handler + + async def AccumulateFn( + self, + request_iterator: AsyncIterable[accumulator_pb2.AccumulatorRequest], + context: NumaflowServicerContext, + ) -> accumulator_pb2.AccumulatorResponse: + """ + Applies a accumulator function to a datum stream. + The pascal case function name comes from the proto accumulator_pb2_grpc.py file. + """ + # Create a task manager instance + task_manager = TaskManager(handler=self.__accumulator_handler) + + # Create a consumer task to read from the result queue + # All the results from the accumulator function will be sent to the result queue + # We will read from the result queue and send the results to the client + consumer = task_manager.global_result_queue.read_iterator() + + # Create an async iterator from the request iterator + datum_iterator = datum_generator(request_iterator=request_iterator) + + # Create a process_input_stream task in the task manager, + # this would read from the datum iterator + # and then create the required tasks to process the data requests + # The results from these tasks are then sent to the result queue + producer = asyncio.create_task(task_manager.process_input_stream(datum_iterator)) + + # Start the consumer task where we read from the result queue + # and send the results to the client + # The task manager can write the following to the result queue: + # 1. A accumulator_pb2.AccumulatorResponse message + # This is the result of the accumulator function, it contains the window and the + # result of the accumulator function + # The result of the accumulator function is a accumulator_pb2.AccumulatorResponse message + # and can be directly sent to the client + # + # 2. An Exception + # Any exceptions that occur during the processing accumulator function tasks are + # sent to the result queue. We then forward these exception to the client + # + # 3. A accumulator_pb2.AccumulatorResponse message with EOF=True + # This is a special message that indicates the end of the processing for a window + # When we get this message, we send an EOF message to the client + try: + async for msg in consumer: + # If the message is an exception, we raise the exception + if isinstance(msg, BaseException): + await handle_async_error(context, msg, ERR_UDF_EXCEPTION_STRING) + return + # Send window EOF response or Window result response + # back to the client + else: + yield msg + except BaseException as e: + await handle_async_error(context, e, ERR_UDF_EXCEPTION_STRING) + return + # Wait for the process_input_stream task to finish for a clean exit + try: + await producer + except BaseException as e: + await handle_async_error(context, e, ERR_UDF_EXCEPTION_STRING) + return + + async def IsReady( + self, request: _empty_pb2.Empty, context: NumaflowServicerContext + ) -> accumulator_pb2.ReadyResponse: + """ + IsReady is the heartbeat endpoint for gRPC. + The pascal case function name comes from the proto accumulator_pb2_grpc.py file. + """ + return accumulator_pb2.ReadyResponse(ready=True) diff --git a/pynumaflow/accumulator/servicer/task_manager.py b/pynumaflow/accumulator/servicer/task_manager.py new file mode 100644 index 00000000..a7c80968 --- /dev/null +++ b/pynumaflow/accumulator/servicer/task_manager.py @@ -0,0 +1,349 @@ +import asyncio +from collections.abc import AsyncIterable +from datetime import datetime +from typing import Union + +from google.protobuf import timestamp_pb2 +from pynumaflow._constants import ( + STREAM_EOF, + DELIMITER, + _LOGGER, +) +from pynumaflow.accumulator._dtypes import ( + AccumulatorResult, + Datum, + _AccumulatorBuilderClass, + AccumulatorAsyncCallable, + WindowOperation, +) +from pynumaflow.proto.accumulator import accumulator_pb2 +from pynumaflow.shared.asynciter import NonBlockingIterator + + +def build_unique_key_name(keys): + """ + Builds a unique key name for the given keys and window. + The key name is used to identify the Accumulator task. + The format is: start_time:end_time:key1:key2:... + """ + return f"{DELIMITER.join(keys)}" + + +def build_window_hash(window): + """ + Builds a hash for the given window. + The hash is used to identify the Accumulator Window + The format is: start_time:end_time + """ + return f"{window.start.ToMilliseconds()}:{window.end.ToMilliseconds()}" + + +class TaskManager: + """ + TaskManager is responsible for managing the Accumulator tasks. + It is created whenever a new accumulator operation is requested. + """ + + def __init__(self, handler: Union[AccumulatorAsyncCallable, _AccumulatorBuilderClass]): + # A dictionary to store the task information + self.tasks: dict[str, AccumulatorResult] = {} + # Collection for storing strong references to all running tasks. + # Event loop only keeps a weak reference, which can cause it to + # get lost during execution. + self.background_tasks = set() + # Handler for the accumulator operation + self.__accumulator_handler = handler + # Queue to store the results of the accumulator operation + # This queue is used to send the results to the client + # once the accumulator operation is completed. + # This queue is also used to send the error/exceptions to the client + # if the accumulator operation fails. + self.global_result_queue = NonBlockingIterator() + + def get_unique_windows(self): + """ + Returns the unique windows that are currently being processed + """ + # Dict to store unique windows + windows = dict() + # Iterate over all the tasks and add the windows + for task in self.tasks.values(): + window_hash = build_window_hash(task.window) + window_found = windows.get(window_hash, None) + # if window not seen yet, add to the dict + if not window_found: + windows[window_hash] = task.window + return windows + + def get_tasks(self): + """ + Returns the list of accumulator tasks that are + currently being processed + """ + return list(self.tasks.values()) + + async def stream_send_eof(self): + """ + Function used to indicate to all processing tasks that no + more requests are expected by sending EOF message to + local input streams of individual tasks. + This is called when the input grpc stream is closed. + """ + # Create a copy of the keys to avoid dictionary size change during iteration + task_keys = list(self.tasks.keys()) + for unified_key in task_keys: + await self.tasks[unified_key].iterator.put(STREAM_EOF) + + async def close_task(self, req): + """ + Closes a running accumulator task for a given key. + Based on the request we compute the unique key, and then + signal the corresponding task for it to closure. + The steps involve + 1. Send a signal to the local request queue of the task to stop reading + 2. Wait for the user function to complete + 3. Wait for all the results from the task to be written to the global result queue + 4. Remove the task from the tracker + """ + d = req.payload + keys = d.keys() + unified_key = build_unique_key_name(keys) + curr_task = self.tasks.get(unified_key, None) + + if curr_task: + await self.tasks[unified_key].iterator.put(STREAM_EOF) + await curr_task.future + await curr_task.consumer_future + self.tasks.pop(unified_key) + else: + _LOGGER.critical("accumulator task not found", exc_info=True) + err = BaseException("accumulator task not found") + # Put the exception in the result queue + await self.global_result_queue.put(err) + + async def create_task(self, req): + """ + Creates a new accumulator task for the given request. + Based on the request we compute a unique key, and then + it creates a new task or appends the request to the existing task. + """ + d = req.payload + keys = d.keys() + unified_key = build_unique_key_name(keys) + curr_task = self.tasks.get(unified_key, None) + + # If the task does not exist, create a new task + if not curr_task: + niter = NonBlockingIterator() + riter = niter.read_iterator() + # Create a new result queue for the current task + # We create a new result queue for each task, so that + # the results of the accumulator operation can be sent to the + # the global result queue, which in turn sends the results + # to the client. + res_queue = NonBlockingIterator() + + # Create a new write_to_global_queue task for the current, this will read from the + # result queue specifically for the current task and update the + # global result queue + consumer = asyncio.create_task( + self.write_to_global_queue(res_queue, self.global_result_queue, unified_key) + ) + # Save a reference to the result of this function, to avoid a + # task disappearing mid-execution. + self.background_tasks.add(consumer) + consumer.add_done_callback(self.clean_background) + + # Create a new task for the accumulator operation, this will invoke the + # Accumulator handler with the given keys, request iterator, and window. + task = asyncio.create_task(self.__invoke_accumulator(riter, res_queue)) + # Save a reference to the result of this function, to avoid a + # task disappearing mid-execution. + self.background_tasks.add(task) + task.add_done_callback(self.clean_background) + + # Create a new AccumulatorResult object to store the task information + curr_task = AccumulatorResult( + task, niter, keys, res_queue, consumer, datetime.fromtimestamp(-1) + ) + + # Save the result of the accumulator operation to the task list + self.tasks[unified_key] = curr_task + + # Put the request in the iterator + await curr_task.iterator.put(d) + + async def send_datum_to_task(self, req): + """ + Appends the request to the existing window reduce task. + If the task does not exist, create it. + """ + d = req.payload + keys = d.keys() + unified_key = build_unique_key_name(keys) + result = self.tasks.get(unified_key, None) + if not result: + await self.create_task(req) + else: + await result.iterator.put(d) + + async def __invoke_accumulator( + self, + request_iterator: AsyncIterable[Datum], + output: NonBlockingIterator, + ): + """ + Invokes the UDF accumulator handler with the given keys, + request iterator, and window. Returns the result of the + accumulator operation. + """ + new_instance = self.__accumulator_handler + + # If the accumulator handler is a class instance, create a new instance of it. + # It is required for a new key to be processed by a + # new instance of the accumulator for a given window + # Otherwise the function handler can be called directly + if isinstance(self.__accumulator_handler, _AccumulatorBuilderClass): + new_instance = self.__accumulator_handler.create() + try: + _ = await new_instance(request_iterator, output) + # send EOF to the output stream + await output.put(STREAM_EOF) + # If there is an error in the accumulator operation, log and + # then send the error to the result queue + except BaseException as err: + _LOGGER.critical("panic inside accumulator handle", exc_info=True) + # Put the exception in the result queue + await self.global_result_queue.put(err) + + async def process_input_stream( + self, request_iterator: AsyncIterable[accumulator_pb2.AccumulatorRequest] + ): + # Start iterating through the request iterator and create tasks + # based on the operation type received. + try: + request_count = 0 + async for request in request_iterator: + request_count += 1 + # check whether the request is an open or append operation + if request.operation is int(WindowOperation.OPEN): + # create a new task for the open operation and + # put the request in the task iterator + await self.create_task(request) + elif request.operation is int(WindowOperation.APPEND): + # append the task data to the existing task + # if the task does not exist, create a new task + await self.send_datum_to_task(request) + elif request.operation is int(WindowOperation.CLOSE): + # close the current task for req + await self.close_task(request) + else: + _LOGGER.debug(f"No operation matched for request: {request}", exc_info=True) + + # If there is an error in the accumulator operation, log and + # then send the error to the result queue + except BaseException as e: + err_msg = f"Accumulator Error: {repr(e)}" + _LOGGER.critical(err_msg, exc_info=True) + # Put the exception in the global result queue + await self.global_result_queue.put(e) + return + + try: + # send EOF to all the tasks once the request iterator is exhausted + # This will signal the tasks to stop reading the data on their + # respective iterators. + await self.stream_send_eof() + + # get the list of accumulator tasks that are currently being processed + # iterate through the tasks and wait for them to complete + for task in self.get_tasks(): + # Once this is done, we know that the task has written all the results + # to the local result queue + fut = task.future + await fut + + # Wait for the local queue to write + # all the results of this task to the global result queue + con_future = task.consumer_future + await con_future + self.tasks.clear() + + # Now send STREAM_EOF to terminate the global result queue iterator + await self.global_result_queue.put(STREAM_EOF) + except BaseException as e: + err_msg = f"Accumulator Streaming Error: {repr(e)}" + _LOGGER.critical(err_msg, exc_info=True) + await self.global_result_queue.put(e) + + async def write_to_global_queue( + self, input_queue: NonBlockingIterator, output_queue: NonBlockingIterator, unified_key: str + ): + """ + This function is used to route the messages from the + local result queue for a given task to the global result queue. + Once all messages are routed, it sends the window EOF messages for the same. + """ + reader = input_queue.read_iterator() + task = self.tasks[unified_key] + + wm: datetime = task.latest_watermark + async for msg in reader: + # Convert the window to a datetime object + # Only update watermark if msg.watermark is not None + if msg.watermark is not None and wm < msg.watermark: + task.update_watermark(msg.watermark) + self.tasks[unified_key] = task + wm = msg.watermark + + # Convert datetime to protobuf timestamp + event_time_pb = timestamp_pb2.Timestamp() + if msg.event_time is not None: + event_time_pb.FromDatetime(msg.event_time) + + watermark_pb = timestamp_pb2.Timestamp() + if msg.watermark is not None: + watermark_pb.FromDatetime(msg.watermark) + + start_dt_pb = timestamp_pb2.Timestamp() + start_dt_pb.FromDatetime(datetime.fromtimestamp(0)) + + end_dt_pb = timestamp_pb2.Timestamp() + end_dt_pb.FromDatetime(wm) + + res = accumulator_pb2.AccumulatorResponse( + payload=accumulator_pb2.Payload( + keys=msg.keys, + value=msg.value, + event_time=event_time_pb, + watermark=watermark_pb, + headers=msg.headers, + id=msg.id, + ), + window=accumulator_pb2.KeyedWindow( + start=start_dt_pb, end=end_dt_pb, slot="slot-0", keys=task.keys + ), + EOF=False, + tags=msg.tags, + ) + await output_queue.put(res) + # send EOF + start_eof_pb = timestamp_pb2.Timestamp() + start_eof_pb.FromDatetime(datetime.fromtimestamp(0)) + + end_eof_pb = timestamp_pb2.Timestamp() + end_eof_pb.FromDatetime(wm) + + res = accumulator_pb2.AccumulatorResponse( + window=accumulator_pb2.KeyedWindow( + start=start_eof_pb, end=end_eof_pb, slot="slot-0", keys=task.keys + ), + EOF=True, + ) + await output_queue.put(res) + + def clean_background(self, task): + """ + Remove the task from the background tasks collection + """ + self.background_tasks.remove(task) diff --git a/pynumaflow/info/types.py b/pynumaflow/info/types.py index 2845c264..12375a70 100644 --- a/pynumaflow/info/types.py +++ b/pynumaflow/info/types.py @@ -71,6 +71,7 @@ class ContainerType(str, Enum): Sessionreducer = "sessionreducer" Sideinput = "sideinput" Fbsinker = "fb-sinker" + Accumulator = "accumulator" # Minimum version of Numaflow required by the current SDK version @@ -86,6 +87,7 @@ class ContainerType(str, Enum): ContainerType.Sessionreducer: "1.4.0-z", ContainerType.Sideinput: "1.4.0-z", ContainerType.Fbsinker: "1.4.0-z", + ContainerType.Accumulator: "1.5.0-z", } diff --git a/pynumaflow/proto/accumulator/__init__.py b/pynumaflow/proto/accumulator/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pynumaflow/proto/accumulator/accumulator.proto b/pynumaflow/proto/accumulator/accumulator.proto new file mode 100644 index 00000000..acde986b --- /dev/null +++ b/pynumaflow/proto/accumulator/accumulator.proto @@ -0,0 +1,90 @@ +/* +Copyright 2022 The Numaproj Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +syntax = "proto3"; + +option go_package = "github.com/numaproj/numaflow-go/pkg/apis/proto/accumulator/v1"; +option java_package = "io.numaproj.numaflow.accumulator.v1"; + +import "google/protobuf/empty.proto"; +import "google/protobuf/timestamp.proto"; + + +package accumulator.v1; + +// AccumulatorWindow describes a special kind of SessionWindow (similar to Global Window) where output should +// always have monotonically increasing WM but it can be manipulated through event-time by reordering the messages. +// NOTE: Quite powerful, should not be abused; it can cause stalling of pipelines and leaks +service Accumulator { + // AccumulateFn applies a accumulate function to a request stream. + rpc AccumulateFn(stream AccumulatorRequest) returns (stream AccumulatorResponse); + + // IsReady is the heartbeat endpoint for gRPC. + rpc IsReady(google.protobuf.Empty) returns (ReadyResponse); +} + +// Payload represents a payload element. +message Payload { + repeated string keys = 1; + bytes value = 2; + google.protobuf.Timestamp event_time = 3; + google.protobuf.Timestamp watermark = 4; + string id = 5; + map headers = 6; +} + +// AccumulatorRequest represents a request element. +message AccumulatorRequest { + // WindowOperation represents a window operation. + // For Unaligned windows, OPEN, APPEND and CLOSE events are sent. + message WindowOperation { + enum Event { + OPEN = 0; + CLOSE = 1; + APPEND = 2; + } + Event event = 1; + KeyedWindow keyedWindow = 2; + } + + Payload payload = 1; + WindowOperation operation = 2; +} + + +// Window represents a window. +message KeyedWindow { + google.protobuf.Timestamp start = 1; + google.protobuf.Timestamp end = 2; + string slot = 3; + repeated string keys = 4; +} + +// AccumulatorResponse represents a response element. +message AccumulatorResponse { + Payload payload = 1; + // window represents a window to which the result belongs. + KeyedWindow window = 2; + repeated string tags = 3; + // EOF represents the end of the response for a window. + bool EOF = 4; +} + + +// ReadyResponse is the health check result. +message ReadyResponse { + bool ready = 1; +} \ No newline at end of file diff --git a/pynumaflow/proto/accumulator/accumulator_pb2.py b/pynumaflow/proto/accumulator/accumulator_pb2.py new file mode 100644 index 00000000..f1e8ec8d --- /dev/null +++ b/pynumaflow/proto/accumulator/accumulator_pb2.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: accumulator.proto +# Protobuf Python Version: 4.25.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x11\x61\x63\x63umulator.proto\x12\x0e\x61\x63\x63umulator.v1\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto"\xf8\x01\n\x07Payload\x12\x0c\n\x04keys\x18\x01 \x03(\t\x12\r\n\x05value\x18\x02 \x01(\x0c\x12.\n\nevent_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12-\n\twatermark\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\n\n\x02id\x18\x05 \x01(\t\x12\x35\n\x07headers\x18\x06 \x03(\x0b\x32$.accumulator.v1.Payload.HeadersEntry\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"\xbe\x02\n\x12\x41\x63\x63umulatorRequest\x12(\n\x07payload\x18\x01 \x01(\x0b\x32\x17.accumulator.v1.Payload\x12\x45\n\toperation\x18\x02 \x01(\x0b\x32\x32.accumulator.v1.AccumulatorRequest.WindowOperation\x1a\xb6\x01\n\x0fWindowOperation\x12G\n\x05\x65vent\x18\x01 \x01(\x0e\x32\x38.accumulator.v1.AccumulatorRequest.WindowOperation.Event\x12\x30\n\x0bkeyedWindow\x18\x02 \x01(\x0b\x32\x1b.accumulator.v1.KeyedWindow"(\n\x05\x45vent\x12\x08\n\x04OPEN\x10\x00\x12\t\n\x05\x43LOSE\x10\x01\x12\n\n\x06\x41PPEND\x10\x02"}\n\x0bKeyedWindow\x12)\n\x05start\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\'\n\x03\x65nd\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04slot\x18\x03 \x01(\t\x12\x0c\n\x04keys\x18\x04 \x03(\t"\x87\x01\n\x13\x41\x63\x63umulatorResponse\x12(\n\x07payload\x18\x01 \x01(\x0b\x32\x17.accumulator.v1.Payload\x12+\n\x06window\x18\x02 \x01(\x0b\x32\x1b.accumulator.v1.KeyedWindow\x12\x0c\n\x04tags\x18\x03 \x03(\t\x12\x0b\n\x03\x45OF\x18\x04 \x01(\x08"\x1e\n\rReadyResponse\x12\r\n\x05ready\x18\x01 \x01(\x08\x32\xac\x01\n\x0b\x41\x63\x63umulator\x12[\n\x0c\x41\x63\x63umulateFn\x12".accumulator.v1.AccumulatorRequest\x1a#.accumulator.v1.AccumulatorResponse(\x01\x30\x01\x12@\n\x07IsReady\x12\x16.google.protobuf.Empty\x1a\x1d.accumulator.v1.ReadyResponseBd\n#io.numaproj.numaflow.accumulator.v1Z=github.com/numaproj/numaflow-go/pkg/apis/proto/accumulator/v1b\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "accumulator_pb2", _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + _globals["DESCRIPTOR"]._options = None + _globals[ + "DESCRIPTOR" + ]._serialized_options = b"\n#io.numaproj.numaflow.accumulator.v1Z=github.com/numaproj/numaflow-go/pkg/apis/proto/accumulator/v1" + _globals["_PAYLOAD_HEADERSENTRY"]._options = None + _globals["_PAYLOAD_HEADERSENTRY"]._serialized_options = b"8\001" + _globals["_PAYLOAD"]._serialized_start = 100 + _globals["_PAYLOAD"]._serialized_end = 348 + _globals["_PAYLOAD_HEADERSENTRY"]._serialized_start = 302 + _globals["_PAYLOAD_HEADERSENTRY"]._serialized_end = 348 + _globals["_ACCUMULATORREQUEST"]._serialized_start = 351 + _globals["_ACCUMULATORREQUEST"]._serialized_end = 669 + _globals["_ACCUMULATORREQUEST_WINDOWOPERATION"]._serialized_start = 487 + _globals["_ACCUMULATORREQUEST_WINDOWOPERATION"]._serialized_end = 669 + _globals["_ACCUMULATORREQUEST_WINDOWOPERATION_EVENT"]._serialized_start = 629 + _globals["_ACCUMULATORREQUEST_WINDOWOPERATION_EVENT"]._serialized_end = 669 + _globals["_KEYEDWINDOW"]._serialized_start = 671 + _globals["_KEYEDWINDOW"]._serialized_end = 796 + _globals["_ACCUMULATORRESPONSE"]._serialized_start = 799 + _globals["_ACCUMULATORRESPONSE"]._serialized_end = 934 + _globals["_READYRESPONSE"]._serialized_start = 936 + _globals["_READYRESPONSE"]._serialized_end = 966 + _globals["_ACCUMULATOR"]._serialized_start = 969 + _globals["_ACCUMULATOR"]._serialized_end = 1141 +# @@protoc_insertion_point(module_scope) diff --git a/pynumaflow/proto/accumulator/accumulator_pb2.pyi b/pynumaflow/proto/accumulator/accumulator_pb2.pyi new file mode 100644 index 00000000..d9f0f7a5 --- /dev/null +++ b/pynumaflow/proto/accumulator/accumulator_pb2.pyi @@ -0,0 +1,122 @@ +from google.protobuf import empty_pb2 as _empty_pb2 +from google.protobuf import timestamp_pb2 as _timestamp_pb2 +from google.protobuf.internal import containers as _containers +from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from typing import ( + ClassVar as _ClassVar, + Iterable as _Iterable, + Mapping as _Mapping, + Optional as _Optional, + Union as _Union, +) + +DESCRIPTOR: _descriptor.FileDescriptor + +class Payload(_message.Message): + __slots__ = ("keys", "value", "event_time", "watermark", "id", "headers") + + class HeadersEntry(_message.Message): + __slots__ = ("key", "value") + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: str + def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... + KEYS_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + EVENT_TIME_FIELD_NUMBER: _ClassVar[int] + WATERMARK_FIELD_NUMBER: _ClassVar[int] + ID_FIELD_NUMBER: _ClassVar[int] + HEADERS_FIELD_NUMBER: _ClassVar[int] + keys: _containers.RepeatedScalarFieldContainer[str] + value: bytes + event_time: _timestamp_pb2.Timestamp + watermark: _timestamp_pb2.Timestamp + id: str + headers: _containers.ScalarMap[str, str] + def __init__( + self, + keys: _Optional[_Iterable[str]] = ..., + value: _Optional[bytes] = ..., + event_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., + watermark: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., + id: _Optional[str] = ..., + headers: _Optional[_Mapping[str, str]] = ..., + ) -> None: ... + +class AccumulatorRequest(_message.Message): + __slots__ = ("payload", "operation") + + class WindowOperation(_message.Message): + __slots__ = ("event", "keyedWindow") + + class Event(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + OPEN: _ClassVar[AccumulatorRequest.WindowOperation.Event] + CLOSE: _ClassVar[AccumulatorRequest.WindowOperation.Event] + APPEND: _ClassVar[AccumulatorRequest.WindowOperation.Event] + OPEN: AccumulatorRequest.WindowOperation.Event + CLOSE: AccumulatorRequest.WindowOperation.Event + APPEND: AccumulatorRequest.WindowOperation.Event + EVENT_FIELD_NUMBER: _ClassVar[int] + KEYEDWINDOW_FIELD_NUMBER: _ClassVar[int] + event: AccumulatorRequest.WindowOperation.Event + keyedWindow: KeyedWindow + def __init__( + self, + event: _Optional[_Union[AccumulatorRequest.WindowOperation.Event, str]] = ..., + keyedWindow: _Optional[_Union[KeyedWindow, _Mapping]] = ..., + ) -> None: ... + PAYLOAD_FIELD_NUMBER: _ClassVar[int] + OPERATION_FIELD_NUMBER: _ClassVar[int] + payload: Payload + operation: AccumulatorRequest.WindowOperation + def __init__( + self, + payload: _Optional[_Union[Payload, _Mapping]] = ..., + operation: _Optional[_Union[AccumulatorRequest.WindowOperation, _Mapping]] = ..., + ) -> None: ... + +class KeyedWindow(_message.Message): + __slots__ = ("start", "end", "slot", "keys") + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + SLOT_FIELD_NUMBER: _ClassVar[int] + KEYS_FIELD_NUMBER: _ClassVar[int] + start: _timestamp_pb2.Timestamp + end: _timestamp_pb2.Timestamp + slot: str + keys: _containers.RepeatedScalarFieldContainer[str] + def __init__( + self, + start: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., + end: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., + slot: _Optional[str] = ..., + keys: _Optional[_Iterable[str]] = ..., + ) -> None: ... + +class AccumulatorResponse(_message.Message): + __slots__ = ("payload", "window", "tags", "EOF") + PAYLOAD_FIELD_NUMBER: _ClassVar[int] + WINDOW_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + EOF_FIELD_NUMBER: _ClassVar[int] + payload: Payload + window: KeyedWindow + tags: _containers.RepeatedScalarFieldContainer[str] + EOF: bool + def __init__( + self, + payload: _Optional[_Union[Payload, _Mapping]] = ..., + window: _Optional[_Union[KeyedWindow, _Mapping]] = ..., + tags: _Optional[_Iterable[str]] = ..., + EOF: bool = ..., + ) -> None: ... + +class ReadyResponse(_message.Message): + __slots__ = ("ready",) + READY_FIELD_NUMBER: _ClassVar[int] + ready: bool + def __init__(self, ready: bool = ...) -> None: ... diff --git a/pynumaflow/proto/accumulator/accumulator_pb2_grpc.py b/pynumaflow/proto/accumulator/accumulator_pb2_grpc.py new file mode 100644 index 00000000..f41606dd --- /dev/null +++ b/pynumaflow/proto/accumulator/accumulator_pb2_grpc.py @@ -0,0 +1,134 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from . import accumulator_pb2 as accumulator__pb2 +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 + + +class AccumulatorStub(object): + """AccumulatorWindow describes a special kind of SessionWindow (similar to Global Window) where output should + always have monotonically increasing WM but it can be manipulated through event-time by reordering the messages. + NOTE: Quite powerful, should not be abused; it can cause stalling of pipelines and leaks + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.AccumulateFn = channel.stream_stream( + "/accumulator.v1.Accumulator/AccumulateFn", + request_serializer=accumulator__pb2.AccumulatorRequest.SerializeToString, + response_deserializer=accumulator__pb2.AccumulatorResponse.FromString, + ) + self.IsReady = channel.unary_unary( + "/accumulator.v1.Accumulator/IsReady", + request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + response_deserializer=accumulator__pb2.ReadyResponse.FromString, + ) + + +class AccumulatorServicer(object): + """AccumulatorWindow describes a special kind of SessionWindow (similar to Global Window) where output should + always have monotonically increasing WM but it can be manipulated through event-time by reordering the messages. + NOTE: Quite powerful, should not be abused; it can cause stalling of pipelines and leaks + """ + + def AccumulateFn(self, request_iterator, context): + """AccumulateFn applies a accumulate function to a request stream.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def IsReady(self, request, context): + """IsReady is the heartbeat endpoint for gRPC.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + +def add_AccumulatorServicer_to_server(servicer, server): + rpc_method_handlers = { + "AccumulateFn": grpc.stream_stream_rpc_method_handler( + servicer.AccumulateFn, + request_deserializer=accumulator__pb2.AccumulatorRequest.FromString, + response_serializer=accumulator__pb2.AccumulatorResponse.SerializeToString, + ), + "IsReady": grpc.unary_unary_rpc_method_handler( + servicer.IsReady, + request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + response_serializer=accumulator__pb2.ReadyResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + "accumulator.v1.Accumulator", rpc_method_handlers + ) + server.add_generic_rpc_handlers((generic_handler,)) + + +# This class is part of an EXPERIMENTAL API. +class Accumulator(object): + """AccumulatorWindow describes a special kind of SessionWindow (similar to Global Window) where output should + always have monotonically increasing WM but it can be manipulated through event-time by reordering the messages. + NOTE: Quite powerful, should not be abused; it can cause stalling of pipelines and leaks + """ + + @staticmethod + def AccumulateFn( + request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.stream_stream( + request_iterator, + target, + "/accumulator.v1.Accumulator/AccumulateFn", + accumulator__pb2.AccumulatorRequest.SerializeToString, + accumulator__pb2.AccumulatorResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + + @staticmethod + def IsReady( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/accumulator.v1.Accumulator/IsReady", + google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + accumulator__pb2.ReadyResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) diff --git a/pynumaflow/proto/mapper/map_pb2.pyi b/pynumaflow/proto/mapper/map_pb2.pyi index e1279ff0..9832bc3e 100644 --- a/pynumaflow/proto/mapper/map_pb2.pyi +++ b/pynumaflow/proto/mapper/map_pb2.pyi @@ -26,7 +26,6 @@ class MapRequest(_message.Message): key: str value: str def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... - KEYS_FIELD_NUMBER: _ClassVar[int] VALUE_FIELD_NUMBER: _ClassVar[int] EVENT_TIME_FIELD_NUMBER: _ClassVar[int] @@ -45,7 +44,6 @@ class MapRequest(_message.Message): watermark: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., headers: _Optional[_Mapping[str, str]] = ..., ) -> None: ... - REQUEST_FIELD_NUMBER: _ClassVar[int] ID_FIELD_NUMBER: _ClassVar[int] HANDSHAKE_FIELD_NUMBER: _ClassVar[int] @@ -91,7 +89,6 @@ class MapResponse(_message.Message): value: _Optional[bytes] = ..., tags: _Optional[_Iterable[str]] = ..., ) -> None: ... - RESULTS_FIELD_NUMBER: _ClassVar[int] ID_FIELD_NUMBER: _ClassVar[int] HANDSHAKE_FIELD_NUMBER: _ClassVar[int] diff --git a/pynumaflow/proto/reducer/reduce_pb2.pyi b/pynumaflow/proto/reducer/reduce_pb2.pyi index 2c4b248c..88b27d53 100644 --- a/pynumaflow/proto/reducer/reduce_pb2.pyi +++ b/pynumaflow/proto/reducer/reduce_pb2.pyi @@ -48,7 +48,6 @@ class ReduceRequest(_message.Message): key: str value: str def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... - KEYS_FIELD_NUMBER: _ClassVar[int] VALUE_FIELD_NUMBER: _ClassVar[int] EVENT_TIME_FIELD_NUMBER: _ClassVar[int] @@ -67,7 +66,6 @@ class ReduceRequest(_message.Message): watermark: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., headers: _Optional[_Mapping[str, str]] = ..., ) -> None: ... - PAYLOAD_FIELD_NUMBER: _ClassVar[int] OPERATION_FIELD_NUMBER: _ClassVar[int] payload: ReduceRequest.Payload @@ -110,7 +108,6 @@ class ReduceResponse(_message.Message): value: _Optional[bytes] = ..., tags: _Optional[_Iterable[str]] = ..., ) -> None: ... - RESULT_FIELD_NUMBER: _ClassVar[int] WINDOW_FIELD_NUMBER: _ClassVar[int] EOF_FIELD_NUMBER: _ClassVar[int] diff --git a/pynumaflow/proto/sinker/sink_pb2.pyi b/pynumaflow/proto/sinker/sink_pb2.pyi index 18d4d3b6..78926321 100644 --- a/pynumaflow/proto/sinker/sink_pb2.pyi +++ b/pynumaflow/proto/sinker/sink_pb2.pyi @@ -37,7 +37,6 @@ class SinkRequest(_message.Message): key: str value: str def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... - KEYS_FIELD_NUMBER: _ClassVar[int] VALUE_FIELD_NUMBER: _ClassVar[int] EVENT_TIME_FIELD_NUMBER: _ClassVar[int] @@ -59,7 +58,6 @@ class SinkRequest(_message.Message): id: _Optional[str] = ..., headers: _Optional[_Mapping[str, str]] = ..., ) -> None: ... - REQUEST_FIELD_NUMBER: _ClassVar[int] STATUS_FIELD_NUMBER: _ClassVar[int] HANDSHAKE_FIELD_NUMBER: _ClassVar[int] @@ -108,7 +106,6 @@ class SinkResponse(_message.Message): status: _Optional[_Union[Status, str]] = ..., err_msg: _Optional[str] = ..., ) -> None: ... - RESULTS_FIELD_NUMBER: _ClassVar[int] HANDSHAKE_FIELD_NUMBER: _ClassVar[int] STATUS_FIELD_NUMBER: _ClassVar[int] diff --git a/pynumaflow/proto/sourcer/source_pb2.pyi b/pynumaflow/proto/sourcer/source_pb2.pyi index 8f588410..f2cdc70e 100644 --- a/pynumaflow/proto/sourcer/source_pb2.pyi +++ b/pynumaflow/proto/sourcer/source_pb2.pyi @@ -32,7 +32,6 @@ class ReadRequest(_message.Message): def __init__( self, num_records: _Optional[int] = ..., timeout_in_ms: _Optional[int] = ... ) -> None: ... - REQUEST_FIELD_NUMBER: _ClassVar[int] HANDSHAKE_FIELD_NUMBER: _ClassVar[int] request: ReadRequest.Request @@ -56,7 +55,6 @@ class ReadResponse(_message.Message): key: str value: str def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... - PAYLOAD_FIELD_NUMBER: _ClassVar[int] OFFSET_FIELD_NUMBER: _ClassVar[int] EVENT_TIME_FIELD_NUMBER: _ClassVar[int] @@ -107,7 +105,6 @@ class ReadResponse(_message.Message): error: _Optional[_Union[ReadResponse.Status.Error, str]] = ..., msg: _Optional[str] = ..., ) -> None: ... - RESULT_FIELD_NUMBER: _ClassVar[int] STATUS_FIELD_NUMBER: _ClassVar[int] HANDSHAKE_FIELD_NUMBER: _ClassVar[int] @@ -131,7 +128,6 @@ class AckRequest(_message.Message): def __init__( self, offsets: _Optional[_Iterable[_Union[Offset, _Mapping]]] = ... ) -> None: ... - REQUEST_FIELD_NUMBER: _ClassVar[int] HANDSHAKE_FIELD_NUMBER: _ClassVar[int] request: AckRequest.Request @@ -152,7 +148,6 @@ class AckResponse(_message.Message): def __init__( self, success: _Optional[_Union[_empty_pb2.Empty, _Mapping]] = ... ) -> None: ... - RESULT_FIELD_NUMBER: _ClassVar[int] HANDSHAKE_FIELD_NUMBER: _ClassVar[int] result: AckResponse.Result @@ -177,7 +172,6 @@ class PendingResponse(_message.Message): COUNT_FIELD_NUMBER: _ClassVar[int] count: int def __init__(self, count: _Optional[int] = ...) -> None: ... - RESULT_FIELD_NUMBER: _ClassVar[int] result: PendingResponse.Result def __init__( @@ -192,7 +186,6 @@ class PartitionsResponse(_message.Message): PARTITIONS_FIELD_NUMBER: _ClassVar[int] partitions: _containers.RepeatedScalarFieldContainer[int] def __init__(self, partitions: _Optional[_Iterable[int]] = ...) -> None: ... - RESULT_FIELD_NUMBER: _ClassVar[int] result: PartitionsResponse.Result def __init__( diff --git a/pynumaflow/proto/sourcetransformer/transform_pb2.pyi b/pynumaflow/proto/sourcetransformer/transform_pb2.pyi index 1fe8cb08..cc8fe420 100644 --- a/pynumaflow/proto/sourcetransformer/transform_pb2.pyi +++ b/pynumaflow/proto/sourcetransformer/transform_pb2.pyi @@ -32,7 +32,6 @@ class SourceTransformRequest(_message.Message): key: str value: str def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... - KEYS_FIELD_NUMBER: _ClassVar[int] VALUE_FIELD_NUMBER: _ClassVar[int] EVENT_TIME_FIELD_NUMBER: _ClassVar[int] @@ -54,7 +53,6 @@ class SourceTransformRequest(_message.Message): headers: _Optional[_Mapping[str, str]] = ..., id: _Optional[str] = ..., ) -> None: ... - REQUEST_FIELD_NUMBER: _ClassVar[int] HANDSHAKE_FIELD_NUMBER: _ClassVar[int] request: SourceTransformRequest.Request @@ -85,7 +83,6 @@ class SourceTransformResponse(_message.Message): event_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., tags: _Optional[_Iterable[str]] = ..., ) -> None: ... - RESULTS_FIELD_NUMBER: _ClassVar[int] ID_FIELD_NUMBER: _ClassVar[int] HANDSHAKE_FIELD_NUMBER: _ClassVar[int] diff --git a/tests/accumulator/__init__.py b/tests/accumulator/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/accumulator/test_async_accumulator.py b/tests/accumulator/test_async_accumulator.py new file mode 100644 index 00000000..292e3687 --- /dev/null +++ b/tests/accumulator/test_async_accumulator.py @@ -0,0 +1,476 @@ +import asyncio +import logging +import threading +import unittest +from collections.abc import AsyncIterable + +import grpc +from google.protobuf import empty_pb2 as _empty_pb2 +from grpc.aio._server import Server + +from pynumaflow import setup_logging +from pynumaflow.accumulator import ( + Message, + Datum, + AccumulatorAsyncServer, + Accumulator, +) +from pynumaflow.proto.accumulator import accumulator_pb2, accumulator_pb2_grpc +from pynumaflow.shared.asynciter import NonBlockingIterator +from tests.testing_utils import ( + mock_message, + mock_interval_window_start, + mock_interval_window_end, + get_time_args, +) + +LOGGER = setup_logging(__name__) + + +def request_generator(count, request, resetkey: bool = False, send_close: bool = False): + for i in range(count): + if resetkey: + # Clear previous keys and add new ones + del request.payload.keys[:] + request.payload.keys.extend([f"key-{i}"]) + + # Set operation based on index - first is OPEN, rest are APPEND + if i == 0: + request.operation.event = accumulator_pb2.AccumulatorRequest.WindowOperation.Event.OPEN + else: + request.operation.event = ( + accumulator_pb2.AccumulatorRequest.WindowOperation.Event.APPEND + ) + yield request + + if send_close: + # Send a close operation after all requests + request.operation.event = accumulator_pb2.AccumulatorRequest.WindowOperation.Event.CLOSE + yield request + + +def request_generator_append_only(count, request, resetkey: bool = False): + for i in range(count): + if resetkey: + # Clear previous keys and add new ones + del request.payload.keys[:] + request.payload.keys.extend([f"key-{i}"]) + + # Set operation to APPEND for all requests + request.operation.event = accumulator_pb2.AccumulatorRequest.WindowOperation.Event.APPEND + yield request + + +def request_generator_mixed(count, request, resetkey: bool = False): + for i in range(count): + if resetkey: + # Clear previous keys and add new ones + del request.payload.keys[:] + request.payload.keys.extend([f"key-{i}"]) + + if i % 2 == 0: + # Set operation to APPEND for even requests + request.operation.event = ( + accumulator_pb2.AccumulatorRequest.WindowOperation.Event.APPEND + ) + else: + # Set operation to CLOSE for odd requests + request.operation.event = accumulator_pb2.AccumulatorRequest.WindowOperation.Event.CLOSE + yield request + + +def start_request() -> accumulator_pb2.AccumulatorRequest: + event_time_timestamp, watermark_timestamp = get_time_args() + window = accumulator_pb2.KeyedWindow( + start=mock_interval_window_start(), + end=mock_interval_window_end(), + slot="slot-0", + keys=["test_key"], + ) + payload = accumulator_pb2.Payload( + keys=["test_key"], + value=mock_message(), + event_time=event_time_timestamp, + watermark=watermark_timestamp, + id="test_id", + ) + operation = accumulator_pb2.AccumulatorRequest.WindowOperation( + event=accumulator_pb2.AccumulatorRequest.WindowOperation.Event.OPEN, + keyedWindow=window, + ) + request = accumulator_pb2.AccumulatorRequest( + payload=payload, + operation=operation, + ) + return request + + +def start_request_without_open() -> accumulator_pb2.AccumulatorRequest: + event_time_timestamp, watermark_timestamp = get_time_args() + + payload = accumulator_pb2.Payload( + keys=["test_key"], + value=mock_message(), + event_time=event_time_timestamp, + watermark=watermark_timestamp, + id="test_id", + ) + + request = accumulator_pb2.AccumulatorRequest( + payload=payload, + ) + return request + + +_s: Server = None +_channel = grpc.insecure_channel("unix:///tmp/accumulator.sock") +_loop = None + + +def startup_callable(loop): + asyncio.set_event_loop(loop) + loop.run_forever() + + +class ExampleClass(Accumulator): + def __init__(self, counter): + self.counter = counter + + async def handler(self, datums: AsyncIterable[Datum], output: NonBlockingIterator): + async for datum in datums: + self.counter += 1 + msg = f"counter:{self.counter}" + await output.put(Message(str.encode(msg), keys=datum.keys(), tags=[])) + + +async def accumulator_handler_func(datums: AsyncIterable[Datum], output: NonBlockingIterator): + counter = 0 + async for datum in datums: + counter += 1 + msg = f"counter:{counter}" + await output.put(Message(str.encode(msg), keys=datum.keys(), tags=[])) + + +def NewAsyncAccumulator(): + server_instance = AccumulatorAsyncServer(ExampleClass, init_args=(0,)) + udfs = server_instance.servicer + return udfs + + +async def start_server(udfs): + server = grpc.aio.server() + accumulator_pb2_grpc.add_AccumulatorServicer_to_server(udfs, server) + listen_addr = "unix:///tmp/accumulator.sock" + server.add_insecure_port(listen_addr) + logging.info("Starting server on %s", listen_addr) + global _s + _s = server + await server.start() + await server.wait_for_termination() + + +class TestAsyncAccumulator(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + global _loop + loop = asyncio.new_event_loop() + _loop = loop + _thread = threading.Thread(target=startup_callable, args=(loop,), daemon=True) + _thread.start() + udfs = NewAsyncAccumulator() + asyncio.run_coroutine_threadsafe(start_server(udfs), loop=loop) + while True: + try: + with grpc.insecure_channel("unix:///tmp/accumulator.sock") as channel: + f = grpc.channel_ready_future(channel) + f.result(timeout=10) + if f.done(): + break + except grpc.FutureTimeoutError as e: + LOGGER.error("error trying to connect to grpc server") + LOGGER.error(e) + + @classmethod + def tearDownClass(cls) -> None: + try: + _loop.stop() + LOGGER.info("stopped the event loop") + except Exception as e: + LOGGER.error(e) + + def test_accumulate(self) -> None: + stub = self.__stub() + request = start_request() + generator_response = None + try: + generator_response = stub.AccumulateFn( + request_iterator=request_generator(count=5, request=request) + ) + except grpc.RpcError as e: + logging.error(e) + + # capture the output from the AccumulateFn generator and assert. + count = 0 + eof_count = 0 + for r in generator_response: + if hasattr(r, "payload") and r.payload.value: + count += 1 + # Each datum should increment the counter + expected_msg = f"counter:{count}" + self.assertEqual( + bytes(expected_msg, encoding="utf-8"), + r.payload.value, + ) + self.assertEqual(r.EOF, False) + # Check that keys are preserved + self.assertEqual(list(r.payload.keys), ["test_key"]) + else: + self.assertEqual(r.EOF, True) + eof_count += 1 + + # We should have received 5 messages (one for each datum) + self.assertEqual(5, count) + self.assertEqual(1, eof_count) + + def test_accumulate_with_multiple_keys(self) -> None: + stub = self.__stub() + request = start_request() + generator_response = None + try: + generator_response = stub.AccumulateFn( + request_iterator=request_generator(count=10, request=request, resetkey=True), + ) + except grpc.RpcError as e: + LOGGER.error(e) + + count = 0 + eof_count = 0 + key_counts = {} + + # capture the output from the AccumulateFn generator and assert. + for r in generator_response: + # Check for responses with values + if r.payload.value: + count += 1 + # Track count per key + key = r.payload.keys[0] if r.payload.keys else "no_key" + key_counts[key] = key_counts.get(key, 0) + 1 + + # Each key should have its own counter starting from 1 + expected_msg = f"counter:{key_counts[key]}" + self.assertEqual( + bytes(expected_msg, encoding="utf-8"), + r.payload.value, + ) + self.assertEqual(r.EOF, False) + else: + eof_count += 1 + self.assertEqual(r.EOF, True) + + # We should have 10 messages (one for each key) + self.assertEqual(10, count) + self.assertEqual(10, eof_count) # Each key/task sends its own EOF + # Each key should appear once + self.assertEqual(len(key_counts), 10) + + def test_accumulate_with_close(self) -> None: + stub = self.__stub() + request = start_request() + generator_response = None + try: + generator_response = stub.AccumulateFn( + request_iterator=request_generator(count=5, request=request, send_close=True) + ) + except grpc.RpcError as e: + logging.error(e) + + # capture the output from the AccumulateFn generator and assert. + count = 0 + eof_count = 0 + for r in generator_response: + if hasattr(r, "payload") and r.payload.value: + count += 1 + # Each datum should increment the counter + expected_msg = f"counter:{count}" + self.assertEqual( + bytes(expected_msg, encoding="utf-8"), + r.payload.value, + ) + self.assertEqual(r.EOF, False) + # Check that keys are preserved + self.assertEqual(list(r.payload.keys), ["test_key"]) + else: + self.assertEqual(r.EOF, True) + eof_count += 1 + + # We should have received 5 messages (one for each datum) + self.assertEqual(5, count) + self.assertEqual(1, eof_count) + + def test_accumulate_append_without_open(self) -> None: + stub = self.__stub() + request = start_request_without_open() + generator_response = None + try: + generator_response = stub.AccumulateFn( + request_iterator=request_generator_append_only(count=5, request=request) + ) + except grpc.RpcError as e: + logging.error(e) + + # capture the output from the AccumulateFn generator and assert. + count = 0 + eof_count = 0 + for r in generator_response: + if hasattr(r, "payload") and r.payload.value: + count += 1 + # Each datum should increment the counter + expected_msg = f"counter:{count}" + self.assertEqual( + bytes(expected_msg, encoding="utf-8"), + r.payload.value, + ) + self.assertEqual(r.EOF, False) + # Check that keys are preserved + self.assertEqual(list(r.payload.keys), ["test_key"]) + else: + self.assertEqual(r.EOF, True) + eof_count += 1 + + # We should have received 5 messages (one for each datum) + self.assertEqual(5, count) + self.assertEqual(1, eof_count) + + def test_accumulate_append_mixed(self) -> None: + stub = self.__stub() + request = start_request() + generator_response = None + try: + generator_response = stub.AccumulateFn( + request_iterator=request_generator_mixed(count=5, request=request) + ) + except grpc.RpcError as e: + logging.error(e) + + # capture the output from the AccumulateFn generator and assert. + count = 0 + eof_count = 0 + for r in generator_response: + if hasattr(r, "payload") and r.payload.value: + count += 1 + # Each datum should increment the counter + expected_msg = "counter:1" + self.assertEqual( + bytes(expected_msg, encoding="utf-8"), + r.payload.value, + ) + self.assertEqual(r.EOF, False) + # Check that keys are preserved + self.assertEqual(list(r.payload.keys), ["test_key"]) + else: + self.assertEqual(r.EOF, True) + eof_count += 1 + + # We should have received 5 messages (one for each datum) + self.assertEqual(3, count) + self.assertEqual(3, eof_count) + + def test_is_ready(self) -> None: + with grpc.insecure_channel("unix:///tmp/accumulator.sock") as channel: + stub = accumulator_pb2_grpc.AccumulatorStub(channel) + + request = _empty_pb2.Empty() + response = None + try: + response = stub.IsReady(request=request) + except grpc.RpcError as e: + logging.error(e) + + self.assertTrue(response.ready) + + def __stub(self): + return accumulator_pb2_grpc.AccumulatorStub(_channel) + + def test_error_init(self): + # Check that accumulator_instance is required + with self.assertRaises(TypeError): + AccumulatorAsyncServer() + # Check that the init_args and init_kwargs are passed + # only with an Accumulator class + with self.assertRaises(TypeError): + AccumulatorAsyncServer(accumulator_handler_func, init_args=(0, 1)) + # Check that an instance is not passed instead of the class + # signature + with self.assertRaises(TypeError): + AccumulatorAsyncServer(ExampleClass(0)) + + # Check that an invalid class is passed + class ExampleBadClass: + pass + + with self.assertRaises(TypeError): + AccumulatorAsyncServer(accumulator_instance=ExampleBadClass) + + def test_max_threads(self): + # max cap at 16 + server = AccumulatorAsyncServer(accumulator_instance=ExampleClass, max_threads=32) + self.assertEqual(server.max_threads, 16) + + # use argument provided + server = AccumulatorAsyncServer(accumulator_instance=ExampleClass, max_threads=5) + self.assertEqual(server.max_threads, 5) + + # defaults to 4 + server = AccumulatorAsyncServer(accumulator_instance=ExampleClass) + self.assertEqual(server.max_threads, 4) + + # zero threads + server = AccumulatorAsyncServer(ExampleClass, max_threads=0) + self.assertEqual(server.max_threads, 0) + + # negative threads + server = AccumulatorAsyncServer(ExampleClass, max_threads=-5) + self.assertEqual(server.max_threads, -5) + + def test_server_info_file_path_handling(self): + """Test AccumulatorAsyncServer with custom server info file path.""" + + server = AccumulatorAsyncServer( + ExampleClass, init_args=(0,), server_info_file="/custom/path/server_info.json" + ) + + self.assertEqual(server.server_info_file, "/custom/path/server_info.json") + + def test_init_kwargs_none_handling(self): + """Test init_kwargs None handling in AccumulatorAsyncServer.""" + + server = AccumulatorAsyncServer( + ExampleClass, init_args=(0,), init_kwargs=None # This should be converted to {} + ) + + # Should not raise any errors and should work correctly + self.assertIsNotNone(server.accumulator_handler) + + def test_server_start_method_logging(self): + """Test server start method includes proper logging.""" + from unittest.mock import patch + + server = AccumulatorAsyncServer(ExampleClass) + + # Mock aiorun.run to prevent actual server startup + with patch("pynumaflow.accumulator.async_server.aiorun") as mock_aiorun, patch( + "pynumaflow.accumulator.async_server._LOGGER" + ) as mock_logger: + server.start() + + # Verify logging was called + mock_logger.info.assert_called_once_with("Starting Async Accumulator Server") + + # Verify aiorun.run was called with correct parameters + mock_aiorun.run.assert_called_once() + self.assertTrue(mock_aiorun.run.call_args[1]["use_uvloop"]) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + unittest.main() diff --git a/tests/accumulator/test_async_accumulator_err.py b/tests/accumulator/test_async_accumulator_err.py new file mode 100644 index 00000000..5b39174c --- /dev/null +++ b/tests/accumulator/test_async_accumulator_err.py @@ -0,0 +1,175 @@ +import asyncio +import logging +import threading +import unittest +from collections.abc import AsyncIterable +from unittest.mock import patch + +import grpc +from grpc.aio._server import Server + +from pynumaflow import setup_logging +from pynumaflow.accumulator import ( + Message, + Datum, + AccumulatorAsyncServer, + Accumulator, +) +from pynumaflow.proto.accumulator import accumulator_pb2, accumulator_pb2_grpc +from pynumaflow.shared.asynciter import NonBlockingIterator +from tests.testing_utils import ( + mock_message, + get_time_args, + mock_terminate_on_stop, +) + +LOGGER = setup_logging(__name__) + + +def request_generator(count, request): + for i in range(count): + yield request + + +def start_request() -> accumulator_pb2.AccumulatorRequest: + event_time_timestamp, watermark_timestamp = get_time_args() + window = accumulator_pb2.KeyedWindow( + start=event_time_timestamp, + end=watermark_timestamp, + slot="slot-0", + keys=["test_key"], + ) + payload = accumulator_pb2.Payload( + keys=["test_key"], + value=mock_message(), + event_time=event_time_timestamp, + watermark=watermark_timestamp, + id="test_id", + headers={"test_header_key": "test_header_value", "source": "test_source"}, + ) + operation = accumulator_pb2.AccumulatorRequest.WindowOperation( + event=accumulator_pb2.AccumulatorRequest.WindowOperation.Event.OPEN, + keyedWindow=window, + ) + request = accumulator_pb2.AccumulatorRequest( + payload=payload, + operation=operation, + ) + return request + + +_s: Server = None +_channel = grpc.insecure_channel("unix:///tmp/accumulator_err.sock") +_loop = None + + +def startup_callable(loop): + asyncio.set_event_loop(loop) + loop.run_forever() + + +class ExampleErrorClass(Accumulator): + def __init__(self, counter): + self.counter = counter + + async def handler(self, datums: AsyncIterable[Datum], output: NonBlockingIterator): + async for datum in datums: + self.counter += 1 + if self.counter == 2: + # Simulate an error on the second datum + raise RuntimeError("Simulated error in accumulator handler") + msg = f"counter:{self.counter}" + await output.put(Message(str.encode(msg), keys=datum.keys(), tags=[])) + + +async def error_accumulator_handler_func(datums: AsyncIterable[Datum], output: NonBlockingIterator): + counter = 0 + async for datum in datums: + counter += 1 + if counter == 2: + # Simulate an error on the second datum + raise RuntimeError("Simulated error in accumulator function") + msg = f"counter:{counter}" + await output.put(Message(str.encode(msg), keys=datum.keys(), tags=[])) + + +def NewAsyncAccumulatorError(): + server_instance = AccumulatorAsyncServer(ExampleErrorClass, init_args=(0,)) + udfs = server_instance.servicer + return udfs + + +@patch("psutil.Process.kill", mock_terminate_on_stop) +async def start_server(udfs): + server = grpc.aio.server() + accumulator_pb2_grpc.add_AccumulatorServicer_to_server(udfs, server) + listen_addr = "unix:///tmp/accumulator_err.sock" + server.add_insecure_port(listen_addr) + logging.info("Starting server on %s", listen_addr) + global _s + _s = server + await server.start() + await server.wait_for_termination() + + +@patch("psutil.Process.kill", mock_terminate_on_stop) +class TestAsyncAccumulatorError(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + global _loop + loop = asyncio.new_event_loop() + _loop = loop + _thread = threading.Thread(target=startup_callable, args=(loop,), daemon=True) + _thread.start() + udfs = NewAsyncAccumulatorError() + asyncio.run_coroutine_threadsafe(start_server(udfs), loop=loop) + while True: + try: + with grpc.insecure_channel("unix:///tmp/accumulator_err.sock") as channel: + f = grpc.channel_ready_future(channel) + f.result(timeout=10) + if f.done(): + break + except grpc.FutureTimeoutError as e: + LOGGER.error("error trying to connect to grpc server") + LOGGER.error(e) + + @classmethod + def tearDownClass(cls) -> None: + try: + _loop.stop() + LOGGER.info("stopped the event loop") + except Exception as e: + LOGGER.error(e) + + @patch("psutil.Process.kill", mock_terminate_on_stop) + def test_accumulate_partial_success(self) -> None: + """Test that the first datum is processed before error occurs""" + stub = self.__stub() + request = start_request() + + try: + generator_response = stub.AccumulateFn( + request_iterator=request_generator(count=5, request=request) + ) + + # Try to consume the generator + counter = 0 + for response in generator_response: + self.assertIsInstance(response, accumulator_pb2.AccumulatorResponse) + self.assertTrue(response.payload.value.startswith(b"counter:")) + counter += 1 + + self.assertEqual(counter, 1, "Expected only one successful response before error") + except BaseException as err: + self.assertTrue("Simulated error in accumulator handler" in str(err)) + return + self.fail("Expected an exception.") + + def __stub(self): + return accumulator_pb2_grpc.AccumulatorStub(_channel) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + unittest.main() diff --git a/tests/accumulator/test_datatypes.py b/tests/accumulator/test_datatypes.py new file mode 100644 index 00000000..a71f3452 --- /dev/null +++ b/tests/accumulator/test_datatypes.py @@ -0,0 +1,339 @@ +import unittest +from collections.abc import AsyncIterable +from datetime import datetime, timezone + +from google.protobuf import timestamp_pb2 as _timestamp_pb2 +from pynumaflow.accumulator import Accumulator + +from pynumaflow.accumulator._dtypes import ( + IntervalWindow, + KeyedWindow, + Datum, + AccumulatorResult, + AccumulatorRequest, + WindowOperation, + Message, +) +from pynumaflow.shared.asynciter import NonBlockingIterator +from tests.testing_utils import ( + mock_message, + mock_event_time, + mock_watermark, + mock_start_time, + mock_end_time, +) + +TEST_KEYS = ["test"] +TEST_ID = "test_id" +TEST_HEADERS = {"key1": "value1", "key2": "value2"} + + +class TestDatum(unittest.TestCase): + def test_err_event_time(self): + ts = _timestamp_pb2.Timestamp() + ts.GetCurrentTime() + headers = {"key1": "value1", "key2": "value2"} + with self.assertRaises(Exception) as context: + Datum( + keys=TEST_KEYS, + value=mock_message(), + event_time=ts, + watermark=mock_watermark(), + id_=TEST_ID, + headers=headers, + ) + self.assertEqual( + "Wrong data type: " + "for Datum.event_time", + str(context.exception), + ) + + def test_err_watermark(self): + ts = _timestamp_pb2.Timestamp() + ts.GetCurrentTime() + headers = {"key1": "value1", "key2": "value2"} + with self.assertRaises(Exception) as context: + Datum( + keys=TEST_KEYS, + value=mock_message(), + event_time=mock_event_time(), + watermark=ts, + id_=TEST_ID, + headers=headers, + ) + self.assertEqual( + "Wrong data type: " + "for Datum.watermark", + str(context.exception), + ) + + def test_properties(self): + d = Datum( + keys=TEST_KEYS, + value=mock_message(), + event_time=mock_event_time(), + watermark=mock_watermark(), + id_=TEST_ID, + headers=TEST_HEADERS, + ) + self.assertEqual(mock_message(), d.value) + self.assertEqual(TEST_KEYS, d.keys()) + self.assertEqual(mock_event_time(), d.event_time) + self.assertEqual(mock_watermark(), d.watermark) + self.assertEqual(TEST_HEADERS, d.headers) + self.assertEqual(TEST_ID, d.id) + + def test_default_values(self): + d = Datum( + keys=None, + value=None, + event_time=mock_event_time(), + watermark=mock_watermark(), + id_=TEST_ID, + ) + self.assertEqual([], d.keys()) + self.assertEqual(b"", d.value) + self.assertEqual({}, d.headers) + + +class TestIntervalWindow(unittest.TestCase): + def test_start(self): + i = IntervalWindow(start=mock_start_time(), end=mock_end_time()) + self.assertEqual(mock_start_time(), i.start) + + def test_end(self): + i = IntervalWindow(start=mock_start_time(), end=mock_end_time()) + self.assertEqual(mock_end_time(), i.end) + + +class TestKeyedWindow(unittest.TestCase): + def test_create_window(self): + kw = KeyedWindow( + start=mock_start_time(), end=mock_end_time(), slot="slot-0", keys=["key1", "key2"] + ) + self.assertEqual(kw.start, mock_start_time()) + self.assertEqual(kw.end, mock_end_time()) + self.assertEqual(kw.slot, "slot-0") + self.assertEqual(kw.keys, ["key1", "key2"]) + + def test_default_values(self): + kw = KeyedWindow(start=mock_start_time(), end=mock_end_time()) + self.assertEqual(kw.slot, "") + self.assertEqual(kw.keys, []) + + def test_window_property(self): + kw = KeyedWindow(start=mock_start_time(), end=mock_end_time()) + self.assertIsInstance(kw.window, IntervalWindow) + self.assertEqual(kw.window.start, mock_start_time()) + self.assertEqual(kw.window.end, mock_end_time()) + + +class TestAccumulatorResult(unittest.TestCase): + def test_create_result(self): + # Create mock objects + future = None # In real usage, this would be an asyncio.Task + iterator = NonBlockingIterator() + keys = ["key1", "key2"] + result_queue = NonBlockingIterator() + consumer_future = None # In real usage, this would be an asyncio.Task + watermark = datetime.fromtimestamp(1662998400, timezone.utc) + + result = AccumulatorResult(future, iterator, keys, result_queue, consumer_future, watermark) + + self.assertEqual(result.future, future) + self.assertEqual(result.iterator, iterator) + self.assertEqual(result.keys, keys) + self.assertEqual(result.result_queue, result_queue) + self.assertEqual(result.consumer_future, consumer_future) + self.assertEqual(result.latest_watermark, watermark) + + def test_update_watermark(self): + result = AccumulatorResult( + None, None, [], None, None, datetime.fromtimestamp(1662998400, timezone.utc) + ) + new_watermark = datetime.fromtimestamp(1662998460, timezone.utc) + result.update_watermark(new_watermark) + self.assertEqual(result.latest_watermark, new_watermark) + + def test_update_watermark_invalid_type(self): + result = AccumulatorResult( + None, None, [], None, None, datetime.fromtimestamp(1662998400, timezone.utc) + ) + with self.assertRaises(TypeError): + result.update_watermark("not a datetime") + + +class TestAccumulatorRequest(unittest.TestCase): + def test_create_request(self): + operation = WindowOperation.OPEN + keyed_window = KeyedWindow(start=mock_start_time(), end=mock_end_time()) + payload = Datum( + keys=TEST_KEYS, + value=mock_message(), + event_time=mock_event_time(), + watermark=mock_watermark(), + id_=TEST_ID, + ) + + request = AccumulatorRequest(operation, keyed_window, payload) + self.assertEqual(request.operation, operation) + self.assertEqual(request.keyed_window, keyed_window) + self.assertEqual(request.payload, payload) + + +class TestWindowOperation(unittest.TestCase): + def test_enum_values(self): + self.assertEqual(WindowOperation.OPEN, 0) + self.assertEqual(WindowOperation.CLOSE, 1) + self.assertEqual(WindowOperation.APPEND, 2) + + +class TestMessage(unittest.TestCase): + def test_create_message(self): + value = b"test_value" + keys = ["key1", "key2"] + tags = ["tag1", "tag2"] + + msg = Message(value=value, keys=keys, tags=tags) + self.assertEqual(msg.value, value) + self.assertEqual(msg.keys, keys) + self.assertEqual(msg.tags, tags) + + def test_default_values(self): + msg = Message(value=b"test") + self.assertEqual(msg.keys, []) + self.assertEqual(msg.tags, []) + + def test_to_drop(self): + msg = Message.to_drop() + self.assertEqual(msg.value, b"") + self.assertEqual(msg.keys, []) + self.assertTrue("U+005C__DROP__" in msg.tags) + + def test_none_values(self): + msg = Message(value=None, keys=None, tags=None) + self.assertEqual(msg.value, b"") + self.assertEqual(msg.keys, []) + self.assertEqual(msg.tags, []) + + def test_from_datum(self): + """Test that Message.from_datum correctly creates a Message from a Datum""" + # Create a sample datum with all properties + test_keys = ["key1", "key2"] + test_value = b"test_message_value" + test_event_time = mock_event_time() + test_watermark = mock_watermark() + test_headers = {"header1": "value1", "header2": "value2"} + test_id = "test_datum_id" + + datum = Datum( + keys=test_keys, + value=test_value, + event_time=test_event_time, + watermark=test_watermark, + id_=test_id, + headers=test_headers, + ) + + # Create message from datum + message = Message.from_datum(datum) + + # Verify all properties are correctly transferred + self.assertEqual(message.value, test_value) + self.assertEqual(message.keys, test_keys) + self.assertEqual(message.event_time, test_event_time) + self.assertEqual(message.watermark, test_watermark) + self.assertEqual(message.headers, test_headers) + self.assertEqual(message.id, test_id) + + # Verify that tags are empty (default for Message) + self.assertEqual(message.tags, []) + + def test_from_datum_minimal(self): + """Test from_datum with minimal Datum (no headers)""" + test_keys = ["minimal_key"] + test_value = b"minimal_value" + test_event_time = mock_event_time() + test_watermark = mock_watermark() + test_id = "minimal_id" + + datum = Datum( + keys=test_keys, + value=test_value, + event_time=test_event_time, + watermark=test_watermark, + id_=test_id, + # headers not provided (will default to {}) + ) + + message = Message.from_datum(datum) + + self.assertEqual(message.value, test_value) + self.assertEqual(message.keys, test_keys) + self.assertEqual(message.event_time, test_event_time) + self.assertEqual(message.watermark, test_watermark) + self.assertEqual(message.headers, {}) + self.assertEqual(message.id, test_id) + self.assertEqual(message.tags, []) + + def test_from_datum_empty_keys(self): + """Test from_datum with empty keys""" + datum = Datum( + keys=None, # Will default to [] + value=b"test_value", + event_time=mock_event_time(), + watermark=mock_watermark(), + id_="test_id", + ) + + message = Message.from_datum(datum) + + self.assertEqual(message.keys, []) + self.assertEqual(message.value, b"test_value") + self.assertEqual(message.id, "test_id") + + +class TestAccumulatorClass(unittest.TestCase): + class ExampleClass(Accumulator): + async def handler(self, datums: AsyncIterable[Datum], output: NonBlockingIterator): + pass + + def __init__(self, test1, test2): + self.test1 = test1 + self.test2 = test2 + self.test3 = self.test1 + + def test_init(self): + r = self.ExampleClass(test1=1, test2=2) + self.assertEqual(1, r.test1) + self.assertEqual(2, r.test2) + self.assertEqual(1, r.test3) + + def test_callable(self): + """Test that accumulator instances can be called directly""" + r = self.ExampleClass(test1=1, test2=2) + # The __call__ method should be callable and delegate to the handler method + self.assertTrue(callable(r)) + # __call__ should return the result of calling handler + # Since handler is an async method, __call__ should return a coroutine + import asyncio + from pynumaflow.shared.asynciter import NonBlockingIterator + + async def test_datums(): + yield Datum( + keys=["test"], + value=b"test", + event_time=mock_event_time(), + watermark=mock_watermark(), + id_="test", + ) + + output = NonBlockingIterator() + result = r(test_datums(), output) + self.assertTrue(asyncio.iscoroutine(result)) + # Clean up the coroutine + result.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/accumulator/utils.py b/tests/accumulator/utils.py new file mode 100644 index 00000000..d0c68fbb --- /dev/null +++ b/tests/accumulator/utils.py @@ -0,0 +1,23 @@ +from datetime import datetime, timezone +from pynumaflow.accumulator import Datum + + +def create_test_datum(keys, value, event_time=None, watermark=None, id_=None, headers=None): + """Create a test Datum object with default values""" + if event_time is None: + event_time = datetime.fromtimestamp(1662998400, timezone.utc) + if watermark is None: + watermark = datetime.fromtimestamp(1662998460, timezone.utc) + if id_ is None: + id_ = "test_id" + if headers is None: + headers = {} + + return Datum( + keys=keys, + value=value, + event_time=event_time, + watermark=watermark, + id_=id_, + headers=headers, + ) From 3e01da23c6f45beb0661966fa6da59890d16e2d5 Mon Sep 17 00:00:00 2001 From: Takashi Menjo Date: Tue, 5 Aug 2025 03:51:36 +0900 Subject: [PATCH 03/11] chore: Datum keys as property (#239) Signed-off-by: Takashi Menjo --- pynumaflow/accumulator/_dtypes.py | 3 ++- pynumaflow/accumulator/servicer/task_manager.py | 6 +++--- pynumaflow/batchmapper/_dtypes.py | 1 + pynumaflow/mapstreamer/_dtypes.py | 1 + pynumaflow/reducer/_dtypes.py | 1 + pynumaflow/reducer/servicer/task_manager.py | 4 ++-- pynumaflow/reducestreamer/_dtypes.py | 1 + pynumaflow/reducestreamer/servicer/task_manager.py | 4 ++-- pynumaflow/sourcetransformer/_dtypes.py | 1 + tests/accumulator/test_async_accumulator.py | 4 ++-- tests/accumulator/test_async_accumulator_err.py | 4 ++-- tests/accumulator/test_datatypes.py | 4 ++-- tests/batchmap/test_datatypes.py | 2 +- tests/reduce/test_datatypes.py | 2 +- tests/reducestreamer/test_datatypes.py | 2 +- 15 files changed, 23 insertions(+), 17 deletions(-) diff --git a/pynumaflow/accumulator/_dtypes.py b/pynumaflow/accumulator/_dtypes.py index 31a0d5fe..52b36a13 100644 --- a/pynumaflow/accumulator/_dtypes.py +++ b/pynumaflow/accumulator/_dtypes.py @@ -76,6 +76,7 @@ def __init__( self._headers = headers or {} self._id = id_ + @property def keys(self) -> list[str]: """Returns the keys of the event. @@ -488,7 +489,7 @@ def from_datum(cls, datum: Datum): """ return cls( value=datum.value, - keys=datum.keys(), + keys=datum.keys, watermark=datum.watermark, event_time=datum.event_time, headers=datum.headers, diff --git a/pynumaflow/accumulator/servicer/task_manager.py b/pynumaflow/accumulator/servicer/task_manager.py index a7c80968..dc333768 100644 --- a/pynumaflow/accumulator/servicer/task_manager.py +++ b/pynumaflow/accumulator/servicer/task_manager.py @@ -106,7 +106,7 @@ async def close_task(self, req): 4. Remove the task from the tracker """ d = req.payload - keys = d.keys() + keys = d.keys unified_key = build_unique_key_name(keys) curr_task = self.tasks.get(unified_key, None) @@ -128,7 +128,7 @@ async def create_task(self, req): it creates a new task or appends the request to the existing task. """ d = req.payload - keys = d.keys() + keys = d.keys unified_key = build_unique_key_name(keys) curr_task = self.tasks.get(unified_key, None) @@ -179,7 +179,7 @@ async def send_datum_to_task(self, req): If the task does not exist, create it. """ d = req.payload - keys = d.keys() + keys = d.keys unified_key = build_unique_key_name(keys) result = self.tasks.get(unified_key, None) if not result: diff --git a/pynumaflow/batchmapper/_dtypes.py b/pynumaflow/batchmapper/_dtypes.py index edeeb08a..f8a9fb82 100644 --- a/pynumaflow/batchmapper/_dtypes.py +++ b/pynumaflow/batchmapper/_dtypes.py @@ -99,6 +99,7 @@ def __init__( self._watermark = watermark self._headers = headers or {} + @property def keys(self) -> list[str]: """Returns the keys of the event""" return self._keys diff --git a/pynumaflow/mapstreamer/_dtypes.py b/pynumaflow/mapstreamer/_dtypes.py index 43089415..a278ab38 100644 --- a/pynumaflow/mapstreamer/_dtypes.py +++ b/pynumaflow/mapstreamer/_dtypes.py @@ -151,6 +151,7 @@ def __init__( self._watermark = watermark self._headers = headers or {} + @property def keys(self) -> list[str]: """Returns the keys of the event""" return self._keys diff --git a/pynumaflow/reducer/_dtypes.py b/pynumaflow/reducer/_dtypes.py index 99fddcae..6a70edb5 100644 --- a/pynumaflow/reducer/_dtypes.py +++ b/pynumaflow/reducer/_dtypes.py @@ -167,6 +167,7 @@ def __init__( self._watermark = watermark self._headers = headers or {} + @property def keys(self) -> list[str]: """Returns the keys of the event""" return self._keys diff --git a/pynumaflow/reducer/servicer/task_manager.py b/pynumaflow/reducer/servicer/task_manager.py index bdb2e0b8..cb453d78 100644 --- a/pynumaflow/reducer/servicer/task_manager.py +++ b/pynumaflow/reducer/servicer/task_manager.py @@ -108,7 +108,7 @@ async def create_task(self, req): raise UDFError("reduce create operation error: invalid number of windows") d = req.payload - keys = d.keys() + keys = d.keys unified_key = build_unique_key_name(keys, req.windows[0]) result = self.tasks.get(unified_key, None) @@ -137,7 +137,7 @@ async def append_task(self, req): if len(req.windows) != 1: raise UDFError("reduce create operation error: invalid number of windows") d = req.payload - keys = d.keys() + keys = d.keys unified_key = build_unique_key_name(keys, req.windows[0]) result = self.tasks.get(unified_key, None) if not result: diff --git a/pynumaflow/reducestreamer/_dtypes.py b/pynumaflow/reducestreamer/_dtypes.py index f0298543..8628a6c0 100644 --- a/pynumaflow/reducestreamer/_dtypes.py +++ b/pynumaflow/reducestreamer/_dtypes.py @@ -73,6 +73,7 @@ def __init__( self._watermark = watermark self._headers = headers or {} + @property def keys(self) -> list[str]: """Returns the keys of the event""" return self._keys diff --git a/pynumaflow/reducestreamer/servicer/task_manager.py b/pynumaflow/reducestreamer/servicer/task_manager.py index 4e078b3e..1216372b 100644 --- a/pynumaflow/reducestreamer/servicer/task_manager.py +++ b/pynumaflow/reducestreamer/servicer/task_manager.py @@ -110,7 +110,7 @@ async def create_task(self, req): raise UDFError("reduce create operation error: invalid number of windows") d = req.payload - keys = d.keys() + keys = d.keys unified_key = build_unique_key_name(keys, req.windows[0]) curr_task = self.tasks.get(unified_key, None) @@ -161,7 +161,7 @@ async def send_datum_to_task(self, req): if len(req.windows) != 1: raise UDFError("reduce append operation error: invalid number of windows") d = req.payload - keys = d.keys() + keys = d.keys unified_key = build_unique_key_name(keys, req.windows[0]) result = self.tasks.get(unified_key, None) if not result: diff --git a/pynumaflow/sourcetransformer/_dtypes.py b/pynumaflow/sourcetransformer/_dtypes.py index bc0ec7b5..4d3da5ad 100644 --- a/pynumaflow/sourcetransformer/_dtypes.py +++ b/pynumaflow/sourcetransformer/_dtypes.py @@ -160,6 +160,7 @@ def __init__( self._watermark = watermark self._headers = headers or {} + @property def keys(self) -> list[str]: """Returns the keys of the event""" return self._keys diff --git a/tests/accumulator/test_async_accumulator.py b/tests/accumulator/test_async_accumulator.py index 292e3687..e0927f8e 100644 --- a/tests/accumulator/test_async_accumulator.py +++ b/tests/accumulator/test_async_accumulator.py @@ -140,7 +140,7 @@ async def handler(self, datums: AsyncIterable[Datum], output: NonBlockingIterato async for datum in datums: self.counter += 1 msg = f"counter:{self.counter}" - await output.put(Message(str.encode(msg), keys=datum.keys(), tags=[])) + await output.put(Message(str.encode(msg), keys=datum.keys, tags=[])) async def accumulator_handler_func(datums: AsyncIterable[Datum], output: NonBlockingIterator): @@ -148,7 +148,7 @@ async def accumulator_handler_func(datums: AsyncIterable[Datum], output: NonBloc async for datum in datums: counter += 1 msg = f"counter:{counter}" - await output.put(Message(str.encode(msg), keys=datum.keys(), tags=[])) + await output.put(Message(str.encode(msg), keys=datum.keys, tags=[])) def NewAsyncAccumulator(): diff --git a/tests/accumulator/test_async_accumulator_err.py b/tests/accumulator/test_async_accumulator_err.py index 5b39174c..a6b49b26 100644 --- a/tests/accumulator/test_async_accumulator_err.py +++ b/tests/accumulator/test_async_accumulator_err.py @@ -79,7 +79,7 @@ async def handler(self, datums: AsyncIterable[Datum], output: NonBlockingIterato # Simulate an error on the second datum raise RuntimeError("Simulated error in accumulator handler") msg = f"counter:{self.counter}" - await output.put(Message(str.encode(msg), keys=datum.keys(), tags=[])) + await output.put(Message(str.encode(msg), keys=datum.keys, tags=[])) async def error_accumulator_handler_func(datums: AsyncIterable[Datum], output: NonBlockingIterator): @@ -90,7 +90,7 @@ async def error_accumulator_handler_func(datums: AsyncIterable[Datum], output: N # Simulate an error on the second datum raise RuntimeError("Simulated error in accumulator function") msg = f"counter:{counter}" - await output.put(Message(str.encode(msg), keys=datum.keys(), tags=[])) + await output.put(Message(str.encode(msg), keys=datum.keys, tags=[])) def NewAsyncAccumulatorError(): diff --git a/tests/accumulator/test_datatypes.py b/tests/accumulator/test_datatypes.py index a71f3452..d82e4bf2 100644 --- a/tests/accumulator/test_datatypes.py +++ b/tests/accumulator/test_datatypes.py @@ -77,7 +77,7 @@ def test_properties(self): headers=TEST_HEADERS, ) self.assertEqual(mock_message(), d.value) - self.assertEqual(TEST_KEYS, d.keys()) + self.assertEqual(TEST_KEYS, d.keys) self.assertEqual(mock_event_time(), d.event_time) self.assertEqual(mock_watermark(), d.watermark) self.assertEqual(TEST_HEADERS, d.headers) @@ -91,7 +91,7 @@ def test_default_values(self): watermark=mock_watermark(), id_=TEST_ID, ) - self.assertEqual([], d.keys()) + self.assertEqual([], d.keys) self.assertEqual(b"", d.value) self.assertEqual({}, d.headers) diff --git a/tests/batchmap/test_datatypes.py b/tests/batchmap/test_datatypes.py index 8649732a..06fb3624 100644 --- a/tests/batchmap/test_datatypes.py +++ b/tests/batchmap/test_datatypes.py @@ -72,7 +72,7 @@ def test_key(self): watermark=mock_watermark(), id=TEST_ID, ) - self.assertEqual(TEST_KEYS, d.keys()) + self.assertEqual(TEST_KEYS, d.keys) def test_event_time(self): d = Datum( diff --git a/tests/reduce/test_datatypes.py b/tests/reduce/test_datatypes.py index a92b27fb..20a520dd 100644 --- a/tests/reduce/test_datatypes.py +++ b/tests/reduce/test_datatypes.py @@ -75,7 +75,7 @@ def test_key(self): event_time=mock_event_time(), watermark=mock_watermark(), ) - self.assertEqual(TEST_KEYS, d.keys()) + self.assertEqual(TEST_KEYS, d.keys) def test_event_time(self): d = Datum( diff --git a/tests/reducestreamer/test_datatypes.py b/tests/reducestreamer/test_datatypes.py index d7d75b6b..4d3ffc97 100644 --- a/tests/reducestreamer/test_datatypes.py +++ b/tests/reducestreamer/test_datatypes.py @@ -76,7 +76,7 @@ def test_key(self): event_time=mock_event_time(), watermark=mock_watermark(), ) - self.assertEqual(TEST_KEYS, d.keys()) + self.assertEqual(TEST_KEYS, d.keys) def test_event_time(self): d = Datum( From 941a5297862edcebdb584c3a5d9fd392a4003f49 Mon Sep 17 00:00:00 2001 From: Sidhant Kohli Date: Mon, 4 Aug 2025 14:08:40 -0700 Subject: [PATCH 04/11] chore: fix cve with setuptools (#240) Signed-off-by: kohlisid --- poetry.lock | 8 ++++---- tests/map/test_multiproc_mapper.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index ab404238..a9ad8fb1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1005,19 +1005,19 @@ files = [ [[package]] name = "setuptools" -version = "75.8.0" +version = "80.9.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "setuptools-75.8.0-py3-none-any.whl", hash = "sha256:e3982f444617239225d675215d51f6ba05f845d4eec313da4418fdbb56fb27e3"}, - {file = "setuptools-75.8.0.tar.gz", hash = "sha256:c5afc8f407c626b8313a86e10311dd3f661c6cd9c09d4bf8c15c0e11f9f2b0e6"}, + {file = "setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922"}, + {file = "setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c"}, ] [package.extras] check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""] -core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.collections", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] diff --git a/tests/map/test_multiproc_mapper.py b/tests/map/test_multiproc_mapper.py index 418d0161..77fda9fa 100644 --- a/tests/map/test_multiproc_mapper.py +++ b/tests/map/test_multiproc_mapper.py @@ -36,7 +36,7 @@ def test_max_process_count(self) -> None: """Max process count is capped at 2 * os.cpu_count, irrespective of what the user provides as input""" default_val = os.cpu_count() - server = MapMultiprocServer(mapper_instance=map_handler, server_count=20) + server = MapMultiprocServer(mapper_instance=map_handler, server_count=100) self.assertEqual(server._process_count, default_val * 2) def test_udf_map_err_handshake(self): From 9b926c78f2a89f7daad628ac298e87378cfa7cf2 Mon Sep 17 00:00:00 2001 From: Sidhant Kohli Date: Mon, 4 Aug 2025 16:54:26 -0700 Subject: [PATCH 05/11] chore: release version 0.10.0 (#241) Signed-off-by: kohlisid --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 41816b73..ed7ddfc2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pynumaflow" -version = "0.10.0a0" +version = "0.10.0" description = "Provides the interfaces of writing Python User Defined Functions and Sinks for NumaFlow." authors = ["NumaFlow Developers"] readme = "README.md" From 17e063b23f25587e292a0f1b086da8ed0a65c2ab Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 4 Aug 2025 20:03:36 -0700 Subject: [PATCH 06/11] docs: updated CHANGELOG.md (#229) --- CHANGELOG.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a52642a..1af24e6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,35 @@ # Changelog +## v0.10.0a0 (2025-06-25) + + +### Contributors + + +## v0.10.0 (2025-08-04) + + * [4982414](https://github.com/numaproj/numaflow-python/commit/49824144e6b1586f612bad601e47aeb2166963ca) feat: add Accumulator (#237) + * [3ed02e6](https://github.com/numaproj/numaflow-python/commit/3ed02e671cf6fee55e19ff53ad716a012482456a) feat: add async source transformer (#230) + +### Contributors + + * Sidhant Kohli + * shrivardhan + +## v0.9.2 (2025-05-24) + + * [6ccd49e](https://github.com/numaproj/numaflow-python/commit/6ccd49ef09ad11eac9b0bb9a6b0f5517d858bea4) feat: utility function to persist critical error (#222) + +### Contributors + + * Adarsh Jain + +## v0.9.1 (2025-02-13) + + +### Contributors + + ## v0.9.0 (2024-11-08) * [f2f7bf6](https://github.com/numaproj/numaflow-python/commit/f2f7bf67ed89d74f51173e907c30abb116bb2cf7) feat: update batchmap and mapstream to use Map proto (#200) @@ -14,6 +44,12 @@ * Keran Yang * Sidhant Kohli +## v0.8.1 (2025-01-29) + + +### Contributors + + ## v0.8.0 (2024-08-19) * [921b7c0](https://github.com/numaproj/numaflow-python/commit/921b7c069d726799ccdfc2f3b2ec3a1ebc27669b) feat: batch-map implementation (#177) From b944a5d5883844c0c7bde0714fc85d60aa0c138a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Th=C3=B8gersen?= Date: Thu, 11 Sep 2025 00:30:50 +0200 Subject: [PATCH 07/11] doc: fix async reducer README examples (#244) --- examples/reduce/README.md | 52 ++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/examples/reduce/README.md b/examples/reduce/README.md index c8da232e..63b9735d 100644 --- a/examples/reduce/README.md +++ b/examples/reduce/README.md @@ -7,11 +7,13 @@ For creating a reducer UDF we can use two different approaches: kwargs that the custom reducer class requires. - Finally we need to call the `start` method on the `ReduceAsyncServer` instance to start the reducer server. ```python - from numaflow import Reducer, ReduceAsyncServer - class Example(Reducer): + from collections.abc import AsyncIterable + from pynumaflow.reducer import Reducer, ReduceAsyncServer, Datum, Message, Messages, Metadata + + class Example(Reducer): def __init__(self, counter): self.counter = counter - + async def handler( self, keys: list[str], datums: AsyncIterable[Datum], md: Metadata ) -> Messages: @@ -25,13 +27,13 @@ For creating a reducer UDF we can use two different approaches: ) return Messages(Message(str.encode(msg), keys=keys)) - if __name__ == "__main__": - # Here we are using the class instance as the reducer_instance - # which will be used to invoke the handler function. - # We are passing the init_args for the class instance. - grpc_server = ReduceAsyncServer(Example, init_args=(0,)) - grpc_server.start() - ``` + if __name__ == "__main__": + # Here we are using the class instance as the reducer_instance + # which will be used to invoke the handler function. + # We are passing the init_args for the class instance. + grpc_server = ReduceAsyncServer(Example, init_args=(0,)) + grpc_server.start() + ``` - Function based reducer For the function based reducer we need to create a function of the signature @@ -43,23 +45,23 @@ For creating a reducer UDF we can use two different approaches: - Finally we need to call the `start` method on the `ReduceAsyncServer` instance to start the reducer server. - We must ensure that no init_args or init_kwargs are passed to the `ReduceAsyncServer` instance as they are not used for function based reducers. ```python - from numaflow import ReduceAsyncServer - async def handler(keys: list[str], datums: AsyncIterable[Datum], md: Metadata) -> Messages: - counter = 0 - interval_window = md.interval_window - async for _ in datums: - counter += 1 - msg = ( - f"counter:{counter} interval_window_start:{interval_window.start} " - f"interval_window_end:{interval_window.end}" - ) - return Messages(Message(str.encode(msg), keys=keys)) - - if __name__ == "__main__": + from collections.abc import AsyncIterable + from pynumaflow.reducer import ReduceAsyncServer, Datum, Message, Messages, Metadata + + async def handler(keys: list[str], datums: AsyncIterable[Datum], md: Metadata) -> Messages: + counter = 0 + interval_window = md.interval_window + async for _ in datums: + counter += 1 + msg = ( + f"counter:{counter} interval_window_start:{interval_window.start} " + f"interval_window_end:{interval_window.end}" + ) + return Messages(Message(str.encode(msg), keys=keys)) + + if __name__ == "__main__": # Here we are using the function as the reducer_instance # which will be used to invoke the handler function. grpc_server = ReduceAsyncServer(handler) grpc_server.start() ``` - - From bf00b2e44dce6edfdfaf78466b9e636eebe961e9 Mon Sep 17 00:00:00 2001 From: Sidhant Kohli Date: Mon, 15 Sep 2025 10:30:59 -0700 Subject: [PATCH 08/11] feat: parallelized mapstreamer (#242) Signed-off-by: kohlisid --- .../mapstreamer/servicer/async_servicer.py | 135 ++++++++++++------ tests/mapstream/test_async_map_stream.py | 76 ++++++---- 2 files changed, 146 insertions(+), 65 deletions(-) diff --git a/pynumaflow/mapstreamer/servicer/async_servicer.py b/pynumaflow/mapstreamer/servicer/async_servicer.py index 0fe58b66..f5a9a999 100644 --- a/pynumaflow/mapstreamer/servicer/async_servicer.py +++ b/pynumaflow/mapstreamer/servicer/async_servicer.py @@ -1,28 +1,27 @@ +import asyncio from collections.abc import AsyncIterable from google.protobuf import empty_pb2 as _empty_pb2 +from pynumaflow.shared.asynciter import NonBlockingIterator +from pynumaflow._constants import _LOGGER, STREAM_EOF, ERR_UDF_EXCEPTION_STRING from pynumaflow.mapstreamer import Datum from pynumaflow.mapstreamer._dtypes import MapStreamCallable, MapStreamError from pynumaflow.proto.mapper import map_pb2_grpc, map_pb2 from pynumaflow.shared.server import handle_async_error from pynumaflow.types import NumaflowServicerContext -from pynumaflow._constants import _LOGGER, ERR_UDF_EXCEPTION_STRING class AsyncMapStreamServicer(map_pb2_grpc.MapServicer): """ - This class is used to create a new grpc Map Stream Servicer instance. - It implements the SyncMapServicer interface from the proto - map_pb2_grpc.py file. - Provides the functionality for the required rpc methods. + Concurrent gRPC Map Stream Servicer. + Spawns one background task per incoming MapRequest; each task streams + results as produced and finally emits an EOT for that request. """ - def __init__( - self, - handler: MapStreamCallable, - ): + def __init__(self, handler: MapStreamCallable): self.__map_stream_handler: MapStreamCallable = handler + self._background_tasks: set[asyncio.Task] = set() async def MapFn( self, @@ -31,51 +30,105 @@ async def MapFn( ) -> AsyncIterable[map_pb2.MapResponse]: """ Applies a map function to a datum stream in streaming mode. - The pascal case function name comes from the proto map_pb2_grpc.py file. + The PascalCase name comes from the generated map_pb2_grpc.py file. """ try: - # The first message to be received should be a valid handshake - req = await request_iterator.__anext__() - # check if it is a valid handshake req - if not (req.handshake and req.handshake.sot): + # First message must be a handshake + first = await request_iterator.__anext__() + if not (first.handshake and first.handshake.sot): raise MapStreamError("MapStreamFn: expected handshake as the first message") + # Acknowledge handshake yield map_pb2.MapResponse(handshake=map_pb2.Handshake(sot=True)) - # read for each input request - async for req in request_iterator: - # yield messages as received from the UDF - async for res in self.__invoke_map_stream( - list(req.request.keys), - Datum( - keys=list(req.request.keys), - value=req.request.value, - event_time=req.request.event_time.ToDatetime(), - watermark=req.request.watermark.ToDatetime(), - headers=dict(req.request.headers), - ), - ): - yield map_pb2.MapResponse(results=[res], id=req.id) - # send EOT to indicate end of transmission for a given message - yield map_pb2.MapResponse(status=map_pb2.TransmissionStatus(eot=True), id=req.id) - except BaseException as err: + # Global non-blocking queue for outbound responses / errors + global_result_queue = NonBlockingIterator() + + # Start producer that turns each inbound request into a background task + producer = asyncio.create_task( + self._process_inputs(request_iterator, global_result_queue) + ) + + # Consume results as they arrive and stream them to the client + async for msg in global_result_queue.read_iterator(): + if isinstance(msg, BaseException): + await handle_async_error(context, msg, ERR_UDF_EXCEPTION_STRING) + return + else: + # msg is a map_pb2.MapResponse, already formed + yield msg + + # Ensure producer has finished (covers graceful shutdown) + await producer + + except BaseException as e: _LOGGER.critical("UDFError, re-raising the error", exc_info=True) - await handle_async_error(context, err, ERR_UDF_EXCEPTION_STRING) + await handle_async_error(context, e, ERR_UDF_EXCEPTION_STRING) return - async def __invoke_map_stream(self, keys: list[str], req: Datum): + async def _process_inputs( + self, + request_iterator: AsyncIterable[map_pb2.MapRequest], + result_queue: NonBlockingIterator, + ) -> None: + """ + Reads MapRequests from the client and spawns a background task per request. + Each task streams results to result_queue as they are produced. + """ try: - # Invoke the user handler for map stream - async for msg in self.__map_stream_handler(keys, req): - yield map_pb2.MapResponse.Result(keys=msg.keys, value=msg.value, tags=msg.tags) + async for req in request_iterator: + task = asyncio.create_task(self._invoke_map_stream(req, result_queue)) + self._background_tasks.add(task) + # Remove from the set when done to avoid memory growth + task.add_done_callback(self._background_tasks.discard) + + # Wait for all in-flight tasks to complete + if self._background_tasks: + await asyncio.gather(*list(self._background_tasks), return_exceptions=False) + + # Signal end-of-stream to the consumer + await result_queue.put(STREAM_EOF) + + except BaseException as e: + _LOGGER.critical("MapFn Error, re-raising the error", exc_info=True) + # Surface the error to the consumer; MapFn will handle and close the RPC + await result_queue.put(e) + + async def _invoke_map_stream( + self, + req: map_pb2.MapRequest, + result_queue: NonBlockingIterator, + ) -> None: + """ + Invokes the user-provided async generator for a single request and + pushes each result onto the global queue, followed by an EOT for this id. + """ + try: + datum = Datum( + keys=list(req.request.keys), + value=req.request.value, + event_time=req.request.event_time.ToDatetime(), + watermark=req.request.watermark.ToDatetime(), + headers=dict(req.request.headers), + ) + + # Stream results from the user handler as they are produced + async for msg in self.__map_stream_handler(list(req.request.keys), datum): + res = map_pb2.MapResponse.Result(keys=msg.keys, value=msg.value, tags=msg.tags) + await result_queue.put(map_pb2.MapResponse(results=[res], id=req.id)) + + # Emit EOT for this request id + await result_queue.put( + map_pb2.MapResponse(status=map_pb2.TransmissionStatus(eot=True), id=req.id) + ) + except BaseException as err: _LOGGER.critical("MapFn handler error", exc_info=True) - raise err + # Surface handler error to the main producer; + # it will call handle_async_error and end the RPC + await result_queue.put(err) async def IsReady( self, request: _empty_pb2.Empty, context: NumaflowServicerContext ) -> map_pb2.ReadyResponse: - """ - IsReady is the heartbeat endpoint for gRPC. - The pascal case function name comes from the proto map_pb2_grpc.py file. - """ + """Heartbeat endpoint for gRPC.""" return map_pb2.ReadyResponse(ready=True) diff --git a/tests/mapstream/test_async_map_stream.py b/tests/mapstream/test_async_map_stream.py index a4b36941..0beae35e 100644 --- a/tests/mapstream/test_async_map_stream.py +++ b/tests/mapstream/test_async_map_stream.py @@ -95,39 +95,67 @@ def tearDownClass(cls) -> None: def test_map_stream(self) -> None: stub = self.__stub() - generator_response = None + + # Send >1 requests + req_count = 3 try: - generator_response = stub.MapFn(request_iterator=request_generator(count=1, session=1)) + generator_response = stub.MapFn( + request_iterator=request_generator(count=req_count, session=1) + ) except grpc.RpcError as e: logging.error(e) + self.fail(f"RPC failed: {e}") + # First message must be the handshake handshake = next(generator_response) - # assert that handshake response is received. self.assertTrue(handshake.handshake.sot) - data_resp = [] - for r in generator_response: - data_resp.append(r) - - self.assertEqual(11, len(data_resp)) - idx = 0 - while idx < len(data_resp) - 1: + # Expected: 10 results per request + 1 EOT per request + expected_result_msgs = req_count * 10 + expected_eots = req_count + + # Prepare expected payload + expected_payload = bytes( + "payload:test_mock_message " + "event_time:2022-09-12 16:00:00 watermark:2022-09-12 16:01:00", + encoding="utf-8", + ) + + from collections import Counter + + id_counter = Counter() + result_msg_count = 0 + eot_count = 0 + + for msg in generator_response: + # Count EOTs wherever they show up + if hasattr(msg, "status") and msg.status.eot: + eot_count += 1 + continue + + # Otherwise, it's a data/result message; validate payload and tally by id + self.assertTrue(msg.results, "Expected results in MapResponse.") + self.assertEqual(expected_payload, msg.results[0].value) + id_counter[msg.id] += 1 + result_msg_count += 1 + + # Validate totals + self.assertEqual( + expected_result_msgs, + result_msg_count, + f"Expected {expected_result_msgs} result messages, got {result_msg_count}", + ) + self.assertEqual( + expected_eots, eot_count, f"Expected {expected_eots} EOT messages, got {eot_count}" + ) + + # Validate 10 messages per request id: test-id-0..test-id-(req_count-1) + for i in range(req_count): self.assertEqual( - bytes( - "payload:test_mock_message " - "event_time:2022-09-12 16:00:00 watermark:2022-09-12 16:01:00", - encoding="utf-8", - ), - data_resp[idx].results[0].value, + 10, + id_counter[f"test-id-{i}"], + f"Expected 10 results for test-id-{i}, got {id_counter[f'test-id-{i}']}", ) - _id = data_resp[idx].id - self.assertEqual(_id, "test-id-0") - # capture the output from the SinkFn generator and assert. - idx += 1 - # EOT Response - self.assertEqual(data_resp[len(data_resp) - 1].status.eot, True) - # 10 sink responses + 1 EOT response - self.assertEqual(11, len(data_resp)) def test_is_ready(self) -> None: with grpc.insecure_channel("unix:///tmp/async_map_stream.sock") as channel: From 123e2f802cab33302eeaa2bf7166352491b008a9 Mon Sep 17 00:00:00 2001 From: Sidhant Kohli Date: Tue, 16 Sep 2025 13:49:03 -0700 Subject: [PATCH 09/11] chore: release v0.10.1 (#246) Signed-off-by: kohlisid --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ed7ddfc2..e6dae906 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pynumaflow" -version = "0.10.0" +version = "0.10.1" description = "Provides the interfaces of writing Python User Defined Functions and Sinks for NumaFlow." authors = ["NumaFlow Developers"] readme = "README.md" From 59a58e0cbd64f1c225e0e82ccc881fb972ae22d5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 16 Sep 2025 16:52:27 -0700 Subject: [PATCH 10/11] docs: updated CHANGELOG.md (#247) Signed-off-by: GitHub Co-authored-by: kohlisid --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1af24e6a..2e0b6805 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## v0.10.1 (2025-09-16) + + * [bf00b2e](https://github.com/numaproj/numaflow-python/commit/bf00b2e44dce6edfdfaf78466b9e636eebe961e9) feat: parallelized mapstreamer (#242) + * [b944a5d](https://github.com/numaproj/numaflow-python/commit/b944a5d5883844c0c7bde0714fc85d60aa0c138a) doc: fix async reducer README examples (#244) + * [4982414](https://github.com/numaproj/numaflow-python/commit/49824144e6b1586f612bad601e47aeb2166963ca) feat: add Accumulator (#237) + +### Contributors + + * Martin Thøgersen + * Sidhant Kohli + * shrivardhan + ## v0.10.0a0 (2025-06-25) From ac2b51353755608a6aa326de69f9d4be6d71b670 Mon Sep 17 00:00:00 2001 From: Kevin Neal Date: Wed, 17 Sep 2025 10:50:00 -0700 Subject: [PATCH 11/11] chore: support Python 3.13 (#245) Signed-off-by: Kevin Neal --- .github/workflows/run-tests.yml | 2 +- poetry.lock | 393 +++++++++++----------- pyproject.toml | 9 +- tests/map/test_multiproc_mapper.py | 5 +- tests/map/test_sync_mapper.py | 5 +- tests/sourcetransform/test_multiproc.py | 5 +- tests/sourcetransform/test_sync_server.py | 5 +- 7 files changed, 219 insertions(+), 205 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index e03fa14b..0387ba79 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v3 diff --git a/poetry.lock b/poetry.lock index a9ad8fb1..c0c992eb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -353,28 +353,31 @@ typing = ["typing-extensions (>=4.12.2) ; python_version < \"3.11\""] [[package]] name = "google-api-core" -version = "2.24.1" +version = "2.25.1" description = "Google API client core library" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "google_api_core-2.24.1-py3-none-any.whl", hash = "sha256:bc78d608f5a5bf853b80bd70a795f703294de656c096c0968320830a4bc280f1"}, - {file = "google_api_core-2.24.1.tar.gz", hash = "sha256:f8b36f5456ab0dd99a1b693a40a31d1e7757beea380ad1b38faaf8941eae9d8a"}, + {file = "google_api_core-2.25.1-py3-none-any.whl", hash = "sha256:8a2a56c1fef82987a524371f99f3bd0143702fecc670c72e600c1cda6bf8dbb7"}, + {file = "google_api_core-2.25.1.tar.gz", hash = "sha256:d2aaa0b13c78c61cb3f4282c464c046e45fbd75755683c9c525e6e8f7ed0a5e8"}, ] [package.dependencies] -google-auth = ">=2.14.1,<3.0.dev0" -googleapis-common-protos = ">=1.56.2,<2.0.dev0" -proto-plus = ">=1.22.3,<2.0.0dev" -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0" -requests = ">=2.18.0,<3.0.0.dev0" +google-auth = ">=2.14.1,<3.0.0" +googleapis-common-protos = ">=1.56.2,<2.0.0" +proto-plus = [ + {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, + {version = ">=1.22.3,<2.0.0", markers = "python_version < \"3.13\""}, +] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" +requests = ">=2.18.0,<3.0.0" [package.extras] -async-rest = ["google-auth[aiohttp] (>=2.35.0,<3.0.dev0)"] -grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev) ; python_version >= \"3.11\"", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0) ; python_version >= \"3.11\""] -grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] -grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +async-rest = ["google-auth[aiohttp] (>=2.35.0,<3.0.0)"] +grpc = ["grpcio (>=1.33.2,<2.0.0)", "grpcio (>=1.49.1,<2.0.0) ; python_version >= \"3.11\"", "grpcio-status (>=1.33.2,<2.0.0)", "grpcio-status (>=1.49.1,<2.0.0) ; python_version >= \"3.11\""] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] [[package]] name = "google-auth" @@ -415,184 +418,186 @@ files = [ [[package]] name = "googleapis-common-protos" -version = "1.66.0" +version = "1.70.0" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "googleapis_common_protos-1.66.0-py2.py3-none-any.whl", hash = "sha256:d7abcd75fabb2e0ec9f74466401f6c119a0b498e27370e9be4c94cb7e382b8ed"}, - {file = "googleapis_common_protos-1.66.0.tar.gz", hash = "sha256:c3e7b33d15fdca5374cc0a7346dd92ffa847425cc4ea941d970f13680052ec8c"}, + {file = "googleapis_common_protos-1.70.0-py3-none-any.whl", hash = "sha256:b8bfcca8c25a2bb253e0e0b0adaf8c00773e5e6af6fd92397576680b807e0fd8"}, + {file = "googleapis_common_protos-1.70.0.tar.gz", hash = "sha256:0e1b44e0ea153e6594f9f394fef15193a68aaaea2d843f83e2742717ca753257"}, ] [package.dependencies] -protobuf = ">=3.20.2,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0" +protobuf = ">=3.20.2,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" [package.extras] -grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] +grpc = ["grpcio (>=1.44.0,<2.0.0)"] [[package]] name = "grpcio" -version = "1.70.0" +version = "1.75.0" description = "HTTP/2-based RPC framework" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "grpcio-1.70.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:95469d1977429f45fe7df441f586521361e235982a0b39e33841549143ae2851"}, - {file = "grpcio-1.70.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:ed9718f17fbdb472e33b869c77a16d0b55e166b100ec57b016dc7de9c8d236bf"}, - {file = "grpcio-1.70.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:374d014f29f9dfdb40510b041792e0e2828a1389281eb590df066e1cc2b404e5"}, - {file = "grpcio-1.70.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2af68a6f5c8f78d56c145161544ad0febbd7479524a59c16b3e25053f39c87f"}, - {file = "grpcio-1.70.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce7df14b2dcd1102a2ec32f621cc9fab6695effef516efbc6b063ad749867295"}, - {file = "grpcio-1.70.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c78b339869f4dbf89881e0b6fbf376313e4f845a42840a7bdf42ee6caed4b11f"}, - {file = "grpcio-1.70.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:58ad9ba575b39edef71f4798fdb5c7b6d02ad36d47949cd381d4392a5c9cbcd3"}, - {file = "grpcio-1.70.0-cp310-cp310-win32.whl", hash = "sha256:2b0d02e4b25a5c1f9b6c7745d4fa06efc9fd6a611af0fb38d3ba956786b95199"}, - {file = "grpcio-1.70.0-cp310-cp310-win_amd64.whl", hash = "sha256:0de706c0a5bb9d841e353f6343a9defc9fc35ec61d6eb6111802f3aa9fef29e1"}, - {file = "grpcio-1.70.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:17325b0be0c068f35770f944124e8839ea3185d6d54862800fc28cc2ffad205a"}, - {file = "grpcio-1.70.0-cp311-cp311-macosx_10_14_universal2.whl", hash = "sha256:dbe41ad140df911e796d4463168e33ef80a24f5d21ef4d1e310553fcd2c4a386"}, - {file = "grpcio-1.70.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:5ea67c72101d687d44d9c56068328da39c9ccba634cabb336075fae2eab0d04b"}, - {file = "grpcio-1.70.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb5277db254ab7586769e490b7b22f4ddab3876c490da0a1a9d7c695ccf0bf77"}, - {file = "grpcio-1.70.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7831a0fc1beeeb7759f737f5acd9fdcda520e955049512d68fda03d91186eea"}, - {file = "grpcio-1.70.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:27cc75e22c5dba1fbaf5a66c778e36ca9b8ce850bf58a9db887754593080d839"}, - {file = "grpcio-1.70.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d63764963412e22f0491d0d32833d71087288f4e24cbcddbae82476bfa1d81fd"}, - {file = "grpcio-1.70.0-cp311-cp311-win32.whl", hash = "sha256:bb491125103c800ec209d84c9b51f1c60ea456038e4734688004f377cfacc113"}, - {file = "grpcio-1.70.0-cp311-cp311-win_amd64.whl", hash = "sha256:d24035d49e026353eb042bf7b058fb831db3e06d52bee75c5f2f3ab453e71aca"}, - {file = "grpcio-1.70.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:ef4c14508299b1406c32bdbb9fb7b47612ab979b04cf2b27686ea31882387cff"}, - {file = "grpcio-1.70.0-cp312-cp312-macosx_10_14_universal2.whl", hash = "sha256:aa47688a65643afd8b166928a1da6247d3f46a2784d301e48ca1cc394d2ffb40"}, - {file = "grpcio-1.70.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:880bfb43b1bb8905701b926274eafce5c70a105bc6b99e25f62e98ad59cb278e"}, - {file = "grpcio-1.70.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e654c4b17d07eab259d392e12b149c3a134ec52b11ecdc6a515b39aceeec898"}, - {file = "grpcio-1.70.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2394e3381071045a706ee2eeb6e08962dd87e8999b90ac15c55f56fa5a8c9597"}, - {file = "grpcio-1.70.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:b3c76701428d2df01964bc6479422f20e62fcbc0a37d82ebd58050b86926ef8c"}, - {file = "grpcio-1.70.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ac073fe1c4cd856ebcf49e9ed6240f4f84d7a4e6ee95baa5d66ea05d3dd0df7f"}, - {file = "grpcio-1.70.0-cp312-cp312-win32.whl", hash = "sha256:cd24d2d9d380fbbee7a5ac86afe9787813f285e684b0271599f95a51bce33528"}, - {file = "grpcio-1.70.0-cp312-cp312-win_amd64.whl", hash = "sha256:0495c86a55a04a874c7627fd33e5beaee771917d92c0e6d9d797628ac40e7655"}, - {file = "grpcio-1.70.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:aa573896aeb7d7ce10b1fa425ba263e8dddd83d71530d1322fd3a16f31257b4a"}, - {file = "grpcio-1.70.0-cp313-cp313-macosx_10_14_universal2.whl", hash = "sha256:d405b005018fd516c9ac529f4b4122342f60ec1cee181788249372524e6db429"}, - {file = "grpcio-1.70.0-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:f32090238b720eb585248654db8e3afc87b48d26ac423c8dde8334a232ff53c9"}, - {file = "grpcio-1.70.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dfa089a734f24ee5f6880c83d043e4f46bf812fcea5181dcb3a572db1e79e01c"}, - {file = "grpcio-1.70.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f19375f0300b96c0117aca118d400e76fede6db6e91f3c34b7b035822e06c35f"}, - {file = "grpcio-1.70.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:7c73c42102e4a5ec76608d9b60227d917cea46dff4d11d372f64cbeb56d259d0"}, - {file = "grpcio-1.70.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:0a5c78d5198a1f0aa60006cd6eb1c912b4a1520b6a3968e677dbcba215fabb40"}, - {file = "grpcio-1.70.0-cp313-cp313-win32.whl", hash = "sha256:fe9dbd916df3b60e865258a8c72ac98f3ac9e2a9542dcb72b7a34d236242a5ce"}, - {file = "grpcio-1.70.0-cp313-cp313-win_amd64.whl", hash = "sha256:4119fed8abb7ff6c32e3d2255301e59c316c22d31ab812b3fbcbaf3d0d87cc68"}, - {file = "grpcio-1.70.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:8058667a755f97407fca257c844018b80004ae8035565ebc2812cc550110718d"}, - {file = "grpcio-1.70.0-cp38-cp38-macosx_10_14_universal2.whl", hash = "sha256:879a61bf52ff8ccacbedf534665bb5478ec8e86ad483e76fe4f729aaef867cab"}, - {file = "grpcio-1.70.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:0ba0a173f4feacf90ee618fbc1a27956bfd21260cd31ced9bc707ef551ff7dc7"}, - {file = "grpcio-1.70.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:558c386ecb0148f4f99b1a65160f9d4b790ed3163e8610d11db47838d452512d"}, - {file = "grpcio-1.70.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:412faabcc787bbc826f51be261ae5fa996b21263de5368a55dc2cf824dc5090e"}, - {file = "grpcio-1.70.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3b0f01f6ed9994d7a0b27eeddea43ceac1b7e6f3f9d86aeec0f0064b8cf50fdb"}, - {file = "grpcio-1.70.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7385b1cb064734005204bc8994eed7dcb801ed6c2eda283f613ad8c6c75cf873"}, - {file = "grpcio-1.70.0-cp38-cp38-win32.whl", hash = "sha256:07269ff4940f6fb6710951116a04cd70284da86d0a4368fd5a3b552744511f5a"}, - {file = "grpcio-1.70.0-cp38-cp38-win_amd64.whl", hash = "sha256:aba19419aef9b254e15011b230a180e26e0f6864c90406fdbc255f01d83bc83c"}, - {file = "grpcio-1.70.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:4f1937f47c77392ccd555728f564a49128b6a197a05a5cd527b796d36f3387d0"}, - {file = "grpcio-1.70.0-cp39-cp39-macosx_10_14_universal2.whl", hash = "sha256:0cd430b9215a15c10b0e7d78f51e8a39d6cf2ea819fd635a7214fae600b1da27"}, - {file = "grpcio-1.70.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:e27585831aa6b57b9250abaf147003e126cd3a6c6ca0c531a01996f31709bed1"}, - {file = "grpcio-1.70.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c1af8e15b0f0fe0eac75195992a63df17579553b0c4af9f8362cc7cc99ccddf4"}, - {file = "grpcio-1.70.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cbce24409beaee911c574a3d75d12ffb8c3e3dd1b813321b1d7a96bbcac46bf4"}, - {file = "grpcio-1.70.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ff4a8112a79464919bb21c18e956c54add43ec9a4850e3949da54f61c241a4a6"}, - {file = "grpcio-1.70.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5413549fdf0b14046c545e19cfc4eb1e37e9e1ebba0ca390a8d4e9963cab44d2"}, - {file = "grpcio-1.70.0-cp39-cp39-win32.whl", hash = "sha256:b745d2c41b27650095e81dea7091668c040457483c9bdb5d0d9de8f8eb25e59f"}, - {file = "grpcio-1.70.0-cp39-cp39-win_amd64.whl", hash = "sha256:a31d7e3b529c94e930a117b2175b2efd179d96eb3c7a21ccb0289a8ab05b645c"}, - {file = "grpcio-1.70.0.tar.gz", hash = "sha256:8d1584a68d5922330025881e63a6c1b54cc8117291d382e4fa69339b6d914c56"}, + {file = "grpcio-1.75.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:1ec9cbaec18d9597c718b1ed452e61748ac0b36ba350d558f9ded1a94cc15ec7"}, + {file = "grpcio-1.75.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:7ee5ee42bfae8238b66a275f9ebcf6f295724375f2fa6f3b52188008b6380faf"}, + {file = "grpcio-1.75.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9146e40378f551eed66c887332afc807fcce593c43c698e21266a4227d4e20d2"}, + {file = "grpcio-1.75.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:0c40f368541945bb664857ecd7400acb901053a1abbcf9f7896361b2cfa66798"}, + {file = "grpcio-1.75.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:50a6e43a9adc6938e2a16c9d9f8a2da9dd557ddd9284b73b07bd03d0e098d1e9"}, + {file = "grpcio-1.75.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dce15597ca11913b78e1203c042d5723e3ea7f59e7095a1abd0621be0e05b895"}, + {file = "grpcio-1.75.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:851194eec47755101962da423f575ea223c9dd7f487828fe5693920e8745227e"}, + {file = "grpcio-1.75.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ca123db0813eef80625a4242a0c37563cb30a3edddebe5ee65373854cf187215"}, + {file = "grpcio-1.75.0-cp310-cp310-win32.whl", hash = "sha256:222b0851e20c04900c63f60153503e918b08a5a0fad8198401c0b1be13c6815b"}, + {file = "grpcio-1.75.0-cp310-cp310-win_amd64.whl", hash = "sha256:bb58e38a50baed9b21492c4b3f3263462e4e37270b7ea152fc10124b4bd1c318"}, + {file = "grpcio-1.75.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:7f89d6d0cd43170a80ebb4605cad54c7d462d21dc054f47688912e8bf08164af"}, + {file = "grpcio-1.75.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:cb6c5b075c2d092f81138646a755f0dad94e4622300ebef089f94e6308155d82"}, + {file = "grpcio-1.75.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:494dcbade5606128cb9f530ce00331a90ecf5e7c5b243d373aebdb18e503c346"}, + {file = "grpcio-1.75.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:050760fd29c8508844a720f06c5827bb00de8f5e02f58587eb21a4444ad706e5"}, + {file = "grpcio-1.75.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:266fa6209b68a537b2728bb2552f970e7e78c77fe43c6e9cbbe1f476e9e5c35f"}, + {file = "grpcio-1.75.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:06d22e1d8645e37bc110f4c589cb22c283fd3de76523065f821d6e81de33f5d4"}, + {file = "grpcio-1.75.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:9880c323595d851292785966cadb6c708100b34b163cab114e3933f5773cba2d"}, + {file = "grpcio-1.75.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:55a2d5ae79cd0f68783fb6ec95509be23746e3c239290b2ee69c69a38daa961a"}, + {file = "grpcio-1.75.0-cp311-cp311-win32.whl", hash = "sha256:352dbdf25495eef584c8de809db280582093bc3961d95a9d78f0dfb7274023a2"}, + {file = "grpcio-1.75.0-cp311-cp311-win_amd64.whl", hash = "sha256:678b649171f229fb16bda1a2473e820330aa3002500c4f9fd3a74b786578e90f"}, + {file = "grpcio-1.75.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:fa35ccd9501ffdd82b861809cbfc4b5b13f4b4c5dc3434d2d9170b9ed38a9054"}, + {file = "grpcio-1.75.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:0fcb77f2d718c1e58cc04ef6d3b51e0fa3b26cf926446e86c7eba105727b6cd4"}, + {file = "grpcio-1.75.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:36764a4ad9dc1eb891042fab51e8cdf7cc014ad82cee807c10796fb708455041"}, + {file = "grpcio-1.75.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:725e67c010f63ef17fc052b261004942763c0b18dcd84841e6578ddacf1f9d10"}, + {file = "grpcio-1.75.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:91fbfc43f605c5ee015c9056d580a70dd35df78a7bad97e05426795ceacdb59f"}, + {file = "grpcio-1.75.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7a9337ac4ce61c388e02019d27fa837496c4b7837cbbcec71b05934337e51531"}, + {file = "grpcio-1.75.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:ee16e232e3d0974750ab5f4da0ab92b59d6473872690b5e40dcec9a22927f22e"}, + {file = "grpcio-1.75.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:55dfb9122973cc69520b23d39867726722cafb32e541435707dc10249a1bdbc6"}, + {file = "grpcio-1.75.0-cp312-cp312-win32.whl", hash = "sha256:fb64dd62face3d687a7b56cd881e2ea39417af80f75e8b36f0f81dfd93071651"}, + {file = "grpcio-1.75.0-cp312-cp312-win_amd64.whl", hash = "sha256:6b365f37a9c9543a9e91c6b4103d68d38d5bcb9965b11d5092b3c157bd6a5ee7"}, + {file = "grpcio-1.75.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:1bb78d052948d8272c820bb928753f16a614bb2c42fbf56ad56636991b427518"}, + {file = "grpcio-1.75.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:9dc4a02796394dd04de0b9673cb79a78901b90bb16bf99ed8cb528c61ed9372e"}, + {file = "grpcio-1.75.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:437eeb16091d31498585d73b133b825dc80a8db43311e332c08facf820d36894"}, + {file = "grpcio-1.75.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:c2c39984e846bd5da45c5f7bcea8fafbe47c98e1ff2b6f40e57921b0c23a52d0"}, + {file = "grpcio-1.75.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:38d665f44b980acdbb2f0e1abf67605ba1899f4d2443908df9ec8a6f26d2ed88"}, + {file = "grpcio-1.75.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2e8e752ab5cc0a9c5b949808c000ca7586223be4f877b729f034b912364c3964"}, + {file = "grpcio-1.75.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:3a6788b30aa8e6f207c417874effe3f79c2aa154e91e78e477c4825e8b431ce0"}, + {file = "grpcio-1.75.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffc33e67cab6141c54e75d85acd5dec616c5095a957ff997b4330a6395aa9b51"}, + {file = "grpcio-1.75.0-cp313-cp313-win32.whl", hash = "sha256:c8cfc780b7a15e06253aae5f228e1e84c0d3c4daa90faf5bc26b751174da4bf9"}, + {file = "grpcio-1.75.0-cp313-cp313-win_amd64.whl", hash = "sha256:0c91d5b16eff3cbbe76b7a1eaaf3d91e7a954501e9d4f915554f87c470475c3d"}, + {file = "grpcio-1.75.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:0b85f4ebe6b56d2a512201bb0e5f192c273850d349b0a74ac889ab5d38959d16"}, + {file = "grpcio-1.75.0-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:68c95b1c1e3bf96ceadf98226e9dfe2bc92155ce352fa0ee32a1603040e61856"}, + {file = "grpcio-1.75.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:153c5a7655022c3626ad70be3d4c2974cb0967f3670ee49ece8b45b7a139665f"}, + {file = "grpcio-1.75.0-cp39-cp39-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:53067c590ac3638ad0c04272f2a5e7e32a99fec8824c31b73bc3ef93160511fa"}, + {file = "grpcio-1.75.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:78dcc025a144319b66df6d088bd0eda69e1719eb6ac6127884a36188f336df19"}, + {file = "grpcio-1.75.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1ec2937fd92b5b4598cbe65f7e57d66039f82b9e2b7f7a5f9149374057dde77d"}, + {file = "grpcio-1.75.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:597340a41ad4b619aaa5c9b94f7e6ba4067885386342ab0af039eda945c255cd"}, + {file = "grpcio-1.75.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:0aa795198b28807d28570c0a5f07bb04d5facca7d3f27affa6ae247bbd7f312a"}, + {file = "grpcio-1.75.0-cp39-cp39-win32.whl", hash = "sha256:585147859ff4603798e92605db28f4a97c821c69908e7754c44771c27b239bbd"}, + {file = "grpcio-1.75.0-cp39-cp39-win_amd64.whl", hash = "sha256:eafbe3563f9cb378370a3fa87ef4870539cf158124721f3abee9f11cd8162460"}, + {file = "grpcio-1.75.0.tar.gz", hash = "sha256:b989e8b09489478c2d19fecc744a298930f40d8b27c3638afbfe84d22f36ce4e"}, ] +[package.dependencies] +typing-extensions = ">=4.12,<5.0" + [package.extras] -protobuf = ["grpcio-tools (>=1.70.0)"] +protobuf = ["grpcio-tools (>=1.75.0)"] [[package]] name = "grpcio-status" -version = "1.62.3" +version = "1.75.0" description = "Status proto mapping for gRPC" optional = false -python-versions = ">=3.6" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "grpcio-status-1.62.3.tar.gz", hash = "sha256:289bdd7b2459794a12cf95dc0cb727bd4a1742c37bd823f760236c937e53a485"}, - {file = "grpcio_status-1.62.3-py3-none-any.whl", hash = "sha256:f9049b762ba8de6b1086789d8315846e094edac2c50beaf462338b301a8fd4b8"}, + {file = "grpcio_status-1.75.0-py3-none-any.whl", hash = "sha256:de62557ef97b7e19c3ce6da19793a12c5f6c1fbbb918d233d9671aba9d9e1d78"}, + {file = "grpcio_status-1.75.0.tar.gz", hash = "sha256:69d5b91be1b8b926f086c1c483519a968c14640773a0ccdd6c04282515dbedf7"}, ] [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.62.3" -protobuf = ">=4.21.6" +grpcio = ">=1.75.0" +protobuf = ">=6.31.1,<7.0.0" [[package]] name = "grpcio-testing" -version = "1.62.3" +version = "1.75.0" description = "Testing utilities for gRPC Python" optional = false python-versions = "*" groups = ["dev"] files = [ - {file = "grpcio-testing-1.62.3.tar.gz", hash = "sha256:f63577f28aaa95ea525124a0fd63c3429d71f769f4179b13f5e6cbc54979bfab"}, - {file = "grpcio_testing-1.62.3-py3-none-any.whl", hash = "sha256:06a4d7eb30d22f91368aa7f48bfc33563da13b9d951314455ca8c9c987fb75bb"}, + {file = "grpcio_testing-1.75.0-py3-none-any.whl", hash = "sha256:ffe05ee52b0e6ca4e78b3e73d4e67c994346725d65301f4ebf718985c8b75422"}, + {file = "grpcio_testing-1.75.0.tar.gz", hash = "sha256:52e0c4f602faad000ae1afcf9d954377277eb7c9614cc5c9f82b290076c405bb"}, ] [package.dependencies] -grpcio = ">=1.62.3" -protobuf = ">=4.21.6" +grpcio = ">=1.75.0" +protobuf = ">=6.31.1,<7.0.0" [[package]] name = "grpcio-tools" -version = "1.62.3" +version = "1.75.0" description = "Protobuf code generator for gRPC" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "grpcio-tools-1.62.3.tar.gz", hash = "sha256:7c7136015c3d62c3eef493efabaf9e3380e3e66d24ee8e94c01cb71377f57833"}, - {file = "grpcio_tools-1.62.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:2f968b049c2849540751ec2100ab05e8086c24bead769ca734fdab58698408c1"}, - {file = "grpcio_tools-1.62.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:0a8c0c4724ae9c2181b7dbc9b186df46e4f62cb18dc184e46d06c0ebeccf569e"}, - {file = "grpcio_tools-1.62.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5782883a27d3fae8c425b29a9d3dcf5f47d992848a1b76970da3b5a28d424b26"}, - {file = "grpcio_tools-1.62.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3d812daffd0c2d2794756bd45a353f89e55dc8f91eb2fc840c51b9f6be62667"}, - {file = "grpcio_tools-1.62.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b47d0dda1bdb0a0ba7a9a6de88e5a1ed61f07fad613964879954961e36d49193"}, - {file = "grpcio_tools-1.62.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ca246dffeca0498be9b4e1ee169b62e64694b0f92e6d0be2573e65522f39eea9"}, - {file = "grpcio_tools-1.62.3-cp310-cp310-win32.whl", hash = "sha256:6a56d344b0bab30bf342a67e33d386b0b3c4e65868ffe93c341c51e1a8853ca5"}, - {file = "grpcio_tools-1.62.3-cp310-cp310-win_amd64.whl", hash = "sha256:710fecf6a171dcbfa263a0a3e7070e0df65ba73158d4c539cec50978f11dad5d"}, - {file = "grpcio_tools-1.62.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:703f46e0012af83a36082b5f30341113474ed0d91e36640da713355cd0ea5d23"}, - {file = "grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:7cc83023acd8bc72cf74c2edbe85b52098501d5b74d8377bfa06f3e929803492"}, - {file = "grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ff7d58a45b75df67d25f8f144936a3e44aabd91afec833ee06826bd02b7fbe7"}, - {file = "grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f2483ea232bd72d98a6dc6d7aefd97e5bc80b15cd909b9e356d6f3e326b6e43"}, - {file = "grpcio_tools-1.62.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:962c84b4da0f3b14b3cdb10bc3837ebc5f136b67d919aea8d7bb3fd3df39528a"}, - {file = "grpcio_tools-1.62.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8ad0473af5544f89fc5a1ece8676dd03bdf160fb3230f967e05d0f4bf89620e3"}, - {file = "grpcio_tools-1.62.3-cp311-cp311-win32.whl", hash = "sha256:db3bc9fa39afc5e4e2767da4459df82b095ef0cab2f257707be06c44a1c2c3e5"}, - {file = "grpcio_tools-1.62.3-cp311-cp311-win_amd64.whl", hash = "sha256:e0898d412a434e768a0c7e365acabe13ff1558b767e400936e26b5b6ed1ee51f"}, - {file = "grpcio_tools-1.62.3-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:d102b9b21c4e1e40af9a2ab3c6d41afba6bd29c0aa50ca013bf85c99cdc44ac5"}, - {file = "grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:0a52cc9444df978438b8d2332c0ca99000521895229934a59f94f37ed896b133"}, - {file = "grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141d028bf5762d4a97f981c501da873589df3f7e02f4c1260e1921e565b376fa"}, - {file = "grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47a5c093ab256dec5714a7a345f8cc89315cb57c298b276fa244f37a0ba507f0"}, - {file = "grpcio_tools-1.62.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:f6831fdec2b853c9daa3358535c55eed3694325889aa714070528cf8f92d7d6d"}, - {file = "grpcio_tools-1.62.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e02d7c1a02e3814c94ba0cfe43d93e872c758bd8fd5c2797f894d0c49b4a1dfc"}, - {file = "grpcio_tools-1.62.3-cp312-cp312-win32.whl", hash = "sha256:b881fd9505a84457e9f7e99362eeedd86497b659030cf57c6f0070df6d9c2b9b"}, - {file = "grpcio_tools-1.62.3-cp312-cp312-win_amd64.whl", hash = "sha256:11c625eebefd1fd40a228fc8bae385e448c7e32a6ae134e43cf13bbc23f902b7"}, - {file = "grpcio_tools-1.62.3-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:ec6fbded0c61afe6f84e3c2a43e6d656791d95747d6d28b73eff1af64108c434"}, - {file = "grpcio_tools-1.62.3-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:bfda6ee8990997a9df95c5606f3096dae65f09af7ca03a1e9ca28f088caca5cf"}, - {file = "grpcio_tools-1.62.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b77f9f9cee87cd798f0fe26b7024344d1b03a7cd2d2cba7035f8433b13986325"}, - {file = "grpcio_tools-1.62.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e02d3b96f2d0e4bab9ceaa30f37d4f75571e40c6272e95364bff3125a64d184"}, - {file = "grpcio_tools-1.62.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1da38070738da53556a4b35ab67c1b9884a5dd48fa2f243db35dc14079ea3d0c"}, - {file = "grpcio_tools-1.62.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ace43b26d88a58dcff16c20d23ff72b04d0a415f64d2820f4ff06b1166f50557"}, - {file = "grpcio_tools-1.62.3-cp37-cp37m-win_amd64.whl", hash = "sha256:350a80485e302daaa95d335a931f97b693e170e02d43767ab06552c708808950"}, - {file = "grpcio_tools-1.62.3-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:c3a1ac9d394f8e229eb28eec2e04b9a6f5433fa19c9d32f1cb6066e3c5114a1d"}, - {file = "grpcio_tools-1.62.3-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:11f363570dea661dde99e04a51bd108a5807b5df32a6f8bdf4860e34e94a4dbf"}, - {file = "grpcio_tools-1.62.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9ad9950119d8ae27634e68b7663cc8d340ae535a0f80d85a55e56a6973ab1f"}, - {file = "grpcio_tools-1.62.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c5d22b252dcef11dd1e0fbbe5bbfb9b4ae048e8880d33338215e8ccbdb03edc"}, - {file = "grpcio_tools-1.62.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:27cd9ef5c5d68d5ed104b6dcb96fe9c66b82050e546c9e255716903c3d8f0373"}, - {file = "grpcio_tools-1.62.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f4b1615adf67bd8bb71f3464146a6f9949972d06d21a4f5e87e73f6464d97f57"}, - {file = "grpcio_tools-1.62.3-cp38-cp38-win32.whl", hash = "sha256:e18e15287c31baf574fcdf8251fb7f997d64e96c6ecf467906e576da0a079af6"}, - {file = "grpcio_tools-1.62.3-cp38-cp38-win_amd64.whl", hash = "sha256:6c3064610826f50bd69410c63101954676edc703e03f9e8f978a135f1aaf97c1"}, - {file = "grpcio_tools-1.62.3-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:8e62cc7164b0b7c5128e637e394eb2ef3db0e61fc798e80c301de3b2379203ed"}, - {file = "grpcio_tools-1.62.3-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:c8ad5cce554e2fcaf8842dee5d9462583b601a3a78f8b76a153c38c963f58c10"}, - {file = "grpcio_tools-1.62.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ec279dcf3518201fc592c65002754f58a6b542798cd7f3ecd4af086422f33f29"}, - {file = "grpcio_tools-1.62.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c989246c2aebc13253f08be32538a4039a64e12d9c18f6d662d7aee641dc8b5"}, - {file = "grpcio_tools-1.62.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ca4f5eeadbb57cf03317d6a2857823239a63a59cc935f5bd6cf6e8b7af7a7ecc"}, - {file = "grpcio_tools-1.62.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0cb3a3436ac119cbd37a7d3331d9bdf85dad21a6ac233a3411dff716dcbf401e"}, - {file = "grpcio_tools-1.62.3-cp39-cp39-win32.whl", hash = "sha256:3eae6ea76d62fcac091e1f15c2dcedf1dc3f114f8df1a972a8a0745e89f4cf61"}, - {file = "grpcio_tools-1.62.3-cp39-cp39-win_amd64.whl", hash = "sha256:eec73a005443061f4759b71a056f745e3b000dc0dc125c9f20560232dfbcbd14"}, + {file = "grpcio_tools-1.75.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:5ca29b0ae735044c6a48072cf7bf53e34ce9ab03eec66acaf2173071d4f66d8a"}, + {file = "grpcio_tools-1.75.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:d1a224887f70981683dfcaacc253c08f3680b919c0b2353fbb57f89b27e1c9b9"}, + {file = "grpcio_tools-1.75.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c49649d2b46a5a09419631adec105b05bcb016e5727c8f1b08ac8e16d9b0e3e0"}, + {file = "grpcio_tools-1.75.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:c944610bc009185f3da399030a2a8a9d550ae3246f93ad20ff63593fa883ddfb"}, + {file = "grpcio_tools-1.75.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:186c11fe9c8ef90b0862013b61876693644c952fda8fffef6ab0de0a83f90479"}, + {file = "grpcio_tools-1.75.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:688668666265a8f3e5eb86f73694e8adac2d2cc5f40c90249ce80bf6c6cec9ea"}, + {file = "grpcio_tools-1.75.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9083fe53cbe17b972d9ede47b1e6c82ec532a91770d41c790c4f9b39291041c3"}, + {file = "grpcio_tools-1.75.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3072b10f4ad82739650aa9d667b536de8d4973083236215b7bf2389ba75bb507"}, + {file = "grpcio_tools-1.75.0-cp310-cp310-win32.whl", hash = "sha256:c42fc86ab55018ba5afe2aa95d6d34e2e763da06eff23c08bed487a556341071"}, + {file = "grpcio_tools-1.75.0-cp310-cp310-win_amd64.whl", hash = "sha256:5e0c8d5d4bdce7f32e2fef3e2304cdca1fbb16a6469c7d3bce38884ee4c449d1"}, + {file = "grpcio_tools-1.75.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:6c3b8dbe8b2ad7df4ba661b5ee29ae8fe79d2715aade519847deaef26f5c1a06"}, + {file = "grpcio_tools-1.75.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:cdbccc5a4809ef9414b7c434dd1aabc94b66a01c01c13ecc1edba9f8f4277b44"}, + {file = "grpcio_tools-1.75.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:16a9597d1bd4143a71bfae341a32952a64c094a63d3d0bdd24b21fdc8b843846"}, + {file = "grpcio_tools-1.75.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:899c46520446ad1935f5899729746b390e13085e9757d043401298b18fa37d99"}, + {file = "grpcio_tools-1.75.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:53c116d0d5df70845330eefb98ef4242ff09be264a22bc5e18f171a3047c9e66"}, + {file = "grpcio_tools-1.75.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:495ce168f996d4c42328e17b788d51d808fc585a80612fe70943c00ac16d0fca"}, + {file = "grpcio_tools-1.75.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:26f1f3cedebe465f97b5aad312fb775a4bd53a0e88d08c4000e588c195519eca"}, + {file = "grpcio_tools-1.75.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:82692be482cdcf7ac9b79563dbea99333835aaa3f5e7f0641689766b64b91543"}, + {file = "grpcio_tools-1.75.0-cp311-cp311-win32.whl", hash = "sha256:fd038847974aeb883ee0f3b5b535d85618ad32789c15c9bf24af6c12a44f67f1"}, + {file = "grpcio_tools-1.75.0-cp311-cp311-win_amd64.whl", hash = "sha256:5c5465cd7b83c34f3c987a235fe3b04012411502d4bc66de5a34b238617ded4c"}, + {file = "grpcio_tools-1.75.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:6ded12c79fb56ceae0ce60e653453159bfc2ccb044922b7e7d721de6c8e04506"}, + {file = "grpcio_tools-1.75.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:ebdac7cc820459874f3b19eddddae19c0c7e7cdf228aee8e7567cec1fddb2ae3"}, + {file = "grpcio_tools-1.75.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:509ec0ce7c4269c2bea6015efcdcde00a5d55d97c88ad17587b4247cdc3d2fe8"}, + {file = "grpcio_tools-1.75.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:a68a8dcbcbd1df33e7c08c2ceeb69ed8fd53e235784ac680dfe3fc1e89aac2ac"}, + {file = "grpcio_tools-1.75.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3ac8a663e955bf3188f76d93d7fdc656f346ff54ea7e512eb034374c6fd61b50"}, + {file = "grpcio_tools-1.75.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3c30cb36ae1a4ed5fb1960f4bc0000548fecb9ff21a51d78a1f54e3424f971c0"}, + {file = "grpcio_tools-1.75.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:35d4368794506db2b0acde60e7e2bae21255cc0d05db9ffc078510ab6a84ff4f"}, + {file = "grpcio_tools-1.75.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:edefbb90bb7ddc4eadac3463d5f7084e1d43b1d713254f668dd55c25db5b5ef2"}, + {file = "grpcio_tools-1.75.0-cp312-cp312-win32.whl", hash = "sha256:c2bad23bd0d43acd9d7032b6ffb04f5eb176d853cd32967eb2c4a39044c81cfe"}, + {file = "grpcio_tools-1.75.0-cp312-cp312-win_amd64.whl", hash = "sha256:0f4f31035a5178acd924a052b8954d5ac71319092b57e3711438ca6518b71017"}, + {file = "grpcio_tools-1.75.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:69742254df93323275b7ee5ac017e3b9fdba8ecc6dca00bd6b2cd1c70c80a9c2"}, + {file = "grpcio_tools-1.75.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:a07aa71ad96103b18bb84dc069dd139897356116d2aaa68d3df84d4d59701ae8"}, + {file = "grpcio_tools-1.75.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:dcfb12654fb1d6ce84f4a55d3dfbc267a04d53dc9b52ee0974b2110d02f68dac"}, + {file = "grpcio_tools-1.75.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:990d183fee5a2ef9d4f3a220b6506f5da740271da175efcb7e4e34ebc3191a12"}, + {file = "grpcio_tools-1.75.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:39c6ff052960a3301cd920549384a2ad7cb3165c778feed601cae2a2131b63f8"}, + {file = "grpcio_tools-1.75.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:60bd449814fe3cebeda11c0cda3a3adffd81941559aa254e6d153751baa0cffc"}, + {file = "grpcio_tools-1.75.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:91e430e9368afc38e94645f744840ab06995cfb7312233623c5d7370f8c0dd7c"}, + {file = "grpcio_tools-1.75.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3351acef4b8897e99bdceae5cfcc300e1e5c1d88c0fc2ffc2b5ca1bd5ce4ced8"}, + {file = "grpcio_tools-1.75.0-cp313-cp313-win32.whl", hash = "sha256:1241f8c65f2429f00d9e15e819aca2138c5aa571f0ac644ab658a0281dc177d9"}, + {file = "grpcio_tools-1.75.0-cp313-cp313-win_amd64.whl", hash = "sha256:193ce6aef33417849289cbb518402fe60c00d0fa66d68ea9a30c98cb8818280c"}, + {file = "grpcio_tools-1.75.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:b9f64ab078f1e8ea09ceb72c3f7a55b9cbec515fd20e804aea78491adf785503"}, + {file = "grpcio_tools-1.75.0-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:7154a35243a49704782b39e8780d9a0adb393a9cedba2ab65c352e94ff42fe8c"}, + {file = "grpcio_tools-1.75.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9a620de24caa85b102d2416c3f679260d1d4103edcc2806d7dda43aad1913e01"}, + {file = "grpcio_tools-1.75.0-cp39-cp39-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:08cc1b8a1364a5b8f975e6a7273684d13630caab76c209a201464ad05f826eb9"}, + {file = "grpcio_tools-1.75.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0a0c899175dd23e96f61b3ab8153642e0ae0182b9c9a582cd0cc4702a056d845"}, + {file = "grpcio_tools-1.75.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:8707b63acb1e08c4031e959936af45487bc185a3fa1ae37fdac465e8ab311774"}, + {file = "grpcio_tools-1.75.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4d28cb03efb871a0ce13dc0fe1416c237ed6d70c42f19a64cef24aba88dd7c5f"}, + {file = "grpcio_tools-1.75.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:93b297f77a3f9fe99ea30597e98fd62d3d40bc2520f3e6c6c12b202710a2581d"}, + {file = "grpcio_tools-1.75.0-cp39-cp39-win32.whl", hash = "sha256:05087b1879b3f32a2182f1365e34233236c22e1a1e8cc448b5d29ea58d661846"}, + {file = "grpcio_tools-1.75.0-cp39-cp39-win_amd64.whl", hash = "sha256:aaec9c9b1cb0ff3823961e74b6cf0a1e6b0e7a82fa2fb0b2bc7b312978bd34f7"}, + {file = "grpcio_tools-1.75.0.tar.gz", hash = "sha256:eb5e4025034d92da3c81fd5e3468c33d5ae7571b07a72c385b5ec1746658573f"}, ] [package.dependencies] -grpcio = ">=1.62.3" -protobuf = ">=4.21.6,<5.0dev" +grpcio = ">=1.75.0" +protobuf = ">=6.31.1,<7.0.0" setuptools = "*" [[package]] @@ -739,41 +744,39 @@ virtualenv = ">=20.10.0" [[package]] name = "proto-plus" -version = "1.26.0" +version = "1.26.1" description = "Beautiful, Pythonic protocol buffers" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "proto_plus-1.26.0-py3-none-any.whl", hash = "sha256:bf2dfaa3da281fc3187d12d224c707cb57214fb2c22ba854eb0c105a3fb2d4d7"}, - {file = "proto_plus-1.26.0.tar.gz", hash = "sha256:6e93d5f5ca267b54300880fff156b6a3386b3fa3f43b1da62e680fc0c586ef22"}, + {file = "proto_plus-1.26.1-py3-none-any.whl", hash = "sha256:13285478c2dcf2abb829db158e1047e2f1e8d63a077d94263c2b88b043c75a66"}, + {file = "proto_plus-1.26.1.tar.gz", hash = "sha256:21a515a4c4c0088a773899e23c7bbade3d18f9c66c73edd4c7ee3816bc96a012"}, ] [package.dependencies] -protobuf = ">=3.19.0,<6.0.0dev" +protobuf = ">=3.19.0,<7.0.0" [package.extras] testing = ["google-api-core (>=1.31.5)"] [[package]] name = "protobuf" -version = "4.25.6" +version = "6.32.1" description = "" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "protobuf-4.25.6-cp310-abi3-win32.whl", hash = "sha256:61df6b5786e2b49fc0055f636c1e8f0aff263808bb724b95b164685ac1bcc13a"}, - {file = "protobuf-4.25.6-cp310-abi3-win_amd64.whl", hash = "sha256:b8f837bfb77513fe0e2f263250f423217a173b6d85135be4d81e96a4653bcd3c"}, - {file = "protobuf-4.25.6-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:6d4381f2417606d7e01750e2729fe6fbcda3f9883aa0c32b51d23012bded6c91"}, - {file = "protobuf-4.25.6-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:5dd800da412ba7f6f26d2c08868a5023ce624e1fdb28bccca2dc957191e81fb5"}, - {file = "protobuf-4.25.6-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:4434ff8bb5576f9e0c78f47c41cdf3a152c0b44de475784cd3fd170aef16205a"}, - {file = "protobuf-4.25.6-cp38-cp38-win32.whl", hash = "sha256:8bad0f9e8f83c1fbfcc34e573352b17dfce7d0519512df8519994168dc015d7d"}, - {file = "protobuf-4.25.6-cp38-cp38-win_amd64.whl", hash = "sha256:b6905b68cde3b8243a198268bb46fbec42b3455c88b6b02fb2529d2c306d18fc"}, - {file = "protobuf-4.25.6-cp39-cp39-win32.whl", hash = "sha256:3f3b0b39db04b509859361ac9bca65a265fe9342e6b9406eda58029f5b1d10b2"}, - {file = "protobuf-4.25.6-cp39-cp39-win_amd64.whl", hash = "sha256:6ef2045f89d4ad8d95fd43cd84621487832a61d15b49500e4c1350e8a0ef96be"}, - {file = "protobuf-4.25.6-py3-none-any.whl", hash = "sha256:07972021c8e30b870cfc0863409d033af940213e0e7f64e27fe017b929d2c9f7"}, - {file = "protobuf-4.25.6.tar.gz", hash = "sha256:f8cfbae7c5afd0d0eaccbe73267339bff605a2315860bb1ba08eb66670a9a91f"}, + {file = "protobuf-6.32.1-cp310-abi3-win32.whl", hash = "sha256:a8a32a84bc9f2aad712041b8b366190f71dde248926da517bde9e832e4412085"}, + {file = "protobuf-6.32.1-cp310-abi3-win_amd64.whl", hash = "sha256:b00a7d8c25fa471f16bc8153d0e53d6c9e827f0953f3c09aaa4331c718cae5e1"}, + {file = "protobuf-6.32.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:d8c7e6eb619ffdf105ee4ab76af5a68b60a9d0f66da3ea12d1640e6d8dab7281"}, + {file = "protobuf-6.32.1-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:2f5b80a49e1eb7b86d85fcd23fe92df154b9730a725c3b38c4e43b9d77018bf4"}, + {file = "protobuf-6.32.1-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:b1864818300c297265c83a4982fd3169f97122c299f56a56e2445c3698d34710"}, + {file = "protobuf-6.32.1-cp39-cp39-win32.whl", hash = "sha256:68ff170bac18c8178f130d1ccb94700cf72852298e016a2443bdb9502279e5f1"}, + {file = "protobuf-6.32.1-cp39-cp39-win_amd64.whl", hash = "sha256:d0975d0b2f3e6957111aa3935d08a0eb7e006b1505d825f862a1fffc8348e122"}, + {file = "protobuf-6.32.1-py3-none-any.whl", hash = "sha256:2601b779fc7d32a866c6b4404f9d42a3f67c5b9f3f15b4db3cccabe06b95c346"}, + {file = "protobuf-6.32.1.tar.gz", hash = "sha256:ee2469e4a021474ab9baafea6cd070e5bf27c7d29433504ddea1a4ee5850f68d"}, ] [[package]] @@ -1073,8 +1076,7 @@ version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" -groups = ["dev"] -markers = "python_version < \"3.11\"" +groups = ["main", "dev"] files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, @@ -1100,48 +1102,55 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "uvloop" -version = "0.19.0" +version = "0.21.0" description = "Fast implementation of asyncio event loop on top of libuv" optional = false python-versions = ">=3.8.0" groups = ["main"] files = [ - {file = "uvloop-0.19.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:de4313d7f575474c8f5a12e163f6d89c0a878bc49219641d49e6f1444369a90e"}, - {file = "uvloop-0.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5588bd21cf1fcf06bded085f37e43ce0e00424197e7c10e77afd4bbefffef428"}, - {file = "uvloop-0.19.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b1fd71c3843327f3bbc3237bedcdb6504fd50368ab3e04d0410e52ec293f5b8"}, - {file = "uvloop-0.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a05128d315e2912791de6088c34136bfcdd0c7cbc1cf85fd6fd1bb321b7c849"}, - {file = "uvloop-0.19.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:cd81bdc2b8219cb4b2556eea39d2e36bfa375a2dd021404f90a62e44efaaf957"}, - {file = "uvloop-0.19.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5f17766fb6da94135526273080f3455a112f82570b2ee5daa64d682387fe0dcd"}, - {file = "uvloop-0.19.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4ce6b0af8f2729a02a5d1575feacb2a94fc7b2e983868b009d51c9a9d2149bef"}, - {file = "uvloop-0.19.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:31e672bb38b45abc4f26e273be83b72a0d28d074d5b370fc4dcf4c4eb15417d2"}, - {file = "uvloop-0.19.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:570fc0ed613883d8d30ee40397b79207eedd2624891692471808a95069a007c1"}, - {file = "uvloop-0.19.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5138821e40b0c3e6c9478643b4660bd44372ae1e16a322b8fc07478f92684e24"}, - {file = "uvloop-0.19.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:91ab01c6cd00e39cde50173ba4ec68a1e578fee9279ba64f5221810a9e786533"}, - {file = "uvloop-0.19.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:47bf3e9312f63684efe283f7342afb414eea4d3011542155c7e625cd799c3b12"}, - {file = "uvloop-0.19.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:da8435a3bd498419ee8c13c34b89b5005130a476bda1d6ca8cfdde3de35cd650"}, - {file = "uvloop-0.19.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:02506dc23a5d90e04d4f65c7791e65cf44bd91b37f24cfc3ef6cf2aff05dc7ec"}, - {file = "uvloop-0.19.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2693049be9d36fef81741fddb3f441673ba12a34a704e7b4361efb75cf30befc"}, - {file = "uvloop-0.19.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7010271303961c6f0fe37731004335401eb9075a12680738731e9c92ddd96ad6"}, - {file = "uvloop-0.19.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5daa304d2161d2918fa9a17d5635099a2f78ae5b5960e742b2fcfbb7aefaa593"}, - {file = "uvloop-0.19.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:7207272c9520203fea9b93843bb775d03e1cf88a80a936ce760f60bb5add92f3"}, - {file = "uvloop-0.19.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:78ab247f0b5671cc887c31d33f9b3abfb88d2614b84e4303f1a63b46c046c8bd"}, - {file = "uvloop-0.19.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:472d61143059c84947aa8bb74eabbace30d577a03a1805b77933d6bd13ddebbd"}, - {file = "uvloop-0.19.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45bf4c24c19fb8a50902ae37c5de50da81de4922af65baf760f7c0c42e1088be"}, - {file = "uvloop-0.19.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:271718e26b3e17906b28b67314c45d19106112067205119dddbd834c2b7ce797"}, - {file = "uvloop-0.19.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:34175c9fd2a4bc3adc1380e1261f60306344e3407c20a4d684fd5f3be010fa3d"}, - {file = "uvloop-0.19.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e27f100e1ff17f6feeb1f33968bc185bf8ce41ca557deee9d9bbbffeb72030b7"}, - {file = "uvloop-0.19.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:13dfdf492af0aa0a0edf66807d2b465607d11c4fa48f4a1fd41cbea5b18e8e8b"}, - {file = "uvloop-0.19.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6e3d4e85ac060e2342ff85e90d0c04157acb210b9ce508e784a944f852a40e67"}, - {file = "uvloop-0.19.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ca4956c9ab567d87d59d49fa3704cf29e37109ad348f2d5223c9bf761a332e7"}, - {file = "uvloop-0.19.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f467a5fd23b4fc43ed86342641f3936a68ded707f4627622fa3f82a120e18256"}, - {file = "uvloop-0.19.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:492e2c32c2af3f971473bc22f086513cedfc66a130756145a931a90c3958cb17"}, - {file = "uvloop-0.19.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2df95fca285a9f5bfe730e51945ffe2fa71ccbfdde3b0da5772b4ee4f2e770d5"}, - {file = "uvloop-0.19.0.tar.gz", hash = "sha256:0246f4fd1bf2bf702e06b0d45ee91677ee5c31242f39aab4ea6fe0c51aedd0fd"}, + {file = "uvloop-0.21.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ec7e6b09a6fdded42403182ab6b832b71f4edaf7f37a9a0e371a01db5f0cb45f"}, + {file = "uvloop-0.21.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:196274f2adb9689a289ad7d65700d37df0c0930fd8e4e743fa4834e850d7719d"}, + {file = "uvloop-0.21.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f38b2e090258d051d68a5b14d1da7203a3c3677321cf32a95a6f4db4dd8b6f26"}, + {file = "uvloop-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87c43e0f13022b998eb9b973b5e97200c8b90823454d4bc06ab33829e09fb9bb"}, + {file = "uvloop-0.21.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:10d66943def5fcb6e7b37310eb6b5639fd2ccbc38df1177262b0640c3ca68c1f"}, + {file = "uvloop-0.21.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:67dd654b8ca23aed0a8e99010b4c34aca62f4b7fce88f39d452ed7622c94845c"}, + {file = "uvloop-0.21.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c0f3fa6200b3108919f8bdabb9a7f87f20e7097ea3c543754cabc7d717d95cf8"}, + {file = "uvloop-0.21.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0878c2640cf341b269b7e128b1a5fed890adc4455513ca710d77d5e93aa6d6a0"}, + {file = "uvloop-0.21.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9fb766bb57b7388745d8bcc53a359b116b8a04c83a2288069809d2b3466c37e"}, + {file = "uvloop-0.21.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a375441696e2eda1c43c44ccb66e04d61ceeffcd76e4929e527b7fa401b90fb"}, + {file = "uvloop-0.21.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:baa0e6291d91649c6ba4ed4b2f982f9fa165b5bbd50a9e203c416a2797bab3c6"}, + {file = "uvloop-0.21.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4509360fcc4c3bd2c70d87573ad472de40c13387f5fda8cb58350a1d7475e58d"}, + {file = "uvloop-0.21.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:359ec2c888397b9e592a889c4d72ba3d6befba8b2bb01743f72fffbde663b59c"}, + {file = "uvloop-0.21.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2"}, + {file = "uvloop-0.21.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baa4dcdbd9ae0a372f2167a207cd98c9f9a1ea1188a8a526431eef2f8116cc8d"}, + {file = "uvloop-0.21.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86975dca1c773a2c9864f4c52c5a55631038e387b47eaf56210f873887b6c8dc"}, + {file = "uvloop-0.21.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:461d9ae6660fbbafedd07559c6a2e57cd553b34b0065b6550685f6653a98c1cb"}, + {file = "uvloop-0.21.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:183aef7c8730e54c9a3ee3227464daed66e37ba13040bb3f350bc2ddc040f22f"}, + {file = "uvloop-0.21.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:bfd55dfcc2a512316e65f16e503e9e450cab148ef11df4e4e679b5e8253a5281"}, + {file = "uvloop-0.21.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:787ae31ad8a2856fc4e7c095341cccc7209bd657d0e71ad0dc2ea83c4a6fa8af"}, + {file = "uvloop-0.21.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ee4d4ef48036ff6e5cfffb09dd192c7a5027153948d85b8da7ff705065bacc6"}, + {file = "uvloop-0.21.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3df876acd7ec037a3d005b3ab85a7e4110422e4d9c1571d4fc89b0fc41b6816"}, + {file = "uvloop-0.21.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bd53ecc9a0f3d87ab847503c2e1552b690362e005ab54e8a48ba97da3924c0dc"}, + {file = "uvloop-0.21.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a5c39f217ab3c663dc699c04cbd50c13813e31d917642d459fdcec07555cc553"}, + {file = "uvloop-0.21.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:17df489689befc72c39a08359efac29bbee8eee5209650d4b9f34df73d22e414"}, + {file = "uvloop-0.21.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bc09f0ff191e61c2d592a752423c767b4ebb2986daa9ed62908e2b1b9a9ae206"}, + {file = "uvloop-0.21.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0ce1b49560b1d2d8a2977e3ba4afb2414fb46b86a1b64056bc4ab929efdafbe"}, + {file = "uvloop-0.21.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e678ad6fe52af2c58d2ae3c73dc85524ba8abe637f134bf3564ed07f555c5e79"}, + {file = "uvloop-0.21.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:460def4412e473896ef179a1671b40c039c7012184b627898eea5072ef6f017a"}, + {file = "uvloop-0.21.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:10da8046cc4a8f12c91a1c39d1dd1585c41162a15caaef165c2174db9ef18bdc"}, + {file = "uvloop-0.21.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c097078b8031190c934ed0ebfee8cc5f9ba9642e6eb88322b9958b649750f72b"}, + {file = "uvloop-0.21.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:46923b0b5ee7fc0020bef24afe7836cb068f5050ca04caf6b487c513dc1a20b2"}, + {file = "uvloop-0.21.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:53e420a3afe22cdcf2a0f4846e377d16e718bc70103d7088a4f7623567ba5fb0"}, + {file = "uvloop-0.21.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88cb67cdbc0e483da00af0b2c3cdad4b7c61ceb1ee0f33fe00e09c81e3a6cb75"}, + {file = "uvloop-0.21.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:221f4f2a1f46032b403bf3be628011caf75428ee3cc204a22addf96f586b19fd"}, + {file = "uvloop-0.21.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2d1f581393673ce119355d56da84fe1dd9d2bb8b3d13ce792524e1607139feff"}, + {file = "uvloop-0.21.0.tar.gz", hash = "sha256:3bf12b0fda68447806a7ad847bfa591613177275d35b6724b1ee573faa3704e3"}, ] [package.extras] +dev = ["Cython (>=3.0,<4.0)", "setuptools (>=60)"] docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] -test = ["Cython (>=0.29.36,<0.30.0)", "aiohttp (==3.9.0b0) ; python_version >= \"3.12\"", "aiohttp (>=3.8.1) ; python_version < \"3.12\"", "flake8 (>=5.0,<6.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=23.0.0,<23.1.0)", "pycodestyle (>=2.9.0,<2.10.0)"] +test = ["aiohttp (>=3.10.5)", "flake8 (>=5.0,<6.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=23.0.0,<23.1.0)", "pycodestyle (>=2.9.0,<2.10.0)"] [[package]] name = "virtualenv" @@ -1166,5 +1175,5 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess [metadata] lock-version = "2.1" -python-versions = ">=3.9, <3.13" -content-hash = "afdf2080de75a2057f967d51fb0e96189ea51ce3a2970f3ee623701c6d4a70d7" +python-versions = ">=3.9, <3.14" +content-hash = "cf36257cde0ef46a94993201cd08d652d1e7ec9a07b3c3a1f667fec042e1b97a" diff --git a/pyproject.toml b/pyproject.toml index e6dae906..9af322ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,20 +16,21 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12" + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13" ] repository = "https://github.com/numaproj/numaflow-python" [tool.poetry.dependencies] -python = ">=3.9, <3.13" +python = ">=3.9, <3.14" grpcio = "^1.48.1" grpcio-tools = "^1.48.1" google-cloud = "^0.34.0" google-api-core = "^2.11.0" grpcio-status = "^1.48.1" -protobuf = ">=3.20,<6.0" +protobuf = ">=3.20,<7.0" aiorun = "^2023.7" -uvloop = "^0.19.0" +uvloop = "^0.21.0" psutil = "^6.0.0" [tool.poetry.group.dev] diff --git a/tests/map/test_multiproc_mapper.py b/tests/map/test_multiproc_mapper.py index 77fda9fa..90eabb9b 100644 --- a/tests/map/test_multiproc_mapper.py +++ b/tests/map/test_multiproc_mapper.py @@ -136,10 +136,10 @@ def test_map_forward_message(self): self.assertTrue(responses[0].handshake.sot) + result_ids = {f"test-id-{id}" for id in range(1, 4)} idx = 1 while idx < len(responses): - _id = "test-id-" + str(idx) - self.assertEqual(_id, responses[idx].id) + result_ids.remove(responses[idx].id) self.assertEqual( bytes( "payload:test_mock_message " @@ -150,6 +150,7 @@ def test_map_forward_message(self): ) self.assertEqual(1, len(responses[idx].results)) idx += 1 + self.assertEqual(len(result_ids), 0) self.assertEqual(code, StatusCode.OK) def test_invalid_input(self): diff --git a/tests/map/test_sync_mapper.py b/tests/map/test_sync_mapper.py index 4eb7149a..edafa835 100644 --- a/tests/map/test_sync_mapper.py +++ b/tests/map/test_sync_mapper.py @@ -130,10 +130,10 @@ def test_map_forward_message(self): self.assertTrue(responses[0].handshake.sot) + result_ids = {f"test-id-{id}" for id in range(1, 4)} idx = 1 while idx < len(responses): - _id = "test-id-" + str(idx) - self.assertEqual(_id, responses[idx].id) + result_ids.remove(responses[idx].id) self.assertEqual( bytes( "payload:test_mock_message " @@ -144,6 +144,7 @@ def test_map_forward_message(self): ) self.assertEqual(1, len(responses[idx].results)) idx += 1 + self.assertEqual(len(result_ids), 0) self.assertEqual(code, StatusCode.OK) def test_invalid_input(self): diff --git a/tests/sourcetransform/test_multiproc.py b/tests/sourcetransform/test_multiproc.py index 03e33bbc..b844231e 100644 --- a/tests/sourcetransform/test_multiproc.py +++ b/tests/sourcetransform/test_multiproc.py @@ -162,10 +162,10 @@ def test_mapt_assign_new_event_time(self): self.assertTrue(responses[0].handshake.sot) + result_ids = {f"test-id-{id}" for id in range(1, 4)} idx = 1 while idx < len(responses): - _id = "test-id-" + str(idx) - self.assertEqual(_id, responses[idx].id) + result_ids.remove(responses[idx].id) self.assertEqual( bytes( "payload:test_mock_message " "event_time:2022-09-12 16:00:00 ", @@ -175,6 +175,7 @@ def test_mapt_assign_new_event_time(self): ) self.assertEqual(1, len(responses[idx].results)) idx += 1 + self.assertEqual(len(result_ids), 0) # Verify new event time gets assigned. updated_event_time_timestamp = _timestamp_pb2.Timestamp() diff --git a/tests/sourcetransform/test_sync_server.py b/tests/sourcetransform/test_sync_server.py index 9ec63cb6..3ffcd0da 100644 --- a/tests/sourcetransform/test_sync_server.py +++ b/tests/sourcetransform/test_sync_server.py @@ -153,10 +153,10 @@ def test_mapt_assign_new_event_time(self): self.assertTrue(responses[0].handshake.sot) + result_ids = {f"test-id-{id}" for id in range(1, 4)} idx = 1 while idx < len(responses): - _id = "test-id-" + str(idx) - self.assertEqual(_id, responses[idx].id) + result_ids.remove(responses[idx].id) self.assertEqual( bytes( "payload:test_mock_message " "event_time:2022-09-12 16:00:00 ", @@ -166,6 +166,7 @@ def test_mapt_assign_new_event_time(self): ) self.assertEqual(1, len(responses[idx].results)) idx += 1 + self.assertEqual(len(result_ids), 0) # Verify new event time gets assigned. updated_event_time_timestamp = _timestamp_pb2.Timestamp()