Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 30e0bf3

Browse files
pyalexpyalex
authored
chore: Make contrib test pluggable (#2654)
* make contrib test plugable Signed-off-by: Oleksii Moskalenko <[email protected]> * no autouse Signed-off-by: Oleksii Moskalenko <[email protected]> * pass request Signed-off-by: Oleksii Moskalenko <[email protected]> * fix fixture name Signed-off-by: Oleksii Moskalenko <[email protected]> * publish trino fixture on package level Signed-off-by: Oleksii Moskalenko <[email protected]> * format Signed-off-by: Oleksii Moskalenko <[email protected]> * disable some tests for contrib tests run Signed-off-by: Oleksii Moskalenko <[email protected]> * address comments Signed-off-by: Oleksii Moskalenko <[email protected]>
1 parent 0e809c2 commit 30e0bf3

File tree

23 files changed

+255
-397
lines changed

23 files changed

+255
-397
lines changed

‎Makefile‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,11 +72,25 @@ test-python-integration-container:
7272
FEAST_USAGE=False IS_TEST=True FEAST_LOCAL_ONLINE_CONTAINER=True python -m pytest -n 8 --integration sdk/python/tests
7373

7474
test-python-universal-contrib:
75-
PYTHONPATH='.' FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.contrib_repo_configuration FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests
75+
PYTHONPATH='.' \
76+
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.contrib_repo_configuration \
77+
PYTEST_PLUGINS=feast.infra.offline_stores.contrib.trino_offline_store.tests \
78+
FEAST_USAGE=False IS_TEST=True \
79+
python -m pytest -n 8 --integration --universal \
80+
-k "not test_historical_retrieval_fails_on_validation and \
81+
not test_historical_retrieval_with_validation and \
82+
not test_historical_features_persisting and \
83+
not test_historical_retrieval_fails_on_validation and \
84+
not test_universal_cli and \
85+
not test_go_feature_server and \
86+
not test_feature_logging and \
87+
not test_universal_types" \
88+
sdk/python/tests
7689

7790
test-python-universal-postgres:
7891
PYTHONPATH='.' \
7992
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.postgres_repo_configuration \
93+
PYTEST_PLUGINS=sdk.python.feast.infra.offline_stores.contrib.postgres_offline_store.tests \
8094
FEAST_USAGE=False \
8195
IS_TEST=True \
8296
python -m pytest -x --integration --universal \

‎sdk/python/feast/infra/offline_stores/contrib/contrib_repo_configuration.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1-
from tests.integration.feature_repos.integration_test_repo_config import (
2-
IntegrationTestRepoConfig,
3-
)
4-
from tests.integration.feature_repos.universal.data_sources.spark_data_source_creator import (
1+
from feast.infra.offline_stores.contrib.spark_offline_store.tests.data_source import (
52
SparkDataSourceCreator,
63
)
7-
from tests.integration.feature_repos.universal.data_sources.trino import (
4+
from feast.infra.offline_stores.contrib.trino_offline_store.tests.data_source import (
85
TrinoSourceCreator,
96
)
7+
from tests.integration.feature_repos.integration_test_repo_config import (
8+
IntegrationTestRepoConfig,
9+
)
1010

1111
FULL_REPO_CONFIGS = [
1212
IntegrationTestRepoConfig(offline_store_creator=SparkDataSourceCreator),

‎sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/tests/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
from .data_source import postgres_container # noqa

‎sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/tests/data_source.py‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
import logging
2+
from typing import Dict, Optional
3+
4+
import pandas as pd
5+
import pytest
6+
from testcontainers.core.container import DockerContainer
7+
from testcontainers.core.waiting_utils import wait_for_logs
8+
9+
from feast.data_source import DataSource
10+
from feast.infra.offline_stores.contrib.postgres_offline_store.postgres import (
11+
PostgreSQLOfflineStoreConfig,
12+
PostgreSQLSource,
13+
)
14+
from feast.infra.utils.postgres.connection_utils import df_to_postgres_table
15+
from tests.integration.feature_repos.universal.data_source_creator import (
16+
DataSourceCreator,
17+
)
18+
from tests.integration.feature_repos.universal.online_store_creator import (
19+
OnlineStoreCreator,
20+
)
21+
22+
logger = logging.getLogger(__name__)
23+
24+
POSTGRES_USER = "test"
25+
POSTGRES_PASSWORD = "test"
26+
POSTGRES_DB = "test"
27+
28+
29+
@pytest.fixture(scope="session")
30+
def postgres_container():
31+
container = (
32+
DockerContainer("postgres:latest")
33+
.with_exposed_ports(5432)
34+
.with_env("POSTGRES_USER", POSTGRES_USER)
35+
.with_env("POSTGRES_PASSWORD", POSTGRES_PASSWORD)
36+
.with_env("POSTGRES_DB", POSTGRES_DB)
37+
)
38+
39+
container.start()
40+
41+
log_string_to_wait_for = "database system is ready to accept connections"
42+
waited = wait_for_logs(
43+
container=container, predicate=log_string_to_wait_for, timeout=30, interval=10,
44+
)
45+
logger.info("Waited for %s seconds until postgres container was up", waited)
46+
47+
yield container
48+
container.stop()
49+
50+
51+
class PostgreSQLDataSourceCreator(DataSourceCreator, OnlineStoreCreator):
52+
def __init__(
53+
self, project_name: str, fixture_request: pytest.FixtureRequest, **kwargs
54+
):
55+
super().__init__(project_name,)
56+
57+
self.project_name = project_name
58+
self.container = fixture_request.getfixturevalue("postgres_container")
59+
if not self.container:
60+
raise RuntimeError(
61+
"In order to use this data source "
62+
"'feast.infra.offline_stores.contrib.postgres_offline_store.tests' "
63+
"must be include into pytest plugins"
64+
)
65+
66+
self.offline_store_config = PostgreSQLOfflineStoreConfig(
67+
type="postgres",
68+
host="localhost",
69+
port=self.container.get_exposed_port(5432),
70+
database=self.container.env["POSTGRES_DB"],
71+
db_schema="public",
72+
user=self.container.env["POSTGRES_USER"],
73+
password=self.container.env["POSTGRES_PASSWORD"],
74+
)
75+
76+
def create_data_source(
77+
self,
78+
df: pd.DataFrame,
79+
destination_name: str,
80+
suffix: Optional[str] = None,
81+
timestamp_field="ts",
82+
created_timestamp_column="created_ts",
83+
field_mapping: Dict[str, str] = None,
84+
) -> DataSource:
85+
destination_name = self.get_prefixed_table_name(destination_name)
86+
87+
if self.offline_store_config:
88+
df_to_postgres_table(self.offline_store_config, df, destination_name)
89+
90+
return PostgreSQLSource(
91+
name=destination_name,
92+
query=f"SELECT * FROM {destination_name}",
93+
timestamp_field=timestamp_field,
94+
created_timestamp_column=created_timestamp_column,
95+
field_mapping=field_mapping or {"ts_1": "ts"},
96+
)
97+
98+
def create_offline_store_config(self) -> PostgreSQLOfflineStoreConfig:
99+
assert self.offline_store_config
100+
return self.offline_store_config
101+
102+
def get_prefixed_table_name(self, suffix: str) -> str:
103+
return f"{self.project_name}_{suffix}"
104+
105+
def create_online_store(self) -> Dict[str, str]:
106+
assert self.container
107+
return {
108+
"type": "postgres",
109+
"host": "localhost",
110+
"port": self.container.get_exposed_port(5432),
111+
"database": POSTGRES_DB,
112+
"db_schema": "feature_store",
113+
"user": POSTGRES_USER,
114+
"password": POSTGRES_PASSWORD,
115+
}
116+
117+
def create_saved_dataset_destination(self):
118+
# FIXME: ...
119+
return None
120+
121+
def teardown(self):
122+
pass

‎sdk/python/feast/infra/offline_stores/contrib/postgres_repo_configuration.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1+
from feast.infra.offline_stores.contrib.postgres_offline_store.tests.data_source import (
2+
PostgreSQLDataSourceCreator,
3+
)
14
from tests.integration.feature_repos.integration_test_repo_config import (
25
IntegrationTestRepoConfig,
36
)
4-
from tests.integration.feature_repos.universal.data_sources.postgres import (
5-
PostgreSQLDataSourceCreator,
6-
)
77

88
FULL_REPO_CONFIGS = [
99
IntegrationTestRepoConfig(

‎sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/__init__.py‎

Whitespace-only changes.

sdk/python/tests/integration/feature_repos/universal/data_sources/spark_data_source_creator.py renamed to sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py

File renamed without changes.

‎sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/test_config/manual_tests.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1+
from feast.infra.offline_stores.contrib.trino_offline_store.tests.data_source import (
2+
TrinoSourceCreator,
3+
)
14
from tests.integration.feature_repos.integration_test_repo_config import (
25
IntegrationTestRepoConfig,
36
)
4-
from tests.integration.feature_repos.universal.data_sources.trino import (
5-
TrinoSourceCreator,
6-
)
77

88
FULL_REPO_CONFIGS = [
99
IntegrationTestRepoConfig(

‎sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
from .data_source import trino_container # noqa

sdk/python/tests/integration/feature_repos/universal/data_sources/catalog/memory.properties renamed to sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/catalog/memory.properties

File renamed without changes.

0 commit comments

Comments
 (0)