Thanks to visit codestin.com
Credit goes to github.com

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# Changelog

## Unreleased
- DMS: Added option `ignore_ddl` to ignore DDL events

## 2025/06/23 v0.0.23
- Dependencies: Migrated from `zyp` to `tikray`. It's effectively the
Expand Down
4 changes: 4 additions & 0 deletions src/commons_codec/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,10 @@ class SQLOperation:
parameters: t.Optional[t.Union[t.Mapping[str, t.Any], t.List[t.Mapping[str, t.Any]]]] = None


class SkipOperation(Exception):
pass


@define
class SQLParameterizedClause:
"""
Expand Down
16 changes: 12 additions & 4 deletions src/commons_codec/transform/aws_dms.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
ColumnType,
ColumnTypeMapStore,
PrimaryKeyStore,
SkipOperation,
SQLOperation,
SQLParameterizedSetClause,
SQLParameterizedWhereClause,
Expand Down Expand Up @@ -83,13 +84,16 @@ def __init__(
self.primary_keys: t.List[str] = self.container.primary_keys[self.address]
self.column_types: t.Dict[str, ColumnType] = self.container.column_types[self.address]

pks = self.control.get("table-def", {}).get("primary-key", [])
for pk in pks:
if pk not in self.primary_keys:
self.primary_keys.append(pk)
if not self.container.ignore_ddl:
pks = self.control.get("table-def", {}).get("primary-key", [])
for pk in pks:
if pk not in self.primary_keys:
self.primary_keys.append(pk)

def to_sql(self) -> SQLOperation:
if self.operation == "create-table":
if self.container.ignore_ddl:
raise SkipOperation("Ignoring DMS DDL event: create-table")
return SQLOperation(
f"CREATE TABLE IF NOT EXISTS {self.address.fqn} ("
f"{self.PK_COLUMN} OBJECT(STRICT){self.pk_clause()}, "
Expand All @@ -98,6 +102,8 @@ def to_sql(self) -> SQLOperation:
)

elif self.operation == "drop-table":
if self.container.ignore_ddl:
raise SkipOperation("Ignoring DMS DDL event: drop-table")
# Remove cached schema information by restoring original so a future CREATE starts clean.
self.container.primary_keys[self.address] = self.container.primary_keys_caller.get(self.address, [])
self.container.column_types[self.address] = self.container.column_types_caller.get(self.address, {})
Expand Down Expand Up @@ -249,9 +255,11 @@ def __init__(
self,
primary_keys: PrimaryKeyStore = None,
column_types: ColumnTypeMapStore = None,
ignore_ddl: bool = False,
):
self.primary_keys = primary_keys or PrimaryKeyStore()
self.column_types = column_types or ColumnTypeMapStore()
self.ignore_ddl = ignore_ddl

# Store caller-provided schema information to restore this state on `DROP TABLE` operations.
self.primary_keys_caller = copy.deepcopy(self.primary_keys)
Expand Down
39 changes: 37 additions & 2 deletions tests/transform/test_aws_dms.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,14 @@
import pytest

from commons_codec.exception import MessageFormatError, UnknownOperationError
from commons_codec.model import ColumnType, ColumnTypeMapStore, SQLOperation, TableAddress
from commons_codec.model import (
ColumnType,
ColumnTypeMapStore,
PrimaryKeyStore,
SkipOperation,
SQLOperation,
TableAddress,
)
from commons_codec.transform.aws_dms import DMSTranslatorCrateDB

RECORD_INSERT = {"age": 31, "attributes": {"baz": "qux"}, "id": 46, "name": "Jane"}
Expand Down Expand Up @@ -179,7 +186,7 @@
@pytest.fixture
def cdc():
"""
Provide fresh translator instance.
Provide a regular translator instance.
"""
column_types = ColumnTypeMapStore().add(
table=TableAddress(schema="public", table="foo"),
Expand All @@ -189,6 +196,22 @@ def cdc():
return DMSTranslatorCrateDB(column_types=column_types)


@pytest.fixture
def cdc_without_ddl():
"""
Provide a translator instance that ignores DDL events.
"""
ta = TableAddress(schema="public", table="foo")
primary_keys = PrimaryKeyStore()
primary_keys[ta] = {"name": "id", "type": "INTEGER"}
column_types = ColumnTypeMapStore().add(
table=ta,
column="attributes",
type_=ColumnType.MAP,
)
return DMSTranslatorCrateDB(column_types=column_types, ignore_ddl=True)


def test_decode_cdc_unknown_source(cdc):
with pytest.raises(MessageFormatError) as ex:
cdc.to_sql(MSG_UNKNOWN_SHAPE)
Expand Down Expand Up @@ -239,6 +262,18 @@ def test_decode_cdc_sql_ddl_regular_drop(cdc):
)


def test_decode_cdc_sql_without_ddl_regular_create(cdc_without_ddl):
with pytest.raises(SkipOperation) as ex:
cdc_without_ddl.to_sql(MSG_CONTROL_CREATE_TABLE)
assert ex.match("Ignoring DMS DDL event: create-table")


def test_decode_cdc_sql_without_ddl_regular_drop(cdc_without_ddl):
with pytest.raises(SkipOperation) as ex:
cdc_without_ddl.to_sql(MSG_CONTROL_DROP_TABLE)
assert ex.match("Ignoring DMS DDL event: drop-table")


def test_decode_cdc_sql_ddl_awsdms(cdc):
assert cdc.to_sql(MSG_CONTROL_AWSDMS) == SQLOperation(
statement="CREATE TABLE IF NOT EXISTS dms.awsdms_apply_exceptions "
Expand Down