From a17dfab10639af288d1e21b048debe28b1276e58 Mon Sep 17 00:00:00 2001 From: Clearvive Date: Wed, 6 Dec 2023 14:36:03 +0800 Subject: [PATCH 1/2] [895#] feature(jdbc-catalog): Jdbc catalog support update column nullability. --- .../mysql/operation/MysqlTableOperations.java | 28 +++++++++++++++++++ .../operation/PostgreSqlTableOperations.java | 20 +++++++++++++ .../jdbc/mysql/TestMysqlTableOperations.java | 25 +++++++++++++++-- .../TestPostgreSqlTableOperations.java | 26 +++++++++++++++++ 4 files changed, 96 insertions(+), 3 deletions(-) diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java index ad74254e425..4f17ee0d49d 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java @@ -322,6 +322,14 @@ protected String generateAlterTableSql( } else if (change instanceof TableChange.DeleteColumn) { TableChange.DeleteColumn deleteColumn = (TableChange.DeleteColumn) change; alterSql.add(deleteColumnFieldDefinition(deleteColumn)); + } else if (change instanceof TableChange.UpdateColumnNullability) { + lazyLoadCreateTable = getOrCreateTable(databaseName, tableName, lazyLoadCreateTable); + alterSql.add( + updateColumnNullabilityDefinition( + (TableChange.UpdateColumnNullability) change, lazyLoadCreateTable)); + } else { + throw new UnsupportedOperationException( + "Unsupported table change type: " + change.getClass().getName()); } } if (!setProperties.isEmpty()) { @@ -350,6 +358,26 @@ protected String generateAlterTableSql( return result; } + private String updateColumnNullabilityDefinition( + TableChange.UpdateColumnNullability change, CreateTable table) { + if (change.fieldName().length > 1) { + throw new UnsupportedOperationException("Mysql does not support nested column names."); + } + String col = change.fieldName()[0]; + JdbcColumn column = getJdbcColumnFromCreateTable(table, col); + column.getProperties().remove(PRIMARY_KEY); + JdbcColumn updateColumn = + new JdbcColumn.Builder() + .withName(col) + .withDefaultValue(column.getDefaultValue()) + .withNullable(change.nullable()) + .withProperties(column.getProperties()) + .withType(column.dataType()) + .withComment(column.comment()) + .build(); + return "MODIFY COLUMN " + col + appendColumnDefinition(updateColumn, new StringBuilder()); + } + private String generateTableProperties(List setProperties) { // TODO #804 Properties will be unified in the future. // return setProperties.stream() diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java index c94b623e724..cb09bb8c066 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java @@ -343,6 +343,13 @@ protected String generateAlterTableSql( } else if (change instanceof TableChange.DeleteColumn) { TableChange.DeleteColumn deleteColumn = (TableChange.DeleteColumn) change; alterSql.add(deleteColumnFieldDefinition(deleteColumn, tableName)); + } else if (change instanceof TableChange.UpdateColumnNullability) { + alterSql.add( + updateColumnNullabilityDefinition( + (TableChange.UpdateColumnNullability) change, tableName)); + } else { + throw new UnsupportedOperationException( + "Unsupported table change type: " + change.getClass().getName()); } } @@ -352,6 +359,19 @@ protected String generateAlterTableSql( return result; } + private String updateColumnNullabilityDefinition( + TableChange.UpdateColumnNullability updateColumnNullability, String tableName) { + if (updateColumnNullability.fieldName().length > 1) { + throw new UnsupportedOperationException("PostgreSQL does not support nested column names."); + } + String col = updateColumnNullability.fieldName()[0]; + if (updateColumnNullability.nullable()) { + return "ALTER TABLE " + tableName + " ALTER COLUMN " + col + " DROP NOT NULL;"; + } else { + return "ALTER TABLE " + tableName + " ALTER COLUMN " + col + " SET NOT NULL;"; + } + } + private String updateCommentDefinition( TableChange.UpdateComment updateComment, JdbcTable jdbcTable) { String newComment = updateComment.getNewComment(); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/TestMysqlTableOperations.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/TestMysqlTableOperations.java index 3c7eb86f6ff..d7635554fb2 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/TestMysqlTableOperations.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/TestMysqlTableOperations.java @@ -322,18 +322,37 @@ public void testAlterTable() { .withNullable(col_2.nullable()) .build()); columns.add(col_1); + JdbcColumn col_3 = + new JdbcColumn.Builder() + .withName("col_3") + .withType(VARCHAR) + .withNullable(true) + .withComment("txt3") + .build(); columns.add( new JdbcColumn.Builder().withName("col_3").withType(VARCHAR).withComment("txt3").build()); - // properties.put("ROW_FORMAT", "DYNAMIC"); assertionsTableInfo(tableName, newComment, columns, properties, load); TABLE_OPERATIONS.alterTable( TEST_DB_NAME, tableName, - TableChange.updateColumnPosition(new String[] {columns.get(0).name()}, null)); + TableChange.updateColumnPosition(new String[] {columns.get(0).name()}, null), + TableChange.updateColumnNullability(new String[] {col_3.name()}, !col_3.nullable())); load = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); - columns.add(columns.remove(0)); + col_2 = columns.remove(0); + columns.clear(); + + columns.add(col_1); + columns.add( + new JdbcColumn.Builder() + .withName("col_3") + .withType(VARCHAR) + .withNullable(false) + .withComment("txt3") + .build()); + columns.add(col_2); + assertionsTableInfo(tableName, newComment, columns, properties, load); } diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java index e675d8b94e0..9642fb1d8fd 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java @@ -151,6 +151,32 @@ public void testOperationTable() { alterColumns.add(newColumn); assertionsTableInfo(newName, tableComment, alterColumns, properties, load); + // alter column Nullability + TABLE_OPERATIONS.alterTable( + TEST_DB_NAME, + newName, + TableChange.updateColumnNullability(new String[] {"col_1_new"}, false), + TableChange.updateColumnNullability(new String[] {"col_2"}, true)); + load = TABLE_OPERATIONS.load(TEST_DB_NAME, newName); + alterColumns.clear(); + alterColumns.add( + new JdbcColumn.Builder() + .withName("col_1_new") + .withType(VARCHAR) + .withComment("test_new_comment") + .withNullable(false) + .build()); + alterColumns.add( + new JdbcColumn.Builder() + .withName("col_2") + .withType(Types.DoubleType.get()) + .withNullable(true) + .withComment("set test key") + .build()); + alterColumns.add(columns.get(3)); + alterColumns.add(newColumn); + assertionsTableInfo(newName, tableComment, alterColumns, properties, load); + // delete column TABLE_OPERATIONS.alterTable( TEST_DB_NAME, newName, TableChange.deleteColumn(new String[] {newColumn.name()}, true)); From b66b9cbfe23636587a8618f36a8624c330e19172 Mon Sep 17 00:00:00 2001 From: Clearvive Date: Wed, 6 Dec 2023 17:02:21 +0800 Subject: [PATCH 2/2] [895#] feature(jdbc-catalog): Jdbc catalog support update column nullability. --- .../gravitino/catalog/mysql/operation/MysqlTableOperations.java | 2 +- .../catalog/postgresql/operation/PostgreSqlTableOperations.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java index 4f17ee0d49d..ba30438ea43 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java @@ -328,7 +328,7 @@ protected String generateAlterTableSql( updateColumnNullabilityDefinition( (TableChange.UpdateColumnNullability) change, lazyLoadCreateTable)); } else { - throw new UnsupportedOperationException( + throw new IllegalArgumentException( "Unsupported table change type: " + change.getClass().getName()); } } diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java index cb09bb8c066..192a45b51e1 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java @@ -348,7 +348,7 @@ protected String generateAlterTableSql( updateColumnNullabilityDefinition( (TableChange.UpdateColumnNullability) change, tableName)); } else { - throw new UnsupportedOperationException( + throw new IllegalArgumentException( "Unsupported table change type: " + change.getClass().getName()); } }