From 2dbf234664f24b6654d1c3bfe00ea98badd17220 Mon Sep 17 00:00:00 2001 From: Ajay Date: Wed, 5 Dec 2018 16:36:44 -0500 Subject: [PATCH 1/3] Fix #3924 ExtractJobConfiguration's setProjectId makes cross-project BQ extracts impossible --- .../google/cloud/bigquery/CopyJobConfiguration.java | 10 ++++++++-- .../google/cloud/bigquery/ExtractJobConfiguration.java | 6 +++++- .../google/cloud/bigquery/LoadJobConfiguration.java | 6 +++++- .../google/cloud/bigquery/QueryJobConfiguration.java | 4 +++- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/CopyJobConfiguration.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/CopyJobConfiguration.java index 3199a29918e5..7403a66131e9 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/CopyJobConfiguration.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/CopyJobConfiguration.java @@ -21,6 +21,7 @@ import com.google.api.services.bigquery.model.JobConfigurationTableCopy; import com.google.common.base.Function; import com.google.common.base.MoreObjects.ToStringHelper; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import java.util.List; @@ -216,10 +217,15 @@ CopyJobConfiguration setProjectId(final String projectId) { new Function() { @Override public TableId apply(TableId tableId) { - return tableId.setProjectId(projectId); + if (Strings.isNullOrEmpty(tableId.getProject())) { + return tableId.setProjectId(projectId); + } + return tableId; } })); - builder.setDestinationTable(getDestinationTable().setProjectId(projectId)); + if (Strings.isNullOrEmpty(getDestinationTable().getProject())) { + builder.setDestinationTable(getDestinationTable().setProjectId(projectId)); + } return builder.build(); } diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExtractJobConfiguration.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExtractJobConfiguration.java index d1ed9b994eaf..bcacbc211fb7 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExtractJobConfiguration.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExtractJobConfiguration.java @@ -20,6 +20,7 @@ import com.google.api.services.bigquery.model.JobConfigurationExtract; import com.google.common.base.MoreObjects.ToStringHelper; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import java.util.List; import java.util.Objects; @@ -215,7 +216,10 @@ public int hashCode() { @Override ExtractJobConfiguration setProjectId(String projectId) { - return toBuilder().setSourceTable(getSourceTable().setProjectId(projectId)).build(); + if (Strings.isNullOrEmpty(getSourceTable().getProject())) { + return toBuilder().setSourceTable(getSourceTable().setProjectId(projectId)).build(); + } + return this; } @Override diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java index 5cfad8dad38e..d11f17081e2e 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java @@ -20,6 +20,7 @@ import com.google.api.services.bigquery.model.JobConfigurationLoad; import com.google.common.base.MoreObjects.ToStringHelper; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.primitives.Ints; import java.util.List; @@ -398,7 +399,10 @@ public int hashCode() { @Override LoadJobConfiguration setProjectId(String projectId) { - return toBuilder().setDestinationTable(getDestinationTable().setProjectId(projectId)).build(); + if (Strings.isNullOrEmpty(getDestinationTable().getProject())) { + return toBuilder().setDestinationTable(getDestinationTable().setProjectId(projectId)).build(); + } + return this; } @Override diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java index af91ba064339..a5e741d7f8f3 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java @@ -26,6 +26,7 @@ import com.google.cloud.bigquery.JobInfo.WriteDisposition; import com.google.common.base.Function; import com.google.common.base.MoreObjects.ToStringHelper; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; @@ -769,7 +770,8 @@ public int hashCode() { @Override QueryJobConfiguration setProjectId(String projectId) { Builder builder = toBuilder(); - if (getDestinationTable() != null) { + if (getDestinationTable() != null + && Strings.isNullOrEmpty(getDestinationTable().getProject())) { builder.setDestinationTable(getDestinationTable().setProjectId(projectId)); } if (getDefaultDataset() != null) { From eb3319156193558b8d8130ff16bbb04664d7f902 Mon Sep 17 00:00:00 2001 From: Ajay Date: Wed, 5 Dec 2018 16:58:16 -0500 Subject: [PATCH 2/3] add test case --- .../bigquery/CopyJobConfigurationTest.java | 32 +++++++++++++++++-- .../bigquery/ExtractJobConfigurationTest.java | 16 ++++++++-- .../bigquery/LoadJobConfigurationTest.java | 16 ++++++++-- .../bigquery/QueryJobConfigurationTest.java | 19 +++++++++-- 4 files changed, 73 insertions(+), 10 deletions(-) diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/CopyJobConfigurationTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/CopyJobConfigurationTest.java index 6e155b0f33d0..87d3d27f08f7 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/CopyJobConfigurationTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/CopyJobConfigurationTest.java @@ -22,12 +22,15 @@ import com.google.cloud.bigquery.JobInfo.CreateDisposition; import com.google.cloud.bigquery.JobInfo.WriteDisposition; +import com.google.common.base.Function; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import java.util.List; import org.junit.Test; public class CopyJobConfigurationTest { + private static final String TEST_PROJECT_ID = "test-project-id"; private static final TableId SOURCE_TABLE = TableId.of("dataset", "sourceTable"); private static final List SOURCE_TABLES = ImmutableList.of( @@ -115,10 +118,33 @@ public void testToPbAndFromPb() { @Test public void testSetProjectId() { - CopyJobConfiguration configuration = COPY_JOB_CONFIGURATION_MULTIPLE_TABLES.setProjectId("p"); - assertEquals("p", configuration.getDestinationTable().getProject()); + CopyJobConfiguration configuration = COPY_JOB_CONFIGURATION_MULTIPLE_TABLES.setProjectId(TEST_PROJECT_ID); + assertEquals(TEST_PROJECT_ID, configuration.getDestinationTable().getProject()); for (TableId sourceTable : configuration.getSourceTables()) { - assertEquals("p", sourceTable.getProject()); + assertEquals(TEST_PROJECT_ID, sourceTable.getProject()); + } + } + + @Test + public void testSetProjectIdDoNotOverride() { + CopyJobConfiguration configuration = + COPY_JOB_CONFIGURATION_MULTIPLE_TABLES + .toBuilder() + .setSourceTables( + Lists.transform( + SOURCE_TABLES, + new Function() { + @Override + public TableId apply(TableId tableId) { + return tableId.setProjectId(TEST_PROJECT_ID); + } + })) + .setDestinationTable(DESTINATION_TABLE.setProjectId(TEST_PROJECT_ID)) + .build() + .setProjectId("do-not-update"); + assertEquals(TEST_PROJECT_ID, configuration.getDestinationTable().getProject()); + for (TableId sourceTable : configuration.getSourceTables()) { + assertEquals(TEST_PROJECT_ID, sourceTable.getProject()); } } diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExtractJobConfigurationTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExtractJobConfigurationTest.java index 3ac686ac14b8..d621a82985fb 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExtractJobConfigurationTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExtractJobConfigurationTest.java @@ -26,6 +26,7 @@ public class ExtractJobConfigurationTest { + private static final String TEST_PROJECT_ID = "test-project-id"; private static final List DESTINATION_URIS = ImmutableList.of("uri1", "uri2"); private static final String DESTINATION_URI = "uri1"; private static final TableId TABLE_ID = TableId.of("dataset", "table"); @@ -118,8 +119,19 @@ public void testToPbAndFromPb() { @Test public void testSetProjectId() { - ExtractJobConfiguration configuration = EXTRACT_CONFIGURATION.setProjectId("p"); - assertEquals("p", configuration.getSourceTable().getProject()); + ExtractJobConfiguration configuration = EXTRACT_CONFIGURATION.setProjectId(TEST_PROJECT_ID); + assertEquals(TEST_PROJECT_ID, configuration.getSourceTable().getProject()); + } + + @Test + public void testSetProjectIdDoNotOverride() { + ExtractJobConfiguration configuration = + EXTRACT_CONFIGURATION + .toBuilder() + .setSourceTable(TABLE_ID.setProjectId(TEST_PROJECT_ID)) + .build() + .setProjectId("do-not-update"); + assertEquals(TEST_PROJECT_ID, configuration.getSourceTable().getProject()); } @Test diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java index d143ce85d794..85184d88e069 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java @@ -29,6 +29,7 @@ public class LoadJobConfigurationTest { + private static final String TEST_PROJECT_ID = "test-project-id"; private static final CsvOptions CSV_OPTIONS = CsvOptions.newBuilder() .setAllowJaggedRows(true) @@ -146,8 +147,19 @@ public void testToPbAndFromPb() { @Test public void testSetProjectId() { - LoadConfiguration configuration = LOAD_CONFIGURATION_CSV.setProjectId("p"); - assertEquals("p", configuration.getDestinationTable().getProject()); + LoadConfiguration configuration = LOAD_CONFIGURATION_CSV.setProjectId(TEST_PROJECT_ID); + assertEquals(TEST_PROJECT_ID, configuration.getDestinationTable().getProject()); + } + + @Test + public void testSetProjectIdDoNotOverride() { + LoadConfiguration configuration = + LOAD_CONFIGURATION_CSV + .toBuilder() + .setDestinationTable(TABLE_ID.setProjectId(TEST_PROJECT_ID)) + .build() + .setProjectId("do-not-update"); + assertEquals(TEST_PROJECT_ID, configuration.getDestinationTable().getProject()); } @Test diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java index 2ecc0c800e94..dee9c7ebcf65 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java @@ -33,6 +33,7 @@ public class QueryJobConfigurationTest { + private static final String TEST_PROJECT_ID = "test-project-id"; private static final String QUERY = "BigQuery SQL"; private static final DatasetId DATASET_ID = DatasetId.of("dataset"); private static final TableId TABLE_ID = TableId.of("dataset", "table"); @@ -140,9 +141,21 @@ public void testToPbAndFromPb() { @Test public void testSetProjectId() { - QueryJobConfiguration configuration = QUERY_JOB_CONFIGURATION.setProjectId("p"); - assertEquals("p", configuration.getDefaultDataset().getProject()); - assertEquals("p", configuration.getDestinationTable().getProject()); + QueryJobConfiguration configuration = QUERY_JOB_CONFIGURATION.setProjectId(TEST_PROJECT_ID); + assertEquals(TEST_PROJECT_ID, configuration.getDefaultDataset().getProject()); + assertEquals(TEST_PROJECT_ID, configuration.getDestinationTable().getProject()); + } + + @Test + public void testSetProjectIdDoNotOverride() { + QueryJobConfiguration configuration = + QUERY_JOB_CONFIGURATION + .toBuilder() + .setDestinationTable(TABLE_ID.setProjectId(TEST_PROJECT_ID)) + .build() + .setProjectId("update-only-on-dataset"); + assertEquals("update-only-on-dataset", configuration.getDefaultDataset().getProject()); + assertEquals(TEST_PROJECT_ID, configuration.getDestinationTable().getProject()); } @Test From d7575b51a22152818ccb450930b60df41a981765 Mon Sep 17 00:00:00 2001 From: Ajay Date: Wed, 5 Dec 2018 20:38:04 -0500 Subject: [PATCH 3/3] fix formatting --- .../com/google/cloud/bigquery/CopyJobConfigurationTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/CopyJobConfigurationTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/CopyJobConfigurationTest.java index 87d3d27f08f7..73a29936de77 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/CopyJobConfigurationTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/CopyJobConfigurationTest.java @@ -118,7 +118,8 @@ public void testToPbAndFromPb() { @Test public void testSetProjectId() { - CopyJobConfiguration configuration = COPY_JOB_CONFIGURATION_MULTIPLE_TABLES.setProjectId(TEST_PROJECT_ID); + CopyJobConfiguration configuration = + COPY_JOB_CONFIGURATION_MULTIPLE_TABLES.setProjectId(TEST_PROJECT_ID); assertEquals(TEST_PROJECT_ID, configuration.getDestinationTable().getProject()); for (TableId sourceTable : configuration.getSourceTables()) { assertEquals(TEST_PROJECT_ID, sourceTable.getProject());