Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 4fb39c1

Browse files
shaomengwangshaomengwang
committed
Merge the dev branch.
1 parent edb543e commit 4fb39c1

573 files changed

Lines changed: 65554 additions & 2883 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎connectors/connector-datahub/src/main/java/com/alibaba/alink/common/io/catalog/datahub/DataHubCatalog.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,9 @@
3636
import com.alibaba.alink.common.io.catalog.datahub.datastream.sink.DatahubPublicSinkFunction;
3737
import com.alibaba.alink.common.io.catalog.datahub.datastream.source.DatahubPublicSourceFunction;
3838
import com.alibaba.alink.common.io.catalog.datahub.datastream.util.DatahubClientProvider;
39-
import com.alibaba.alink.params.io.DataHubParams;
4039
import com.alibaba.alink.params.io.shared.HasCatalogName;
4140
import com.alibaba.alink.params.io.shared.HasDefaultDatabase;
41+
import com.alibaba.alink.params.io.DataHubParams;
4242
import com.aliyun.datahub.client.exception.ResourceNotFoundException;
4343
import com.aliyun.datahub.client.model.Field;
4444
import com.aliyun.datahub.client.model.GetProjectResult;
@@ -308,15 +308,15 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig
308308

309309
String comment = table.getComment();
310310

311-
String sharedCountStr = table.getOptions().get("shareCount");
311+
String sharedCountStr = table.getProperties().get("shareCount");
312312

313313
if (sharedCountStr == null) {
314314
sharedCountStr = "1";
315315
}
316316

317317
int sharedCount = Integer.parseInt(sharedCountStr);
318318

319-
String lifeCycleStr = table.getOptions().get("lifeCycle");
319+
String lifeCycleStr = table.getProperties().get("lifeCycle");
320320

321321
if (lifeCycleStr == null) {
322322
lifeCycleStr = "3";

‎connectors/connector-datahub/src/main/java/com/alibaba/alink/common/io/catalog/datahub/descriptors/DataHubCatalogValidator.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
package com.alibaba.alink.common.io.catalog.datahub.descriptors;
2+
3+
import org.apache.flink.table.descriptors.CatalogDescriptorValidator;
4+
import org.apache.flink.table.descriptors.DescriptorProperties;
5+
6+
public class DataHubCatalogValidator extends CatalogDescriptorValidator {
7+
public static final String CATALOG_TYPE_VALUE_DATAHUB = "datahub";
8+
public static final String CATALOG_DATAHUB_ACCESS_ID = "accessId";
9+
public static final String CATALOG_DATAHUB_ACCESS_KEY = "accessKey";
10+
public static final String CATALOG_DATAHUB_PROJECT = "project";
11+
public static final String CATALOG_DATAHUB_ENDPOINT = "endpoint";
12+
13+
@Override
14+
public void validate(DescriptorProperties properties) {
15+
super.validate(properties);
16+
properties.validateValue(CATALOG_TYPE, CATALOG_TYPE_VALUE_DATAHUB, false);
17+
properties.validateString(CATALOG_DATAHUB_ACCESS_ID, false, 1);
18+
properties.validateString(CATALOG_DATAHUB_ACCESS_KEY, false, 1);
19+
properties.validateString(CATALOG_DATAHUB_PROJECT, false, 1);
20+
properties.validateString(CATALOG_DATAHUB_ENDPOINT, false, 1);
21+
}
22+
}

‎connectors/connector-datahub/src/main/java/com/alibaba/alink/common/io/catalog/datahub/factories/DataHubCatalogFactory.java‎

Lines changed: 48 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,72 @@
11
package com.alibaba.alink.common.io.catalog.datahub.factories;
22

3-
import org.apache.flink.configuration.ConfigOption;
43
import org.apache.flink.table.catalog.Catalog;
4+
import org.apache.flink.table.descriptors.CatalogDescriptorValidator;
5+
import org.apache.flink.table.descriptors.DescriptorProperties;
56
import org.apache.flink.table.factories.CatalogFactory;
6-
import org.apache.flink.table.factories.FactoryUtil;
77

88
import com.alibaba.alink.common.io.catalog.datahub.DataHubCatalog;
9+
import com.alibaba.alink.common.io.catalog.datahub.descriptors.DataHubCatalogValidator;
910

10-
import java.util.HashSet;
11-
import java.util.Set;
11+
import java.util.ArrayList;
12+
import java.util.HashMap;
13+
import java.util.List;
14+
import java.util.Map;
1215

1316
public class DataHubCatalogFactory implements CatalogFactory {
1417

1518
@Override
16-
public Catalog createCatalog(Context context) {
19+
public Catalog createCatalog(String name, Map <String, String> properties) {
1720

18-
final FactoryUtil.CatalogFactoryHelper helper =
19-
FactoryUtil.createCatalogFactoryHelper(this, context);
20-
helper.validate();
21+
final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
2122

22-
return new DataHubCatalog(
23-
helper.getOptions().get(DataHubCatalogFactoryOptions.ACCESS_ID),
24-
helper.getOptions().get(DataHubCatalogFactoryOptions.ACCESS_KEY),
25-
helper.getOptions().get(DataHubCatalogFactoryOptions.PROJECT),
26-
helper.getOptions().get(DataHubCatalogFactoryOptions.ENDPOINT)
23+
final String accessId = descriptorProperties.getString(
24+
DataHubCatalogValidator.CATALOG_DATAHUB_ACCESS_ID
2725
);
26+
27+
final String accessKey = descriptorProperties.getString(
28+
DataHubCatalogValidator.CATALOG_DATAHUB_ACCESS_KEY
29+
);
30+
31+
final String project = descriptorProperties.getString(
32+
DataHubCatalogValidator.CATALOG_DATAHUB_PROJECT
33+
);
34+
35+
final String endPoint = descriptorProperties.getString(
36+
DataHubCatalogValidator.CATALOG_DATAHUB_ENDPOINT);
37+
38+
return new DataHubCatalog(accessId, accessKey, project, endPoint);
2839
}
2940

3041
@Override
31-
public String factoryIdentifier() {
32-
return DataHubCatalogFactoryOptions.IDENTIFIER;
42+
public Map <String, String> requiredContext() {
43+
Map <String, String> context = new HashMap <>();
44+
context.put(CatalogDescriptorValidator.CATALOG_TYPE, DataHubCatalogValidator.CATALOG_TYPE_VALUE_DATAHUB);
45+
context.put(CatalogDescriptorValidator.CATALOG_PROPERTY_VERSION, "1");
46+
return context;
3347
}
3448

3549
@Override
36-
public Set <ConfigOption <?>> requiredOptions() {
37-
final Set <ConfigOption <?>> options = new HashSet <>();
38-
options.add(DataHubCatalogFactoryOptions.ACCESS_ID);
39-
options.add(DataHubCatalogFactoryOptions.ACCESS_KEY);
40-
options.add(DataHubCatalogFactoryOptions.PROJECT);
41-
options.add(DataHubCatalogFactoryOptions.ENDPOINT);
42-
return options;
50+
public List <String> supportedProperties() {
51+
List <String> properties = new ArrayList <>();
52+
53+
properties.add(DataHubCatalogValidator.CATALOG_DATAHUB_ACCESS_ID);
54+
55+
properties.add(DataHubCatalogValidator.CATALOG_DATAHUB_ACCESS_KEY);
56+
57+
properties.add(DataHubCatalogValidator.CATALOG_DATAHUB_PROJECT);
58+
59+
properties.add(DataHubCatalogValidator.CATALOG_DATAHUB_ENDPOINT);
60+
61+
return properties;
4362
}
4463

45-
@Override
46-
public Set <ConfigOption <?>> optionalOptions() {
47-
final Set <ConfigOption <?>> options = new HashSet <>();
48-
options.add(DataHubCatalogFactoryOptions.DEFAULT_DATABASE);
49-
return options;
64+
private static DescriptorProperties getValidatedProperties(Map <String, String> properties) {
65+
final DescriptorProperties descriptorProperties = new DescriptorProperties(false);
66+
descriptorProperties.putProperties(properties);
67+
68+
new DataHubCatalogValidator().validate(descriptorProperties);
69+
70+
return descriptorProperties;
5071
}
5172
}

‎connectors/connector-datahub/src/main/java/com/alibaba/alink/common/io/catalog/datahub/factories/DataHubCatalogFactoryOptions.java‎

Lines changed: 0 additions & 30 deletions
This file was deleted.

connectors/connector-datahub/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory renamed to connectors/connector-datahub/src/main/resources/META-INF/services/org.apache.flink.table.factories.TableFactory

File renamed without changes.

‎connectors/connector-hive/hive-bridge/pom.xml‎

Lines changed: 39 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,12 @@
2020
</properties>
2121

2222
<dependencies>
23+
24+
<!-- core dependencies -->
2325
<dependency>
2426
<groupId>org.apache.flink</groupId>
25-
<artifactId>flink-shaded-hadoop-2-uber</artifactId>
26-
<version>${hivemetastore.hadoop.version}-${flink.shaded.version}</version>
27+
<artifactId>flink-connector-hive_${alink.scala.major.version}</artifactId>
28+
<version>${flink.version}</version>
2729
<exclusions>
2830
<exclusion>
2931
<groupId>log4j</groupId>
@@ -36,52 +38,76 @@
3638
</exclusions>
3739
<scope>provided</scope>
3840
</dependency>
41+
3942
<dependency>
40-
<groupId>org.apache.commons</groupId>
41-
<artifactId>commons-lang3</artifactId>
42-
<version>3.4</version>
43+
<groupId>org.apache.flink</groupId>
44+
<artifactId>flink-table-common</artifactId>
45+
<version>${flink.version}</version>
4346
<scope>provided</scope>
4447
</dependency>
48+
4549
<dependency>
4650
<groupId>org.apache.flink</groupId>
47-
<artifactId>flink-runtime_${alink.scala.major.version}</artifactId>
51+
<artifactId>flink-hadoop-compatibility_${alink.scala.major.version}</artifactId>
4852
<version>${flink.version}</version>
4953
<scope>provided</scope>
5054
</dependency>
55+
5156
<dependency>
5257
<groupId>org.apache.flink</groupId>
53-
<artifactId>flink-core</artifactId>
58+
<artifactId>flink-table-api-java-bridge_${alink.scala.major.version}</artifactId>
5459
<version>${flink.version}</version>
5560
<scope>provided</scope>
5661
</dependency>
62+
5763
<dependency>
5864
<groupId>org.apache.flink</groupId>
59-
<artifactId>flink-table-api-java</artifactId>
65+
<artifactId>flink-table-runtime-blink_${alink.scala.major.version}</artifactId>
6066
<version>${flink.version}</version>
6167
<scope>provided</scope>
6268
</dependency>
69+
6370
<dependency>
6471
<groupId>org.apache.flink</groupId>
65-
<artifactId>flink-table-common</artifactId>
72+
<artifactId>flink-table-planner-blink_${alink.scala.major.version}</artifactId>
6673
<version>${flink.version}</version>
6774
<scope>provided</scope>
6875
</dependency>
76+
77+
<!-- format dependencies -->
78+
6979
<dependency>
7080
<groupId>org.apache.flink</groupId>
71-
<artifactId>flink-connector-hive_${alink.scala.major.version}</artifactId>
81+
<artifactId>flink-orc_${alink.scala.major.version}</artifactId>
7282
<version>${flink.version}</version>
7383
<scope>provided</scope>
84+
<exclusions>
85+
<exclusion>
86+
<groupId>org.apache.orc</groupId>
87+
<artifactId>orc-core</artifactId>
88+
</exclusion>
89+
</exclusions>
7490
</dependency>
91+
7592
<dependency>
7693
<groupId>org.apache.flink</groupId>
77-
<artifactId>flink-table-runtime-blink_${alink.scala.major.version}</artifactId>
94+
<artifactId>flink-hadoop-fs</artifactId>
7895
<version>${flink.version}</version>
7996
<scope>provided</scope>
8097
</dependency>
98+
99+
<!-- Hadoop dependency -->
100+
<!-- Hadoop as provided dependencies, so we can depend on them without pulling in Hadoop -->
101+
102+
<!--
103+
Hive 2.3.4 relies on Hadoop 2.7.2 and later versions.
104+
For Hadoop 2.7, the minor Hadoop version supported for flink-shaded-hadoop-2-uber is 2.7.5,
105+
thus override the default hadoop version from 2.4.1 to 2.7.5
106+
-->
81107
<dependency>
82108
<groupId>org.apache.flink</groupId>
83-
<artifactId>flink-parquet_${alink.scala.major.version}</artifactId>
84-
<version>${flink.version}</version>
109+
<artifactId>flink-shaded-hadoop-2-uber</artifactId>
110+
<version>${hivemetastore.hadoop.version}-${flink.shaded.version}</version>
85111
<scope>provided</scope>
86112
</dependency>
87113

@@ -235,14 +261,6 @@
235261
<groupId>org.apache.parquet</groupId>
236262
<artifactId>parquet-hadoop-bundle</artifactId>
237263
</exclusion>
238-
<exclusion>
239-
<groupId>org.slf4j</groupId>
240-
<artifactId>slf4j-log4j12</artifactId>
241-
</exclusion>
242-
<exclusion>
243-
<groupId>org.apache.derby</groupId>
244-
<artifactId>derby</artifactId>
245-
</exclusion>
246264
</exclusions>
247265
</dependency>
248266

@@ -359,12 +377,6 @@
359377
</exclusions>
360378
</dependency>
361379

362-
<dependency>
363-
<groupId>org.slf4j</groupId>
364-
<artifactId>slf4j-api</artifactId>
365-
<version>1.7.21</version>
366-
<scope>provided</scope>
367-
</dependency>
368380
</dependencies>
369381

370382
<build>

0 commit comments

Comments
 (0)