2

apache · morningman · Dec 1, 2023 · Nov 17, 2023 · Nov 21, 2023 · Nov 21, 2023
commit 87daa756542758c421f0afe7fcdaffc72582ea1b
diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp
@@ -747,9 +747,8 @@ Status VFileScanner::_get_next_reader() {
         case TFileFormatType::FORMAT_JNI: {
             if (range.__isset.table_format_params &&
                 range.table_format_params.table_format_type == "max_compute") {
-                const MaxComputeTableDescriptor* mc_desc =
-                        static_cast<const MaxComputeTableDescriptor*>(
-                                _real_tuple_desc->table_desc());
+                const auto* mc_desc = static_cast<const MaxComputeTableDescriptor*>(
+                        _real_tuple_desc->table_desc());
                 std::unique_ptr<MaxComputeJniReader> mc_reader = MaxComputeJniReader::create_unique(
                         mc_desc, range.table_format_params.max_compute_params, _file_slot_descs,
                         range, _state, _profile);

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/MaxComputeExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/MaxComputeExternalTable.java
@@ -47,7 +47,7 @@
 import com.google.common.collect.Lists;
 
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -159,7 +159,8 @@ private void initTablePartitions() {
         } else {
             partitionSpecs = ImmutableList.of();
         }
-        partitionNameToColumns = new HashMap<>();
+        // sort partition columns to align partitionTypes and partitionName.
+        partitionNameToColumns = new LinkedHashMap<>();
         for (com.aliyun.odps.Column partColumn : partitionColumns) {
             Column dorisCol = new Column(partColumn.getName(),
                     mcTypeToDorisType(partColumn.getTypeInfo()), true, null,

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/MaxComputeExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/MaxComputeExternalCatalog.java
@@ -24,6 +24,7 @@
 
 import com.aliyun.odps.Odps;
 import com.aliyun.odps.OdpsException;
+import com.aliyun.odps.Partition;
 import com.aliyun.odps.PartitionSpec;
 import com.aliyun.odps.account.Account;
 import com.aliyun.odps.account.AliyunAccount;
@@ -34,6 +35,7 @@
 import com.google.gson.annotations.SerializedName;
 
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -144,10 +146,32 @@ public boolean tableExist(SessionContext ctx, String dbName, String tblName) {
     }
 
     public List<String> listPartitionNames(String dbName, String tbl) {
+        return listPartitionNames(dbName, tbl, 0, -1);
+    }
+
+    public List<String> listPartitionNames(String dbName, String tbl, long skip, long limit) {
         try {
             if (getClient().projects().exists(dbName)) {
-                return getClient().tables().get(tbl).getPartitions().stream()
-                        .map(p -> p.getPartitionSpec().toString(false, true))
+                List<Partition> parts;
+                if (limit < 0) {
+                    parts = getClient().tables().get(tbl).getPartitions();
+                } else {
+                    skip = skip < 0 ? 0 : skip;
+                    parts = new ArrayList<>();
+                    Iterator<Partition> it = getClient().tables().get(tbl).getPartitionIterator();
+                    int count = 0;
+                    while (it.hasNext()) {
+                        if (count < skip) {
+                            count++;
+                            it.next();
+                        } else if (parts.size() >= limit) {
+                            break;
+                        } else {
+                            parts.add(it.next());
+                        }
+                    }
+                }
+                return parts.stream().map(p -> p.getPartitionSpec().toString(false, true))
                         .collect(Collectors.toList());
             } else {
                 throw new OdpsException("Max compute project: " + dbName + " not exists.");

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java
@@ -27,7 +27,6 @@
 import com.amazonaws.glue.catalog.metastore.AWSCatalogMetastoreClient;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
-import org.apache.hadoop.hive.common.ValidReadTxnList;
 import org.apache.hadoop.hive.common.ValidReaderWriteIdList;
 import org.apache.hadoop.hive.common.ValidTxnList;
 import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
@@ -126,9 +125,13 @@ public boolean tableExists(String dbName, String tblName) {
     }
 
     public List<String> listPartitionNames(String dbName, String tblName) {
+        return listPartitionNames(dbName, tblName, MAX_LIST_PARTITION_NUM);
+    }
+
+    public List<String> listPartitionNames(String dbName, String tblName, short max) {
         try (CachedClient client = getClient()) {
             try {
-                return client.client.listPartitionNames(dbName, tblName, MAX_LIST_PARTITION_NUM);
+                return client.client.listPartitionNames(dbName, tblName, max);
             } catch (Exception e) {
                 client.setThrowable(e);
                 throw e;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -25,6 +25,7 @@
 import org.apache.doris.analysis.AdminShowTabletStorageFormatStmt;
 import org.apache.doris.analysis.DescribeStmt;
 import org.apache.doris.analysis.HelpStmt;
+import org.apache.doris.analysis.LimitElement;
 import org.apache.doris.analysis.PartitionNames;
 import org.apache.doris.analysis.ShowAlterStmt;
 import org.apache.doris.analysis.ShowAnalyzeStmt;
@@ -1659,31 +1660,48 @@ private void handleShowPartitions() throws AnalysisException {
                     showStmt.getOrderByPairs(), showStmt.getLimitElement()).getRows();
             resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
         } else if (showStmt.getCatalog() instanceof MaxComputeExternalCatalog) {
-            MaxComputeExternalCatalog catalog = (MaxComputeExternalCatalog) (showStmt.getCatalog());
-            List<List<String>> rows = new ArrayList<>();
-            String dbName = ClusterNamespace.getNameFromFullName(showStmt.getTableName().getDb());
-            List<String> partitionNames = catalog.listPartitionNames(dbName,
-                    showStmt.getTableName().getTbl());
-            for (String partition : partitionNames) {
-                List<String> list = new ArrayList<>();
-                list.add(partition);
-                rows.add(list);
-            }
-            // sort by partition name
-            rows.sort(Comparator.comparing(x -> x.get(0)));
-            resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
+            handleShowMaxComputeTablePartitions(showStmt);
         } else {
             handleShowHMSTablePartitions(showStmt);
         }
     }
 
+    private void handleShowMaxComputeTablePartitions(ShowPartitionsStmt showStmt) {
+        MaxComputeExternalCatalog catalog = (MaxComputeExternalCatalog) (showStmt.getCatalog());
+        List<List<String>> rows = new ArrayList<>();
+        String dbName = ClusterNamespace.getNameFromFullName(showStmt.getTableName().getDb());
+        List<String> partitionNames;
+        LimitElement limit = showStmt.getLimitElement();
+        if (limit != null && limit.hasLimit()) {
+            partitionNames = catalog.listPartitionNames(dbName,
+                    showStmt.getTableName().getTbl(), limit.getOffset(), limit.getLimit());
+        } else {
+            partitionNames = catalog.listPartitionNames(dbName, showStmt.getTableName().getTbl());
+        }
+        for (String partition : partitionNames) {
+            List<String> list = new ArrayList<>();
+            list.add(partition);
+            rows.add(list);
+        }
+        // sort by partition name
+        rows.sort(Comparator.comparing(x -> x.get(0)));
+        resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
+    }
+
     private void handleShowHMSTablePartitions(ShowPartitionsStmt showStmt) {
         HMSExternalCatalog catalog = (HMSExternalCatalog) (showStmt.getCatalog());
         List<List<String>> rows = new ArrayList<>();
         String dbName = ClusterNamespace.getNameFromFullName(showStmt.getTableName().getDb());
 
-        List<String> partitionNames = catalog.getClient().listPartitionNames(dbName,
-                showStmt.getTableName().getTbl());
+        List<String> partitionNames;
+        LimitElement limit = showStmt.getLimitElement();
+        if (limit != null && limit.hasLimit()) {
+            // only short limit is valid on Hive
+            short limited = (short) limit.getLimit();
+            partitionNames = catalog.getClient().listPartitionNames(dbName, showStmt.getTableName().getTbl(), limited);
+        } else {
+            partitionNames = catalog.getClient().listPartitionNames(dbName, showStmt.getTableName().getTbl());
+        }
         for (String partition : partitionNames) {
             List<String> list = new ArrayList<>();
             list.add(partition);

diff --git a/regression-test/data/external_table_p2/maxcompute/test_external_catalog_maxcompute.out b/regression-test/data/external_table_p2/maxcompute/test_external_catalog_maxcompute.out
@@ -25,3 +25,42 @@ true	77	8920	182239402452
 
 -- !replay_q6 --
 9601	qewtoll	2020-09-21
+
+-- !multi_partition_q1 --
+pt=11/yy=2021/mm=12/dd=21
+pt=11/yy=2021/mm=12/dd=22
+pt=12/yy=2021/mm=12/dd=21
+pt=12/yy=2021/mm=12/dd=22
+pt=13/yy=2021/mm=12/dd=21
+pt=13/yy=2021/mm=12/dd=22
+pt=14/yy=2021/mm=12/dd=21
+pt=14/yy=2021/mm=12/dd=22
+pt=14/yy=2021/mm=12/dd=23
+pt=14/yy=2022/mm=01/dd=01
+
+-- !multi_partition_q2 --
+17	2022-04-23 19:12:30	2021	12	22
+17	2022-04-23 19:12:30	2021	12	21
+16	2022-04-23 19:12:30	2021	12	22
+
+-- !multi_partition_q3 --
+14	2022-04-23 19:12:30	2022	01	01
+14	2022-04-23 19:12:30	2022	01	02
+98	2022-04-23 19:12:30	2021	12	21
+
+-- !multi_partition_q4 --
+17
+
+-- !multi_partition_q5 --
+2022-04-23 19:12:30	2021	12	21
+2022-04-23 19:12:30	2021	12	21
+2022-04-23 19:12:30	2021	12	21
+
+-- !multi_partition_q6 --
+17	2021	12
+
+-- !multi_partition_q7 --
+15
+
+-- !multi_partition_q8 --
+11
diff --git a/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy b/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy
@@ -73,5 +73,19 @@ suite("test_external_catalog_maxcompute", "p2,external,maxcompute,external_remot
         sql """ switch `${mc_catalog_name}`; """
         sql """ use `${mc_db}`; """
         qt_replay_q6 """ select * from mc_parts where dt = '2020-09-21' and mc_bigint > 6223 """
+
+        // test multi partitions prune
+        sql """ refresh catalog ${mc_catalog_name} """
+        sql """ switch `${mc_catalog_name}`; """
+        sql """ use `${mc_db}`; """
+        qt_multi_partition_q1 """ show partitions from multi_partitions limit 10; """
+        qt_multi_partition_q2 """ select pt, create_time, yy, mm, dd from multi_partitions where pt>-1 and yy > '' and mm > '' and dd >'' order by pt desc limit 3; """
+        qt_multi_partition_q3 """ select sum(pt), create_time, yy, mm, dd from multi_partitions where yy > '' and mm > '' and dd >'' group by create_time, yy, mm, dd order by dd limit 3; """
+        qt_multi_partition_q4 """ select count(*) from multi_partitions where pt>-1 and yy > '' and mm > '' and dd <= '30'; """
+        qt_multi_partition_q5 """ select create_time, yy, mm, dd from multi_partitions where yy = '2021' and mm='12' and dd='21' order by pt limit 3; """
+        qt_multi_partition_q6 """ select max(pt), yy, mm from multi_partitions where yy = '2021' and mm='12' group by yy, mm order by yy, mm; """
+        qt_multi_partition_q7 """ select count(*) from multi_partitions where yy < '2022'; """
+        qt_multi_partition_q8 """ select count(*) from multi_partitions where pt>=14; """
+
     }
 }