From 216b6c4766ca717aa978fbeec7ac8c19955424c0 Mon Sep 17 00:00:00 2001 From: re20052 <51275902044@stu.ecnu.edu.cn> Date: Wed, 29 Jul 2026 19:04:48 +0800 Subject: [PATCH] [feature](query) support max_visible_partition() syntax --- .../org/apache/doris/catalog/OlapTable.java | 61 ++ .../org/apache/doris/catalog/TableIf.java | 4 + .../doris/nereids/StatementContext.java | 12 + .../nereids/analyzer/UnboundRelation.java | 53 +- .../nereids/parser/LogicalPlanBuilder.java | 3 +- .../nereids/rules/analysis/BindRelation.java | 24 +- ...calResultSinkToShortCircuitPointQuery.java | 3 + .../parser/NereidsParserDigestTest.java | 5 + .../org/apache/doris/nereids/DorisLexer.g4 | 1 + .../org/apache/doris/nereids/DorisParser.g4 | 8 +- .../max_visible_partition.out | 83 +++ .../max_visible_partition.groovy | 540 ++++++++++++++++++ 12 files changed, 779 insertions(+), 18 deletions(-) create mode 100644 regression-test/data/nereids_syntax_p0/max_visible_partition.out create mode 100644 regression-test/suites/nereids_syntax_p0/max_visible_partition.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index bcd9e76e7c434c..e65f4ee053dc70 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -1478,6 +1478,67 @@ public Partition dropPartitionForTruncate(long dbId, boolean isForceDrop, * */ + @Override + public Partition getMaxVisiblePartition() { + PartitionInfo partitionInfo = getPartitionInfo(); + PartitionType type = partitionInfo.getType(); + // Cloud mode: prefetch all visible versions in one batch RPC so the loop below hits + // cache instead of firing N sequential meta-service RPCs under metadata locks. + if (Config.isCloudMode()) { + try { + getVersionInBatchForCloudMode(nameToPartition.values().stream() + .map(Partition::getId).collect(Collectors.toList())); + } catch (RpcException e) { + LOG.warn("batch prefetch visible version failed, fallback to per-partition lookup", e); + } + } + if (type == PartitionType.UNPARTITIONED) { + for (Partition partition : nameToPartition.values()) { + if (partition.getVisibleVersion() > Partition.PARTITION_INIT_VERSION) { + return partition; + } + } + return null; + } + // RANGE/LIST: pick the visible partition with the greatest key (RANGE by upper bound, + // LIST by max discrete key). A LIST default partition is only a last-resort fallback. + Partition result = null; + PartitionKey maxKey = null; + Partition defaultFallback = null; + for (Partition partition : nameToPartition.values()) { + if (partition.getVisibleVersion() <= Partition.PARTITION_INIT_VERSION) { + continue; + } + PartitionItem item = partitionInfo.getItem(partition.getId()); + // LIST default partition's key is a synthetic MIN placeholder, not real data; + // keep it only as a fallback so a keyed partition with data always wins. + if (item instanceof ListPartitionItem && ((ListPartitionItem) item).isDefaultPartition()) { + defaultFallback = partition; + continue; + } + PartitionKey key = maxPartitionKey(item); + if (maxKey == null || key.compareTo(maxKey) > 0) { + maxKey = key; + result = partition; + } + } + return result != null ? result : defaultFallback; + } + + // Greatest partition key of an item: RANGE upper bound, or LIST max discrete key. + private PartitionKey maxPartitionKey(PartitionItem item) { + if (item instanceof RangePartitionItem) { + return ((RangePartitionItem) item).getItems().upperEndpoint(); + } + PartitionKey max = null; + for (PartitionKey key : ((ListPartitionItem) item).getItems()) { + if (max == null || key.compareTo(max) > 0) { + max = key; + } + } + return max; + } + // Priority is given to querying from the partition. If not found, query from the tempPartition @Override public Partition getPartition(String partitionName) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java index a98aa7211bb4ff..97a9c699d56cb0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java @@ -376,6 +376,10 @@ default Partition getPartition(String name) { return null; } + default Partition getMaxVisiblePartition() { + return null; + } + default List getFullQualifiers() { return ImmutableList.of(getDatabase().getCatalog().getName(), getDatabase().getFullName(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 941eeda143f442..b0ea2eaffa271c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -269,6 +269,10 @@ public enum TableFrom { // for high speed/concurrency point queries private boolean isShortCircuitQuery; + // max_visible_partition() is data-dependent, so such a query must not be cached/reused + // as a prepared short-circuit point plan. + private boolean hasMaxVisiblePartition; + private ShortCircuitQueryContext shortCircuitQueryContext; private FormatOptions formatOptions = FormatOptions.getDefault(); @@ -579,6 +583,14 @@ public void setShortCircuitQuery(boolean shortCircuitQuery) { isShortCircuitQuery = shortCircuitQuery; } + public boolean hasMaxVisiblePartition() { + return hasMaxVisiblePartition; + } + + public void setHasMaxVisiblePartition(boolean hasMaxVisiblePartition) { + this.hasMaxVisiblePartition = hasMaxVisiblePartition; + } + public ShortCircuitQueryContext getShortCircuitQueryContext() { return shortCircuitQueryContext; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java index 717c55c8eb8c63..522550e4e44bdf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java @@ -53,6 +53,7 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu private final List partNames; private final List tabletIds; private final boolean isTempPart; + private final boolean isMaxVisiblePartition; private final List hints; private final Optional tableSample; private final Optional indexName; @@ -64,14 +65,14 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu public UnboundRelation(RelationId id, List nameParts) { this(id, nameParts, Optional.empty(), Optional.empty(), - ImmutableList.of(), false, ImmutableList.of(), + ImmutableList.of(), false, false, ImmutableList.of(), ImmutableList.of(), Optional.empty(), Optional.empty(), null, Optional.empty(), Optional.empty()); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart) { - this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, ImmutableList.of(), + this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, false, ImmutableList.of(), ImmutableList.of(), Optional.empty(), Optional.empty(), null, Optional.empty(), Optional.empty()); } @@ -79,7 +80,7 @@ public UnboundRelation(RelationId id, List nameParts, List partN boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName) { this(id, nameParts, Optional.empty(), Optional.empty(), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, null, Optional.empty(), + partNames, isTempPart, false, tabletIds, hints, tableSample, indexName, null, Optional.empty(), Optional.empty()); } @@ -87,7 +88,7 @@ public UnboundRelation(RelationId id, List nameParts, List partN boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName, TableScanParams scanParams, Optional tableSnapshot) { this(id, nameParts, Optional.empty(), Optional.empty(), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, Optional.empty(), + partNames, isTempPart, false, tabletIds, hints, tableSample, indexName, scanParams, Optional.empty(), tableSnapshot); } @@ -96,7 +97,7 @@ public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName) { this(id, nameParts, groupExpression, logicalProperties, partNames, - isTempPart, tabletIds, hints, tableSample, indexName, null, Optional.empty(), Optional.empty()); + isTempPart, false, tabletIds, hints, tableSample, indexName, null, Optional.empty(), Optional.empty()); } public UnboundRelation(RelationId id, List nameParts, List partNames, @@ -104,16 +105,29 @@ public UnboundRelation(RelationId id, List nameParts, List partN Optional indexName, TableScanParams scanParams, Optional> indexInSqlString, Optional tableSnapshot) { this(id, nameParts, Optional.empty(), Optional.empty(), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, indexInSqlString, + partNames, isTempPart, false, tabletIds, hints, tableSample, indexName, scanParams, indexInSqlString, tableSnapshot); } /** - * constructor of UnboundRelation + * constructor of UnboundRelation with max_visible_partition() flag. + */ + public UnboundRelation(RelationId id, List nameParts, + List partNames, boolean isTempPart, boolean isMaxVisiblePartition, + List tabletIds, List hints, + Optional tableSample, Optional indexName, TableScanParams scanParams, + Optional tableSnapshot) { + this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, isMaxVisiblePartition, + tabletIds, hints, tableSample, indexName, scanParams, Optional.empty(), tableSnapshot); + } + + /** + * constructor of UnboundRelation with max_visible_partition() flag. */ public UnboundRelation(RelationId id, List nameParts, Optional groupExpression, Optional logicalProperties, - List partNames, boolean isTempPart, List tabletIds, List hints, + List partNames, boolean isTempPart, boolean isMaxVisiblePartition, + List tabletIds, List hints, Optional tableSample, Optional indexName, TableScanParams scanParams, Optional> indexInSqlString, Optional tableSnapshot) { @@ -122,6 +136,7 @@ public UnboundRelation(RelationId id, List nameParts, this.partNames = ImmutableList.copyOf(Objects.requireNonNull(partNames, "partNames should not null")); this.tabletIds = ImmutableList.copyOf(Objects.requireNonNull(tabletIds, "tabletIds should not null")); this.isTempPart = isTempPart; + this.isMaxVisiblePartition = isMaxVisiblePartition; this.hints = ImmutableList.copyOf(Objects.requireNonNull(hints, "hints should not be null.")); this.tableSample = tableSample; this.indexName = indexName; @@ -148,23 +163,23 @@ public LogicalProperties computeLogicalProperties() { public Plan withGroupExpression(Optional groupExpression) { return new UnboundRelation(relationId, nameParts, groupExpression, Optional.of(getLogicalProperties()), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, - indexInSqlString, tableSnapshot); + partNames, isTempPart, isMaxVisiblePartition, tabletIds, hints, tableSample, indexName, + scanParams, indexInSqlString, tableSnapshot); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new UnboundRelation(relationId, nameParts, groupExpression, - logicalProperties, partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, - indexInSqlString, tableSnapshot); + logicalProperties, partNames, isTempPart, isMaxVisiblePartition, tabletIds, hints, tableSample, + indexName, scanParams, indexInSqlString, tableSnapshot); } public UnboundRelation withIndexInSql(Pair index) { // SQL source indexing annotates relation identity only; it must not erase scan semantics. return new UnboundRelation(relationId, nameParts, groupExpression, - Optional.of(getLogicalProperties()), partNames, isTempPart, tabletIds, hints, tableSample, indexName, - scanParams, Optional.of(index), tableSnapshot); + Optional.of(getLogicalProperties()), partNames, isTempPart, isMaxVisiblePartition, tabletIds, + hints, tableSample, indexName, scanParams, Optional.of(index), tableSnapshot); } @Override @@ -197,6 +212,9 @@ public String toDigest() { sb.append(nameParts.stream().collect(Collectors.joining("."))); sb.append(" "); } + if (isMaxVisiblePartition) { + sb.append("MAX_VISIBLE_PARTITION()").append(" "); + } if (indexName.isPresent()) { sb.append("INDEX ").append(indexName.get()).append(" "); } @@ -221,7 +239,8 @@ public boolean equals(Object o) { return false; } UnboundRelation that = (UnboundRelation) o; - return isTempPart == that.isTempPart && Objects.equals(nameParts, that.nameParts) + return isTempPart == that.isTempPart && isMaxVisiblePartition == that.isMaxVisiblePartition + && Objects.equals(nameParts, that.nameParts) && Objects.equals(partNames, that.partNames) && Objects.equals(tabletIds, that.tabletIds) && Objects.equals(hints, that.hints) && Objects.equals(tableSample, that.tableSample) && Objects.equals(indexName, that.indexName) && Objects.equals( @@ -247,6 +266,10 @@ public boolean isTempPart() { return isTempPart; } + public boolean isMaxVisiblePartition() { + return isMaxVisiblePartition; + } + public List getTabletIds() { return tabletIds; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 0912f8dcccf7ee..d611a1b15918d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -2749,6 +2749,7 @@ public LogicalPlan visitTableName(TableNameContext ctx) { List nameParts = visitMultipartIdentifier(ctx.multipartIdentifier()); List partitionNames = new ArrayList<>(); boolean isTempPart = false; + boolean isMaxVisiblePartition = ctx.maxVisiblePartition() != null; if (ctx.specifiedPartition() != null) { isTempPart = ctx.specifiedPartition().TEMPORARY() != null; if (ctx.specifiedPartition().identifier() != null) { @@ -2801,7 +2802,7 @@ public LogicalPlan visitTableName(TableNameContext ctx) { TableSample tableSample = ctx.sample() == null ? null : (TableSample) visit(ctx.sample()); UnboundRelation relation = new UnboundRelation( StatementScopeIdGenerator.newRelationId(), - nameParts, partitionNames, isTempPart, tabletIdLists, relationHints, + nameParts, partitionNames, isTempPart, isMaxVisiblePartition, tabletIdLists, relationHints, Optional.ofNullable(tableSample), indexName, scanParams, Optional.ofNullable(tableSnapshot)); LogicalPlan checkedRelation = LogicalPlanBuilderAssistant.withCheckPolicy(relation); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index 73a5bfd844aef6..4092c420c3d2f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -232,6 +232,12 @@ private void rejectScanParamsOnCte(UnboundRelation unboundRelation) { // A CTE reference has no physical table handle on which scan parameters can be applied. throw new AnalysisException("Table scan parameters are not supported on CTE references."); } + if (unboundRelation.isMaxVisiblePartition()) { + String tableName = unboundRelation.getNameParts().get(unboundRelation.getNameParts().size() - 1); + throw new AnalysisException("max_visible_partition() is not supported on CTE reference '" + + tableName + "', please move it inside the CTE definition, " + + "e.g. WITH " + tableName + " AS (SELECT ... FROM base_table max_visible_partition()) ..."); + } } private LogicalPlan bind(CascadesContext cascadesContext, UnboundRelation unboundRelation) { @@ -731,6 +737,18 @@ private boolean isScanAppendOnlyTableStream(OlapTableStream table) { private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelation, List qualifiedTableName, CascadesContext cascadesContext) { + if (unboundRelation.isMaxVisiblePartition()) { + TableIf.TableType type = table.getType(); + if (type != TableIf.TableType.OLAP && type != TableIf.TableType.MATERIALIZED_VIEW) { + throw new AnalysisException("max_visible_partition() is only supported on OLAP table " + + "or materialized view, but table " + table.getName() + " is " + type); + } + if (unboundRelation.getTableSnapshot().isPresent()) { + throw new AnalysisException("max_visible_partition() cannot be used with FOR VERSION/TIME AS OF"); + } + cascadesContext.getStatementContext().setHasMaxVisiblePartition(true); + } + // for create view stmt replace tableName to ctl.db.tableName unboundRelation.getIndexInSqlString().ifPresent(pair -> { StatementContext statementContext = cascadesContext.getStatementContext(); @@ -982,7 +1000,7 @@ private Plan parseAndAnalyzeView(TableIf view, String ddlSql, CascadesContext pa private List getPartitionIds(TableIf t, UnboundRelation unboundRelation, List qualifier) { List parts = unboundRelation.getPartNames(); - if (CollectionUtils.isEmpty(parts)) { + if (CollectionUtils.isEmpty(parts) && !unboundRelation.isMaxVisiblePartition()) { return ImmutableList.of(); } if (!t.isManagedTable()) { @@ -990,6 +1008,10 @@ private List getPartitionIds(TableIf t, UnboundRelation unboundRelation, L "Only OLAP table is support select by partition for now," + "Table: %s is not OLAP table", t.getName())); } + if (unboundRelation.isMaxVisiblePartition()) { + Partition partition = t.getMaxVisiblePartition(); + return partition == null ? ImmutableList.of() : ImmutableList.of(partition.getId()); + } return parts.stream().map(name -> { Partition part = ((OlapTable) t).getPartition(name, unboundRelation.isTempPart()); if (part == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java index dfcd2ea289cc6f..7746020619bfe2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java @@ -64,6 +64,9 @@ private boolean scanMatchShortCircuitCondition(LogicalOlapScan olapScan) { if (!ConnectContext.get().getSessionVariable().isEnableShortCircuitQuery()) { return false; } + if (ConnectContext.get().getStatementContext().hasMaxVisiblePartition()) { + return false; + } OlapTable olapTable = olapScan.getTable(); if (olapTable.hasVariantColumns()) { return false; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserDigestTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserDigestTest.java index be2bd718f11571..98580ecf223d37 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserDigestTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserDigestTest.java @@ -79,6 +79,11 @@ public void testDigest() { logicalPlanList = nereidsParser.parseMultiple(sql); assertDigestEquals("SELECT * FROM test TABLET(?)", logicalPlanList); + // test max_visible_partition + sql = "select * from test max_visible_partition()"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT * FROM test MAX_VISIBLE_PARTITION()", logicalPlanList); + // test except sql = "select * except(age) from student;"; logicalPlanList = nereidsParser.parseMultiple(sql); diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 446685892295e4..ba98be4d99e01b 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -370,6 +370,7 @@ MATCH_PHRASE_PREFIX: 'MATCH_PHRASE_PREFIX'; MATCH_REGEXP: 'MATCH_REGEXP'; MATERIALIZED: 'MATERIALIZED'; MAX: 'MAX'; +MAX_VISIBLE_PARTITION: 'MAX_VISIBLE_PARTITION'; MAXVALUE: 'MAXVALUE'; MEMO:'MEMO'; MERGE: 'MERGE'; diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 9dfd4630ed8890..edf6764416e9b0 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1485,7 +1485,8 @@ optScanParams ; relationPrimary - : multipartIdentifier optScanParams? materializedViewName? tableSnapshot? specifiedPartition? + : multipartIdentifier optScanParams? materializedViewName? tableSnapshot? + (specifiedPartition | maxVisiblePartition)? tabletList? tableAlias sample? relationHint? lateralView* #tableName | LEFT_PAREN query RIGHT_PAREN tableAlias lateralView* #aliasedQuery | tvfName=identifier LEFT_PAREN @@ -1830,6 +1831,10 @@ specifiedPartition | TEMPORARY? PARTITIONS identifierList ; +maxVisiblePartition + : MAX_VISIBLE_PARTITION LEFT_PAREN RIGHT_PAREN + ; + constant : NULL #nullLiteral | type=(DATE | DATEV1 | DATEV2 | TIMESTAMP) STRING_LITERAL #typeConstructor @@ -2216,6 +2221,7 @@ nonReserved | MATCH_NAME_GLOB | MATERIALIZED | MAX + | MAX_VISIBLE_PARTITION | MEMO | MERGE | MID diff --git a/regression-test/data/nereids_syntax_p0/max_visible_partition.out b/regression-test/data/nereids_syntax_p0/max_visible_partition.out new file mode 100644 index 00000000000000..d49e495f4f16e3 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/max_visible_partition.out @@ -0,0 +1,83 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_range -- +7 20 + +-- !sql_range_date -- +2026-01-02 200 + +-- !sql_range_multi -- +2026-01-02T12:00 15 200 + +-- !sql_mtmv -- +7 20 + +-- !sql_unpart -- +1 10 +2 20 + +-- !sql_unpart_empty -- + +-- !sql_list -- +5 50 + +-- !sql_list_low -- +2 20 + +-- !sql_list_empty -- + +-- !sql_list_multi -- +us 1 300 + +-- !sql_list_def -- +1 10 + +-- !sql_list_def_empty -- +2 20 + +-- !sql_list_def_only -- +7 70 + +-- !sql_sc_mvp -- +7 70 + +-- !sql_td_truncate -- +1 10 + +-- !sql_td2_delete -- + +-- !sql_cte_inside -- +7 20 + +-- !sql_cte_inside_filter -- +7 20 + +-- !sql_join_same -- +7 20 20 + +-- !sql_join_diff -- + +-- !sql_alias_bare -- +7 20 + +-- !sql_alias_as -- +7 20 + +-- !sql_subquery -- +1 20 + +-- !sql_union_all -- +7 20 +7 70 + +-- !sql_agg -- +1 20 + +-- !sql_group_by -- +7 20 + +-- !sql_explicit_partition -- +1 10 + +-- !sql_max_visible -- +7 20 + diff --git a/regression-test/suites/nereids_syntax_p0/max_visible_partition.groovy b/regression-test/suites/nereids_syntax_p0/max_visible_partition.groovy new file mode 100644 index 00000000000000..c877fe36bd5e8d --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/max_visible_partition.groovy @@ -0,0 +1,540 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("max_visible_partition") { + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + // ============================================================ + // Case 1: RANGE partition (BIGINT) - prunes to the greatest upper bound with data + // ============================================================ + sql """DROP TABLE IF EXISTS mvp_range;""" + sql """ + CREATE TABLE mvp_range ( + id BIGINT, + val BIGINT + ) DUPLICATE KEY(`id`) + PARTITION BY RANGE(`id`) + ( + PARTITION `p1` VALUES LESS THAN ('5'), + PARTITION `p2` VALUES LESS THAN ('10'), + PARTITION `p3` VALUES LESS THAN ('15') + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES ("replication_num" = "1"); + """ + sql "INSERT INTO mvp_range VALUES(1, 10)" + sql "INSERT INTO mvp_range VALUES(7, 20)" + // p3 has no data -> latest visible is p2 + qt_sql_range """select * from mvp_range max_visible_partition();""" + explain { + sql "select * from mvp_range max_visible_partition();" + contains "partitions=1/3 (p2)" + } + + // ============================================================ + // Case 1b: RANGE partition (DATE) - most common business case + // ============================================================ + sql """DROP TABLE IF EXISTS mvp_range_date;""" + sql """ + CREATE TABLE mvp_range_date ( + dt DATE, + val BIGINT + ) DUPLICATE KEY(`dt`) + PARTITION BY RANGE(`dt`) + ( + PARTITION p20260101 VALUES [('2026-01-01'), ('2026-01-02')), + PARTITION p20260102 VALUES [('2026-01-02'), ('2026-01-03')), + PARTITION p20260103 VALUES [('2026-01-03'), ('2026-01-04')) + ) + DISTRIBUTED BY HASH(`dt`) BUCKETS 3 + PROPERTIES ("replication_num" = "1"); + """ + sql "INSERT INTO mvp_range_date VALUES('2026-01-01', 100)" + sql "INSERT INTO mvp_range_date VALUES('2026-01-02', 200)" + // p20260103 has no data -> latest visible is p20260102 + qt_sql_range_date """select * from mvp_range_date max_visible_partition();""" + explain { + sql "select * from mvp_range_date max_visible_partition();" + contains "partitions=1/3 (p20260102)" + } + + // ============================================================ + // Case 1c: multi-column RANGE partition (DATETIME + INT) + // ============================================================ + sql """DROP TABLE IF EXISTS mvp_range_multi;""" + sql """ + CREATE TABLE mvp_range_multi ( + dt DATETIME, + region INT, + val BIGINT + ) DUPLICATE KEY(`dt`, `region`) + PARTITION BY RANGE(`dt`, `region`) + ( + PARTITION p_low VALUES [('2026-01-01 00:00:00', 1), ('2026-01-02 00:00:00', 10)), + PARTITION p_mid VALUES [('2026-01-02 00:00:00', 10), ('2026-01-03 00:00:00', 20)), + PARTITION p_high VALUES [('2026-01-03 00:00:00', 20), ('2026-01-04 00:00:00', 30)) + ) + DISTRIBUTED BY HASH(`region`) BUCKETS 3 + PROPERTIES ("replication_num" = "1"); + """ + sql "INSERT INTO mvp_range_multi VALUES('2026-01-01 12:00:00', 5, 100)" + sql "INSERT INTO mvp_range_multi VALUES('2026-01-02 12:00:00', 15, 200)" + // p_high has no data -> latest visible is p_mid + qt_sql_range_multi """select * from mvp_range_multi max_visible_partition();""" + explain { + sql "select * from mvp_range_multi max_visible_partition();" + contains "partitions=1/3 (p_mid)" + } + + // ============================================================ + // Case 2: Async materialized view (MTMV) + // ============================================================ + sql """DROP MATERIALIZED VIEW IF EXISTS mvp_mtmv;""" + sql """ + CREATE MATERIALIZED VIEW mvp_mtmv + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + PARTITION BY (id) + DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES ("replication_num" = "1") + AS SELECT id, val FROM mvp_range; + """ + waitingMTMVTaskFinishedByMvName("mvp_mtmv") + qt_sql_mtmv """select * from mvp_mtmv max_visible_partition();""" + explain { + sql "select * from mvp_mtmv max_visible_partition();" + contains "partitions=1/" + } + + // ============================================================ + // Case 3: View is not OLAP-backed -> reject explicitly + // ============================================================ + sql """DROP VIEW IF EXISTS v_mvp;""" + sql """CREATE VIEW v_mvp AS SELECT * FROM mvp_range;""" + test { + sql "select * from v_mvp max_visible_partition();" + exception "max_visible_partition() is only supported on OLAP table or materialized view" + } + + // ============================================================ + // Case 4: FOR VERSION / TIME AS OF conflict -> reject + // ============================================================ + test { + sql "SELECT * FROM mvp_range FOR VERSION AS OF 100 max_visible_partition();" + exception "max_visible_partition() cannot be used with FOR VERSION/TIME AS OF" + } + test { + sql "SELECT * FROM mvp_range FOR TIME AS OF '2026-01-01 00:00:00' max_visible_partition();" + exception "max_visible_partition() cannot be used with FOR VERSION/TIME AS OF" + } + + // ============================================================ + // Case 5a: UNPARTITIONED with data -> full data + // ============================================================ + sql """DROP TABLE IF EXISTS mvp_unpart;""" + sql """ + CREATE TABLE mvp_unpart ( + id BIGINT, + val BIGINT + ) DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES ("replication_num" = "1"); + """ + sql "INSERT INTO mvp_unpart VALUES(1, 10), (2, 20)" + order_qt_sql_unpart """select * from mvp_unpart max_visible_partition();""" + + // ============================================================ + // Case 5b: UNPARTITIONED with no data -> result empty + // ============================================================ + sql """DROP TABLE IF EXISTS mvp_unpart_empty;""" + sql """ + CREATE TABLE mvp_unpart_empty ( + id BIGINT, + val BIGINT + ) DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES ("replication_num" = "1"); + """ + qt_sql_unpart_empty """select * from mvp_unpart_empty max_visible_partition();""" + + // ============================================================ + // Case 6a: LIST partition (both have data) - pick partition of max key + // ============================================================ + sql """DROP TABLE IF EXISTS mvp_list;""" + sql """ + CREATE TABLE mvp_list ( + id INT, + val BIGINT + ) DUPLICATE KEY(`id`) + PARTITION BY LIST(`id`) + ( + PARTITION p1 VALUES IN ("1","2","3"), + PARTITION p2 VALUES IN ("4","5","6") + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES ("replication_num" = "1"); + """ + sql "INSERT INTO mvp_list VALUES(1, 10), (5, 50)" + qt_sql_list """select * from mvp_list max_visible_partition();""" + explain { + sql "select * from mvp_list max_visible_partition();" + contains "partitions=1/2 (p2)" + } + + // ============================================================ + // Case 6b: LIST - only low partition has data -> pick that one + // ============================================================ + sql """DROP TABLE IF EXISTS mvp_list_low;""" + sql """ + CREATE TABLE mvp_list_low ( + id INT, + val BIGINT + ) DUPLICATE KEY(`id`) + PARTITION BY LIST(`id`) + ( + PARTITION p1 VALUES IN ("1","2","3"), + PARTITION p2 VALUES IN ("4","5","6") + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES ("replication_num" = "1"); + """ + sql "INSERT INTO mvp_list_low VALUES(2, 20)" + qt_sql_list_low """select * from mvp_list_low max_visible_partition();""" + explain { + sql "select * from mvp_list_low max_visible_partition();" + contains "partitions=1/2 (p1)" + } + + // ============================================================ + // Case 6c: LIST - all empty -> result empty + // ============================================================ + sql """DROP TABLE IF EXISTS mvp_list_empty;""" + sql """ + CREATE TABLE mvp_list_empty ( + id INT, + val BIGINT + ) DUPLICATE KEY(`id`) + PARTITION BY LIST(`id`) + ( + PARTITION p1 VALUES IN ("1","2","3") + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES ("replication_num" = "1"); + """ + qt_sql_list_empty """select * from mvp_list_empty max_visible_partition();""" + + // ============================================================ + // Case 6d: multi-column LIST partition + // ============================================================ + sql """DROP TABLE IF EXISTS mvp_list_multi;""" + sql """ + CREATE TABLE mvp_list_multi ( + region VARCHAR(16), + id INT, + val BIGINT + ) DUPLICATE KEY(`region`, `id`) + PARTITION BY LIST(`region`, `id`) + ( + PARTITION p_cn_low VALUES IN (("cn", "1"), ("cn", "2")), + PARTITION p_cn_high VALUES IN (("cn", "8"), ("cn", "9")), + PARTITION p_us_low VALUES IN (("us", "1"), ("us", "2")) + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES ("replication_num" = "1"); + """ + sql "INSERT INTO mvp_list_multi VALUES('cn', 1, 100), ('cn', 8, 200), ('us', 1, 300)" + // max key ('us','1') -> partition p_us_low + qt_sql_list_multi """select * from mvp_list_multi max_visible_partition();""" + explain { + sql "select * from mvp_list_multi max_visible_partition();" + contains "partitions=1/3 (p_us_low)" + } + + // ============================================================ + // Case 6e: LIST with a DEFAULT partition alongside a keyed partition. + // The default partition is only a last-resort fallback; when a keyed partition + // has data it always wins, so the default (even with larger values) is not chosen. + // ============================================================ + sql """DROP TABLE IF EXISTS mvp_list_def;""" + sql """ + CREATE TABLE mvp_list_def ( + id INT, + val BIGINT + ) DUPLICATE KEY(`id`) + PARTITION BY LIST(`id`) + ( + PARTITION p_ex VALUES IN ("1","2","3"), + PARTITION p_def + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES ("replication_num" = "1"); + """ + // 1 -> p_ex, 100 -> p_def(default); keyed p_ex wins over the default fallback + sql "INSERT INTO mvp_list_def VALUES(1, 10), (100, 999)" + qt_sql_list_def """select * from mvp_list_def max_visible_partition();""" + explain { + sql "select * from mvp_list_def max_visible_partition();" + contains "partitions=1/2 (p_ex)" + } + + // ============================================================ + // Case 6f: LIST with DEFAULT partition but only the keyed partition has data. + // The keyed partition is chosen; the empty default fallback is not needed. + // ============================================================ + sql """DROP TABLE IF EXISTS mvp_list_def_empty;""" + sql """ + CREATE TABLE mvp_list_def_empty ( + id INT, + val BIGINT + ) DUPLICATE KEY(`id`) + PARTITION BY LIST(`id`) + ( + PARTITION p_ex VALUES IN ("1","2","3"), + PARTITION p_def + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES ("replication_num" = "1"); + """ + sql "INSERT INTO mvp_list_def_empty VALUES(2, 20)" + qt_sql_list_def_empty """select * from mvp_list_def_empty max_visible_partition();""" + explain { + sql "select * from mvp_list_def_empty max_visible_partition();" + contains "partitions=1/2 (p_ex)" + } + + // ============================================================ + // Case 6g: LIST with only a DEFAULT partition holding data -> fall back to default. + // ============================================================ + sql """DROP TABLE IF EXISTS mvp_list_def_only;""" + sql """ + CREATE TABLE mvp_list_def_only ( + id INT, + val BIGINT + ) DUPLICATE KEY(`id`) + PARTITION BY LIST(`id`) + ( + PARTITION p_def + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES ("replication_num" = "1"); + """ + sql "INSERT INTO mvp_list_def_only VALUES(7, 70)" + qt_sql_list_def_only """select * from mvp_list_def_only max_visible_partition();""" + explain { + sql "select * from mvp_list_def_only max_visible_partition();" + contains "partitions=1/1 (p_def)" + } + + // ============================================================ + // Case 7: Short-circuit point query MUST be excluded. + // Caching a resolved partition into a prepared point-query plan + // would return stale data as new partitions become visible. + // ============================================================ + sql """DROP TABLE IF EXISTS mvp_sc;""" + sql """ + CREATE TABLE mvp_sc ( + id BIGINT, + val BIGINT + ) UNIQUE KEY(`id`) + PARTITION BY RANGE(`id`) + ( + PARTITION `p1` VALUES LESS THAN ('5'), + PARTITION `p2` VALUES LESS THAN ('10') + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "store_row_column" = "true" + ); + """ + sql "INSERT INTO mvp_sc VALUES(1, 10)" + sql "INSERT INTO mvp_sc VALUES(7, 70)" + sql "SET enable_short_circuit_query=true" + + // baseline: plain full-key point query short-circuits + explain { + sql "select * from mvp_sc where id = 7" + contains "SHORT-CIRCUIT" + } + // with max_visible_partition(): must NOT short-circuit + explain { + sql "select * from mvp_sc max_visible_partition() where id = 7" + notContains "SHORT-CIRCUIT" + } + // still resolves to latest visible partition (p2) + explain { + sql "select * from mvp_sc max_visible_partition() where id = 7" + contains "partitions=1/2 (p2)" + } + qt_sql_sc_mvp """select * from mvp_sc max_visible_partition() where id = 7;""" + + sql "SET enable_short_circuit_query=false" + + // ============================================================ + // Case 8a: TRUNCATE partition -> version reset -> partition invisible + // ============================================================ + sql """DROP TABLE IF EXISTS mvp_td;""" + sql """ + CREATE TABLE mvp_td ( + id BIGINT, + val BIGINT + ) DUPLICATE KEY(`id`) + PARTITION BY RANGE(`id`) + ( + PARTITION `p1` VALUES LESS THAN ('5'), + PARTITION `p2` VALUES LESS THAN ('10') + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_num" = "1"); + """ + sql "INSERT INTO mvp_td VALUES(1, 10)" + sql "INSERT INTO mvp_td VALUES(7, 70)" + // both partitions have data -> latest visible is p2 + explain { + sql "select * from mvp_td max_visible_partition();" + contains "partitions=1/2 (p2)" + } + sql "truncate table mvp_td partitions (p2);" + order_qt_sql_td_truncate """select * from mvp_td max_visible_partition();""" + explain { + sql "select * from mvp_td max_visible_partition();" + contains "partitions=1/2 (p1)" + } + + // ============================================================ + // Case 8b: DELETE partition -> version still advances -> partition visible but empty + // ============================================================ + sql """DROP TABLE IF EXISTS mvp_td2;""" + sql """ + CREATE TABLE mvp_td2 ( + id BIGINT, + val BIGINT + ) DUPLICATE KEY(`id`) + PARTITION BY RANGE(`id`) + ( + PARTITION `p1` VALUES LESS THAN ('5'), + PARTITION `p2` VALUES LESS THAN ('10') + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_num" = "1"); + """ + sql "INSERT INTO mvp_td2 VALUES(1, 10)" + sql "INSERT INTO mvp_td2 VALUES(7, 70)" + sql "DELETE FROM mvp_td2 PARTITION p2 WHERE id = 7;" + explain { + sql "select * from mvp_td2 max_visible_partition();" + contains "partitions=1/2 (p2)" + } + qt_sql_td2_delete """select * from mvp_td2 max_visible_partition();""" + + // ============================================================ + // Case 9: CTE - modifier INSIDE CTE definition (on the base table) is supported + // ============================================================ + order_qt_sql_cte_inside """ + WITH t AS ( + SELECT * FROM mvp_range max_visible_partition() + ) + SELECT * FROM t; + """ + // nested CTE + outer filter + order_qt_sql_cte_inside_filter """ + WITH t AS ( + SELECT * FROM mvp_range max_visible_partition() + ) + SELECT id, val FROM t WHERE val > 0; + """ + + // ============================================================ + // Case 9b: CTE - modifier on the CTE ALIAS is rejected + // (CTEConsumer holds an already-analyzed subplan, not a base table) + // ============================================================ + test { + sql """ + WITH t AS (SELECT * FROM mvp_range) + SELECT * FROM t max_visible_partition(); + """ + exception "max_visible_partition() is not supported on CTE reference" + } + + // ============================================================ + // Case 10: JOIN - each side independently prunes to its latest visible partition + // ============================================================ + order_qt_sql_join_same """ + SELECT a.id, a.val AS a_val, b.val AS b_val + FROM mvp_range max_visible_partition() a + JOIN mvp_range max_visible_partition() b + ON a.id = b.id; + """ + order_qt_sql_join_diff """ + SELECT r.dt, r.val AS r_val, m.val AS m_val + FROM mvp_range_date max_visible_partition() r + JOIN mvp_range max_visible_partition() m + ON r.val = m.val; + """ + + // ============================================================ + // Case 11: table alias placement - modifier goes between table name and alias + // ============================================================ + order_qt_sql_alias_bare """ + SELECT r.id, r.val + FROM mvp_range max_visible_partition() r + WHERE r.id > 0; + """ + order_qt_sql_alias_as """ + SELECT x.id, x.val + FROM mvp_range max_visible_partition() AS x + WHERE x.id > 0; + """ + + // ============================================================ + // Case 12: subquery / derived table + // ============================================================ + qt_sql_subquery """ + SELECT COUNT(*) AS cnt, MAX(val) AS max_val + FROM ( + SELECT * FROM mvp_range max_visible_partition() + ) sub; + """ + + // ============================================================ + // Case 13: UNION ALL - each branch prunes independently + // ============================================================ + order_qt_sql_union_all """ + SELECT id, val FROM mvp_range max_visible_partition() + UNION ALL + SELECT id, val FROM mvp_sc max_visible_partition() WHERE id = 7; + """ + + // ============================================================ + // Case 14: aggregation / GROUP BY on the pruned partition only + // ============================================================ + qt_sql_agg """SELECT COUNT(*) AS cnt, SUM(val) AS sum_val FROM mvp_range max_visible_partition();""" + qt_sql_group_by """ + SELECT id, SUM(val) AS sum_val + FROM mvp_range max_visible_partition() + GROUP BY id + ORDER BY id; + """ + + // ============================================================ + // Case 15: baseline sanity - explicit PARTITION() vs max_visible_partition() + // ============================================================ + order_qt_sql_explicit_partition """select * from mvp_range PARTITION(p1);""" + order_qt_sql_max_visible """select * from mvp_range max_visible_partition();""" +}