diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index bcd9e76e7c434c..187e697a3d7adf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -1478,6 +1478,63 @@ public Partition dropPartitionForTruncate(long dbId, boolean isForceDrop, * */ + @Override + public Partition getMaxVisiblePartition() { + PartitionInfo partitionInfo = getPartitionInfo(); + PartitionType type = partitionInfo.getType(); + // Cloud mode: prefetch all visible versions in one batch RPC so the loop below hits + // cache instead of firing N sequential meta-service RPCs under metadata locks. + if (Config.isCloudMode()) { + try { + getVersionInBatchForCloudMode(nameToPartition.values().stream() + .map(Partition::getId).collect(Collectors.toList())); + } catch (RpcException e) { + LOG.warn("batch prefetch visible version failed, fallback to per-partition lookup", e); + } + } + if (type == PartitionType.UNPARTITIONED) { + for (Partition partition : nameToPartition.values()) { + if (partition.getVisibleVersion() > Partition.PARTITION_INIT_VERSION) { + return partition; + } + } + return null; + } + // RANGE/LIST: pick the visible partition with the greatest key (RANGE by upper bound, + // LIST by max discrete key), giving LIST a deterministic "latest" in dictionary order. + Partition result = null; + PartitionKey maxKey = null; + for (Partition partition : nameToPartition.values()) { + if (partition.getVisibleVersion() <= Partition.PARTITION_INIT_VERSION) { + continue; + } + PartitionKey key = maxPartitionKey(partitionInfo.getItem(partition.getId())); + if (key == null) { + // LIST default partition has no discrete key to order by + continue; + } + if (maxKey == null || key.compareTo(maxKey) > 0) { + maxKey = key; + result = partition; + } + } + return result; + } + + // Greatest partition key of an item: RANGE upper bound, or LIST max discrete key (null if none). + private PartitionKey maxPartitionKey(PartitionItem item) { + if (item instanceof RangePartitionItem) { + return ((RangePartitionItem) item).getItems().upperEndpoint(); + } + PartitionKey max = null; + for (PartitionKey key : ((ListPartitionItem) item).getItems()) { + if (max == null || key.compareTo(max) > 0) { + max = key; + } + } + return max; + } + // Priority is given to querying from the partition. If not found, query from the tempPartition @Override public Partition getPartition(String partitionName) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java index a201617b4f30b5..4b1a6b55252f14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java @@ -375,6 +375,10 @@ default Partition getPartition(String name) { return null; } + default Partition getMaxVisiblePartition() { + return null; + } + default List getFullQualifiers() { return ImmutableList.of(getDatabase().getCatalog().getName(), getDatabase().getFullName(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 87e447190d4f96..8cdd0b915075f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -261,6 +261,10 @@ public enum TableFrom { // for high speed/concurrency point queries private boolean isShortCircuitQuery; + // max_visible_partition() is data-dependent, so such a query must not be cached/reused + // as a prepared short-circuit point plan. + private boolean hasMaxVisiblePartition; + private ShortCircuitQueryContext shortCircuitQueryContext; private FormatOptions formatOptions = FormatOptions.getDefault(); @@ -567,6 +571,14 @@ public void setShortCircuitQuery(boolean shortCircuitQuery) { isShortCircuitQuery = shortCircuitQuery; } + public boolean hasMaxVisiblePartition() { + return hasMaxVisiblePartition; + } + + public void setHasMaxVisiblePartition(boolean hasMaxVisiblePartition) { + this.hasMaxVisiblePartition = hasMaxVisiblePartition; + } + public ShortCircuitQueryContext getShortCircuitQueryContext() { return shortCircuitQueryContext; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java index 717c55c8eb8c63..865bdab41e0dc4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java @@ -53,6 +53,7 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu private final List partNames; private final List tabletIds; private final boolean isTempPart; + private final boolean isMaxVisiblePartition; private final List hints; private final Optional tableSample; private final Optional indexName; @@ -64,14 +65,14 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu public UnboundRelation(RelationId id, List nameParts) { this(id, nameParts, Optional.empty(), Optional.empty(), - ImmutableList.of(), false, ImmutableList.of(), + ImmutableList.of(), false, false, ImmutableList.of(), ImmutableList.of(), Optional.empty(), Optional.empty(), null, Optional.empty(), Optional.empty()); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart) { - this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, ImmutableList.of(), + this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, false, ImmutableList.of(), ImmutableList.of(), Optional.empty(), Optional.empty(), null, Optional.empty(), Optional.empty()); } @@ -79,7 +80,7 @@ public UnboundRelation(RelationId id, List nameParts, List partN boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName) { this(id, nameParts, Optional.empty(), Optional.empty(), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, null, Optional.empty(), + partNames, isTempPart, false, tabletIds, hints, tableSample, indexName, null, Optional.empty(), Optional.empty()); } @@ -87,7 +88,7 @@ public UnboundRelation(RelationId id, List nameParts, List partN boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName, TableScanParams scanParams, Optional tableSnapshot) { this(id, nameParts, Optional.empty(), Optional.empty(), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, Optional.empty(), + partNames, isTempPart, false, tabletIds, hints, tableSample, indexName, scanParams, Optional.empty(), tableSnapshot); } @@ -96,7 +97,7 @@ public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName) { this(id, nameParts, groupExpression, logicalProperties, partNames, - isTempPart, tabletIds, hints, tableSample, indexName, null, Optional.empty(), Optional.empty()); + isTempPart, false, tabletIds, hints, tableSample, indexName, null, Optional.empty(), Optional.empty()); } public UnboundRelation(RelationId id, List nameParts, List partNames, @@ -104,16 +105,29 @@ public UnboundRelation(RelationId id, List nameParts, List partN Optional indexName, TableScanParams scanParams, Optional> indexInSqlString, Optional tableSnapshot) { this(id, nameParts, Optional.empty(), Optional.empty(), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, indexInSqlString, + partNames, isTempPart, false, tabletIds, hints, tableSample, indexName, scanParams, indexInSqlString, tableSnapshot); } /** - * constructor of UnboundRelation + * constructor of UnboundRelation with max_visible_partition() flag. + */ + public UnboundRelation(RelationId id, List nameParts, + List partNames, boolean isTempPart, boolean isMaxVisiblePartition, + List tabletIds, List hints, + Optional tableSample, Optional indexName, TableScanParams scanParams, + Optional tableSnapshot) { + this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, isMaxVisiblePartition, + tabletIds, hints, tableSample, indexName, scanParams, Optional.empty(), tableSnapshot); + } + + /** + * constructor of UnboundRelation with max_visible_partition() flag. */ public UnboundRelation(RelationId id, List nameParts, Optional groupExpression, Optional logicalProperties, - List partNames, boolean isTempPart, List tabletIds, List hints, + List partNames, boolean isTempPart, boolean isMaxVisiblePartition, + List tabletIds, List hints, Optional tableSample, Optional indexName, TableScanParams scanParams, Optional> indexInSqlString, Optional tableSnapshot) { @@ -122,6 +136,7 @@ public UnboundRelation(RelationId id, List nameParts, this.partNames = ImmutableList.copyOf(Objects.requireNonNull(partNames, "partNames should not null")); this.tabletIds = ImmutableList.copyOf(Objects.requireNonNull(tabletIds, "tabletIds should not null")); this.isTempPart = isTempPart; + this.isMaxVisiblePartition = isMaxVisiblePartition; this.hints = ImmutableList.copyOf(Objects.requireNonNull(hints, "hints should not be null.")); this.tableSample = tableSample; this.indexName = indexName; @@ -148,7 +163,7 @@ public LogicalProperties computeLogicalProperties() { public Plan withGroupExpression(Optional groupExpression) { return new UnboundRelation(relationId, nameParts, groupExpression, Optional.of(getLogicalProperties()), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, + partNames, isTempPart, isMaxVisiblePartition, tabletIds, hints, tableSample, indexName, null, indexInSqlString, tableSnapshot); } @@ -156,15 +171,14 @@ public Plan withGroupExpression(Optional groupExpression) { public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new UnboundRelation(relationId, nameParts, groupExpression, - logicalProperties, partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, - indexInSqlString, tableSnapshot); + logicalProperties, partNames, isTempPart, isMaxVisiblePartition, tabletIds, hints, tableSample, + indexName, null, indexInSqlString, tableSnapshot); } public UnboundRelation withIndexInSql(Pair index) { - // SQL source indexing annotates relation identity only; it must not erase scan semantics. return new UnboundRelation(relationId, nameParts, groupExpression, - Optional.of(getLogicalProperties()), partNames, isTempPart, tabletIds, hints, tableSample, indexName, - scanParams, Optional.of(index), tableSnapshot); + Optional.of(getLogicalProperties()), partNames, isTempPart, isMaxVisiblePartition, tabletIds, hints, + tableSample, indexName, null, Optional.of(index), tableSnapshot); } @Override @@ -197,6 +211,9 @@ public String toDigest() { sb.append(nameParts.stream().collect(Collectors.joining("."))); sb.append(" "); } + if (isMaxVisiblePartition) { + sb.append("MAX_VISIBLE_PARTITION()").append(" "); + } if (indexName.isPresent()) { sb.append("INDEX ").append(indexName.get()).append(" "); } @@ -221,7 +238,8 @@ public boolean equals(Object o) { return false; } UnboundRelation that = (UnboundRelation) o; - return isTempPart == that.isTempPart && Objects.equals(nameParts, that.nameParts) + return isTempPart == that.isTempPart && isMaxVisiblePartition == that.isMaxVisiblePartition + && Objects.equals(nameParts, that.nameParts) && Objects.equals(partNames, that.partNames) && Objects.equals(tabletIds, that.tabletIds) && Objects.equals(hints, that.hints) && Objects.equals(tableSample, that.tableSample) && Objects.equals(indexName, that.indexName) && Objects.equals( @@ -247,6 +265,10 @@ public boolean isTempPart() { return isTempPart; } + public boolean isMaxVisiblePartition() { + return isMaxVisiblePartition; + } + public List getTabletIds() { return tabletIds; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 51dacb59e83110..f8825456f94ef8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -2752,6 +2752,7 @@ public LogicalPlan visitTableName(TableNameContext ctx) { List nameParts = visitMultipartIdentifier(ctx.multipartIdentifier()); List partitionNames = new ArrayList<>(); boolean isTempPart = false; + boolean isMaxVisiblePartition = ctx.maxVisiblePartition() != null; if (ctx.specifiedPartition() != null) { isTempPart = ctx.specifiedPartition().TEMPORARY() != null; if (ctx.specifiedPartition().identifier() != null) { @@ -2804,7 +2805,7 @@ public LogicalPlan visitTableName(TableNameContext ctx) { TableSample tableSample = ctx.sample() == null ? null : (TableSample) visit(ctx.sample()); UnboundRelation relation = new UnboundRelation( StatementScopeIdGenerator.newRelationId(), - nameParts, partitionNames, isTempPart, tabletIdLists, relationHints, + nameParts, partitionNames, isTempPart, isMaxVisiblePartition, tabletIdLists, relationHints, Optional.ofNullable(tableSample), indexName, scanParams, Optional.ofNullable(tableSnapshot)); LogicalPlan checkedRelation = LogicalPlanBuilderAssistant.withCheckPolicy(relation); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index 2348453c52d7c7..5108179a5b05a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -102,6 +102,7 @@ import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalHudiScan; @@ -302,6 +303,9 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, // This tabletIds is set manually, so need to set specifiedTabletIds scan = scan.withManuallySpecifiedTabletIds(tabletIds); } + if (unboundRelation.isMaxVisiblePartition() && CollectionUtils.isEmpty(partIds)) { + return new LogicalEmptyRelation(unboundRelation.getRelationId(), scan.getOutput()); + } if (changeScanType != null) { if (cascadesContext.getStatementContext().isHintForcePreAggOn()) { throw new AnalysisException( @@ -734,6 +738,18 @@ private boolean isScanAppendOnlyTableStream(OlapTableStream table) { private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelation, List qualifiedTableName, CascadesContext cascadesContext) { + if (unboundRelation.isMaxVisiblePartition()) { + TableIf.TableType type = table.getType(); + if (type != TableIf.TableType.OLAP && type != TableIf.TableType.MATERIALIZED_VIEW) { + throw new AnalysisException("max_visible_partition() is only supported on OLAP table " + + "or materialized view, but table " + table.getName() + " is " + type); + } + if (unboundRelation.getTableSnapshot().isPresent()) { + throw new AnalysisException("max_visible_partition() cannot be used with FOR VERSION/TIME AS OF"); + } + cascadesContext.getStatementContext().setHasMaxVisiblePartition(true); + } + // for create view stmt replace tableName to ctl.db.tableName unboundRelation.getIndexInSqlString().ifPresent(pair -> { StatementContext statementContext = cascadesContext.getStatementContext(); @@ -1014,7 +1030,7 @@ private Plan parseAndAnalyzeView(TableIf view, String ddlSql, CascadesContext pa private List getPartitionIds(TableIf t, UnboundRelation unboundRelation, List qualifier) { List parts = unboundRelation.getPartNames(); - if (CollectionUtils.isEmpty(parts)) { + if (CollectionUtils.isEmpty(parts) && !unboundRelation.isMaxVisiblePartition()) { return ImmutableList.of(); } if (!t.isManagedTable()) { @@ -1022,6 +1038,10 @@ private List getPartitionIds(TableIf t, UnboundRelation unboundRelation, L "Only OLAP table is support select by partition for now," + "Table: %s is not OLAP table", t.getName())); } + if (unboundRelation.isMaxVisiblePartition()) { + Partition partition = t.getMaxVisiblePartition(); + return partition == null ? ImmutableList.of() : ImmutableList.of(partition.getId()); + } return parts.stream().map(name -> { Partition part = ((OlapTable) t).getPartition(name, unboundRelation.isTempPart()); if (part == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java index dfcd2ea289cc6f..7746020619bfe2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java @@ -64,6 +64,9 @@ private boolean scanMatchShortCircuitCondition(LogicalOlapScan olapScan) { if (!ConnectContext.get().getSessionVariable().isEnableShortCircuitQuery()) { return false; } + if (ConnectContext.get().getStatementContext().hasMaxVisiblePartition()) { + return false; + } OlapTable olapTable = olapScan.getTable(); if (olapTable.hasVariantColumns()) { return false; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserDigestTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserDigestTest.java index be2bd718f11571..98580ecf223d37 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserDigestTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserDigestTest.java @@ -79,6 +79,11 @@ public void testDigest() { logicalPlanList = nereidsParser.parseMultiple(sql); assertDigestEquals("SELECT * FROM test TABLET(?)", logicalPlanList); + // test max_visible_partition + sql = "select * from test max_visible_partition()"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT * FROM test MAX_VISIBLE_PARTITION()", logicalPlanList); + // test except sql = "select * except(age) from student;"; logicalPlanList = nereidsParser.parseMultiple(sql); diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 446685892295e4..ba98be4d99e01b 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -370,6 +370,7 @@ MATCH_PHRASE_PREFIX: 'MATCH_PHRASE_PREFIX'; MATCH_REGEXP: 'MATCH_REGEXP'; MATERIALIZED: 'MATERIALIZED'; MAX: 'MAX'; +MAX_VISIBLE_PARTITION: 'MAX_VISIBLE_PARTITION'; MAXVALUE: 'MAXVALUE'; MEMO:'MEMO'; MERGE: 'MERGE'; diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index b300b6185c8730..6f8230d9f8b24f 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1486,7 +1486,8 @@ optScanParams ; relationPrimary - : multipartIdentifier optScanParams? materializedViewName? tableSnapshot? specifiedPartition? + : multipartIdentifier optScanParams? materializedViewName? tableSnapshot? + (specifiedPartition | maxVisiblePartition)? tabletList? tableAlias sample? relationHint? lateralView* #tableName | LEFT_PAREN query RIGHT_PAREN tableAlias lateralView* #aliasedQuery | tvfName=identifier LEFT_PAREN @@ -1831,6 +1832,10 @@ specifiedPartition | TEMPORARY? PARTITIONS identifierList ; +maxVisiblePartition + : MAX_VISIBLE_PARTITION LEFT_PAREN RIGHT_PAREN + ; + constant : NULL #nullLiteral | type=(DATE | DATEV1 | DATEV2 | TIMESTAMP) STRING_LITERAL #typeConstructor @@ -2217,6 +2222,7 @@ nonReserved | MATCH_NAME_GLOB | MATERIALIZED | MAX + | MAX_VISIBLE_PARTITION | MEMO | MERGE | MID diff --git a/regression-test/data/nereids_syntax_p0/max_visible_partition.out b/regression-test/data/nereids_syntax_p0/max_visible_partition.out new file mode 100644 index 00000000000000..4c67aaea8ca6b3 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/max_visible_partition.out @@ -0,0 +1,29 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_range -- +7 20 + +-- !sql_mtmv -- +7 20 + +-- !sql_unpart -- +1 10 +2 20 + +-- !sql_unpart_empty -- + +-- !sql_list -- +5 50 + +-- !sql_list_low -- +2 20 + +-- !sql_list_empty -- + +-- !sql_sc_mvp -- +7 70 + +-- !sql_td_truncate -- +1 10 + +-- !sql_td2_delete -- + diff --git a/regression-test/suites/nereids_syntax_p0/max_visible_partition.groovy b/regression-test/suites/nereids_syntax_p0/max_visible_partition.groovy new file mode 100644 index 00000000000000..067f5b1d430999 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/max_visible_partition.groovy @@ -0,0 +1,273 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("max_visible_partition") { + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + // 1) OLAP RANGE table: prunes to the partition with the greatest upper bound + sql """DROP TABLE IF EXISTS mvp_range;""" + sql """ + CREATE TABLE mvp_range ( + id BIGINT, + val BIGINT + ) DUPLICATE KEY(`id`) + PARTITION BY RANGE(`id`) + ( + PARTITION `p1` VALUES LESS THAN ('5'), + PARTITION `p2` VALUES LESS THAN ('10'), + PARTITION `p3` VALUES LESS THAN ('15') + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES ("replication_num" = "1"); + """ + sql "INSERT INTO mvp_range VALUES(1, 10)" + sql "INSERT INTO mvp_range VALUES(7, 20)" + // p3 has no data yet -> latest visible is p2 + qt_sql_range """select * from mvp_range max_visible_partition();""" + explain { + sql "select * from mvp_range max_visible_partition();" + contains "partitions=1/3 (p2)" + } + + // 2) Async materialized view (MTMV): supported + sql """DROP MATERIALIZED VIEW IF EXISTS mvp_mtmv;""" + sql """ + CREATE MATERIALIZED VIEW mvp_mtmv + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + PARTITION BY (id) + DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES ("replication_num" = "1") + AS SELECT id, val FROM mvp_range; + """ + waitingMTMVTaskFinishedByMvName("mvp_mtmv") + qt_sql_mtmv """select * from mvp_mtmv max_visible_partition();""" + explain { + sql "select * from mvp_mtmv max_visible_partition();" + contains "partitions=1/" + } + + // 3) View: not OLAP-backed -> reject explicitly + sql """DROP VIEW IF EXISTS v_mvp;""" + sql """CREATE VIEW v_mvp AS SELECT * FROM mvp_range;""" + test { + sql "select * from v_mvp max_visible_partition();" + exception "max_visible_partition() is only supported on OLAP table or materialized view" + } + + // Snapshot/Time Travel rejection + test { + sql "SELECT * FROM mvp_range FOR VERSION AS OF 100 max_visible_partition();" + exception "max_visible_partition() cannot be used with FOR VERSION/TIME AS OF" + } + + // 4a) UNPARTITIONED with data -> full data + sql """DROP TABLE IF EXISTS mvp_unpart;""" + sql """ + CREATE TABLE mvp_unpart ( + id BIGINT, + val BIGINT + ) DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES ("replication_num" = "1"); + """ + sql "INSERT INTO mvp_unpart VALUES(1, 10), (2, 20)" + order_qt_sql_unpart """select * from mvp_unpart max_visible_partition();""" + + // 4b) UNPARTITIONED with no data -> empty + sql """DROP TABLE IF EXISTS mvp_unpart_empty;""" + sql """ + CREATE TABLE mvp_unpart_empty ( + id BIGINT, + val BIGINT + ) DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES ("replication_num" = "1"); + """ + qt_sql_unpart_empty """select * from mvp_unpart_empty max_visible_partition();""" + explain { + sql "select * from mvp_unpart_empty max_visible_partition();" + contains "VEMPTYSET" + } + + // 5a) LIST with data -> latest by dictionary order of partition key. + // p1 IN (1,2,3), p2 IN (4,5,6); both have data -> max key is 5 (in p2) -> prune to p2. + sql """DROP TABLE IF EXISTS mvp_list;""" + sql """ + CREATE TABLE mvp_list ( + id INT, + val BIGINT + ) DUPLICATE KEY(`id`) + PARTITION BY LIST(`id`) + ( + PARTITION p1 VALUES IN ("1","2","3"), + PARTITION p2 VALUES IN ("4","5","6") + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES ("replication_num" = "1"); + """ + sql "INSERT INTO mvp_list VALUES(1, 10), (5, 50)" + qt_sql_list """select * from mvp_list max_visible_partition();""" + explain { + sql "select * from mvp_list max_visible_partition();" + contains "partitions=1/2 (p2)" + } + + // 5a2) LIST where only the lower partition has data -> pick that one (p1), not empty. + sql """DROP TABLE IF EXISTS mvp_list_low;""" + sql """ + CREATE TABLE mvp_list_low ( + id INT, + val BIGINT + ) DUPLICATE KEY(`id`) + PARTITION BY LIST(`id`) + ( + PARTITION p1 VALUES IN ("1","2","3"), + PARTITION p2 VALUES IN ("4","5","6") + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES ("replication_num" = "1"); + """ + sql "INSERT INTO mvp_list_low VALUES(2, 20)" + qt_sql_list_low """select * from mvp_list_low max_visible_partition();""" + explain { + sql "select * from mvp_list_low max_visible_partition();" + contains "partitions=1/2 (p1)" + } + + // 5b) LIST empty -> empty + sql """DROP TABLE IF EXISTS mvp_list_empty;""" + sql """ + CREATE TABLE mvp_list_empty ( + id INT, + val BIGINT + ) DUPLICATE KEY(`id`) + PARTITION BY LIST(`id`) + ( + PARTITION p1 VALUES IN ("1","2","3") + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES ("replication_num" = "1"); + """ + qt_sql_list_empty """select * from mvp_list_empty max_visible_partition();""" + explain { + sql "select * from mvp_list_empty max_visible_partition();" + contains "VEMPTYSET" + } + + // 6) max_visible_partition() must NOT be a short-circuit point query: caching the resolved + // partition into a prepared plan would return stale results as data drifts. Use a MOW + + // row-store PK table (so the plain point query qualifies) and verify the variant is excluded. + sql """DROP TABLE IF EXISTS mvp_sc;""" + sql """ + CREATE TABLE mvp_sc ( + id BIGINT, + val BIGINT + ) UNIQUE KEY(`id`) + PARTITION BY RANGE(`id`) + ( + PARTITION `p1` VALUES LESS THAN ('5'), + PARTITION `p2` VALUES LESS THAN ('10') + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "store_row_column" = "true" + ); + """ + sql "INSERT INTO mvp_sc VALUES(1, 10)" + sql "INSERT INTO mvp_sc VALUES(7, 70)" + sql "SET enable_short_circuit_query=true" + + // baseline: plain full-key point query does short-circuit + explain { + sql "select * from mvp_sc where id = 7" + contains "SHORT-CIRCUIT" + } + // with max_visible_partition(): must fall back to normal planning (no short-circuit) + explain { + sql "select * from mvp_sc max_visible_partition() where id = 7" + notContains "SHORT-CIRCUIT" + } + // and it still resolves to the latest visible partition (p2) with correct result + explain { + sql "select * from mvp_sc max_visible_partition() where id = 7" + contains "partitions=1/2 (p2)" + } + qt_sql_sc_mvp """select * from mvp_sc max_visible_partition() where id = 7;""" + + // 7) TRUNCATE vs DELETE visibility: TRUNCATE resets the partition version to + // PARTITION_INIT_VERSION (partition becomes "no data"), while DELETE advances the version + // (partition still counts as visible even with all rows gone). + sql """DROP TABLE IF EXISTS mvp_td;""" + sql """ + CREATE TABLE mvp_td ( + id BIGINT, + val BIGINT + ) DUPLICATE KEY(`id`) + PARTITION BY RANGE(`id`) + ( + PARTITION `p1` VALUES LESS THAN ('5'), + PARTITION `p2` VALUES LESS THAN ('10') + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_num" = "1"); + """ + sql "INSERT INTO mvp_td VALUES(1, 10)" + sql "INSERT INTO mvp_td VALUES(7, 70)" + // both partitions have data -> latest visible is p2 + explain { + sql "select * from mvp_td max_visible_partition();" + contains "partitions=1/2 (p2)" + } + + // 7a) TRUNCATE p2 resets its version -> p2 no longer visible -> max falls back to p1 + sql "truncate table mvp_td partitions (p2);" + order_qt_sql_td_truncate """select * from mvp_td max_visible_partition();""" + explain { + sql "select * from mvp_td max_visible_partition();" + contains "partitions=1/2 (p1)" + } + + // 7b) DELETE keeps the version advanced -> p2 still the latest visible partition (rows gone) + sql """DROP TABLE IF EXISTS mvp_td2;""" + sql """ + CREATE TABLE mvp_td2 ( + id BIGINT, + val BIGINT + ) DUPLICATE KEY(`id`) + PARTITION BY RANGE(`id`) + ( + PARTITION `p1` VALUES LESS THAN ('5'), + PARTITION `p2` VALUES LESS THAN ('10') + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_num" = "1"); + """ + sql "INSERT INTO mvp_td2 VALUES(1, 10)" + sql "INSERT INTO mvp_td2 VALUES(7, 70)" + sql "DELETE FROM mvp_td2 PARTITION p2 WHERE id = 7;" + // p2 version advanced by DELETE -> still the latest visible partition + explain { + sql "select * from mvp_td2 max_visible_partition();" + contains "partitions=1/2 (p2)" + } + // rows are gone, so scanning the latest visible partition returns empty + qt_sql_td2_delete """select * from mvp_td2 max_visible_partition();""" +}