Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
Original file line number Diff line number Diff line change
Expand Up @@ -1508,6 +1508,42 @@ public Partition getPartition(String partitionName, boolean isTempPartition) {
}
}

@Override
public Partition getMaxVisiblePartition() {
PartitionInfo partitionInfo = getPartitionInfo();
PartitionType type = partitionInfo.getType();
// LIST has no natural "latest"; UNPARTITIONED has a single partition.
// Neither participates in the range-max scan below.
if (type == PartitionType.LIST) {
// no meaningful "max" partition -> caller builds an empty query
return null;
}
if (type == PartitionType.UNPARTITIONED) {
for (Partition partition : nameToPartition.values()) {
if (partition.getVisibleVersion() > Partition.PARTITION_INIT_VERSION) {
return partition;
}
}
return null;
}
Partition result = null;
PartitionKey maxUpperBound = null;
for (Partition partition : nameToPartition.values()) {
if (partition.getVisibleVersion() <= Partition.PARTITION_INIT_VERSION) {
continue;
}
// RANGE: pick the partition with the greatest range upper bound, so it works for
// date/number partitions regardless of the partition name width
PartitionKey upper = ((RangePartitionItem) partitionInfo.getItem(partition.getId()))
.getItems().upperEndpoint();
if (maxUpperBound == null || upper.compareTo(maxUpperBound) > 0) {
maxUpperBound = upper;
result = partition;
}
}
return result;
}

// Priority is given to querying from the partition. If not found, query from the tempPartition
public Partition getPartition(long partitionId) {
Partition partition = idToPartition.get(partitionId);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,10 @@ default Partition getPartition(String name) {
return null;
}

default Partition getMaxVisiblePartition() {
return null;
}

default List<String> getFullQualifiers() {
return ImmutableList.of(getDatabase().getCatalog().getName(),
getDatabase().getFullName(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu
private final List<String> partNames;
private final List<Long> tabletIds;
private final boolean isTempPart;
private final boolean isMaxVisiblePartition;
private final List<String> hints;
private final Optional<TableSample> tableSample;
private final Optional<String> indexName;
Expand Down Expand Up @@ -91,6 +92,18 @@ public UnboundRelation(RelationId id, List<String> nameParts, List<String> partN
tableSnapshot);
}

/**
* Constructor of UnboundRelation with max visible partition flag.
*/
public UnboundRelation(RelationId id, List<String> nameParts, List<String> partNames,
boolean isTempPart, boolean isMaxVisiblePartition, List<Long> tabletIds, List<String> hints,
Optional<TableSample> tableSample, Optional<String> indexName, TableScanParams scanParams,
Optional<TableSnapshot> tableSnapshot) {
this(id, nameParts, Optional.empty(), Optional.empty(),
partNames, isTempPart, isMaxVisiblePartition, tabletIds, hints, tableSample, indexName, scanParams,
Optional.empty(), tableSnapshot);
}

public UnboundRelation(RelationId id, List<String> nameParts,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
List<String> partNames, boolean isTempPart, List<Long> tabletIds, List<String> hints,
Expand All @@ -117,11 +130,22 @@ public UnboundRelation(RelationId id, List<String> nameParts,
Optional<TableSample> tableSample, Optional<String> indexName, TableScanParams scanParams,
Optional<Pair<Integer, Integer>> indexInSqlString,
Optional<TableSnapshot> tableSnapshot) {
this(id, nameParts, groupExpression, logicalProperties, partNames, isTempPart, false, tabletIds, hints,
tableSample, indexName, scanParams, indexInSqlString, tableSnapshot);
}

private UnboundRelation(RelationId id, List<String> nameParts,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
List<String> partNames, boolean isTempPart, boolean isMaxVisiblePartition, List<Long> tabletIds,
List<String> hints, Optional<TableSample> tableSample, Optional<String> indexName,
TableScanParams scanParams, Optional<Pair<Integer, Integer>> indexInSqlString,
Optional<TableSnapshot> tableSnapshot) {
super(id, PlanType.LOGICAL_UNBOUND_RELATION, groupExpression, logicalProperties);
this.nameParts = ImmutableList.copyOf(Objects.requireNonNull(nameParts, "nameParts should not null"));
this.partNames = ImmutableList.copyOf(Objects.requireNonNull(partNames, "partNames should not null"));
this.tabletIds = ImmutableList.copyOf(Objects.requireNonNull(tabletIds, "tabletIds should not null"));
this.isTempPart = isTempPart;
this.isMaxVisiblePartition = isMaxVisiblePartition;
this.hints = ImmutableList.copyOf(Objects.requireNonNull(hints, "hints should not be null."));
this.tableSample = tableSample;
this.indexName = indexName;
Expand All @@ -148,23 +172,23 @@ public LogicalProperties computeLogicalProperties() {
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new UnboundRelation(relationId, nameParts,
groupExpression, Optional.of(getLogicalProperties()),
partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams,
partNames, isTempPart, isMaxVisiblePartition, tabletIds, hints, tableSample, indexName, scanParams,
indexInSqlString, tableSnapshot);
}

@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new UnboundRelation(relationId, nameParts, groupExpression,
logicalProperties, partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams,
indexInSqlString, tableSnapshot);
logicalProperties, partNames, isTempPart, isMaxVisiblePartition, tabletIds, hints, tableSample,
indexName, scanParams, indexInSqlString, tableSnapshot);
}

public UnboundRelation withIndexInSql(Pair<Integer, Integer> index) {
// SQL source indexing annotates relation identity only; it must not erase scan semantics.
return new UnboundRelation(relationId, nameParts, groupExpression,
Optional.of(getLogicalProperties()), partNames, isTempPart, tabletIds, hints, tableSample, indexName,
scanParams, Optional.of(index), tableSnapshot);
Optional.of(getLogicalProperties()), partNames, isTempPart, isMaxVisiblePartition, tabletIds, hints,
tableSample, indexName, scanParams, Optional.of(index), tableSnapshot);
}

@Override
Expand Down Expand Up @@ -221,7 +245,8 @@ public boolean equals(Object o) {
return false;
}
UnboundRelation that = (UnboundRelation) o;
return isTempPart == that.isTempPart && Objects.equals(nameParts, that.nameParts)
return isTempPart == that.isTempPart && isMaxVisiblePartition == that.isMaxVisiblePartition
&& Objects.equals(nameParts, that.nameParts)
&& Objects.equals(partNames, that.partNames) && Objects.equals(tabletIds,
that.tabletIds) && Objects.equals(hints, that.hints) && Objects.equals(tableSample,
that.tableSample) && Objects.equals(indexName, that.indexName) && Objects.equals(
Expand All @@ -247,6 +272,10 @@ public boolean isTempPart() {
return isTempPart;
}

public boolean isMaxVisiblePartition() {
return isMaxVisiblePartition;
}

public List<Long> getTabletIds() {
return tabletIds;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2752,6 +2752,7 @@ public LogicalPlan visitTableName(TableNameContext ctx) {
List<String> nameParts = visitMultipartIdentifier(ctx.multipartIdentifier());
List<String> partitionNames = new ArrayList<>();
boolean isTempPart = false;
boolean isMaxVisiblePartition = ctx.maxVisiblePartition() != null;
if (ctx.specifiedPartition() != null) {
isTempPart = ctx.specifiedPartition().TEMPORARY() != null;
if (ctx.specifiedPartition().identifier() != null) {
Expand Down Expand Up @@ -2804,7 +2805,7 @@ public LogicalPlan visitTableName(TableNameContext ctx) {
TableSample tableSample = ctx.sample() == null ? null : (TableSample) visit(ctx.sample());
UnboundRelation relation = new UnboundRelation(
StatementScopeIdGenerator.newRelationId(),
nameParts, partitionNames, isTempPart, tabletIdLists, relationHints,
nameParts, partitionNames, isTempPart, isMaxVisiblePartition, tabletIdLists, relationHints,
Optional.ofNullable(tableSample), indexName, scanParams, Optional.ofNullable(tableSnapshot));

LogicalPlan checkedRelation = LogicalPlanBuilderAssistant.withCheckPolicy(relation);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@
import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer;
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalHudiScan;
Expand Down Expand Up @@ -260,6 +261,15 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation,
} else if (unboundRelation.getScanParams() != null) {
unboundRelation.getScanParams().validateOlapTable();
}
// max_visible_partition() with no visible partition -> empty result instead of
// falling through to a full-table scan (empty partIds otherwise means "all partitions")
if (unboundRelation.isMaxVisiblePartition() && CollectionUtils.isEmpty(partIds)) {
scan = new LogicalOlapScan(unboundRelation.getRelationId(),
(OlapTable) table, qualifier, ImmutableList.of(),
tabletIds, unboundRelation.getHints(),
unboundRelation.getTableSample(), ImmutableList.of());
return new LogicalEmptyRelation(unboundRelation.getRelationId(), scan.getOutput());
}
if (!CollectionUtils.isEmpty(partIds) && !unboundRelation.getIndexName().isPresent()) {
scan = new LogicalOlapScan(unboundRelation.getRelationId(),
(OlapTable) table, qualifier, partIds,
Expand Down Expand Up @@ -751,6 +761,15 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio
}

List<String> qualifierWithoutTableName = qualifiedTableName.subList(0, qualifiedTableName.size() - 1);
// max_visible_partition() only works on OLAP-backed tables; reject others explicitly
// so the flag is not silently ignored.
if (unboundRelation.isMaxVisiblePartition()) {
TableIf.TableType type = table.getType();
if (type != TableIf.TableType.OLAP && type != TableIf.TableType.MATERIALIZED_VIEW) {
throw new AnalysisException("max_visible_partition() is only supported on OLAP table "
+ "or materialized view, but table " + table.getName() + " is " + type);
}
}
cascadesContext.getStatementContext().loadSnapshots(
table,
unboundRelation.getTableSnapshot(),
Expand Down Expand Up @@ -1014,14 +1033,19 @@ private Plan parseAndAnalyzeView(TableIf view, String ddlSql, CascadesContext pa

private List<Long> getPartitionIds(TableIf t, UnboundRelation unboundRelation, List<String> qualifier) {
List<String> parts = unboundRelation.getPartNames();
if (CollectionUtils.isEmpty(parts)) {
if (CollectionUtils.isEmpty(parts) && !unboundRelation.isMaxVisiblePartition()) {
return ImmutableList.of();
}
if (!t.isManagedTable()) {
throw new AnalysisException(String.format(
"Only OLAP table is support select by partition for now,"
+ "Table: %s is not OLAP table", t.getName()));
}
if (unboundRelation.isMaxVisiblePartition()) {
Partition partition = t.getMaxVisiblePartition();
// no visible partition means empty data, just return empty result
return partition == null ? ImmutableList.of() : ImmutableList.of(partition.getId());
}
return parts.stream().map(name -> {
Partition part = ((OlapTable) t).getPartition(name, unboundRelation.isTempPart());
if (part == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,7 @@ MATCH_PHRASE_PREFIX: 'MATCH_PHRASE_PREFIX';
MATCH_REGEXP: 'MATCH_REGEXP';
MATERIALIZED: 'MATERIALIZED';
MAX: 'MAX';
MAX_VISIBLE_PARTITION: 'MAX_VISIBLE_PARTITION';
MAXVALUE: 'MAXVALUE';
MEMO:'MEMO';
MERGE: 'MERGE';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1486,8 +1486,8 @@ optScanParams
;

relationPrimary
: multipartIdentifier optScanParams? materializedViewName? tableSnapshot? specifiedPartition?
tabletList? tableAlias sample? relationHint? lateralView* #tableName
: multipartIdentifier optScanParams? materializedViewName? tableSnapshot?
(specifiedPartition | maxVisiblePartition)? tabletList? tableAlias sample? relationHint? lateralView* #tableName
| LEFT_PAREN query RIGHT_PAREN tableAlias lateralView* #aliasedQuery
| tvfName=identifier LEFT_PAREN
(properties=propertyItemList)?
Expand Down Expand Up @@ -1831,6 +1831,10 @@ specifiedPartition
| TEMPORARY? PARTITIONS identifierList
;

maxVisiblePartition
: MAX_VISIBLE_PARTITION LEFT_PAREN RIGHT_PAREN
;

constant
: NULL #nullLiteral
| type=(DATE | DATEV1 | DATEV2 | TIMESTAMP) STRING_LITERAL #typeConstructor
Expand Down Expand Up @@ -2217,6 +2221,7 @@ nonReserved
| MATCH_NAME_GLOB
| MATERIALIZED
| MAX
| MAX_VISIBLE_PARTITION
| MEMO
| MERGE
| MID
Expand Down
16 changes: 16 additions & 0 deletions regression-test/data/nereids_syntax_p0/max_visible_partition.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql_range --
7 20

-- !sql_mtmv --
7 20

-- !sql_unpart --
1 10
2 20

-- !sql_unpart_empty --

-- !sql_list --

-- !sql_list_empty --
Loading
Loading