diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/joinhint/DistributeHintTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/joinhint/DistributeHintTest.java index 7f3b6aa12c5338..ee60a9eec86359 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/joinhint/DistributeHintTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/joinhint/DistributeHintTest.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.datasets.tpch.TPCHTestBase; import org.apache.doris.nereids.properties.SelectHint; import org.apache.doris.nereids.properties.SelectHintLeading; +import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalSelectHint; @@ -30,124 +31,130 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Random; import java.util.Set; +/** + * A hint only tells the optimizer which plan to prefer; it must never change the result of the + * query. Both tests below build a random join graph, evaluate it with an in-memory join evaluator, + * optimize it, and assert the evaluated result is unchanged. + */ public class DistributeHintTest extends TPCHTestBase { - private int used = 0; - private int unused = 0; - - private int syntaxError = 0; - - private int successCases = 0; - - private int unsuccessCases = 0; - - private List failCases = new ArrayList<>(); + // Leading order is only unconditionally valid for inner joins, so the permutations are drawn + // from a fixed seed to keep a failure reproducible from the test output. + private static final long LEADING_SEED = 20260729L; + /** + * A distribute hint (broadcast / shuffle) picks a physical strategy, so it can never change the + * result. This is the only place that covers distribute hints against random join graphs. + */ @Test - public void testLeading() { + public void testHintJoin() { for (int t = 3; t < 10; t++) { for (int e = t - 1; e <= (t * (t - 1)) / 2; e++) { for (int i = 0; i < 10; i++) { - System.out.println("TableNumber: " + String.valueOf(t) + " EdgeNumber: " + e + " Iteration: " + i); - randomTest(t, e, false, true); + HyperGraphBuilderOld builder = new HyperGraphBuilderOld(); + Plan plan = builder.buildJoinPlanWithJoinHint(t, e); + plan = new LogicalProject(plan.getOutput(), plan); + assertOptimizeKeepsResult(builder, plan, plan, + String.format("distribute hint changed the result (tables=%d, edges=%d, iter=%d)", + t, e, i)); } } } - int totalCases = successCases + unsuccessCases; - System.out.println("TotalCases: " + totalCases + "\tSuccessCases: " + successCases + unsuccessCases + "\tUnSuccessCases: " + 0); - for (String treePlan : failCases) { - System.out.println(treePlan); - } } + /** + * Reordering inner joins is always semantics-preserving, so any leading order must produce the + * same result as the un-hinted plan. + */ @Test - public void testHintJoin() { - for (int t = 3; t < 10; t++) { + public void testLeading() { + Random random = new Random(LEADING_SEED); + for (int t = 3; t < 7; t++) { for (int e = t - 1; e <= (t * (t - 1)) / 2; e++) { - for (int i = 0; i < 10; i++) { - System.out.println("TableNumber: " + String.valueOf(t) + " EdgeNumber: " + e + " Iteration: " + i); - randomTest(t, e, true, false); + for (int i = 0; i < 3; i++) { + HyperGraphBuilderOld builder = + new HyperGraphBuilderOld(ImmutableSet.of(JoinType.INNER_JOIN)); + Plan plan = builder.randomBuildPlanWith(t, e); + plan = new LogicalProject(plan.getOutput(), plan); + for (int p = 0; p < Math.min(t, 4); p++) { + List order = leadingOrder(t, random); + Plan leadingPlan = withLeadingHint(order, plan); + assertOptimizeKeepsResult(builder, plan, leadingPlan, + String.format("leading %s changed the result (tables=%d, edges=%d, iter=%d)", + order, t, e, i)); + } } } } - int totalCases = successCases + unsuccessCases; - System.out.println("TotalCases: " + totalCases + "\tSuccessCases: " + successCases + "\tUnSuccessCases: " + unsuccessCases); - for (String treePlan : failCases) { - System.out.println(treePlan); + } + + /** + * Same property as {@link #testLeading}, but the graph also contains outer / semi / anti joins, + * for which an arbitrary leading order is generally not a valid join order. Today Doris applies + * such a hint anyway and the result changes -- extra null-padded rows appear. Whether the hint + * should be rejected and ignored instead is an open question for the Nereids owners, so this + * test records the case rather than asserting a behaviour nobody has signed off on. + * + *

Reproduces at 3 tables. Re-enable once the intended behaviour is decided. + */ + @Disabled("leading hint over outer joins is not result-preserving today; intended behaviour undecided") + @Test + public void testLeadingWithOuterJoin() { + Random random = new Random(LEADING_SEED); + for (int t = 3; t < 6; t++) { + for (int e = t - 1; e <= (t * (t - 1)) / 2; e++) { + HyperGraphBuilderOld builder = new HyperGraphBuilderOld(); + Plan plan = builder.randomBuildPlanWith(t, e); + plan = new LogicalProject(plan.getOutput(), plan); + List order = leadingOrder(t, random); + assertOptimizeKeepsResult(builder, plan, withLeadingHint(order, plan), + String.format("leading %s over outer joins changed the result (tables=%d, edges=%d)", + order, t, e)); + } } } - private Plan generateLeadingHintPlan(int tableNum, Plan childPlan) { - ImmutableList.Builder hints = ImmutableList.builder(); - List leadingParameters = new ArrayList(); + private List leadingOrder(int tableNum, Random random) { + List order = new ArrayList<>(); for (int i = 0; i < tableNum; i++) { - leadingParameters.add(String.valueOf(i)); + order.add(String.valueOf(i)); } - Collections.shuffle(leadingParameters); - System.out.println("LeadingHint: " + leadingParameters.toString()); - hints.add(new SelectHintLeading("Leading", leadingParameters, ImmutableMap.of())); - return new LogicalSelectHint<>(hints.build(), childPlan); + Collections.shuffle(order, random); + return order; } - private void randomTest(int tableNum, int edgeNum, boolean withJoinHint, boolean withLeading) { - HyperGraphBuilderOld hyperGraphBuilder = new HyperGraphBuilderOld(); - Plan plan = withJoinHint ? hyperGraphBuilder.buildJoinPlanWithJoinHint(tableNum, edgeNum) : - hyperGraphBuilder.randomBuildPlanWith(tableNum, edgeNum); - plan = new LogicalProject(plan.getOutput(), plan); - Set> res1 = hyperGraphBuilder.evaluate(plan); - if (!withLeading) { - CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan); - hyperGraphBuilder.initStats("tpch", cascadesContext); - Plan optimizedPlan = PlanChecker.from(cascadesContext) - .analyze() - .optimize() - .getBestPlanTree(); - - Set> res2 = hyperGraphBuilder.evaluate(optimizedPlan); - if (!res1.equals(res2)) { - System.out.println(plan.treeString()); - System.out.println(optimizedPlan.treeString()); - cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan); - PlanChecker.from(cascadesContext).dpHypOptimize().getBestPlanTree(); - System.out.println(res1); - System.out.println(res2); - unsuccessCases++; - failCases.add(plan.treeString()); - failCases.add(optimizedPlan.treeString()); - } - successCases++; - } else { - // generate select hint - for (int i = 0; i < (tableNum * tableNum - 1); i++) { - Plan leadingPlan = generateLeadingHintPlan(tableNum, plan); - CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(connectContext, leadingPlan); - hyperGraphBuilder.initStats("tpch", cascadesContext); - Plan optimizedPlan = PlanChecker.from(cascadesContext) - .analyze() - .optimize() - .getBestPlanTree(); + private Plan withLeadingHint(List order, Plan childPlan) { + ImmutableList.Builder hints = ImmutableList.builder(); + hints.add(new SelectHintLeading("Leading", order, ImmutableMap.of())); + return new LogicalSelectHint<>(hints.build(), childPlan); + } - Set> res2 = hyperGraphBuilder.evaluate(optimizedPlan); - if (!res1.equals(res2)) { - System.out.println(leadingPlan.treeString()); - System.out.println(optimizedPlan.treeString()); - cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan); - PlanChecker.from(cascadesContext).dpHypOptimize().getBestPlanTree(); - System.out.println(res1); - System.out.println(res2); - unsuccessCases++; - failCases.add(leadingPlan.treeString()); - failCases.add(optimizedPlan.treeString()); - } - successCases++; - } - } + /** + * Evaluates {@code originalPlan}, optimizes {@code planToOptimize}, and asserts both produce the + * same tuples. + */ + private void assertOptimizeKeepsResult(HyperGraphBuilderOld builder, Plan originalPlan, + Plan planToOptimize, String message) { + Set> expected = builder.evaluate(originalPlan); + CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(connectContext, planToOptimize); + builder.initStats("tpch", cascadesContext); + Plan optimizedPlan = PlanChecker.from(cascadesContext) + .analyze() + .optimize() + .getBestPlanTree(); + Assertions.assertEquals(expected, builder.evaluate(optimizedPlan), + message + "\noriginal plan:\n" + originalPlan.treeString() + + "\noptimized plan:\n" + optimizedPlan.treeString()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java deleted file mode 100644 index bfecdbd4181049..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java +++ /dev/null @@ -1,207 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.exploration.mv; - -import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; -import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; -import org.apache.doris.nereids.sqltest.SqlTestBase; -import org.apache.doris.nereids.trees.plans.GroupPlan; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.util.PlanChecker; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -class BuildStructInfoTest extends SqlTestBase { - - @Test - void testSimpleSQL() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "select * from T1, T2, T3, T4 " - + "where " - + "T1.id = T2.id and " - + "T2.score = T3.score and " - + "T3.id = T4.id"; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .deriveStats() - .matches(logicalJoin() - .when(j -> { - HyperGraph.builderForMv(j); - return true; - })); - - } - - @Test - void testStructInfoNode() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "select * from T1 inner join " - + "(select sum(id) as id from T2 where id = 1) T2 " - + "on T1.id = T2.id"; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .deriveStats() - .matches(logicalJoin() - .when(j -> { - HyperGraph hyperGraph = HyperGraph.builderForMv(j).build(); - Assertions.assertTrue(hyperGraph.getNodes().stream() - .allMatch(n -> n.getPlan() - .collectToList(GroupPlan.class::isInstance).isEmpty())); - return true; - })); - - } - - @Test - void testFilter() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "select * from T1 left outer join " - + " (select id from T2 where id = 1) T2 " - + "on T1.id = T2.id "; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .matches(logicalJoin() - .when(j -> { - HyperGraph structInfo = HyperGraph.builderForMv(j).build(); - Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin()); - Assertions.assertEquals(0, structInfo.getFilterEdge(0).getLeftRejectEdge().size()); - Assertions.assertEquals(1, structInfo.getFilterEdge(0).getRightRejectEdge().size()); - return true; - })); - - sql = "select * from (select id from T1 where id = 0) T1 left outer join T2 " - + "on T1.id = T2.id "; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .matches(logicalJoin() - .when(j -> { - HyperGraph structInfo = HyperGraph.builderForMv(j).build(); - Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin()); - return true; - })); - } - - @Test - void testPlanCheckerWithJoin() { - // Should not make scan to empty relation when the table used by materialized view has no data - connectContext.getSessionVariable().setDisableNereidsRules( - "OLAP_SCAN_PARTITION_PRUNE" - + ",PRUNE_EMPTY_PARTITION" - + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" - + ",ELIMINATE_CONST_JOIN_CONDITION" - + ",CONSTANT_PROPAGATION" - ); - PlanChecker.from(connectContext) - .checkExplain("select * from " - + "(select * from lineitem " - + "where l_shipdate >= \"2023-12-01\" and l_shipdate <= \"2023-12-03\") t1 " - + "left join " - + "(select * from orders where o_orderdate >= \"2023-12-01\" and o_orderdate <= \"2023-12-03\" ) t2 " - + "on t1.l_orderkey = o_orderkey;", - nereidsPlanner -> { - Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - PlanCheckContext checkContext = PlanCheckContext.of( - AbstractMaterializedViewRule.SUPPORTED_JOIN_TYPE_SET); - Boolean result = rewrittenPlan.child(0).accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext); - Assertions.assertTrue(result); - Assertions.assertFalse(checkContext.isContainsTopAggregate()); - }); - } - - @Test - void testPlanCheckerWithAggregate() { - // Should not make scan to empty relation when the table used by materialized view has no data - connectContext.getSessionVariable().setDisableNereidsRules( - "OLAP_SCAN_PARTITION_PRUNE" - + ",PRUNE_EMPTY_PARTITION" - + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" - + ",ELIMINATE_CONST_JOIN_CONDITION" - + ",CONSTANT_PROPAGATION" - ); - PlanChecker.from(connectContext) - .checkExplain("SELECT l.L_SHIPDATE AS ship_data_alias, o.O_ORDERDATE, count(*) " - + "FROM " - + "lineitem as l " - + "LEFT JOIN " - + "(SELECT abs(O_TOTALPRICE + 10) as c1_abs, O_CUSTKEY, O_ORDERDATE, O_ORDERKEY " - + "FROM orders) as o " - + "ON l.L_ORDERKEY = o.O_ORDERKEY " - + "JOIN " - + "(SELECT abs(sqrt(PS_SUPPLYCOST)) as c2_abs, PS_AVAILQTY, PS_PARTKEY, PS_SUPPKEY " - + "FROM partsupp) as ps " - + "ON l.L_PARTKEY = ps.PS_PARTKEY and l.L_SUPPKEY = ps.PS_SUPPKEY " - + "GROUP BY l.L_SHIPDATE, o.O_ORDERDATE ", - nereidsPlanner -> { - Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - PlanCheckContext checkContext = PlanCheckContext.of( - AbstractMaterializedViewRule.SUPPORTED_JOIN_TYPE_SET); - Boolean result = rewrittenPlan.child(0).accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext); - Assertions.assertTrue(result); - Assertions.assertTrue(checkContext.isContainsTopAggregate()); - }); - } - - @Test - void testPlanCheckerScanAggregate() { - // Should not make scan to empty relation when the table used by materialized view has no data - connectContext.getSessionVariable().setDisableNereidsRules( - "OLAP_SCAN_PARTITION_PRUNE" - + ",PRUNE_EMPTY_PARTITION" - + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" - + ",ELIMINATE_CONST_JOIN_CONDITION" - + ",CONSTANT_PROPAGATION" - ); - PlanChecker.from(connectContext) - .checkExplain("select l.L_SHIPDATE, count(*) from lineitem l " - + "GROUP BY l.L_SHIPDATE", - nereidsPlanner -> { - Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - PlanCheckContext checkContext = PlanCheckContext.of( - AbstractMaterializedViewRule.SUPPORTED_JOIN_TYPE_SET); - Boolean result = rewrittenPlan.child(0).accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext); - Assertions.assertFalse(result); - }); - } - - @Test - void testPlanCheckerOnlyScan() { - // Should not make scan to empty relation when the table used by materialized view has no data - connectContext.getSessionVariable().setDisableNereidsRules( - "OLAP_SCAN_PARTITION_PRUNE" - + ",PRUNE_EMPTY_PARTITION" - + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" - + ",ELIMINATE_CONST_JOIN_CONDITION" - + ",CONSTANT_PROPAGATION" - ); - PlanChecker.from(connectContext) - .checkExplain("select l.L_SHIPDATE from lineitem l ", - nereidsPlanner -> { - Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - PlanCheckContext checkContext = PlanCheckContext.of( - AbstractMaterializedViewRule.SUPPORTED_JOIN_TYPE_SET); - Boolean result = rewrittenPlan.child(0).accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext); - Assertions.assertTrue(result); - Assertions.assertFalse(checkContext.isContainsTopAggregate()); - }); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java deleted file mode 100644 index 20697e78912d20..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java +++ /dev/null @@ -1,341 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.exploration.mv; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; -import org.apache.doris.nereids.rules.RuleSet; -import org.apache.doris.nereids.sqltest.SqlTestBase; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.util.PlanChecker; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -class EliminateJoinTest extends SqlTestBase { - @Test - void testLOJWithGroupBy() { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select * from T1", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - CascadesContext c2 = createCascadesContext( - "select * from T1 left outer join (select id from T2 group by id) T2 " - + "on T1.id = T2.id ", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - CascadesContext c3 = createCascadesContext( - "select * from T1 left outer join (select id as id2 from T2 group by id) T2 " - + "on T1.id = T2.id2 ", - connectContext - ); - Plan p3 = PlanChecker.from(c3) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - HyperGraph h3 = HyperGraph.builderForMv(p3).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertTrue(!res.isInvalid()); - Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3, - constructContext(p1, p2, c1)).isInvalid()); - Assertions.assertTrue(res.getViewExpressions().isEmpty()); - } - - @Test - void testLOJWithUK() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select * from T1", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - addConstraint("alter table T2 add constraint uk unique (id)"); - CascadesContext c2 = createCascadesContext( - "select * from T1 left outer join T2 " - + "on T1.id = T2.id ", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertTrue(!res.isInvalid()); - Assertions.assertTrue(res.getViewExpressions().isEmpty()); - dropConstraint("alter table T2 drop constraint uk"); - } - - @Test - void testLOJWithUKAndOtherJoinConjuncts() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select * from T1", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - addConstraint("alter table T2 add constraint uk_other_join_conjunct unique (id)"); - try { - CascadesContext c2 = createCascadesContext( - "select * from T1 left outer join T2 " - + "on T1.id = T2.id and T2.id = 1", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertFalse(res.isInvalid()); - Assertions.assertTrue(res.getViewExpressions().isEmpty()); - } finally { - dropConstraint("alter table T2 drop constraint uk_other_join_conjunct"); - } - } - - @Test - void testLOJWithUKAndFilterOnEliminatedNode() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select T1.id from T1", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - addConstraint("alter table T2 add constraint uk_loj_filter unique (id)"); - try { - CascadesContext c2 = createCascadesContext( - "select T1.id from T1 left outer join T2 " - + "on T1.id = T2.id where T2.id is null", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertTrue(res.isInvalid()); - } finally { - dropConstraint("alter table T2 drop constraint uk_loj_filter"); - } - } - - @Test - void testInnerJoinWithPKFKAndSlotFreeFilterOnEliminatedNode() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select * from T1", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - addConstraint("alter table T2 add constraint pk_slot_free_filter primary key (id)"); - addConstraint("alter table T1 add constraint fk_slot_free_filter foreign key (id) references T2(id)"); - try { - CascadesContext c2 = createCascadesContext( - "select * from T1 inner join (select * from T2 where 1 = 0) T2 " - + "on T1.id = T2.id", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertTrue(res.isInvalid()); - } finally { - dropConstraint("alter table T1 drop constraint fk_slot_free_filter"); - dropConstraint("alter table T2 drop constraint pk_slot_free_filter"); - } - } - - @Test - void testLOJWithPKFK() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select * from T1", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - addConstraint("alter table T2 add constraint pk primary key (id)"); - addConstraint("alter table T1 add constraint fk foreign key (id) references T2(id)"); - CascadesContext c2 = createCascadesContext( - "select * from T1 inner join T2 " - + "on T1.id = T2.id ", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - CascadesContext c3 = createCascadesContext( - "select * from T1 inner join (select id as id2 from T2) T2 " - + "on T1.id = T2.id2 ", - connectContext - ); - Plan p3 = PlanChecker.from(c3) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - HyperGraph h3 = HyperGraph.builderForMv(p3).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertTrue(!res.isInvalid()); - Assertions.assertTrue(res.getViewExpressions().isEmpty()); - Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3, constructContext(p1, p2, c1)).isInvalid()); - dropConstraint("alter table T2 drop constraint pk"); - } - - @Test - void testInnerJoinWithPKFKAndMultiNodeResidualFilter() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select * from T1", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - addConstraint("alter table T2 add constraint pk primary key (id)"); - addConstraint("alter table T1 add constraint fk foreign key (id) references T2(id)"); - CascadesContext c2 = createCascadesContext( - "select * from T1 inner join T2 " - + "on T1.id = T2.id where T1.score > T2.score", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertTrue(res.isInvalid()); - dropConstraint("alter table T2 drop constraint pk"); - } - - @Disabled - @Test - void testLOJWithPKFKAndUK1() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select * from T1", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - addConstraint("alter table T2 add constraint pk primary key (id)"); - addConstraint("alter table T1 add constraint fk foreign key (id) references T2(id)"); - addConstraint("alter table T3 add constraint uk unique (id)"); - CascadesContext c2 = createCascadesContext( - "select * from (select T1.*, T3.id as id3 from T1 left outer join T3 on T1.id = T3.id) T1 inner join T2 " - + "on T1.id = T2.id ", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertTrue(!res.isInvalid()); - Assertions.assertTrue(res.getViewExpressions().isEmpty()); - dropConstraint("alter table T2 drop constraint pk"); - dropConstraint("alter table T3 drop constraint uk"); - } - - @Test - void testLOJWithPKFKAndUK2() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select * from T1", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - addConstraint("alter table T2 add constraint pk primary key (id)"); - addConstraint("alter table T1 add constraint fk foreign key (id) references T2(id)"); - addConstraint("alter table T3 add constraint uk unique (id)"); - CascadesContext c2 = createCascadesContext( - "select * from (select T1.*, T2.id as id2 from T1 inner join T2 on T1.id = T2.id) T1 left outer join T3 " - + "on T1.id = T3.id ", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertTrue(!res.isInvalid()); - Assertions.assertTrue(res.getViewExpressions().isEmpty()); - dropConstraint("alter table T2 drop constraint pk"); - dropConstraint("alter table T3 drop constraint uk"); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java deleted file mode 100644 index af36afe715112d..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java +++ /dev/null @@ -1,98 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.exploration.mv; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; -import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; -import org.apache.doris.nereids.rules.RuleSet; -import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; -import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; -import org.apache.doris.nereids.sqltest.SqlTestBase; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.util.PlanChecker; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import java.util.Objects; - -class HyperGraphAggTest extends SqlTestBase { - @Test - void testJoinWithAgg() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - CascadesContext c2 = createCascadesContext( - "select * from T1 inner join" - + "(" - + "select id from T2 group by id" - + ") T2 " - + "on T1.id = T2.id ", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p2).build(); - Assertions.assertEquals("id", Objects.requireNonNull(((StructInfoNode) h1.getNode(1)).getExpressions()).get(0).toSql()); - } - - @Disabled - @Test - void testIJWithAgg() { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES"); - CascadesContext c1 = createCascadesContext( - "select * from T1 inner join T2 " - + "on T1.id = T2.id", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - CascadesContext c2 = createCascadesContext( - "select * from T1 inner join" - + "(" - + "select id from T2 group by id" - + ") T2 " - + "on T1.id = T2.id ", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); - Assertions.assertTrue(!res.isInvalid()); - Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); - } - - LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { - StructInfo st1 = StructInfo.of(p1, p1, null); - StructInfo st2 = StructInfo.of(p2, p2, null); - RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations(), 8) - .get(0); - SlotMapping sm = SlotMapping.generate(rm); - return LogicalCompatibilityContext.from(rm, sm, st1, st2); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java deleted file mode 100644 index db02e45dd48ffc..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java +++ /dev/null @@ -1,185 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.exploration.mv; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; -import org.apache.doris.nereids.rules.RuleSet; -import org.apache.doris.nereids.sqltest.SqlTestBase; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.util.PlanChecker; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -class HyperGraphComparatorTest extends SqlTestBase { - - @Override - protected String getDisableNereidsRules() { - return "INFER_PREDICATES,CONSTANT_PROPAGATION,PRUNE_EMPTY_PARTITION"; - } - - @Test - void testInnerJoinAndLOJ() { - CascadesContext c1 = createCascadesContext( - "select * from T1 inner join T2 " - + "on T1.id = T2.id " - + "inner join T3 on T1.id = T3.id", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - CascadesContext c2 = createCascadesContext( - "select * from T1 left outer join T2 " - + "on T1.id = T2.id " - + "left outer join T3 on T1.id = T3.id", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertFalse(res.isInvalid()); - Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); - } - - @Test - void testIJAndLojAssoc() { - CascadesContext c1 = createCascadesContext( - "select * from T1 inner join T3 " - + "on T1.id = T3.id " - + "inner join T2 on T1.id = T2.id", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - CascadesContext c2 = createCascadesContext( - "select * from T1 left outer join T2 " - + "on T1.id = T2.id " - + "left outer join T3 on T1.id = T3.id", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertFalse(res.isInvalid()); - Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); - } - - @Test - void testIJAndLojAssocWithFilter() { - CascadesContext c1 = createCascadesContext( - "select * from T1 inner join T3 " - + "on T1.id = T3.id " - + "inner join T2 on T1.id = T2.id", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - CascadesContext c2 = createCascadesContext( - "select * from T1 left outer join " - + "(select * from T2 where T2.id = 1) T2 " - + "on T1.id = T2.id " - + "left outer join T3 on T1.id = T3.id", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertFalse(res.isInvalid()); - Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); - } - - @Test - void testIJAndLojAssocWithJoinCond() { - CascadesContext c1 = createCascadesContext( - "select * from T1 inner join T3 " - + "on T1.id = T3.id " - + "inner join T2 on T1.id = T2.id", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - CascadesContext c2 = createCascadesContext( - "select * from T1 left join T2 " - + "on T1.id = T2.id " - + "left join T3 on T1.id = T3.id", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertFalse(res.isInvalid()); - Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); - } - - @Test - void testJoinEliminateShouldFail() { - CascadesContext c1 = createCascadesContext( - "select * from T1 inner join T2 " - + "on T1.id = T2.id", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - CascadesContext c2 = createCascadesContext( - "select * from T1 inner join T2 " - + "on T1.id = T2.id " - + "inner join T3 on T1.id = T3.id", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertTrue(res.isInvalid()); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MvExplorationSuiteTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MvExplorationSuiteTest.java new file mode 100644 index 00000000000000..c3ce56772a4d3c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MvExplorationSuiteTest.java @@ -0,0 +1,951 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleSet; +import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; +import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.sqltest.SqlTestBase; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.Not; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.util.PlanChecker; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Objects; + +/** + * Merged suite: these tests only need the shared fixture provided by the base class, so they are + * kept in one class on purpose. Every extra test class pays a full FE startup, which dominates the + * runtime of tests this small. + * + *

Replaces the former standalone classes: + *

+ */ +public class MvExplorationSuiteTest extends SqlTestBase { + + private static final TestMaterializedViewRule TEST_RULE = new TestMaterializedViewRule(); + + // ------------------------------------------------------------------------- + // from BuildStructInfoTest + // ------------------------------------------------------------------------- + + @Test + void testSimpleSQL() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "select * from T1, T2, T3, T4 " + + "where " + + "T1.id = T2.id and " + + "T2.score = T3.score and " + + "T3.id = T4.id"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .deriveStats() + .matches(logicalJoin() + .when(j -> { + HyperGraph.builderForMv(j); + return true; + })); + + } + + @Test + void testStructInfoNode() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "select * from T1 inner join " + + "(select sum(id) as id from T2 where id = 1) T2 " + + "on T1.id = T2.id"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .deriveStats() + .matches(logicalJoin() + .when(j -> { + HyperGraph hyperGraph = HyperGraph.builderForMv(j).build(); + Assertions.assertTrue(hyperGraph.getNodes().stream() + .allMatch(n -> n.getPlan() + .collectToList(GroupPlan.class::isInstance).isEmpty())); + return true; + })); + + } + + @Test + void testFilter() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "select * from T1 left outer join " + + " (select id from T2 where id = 1) T2 " + + "on T1.id = T2.id "; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches(logicalJoin() + .when(j -> { + HyperGraph structInfo = HyperGraph.builderForMv(j).build(); + Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin()); + Assertions.assertEquals(0, structInfo.getFilterEdge(0).getLeftRejectEdge().size()); + Assertions.assertEquals(1, structInfo.getFilterEdge(0).getRightRejectEdge().size()); + return true; + })); + + sql = "select * from (select id from T1 where id = 0) T1 left outer join T2 " + + "on T1.id = T2.id "; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches(logicalJoin() + .when(j -> { + HyperGraph structInfo = HyperGraph.builderForMv(j).build(); + Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin()); + return true; + })); + } + + @Test + void testPlanCheckerWithJoin() { + // Should not make scan to empty relation when the table used by materialized view has no data + connectContext.getSessionVariable().setDisableNereidsRules( + "OLAP_SCAN_PARTITION_PRUNE" + + ",PRUNE_EMPTY_PARTITION" + + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" + + ",ELIMINATE_CONST_JOIN_CONDITION" + + ",CONSTANT_PROPAGATION" + ); + PlanChecker.from(connectContext) + .checkExplain("select * from " + + "(select * from lineitem " + + "where l_shipdate >= \"2023-12-01\" and l_shipdate <= \"2023-12-03\") t1 " + + "left join " + + "(select * from orders where o_orderdate >= \"2023-12-01\" and o_orderdate <= \"2023-12-03\" ) t2 " + + "on t1.l_orderkey = o_orderkey;", + nereidsPlanner -> { + Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); + PlanCheckContext checkContext = PlanCheckContext.of( + AbstractMaterializedViewRule.SUPPORTED_JOIN_TYPE_SET); + Boolean result = rewrittenPlan.child(0).accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext); + Assertions.assertTrue(result); + Assertions.assertFalse(checkContext.isContainsTopAggregate()); + }); + } + + @Test + void testPlanCheckerWithAggregate() { + // Should not make scan to empty relation when the table used by materialized view has no data + connectContext.getSessionVariable().setDisableNereidsRules( + "OLAP_SCAN_PARTITION_PRUNE" + + ",PRUNE_EMPTY_PARTITION" + + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" + + ",ELIMINATE_CONST_JOIN_CONDITION" + + ",CONSTANT_PROPAGATION" + ); + PlanChecker.from(connectContext) + .checkExplain("SELECT l.L_SHIPDATE AS ship_data_alias, o.O_ORDERDATE, count(*) " + + "FROM " + + "lineitem as l " + + "LEFT JOIN " + + "(SELECT abs(O_TOTALPRICE + 10) as c1_abs, O_CUSTKEY, O_ORDERDATE, O_ORDERKEY " + + "FROM orders) as o " + + "ON l.L_ORDERKEY = o.O_ORDERKEY " + + "JOIN " + + "(SELECT abs(sqrt(PS_SUPPLYCOST)) as c2_abs, PS_AVAILQTY, PS_PARTKEY, PS_SUPPKEY " + + "FROM partsupp) as ps " + + "ON l.L_PARTKEY = ps.PS_PARTKEY and l.L_SUPPKEY = ps.PS_SUPPKEY " + + "GROUP BY l.L_SHIPDATE, o.O_ORDERDATE ", + nereidsPlanner -> { + Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); + PlanCheckContext checkContext = PlanCheckContext.of( + AbstractMaterializedViewRule.SUPPORTED_JOIN_TYPE_SET); + Boolean result = rewrittenPlan.child(0).accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext); + Assertions.assertTrue(result); + Assertions.assertTrue(checkContext.isContainsTopAggregate()); + }); + } + + @Test + void testPlanCheckerScanAggregate() { + // Should not make scan to empty relation when the table used by materialized view has no data + connectContext.getSessionVariable().setDisableNereidsRules( + "OLAP_SCAN_PARTITION_PRUNE" + + ",PRUNE_EMPTY_PARTITION" + + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" + + ",ELIMINATE_CONST_JOIN_CONDITION" + + ",CONSTANT_PROPAGATION" + ); + PlanChecker.from(connectContext) + .checkExplain("select l.L_SHIPDATE, count(*) from lineitem l " + + "GROUP BY l.L_SHIPDATE", + nereidsPlanner -> { + Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); + PlanCheckContext checkContext = PlanCheckContext.of( + AbstractMaterializedViewRule.SUPPORTED_JOIN_TYPE_SET); + Boolean result = rewrittenPlan.child(0).accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext); + Assertions.assertFalse(result); + }); + } + + @Test + void testPlanCheckerOnlyScan() { + // Should not make scan to empty relation when the table used by materialized view has no data + connectContext.getSessionVariable().setDisableNereidsRules( + "OLAP_SCAN_PARTITION_PRUNE" + + ",PRUNE_EMPTY_PARTITION" + + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" + + ",ELIMINATE_CONST_JOIN_CONDITION" + + ",CONSTANT_PROPAGATION" + ); + PlanChecker.from(connectContext) + .checkExplain("select l.L_SHIPDATE from lineitem l ", + nereidsPlanner -> { + Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); + PlanCheckContext checkContext = PlanCheckContext.of( + AbstractMaterializedViewRule.SUPPORTED_JOIN_TYPE_SET); + Boolean result = rewrittenPlan.child(0).accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext); + Assertions.assertTrue(result); + Assertions.assertFalse(checkContext.isContainsTopAggregate()); + }); + } + + // ------------------------------------------------------------------------- + // from EliminateJoinTest + // ------------------------------------------------------------------------- + + @Test + void testLOJWithGroupBy() { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext c1 = createCascadesContext( + "select * from T1", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 left outer join (select id from T2 group by id) T2 " + + "on T1.id = T2.id ", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + CascadesContext c3 = createCascadesContext( + "select * from T1 left outer join (select id as id2 from T2 group by id) T2 " + + "on T1.id = T2.id2 ", + connectContext + ); + Plan p3 = PlanChecker.from(c3) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + HyperGraph h3 = HyperGraph.builderForMv(p3).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertTrue(!res.isInvalid()); + Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3, + constructContext(p1, p2, c1)).isInvalid()); + Assertions.assertTrue(res.getViewExpressions().isEmpty()); + } + + @Test + void testLOJWithUK() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext c1 = createCascadesContext( + "select * from T1", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + addConstraint("alter table T2 add constraint uk unique (id)"); + CascadesContext c2 = createCascadesContext( + "select * from T1 left outer join T2 " + + "on T1.id = T2.id ", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertTrue(!res.isInvalid()); + Assertions.assertTrue(res.getViewExpressions().isEmpty()); + dropConstraint("alter table T2 drop constraint uk"); + } + + @Test + void testLOJWithUKAndOtherJoinConjuncts() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext c1 = createCascadesContext( + "select * from T1", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + addConstraint("alter table T2 add constraint uk_other_join_conjunct unique (id)"); + try { + CascadesContext c2 = createCascadesContext( + "select * from T1 left outer join T2 " + + "on T1.id = T2.id and T2.id = 1", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertFalse(res.isInvalid()); + Assertions.assertTrue(res.getViewExpressions().isEmpty()); + } finally { + dropConstraint("alter table T2 drop constraint uk_other_join_conjunct"); + } + } + + @Test + void testLOJWithUKAndFilterOnEliminatedNode() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext c1 = createCascadesContext( + "select T1.id from T1", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + addConstraint("alter table T2 add constraint uk_loj_filter unique (id)"); + try { + CascadesContext c2 = createCascadesContext( + "select T1.id from T1 left outer join T2 " + + "on T1.id = T2.id where T2.id is null", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertTrue(res.isInvalid()); + } finally { + dropConstraint("alter table T2 drop constraint uk_loj_filter"); + } + } + + @Test + void testInnerJoinWithPKFKAndSlotFreeFilterOnEliminatedNode() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext c1 = createCascadesContext( + "select * from T1", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + addConstraint("alter table T2 add constraint pk_slot_free_filter primary key (id)"); + addConstraint("alter table T1 add constraint fk_slot_free_filter foreign key (id) references T2(id)"); + try { + CascadesContext c2 = createCascadesContext( + "select * from T1 inner join (select * from T2 where 1 = 0) T2 " + + "on T1.id = T2.id", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertTrue(res.isInvalid()); + } finally { + dropConstraint("alter table T1 drop constraint fk_slot_free_filter"); + dropConstraint("alter table T2 drop constraint pk_slot_free_filter"); + } + } + + @Test + void testLOJWithPKFK() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext c1 = createCascadesContext( + "select * from T1", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + addConstraint("alter table T2 add constraint pk primary key (id)"); + addConstraint("alter table T1 add constraint fk foreign key (id) references T2(id)"); + CascadesContext c2 = createCascadesContext( + "select * from T1 inner join T2 " + + "on T1.id = T2.id ", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + CascadesContext c3 = createCascadesContext( + "select * from T1 inner join (select id as id2 from T2) T2 " + + "on T1.id = T2.id2 ", + connectContext + ); + Plan p3 = PlanChecker.from(c3) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + HyperGraph h3 = HyperGraph.builderForMv(p3).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertTrue(!res.isInvalid()); + Assertions.assertTrue(res.getViewExpressions().isEmpty()); + Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3, constructContext(p1, p2, c1)).isInvalid()); + dropConstraint("alter table T2 drop constraint pk"); + } + + @Test + void testInnerJoinWithPKFKAndMultiNodeResidualFilter() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext c1 = createCascadesContext( + "select * from T1", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + addConstraint("alter table T2 add constraint pk primary key (id)"); + addConstraint("alter table T1 add constraint fk foreign key (id) references T2(id)"); + CascadesContext c2 = createCascadesContext( + "select * from T1 inner join T2 " + + "on T1.id = T2.id where T1.score > T2.score", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertTrue(res.isInvalid()); + dropConstraint("alter table T2 drop constraint pk"); + } + + @Disabled + @Test + void testLOJWithPKFKAndUK1() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext c1 = createCascadesContext( + "select * from T1", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + addConstraint("alter table T2 add constraint pk primary key (id)"); + addConstraint("alter table T1 add constraint fk foreign key (id) references T2(id)"); + addConstraint("alter table T3 add constraint uk unique (id)"); + CascadesContext c2 = createCascadesContext( + "select * from (select T1.*, T3.id as id3 from T1 left outer join T3 on T1.id = T3.id) T1 inner join T2 " + + "on T1.id = T2.id ", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertTrue(!res.isInvalid()); + Assertions.assertTrue(res.getViewExpressions().isEmpty()); + dropConstraint("alter table T2 drop constraint pk"); + dropConstraint("alter table T3 drop constraint uk"); + } + + @Test + void testLOJWithPKFKAndUK2() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext c1 = createCascadesContext( + "select * from T1", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + addConstraint("alter table T2 add constraint pk primary key (id)"); + addConstraint("alter table T1 add constraint fk foreign key (id) references T2(id)"); + addConstraint("alter table T3 add constraint uk unique (id)"); + CascadesContext c2 = createCascadesContext( + "select * from (select T1.*, T2.id as id2 from T1 inner join T2 on T1.id = T2.id) T1 left outer join T3 " + + "on T1.id = T3.id ", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertTrue(!res.isInvalid()); + Assertions.assertTrue(res.getViewExpressions().isEmpty()); + dropConstraint("alter table T2 drop constraint pk"); + dropConstraint("alter table T3 drop constraint uk"); + } + + // ------------------------------------------------------------------------- + // from HyperGraphAggTest + // ------------------------------------------------------------------------- + + @Test + void testJoinWithAgg() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + CascadesContext c2 = createCascadesContext( + "select * from T1 inner join" + + "(" + + "select id from T2 group by id" + + ") T2 " + + "on T1.id = T2.id ", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p2).build(); + Assertions.assertEquals("id", Objects.requireNonNull(((StructInfoNode) h1.getNode(1)).getExpressions()).get(0).toSql()); + } + + @Disabled + @Test + void testIJWithAgg() { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES"); + CascadesContext c1 = createCascadesContext( + "select * from T1 inner join T2 " + + "on T1.id = T2.id", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 inner join" + + "(" + + "select id from T2 group by id" + + ") T2 " + + "on T1.id = T2.id ", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + Assertions.assertTrue(!res.isInvalid()); + Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); + } + + LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { + StructInfo st1 = StructInfo.of(p1, p1, null); + StructInfo st2 = StructInfo.of(p2, p2, null); + RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations(), 8) + .get(0); + SlotMapping sm = SlotMapping.generate(rm); + return LogicalCompatibilityContext.from(rm, sm, st1, st2); + } + + // ------------------------------------------------------------------------- + // from HyperGraphComparatorTest + // ------------------------------------------------------------------------- + + @Override + protected String getDisableNereidsRules() { + return "INFER_PREDICATES,CONSTANT_PROPAGATION,PRUNE_EMPTY_PARTITION"; + } + + @Test + void testInnerJoinAndLOJ() { + CascadesContext c1 = createCascadesContext( + "select * from T1 inner join T2 " + + "on T1.id = T2.id " + + "inner join T3 on T1.id = T3.id", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 left outer join T2 " + + "on T1.id = T2.id " + + "left outer join T3 on T1.id = T3.id", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertFalse(res.isInvalid()); + Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); + } + + @Test + void testIJAndLojAssoc() { + CascadesContext c1 = createCascadesContext( + "select * from T1 inner join T3 " + + "on T1.id = T3.id " + + "inner join T2 on T1.id = T2.id", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 left outer join T2 " + + "on T1.id = T2.id " + + "left outer join T3 on T1.id = T3.id", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertFalse(res.isInvalid()); + Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); + } + + @Test + void testIJAndLojAssocWithFilter() { + CascadesContext c1 = createCascadesContext( + "select * from T1 inner join T3 " + + "on T1.id = T3.id " + + "inner join T2 on T1.id = T2.id", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 left outer join " + + "(select * from T2 where T2.id = 1) T2 " + + "on T1.id = T2.id " + + "left outer join T3 on T1.id = T3.id", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertFalse(res.isInvalid()); + Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); + } + + @Test + void testIJAndLojAssocWithJoinCond() { + CascadesContext c1 = createCascadesContext( + "select * from T1 inner join T3 " + + "on T1.id = T3.id " + + "inner join T2 on T1.id = T2.id", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 left join T2 " + + "on T1.id = T2.id " + + "left join T3 on T1.id = T3.id", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertFalse(res.isInvalid()); + Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); + } + + @Test + void testJoinEliminateShouldFail() { + CascadesContext c1 = createCascadesContext( + "select * from T1 inner join T2 " + + "on T1.id = T2.id", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 inner join T2 " + + "on T1.id = T2.id " + + "inner join T3 on T1.id = T3.id", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertTrue(res.isInvalid()); + } + + // ------------------------------------------------------------------------- + // from NullRejectInferenceTest + // ------------------------------------------------------------------------- + + @Test + void testTwoHopNullRejectFromInnerJoinConditions() { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext queryContext = createCascadesContext( + "select lineitem.l_orderkey, supplier.s_name, nation.n_name from lineitem " + + "inner join supplier on lineitem.l_suppkey = supplier.s_suppkey " + + "inner join nation on supplier.s_nationkey = nation.n_nationkey " + + "where nation.n_name = 'CHINA'", + connectContext + ); + Plan queryPlan = PlanChecker.from(queryContext) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + + CascadesContext viewContext = createCascadesContext( + "select lineitem.l_orderkey, supplier.s_name, nation.n_name from lineitem " + + "left outer join supplier on lineitem.l_suppkey = supplier.s_suppkey " + + "left outer join nation on supplier.s_nationkey = nation.n_nationkey", + connectContext + ); + Plan viewPlan = PlanChecker.from(viewContext) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + + StructInfo queryStructInfo = StructInfo.of(queryPlan, queryPlan, queryContext); + StructInfo viewStructInfo = StructInfo.of(viewPlan, viewPlan, viewContext); + RelationMapping relationMapping = RelationMapping.generate( + queryStructInfo.getRelations(), viewStructInfo.getRelations(), 8).get(0); + SlotMapping queryToView = SlotMapping.generate(relationMapping); + SlotMapping viewToQuery = queryToView.inverse(); + LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( + relationMapping, viewToQuery, queryStructInfo, viewStructInfo); + ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals( + queryStructInfo, viewStructInfo, compatibilityContext); + + Assertions.assertFalse(comparisonResult.isInvalid()); + Assertions.assertFalse(comparisonResult.getViewNoNullableSlot().isEmpty()); + + SplitPredicate compensatePredicates = TEST_RULE.predicatesCompensateForTest( + queryStructInfo, viewStructInfo, viewToQuery, comparisonResult, queryContext); + Assertions.assertFalse(compensatePredicates.isInvalid()); + Assertions.assertTrue(compensatePredicates.toList().stream() + .anyMatch(expression -> isNotNullOnSlot(expression, "s_name"))); + } + + @Test + void testNullRejectCompensationForInnerJoinFullJoinRewrite() { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext queryContext = createCascadesContext( + "select lineitem.l_shipdate, orders.o_orderdate from lineitem " + + "inner join orders on lineitem.l_orderkey = orders.o_orderkey " + + "where orders.o_orderdate = '2023-10-17'", + connectContext + ); + Plan queryPlan = PlanChecker.from(queryContext) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + + CascadesContext viewContext = createCascadesContext( + "select lineitem.l_shipdate, orders.o_orderdate from lineitem " + + "full outer join orders on lineitem.l_orderkey = orders.o_orderkey", + connectContext + ); + Plan viewPlan = PlanChecker.from(viewContext) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + + StructInfo queryStructInfo = StructInfo.of(queryPlan, queryPlan, queryContext); + StructInfo viewStructInfo = StructInfo.of(viewPlan, viewPlan, viewContext); + RelationMapping relationMapping = RelationMapping.generate( + queryStructInfo.getRelations(), viewStructInfo.getRelations(), 8).get(0); + SlotMapping queryToView = SlotMapping.generate(relationMapping); + SlotMapping viewToQuery = queryToView.inverse(); + LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( + relationMapping, viewToQuery, queryStructInfo, viewStructInfo); + ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals( + queryStructInfo, viewStructInfo, compatibilityContext); + + Assertions.assertFalse(comparisonResult.isInvalid()); + Assertions.assertFalse(comparisonResult.getViewNoNullableSlot().isEmpty()); + + SplitPredicate compensatePredicates = TEST_RULE.predicatesCompensateForTest( + queryStructInfo, viewStructInfo, viewToQuery, comparisonResult, queryContext); + Assertions.assertFalse(compensatePredicates.isInvalid()); + Assertions.assertTrue(compensatePredicates.toList().stream() + .anyMatch(expression -> isNotNullOnSlot(expression, "l_shipdate"))); + } + + @Test + void testNullRejectCompensationForInnerJoinFullJoinRewriteOnRightSide() { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext queryContext = createCascadesContext( + "select lineitem.l_shipdate, orders.o_orderdate from lineitem " + + "inner join orders on lineitem.l_orderkey = orders.o_orderkey " + + "where lineitem.l_shipdate = '2023-10-17'", + connectContext + ); + Plan queryPlan = PlanChecker.from(queryContext) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + + CascadesContext viewContext = createCascadesContext( + "select lineitem.l_shipdate, orders.o_orderdate from lineitem " + + "full outer join orders on lineitem.l_orderkey = orders.o_orderkey", + connectContext + ); + Plan viewPlan = PlanChecker.from(viewContext) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + + StructInfo queryStructInfo = StructInfo.of(queryPlan, queryPlan, queryContext); + StructInfo viewStructInfo = StructInfo.of(viewPlan, viewPlan, viewContext); + RelationMapping relationMapping = RelationMapping.generate( + queryStructInfo.getRelations(), viewStructInfo.getRelations(), 8).get(0); + SlotMapping queryToView = SlotMapping.generate(relationMapping); + SlotMapping viewToQuery = queryToView.inverse(); + LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( + relationMapping, viewToQuery, queryStructInfo, viewStructInfo); + ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals( + queryStructInfo, viewStructInfo, compatibilityContext); + + Assertions.assertFalse(comparisonResult.isInvalid()); + Assertions.assertFalse(comparisonResult.getViewNoNullableSlot().isEmpty()); + + SplitPredicate compensatePredicates = TEST_RULE.predicatesCompensateForTest( + queryStructInfo, viewStructInfo, viewToQuery, comparisonResult, queryContext); + Assertions.assertFalse(compensatePredicates.isInvalid()); + Assertions.assertTrue(compensatePredicates.toList().stream() + .anyMatch(expression -> isNotNullOnSlot(expression, "o_orderdate"))); + } + + private static boolean isNotNullOnSlot(Expression expression, String slotName) { + if (!(expression instanceof Not) || ((Not) expression).isGeneratedIsNotNull() + || !(((Not) expression).child() instanceof IsNull)) { + return false; + } + Expression slot = ((IsNull) ((Not) expression).child()).child(); + return slot instanceof SlotReference && slotName.equals(((SlotReference) slot).getName()); + } + + private static class TestMaterializedViewRule extends AbstractMaterializedViewRule { + @Override + public List buildRules() { + return ImmutableList.of(); + } + + private SplitPredicate predicatesCompensateForTest(StructInfo queryStructInfo, + StructInfo viewStructInfo, SlotMapping viewToQuerySlotMapping, + ComparisonResult comparisonResult, CascadesContext cascadesContext) { + return predicatesCompensate(queryStructInfo, viewStructInfo, viewToQuerySlotMapping, + comparisonResult, cascadesContext); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/NullRejectInferenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/NullRejectInferenceTest.java deleted file mode 100644 index 5fd7628096b5b5..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/NullRejectInferenceTest.java +++ /dev/null @@ -1,209 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.exploration.mv; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleSet; -import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; -import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; -import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; -import org.apache.doris.nereids.sqltest.SqlTestBase; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.IsNull; -import org.apache.doris.nereids.trees.expressions.Not; -import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.util.PlanChecker; - -import com.google.common.collect.ImmutableList; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.List; - -class NullRejectInferenceTest extends SqlTestBase { - - private static final TestMaterializedViewRule TEST_RULE = new TestMaterializedViewRule(); - - @Test - void testTwoHopNullRejectFromInnerJoinConditions() { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext queryContext = createCascadesContext( - "select lineitem.l_orderkey, supplier.s_name, nation.n_name from lineitem " - + "inner join supplier on lineitem.l_suppkey = supplier.s_suppkey " - + "inner join nation on supplier.s_nationkey = nation.n_nationkey " - + "where nation.n_name = 'CHINA'", - connectContext - ); - Plan queryPlan = PlanChecker.from(queryContext) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - - CascadesContext viewContext = createCascadesContext( - "select lineitem.l_orderkey, supplier.s_name, nation.n_name from lineitem " - + "left outer join supplier on lineitem.l_suppkey = supplier.s_suppkey " - + "left outer join nation on supplier.s_nationkey = nation.n_nationkey", - connectContext - ); - Plan viewPlan = PlanChecker.from(viewContext) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - - StructInfo queryStructInfo = StructInfo.of(queryPlan, queryPlan, queryContext); - StructInfo viewStructInfo = StructInfo.of(viewPlan, viewPlan, viewContext); - RelationMapping relationMapping = RelationMapping.generate( - queryStructInfo.getRelations(), viewStructInfo.getRelations(), 8).get(0); - SlotMapping queryToView = SlotMapping.generate(relationMapping); - SlotMapping viewToQuery = queryToView.inverse(); - LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( - relationMapping, viewToQuery, queryStructInfo, viewStructInfo); - ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals( - queryStructInfo, viewStructInfo, compatibilityContext); - - Assertions.assertFalse(comparisonResult.isInvalid()); - Assertions.assertFalse(comparisonResult.getViewNoNullableSlot().isEmpty()); - - SplitPredicate compensatePredicates = TEST_RULE.predicatesCompensateForTest( - queryStructInfo, viewStructInfo, viewToQuery, comparisonResult, queryContext); - Assertions.assertFalse(compensatePredicates.isInvalid()); - Assertions.assertTrue(compensatePredicates.toList().stream() - .anyMatch(expression -> isNotNullOnSlot(expression, "s_name"))); - } - - @Test - void testNullRejectCompensationForInnerJoinFullJoinRewrite() { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext queryContext = createCascadesContext( - "select lineitem.l_shipdate, orders.o_orderdate from lineitem " - + "inner join orders on lineitem.l_orderkey = orders.o_orderkey " - + "where orders.o_orderdate = '2023-10-17'", - connectContext - ); - Plan queryPlan = PlanChecker.from(queryContext) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - - CascadesContext viewContext = createCascadesContext( - "select lineitem.l_shipdate, orders.o_orderdate from lineitem " - + "full outer join orders on lineitem.l_orderkey = orders.o_orderkey", - connectContext - ); - Plan viewPlan = PlanChecker.from(viewContext) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - - StructInfo queryStructInfo = StructInfo.of(queryPlan, queryPlan, queryContext); - StructInfo viewStructInfo = StructInfo.of(viewPlan, viewPlan, viewContext); - RelationMapping relationMapping = RelationMapping.generate( - queryStructInfo.getRelations(), viewStructInfo.getRelations(), 8).get(0); - SlotMapping queryToView = SlotMapping.generate(relationMapping); - SlotMapping viewToQuery = queryToView.inverse(); - LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( - relationMapping, viewToQuery, queryStructInfo, viewStructInfo); - ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals( - queryStructInfo, viewStructInfo, compatibilityContext); - - Assertions.assertFalse(comparisonResult.isInvalid()); - Assertions.assertFalse(comparisonResult.getViewNoNullableSlot().isEmpty()); - - SplitPredicate compensatePredicates = TEST_RULE.predicatesCompensateForTest( - queryStructInfo, viewStructInfo, viewToQuery, comparisonResult, queryContext); - Assertions.assertFalse(compensatePredicates.isInvalid()); - Assertions.assertTrue(compensatePredicates.toList().stream() - .anyMatch(expression -> isNotNullOnSlot(expression, "l_shipdate"))); - } - - @Test - void testNullRejectCompensationForInnerJoinFullJoinRewriteOnRightSide() { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext queryContext = createCascadesContext( - "select lineitem.l_shipdate, orders.o_orderdate from lineitem " - + "inner join orders on lineitem.l_orderkey = orders.o_orderkey " - + "where lineitem.l_shipdate = '2023-10-17'", - connectContext - ); - Plan queryPlan = PlanChecker.from(queryContext) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - - CascadesContext viewContext = createCascadesContext( - "select lineitem.l_shipdate, orders.o_orderdate from lineitem " - + "full outer join orders on lineitem.l_orderkey = orders.o_orderkey", - connectContext - ); - Plan viewPlan = PlanChecker.from(viewContext) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - - StructInfo queryStructInfo = StructInfo.of(queryPlan, queryPlan, queryContext); - StructInfo viewStructInfo = StructInfo.of(viewPlan, viewPlan, viewContext); - RelationMapping relationMapping = RelationMapping.generate( - queryStructInfo.getRelations(), viewStructInfo.getRelations(), 8).get(0); - SlotMapping queryToView = SlotMapping.generate(relationMapping); - SlotMapping viewToQuery = queryToView.inverse(); - LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( - relationMapping, viewToQuery, queryStructInfo, viewStructInfo); - ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals( - queryStructInfo, viewStructInfo, compatibilityContext); - - Assertions.assertFalse(comparisonResult.isInvalid()); - Assertions.assertFalse(comparisonResult.getViewNoNullableSlot().isEmpty()); - - SplitPredicate compensatePredicates = TEST_RULE.predicatesCompensateForTest( - queryStructInfo, viewStructInfo, viewToQuery, comparisonResult, queryContext); - Assertions.assertFalse(compensatePredicates.isInvalid()); - Assertions.assertTrue(compensatePredicates.toList().stream() - .anyMatch(expression -> isNotNullOnSlot(expression, "o_orderdate"))); - } - - private static boolean isNotNullOnSlot(Expression expression, String slotName) { - if (!(expression instanceof Not) || ((Not) expression).isGeneratedIsNotNull() - || !(((Not) expression).child() instanceof IsNull)) { - return false; - } - Expression slot = ((IsNull) ((Not) expression).child()).child(); - return slot instanceof SlotReference && slotName.equals(((SlotReference) slot).getName()); - } - - private static class TestMaterializedViewRule extends AbstractMaterializedViewRule { - @Override - public List buildRules() { - return ImmutableList.of(); - } - - private SplitPredicate predicatesCompensateForTest(StructInfo queryStructInfo, - StructInfo viewStructInfo, SlotMapping viewToQuerySlotMapping, - ComparisonResult comparisonResult, CascadesContext cascadesContext) { - return predicatesCompensate(queryStructInfo, viewStructInfo, viewToQuerySlotMapping, - comparisonResult, cascadesContext); - } - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateConstHashJoinConditionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateConstHashJoinConditionTest.java deleted file mode 100644 index 977c216c4e7aab..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateConstHashJoinConditionTest.java +++ /dev/null @@ -1,74 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.sqltest.SqlTestBase; -import org.apache.doris.nereids.util.PlanChecker; - -import org.junit.jupiter.api.Test; - -public class EliminateConstHashJoinConditionTest extends SqlTestBase { - - @Test - void testEliminate() { - CascadesContext c1 = createCascadesContext( - "select * from T1 join T2 on T1.id = T2.id and T1.id = 1", - connectContext - ); - PlanChecker.from(c1) - .analyze() - .rewrite() - .matches( - logicalJoin().when(join -> - join.getHashJoinConjuncts().isEmpty() && join.getOtherJoinConjuncts().isEmpty()) - ); - } - - @Test - void testNotEliminateNonInnerJoin() { - CascadesContext c1 = createCascadesContext( - "select * from T1 left join T2 on T1.id = T2.id where T1.id = 1", - connectContext - ); - PlanChecker.from(c1) - .analyze() - .rewrite() - .matches( - logicalJoin().when(join -> - !join.getHashJoinConjuncts().isEmpty()) - ); - } - - @Test - void testNotEliminateAsofJoin() { - CascadesContext c1 = createCascadesContext( - "select * from T1 asof inner join T2 match_condition(cast(T1.score as datetime) " - + "> cast(T2.score as datetime)) on T1.id = T2.id where T1.id = 1", - connectContext - ); - PlanChecker.from(c1) - .analyze() - .rewrite() - .matches( - logicalJoin().when(join -> - !join.getHashJoinConjuncts().isEmpty()) - ); - } - -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteRuleSuiteTest.java similarity index 75% rename from fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java rename to fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteRuleSuiteTest.java index 6bcb1486a69306..1a87a28fee6cd5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteRuleSuiteTest.java @@ -28,6 +28,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.sqltest.SqlTestBase; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; @@ -38,7 +39,6 @@ import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; -import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.planner.PartitionColumnFilter; @@ -53,7 +53,23 @@ import java.util.List; import java.util.Objects; -class PruneOlapScanTabletTest extends SqlTestBase implements MemoPatternMatchSupported { +/** + * Rewrite-rule tests that only need the shared {@link SqlTestBase} fixture (database "test" with + * tables T0..T4). They are kept in one class on purpose: every extra test class pays a full FE + * startup, which dominates the runtime of tests this small. + * + *

Replaces the former standalone classes: + *

    + *
  • PruneOlapScanTabletTest
    • + *
  • EliminateConstHashJoinConditionTest
    • + *
+ * Add a new section below when moving another shared-fixture rewrite test here. + */ +public class RewriteRuleSuiteTest extends SqlTestBase { + + // ------------------------------------------------------------------------- + // from PruneOlapScanTabletTest + // ------------------------------------------------------------------------- @Test void testPruneOlapScanTablet() { @@ -159,4 +175,54 @@ void testPruneOlapScanTabletWithManually() { ) ); } + + // ------------------------------------------------------------------------- + // from EliminateConstHashJoinConditionTest + // ------------------------------------------------------------------------- + + @Test + void testEliminate() { + CascadesContext c1 = createCascadesContext( + "select * from T1 join T2 on T1.id = T2.id and T1.id = 1", + connectContext + ); + PlanChecker.from(c1) + .analyze() + .rewrite() + .matches( + logicalJoin().when(join -> + join.getHashJoinConjuncts().isEmpty() && join.getOtherJoinConjuncts().isEmpty()) + ); + } + + @Test + void testNotEliminateNonInnerJoin() { + CascadesContext c1 = createCascadesContext( + "select * from T1 left join T2 on T1.id = T2.id where T1.id = 1", + connectContext + ); + PlanChecker.from(c1) + .analyze() + .rewrite() + .matches( + logicalJoin().when(join -> + !join.getHashJoinConjuncts().isEmpty()) + ); + } + + @Test + void testNotEliminateAsofJoin() { + CascadesContext c1 = createCascadesContext( + "select * from T1 asof inner join T2 match_condition(cast(T1.score as datetime) " + + "> cast(T2.score as datetime)) on T1.id = T2.id where T1.id = 1", + connectContext + ); + PlanChecker.from(c1) + .analyze() + .rewrite() + .matches( + logicalJoin().when(join -> + !join.getHashJoinConjuncts().isEmpty()) + ); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/CascadesJoinReorderTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/CascadesJoinReorderTest.java deleted file mode 100644 index 00dce9f48a6b85..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/CascadesJoinReorderTest.java +++ /dev/null @@ -1,192 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.sqltest; - -import org.apache.doris.nereids.rules.RuleSet; -import org.apache.doris.nereids.util.PlanChecker; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -/** - * Paper: Measuring the Complexity of Join Enumeration in Query Optimization - * 
- * Star query without products
- * Join tree Number：
- *   left-deep: 2(n-1)! * 1
- *   zig-zag: (n-1)! * 2^(n-1)
- *   bushy: star graph can't be a bushy, it can only form a zig-zag (because the center must be joined first)
- *
- */ -class CascadesJoinReorderTest extends SqlTestBase { - @Test - void testStartThreeJoin() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - // Three join - // (n-1)! * 2^(n-1) = 8 - String sql = "SELECT * FROM T1 " - + "JOIN T2 ON T1.id = T2.id " - + "JOIN T3 ON T1.id = T3.id"; - - int plansNumber = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .applyExploration(RuleSet.ZIG_ZAG_TREE_JOIN_REORDER) - .applyExploration(RuleSet.ZIG_ZAG_TREE_JOIN_REORDER) - .applyExploration(RuleSet.ZIG_ZAG_TREE_JOIN_REORDER) - .applyExploration(RuleSet.ZIG_ZAG_TREE_JOIN_REORDER) - .plansNumber(); - - Assertions.assertEquals(8, plansNumber); - } - - @Test - void testStartThreeJoinBushy() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - // Three join - // (n-1)! * 2^(n-1) = 8 - String sql = "SELECT * FROM T1 " - + "JOIN T2 ON T1.id = T2.id " - + "JOIN T3 ON T1.id = T3.id"; - - int plansNumber = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .printlnAllTree() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .plansNumber(); - - Assertions.assertEquals(8, plansNumber); - } - - @Test - void testStarFourJoinZigzag() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - // Four join - // (n-1)! * 2^(n-1) = 48 - String sql = "SELECT * FROM T1 " - + "JOIN T2 ON T1.id = T2.id " - + "JOIN T3 ON T1.id = T3.id " - + "JOIN T4 ON T1.id = T4.id "; - - int plansNumber = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .plansNumber(); - - Assertions.assertEquals(48, plansNumber); - } - - @Test - void testStarFourJoinBushy() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - // Four join - // (n-1)! * 2^(n-1) = 48 - String sql = "SELECT * FROM T1 " - + "JOIN T2 ON T1.id = T2.id " - + "JOIN T3 ON T1.id = T3.id " - + "JOIN T4 ON T1.id = T4.id "; - - int plansNumber = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .plansNumber(); - - Assertions.assertEquals(48, plansNumber); - } - - @Test - void testChainFourJoinBushy() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - // Four join - // 2^(n-1) * C(n-1) = 40 - String sql = "SELECT * FROM T1 " - + "JOIN T2 ON T1.id = T2.id " - + "JOIN T3 ON T2.id = T3.id " - + "JOIN T4 ON T3.id = T4.id "; - - int plansNumber = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .plansNumber(); - - Assertions.assertEquals(40, plansNumber); - } - - @Test - void testChainFiveJoinBushy() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - // Five join - // 2^(n-1) * C(n-1) = 224 - String sql = "SELECT * FROM T1 " - + "JOIN T2 ON T1.id = T2.id " - + "JOIN T3 ON T2.id = T3.id " - + "JOIN T4 ON T3.id = T4.id " - + "JOIN T1 T5 ON T4.ID = T5.ID"; - - int plansNumber = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .plansNumber(); - - Assertions.assertEquals(224, plansNumber); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java deleted file mode 100644 index cdc36164ae9f95..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java +++ /dev/null @@ -1,122 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.sqltest; - -import org.apache.doris.nereids.util.ExpressionUtils; -import org.apache.doris.nereids.util.PlanChecker; - -import org.junit.jupiter.api.Test; - -import java.util.stream.Collectors; - -public class InferTest extends SqlTestBase { - @Test - void testInferNotNullAndInferPredicates() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - // Test InferNotNull, EliminateOuter, InferPredicate together - String sql = "select * from T1 left outer join T2 on T1.id = T2.id where T2.id = 4"; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .matches( - innerLogicalJoin( - logicalFilter().when(f -> f.getPredicate().toString().equals("(id#0 = 4)")), - logicalFilter().when(f -> f.getPredicate().toString().equals("(id#2 = 4)")) - ) - ); - } - - @Test - void testInferNotNullFromFilterAndEliminateOuter2() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql - = "select * from T1 right outer join T2 on T1.id = T2.id where T1.id = 4 OR (T1.id > 4 AND T2.score IS NULL)"; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .printlnTree() - .matches( - innerLogicalJoin( - logicalFilter().when( - f -> f.getPredicate().toString().equals("(id#2 >= 4)")), - logicalFilter().when( - f -> ExpressionUtils.and(f.getConjuncts().stream() - .sorted((a, b) -> a.toString().compareTo(b.toString())) - .collect(Collectors.toList())) - .toString().equals("(id#0 >= 4)")) - ) - - ); - } - - @Test - void testInferNotNullFromFilterAndEliminateOuter3() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql - = "select * from T1 full outer join T2 on T1.id = T2.id where T1.id = 4 OR (T1.id > 4 AND T2.score IS NULL)"; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .matches( - logicalFilter( - leftOuterLogicalJoin( - logicalFilter().when( - f -> f.getPredicate().toString().equals("(id#0 >= 4)")), - logicalFilter().when( - f -> f.getPredicate().toString().equals("(id#2 >= 4)") - ) - ) - ).when(f -> f.getPredicate().toString() - .equals("OR[(id#0 = 4),AND[(id#0 > 4),score#3 IS NULL]]")) - ); - } - - @Test - void testInferNotNullFromJoinAndEliminateOuter() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - // Is Not Null will infer from semi join, so right outer join can be eliminated. - String sql - = "select * from (select T1.id from T1 right outer join T2 on T1.id = T2.id) T1 left semi join T3 on T1.id = T3.id"; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .matches( - innerLogicalJoin( - logicalProject(), - logicalProject(leftSemiLogicalJoin()) - ) - ); - } - - @Test - void aggEliminateOuterJoin() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "select count(T2.score) from T1 left Join T2 on T1.id = T2.id"; - - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .matches( - logicalAggregate( - logicalProject( - innerLogicalJoin() - ) - ) - ); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/JoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/JoinTest.java deleted file mode 100644 index 899e35f2b63647..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/JoinTest.java +++ /dev/null @@ -1,106 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.sqltest; - -import org.apache.doris.nereids.properties.DistributionSpecHash; -import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType; -import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.rules.rewrite.ReorderJoin; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; -import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; -import org.apache.doris.nereids.util.PlanChecker; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class JoinTest extends SqlTestBase { - @Test - void testJoinUsing() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "SELECT * FROM T1 JOIN T2 using (id)"; - PlanChecker.from(connectContext) - .analyze(sql) - .applyBottomUp(new ReorderJoin()) - .matches( - innerLogicalJoin().when(j -> j.getHashJoinConjuncts().size() == 1) - ); - } - - @Test - void testColocatedJoin() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "select * from T2 join T2 b on T2.id = b.id and T2.id = b.id;"; - PhysicalPlan plan = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .optimize() - .getBestPlanTree(); - // generate colocate join plan without physicalDistribute - System.out.println(plan.treeString()); - Assertions.assertFalse(plan.anyMatch(p -> p instanceof PhysicalDistribute - && ((PhysicalDistribute) p).getDistributionSpec() instanceof DistributionSpecHash)); - sql = "select * from T1 join T0 on T1.score = T0.score and T1.id = T0.id;"; - plan = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .optimize() - .getBestPlanTree(); - // generate colocate join plan without physicalDistribute - Assertions.assertFalse(plan.anyMatch(p -> p instanceof PhysicalDistribute - && ((PhysicalDistribute) p).getDistributionSpec() instanceof DistributionSpecHash)); - } - - @Test - void testDedupConjuncts() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "select * from T1 join T2 on T1.id = T2.id and T1.id = T2.id;"; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .matches( - innerLogicalJoin().when(j -> j.getHashJoinConjuncts().size() == 1) - ); - - String sql1 = "select * from T1 left join T2 on T1.id = T2.id and T1.id = T2.id;"; - PlanChecker.from(connectContext) - .analyze(sql1) - .rewrite() - .matches( - leftOuterLogicalJoin().when(j -> j.getHashJoinConjuncts().size() == 1) - ); - } - - @Test - void testBucketJoinWithAgg() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "select * from " - + "(select distinct id as cnt from T2) T1 inner join" - + "(select distinct id as cnt from T2) T2 " - + "on T1.cnt = T2.cnt"; - PhysicalPlan plan = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .optimize() - .getBestPlanTree(PhysicalProperties.ANY); - Assertions.assertEquals( - ShuffleType.NATURAL, - ((DistributionSpecHash) ((PhysicalPlan) (plan.child(0).child(0))) - .getPhysicalProperties().getDistributionSpec()).getShuffleType() - ); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/MultiJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/MultiJoinTest.java deleted file mode 100644 index 26539fb356a1e2..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/MultiJoinTest.java +++ /dev/null @@ -1,131 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.sqltest; - -import org.apache.doris.nereids.rules.rewrite.ReorderJoin; -import org.apache.doris.nereids.util.PlanChecker; - -import com.google.common.collect.ImmutableList; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import java.util.List; - -class MultiJoinTest extends SqlTestBase { - @Test - void testMultiJoinEliminateCross() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - List sqls = ImmutableList.builder() - .add("SELECT * FROM T2 LEFT JOIN T3 ON T2.id = T3.id, T1 WHERE T1.id = T2.id") - .add("SELECT * FROM T2 LEFT JOIN T3 ON T2.id = T3.id, T1 WHERE T1.id = T2.id AND T1.score > 0") - .add("SELECT * FROM T2 LEFT JOIN T3 ON T2.id = T3.id, T1 WHERE T1.id = T2.id AND T1.score > 0 AND T1.id + T2.id + T3.id > 0") - .build(); - - for (String sql : sqls) { - PlanChecker.from(connectContext) - .analyze(sql) - .applyBottomUp(new ReorderJoin()) - .matches( - logicalJoin( - logicalJoin().whenNot(join -> join.getJoinType().isCrossJoin()), - leafPlan() - ).whenNot(join -> join.getJoinType().isCrossJoin()) - ) - .printlnTree(); - } - } - - @Test - @Disabled - void testEliminateBelowOuter() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - // FIXME: MultiJoin And EliminateOuter - String sql = "SELECT * FROM T1, T2 LEFT JOIN T3 ON T2.id = T3.id WHERE T1.id = T2.id"; - PlanChecker.from(connectContext) - .analyze(sql) - .applyBottomUp(new ReorderJoin()) - .printlnTree(); - } - - @Test - void testMultiJoinExistCross() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - List sqls = ImmutableList.builder() - .add("SELECT * FROM T2 LEFT SEMI JOIN T3 ON T2.id = T3.id, T1 WHERE T1.id > T2.id") - .build(); - - for (String sql : sqls) { - PlanChecker.from(connectContext) - .analyze(sql) - .applyBottomUp(new ReorderJoin()) - .matches( - logicalJoin( - logicalJoin().whenNot(join -> join.getJoinType().isCrossJoin()), - leafPlan() - ).when(join -> join.getJoinType().isCrossJoin()) - .whenNot(join -> join.getOtherJoinConjuncts().isEmpty()) - ) - .printlnTree(); - } - } - - @Test - void testOuterJoin() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "SELECT * FROM T1 LEFT OUTER JOIN T2 ON T1.id = T2.id, T3 WHERE T2.score > 0"; - PlanChecker.from(connectContext) - .analyze(sql) - .applyBottomUp(new ReorderJoin()) - .printlnTree() - .matches( - crossLogicalJoin( - leftOuterLogicalJoin() - .when(join -> join.getOtherJoinConjuncts().size() == 1), - logicalOlapScan() - ) - ); - } - - @Test - @Disabled - void testNoFilter() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "Select * FROM T1 INNER JOIN T2 On true"; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .matches( - crossLogicalJoin() - ); - } - - @Test - void test() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "select T1.score, T2.score from T1 inner join T2 on T1.id = T2.id where T1.score - 2 > T2.score"; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .matches( - logicalProject( - innerLogicalJoin() - ) - ); - - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SortTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SortTest.java deleted file mode 100644 index f042fee4de2c77..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SortTest.java +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.sqltest; - -import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; -import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; -import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort; -import org.apache.doris.nereids.util.PlanChecker; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class SortTest extends SqlTestBase { - @Test - public void testTwoPhaseSort() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "select * from\n" - + "(select score from T1 order by id) as t order by score\n"; - PhysicalPlan plan = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .optimize() - .getBestPlanTree(); - System.out.println(plan.treeString()); - Assertions.assertTrue(plan.anyMatch(e -> e instanceof PhysicalQuickSort - && ((PhysicalQuickSort) e).getSortPhase().isMerge() && e.child(0) instanceof PhysicalDistribute)); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlPlanSuiteTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlPlanSuiteTest.java new file mode 100644 index 00000000000000..879d724c69e5ac --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlPlanSuiteTest.java @@ -0,0 +1,517 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.sqltest; + +import org.apache.doris.nereids.properties.DistributionSpecHash; +import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.RuleSet; +import org.apache.doris.nereids.rules.rewrite.ReorderJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort; +import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.PlanChecker; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Merged suite: these tests only need the shared fixture provided by the base class, so they are + * kept in one class on purpose. Every extra test class pays a full FE startup, which dominates the + * runtime of tests this small. + * + *

Replaces the former standalone classes: + *

    + *
  • CascadesJoinReorderTest
    • + *
  • InferTest
    • + *
  • JoinTest
    • + *
  • MultiJoinTest
    • + *
  • SortTest
    • + *
+ */ +public class SqlPlanSuiteTest extends SqlTestBase { + + // ------------------------------------------------------------------------- + // from CascadesJoinReorderTest + // ------------------------------------------------------------------------- + + @Test + void testStartThreeJoin() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + // Three join + // (n-1)! * 2^(n-1) = 8 + String sql = "SELECT * FROM T1 " + + "JOIN T2 ON T1.id = T2.id " + + "JOIN T3 ON T1.id = T3.id"; + + int plansNumber = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .applyExploration(RuleSet.ZIG_ZAG_TREE_JOIN_REORDER) + .applyExploration(RuleSet.ZIG_ZAG_TREE_JOIN_REORDER) + .applyExploration(RuleSet.ZIG_ZAG_TREE_JOIN_REORDER) + .applyExploration(RuleSet.ZIG_ZAG_TREE_JOIN_REORDER) + .plansNumber(); + + Assertions.assertEquals(8, plansNumber); + } + + @Test + void testStartThreeJoinBushy() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + // Three join + // (n-1)! * 2^(n-1) = 8 + String sql = "SELECT * FROM T1 " + + "JOIN T2 ON T1.id = T2.id " + + "JOIN T3 ON T1.id = T3.id"; + + int plansNumber = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .printlnAllTree() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .plansNumber(); + + Assertions.assertEquals(8, plansNumber); + } + + @Test + void testStarFourJoinZigzag() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + // Four join + // (n-1)! * 2^(n-1) = 48 + String sql = "SELECT * FROM T1 " + + "JOIN T2 ON T1.id = T2.id " + + "JOIN T3 ON T1.id = T3.id " + + "JOIN T4 ON T1.id = T4.id "; + + int plansNumber = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .plansNumber(); + + Assertions.assertEquals(48, plansNumber); + } + + @Test + void testStarFourJoinBushy() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + // Four join + // (n-1)! * 2^(n-1) = 48 + String sql = "SELECT * FROM T1 " + + "JOIN T2 ON T1.id = T2.id " + + "JOIN T3 ON T1.id = T3.id " + + "JOIN T4 ON T1.id = T4.id "; + + int plansNumber = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .plansNumber(); + + Assertions.assertEquals(48, plansNumber); + } + + @Test + void testChainFourJoinBushy() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + // Four join + // 2^(n-1) * C(n-1) = 40 + String sql = "SELECT * FROM T1 " + + "JOIN T2 ON T1.id = T2.id " + + "JOIN T3 ON T2.id = T3.id " + + "JOIN T4 ON T3.id = T4.id "; + + int plansNumber = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .plansNumber(); + + Assertions.assertEquals(40, plansNumber); + } + + @Test + void testChainFiveJoinBushy() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + // Five join + // 2^(n-1) * C(n-1) = 224 + String sql = "SELECT * FROM T1 " + + "JOIN T2 ON T1.id = T2.id " + + "JOIN T3 ON T2.id = T3.id " + + "JOIN T4 ON T3.id = T4.id " + + "JOIN T1 T5 ON T4.ID = T5.ID"; + + int plansNumber = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .plansNumber(); + + Assertions.assertEquals(224, plansNumber); + } + + // ------------------------------------------------------------------------- + // from InferTest + // ------------------------------------------------------------------------- + + @Test + void testInferNotNullAndInferPredicates() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + // Test InferNotNull, EliminateOuter, InferPredicate together + String sql = "select * from T1 left outer join T2 on T1.id = T2.id where T2.id = 4"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches( + innerLogicalJoin( + logicalFilter().when(f -> f.getPredicate().toString().equals("(id#0 = 4)")), + logicalFilter().when(f -> f.getPredicate().toString().equals("(id#2 = 4)")) + ) + ); + } + + @Test + void testInferNotNullFromFilterAndEliminateOuter2() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql + = "select * from T1 right outer join T2 on T1.id = T2.id where T1.id = 4 OR (T1.id > 4 AND T2.score IS NULL)"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .printlnTree() + .matches( + innerLogicalJoin( + logicalFilter().when( + f -> f.getPredicate().toString().equals("(id#2 >= 4)")), + logicalFilter().when( + f -> ExpressionUtils.and(f.getConjuncts().stream() + .sorted((a, b) -> a.toString().compareTo(b.toString())) + .collect(Collectors.toList())) + .toString().equals("(id#0 >= 4)")) + ) + + ); + } + + @Test + void testInferNotNullFromFilterAndEliminateOuter3() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql + = "select * from T1 full outer join T2 on T1.id = T2.id where T1.id = 4 OR (T1.id > 4 AND T2.score IS NULL)"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches( + logicalFilter( + leftOuterLogicalJoin( + logicalFilter().when( + f -> f.getPredicate().toString().equals("(id#0 >= 4)")), + logicalFilter().when( + f -> f.getPredicate().toString().equals("(id#2 >= 4)") + ) + ) + ).when(f -> f.getPredicate().toString() + .equals("OR[(id#0 = 4),AND[(id#0 > 4),score#3 IS NULL]]")) + ); + } + + @Test + void testInferNotNullFromJoinAndEliminateOuter() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + // Is Not Null will infer from semi join, so right outer join can be eliminated. + String sql + = "select * from (select T1.id from T1 right outer join T2 on T1.id = T2.id) T1 left semi join T3 on T1.id = T3.id"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches( + innerLogicalJoin( + logicalProject(), + logicalProject(leftSemiLogicalJoin()) + ) + ); + } + + @Test + void aggEliminateOuterJoin() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "select count(T2.score) from T1 left Join T2 on T1.id = T2.id"; + + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches( + logicalAggregate( + logicalProject( + innerLogicalJoin() + ) + ) + ); + } + + // ------------------------------------------------------------------------- + // from JoinTest + // ------------------------------------------------------------------------- + + @Test + void testJoinUsing() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "SELECT * FROM T1 JOIN T2 using (id)"; + PlanChecker.from(connectContext) + .analyze(sql) + .applyBottomUp(new ReorderJoin()) + .matches( + innerLogicalJoin().when(j -> j.getHashJoinConjuncts().size() == 1) + ); + } + + @Test + void testColocatedJoin() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "select * from T2 join T2 b on T2.id = b.id and T2.id = b.id;"; + PhysicalPlan plan = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .optimize() + .getBestPlanTree(); + // generate colocate join plan without physicalDistribute + System.out.println(plan.treeString()); + Assertions.assertFalse(plan.anyMatch(p -> p instanceof PhysicalDistribute + && ((PhysicalDistribute) p).getDistributionSpec() instanceof DistributionSpecHash)); + sql = "select * from T1 join T0 on T1.score = T0.score and T1.id = T0.id;"; + plan = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .optimize() + .getBestPlanTree(); + // generate colocate join plan without physicalDistribute + Assertions.assertFalse(plan.anyMatch(p -> p instanceof PhysicalDistribute + && ((PhysicalDistribute) p).getDistributionSpec() instanceof DistributionSpecHash)); + } + + @Test + void testDedupConjuncts() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "select * from T1 join T2 on T1.id = T2.id and T1.id = T2.id;"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches( + innerLogicalJoin().when(j -> j.getHashJoinConjuncts().size() == 1) + ); + + String sql1 = "select * from T1 left join T2 on T1.id = T2.id and T1.id = T2.id;"; + PlanChecker.from(connectContext) + .analyze(sql1) + .rewrite() + .matches( + leftOuterLogicalJoin().when(j -> j.getHashJoinConjuncts().size() == 1) + ); + } + + @Test + void testBucketJoinWithAgg() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "select * from " + + "(select distinct id as cnt from T2) T1 inner join" + + "(select distinct id as cnt from T2) T2 " + + "on T1.cnt = T2.cnt"; + PhysicalPlan plan = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .optimize() + .getBestPlanTree(PhysicalProperties.ANY); + Assertions.assertEquals( + ShuffleType.NATURAL, + ((DistributionSpecHash) ((PhysicalPlan) (plan.child(0).child(0))) + .getPhysicalProperties().getDistributionSpec()).getShuffleType() + ); + } + + // ------------------------------------------------------------------------- + // from MultiJoinTest + // ------------------------------------------------------------------------- + + @Test + void testMultiJoinEliminateCross() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + List sqls = ImmutableList.builder() + .add("SELECT * FROM T2 LEFT JOIN T3 ON T2.id = T3.id, T1 WHERE T1.id = T2.id") + .add("SELECT * FROM T2 LEFT JOIN T3 ON T2.id = T3.id, T1 WHERE T1.id = T2.id AND T1.score > 0") + .add("SELECT * FROM T2 LEFT JOIN T3 ON T2.id = T3.id, T1 WHERE T1.id = T2.id AND T1.score > 0 AND T1.id + T2.id + T3.id > 0") + .build(); + + for (String sql : sqls) { + PlanChecker.from(connectContext) + .analyze(sql) + .applyBottomUp(new ReorderJoin()) + .matches( + logicalJoin( + logicalJoin().whenNot(join -> join.getJoinType().isCrossJoin()), + leafPlan() + ).whenNot(join -> join.getJoinType().isCrossJoin()) + ) + .printlnTree(); + } + } + + @Test + @Disabled + void testEliminateBelowOuter() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + // FIXME: MultiJoin And EliminateOuter + String sql = "SELECT * FROM T1, T2 LEFT JOIN T3 ON T2.id = T3.id WHERE T1.id = T2.id"; + PlanChecker.from(connectContext) + .analyze(sql) + .applyBottomUp(new ReorderJoin()) + .printlnTree(); + } + + @Test + void testMultiJoinExistCross() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + List sqls = ImmutableList.builder() + .add("SELECT * FROM T2 LEFT SEMI JOIN T3 ON T2.id = T3.id, T1 WHERE T1.id > T2.id") + .build(); + + for (String sql : sqls) { + PlanChecker.from(connectContext) + .analyze(sql) + .applyBottomUp(new ReorderJoin()) + .matches( + logicalJoin( + logicalJoin().whenNot(join -> join.getJoinType().isCrossJoin()), + leafPlan() + ).when(join -> join.getJoinType().isCrossJoin()) + .whenNot(join -> join.getOtherJoinConjuncts().isEmpty()) + ) + .printlnTree(); + } + } + + @Test + void testOuterJoin() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "SELECT * FROM T1 LEFT OUTER JOIN T2 ON T1.id = T2.id, T3 WHERE T2.score > 0"; + PlanChecker.from(connectContext) + .analyze(sql) + .applyBottomUp(new ReorderJoin()) + .printlnTree() + .matches( + crossLogicalJoin( + leftOuterLogicalJoin() + .when(join -> join.getOtherJoinConjuncts().size() == 1), + logicalOlapScan() + ) + ); + } + + @Test + @Disabled + void testNoFilter() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "Select * FROM T1 INNER JOIN T2 On true"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches( + crossLogicalJoin() + ); + } + + @Test + void test() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "select T1.score, T2.score from T1 inner join T2 on T1.id = T2.id where T1.score - 2 > T2.score"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches( + logicalProject( + innerLogicalJoin() + ) + ); + + } + + // ------------------------------------------------------------------------- + // from SortTest + // ------------------------------------------------------------------------- + + @Test + public void testTwoPhaseSort() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "select * from\n" + + "(select score from T1 order by id) as t order by score\n"; + PhysicalPlan plan = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .optimize() + .getBestPlanTree(); + System.out.println(plan.treeString()); + Assertions.assertTrue(plan.anyMatch(e -> e instanceof PhysicalQuickSort + && ((PhysicalQuickSort) e).getSortPhase().isMerge() && e.child(0) instanceof PhysicalDistribute)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AdminCopyTabletCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AdminCopyTabletCommandTest.java deleted file mode 100644 index 0ca4823a463426..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AdminCopyTabletCommandTest.java +++ /dev/null @@ -1,86 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.commands; - -import org.apache.doris.catalog.Env; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.mysql.privilege.AccessControllerManager; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.utframe.TestWithFeService; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -public class AdminCopyTabletCommandTest extends TestWithFeService { - private ConnectContext connectContext; - private Env env; - private AccessControllerManager accessControllerManager; - - private void runBefore() throws IOException { - connectContext = createDefaultCtx(); - env = Env.getCurrentEnv(); - accessControllerManager = env.getAccessManager(); - } - - @Test - public void testValidateNormal() throws Exception { - runBefore(); - connectContext.setSkipAuth(true); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(true).when(spyAcm).checkGlobalPriv( - Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - Map properties = new HashMap<>(); - properties.put("version", "0"); - AdminCopyTabletCommand command = new AdminCopyTabletCommand(100, properties); - Assertions.assertDoesNotThrow(() -> command.validate()); - - Map properties2 = new HashMap<>(); - properties2.put("backend_id", "10"); - AdminCopyTabletCommand command2 = new AdminCopyTabletCommand(100, properties2); - Assertions.assertDoesNotThrow(() -> command2.validate()); - - Map properties3 = new HashMap<>(); - properties3.put("expiration_minutes", "10"); - AdminCopyTabletCommand command3 = new AdminCopyTabletCommand(100, properties3); - Assertions.assertDoesNotThrow(() -> command3.validate()); - } - - @Test - public void testNoPriviledge() throws Exception { - runBefore(); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(false).when(spyAcm).checkGlobalPriv( - Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - Map properties = new HashMap<>(); - properties.put("version", "0"); - AdminCopyTabletCommand command = new AdminCopyTabletCommand(100, properties); - Assertions.assertThrows(AnalysisException.class, () -> command.validate(), - "Access denied; you need (at least one of) the (Admin_priv) privilege(s) for this operation"); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AdminRepairTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AdminRepairTableCommandTest.java deleted file mode 100644 index 2e45dbaf160147..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AdminRepairTableCommandTest.java +++ /dev/null @@ -1,101 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.commands; - -import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.info.PartitionNamesInfo; -import org.apache.doris.catalog.info.TableNameInfo; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.datasource.InternalCatalog; -import org.apache.doris.info.TableRefInfo; -import org.apache.doris.mysql.privilege.AccessControllerManager; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.utframe.TestWithFeService; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -public class AdminRepairTableCommandTest extends TestWithFeService { - private static final String internalCtl = InternalCatalog.INTERNAL_CATALOG_NAME; - private ConnectContext connectContext; - private Env env; - private AccessControllerManager accessControllerManager; - - private void runBefore() throws IOException { - connectContext = createDefaultCtx(); - env = Env.getCurrentEnv(); - accessControllerManager = env.getAccessManager(); - } - - @Test - public void testValidateNormal() throws Exception { - runBefore(); - connectContext.setSkipAuth(true); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(true).when(spyAcm).checkGlobalPriv( - Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - TableNameInfo tableNameInfo = new TableNameInfo(internalCtl, "test_db", "test_tbl"); - List partitionNames = new ArrayList<>(); - partitionNames.add("p1"); - partitionNames.add("p2"); - PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, partitionNames); - TableRefInfo tableRefInfo = new TableRefInfo(tableNameInfo, null, null, partitionNamesInfo, null, null, null, null); - AdminRepairTableCommand command = new AdminRepairTableCommand(tableRefInfo); - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - - //test external catalog - TableNameInfo tableNameInfo2 = new TableNameInfo("hive", "test_db", "test_tbl"); - TableRefInfo tableRefInfo2 = new TableRefInfo(tableNameInfo2, null, null, partitionNamesInfo, null, null, null, null); - AdminRepairTableCommand command2 = new AdminRepairTableCommand(tableRefInfo2); - Assertions.assertThrows(AnalysisException.class, () -> command2.validate(connectContext), - "External catalog 'hive' is not allowed in 'AdminCancelRepairTableCommand.class'"); - - //test partitionNameInfo isTemp - PartitionNamesInfo partitionNamesInfo2 = new PartitionNamesInfo(true, partitionNames); - TableRefInfo tableRefInfo3 = new TableRefInfo(tableNameInfo, null, null, partitionNamesInfo2, null, null, null, null); - AdminRepairTableCommand command3 = new AdminRepairTableCommand(tableRefInfo3); - Assertions.assertThrows(AnalysisException.class, () -> command3.validate(connectContext), - "Do not support (cancel)repair temporary partitions"); - } - - @Test - public void testValidateNoPriviledge() throws Exception { - runBefore(); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(false).when(spyAcm).checkGlobalPriv( - Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - TableNameInfo tableNameInfo = new TableNameInfo(internalCtl, "test_db", "test_tbl"); - List partitionNames = new ArrayList<>(); - PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, partitionNames); - TableRefInfo tableRefInfo = new TableRefInfo(tableNameInfo, null, null, partitionNamesInfo, null, null, null, null); - AdminRepairTableCommand command = new AdminRepairTableCommand(tableRefInfo); - Assertions.assertThrows(AnalysisException.class, () -> command.validate(connectContext), - "Access denied; you need (at least one of) the (ADMIN) privilege(s) for this operation"); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterColumnStatsCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterColumnStatsCommandTest.java deleted file mode 100644 index d5efeec9d6b46d..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterColumnStatsCommandTest.java +++ /dev/null @@ -1,149 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.commands; - -import org.apache.doris.backup.CatalogMocker; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.info.PartitionNamesInfo; -import org.apache.doris.catalog.info.TableNameInfo; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.UserException; -import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.datasource.InternalCatalog; -import org.apache.doris.mysql.privilege.AccessControllerManager; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.utframe.TestWithFeService; - -import com.google.common.collect.ImmutableList; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -public class AlterColumnStatsCommandTest extends TestWithFeService { - private static final String internalCtl = InternalCatalog.INTERNAL_CATALOG_NAME; - private ConnectContext connectContext; - private Env env; - private AccessControllerManager accessControllerManager; - private Database db; - - private void runBefore() throws IOException { - connectContext = createDefaultCtx(); - env = Env.getCurrentEnv(); - accessControllerManager = env.getAccessManager(); - } - - @Test - public void testValidateNormal() throws Exception { - runBefore(); - connectContext.setSkipAuth(true); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(true).when(spyAcm).checkTblPriv(Mockito.nullable(ConnectContext.class), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), - Mockito.any(PrivPredicate.class)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - //test normal - connectContext.getSessionVariable().enableStats = true; - createDatabase("test_db"); - createTable("create table test_db.test_tbl\n" + "(k1 int, k2 int)\n" - + "duplicate key(k1)\n" + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" - + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1'); "); - - TableNameInfo tableNameInfo = - new TableNameInfo("test_db", "test_tbl"); - PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, - ImmutableList.of("p1")); - String indexName = null; - String columnName = "k1"; - Map properties = new HashMap<>(); - properties.put("row_count", "5"); - properties.put("avg_size", "100000"); - AlterColumnStatsCommand command = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties); - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - - //test not a partitioned table - createTable("create table test_db.test_tbl2(k1 int) distributed by hash(k1) buckets 3 properties('replication_num' = '1');"); - TableNameInfo tableNameInfo2 = - new TableNameInfo("test_db", "test_tbl2"); - AlterColumnStatsCommand command2 = new AlterColumnStatsCommand(tableNameInfo2, partitionNamesInfo, indexName, columnName, properties); - Assertions.assertThrows(AnalysisException.class, () -> command2.validate(connectContext), - "Not a partitioned table: test_tbl2"); - - //test partition does not exist - PartitionNamesInfo partitionNamesInfo2 = new PartitionNamesInfo(false, - ImmutableList.of("k3")); - AlterColumnStatsCommand command3 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo2, indexName, columnName, properties); - Assertions.assertThrows(AnalysisException.class, () -> command3.validate(connectContext), - "Partition does not exist: k3"); - - //test indexId not exist in OlapTable - String indexName3 = "invalid_index"; - AlterColumnStatsCommand command4 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName3, columnName, properties); - Assertions.assertThrows(AnalysisException.class, () -> command4.validate(connectContext), - "Index invalid_index not exist in table test_tbl"); - - //test invalid statistics - Map properties2 = new HashMap<>(); - properties2.put("histogram", "invalide_value"); - AlterColumnStatsCommand command5 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties2); - Assertions.assertThrows(AnalysisException.class, () -> command5.validate(connectContext), - "histogram is invalid statistics"); - - //row_count is not exist - Map properties3 = new HashMap<>(); - properties2.put("avg_size", "100000"); - properties2.put("max_size", "100000000"); - AlterColumnStatsCommand command6 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties3); - Assertions.assertThrows(AnalysisException.class, () -> command6.validate(connectContext), - "Set column stats must set row_count. e.g. 'row_count'='5'"); - - //test enable stats - connectContext.getSessionVariable().enableStats = false; - AlterColumnStatsCommand command7 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties); - Assertions.assertThrows(UserException.class, () -> command7.validate(connectContext), - "Analyze function is forbidden, you should add `enable_stats=true` in your FE conf file"); - } - - @Test - void testValidateNoPrivilege() throws IOException { - runBefore(); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(false).when(spyAcm).checkTblPriv(Mockito.nullable(ConnectContext.class), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), - Mockito.any(PrivPredicate.class)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - TableNameInfo tableNameInfo = - new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL2_NAME); - PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, - ImmutableList.of(CatalogMocker.TEST_PARTITION1_NAME)); - - String indexName = "index1"; - String columnName = "k1"; - Map properties = new HashMap<>(); - - AlterColumnStatsCommand command = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties); - connectContext.getSessionVariable().enableStats = true; - Assertions.assertThrows(AnalysisException.class, () -> command.validate(connectContext), - "ALTER TABLE STATS command denied to user 'null'@'null' for table 'test_db: test_tbl2'"); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CleanQueryStatsCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CleanQueryStatsCommandTest.java deleted file mode 100644 index 2b8659a0e85ec7..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CleanQueryStatsCommandTest.java +++ /dev/null @@ -1,86 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.commands; - -import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.info.TableNameInfo; -import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.datasource.InternalCatalog; -import org.apache.doris.mysql.privilege.AccessControllerManager; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.utframe.TestWithFeService; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.io.IOException; - -public class CleanQueryStatsCommandTest extends TestWithFeService { - private static final String internalCtl = InternalCatalog.INTERNAL_CATALOG_NAME; - private ConnectContext connectContext; - private Env env; - private AccessControllerManager accessControllerManager; - - public void runBefore() throws IOException { - connectContext = createDefaultCtx(); - env = Env.getCurrentEnv(); - accessControllerManager = env.getAccessManager(); - } - - @Test - public void testAllNormal() throws IOException { - runBefore(); - connectContext.setSkipAuth(true); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(true).when(spyAcm).checkGlobalPriv(Mockito.nullable(ConnectContext.class), Mockito.any(PrivPredicate.class)); - Deencapsulation.setField(env, "accessManager", spyAcm); - CleanQueryStatsCommand command = new CleanQueryStatsCommand(); - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - } - - @Test - public void testDB() throws Exception { - runBefore(); - connectContext.setDatabase("test_db"); - connectContext.setSkipAuth(true); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(true).when(spyAcm).checkDbPriv(Mockito.nullable(ConnectContext.class), Mockito.anyString(), Mockito.anyString(), Mockito.any(PrivPredicate.class)); - Deencapsulation.setField(env, "accessManager", spyAcm); - CleanQueryStatsCommand command = new CleanQueryStatsCommand("test_db"); - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - } - - @Test - public void testTbl() throws Exception { - runBefore(); - createDatabase("test_db"); - createTable("create table test_db.test_tbl\n" + "(k1 int, k2 int)\n" - + "duplicate key(k1)\n" + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" - + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1'); "); - TableNameInfo tableNameInfo = new TableNameInfo("test_db", "test_tbl"); - connectContext.setDatabase("test_db"); - connectContext.setSkipAuth(true); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(true).when(spyAcm).checkTblPriv(Mockito.nullable(ConnectContext.class), Mockito.any(TableNameInfo.class), Mockito.any(PrivPredicate.class)); - Deencapsulation.setField(env, "accessManager", spyAcm); - CleanQueryStatsCommand command = new CleanQueryStatsCommand(tableNameInfo); - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CommandPrivilegeValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CommandPrivilegeValidationTest.java new file mode 100644 index 00000000000000..6d27470e2d92c2 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CommandPrivilegeValidationTest.java @@ -0,0 +1,490 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.backup.CatalogMocker; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.info.PartitionNamesInfo; +import org.apache.doris.catalog.info.TableNameInfo; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.UserException; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.info.TableRefInfo; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.analyzer.UnboundSlot; +import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Or; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Merged suite: each of these classes only needed a bare FE and a spied access manager, so they all + * paid a full FE startup for two or three assertions. Members that were byte-identical across the + * originals are shared; test methods whose names collided are prefixed with their original class so + * grepping the old name still finds them. + * + *

Replaces the former standalone classes: + *

    + *
  • AdminRepairTableCommandTest
    • + *
  • AdminCopyTabletCommandTest
    • + *
  • AlterColumnStatsCommandTest
    • + *
  • CleanQueryStatsCommandTest
    • + *
  • ShowQueryStatsCommandTest
    • + *
  • ShowTabletsFromTableCommandTest
    • + *
+ */ +public class CommandPrivilegeValidationTest extends TestWithFeService { + + private static final String internalCtl = InternalCatalog.INTERNAL_CATALOG_NAME; + private static boolean dbInitialized = false; + private AccessControllerManager accessControllerManager; + private ConnectContext connectContext; + private Env env; + private Database db; + + // ------------------------------------------------------------------------- + // from AdminRepairTableCommandTest + // ------------------------------------------------------------------------- + + private void adminRepairTable_runBefore() throws IOException { + connectContext = createDefaultCtx(); + env = Env.getCurrentEnv(); + accessControllerManager = env.getAccessManager(); + } + + @Test + public void adminRepairTable_testValidateNormal() throws Exception { + adminRepairTable_runBefore(); + connectContext.setSkipAuth(true); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(true).when(spyAcm).checkGlobalPriv( + Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + TableNameInfo tableNameInfo = new TableNameInfo(internalCtl, "test_db", "test_tbl"); + List partitionNames = new ArrayList<>(); + partitionNames.add("p1"); + partitionNames.add("p2"); + PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, partitionNames); + TableRefInfo tableRefInfo = new TableRefInfo(tableNameInfo, null, null, partitionNamesInfo, null, null, null, null); + AdminRepairTableCommand command = new AdminRepairTableCommand(tableRefInfo); + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + + //test external catalog + TableNameInfo tableNameInfo2 = new TableNameInfo("hive", "test_db", "test_tbl"); + TableRefInfo tableRefInfo2 = new TableRefInfo(tableNameInfo2, null, null, partitionNamesInfo, null, null, null, null); + AdminRepairTableCommand command2 = new AdminRepairTableCommand(tableRefInfo2); + Assertions.assertThrows(AnalysisException.class, () -> command2.validate(connectContext), + "External catalog 'hive' is not allowed in 'AdminCancelRepairTableCommand.class'"); + + //test partitionNameInfo isTemp + PartitionNamesInfo partitionNamesInfo2 = new PartitionNamesInfo(true, partitionNames); + TableRefInfo tableRefInfo3 = new TableRefInfo(tableNameInfo, null, null, partitionNamesInfo2, null, null, null, null); + AdminRepairTableCommand command3 = new AdminRepairTableCommand(tableRefInfo3); + Assertions.assertThrows(AnalysisException.class, () -> command3.validate(connectContext), + "Do not support (cancel)repair temporary partitions"); + } + + @Test + public void testValidateNoPriviledge() throws Exception { + adminRepairTable_runBefore(); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(false).when(spyAcm).checkGlobalPriv( + Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + TableNameInfo tableNameInfo = new TableNameInfo(internalCtl, "test_db", "test_tbl"); + List partitionNames = new ArrayList<>(); + PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, partitionNames); + TableRefInfo tableRefInfo = new TableRefInfo(tableNameInfo, null, null, partitionNamesInfo, null, null, null, null); + AdminRepairTableCommand command = new AdminRepairTableCommand(tableRefInfo); + Assertions.assertThrows(AnalysisException.class, () -> command.validate(connectContext), + "Access denied; you need (at least one of) the (ADMIN) privilege(s) for this operation"); + } + + // ------------------------------------------------------------------------- + // from AdminCopyTabletCommandTest + // ------------------------------------------------------------------------- + + private void adminCopyTablet_runBefore() throws IOException { + connectContext = createDefaultCtx(); + env = Env.getCurrentEnv(); + accessControllerManager = env.getAccessManager(); + } + + @Test + public void adminCopyTablet_testValidateNormal() throws Exception { + adminCopyTablet_runBefore(); + connectContext.setSkipAuth(true); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(true).when(spyAcm).checkGlobalPriv( + Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + Map properties = new HashMap<>(); + properties.put("version", "0"); + AdminCopyTabletCommand command = new AdminCopyTabletCommand(100, properties); + Assertions.assertDoesNotThrow(() -> command.validate()); + + Map properties2 = new HashMap<>(); + properties2.put("backend_id", "10"); + AdminCopyTabletCommand command2 = new AdminCopyTabletCommand(100, properties2); + Assertions.assertDoesNotThrow(() -> command2.validate()); + + Map properties3 = new HashMap<>(); + properties3.put("expiration_minutes", "10"); + AdminCopyTabletCommand command3 = new AdminCopyTabletCommand(100, properties3); + Assertions.assertDoesNotThrow(() -> command3.validate()); + } + + @Test + public void testNoPriviledge() throws Exception { + adminCopyTablet_runBefore(); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(false).when(spyAcm).checkGlobalPriv( + Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + Map properties = new HashMap<>(); + properties.put("version", "0"); + AdminCopyTabletCommand command = new AdminCopyTabletCommand(100, properties); + Assertions.assertThrows(AnalysisException.class, () -> command.validate(), + "Access denied; you need (at least one of) the (Admin_priv) privilege(s) for this operation"); + } + + // ------------------------------------------------------------------------- + // from AlterColumnStatsCommandTest + // ------------------------------------------------------------------------- + + private void alterColumnStats_runBefore() throws IOException { + connectContext = createDefaultCtx(); + env = Env.getCurrentEnv(); + accessControllerManager = env.getAccessManager(); + } + + @Test + public void alterColumnStats_testValidateNormal() throws Exception { + alterColumnStats_runBefore(); + connectContext.setSkipAuth(true); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(true).when(spyAcm).checkTblPriv(Mockito.nullable(ConnectContext.class), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), + Mockito.any(PrivPredicate.class)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + //test normal + connectContext.getSessionVariable().enableStats = true; + createDatabase("alter_column_stats_db"); + createTable("create table alter_column_stats_db.test_tbl\n" + "(k1 int, k2 int)\n" + + "duplicate key(k1)\n" + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" + + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1'); "); + + TableNameInfo tableNameInfo = + new TableNameInfo("alter_column_stats_db", "test_tbl"); + PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, + ImmutableList.of("p1")); + String indexName = null; + String columnName = "k1"; + Map properties = new HashMap<>(); + properties.put("row_count", "5"); + properties.put("avg_size", "100000"); + AlterColumnStatsCommand command = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties); + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + + //test not a partitioned table + createTable("create table alter_column_stats_db.test_tbl2(k1 int) distributed by hash(k1) buckets 3 properties('replication_num' = '1');"); + TableNameInfo tableNameInfo2 = + new TableNameInfo("alter_column_stats_db", "test_tbl2"); + AlterColumnStatsCommand command2 = new AlterColumnStatsCommand(tableNameInfo2, partitionNamesInfo, indexName, columnName, properties); + Assertions.assertThrows(AnalysisException.class, () -> command2.validate(connectContext), + "Not a partitioned table: test_tbl2"); + + //test partition does not exist + PartitionNamesInfo partitionNamesInfo2 = new PartitionNamesInfo(false, + ImmutableList.of("k3")); + AlterColumnStatsCommand command3 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo2, indexName, columnName, properties); + Assertions.assertThrows(AnalysisException.class, () -> command3.validate(connectContext), + "Partition does not exist: k3"); + + //test indexId not exist in OlapTable + String indexName3 = "invalid_index"; + AlterColumnStatsCommand command4 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName3, columnName, properties); + Assertions.assertThrows(AnalysisException.class, () -> command4.validate(connectContext), + "Index invalid_index not exist in table test_tbl"); + + //test invalid statistics + Map properties2 = new HashMap<>(); + properties2.put("histogram", "invalide_value"); + AlterColumnStatsCommand command5 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties2); + Assertions.assertThrows(AnalysisException.class, () -> command5.validate(connectContext), + "histogram is invalid statistics"); + + //row_count is not exist + Map properties3 = new HashMap<>(); + properties2.put("avg_size", "100000"); + properties2.put("max_size", "100000000"); + AlterColumnStatsCommand command6 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties3); + Assertions.assertThrows(AnalysisException.class, () -> command6.validate(connectContext), + "Set column stats must set row_count. e.g. 'row_count'='5'"); + + //test enable stats + connectContext.getSessionVariable().enableStats = false; + AlterColumnStatsCommand command7 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties); + Assertions.assertThrows(UserException.class, () -> command7.validate(connectContext), + "Analyze function is forbidden, you should add `enable_stats=true` in your FE conf file"); + } + + @Test + void alterColumnStats_testValidateNoPrivilege() throws IOException { + alterColumnStats_runBefore(); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(false).when(spyAcm).checkTblPriv(Mockito.nullable(ConnectContext.class), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), + Mockito.any(PrivPredicate.class)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + TableNameInfo tableNameInfo = + new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL2_NAME); + PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, + ImmutableList.of(CatalogMocker.TEST_PARTITION1_NAME)); + + String indexName = "index1"; + String columnName = "k1"; + Map properties = new HashMap<>(); + + AlterColumnStatsCommand command = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties); + connectContext.getSessionVariable().enableStats = true; + Assertions.assertThrows(AnalysisException.class, () -> command.validate(connectContext), + "ALTER TABLE STATS command denied to user 'null'@'null' for table 'alter_column_stats_db: test_tbl2'"); + } + + // ------------------------------------------------------------------------- + // from CleanQueryStatsCommandTest + // ------------------------------------------------------------------------- + + public void cleanQueryStats_runBefore() throws IOException { + connectContext = createDefaultCtx(); + env = Env.getCurrentEnv(); + accessControllerManager = env.getAccessManager(); + } + + @Test + public void testAllNormal() throws IOException { + cleanQueryStats_runBefore(); + connectContext.setSkipAuth(true); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(true).when(spyAcm).checkGlobalPriv(Mockito.nullable(ConnectContext.class), Mockito.any(PrivPredicate.class)); + Deencapsulation.setField(env, "accessManager", spyAcm); + CleanQueryStatsCommand command = new CleanQueryStatsCommand(); + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + } + + @Test + public void testDB() throws Exception { + cleanQueryStats_runBefore(); + connectContext.setDatabase("clean_query_stats_db"); + connectContext.setSkipAuth(true); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(true).when(spyAcm).checkDbPriv(Mockito.nullable(ConnectContext.class), Mockito.anyString(), Mockito.anyString(), Mockito.any(PrivPredicate.class)); + Deencapsulation.setField(env, "accessManager", spyAcm); + CleanQueryStatsCommand command = new CleanQueryStatsCommand("clean_query_stats_db"); + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + } + + @Test + public void testTbl() throws Exception { + cleanQueryStats_runBefore(); + createDatabase("clean_query_stats_db"); + createTable("create table clean_query_stats_db.test_tbl\n" + "(k1 int, k2 int)\n" + + "duplicate key(k1)\n" + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" + + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1'); "); + TableNameInfo tableNameInfo = new TableNameInfo("clean_query_stats_db", "test_tbl"); + connectContext.setDatabase("clean_query_stats_db"); + connectContext.setSkipAuth(true); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(true).when(spyAcm).checkTblPriv(Mockito.nullable(ConnectContext.class), Mockito.any(TableNameInfo.class), Mockito.any(PrivPredicate.class)); + Deencapsulation.setField(env, "accessManager", spyAcm); + CleanQueryStatsCommand command = new CleanQueryStatsCommand(tableNameInfo); + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + } + + // ------------------------------------------------------------------------- + // from ShowQueryStatsCommandTest + // ------------------------------------------------------------------------- + + private void showQueryStats_runBefore() throws Exception { + connectContext = createDefaultCtx(); + env = Env.getCurrentEnv(); + accessControllerManager = env.getAccessManager(); + if (!dbInitialized) { + createDatabaseWithSql("CREATE DATABASE IF NOT EXISTS " + CatalogMocker.TEST_DB_NAME); + createTable("CREATE TABLE IF NOT EXISTS " + CatalogMocker.TEST_DB_NAME + "." + CatalogMocker.TEST_TBL_NAME + + " (k1 INT, k2 INT) DISTRIBUTED BY HASH(k1) BUCKETS 1" + + " PROPERTIES ('replication_num' = '1')"); + dbInitialized = true; + } + } + + @Test + public void showQueryStats_testValidateWithPrivilege() throws Exception { + showQueryStats_runBefore(); + connectContext.setSkipAuth(true); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(true).when(spyAcm).checkGlobalPriv( + Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); + Mockito.doReturn(true).when(spyAcm).checkTblPriv( + Mockito.nullable(ConnectContext.class), Mockito.any(TableNameInfo.class), + Mockito.eq(PrivPredicate.SHOW)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + TableNameInfo tableNameInfo = + new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME); + + // normal + ShowQueryStatsCommand command = new ShowQueryStatsCommand(tableNameInfo.getDb(), + tableNameInfo, false, false); + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + } + + @Test + void showQueryStats_testValidateNoPrivilege() throws Exception { + showQueryStats_runBefore(); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(false).when(spyAcm).checkGlobalPriv( + Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); + Mockito.doReturn(false).when(spyAcm).checkTblPriv( + Mockito.nullable(ConnectContext.class), Mockito.anyString(), + Mockito.anyString(), Mockito.anyString(), Mockito.eq(PrivPredicate.SHOW)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + TableNameInfo tableNameInfo = + new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME); + ShowQueryStatsCommand command = new ShowQueryStatsCommand(tableNameInfo.getDb(), + tableNameInfo, false, false); + Assertions.assertThrows(AnalysisException.class, () -> command.validate(connectContext)); + } + + // ------------------------------------------------------------------------- + // from ShowTabletsFromTableCommandTest + // ------------------------------------------------------------------------- + + private void showTabletsFromTable_runBefore() throws IOException { + connectContext = createDefaultCtx(); + env = Env.getCurrentEnv(); + accessControllerManager = env.getAccessManager(); + } + + @Test + public void showTabletsFromTable_testValidateWithPrivilege() throws Exception { + showTabletsFromTable_runBefore(); + connectContext.setSkipAuth(true); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(true).when(spyAcm).checkGlobalPriv( + Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); + Mockito.doReturn(true).when(spyAcm).checkTblPriv( + Mockito.nullable(ConnectContext.class), Mockito.anyString(), + Mockito.anyString(), Mockito.anyString(), Mockito.eq(PrivPredicate.ADMIN)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + Expression version = new UnboundSlot("version"); + + Expression whereClauseNormal = new EqualTo(version, new IntegerLiteral(2)); + + List orderKeysNormal = new ArrayList<>(); + orderKeysNormal.add(new OrderKey(version, true, true)); + + TableNameInfo tableNameInfo = + new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME); + PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, + ImmutableList.of(CatalogMocker.TEST_SINGLE_PARTITION_NAME)); + + // normal + ShowTabletsFromTableCommand command = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, + whereClauseNormal, orderKeysNormal, 5, 0); + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + + // where clause error + Expression error = new UnboundSlot("error"); + Expression whereClauseError = new EqualTo(error, new IntegerLiteral(2)); + + ShowTabletsFromTableCommand command2 = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, + whereClauseError, orderKeysNormal, 5, 0); + Assertions.assertThrows(AnalysisException.class, () -> command2.validate(connectContext)); + + // where clause contains or + Expression backendId = new UnboundSlot("BackendId"); + Expression whereClauseOr = new Or( + new EqualTo(version, new IntegerLiteral(2)), + new EqualTo(backendId, new IntegerLiteral(2))); + + ShowTabletsFromTableCommand command3 = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, + whereClauseOr, orderKeysNormal, 5, 0); + Assertions.assertThrows(AnalysisException.class, () -> command3.validate(connectContext)); + + // order by error + List orderKeysError = new ArrayList<>(); + orderKeysError.add(new OrderKey(error, true, true)); + + ShowTabletsFromTableCommand command4 = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, + whereClauseNormal, orderKeysError, 5, 0); + Assertions.assertThrows(AnalysisException.class, () -> command4.validate(connectContext)); + } + + @Test + void showTabletsFromTable_testValidateNoPrivilege() throws Exception { + showTabletsFromTable_runBefore(); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(false).when(spyAcm).checkGlobalPriv( + Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); + Mockito.doReturn(false).when(spyAcm).checkTblPriv( + Mockito.nullable(ConnectContext.class), Mockito.anyString(), + Mockito.anyString(), Mockito.anyString(), Mockito.eq(PrivPredicate.ADMIN)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + Expression version = new UnboundSlot("version"); + + Expression whereClauseNormal = new EqualTo(version, new IntegerLiteral(2)); + + List orderKeysNormal = new ArrayList<>(); + orderKeysNormal.add(new OrderKey(version, true, true)); + + TableNameInfo tableNameInfo = + new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME); + PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, + ImmutableList.of(CatalogMocker.TEST_SINGLE_PARTITION_NAME)); + + ShowTabletsFromTableCommand command = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, + whereClauseNormal, orderKeysNormal, 5, 0); + Assertions.assertThrows(AnalysisException.class, () -> command.validate(connectContext)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowQueryStatsCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowQueryStatsCommandTest.java deleted file mode 100644 index 472d90cea95502..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowQueryStatsCommandTest.java +++ /dev/null @@ -1,94 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.commands; - -import org.apache.doris.backup.CatalogMocker; -import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.info.TableNameInfo; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.datasource.InternalCatalog; -import org.apache.doris.mysql.privilege.AccessControllerManager; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.utframe.TestWithFeService; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -public class ShowQueryStatsCommandTest extends TestWithFeService { - private static final String internalCtl = InternalCatalog.INTERNAL_CATALOG_NAME; - private static boolean dbInitialized = false; - - private ConnectContext connectContext; - private Env env; - private AccessControllerManager accessControllerManager; - - private void runBefore() throws Exception { - connectContext = createDefaultCtx(); - env = Env.getCurrentEnv(); - accessControllerManager = env.getAccessManager(); - if (!dbInitialized) { - createDatabaseWithSql("CREATE DATABASE IF NOT EXISTS " + CatalogMocker.TEST_DB_NAME); - createTable("CREATE TABLE IF NOT EXISTS " + CatalogMocker.TEST_DB_NAME + "." + CatalogMocker.TEST_TBL_NAME - + " (k1 INT, k2 INT) DISTRIBUTED BY HASH(k1) BUCKETS 1" - + " PROPERTIES ('replication_num' = '1')"); - dbInitialized = true; - } - } - - @Test - public void testValidateWithPrivilege() throws Exception { - runBefore(); - connectContext.setSkipAuth(true); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(true).when(spyAcm).checkGlobalPriv( - Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); - Mockito.doReturn(true).when(spyAcm).checkTblPriv( - Mockito.nullable(ConnectContext.class), Mockito.any(TableNameInfo.class), - Mockito.eq(PrivPredicate.SHOW)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - TableNameInfo tableNameInfo = - new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME); - - // normal - ShowQueryStatsCommand command = new ShowQueryStatsCommand(tableNameInfo.getDb(), - tableNameInfo, false, false); - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - } - - @Test - void testValidateNoPrivilege() throws Exception { - runBefore(); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(false).when(spyAcm).checkGlobalPriv( - Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); - Mockito.doReturn(false).when(spyAcm).checkTblPriv( - Mockito.nullable(ConnectContext.class), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString(), Mockito.eq(PrivPredicate.SHOW)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - TableNameInfo tableNameInfo = - new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME); - ShowQueryStatsCommand command = new ShowQueryStatsCommand(tableNameInfo.getDb(), - tableNameInfo, false, false); - Assertions.assertThrows(AnalysisException.class, () -> command.validate(connectContext)); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletsFromTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletsFromTableCommandTest.java deleted file mode 100644 index 17613cd057ce02..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletsFromTableCommandTest.java +++ /dev/null @@ -1,142 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.commands; - -import org.apache.doris.backup.CatalogMocker; -import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.info.PartitionNamesInfo; -import org.apache.doris.catalog.info.TableNameInfo; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.datasource.InternalCatalog; -import org.apache.doris.mysql.privilege.AccessControllerManager; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.nereids.analyzer.UnboundSlot; -import org.apache.doris.nereids.properties.OrderKey; -import org.apache.doris.nereids.trees.expressions.EqualTo; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.Or; -import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.utframe.TestWithFeService; - -import com.google.common.collect.ImmutableList; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -public class ShowTabletsFromTableCommandTest extends TestWithFeService { - private static final String internalCtl = InternalCatalog.INTERNAL_CATALOG_NAME; - private ConnectContext connectContext; - private Env env; - private AccessControllerManager accessControllerManager; - - private void runBefore() throws IOException { - connectContext = createDefaultCtx(); - env = Env.getCurrentEnv(); - accessControllerManager = env.getAccessManager(); - } - - @Test - public void testValidateWithPrivilege() throws Exception { - runBefore(); - connectContext.setSkipAuth(true); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(true).when(spyAcm).checkGlobalPriv( - Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); - Mockito.doReturn(true).when(spyAcm).checkTblPriv( - Mockito.nullable(ConnectContext.class), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString(), Mockito.eq(PrivPredicate.ADMIN)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - Expression version = new UnboundSlot("version"); - - Expression whereClauseNormal = new EqualTo(version, new IntegerLiteral(2)); - - List orderKeysNormal = new ArrayList<>(); - orderKeysNormal.add(new OrderKey(version, true, true)); - - TableNameInfo tableNameInfo = - new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME); - PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, - ImmutableList.of(CatalogMocker.TEST_SINGLE_PARTITION_NAME)); - - // normal - ShowTabletsFromTableCommand command = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, - whereClauseNormal, orderKeysNormal, 5, 0); - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - - // where clause error - Expression error = new UnboundSlot("error"); - Expression whereClauseError = new EqualTo(error, new IntegerLiteral(2)); - - ShowTabletsFromTableCommand command2 = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, - whereClauseError, orderKeysNormal, 5, 0); - Assertions.assertThrows(AnalysisException.class, () -> command2.validate(connectContext)); - - // where clause contains or - Expression backendId = new UnboundSlot("BackendId"); - Expression whereClauseOr = new Or( - new EqualTo(version, new IntegerLiteral(2)), - new EqualTo(backendId, new IntegerLiteral(2))); - - ShowTabletsFromTableCommand command3 = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, - whereClauseOr, orderKeysNormal, 5, 0); - Assertions.assertThrows(AnalysisException.class, () -> command3.validate(connectContext)); - - // order by error - List orderKeysError = new ArrayList<>(); - orderKeysError.add(new OrderKey(error, true, true)); - - ShowTabletsFromTableCommand command4 = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, - whereClauseNormal, orderKeysError, 5, 0); - Assertions.assertThrows(AnalysisException.class, () -> command4.validate(connectContext)); - } - - @Test - void testValidateNoPrivilege() throws Exception { - runBefore(); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(false).when(spyAcm).checkGlobalPriv( - Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); - Mockito.doReturn(false).when(spyAcm).checkTblPriv( - Mockito.nullable(ConnectContext.class), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString(), Mockito.eq(PrivPredicate.ADMIN)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - Expression version = new UnboundSlot("version"); - - Expression whereClauseNormal = new EqualTo(version, new IntegerLiteral(2)); - - List orderKeysNormal = new ArrayList<>(); - orderKeysNormal.add(new OrderKey(version, true, true)); - - TableNameInfo tableNameInfo = - new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME); - PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, - ImmutableList.of(CatalogMocker.TEST_SINGLE_PARTITION_NAME)); - - ShowTabletsFromTableCommand command = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, - whereClauseNormal, orderKeysNormal, 5, 0); - Assertions.assertThrows(AnalysisException.class, () -> command.validate(connectContext)); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java index e82c56942cdcce..da9e9aea08bb77 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java @@ -35,6 +35,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.job.base.AbstractJob; import org.apache.doris.nereids.CascadesContext; @@ -99,6 +100,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -144,12 +146,24 @@ public abstract class TestWithFeService { protected static final String DEFAULT_CLUSTER_PREFIX = ""; + // Env fields that tests routinely swap for a Mockito spy. They are captured once class setup has + // finished and restored after every test method, so a spy installed by one test method cannot leak + // into the next one. Capturing after runBeforeAll() is deliberate: a class that installs a spy for + // the whole class (CheckRowPolicyTest) has that spy as its baseline, making the restore a no-op there. + private static final String[] ENV_FIELDS_RESTORED_PER_TEST = new String[] { + "accessManager", "systemInfo", "authenticationIntegrationMgr"}; + private final Map envFieldBaseline = Maps.newHashMap(); + @BeforeAll public final void beforeAll() throws Exception { // this.enableAdvanceNextId may be reset by children classes Config.enable_advance_next_id = this.enableAdvanceNextId; FeConstants.enableInternalSchemaDb = false; FeConstants.disableWGCheckerForUT = true; + // HeartbeatMgr runs its first cycle before createDorisCluster() registers any BE, so with the + // production default (10s) every test class blocks a full interval waiting for the second cycle + // to mark BEs alive. Set before beforeCreatingConnectContext() so subclasses can still override. + Config.heartbeat_interval_second = 1; beforeCreatingConnectContext(); connectContext = createDefaultCtx(); connectContext.getSessionVariable().feDebug = true; @@ -157,6 +171,10 @@ public final void beforeAll() throws Exception { createDorisCluster(); Env.getCurrentEnv().getWorkloadGroupMgr().createNormalWorkloadGroupForUT(); runBeforeAll(); + Env env = Env.getCurrentEnv(); + for (String field : ENV_FIELDS_RESTORED_PER_TEST) { + envFieldBaseline.put(field, Deencapsulation.getField(env, field)); + } } protected void beforeCluster() { @@ -177,6 +195,14 @@ public final void beforeEach() throws Exception { runBeforeEach(); } + @AfterEach + public final void afterEach() { + Env env = Env.getCurrentEnv(); + for (Map.Entry baseline : envFieldBaseline.entrySet()) { + Deencapsulation.setField(env, baseline.getKey(), baseline.getValue()); + } + } + protected void beforeCreatingConnectContext() throws Exception { } @@ -424,16 +450,18 @@ protected boolean checkBELostHeartbeat(List bes) { } private boolean checkBEHeartbeatStatus(List bes, boolean isAlive) { - int maxTry = Config.heartbeat_interval_second + 2; - while (maxTry-- > 0) { + // Poll instead of sleeping a fixed 1s before each check: the status flips as soon as + // HeartbeatMgr finishes a cycle, so a coarse sleep just wastes wall clock in every test class. + long deadline = System.currentTimeMillis() + (Config.heartbeat_interval_second + 2) * 1000L; + while (System.currentTimeMillis() < deadline) { + if (bes.stream().allMatch(be -> be.isAlive() == isAlive)) { + return true; + } try { - Thread.sleep(1000); + Thread.sleep(20); } catch (InterruptedException e) { // no exception } - if (bes.stream().allMatch(be -> be.isAlive() == isAlive)) { - return true; - } } return false;