From 86d1820bb0fac79eceddbfb24466f9b4320ec767 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 28 Jul 2026 23:04:34 +0800 Subject: [PATCH 1/5] [opt](test) Avoid a 10s heartbeat wait in every FE unit test class Every test class deriving from TestWithFeService spends ~12s in createDorisCluster(), of which ~10s is pure idle waiting. Daemon.run() executes one cycle before sleeping, so HeartbeatMgr's first cycle completes while createDorisCluster() has not registered any backend yet. checkBEHeartbeat() therefore has to wait for the second cycle, which is heartbeat_interval_second (10s by default) away. Two changes: - Set heartbeat_interval_second to 1 at the start of beforeAll(). It is placed before beforeCreatingConnectContext() so subclasses that set the interval explicitly (DecommissionBackendTest, BeDownCancelCloneTest) still win, and before the Env singleton is created so HeartbeatMgr picks up the new interval. It is deliberately not put into MockedFrontend's MIN_FE_CONF, because DorisFE.start() would then load it from fe.conf and silently override those subclasses. - Make checkBEHeartbeatStatus() check before sleeping and poll at 20ms instead of sleeping a fixed 1s before each check. The overall timeout budget is unchanged. Measured on 20 nereids command test classes (43 tests, all passing before and after): fork=1 wall 399s -> 211s, reported test time 283s -> 95s fork=4 wall 399s -> 67s A synthetic class that only starts the FE and asserts once goes from 12.0s to 2.83s. Co-Authored-By: Claude Opus 5 (1M context) --- .../doris/utframe/TestWithFeService.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java index e82c56942cdcce..a3591281d20369 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java @@ -150,6 +150,10 @@ public final void beforeAll() throws Exception { Config.enable_advance_next_id = this.enableAdvanceNextId; FeConstants.enableInternalSchemaDb = false; FeConstants.disableWGCheckerForUT = true; + // HeartbeatMgr runs its first cycle before createDorisCluster() registers any BE, so with the + // production default (10s) every test class blocks a full interval waiting for the second cycle + // to mark BEs alive. Set before beforeCreatingConnectContext() so subclasses can still override. + Config.heartbeat_interval_second = 1; beforeCreatingConnectContext(); connectContext = createDefaultCtx(); connectContext.getSessionVariable().feDebug = true; @@ -424,16 +428,18 @@ protected boolean checkBELostHeartbeat(List bes) { } private boolean checkBEHeartbeatStatus(List bes, boolean isAlive) { - int maxTry = Config.heartbeat_interval_second + 2; - while (maxTry-- > 0) { + // Poll instead of sleeping a fixed 1s before each check: the status flips as soon as + // HeartbeatMgr finishes a cycle, so a coarse sleep just wastes wall clock in every test class. + long deadline = System.currentTimeMillis() + (Config.heartbeat_interval_second + 2) * 1000L; + while (System.currentTimeMillis() < deadline) { + if (bes.stream().allMatch(be -> be.isAlive() == isAlive)) { + return true; + } try { - Thread.sleep(1000); + Thread.sleep(20); } catch (InterruptedException e) { // no exception } - if (bes.stream().allMatch(be -> be.isAlive() == isAlive)) { - return true; - } } return false; From 9b4f4b1735b68b4ad2422b822f7067d8c95eafa4 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 29 Jul 2026 08:39:14 +0800 Subject: [PATCH 2/5] [opt](test) Merge 12 tiny nereids test classes into 3 suites Every FE unit test class pays a full FE startup. Measured on the FE UT CI agent (build 1008420), that fixed cost is about 54s per class: ~16s of JVM fork and class loading (measured from test classes that never start an FE) plus ~38s of FE startup. 210 of the 214 nereids test classes that start an FE finish in under 90s, i.e. they are almost entirely that fixed cost. Merging classes that already share the exact same fixture removes it without changing any test. This first batch merges 12 classes that all extend SqlTestBase and override no setup at all, so they already ran against an identical fixture (database "test" with tables T0..T4): sqltest/SqlPlanSuiteTest <- CascadesJoinReorderTest, InferTest, JoinTest, MultiJoinTest, SortTest rules/exploration/mv/MvExplorationSuiteTest <- BuildStructInfoTest, EliminateJoinTest, HyperGraphAggTest, HyperGraphComparatorTest, NullRejectInferenceTest rules/rewrite/RewriteRuleSuiteTest <- PruneOlapScanTabletTest, EliminateConstHashJoinConditionTest Test bodies are moved verbatim; method names are unchanged so test reports keep the same identifiers. Each merged file keeps a "from " section header, so grepping the old class name still lands on its tests. Verified: 53 tests before, 53 tests after, all passing. Deliberately not merged here: - nereids/mv (6 classes): they create materialized views with colliding names inside their test methods, so sharing one FE makes them interfere. Needs namespace isolation first. - jobs/joinorder/hypergraph (4 classes): testPullUpQueryFilter exists with different bodies in two of them. - Classes with real work (DistributeHintTest 764s, OtherJoinTest 256s, GraphSimplifierConsistencyTest 223s, PreMaterializedViewRewriterTest 205s) are intentionally left alone: merging them would create a straggler that hurts wall clock under the 12-way parallel CI run. Co-Authored-By: Claude Opus 5 (1M context) --- .../exploration/mv/BuildStructInfoTest.java | 207 ---- .../exploration/mv/EliminateJoinTest.java | 341 ------- .../exploration/mv/HyperGraphAggTest.java | 98 -- .../mv/HyperGraphComparatorTest.java | 185 ---- .../mv/MvExplorationSuiteTest.java | 951 ++++++++++++++++++ .../mv/NullRejectInferenceTest.java | 209 ---- .../EliminateConstHashJoinConditionTest.java | 74 -- ...letTest.java => RewriteRuleSuiteTest.java} | 70 +- .../sqltest/CascadesJoinReorderTest.java | 192 ---- .../doris/nereids/sqltest/InferTest.java | 122 --- .../doris/nereids/sqltest/JoinTest.java | 106 -- .../doris/nereids/sqltest/MultiJoinTest.java | 131 --- .../doris/nereids/sqltest/SortTest.java | 43 - .../nereids/sqltest/SqlPlanSuiteTest.java | 517 ++++++++++ 14 files changed, 1536 insertions(+), 1710 deletions(-) delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MvExplorationSuiteTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/NullRejectInferenceTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateConstHashJoinConditionTest.java rename fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/{PruneOlapScanTabletTest.java => RewriteRuleSuiteTest.java} (75%) delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/CascadesJoinReorderTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/JoinTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/MultiJoinTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SortTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlPlanSuiteTest.java diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java deleted file mode 100644 index bfecdbd4181049..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java +++ /dev/null @@ -1,207 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.exploration.mv; - -import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; -import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; -import org.apache.doris.nereids.sqltest.SqlTestBase; -import org.apache.doris.nereids.trees.plans.GroupPlan; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.util.PlanChecker; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -class BuildStructInfoTest extends SqlTestBase { - - @Test - void testSimpleSQL() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "select * from T1, T2, T3, T4 " - + "where " - + "T1.id = T2.id and " - + "T2.score = T3.score and " - + "T3.id = T4.id"; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .deriveStats() - .matches(logicalJoin() - .when(j -> { - HyperGraph.builderForMv(j); - return true; - })); - - } - - @Test - void testStructInfoNode() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "select * from T1 inner join " - + "(select sum(id) as id from T2 where id = 1) T2 " - + "on T1.id = T2.id"; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .deriveStats() - .matches(logicalJoin() - .when(j -> { - HyperGraph hyperGraph = HyperGraph.builderForMv(j).build(); - Assertions.assertTrue(hyperGraph.getNodes().stream() - .allMatch(n -> n.getPlan() - .collectToList(GroupPlan.class::isInstance).isEmpty())); - return true; - })); - - } - - @Test - void testFilter() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "select * from T1 left outer join " - + " (select id from T2 where id = 1) T2 " - + "on T1.id = T2.id "; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .matches(logicalJoin() - .when(j -> { - HyperGraph structInfo = HyperGraph.builderForMv(j).build(); - Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin()); - Assertions.assertEquals(0, structInfo.getFilterEdge(0).getLeftRejectEdge().size()); - Assertions.assertEquals(1, structInfo.getFilterEdge(0).getRightRejectEdge().size()); - return true; - })); - - sql = "select * from (select id from T1 where id = 0) T1 left outer join T2 " - + "on T1.id = T2.id "; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .matches(logicalJoin() - .when(j -> { - HyperGraph structInfo = HyperGraph.builderForMv(j).build(); - Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin()); - return true; - })); - } - - @Test - void testPlanCheckerWithJoin() { - // Should not make scan to empty relation when the table used by materialized view has no data - connectContext.getSessionVariable().setDisableNereidsRules( - "OLAP_SCAN_PARTITION_PRUNE" - + ",PRUNE_EMPTY_PARTITION" - + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" - + ",ELIMINATE_CONST_JOIN_CONDITION" - + ",CONSTANT_PROPAGATION" - ); - PlanChecker.from(connectContext) - .checkExplain("select * from " - + "(select * from lineitem " - + "where l_shipdate >= \"2023-12-01\" and l_shipdate <= \"2023-12-03\") t1 " - + "left join " - + "(select * from orders where o_orderdate >= \"2023-12-01\" and o_orderdate <= \"2023-12-03\" ) t2 " - + "on t1.l_orderkey = o_orderkey;", - nereidsPlanner -> { - Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - PlanCheckContext checkContext = PlanCheckContext.of( - AbstractMaterializedViewRule.SUPPORTED_JOIN_TYPE_SET); - Boolean result = rewrittenPlan.child(0).accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext); - Assertions.assertTrue(result); - Assertions.assertFalse(checkContext.isContainsTopAggregate()); - }); - } - - @Test - void testPlanCheckerWithAggregate() { - // Should not make scan to empty relation when the table used by materialized view has no data - connectContext.getSessionVariable().setDisableNereidsRules( - "OLAP_SCAN_PARTITION_PRUNE" - + ",PRUNE_EMPTY_PARTITION" - + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" - + ",ELIMINATE_CONST_JOIN_CONDITION" - + ",CONSTANT_PROPAGATION" - ); - PlanChecker.from(connectContext) - .checkExplain("SELECT l.L_SHIPDATE AS ship_data_alias, o.O_ORDERDATE, count(*) " - + "FROM " - + "lineitem as l " - + "LEFT JOIN " - + "(SELECT abs(O_TOTALPRICE + 10) as c1_abs, O_CUSTKEY, O_ORDERDATE, O_ORDERKEY " - + "FROM orders) as o " - + "ON l.L_ORDERKEY = o.O_ORDERKEY " - + "JOIN " - + "(SELECT abs(sqrt(PS_SUPPLYCOST)) as c2_abs, PS_AVAILQTY, PS_PARTKEY, PS_SUPPKEY " - + "FROM partsupp) as ps " - + "ON l.L_PARTKEY = ps.PS_PARTKEY and l.L_SUPPKEY = ps.PS_SUPPKEY " - + "GROUP BY l.L_SHIPDATE, o.O_ORDERDATE ", - nereidsPlanner -> { - Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - PlanCheckContext checkContext = PlanCheckContext.of( - AbstractMaterializedViewRule.SUPPORTED_JOIN_TYPE_SET); - Boolean result = rewrittenPlan.child(0).accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext); - Assertions.assertTrue(result); - Assertions.assertTrue(checkContext.isContainsTopAggregate()); - }); - } - - @Test - void testPlanCheckerScanAggregate() { - // Should not make scan to empty relation when the table used by materialized view has no data - connectContext.getSessionVariable().setDisableNereidsRules( - "OLAP_SCAN_PARTITION_PRUNE" - + ",PRUNE_EMPTY_PARTITION" - + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" - + ",ELIMINATE_CONST_JOIN_CONDITION" - + ",CONSTANT_PROPAGATION" - ); - PlanChecker.from(connectContext) - .checkExplain("select l.L_SHIPDATE, count(*) from lineitem l " - + "GROUP BY l.L_SHIPDATE", - nereidsPlanner -> { - Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - PlanCheckContext checkContext = PlanCheckContext.of( - AbstractMaterializedViewRule.SUPPORTED_JOIN_TYPE_SET); - Boolean result = rewrittenPlan.child(0).accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext); - Assertions.assertFalse(result); - }); - } - - @Test - void testPlanCheckerOnlyScan() { - // Should not make scan to empty relation when the table used by materialized view has no data - connectContext.getSessionVariable().setDisableNereidsRules( - "OLAP_SCAN_PARTITION_PRUNE" - + ",PRUNE_EMPTY_PARTITION" - + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" - + ",ELIMINATE_CONST_JOIN_CONDITION" - + ",CONSTANT_PROPAGATION" - ); - PlanChecker.from(connectContext) - .checkExplain("select l.L_SHIPDATE from lineitem l ", - nereidsPlanner -> { - Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - PlanCheckContext checkContext = PlanCheckContext.of( - AbstractMaterializedViewRule.SUPPORTED_JOIN_TYPE_SET); - Boolean result = rewrittenPlan.child(0).accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext); - Assertions.assertTrue(result); - Assertions.assertFalse(checkContext.isContainsTopAggregate()); - }); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java deleted file mode 100644 index 20697e78912d20..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java +++ /dev/null @@ -1,341 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.exploration.mv; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; -import org.apache.doris.nereids.rules.RuleSet; -import org.apache.doris.nereids.sqltest.SqlTestBase; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.util.PlanChecker; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -class EliminateJoinTest extends SqlTestBase { - @Test - void testLOJWithGroupBy() { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select * from T1", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - CascadesContext c2 = createCascadesContext( - "select * from T1 left outer join (select id from T2 group by id) T2 " - + "on T1.id = T2.id ", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - CascadesContext c3 = createCascadesContext( - "select * from T1 left outer join (select id as id2 from T2 group by id) T2 " - + "on T1.id = T2.id2 ", - connectContext - ); - Plan p3 = PlanChecker.from(c3) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - HyperGraph h3 = HyperGraph.builderForMv(p3).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertTrue(!res.isInvalid()); - Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3, - constructContext(p1, p2, c1)).isInvalid()); - Assertions.assertTrue(res.getViewExpressions().isEmpty()); - } - - @Test - void testLOJWithUK() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select * from T1", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - addConstraint("alter table T2 add constraint uk unique (id)"); - CascadesContext c2 = createCascadesContext( - "select * from T1 left outer join T2 " - + "on T1.id = T2.id ", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertTrue(!res.isInvalid()); - Assertions.assertTrue(res.getViewExpressions().isEmpty()); - dropConstraint("alter table T2 drop constraint uk"); - } - - @Test - void testLOJWithUKAndOtherJoinConjuncts() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select * from T1", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - addConstraint("alter table T2 add constraint uk_other_join_conjunct unique (id)"); - try { - CascadesContext c2 = createCascadesContext( - "select * from T1 left outer join T2 " - + "on T1.id = T2.id and T2.id = 1", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertFalse(res.isInvalid()); - Assertions.assertTrue(res.getViewExpressions().isEmpty()); - } finally { - dropConstraint("alter table T2 drop constraint uk_other_join_conjunct"); - } - } - - @Test - void testLOJWithUKAndFilterOnEliminatedNode() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select T1.id from T1", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - addConstraint("alter table T2 add constraint uk_loj_filter unique (id)"); - try { - CascadesContext c2 = createCascadesContext( - "select T1.id from T1 left outer join T2 " - + "on T1.id = T2.id where T2.id is null", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertTrue(res.isInvalid()); - } finally { - dropConstraint("alter table T2 drop constraint uk_loj_filter"); - } - } - - @Test - void testInnerJoinWithPKFKAndSlotFreeFilterOnEliminatedNode() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select * from T1", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - addConstraint("alter table T2 add constraint pk_slot_free_filter primary key (id)"); - addConstraint("alter table T1 add constraint fk_slot_free_filter foreign key (id) references T2(id)"); - try { - CascadesContext c2 = createCascadesContext( - "select * from T1 inner join (select * from T2 where 1 = 0) T2 " - + "on T1.id = T2.id", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertTrue(res.isInvalid()); - } finally { - dropConstraint("alter table T1 drop constraint fk_slot_free_filter"); - dropConstraint("alter table T2 drop constraint pk_slot_free_filter"); - } - } - - @Test - void testLOJWithPKFK() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select * from T1", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - addConstraint("alter table T2 add constraint pk primary key (id)"); - addConstraint("alter table T1 add constraint fk foreign key (id) references T2(id)"); - CascadesContext c2 = createCascadesContext( - "select * from T1 inner join T2 " - + "on T1.id = T2.id ", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - CascadesContext c3 = createCascadesContext( - "select * from T1 inner join (select id as id2 from T2) T2 " - + "on T1.id = T2.id2 ", - connectContext - ); - Plan p3 = PlanChecker.from(c3) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - HyperGraph h3 = HyperGraph.builderForMv(p3).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertTrue(!res.isInvalid()); - Assertions.assertTrue(res.getViewExpressions().isEmpty()); - Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3, constructContext(p1, p2, c1)).isInvalid()); - dropConstraint("alter table T2 drop constraint pk"); - } - - @Test - void testInnerJoinWithPKFKAndMultiNodeResidualFilter() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select * from T1", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - addConstraint("alter table T2 add constraint pk primary key (id)"); - addConstraint("alter table T1 add constraint fk foreign key (id) references T2(id)"); - CascadesContext c2 = createCascadesContext( - "select * from T1 inner join T2 " - + "on T1.id = T2.id where T1.score > T2.score", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertTrue(res.isInvalid()); - dropConstraint("alter table T2 drop constraint pk"); - } - - @Disabled - @Test - void testLOJWithPKFKAndUK1() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select * from T1", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - addConstraint("alter table T2 add constraint pk primary key (id)"); - addConstraint("alter table T1 add constraint fk foreign key (id) references T2(id)"); - addConstraint("alter table T3 add constraint uk unique (id)"); - CascadesContext c2 = createCascadesContext( - "select * from (select T1.*, T3.id as id3 from T1 left outer join T3 on T1.id = T3.id) T1 inner join T2 " - + "on T1.id = T2.id ", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertTrue(!res.isInvalid()); - Assertions.assertTrue(res.getViewExpressions().isEmpty()); - dropConstraint("alter table T2 drop constraint pk"); - dropConstraint("alter table T3 drop constraint uk"); - } - - @Test - void testLOJWithPKFKAndUK2() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select * from T1", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - addConstraint("alter table T2 add constraint pk primary key (id)"); - addConstraint("alter table T1 add constraint fk foreign key (id) references T2(id)"); - addConstraint("alter table T3 add constraint uk unique (id)"); - CascadesContext c2 = createCascadesContext( - "select * from (select T1.*, T2.id as id2 from T1 inner join T2 on T1.id = T2.id) T1 left outer join T3 " - + "on T1.id = T3.id ", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertTrue(!res.isInvalid()); - Assertions.assertTrue(res.getViewExpressions().isEmpty()); - dropConstraint("alter table T2 drop constraint pk"); - dropConstraint("alter table T3 drop constraint uk"); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java deleted file mode 100644 index af36afe715112d..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java +++ /dev/null @@ -1,98 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.exploration.mv; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; -import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; -import org.apache.doris.nereids.rules.RuleSet; -import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; -import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; -import org.apache.doris.nereids.sqltest.SqlTestBase; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.util.PlanChecker; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import java.util.Objects; - -class HyperGraphAggTest extends SqlTestBase { - @Test - void testJoinWithAgg() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - CascadesContext c2 = createCascadesContext( - "select * from T1 inner join" - + "(" - + "select id from T2 group by id" - + ") T2 " - + "on T1.id = T2.id ", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p2).build(); - Assertions.assertEquals("id", Objects.requireNonNull(((StructInfoNode) h1.getNode(1)).getExpressions()).get(0).toSql()); - } - - @Disabled - @Test - void testIJWithAgg() { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES"); - CascadesContext c1 = createCascadesContext( - "select * from T1 inner join T2 " - + "on T1.id = T2.id", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - CascadesContext c2 = createCascadesContext( - "select * from T1 inner join" - + "(" - + "select id from T2 group by id" - + ") T2 " - + "on T1.id = T2.id ", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); - Assertions.assertTrue(!res.isInvalid()); - Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); - } - - LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { - StructInfo st1 = StructInfo.of(p1, p1, null); - StructInfo st2 = StructInfo.of(p2, p2, null); - RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations(), 8) - .get(0); - SlotMapping sm = SlotMapping.generate(rm); - return LogicalCompatibilityContext.from(rm, sm, st1, st2); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java deleted file mode 100644 index db02e45dd48ffc..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java +++ /dev/null @@ -1,185 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.exploration.mv; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; -import org.apache.doris.nereids.rules.RuleSet; -import org.apache.doris.nereids.sqltest.SqlTestBase; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.util.PlanChecker; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -class HyperGraphComparatorTest extends SqlTestBase { - - @Override - protected String getDisableNereidsRules() { - return "INFER_PREDICATES,CONSTANT_PROPAGATION,PRUNE_EMPTY_PARTITION"; - } - - @Test - void testInnerJoinAndLOJ() { - CascadesContext c1 = createCascadesContext( - "select * from T1 inner join T2 " - + "on T1.id = T2.id " - + "inner join T3 on T1.id = T3.id", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - CascadesContext c2 = createCascadesContext( - "select * from T1 left outer join T2 " - + "on T1.id = T2.id " - + "left outer join T3 on T1.id = T3.id", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertFalse(res.isInvalid()); - Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); - } - - @Test - void testIJAndLojAssoc() { - CascadesContext c1 = createCascadesContext( - "select * from T1 inner join T3 " - + "on T1.id = T3.id " - + "inner join T2 on T1.id = T2.id", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - CascadesContext c2 = createCascadesContext( - "select * from T1 left outer join T2 " - + "on T1.id = T2.id " - + "left outer join T3 on T1.id = T3.id", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertFalse(res.isInvalid()); - Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); - } - - @Test - void testIJAndLojAssocWithFilter() { - CascadesContext c1 = createCascadesContext( - "select * from T1 inner join T3 " - + "on T1.id = T3.id " - + "inner join T2 on T1.id = T2.id", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - CascadesContext c2 = createCascadesContext( - "select * from T1 left outer join " - + "(select * from T2 where T2.id = 1) T2 " - + "on T1.id = T2.id " - + "left outer join T3 on T1.id = T3.id", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertFalse(res.isInvalid()); - Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); - } - - @Test - void testIJAndLojAssocWithJoinCond() { - CascadesContext c1 = createCascadesContext( - "select * from T1 inner join T3 " - + "on T1.id = T3.id " - + "inner join T2 on T1.id = T2.id", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - CascadesContext c2 = createCascadesContext( - "select * from T1 left join T2 " - + "on T1.id = T2.id " - + "left join T3 on T1.id = T3.id", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertFalse(res.isInvalid()); - Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); - } - - @Test - void testJoinEliminateShouldFail() { - CascadesContext c1 = createCascadesContext( - "select * from T1 inner join T2 " - + "on T1.id = T2.id", - connectContext - ); - Plan p1 = PlanChecker.from(c1) - .analyze() - .rewrite() - .getPlan().child(0); - CascadesContext c2 = createCascadesContext( - "select * from T1 inner join T2 " - + "on T1.id = T2.id " - + "inner join T3 on T1.id = T3.id", - connectContext - ); - Plan p2 = PlanChecker.from(c2) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).build(); - HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); - Assertions.assertTrue(res.isInvalid()); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MvExplorationSuiteTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MvExplorationSuiteTest.java new file mode 100644 index 00000000000000..c3ce56772a4d3c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MvExplorationSuiteTest.java @@ -0,0 +1,951 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleSet; +import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; +import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.sqltest.SqlTestBase; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.Not; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.util.PlanChecker; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Objects; + +/** + * Merged suite: these tests only need the shared fixture provided by the base class, so they are + * kept in one class on purpose. Every extra test class pays a full FE startup, which dominates the + * runtime of tests this small. + * + *

Replaces the former standalone classes: + *

    + *
  • BuildStructInfoTest
    • + *
  • EliminateJoinTest
    • + *
  • HyperGraphAggTest
    • + *
  • HyperGraphComparatorTest
    • + *
  • NullRejectInferenceTest
    • + *
+ */ +public class MvExplorationSuiteTest extends SqlTestBase { + + private static final TestMaterializedViewRule TEST_RULE = new TestMaterializedViewRule(); + + // ------------------------------------------------------------------------- + // from BuildStructInfoTest + // ------------------------------------------------------------------------- + + @Test + void testSimpleSQL() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "select * from T1, T2, T3, T4 " + + "where " + + "T1.id = T2.id and " + + "T2.score = T3.score and " + + "T3.id = T4.id"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .deriveStats() + .matches(logicalJoin() + .when(j -> { + HyperGraph.builderForMv(j); + return true; + })); + + } + + @Test + void testStructInfoNode() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "select * from T1 inner join " + + "(select sum(id) as id from T2 where id = 1) T2 " + + "on T1.id = T2.id"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .deriveStats() + .matches(logicalJoin() + .when(j -> { + HyperGraph hyperGraph = HyperGraph.builderForMv(j).build(); + Assertions.assertTrue(hyperGraph.getNodes().stream() + .allMatch(n -> n.getPlan() + .collectToList(GroupPlan.class::isInstance).isEmpty())); + return true; + })); + + } + + @Test + void testFilter() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "select * from T1 left outer join " + + " (select id from T2 where id = 1) T2 " + + "on T1.id = T2.id "; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches(logicalJoin() + .when(j -> { + HyperGraph structInfo = HyperGraph.builderForMv(j).build(); + Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin()); + Assertions.assertEquals(0, structInfo.getFilterEdge(0).getLeftRejectEdge().size()); + Assertions.assertEquals(1, structInfo.getFilterEdge(0).getRightRejectEdge().size()); + return true; + })); + + sql = "select * from (select id from T1 where id = 0) T1 left outer join T2 " + + "on T1.id = T2.id "; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches(logicalJoin() + .when(j -> { + HyperGraph structInfo = HyperGraph.builderForMv(j).build(); + Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin()); + return true; + })); + } + + @Test + void testPlanCheckerWithJoin() { + // Should not make scan to empty relation when the table used by materialized view has no data + connectContext.getSessionVariable().setDisableNereidsRules( + "OLAP_SCAN_PARTITION_PRUNE" + + ",PRUNE_EMPTY_PARTITION" + + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" + + ",ELIMINATE_CONST_JOIN_CONDITION" + + ",CONSTANT_PROPAGATION" + ); + PlanChecker.from(connectContext) + .checkExplain("select * from " + + "(select * from lineitem " + + "where l_shipdate >= \"2023-12-01\" and l_shipdate <= \"2023-12-03\") t1 " + + "left join " + + "(select * from orders where o_orderdate >= \"2023-12-01\" and o_orderdate <= \"2023-12-03\" ) t2 " + + "on t1.l_orderkey = o_orderkey;", + nereidsPlanner -> { + Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); + PlanCheckContext checkContext = PlanCheckContext.of( + AbstractMaterializedViewRule.SUPPORTED_JOIN_TYPE_SET); + Boolean result = rewrittenPlan.child(0).accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext); + Assertions.assertTrue(result); + Assertions.assertFalse(checkContext.isContainsTopAggregate()); + }); + } + + @Test + void testPlanCheckerWithAggregate() { + // Should not make scan to empty relation when the table used by materialized view has no data + connectContext.getSessionVariable().setDisableNereidsRules( + "OLAP_SCAN_PARTITION_PRUNE" + + ",PRUNE_EMPTY_PARTITION" + + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" + + ",ELIMINATE_CONST_JOIN_CONDITION" + + ",CONSTANT_PROPAGATION" + ); + PlanChecker.from(connectContext) + .checkExplain("SELECT l.L_SHIPDATE AS ship_data_alias, o.O_ORDERDATE, count(*) " + + "FROM " + + "lineitem as l " + + "LEFT JOIN " + + "(SELECT abs(O_TOTALPRICE + 10) as c1_abs, O_CUSTKEY, O_ORDERDATE, O_ORDERKEY " + + "FROM orders) as o " + + "ON l.L_ORDERKEY = o.O_ORDERKEY " + + "JOIN " + + "(SELECT abs(sqrt(PS_SUPPLYCOST)) as c2_abs, PS_AVAILQTY, PS_PARTKEY, PS_SUPPKEY " + + "FROM partsupp) as ps " + + "ON l.L_PARTKEY = ps.PS_PARTKEY and l.L_SUPPKEY = ps.PS_SUPPKEY " + + "GROUP BY l.L_SHIPDATE, o.O_ORDERDATE ", + nereidsPlanner -> { + Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); + PlanCheckContext checkContext = PlanCheckContext.of( + AbstractMaterializedViewRule.SUPPORTED_JOIN_TYPE_SET); + Boolean result = rewrittenPlan.child(0).accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext); + Assertions.assertTrue(result); + Assertions.assertTrue(checkContext.isContainsTopAggregate()); + }); + } + + @Test + void testPlanCheckerScanAggregate() { + // Should not make scan to empty relation when the table used by materialized view has no data + connectContext.getSessionVariable().setDisableNereidsRules( + "OLAP_SCAN_PARTITION_PRUNE" + + ",PRUNE_EMPTY_PARTITION" + + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" + + ",ELIMINATE_CONST_JOIN_CONDITION" + + ",CONSTANT_PROPAGATION" + ); + PlanChecker.from(connectContext) + .checkExplain("select l.L_SHIPDATE, count(*) from lineitem l " + + "GROUP BY l.L_SHIPDATE", + nereidsPlanner -> { + Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); + PlanCheckContext checkContext = PlanCheckContext.of( + AbstractMaterializedViewRule.SUPPORTED_JOIN_TYPE_SET); + Boolean result = rewrittenPlan.child(0).accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext); + Assertions.assertFalse(result); + }); + } + + @Test + void testPlanCheckerOnlyScan() { + // Should not make scan to empty relation when the table used by materialized view has no data + connectContext.getSessionVariable().setDisableNereidsRules( + "OLAP_SCAN_PARTITION_PRUNE" + + ",PRUNE_EMPTY_PARTITION" + + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" + + ",ELIMINATE_CONST_JOIN_CONDITION" + + ",CONSTANT_PROPAGATION" + ); + PlanChecker.from(connectContext) + .checkExplain("select l.L_SHIPDATE from lineitem l ", + nereidsPlanner -> { + Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); + PlanCheckContext checkContext = PlanCheckContext.of( + AbstractMaterializedViewRule.SUPPORTED_JOIN_TYPE_SET); + Boolean result = rewrittenPlan.child(0).accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext); + Assertions.assertTrue(result); + Assertions.assertFalse(checkContext.isContainsTopAggregate()); + }); + } + + // ------------------------------------------------------------------------- + // from EliminateJoinTest + // ------------------------------------------------------------------------- + + @Test + void testLOJWithGroupBy() { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext c1 = createCascadesContext( + "select * from T1", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 left outer join (select id from T2 group by id) T2 " + + "on T1.id = T2.id ", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + CascadesContext c3 = createCascadesContext( + "select * from T1 left outer join (select id as id2 from T2 group by id) T2 " + + "on T1.id = T2.id2 ", + connectContext + ); + Plan p3 = PlanChecker.from(c3) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + HyperGraph h3 = HyperGraph.builderForMv(p3).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertTrue(!res.isInvalid()); + Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3, + constructContext(p1, p2, c1)).isInvalid()); + Assertions.assertTrue(res.getViewExpressions().isEmpty()); + } + + @Test + void testLOJWithUK() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext c1 = createCascadesContext( + "select * from T1", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + addConstraint("alter table T2 add constraint uk unique (id)"); + CascadesContext c2 = createCascadesContext( + "select * from T1 left outer join T2 " + + "on T1.id = T2.id ", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertTrue(!res.isInvalid()); + Assertions.assertTrue(res.getViewExpressions().isEmpty()); + dropConstraint("alter table T2 drop constraint uk"); + } + + @Test + void testLOJWithUKAndOtherJoinConjuncts() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext c1 = createCascadesContext( + "select * from T1", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + addConstraint("alter table T2 add constraint uk_other_join_conjunct unique (id)"); + try { + CascadesContext c2 = createCascadesContext( + "select * from T1 left outer join T2 " + + "on T1.id = T2.id and T2.id = 1", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertFalse(res.isInvalid()); + Assertions.assertTrue(res.getViewExpressions().isEmpty()); + } finally { + dropConstraint("alter table T2 drop constraint uk_other_join_conjunct"); + } + } + + @Test + void testLOJWithUKAndFilterOnEliminatedNode() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext c1 = createCascadesContext( + "select T1.id from T1", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + addConstraint("alter table T2 add constraint uk_loj_filter unique (id)"); + try { + CascadesContext c2 = createCascadesContext( + "select T1.id from T1 left outer join T2 " + + "on T1.id = T2.id where T2.id is null", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertTrue(res.isInvalid()); + } finally { + dropConstraint("alter table T2 drop constraint uk_loj_filter"); + } + } + + @Test + void testInnerJoinWithPKFKAndSlotFreeFilterOnEliminatedNode() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext c1 = createCascadesContext( + "select * from T1", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + addConstraint("alter table T2 add constraint pk_slot_free_filter primary key (id)"); + addConstraint("alter table T1 add constraint fk_slot_free_filter foreign key (id) references T2(id)"); + try { + CascadesContext c2 = createCascadesContext( + "select * from T1 inner join (select * from T2 where 1 = 0) T2 " + + "on T1.id = T2.id", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertTrue(res.isInvalid()); + } finally { + dropConstraint("alter table T1 drop constraint fk_slot_free_filter"); + dropConstraint("alter table T2 drop constraint pk_slot_free_filter"); + } + } + + @Test + void testLOJWithPKFK() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext c1 = createCascadesContext( + "select * from T1", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + addConstraint("alter table T2 add constraint pk primary key (id)"); + addConstraint("alter table T1 add constraint fk foreign key (id) references T2(id)"); + CascadesContext c2 = createCascadesContext( + "select * from T1 inner join T2 " + + "on T1.id = T2.id ", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + CascadesContext c3 = createCascadesContext( + "select * from T1 inner join (select id as id2 from T2) T2 " + + "on T1.id = T2.id2 ", + connectContext + ); + Plan p3 = PlanChecker.from(c3) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + HyperGraph h3 = HyperGraph.builderForMv(p3).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertTrue(!res.isInvalid()); + Assertions.assertTrue(res.getViewExpressions().isEmpty()); + Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3, constructContext(p1, p2, c1)).isInvalid()); + dropConstraint("alter table T2 drop constraint pk"); + } + + @Test + void testInnerJoinWithPKFKAndMultiNodeResidualFilter() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext c1 = createCascadesContext( + "select * from T1", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + addConstraint("alter table T2 add constraint pk primary key (id)"); + addConstraint("alter table T1 add constraint fk foreign key (id) references T2(id)"); + CascadesContext c2 = createCascadesContext( + "select * from T1 inner join T2 " + + "on T1.id = T2.id where T1.score > T2.score", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertTrue(res.isInvalid()); + dropConstraint("alter table T2 drop constraint pk"); + } + + @Disabled + @Test + void testLOJWithPKFKAndUK1() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext c1 = createCascadesContext( + "select * from T1", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + addConstraint("alter table T2 add constraint pk primary key (id)"); + addConstraint("alter table T1 add constraint fk foreign key (id) references T2(id)"); + addConstraint("alter table T3 add constraint uk unique (id)"); + CascadesContext c2 = createCascadesContext( + "select * from (select T1.*, T3.id as id3 from T1 left outer join T3 on T1.id = T3.id) T1 inner join T2 " + + "on T1.id = T2.id ", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertTrue(!res.isInvalid()); + Assertions.assertTrue(res.getViewExpressions().isEmpty()); + dropConstraint("alter table T2 drop constraint pk"); + dropConstraint("alter table T3 drop constraint uk"); + } + + @Test + void testLOJWithPKFKAndUK2() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext c1 = createCascadesContext( + "select * from T1", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + addConstraint("alter table T2 add constraint pk primary key (id)"); + addConstraint("alter table T1 add constraint fk foreign key (id) references T2(id)"); + addConstraint("alter table T3 add constraint uk unique (id)"); + CascadesContext c2 = createCascadesContext( + "select * from (select T1.*, T2.id as id2 from T1 inner join T2 on T1.id = T2.id) T1 left outer join T3 " + + "on T1.id = T3.id ", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertTrue(!res.isInvalid()); + Assertions.assertTrue(res.getViewExpressions().isEmpty()); + dropConstraint("alter table T2 drop constraint pk"); + dropConstraint("alter table T3 drop constraint uk"); + } + + // ------------------------------------------------------------------------- + // from HyperGraphAggTest + // ------------------------------------------------------------------------- + + @Test + void testJoinWithAgg() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + CascadesContext c2 = createCascadesContext( + "select * from T1 inner join" + + "(" + + "select id from T2 group by id" + + ") T2 " + + "on T1.id = T2.id ", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p2).build(); + Assertions.assertEquals("id", Objects.requireNonNull(((StructInfoNode) h1.getNode(1)).getExpressions()).get(0).toSql()); + } + + @Disabled + @Test + void testIJWithAgg() { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES"); + CascadesContext c1 = createCascadesContext( + "select * from T1 inner join T2 " + + "on T1.id = T2.id", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 inner join" + + "(" + + "select id from T2 group by id" + + ") T2 " + + "on T1.id = T2.id ", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + Assertions.assertTrue(!res.isInvalid()); + Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); + } + + LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { + StructInfo st1 = StructInfo.of(p1, p1, null); + StructInfo st2 = StructInfo.of(p2, p2, null); + RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations(), 8) + .get(0); + SlotMapping sm = SlotMapping.generate(rm); + return LogicalCompatibilityContext.from(rm, sm, st1, st2); + } + + // ------------------------------------------------------------------------- + // from HyperGraphComparatorTest + // ------------------------------------------------------------------------- + + @Override + protected String getDisableNereidsRules() { + return "INFER_PREDICATES,CONSTANT_PROPAGATION,PRUNE_EMPTY_PARTITION"; + } + + @Test + void testInnerJoinAndLOJ() { + CascadesContext c1 = createCascadesContext( + "select * from T1 inner join T2 " + + "on T1.id = T2.id " + + "inner join T3 on T1.id = T3.id", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 left outer join T2 " + + "on T1.id = T2.id " + + "left outer join T3 on T1.id = T3.id", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertFalse(res.isInvalid()); + Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); + } + + @Test + void testIJAndLojAssoc() { + CascadesContext c1 = createCascadesContext( + "select * from T1 inner join T3 " + + "on T1.id = T3.id " + + "inner join T2 on T1.id = T2.id", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 left outer join T2 " + + "on T1.id = T2.id " + + "left outer join T3 on T1.id = T3.id", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertFalse(res.isInvalid()); + Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); + } + + @Test + void testIJAndLojAssocWithFilter() { + CascadesContext c1 = createCascadesContext( + "select * from T1 inner join T3 " + + "on T1.id = T3.id " + + "inner join T2 on T1.id = T2.id", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 left outer join " + + "(select * from T2 where T2.id = 1) T2 " + + "on T1.id = T2.id " + + "left outer join T3 on T1.id = T3.id", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertFalse(res.isInvalid()); + Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); + } + + @Test + void testIJAndLojAssocWithJoinCond() { + CascadesContext c1 = createCascadesContext( + "select * from T1 inner join T3 " + + "on T1.id = T3.id " + + "inner join T2 on T1.id = T2.id", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 left join T2 " + + "on T1.id = T2.id " + + "left join T3 on T1.id = T3.id", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertFalse(res.isInvalid()); + Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); + } + + @Test + void testJoinEliminateShouldFail() { + CascadesContext c1 = createCascadesContext( + "select * from T1 inner join T2 " + + "on T1.id = T2.id", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 inner join T2 " + + "on T1.id = T2.id " + + "inner join T3 on T1.id = T3.id", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); + Assertions.assertTrue(res.isInvalid()); + } + + // ------------------------------------------------------------------------- + // from NullRejectInferenceTest + // ------------------------------------------------------------------------- + + @Test + void testTwoHopNullRejectFromInnerJoinConditions() { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext queryContext = createCascadesContext( + "select lineitem.l_orderkey, supplier.s_name, nation.n_name from lineitem " + + "inner join supplier on lineitem.l_suppkey = supplier.s_suppkey " + + "inner join nation on supplier.s_nationkey = nation.n_nationkey " + + "where nation.n_name = 'CHINA'", + connectContext + ); + Plan queryPlan = PlanChecker.from(queryContext) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + + CascadesContext viewContext = createCascadesContext( + "select lineitem.l_orderkey, supplier.s_name, nation.n_name from lineitem " + + "left outer join supplier on lineitem.l_suppkey = supplier.s_suppkey " + + "left outer join nation on supplier.s_nationkey = nation.n_nationkey", + connectContext + ); + Plan viewPlan = PlanChecker.from(viewContext) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + + StructInfo queryStructInfo = StructInfo.of(queryPlan, queryPlan, queryContext); + StructInfo viewStructInfo = StructInfo.of(viewPlan, viewPlan, viewContext); + RelationMapping relationMapping = RelationMapping.generate( + queryStructInfo.getRelations(), viewStructInfo.getRelations(), 8).get(0); + SlotMapping queryToView = SlotMapping.generate(relationMapping); + SlotMapping viewToQuery = queryToView.inverse(); + LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( + relationMapping, viewToQuery, queryStructInfo, viewStructInfo); + ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals( + queryStructInfo, viewStructInfo, compatibilityContext); + + Assertions.assertFalse(comparisonResult.isInvalid()); + Assertions.assertFalse(comparisonResult.getViewNoNullableSlot().isEmpty()); + + SplitPredicate compensatePredicates = TEST_RULE.predicatesCompensateForTest( + queryStructInfo, viewStructInfo, viewToQuery, comparisonResult, queryContext); + Assertions.assertFalse(compensatePredicates.isInvalid()); + Assertions.assertTrue(compensatePredicates.toList().stream() + .anyMatch(expression -> isNotNullOnSlot(expression, "s_name"))); + } + + @Test + void testNullRejectCompensationForInnerJoinFullJoinRewrite() { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext queryContext = createCascadesContext( + "select lineitem.l_shipdate, orders.o_orderdate from lineitem " + + "inner join orders on lineitem.l_orderkey = orders.o_orderkey " + + "where orders.o_orderdate = '2023-10-17'", + connectContext + ); + Plan queryPlan = PlanChecker.from(queryContext) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + + CascadesContext viewContext = createCascadesContext( + "select lineitem.l_shipdate, orders.o_orderdate from lineitem " + + "full outer join orders on lineitem.l_orderkey = orders.o_orderkey", + connectContext + ); + Plan viewPlan = PlanChecker.from(viewContext) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + + StructInfo queryStructInfo = StructInfo.of(queryPlan, queryPlan, queryContext); + StructInfo viewStructInfo = StructInfo.of(viewPlan, viewPlan, viewContext); + RelationMapping relationMapping = RelationMapping.generate( + queryStructInfo.getRelations(), viewStructInfo.getRelations(), 8).get(0); + SlotMapping queryToView = SlotMapping.generate(relationMapping); + SlotMapping viewToQuery = queryToView.inverse(); + LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( + relationMapping, viewToQuery, queryStructInfo, viewStructInfo); + ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals( + queryStructInfo, viewStructInfo, compatibilityContext); + + Assertions.assertFalse(comparisonResult.isInvalid()); + Assertions.assertFalse(comparisonResult.getViewNoNullableSlot().isEmpty()); + + SplitPredicate compensatePredicates = TEST_RULE.predicatesCompensateForTest( + queryStructInfo, viewStructInfo, viewToQuery, comparisonResult, queryContext); + Assertions.assertFalse(compensatePredicates.isInvalid()); + Assertions.assertTrue(compensatePredicates.toList().stream() + .anyMatch(expression -> isNotNullOnSlot(expression, "l_shipdate"))); + } + + @Test + void testNullRejectCompensationForInnerJoinFullJoinRewriteOnRightSide() { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); + CascadesContext queryContext = createCascadesContext( + "select lineitem.l_shipdate, orders.o_orderdate from lineitem " + + "inner join orders on lineitem.l_orderkey = orders.o_orderkey " + + "where lineitem.l_shipdate = '2023-10-17'", + connectContext + ); + Plan queryPlan = PlanChecker.from(queryContext) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + + CascadesContext viewContext = createCascadesContext( + "select lineitem.l_shipdate, orders.o_orderdate from lineitem " + + "full outer join orders on lineitem.l_orderkey = orders.o_orderkey", + connectContext + ); + Plan viewPlan = PlanChecker.from(viewContext) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + + StructInfo queryStructInfo = StructInfo.of(queryPlan, queryPlan, queryContext); + StructInfo viewStructInfo = StructInfo.of(viewPlan, viewPlan, viewContext); + RelationMapping relationMapping = RelationMapping.generate( + queryStructInfo.getRelations(), viewStructInfo.getRelations(), 8).get(0); + SlotMapping queryToView = SlotMapping.generate(relationMapping); + SlotMapping viewToQuery = queryToView.inverse(); + LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( + relationMapping, viewToQuery, queryStructInfo, viewStructInfo); + ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals( + queryStructInfo, viewStructInfo, compatibilityContext); + + Assertions.assertFalse(comparisonResult.isInvalid()); + Assertions.assertFalse(comparisonResult.getViewNoNullableSlot().isEmpty()); + + SplitPredicate compensatePredicates = TEST_RULE.predicatesCompensateForTest( + queryStructInfo, viewStructInfo, viewToQuery, comparisonResult, queryContext); + Assertions.assertFalse(compensatePredicates.isInvalid()); + Assertions.assertTrue(compensatePredicates.toList().stream() + .anyMatch(expression -> isNotNullOnSlot(expression, "o_orderdate"))); + } + + private static boolean isNotNullOnSlot(Expression expression, String slotName) { + if (!(expression instanceof Not) || ((Not) expression).isGeneratedIsNotNull() + || !(((Not) expression).child() instanceof IsNull)) { + return false; + } + Expression slot = ((IsNull) ((Not) expression).child()).child(); + return slot instanceof SlotReference && slotName.equals(((SlotReference) slot).getName()); + } + + private static class TestMaterializedViewRule extends AbstractMaterializedViewRule { + @Override + public List buildRules() { + return ImmutableList.of(); + } + + private SplitPredicate predicatesCompensateForTest(StructInfo queryStructInfo, + StructInfo viewStructInfo, SlotMapping viewToQuerySlotMapping, + ComparisonResult comparisonResult, CascadesContext cascadesContext) { + return predicatesCompensate(queryStructInfo, viewStructInfo, viewToQuerySlotMapping, + comparisonResult, cascadesContext); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/NullRejectInferenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/NullRejectInferenceTest.java deleted file mode 100644 index 5fd7628096b5b5..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/NullRejectInferenceTest.java +++ /dev/null @@ -1,209 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.exploration.mv; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleSet; -import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; -import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; -import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; -import org.apache.doris.nereids.sqltest.SqlTestBase; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.IsNull; -import org.apache.doris.nereids.trees.expressions.Not; -import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.util.PlanChecker; - -import com.google.common.collect.ImmutableList; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.List; - -class NullRejectInferenceTest extends SqlTestBase { - - private static final TestMaterializedViewRule TEST_RULE = new TestMaterializedViewRule(); - - @Test - void testTwoHopNullRejectFromInnerJoinConditions() { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext queryContext = createCascadesContext( - "select lineitem.l_orderkey, supplier.s_name, nation.n_name from lineitem " - + "inner join supplier on lineitem.l_suppkey = supplier.s_suppkey " - + "inner join nation on supplier.s_nationkey = nation.n_nationkey " - + "where nation.n_name = 'CHINA'", - connectContext - ); - Plan queryPlan = PlanChecker.from(queryContext) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - - CascadesContext viewContext = createCascadesContext( - "select lineitem.l_orderkey, supplier.s_name, nation.n_name from lineitem " - + "left outer join supplier on lineitem.l_suppkey = supplier.s_suppkey " - + "left outer join nation on supplier.s_nationkey = nation.n_nationkey", - connectContext - ); - Plan viewPlan = PlanChecker.from(viewContext) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - - StructInfo queryStructInfo = StructInfo.of(queryPlan, queryPlan, queryContext); - StructInfo viewStructInfo = StructInfo.of(viewPlan, viewPlan, viewContext); - RelationMapping relationMapping = RelationMapping.generate( - queryStructInfo.getRelations(), viewStructInfo.getRelations(), 8).get(0); - SlotMapping queryToView = SlotMapping.generate(relationMapping); - SlotMapping viewToQuery = queryToView.inverse(); - LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( - relationMapping, viewToQuery, queryStructInfo, viewStructInfo); - ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals( - queryStructInfo, viewStructInfo, compatibilityContext); - - Assertions.assertFalse(comparisonResult.isInvalid()); - Assertions.assertFalse(comparisonResult.getViewNoNullableSlot().isEmpty()); - - SplitPredicate compensatePredicates = TEST_RULE.predicatesCompensateForTest( - queryStructInfo, viewStructInfo, viewToQuery, comparisonResult, queryContext); - Assertions.assertFalse(compensatePredicates.isInvalid()); - Assertions.assertTrue(compensatePredicates.toList().stream() - .anyMatch(expression -> isNotNullOnSlot(expression, "s_name"))); - } - - @Test - void testNullRejectCompensationForInnerJoinFullJoinRewrite() { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext queryContext = createCascadesContext( - "select lineitem.l_shipdate, orders.o_orderdate from lineitem " - + "inner join orders on lineitem.l_orderkey = orders.o_orderkey " - + "where orders.o_orderdate = '2023-10-17'", - connectContext - ); - Plan queryPlan = PlanChecker.from(queryContext) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - - CascadesContext viewContext = createCascadesContext( - "select lineitem.l_shipdate, orders.o_orderdate from lineitem " - + "full outer join orders on lineitem.l_orderkey = orders.o_orderkey", - connectContext - ); - Plan viewPlan = PlanChecker.from(viewContext) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - - StructInfo queryStructInfo = StructInfo.of(queryPlan, queryPlan, queryContext); - StructInfo viewStructInfo = StructInfo.of(viewPlan, viewPlan, viewContext); - RelationMapping relationMapping = RelationMapping.generate( - queryStructInfo.getRelations(), viewStructInfo.getRelations(), 8).get(0); - SlotMapping queryToView = SlotMapping.generate(relationMapping); - SlotMapping viewToQuery = queryToView.inverse(); - LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( - relationMapping, viewToQuery, queryStructInfo, viewStructInfo); - ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals( - queryStructInfo, viewStructInfo, compatibilityContext); - - Assertions.assertFalse(comparisonResult.isInvalid()); - Assertions.assertFalse(comparisonResult.getViewNoNullableSlot().isEmpty()); - - SplitPredicate compensatePredicates = TEST_RULE.predicatesCompensateForTest( - queryStructInfo, viewStructInfo, viewToQuery, comparisonResult, queryContext); - Assertions.assertFalse(compensatePredicates.isInvalid()); - Assertions.assertTrue(compensatePredicates.toList().stream() - .anyMatch(expression -> isNotNullOnSlot(expression, "l_shipdate"))); - } - - @Test - void testNullRejectCompensationForInnerJoinFullJoinRewriteOnRightSide() { - connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION"); - CascadesContext queryContext = createCascadesContext( - "select lineitem.l_shipdate, orders.o_orderdate from lineitem " - + "inner join orders on lineitem.l_orderkey = orders.o_orderkey " - + "where lineitem.l_shipdate = '2023-10-17'", - connectContext - ); - Plan queryPlan = PlanChecker.from(queryContext) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - - CascadesContext viewContext = createCascadesContext( - "select lineitem.l_shipdate, orders.o_orderdate from lineitem " - + "full outer join orders on lineitem.l_orderkey = orders.o_orderkey", - connectContext - ); - Plan viewPlan = PlanChecker.from(viewContext) - .analyze() - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .getAllPlan().get(0).child(0); - - StructInfo queryStructInfo = StructInfo.of(queryPlan, queryPlan, queryContext); - StructInfo viewStructInfo = StructInfo.of(viewPlan, viewPlan, viewContext); - RelationMapping relationMapping = RelationMapping.generate( - queryStructInfo.getRelations(), viewStructInfo.getRelations(), 8).get(0); - SlotMapping queryToView = SlotMapping.generate(relationMapping); - SlotMapping viewToQuery = queryToView.inverse(); - LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( - relationMapping, viewToQuery, queryStructInfo, viewStructInfo); - ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals( - queryStructInfo, viewStructInfo, compatibilityContext); - - Assertions.assertFalse(comparisonResult.isInvalid()); - Assertions.assertFalse(comparisonResult.getViewNoNullableSlot().isEmpty()); - - SplitPredicate compensatePredicates = TEST_RULE.predicatesCompensateForTest( - queryStructInfo, viewStructInfo, viewToQuery, comparisonResult, queryContext); - Assertions.assertFalse(compensatePredicates.isInvalid()); - Assertions.assertTrue(compensatePredicates.toList().stream() - .anyMatch(expression -> isNotNullOnSlot(expression, "o_orderdate"))); - } - - private static boolean isNotNullOnSlot(Expression expression, String slotName) { - if (!(expression instanceof Not) || ((Not) expression).isGeneratedIsNotNull() - || !(((Not) expression).child() instanceof IsNull)) { - return false; - } - Expression slot = ((IsNull) ((Not) expression).child()).child(); - return slot instanceof SlotReference && slotName.equals(((SlotReference) slot).getName()); - } - - private static class TestMaterializedViewRule extends AbstractMaterializedViewRule { - @Override - public List buildRules() { - return ImmutableList.of(); - } - - private SplitPredicate predicatesCompensateForTest(StructInfo queryStructInfo, - StructInfo viewStructInfo, SlotMapping viewToQuerySlotMapping, - ComparisonResult comparisonResult, CascadesContext cascadesContext) { - return predicatesCompensate(queryStructInfo, viewStructInfo, viewToQuerySlotMapping, - comparisonResult, cascadesContext); - } - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateConstHashJoinConditionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateConstHashJoinConditionTest.java deleted file mode 100644 index 977c216c4e7aab..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateConstHashJoinConditionTest.java +++ /dev/null @@ -1,74 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.sqltest.SqlTestBase; -import org.apache.doris.nereids.util.PlanChecker; - -import org.junit.jupiter.api.Test; - -public class EliminateConstHashJoinConditionTest extends SqlTestBase { - - @Test - void testEliminate() { - CascadesContext c1 = createCascadesContext( - "select * from T1 join T2 on T1.id = T2.id and T1.id = 1", - connectContext - ); - PlanChecker.from(c1) - .analyze() - .rewrite() - .matches( - logicalJoin().when(join -> - join.getHashJoinConjuncts().isEmpty() && join.getOtherJoinConjuncts().isEmpty()) - ); - } - - @Test - void testNotEliminateNonInnerJoin() { - CascadesContext c1 = createCascadesContext( - "select * from T1 left join T2 on T1.id = T2.id where T1.id = 1", - connectContext - ); - PlanChecker.from(c1) - .analyze() - .rewrite() - .matches( - logicalJoin().when(join -> - !join.getHashJoinConjuncts().isEmpty()) - ); - } - - @Test - void testNotEliminateAsofJoin() { - CascadesContext c1 = createCascadesContext( - "select * from T1 asof inner join T2 match_condition(cast(T1.score as datetime) " - + "> cast(T2.score as datetime)) on T1.id = T2.id where T1.id = 1", - connectContext - ); - PlanChecker.from(c1) - .analyze() - .rewrite() - .matches( - logicalJoin().when(join -> - !join.getHashJoinConjuncts().isEmpty()) - ); - } - -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteRuleSuiteTest.java similarity index 75% rename from fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java rename to fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteRuleSuiteTest.java index 6bcb1486a69306..1a87a28fee6cd5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteRuleSuiteTest.java @@ -28,6 +28,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.sqltest.SqlTestBase; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; @@ -38,7 +39,6 @@ import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; -import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.planner.PartitionColumnFilter; @@ -53,7 +53,23 @@ import java.util.List; import java.util.Objects; -class PruneOlapScanTabletTest extends SqlTestBase implements MemoPatternMatchSupported { +/** + * Rewrite-rule tests that only need the shared {@link SqlTestBase} fixture (database "test" with + * tables T0..T4). They are kept in one class on purpose: every extra test class pays a full FE + * startup, which dominates the runtime of tests this small. + * + *

Replaces the former standalone classes: + *

    + *
  • PruneOlapScanTabletTest
    • + *
  • EliminateConstHashJoinConditionTest
    • + *
+ * Add a new section below when moving another shared-fixture rewrite test here. + */ +public class RewriteRuleSuiteTest extends SqlTestBase { + + // ------------------------------------------------------------------------- + // from PruneOlapScanTabletTest + // ------------------------------------------------------------------------- @Test void testPruneOlapScanTablet() { @@ -159,4 +175,54 @@ void testPruneOlapScanTabletWithManually() { ) ); } + + // ------------------------------------------------------------------------- + // from EliminateConstHashJoinConditionTest + // ------------------------------------------------------------------------- + + @Test + void testEliminate() { + CascadesContext c1 = createCascadesContext( + "select * from T1 join T2 on T1.id = T2.id and T1.id = 1", + connectContext + ); + PlanChecker.from(c1) + .analyze() + .rewrite() + .matches( + logicalJoin().when(join -> + join.getHashJoinConjuncts().isEmpty() && join.getOtherJoinConjuncts().isEmpty()) + ); + } + + @Test + void testNotEliminateNonInnerJoin() { + CascadesContext c1 = createCascadesContext( + "select * from T1 left join T2 on T1.id = T2.id where T1.id = 1", + connectContext + ); + PlanChecker.from(c1) + .analyze() + .rewrite() + .matches( + logicalJoin().when(join -> + !join.getHashJoinConjuncts().isEmpty()) + ); + } + + @Test + void testNotEliminateAsofJoin() { + CascadesContext c1 = createCascadesContext( + "select * from T1 asof inner join T2 match_condition(cast(T1.score as datetime) " + + "> cast(T2.score as datetime)) on T1.id = T2.id where T1.id = 1", + connectContext + ); + PlanChecker.from(c1) + .analyze() + .rewrite() + .matches( + logicalJoin().when(join -> + !join.getHashJoinConjuncts().isEmpty()) + ); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/CascadesJoinReorderTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/CascadesJoinReorderTest.java deleted file mode 100644 index 00dce9f48a6b85..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/CascadesJoinReorderTest.java +++ /dev/null @@ -1,192 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.sqltest; - -import org.apache.doris.nereids.rules.RuleSet; -import org.apache.doris.nereids.util.PlanChecker; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -/** - * Paper: Measuring the Complexity of Join Enumeration in Query Optimization - * 
- * Star query without products
- * Join tree Number：
- *   left-deep: 2(n-1)! * 1
- *   zig-zag: (n-1)! * 2^(n-1)
- *   bushy: star graph can't be a bushy, it can only form a zig-zag (because the center must be joined first)
- *
- */ -class CascadesJoinReorderTest extends SqlTestBase { - @Test - void testStartThreeJoin() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - // Three join - // (n-1)! * 2^(n-1) = 8 - String sql = "SELECT * FROM T1 " - + "JOIN T2 ON T1.id = T2.id " - + "JOIN T3 ON T1.id = T3.id"; - - int plansNumber = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .applyExploration(RuleSet.ZIG_ZAG_TREE_JOIN_REORDER) - .applyExploration(RuleSet.ZIG_ZAG_TREE_JOIN_REORDER) - .applyExploration(RuleSet.ZIG_ZAG_TREE_JOIN_REORDER) - .applyExploration(RuleSet.ZIG_ZAG_TREE_JOIN_REORDER) - .plansNumber(); - - Assertions.assertEquals(8, plansNumber); - } - - @Test - void testStartThreeJoinBushy() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - // Three join - // (n-1)! * 2^(n-1) = 8 - String sql = "SELECT * FROM T1 " - + "JOIN T2 ON T1.id = T2.id " - + "JOIN T3 ON T1.id = T3.id"; - - int plansNumber = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .printlnAllTree() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .plansNumber(); - - Assertions.assertEquals(8, plansNumber); - } - - @Test - void testStarFourJoinZigzag() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - // Four join - // (n-1)! * 2^(n-1) = 48 - String sql = "SELECT * FROM T1 " - + "JOIN T2 ON T1.id = T2.id " - + "JOIN T3 ON T1.id = T3.id " - + "JOIN T4 ON T1.id = T4.id "; - - int plansNumber = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .plansNumber(); - - Assertions.assertEquals(48, plansNumber); - } - - @Test - void testStarFourJoinBushy() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - // Four join - // (n-1)! * 2^(n-1) = 48 - String sql = "SELECT * FROM T1 " - + "JOIN T2 ON T1.id = T2.id " - + "JOIN T3 ON T1.id = T3.id " - + "JOIN T4 ON T1.id = T4.id "; - - int plansNumber = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .plansNumber(); - - Assertions.assertEquals(48, plansNumber); - } - - @Test - void testChainFourJoinBushy() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - // Four join - // 2^(n-1) * C(n-1) = 40 - String sql = "SELECT * FROM T1 " - + "JOIN T2 ON T1.id = T2.id " - + "JOIN T3 ON T2.id = T3.id " - + "JOIN T4 ON T3.id = T4.id "; - - int plansNumber = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .plansNumber(); - - Assertions.assertEquals(40, plansNumber); - } - - @Test - void testChainFiveJoinBushy() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - // Five join - // 2^(n-1) * C(n-1) = 224 - String sql = "SELECT * FROM T1 " - + "JOIN T2 ON T1.id = T2.id " - + "JOIN T3 ON T2.id = T3.id " - + "JOIN T4 ON T3.id = T4.id " - + "JOIN T1 T5 ON T4.ID = T5.ID"; - - int plansNumber = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) - .plansNumber(); - - Assertions.assertEquals(224, plansNumber); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java deleted file mode 100644 index cdc36164ae9f95..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java +++ /dev/null @@ -1,122 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.sqltest; - -import org.apache.doris.nereids.util.ExpressionUtils; -import org.apache.doris.nereids.util.PlanChecker; - -import org.junit.jupiter.api.Test; - -import java.util.stream.Collectors; - -public class InferTest extends SqlTestBase { - @Test - void testInferNotNullAndInferPredicates() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - // Test InferNotNull, EliminateOuter, InferPredicate together - String sql = "select * from T1 left outer join T2 on T1.id = T2.id where T2.id = 4"; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .matches( - innerLogicalJoin( - logicalFilter().when(f -> f.getPredicate().toString().equals("(id#0 = 4)")), - logicalFilter().when(f -> f.getPredicate().toString().equals("(id#2 = 4)")) - ) - ); - } - - @Test - void testInferNotNullFromFilterAndEliminateOuter2() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql - = "select * from T1 right outer join T2 on T1.id = T2.id where T1.id = 4 OR (T1.id > 4 AND T2.score IS NULL)"; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .printlnTree() - .matches( - innerLogicalJoin( - logicalFilter().when( - f -> f.getPredicate().toString().equals("(id#2 >= 4)")), - logicalFilter().when( - f -> ExpressionUtils.and(f.getConjuncts().stream() - .sorted((a, b) -> a.toString().compareTo(b.toString())) - .collect(Collectors.toList())) - .toString().equals("(id#0 >= 4)")) - ) - - ); - } - - @Test - void testInferNotNullFromFilterAndEliminateOuter3() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql - = "select * from T1 full outer join T2 on T1.id = T2.id where T1.id = 4 OR (T1.id > 4 AND T2.score IS NULL)"; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .matches( - logicalFilter( - leftOuterLogicalJoin( - logicalFilter().when( - f -> f.getPredicate().toString().equals("(id#0 >= 4)")), - logicalFilter().when( - f -> f.getPredicate().toString().equals("(id#2 >= 4)") - ) - ) - ).when(f -> f.getPredicate().toString() - .equals("OR[(id#0 = 4),AND[(id#0 > 4),score#3 IS NULL]]")) - ); - } - - @Test - void testInferNotNullFromJoinAndEliminateOuter() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - // Is Not Null will infer from semi join, so right outer join can be eliminated. - String sql - = "select * from (select T1.id from T1 right outer join T2 on T1.id = T2.id) T1 left semi join T3 on T1.id = T3.id"; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .matches( - innerLogicalJoin( - logicalProject(), - logicalProject(leftSemiLogicalJoin()) - ) - ); - } - - @Test - void aggEliminateOuterJoin() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "select count(T2.score) from T1 left Join T2 on T1.id = T2.id"; - - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .matches( - logicalAggregate( - logicalProject( - innerLogicalJoin() - ) - ) - ); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/JoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/JoinTest.java deleted file mode 100644 index 899e35f2b63647..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/JoinTest.java +++ /dev/null @@ -1,106 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.sqltest; - -import org.apache.doris.nereids.properties.DistributionSpecHash; -import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType; -import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.rules.rewrite.ReorderJoin; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; -import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; -import org.apache.doris.nereids.util.PlanChecker; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class JoinTest extends SqlTestBase { - @Test - void testJoinUsing() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "SELECT * FROM T1 JOIN T2 using (id)"; - PlanChecker.from(connectContext) - .analyze(sql) - .applyBottomUp(new ReorderJoin()) - .matches( - innerLogicalJoin().when(j -> j.getHashJoinConjuncts().size() == 1) - ); - } - - @Test - void testColocatedJoin() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "select * from T2 join T2 b on T2.id = b.id and T2.id = b.id;"; - PhysicalPlan plan = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .optimize() - .getBestPlanTree(); - // generate colocate join plan without physicalDistribute - System.out.println(plan.treeString()); - Assertions.assertFalse(plan.anyMatch(p -> p instanceof PhysicalDistribute - && ((PhysicalDistribute) p).getDistributionSpec() instanceof DistributionSpecHash)); - sql = "select * from T1 join T0 on T1.score = T0.score and T1.id = T0.id;"; - plan = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .optimize() - .getBestPlanTree(); - // generate colocate join plan without physicalDistribute - Assertions.assertFalse(plan.anyMatch(p -> p instanceof PhysicalDistribute - && ((PhysicalDistribute) p).getDistributionSpec() instanceof DistributionSpecHash)); - } - - @Test - void testDedupConjuncts() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "select * from T1 join T2 on T1.id = T2.id and T1.id = T2.id;"; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .matches( - innerLogicalJoin().when(j -> j.getHashJoinConjuncts().size() == 1) - ); - - String sql1 = "select * from T1 left join T2 on T1.id = T2.id and T1.id = T2.id;"; - PlanChecker.from(connectContext) - .analyze(sql1) - .rewrite() - .matches( - leftOuterLogicalJoin().when(j -> j.getHashJoinConjuncts().size() == 1) - ); - } - - @Test - void testBucketJoinWithAgg() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "select * from " - + "(select distinct id as cnt from T2) T1 inner join" - + "(select distinct id as cnt from T2) T2 " - + "on T1.cnt = T2.cnt"; - PhysicalPlan plan = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .optimize() - .getBestPlanTree(PhysicalProperties.ANY); - Assertions.assertEquals( - ShuffleType.NATURAL, - ((DistributionSpecHash) ((PhysicalPlan) (plan.child(0).child(0))) - .getPhysicalProperties().getDistributionSpec()).getShuffleType() - ); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/MultiJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/MultiJoinTest.java deleted file mode 100644 index 26539fb356a1e2..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/MultiJoinTest.java +++ /dev/null @@ -1,131 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.sqltest; - -import org.apache.doris.nereids.rules.rewrite.ReorderJoin; -import org.apache.doris.nereids.util.PlanChecker; - -import com.google.common.collect.ImmutableList; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import java.util.List; - -class MultiJoinTest extends SqlTestBase { - @Test - void testMultiJoinEliminateCross() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - List sqls = ImmutableList.builder() - .add("SELECT * FROM T2 LEFT JOIN T3 ON T2.id = T3.id, T1 WHERE T1.id = T2.id") - .add("SELECT * FROM T2 LEFT JOIN T3 ON T2.id = T3.id, T1 WHERE T1.id = T2.id AND T1.score > 0") - .add("SELECT * FROM T2 LEFT JOIN T3 ON T2.id = T3.id, T1 WHERE T1.id = T2.id AND T1.score > 0 AND T1.id + T2.id + T3.id > 0") - .build(); - - for (String sql : sqls) { - PlanChecker.from(connectContext) - .analyze(sql) - .applyBottomUp(new ReorderJoin()) - .matches( - logicalJoin( - logicalJoin().whenNot(join -> join.getJoinType().isCrossJoin()), - leafPlan() - ).whenNot(join -> join.getJoinType().isCrossJoin()) - ) - .printlnTree(); - } - } - - @Test - @Disabled - void testEliminateBelowOuter() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - // FIXME: MultiJoin And EliminateOuter - String sql = "SELECT * FROM T1, T2 LEFT JOIN T3 ON T2.id = T3.id WHERE T1.id = T2.id"; - PlanChecker.from(connectContext) - .analyze(sql) - .applyBottomUp(new ReorderJoin()) - .printlnTree(); - } - - @Test - void testMultiJoinExistCross() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - List sqls = ImmutableList.builder() - .add("SELECT * FROM T2 LEFT SEMI JOIN T3 ON T2.id = T3.id, T1 WHERE T1.id > T2.id") - .build(); - - for (String sql : sqls) { - PlanChecker.from(connectContext) - .analyze(sql) - .applyBottomUp(new ReorderJoin()) - .matches( - logicalJoin( - logicalJoin().whenNot(join -> join.getJoinType().isCrossJoin()), - leafPlan() - ).when(join -> join.getJoinType().isCrossJoin()) - .whenNot(join -> join.getOtherJoinConjuncts().isEmpty()) - ) - .printlnTree(); - } - } - - @Test - void testOuterJoin() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "SELECT * FROM T1 LEFT OUTER JOIN T2 ON T1.id = T2.id, T3 WHERE T2.score > 0"; - PlanChecker.from(connectContext) - .analyze(sql) - .applyBottomUp(new ReorderJoin()) - .printlnTree() - .matches( - crossLogicalJoin( - leftOuterLogicalJoin() - .when(join -> join.getOtherJoinConjuncts().size() == 1), - logicalOlapScan() - ) - ); - } - - @Test - @Disabled - void testNoFilter() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "Select * FROM T1 INNER JOIN T2 On true"; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .matches( - crossLogicalJoin() - ); - } - - @Test - void test() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "select T1.score, T2.score from T1 inner join T2 on T1.id = T2.id where T1.score - 2 > T2.score"; - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .matches( - logicalProject( - innerLogicalJoin() - ) - ); - - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SortTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SortTest.java deleted file mode 100644 index f042fee4de2c77..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SortTest.java +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.sqltest; - -import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; -import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; -import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort; -import org.apache.doris.nereids.util.PlanChecker; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class SortTest extends SqlTestBase { - @Test - public void testTwoPhaseSort() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - String sql = "select * from\n" - + "(select score from T1 order by id) as t order by score\n"; - PhysicalPlan plan = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .optimize() - .getBestPlanTree(); - System.out.println(plan.treeString()); - Assertions.assertTrue(plan.anyMatch(e -> e instanceof PhysicalQuickSort - && ((PhysicalQuickSort) e).getSortPhase().isMerge() && e.child(0) instanceof PhysicalDistribute)); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlPlanSuiteTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlPlanSuiteTest.java new file mode 100644 index 00000000000000..879d724c69e5ac --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlPlanSuiteTest.java @@ -0,0 +1,517 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.sqltest; + +import org.apache.doris.nereids.properties.DistributionSpecHash; +import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.RuleSet; +import org.apache.doris.nereids.rules.rewrite.ReorderJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort; +import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.PlanChecker; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Merged suite: these tests only need the shared fixture provided by the base class, so they are + * kept in one class on purpose. Every extra test class pays a full FE startup, which dominates the + * runtime of tests this small. + * + *

Replaces the former standalone classes: + *

    + *
  • CascadesJoinReorderTest
    • + *
  • InferTest
    • + *
  • JoinTest
    • + *
  • MultiJoinTest
    • + *
  • SortTest
    • + *
+ */ +public class SqlPlanSuiteTest extends SqlTestBase { + + // ------------------------------------------------------------------------- + // from CascadesJoinReorderTest + // ------------------------------------------------------------------------- + + @Test + void testStartThreeJoin() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + // Three join + // (n-1)! * 2^(n-1) = 8 + String sql = "SELECT * FROM T1 " + + "JOIN T2 ON T1.id = T2.id " + + "JOIN T3 ON T1.id = T3.id"; + + int plansNumber = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .applyExploration(RuleSet.ZIG_ZAG_TREE_JOIN_REORDER) + .applyExploration(RuleSet.ZIG_ZAG_TREE_JOIN_REORDER) + .applyExploration(RuleSet.ZIG_ZAG_TREE_JOIN_REORDER) + .applyExploration(RuleSet.ZIG_ZAG_TREE_JOIN_REORDER) + .plansNumber(); + + Assertions.assertEquals(8, plansNumber); + } + + @Test + void testStartThreeJoinBushy() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + // Three join + // (n-1)! * 2^(n-1) = 8 + String sql = "SELECT * FROM T1 " + + "JOIN T2 ON T1.id = T2.id " + + "JOIN T3 ON T1.id = T3.id"; + + int plansNumber = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .printlnAllTree() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .plansNumber(); + + Assertions.assertEquals(8, plansNumber); + } + + @Test + void testStarFourJoinZigzag() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + // Four join + // (n-1)! * 2^(n-1) = 48 + String sql = "SELECT * FROM T1 " + + "JOIN T2 ON T1.id = T2.id " + + "JOIN T3 ON T1.id = T3.id " + + "JOIN T4 ON T1.id = T4.id "; + + int plansNumber = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .plansNumber(); + + Assertions.assertEquals(48, plansNumber); + } + + @Test + void testStarFourJoinBushy() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + // Four join + // (n-1)! * 2^(n-1) = 48 + String sql = "SELECT * FROM T1 " + + "JOIN T2 ON T1.id = T2.id " + + "JOIN T3 ON T1.id = T3.id " + + "JOIN T4 ON T1.id = T4.id "; + + int plansNumber = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .plansNumber(); + + Assertions.assertEquals(48, plansNumber); + } + + @Test + void testChainFourJoinBushy() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + // Four join + // 2^(n-1) * C(n-1) = 40 + String sql = "SELECT * FROM T1 " + + "JOIN T2 ON T1.id = T2.id " + + "JOIN T3 ON T2.id = T3.id " + + "JOIN T4 ON T3.id = T4.id "; + + int plansNumber = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .plansNumber(); + + Assertions.assertEquals(40, plansNumber); + } + + @Test + void testChainFiveJoinBushy() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + // Five join + // 2^(n-1) * C(n-1) = 224 + String sql = "SELECT * FROM T1 " + + "JOIN T2 ON T1.id = T2.id " + + "JOIN T3 ON T2.id = T3.id " + + "JOIN T4 ON T3.id = T4.id " + + "JOIN T1 T5 ON T4.ID = T5.ID"; + + int plansNumber = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .plansNumber(); + + Assertions.assertEquals(224, plansNumber); + } + + // ------------------------------------------------------------------------- + // from InferTest + // ------------------------------------------------------------------------- + + @Test + void testInferNotNullAndInferPredicates() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + // Test InferNotNull, EliminateOuter, InferPredicate together + String sql = "select * from T1 left outer join T2 on T1.id = T2.id where T2.id = 4"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches( + innerLogicalJoin( + logicalFilter().when(f -> f.getPredicate().toString().equals("(id#0 = 4)")), + logicalFilter().when(f -> f.getPredicate().toString().equals("(id#2 = 4)")) + ) + ); + } + + @Test + void testInferNotNullFromFilterAndEliminateOuter2() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql + = "select * from T1 right outer join T2 on T1.id = T2.id where T1.id = 4 OR (T1.id > 4 AND T2.score IS NULL)"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .printlnTree() + .matches( + innerLogicalJoin( + logicalFilter().when( + f -> f.getPredicate().toString().equals("(id#2 >= 4)")), + logicalFilter().when( + f -> ExpressionUtils.and(f.getConjuncts().stream() + .sorted((a, b) -> a.toString().compareTo(b.toString())) + .collect(Collectors.toList())) + .toString().equals("(id#0 >= 4)")) + ) + + ); + } + + @Test + void testInferNotNullFromFilterAndEliminateOuter3() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql + = "select * from T1 full outer join T2 on T1.id = T2.id where T1.id = 4 OR (T1.id > 4 AND T2.score IS NULL)"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches( + logicalFilter( + leftOuterLogicalJoin( + logicalFilter().when( + f -> f.getPredicate().toString().equals("(id#0 >= 4)")), + logicalFilter().when( + f -> f.getPredicate().toString().equals("(id#2 >= 4)") + ) + ) + ).when(f -> f.getPredicate().toString() + .equals("OR[(id#0 = 4),AND[(id#0 > 4),score#3 IS NULL]]")) + ); + } + + @Test + void testInferNotNullFromJoinAndEliminateOuter() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + // Is Not Null will infer from semi join, so right outer join can be eliminated. + String sql + = "select * from (select T1.id from T1 right outer join T2 on T1.id = T2.id) T1 left semi join T3 on T1.id = T3.id"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches( + innerLogicalJoin( + logicalProject(), + logicalProject(leftSemiLogicalJoin()) + ) + ); + } + + @Test + void aggEliminateOuterJoin() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "select count(T2.score) from T1 left Join T2 on T1.id = T2.id"; + + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches( + logicalAggregate( + logicalProject( + innerLogicalJoin() + ) + ) + ); + } + + // ------------------------------------------------------------------------- + // from JoinTest + // ------------------------------------------------------------------------- + + @Test + void testJoinUsing() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "SELECT * FROM T1 JOIN T2 using (id)"; + PlanChecker.from(connectContext) + .analyze(sql) + .applyBottomUp(new ReorderJoin()) + .matches( + innerLogicalJoin().when(j -> j.getHashJoinConjuncts().size() == 1) + ); + } + + @Test + void testColocatedJoin() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "select * from T2 join T2 b on T2.id = b.id and T2.id = b.id;"; + PhysicalPlan plan = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .optimize() + .getBestPlanTree(); + // generate colocate join plan without physicalDistribute + System.out.println(plan.treeString()); + Assertions.assertFalse(plan.anyMatch(p -> p instanceof PhysicalDistribute + && ((PhysicalDistribute) p).getDistributionSpec() instanceof DistributionSpecHash)); + sql = "select * from T1 join T0 on T1.score = T0.score and T1.id = T0.id;"; + plan = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .optimize() + .getBestPlanTree(); + // generate colocate join plan without physicalDistribute + Assertions.assertFalse(plan.anyMatch(p -> p instanceof PhysicalDistribute + && ((PhysicalDistribute) p).getDistributionSpec() instanceof DistributionSpecHash)); + } + + @Test + void testDedupConjuncts() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "select * from T1 join T2 on T1.id = T2.id and T1.id = T2.id;"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches( + innerLogicalJoin().when(j -> j.getHashJoinConjuncts().size() == 1) + ); + + String sql1 = "select * from T1 left join T2 on T1.id = T2.id and T1.id = T2.id;"; + PlanChecker.from(connectContext) + .analyze(sql1) + .rewrite() + .matches( + leftOuterLogicalJoin().when(j -> j.getHashJoinConjuncts().size() == 1) + ); + } + + @Test + void testBucketJoinWithAgg() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "select * from " + + "(select distinct id as cnt from T2) T1 inner join" + + "(select distinct id as cnt from T2) T2 " + + "on T1.cnt = T2.cnt"; + PhysicalPlan plan = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .optimize() + .getBestPlanTree(PhysicalProperties.ANY); + Assertions.assertEquals( + ShuffleType.NATURAL, + ((DistributionSpecHash) ((PhysicalPlan) (plan.child(0).child(0))) + .getPhysicalProperties().getDistributionSpec()).getShuffleType() + ); + } + + // ------------------------------------------------------------------------- + // from MultiJoinTest + // ------------------------------------------------------------------------- + + @Test + void testMultiJoinEliminateCross() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + List sqls = ImmutableList.builder() + .add("SELECT * FROM T2 LEFT JOIN T3 ON T2.id = T3.id, T1 WHERE T1.id = T2.id") + .add("SELECT * FROM T2 LEFT JOIN T3 ON T2.id = T3.id, T1 WHERE T1.id = T2.id AND T1.score > 0") + .add("SELECT * FROM T2 LEFT JOIN T3 ON T2.id = T3.id, T1 WHERE T1.id = T2.id AND T1.score > 0 AND T1.id + T2.id + T3.id > 0") + .build(); + + for (String sql : sqls) { + PlanChecker.from(connectContext) + .analyze(sql) + .applyBottomUp(new ReorderJoin()) + .matches( + logicalJoin( + logicalJoin().whenNot(join -> join.getJoinType().isCrossJoin()), + leafPlan() + ).whenNot(join -> join.getJoinType().isCrossJoin()) + ) + .printlnTree(); + } + } + + @Test + @Disabled + void testEliminateBelowOuter() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + // FIXME: MultiJoin And EliminateOuter + String sql = "SELECT * FROM T1, T2 LEFT JOIN T3 ON T2.id = T3.id WHERE T1.id = T2.id"; + PlanChecker.from(connectContext) + .analyze(sql) + .applyBottomUp(new ReorderJoin()) + .printlnTree(); + } + + @Test + void testMultiJoinExistCross() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + List sqls = ImmutableList.builder() + .add("SELECT * FROM T2 LEFT SEMI JOIN T3 ON T2.id = T3.id, T1 WHERE T1.id > T2.id") + .build(); + + for (String sql : sqls) { + PlanChecker.from(connectContext) + .analyze(sql) + .applyBottomUp(new ReorderJoin()) + .matches( + logicalJoin( + logicalJoin().whenNot(join -> join.getJoinType().isCrossJoin()), + leafPlan() + ).when(join -> join.getJoinType().isCrossJoin()) + .whenNot(join -> join.getOtherJoinConjuncts().isEmpty()) + ) + .printlnTree(); + } + } + + @Test + void testOuterJoin() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "SELECT * FROM T1 LEFT OUTER JOIN T2 ON T1.id = T2.id, T3 WHERE T2.score > 0"; + PlanChecker.from(connectContext) + .analyze(sql) + .applyBottomUp(new ReorderJoin()) + .printlnTree() + .matches( + crossLogicalJoin( + leftOuterLogicalJoin() + .when(join -> join.getOtherJoinConjuncts().size() == 1), + logicalOlapScan() + ) + ); + } + + @Test + @Disabled + void testNoFilter() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "Select * FROM T1 INNER JOIN T2 On true"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches( + crossLogicalJoin() + ); + } + + @Test + void test() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "select T1.score, T2.score from T1 inner join T2 on T1.id = T2.id where T1.score - 2 > T2.score"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches( + logicalProject( + innerLogicalJoin() + ) + ); + + } + + // ------------------------------------------------------------------------- + // from SortTest + // ------------------------------------------------------------------------- + + @Test + public void testTwoPhaseSort() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + String sql = "select * from\n" + + "(select score from T1 order by id) as t order by score\n"; + PhysicalPlan plan = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .optimize() + .getBestPlanTree(); + System.out.println(plan.treeString()); + Assertions.assertTrue(plan.anyMatch(e -> e instanceof PhysicalQuickSort + && ((PhysicalQuickSort) e).getSortPhase().isMerge() && e.child(0) instanceof PhysicalDistribute)); + } +} From 022a9607d4e7f3725dbbe0c05548d4897a674379 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 29 Jul 2026 16:52:16 +0800 Subject: [PATCH 3/5] [opt](test) Give DistributeHintTest assertions and cut its runtime by 95% nereids/jobs/joinorder/joinhint/DistributeHintTest was the slowest class in the whole FE UT suite: 764s in CI build 1008420, 459s in 1008517. The 40% swing is not noise, the test is unseeded and the amount of work it does varies per run. It also had no assertions at all, and never had any since it was added in #28562: mismatches were counted into a field and printed to stdout, so no optimizer regression could ever fail it. Two of its 182 local seconds were useful; the rest was enumeration that could not report anything. Both properties it covers are worth keeping, so the tests are kept and made able to fail: - testHintJoin: a distribute hint (broadcast / shuffle) selects a physical strategy and must never change the result. This is the only coverage of distribute hints over random join graphs. Enumeration unchanged (~910 optimizations, ~5s); it now asserts. - testLeading: reordering *inner* joins is always semantics-preserving, so any leading order must give the same result as the un-hinted plan. It now builds inner-join-only graphs via the existing HyperGraphBuilderOld(Set) constructor, draws permutations from a fixed seed, and asserts. Enumeration is cut from ~51000 optimizations to ~280, which is what made the class slow. The previous testLeading applied a random permutation to graphs that also contained outer / semi / anti joins, where an arbitrary leading order is generally not a valid join order. Asserting on that case fails reproducibly (3/3 runs, down to 3 tables): extra null-padded rows appear. Whether Doris should reject such a hint instead of applying it is a question for the Nereids owners, so that case is kept as a @Disabled test documenting the repro rather than asserting behaviour nobody has decided. Local runtime: 182s -> 9-10s over three runs, all passing. Co-Authored-By: Claude Opus 5 (1M context) --- .../joinhint/DistributeHintTest.java | 181 +++++++++--------- 1 file changed, 94 insertions(+), 87 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/joinhint/DistributeHintTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/joinhint/DistributeHintTest.java index 7f3b6aa12c5338..ee60a9eec86359 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/joinhint/DistributeHintTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/joinhint/DistributeHintTest.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.datasets.tpch.TPCHTestBase; import org.apache.doris.nereids.properties.SelectHint; import org.apache.doris.nereids.properties.SelectHintLeading; +import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalSelectHint; @@ -30,124 +31,130 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Random; import java.util.Set; +/** + * A hint only tells the optimizer which plan to prefer; it must never change the result of the + * query. Both tests below build a random join graph, evaluate it with an in-memory join evaluator, + * optimize it, and assert the evaluated result is unchanged. + */ public class DistributeHintTest extends TPCHTestBase { - private int used = 0; - private int unused = 0; - - private int syntaxError = 0; - - private int successCases = 0; - - private int unsuccessCases = 0; - - private List failCases = new ArrayList<>(); + // Leading order is only unconditionally valid for inner joins, so the permutations are drawn + // from a fixed seed to keep a failure reproducible from the test output. + private static final long LEADING_SEED = 20260729L; + /** + * A distribute hint (broadcast / shuffle) picks a physical strategy, so it can never change the + * result. This is the only place that covers distribute hints against random join graphs. + */ @Test - public void testLeading() { + public void testHintJoin() { for (int t = 3; t < 10; t++) { for (int e = t - 1; e <= (t * (t - 1)) / 2; e++) { for (int i = 0; i < 10; i++) { - System.out.println("TableNumber: " + String.valueOf(t) + " EdgeNumber: " + e + " Iteration: " + i); - randomTest(t, e, false, true); + HyperGraphBuilderOld builder = new HyperGraphBuilderOld(); + Plan plan = builder.buildJoinPlanWithJoinHint(t, e); + plan = new LogicalProject(plan.getOutput(), plan); + assertOptimizeKeepsResult(builder, plan, plan, + String.format("distribute hint changed the result (tables=%d, edges=%d, iter=%d)", + t, e, i)); } } } - int totalCases = successCases + unsuccessCases; - System.out.println("TotalCases: " + totalCases + "\tSuccessCases: " + successCases + unsuccessCases + "\tUnSuccessCases: " + 0); - for (String treePlan : failCases) { - System.out.println(treePlan); - } } + /** + * Reordering inner joins is always semantics-preserving, so any leading order must produce the + * same result as the un-hinted plan. + */ @Test - public void testHintJoin() { - for (int t = 3; t < 10; t++) { + public void testLeading() { + Random random = new Random(LEADING_SEED); + for (int t = 3; t < 7; t++) { for (int e = t - 1; e <= (t * (t - 1)) / 2; e++) { - for (int i = 0; i < 10; i++) { - System.out.println("TableNumber: " + String.valueOf(t) + " EdgeNumber: " + e + " Iteration: " + i); - randomTest(t, e, true, false); + for (int i = 0; i < 3; i++) { + HyperGraphBuilderOld builder = + new HyperGraphBuilderOld(ImmutableSet.of(JoinType.INNER_JOIN)); + Plan plan = builder.randomBuildPlanWith(t, e); + plan = new LogicalProject(plan.getOutput(), plan); + for (int p = 0; p < Math.min(t, 4); p++) { + List order = leadingOrder(t, random); + Plan leadingPlan = withLeadingHint(order, plan); + assertOptimizeKeepsResult(builder, plan, leadingPlan, + String.format("leading %s changed the result (tables=%d, edges=%d, iter=%d)", + order, t, e, i)); + } } } } - int totalCases = successCases + unsuccessCases; - System.out.println("TotalCases: " + totalCases + "\tSuccessCases: " + successCases + "\tUnSuccessCases: " + unsuccessCases); - for (String treePlan : failCases) { - System.out.println(treePlan); + } + + /** + * Same property as {@link #testLeading}, but the graph also contains outer / semi / anti joins, + * for which an arbitrary leading order is generally not a valid join order. Today Doris applies + * such a hint anyway and the result changes -- extra null-padded rows appear. Whether the hint + * should be rejected and ignored instead is an open question for the Nereids owners, so this + * test records the case rather than asserting a behaviour nobody has signed off on. + * + *

Reproduces at 3 tables. Re-enable once the intended behaviour is decided. + */ + @Disabled("leading hint over outer joins is not result-preserving today; intended behaviour undecided") + @Test + public void testLeadingWithOuterJoin() { + Random random = new Random(LEADING_SEED); + for (int t = 3; t < 6; t++) { + for (int e = t - 1; e <= (t * (t - 1)) / 2; e++) { + HyperGraphBuilderOld builder = new HyperGraphBuilderOld(); + Plan plan = builder.randomBuildPlanWith(t, e); + plan = new LogicalProject(plan.getOutput(), plan); + List order = leadingOrder(t, random); + assertOptimizeKeepsResult(builder, plan, withLeadingHint(order, plan), + String.format("leading %s over outer joins changed the result (tables=%d, edges=%d)", + order, t, e)); + } } } - private Plan generateLeadingHintPlan(int tableNum, Plan childPlan) { - ImmutableList.Builder hints = ImmutableList.builder(); - List leadingParameters = new ArrayList(); + private List leadingOrder(int tableNum, Random random) { + List order = new ArrayList<>(); for (int i = 0; i < tableNum; i++) { - leadingParameters.add(String.valueOf(i)); + order.add(String.valueOf(i)); } - Collections.shuffle(leadingParameters); - System.out.println("LeadingHint: " + leadingParameters.toString()); - hints.add(new SelectHintLeading("Leading", leadingParameters, ImmutableMap.of())); - return new LogicalSelectHint<>(hints.build(), childPlan); + Collections.shuffle(order, random); + return order; } - private void randomTest(int tableNum, int edgeNum, boolean withJoinHint, boolean withLeading) { - HyperGraphBuilderOld hyperGraphBuilder = new HyperGraphBuilderOld(); - Plan plan = withJoinHint ? hyperGraphBuilder.buildJoinPlanWithJoinHint(tableNum, edgeNum) : - hyperGraphBuilder.randomBuildPlanWith(tableNum, edgeNum); - plan = new LogicalProject(plan.getOutput(), plan); - Set> res1 = hyperGraphBuilder.evaluate(plan); - if (!withLeading) { - CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan); - hyperGraphBuilder.initStats("tpch", cascadesContext); - Plan optimizedPlan = PlanChecker.from(cascadesContext) - .analyze() - .optimize() - .getBestPlanTree(); - - Set> res2 = hyperGraphBuilder.evaluate(optimizedPlan); - if (!res1.equals(res2)) { - System.out.println(plan.treeString()); - System.out.println(optimizedPlan.treeString()); - cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan); - PlanChecker.from(cascadesContext).dpHypOptimize().getBestPlanTree(); - System.out.println(res1); - System.out.println(res2); - unsuccessCases++; - failCases.add(plan.treeString()); - failCases.add(optimizedPlan.treeString()); - } - successCases++; - } else { - // generate select hint - for (int i = 0; i < (tableNum * tableNum - 1); i++) { - Plan leadingPlan = generateLeadingHintPlan(tableNum, plan); - CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(connectContext, leadingPlan); - hyperGraphBuilder.initStats("tpch", cascadesContext); - Plan optimizedPlan = PlanChecker.from(cascadesContext) - .analyze() - .optimize() - .getBestPlanTree(); + private Plan withLeadingHint(List order, Plan childPlan) { + ImmutableList.Builder hints = ImmutableList.builder(); + hints.add(new SelectHintLeading("Leading", order, ImmutableMap.of())); + return new LogicalSelectHint<>(hints.build(), childPlan); + } - Set> res2 = hyperGraphBuilder.evaluate(optimizedPlan); - if (!res1.equals(res2)) { - System.out.println(leadingPlan.treeString()); - System.out.println(optimizedPlan.treeString()); - cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan); - PlanChecker.from(cascadesContext).dpHypOptimize().getBestPlanTree(); - System.out.println(res1); - System.out.println(res2); - unsuccessCases++; - failCases.add(leadingPlan.treeString()); - failCases.add(optimizedPlan.treeString()); - } - successCases++; - } - } + /** + * Evaluates {@code originalPlan}, optimizes {@code planToOptimize}, and asserts both produce the + * same tuples. + */ + private void assertOptimizeKeepsResult(HyperGraphBuilderOld builder, Plan originalPlan, + Plan planToOptimize, String message) { + Set> expected = builder.evaluate(originalPlan); + CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(connectContext, planToOptimize); + builder.initStats("tpch", cascadesContext); + Plan optimizedPlan = PlanChecker.from(cascadesContext) + .analyze() + .optimize() + .getBestPlanTree(); + Assertions.assertEquals(expected, builder.evaluate(optimizedPlan), + message + "\noriginal plan:\n" + originalPlan.treeString() + + "\noptimized plan:\n" + optimizedPlan.treeString()); } } From 5c459c97e34088cc865eee49e6e4fee7eb335763 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 29 Jul 2026 23:04:54 +0800 Subject: [PATCH 4/5] [opt](test) Restore spied Env fields after every test method 26 test classes install a Mockito spy over Env.accessManager (and a few over systemInfo / authenticationIntegrationMgr) with Deencapsulation.setField and never put the original back. Each subsequent test method in the same JVM then spies the previous spy, so the wrappers stack up for the life of the class. TestWithFeService now captures those three fields once class setup is done and restores them after every test method. Capturing after runBeforeAll() rather than before each test is deliberate: CheckRowPolicyTest installs a spy for the whole class inside runBeforeAll(), so that spy becomes its baseline and the restore is a no-op there. Capturing earlier would have torn its spy down after the first test method. Honest scope note: this was written as a prerequisite for merging spy-using test classes, but it turned out not to be required for that. Merging six of them passes with this commit reverted, because each test re-reads the manager and re-stubs what it needs. What this commit fixes is the unbounded spy nesting and the latent cross-test contamination, not a failure observed today. Co-Authored-By: Claude Opus 5 (1M context) --- .../commands/AdminCopyTabletCommandTest.java | 86 ---------- .../commands/AdminRepairTableCommandTest.java | 101 ------------ .../commands/AlterColumnStatsCommandTest.java | 149 ------------------ .../commands/CleanQueryStatsCommandTest.java | 86 ---------- .../commands/ShowQueryStatsCommandTest.java | 94 ----------- .../ShowTabletsFromTableCommandTest.java | 142 ----------------- .../doris/utframe/TestWithFeService.java | 22 +++ 7 files changed, 22 insertions(+), 658 deletions(-) delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AdminCopyTabletCommandTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AdminRepairTableCommandTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterColumnStatsCommandTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CleanQueryStatsCommandTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowQueryStatsCommandTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletsFromTableCommandTest.java diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AdminCopyTabletCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AdminCopyTabletCommandTest.java deleted file mode 100644 index 0ca4823a463426..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AdminCopyTabletCommandTest.java +++ /dev/null @@ -1,86 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.commands; - -import org.apache.doris.catalog.Env; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.mysql.privilege.AccessControllerManager; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.utframe.TestWithFeService; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -public class AdminCopyTabletCommandTest extends TestWithFeService { - private ConnectContext connectContext; - private Env env; - private AccessControllerManager accessControllerManager; - - private void runBefore() throws IOException { - connectContext = createDefaultCtx(); - env = Env.getCurrentEnv(); - accessControllerManager = env.getAccessManager(); - } - - @Test - public void testValidateNormal() throws Exception { - runBefore(); - connectContext.setSkipAuth(true); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(true).when(spyAcm).checkGlobalPriv( - Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - Map properties = new HashMap<>(); - properties.put("version", "0"); - AdminCopyTabletCommand command = new AdminCopyTabletCommand(100, properties); - Assertions.assertDoesNotThrow(() -> command.validate()); - - Map properties2 = new HashMap<>(); - properties2.put("backend_id", "10"); - AdminCopyTabletCommand command2 = new AdminCopyTabletCommand(100, properties2); - Assertions.assertDoesNotThrow(() -> command2.validate()); - - Map properties3 = new HashMap<>(); - properties3.put("expiration_minutes", "10"); - AdminCopyTabletCommand command3 = new AdminCopyTabletCommand(100, properties3); - Assertions.assertDoesNotThrow(() -> command3.validate()); - } - - @Test - public void testNoPriviledge() throws Exception { - runBefore(); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(false).when(spyAcm).checkGlobalPriv( - Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - Map properties = new HashMap<>(); - properties.put("version", "0"); - AdminCopyTabletCommand command = new AdminCopyTabletCommand(100, properties); - Assertions.assertThrows(AnalysisException.class, () -> command.validate(), - "Access denied; you need (at least one of) the (Admin_priv) privilege(s) for this operation"); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AdminRepairTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AdminRepairTableCommandTest.java deleted file mode 100644 index 2e45dbaf160147..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AdminRepairTableCommandTest.java +++ /dev/null @@ -1,101 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.commands; - -import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.info.PartitionNamesInfo; -import org.apache.doris.catalog.info.TableNameInfo; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.datasource.InternalCatalog; -import org.apache.doris.info.TableRefInfo; -import org.apache.doris.mysql.privilege.AccessControllerManager; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.utframe.TestWithFeService; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -public class AdminRepairTableCommandTest extends TestWithFeService { - private static final String internalCtl = InternalCatalog.INTERNAL_CATALOG_NAME; - private ConnectContext connectContext; - private Env env; - private AccessControllerManager accessControllerManager; - - private void runBefore() throws IOException { - connectContext = createDefaultCtx(); - env = Env.getCurrentEnv(); - accessControllerManager = env.getAccessManager(); - } - - @Test - public void testValidateNormal() throws Exception { - runBefore(); - connectContext.setSkipAuth(true); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(true).when(spyAcm).checkGlobalPriv( - Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - TableNameInfo tableNameInfo = new TableNameInfo(internalCtl, "test_db", "test_tbl"); - List partitionNames = new ArrayList<>(); - partitionNames.add("p1"); - partitionNames.add("p2"); - PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, partitionNames); - TableRefInfo tableRefInfo = new TableRefInfo(tableNameInfo, null, null, partitionNamesInfo, null, null, null, null); - AdminRepairTableCommand command = new AdminRepairTableCommand(tableRefInfo); - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - - //test external catalog - TableNameInfo tableNameInfo2 = new TableNameInfo("hive", "test_db", "test_tbl"); - TableRefInfo tableRefInfo2 = new TableRefInfo(tableNameInfo2, null, null, partitionNamesInfo, null, null, null, null); - AdminRepairTableCommand command2 = new AdminRepairTableCommand(tableRefInfo2); - Assertions.assertThrows(AnalysisException.class, () -> command2.validate(connectContext), - "External catalog 'hive' is not allowed in 'AdminCancelRepairTableCommand.class'"); - - //test partitionNameInfo isTemp - PartitionNamesInfo partitionNamesInfo2 = new PartitionNamesInfo(true, partitionNames); - TableRefInfo tableRefInfo3 = new TableRefInfo(tableNameInfo, null, null, partitionNamesInfo2, null, null, null, null); - AdminRepairTableCommand command3 = new AdminRepairTableCommand(tableRefInfo3); - Assertions.assertThrows(AnalysisException.class, () -> command3.validate(connectContext), - "Do not support (cancel)repair temporary partitions"); - } - - @Test - public void testValidateNoPriviledge() throws Exception { - runBefore(); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(false).when(spyAcm).checkGlobalPriv( - Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - TableNameInfo tableNameInfo = new TableNameInfo(internalCtl, "test_db", "test_tbl"); - List partitionNames = new ArrayList<>(); - PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, partitionNames); - TableRefInfo tableRefInfo = new TableRefInfo(tableNameInfo, null, null, partitionNamesInfo, null, null, null, null); - AdminRepairTableCommand command = new AdminRepairTableCommand(tableRefInfo); - Assertions.assertThrows(AnalysisException.class, () -> command.validate(connectContext), - "Access denied; you need (at least one of) the (ADMIN) privilege(s) for this operation"); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterColumnStatsCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterColumnStatsCommandTest.java deleted file mode 100644 index d5efeec9d6b46d..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterColumnStatsCommandTest.java +++ /dev/null @@ -1,149 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.commands; - -import org.apache.doris.backup.CatalogMocker; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.info.PartitionNamesInfo; -import org.apache.doris.catalog.info.TableNameInfo; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.UserException; -import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.datasource.InternalCatalog; -import org.apache.doris.mysql.privilege.AccessControllerManager; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.utframe.TestWithFeService; - -import com.google.common.collect.ImmutableList; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -public class AlterColumnStatsCommandTest extends TestWithFeService { - private static final String internalCtl = InternalCatalog.INTERNAL_CATALOG_NAME; - private ConnectContext connectContext; - private Env env; - private AccessControllerManager accessControllerManager; - private Database db; - - private void runBefore() throws IOException { - connectContext = createDefaultCtx(); - env = Env.getCurrentEnv(); - accessControllerManager = env.getAccessManager(); - } - - @Test - public void testValidateNormal() throws Exception { - runBefore(); - connectContext.setSkipAuth(true); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(true).when(spyAcm).checkTblPriv(Mockito.nullable(ConnectContext.class), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), - Mockito.any(PrivPredicate.class)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - //test normal - connectContext.getSessionVariable().enableStats = true; - createDatabase("test_db"); - createTable("create table test_db.test_tbl\n" + "(k1 int, k2 int)\n" - + "duplicate key(k1)\n" + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" - + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1'); "); - - TableNameInfo tableNameInfo = - new TableNameInfo("test_db", "test_tbl"); - PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, - ImmutableList.of("p1")); - String indexName = null; - String columnName = "k1"; - Map properties = new HashMap<>(); - properties.put("row_count", "5"); - properties.put("avg_size", "100000"); - AlterColumnStatsCommand command = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties); - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - - //test not a partitioned table - createTable("create table test_db.test_tbl2(k1 int) distributed by hash(k1) buckets 3 properties('replication_num' = '1');"); - TableNameInfo tableNameInfo2 = - new TableNameInfo("test_db", "test_tbl2"); - AlterColumnStatsCommand command2 = new AlterColumnStatsCommand(tableNameInfo2, partitionNamesInfo, indexName, columnName, properties); - Assertions.assertThrows(AnalysisException.class, () -> command2.validate(connectContext), - "Not a partitioned table: test_tbl2"); - - //test partition does not exist - PartitionNamesInfo partitionNamesInfo2 = new PartitionNamesInfo(false, - ImmutableList.of("k3")); - AlterColumnStatsCommand command3 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo2, indexName, columnName, properties); - Assertions.assertThrows(AnalysisException.class, () -> command3.validate(connectContext), - "Partition does not exist: k3"); - - //test indexId not exist in OlapTable - String indexName3 = "invalid_index"; - AlterColumnStatsCommand command4 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName3, columnName, properties); - Assertions.assertThrows(AnalysisException.class, () -> command4.validate(connectContext), - "Index invalid_index not exist in table test_tbl"); - - //test invalid statistics - Map properties2 = new HashMap<>(); - properties2.put("histogram", "invalide_value"); - AlterColumnStatsCommand command5 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties2); - Assertions.assertThrows(AnalysisException.class, () -> command5.validate(connectContext), - "histogram is invalid statistics"); - - //row_count is not exist - Map properties3 = new HashMap<>(); - properties2.put("avg_size", "100000"); - properties2.put("max_size", "100000000"); - AlterColumnStatsCommand command6 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties3); - Assertions.assertThrows(AnalysisException.class, () -> command6.validate(connectContext), - "Set column stats must set row_count. e.g. 'row_count'='5'"); - - //test enable stats - connectContext.getSessionVariable().enableStats = false; - AlterColumnStatsCommand command7 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties); - Assertions.assertThrows(UserException.class, () -> command7.validate(connectContext), - "Analyze function is forbidden, you should add `enable_stats=true` in your FE conf file"); - } - - @Test - void testValidateNoPrivilege() throws IOException { - runBefore(); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(false).when(spyAcm).checkTblPriv(Mockito.nullable(ConnectContext.class), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), - Mockito.any(PrivPredicate.class)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - TableNameInfo tableNameInfo = - new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL2_NAME); - PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, - ImmutableList.of(CatalogMocker.TEST_PARTITION1_NAME)); - - String indexName = "index1"; - String columnName = "k1"; - Map properties = new HashMap<>(); - - AlterColumnStatsCommand command = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties); - connectContext.getSessionVariable().enableStats = true; - Assertions.assertThrows(AnalysisException.class, () -> command.validate(connectContext), - "ALTER TABLE STATS command denied to user 'null'@'null' for table 'test_db: test_tbl2'"); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CleanQueryStatsCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CleanQueryStatsCommandTest.java deleted file mode 100644 index 2b8659a0e85ec7..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CleanQueryStatsCommandTest.java +++ /dev/null @@ -1,86 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.commands; - -import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.info.TableNameInfo; -import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.datasource.InternalCatalog; -import org.apache.doris.mysql.privilege.AccessControllerManager; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.utframe.TestWithFeService; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.io.IOException; - -public class CleanQueryStatsCommandTest extends TestWithFeService { - private static final String internalCtl = InternalCatalog.INTERNAL_CATALOG_NAME; - private ConnectContext connectContext; - private Env env; - private AccessControllerManager accessControllerManager; - - public void runBefore() throws IOException { - connectContext = createDefaultCtx(); - env = Env.getCurrentEnv(); - accessControllerManager = env.getAccessManager(); - } - - @Test - public void testAllNormal() throws IOException { - runBefore(); - connectContext.setSkipAuth(true); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(true).when(spyAcm).checkGlobalPriv(Mockito.nullable(ConnectContext.class), Mockito.any(PrivPredicate.class)); - Deencapsulation.setField(env, "accessManager", spyAcm); - CleanQueryStatsCommand command = new CleanQueryStatsCommand(); - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - } - - @Test - public void testDB() throws Exception { - runBefore(); - connectContext.setDatabase("test_db"); - connectContext.setSkipAuth(true); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(true).when(spyAcm).checkDbPriv(Mockito.nullable(ConnectContext.class), Mockito.anyString(), Mockito.anyString(), Mockito.any(PrivPredicate.class)); - Deencapsulation.setField(env, "accessManager", spyAcm); - CleanQueryStatsCommand command = new CleanQueryStatsCommand("test_db"); - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - } - - @Test - public void testTbl() throws Exception { - runBefore(); - createDatabase("test_db"); - createTable("create table test_db.test_tbl\n" + "(k1 int, k2 int)\n" - + "duplicate key(k1)\n" + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" - + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1'); "); - TableNameInfo tableNameInfo = new TableNameInfo("test_db", "test_tbl"); - connectContext.setDatabase("test_db"); - connectContext.setSkipAuth(true); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(true).when(spyAcm).checkTblPriv(Mockito.nullable(ConnectContext.class), Mockito.any(TableNameInfo.class), Mockito.any(PrivPredicate.class)); - Deencapsulation.setField(env, "accessManager", spyAcm); - CleanQueryStatsCommand command = new CleanQueryStatsCommand(tableNameInfo); - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowQueryStatsCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowQueryStatsCommandTest.java deleted file mode 100644 index 472d90cea95502..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowQueryStatsCommandTest.java +++ /dev/null @@ -1,94 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.commands; - -import org.apache.doris.backup.CatalogMocker; -import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.info.TableNameInfo; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.datasource.InternalCatalog; -import org.apache.doris.mysql.privilege.AccessControllerManager; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.utframe.TestWithFeService; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -public class ShowQueryStatsCommandTest extends TestWithFeService { - private static final String internalCtl = InternalCatalog.INTERNAL_CATALOG_NAME; - private static boolean dbInitialized = false; - - private ConnectContext connectContext; - private Env env; - private AccessControllerManager accessControllerManager; - - private void runBefore() throws Exception { - connectContext = createDefaultCtx(); - env = Env.getCurrentEnv(); - accessControllerManager = env.getAccessManager(); - if (!dbInitialized) { - createDatabaseWithSql("CREATE DATABASE IF NOT EXISTS " + CatalogMocker.TEST_DB_NAME); - createTable("CREATE TABLE IF NOT EXISTS " + CatalogMocker.TEST_DB_NAME + "." + CatalogMocker.TEST_TBL_NAME - + " (k1 INT, k2 INT) DISTRIBUTED BY HASH(k1) BUCKETS 1" - + " PROPERTIES ('replication_num' = '1')"); - dbInitialized = true; - } - } - - @Test - public void testValidateWithPrivilege() throws Exception { - runBefore(); - connectContext.setSkipAuth(true); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(true).when(spyAcm).checkGlobalPriv( - Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); - Mockito.doReturn(true).when(spyAcm).checkTblPriv( - Mockito.nullable(ConnectContext.class), Mockito.any(TableNameInfo.class), - Mockito.eq(PrivPredicate.SHOW)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - TableNameInfo tableNameInfo = - new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME); - - // normal - ShowQueryStatsCommand command = new ShowQueryStatsCommand(tableNameInfo.getDb(), - tableNameInfo, false, false); - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - } - - @Test - void testValidateNoPrivilege() throws Exception { - runBefore(); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(false).when(spyAcm).checkGlobalPriv( - Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); - Mockito.doReturn(false).when(spyAcm).checkTblPriv( - Mockito.nullable(ConnectContext.class), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString(), Mockito.eq(PrivPredicate.SHOW)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - TableNameInfo tableNameInfo = - new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME); - ShowQueryStatsCommand command = new ShowQueryStatsCommand(tableNameInfo.getDb(), - tableNameInfo, false, false); - Assertions.assertThrows(AnalysisException.class, () -> command.validate(connectContext)); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletsFromTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletsFromTableCommandTest.java deleted file mode 100644 index 17613cd057ce02..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletsFromTableCommandTest.java +++ /dev/null @@ -1,142 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.commands; - -import org.apache.doris.backup.CatalogMocker; -import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.info.PartitionNamesInfo; -import org.apache.doris.catalog.info.TableNameInfo; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.datasource.InternalCatalog; -import org.apache.doris.mysql.privilege.AccessControllerManager; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.nereids.analyzer.UnboundSlot; -import org.apache.doris.nereids.properties.OrderKey; -import org.apache.doris.nereids.trees.expressions.EqualTo; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.Or; -import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.utframe.TestWithFeService; - -import com.google.common.collect.ImmutableList; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -public class ShowTabletsFromTableCommandTest extends TestWithFeService { - private static final String internalCtl = InternalCatalog.INTERNAL_CATALOG_NAME; - private ConnectContext connectContext; - private Env env; - private AccessControllerManager accessControllerManager; - - private void runBefore() throws IOException { - connectContext = createDefaultCtx(); - env = Env.getCurrentEnv(); - accessControllerManager = env.getAccessManager(); - } - - @Test - public void testValidateWithPrivilege() throws Exception { - runBefore(); - connectContext.setSkipAuth(true); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(true).when(spyAcm).checkGlobalPriv( - Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); - Mockito.doReturn(true).when(spyAcm).checkTblPriv( - Mockito.nullable(ConnectContext.class), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString(), Mockito.eq(PrivPredicate.ADMIN)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - Expression version = new UnboundSlot("version"); - - Expression whereClauseNormal = new EqualTo(version, new IntegerLiteral(2)); - - List orderKeysNormal = new ArrayList<>(); - orderKeysNormal.add(new OrderKey(version, true, true)); - - TableNameInfo tableNameInfo = - new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME); - PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, - ImmutableList.of(CatalogMocker.TEST_SINGLE_PARTITION_NAME)); - - // normal - ShowTabletsFromTableCommand command = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, - whereClauseNormal, orderKeysNormal, 5, 0); - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - - // where clause error - Expression error = new UnboundSlot("error"); - Expression whereClauseError = new EqualTo(error, new IntegerLiteral(2)); - - ShowTabletsFromTableCommand command2 = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, - whereClauseError, orderKeysNormal, 5, 0); - Assertions.assertThrows(AnalysisException.class, () -> command2.validate(connectContext)); - - // where clause contains or - Expression backendId = new UnboundSlot("BackendId"); - Expression whereClauseOr = new Or( - new EqualTo(version, new IntegerLiteral(2)), - new EqualTo(backendId, new IntegerLiteral(2))); - - ShowTabletsFromTableCommand command3 = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, - whereClauseOr, orderKeysNormal, 5, 0); - Assertions.assertThrows(AnalysisException.class, () -> command3.validate(connectContext)); - - // order by error - List orderKeysError = new ArrayList<>(); - orderKeysError.add(new OrderKey(error, true, true)); - - ShowTabletsFromTableCommand command4 = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, - whereClauseNormal, orderKeysError, 5, 0); - Assertions.assertThrows(AnalysisException.class, () -> command4.validate(connectContext)); - } - - @Test - void testValidateNoPrivilege() throws Exception { - runBefore(); - AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); - Mockito.doReturn(false).when(spyAcm).checkGlobalPriv( - Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); - Mockito.doReturn(false).when(spyAcm).checkTblPriv( - Mockito.nullable(ConnectContext.class), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString(), Mockito.eq(PrivPredicate.ADMIN)); - Deencapsulation.setField(env, "accessManager", spyAcm); - - Expression version = new UnboundSlot("version"); - - Expression whereClauseNormal = new EqualTo(version, new IntegerLiteral(2)); - - List orderKeysNormal = new ArrayList<>(); - orderKeysNormal.add(new OrderKey(version, true, true)); - - TableNameInfo tableNameInfo = - new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME); - PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, - ImmutableList.of(CatalogMocker.TEST_SINGLE_PARTITION_NAME)); - - ShowTabletsFromTableCommand command = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, - whereClauseNormal, orderKeysNormal, 5, 0); - Assertions.assertThrows(AnalysisException.class, () -> command.validate(connectContext)); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java index a3591281d20369..da9e9aea08bb77 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java @@ -35,6 +35,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.job.base.AbstractJob; import org.apache.doris.nereids.CascadesContext; @@ -99,6 +100,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -144,6 +146,14 @@ public abstract class TestWithFeService { protected static final String DEFAULT_CLUSTER_PREFIX = ""; + // Env fields that tests routinely swap for a Mockito spy. They are captured once class setup has + // finished and restored after every test method, so a spy installed by one test method cannot leak + // into the next one. Capturing after runBeforeAll() is deliberate: a class that installs a spy for + // the whole class (CheckRowPolicyTest) has that spy as its baseline, making the restore a no-op there. + private static final String[] ENV_FIELDS_RESTORED_PER_TEST = new String[] { + "accessManager", "systemInfo", "authenticationIntegrationMgr"}; + private final Map envFieldBaseline = Maps.newHashMap(); + @BeforeAll public final void beforeAll() throws Exception { // this.enableAdvanceNextId may be reset by children classes @@ -161,6 +171,10 @@ public final void beforeAll() throws Exception { createDorisCluster(); Env.getCurrentEnv().getWorkloadGroupMgr().createNormalWorkloadGroupForUT(); runBeforeAll(); + Env env = Env.getCurrentEnv(); + for (String field : ENV_FIELDS_RESTORED_PER_TEST) { + envFieldBaseline.put(field, Deencapsulation.getField(env, field)); + } } protected void beforeCluster() { @@ -181,6 +195,14 @@ public final void beforeEach() throws Exception { runBeforeEach(); } + @AfterEach + public final void afterEach() { + Env env = Env.getCurrentEnv(); + for (Map.Entry baseline : envFieldBaseline.entrySet()) { + Deencapsulation.setField(env, baseline.getKey(), baseline.getValue()); + } + } + protected void beforeCreatingConnectContext() throws Exception { } From 6c35909519abf06dfbcb99a23ae9904a7fe66c38 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 29 Jul 2026 23:04:54 +0800 Subject: [PATCH 5/5] [opt](test) Merge six command privilege tests into one suite Six classes under nereids/trees/plans/commands each started a full FE to run two or three validate() assertions with a spied access manager. On the CI agent that fixed cost is about 50s per class, so the six cost roughly 300s of test time for 13 assertions. They are merged into CommandPrivilegeValidationTest: AdminRepairTableCommandTest, AdminCopyTabletCommandTest, AlterColumnStatsCommandTest, CleanQueryStatsCommandTest, ShowQueryStatsCommandTest, ShowTabletsFromTableCommandTest Members that were byte-identical across the originals (connectContext, env, accessControllerManager, internalCtl) are declared once. Test methods whose names collided -- testValidateNormal, testValidateNoPrivilege, testValidateWithPrivilege and the per-class runBefore() helper -- are prefixed with their original class, so grepping the old class name still lands on its tests and the names now say which command they cover. Two databases had to be renamed. AlterColumnStats and CleanQueryStats both created a database literally called "test_db", and CatalogMocker.TEST_DB_NAME is also "test_db", so sharing one FE made the second createDatabase() fail with "database exists". Each section now creates its own database. Verified: 13 tests before, 13 tests after, passing on two consecutive runs. Co-Authored-By: Claude Opus 5 (1M context) --- .../CommandPrivilegeValidationTest.java | 490 ++++++++++++++++++ 1 file changed, 490 insertions(+) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CommandPrivilegeValidationTest.java diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CommandPrivilegeValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CommandPrivilegeValidationTest.java new file mode 100644 index 00000000000000..6d27470e2d92c2 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CommandPrivilegeValidationTest.java @@ -0,0 +1,490 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.backup.CatalogMocker; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.info.PartitionNamesInfo; +import org.apache.doris.catalog.info.TableNameInfo; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.UserException; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.info.TableRefInfo; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.analyzer.UnboundSlot; +import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Or; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Merged suite: each of these classes only needed a bare FE and a spied access manager, so they all + * paid a full FE startup for two or three assertions. Members that were byte-identical across the + * originals are shared; test methods whose names collided are prefixed with their original class so + * grepping the old name still finds them. + * + *

Replaces the former standalone classes: + *

    + *
  • AdminRepairTableCommandTest
    • + *
  • AdminCopyTabletCommandTest
    • + *
  • AlterColumnStatsCommandTest
    • + *
  • CleanQueryStatsCommandTest
    • + *
  • ShowQueryStatsCommandTest
    • + *
  • ShowTabletsFromTableCommandTest
    • + *
+ */ +public class CommandPrivilegeValidationTest extends TestWithFeService { + + private static final String internalCtl = InternalCatalog.INTERNAL_CATALOG_NAME; + private static boolean dbInitialized = false; + private AccessControllerManager accessControllerManager; + private ConnectContext connectContext; + private Env env; + private Database db; + + // ------------------------------------------------------------------------- + // from AdminRepairTableCommandTest + // ------------------------------------------------------------------------- + + private void adminRepairTable_runBefore() throws IOException { + connectContext = createDefaultCtx(); + env = Env.getCurrentEnv(); + accessControllerManager = env.getAccessManager(); + } + + @Test + public void adminRepairTable_testValidateNormal() throws Exception { + adminRepairTable_runBefore(); + connectContext.setSkipAuth(true); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(true).when(spyAcm).checkGlobalPriv( + Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + TableNameInfo tableNameInfo = new TableNameInfo(internalCtl, "test_db", "test_tbl"); + List partitionNames = new ArrayList<>(); + partitionNames.add("p1"); + partitionNames.add("p2"); + PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, partitionNames); + TableRefInfo tableRefInfo = new TableRefInfo(tableNameInfo, null, null, partitionNamesInfo, null, null, null, null); + AdminRepairTableCommand command = new AdminRepairTableCommand(tableRefInfo); + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + + //test external catalog + TableNameInfo tableNameInfo2 = new TableNameInfo("hive", "test_db", "test_tbl"); + TableRefInfo tableRefInfo2 = new TableRefInfo(tableNameInfo2, null, null, partitionNamesInfo, null, null, null, null); + AdminRepairTableCommand command2 = new AdminRepairTableCommand(tableRefInfo2); + Assertions.assertThrows(AnalysisException.class, () -> command2.validate(connectContext), + "External catalog 'hive' is not allowed in 'AdminCancelRepairTableCommand.class'"); + + //test partitionNameInfo isTemp + PartitionNamesInfo partitionNamesInfo2 = new PartitionNamesInfo(true, partitionNames); + TableRefInfo tableRefInfo3 = new TableRefInfo(tableNameInfo, null, null, partitionNamesInfo2, null, null, null, null); + AdminRepairTableCommand command3 = new AdminRepairTableCommand(tableRefInfo3); + Assertions.assertThrows(AnalysisException.class, () -> command3.validate(connectContext), + "Do not support (cancel)repair temporary partitions"); + } + + @Test + public void testValidateNoPriviledge() throws Exception { + adminRepairTable_runBefore(); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(false).when(spyAcm).checkGlobalPriv( + Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + TableNameInfo tableNameInfo = new TableNameInfo(internalCtl, "test_db", "test_tbl"); + List partitionNames = new ArrayList<>(); + PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, partitionNames); + TableRefInfo tableRefInfo = new TableRefInfo(tableNameInfo, null, null, partitionNamesInfo, null, null, null, null); + AdminRepairTableCommand command = new AdminRepairTableCommand(tableRefInfo); + Assertions.assertThrows(AnalysisException.class, () -> command.validate(connectContext), + "Access denied; you need (at least one of) the (ADMIN) privilege(s) for this operation"); + } + + // ------------------------------------------------------------------------- + // from AdminCopyTabletCommandTest + // ------------------------------------------------------------------------- + + private void adminCopyTablet_runBefore() throws IOException { + connectContext = createDefaultCtx(); + env = Env.getCurrentEnv(); + accessControllerManager = env.getAccessManager(); + } + + @Test + public void adminCopyTablet_testValidateNormal() throws Exception { + adminCopyTablet_runBefore(); + connectContext.setSkipAuth(true); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(true).when(spyAcm).checkGlobalPriv( + Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + Map properties = new HashMap<>(); + properties.put("version", "0"); + AdminCopyTabletCommand command = new AdminCopyTabletCommand(100, properties); + Assertions.assertDoesNotThrow(() -> command.validate()); + + Map properties2 = new HashMap<>(); + properties2.put("backend_id", "10"); + AdminCopyTabletCommand command2 = new AdminCopyTabletCommand(100, properties2); + Assertions.assertDoesNotThrow(() -> command2.validate()); + + Map properties3 = new HashMap<>(); + properties3.put("expiration_minutes", "10"); + AdminCopyTabletCommand command3 = new AdminCopyTabletCommand(100, properties3); + Assertions.assertDoesNotThrow(() -> command3.validate()); + } + + @Test + public void testNoPriviledge() throws Exception { + adminCopyTablet_runBefore(); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(false).when(spyAcm).checkGlobalPriv( + Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + Map properties = new HashMap<>(); + properties.put("version", "0"); + AdminCopyTabletCommand command = new AdminCopyTabletCommand(100, properties); + Assertions.assertThrows(AnalysisException.class, () -> command.validate(), + "Access denied; you need (at least one of) the (Admin_priv) privilege(s) for this operation"); + } + + // ------------------------------------------------------------------------- + // from AlterColumnStatsCommandTest + // ------------------------------------------------------------------------- + + private void alterColumnStats_runBefore() throws IOException { + connectContext = createDefaultCtx(); + env = Env.getCurrentEnv(); + accessControllerManager = env.getAccessManager(); + } + + @Test + public void alterColumnStats_testValidateNormal() throws Exception { + alterColumnStats_runBefore(); + connectContext.setSkipAuth(true); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(true).when(spyAcm).checkTblPriv(Mockito.nullable(ConnectContext.class), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), + Mockito.any(PrivPredicate.class)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + //test normal + connectContext.getSessionVariable().enableStats = true; + createDatabase("alter_column_stats_db"); + createTable("create table alter_column_stats_db.test_tbl\n" + "(k1 int, k2 int)\n" + + "duplicate key(k1)\n" + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" + + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1'); "); + + TableNameInfo tableNameInfo = + new TableNameInfo("alter_column_stats_db", "test_tbl"); + PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, + ImmutableList.of("p1")); + String indexName = null; + String columnName = "k1"; + Map properties = new HashMap<>(); + properties.put("row_count", "5"); + properties.put("avg_size", "100000"); + AlterColumnStatsCommand command = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties); + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + + //test not a partitioned table + createTable("create table alter_column_stats_db.test_tbl2(k1 int) distributed by hash(k1) buckets 3 properties('replication_num' = '1');"); + TableNameInfo tableNameInfo2 = + new TableNameInfo("alter_column_stats_db", "test_tbl2"); + AlterColumnStatsCommand command2 = new AlterColumnStatsCommand(tableNameInfo2, partitionNamesInfo, indexName, columnName, properties); + Assertions.assertThrows(AnalysisException.class, () -> command2.validate(connectContext), + "Not a partitioned table: test_tbl2"); + + //test partition does not exist + PartitionNamesInfo partitionNamesInfo2 = new PartitionNamesInfo(false, + ImmutableList.of("k3")); + AlterColumnStatsCommand command3 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo2, indexName, columnName, properties); + Assertions.assertThrows(AnalysisException.class, () -> command3.validate(connectContext), + "Partition does not exist: k3"); + + //test indexId not exist in OlapTable + String indexName3 = "invalid_index"; + AlterColumnStatsCommand command4 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName3, columnName, properties); + Assertions.assertThrows(AnalysisException.class, () -> command4.validate(connectContext), + "Index invalid_index not exist in table test_tbl"); + + //test invalid statistics + Map properties2 = new HashMap<>(); + properties2.put("histogram", "invalide_value"); + AlterColumnStatsCommand command5 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties2); + Assertions.assertThrows(AnalysisException.class, () -> command5.validate(connectContext), + "histogram is invalid statistics"); + + //row_count is not exist + Map properties3 = new HashMap<>(); + properties2.put("avg_size", "100000"); + properties2.put("max_size", "100000000"); + AlterColumnStatsCommand command6 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties3); + Assertions.assertThrows(AnalysisException.class, () -> command6.validate(connectContext), + "Set column stats must set row_count. e.g. 'row_count'='5'"); + + //test enable stats + connectContext.getSessionVariable().enableStats = false; + AlterColumnStatsCommand command7 = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties); + Assertions.assertThrows(UserException.class, () -> command7.validate(connectContext), + "Analyze function is forbidden, you should add `enable_stats=true` in your FE conf file"); + } + + @Test + void alterColumnStats_testValidateNoPrivilege() throws IOException { + alterColumnStats_runBefore(); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(false).when(spyAcm).checkTblPriv(Mockito.nullable(ConnectContext.class), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), + Mockito.any(PrivPredicate.class)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + TableNameInfo tableNameInfo = + new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL2_NAME); + PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, + ImmutableList.of(CatalogMocker.TEST_PARTITION1_NAME)); + + String indexName = "index1"; + String columnName = "k1"; + Map properties = new HashMap<>(); + + AlterColumnStatsCommand command = new AlterColumnStatsCommand(tableNameInfo, partitionNamesInfo, indexName, columnName, properties); + connectContext.getSessionVariable().enableStats = true; + Assertions.assertThrows(AnalysisException.class, () -> command.validate(connectContext), + "ALTER TABLE STATS command denied to user 'null'@'null' for table 'alter_column_stats_db: test_tbl2'"); + } + + // ------------------------------------------------------------------------- + // from CleanQueryStatsCommandTest + // ------------------------------------------------------------------------- + + public void cleanQueryStats_runBefore() throws IOException { + connectContext = createDefaultCtx(); + env = Env.getCurrentEnv(); + accessControllerManager = env.getAccessManager(); + } + + @Test + public void testAllNormal() throws IOException { + cleanQueryStats_runBefore(); + connectContext.setSkipAuth(true); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(true).when(spyAcm).checkGlobalPriv(Mockito.nullable(ConnectContext.class), Mockito.any(PrivPredicate.class)); + Deencapsulation.setField(env, "accessManager", spyAcm); + CleanQueryStatsCommand command = new CleanQueryStatsCommand(); + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + } + + @Test + public void testDB() throws Exception { + cleanQueryStats_runBefore(); + connectContext.setDatabase("clean_query_stats_db"); + connectContext.setSkipAuth(true); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(true).when(spyAcm).checkDbPriv(Mockito.nullable(ConnectContext.class), Mockito.anyString(), Mockito.anyString(), Mockito.any(PrivPredicate.class)); + Deencapsulation.setField(env, "accessManager", spyAcm); + CleanQueryStatsCommand command = new CleanQueryStatsCommand("clean_query_stats_db"); + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + } + + @Test + public void testTbl() throws Exception { + cleanQueryStats_runBefore(); + createDatabase("clean_query_stats_db"); + createTable("create table clean_query_stats_db.test_tbl\n" + "(k1 int, k2 int)\n" + + "duplicate key(k1)\n" + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" + + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1'); "); + TableNameInfo tableNameInfo = new TableNameInfo("clean_query_stats_db", "test_tbl"); + connectContext.setDatabase("clean_query_stats_db"); + connectContext.setSkipAuth(true); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(true).when(spyAcm).checkTblPriv(Mockito.nullable(ConnectContext.class), Mockito.any(TableNameInfo.class), Mockito.any(PrivPredicate.class)); + Deencapsulation.setField(env, "accessManager", spyAcm); + CleanQueryStatsCommand command = new CleanQueryStatsCommand(tableNameInfo); + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + } + + // ------------------------------------------------------------------------- + // from ShowQueryStatsCommandTest + // ------------------------------------------------------------------------- + + private void showQueryStats_runBefore() throws Exception { + connectContext = createDefaultCtx(); + env = Env.getCurrentEnv(); + accessControllerManager = env.getAccessManager(); + if (!dbInitialized) { + createDatabaseWithSql("CREATE DATABASE IF NOT EXISTS " + CatalogMocker.TEST_DB_NAME); + createTable("CREATE TABLE IF NOT EXISTS " + CatalogMocker.TEST_DB_NAME + "." + CatalogMocker.TEST_TBL_NAME + + " (k1 INT, k2 INT) DISTRIBUTED BY HASH(k1) BUCKETS 1" + + " PROPERTIES ('replication_num' = '1')"); + dbInitialized = true; + } + } + + @Test + public void showQueryStats_testValidateWithPrivilege() throws Exception { + showQueryStats_runBefore(); + connectContext.setSkipAuth(true); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(true).when(spyAcm).checkGlobalPriv( + Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); + Mockito.doReturn(true).when(spyAcm).checkTblPriv( + Mockito.nullable(ConnectContext.class), Mockito.any(TableNameInfo.class), + Mockito.eq(PrivPredicate.SHOW)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + TableNameInfo tableNameInfo = + new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME); + + // normal + ShowQueryStatsCommand command = new ShowQueryStatsCommand(tableNameInfo.getDb(), + tableNameInfo, false, false); + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + } + + @Test + void showQueryStats_testValidateNoPrivilege() throws Exception { + showQueryStats_runBefore(); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(false).when(spyAcm).checkGlobalPriv( + Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); + Mockito.doReturn(false).when(spyAcm).checkTblPriv( + Mockito.nullable(ConnectContext.class), Mockito.anyString(), + Mockito.anyString(), Mockito.anyString(), Mockito.eq(PrivPredicate.SHOW)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + TableNameInfo tableNameInfo = + new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME); + ShowQueryStatsCommand command = new ShowQueryStatsCommand(tableNameInfo.getDb(), + tableNameInfo, false, false); + Assertions.assertThrows(AnalysisException.class, () -> command.validate(connectContext)); + } + + // ------------------------------------------------------------------------- + // from ShowTabletsFromTableCommandTest + // ------------------------------------------------------------------------- + + private void showTabletsFromTable_runBefore() throws IOException { + connectContext = createDefaultCtx(); + env = Env.getCurrentEnv(); + accessControllerManager = env.getAccessManager(); + } + + @Test + public void showTabletsFromTable_testValidateWithPrivilege() throws Exception { + showTabletsFromTable_runBefore(); + connectContext.setSkipAuth(true); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(true).when(spyAcm).checkGlobalPriv( + Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); + Mockito.doReturn(true).when(spyAcm).checkTblPriv( + Mockito.nullable(ConnectContext.class), Mockito.anyString(), + Mockito.anyString(), Mockito.anyString(), Mockito.eq(PrivPredicate.ADMIN)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + Expression version = new UnboundSlot("version"); + + Expression whereClauseNormal = new EqualTo(version, new IntegerLiteral(2)); + + List orderKeysNormal = new ArrayList<>(); + orderKeysNormal.add(new OrderKey(version, true, true)); + + TableNameInfo tableNameInfo = + new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME); + PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, + ImmutableList.of(CatalogMocker.TEST_SINGLE_PARTITION_NAME)); + + // normal + ShowTabletsFromTableCommand command = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, + whereClauseNormal, orderKeysNormal, 5, 0); + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + + // where clause error + Expression error = new UnboundSlot("error"); + Expression whereClauseError = new EqualTo(error, new IntegerLiteral(2)); + + ShowTabletsFromTableCommand command2 = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, + whereClauseError, orderKeysNormal, 5, 0); + Assertions.assertThrows(AnalysisException.class, () -> command2.validate(connectContext)); + + // where clause contains or + Expression backendId = new UnboundSlot("BackendId"); + Expression whereClauseOr = new Or( + new EqualTo(version, new IntegerLiteral(2)), + new EqualTo(backendId, new IntegerLiteral(2))); + + ShowTabletsFromTableCommand command3 = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, + whereClauseOr, orderKeysNormal, 5, 0); + Assertions.assertThrows(AnalysisException.class, () -> command3.validate(connectContext)); + + // order by error + List orderKeysError = new ArrayList<>(); + orderKeysError.add(new OrderKey(error, true, true)); + + ShowTabletsFromTableCommand command4 = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, + whereClauseNormal, orderKeysError, 5, 0); + Assertions.assertThrows(AnalysisException.class, () -> command4.validate(connectContext)); + } + + @Test + void showTabletsFromTable_testValidateNoPrivilege() throws Exception { + showTabletsFromTable_runBefore(); + AccessControllerManager spyAcm = Mockito.spy(accessControllerManager); + Mockito.doReturn(false).when(spyAcm).checkGlobalPriv( + Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN)); + Mockito.doReturn(false).when(spyAcm).checkTblPriv( + Mockito.nullable(ConnectContext.class), Mockito.anyString(), + Mockito.anyString(), Mockito.anyString(), Mockito.eq(PrivPredicate.ADMIN)); + Deencapsulation.setField(env, "accessManager", spyAcm); + + Expression version = new UnboundSlot("version"); + + Expression whereClauseNormal = new EqualTo(version, new IntegerLiteral(2)); + + List orderKeysNormal = new ArrayList<>(); + orderKeysNormal.add(new OrderKey(version, true, true)); + + TableNameInfo tableNameInfo = + new TableNameInfo(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME); + PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false, + ImmutableList.of(CatalogMocker.TEST_SINGLE_PARTITION_NAME)); + + ShowTabletsFromTableCommand command = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, + whereClauseNormal, orderKeysNormal, 5, 0); + Assertions.assertThrows(AnalysisException.class, () -> command.validate(connectContext)); + } +}