+ * This is an emergency recovery tool for debugging when a Cassandra instance cannot + * start due to cluster metadata issues. It reads the local_metadata_log and metadata_snapshots + * tables from the system keyspace to reconstruct and display the cluster metadata state. + *
+ * NOTE: This tool is for offline use only. Do not run on a live cluster. + *
+ * Usage: + *
+ * # Dump cluster metadata as binary (default) + * offlineclustermetadatadump metadata --data-dir /path/to/data + * + * # Dump cluster metadata as toString output + * offlineclustermetadatadump metadata --data-dir /path/to/data --to-string + * + * # Dump local log entries + * offlineclustermetadatadump log --data-dir /path/to/data --from-epoch 1 --to-epoch 50 + * + * # Dump distributed log (CMS nodes) + * offlineclustermetadatadump distributed-log --data-dir /path/to/data + *+ */ +@Command(name = "offlineclustermetadatadump", +mixinStandardHelpOptions = true, +description = "Offline tool to dump cluster metadata from local SSTables. NOTE: For offline use only.", +subcommands = { OfflineClusterMetadataDump.MetadataCommand.class, OfflineClusterMetadataDump.LogCommand.class, OfflineClusterMetadataDump.DistributedLogCommand.class }) +public class OfflineClusterMetadataDump implements Runnable +{ + private static final Output output = Output.CONSOLE; + + public static void main(String... args) + { + Util.initDatabaseDescriptor(); + + CommandLine cli = new CommandLine(OfflineClusterMetadataDump.class).setExecutionExceptionHandler((ex, cmd, parseResult) -> { + err(ex); + return 2; + }); + int status = cli.execute(args); + System.exit(status); + } + + protected static void err(Throwable e) + { + output.err.println("error: " + e.getMessage()); + output.err.println("-- StackTrace --"); + output.err.println(getStackTraceAsString(e)); + } + + @Override + public void run() + { + CommandLine.usage(this, output.out); + } + + /** + * Base class with common options and methods shared by all subcommands. + */ + static abstract class BaseCommand implements Runnable + { + @Option(names = { "-d", "--data-dir" }, description = "Data directory containing system keyspace") + public String dataDir; + + @Option(names = { "-s", "--sstables" }, description = "Path to SSTable directory for metadata tables (can be specified multiple times)", arity = "1..*") + public List
+ * These tests write entries directly to the system keyspace storage and then + * call OfflineClusterMetadataDump.BaseCommand.getLogState() directly to verify gap detection behavior. + */ +public class OfflineClusterMetadataDumpIntegrationTest extends OfflineToolUtils +{ + private SystemKeyspaceStorage storage; + private MetadataSnapshots snapshotManager; + + @BeforeClass + public static void setupClass() throws IOException + { + DatabaseDescriptor.daemonInitialization(); + StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance); + ServerTestUtils.prepareServerNoRegister(); + CommitLog.instance.start(); + } + + @Before + public void setup() + { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(SYSTEM_KEYSPACE_NAME, METADATA_LOG); + if (cfs != null) + cfs.truncateBlockingWithoutSnapshot(); + + snapshotManager = new MetadataSnapshots.SystemKeyspaceMetadataSnapshots(); + storage = new SystemKeyspaceStorage(() -> snapshotManager); + } + + private Entry entry(long epoch) + { + return new Entry(new Entry.Id(epoch), + Epoch.create(epoch), + CustomTransformation.make((int) epoch)); + } + + @Test + public void testGapDetectionInEpochs() + { + // Write entries with gap at epoch 3 + storage.append(entry(1)); + storage.append(entry(2)); + storage.append(entry(4)); // Gap: skipping 3 + storage.append(entry(5)); + + TestOutput testOutput = new TestOutput(); + LogState logState = OfflineClusterMetadataDump.BaseCommand.getLogState(storage, snapshotManager, null, testOutput.getOutput()); + String stderr = testOutput.getStderr(); + + // Verify gap is detected and reported + assertThat(stderr).contains("Gap detected"); + assertThat(stderr).contains("expected epoch 3 but found 4"); + + // All epochs should still be in the log state + assertThat(logState.entries).hasSize(4); + assertThat(logState.entries.get(0).epoch.getEpoch()).isEqualTo(1); + assertThat(logState.entries.get(1).epoch.getEpoch()).isEqualTo(2); + assertThat(logState.entries.get(2).epoch.getEpoch()).isEqualTo(4); + assertThat(logState.entries.get(3).epoch.getEpoch()).isEqualTo(5); + } + + @Test + public void testMultipleGapsDetection() + { + // Write entries with multiple gaps: missing 2, 4, 6, 7 + storage.append(entry(1)); + storage.append(entry(3)); // Gap: skipping 2 + storage.append(entry(5)); // Gap: skipping 4 + storage.append(entry(8)); // Gap: skipping 6, 7 + + TestOutput testOutput = new TestOutput(); + LogState logState = OfflineClusterMetadataDump.BaseCommand.getLogState(storage, snapshotManager, null, testOutput.getOutput()); + String stderr = testOutput.getStderr(); + + // Verify multiple gaps are detected + assertThat(stderr).contains("Gap detected"); + assertThat(stderr).contains("expected epoch 2 but found 3"); + assertThat(stderr).contains("expected epoch 4 but found 5"); + assertThat(stderr).contains("expected epoch 6 but found 8"); + + // All available epochs should still be in the log state + assertThat(logState.entries).hasSize(4); + assertThat(logState.entries.get(0).epoch.getEpoch()).isEqualTo(1); + assertThat(logState.entries.get(1).epoch.getEpoch()).isEqualTo(3); + assertThat(logState.entries.get(2).epoch.getEpoch()).isEqualTo(5); + assertThat(logState.entries.get(3).epoch.getEpoch()).isEqualTo(8); + } + + @Test + public void testNoGapsNoWarnings() + { + // No gaps + storage.append(entry(1)); + storage.append(entry(2)); + storage.append(entry(3)); + storage.append(entry(4)); + storage.append(entry(5)); + + TestOutput testOutput = new TestOutput(); + LogState logState = OfflineClusterMetadataDump.BaseCommand.getLogState(storage, snapshotManager, null, testOutput.getOutput()); + String stderr = testOutput.getStderr(); + + // Gap warnings should not appear + assertThat(stderr).doesNotContain("Gap detected"); + assertThat(stderr).doesNotContain("WARNING"); + + // All entries should be in the log state + assertThat(logState.entries).hasSize(5); + for (int i = 0; i < 5; i++) + { + assertThat(logState.entries.get(i).epoch.getEpoch()).isEqualTo(i + 1); + } + } + + @Test + public void testEmptyLogReturnsEmptyState() + { + // Don't write any entries - log is empty + TestOutput testOutput = new TestOutput(); + LogState logState = OfflineClusterMetadataDump.BaseCommand.getLogState(storage, snapshotManager, null, testOutput.getOutput()); + + // Should return empty log state + assertThat(logState.isEmpty()).isTrue(); + assertThat(logState.entries).isEmpty(); + } + + @Test + public void testSingleEntryNoGap() + { + // Single entry at epoch 1 + storage.append(entry(1)); + + TestOutput testOutput = new TestOutput(); + LogState logState = OfflineClusterMetadataDump.BaseCommand.getLogState(storage, snapshotManager, null, testOutput.getOutput()); + String stderr = testOutput.getStderr(); + + // No gap warnings + assertThat(stderr).doesNotContain("Gap detected"); + + // Single entry should be present + assertThat(logState.entries).hasSize(1); + assertThat(logState.entries.get(0).epoch.getEpoch()).isEqualTo(1); + } + + @Test + public void testGapAtBeginning() + { + // Start with epoch 3 instead of 1 - gap at the beginning + storage.append(entry(3)); + storage.append(entry(4)); + storage.append(entry(5)); + + TestOutput testOutput = new TestOutput(); + LogState logState = OfflineClusterMetadataDump.BaseCommand.getLogState(storage, snapshotManager, null, testOutput.getOutput()); + String stderr = testOutput.getStderr(); + + // Should detect gap at beginning (expected 1 but found 3) + assertThat(stderr).contains("Gap detected"); + assertThat(stderr).contains("expected epoch 1 but found 3"); + + // All entries should still be present + assertThat(logState.entries).hasSize(3); + assertThat(logState.entries.get(0).epoch.getEpoch()).isEqualTo(3); + assertThat(logState.entries.get(1).epoch.getEpoch()).isEqualTo(4); + assertThat(logState.entries.get(2).epoch.getEpoch()).isEqualTo(5); + } + + @Test + public void testTargetEpochFilter() + { + // Write entries 1-10 + for (int i = 1; i <= 10; i++) + { + storage.append(entry(i)); + } + + // Get log state up to epoch 5 + TestOutput testOutput = new TestOutput(); + LogState logState = OfflineClusterMetadataDump.BaseCommand.getLogState(storage, snapshotManager, 5L, testOutput.getOutput()); + String stderr = testOutput.getStderr(); + + // No gaps + assertThat(stderr).doesNotContain("Gap detected"); + + // Should only have epochs up to 5 + assertThat(logState.entries).hasSizeLessThanOrEqualTo(5); + for (Entry e : logState.entries) + { + assertThat(e.epoch.getEpoch()).isLessThanOrEqualTo(5); + } + } + + /** + * Helper class to capture output from the gap detection logic. + */ + private static class TestOutput + { + private final ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errStream = new ByteArrayOutputStream(); + private final Output output = new Output(new PrintStream(outStream), new PrintStream(errStream)); + + public Output getOutput() + { + return output; + } + + public String getStderr() + { + return errStream.toString(); + } + } +} diff --git a/test/unit/org/apache/cassandra/tools/OfflineClusterMetadataDumpTest.java b/test/unit/org/apache/cassandra/tools/OfflineClusterMetadataDumpTest.java new file mode 100644 index 000000000000..072268ac64c9 --- /dev/null +++ b/test/unit/org/apache/cassandra/tools/OfflineClusterMetadataDumpTest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tools; + +import org.assertj.core.api.Assertions; +import org.hamcrest.CoreMatchers; +import org.junit.Test; + +import org.apache.cassandra.tools.ToolRunner.ToolResult; + +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * Tests for OfflineClusterMetadataDump tool. + *
+ * Note: This tool requires some initialization (DatabaseDescriptor, Schema) even for help, + * similar to StandaloneJournalUtil and other cluster metadata-related tools. + */ +public class OfflineClusterMetadataDumpTest extends OfflineToolUtils +{ + @Test + public void testMainHelpOption() + { + // Main command help shows subcommands + ToolResult tool = ToolRunner.invokeClass(OfflineClusterMetadataDump.class, "-h"); + String output = tool.getStdout() + tool.getStderr(); + assertThat("Help should show usage", output, CoreMatchers.containsStringIgnoringCase("Usage:")); + assertThat("Help should mention metadata subcommand", output, CoreMatchers.containsStringIgnoringCase("metadata")); + assertThat("Help should mention log subcommand", output, CoreMatchers.containsStringIgnoringCase("log")); + assertThat("Help should mention distributed-log subcommand", output, CoreMatchers.containsStringIgnoringCase("distributed-log")); + } + + @Test + public void testMetadataSubcommandHelpOption() + { + // Metadata subcommand help shows all the options + ToolResult tool = ToolRunner.invokeClass(OfflineClusterMetadataDump.class, "metadata", "-h"); + String output = tool.getStdout() + tool.getStderr(); + + assertThat("Help should show usage", output, CoreMatchers.containsStringIgnoringCase("Usage:")); + Assertions.assertThat(output).containsIgnoringCase("--data-dir"); + Assertions.assertThat(output).containsIgnoringCase("--to-string"); + Assertions.assertThat(output).containsIgnoringCase("--output"); + Assertions.assertThat(output).containsIgnoringCase("--epoch"); + } + + @Test + public void testLogSubcommandHelpOption() + { + // Log subcommand help shows all the options + ToolResult tool = ToolRunner.invokeClass(OfflineClusterMetadataDump.class, "log", "-h"); + String output = tool.getStdout() + tool.getStderr(); + + assertThat("Help should show usage", output, CoreMatchers.containsStringIgnoringCase("Usage:")); + Assertions.assertThat(output).containsIgnoringCase("--data-dir"); + Assertions.assertThat(output).containsIgnoringCase("--from-epoch"); + Assertions.assertThat(output).containsIgnoringCase("--to-epoch"); + } + + @Test + public void testDistributedLogSubcommandHelpOption() + { + // Distributed-log subcommand help shows all the options + ToolResult tool = ToolRunner.invokeClass(OfflineClusterMetadataDump.class, "distributed-log", "-h"); + String output = tool.getStdout() + tool.getStderr(); + + assertThat("Help should show usage", output, CoreMatchers.containsStringIgnoringCase("Usage:")); + Assertions.assertThat(output).containsIgnoringCase("--data-dir"); + Assertions.assertThat(output).containsIgnoringCase("--from-epoch"); + Assertions.assertThat(output).containsIgnoringCase("--to-epoch"); + } + + @Test + public void testWrongArgFailsAndPrintsHelp() + { + ToolResult tool = ToolRunner.invokeClass(OfflineClusterMetadataDump.class, "metadata", "--invalid-option"); + String output = tool.getStdout() + tool.getStderr(); + assertThat("Should mention unknown option", output, CoreMatchers.containsStringIgnoringCase("Unknown")); + assertTrue("Expected non-zero exit code", tool.getExitCode() != 0); + } + + @Test + public void testNonExistentDataDirectory() + { + // When running with a non-existent directory, should fail gracefully + ToolResult tool = ToolRunner.invokeClass(OfflineClusterMetadataDump.class, "metadata", + "--data-dir", "/nonexistent/path/to/data", + "--to-string"); + String output = tool.getStdout() + tool.getStderr(); + // Tool should fail gracefully when directory doesn't exist or no SSTables found + assertTrue("Expected error or no metadata message", + tool.getExitCode() != 0 || + output.toLowerCase().contains("no metadata") || + output.toLowerCase().contains("not found") || + output.toLowerCase().contains("does not exist") || + output.toLowerCase().contains("error")); + } + + @Test + public void testMetadataSubcommandFlags() + { + // Test that --to-string flag is recognized + ToolResult toStringFlag = ToolRunner.invokeClass(OfflineClusterMetadataDump.class, "metadata", "--to-string", "-h"); + String toStringOutput = toStringFlag.getStdout() + toStringFlag.getStderr(); + assertThat("Should show help with --to-string", toStringOutput, CoreMatchers.containsStringIgnoringCase("Usage:")); + + // Test that -o/--output flag is recognized + ToolResult outputFlag = ToolRunner.invokeClass(OfflineClusterMetadataDump.class, "metadata", "-o", "/tmp/test.dump", "-h"); + String outputOutput = outputFlag.getStdout() + outputFlag.getStderr(); + assertThat("Should show help with -o", outputOutput, CoreMatchers.containsStringIgnoringCase("Usage:")); + + ToolResult outputLongFlag = ToolRunner.invokeClass(OfflineClusterMetadataDump.class, "metadata", "--output", "/tmp/test.dump", "-h"); + String outputLongOutput = outputLongFlag.getStdout() + outputLongFlag.getStderr(); + assertThat("Should show help with --output", outputLongOutput, CoreMatchers.containsStringIgnoringCase("Usage:")); + + // Test --epoch flag + ToolResult epochTool = ToolRunner.invokeClass(OfflineClusterMetadataDump.class, "metadata", "--epoch", "100", "-h"); + String epochOutput = epochTool.getStdout() + epochTool.getStderr(); + assertThat("--epoch flag should be recognized", epochOutput, CoreMatchers.containsStringIgnoringCase("Usage:")); + } + + @Test + public void testLogSubcommandEpochFilterFlags() + { + // Test that epoch filter flags are recognized on log subcommand + ToolResult fromTool = ToolRunner.invokeClass(OfflineClusterMetadataDump.class, "log", "--from-epoch", "50", "-h"); + String fromOutput = fromTool.getStdout() + fromTool.getStderr(); + assertThat("--from-epoch flag should be recognized", fromOutput, CoreMatchers.containsStringIgnoringCase("Usage:")); + + ToolResult toTool = ToolRunner.invokeClass(OfflineClusterMetadataDump.class, "log", "--to-epoch", "150", "-h"); + String toOutput = toTool.getStdout() + toTool.getStderr(); + assertThat("--to-epoch flag should be recognized", toOutput, CoreMatchers.containsStringIgnoringCase("Usage:")); + } + + @Test + public void testVerboseAndDebugFlags() + { + // Test verbose flags on metadata subcommand + ToolResult verboseShort = ToolRunner.invokeClass(OfflineClusterMetadataDump.class, "metadata", "-v", "-h"); + String verboseShortOutput = verboseShort.getStdout() + verboseShort.getStderr(); + assertThat("-v flag should be recognized", verboseShortOutput, CoreMatchers.containsStringIgnoringCase("Usage:")); + + ToolResult verboseLong = ToolRunner.invokeClass(OfflineClusterMetadataDump.class, "metadata", "--verbose", "-h"); + String verboseLongOutput = verboseLong.getStdout() + verboseLong.getStderr(); + assertThat("--verbose flag should be recognized", verboseLongOutput, CoreMatchers.containsStringIgnoringCase("Usage:")); + + // Test debug flag + ToolResult debug = ToolRunner.invokeClass(OfflineClusterMetadataDump.class, "metadata", "--debug", "-h"); + String debugOutput = debug.getStdout() + debug.getStderr(); + assertThat("--debug flag should be recognized", debugOutput, CoreMatchers.containsStringIgnoringCase("Usage:")); + } + + @Test + public void testPartitionerFlag() + { + // Test partitioner flags on metadata subcommand + ToolResult shortFlag = ToolRunner.invokeClass(OfflineClusterMetadataDump.class, "metadata", + "-p", "org.apache.cassandra.dht.Murmur3Partitioner", "-h"); + String shortOutput = shortFlag.getStdout() + shortFlag.getStderr(); + assertThat("-p flag should be recognized", shortOutput, CoreMatchers.containsStringIgnoringCase("Usage:")); + + ToolResult longFlag = ToolRunner.invokeClass(OfflineClusterMetadataDump.class, "metadata", + "--partitioner", "org.apache.cassandra.dht.Murmur3Partitioner", "-h"); + String longOutput = longFlag.getStdout() + longFlag.getStderr(); + assertThat("--partitioner flag should be recognized", longOutput, CoreMatchers.containsStringIgnoringCase("Usage:")); + } +} diff --git a/tools/bin/offlineclustermetadatadump b/tools/bin/offlineclustermetadatadump new file mode 100755 index 000000000000..175b126d750b --- /dev/null +++ b/tools/bin/offlineclustermetadatadump @@ -0,0 +1,49 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "x$CASSANDRA_INCLUDE" = "x" ]; then + # Locations (in order) to use when searching for an include file. + for include in "`dirname "$0"`/cassandra.in.sh" \ + "$HOME/.cassandra.in.sh" \ + /usr/share/cassandra/cassandra.in.sh \ + /usr/local/share/cassandra/cassandra.in.sh \ + /opt/cassandra/cassandra.in.sh; do + if [ -r "$include" ]; then + . "$include" + break + fi + done +elif [ -r "$CASSANDRA_INCLUDE" ]; then + . "$CASSANDRA_INCLUDE" +fi + +if [ -z "$CLASSPATH" ]; then + echo "You must set the CLASSPATH var" >&2 + exit 1 +fi + +if [ "x$MAX_HEAP_SIZE" = "x" ]; then + MAX_HEAP_SIZE="256M" +fi + +"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \ + -Dcassandra.storagedir="$cassandra_storagedir" \ + -Dlogback.configurationFile=logback-tools.xml \ + org.apache.cassandra.tools.OfflineClusterMetadataDump "$@" + +# vi:ai sw=4 ts=4 tw=0 et