Skip to content

fix(load): validate TsFile before async move #18531

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
luoluoyuyu wants to merge 2 commits into
base: master
Choose a base branch
from
Open

fix(load): validate TsFile before async move #18531

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7304c13
fix(load): validate TsFile before async move
luoluoyuyu Aug 27, 2026
44a8818
fix(load): report invalid tsfile in async pipe path
luoluoyuyu Aug 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileAuthIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import org.junit.runner.RunWith;

import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.sql.Connection;
import java.sql.ResultSet;
Expand Down Expand Up @@ -180,6 +181,34 @@ public void testAsyncLoadShouldCheckWriteDataPermissionWithStoredUser() throws E
assertCountEventually(10, TimeUnit.SECONDS.toMillis(60));
}

@Test
public void testAsyncLoadShouldNotMoveFileWithoutTsFileSuffix() throws Exception {
assertAsyncLoadDoesNotMoveInvalidTsFile(new File(tmpDir, "not-a-tsfile.txt"), "Can not find");
}

@Test
public void testAsyncLoadShouldNotMoveInvalidTsFile() throws Exception {
assertAsyncLoadDoesNotMoveInvalidTsFile(new File(tmpDir, "invalid.tsfile"), "Loading file");
}

private static void assertAsyncLoadDoesNotMoveInvalidTsFile(
final File sourceFile, final String expectedErrorMessage) throws Exception {
final String originalContent = "ordinary file content";
Files.write(sourceFile.toPath(), originalContent.getBytes(StandardCharsets.UTF_8));

assertNonQueryTestFail(
String.format(
"load \"%s\" with ('async'='true', 'on-success'='delete')",
sourceFile.getAbsolutePath()),
expectedErrorMessage);

Assert.assertTrue(
"Non-TsFile source must remain in its original location", sourceFile.isFile());
Assert.assertEquals(
originalContent,
new String(Files.readAllBytes(sourceFile.toPath()), StandardCharsets.UTF_8));
}

private static void prepareSchemaAndTsFile(final File tsFile) throws Exception {
prepareSchema(MEASUREMENT.getType());
generateTsFile(tsFile);
Expand Down
2 changes: 1 addition & 1 deletion ...rc/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ private static List<File> processTsFile(
}

final List<File> tsFiles = new ArrayList<>();
if (file.isFile()) {
if (file.isFile() && file.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) {
tsFiles.add(file);
} else {
if (file.listFiles() == null) {
Expand Down
38 changes: 38 additions & 0 deletions iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/util/LoadUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.iotdb.commons.utils.RetryUtils;
import org.apache.iotdb.db.auth.AuthorityChecker;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
import org.apache.iotdb.db.protocol.session.IClientSession;
import org.apache.iotdb.db.protocol.session.SessionManager;
Expand All @@ -35,7 +36,9 @@
import org.apache.iotdb.db.storageengine.load.active.ActiveLoadPathHelper;
import org.apache.iotdb.db.storageengine.load.disk.ILoadDiskSelector;

import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.read.TsFileSequenceReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -67,6 +70,13 @@ public static boolean loadTsFileAsyncToActiveDir(
}

try {
// Validate the complete batch before moving any source file. Otherwise a later malformed
// file could leave earlier files queued in the active-load directory.
for (final File file : tsFiles) {
if (file != null && !isValidTsFile(file)) {
return false;
}
}
for (File file : tsFiles) {
if (!loadTsFilesToActiveDir(loadAttributes, file, isDeleteAfterLoad)) {
return false;
Expand Down Expand Up @@ -118,6 +128,11 @@ private static boolean loadTsFilesToActiveDir(
return true;
}

// Validate before moving the source so ordinary or malformed files remain in place.
if (!isValidTsFile(file)) {

@Caideyipi Caideyipi Aug 27, 2026 •
edited
Loading

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Validate the complete batch before moving any file

loadTsFileAsyncToActiveDir calls this method once per *.tsfile. If a directory contains a valid *.tsfile followed by a malformed *.tsfile, the valid file has already been copied/linked into the active-load directory before this call returns false for the malformed one. doAsyncLoad then falls back to normal analysis, but the first file remains queued; with on-success='delete', its source may already be deleted as well. This can cause partial ingestion and unsafe retries. Validate all files before starting the transfer, or roll back every transfer when a later validation fails.

return false;
}

final File targetFilePath;
try {
targetFilePath =
Expand Down Expand Up @@ -145,6 +160,19 @@ private static boolean loadTsFilesToActiveDir(
return true;
}

private static boolean isValidTsFile(final File file) {

@Caideyipi Caideyipi Aug 27, 2026 •
edited
Loading

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Apply this validation to the Pipe async path too

IoTDBDataNodeReceiver.loadTsFileAsync calls LoadUtil.loadFilesToActiveDir, but isValidTsFile is only invoked from loadTsFilesToActiveDir. The Pipe seal path therefore still transfers and deletes the main TsFile even when its magic is invalid, then returns SUCCESS_STATUS; the active loader fails later, after the sender has already been acknowledged. Identify the main .tsfile entry and apply the same validation in loadFilesToActiveDir before transferFilesToActiveDir (while preserving valid .resource/.mods sidecars), and add coverage for this path.

if (!file.isFile() || !file.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) {
return false;
}
try (final TsFileSequenceReader reader =
new TsFileSequenceReader(file.getAbsolutePath(), false)) {
return TSFileConfig.MAGIC_STRING.equals(reader.readHeadMagic())
&& TSFileConfig.MAGIC_STRING.equals(reader.readTailMagic());
} catch (Exception e) {
return false;
}
}

private static Map<String, String> appendCurrentUserIfAbsent(
final Map<String, String> loadAttributes) {
final Map<String, String> attributes =
Expand Down Expand Up @@ -192,6 +220,16 @@ public static boolean loadFilesToActiveDir(
for (final String file : files) {
sourceFiles.add(new File(file));
}
// The main TsFile must be valid before any TsFile or sidecar is transferred. Pipe acknowledges
// this method immediately, so deferring validation to the active loader is too late.
for (final File sourceFile : sourceFiles) {
if (isTsFile(sourceFile) && !isValidTsFile(sourceFile)) {
throw new IOException(
String.format(
DataNodeQueryMessages.THE_FILE_S_IS_NOT_A_VALID_TSFILE_PLEASE_CHECK_THE_INPUT_FILE,
sourceFile.getAbsolutePath()));
}
}
sourceFiles.sort(Comparator.comparing(LoadUtil::isTsFile));
transferFilesToActiveDir(targetDir, sourceFiles, isDeleteAfterLoad);
return true;
Expand Down
59 changes: 59 additions & 0 deletions ...core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/util/LoadUtilTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,14 @@

package org.apache.iotdb.db.storageengine.load.util;

import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;

import org.apache.tsfile.write.writer.TsFileIOWriter;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
Expand All @@ -39,6 +43,8 @@

public class LoadUtilTest {

private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
private String[] originalListeningDirs;
private File tempDir;
private File sourceDir;
private File targetDir;
Expand All @@ -50,13 +56,58 @@ public void setUp() throws Exception {
targetDir = new File(tempDir, "target");
Assert.assertTrue(sourceDir.mkdirs());
Assert.assertTrue(targetDir.mkdirs());
originalListeningDirs = config.getLoadActiveListeningDirs();
config.setLoadActiveListeningDirs(new String[] {targetDir.getAbsolutePath()});
LoadUtil.updateLoadDiskSelector();
}

@After
public void tearDown() {
config.setLoadActiveListeningDirs(originalListeningDirs);
LoadUtil.updateLoadDiskSelector();
deleteRecursively(tempDir);
}

@Test
public void testAsyncLoadValidatesCompleteBatchBeforeTransfer() throws Exception {
final File validTsFile = createCompletedTsFile("valid.tsfile");
final File invalidTsFile = new File(sourceDir, "invalid.tsfile");
Files.write(invalidTsFile.toPath(), "invalid".getBytes(StandardCharsets.UTF_8));

Assert.assertFalse(
LoadUtil.loadTsFileAsyncToActiveDir(Arrays.asList(validTsFile, invalidTsFile), null, true));
Assert.assertTrue(validTsFile.exists());
Assert.assertTrue(invalidTsFile.exists());
Assert.assertEquals(0, targetDir.listFiles().length);
}

@Test
public void testPipeAsyncLoadReportsInvalidTsFileAndKeepsSources() throws Exception {
final File invalidTsFile = new File(sourceDir, "invalid.tsfile");
final File resourceFile =
new File(invalidTsFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX);
Files.write(invalidTsFile.toPath(), "invalid".getBytes(StandardCharsets.UTF_8));
Files.write(resourceFile.toPath(), "resource".getBytes(StandardCharsets.UTF_8));

try {
LoadUtil.loadFilesToActiveDir(
null,
Arrays.asList(resourceFile.getAbsolutePath(), invalidTsFile.getAbsolutePath()),
true);
Assert.fail("Expected invalid TsFile error");
} catch (final IOException e) {
Assert.assertEquals(
String.format(
DataNodeQueryMessages.THE_FILE_S_IS_NOT_A_VALID_TSFILE_PLEASE_CHECK_THE_INPUT_FILE,
invalidTsFile.getAbsolutePath()),
e.getMessage());
}

Assert.assertTrue(invalidTsFile.exists());
Assert.assertTrue(resourceFile.exists());
Assert.assertEquals(0, targetDir.listFiles().length);
}

@Test
public void testTransferFilesKeepsSameNamedGroupsIsolatedAndDeletesSourcesAfterHandoff()
throws Exception {
Expand Down Expand Up @@ -126,6 +177,14 @@ private List<File> createTsFileAndCompanions() throws Exception {
return sourceFiles;
}

private File createCompletedTsFile(final String fileName) throws Exception {
final File tsFile = new File(sourceDir, fileName);
try (final TsFileIOWriter writer = new TsFileIOWriter(tsFile)) {
writer.endFile();
}
return tsFile;
}

private static void deleteRecursively(final File file) {
if (file == null || !file.exists()) {
return;
Expand Down
Loading