From 964f60b8d04a07d9044f0546bd2b08e7a721c2f5 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sun, 26 Jul 2026 13:31:54 -0300 Subject: [PATCH 1/4] SOLR-18320: Use Configsets V2 API instead of direct ZooKeeper access when uploading a configset in bin/solr create. createCollection/createCore now just take a SolrClient; the only remaining ZK-specific step is resolving a live node's URL when only --zk-host is given. Co-Authored-By: Claude Sonnet 5 --- ...OLR-18320-skip-zk-for-configset-upload.yml | 7 ++ .../java/org/apache/solr/cli/CreateTool.java | 96 ++++++++----------- .../org/apache/solr/cli/CreateToolTest.java | 20 ++++ 3 files changed, 67 insertions(+), 56 deletions(-) create mode 100644 changelog/unreleased/SOLR-18320-skip-zk-for-configset-upload.yml diff --git a/changelog/unreleased/SOLR-18320-skip-zk-for-configset-upload.yml b/changelog/unreleased/SOLR-18320-skip-zk-for-configset-upload.yml new file mode 100644 index 000000000000..fa9d69fa2f78 --- /dev/null +++ b/changelog/unreleased/SOLR-18320-skip-zk-for-configset-upload.yml @@ -0,0 +1,7 @@ +title: bin/solr create no longer requires a direct ZooKeeper connection to upload a configset; it now uses the Configsets V2 API. +type: changed +authors: + - name: Eric Pugh +links: + - name: SOLR-18320 + url: https://issues.apache.org/jira/browse/SOLR-18320 diff --git a/solr/core/src/java/org/apache/solr/cli/CreateTool.java b/solr/core/src/java/org/apache/solr/cli/CreateTool.java index 6b7c6549503f..4ca4d8733454 100644 --- a/solr/core/src/java/org/apache/solr/cli/CreateTool.java +++ b/solr/core/src/java/org/apache/solr/cli/CreateTool.java @@ -16,13 +16,18 @@ */ package org.apache.solr.cli; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.BasicFileAttributes; import java.util.Locale; -import java.util.Set; -import java.util.concurrent.TimeUnit; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; @@ -30,14 +35,11 @@ import org.apache.solr.cli.CommonCLIOptions.DefaultValues; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; import org.apache.solr.client.solrj.request.CollectionsApi; +import org.apache.solr.client.solrj.request.ConfigsetsApi; import org.apache.solr.client.solrj.request.CoresApi; import org.apache.solr.client.solrj.request.SystemInfoRequest; import org.apache.solr.client.solrj.response.SystemInfoResponse; -import org.apache.solr.cloud.ZkConfigSetService; -import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.EnvUtils; import org.apache.solr.core.ConfigSetService; @@ -125,7 +127,7 @@ public Options getOptions() { public void runImpl(CommandLine cli) throws Exception { try (var solrClient = CLIUtils.getSolrClient(cli)) { if (CLIUtils.isCloudMode(solrClient)) { - createCollection(cli); + createCollection(cli, solrClient); } else { createCore(cli, solrClient); } @@ -193,29 +195,7 @@ protected void createCore(CommandLine cli, SolrClient solrClient) throws Excepti } } - protected void createCollection(CommandLine cli) throws Exception { - var builder = - new HttpJettySolrClient.Builder() - .withIdleTimeout(30, TimeUnit.SECONDS) - .withConnectionTimeout(15, TimeUnit.SECONDS) - .withKeyStoreReloadInterval(-1, TimeUnit.SECONDS) - .withOptionalBasicAuthCredentials( - cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION)); - String zkHost = CLIUtils.getZkHost(cli); - echoIfVerbose("Connecting to ZooKeeper at " + zkHost); - var zkSolrConnection = CloudSolrClient.CloudSolrClientConnection.parse(zkHost); - if (!zkSolrConnection.isZookeeper()) { - throw new IOException( - String.format( - Locale.ROOT, "Expected ZooKeeper connection string, but got: '%s'.", zkHost)); - } - try (var cloudSolrClient = CLIUtils.getCloudSolrClient(zkSolrConnection, builder)) { - createCollection(cloudSolrClient, cli); - } - } - - protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli) - throws Exception { + protected void createCollection(CommandLine cli, SolrClient solrClient) throws Exception { String collectionName = cli.getOptionValue(COLLECTION_NAME_OPTION); final String solrInstallDir = EnvUtils.getProperty("solr.install.dir"); @@ -226,42 +206,25 @@ protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli ensureConfDirExists(solrInstallDirPath, confDirPath); printDefaultConfigsetWarningIfNecessary(cli); - Set liveNodes = cloudSolrClient.getClusterState().getLiveNodes(); - if (liveNodes.isEmpty()) - throw new IllegalStateException( - "No live nodes found! Cannot create a collection until " - + "there is at least 1 live node in the cluster."); - - String solrUrl; - if (CLIUtils.hasConnectionOption(cli)) { - solrUrl = CLIUtils.normalizeSolrUrl(cli); - } else { - String firstLiveNode = liveNodes.iterator().next(); - solrUrl = ZkStateReader.from(cloudSolrClient).getBaseUrlForNodeName(firstLiveNode); - } + String solrUrl = CLIUtils.normalizeSolrUrl(cli); // build a URL to create the collection int numShards = cli.getParsedOptionValue(SHARDS_OPTION, 1); int replicationFactor = cli.getParsedOptionValue(REPLICATION_FACTOR_OPTION, 1); - boolean configExistsInZk = + boolean configExists = confName != null && !confName.trim().isEmpty() - && ZkStateReader.from(cloudSolrClient).getZkClient().exists("/configs/" + confName); + && new ConfigsetsApi.ListConfigSet().process(solrClient).configSets.contains(confName); - if (configExistsInZk) { + if (configExists) { echo("Re-using existing configuration directory " + confName); } else { // if (confdir != null && !confdir.trim().isEmpty()) { if (confName == null || confName.trim().isEmpty()) { confName = collectionName; } - // TODO: This should be done using the configSet API. This would let us remove - // the direct dependency on ZooKeeper APIs. Unlike the bin/solr zk comamnds that - // work directly with ZooKeeper. final Path configsetsDirPath = CLIUtils.getConfigSetsDir(solrInstallDirPath); - ConfigSetService configSetService = - new ZkConfigSetService(ZkStateReader.from(cloudSolrClient).getZkClient()); Path confPath = ConfigSetService.getConfigsetPath(confDir, configsetsDirPath.toString()); echoIfVerbose( @@ -269,10 +232,11 @@ protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli + confPath.toAbsolutePath() + " for config " + confName - + " to ZooKeeper at " - + cloudSolrClient.getClusterStateProvider().getQuorumHosts()); - // We will trust the config since we have the Zookeeper Address - configSetService.uploadConfig(confName, confPath); + + " using the Configsets V2 API"); + var uploadReq = + new ConfigsetsApi.UploadConfigSet( + confName, new ByteArrayInputStream(zipConfigSet(confPath))); + uploadReq.process(solrClient); } // since creating a collection is a heavy-weight operation, check for existence first @@ -293,7 +257,7 @@ protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli req.setConfig(confName); req.setNumShards(numShards); req.setReplicationFactor(replicationFactor); - var response = req.process(cloudSolrClient); + var response = req.process(solrClient); echoIfVerbose(response); } catch (SolrServerException sse) { throw new Exception( @@ -314,6 +278,26 @@ protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli echo(endMessage); } + /** Zips the contents of a configset directory for upload via the Configsets V2 API. */ + private static byte[] zipConfigSet(Path confPath) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { + Files.walkFileTree( + confPath, + new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + zipOut.putNextEntry(new ZipEntry(confPath.relativize(file).toString())); + Files.copy(file, zipOut); + zipOut.closeEntry(); + return FileVisitResult.CONTINUE; + } + }); + } + return baos.toByteArray(); + } + private Path getFullConfDir(Path solrInstallDir, Path confDirName) { return CLIUtils.getConfigSetsDir(solrInstallDir).resolve(confDirName); } diff --git a/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java b/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java index 74c9a6411668..b95692da2444 100644 --- a/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java +++ b/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java @@ -52,4 +52,24 @@ public void testCreateCollectionWithBasicAuth() throws Exception { assertEquals(0, CLITestHelper.runTool(args, CreateTool.class)); } + + @Test + public void testCreateCollectionUploadsNewConfigSet() throws Exception { + String[] args = { + "create", + "-c", + "testCreateCollectionUploadsNewConfigSet", + "-d", + configset("cloud-minimal").toString(), + "-n", + "cloud-minimal-uploaded", + "-z", + cluster.getZkClient().getZkServerAddress(), + "--credentials", + SecurityJson.USER_PASS, + "--verbose" + }; + + assertEquals(0, CLITestHelper.runTool(args, CreateTool.class)); + } } From 20f76414b70324868dc4106fd6ee393984c71e9e Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Tue, 28 Jul 2026 13:40:03 -0300 Subject: [PATCH 2/4] don't tie commment to specific api, but still leave in why we have this here, its for uploading the configset. --- solr/core/src/java/org/apache/solr/cli/CreateTool.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/cli/CreateTool.java b/solr/core/src/java/org/apache/solr/cli/CreateTool.java index 4ca4d8733454..e9f4c82cb099 100644 --- a/solr/core/src/java/org/apache/solr/cli/CreateTool.java +++ b/solr/core/src/java/org/apache/solr/cli/CreateTool.java @@ -278,7 +278,7 @@ protected void createCollection(CommandLine cli, SolrClient solrClient) throws E echo(endMessage); } - /** Zips the contents of a configset directory for upload via the Configsets V2 API. */ + /** Zips the contents of a configset directory for upload. */ private static byte[] zipConfigSet(Path confPath) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { From 7fc5b08b43e0ab40406cf4244d4f2c0859d6b678 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Tue, 28 Jul 2026 13:50:52 -0300 Subject: [PATCH 3/4] Tighten up logic on what gets put into a configset. --- .../java/org/apache/solr/cli/CreateTool.java | 39 ++++++++++++++-- .../org/apache/solr/cli/CreateToolTest.java | 45 +++++++++++++++++++ 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cli/CreateTool.java b/solr/core/src/java/org/apache/solr/cli/CreateTool.java index e9f4c82cb099..2737a1d2a3ed 100644 --- a/solr/core/src/java/org/apache/solr/cli/CreateTool.java +++ b/solr/core/src/java/org/apache/solr/cli/CreateTool.java @@ -278,17 +278,50 @@ protected void createCollection(CommandLine cli, SolrClient solrClient) throws E echo(endMessage); } - /** Zips the contents of a configset directory for upload. */ - private static byte[] zipConfigSet(Path confPath) throws IOException { + /** + * Zips the contents of a configset directory for upload. + * + *

Mirrors the hidden-file skipping, directory-entry, and forbidden-file-type logic in {@link + * org.apache.solr.handler.configsets.DownloadConfigSet#zipConfigSet}. + */ + static byte[] zipConfigSet(Path confPath) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { Files.walkFileTree( confPath, new SimpleFileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) + throws IOException { + if (Files.isHidden(dir)) { + return FileVisitResult.SKIP_SUBTREE; + } + String dirName = confPath.relativize(dir).toString().replace('\\', '/'); + if (!dirName.isEmpty()) { + if (!dirName.endsWith("/")) { + dirName += "/"; + } + zipOut.putNextEntry(new ZipEntry(dirName)); + zipOut.closeEntry(); + } + return FileVisitResult.CONTINUE; + } + @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - zipOut.putNextEntry(new ZipEntry(confPath.relativize(file).toString())); + if (Files.isHidden(file)) { + return FileVisitResult.CONTINUE; + } + String filename = file.getFileName().toString(); + if (ConfigSetService.isFileForbiddenInConfigSets(filename)) { + throw new IOException( + "The file type provided for upload, '" + + filename + + "', is forbidden for use in uploading configsets."); + } + String entryName = confPath.relativize(file).toString().replace('\\', '/'); + zipOut.putNextEntry(new ZipEntry(entryName)); Files.copy(file, zipOut); zipOut.closeEntry(); return FileVisitResult.CONTINUE; diff --git a/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java b/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java index b95692da2444..13206286145f 100644 --- a/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java +++ b/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java @@ -17,6 +17,14 @@ package org.apache.solr.cli; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.util.SecurityJson; import org.junit.BeforeClass; @@ -71,5 +79,42 @@ public void testCreateCollectionUploadsNewConfigSet() throws Exception { }; assertEquals(0, CLITestHelper.runTool(args, CreateTool.class)); + assertTrue(cluster.getZkClient().exists("/configs/cloud-minimal-uploaded")); + } + + @Test + public void testZipConfigSetSkipsHiddenFilesAndIncludesDirectoryEntries() throws Exception { + Path confDir = createTempDir("zipConfigSetTest"); + Files.writeString(confDir.resolve("solrconfig.xml"), ""); + Files.writeString(confDir.resolve(".hidden-file"), "should not be zipped"); + Path langDir = Files.createDirectory(confDir.resolve("lang")); + Files.writeString(langDir.resolve("stopwords.txt"), "the\na\n"); + Path hiddenDir = Files.createDirectory(confDir.resolve(".hiddenDir")); + Files.writeString(hiddenDir.resolve("nope.txt"), "should not be zipped either"); + + byte[] zipBytes = CreateTool.zipConfigSet(confDir); + + Set entryNames = new HashSet<>(); + try (ZipInputStream zipIn = new ZipInputStream(new ByteArrayInputStream(zipBytes))) { + ZipEntry entry; + while ((entry = zipIn.getNextEntry()) != null) { + entryNames.add(entry.getName()); + } + } + + assertTrue(entryNames.contains("solrconfig.xml")); + assertTrue(entryNames.contains("lang/")); + assertTrue(entryNames.contains("lang/stopwords.txt")); + assertFalse(entryNames.contains(".hidden-file")); + assertTrue(entryNames.stream().noneMatch(name -> name.startsWith(".hiddenDir"))); + } + + @Test + public void testZipConfigSetRejectsForbiddenFileType() throws Exception { + Path confDir = createTempDir("zipConfigSetForbiddenTest"); + Files.writeString(confDir.resolve("evil.jar"), "not really a jar"); + + IOException thrown = expectThrows(IOException.class, () -> CreateTool.zipConfigSet(confDir)); + assertTrue(thrown.getMessage().contains("forbidden")); } } From adda596e0acd341e910f51aa20b8d6df33b484d8 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Tue, 28 Jul 2026 14:00:40 -0300 Subject: [PATCH 4/4] Pull all the various configset zipping methods into one helper. --- .../java/org/apache/solr/cli/CreateTool.java | 56 ++-------------- .../apache/solr/core/ConfigSetService.java | 65 +++++++++++++++++++ .../handler/configsets/DownloadConfigSet.java | 45 +------------ .../SchemaDesignerConfigSetHelper.java | 44 +------------ 4 files changed, 72 insertions(+), 138 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cli/CreateTool.java b/solr/core/src/java/org/apache/solr/cli/CreateTool.java index 2737a1d2a3ed..b65ba8b0d0d2 100644 --- a/solr/core/src/java/org/apache/solr/cli/CreateTool.java +++ b/solr/core/src/java/org/apache/solr/cli/CreateTool.java @@ -17,17 +17,11 @@ package org.apache.solr.cli; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; import java.nio.file.StandardCopyOption; -import java.nio.file.attribute.BasicFileAttributes; import java.util.Locale; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; @@ -281,54 +275,12 @@ protected void createCollection(CommandLine cli, SolrClient solrClient) throws E /** * Zips the contents of a configset directory for upload. * - *

Mirrors the hidden-file skipping, directory-entry, and forbidden-file-type logic in {@link - * org.apache.solr.handler.configsets.DownloadConfigSet#zipConfigSet}. + *

Delegates to {@link ConfigSetService#zipDirectory}, which is shared with {@link + * org.apache.solr.handler.configsets.DownloadConfigSet#zipConfigSet} and {@link + * org.apache.solr.handler.designer.SchemaDesignerConfigSetHelper#downloadAndZipConfigSet}. */ static byte[] zipConfigSet(Path confPath) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { - Files.walkFileTree( - confPath, - new SimpleFileVisitor<>() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) - throws IOException { - if (Files.isHidden(dir)) { - return FileVisitResult.SKIP_SUBTREE; - } - String dirName = confPath.relativize(dir).toString().replace('\\', '/'); - if (!dirName.isEmpty()) { - if (!dirName.endsWith("/")) { - dirName += "/"; - } - zipOut.putNextEntry(new ZipEntry(dirName)); - zipOut.closeEntry(); - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) - throws IOException { - if (Files.isHidden(file)) { - return FileVisitResult.CONTINUE; - } - String filename = file.getFileName().toString(); - if (ConfigSetService.isFileForbiddenInConfigSets(filename)) { - throw new IOException( - "The file type provided for upload, '" - + filename - + "', is forbidden for use in uploading configsets."); - } - String entryName = confPath.relativize(file).toString().replace('\\', '/'); - zipOut.putNextEntry(new ZipEntry(entryName)); - Files.copy(file, zipOut); - zipOut.closeEntry(); - return FileVisitResult.CONTINUE; - } - }); - } - return baos.toByteArray(); + return ConfigSetService.zipDirectory(confPath, true); } private Path getFullConfDir(Path solrInstallDir, Path confDirName) { diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java index 6976e65c84ba..9d0dc7da6518 100644 --- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java +++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java @@ -18,16 +18,22 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.lang.reflect.Constructor; +import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.regex.Pattern; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import org.apache.solr.cloud.ZkConfigSetService; import org.apache.solr.cloud.ZkController; import org.apache.solr.common.ConfigNode; @@ -75,6 +81,65 @@ public static boolean isFileForbiddenInConfigSets(String filePath) { return lastDot >= 0 && USE_FORBIDDEN_FILE_TYPES.contains(filePath.substring(lastDot + 1)); } + /** + * Zips the contents of {@code rootPath} into an in-memory archive. Hidden files and directories + * (as determined by {@link Files#isHidden}) are skipped, directory entries are written for + * non-empty subdirectories, and zip entry names are normalized to use {@code /} separators + * regardless of platform. + * + * @param rootPath the directory to zip + * @param validateFileTypes if true, a file with a forbidden extension (see {@link + * #isFileForbiddenInConfigSets}) causes an {@link IOException} instead of being silently + * included + * @return the zipped bytes + */ + public static byte[] zipDirectory(Path rootPath, boolean validateFileTypes) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { + Files.walkFileTree( + rootPath, + new SimpleFileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) + throws IOException { + if (Files.isHidden(dir)) { + return FileVisitResult.SKIP_SUBTREE; + } + String dirName = rootPath.relativize(dir).toString().replace('\\', '/'); + if (!dirName.isEmpty()) { + if (!dirName.endsWith("/")) { + dirName += "/"; + } + zipOut.putNextEntry(new ZipEntry(dirName)); + zipOut.closeEntry(); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + if (Files.isHidden(file)) { + return FileVisitResult.CONTINUE; + } + String filename = file.getFileName().toString(); + if (validateFileTypes && isFileForbiddenInConfigSets(filename)) { + throw new IOException( + "The file type provided for upload, '" + + filename + + "', is forbidden for use in uploading configsets."); + } + String entryName = rootPath.relativize(file).toString().replace('\\', '/'); + zipOut.putNextEntry(new ZipEntry(entryName)); + Files.copy(file, zipOut); + zipOut.closeEntry(); + return FileVisitResult.CONTINUE; + } + }); + } + return baos.toByteArray(); + } + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static ConfigSetService createConfigSetService(CoreContainer coreContainer) { diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java b/solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java index 729aaf00d914..cdd695d9975c 100644 --- a/solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java +++ b/solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java @@ -21,16 +21,9 @@ import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.StreamingOutput; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; import org.apache.commons.io.file.PathUtils; import org.apache.solr.client.api.endpoint.ConfigsetsApi; import org.apache.solr.common.SolrException; @@ -86,48 +79,12 @@ public static Response buildZipResponse(ConfigSetService configSetService, Strin */ public static byte[] zipConfigSet(ConfigSetService configSetService, String configSetName) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); Path tmpDirectory = Files.createTempDirectory("configset-download-"); try { configSetService.downloadConfig(configSetName, tmpDirectory); - try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { - Files.walkFileTree( - tmpDirectory, - new SimpleFileVisitor<>() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) - throws IOException { - if (Files.isHidden(dir)) { - return FileVisitResult.SKIP_SUBTREE; - } - String dirName = tmpDirectory.relativize(dir).toString(); - if (!dirName.isEmpty()) { - if (!dirName.endsWith("/")) { - dirName += "/"; - } - zipOut.putNextEntry(new ZipEntry(dirName)); - zipOut.closeEntry(); - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) - throws IOException { - if (!Files.isHidden(file)) { - try (InputStream fis = Files.newInputStream(file)) { - ZipEntry zipEntry = new ZipEntry(tmpDirectory.relativize(file).toString()); - zipOut.putNextEntry(zipEntry); - fis.transferTo(zipOut); - } - } - return FileVisitResult.CONTINUE; - } - }); - } + return ConfigSetService.zipDirectory(tmpDirectory, false); } finally { PathUtils.deleteDirectory(tmpDirectory); } - return baos.toByteArray(); } } diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java index 1af273cc9d09..a7c979d61541 100644 --- a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java +++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java @@ -26,18 +26,14 @@ import static org.apache.solr.schema.IndexSchema.ROOT_FIELD_NAME; import static org.apache.solr.schema.ManagedIndexSchemaFactory.DEFAULT_MANAGED_SCHEMA_RESOURCE_NAME; -import java.io.ByteArrayOutputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.lang.invoke.MethodHandles; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -52,8 +48,6 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.file.PathUtils; import org.apache.lucene.util.IOSupplier; @@ -81,6 +75,7 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.Utils; +import org.apache.solr.core.ConfigSetService; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrResourceLoader; @@ -1100,49 +1095,14 @@ List listConfigsInZk() throws IOException { } byte[] downloadAndZipConfigSet(String configId) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); Path tmpDirectory = Files.createTempDirectory("schema-designer-" + FilenameUtils.getName(configId)); try { cc.getConfigSetService().downloadConfig(configId, tmpDirectory); - try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { - Files.walkFileTree( - tmpDirectory, - new SimpleFileVisitor<>() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) - throws IOException { - if (Files.isHidden(dir)) { - return FileVisitResult.SKIP_SUBTREE; - } - - String dirName = tmpDirectory.relativize(dir).toString(); - if (!dirName.endsWith("/")) { - dirName += "/"; - } - zipOut.putNextEntry(new ZipEntry(dirName)); - zipOut.closeEntry(); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) - throws IOException { - if (!Files.isHidden(file)) { - try (InputStream fis = Files.newInputStream(file)) { - ZipEntry zipEntry = new ZipEntry(tmpDirectory.relativize(file).toString()); - zipOut.putNextEntry(zipEntry); - fis.transferTo(zipOut); - } - } - return FileVisitResult.CONTINUE; - } - }); - } + return ConfigSetService.zipDirectory(tmpDirectory, false); } finally { PathUtils.deleteDirectory(tmpDirectory); } - return baos.toByteArray(); } protected ZkSolrResourceLoader zkLoaderForConfigSet(final String configSet) {