SOLR-18320: Use Configsets V2 API instead of direct ZooKeeper access in bin/solr create

changelog/unreleased/SOLR-18320-skip-zk-for-configset-upload.yml

@@ -0,0 +1,7 @@
+title: bin/solr create no longer requires a direct ZooKeeper connection to upload a configset; it now uses the Configsets V2 API.
+type: changed
+authors:
+  - name: Eric Pugh
+links:
+  - name: SOLR-18320
+    url: https://issues.apache.org/jira/browse/SOLR-18320

solr/core/src/java/org/apache/solr/cli/CreateTool.java

@@ -16,28 +16,24 @@
  */
 package org.apache.solr.cli;
 
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.util.Locale;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.io.file.PathUtils;
 import org.apache.solr.cli.CommonCLIOptions.DefaultValues;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.jetty.HttpJettySolrClient;
 import org.apache.solr.client.solrj.request.CollectionsApi;
+import org.apache.solr.client.solrj.request.ConfigsetsApi;
 import org.apache.solr.client.solrj.request.CoresApi;
 import org.apache.solr.client.solrj.request.SystemInfoRequest;
 import org.apache.solr.client.solrj.response.SystemInfoResponse;
-import org.apache.solr.cloud.ZkConfigSetService;
-import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.util.EnvUtils;
 import org.apache.solr.core.ConfigSetService;
 
@@ -125,7 +121,7 @@ public Options getOptions() {
   public void runImpl(CommandLine cli) throws Exception {
     try (var solrClient = CLIUtils.getSolrClient(cli)) {
       if (CLIUtils.isCloudMode(solrClient)) {
-        createCollection(cli);
+        createCollection(cli, solrClient);
       } else {
         createCore(cli, solrClient);
       }
@@ -193,29 +189,7 @@ protected void createCore(CommandLine cli, SolrClient solrClient) throws Excepti
     }
   }
 
-  protected void createCollection(CommandLine cli) throws Exception {
-    var builder =
-        new HttpJettySolrClient.Builder()
-            .withIdleTimeout(30, TimeUnit.SECONDS)
-            .withConnectionTimeout(15, TimeUnit.SECONDS)
-            .withKeyStoreReloadInterval(-1, TimeUnit.SECONDS)
-            .withOptionalBasicAuthCredentials(
-                cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION));
-    String zkHost = CLIUtils.getZkHost(cli);
-    echoIfVerbose("Connecting to ZooKeeper at " + zkHost);
-    var zkSolrConnection = CloudSolrClient.CloudSolrClientConnection.parse(zkHost);
-    if (!zkSolrConnection.isZookeeper()) {
-      throw new IOException(
-          String.format(
-              Locale.ROOT, "Expected ZooKeeper connection string, but got: '%s'.", zkHost));
-    }
-    try (var cloudSolrClient = CLIUtils.getCloudSolrClient(zkSolrConnection, builder)) {
-      createCollection(cloudSolrClient, cli);
-    }
-  }
-
-  protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli)
-      throws Exception {
+  protected void createCollection(CommandLine cli, SolrClient solrClient) throws Exception {
 
     String collectionName = cli.getOptionValue(COLLECTION_NAME_OPTION);
     final String solrInstallDir = EnvUtils.getProperty("solr.install.dir");
@@ -226,53 +200,37 @@ protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli
     ensureConfDirExists(solrInstallDirPath, confDirPath);
     printDefaultConfigsetWarningIfNecessary(cli);
 
-    Set<String> liveNodes = cloudSolrClient.getClusterState().getLiveNodes();
-    if (liveNodes.isEmpty())
-      throw new IllegalStateException(
-          "No live nodes found! Cannot create a collection until "
-              + "there is at least 1 live node in the cluster.");
-
-    String solrUrl;
-    if (CLIUtils.hasConnectionOption(cli)) {
-      solrUrl = CLIUtils.normalizeSolrUrl(cli);
-    } else {
-      String firstLiveNode = liveNodes.iterator().next();
-      solrUrl = ZkStateReader.from(cloudSolrClient).getBaseUrlForNodeName(firstLiveNode);
-    }
+    String solrUrl = CLIUtils.normalizeSolrUrl(cli);
 
     // build a URL to create the collection
     int numShards = cli.getParsedOptionValue(SHARDS_OPTION, 1);
     int replicationFactor = cli.getParsedOptionValue(REPLICATION_FACTOR_OPTION, 1);
 
-    boolean configExistsInZk =
+    boolean configExists =
         confName != null
             && !confName.trim().isEmpty()
-            && ZkStateReader.from(cloudSolrClient).getZkClient().exists("/configs/" + confName);
+            && new ConfigsetsApi.ListConfigSet().process(solrClient).configSets.contains(confName);
 
-    if (configExistsInZk) {
+    if (configExists) {
       echo("Re-using existing configuration directory " + confName);
     } else { // if (confdir != null && !confdir.trim().isEmpty()) {
       if (confName == null || confName.trim().isEmpty()) {
         confName = collectionName;
       }
 
-      // TODO: This should be done using the configSet API.  This would let us remove
-      // the direct dependency on ZooKeeper APIs.  Unlike the bin/solr zk comamnds that
-      // work directly with ZooKeeper.
       final Path configsetsDirPath = CLIUtils.getConfigSetsDir(solrInstallDirPath);
-      ConfigSetService configSetService =
-          new ZkConfigSetService(ZkStateReader.from(cloudSolrClient).getZkClient());
       Path confPath = ConfigSetService.getConfigsetPath(confDir, configsetsDirPath.toString());
 
       echoIfVerbose(
           "Uploading "
               + confPath.toAbsolutePath()
               + " for config "
               + confName
-              + " to ZooKeeper at "
-              + cloudSolrClient.getClusterStateProvider().getQuorumHosts());
-      // We will trust the config since we have the Zookeeper Address
-      configSetService.uploadConfig(confName, confPath);
+              + " using the Configsets V2 API");
+      var uploadReq =
+          new ConfigsetsApi.UploadConfigSet(
+              confName, new ByteArrayInputStream(zipConfigSet(confPath)));
+      uploadReq.process(solrClient);
     }
 
     // since creating a collection is a heavy-weight operation, check for existence first
@@ -293,7 +251,7 @@ protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli
       req.setConfig(confName);
       req.setNumShards(numShards);
       req.setReplicationFactor(replicationFactor);
-      var response = req.process(cloudSolrClient);
+      var response = req.process(solrClient);
       echoIfVerbose(response);
     } catch (SolrServerException sse) {
       throw new Exception(
@@ -314,6 +272,17 @@ protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli
     echo(endMessage);
   }
 
+  /**
+   * Zips the contents of a configset directory for upload.
+   *
+   * <p>Delegates to {@link ConfigSetService#zipDirectory}, which is shared with {@link
+   * org.apache.solr.handler.configsets.DownloadConfigSet#zipConfigSet} and {@link
+   * org.apache.solr.handler.designer.SchemaDesignerConfigSetHelper#downloadAndZipConfigSet}.
+   */
+  static byte[] zipConfigSet(Path confPath) throws IOException {
+    return ConfigSetService.zipDirectory(confPath, true);
+  }
+
   private Path getFullConfDir(Path solrInstallDir, Path confDirName) {
     return CLIUtils.getConfigSetsDir(solrInstallDir).resolve(confDirName);
   }

solr/core/src/java/org/apache/solr/core/ConfigSetService.java

@@ -18,16 +18,22 @@
 
 import com.github.benmanes.caffeine.cache.Cache;
 import com.github.benmanes.caffeine.cache.Caffeine;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.lang.reflect.Constructor;
+import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.regex.Pattern;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
 import org.apache.solr.cloud.ZkConfigSetService;
 import org.apache.solr.cloud.ZkController;
 import org.apache.solr.common.ConfigNode;
@@ -75,6 +81,65 @@ public static boolean isFileForbiddenInConfigSets(String filePath) {
     return lastDot >= 0 && USE_FORBIDDEN_FILE_TYPES.contains(filePath.substring(lastDot + 1));
   }
 
+  /**
+   * Zips the contents of {@code rootPath} into an in-memory archive. Hidden files and directories
+   * (as determined by {@link Files#isHidden}) are skipped, directory entries are written for
+   * non-empty subdirectories, and zip entry names are normalized to use {@code /} separators
+   * regardless of platform.
+   *
+   * @param rootPath the directory to zip
+   * @param validateFileTypes if true, a file with a forbidden extension (see {@link
+   *     #isFileForbiddenInConfigSets}) causes an {@link IOException} instead of being silently
+   *     included
+   * @return the zipped bytes
+   */
+  public static byte[] zipDirectory(Path rootPath, boolean validateFileTypes) throws IOException {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    try (ZipOutputStream zipOut = new ZipOutputStream(baos)) {
+      Files.walkFileTree(
+          rootPath,
+          new SimpleFileVisitor<>() {
+            @Override
+            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
+                throws IOException {
+              if (Files.isHidden(dir)) {
+                return FileVisitResult.SKIP_SUBTREE;
+              }
+              String dirName = rootPath.relativize(dir).toString().replace('\\', '/');
+              if (!dirName.isEmpty()) {
+                if (!dirName.endsWith("/")) {
+                  dirName += "/";
+                }
+                zipOut.putNextEntry(new ZipEntry(dirName));
+                zipOut.closeEntry();
+              }
+              return FileVisitResult.CONTINUE;
+            }
+
+            @Override
+            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
+                throws IOException {
+              if (Files.isHidden(file)) {
+                return FileVisitResult.CONTINUE;
+              }
+              String filename = file.getFileName().toString();
+              if (validateFileTypes && isFileForbiddenInConfigSets(filename)) {
+                throw new IOException(
+                    "The file type provided for upload, '"
+                        + filename
+                        + "', is forbidden for use in uploading configsets.");
+              }
+              String entryName = rootPath.relativize(file).toString().replace('\\', '/');
+              zipOut.putNextEntry(new ZipEntry(entryName));
+              Files.copy(file, zipOut);
+              zipOut.closeEntry();
+              return FileVisitResult.CONTINUE;
+            }
+          });
+    }
+    return baos.toByteArray();
+  }
+
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
   public static ConfigSetService createConfigSetService(CoreContainer coreContainer) {

solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java

@@ -21,16 +21,9 @@
 import jakarta.inject.Inject;
 import jakarta.ws.rs.core.Response;
 import jakarta.ws.rs.core.StreamingOutput;
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
 import org.apache.commons.io.file.PathUtils;
 import org.apache.solr.client.api.endpoint.ConfigsetsApi;
 import org.apache.solr.common.SolrException;
@@ -86,48 +79,12 @@ public static Response buildZipResponse(ConfigSetService configSetService, Strin
    */
   public static byte[] zipConfigSet(ConfigSetService configSetService, String configSetName)
       throws IOException {
-    ByteArrayOutputStream baos = new ByteArrayOutputStream();
     Path tmpDirectory = Files.createTempDirectory("configset-download-");
     try {
       configSetService.downloadConfig(configSetName, tmpDirectory);
-      try (ZipOutputStream zipOut = new ZipOutputStream(baos)) {
-        Files.walkFileTree(
-            tmpDirectory,
-            new SimpleFileVisitor<>() {
-              @Override
-              public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
-                  throws IOException {
-                if (Files.isHidden(dir)) {
-                  return FileVisitResult.SKIP_SUBTREE;
-                }
-                String dirName = tmpDirectory.relativize(dir).toString();
-                if (!dirName.isEmpty()) {
-                  if (!dirName.endsWith("/")) {
-                    dirName += "/";
-                  }
-                  zipOut.putNextEntry(new ZipEntry(dirName));
-                  zipOut.closeEntry();
-                }
-                return FileVisitResult.CONTINUE;
-              }
-
-              @Override
-              public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
-                  throws IOException {
-                if (!Files.isHidden(file)) {
-                  try (InputStream fis = Files.newInputStream(file)) {
-                    ZipEntry zipEntry = new ZipEntry(tmpDirectory.relativize(file).toString());
-                    zipOut.putNextEntry(zipEntry);
-                    fis.transferTo(zipOut);
-                  }
-                }
-                return FileVisitResult.CONTINUE;
-              }
-            });
-      }
+      return ConfigSetService.zipDirectory(tmpDirectory, false);
     } finally {
       PathUtils.deleteDirectory(tmpDirectory);
     }
-    return baos.toByteArray();
   }
 }