Merge pull request #17 from StackVista/stac-23446

Use s3proxy for all backups

Author: rb3ckers

cmd/clickhouse/check_and_finalize.go

@@ -33,7 +33,7 @@ func checkAndFinalizeCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command {
 This command is useful when a restore was started without --wait flag or was interrupted.
 It will check the restore status and if complete, execute post-restore tasks and scale up resources.`,
 		Run: func(_ *cobra.Command, _ []string) {
-			cmdutils.Run(globalFlags, runCheckAndFinalize, cmdutils.MinioIsRequired)
+			cmdutils.Run(globalFlags, runCheckAndFinalize, cmdutils.StorageIsRequired)
 		},
 	}
 

cmd/clickhouse/list.go

@@ -18,7 +18,7 @@ func listCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command {
 		Short: "List available Clickhouse backups",
 		Long:  `List all Clickhouse backups from the ClickHouse Backup API.`,
 		Run: func(_ *cobra.Command, _ []string) {
-			cmdutils.Run(globalFlags, runList, cmdutils.MinioIsRequired)
+			cmdutils.Run(globalFlags, runList, cmdutils.StorageIsRequired)
 		},
 	}
 }

cmd/clickhouse/restore.go

@@ -27,7 +27,7 @@ func restoreCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command {
 		Short: "Restore ClickHouse from a backup archive",
 		Long:  `Restore ClickHouse data from a backup archive via ClickHouse Backup API. Waits for completion by default; use --background to run asynchronously.`,
 		Run: func(_ *cobra.Command, _ []string) {
-			cmdutils.Run(globalFlags, runRestore, cmdutils.MinioIsRequired)
+			cmdutils.Run(globalFlags, runRestore, cmdutils.StorageIsRequired)
 		},
 	}
 

cmd/cmdutils/common.go

@@ -9,18 +9,18 @@ import (
 )
 
 const (
-	MinioIsRequired    bool = true
-	MinioIsNotRequired bool = false
+	StorageIsRequired    bool = true
+	StorageIsNotRequired bool = false
 )
 
-func Run(globalFlags *config.CLIGlobalFlags, runFunc func(ctx *app.Context) error, minioRequired bool) {
+func Run(globalFlags *config.CLIGlobalFlags, runFunc func(ctx *app.Context) error, storageRequired bool) {
 	appCtx, err := app.NewContext(globalFlags)
 	if err != nil {
 		_, _ = fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
 		os.Exit(1)
 	}
-	if minioRequired && !appCtx.Config.Minio.Enabled {
-		appCtx.Logger.Errorf("commands that interact with Minio require SUSE Observability to be deployed with .Values.global.backup.enabled=true")
+	if storageRequired && !appCtx.Config.GlobalBackupEnabled() {
+		appCtx.Logger.Errorf("commands that interact with S3-compatible storage require SUSE Observability to be deployed with .Values.global.backup.enabled=true")
 		os.Exit(1)
 	}
 	if err := runFunc(appCtx); err != nil {

cmd/elasticsearch/check_and_finalize.go

@@ -26,7 +26,7 @@ func checkAndFinalizeCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command {
 		Long: `Check the status of a restore operation and perform finalization (scale up deployments) if complete.
 If the restore is still running and --wait is specified, wait for completion before finalizing.`,
 		Run: func(_ *cobra.Command, _ []string) {
-			cmdutils.Run(globalFlags, runCheckAndFinalize, cmdutils.MinioIsRequired)
+			cmdutils.Run(globalFlags, runCheckAndFinalize, cmdutils.StorageIsRequired)
 		},
 	}
 

cmd/elasticsearch/configure.go

@@ -16,7 +16,7 @@ func configureCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command {
 		Short: "Configure Elasticsearch snapshot repository and SLM policy",
 		Long:  `Configure Elasticsearch snapshot repository and Snapshot Lifecycle Management (SLM) policy for automated backups.`,
 		Run: func(_ *cobra.Command, _ []string) {
-			cmdutils.Run(globalFlags, runConfigure, cmdutils.MinioIsRequired)
+			cmdutils.Run(globalFlags, runConfigure, cmdutils.StorageIsRequired)
 		},
 	}
 }

cmd/elasticsearch/configure_test.go

@@ -267,6 +267,162 @@ minio:
 	}
 }
 
+// TestConfigureCmd_StorageIntegration tests the integration with Kubernetes client using StorageConfig
+//
+//nolint:funlen
+func TestConfigureCmd_StorageIntegration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	tests := []struct {
+		name          string
+		configData    string
+		secretData    string
+		expectError   bool
+		errorContains string
+	}{
+		{
+			name: "successful configuration with complete data (storage mode)",
+			configData: `
+elasticsearch:
+  service:
+    name: elasticsearch-master
+    port: 9200
+    localPortForwardPort: 9200
+  restore:
+    scaleDownLabelSelector: app=test
+    indexPrefix: sts_
+    datastreamIndexPrefix: sts_k8s_logs
+    datastreamName: sts_k8s_logs
+    indicesPattern: "sts_*"
+    repository: backup-repo
+  snapshotRepository:
+    name: backup-repo
+    bucket: backups
+    endpoint: storage:9000
+    basepath: snapshots
+    accessKey: test-key
+    secretKey: test-secret
+  slm:
+    name: daily
+    schedule: "0 1 * * *"
+    snapshotTemplateName: "<snap-{now/d}>"
+    repository: backup-repo
+    indices: "sts_*"
+    retentionExpireAfter: 30d
+    retentionMinCount: 5
+    retentionMaxCount: 50
+` + minimalStorageStackgraphConfig,
+			secretData:  "",
+			expectError: false,
+		},
+		{
+			name: "missing credentials in config with secret override (storage mode)",
+			configData: `
+elasticsearch:
+  service:
+    name: elasticsearch-master
+    port: 9200
+    localPortForwardPort: 9200
+  restore:
+    scaleDownLabelSelector: app=test
+    indexPrefix: sts_
+    datastreamIndexPrefix: sts_k8s_logs
+    datastreamName: sts_k8s_logs
+    indicesPattern: "sts_*"
+    repository: backup-repo
+  snapshotRepository:
+    name: backup-repo
+    bucket: backups
+    endpoint: storage:9000
+    basepath: snapshots
+    accessKey: ""
+    secretKey: ""
+  slm:
+    name: daily
+    schedule: "0 1 * * *"
+    snapshotTemplateName: "<snap-{now/d}>"
+    repository: backup-repo
+    indices: "sts_*"
+    retentionExpireAfter: 30d
+    retentionMinCount: 5
+    retentionMaxCount: 50
+` + minimalStorageStackgraphConfig,
+			secretData: `
+elasticsearch:
+  snapshotRepository:
+    accessKey: secret-key
+    secretKey: secret-value
+storage:
+  accessKey: secret-storage-key
+  secretKey: secret-storage-value
+`,
+			expectError: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			fakeClient := fake.NewClientset()
+
+			// Create ConfigMap
+			cm := &corev1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      testConfigMapName,
+					Namespace: testNamespace,
+				},
+				Data: map[string]string{
+					"config": tt.configData,
+				},
+			}
+			_, err := fakeClient.CoreV1().ConfigMaps(testNamespace).Create(
+				context.Background(), cm, metav1.CreateOptions{},
+			)
+			require.NoError(t, err)
+
+			// Create Secret if provided
+			if tt.secretData != "" {
+				secret := &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      testSecretName,
+						Namespace: testNamespace,
+					},
+					Data: map[string][]byte{
+						"config": []byte(tt.secretData),
+					},
+				}
+				_, err := fakeClient.CoreV1().Secrets(testNamespace).Create(
+					context.Background(), secret, metav1.CreateOptions{},
+				)
+				require.NoError(t, err)
+			}
+
+			// Test that config loading works
+			secretName := ""
+			if tt.secretData != "" {
+				secretName = testSecretName
+			}
+			cfg, err := config.LoadConfig(fakeClient, testNamespace, testConfigMapName, secretName)
+
+			if tt.expectError {
+				assert.Error(t, err)
+				if tt.errorContains != "" {
+					assert.Contains(t, err.Error(), tt.errorContains)
+				}
+			} else {
+				require.NoError(t, err)
+				assert.NotNil(t, cfg)
+				// Verify storage mode
+				assert.False(t, cfg.IsLegacyMode())
+				assert.True(t, cfg.GlobalBackupEnabled())
+				assert.NotEmpty(t, cfg.Elasticsearch.SnapshotRepository.AccessKey)
+				assert.NotEmpty(t, cfg.Elasticsearch.SnapshotRepository.SecretKey)
+			}
+		})
+	}
+}
+
 // TestMockESClientForConfigure demonstrates mock usage for configure
 //
 //nolint:funlen // Table-driven test

cmd/elasticsearch/list-indices.go

@@ -16,7 +16,7 @@ func listIndicesCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command {
 		Use:   "list-indices",
 		Short: "List Elasticsearch indices",
 		Run: func(_ *cobra.Command, _ []string) {
-			cmdutils.Run(globalFlags, runListIndices, cmdutils.MinioIsRequired)
+			cmdutils.Run(globalFlags, runListIndices, cmdutils.StorageIsRequired)
 		},
 	}
 }

cmd/elasticsearch/list.go

@@ -17,7 +17,7 @@ func listCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command {
 		Use:   "list",
 		Short: "List available Elasticsearch snapshots",
 		Run: func(_ *cobra.Command, _ []string) {
-			cmdutils.Run(globalFlags, runListSnapshots, cmdutils.MinioIsRequired)
+			cmdutils.Run(globalFlags, runListSnapshots, cmdutils.StorageIsRequired)
 		},
 	}
 }