chore: Bump OpenSearch and OpenSearch Dashboards to 3.4.0 (#360)

siegfriedweber · web-flow · commit d21a93837317 · 2026-01-29T14:34:12.000+01:00
* chore: Upgrade OpenSearch to version 3.4.0

* chore: Update opensearch-rag stack and demo for SDP 26.3

* demos(opensearch-rag): Wait for OpenSearch Dashboards to become ready

* Disable multi-tenancy in OpenSearch
diff --git a/demos/opensearch-rag/load-embeddings-from-git.yaml b/demos/opensearch-rag/load-embeddings-from-git.yaml
@@ -18,21 +18,28 @@ data:
     from requests.auth import HTTPBasicAuth
 
     # Configuration
-    OPENSEARCH_HOST = os.getenv('OPENSEARCH_HOST', 'opensearch')
-    OPENSEARCH_PORT = int(os.getenv('OPENSEARCH_PORT', '9200'))
+    OPENSEARCH_HOSTS = os.getenv('OPENSEARCH_HOSTS', 'https://opensearch:9200')
+    OPENSEARCH_HOSTNAME = os.getenv('OPENSEARCH_HOSTNAME', 'opensearch')
+    OPENSEARCH_PORT = os.getenv('OPENSEARCH_PORT', '9200')
+    OPENSEARCH_PROTOCOL = os.getenv('OPENSEARCH_PROTOCOL', 'https')
     OPENSEARCH_USER = os.getenv('OPENSEARCH_USER', 'admin')
     OPENSEARCH_PASSWORD = os.getenv('OPENSEARCH_PASSWORD', 'adminadmin')
+    OPENSEARCH_DASHBOARDS_HOST = os.getenv('DASHBOARDS_HOST', 'opensearch-dashboards')
+    OPENSEARCH_DASHBOARDS_PORT = os.getenv('DASHBOARDS_PORT', '5601')
     INDEX_NAME = os.getenv('INDEX_NAME', 'rag-documents')
     EMBEDDINGS_FILE = os.getenv('EMBEDDINGS_FILE', '/data/stackable-docs-embeddings.json')
 
+    OPENSEARCH_BASIC_AUTH = HTTPBasicAuth(OPENSEARCH_USER, OPENSEARCH_PASSWORD)
+    OPENSEARCH_DASHBOARDS_BASE_URL = f'http://{OPENSEARCH_DASHBOARDS_HOST}:{OPENSEARCH_DASHBOARDS_PORT}'
+
     def wait_for_opensearch():
         """Wait for OpenSearch to be ready."""
         print("Waiting for OpenSearch to be ready...")
         for i in range(60):
             try:
                 response = requests.get(
-                    f'https://{OPENSEARCH_HOST}:{OPENSEARCH_PORT}',
-                    auth=HTTPBasicAuth(OPENSEARCH_USER, OPENSEARCH_PASSWORD),
+                    OPENSEARCH_HOSTS,
+                    auth=OPENSEARCH_BASIC_AUTH,
                     verify=False,
                     timeout=5
                 )
@@ -46,6 +53,27 @@ data:
         print("[ERROR] OpenSearch did not become ready in time")
         return False
 
+    def wait_for_opensearch_dashboards():
+        """Wait for OpenSearch Dashboards to be ready."""
+        print("Waiting for OpenSearch Dashboards to be ready...")
+        for i in range(60):
+            try:
+                response = requests.get(
+                    OPENSEARCH_DASHBOARDS_BASE_URL,
+                    auth=OPENSEARCH_BASIC_AUTH,
+                    verify=False,
+                    timeout=5
+                )
+                if response.status_code == 200:
+                    print("[OK] OpenSearch Dashboards is ready!")
+                    return True
+            except Exception:
+                print(f"Waiting for OpenSearch Dashboards... ({i+1}/60)")
+                time.sleep(5)
+
+        print("[ERROR] OpenSearch Dashboards did not become ready in time")
+        return False
+
     def index_needs_loading(client):
         """Check if index exists and has documents. Returns True if loading is needed."""
         if not client.indices.exists(index=INDEX_NAME):
@@ -135,17 +163,13 @@ data:
 
     def create_index_pattern():
         """Create index pattern in OpenSearch Dashboards and set as default."""
-        dashboards_host = os.getenv('DASHBOARDS_HOST', 'opensearch-dashboards')
-        dashboards_port = os.getenv('DASHBOARDS_PORT', '5601')
-        base_url = f'http://{dashboards_host}:{dashboards_port}'
         headers = {"osd-xsrf": "true"}
-        auth = HTTPBasicAuth(OPENSEARCH_USER, OPENSEARCH_PASSWORD)
 
         # Create index pattern
-        response = requests.put(
-            f'{base_url}/api/saved_objects/index-pattern/{INDEX_NAME}?overwrite=true',
+        response = requests.post(
+            f'{OPENSEARCH_DASHBOARDS_BASE_URL}/api/saved_objects/index-pattern/{INDEX_NAME}?overwrite=true',
             json={"attributes": {"title": INDEX_NAME, "timeFieldName": "timestamp"}},
-            auth=auth, headers=headers, timeout=10
+            auth=OPENSEARCH_BASIC_AUTH, headers=headers, timeout=10
         )
         if response.status_code not in (200, 201, 409):
             print(f"[ERROR] Failed to create index pattern: {response.status_code} - {response.text}")
@@ -154,9 +178,9 @@ data:
 
         # Set as default
         response = requests.post(
-            f'{base_url}/api/opensearch-dashboards/settings',
+            f'{OPENSEARCH_DASHBOARDS_BASE_URL}/api/opensearch-dashboards/settings',
             json={"changes": {"defaultIndex": INDEX_NAME}},
-            auth=auth, headers=headers, timeout=10
+            auth=OPENSEARCH_BASIC_AUTH, headers=headers, timeout=10
         )
         if response.status_code in (200, 201):
             print(f"[OK] Set {INDEX_NAME} as default index pattern")
@@ -201,10 +225,13 @@ data:
         if not wait_for_opensearch():
             return 1
 
+        if not wait_for_opensearch_dashboards():
+            return 1
+
         client = OpenSearch(
-            hosts=[{'host': OPENSEARCH_HOST, 'port': OPENSEARCH_PORT}],
+            hosts=[{'host': OPENSEARCH_HOSTNAME, 'port': OPENSEARCH_PORT}],
             http_auth=(OPENSEARCH_USER, OPENSEARCH_PASSWORD),
-            use_ssl=True,
+            use_ssl=OPENSEARCH_PROTOCOL == 'https',
             verify_certs=False,
             ssl_show_warn=False
         )
@@ -277,10 +304,6 @@ spec:
           pip install -q opensearch-py requests urllib3
           python -u /scripts/load.py
         env:
-        - name: OPENSEARCH_HOST
-          value: "opensearch"
-        - name: OPENSEARCH_PORT
-          value: "9200"
         - name: OPENSEARCH_USER
           value: "admin"
         - name: OPENSEARCH_PASSWORD
@@ -292,6 +315,9 @@ spec:
           value: "rag-documents"
         - name: EMBEDDINGS_FILE
           value: "/data/stackable-docs-embeddings.json"
+        envFrom:
+        - configMapRef:
+            name: opensearch
         volumeMounts:
         - name: script
           mountPath: /scripts
diff --git a/docs/modules/demos/images/logging/tenant.png b/docs/modules/demos/images/logging/tenant.png
diff --git a/docs/modules/demos/pages/logging.adoc b/docs/modules/demos/pages/logging.adoc
@@ -56,7 +56,8 @@ $ stackablectl stacklet list
 ┌───────────────────────┬───────────────────────┬───────────┬────────────────────────────────────────────────────────────────────────────────────┬─────────────────────────────────┐
 │ PRODUCT               ┆ NAME                  ┆ NAMESPACE ┆ ENDPOINTS                                                                          ┆ CONDITIONS                      │
 ╞═══════════════════════╪═══════════════════════╪═══════════╪════════════════════════════════════════════════════════════════════════════════════╪═════════════════════════════════╡
-│ opensearch            ┆ opensearch            ┆ default   ┆ nodes-default-http  http://opensearch-nodes-default.default.svc.cluster.local:9200 ┆ Available, Reconciling, Running │
+│ opensearch            ┆ opensearch            ┆ default   ┆ -http               http://opensearch.default.svc.cluster.local:9200               ┆ Available, Reconciling, Running │
+│                       ┆                       ┆           ┆ nodes-default-http  http://opensearch-nodes-default.default.svc.cluster.local:9200 ┆                                 │
 ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
 │ zookeeper             ┆ simple-zk             ┆ default   ┆ server-zk           simple-zk-server.default.svc.cluster.local:2282                ┆ Available, Reconciling, Running │
 ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
@@ -79,10 +80,6 @@ Log in with the username `admin` and password `adminadmin`.
 
 NOTE: On first login, you will be presented with some options. Feel free to bypass them to get to the logs.
 
-Select the `Global` tenant to have access to the pre-configured `vector-*` index pattern.
-
-image::logging/tenant.png[]
-
 Click _Discover_ in the menu on the left to view the recent logs.
 If you do not see anything, increase the search window to greater than _Last 15 minutes_.
 
diff --git a/docs/modules/demos/pages/opensearch-rag.adoc b/docs/modules/demos/pages/opensearch-rag.adoc
@@ -274,7 +274,7 @@ Log in at http://localhost:5601 with:
 * *Username*: `admin`
 * *Password*: `adminadmin`
 
-Select the *Global* tenant, then navigate to *Discover* to browse the `rag-documents` index:
+Navigate to *Discover* to browse the `rag-documents` index:
 
 * Document titles and content
 * Categories (airflow-operator, kafka-operator, trino-operator, etc.)
diff --git a/stacks/_templates/opensearch-dashboards.yaml b/stacks/_templates/opensearch-dashboards.yaml
@@ -6,7 +6,6 @@ repo:
   url: https://opensearch-project.github.io/helm-charts
 version: {{ opensearchVersion }}
 options:
-  opensearchHosts: https://opensearch:9200
   image:
     repository: oci.stackable.tech/sdp/opensearch-dashboards
     tag: "{{ opensearchVersion }}-stackable{{ stackableReleaseVersion }}"
@@ -16,15 +15,23 @@ options:
     type: NodePort
     port: 5601
     annotations:
-      stackable.tech/logging-view-logs: |-
-        /app/discover?security_tenant=global#/view/logs
+      stackable.tech/logging-view-logs: /app/discover#/view/logs
       stackable.tech/logging-credentials-secret: opensearch-user
     labels:
       stackable.tech/vendor: Stackable
+  opensearchHosts: null # Use the discovery ConfigMap instead
+  extraEnvs:
+    - name: OPENSEARCH_HOSTS
+      valueFrom:
+        configMapKeyRef:
+          name: opensearch
+          key: OPENSEARCH_HOSTS
   opensearchAccount:
     secret: opensearch-dashboard-user
   serviceAccount:
     create: false
     # Use the ServiceAccount of OpenSearch because its permissions are already configured to work on
     # OpenShift.
     name: opensearch-serviceaccount
+config:
+  opensearch_security.multitenancy.enabled: false
diff --git a/stacks/logging/opensearch.yaml b/stacks/logging/opensearch.yaml
@@ -88,6 +88,8 @@ stringData:
             authentication_backend:
               type: intern
         authz: {}
+        kibana:
+          multitenancy_enabled: false
   internal_users.yml: |
     ---
     _meta:
diff --git a/stacks/logging/setup-opensearch-dashboards.yaml b/stacks/logging/setup-opensearch-dashboards.yaml
@@ -28,7 +28,6 @@ spec:
                   --retry-delay 5 \
                   --header "Content-Type:application/json" \
                   --header "osd-xsrf:true" \
-                  --header "securitytenant: global" \
                   --data '{
                             "attributes": {
                               "title": "vector-*",
@@ -39,7 +38,6 @@ spec:
               curl \
                   --header "Content-Type:application/json" \
                   --header "osd-xsrf:true" \
-                  --header "securitytenant: global" \
                   --data '{
                             "attributes": {
                               "title":"Logs",
diff --git a/stacks/opensearch-rag/jupyterlab.yaml b/stacks/opensearch-rag/jupyterlab.yaml
@@ -31,10 +31,6 @@ spec:
           env:
             - name: JUPYTER_PORT
               value: "8888"
-            - name: OPENSEARCH_HOST
-              value: "opensearch"
-            - name: OPENSEARCH_PORT
-              value: "9200"
             - name: OPENSEARCH_USER
               value: "admin"
             - name: OPENSEARCH_PASSWORD
@@ -48,6 +44,9 @@ spec:
               value: "11434"
             - name: OLLAMA_LLM_MODEL
               value: "{{ ollamaLlmModel }}"
+          envFrom:
+            - configMapRef:
+                name: opensearch
           ports:
             - name: http
               containerPort: 8888
diff --git a/stacks/opensearch-rag/opensearch-rag.ipynb b/stacks/opensearch-rag/opensearch-rag.ipynb
@@ -68,16 +68,18 @@
     "warnings.filterwarnings('ignore')\n",
     "\n",
     "# Configuration from environment variables\n",
-    "OPENSEARCH_HOST = os.getenv('OPENSEARCH_HOST')\n",
+    "OPENSEARCH_HOSTS = os.getenv('OPENSEARCH_HOSTS')\n",
+    "OPENSEARCH_HOSTNAME = os.getenv('OPENSEARCH_HOSTNAME')\n",
     "OPENSEARCH_PORT = int(os.getenv('OPENSEARCH_PORT'))\n",
+    "OPENSEARCH_PROTOCOL = os.getenv('OPENSEARCH_PROTOCOL')\n",
     "OPENSEARCH_USER = os.getenv('OPENSEARCH_USER')\n",
     "OPENSEARCH_PASSWORD = os.getenv('OPENSEARCH_PASSWORD')\n",
     "OLLAMA_HOST = os.getenv('OLLAMA_HOST')\n",
     "OLLAMA_PORT = int(os.getenv('OLLAMA_PORT'))\n",
     "OLLAMA_LLM_MODEL = os.getenv('OLLAMA_LLM_MODEL')\n",
     "INDEX_NAME = 'rag-documents'\n",
     "\n",
-    "print(f\"OpenSearch: {OPENSEARCH_HOST}:{OPENSEARCH_PORT}\")\n",
+    "print(f\"OpenSearch: {OPENSEARCH_HOSTS}\")\n",
     "print(f\"Ollama: {OLLAMA_HOST}:{OLLAMA_PORT} using {OLLAMA_LLM_MODEL} to generate responses\")\n",
     "print(f\"Index: {INDEX_NAME}\")"
    ]
@@ -101,9 +103,9 @@
    "source": [
     "# Initialize OpenSearch client\n",
     "opensearch_client = OpenSearch(\n",
-    "    hosts=[{'host': OPENSEARCH_HOST, 'port': OPENSEARCH_PORT}],\n",
+    "    hosts=[{'host': OPENSEARCH_HOSTNAME, 'port': OPENSEARCH_PORT}],\n",
     "    http_auth=(OPENSEARCH_USER, OPENSEARCH_PASSWORD),\n",
-    "    use_ssl=True,\n",
+    "    use_ssl=OPENSEARCH_PROTOCOL == 'https',\n",
     "    verify_certs=False,\n",
     "    ssl_show_warn=False\n",
     ")\n",
diff --git a/stacks/opensearch-rag/opensearch.yaml b/stacks/opensearch-rag/opensearch.yaml
@@ -90,6 +90,8 @@ stringData:
             authentication_backend:
               type: intern
         authz: {}
+        kibana:
+          multitenancy_enabled: false
   internal_users.yml: |
     ---
     _meta:
diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml
@@ -101,7 +101,7 @@ stacks:
         default: kibanaserverkibanaserver
       - name: opensearchVersion
         description: Version of OpenSearch and OpenSearch Dashboards to deploy
-        default: 3.1.0
+        default: 3.4.0
       - name: stackableReleaseVersion
         description: The Stackable release to be used for the OpenSearch Dashboards image tag
         default: 0.0.0-dev
@@ -725,7 +725,7 @@ stacks:
         default: kibanaserverkibanaserver
       - name: opensearchVersion
         description: Version of OpenSearch and OpenSearch Dashboards to deploy
-        default: 3.1.0
+        default: 3.4.0
       - name: stackableReleaseVersion
         description: The Stackable release to be used for the OpenSearch Dashboards image tag
         default: 0.0.0-dev
