Add cloud logging service feature

multiapps-controller-core/src/main/java/module-info.java

@@ -16,6 +16,7 @@
     exports org.cloudfoundry.multiapps.controller.core.cf.util;
     exports org.cloudfoundry.multiapps.controller.core.cf.v2;
     exports org.cloudfoundry.multiapps.controller.core.cf.v3;
+    exports org.cloudfoundry.multiapps.controller.core.cloudlogging;
     exports org.cloudfoundry.multiapps.controller.core.configuration;
     exports org.cloudfoundry.multiapps.controller.core.health;
     exports org.cloudfoundry.multiapps.controller.core.health.model;
@@ -77,6 +78,8 @@
     requires spring.webflux;
     requires spring.security.oauth2.core;
     requires reactor.netty;
+    requires reactor.netty.core;
+    requires reactor.netty.http;
     requires io.netty.handler;
 
     requires static java.compiler;

multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/Messages.java

@@ -95,6 +95,7 @@ public final class Messages {
     public static final String UNSUPPORTED_FILE_FORMAT = "Unsupported file format! \"{0}\" detected";
     public static final String ENCRYPTION_HAS_FAILED = "Encryption has failed! Errored with \"{0}\"";
     public static final String DECRYPTION_HAS_FAILED = "Decryption has failed! Errored with \"{0}\"";
+    public static final String COULD_NOT_CREATE_SSL_CONTEXT_FOR_CLOUD_LOGGING_SERVICE = "Could not create SSL context for Cloud Logging service";
 
     // Warning messages
     public static final String ENVIRONMENT_VARIABLE_IS_NOT_SET_USING_DEFAULT = "Environment variable \"{0}\" is not set. Using default \"{1}\"...";
@@ -287,6 +288,17 @@ public final class Messages {
     public static final String ENTRY_CREATE_AUDIT_LOG_CONFIG = "Configuration entry create";
     public static final String ENTRY_UPDATE_AUDIT_LOG_CONFIG = "Configuration entry update";
 
+    public static final String LOGGING_CONFIGURATION_CREATE = "Create cloud-logging-configuration in space with id: {0}";
+    public static final String LOGGING_CONFIGURATION_UPDATE = "Update cloud-logging-configuration in space with id: {0}";
+    public static final String LOGGING_CONFIGURATION_DELETE = "Delete cloud-logging-configuration in space with id: {0}";
+    public static final String LOGGING_CONFIGURATION_GET = "Get cloud-logging-configuration in space with id: {0}";
+    public static final String LOGGING_CONFIGURATION_LIST = "List cloud-logging-configurations in space with id: {0}";
+    public static final String LOGGING_CONFIGURATION_CREATE_AUDIT_LOG_CONFIG = "Cloud logging configuration create";
+    public static final String LOGGING_CONFIGURATION_UPDATE_AUDIT_LOG_CONFIG = "Cloud logging configuration update";
+    public static final String LOGGING_CONFIGURATION_DELETE_AUDIT_LOG_CONFIG = "Cloud logging configuration delete";
+    public static final String LOGGING_CONFIGURATION_GET_AUDIT_LOG_CONFIG = "Cloud logging configuration get";
+    public static final String LOGGING_CONFIGURATION_LIST_AUDIT_LOG_CONFIG = "Cloud logging configuration list";
+
     public static final String API_INFO_AUDIT_LOG_CONFIG = "Api info";
     public static final String IGNORING_NAMESPACE_PARAMETERS = "Ignoring parameter \"{0}\" , as the MTA is not deployed with namespace!";
     public static final String NAMESPACE_PARSING_ERROR_MESSAGE = "Cannot parse \"{0}\" flag - expected a boolean format.";

multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/auditlogging/AuditLogBean.java

@@ -3,6 +3,7 @@
 import javax.sql.DataSource;
 
 import org.cloudfoundry.multiapps.controller.core.auditlogging.impl.AuditLoggingFacadeSLImpl;
+import org.cloudfoundry.multiapps.controller.core.auditlogging.impl.DefaultCloudLoggingServiceConfigurationAuditLog;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
@@ -58,4 +59,9 @@ public ConfigurationSubscriptionServiceAuditLog buildAConfigurationSubscriptionS
     public ConfigurationEntryServiceAuditLog buildAConfigurationEntryServiceAuditLog(AuditLoggingFacade auditLoggingFacade) {
         return new ConfigurationEntryServiceAuditLog(auditLoggingFacade);
     }
+
+    @Bean
+    public CloudLoggingServiceConfigurationAuditLog buildCloudLoggingServiceConfigurationAuditLog(AuditLoggingFacade auditLoggingFacade) {
+        return new DefaultCloudLoggingServiceConfigurationAuditLog(auditLoggingFacade);
+    }
 }

multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/auditlogging/CloudLoggingServiceConfigurationAuditLog.java

@@ -0,0 +1,14 @@
+package org.cloudfoundry.multiapps.controller.core.auditlogging;
+
+import org.cloudfoundry.multiapps.controller.persistence.model.LoggingConfiguration;
+
+public interface CloudLoggingServiceConfigurationAuditLog {
+
+    void logCreateLoggingConfiguration(String username, String spaceId, LoggingConfiguration loggingConfiguration);
+
+    void logUpdateLoggingConfiguration(String username, String spaceId, LoggingConfiguration newConfiguration);
+
+    void logDeleteLoggingConfiguration(String username, String spaceId, LoggingConfiguration loggingConfiguration);
+
+    void logGetLoggingConfiguration(String username, String spaceId, LoggingConfiguration loggingConfiguration);
+}

multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/auditlogging/impl/DefaultCloudLoggingServiceConfigurationAuditLog.java

@@ -0,0 +1,52 @@
+package org.cloudfoundry.multiapps.controller.core.auditlogging.impl;
+
+import org.cloudfoundry.multiapps.controller.core.Messages;
+import org.cloudfoundry.multiapps.controller.core.auditlogging.AuditLoggingFacade;
+import org.cloudfoundry.multiapps.controller.core.auditlogging.CloudLoggingServiceConfigurationAuditLog;
+import org.cloudfoundry.multiapps.controller.core.auditlogging.model.AuditLogConfiguration;
+import org.cloudfoundry.multiapps.controller.core.auditlogging.model.ConfigurationChangeActions;
+import org.cloudfoundry.multiapps.controller.persistence.model.LoggingConfiguration;
+
+import static org.apache.commons.lang3.StringUtils.EMPTY;
+
+public class DefaultCloudLoggingServiceConfigurationAuditLog implements CloudLoggingServiceConfigurationAuditLog {
+
+    private final AuditLoggingFacade auditLoggingFacade;
+
+    public DefaultCloudLoggingServiceConfigurationAuditLog(AuditLoggingFacade auditLoggingFacade) {
+        this.auditLoggingFacade = auditLoggingFacade;
+    }
+
+    @Override
+    public void logCreateLoggingConfiguration(String username, String spaceId, LoggingConfiguration loggingConfiguration) {
+        auditLoggingFacade.logConfigurationChangeAuditLog(
+            createAuditLogConfiguration(Messages.LOGGING_CONFIGURATION_CREATE_AUDIT_LOG_CONFIG),
+            ConfigurationChangeActions.CONFIGURATION_CREATE);
+    }
+
+    @Override
+    public void logUpdateLoggingConfiguration(String username, String spaceId, LoggingConfiguration newConfiguration) {
+        auditLoggingFacade.logConfigurationChangeAuditLog(
+            createAuditLogConfiguration(Messages.LOGGING_CONFIGURATION_UPDATE_AUDIT_LOG_CONFIG),
+            ConfigurationChangeActions.CONFIGURATION_UPDATE);
+    }
+
+    @Override
+    public void logDeleteLoggingConfiguration(String username, String spaceId, LoggingConfiguration loggingConfiguration) {
+        auditLoggingFacade.logConfigurationChangeAuditLog(
+            createAuditLogConfiguration(Messages.LOGGING_CONFIGURATION_DELETE_AUDIT_LOG_CONFIG),
+            ConfigurationChangeActions.CONFIGURATION_DELETE);
+    }
+
+    @Override
+    public void logGetLoggingConfiguration(String username, String spaceId, LoggingConfiguration loggingConfiguration) {
+        auditLoggingFacade.logDataAccessAuditLog(createAuditLogConfiguration(Messages.LOGGING_CONFIGURATION_GET_AUDIT_LOG_CONFIG));
+    }
+
+    private AuditLogConfiguration createAuditLogConfiguration(String message) {
+        return new AuditLogConfiguration(EMPTY,
+                                         EMPTY,
+                                         message,
+                                         Messages.LOGGING_CONFIGURATION_GET_AUDIT_LOG_CONFIG);
+    }
+}

multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/cf/v2/ResourceType.java

@@ -9,7 +9,8 @@
 
 public enum ResourceType {
     MANAGED_SERVICE("managed-service", SupportedParameters.SERVICE, SupportedParameters.SERVICE_PLAN), USER_PROVIDED_SERVICE(
-        "user-provided-service"), EXISTING_SERVICE("existing-service"), EXISTING_SERVICE_KEY("existing-service-key");
+        "user-provided-service"), EXISTING_SERVICE("existing-service"), EXISTING_SERVICE_KEY("existing-service-key"),
+    CLOUD_LOGGING_SERVICE("cloud-logging-service");
 
     private final String name;
     private final Set<String> requiredParameters = new HashSet<>();
@@ -33,7 +34,7 @@ public static ResourceType get(String value) {
     }
 
     public static Set<ResourceType> getServiceTypes() {
-        return EnumSet.of(MANAGED_SERVICE, USER_PROVIDED_SERVICE, EXISTING_SERVICE);
+        return EnumSet.of(MANAGED_SERVICE, USER_PROVIDED_SERVICE, EXISTING_SERVICE, CLOUD_LOGGING_SERVICE);
     }
 
     public Set<String> getRequiredParameters() {

multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/cloudlogging/CloudLoggingServiceClient.java

@@ -0,0 +1,155 @@
+package org.cloudfoundry.multiapps.controller.core.cloudlogging;
+
+import java.io.IOException;
+import java.text.MessageFormat;
+import java.time.Duration;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeoutException;
+
+import jakarta.inject.Inject;
+import jakarta.inject.Named;
+import org.cloudfoundry.multiapps.common.util.JsonUtil;
+import org.cloudfoundry.multiapps.controller.persistence.Messages;
+import org.cloudfoundry.multiapps.controller.persistence.model.ExternalOperationLogEntry;
+import org.cloudfoundry.multiapps.controller.persistence.model.LoggingConfiguration;
+import org.cloudfoundry.multiapps.controller.persistence.util.CloudLoggingServiceUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.ResponseEntity;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.web.reactive.function.client.WebClient;
+import org.springframework.web.reactive.function.client.WebClientException;
+import org.springframework.web.reactive.function.client.WebClientRequestException;
+import org.springframework.web.reactive.function.client.WebClientResponseException;
+import reactor.core.Exceptions;
+import reactor.util.retry.Retry;
+
+import static org.springframework.http.HttpHeaders.CONTENT_TYPE;
+import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
+
+@Named("cloudLoggingServiceClient")
+public class CloudLoggingServiceClient {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(CloudLoggingServiceClient.class);
+    private static final int MAX_RETRY_ATTEMPTS = 4;
+    private static final Duration INITIAL_RETRY_BACKOFF = Duration.ofMillis(500);
+    private static final Duration MAX_RETRY_BACKOFF = Duration.ofSeconds(10);
+    private static final Duration REQUEST_TIMEOUT = Duration.ofSeconds(30);
+    private static final Set<Integer> RETRYABLE_STATUS_CODES = Set.of(408, 425, 429, 500, 502, 503, 504);
+
+    private final CloudLoggingServiceWebClientFactory webClientFactory;
+    private final CloudLoggingServiceWebClientCache webClientCache;
+    private Retry retrySpec;
+
+    public CloudLoggingServiceClient() {
+        this(new DefaultCloudLoggingServiceWebClientFactory(), new CloudLoggingServiceWebClientCache());
+    }
+
+    @Inject
+    public CloudLoggingServiceClient(CloudLoggingServiceWebClientFactory webClientFactory,
+                                     CloudLoggingServiceWebClientCache webClientCache) {
+        this.webClientFactory = webClientFactory;
+        this.webClientCache = webClientCache;
+    }
+
+    public void withRetrySpec(Retry retrySpec) {
+        this.retrySpec = retrySpec;
+    }
+
+    public void removeClientFromCache(String operationId) {
+        webClientCache.remove(operationId);
+    }
+
+    @Async("asyncExecutor")
+    public void sendLogsToCloudLoggingService(LoggingConfiguration loggingConfiguration,
+                                              List<ExternalOperationLogEntry> logEntryBatch) {
+        WebClient webClient;
+        try {
+            webClient = webClientCache.getOrCreate(loggingConfiguration, webClientFactory::createWebClientWithMtls);
+        } catch (Exception e) {
+            handleSendLogFailure(loggingConfiguration, e);
+            return;
+        }
+        try {
+            ResponseEntity<Void> response = executeSendLogHttpRequest(webClient, logEntryBatch);
+            if (hasRequestFailed(response)) {
+                CloudLoggingServiceUtil.logErrorOrThrowExceptionBasedOnFailSafe(loggingConfiguration, LOGGER,
+                                                                                Messages.FAILED_TO_SEND_LOG_MESSAGE_TO_CLS);
+            }
+        } catch (IllegalStateException | WebClientResponseException e) {
+            handleSendLogFailure(loggingConfiguration, e);
+        } catch (WebClientException e) {
+            if (isTransportFailure(Exceptions.unwrap(e))) {
+                handleSendLogFailure(loggingConfiguration, e);
+            } else {
+                throw e;
+            }
+        }
+    }
+
+    private boolean isTransportFailure(Throwable throwable) {
+        return throwable instanceof IOException || throwable instanceof TimeoutException;
+    }
+
+    private void handleSendLogFailure(LoggingConfiguration loggingConfiguration, Throwable failure) {
+        CloudLoggingServiceUtil.logErrorOrThrowExceptionBasedOnFailSafe(loggingConfiguration, LOGGER,
+                                                                        Messages.FAILED_TO_SEND_LOG_MESSAGE_TO_CLS + ": "
+                                                                            + describeFailure(failure));
+    }
+
+    private ResponseEntity<Void> executeSendLogHttpRequest(WebClient webClient, List<ExternalOperationLogEntry> logEntryBatch) {
+        return webClient.post()
+                        .header(CONTENT_TYPE, APPLICATION_JSON_VALUE)
+                        .bodyValue(JsonUtil.toJson(logEntryBatch))
+                        .retrieve()
+                        .toBodilessEntity()
+                        .timeout(REQUEST_TIMEOUT)
+                        .retryWhen(retrySpec != null ? retrySpec : buildRetrySpec())
+                        .block();
+    }
+
+    private Retry buildRetrySpec() {
+        return Retry.backoff(MAX_RETRY_ATTEMPTS, INITIAL_RETRY_BACKOFF)
+                    .maxBackoff(MAX_RETRY_BACKOFF)
+                    .jitter(0.5d)
+                    .filter(this::isRetryableError)
+                    .doBeforeRetry(retrySignal -> LOGGER.warn(MessageFormat.format(Messages.RETRYING_SEND_LOGS_TO_CLS,
+                                                                                   describeFailure(retrySignal.failure()))))
+                    .onRetryExhaustedThrow((spec, retrySignal) -> retrySignal.failure());
+    }
+
+    boolean isRetryableError(Throwable throwable) {
+        if (throwable instanceof WebClientResponseException responseException) {
+            return RETRYABLE_STATUS_CODES.contains(responseException.getStatusCode()
+                                                                    .value());
+        }
+        if (throwable instanceof WebClientRequestException) {
+            return true;
+        }
+        return throwable instanceof IOException;
+    }
+
+    private String describeFailure(Throwable throwable) {
+        if (throwable instanceof WebClientResponseException responseException) {
+            String retryAfter = responseException.getHeaders()
+                                                 .getFirst(HttpHeaders.RETRY_AFTER);
+            return MessageFormat.format("HTTP {0} {1}{2}", responseException.getStatusCode()
+                                                                            .value(),
+                                        responseException.getStatusText(),
+                                        retryAfter != null ? " (Retry-After=" + retryAfter + ")" : "");
+        }
+        return throwable.getClass()
+                        .getSimpleName() + ": " + throwable.getMessage();
+    }
+
+    private boolean hasRequestFailed(ResponseEntity<Void> response) {
+        if (response == null) {
+            return true;
+        }
+        int statusCode = response.getStatusCode()
+                                 .value();
+        return statusCode < 200 || statusCode > 299;
+    }
+}