feat: update with latest from main (#4034)

* chore: sync with last release from main branch

* feat: Introduce DataFormatOptions to configure the output of BigQuery data types (#4010)

* feat: Create DataFormatOptions in BigQuery

* feat: Add Builder class for DataFormatOptions

* fix: Update existing references of useInt64Timestamp to use DataFormatOption's variant

* chore: Fix lint issues

* chore: Address PR feedback

* chore: Add tests for useInt64Timestamp behavior

* chore: Address failing tests and GCA

* chore: Remove unused fromPb method

* feat: Add timestamp_precision to Field (#4014)

* feat: Add timestamp_precision to Field

* Apply suggestion from @gemini-code-assist[bot]

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* chore: Address GCA PR feedback

* chore: Fix typo

* chore: Remove default value

* chore: Address PR feedback

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* chore: Use custom timestamp validator for ISO8601 timestamps with more than nanosecond precision (#4017)

* chore: Use custom timestamp validator for ISO8601 timestamps with more than nanosecond precision

* chore: Rename helper method to validateTimestamp

* chore: Address GCA comments

* chore: Address last GCA comment

* chore: Update to use assert helper method

* test: Clean up resources created in ITs (#4024)

* chore: Cleanup initialized resources

* chore: Delete created dataset

* chore: Use try-with to close stream

* chore: Cleanup Intellij test warnings (#4026)

* chore: Cleanup initialized resources

* chore: Use try-with to close stream

* chore: Cleanup Intellij test warnings

* chore: Allow for floating point inaccuracies

* chore: Add otel delete operation

* deps: update actions/upload-artifact action to v6 (#4027)

Co-authored-by: Blake Li <[email protected]>

* chore: Use assertThrows in tests (#4028)

* chore: Cleanup initialized resources

* chore: Use try-with to close stream

* chore: Cleanup Intellij test warnings

* chore: Allow for floating point inaccuracies

* chore: Add otel delete operation

* chore: Clean up rest of the IT test file

* chore: Add GCA feedback

* chore: Update renovate.json (#4031)

deps update from bigquerystorage was incorrectly marked as chore(deps). This would cause the upgrade not show up in the release notes and also not trigger release please.

This is because the pattern /^com.google.cloud:google-cloud-bigquery is configured in renovate.json, which matches both bigquery and bigquerystorage. However, I believe the original intention is only for bigquery update in samples, not for bigquerystorage update.

Fixing the pattern so that the String must ends with bigquery

* chore(main): release 2.57.2-SNAPSHOT (#4032)

Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com>

* chore: fix unresolved conflicts

---------

Co-authored-by: Lawrence Qiu <[email protected]>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Mend Renovate <[email protected]>
Co-authored-by: Blake Li <[email protected]>
Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com>

Author: 6 people

.github/workflows/scorecard.yml

@@ -59,7 +59,7 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
         with:
           name: SARIF file
           path: results.sarif

.release-please-manifest.json

@@ -1,3 +1,3 @@
 {
-  ".": "2.56.0"
+  ".": "2.57.1"
 }

benchmark/pom.xml

@@ -6,7 +6,7 @@
   <parent>
     <artifactId>google-cloud-bigquery-parent</artifactId>
     <groupId>com.google.cloud</groupId>
-    <version>2.56.1-rc1-SNAPSHOT</version><!-- {x-version-update:google-cloud-bigquery:current} -->
+    <version>2.57.2-SNAPSHOT</version><!-- {x-version-update:google-cloud-bigquery:current} -->
   </parent>
 
   <properties>

google-cloud-bigquery-bom/pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.google.cloud</groupId>
     <artifactId>google-cloud-bigquery-bom</artifactId>
-    <version>2.56.1-rc1-SNAPSHOT</version><!-- {x-version-update:google-cloud-bigquery:current} -->
+    <version>2.57.2-SNAPSHOT</version><!-- {x-version-update:google-cloud-bigquery:current} -->
     <packaging>pom</packaging>
     <parent>
         <groupId>com.google.cloud</groupId>
@@ -54,7 +54,7 @@
         <dependency>
           <groupId>com.google.cloud</groupId>
           <artifactId>google-cloud-bigquery</artifactId>
-          <version>2.56.1-rc1-SNAPSHOT</version><!-- {x-version-update:google-cloud-bigquery:current} -->
+          <version>2.57.2-SNAPSHOT</version><!-- {x-version-update:google-cloud-bigquery:current} -->
         </dependency>
       </dependencies>
     </dependencyManagement>

google-cloud-bigquery/pom.xml

@@ -3,15 +3,15 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>com.google.cloud</groupId>
   <artifactId>google-cloud-bigquery</artifactId>
-  <version>2.56.1-rc1-SNAPSHOT</version><!-- {x-version-update:google-cloud-bigquery:current} -->
+  <version>2.57.2-SNAPSHOT</version><!-- {x-version-update:google-cloud-bigquery:current} -->
   <packaging>jar</packaging>
   <name>BigQuery</name>
   <url>https://github.com/googleapis/java-bigquery</url>
   <description>BigQuery</description>
   <parent>
     <groupId>com.google.cloud</groupId>
     <artifactId>google-cloud-bigquery-parent</artifactId>
-    <version>2.56.1-rc1-SNAPSHOT</version><!-- {x-version-update:google-cloud-bigquery:current} -->
+    <version>2.57.2-SNAPSHOT</version><!-- {x-version-update:google-cloud-bigquery:current} -->
   </parent>
   <properties>
     <site.installationModule>google-cloud-bigquery</site.installationModule>

google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java

@@ -1731,7 +1731,10 @@ public TableDataList call() throws IOException {
           new PageImpl<>(
               new TableDataPageFetcher(tableId, schema, serviceOptions, cursor, pageOptionMap),
               cursor,
-              transformTableData(result.getRows(), schema, serviceOptions.getUseInt64Timestamps())),
+              transformTableData(
+                  result.getRows(),
+                  schema,
+                  serviceOptions.getDataFormatOptions().useInt64Timestamp())),
           result.getTotalRows());
     } catch (BigQueryRetryHelperException e) {
       throw BigQueryException.translateAndThrow(e);
@@ -2007,7 +2010,9 @@ public com.google.api.services.bigquery.model.QueryResponse call()
                   new QueryPageFetcher(jobId, schema, getOptions(), cursor, optionMap(options)),
                   cursor,
                   transformTableData(
-                      results.getRows(), schema, getOptions().getUseInt64Timestamps())))
+                      results.getRows(),
+                      schema,
+                      getOptions().getDataFormatOptions().useInt64Timestamp())))
           .setJobId(jobId)
           .setQueryId(results.getQueryId())
           .build();
@@ -2021,7 +2026,9 @@ public com.google.api.services.bigquery.model.QueryResponse call()
                 new TableDataPageFetcher(null, schema, getOptions(), null, optionMap(options)),
                 null,
                 transformTableData(
-                    results.getRows(), schema, getOptions().getUseInt64Timestamps())))
+                    results.getRows(),
+                    schema,
+                    getOptions().getDataFormatOptions().useInt64Timestamp())))
         // Return the JobID of the successful job
         .setJobId(
             results.getJobReference() != null ? JobId.fromPb(results.getJobReference()) : null)
@@ -2066,10 +2073,9 @@ && getOptions().getOpenTelemetryTracer() != null) {
     }
     try (Scope queryScope = querySpan != null ? querySpan.makeCurrent() : null) {
       // If all parameters passed in configuration are supported by the query() method on the
-      // backend,
-      // put on fast path
+      // backend, put on fast path
       QueryRequestInfo requestInfo =
-          new QueryRequestInfo(configuration, getOptions().getUseInt64Timestamps());
+          new QueryRequestInfo(configuration, getOptions().getDataFormatOptions());
       if (requestInfo.isFastQuerySupported(jobId)) {
         // Be careful when setting the projectID in JobId, if a projectID is specified in the JobId,
         // the job created by the query method will use that project. This may cause the query to

google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryOptions.java

@@ -17,6 +17,7 @@
 package com.google.cloud.bigquery;
 
 import com.google.api.core.BetaApi;
+import com.google.api.core.ObsoleteApi;
 import com.google.api.gax.retrying.ResultRetryAlgorithm;
 import com.google.cloud.ServiceDefaults;
 import com.google.cloud.ServiceOptions;
@@ -26,6 +27,7 @@
 import com.google.cloud.bigquery.spi.BigQueryRpcFactory;
 import com.google.cloud.bigquery.spi.v2.HttpBigQueryRpc;
 import com.google.cloud.http.HttpTransportOptions;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableSet;
 import io.opentelemetry.api.trace.Tracer;
 import java.util.Set;
@@ -41,6 +43,7 @@ public class BigQueryOptions extends ServiceOptions<BigQuery, BigQueryOptions> {
   // set the option ThrowNotFound when you want to throw the exception when the value not found
   private boolean setThrowNotFound;
   private boolean useInt64Timestamps;
+  private DataFormatOptions dataFormatOptions;
   private JobCreationMode defaultJobCreationMode = JobCreationMode.JOB_CREATION_MODE_UNSPECIFIED;
   private boolean enableOpenTelemetryTracing;
   private Tracer openTelemetryTracer;
@@ -70,6 +73,7 @@ public static class Builder extends ServiceOptions.Builder<BigQuery, BigQueryOpt
 
     private String location;
     private boolean useInt64Timestamps;
+    private DataFormatOptions dataFormatOptions;
     private boolean enableOpenTelemetryTracing;
     private Tracer openTelemetryTracer;
     private ResultRetryAlgorithm<?> resultRetryAlgorithm;
@@ -94,11 +98,32 @@ public Builder setLocation(String location) {
       return this;
     }
 
+    /**
+     * This setter is marked as Obsolete. Prefer {@link #setDataFormatOptions(DataFormatOptions)} to
+     * set the int64timestamp configuration instead.
+     *
+     * <p>If useInt64Timestamps value is set in here and via DataFormatOptions, the
+     * DataFormatOptions configuration value is used.
+     *
+     * <p>{@code DataFormatOptions.newBuilder().setUseInt64Timestamp(...).build()}
+     */
+    @ObsoleteApi("Use setDataFormatOptions(DataFormatOptions) instead")
     public Builder setUseInt64Timestamps(boolean useInt64Timestamps) {
       this.useInt64Timestamps = useInt64Timestamps;
       return this;
     }
 
+    /**
+     * Set the format options for the BigQuery data types
+     *
+     * @param dataFormatOptions Configuration of the formatting options
+     */
+    public Builder setDataFormatOptions(DataFormatOptions dataFormatOptions) {
+      Preconditions.checkNotNull(dataFormatOptions, "DataFormatOptions cannot be null");
+      this.dataFormatOptions = dataFormatOptions;
+      return this;
+    }
+
     /**
      * Enables OpenTelemetry tracing functionality for this BigQuery instance
      *
@@ -143,6 +168,15 @@ private BigQueryOptions(Builder builder) {
     } else {
       this.resultRetryAlgorithm = BigQueryBaseService.DEFAULT_BIGQUERY_EXCEPTION_HANDLER;
     }
+
+    // If dataFormatOptions is not set, then create a new instance and set it with the
+    // useInt64Timestamps configured in BigQueryOptions
+    if (builder.dataFormatOptions == null) {
+      this.dataFormatOptions =
+          DataFormatOptions.newBuilder().useInt64Timestamp(builder.useInt64Timestamps).build();
+    } else {
+      this.dataFormatOptions = builder.dataFormatOptions;
+    }
   }
 
   private static class BigQueryDefaults implements ServiceDefaults<BigQuery, BigQueryOptions> {
@@ -191,8 +225,23 @@ public void setThrowNotFound(boolean setThrowNotFound) {
     this.setThrowNotFound = setThrowNotFound;
   }
 
+  /**
+   * This setter is marked as Obsolete. Prefer {@link
+   * Builder#setDataFormatOptions(DataFormatOptions)} to set the int64timestamp configuration
+   * instead.
+   *
+   * <p>If useInt64Timestamps is set via DataFormatOptions, then the value in DataFormatOptions will
+   * be used. Otherwise, this value will be passed to DataFormatOptions.
+   *
+   * <p>Alternative: {@code DataFormatOptions.newBuilder().setUseInt64Timestamp(...).build()}
+   */
+  @ObsoleteApi("Use Builder#setDataFormatOptions(DataFormatOptions) instead")
   public void setUseInt64Timestamps(boolean useInt64Timestamps) {
     this.useInt64Timestamps = useInt64Timestamps;
+    // Because this setter exists outside the Builder, DataFormatOptions needs be rebuilt to
+    // account for this setting.
+    this.dataFormatOptions =
+        dataFormatOptions.toBuilder().useInt64Timestamp(useInt64Timestamps).build();
   }
 
   @Deprecated
@@ -206,8 +255,22 @@ public boolean getThrowNotFound() {
     return setThrowNotFound;
   }
 
+  /**
+   * This getter is marked as Obsolete. Prefer {@link
+   * DataFormatOptions.Builder#useInt64Timestamp(boolean)} to set the int64timestamp configuration
+   * instead.
+   *
+   * <p>Warning: DataFormatOptions values have precedence. Use {@link
+   * DataFormatOptions#useInt64Timestamp()} to get `useInt64Timestamp` value used by the BigQuery
+   * client.
+   */
+  @ObsoleteApi("Use getDataFormatOptions().isUseInt64Timestamp() instead")
   public boolean getUseInt64Timestamps() {
-    return useInt64Timestamps;
+    return dataFormatOptions.useInt64Timestamp();
+  }
+
+  public DataFormatOptions getDataFormatOptions() {
+    return dataFormatOptions;
   }
 
   public JobCreationMode getDefaultJobCreationMode() {

google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DataFormatOptions.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigquery;
+
+import com.google.auto.value.AutoValue;
+import java.io.Serializable;
+
+/**
+ * Google BigQuery DataFormatOptions. Configures the output format for data types returned from
+ * BigQuery.
+ */
+@AutoValue
+public abstract class DataFormatOptions implements Serializable {
+  public enum TimestampFormatOptions {
+    TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED("TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED"),
+    FLOAT64("FLOAT64"),
+    INT64("INT64"),
+    ISO8601_STRING("ISO8601_STRING");
+
+    private final String format;
+
+    TimestampFormatOptions(String format) {
+      this.format = format;
+    }
+
+    @Override
+    public String toString() {
+      return format;
+    }
+  }
+
+  public abstract boolean useInt64Timestamp();
+
+  public abstract TimestampFormatOptions timestampFormatOptions();
+
+  public static Builder newBuilder() {
+    return new AutoValue_DataFormatOptions.Builder()
+        .useInt64Timestamp(false)
+        .timestampFormatOptions(TimestampFormatOptions.TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED);
+  }
+
+  public abstract Builder toBuilder();
+
+  @AutoValue.Builder
+  public abstract static class Builder {
+    public abstract Builder useInt64Timestamp(boolean useInt64Timestamp);
+
+    public abstract Builder timestampFormatOptions(TimestampFormatOptions timestampFormatOptions);
+
+    public abstract DataFormatOptions build();
+  }
+
+  com.google.api.services.bigquery.model.DataFormatOptions toPb() {
+    com.google.api.services.bigquery.model.DataFormatOptions request =
+        new com.google.api.services.bigquery.model.DataFormatOptions();
+    request.setUseInt64Timestamp(useInt64Timestamp());
+    request.setTimestampOutputFormat(timestampFormatOptions().toString());
+    return request;
+  }
+}