feat: add support for client-side metrics in AppendRowsRequest

google-cloud-bigquery-storage-v1/lib/google/cloud/bigquery/storage/v1/big_query_read/client.rb

@@ -346,7 +346,7 @@ def create_read_session request, options = nil
               #   @param options [::Gapic::CallOptions, ::Hash]
               #     Overrides the default settings for this call, e.g, timeout, retries, etc. Optional.
               #
-              # @overload read_rows(read_stream: nil, offset: nil)
+              # @overload read_rows(read_stream: nil, offset: nil, arrow_serialization_options: nil)
               #   Pass arguments to `read_rows` via keyword arguments. Note that at
               #   least one keyword argument is required. To specify no parameters, or to keep all
               #   the default parameter values, pass an empty Hash as a request object (see above).
@@ -357,6 +357,10 @@ def create_read_session request, options = nil
               #     The offset requested must be less than the last row read from Read.
               #     Requesting a larger offset is undefined. If not specified, start reading
               #     from offset zero.
+              #   @param arrow_serialization_options [::Google::Cloud::Bigquery::Storage::V1::ArrowSerializationOptions, ::Hash]
+              #     Optional. Options specific to the Apache Arrow output format.
+              #
+              #     This feature is not yet available.
               #
               # @yield [response, operation] Access the result along with the RPC operation
               # @yieldparam response [::Enumerable<::Google::Cloud::Bigquery::Storage::V1::ReadRowsResponse>]

google-cloud-bigquery-storage-v1/proto_docs/google/cloud/bigquery/storage/v1/storage.rb

@@ -66,6 +66,11 @@ class CreateReadSessionRequest
           #     The offset requested must be less than the last row read from Read.
           #     Requesting a larger offset is undefined. If not specified, start reading
           #     from offset zero.
+          # @!attribute [rw] arrow_serialization_options
+          #   @return [::Google::Cloud::Bigquery::Storage::V1::ArrowSerializationOptions]
+          #     Optional. Options specific to the Apache Arrow output format.
+          #
+          #     This feature is not yet available.
           class ReadRowsRequest
             include ::Google::Protobuf::MessageExts
             extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -161,6 +166,12 @@ class Progress
           #     follows is not compressed, which can be useful for cases where compression
           #     does not yield appreciable savings. When uncompressed_byte_size is not
           #     greater than 0, the client should skip decompression.
+          # @!attribute [r] total_estimated_row_count
+          #   @return [::Integer]
+          #     Output only. The total estimated number of rows in the query results.
+          #     Only populated when reading data from a BigQuery job.
+          #
+          #     This feature is not yet available.
           class ReadRowsResponse
             include ::Google::Protobuf::MessageExts
             extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -219,7 +230,7 @@ class CreateWriteStreamRequest
           # switching table destinations. You can also switch table destinations within
           # the same connection for the default stream.
           #
-          # The size of a single AppendRowsRequest must be less than 10 MB in size.
+          # The size of a single AppendRowsRequest must be less than 20 MB in size.
           # Requests larger than this return an error, typically `INVALID_ARGUMENT`.
           # @!attribute [rw] write_stream
           #   @return [::String]
@@ -306,6 +317,9 @@ class CreateWriteStreamRequest
           #     `NULL` instead of using default values for some columns, you can set
           #     `default_missing_value_interpretation` to `DEFAULT_VALUE` and at the same
           #     time, set `missing_value_interpretations` to `NULL_VALUE` on those columns.
+          # @!attribute [rw] client_stats
+          #   @return [::Google::Cloud::Bigquery::Storage::V1::ClientStats]
+          #     Optional. Stats and telemetry data gathered on the client side.
           class AppendRowsRequest
             include ::Google::Protobuf::MessageExts
             extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -611,6 +625,71 @@ module RowErrorCode
               FIELDS_ERROR = 1
             end
           end
+
+          # Stats and telemetry data gathered on the client side about requests
+          # being sent to the BigQuery Storage service, for internal use only.
+          # @!attribute [rw] request_stats
+          #   @return [::Google::Cloud::Bigquery::Storage::V1::ClientStats::RequestStats]
+          #     Optional. Per-request stats.
+          # @!attribute [rw] window_stats
+          #   @return [::Google::Cloud::Bigquery::Storage::V1::ClientStats::WindowStats]
+          #     Optional. Windowed stats.
+          class ClientStats
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+
+            # Stats and telemetry data gathered on the client side about a single
+            # request.
+            # @!attribute [rw] send_time_millis
+            #   @return [::Integer]
+            #     Optional. Timestamp indicating when the request was sent over the
+            #     network, expressed in epoch milliseconds.
+            # @!attribute [rw] queued_requests_count
+            #   @return [::Integer]
+            #     Optional. Number of pending requests at the moment this request was sent.
+            #     This includes requests waiting to be sent, and those that are inflight.
+            class RequestStats
+              include ::Google::Protobuf::MessageExts
+              extend ::Google::Protobuf::MessageExts::ClassMethods
+            end
+
+            # Aggregate connection metrics over a window interval.
+            # @!attribute [rw] max_response_latency_millis
+            #   @return [::Integer]
+            #     Optional. The maximum response latency observed in the window, expressed
+            #     in milliseconds.
+            # @!attribute [rw] avg_response_latency_millis
+            #   @return [::Integer]
+            #     Optional. The average response latency observed in the window, expressed
+            #     in milliseconds.
+            # @!attribute [rw] longest_wait_no_response_millis
+            #   @return [::Integer]
+            #     Optional. The longest time spent waiting without receiving a response in
+            #     the window. This could exceed max_response_latency_millis because the
+            #     latter is evaluated only when a response is received. Expressed in
+            #     milliseconds.
+            # @!attribute [rw] requests_sent_count
+            #   @return [::Integer]
+            #     Optional. How many requests were sent in the window.
+            # @!attribute [rw] responses_received_count
+            #   @return [::Integer]
+            #     Optional. How many responses were received in the window.
+            # @!attribute [rw] bytes_sent_count
+            #   @return [::Integer]
+            #     Optional. How many bytes were sent in the window.
+            # @!attribute [rw] window_start_time_epoch_millis
+            #   @return [::Integer]
+            #     Optional. Start time of the window interval for which these stats are
+            #     aggregated, expressed in epoch milliseconds.
+            # @!attribute [rw] window_millis
+            #   @return [::Integer]
+            #     Optional. Duration of the window interval for which these stats are
+            #     aggregated, expressed in milliseconds.
+            class WindowStats
+              include ::Google::Protobuf::MessageExts
+              extend ::Google::Protobuf::MessageExts::ClassMethods
+            end
+          end
         end
       end
     end

google-cloud-bigquery-storage-v1/test/google/cloud/bigquery/storage/v1/big_query_read_test.rb

@@ -137,12 +137,15 @@ def test_read_rows
     # Create request parameters for a server streaming method.
     read_stream = "hello world"
     offset = 42
+    arrow_serialization_options = {}
 
     read_rows_client_stub = ClientStub.new [grpc_response].to_enum, grpc_operation do |name, request, options:|
       assert_equal :read_rows, name
       assert_kind_of ::Google::Cloud::Bigquery::Storage::V1::ReadRowsRequest, request
       assert_equal "hello world", request["read_stream"]
       assert_equal 42, request["offset"]
+      assert_equal Gapic::Protobuf.coerce({}, to: ::Google::Cloud::Bigquery::Storage::V1::ArrowSerializationOptions), request["arrow_serialization_options"]
+      assert_equal :arrow_serialization_options, request.output_format_serialization_options
       refute_nil options
     end
 
@@ -153,7 +156,7 @@ def test_read_rows
       end
 
       # Use hash object
-      c.read_rows({ read_stream: read_stream, offset: offset }) do |response, operation|
+      c.read_rows({ read_stream: read_stream, offset: offset, arrow_serialization_options: arrow_serialization_options }) do |response, operation|
         assert_kind_of Enumerable, response
         response.to_a.each do |r|
           assert_kind_of ::Google::Cloud::Bigquery::Storage::V1::ReadRowsResponse, r
@@ -162,7 +165,7 @@ def test_read_rows
       end
 
       # Use named arguments
-      c.read_rows read_stream: read_stream, offset: offset do |response, operation|
+      c.read_rows read_stream: read_stream, offset: offset, arrow_serialization_options: arrow_serialization_options do |response, operation|
         assert_kind_of Enumerable, response
         response.to_a.each do |r|
           assert_kind_of ::Google::Cloud::Bigquery::Storage::V1::ReadRowsResponse, r
@@ -171,7 +174,7 @@ def test_read_rows
       end
 
       # Use protobuf object
-      c.read_rows ::Google::Cloud::Bigquery::Storage::V1::ReadRowsRequest.new(read_stream: read_stream, offset: offset) do |response, operation|
+      c.read_rows ::Google::Cloud::Bigquery::Storage::V1::ReadRowsRequest.new(read_stream: read_stream, offset: offset, arrow_serialization_options: arrow_serialization_options) do |response, operation|
         assert_kind_of Enumerable, response
         response.to_a.each do |r|
           assert_kind_of ::Google::Cloud::Bigquery::Storage::V1::ReadRowsResponse, r
@@ -180,7 +183,7 @@ def test_read_rows
       end
 
       # Use hash object with options
-      c.read_rows({ read_stream: read_stream, offset: offset }, grpc_options) do |response, operation|
+      c.read_rows({ read_stream: read_stream, offset: offset, arrow_serialization_options: arrow_serialization_options }, grpc_options) do |response, operation|
         assert_kind_of Enumerable, response
         response.to_a.each do |r|
           assert_kind_of ::Google::Cloud::Bigquery::Storage::V1::ReadRowsResponse, r
@@ -189,7 +192,7 @@ def test_read_rows
       end
 
       # Use protobuf object with options
-      c.read_rows(::Google::Cloud::Bigquery::Storage::V1::ReadRowsRequest.new(read_stream: read_stream, offset: offset), grpc_options) do |response, operation|
+      c.read_rows(::Google::Cloud::Bigquery::Storage::V1::ReadRowsRequest.new(read_stream: read_stream, offset: offset, arrow_serialization_options: arrow_serialization_options), grpc_options) do |response, operation|
         assert_kind_of Enumerable, response
         response.to_a.each do |r|
           assert_kind_of ::Google::Cloud::Bigquery::Storage::V1::ReadRowsResponse, r

google-cloud-bigquery-storage-v1/test/google/cloud/bigquery/storage/v1/big_query_write_test.rb

@@ -137,6 +137,7 @@ def test_append_rows
     trace_id = "hello world"
     missing_value_interpretations = {}
     default_missing_value_interpretation = :MISSING_VALUE_INTERPRETATION_UNSPECIFIED
+    client_stats = {}
 
     append_rows_client_stub = ClientStub.new [grpc_response].to_enum, grpc_operation do |name, request, options:|
       assert_equal :append_rows, name
@@ -152,8 +153,8 @@ def test_append_rows
       end
 
       # Use enumerable object with hash and protobuf object.
-      request_hash = { write_stream: write_stream, offset: offset, proto_rows: proto_rows, trace_id: trace_id, missing_value_interpretations: missing_value_interpretations, default_missing_value_interpretation: default_missing_value_interpretation }
-      request_proto = ::Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest.new write_stream: write_stream, offset: offset, proto_rows: proto_rows, trace_id: trace_id, missing_value_interpretations: missing_value_interpretations, default_missing_value_interpretation: default_missing_value_interpretation
+      request_hash = { write_stream: write_stream, offset: offset, proto_rows: proto_rows, trace_id: trace_id, missing_value_interpretations: missing_value_interpretations, default_missing_value_interpretation: default_missing_value_interpretation, client_stats: client_stats }
+      request_proto = ::Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest.new write_stream: write_stream, offset: offset, proto_rows: proto_rows, trace_id: trace_id, missing_value_interpretations: missing_value_interpretations, default_missing_value_interpretation: default_missing_value_interpretation, client_stats: client_stats
       enum_input = [request_hash, request_proto].to_enum
       c.append_rows enum_input do |response, operation|
         assert_kind_of Enumerable, response
@@ -164,8 +165,8 @@ def test_append_rows
       end
 
       # Use stream input object (from gapic-common).
-      request_hash = { write_stream: write_stream, offset: offset, proto_rows: proto_rows, trace_id: trace_id, missing_value_interpretations: missing_value_interpretations, default_missing_value_interpretation: default_missing_value_interpretation }
-      request_proto = ::Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest.new write_stream: write_stream, offset: offset, proto_rows: proto_rows, trace_id: trace_id, missing_value_interpretations: missing_value_interpretations, default_missing_value_interpretation: default_missing_value_interpretation
+      request_hash = { write_stream: write_stream, offset: offset, proto_rows: proto_rows, trace_id: trace_id, missing_value_interpretations: missing_value_interpretations, default_missing_value_interpretation: default_missing_value_interpretation, client_stats: client_stats }
+      request_proto = ::Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest.new write_stream: write_stream, offset: offset, proto_rows: proto_rows, trace_id: trace_id, missing_value_interpretations: missing_value_interpretations, default_missing_value_interpretation: default_missing_value_interpretation, client_stats: client_stats
       stream_input = Gapic::StreamInput.new
       c.append_rows stream_input do |response, operation|
         assert_kind_of Enumerable, response
@@ -179,8 +180,8 @@ def test_append_rows
       stream_input.close
 
       # Use enumerable object with hash and protobuf object with options.
-      request_hash = { write_stream: write_stream, offset: offset, proto_rows: proto_rows, trace_id: trace_id, missing_value_interpretations: missing_value_interpretations, default_missing_value_interpretation: default_missing_value_interpretation }
-      request_proto = ::Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest.new write_stream: write_stream, offset: offset, proto_rows: proto_rows, trace_id: trace_id, missing_value_interpretations: missing_value_interpretations, default_missing_value_interpretation: default_missing_value_interpretation
+      request_hash = { write_stream: write_stream, offset: offset, proto_rows: proto_rows, trace_id: trace_id, missing_value_interpretations: missing_value_interpretations, default_missing_value_interpretation: default_missing_value_interpretation, client_stats: client_stats }
+      request_proto = ::Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest.new write_stream: write_stream, offset: offset, proto_rows: proto_rows, trace_id: trace_id, missing_value_interpretations: missing_value_interpretations, default_missing_value_interpretation: default_missing_value_interpretation, client_stats: client_stats
       enum_input = [request_hash, request_proto].to_enum
       c.append_rows enum_input, grpc_options do |response, operation|
         assert_kind_of Enumerable, response
@@ -191,8 +192,8 @@ def test_append_rows
       end
 
       # Use stream input object (from gapic-common) with options.
-      request_hash = { write_stream: write_stream, offset: offset, proto_rows: proto_rows, trace_id: trace_id, missing_value_interpretations: missing_value_interpretations, default_missing_value_interpretation: default_missing_value_interpretation }
-      request_proto = ::Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest.new write_stream: write_stream, offset: offset, proto_rows: proto_rows, trace_id: trace_id, missing_value_interpretations: missing_value_interpretations, default_missing_value_interpretation: default_missing_value_interpretation
+      request_hash = { write_stream: write_stream, offset: offset, proto_rows: proto_rows, trace_id: trace_id, missing_value_interpretations: missing_value_interpretations, default_missing_value_interpretation: default_missing_value_interpretation, client_stats: client_stats }
+      request_proto = ::Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest.new write_stream: write_stream, offset: offset, proto_rows: proto_rows, trace_id: trace_id, missing_value_interpretations: missing_value_interpretations, default_missing_value_interpretation: default_missing_value_interpretation, client_stats: client_stats
       stream_input = Gapic::StreamInput.new
       c.append_rows stream_input, grpc_options do |response, operation|
         assert_kind_of Enumerable, response
@@ -217,6 +218,7 @@ def test_append_rows
           assert_equal "hello world", r["trace_id"]
           assert_equal({}, r["missing_value_interpretations"].to_h)
           assert_equal :MISSING_VALUE_INTERPRETATION_UNSPECIFIED, r["default_missing_value_interpretation"]
+          assert_equal Gapic::Protobuf.coerce({}, to: ::Google::Cloud::Bigquery::Storage::V1::ClientStats), r["client_stats"]
         end
       end
     end