Split Parquet tail work into morsels

datafusion/common/src/config.rs

@@ -979,6 +979,13 @@ config_namespace! {
         /// parquet reader setting. 0 means no caching.
         pub max_predicate_cache_size: Option<usize>, default = None
 
+        /// (reading) Target size, in compressed bytes, of the morsels a file
+        /// scan is split into when too little work remains to keep all output
+        /// streams busy (the tail of a scan). Row groups larger than this are
+        /// split at page-aligned row ranges and the surplus morsels are shared
+        /// with idle sibling streams. If NULL, splitting is disabled.
+        pub morsel_split_size: Option<usize>, default = Some(1024 * 1024)
+
         // The following options affect writing to parquet files
         // and map to parquet::file::properties::WriterProperties
 

datafusion/common/src/file_options/parquet_writer.rs

@@ -248,6 +248,7 @@ impl ParquetOptions {
             coerce_int96_tz: _,  // not used for writer props
             skip_arrow_metadata: _,
             max_predicate_cache_size: _,
+            morsel_split_size: _, // reads not used for writer props
         } = self;
 
         let mut builder = WriterProperties::builder()
@@ -506,6 +507,7 @@ mod tests {
             coerce_int96: None,
             coerce_int96_tz: None,
             max_predicate_cache_size: defaults.max_predicate_cache_size,
+            morsel_split_size: defaults.morsel_split_size,
             content_defined_chunking: defaults.content_defined_chunking.clone(),
         }
     }
@@ -622,6 +624,7 @@ mod tests {
                 bloom_filter_on_read: global_options_defaults.bloom_filter_on_read,
                 max_predicate_cache_size: global_options_defaults
                     .max_predicate_cache_size,
+                morsel_split_size: global_options_defaults.morsel_split_size,
                 schema_force_view_types: global_options_defaults.schema_force_view_types,
                 binary_as_string: global_options_defaults.binary_as_string,
                 skip_arrow_metadata: global_options_defaults.skip_arrow_metadata,