diff --git a/src/aws/compression/arrow/compress.c b/src/aws/compression/arrow/compress.c index 000d629b553..1cae74c30e6 100644 --- a/src/aws/compression/arrow/compress.c +++ b/src/aws/compression/arrow/compress.c @@ -15,6 +15,20 @@ #include #include +static int choose_block_size(size_t size) +{ + int block_size = 8 * 1024 * 1024; + + while ((size_t) block_size <= size) { + block_size *= 2; + if (block_size > 64 * 1024 * 1024) { + return 64 * 1024 * 1024; + } + } + + return block_size; +} + /* * GArrowTable is the central structure that represents "table" (a.k.a. * data frame). @@ -46,6 +60,10 @@ static GArrowTable* parse_json(uint8_t *json, int size) return NULL; } + g_object_set(options, + "block-size", choose_block_size(size), + NULL); + reader = garrow_json_reader_new(GARROW_INPUT_STREAM(input), options, &error); if (reader == NULL) { g_error_free(error);