diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d05fba..10c0222 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,6 +84,7 @@ jobs: - name: Verify Java lanes against shared fidelity fixtures env: + RAINCLOUD_SIDECAR_PARQUET_JAVA: ${{ github.workspace }}/sidecars/java/parquet-java/build/install/raincloud-export-parquet-java/bin/raincloud-export-parquet-java RAINCLOUD_READER_PARQUET_JAVA: ${{ github.workspace }}/sidecars/java/parquet-java/build/install/raincloud-export-parquet-java/bin/raincloud-read-parquet-java RAINCLOUD_SIDECAR_PARQUET_HARDWOOD: ${{ github.workspace }}/sidecars/java/parquet-hardwood/build/install/raincloud-export-parquet-hardwood/bin/raincloud-export-parquet-hardwood RAINCLOUD_READER_PARQUET_HARDWOOD: ${{ github.workspace }}/sidecars/java/parquet-hardwood/build/install/raincloud-export-parquet-hardwood/bin/raincloud-read-parquet-hardwood @@ -91,7 +92,7 @@ jobs: RAINCLOUD_READER_VORTEX_JNI: ${{ github.workspace }}/sidecars/java/vortex-jni-reader/build/install/raincloud-read-vortex-jni/bin/raincloud-read-vortex-jni RAINCLOUD_SIDECAR_AVRO_JAVA: ${{ github.workspace }}/sidecars/java/avro-java/build/install/raincloud-export-avro-java/bin/raincloud-export-avro-java RAINCLOUD_READER_AVRO_JAVA: ${{ github.workspace }}/sidecars/java/avro-java/build/install/raincloud-export-avro-java/bin/raincloud-read-avro-java - run: uv run --no-sync pytest -q tests/test_reader_fidelity.py tests/test_java_reader_schema_compare.py tests/test_jvm_sidecar_lanes.py + run: uv run --no-sync pytest -q tests/test_reader_fidelity.py tests/test_java_reader_schema_compare.py tests/test_jvm_sidecar_lanes.py tests/test_parquet_options.py wheel: runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index dcb7cea..3f83bbb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -149,9 +149,10 @@ Every Parquet writer is given one set of options (`spec.parquet_options`): the r `write.compression`, `write.statistics` and row cap, and the install's `RAINCLOUD_PARQUET_*` settings in the table below (compression level, statistics and the page index for all or the first N columns, page size and rows, dictionaries, page -checksums). An unset setting leaves each library's own default, which differ, except -that `parquet@py` writes a page index (when statistics are on) and page checksums, as -parquet-java does, where pyarrow's own default writes neither. A set one reaches every writer and +checksums). Page indexes and checksums default on wherever supported: pyarrow and +parquet-java write both, arrow-rs cannot write checksums, and Hardwood cannot write +indexes. Page statistics require the recipe's statistics to be enabled. Other unset +settings use each library's defaults. A set one reaches every writer and becomes part of its toolchain, and a writer whose library cannot do what it asks fails that export as a measurement rather than writing something else; `sidecars/README.md` tabulates which writer honours what, and the `/raincloud-write-settings` skill is the @@ -279,13 +280,13 @@ its own `outputs/` rather than a machine's shared store. | `RAINCLOUD_ROW_GROUP_PROBE_ROWS` | rows the Python Parquet writer samples to size its groups (must be > 0) | 262,144 | | `RAINCLOUD_PARQUET_COMPRESSION_LEVEL` | the recipe's codec at this level in every Parquet writer (zstd 1-22, gzip 0-9, brotli 0-11) | unset (each writer's own) | | `RAINCLOUD_PARQUET_STATISTICS_COLUMNS` | statistics (chunk and page) only for the first N leaf columns (`0`: every column) | unset (each writer's own: every column) | -| `RAINCLOUD_PARQUET_PAGE_INDEX` | `1`: every Parquet writer writes a page index (ColumnIndex + OffsetIndex); `0`: none | unset (each writer's own: pyarrow (raincloud's default for `parquet@py`), arrow-rs and parquet-java one, Hardwood none) | +| `RAINCLOUD_PARQUET_PAGE_INDEX` | `1`: every Parquet writer writes a page index (ColumnIndex + OffsetIndex); `0`: none | on where supported: pyarrow, arrow-rs and parquet-java; Hardwood cannot write one; page statistics require statistics on | | `RAINCLOUD_PARQUET_PAGE_INDEX_COLUMNS` | page statistics only for the first N leaf columns, chunk statistics for every column (`0`: every column) | unset (each writer's own) | | `RAINCLOUD_PARQUET_PAGE_BYTES` | data page size target in every Parquet writer, each measuring a page its own way (`0`: no limit) | unset (each writer's own) | | `RAINCLOUD_PARQUET_PAGE_ROWS` | data page row limit in every Parquet writer (`0`: no limit) | unset (each writer's own) | | `RAINCLOUD_PARQUET_DICTIONARY` | `0`: no dictionary encoding (PLAIN); `1`: dictionaries | unset (each writer's own: on) | | `RAINCLOUD_PARQUET_DICTIONARY_PAGE_BYTES` | dictionary page size limit, past which a column falls back to PLAIN (`0`: no limit) | unset (each writer's own) | -| `RAINCLOUD_PARQUET_PAGE_CHECKSUMS` | `1`: a CRC in every page header; `0`: none | unset (each writer's own: pyarrow (raincloud's default for `parquet@py`), parquet-java and Hardwood write them, arrow-rs does not) | +| `RAINCLOUD_PARQUET_PAGE_CHECKSUMS` | `1`: a CRC in every page header; `0`: none | on where supported: pyarrow, parquet-java and Hardwood; arrow-rs cannot write them | | `RAINCLOUD_ORC_COMPRESSION` | ORC codec in every ORC writer: `zstd`, `snappy`, `zlib`, `lz4`, `none` | unset (zstd, as always) | | `RAINCLOUD_ORC_COMPRESSION_STRATEGY` | `speed` or `compression` (ORC C++ only; orc-rust refuses) | unset (each writer's own) | | `RAINCLOUD_ORC_STRIPE_BYTES` | ORC stripe size target, measured encoded and compressed | unset (each writer's own) | diff --git a/CHANGELOG.md b/CHANGELOG.md index aba1879..f1f9005 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,12 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht pyarrow's own default writes neither. Readers can prune pages by their statistics without a sidecar writer. Set either to `0` for the old files. Every Parquet artifact `parquet@py` writes changes bytes and checksums. +- **Parquet page defaults are explicit across writers.** Page indexes and checksums + default on wherever the pinned writer supports them. Parquet-java explicitly + enables checksums; arrow-rs explicitly enables page indexes with statistics and + now disables the OffsetIndex too when the recipe turns statistics off. Arrow-rs + cannot write page checksums, and Hardwood cannot write page indexes; those + limitations remain. Explicit settings still win or are refused when unsupported. ## [0.3.1] - 2026-10-08 diff --git a/README.md b/README.md index 5a26f28..7c4fff5 100644 --- a/README.md +++ b/README.md @@ -281,10 +281,12 @@ is measured in days. Build what you need. for Parquet a page index (ColumnIndex and OffsetIndex) on every column or only the first N, statistics for only the first N columns, the compression level, page size and rows, dictionaries and page checksums; for ORC and Avro the codec, level and block sizes; for -Vortex compact encodings. Unset, each writer library's default applies, with one -exception: pyarrow, the default Parquet writer, writes a page index and page checksums, -as parquet-java does, though pyarrow's own default writes neither. To build a dataset's -Parquet without them: +Vortex compact encodings. Parquet page indexes and checksums default on wherever the +writer supports them: pyarrow and parquet-java write both, arrow-rs writes indexes +but cannot write checksums, and Hardwood writes checksums but cannot write indexes. +Page statistics require the recipe's statistics to be enabled. Other unset settings +use each library's defaults. To build a dataset's Parquet without indexes or checksums +using the default pyarrow writer: ```bash RAINCLOUD_PARQUET_PAGE_INDEX=0 RAINCLOUD_PARQUET_PAGE_CHECKSUMS=0 raincloud build uci-iris --format parquet @@ -293,6 +295,7 @@ RAINCLOUD_PARQUET_PAGE_INDEX=0 RAINCLOUD_PARQUET_PAGE_CHECKSUMS=0 raincloud buil ```python import os, raincloud os.environ["RAINCLOUD_PARQUET_PAGE_INDEX"] = "0" # before the build runs +os.environ["RAINCLOUD_PARQUET_PAGE_CHECKSUMS"] = "0" ds = raincloud.load("uci-iris", format="parquet", build=True) ``` diff --git a/raincloud/pipeline/spec.py b/raincloud/pipeline/spec.py index 8aabe98..b2c707a 100644 --- a/raincloud/pipeline/spec.py +++ b/raincloud/pipeline/spec.py @@ -234,13 +234,11 @@ def row_group_probe_rows() -> int: # Parquet write options every Parquet writer receives the same way. The install -# settings differ from the row-group knobs in one respect: UNSET means each -# writer's own default, not a raincloud figure, because the four libraries' -# defaults differ (arrow-rs and parquet-java write a page index, Hardwood does -# not) and no one figure leaves every lane's files as they are. The exception is -# parquet@py, which writes a page index and page checksums when unset, as -# parquet-java does (`exporters._writer_options`); pyarrow's own default writes -# neither. +# settings retain None when unset so each lane can apply the defaults its +# library supports: page indexes and checksums on, except Hardwood cannot +# write indexes and arrow-rs cannot write checksums. Other unset options use +# the library's defaults. Resolving those here would turn an unsupported +# default into an explicit request that the lane must refuse. # Set, a setting reaches every lane, and a writer whose library cannot do what # it asks fails that export rather than writing something else. PARQUET_COMPRESSION = "RAINCLOUD_PARQUET_COMPRESSION" @@ -327,7 +325,8 @@ class ParquetOptions: `compression` and `statistics` are the recipe's `write.compression` and `write.statistics`. The rest are install settings (`_PARQUET_SETTINGS`); - None leaves the writer's own default: + None leaves the lane's default (page indexes and checksums on where supported, + other options at the library's defaults): - `compression_level`: the codec's level (zstd 1-22, gzip 0-9, brotli 0-11); - `statistics_columns`: statistics only for the first N leaf columns; diff --git a/scripts/test_native_readers.py b/scripts/test_native_readers.py index d04ddb3..c9837a3 100644 --- a/scripts/test_native_readers.py +++ b/scripts/test_native_readers.py @@ -38,7 +38,7 @@ "tests/test_client_boundaries.py", "tests/test_export_conformance.py", "tests/test_rust_sidecars.py", "tests/test_native_protocol.py", "tests/test_jvm_sidecar_lanes.py", "tests/test_parquet_variant_sidecars.py", "tests/test_parquet_java_list_naming.py", - "tests/test_variant_storage_nullability.py", + "tests/test_variant_storage_nullability.py", "tests/test_parquet_options.py", ) LIBRARY = "libraincloud_reader.so" diff --git a/sidecars/README.md b/sidecars/README.md index 92f9ed4..65931fd 100644 --- a/sidecars/README.md +++ b/sidecars/README.md @@ -170,10 +170,13 @@ passes to a sidecar in this form: | `RAINCLOUD_PARQUET_DICTIONARY_PAGE_BYTES` | dictionary page size limit | the install, only when set | | `RAINCLOUD_PARQUET_PAGE_CHECKSUMS` | `1` or `0`: a CRC in every page header, or none | the install, only when set | -An unset variable is the lane's library default (zstd and statistics on, for the first -two), except that `parquet@py` writes a page index and page checksums when the two are -unset, though pyarrow's own default writes neither. Counts use the count grammar, `0` meaning no limit (every column, for the two -column counts). Switches read `1/true/yes/on` and `0/false/no/off` in any case, empty as +Page indexes and checksums default on wherever the pinned writer supports them: +`parquet@py` and `parquet@java` write both, `parquet@rs` writes indexes but cannot write +checksums, and `parquet@hardwood` writes checksums but cannot write indexes. Page +statistics require the recipe's statistics to be enabled. Other unset variables use +the library's defaults (zstd and statistics on, for the first two). Counts use the +count grammar, `0` meaning no limit (every column, for the two column counts). +Switches read `1/true/yes/on` and `0/false/no/off` in any case, empty as unset, in every lane. Settings that contradict each other or the recipe (a page index with statistics off; `PAGE_INDEX=0` with `PAGE_INDEX_COLUMNS`; a level for snappy, lz4 or none, or out of the codec's range) are refused for every lane before any writer runs. @@ -186,19 +189,21 @@ cannot honour: | compression level | yes | yes | yes | no | | statistics off | yes | yes | yes | no | | statistics, first N columns | yes | yes | yes | no | -| page index on | yes (raincloud's default) | yes | yes | no | +| page index on | yes (default) | yes (default) | yes (default) | no | | page index off | yes | yes | no, with statistics on | yes | | page index, first N columns | no: all or none | yes | no | no | | page bytes | yes | yes | yes | yes | | page rows | yes | yes, checked every 1,024 values | yes | no | | dictionary on / off | yes | yes | yes | yes (off is PLAIN) | | dictionary page bytes | yes | yes | yes | no | -| page checksums on | yes (raincloud's default) | no | yes (its default) | yes (always) | +| page checksums on | yes (default) | no | yes (default) | yes (always) | | page checksums off | yes | yes (always) | yes | no | Each library measures a page its own way, so the same `RAINCLOUD_PARQUET_PAGE_BYTES` does not give identical pages in every lane. The java column's page index gaps are -parquet-java's: it writes a page index for every column that has statistics. +parquet-java's: it writes a ColumnIndex for every column that has statistics and +may retain an OffsetIndex with statistics off. Pyarrow and arrow-rs disable both +indexes when the recipe turns statistics off and no index setting is given. The ORC, Avro and Vortex writers read their own settings the same way (`spec.FORMAT_SETTINGS`): the build passes each set one, in canonical form, under diff --git a/sidecars/java/conformance-common/src/main/java/dev/raincloud/sidecar/common/ParquetKnobs.java b/sidecars/java/conformance-common/src/main/java/dev/raincloud/sidecar/common/ParquetKnobs.java index 99e483f..e0a564b 100644 --- a/sidecars/java/conformance-common/src/main/java/dev/raincloud/sidecar/common/ParquetKnobs.java +++ b/sidecars/java/conformance-common/src/main/java/dev/raincloud/sidecar/common/ParquetKnobs.java @@ -11,7 +11,8 @@ * {@code ParquetOptions} in the Rust lane). The sidecar never sees the recipe: * {@code SidecarExporter} passes its {@code write.compression} and {@code write.statistics} as * {@link #COMPRESSION} and {@link #STATISTICS}, and each install setting only when it is set. An - * unset setting ({@code null} here) leaves the writer library's own default; a lane whose library + * unset setting ({@code null} here) leaves the lane's default: page indexes and checksums are + * enabled where supported, other settings use the library's defaults. A lane whose library * cannot do what a set one asks refuses it ({@link #unsupported}) rather than writing something * else. A count of {@link #NO_LIMIT} is no limit ({@code 0} in the environment). */ @@ -34,7 +35,7 @@ public record ParquetKnobs(String compression, Integer compressionLevel, boolean /** What 0 (no limit) means for a count. */ public static final int NO_LIMIT = WriteSettings.NO_LIMIT; - /** Every option unset: zstd, statistics on, each library's own defaults. */ + /** Every option unset: zstd, statistics on, each lane's defaults. */ public static final ParquetKnobs DEFAULT = new ParquetKnobs("zstd", null, true, null, null, null, null, null, null, null, null); diff --git a/sidecars/java/parquet-java/src/main/java/dev/raincloud/sidecar/parquetjava/ParquetArrowIo.java b/sidecars/java/parquet-java/src/main/java/dev/raincloud/sidecar/parquetjava/ParquetArrowIo.java index 5d67650..a3dd312 100644 --- a/sidecars/java/parquet-java/src/main/java/dev/raincloud/sidecar/parquetjava/ParquetArrowIo.java +++ b/sidecars/java/parquet-java/src/main/java/dev/raincloud/sidecar/parquetjava/ParquetArrowIo.java @@ -109,6 +109,7 @@ private static WriteOptions.Builder optionsBuilder(String maxRows, String target WriteOptions.Builder builder = WriteOptions.builder() .compression(compression(knobs.compression())) .statisticsEnabled(knobs.statistics()) + .pageChecksums(knobs.pageChecksums() == null || knobs.pageChecksums()) .maxRowGroupRows(rowGroupMaxRows(maxRows)) .targetRowGroupBytes(rowGroupTargetEncodedBytes(targetEncodedBytes)); if (knobs.compressionLevel() != null) { @@ -126,9 +127,6 @@ private static WriteOptions.Builder optionsBuilder(String maxRows, String target if (knobs.dictionaryPageBytes() != null) { builder.dictionaryPageSizeBytes(knobs.dictionaryPageBytes()); } - if (knobs.pageChecksums() != null) { - builder.pageChecksums(knobs.pageChecksums()); - } return builder; } diff --git a/sidecars/java/parquet-java/src/test/java/dev/raincloud/sidecar/parquetjava/ParquetArrowIoTest.java b/sidecars/java/parquet-java/src/test/java/dev/raincloud/sidecar/parquetjava/ParquetArrowIoTest.java index b5884ba..f6fbc04 100644 --- a/sidecars/java/parquet-java/src/test/java/dev/raincloud/sidecar/parquetjava/ParquetArrowIoTest.java +++ b/sidecars/java/parquet-java/src/test/java/dev/raincloud/sidecar/parquetjava/ParquetArrowIoTest.java @@ -193,12 +193,12 @@ void parquetKnobs_reachTheWriter() { WriteOptions leveled = ParquetArrowIo.writeOptions(null, null, knobs("COMPRESSION_LEVEL", "9")); assertEquals(9, leveled.compressionLevel().getAsInt()); assertEquals(Compression.LZ4_RAW, ParquetArrowIo.writeOptions(null, null, knobs("COMPRESSION", "lz4")).compression()); - // Unset leaves parquet-arrow-java's defaults. + // Page sizes remain the library's defaults; checksums default on in Raincloud. WriteOptions defaults = WriteOptions.builder().build(); WriteOptions unset = ParquetArrowIo.writeOptions(null, null, ParquetKnobs.DEFAULT); assertEquals(defaults.pageSizeBytes(), unset.pageSizeBytes()); assertEquals(defaults.pageRowLimit(), unset.pageRowLimit()); - assertEquals(defaults.pageChecksums(), unset.pageChecksums()); + assertTrue(unset.pageChecksums()); } @Test diff --git a/sidecars/rust/src/lib.rs b/sidecars/rust/src/lib.rs index f570591..9b1f556 100644 --- a/sidecars/rust/src/lib.rs +++ b/sidecars/rust/src/lib.rs @@ -324,8 +324,9 @@ fn refuse_set( /// sidecar never sees the recipe: `SidecarExporter` passes its /// `write.compression` and `write.statistics` as `RAINCLOUD_PARQUET_COMPRESSION` /// and `RAINCLOUD_PARQUET_STATISTICS`, and each install setting only when it is -/// set. Unset leaves arrow-rs's own default: a page index, 1 MiB pages, 20,000 -/// rows a page, dictionaries on, no page checksums. +/// set. A page index defaults on when statistics are enabled. Other unset +/// settings leave arrow-rs's defaults: 1 MiB pages, 20,000 rows a page, +/// dictionaries on. This arrow-rs version cannot write page checksums. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ParquetOptions { /// The codec, at `RAINCLOUD_PARQUET_COMPRESSION_LEVEL` when that is set. @@ -480,21 +481,18 @@ impl ParquetOptions { columns: &parquet::schema::types::SchemaDescriptor, ) -> parquet::file::properties::WriterPropertiesBuilder { use parquet::file::properties::EnabledStatistics; - if !self.statistics { - builder = builder.set_statistics_enabled(EnabledStatistics::None); - } - match self.page_index { - Some(true) => builder = builder.set_statistics_enabled(EnabledStatistics::Page), - // Neither index, as pyarrow writes without one: Chunk statistics alone - // still write an OffsetIndex. - Some(false) => { - if self.statistics { - builder = builder.set_statistics_enabled(EnabledStatistics::Chunk); - } - builder = builder.set_offset_index_disabled(true); - } - None => {} - } + let page_index = self.page_index.unwrap_or(self.statistics); + builder = builder + .set_statistics_enabled(if !self.statistics { + EnabledStatistics::None + } else if page_index { + EnabledStatistics::Page + } else { + EnabledStatistics::Chunk + }) + // Neither index when disabled: chunk statistics alone still write + // an OffsetIndex. Match parquet@py when statistics are off too. + .set_offset_index_disabled(!page_index); if self.statistics { for (i, column) in columns.columns().iter().enumerate() { let path = column.path().clone(); @@ -2522,6 +2520,14 @@ mod tests { ..Default::default() }; assert_eq!(written("without.parquet", without), (false, None)); + let no_statistics = ParquetOptions { + statistics: false, + ..Default::default() + }; + assert_eq!( + written("no-statistics.parquet", no_statistics), + (false, None) + ); } #[test] diff --git a/tests/test_docs_contracts.py b/tests/test_docs_contracts.py index f9ff3a9..eaa77ab 100644 --- a/tests/test_docs_contracts.py +++ b/tests/test_docs_contracts.py @@ -363,13 +363,16 @@ def test_agents_row_group_defaults_match_the_code(monkeypatch): assert row.rstrip(" |").endswith(spelled), row -def test_agents_parquet_settings_default_to_each_writers_own(monkeypatch): +def test_agents_writer_settings_document_supported_defaults(monkeypatch): from raincloud.pipeline import spec table = (ROOT / "AGENTS.md").read_text() for var in spec.PARQUET_SETTING_VARS: monkeypatch.delenv(var, raising=False) row = next(line for line in table.splitlines() if line.startswith(f"| `{var}`")) - assert "unset (each writer's own" in row, row + if var in ("RAINCLOUD_PARQUET_PAGE_INDEX", "RAINCLOUD_PARQUET_PAGE_CHECKSUMS"): + assert "on where supported:" in row, row + else: + assert "unset (each writer's own" in row, row assert spec.parquet_page_options().chosen() == {} for settings in spec.FORMAT_SETTINGS.values(): for _, var, _ in settings: diff --git a/tests/test_parquet_options.py b/tests/test_parquet_options.py index 3cae9fb..7ac6e51 100644 --- a/tests/test_parquet_options.py +++ b/tests/test_parquet_options.py @@ -1,8 +1,8 @@ # SPDX-FileCopyrightText: 2026 Raincloud Maintainers # SPDX-License-Identifier: Apache-2.0 """The Parquet write options: one set (`spec.ParquetOptions`) given the same way -to every Parquet writer, each page knob leaving the writer's own default when -unset, and a writer that cannot do what a set knob asks failing rather than +to every Parquet writer, page indexes and checksums on by default where supported, +and a writer that cannot do what a set knob asks failing rather than writing something else.""" from __future__ import annotations @@ -27,22 +27,25 @@ def _unset(monkeypatch): monkeypatch.delenv(var, raising=False) -def _data_pages(path, column: int = 0) -> list[dict]: - """The data page headers of one column chunk of the first row group, read +def _pages(path, column: int = 0, row_group: int = 0) -> list[dict]: + """The page headers of one column chunk, including its dictionary, read from the file: {thrift field id: value}, as parquet.thrift numbers them.""" meta = pq.ParquetFile(path).metadata - chunk = meta.row_group(0).column(column) + chunk = meta.row_group(row_group).column(column) data = path.read_bytes() start = chunk.dictionary_page_offset if chunk.has_dictionary_page else chunk.data_page_offset pos, end, pages = start, start + chunk.total_compressed_size, [] while pos < end: header, pos = _thrift_struct(data, pos) pos += header[3] # compressed_page_size - if header[1] in (0, 3): # DATA_PAGE, DATA_PAGE_V2 - pages.append(header) + pages.append(header) return pages +def _data_pages(path, column: int = 0) -> list[dict]: + return [page for page in _pages(path, column) if page[1] in (0, 3)] # DATA_PAGE, DATA_PAGE_V2 + + def _thrift_struct(data: bytes, pos: int) -> tuple[dict, int]: """A thrift compact-protocol struct at `pos`: ({field id: value}, end).""" def varint(): @@ -227,17 +230,44 @@ def test_no_page_index_is_written_on_request_or_refused(tmp_path, monkeypatch, c assert _layout(dest)[0] is False -def test_parquet_py_writes_a_page_index_and_page_checksums_when_unset(tmp_path): - roundtrip, note, dest = _write(tmp_path, "parquet@py") +@pytest.mark.parametrize("cell", CELLS) +def test_page_indexes_and_checksums_default_on_where_supported(tmp_path, cell): + written = _write(tmp_path, cell) + if written is None: + pytest.skip(f"{cell} not installed") + roundtrip, note, dest = written assert roundtrip is True, note - assert _layout(dest)[0] is True - assert all(4 in page for page in _data_pages(dest)) # crc - # A page index is page statistics: with the recipe's statistics off there is none. - (tmp_path / "bare").mkdir() + meta = pq.ParquetFile(dest).metadata + for g in range(meta.num_row_groups): + for c in range(meta.num_columns): + chunk = meta.row_group(g).column(c) + assert chunk.has_column_index == (cell != "parquet@hardwood") + assert chunk.has_offset_index == (cell != "parquet@hardwood") + pages = _pages(dest, c, g) + assert pages + assert all((4 in page) == (cell != "parquet@rs") for page in pages) # crc + + +@pytest.mark.parametrize("cell", CELLS) +def test_default_page_index_respects_statistics_off(tmp_path, cell): bare = {**SPEC, "write": {**SPEC["write"], "statistics": False}} - roundtrip, note, dest = _write(tmp_path / "bare", "parquet@py", bare) + written = _write(tmp_path, cell, bare) + if written is None: + pytest.skip(f"{cell} not installed") + roundtrip, note, dest = written + if cell == "parquet@hardwood": + assert roundtrip is False and "cannot honour RAINCLOUD_PARQUET_STATISTICS=0" in note, note + assert not dest.exists() + return assert roundtrip is True, note - assert _layout(dest)[0] is False + meta = pq.ParquetFile(dest).metadata + for g in range(meta.num_row_groups): + for c in range(meta.num_columns): + chunk = meta.row_group(g).column(c) + assert not chunk.is_stats_set + assert not chunk.has_column_index + if cell != "parquet@java": # parquet-java may keep the positional OffsetIndex. + assert not chunk.has_offset_index @pytest.mark.parametrize("cell", CELLS)