Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.12.7
rev: v0.16.4
hooks:
- id: ruff-check
args: [--fix]
- id: ruff-format
- repo: https://github.com/astral-sh/uv-pre-commit
rev: 0.7.15
rev: 0.12.5
hooks:
- id: uv-lock
- repo: local
Expand All @@ -18,7 +18,7 @@ repos:
types_or: [python, pyi]
pass_filenames: false
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
rev: v6.0.0
hooks:
- id: mixed-line-ending
- id: end-of-file-fixer
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ exclude = ["src/resolver_athena_client/generated/*", "docs"]

[tool.ruff.lint]
select = ["ALL"]
ignore = ["COM812", "D213", "D211", "D203", "S324", "ASYNC109"]
ignore = ["COM812", "D213", "D211", "D203", "S324", "ASYNC109", "CPY001"]

[tool.ruff.lint.per-file-ignores]
# Ignore doc lint rules in tests.
Expand Down
9 changes: 6 additions & 3 deletions src/resolver_athena_client/client/athena_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
)
from resolver_athena_client.client.transformers.worker_batcher import (
WorkerBatcher,
WorkerBatcherOptions,
)
from resolver_athena_client.generated.athena.models_pb2 import (
ClassificationInput,
Expand Down Expand Up @@ -289,9 +290,11 @@ async def transform_image(image_data: ImageData) -> ClassificationInput:
source=images,
transformer_func=transform_image,
deployment_id=self.options.deployment_id,
max_batch_size=self.options.max_batch_size,
num_workers=self.options.num_workers,
keepalive_interval=self.options.keepalive_interval or 30.0,
options=WorkerBatcherOptions(
max_batch_size=self.options.max_batch_size,
num_workers=self.options.num_workers,
keepalive_interval=self.options.keepalive_interval or 30.0,
),
)

# Track the worker for cleanup
Expand Down
73 changes: 31 additions & 42 deletions src/resolver_athena_client/client/image_format_detector.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
"""Utility for detecting image formats from raw bytes."""

from collections.abc import Callable

from resolver_athena_client.generated.athena.models_pb2 import ImageFormat

PNG_MAGIC_BYTES = b"\x89PNG"
Expand All @@ -13,7 +15,32 @@
TIFF_BE_MAGIC_BYTES = b"MM\x00*"


def detect_image_format(data: bytes) -> ImageFormat.ValueType: # noqa: PLR0911
def _is_webp(data: bytes) -> bool:
"""Check for the RIFF....WEBP signature (12 bytes minimum)."""
return (
data[:4] == WEBP_RIFF_MAGIC_BYTES
and data[8:12] == WEBP_WEBP_MAGIC_BYTES
)


_ImageFormatDetector = tuple[Callable[[bytes], bool], ImageFormat.ValueType]
_FORMAT_DETECTORS: list[_ImageFormatDetector] = [
(lambda d: d.startswith(PNG_MAGIC_BYTES), ImageFormat.IMAGE_FORMAT_PNG),
(lambda d: d.startswith(JPEG_MAGIC_BYTES), ImageFormat.IMAGE_FORMAT_JPEG),
(
lambda d: d.startswith((GIF87A_MAGIC_BYTES, GIF89A_MAGIC_BYTES)),
ImageFormat.IMAGE_FORMAT_GIF,
),
(lambda d: d.startswith(BMP_MAGIC_BYTES), ImageFormat.IMAGE_FORMAT_BMP),
(_is_webp, ImageFormat.IMAGE_FORMAT_WEBP),
(
lambda d: d.startswith((TIFF_LE_MAGIC_BYTES, TIFF_BE_MAGIC_BYTES)),
ImageFormat.IMAGE_FORMAT_TIFF,
),
]


def detect_image_format(data: bytes) -> ImageFormat.ValueType:
"""Detect image format from raw bytes using magic number signatures.

Args:
Expand All @@ -28,46 +55,8 @@ def detect_image_format(data: bytes) -> ImageFormat.ValueType: # noqa: PLR0911
if not data:
return ImageFormat.IMAGE_FORMAT_UNSPECIFIED

# Check magic numbers for common image formats
# PNG: starts with PNG_MAGIC_BYTES
png_len = len(PNG_MAGIC_BYTES)
if len(data) >= png_len and data[:png_len] == PNG_MAGIC_BYTES:
return ImageFormat.IMAGE_FORMAT_PNG

# JPEG: starts with JPEG_MAGIC_BYTES
jpeg_len = len(JPEG_MAGIC_BYTES)
if len(data) >= jpeg_len and data[:jpeg_len] == JPEG_MAGIC_BYTES:
return ImageFormat.IMAGE_FORMAT_JPEG

# GIF: starts with GIF87A_MAGIC_BYTES or GIF89A_MAGIC_BYTES
gif_len = len(GIF87A_MAGIC_BYTES)
if len(data) >= gif_len and data[:gif_len] in (
GIF87A_MAGIC_BYTES,
GIF89A_MAGIC_BYTES,
):
return ImageFormat.IMAGE_FORMAT_GIF

# BMP: starts with BMP_MAGIC_BYTES
bmp_len = len(BMP_MAGIC_BYTES)
if len(data) >= bmp_len and data[:bmp_len] == BMP_MAGIC_BYTES:
return ImageFormat.IMAGE_FORMAT_BMP

# WebP: RIFF....WEBP (12 bytes minimum for full signature)
webp_min_len = len(WEBP_RIFF_MAGIC_BYTES) + len(WEBP_WEBP_MAGIC_BYTES) + 4
if (
len(data) >= webp_min_len
and data[:4] == WEBP_RIFF_MAGIC_BYTES
and data[8:12] == WEBP_WEBP_MAGIC_BYTES
):
return ImageFormat.IMAGE_FORMAT_WEBP

# TIFF: little-endian or big-endian magic bytes
tiff_len = len(TIFF_LE_MAGIC_BYTES)
if len(data) >= tiff_len and (
data[:tiff_len] == TIFF_LE_MAGIC_BYTES
or data[:tiff_len] == TIFF_BE_MAGIC_BYTES
):
return ImageFormat.IMAGE_FORMAT_TIFF
for matches, image_format in _FORMAT_DETECTORS:
if matches(data):
return image_format

# Fallback when format cannot be determined
return ImageFormat.IMAGE_FORMAT_UNSPECIFIED
41 changes: 24 additions & 17 deletions src/resolver_athena_client/client/transformers/worker_batcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import logging
import time
from collections.abc import AsyncIterator, Awaitable, Callable
from dataclasses import dataclass
from typing import Generic, TypeVar

from resolver_athena_client.generated.athena.models_pb2 import (
Expand All @@ -15,19 +16,26 @@
T = TypeVar("T")


@dataclass
class WorkerBatcherOptions:
"""Tuning options for `WorkerBatcher`."""

max_batch_size: int = 10
num_workers: int = 4
queue_size: int = 100
keepalive_interval: float = 30.0
batch_timeout: float = 0.1


class WorkerBatcher(Generic[T]):
"""Asyncio worker-based batcher with concurrent processing and buffering."""

def __init__( # noqa: PLR0913
def __init__(
self,
source: AsyncIterator[T],
transformer_func: Callable[[T], Awaitable[ClassificationInput]],
deployment_id: str,
max_batch_size: int = 10,
num_workers: int = 4,
queue_size: int = 100,
keepalive_interval: float = 30.0,
batch_timeout: float = 0.1,
options: WorkerBatcherOptions | None = None,
) -> None:
"""Initialize the worker batcher.

Expand All @@ -37,29 +45,28 @@ def __init__( # noqa: PLR0913
transformer_func: Function to transform items (e.g., image
processing)
deployment_id: Deployment ID for requests
max_batch_size: Maximum items per batch
num_workers: Number of concurrent worker tasks
queue_size: Size of internal processing queue
keepalive_interval: Seconds between keepalive requests
batch_timeout: Max seconds to wait before sending partial batch
options: Tuning options for batch size, worker count, and
timing. Defaults to `WorkerBatcherOptions()`.

"""
options = options or WorkerBatcherOptions()

self.source: AsyncIterator[T] = source
self.transformer_func: Callable[[T], Awaitable[ClassificationInput]] = (
transformer_func
)
self.deployment_id: str = deployment_id
self.max_batch_size: int = max_batch_size
self.num_workers: int = num_workers
self.keepalive_interval: float = keepalive_interval
self.batch_timeout: float = batch_timeout
self.max_batch_size: int = options.max_batch_size
self.num_workers: int = options.num_workers
self.keepalive_interval: float = options.keepalive_interval
self.batch_timeout: float = options.batch_timeout

# Internal queues and state - use Optional[T] to handle None sentinel
self.input_queue: asyncio.Queue[T | None] = asyncio.Queue(
maxsize=queue_size
maxsize=options.queue_size
)
self.output_queue: asyncio.Queue[ClassificationInput] = asyncio.Queue(
maxsize=queue_size
maxsize=options.queue_size
)
self.processed_items: list[ClassificationInput] = []

Expand Down
6 changes: 4 additions & 2 deletions tests/client/models/test_image_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
from resolver_athena_client.client.models import ImageData
from resolver_athena_client.generated.athena.models_pb2 import ImageFormat

EXPECTED_HASH_COUNT = 2


def test_image_data_detects_png_format() -> None:
"""Test that PNG format is detected during initialization."""
Expand Down Expand Up @@ -78,8 +80,8 @@ def test_image_data_transformation_preserves_format() -> None:

# Format should still be PNG (transformers will update it if needed)
assert image_data.image_format == ImageFormat.IMAGE_FORMAT_PNG
assert len(image_data.sha256_hashes) == 2 # noqa: PLR2004
assert len(image_data.md5_hashes) == 2 # noqa: PLR2004
assert len(image_data.sha256_hashes) == EXPECTED_HASH_COUNT
assert len(image_data.md5_hashes) == EXPECTED_HASH_COUNT


@pytest.mark.parametrize(
Expand Down
10 changes: 8 additions & 2 deletions tests/client/test_athena_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import asyncio
import contextlib
from typing import cast
from typing import TYPE_CHECKING, cast
from unittest import mock

import pytest
Expand All @@ -23,6 +23,11 @@
)
from tests.utils.mock_async_iterator import MockAsyncIterator

if TYPE_CHECKING:
from resolver_athena_client.client.transformers.worker_batcher import (
WorkerBatcherOptions,
)
Comment thread
tcarroll-kroll marked this conversation as resolved.
Dismissed


@pytest.fixture
def mock_channel() -> mock.Mock:
Expand Down Expand Up @@ -388,7 +393,8 @@ async def start_classification() -> None:
# Verify WorkerBatcher was created with correct num_workers
mock_worker_batcher_cls.assert_called_once()
call_kwargs = mock_worker_batcher_cls.call_args.kwargs
assert call_kwargs["num_workers"] == custom_num_workers
batcher_options = cast("WorkerBatcherOptions", call_kwargs["options"])
assert batcher_options.num_workers == custom_num_workers


@pytest.mark.asyncio
Expand Down
8 changes: 4 additions & 4 deletions tests/client/test_timeout_behavior.py
Original file line number Diff line number Diff line change
Expand Up @@ -325,11 +325,11 @@ async def test_timeout_with_cancellation() -> None:
responses: list[ClassifyResponse] = []
classify_task = None

try:
def cancel_after_target() -> None:
"""Cancel processing after target responses."""
raise asyncio.CancelledError

def cancel_after_target() -> None:
"""Cancel processing after target responses."""
raise asyncio.CancelledError # noqa: TRY301
try:

async def collect_responses() -> None:
response_iter = aiter(client.classify_images(image_stream))
Expand Down
Loading
Loading