From 510aebdfb202a2fec1496ba9e02dbab213bc3251 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Wed, 8 Jul 2026 15:52:17 -0400 Subject: [PATCH 1/2] Add migrate endpoint for push repositories. Provide an async API to convert legacy ContainerPushRepository instances into ContainerRepository while preserving content and distribution links. Co-authored-by: Cursor --- CHANGES/+migrate-push-repository.feature | 1 + pulp_container/app/serializers.py | 15 ++++ pulp_container/app/tasks/__init__.py | 1 + .../app/tasks/migrate_push_repository.py | 72 +++++++++++++++++++ pulp_container/app/viewsets.py | 38 +++++++++- .../api/test_migrate_push_repository.py | 59 +++++++++++++++ 6 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 CHANGES/+migrate-push-repository.feature create mode 100644 pulp_container/app/tasks/migrate_push_repository.py create mode 100644 pulp_container/tests/functional/api/test_migrate_push_repository.py diff --git a/CHANGES/+migrate-push-repository.feature b/CHANGES/+migrate-push-repository.feature new file mode 100644 index 000000000..0b375e535 --- /dev/null +++ b/CHANGES/+migrate-push-repository.feature @@ -0,0 +1 @@ +Added a `migrate` endpoint to push repositories that converts a `ContainerPushRepository` into a `ContainerRepository`. diff --git a/pulp_container/app/serializers.py b/pulp_container/app/serializers.py index 657f1d650..3cc1d0ac5 100644 --- a/pulp_container/app/serializers.py +++ b/pulp_container/app/serializers.py @@ -290,6 +290,21 @@ class Meta: model = models.ContainerPushRepository +class MigratePushRepositorySerializer(ValidateFieldsMixin, serializers.Serializer): + """ + Serializer for migrating a push repository to a container repository. + """ + + copy_versions = serializers.BooleanField( + required=False, + default=False, + help_text=_( + "If True, copy the full repository version history. " + "If False, only the latest repository version content is copied." + ), + ) + + class ContainerRemoteSerializer(RemoteSerializer): """ A Serializer for ContainerRemote. diff --git a/pulp_container/app/tasks/__init__.py b/pulp_container/app/tasks/__init__.py index 794c190c3..f23c1b9ec 100644 --- a/pulp_container/app/tasks/__init__.py +++ b/pulp_container/app/tasks/__init__.py @@ -1,5 +1,6 @@ from .download_image_data import aadd_and_remove, download_image_data # noqa from .builder import build_image_from_containerfile, build_image # noqa +from .migrate_push_repository import migrate_push_repository # noqa from .recursive_add import recursive_add_content # noqa from .recursive_remove import recursive_remove_content # noqa from .sign import sign # noqa diff --git a/pulp_container/app/tasks/migrate_push_repository.py b/pulp_container/app/tasks/migrate_push_repository.py new file mode 100644 index 000000000..a39c33dc3 --- /dev/null +++ b/pulp_container/app/tasks/migrate_push_repository.py @@ -0,0 +1,72 @@ +from django.db import transaction + +from pulpcore.plugin.models import CreatedResource + +from pulp_container.app.models import ( + ContainerDistribution, + ContainerPushRepository, + ContainerRepository, +) +from pulp_container.app.serializers import ContainerRepositorySerializer + + +def migrate_push_repository(push_repository_pk, copy_versions=False): + """ + Convert a ContainerPushRepository into a ContainerRepository. + + Creates a new container repository with the same name and metadata, copies + content from the push repository, reassociates any distributions, and deletes + the original push repository. + + Args: + push_repository_pk (str): The primary key for the push repository to migrate. + copy_versions (bool): If True, copy the full repository version history. If False, + only copy content from the latest repository version. + """ + push_repository = ContainerPushRepository.objects.get(pk=push_repository_pk) + + original_name = push_repository.name + temp_name = f"{original_name}__migrating__{push_repository.pk}" + + with transaction.atomic(): + push_repository.name = temp_name + push_repository.save(update_fields=["name"]) + + container_repository = ContainerRepository( + name=original_name, + pulp_domain=push_repository.pulp_domain, + pulp_labels=push_repository.pulp_labels, + description=push_repository.description, + retain_repo_versions=push_repository.retain_repo_versions, + retain_checkpoints=push_repository.retain_checkpoints, + user_hidden=push_repository.user_hidden, + manifest_signing_service=push_repository.manifest_signing_service, + ) + container_repository.save() + + container_repository.pending_blobs.set(push_repository.pending_blobs.all()) + container_repository.pending_manifests.set(push_repository.pending_manifests.all()) + + if copy_versions: + for push_version in push_repository.versions.complete().order_by("number"): + if push_version.number == 0 and not push_version.content.exists(): + continue + with container_repository.new_version() as new_version: + new_version.set_content(push_version.content.all()) + else: + latest_version = push_repository.latest_version() + if latest_version: + with container_repository.new_version() as new_version: + new_version.set_content(latest_version.content.all()) + + ContainerDistribution.objects.filter(repository=push_repository).update( + repository=container_repository + ) + + push_repository.delete() + + CreatedResource(content_object=container_repository).save() + + return ContainerRepositorySerializer( + instance=container_repository, context={"request": None} + ).data diff --git a/pulp_container/app/viewsets.py b/pulp_container/app/viewsets.py index c5df96646..8bf439da1 100644 --- a/pulp_container/app/viewsets.py +++ b/pulp_container/app/viewsets.py @@ -1135,7 +1135,7 @@ class ContainerPushRepositoryViewSet( ], }, { - "action": ["update", "partial_update", "set_label", "unset_label"], + "action": ["update", "partial_update", "set_label", "unset_label", "migrate"], "principal": "authenticated", "effect": "allow", "condition_expression": [ @@ -1154,6 +1154,42 @@ class ContainerPushRepositoryViewSet( } LOCKED_ROLES = {} + @extend_schema( + description=( + "Trigger an asynchronous task to convert this push repository into a " + "container repository." + ), + summary="Migrate push repository to container repository", + request=serializers.MigratePushRepositorySerializer, + responses={202: AsyncOperationResponseSerializer}, + ) + @action( + detail=True, methods=["post"], serializer_class=serializers.MigratePushRepositorySerializer + ) + def migrate(self, request, pk): + """ + Create a task which converts a push repository into a container repository. + """ + repository = self.get_object() + + serializer = serializers.MigratePushRepositorySerializer( + data=request.data, context={"request": request} + ) + serializer.is_valid(raise_exception=True) + + distributions = models.ContainerDistribution.objects.filter(repository=repository) + exclusive_resources = [repository, *distributions] + + result = dispatch( + tasks.migrate_push_repository, + exclusive_resources=exclusive_resources, + kwargs={ + "push_repository_pk": str(repository.pk), + "copy_versions": serializer.validated_data["copy_versions"], + }, + ) + return OperationPostponedResponse(result, request) + @extend_schema( description=( "Trigger an asynchronous task to remove a manifest and all its associated " diff --git a/pulp_container/tests/functional/api/test_migrate_push_repository.py b/pulp_container/tests/functional/api/test_migrate_push_repository.py new file mode 100644 index 000000000..75eb69b5e --- /dev/null +++ b/pulp_container/tests/functional/api/test_migrate_push_repository.py @@ -0,0 +1,59 @@ +"""Tests for migrating push repositories to container repositories.""" + +import uuid + +from pulp_container.tests.functional.constants import REGISTRY_V2_REPO_PULP + + +def test_migrate_push_repository( + add_to_cleanup, + container_push_repository_factory, + registry_client, + local_registry, + container_bindings, + full_path, + monitor_task, +): + """A push repository can be migrated to a container repository.""" + namespace_name = str(uuid.uuid4()) + repo_name = f"{namespace_name}/migrate" + local_url = full_path(f"{repo_name}:1.0") + + container_push_repository_factory(name=repo_name) + image_path = f"{REGISTRY_V2_REPO_PULP}:manifest_a" + registry_client.pull(image_path) + local_registry.tag_and_push(image_path, local_url) + + distribution = container_bindings.DistributionsContainerApi.list(name=repo_name).results[0] + namespace = container_bindings.PulpContainerNamespacesApi.read(distribution.namespace) + add_to_cleanup(container_bindings.PulpContainerNamespacesApi, namespace.pulp_href) + + push_repository = container_bindings.RepositoriesContainerPushApi.list(name=repo_name).results[ + 0 + ] + tags_before = container_bindings.ContentTagsApi.list( + repository_version=push_repository.latest_version_href + ) + assert tags_before.count == 1 + + migrate_response = container_bindings.RepositoriesContainerPushApi.migrate( + push_repository.pulp_href, {"copy_versions": False} + ) + task = monitor_task(migrate_response.task) + container_repository = container_bindings.RepositoriesContainerApi.read( + task.result["pulp_href"] + ) + assert container_repository.name == repo_name + assert container_repository.pulp_href in task.created_resources + assert container_repository.prn == task.result["prn"] + tags_after = container_bindings.ContentTagsApi.list( + repository_version=container_repository.latest_version_href + ) + assert tags_after.count == 1 + assert tags_after.results[0].name == tags_before.results[0].name + assert tags_after.results[0].prn == tags_before.results[0].prn + + assert container_bindings.RepositoriesContainerPushApi.list(name=repo_name).count == 0 + + distribution = container_bindings.DistributionsContainerApi.list(name=repo_name).results[0] + assert distribution.repository == container_repository.pulp_href From 16a52e8d9c7c4f99e406fb5ca6ae4c33260e9502 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Thu, 9 Jul 2026 14:22:55 -0400 Subject: [PATCH 2/2] Document push repository migration and expand test coverage. Clarify copy_versions semantics, warn about in-flight uploads, and cover post-migrate push plus full version history replay. Co-authored-by: Cursor --- CHANGES/+migrate-push-repository.feature | 6 +- docs/user/guides/push-image.md | 28 ++++- pulp_container/app/serializers.py | 3 +- .../app/tasks/migrate_push_repository.py | 5 +- .../api/test_migrate_push_repository.py | 117 +++++++++++++++++- 5 files changed, 148 insertions(+), 11 deletions(-) diff --git a/CHANGES/+migrate-push-repository.feature b/CHANGES/+migrate-push-repository.feature index 0b375e535..19e4e11e2 100644 --- a/CHANGES/+migrate-push-repository.feature +++ b/CHANGES/+migrate-push-repository.feature @@ -1 +1,5 @@ -Added a `migrate` endpoint to push repositories that converts a `ContainerPushRepository` into a `ContainerRepository`. +Added a `migrate` endpoint on push repositories + (`POST .../repositories/container/container-push/{pk}/migrate/`) that converts a legacy + `ContainerPushRepository` into a `ContainerRepository`. Optional `copy_versions` preserves + repository version history; by default only the latest version content is copied. + Distributions keep the same registry path after migration. diff --git a/docs/user/guides/push-image.md b/docs/user/guides/push-image.md index 2c4a25b0b..28f886dea 100644 --- a/docs/user/guides/push-image.md +++ b/docs/user/guides/push-image.md @@ -46,7 +46,8 @@ X-Frame-Options: SAMEORIGIN !!! note - Content is pushed to a push repository type. A push repository does not support mirroring of the + New registry pushes create a regular `ContainerRepository`. Older deployments may still have + legacy `ContainerPushRepository` instances. A push repository does not support mirroring of remote content via the Pulp API. Trying to push content with the same name as an existing "regular" repository will fail. @@ -54,6 +55,31 @@ X-Frame-Options: SAMEORIGIN Rollback to the previous repository versions is not possible with a push repository. Its latest version will always be served. +## Migrating legacy push repositories + +Legacy push repositories can be converted to regular container repositories with the `migrate` +action. Migration preserves the repository name, metadata, content, and distribution association, +so registry paths and tags stay the same. The new repository is returned in the task result. + +```bash +http POST $BASE_ADDR/pulp/api/v3/repositories/container/container-push/$PUSH_REPO_PK/migrate/ \ + copy_versions:=false +``` + +By default (`copy_versions=false`), only the latest repository version content is copied. Set +`copy_versions=true` to replay the full version history into the new repository. Version numbers +and timestamps on the new repository are created during migration; they are not a literal clone of +the original versions. The registry API does not expose repository version timestamps. + +After migration, the push repository is deleted and the distribution points at the new container +repository. You can continue to push and pull through the same registry path, and you gain standard +repository operations such as sync and version management. + +!!! warning + + Do not push or upload content to a repository while it is being migrated. In-flight blob uploads + tied to the push repository can fail when the repository is deleted at the end of migration. + !!! warning Image that has been pulled from a registry and then subsequently pushed to another registy can lead to the blobs digest change. diff --git a/pulp_container/app/serializers.py b/pulp_container/app/serializers.py index 3cc1d0ac5..6de999b4d 100644 --- a/pulp_container/app/serializers.py +++ b/pulp_container/app/serializers.py @@ -299,7 +299,8 @@ class MigratePushRepositorySerializer(ValidateFieldsMixin, serializers.Serialize required=False, default=False, help_text=_( - "If True, copy the full repository version history. " + "If True, replay the full repository version history into the new repository. " + "Version numbers and timestamps are newly created. " "If False, only the latest repository version content is copied." ), ) diff --git a/pulp_container/app/tasks/migrate_push_repository.py b/pulp_container/app/tasks/migrate_push_repository.py index a39c33dc3..d2c7ad1f4 100644 --- a/pulp_container/app/tasks/migrate_push_repository.py +++ b/pulp_container/app/tasks/migrate_push_repository.py @@ -20,8 +20,9 @@ def migrate_push_repository(push_repository_pk, copy_versions=False): Args: push_repository_pk (str): The primary key for the push repository to migrate. - copy_versions (bool): If True, copy the full repository version history. If False, - only copy content from the latest repository version. + copy_versions (bool): If True, replay the full repository version history into the + new repository (new version numbers/timestamps). If False, only copy content + from the latest repository version. """ push_repository = ContainerPushRepository.objects.get(pk=push_repository_pk) diff --git a/pulp_container/tests/functional/api/test_migrate_push_repository.py b/pulp_container/tests/functional/api/test_migrate_push_repository.py index 75eb69b5e..af5ffa3ad 100644 --- a/pulp_container/tests/functional/api/test_migrate_push_repository.py +++ b/pulp_container/tests/functional/api/test_migrate_push_repository.py @@ -5,19 +5,18 @@ from pulp_container.tests.functional.constants import REGISTRY_V2_REPO_PULP -def test_migrate_push_repository( +def _setup_push_repository( add_to_cleanup, container_push_repository_factory, registry_client, local_registry, container_bindings, full_path, - monitor_task, + repo_name, + tag="1.0", ): - """A push repository can be migrated to a container repository.""" - namespace_name = str(uuid.uuid4()) - repo_name = f"{namespace_name}/migrate" - local_url = full_path(f"{repo_name}:1.0") + """Create a legacy push repository, push an image, and register cleanup.""" + local_url = full_path(f"{repo_name}:{tag}") container_push_repository_factory(name=repo_name) image_path = f"{REGISTRY_V2_REPO_PULP}:manifest_a" @@ -31,6 +30,31 @@ def test_migrate_push_repository( push_repository = container_bindings.RepositoriesContainerPushApi.list(name=repo_name).results[ 0 ] + return push_repository, local_url + + +def test_migrate_push_repository( + add_to_cleanup, + container_push_repository_factory, + registry_client, + local_registry, + container_bindings, + full_path, + monitor_task, +): + """A push repository can be migrated to a container repository and still accept pushes.""" + namespace_name = str(uuid.uuid4()) + repo_name = f"{namespace_name}/migrate" + + push_repository, local_url = _setup_push_repository( + add_to_cleanup, + container_push_repository_factory, + registry_client, + local_registry, + container_bindings, + full_path, + repo_name, + ) tags_before = container_bindings.ContentTagsApi.list( repository_version=push_repository.latest_version_href ) @@ -57,3 +81,84 @@ def test_migrate_push_repository( distribution = container_bindings.DistributionsContainerApi.list(name=repo_name).results[0] assert distribution.repository == container_repository.pulp_href + + image_path_b = f"{REGISTRY_V2_REPO_PULP}:manifest_b" + registry_client.pull(image_path_b) + local_registry.tag_and_push(image_path_b, full_path(f"{repo_name}:2.0")) + + container_repository = container_bindings.RepositoriesContainerApi.read( + container_repository.pulp_href + ) + tags_after_push = container_bindings.ContentTagsApi.list( + repository_version=container_repository.latest_version_href + ) + assert tags_after_push.count == 2 + assert {tag.name for tag in tags_after_push.results} == {"1.0", "2.0"} + local_registry.pull(local_url) + + +def test_migrate_push_repository_copy_versions( + add_to_cleanup, + container_push_repository_factory, + registry_client, + local_registry, + container_bindings, + full_path, + monitor_task, +): + """Migrating with copy_versions=True preserves repository version history content.""" + namespace_name = str(uuid.uuid4()) + repo_name = f"{namespace_name}/migrate-versions" + + push_repository, _ = _setup_push_repository( + add_to_cleanup, + container_push_repository_factory, + registry_client, + local_registry, + container_bindings, + full_path, + repo_name, + tag="1.0", + ) + + image_path_b = f"{REGISTRY_V2_REPO_PULP}:manifest_b" + registry_client.pull(image_path_b) + local_registry.tag_and_push(image_path_b, full_path(f"{repo_name}:2.0")) + + push_repository = container_bindings.RepositoriesContainerPushApi.read( + push_repository.pulp_href + ) + push_versions = container_bindings.RepositoriesContainerPushVersionsApi.list( + push_repository.pulp_href + ) + # version 0 (empty) plus one version per push + assert push_versions.count >= 3 + + version_tag_counts = [] + for version in sorted(push_versions.results, key=lambda v: v.number): + if version.number == 0: + continue + tags = container_bindings.ContentTagsApi.list(repository_version=version.pulp_href) + version_tag_counts.append((version.number, tags.count, {t.name for t in tags.results})) + + migrate_response = container_bindings.RepositoriesContainerPushApi.migrate( + push_repository.pulp_href, {"copy_versions": True} + ) + task = monitor_task(migrate_response.task) + container_repository = container_bindings.RepositoriesContainerApi.read( + task.result["pulp_href"] + ) + + container_versions = container_bindings.RepositoriesContainerVersionsApi.list( + container_repository.pulp_href + ) + migrated_versions = sorted( + [v for v in container_versions.results if v.number != 0], + key=lambda v: v.number, + ) + assert len(migrated_versions) == len(version_tag_counts) + + for (_, expected_count, expected_names), migrated in zip(version_tag_counts, migrated_versions): + tags = container_bindings.ContentTagsApi.list(repository_version=migrated.pulp_href) + assert tags.count == expected_count + assert {t.name for t in tags.results} == expected_names