Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ jobs:
username: ${{ secrets.DOCKER_USER }}
password: ${{ secrets.DOCKER_PASSWORD }}

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Set up GO 1.25.8
uses: actions/setup-go@v5
with:
Expand Down Expand Up @@ -71,6 +74,7 @@ jobs:
VERSION: ${{ github.event.release.tag_name }}
IMAGE_TAG_BASE: streamnative/function-mesh
CATALOG_BRANCH_TAG: latest
PLATFORMS: linux/amd64,linux/arm64
run: |
# convert vx.y.z to x.y.z because a valid semver is needed in creating the bundle
VERSION=$(echo $VERSION|cut -c 2-)
Expand Down
7 changes: 5 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
# Build the manager binary
FROM golang:1.25.8-trixie AS builder
FROM --platform=$BUILDPLATFORM golang:1.25.8-trixie AS builder

ARG TARGETOS
ARG TARGETARCH

WORKDIR /workspace/api
COPY api/ .
Expand All @@ -20,7 +23,7 @@ COPY controllers/ controllers/
COPY utils/ utils/

# Build
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go
RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} GO111MODULE=on go build -a -o manager main.go

# Use distroless as minimal base image to package the manager binary
# Refer to https://github.com/GoogleContainerTools/distroless for more details
Expand Down
58 changes: 46 additions & 12 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,12 @@ GOARCH := $(if $(GOARCH),$(GOARCH),amd64)
GOENV := CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH)
GO := $(GOENV) go
GO_BUILD := $(GO) build -trimpath
PLATFORMS ?= linux/amd64
comma := ,
PLATFORM_LIST := $(subst $(comma), ,$(PLATFORMS))
PRIMARY_PLATFORM := $(word 1,$(PLATFORM_LIST))
MULTI_PLATFORM_BUILD := $(if $(filter 1,$(words $(PLATFORM_LIST))),false,true)
IMAGE_BUILD_PUSH ?= $(MULTI_PLATFORM_BUILD)
GO_MAJOR_VERSION := $(shell $(GO) version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f1)
GO_MINOR_VERSION := $(shell $(GO) version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f2)

Expand Down Expand Up @@ -133,15 +139,27 @@ generate: controller-gen

# Build the docker image
docker-build: test
docker build --platform linux/amd64 . -t ${IMG}
ifeq ($(MULTI_PLATFORM_BUILD),true)
docker buildx build --platform $(PLATFORMS) --push -t ${IMG} .
else
docker build --platform $(PRIMARY_PLATFORM) -t ${IMG} .
endif

# Build image for red hat certification
docker-build-redhat:
docker build --platform linux/amd64 -f redhat.Dockerfile . -t ${IMG} --build-arg VERSION=${VERSION} --no-cache
ifeq ($(MULTI_PLATFORM_BUILD),true)
docker buildx build --platform $(PLATFORMS) --push -f redhat.Dockerfile -t ${IMG} --build-arg VERSION=${VERSION} --no-cache .
else
docker build --platform $(PRIMARY_PLATFORM) -f redhat.Dockerfile -t ${IMG} --build-arg VERSION=${VERSION} --no-cache .
endif

# Push the docker image
image-push:
ifeq ($(IMAGE_BUILD_PUSH),true)
@echo "image already pushed during multi-platform build: ${IMG}"
else
docker push ${IMG}
endif

# find or download controller-gen
# download controller-gen if necessary
Expand Down Expand Up @@ -171,12 +189,12 @@ bundle: yq kustomize manifests
# Build the bundle image.
.PHONY: bundle-build
bundle-build:
docker build --platform linux/amd64 -f bundle.Dockerfile -t $(BUNDLE_IMG) .
docker build --platform $(PRIMARY_PLATFORM) -f bundle.Dockerfile -t $(BUNDLE_IMG) .

.PHONY: bundle-push
bundle-push: ## Push the bundle image.
echo $(BUNDLE_IMG)
$(MAKE) image-push IMG=$(BUNDLE_IMG)
$(MAKE) image-push IMG=$(BUNDLE_IMG) IMAGE_BUILD_PUSH=false

crd: manifests
$(KUSTOMIZE) build config/crd > manifests/crd.yaml
Expand All @@ -186,13 +204,21 @@ rbac: manifests

release: manifests kustomize crd rbac manager operator-docker-image helm-crds

operator-docker-image: manager test
docker build --platform linux/amd64 -f operator.Dockerfile -t $(OPERATOR_IMG) .
operator-docker-image: test
ifeq ($(MULTI_PLATFORM_BUILD),true)
docker buildx build --platform $(PLATFORMS) --push -f operator.Dockerfile -t $(OPERATOR_IMG) -t $(OPERATOR_IMG_LATEST) .
else
docker build --platform $(PRIMARY_PLATFORM) -f operator.Dockerfile -t $(OPERATOR_IMG) .
docker tag $(OPERATOR_IMG) $(OPERATOR_IMG_LATEST)
endif

docker-push:
ifeq ($(MULTI_PLATFORM_BUILD),true)
@echo "operator images already pushed during multi-platform build: $(OPERATOR_IMG), $(OPERATOR_IMG_LATEST)"
else
docker push $(OPERATOR_IMG)
docker push $(OPERATOR_IMG_LATEST)
endif

.PHONY: opm
OPM = ./bin/opm
Expand Down Expand Up @@ -240,9 +266,9 @@ endif
# Push the catalog image.
.PHONY: catalog-push
catalog-push: ## Push a catalog image.
$(MAKE) image-push IMG=$(CATALOG_IMG)
$(MAKE) image-push IMG=$(CATALOG_IMG) IMAGE_BUILD_PUSH=false
ifneq ($(origin CATALOG_BRANCH_TAG), undefined)
$(MAKE) image-push IMG=$(CATALOG_BRANCH_IMG)
$(MAKE) image-push IMG=$(CATALOG_BRANCH_IMG) IMAGE_BUILD_PUSH=false
endif

version:
Expand All @@ -256,7 +282,11 @@ function-mesh-docker-image-name:

# Build the docker image without tests
docker-build-skip-test:
docker build --platform linux/amd64 . -t ${IMG}
ifeq ($(MULTI_PLATFORM_BUILD),true)
docker buildx build --platform $(PLATFORMS) --push -t ${IMG} .
else
docker build --platform $(PRIMARY_PLATFORM) -t ${IMG} .
endif

e2e: skywalking-e2e yq
$(E2E) run -c .ci/tests/integration/e2e.yaml
Expand Down Expand Up @@ -297,17 +327,21 @@ redhat-certificated-bundle: yq kustomize manifests
# Build the bundle image.
.PHONY: redhat-certificated-bundle-build
redhat-certificated-bundle-build:
docker build --platform linux/amd64 -f bundle.Dockerfile -t $(BUNDLE_IMG) .
docker build --platform $(PRIMARY_PLATFORM) -f bundle.Dockerfile -t $(BUNDLE_IMG) .

.PHONY: redhat-certificated-bundle-push
redhat-certificated-bundle-push: ## Push the bundle image.
echo $(BUNDLE_IMG)
$(MAKE) image-push IMG=$(BUNDLE_IMG)
$(MAKE) image-push IMG=$(BUNDLE_IMG) IMAGE_BUILD_PUSH=false

# Build the bundle image.
.PHONY: redhat-certificated-image-build
redhat-certificated-image-build:
docker build --platform linux/amd64 -f redhat.Dockerfile . -t ${OPERATOR_IMG} --build-arg VERSION=${VERSION} --no-cache
ifeq ($(MULTI_PLATFORM_BUILD),true)
docker buildx build --platform $(PLATFORMS) --push -f redhat.Dockerfile -t ${OPERATOR_IMG} --build-arg VERSION=${VERSION} --no-cache .
else
docker build --platform $(PRIMARY_PLATFORM) -f redhat.Dockerfile -t ${OPERATOR_IMG} --build-arg VERSION=${VERSION} --no-cache .
endif

.PHONY: redhat-certificated-image-push
redhat-certificated-image-push: ## Push the bundle image.
Expand Down
12 changes: 12 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,18 @@ operator-sdk create api --group compute --version v1alpha1 --kind Function --res
operator-sdk create webhook --group compute.functionmesh.io --version v1alpha1 --kind Function --defaulting --programmatic-validation
```

### Multi-platform images

The image build targets accept `PLATFORMS` as a comma-separated Docker platform list.

```bash
make docker-build PLATFORMS=linux/amd64,linux/arm64
make operator-docker-image PLATFORMS=linux/amd64,linux/arm64
PLATFORMS=linux/amd64,linux/arm64 PUSH=true images/build.sh
```

Single-platform builds still default to `linux/amd64`. Multi-platform builds use `docker buildx` and push a manifest list directly, so they require an authenticated registry session.

## Deployment

1. make sure connected to a kubernetes cluster(gke, mini-kube etc.)
Expand Down
2 changes: 2 additions & 0 deletions docs/release_proocess.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,5 @@ git push origin vX.Y.Z
3. Click the release button

Click the release button and draft a new release. When publish the release, the Action CI will automatically trigger the release process, build the corresponding image, and push it to docker_hub.

The release workflow publishes multi-platform Docker images for `linux/amd64` and `linux/arm64`.
146 changes: 115 additions & 31 deletions images/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
# specific language governing permissions and limitations
# under the License.
#
set -e
set -euo pipefail

PULSAR_IMAGE=${PULSAR_IMAGE:-"streamnative/sn-platform"}
PULSAR_IMAGE_TAG=${PULSAR_IMAGE_TAG:-"2.7.1"}
Expand All @@ -34,53 +34,137 @@ PULSARCTL_PYTHON_RUNNER="pulsar-functions-pulsarctl-python-runner"
RUNNER_TAG=${RUNNER_TAG:-$PULSAR_IMAGE_TAG}
KIND_PUSH=${KIND_PUSH:-false}
CI_TEST=${CI_TEST:-false}
PLATFORMS=${PLATFORMS:-"linux/amd64"}
PRIMARY_PLATFORM=${PLATFORMS%%,*}
RUNNER_BASE_IMAGE="${DOCKER_REPO}/${RUNNER_BASE}:${RUNNER_TAG}"
PULSARCTL_RUNNER_BASE_IMAGE="${DOCKER_REPO}/${PULSARCTL_RUNNER_BASE}:${RUNNER_TAG}"
JAVA_RUNNER_IMAGE="${DOCKER_REPO}/${JAVA_RUNNER}:${RUNNER_TAG}"
PULSARCTL_JAVA_RUNNER_IMAGE="${DOCKER_REPO}/${PULSARCTL_JAVA_RUNNER}:${RUNNER_TAG}"
GO_RUNNER_IMAGE="${DOCKER_REPO}/${GO_RUNNER}:${RUNNER_TAG}"
PULSARCTL_GO_RUNNER_IMAGE="${DOCKER_REPO}/${PULSARCTL_GO_RUNNER}:${RUNNER_TAG}"
PYTHON_RUNNER_IMAGE="${DOCKER_REPO}/${PYTHON_RUNNER}:${RUNNER_TAG}"
PULSARCTL_PYTHON_RUNNER_IMAGE="${DOCKER_REPO}/${PULSARCTL_PYTHON_RUNNER}:${RUNNER_TAG}"

MULTI_PLATFORM=false
if [[ "${PLATFORMS}" == *,* ]]; then
MULTI_PLATFORM=true
fi

PUSH_DEFAULT=false
if [[ "${DOCKER_REPO}" == "localhost:5000" || "${MULTI_PLATFORM}" == "true" ]]; then
PUSH_DEFAULT=true
fi
PUSH=${PUSH:-$PUSH_DEFAULT}

if [[ "${MULTI_PLATFORM}" == "true" && "${PUSH}" != "true" ]]; then
echo "multi-platform builds require PUSH=true so dependent images can resolve their base images" >&2
exit 1
fi

if [[ "${MULTI_PLATFORM}" == "true" && "${KIND_PUSH}" == "true" ]]; then
echo "KIND_PUSH=true is only supported for single-platform builds" >&2
exit 1
fi

build_image() {
local image=$1
local context=$2
shift 2

if [[ "${MULTI_PLATFORM}" == "true" ]]; then
docker buildx build --platform "${PLATFORMS}" --push -t "${image}" "$@" "${context}"
else
docker build --platform "${PRIMARY_PLATFORM}" -t "${image}" "$@" "${context}"
fi
}

tag_local_aliases() {
local source_image=$1
local local_name=$2

if [[ "${MULTI_PLATFORM}" != "true" ]]; then
docker tag "${source_image}" "${local_name}:latest"
fi
}

echo "build runner base"
docker build --platform linux/amd64 -t ${RUNNER_BASE} images/pulsar-functions-base-runner --build-arg PULSAR_IMAGE="$PULSAR_IMAGE" --build-arg PULSAR_IMAGE_TAG="$PULSAR_IMAGE_TAG" --progress=plain
docker build --platform linux/amd64 -t ${PULSARCTL_RUNNER_BASE} images/pulsar-functions-base-runner -f images/pulsar-functions-base-runner/pulsarctl.Dockerfile --build-arg PULSAR_IMAGE="$PULSAR_IMAGE" --build-arg PULSAR_IMAGE_TAG="$PULSAR_IMAGE_TAG" --progress=plain
docker tag ${RUNNER_BASE} "${DOCKER_REPO}"/${RUNNER_BASE}:"${RUNNER_TAG}"
docker tag ${PULSARCTL_RUNNER_BASE} "${DOCKER_REPO}"/${PULSARCTL_RUNNER_BASE}:"${RUNNER_TAG}"
build_image "${RUNNER_BASE_IMAGE}" images/pulsar-functions-base-runner \
--build-arg PULSAR_IMAGE="${PULSAR_IMAGE}" \
--build-arg PULSAR_IMAGE_TAG="${PULSAR_IMAGE_TAG}" \
--progress=plain
build_image "${PULSARCTL_RUNNER_BASE_IMAGE}" images/pulsar-functions-base-runner \
-f images/pulsar-functions-base-runner/pulsarctl.Dockerfile \
--build-arg PULSAR_IMAGE="${PULSAR_IMAGE}" \
--build-arg PULSAR_IMAGE_TAG="${PULSAR_IMAGE_TAG}" \
--progress=plain
tag_local_aliases "${RUNNER_BASE_IMAGE}" "${RUNNER_BASE}"
tag_local_aliases "${PULSARCTL_RUNNER_BASE_IMAGE}" "${PULSARCTL_RUNNER_BASE}"

echo "build java runner"
docker build --platform linux/amd64 -t ${JAVA_RUNNER} images/pulsar-functions-java-runner --build-arg PULSAR_IMAGE="$PULSAR_IMAGE" --build-arg PULSAR_IMAGE_TAG="$PULSAR_IMAGE_TAG" --progress=plain
docker build --platform linux/amd64 -t ${PULSARCTL_JAVA_RUNNER} images/pulsar-functions-java-runner -f images/pulsar-functions-java-runner/pulsarctl.Dockerfile --build-arg PULSAR_IMAGE="$PULSAR_IMAGE" --build-arg PULSAR_IMAGE_TAG="$PULSAR_IMAGE_TAG" --progress=plain
docker tag ${JAVA_RUNNER} "${DOCKER_REPO}"/${JAVA_RUNNER}:"${RUNNER_TAG}"
docker tag ${PULSARCTL_JAVA_RUNNER} "${DOCKER_REPO}"/${PULSARCTL_JAVA_RUNNER}:"${RUNNER_TAG}"
build_image "${JAVA_RUNNER_IMAGE}" images/pulsar-functions-java-runner \
--build-arg BASE_IMAGE="${RUNNER_BASE_IMAGE}" \
--build-arg PULSAR_IMAGE="${PULSAR_IMAGE}" \
--build-arg PULSAR_IMAGE_TAG="${PULSAR_IMAGE_TAG}" \
--progress=plain
build_image "${PULSARCTL_JAVA_RUNNER_IMAGE}" images/pulsar-functions-java-runner \
-f images/pulsar-functions-java-runner/pulsarctl.Dockerfile \
--build-arg BASE_IMAGE="${PULSARCTL_RUNNER_BASE_IMAGE}" \
--build-arg PULSAR_IMAGE="${PULSAR_IMAGE}" \
--build-arg PULSAR_IMAGE_TAG="${PULSAR_IMAGE_TAG}" \
--progress=plain
tag_local_aliases "${JAVA_RUNNER_IMAGE}" "${JAVA_RUNNER}"
tag_local_aliases "${PULSARCTL_JAVA_RUNNER_IMAGE}" "${PULSARCTL_JAVA_RUNNER}"

echo "build python runner"
docker build --platform linux/amd64 -t ${PYTHON_RUNNER} images/pulsar-functions-python-runner --build-arg PULSAR_IMAGE="$PULSAR_IMAGE" --build-arg PULSAR_IMAGE_TAG="$PULSAR_IMAGE_TAG" --progress=plain
docker build --platform linux/amd64 -t ${PULSARCTL_PYTHON_RUNNER} images/pulsar-functions-python-runner -f images/pulsar-functions-python-runner/pulsarctl.Dockerfile --build-arg PULSAR_IMAGE="$PULSAR_IMAGE" --build-arg PULSAR_IMAGE_TAG="$PULSAR_IMAGE_TAG" --build-arg PYTHON_VERSION="$PYTHON_VERSION" --progress=plain
docker tag ${PYTHON_RUNNER} "${DOCKER_REPO}"/${PYTHON_RUNNER}:"${RUNNER_TAG}"
docker tag ${PULSARCTL_PYTHON_RUNNER} "${DOCKER_REPO}"/${PULSARCTL_PYTHON_RUNNER}:"${RUNNER_TAG}"
build_image "${PYTHON_RUNNER_IMAGE}" images/pulsar-functions-python-runner \
--build-arg BASE_IMAGE="${RUNNER_BASE_IMAGE}" \
--build-arg PULSAR_IMAGE="${PULSAR_IMAGE}" \
--build-arg PULSAR_IMAGE_TAG="${PULSAR_IMAGE_TAG}" \
--progress=plain
build_image "${PULSARCTL_PYTHON_RUNNER_IMAGE}" images/pulsar-functions-python-runner \
-f images/pulsar-functions-python-runner/pulsarctl.Dockerfile \
--build-arg PULSAR_IMAGE="${PULSAR_IMAGE}" \
--build-arg PULSAR_IMAGE_TAG="${PULSAR_IMAGE_TAG}" \
--build-arg PYTHON_VERSION="${PYTHON_VERSION}" \
--progress=plain
tag_local_aliases "${PYTHON_RUNNER_IMAGE}" "${PYTHON_RUNNER}"
tag_local_aliases "${PULSARCTL_PYTHON_RUNNER_IMAGE}" "${PULSARCTL_PYTHON_RUNNER}"

echo "build go runner"
docker build --platform linux/amd64 -t ${GO_RUNNER} images/pulsar-functions-go-runner --progress=plain # go runner is almost the same as runner base, so we no need to given build args for go runner
docker build --platform linux/amd64 -t ${PULSARCTL_GO_RUNNER} images/pulsar-functions-go-runner -f images/pulsar-functions-go-runner/pulsarctl.Dockerfile --progress=plain # go runner is almost the same as runner base, so we no need to given build args for go runner
docker tag ${GO_RUNNER} "${DOCKER_REPO}"/${GO_RUNNER}:"${RUNNER_TAG}"
docker tag ${PULSARCTL_GO_RUNNER} "${DOCKER_REPO}"/${PULSARCTL_GO_RUNNER}:"${RUNNER_TAG}"
build_image "${GO_RUNNER_IMAGE}" images/pulsar-functions-go-runner \
--build-arg BASE_IMAGE="${RUNNER_BASE_IMAGE}" \
--progress=plain
build_image "${PULSARCTL_GO_RUNNER_IMAGE}" images/pulsar-functions-go-runner \
-f images/pulsar-functions-go-runner/pulsarctl.Dockerfile \
--build-arg BASE_IMAGE="${PULSARCTL_RUNNER_BASE_IMAGE}" \
--progress=plain
tag_local_aliases "${GO_RUNNER_IMAGE}" "${GO_RUNNER}"
tag_local_aliases "${PULSARCTL_GO_RUNNER_IMAGE}" "${PULSARCTL_GO_RUNNER}"

if [ "$KIND_PUSH" = true ] ; then
echo "push images to kind"
clusters=$(kind get clusters)
echo $clusters
for cluster in $clusters
do
kind load docker-image "${DOCKER_REPO}"/${JAVA_RUNNER}:"${RUNNER_TAG}" --name $cluster
kind load docker-image "${DOCKER_REPO}"/${PULSARCTL_JAVA_RUNNER}:"${RUNNER_TAG}" --name $cluster
kind load docker-image "${DOCKER_REPO}"/${PYTHON_RUNNER}:"${RUNNER_TAG}" --name $cluster
kind load docker-image "${DOCKER_REPO}"/${PULSARCTL_PYTHON_RUNNER}:"${RUNNER_TAG}" --name $cluster
kind load docker-image "${DOCKER_REPO}"/${GO_RUNNER}:"${RUNNER_TAG}" --name $cluster
kind load docker-image "${DOCKER_REPO}"/${PULSARCTL_GO_RUNNER}:"${RUNNER_TAG}" --name $cluster
kind load docker-image "${JAVA_RUNNER_IMAGE}" --name $cluster
kind load docker-image "${PULSARCTL_JAVA_RUNNER_IMAGE}" --name $cluster
kind load docker-image "${PYTHON_RUNNER_IMAGE}" --name $cluster
kind load docker-image "${PULSARCTL_PYTHON_RUNNER_IMAGE}" --name $cluster
kind load docker-image "${GO_RUNNER_IMAGE}" --name $cluster
kind load docker-image "${PULSARCTL_GO_RUNNER_IMAGE}" --name $cluster
done
fi

if [ "$DOCKER_REPO" = "localhost:5000" ]; then
docker push "${DOCKER_REPO}"/${JAVA_RUNNER}:"${RUNNER_TAG}"
docker push "${DOCKER_REPO}"/${PULSARCTL_JAVA_RUNNER}:"${RUNNER_TAG}"
docker push "${DOCKER_REPO}"/${PYTHON_RUNNER}:"${RUNNER_TAG}"
docker push "${DOCKER_REPO}"/${PULSARCTL_PYTHON_RUNNER}:"${RUNNER_TAG}"
docker push "${DOCKER_REPO}"/${GO_RUNNER}:"${RUNNER_TAG}"
docker push "${DOCKER_REPO}"/${PULSARCTL_GO_RUNNER}:"${RUNNER_TAG}"
if [[ "${PUSH}" == "true" && "${MULTI_PLATFORM}" != "true" ]]; then
docker push "${RUNNER_BASE_IMAGE}"
docker push "${PULSARCTL_RUNNER_BASE_IMAGE}"
docker push "${JAVA_RUNNER_IMAGE}"
docker push "${PULSARCTL_JAVA_RUNNER_IMAGE}"
docker push "${PYTHON_RUNNER_IMAGE}"
docker push "${PULSARCTL_PYTHON_RUNNER_IMAGE}"
docker push "${GO_RUNNER_IMAGE}"
docker push "${PULSARCTL_GO_RUNNER_IMAGE}"
fi
#
#if [ "$CI_TEST" = true ] ; then
Expand All @@ -93,4 +177,4 @@ fi
# kind load docker-image "${DOCKER_REPO}"/${PYTHON_RUNNER}:"${RUNNER_TAG}" --name $cluster
# kind load docker-image "${DOCKER_REPO}"/${GO_RUNNER}:"${RUNNER_TAG}" --name $cluster
# done
#fi
#fi
Loading
Loading