use AWS_ENDPOINT_URL with an updated provider to avoid patching endpoints in terraform

.github/workflows/main.yml

@@ -148,9 +148,9 @@ jobs:
           options="-s -v --ls-start --gather-metrics"
         fi
 
-        if [[ ${{ matrix.service_partition.service }} == "lambda" ]]
+        if [[ ${{ matrix.service_partition.service }} == "organizations" ]]
         then
-          make prepare-lambda
+          make prepare-organizations
         fi
 
         if [[ ${{ matrix.service_partition.partition }} == "All" ]] 

.pre-commit-config.yaml

@@ -1,13 +1,16 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
-  - repo: https://github.com/psf/black
-    rev: 22.3.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.12.2
     hooks:
-      - id: black
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+      - id: ruff-format
 
   - repo: https://github.com/pycqa/isort
-    rev: 5.9.1
+    rev: 6.0.1
     hooks:
       - id: isort
         name: isort (python)

Makefile

@@ -31,21 +31,21 @@ init_precommit:				## Install the pre-commit hook into your local git repository
 	($(VENV_RUN); pre-commit install)
 
 lint:						## Run linting
-	@echo "Running black... "
-	$(VENV_RUN); black --check .
+	@echo "Running ruff... "
+	$(VENV_RUN); python -m ruff check .
 
 format:						## Run formatting
-	$(VENV_RUN); python -m isort .; python -m black .
+	$(VENV_RUN); python -m isort .; python -m ruff format .
 
 reset-submodules:			## Reset the submodules to the specified commit
 	git submodule foreach git reset --hard
 
 get-submodules:				## Get the submodules
 	git submodule update --init --recursive
 
-prepare-lambda:				## Prepare the lambda function for deployment
-	@test -d terraform-provider-aws || echo "Please run 'git submodule update --init --recursive' to get the terraform-provider-aws submodule"
-	@cp -r terraform-provider-aws/internal/service/lambda/test-fixtures ./test-bin/ && echo "Copied test-fixtures to test-bin"
 
+prepare-organizations: install
+	$(VENV_RUN); awslocal organizations create-organization --feature-set ALL
 
-.PHONY: usage venv install init_precommit lint format reset-submodules
+
+.PHONY: usage venv install init_precommit lint format reset-submodules

README.md

@@ -42,12 +42,6 @@ make install
 ## 🏃‍♂️ **How to Run**
 
 - 🔑 (Pro-image only) Set the `LOCALSTACK_AUTH_TOKEN` environment variable.
-- Apply the patch to the Terraform provider AWS:
-```
-python -m terraform_pytest.main patch
-```
-⚠️ _Note: The above operation isn't idempotent. Ensure you apply the patch only once._
-
 - Construct a testing binary for the Golang module:
 ```
 python -m terraform_pytest.main build -s s3
@@ -79,18 +73,19 @@ AWS_ALTERNATE_REGION='us-west-2' python -m pytest terraform-provider-aws/interna
 
 ## 🔢 **Default Environment Variables for Terraform Tests**
 
-| Variable                             | Default Value |
-|--------------------------------------|---------------|
-| `TF_ACC`                             | 1             |
-| `AWS_ACCESS_KEY_ID`                  | test          |
-| `AWS_SECRET_ACCESS_KEY`              | test          |
-| `AWS_DEFAULT_REGION`                 | us-west-1     |
-| `AWS_ALTERNATE_ACCESS_KEY_ID`        | test          |
-| `AWS_ALTERNATE_SECRET_ACCESS_KEY`    | test          |
-| `AWS_ALTERNATE_REGION`               | us-east-2     |
-| `AWS_THIRD_SECRET_ACCESS_KEY`        | test          |
-| `AWS_THIRD_ACCESS_KEY_ID`            | test          |
-| `AWS_THIRD_REGION`                   | eu-west-1     |
+| Variable                          | Default Value         |
+|-----------------------------------|-----------------------|
+| `TF_ACC`                          | 1                     |
+| `AWS_ACCESS_KEY_ID`               | test                  |
+| `AWS_SECRET_ACCESS_KEY`           | test                  |
+| `AWS_DEFAULT_REGION`              | us-west-1             |
+| `AWS_ALTERNATE_ACCESS_KEY_ID`     | test                  |
+| `AWS_ALTERNATE_SECRET_ACCESS_KEY` | test                  |
+| `AWS_ALTERNATE_REGION`            | us-east-2             |
+| `AWS_THIRD_SECRET_ACCESS_KEY`     | test                  |
+| `AWS_THIRD_ACCESS_KEY_ID`         | test                  |
+| `AWS_THIRD_REGION`                | eu-west-1             |
+| `AWS_ENDPOINT_URL`                | http://localhost:4566 |
 
 ---
 

conftest.py

@@ -2,7 +2,6 @@
 import json
 import os
 import re
-from os.path import dirname, realpath, relpath
 from pathlib import Path
 
 import pytest
@@ -15,7 +14,10 @@
 def pytest_addoption(parser):
     """Add command line options to pytest"""
     parser.addoption(
-        "--ls-start", action="store_true", default=False, help="Start localstack service"
+        "--ls-start",
+        action="store_true",
+        default=False,
+        help="Start localstack service",
     )
     parser.addoption(
         "--gather-metrics",
@@ -55,8 +57,8 @@ def runtest(self):
         """Run the test case"""
 
         cwd = os.getcwd()
-        service_path = dirname(Path(*relpath(self.path).split(os.sep)[1:]))
-        service = service_path.split(os.sep)[-1]
+        service_dir = self.path.parent
+        service = service_dir.name
 
         env = dict(os.environ)
         env.update(
@@ -71,18 +73,20 @@ def runtest(self):
                 "AWS_THIRD_SECRET_ACCESS_KEY": "test",
                 "AWS_THIRD_ACCESS_KEY_ID": "test",
                 "AWS_THIRD_REGION": "eu-west-1",
+                "AWS_ENDPOINT_URL": "http://localhost:4566",
             }
         )
 
+        test_binary = os.path.join(cwd, "test-bin", f"{service}.test")
         cmd = [
-            f"./{service}.test",
+            test_binary,
             "-test.v",
             "-test.parallel=1",
             "-test.count=1",
             "-test.timeout=60m",
             f"-test.run={self.name}",
         ]
-        return_code, stdout = execute_command(cmd, env, f"{cwd}/test-bin")
+        return_code, stdout = execute_command(cmd, env, str(service_dir))
         if return_code != 0:
             raise GoException(returncode=return_code, stderr=stdout)
         elif IS_GATHER_METRICS:
@@ -216,7 +220,7 @@ def pytest_sessionfinish(session, exitstatus):
 def _startup_localstack():
     try:
         _localstack_health_check()
-    except:
+    except Exception:
         os.system(
             "DEBUG=1 FAIL_FAST=1 DNS_ADDRESS=127.0.0.1 EXTENSION_DEV_MODE=1 DISABLE_EVENTS=1 LOCALSTACK_AUTH_TOKEN=$LOCALSTACK_AUTH_TOKEN localstack start -d"
         )

etc/0002-Fast-fail-timeouts-and-retries.patch

@@ -0,0 +1,13 @@
+diff --git a/internal/provider/sdkv2/provider.go b/internal/provider/sdkv2/provider.go
+index 7b5a2f1d86b..a7c3e5c8e0d 100644
+--- a/internal/provider/sdkv2/provider.go
++++ b/internal/provider/sdkv2/provider.go
+@@ -349,7 +349,7 @@ func (p *sdkProvider) configure(ctx context.Context, d *schema.ResourceData) (a
+ 		EC2MetadataServiceEndpointMode: d.Get("ec2_metadata_service_endpoint_mode").(string),
+ 		Endpoints:                      make(map[string]string),
+ 		Insecure:                       d.Get("insecure").(bool),
+-		MaxRetries:                     25, // Set default here, not in schema (muxing with v6 provider).
++		MaxRetries:                     1, // LocalStack fast-fail: reduce retries
+ 		Profile:                        d.Get("profile").(string),
+ 		Region:                         d.Get("region").(string),
+ 		S3UsePathStyle:                 d.Get("s3_use_path_style").(bool),

requirements.txt

@@ -1,10 +1,11 @@
 click>=8.1.3
-pytest>=7.2.0
-docker>=6.0.1
-requests>=2.28.2
-black>=22.1
-isort>=5.10
+pytest>=8.4.1
+docker>=7.1.0
+requests>=2.32.4
+ruff>=0.12.1
+isort>=6.0.1
 pytest-xdist>=3.1.0
-pre-commit>=2.21.0
-localstack>=1.4.0.dev
-PyYAML~=6.0
+pre-commit>=4.2.0
+localstack>=4.6.0
+PyYAML~=6.0
+awscli-local

terraform_pytest/constants.py

@@ -10,7 +10,7 @@
 TF_REPO_SERVICE_PATH = os.path.join(TF_REPO_PATH, TF_REPO_SERVICE_FOLDER)
 
 # list of patch files to apply to the terraform repo
-TF_REPO_PATCH_FILES = ["etc/0001-Patch-Hardcode-endpoints-to-local-server.patch"]
+TF_REPO_PATCH_FILES = ["etc/0002-Fast-fail-timeouts-and-retries.patch"]
 
 # list of services that are supported by the localstack community edition
 LS_COMMUNITY_SERVICES = [

terraform_pytest/get_tf_partitions.py

@@ -1,5 +1,4 @@
 # Prints a JSON dict mapping the different partitions in the terraform-tests.yaml to their service
-import json
 import sys
 
 import yaml

terraform_pytest/main.py

@@ -5,7 +5,12 @@
 import click
 
 from terraform_pytest.constants import TF_REPO_PATH, TF_TEST_BINARY_PATH
-from terraform_pytest.utils import build_test_binary, get_services, patch_repository
+from terraform_pytest.utils import (
+    build_test_binary,
+    get_services,
+    patch_repository,
+    unpatch_repository,
+)
 
 logging.basicConfig(level=logging.INFO)
 
@@ -15,11 +20,16 @@ def cli():
     pass
 
 
-@click.command(name="patch", help="Patch the golang test runner")
+@click.command(name="patch", help="Patch the terraform provider for LocalStack testing")
 def patch_command():
     patch_repository()
 
 
+@click.command(name="unpatch", help="Revert patches to the terraform provider")
+def unpatch_command():
+    unpatch_repository()
+
+
 @click.command(name="build", help="Build binary for testing")
 @click.option(
     "--service",
@@ -29,15 +39,19 @@ def patch_command():
     help="""Service to build; use "ls-all", "ls-community", "ls-pro" to build all services, example: 
 --service=ls-all; --service=ec2; --service=ec2,iam""",
 )
-@click.option("--force-build", "-f", is_flag=True, default=False, help="Force rebuilds binary")
+@click.option(
+    "--force-build", "-f", is_flag=True, default=False, help="Force rebuilds binary"
+)
 def build_command(service, force_build):
     services = get_services(service)
 
     for service in services:
         logging.info(f"Building {service}...")
         try:
             start_time = timer()
-            build_test_binary(service=service, tf_root_path=TF_REPO_PATH, force_build=force_build)
+            build_test_binary(
+                service=service, tf_root_path=TF_REPO_PATH, force_build=force_build
+            )
             end_time = timer()
             logging.info(f"Build completed in {end_time - start_time:.2f} seconds")
         except KeyboardInterrupt:
@@ -65,7 +79,8 @@ def clean_command():
 
 
 if __name__ == "__main__":
-    cli.add_command(build_command)
     cli.add_command(patch_command)
+    cli.add_command(unpatch_command)
+    cli.add_command(build_command)
     cli.add_command(clean_command)
     cli()