Add CodeQL native C build scanning to storage pipeline

eng/pipelines/templates/stages/1es-redirect.yml

@@ -19,6 +19,10 @@ parameters:
 - name: oneESTemplateTag
   type: string
   default: release
+- name: EnableCompiledCodeql
+  type: boolean
+  default: false
+
 
 extends:
   ${{ if and(parameters.Use1ESOfficial, eq(parameters.oneESTemplateTag, 'canary')) }}:
@@ -55,9 +59,16 @@ extends:
         enabled: false
         justificationForDisabling: "ESLint injected task has failures because it uses an old version of mkdirp. We should not fail for tools not controlled by the repo. See: https://dev.azure.com/azur     19 e-sdk/internal/_build/results?buildId=3556850"
       codeql:
-        compiled:
-          enabled: false
-          justificationForDisabling: "To reduce redundant CG runs across all our pipeline jobs we are disabling and only running in our main build job."
+        ${{ if eq(parameters.EnableCompiledCodeql, true) }}:
+          # "cpp" covers both C and C++ code. Language is specified because 
+          # checkout happens after the injected "CodeQL Initialize" step
+          language: cpp
+          compiled:
+            enabled: true
+        ${{ else }}:
+          compiled: 
+            enabled: false
+            justificationForDisabling: "To reduce redundant CG runs across all our pipeline jobs we are disabling and only running in our main build job."
       componentgovernance:
         enabled: false
         justificationForDisabling: "To reduce redundant CG runs across all our pipeline jobs we are disabling and only running in our main build job."

eng/pipelines/templates/stages/archetype-sdk-client.yml

@@ -81,6 +81,9 @@ parameters:
   - name: oneESTemplateTag
     type: string
     default: release
+  - name: EnableCompiledCodeql
+    type: boolean
+    default: false
   - name: EnvVars
     type: object
     default: {}
@@ -89,6 +92,7 @@ extends:
   template: /eng/pipelines/templates/stages/1es-redirect.yml
   parameters:
     oneESTemplateTag: ${{ parameters.oneESTemplateTag }}
+    EnableCompiledCodeql: ${{ parameters.EnableCompiledCodeql }}
     stages:
       - ${{ if and(eq(parameters.SkipPrValidation, true), eq(variables['Build.Reason'], 'Manual')) }}:
         - stage: NoOp

eng/pipelines/templates/steps/build-package-artifacts.yml

@@ -136,6 +136,44 @@ steps:
     displayName: 'Install QEMU Dependencies'
     condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
 
+  # Windows: cibuildwheel fetches CPython from api.nuget.org, which is blocked.
+  # Pre-provision the interpreters from the Azure DevOps NuGet feed into the
+  # cibuildwheel cache so it skips the blocked download.
+  - task: NuGetAuthenticate@1
+    displayName: 'Authenticate to NuGet feed'
+    condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['ENABLE_EXTENSION_BUILD'], 'true'))
+
+  - pwsh: |
+      $ErrorActionPreference = 'Stop'
+      $feed = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json"
+      $version = "3.10.11"
+      $cache = Join-Path "$(Agent.TempDirectory)" "cibw-cache"
+      $nugetCpython = Join-Path $cache "nuget-cpython"
+      New-Item -ItemType Directory -Force -Path $nugetCpython | Out-Null
+      $nuget = (Get-Command nuget).Source
+      & $nuget install python -Version $version -Source $feed -OutputDirectory $nugetCpython
+      if ($LASTEXITCODE) { 
+        Write-Error "Failed to download python $version"
+        exit 1
+      }
+      & $nuget install pythonarm64 -Version $version -Source $feed -OutputDirectory $nugetCpython
+      if ($LASTEXITCODE) { 
+        Write-Error "Failed to download pythonarm64 $version"
+        exit 1
+      }
+      Write-Host "##vso[task.setvariable variable=CIBW_CACHE_PATH]$cache"
+    displayName: 'Pre-provision CPython for cibuildwheel'
+    condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['ENABLE_EXTENSION_BUILD'], 'true'))
+
+  # Linux: forward PIP_INDEX_URL into the build container so pip fetches build
+  # dependencies from the Azure DevOps feed instead of PyPI. Using
+  # CIBW_ENVIRONMENT_PASS_LINUX (not CIBW_ENVIRONMENT) preserves the package's
+  # own [tool.cibuildwheel].environment (CFLAGS) setting.
+  - pwsh: |
+      Write-Host "##vso[task.setvariable variable=CIBW_ENVIRONMENT_PASS_LINUX]PIP_INDEX_URL"
+    displayName: 'Forward PIP_INDEX_URL to cibuildwheel'
+    condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux')))
+
   - pwsh: |
       which python
       sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --inactive --service="${{ parameters.ServiceDirectory }}"

eng/tools/azure-sdk-tools/ci_tools/build.py

@@ -221,8 +221,16 @@ def create_package(
     setup_directory_or_file: str, dest_folder: str, enable_wheel: bool = True, enable_sdist: bool = True
 ):
     """
-    Uses the invoking python executable to build a wheel and sdist file given a setup.py or setup.py directory. Outputs
-    into a distribution directory and defaults to the value of get_artifact_directory().
+    Builds a wheel and/or sdist file given a setup.py, pyproject.toml, or directory containing either.
+
+    For packages with compiled extensions (ext_modules):
+    - setup.py: uses cibuildwheel to build platform-specific wheels
+    - pyproject.toml: uses cibuildwheel to build platform-specific wheels (respects [tool.cibuildwheel] config)
+
+    For pure Python packages:
+    - Uses python -m build
+
+    Outputs into a distribution directory and defaults to get_artifact_directory().
     """
 
     dist = get_artifact_directory(dest_folder)
@@ -231,36 +239,54 @@ def create_package(
     should_log_build_output = logger.getEffectiveLevel() <= logging.DEBUG
 
     if setup_parsed.is_pyproject:
-        # when building with pyproject, we will use `python -m build` to build the package
-        # -n argument will not use an isolated environment, which means the current environment must have all the dependencies of the package installed, to successfully
-        # pull in the dynamic `__version__` attribute. This is because setuptools is actually walking the __init__.py to get that attribute, which will fail
-        # if the imports within the setup.py don't work. Perhaps an isolated environment is better, pulling all the "dependencies" into the [build-system].requires list
-
-        # given the additional requirements of the package, we should install them in the current environment before attempting to build the package
-        # we assume the presence of `wheel`, `build`, `setuptools>=61.0.0`
-        pip_output = get_pip_list_output(sys.executable)
-        necessary_install_requirements = [
-            req for req in setup_parsed.requires if parse_require(req).name not in pip_output.keys()
-        ]
-        run_logged(
-            [sys.executable, "-m", "pip", "install", *necessary_install_requirements],
-            cwd=setup_parsed.folder,
-            check=False,
-            should_stream_to_console=should_log_build_output,
-        )
-        run_logged(
-            [
-                sys.executable,
-                "-m",
-                "build",
-                f"-n{'s' if enable_sdist else ''}{'w' if enable_wheel else ''}",
-                "-o",
-                dist,
-            ],
-            cwd=setup_parsed.folder,
-            check=True,
-            should_stream_to_console=should_log_build_output,
-        )
+        # when building with pyproject, check if package has compiled extensions
+        if enable_wheel and setup_parsed.ext_modules:
+            # Use cibuildwheel for compiled extensions (respects [tool.cibuildwheel] config)
+            run_logged(
+                [sys.executable, "-m", "cibuildwheel", "--output-dir", dist],
+                cwd=setup_parsed.folder,
+                check=True,
+                should_stream_to_console=should_log_build_output,
+            )
+            if enable_sdist:
+                # Build sdist separately with python -m build
+                run_logged(
+                    [sys.executable, "-m", "build", "-s", "-o", dist],
+                    cwd=setup_parsed.folder,
+                    check=True,
+                    should_stream_to_console=should_log_build_output,
+                )
+        else:
+            # Use python -m build for pure Python packages
+            # -n argument will not use an isolated environment, which means the current environment must have all the dependencies of the package installed, to successfully
+            # pull in the dynamic `__version__` attribute. This is because setuptools is actually walking the __init__.py to get that attribute, which will fail
+            # if the imports within the setup.py don't work. Perhaps an isolated environment is better, pulling all the "dependencies" into the [build-system].requires list
+
+            # given the additional requirements of the package, we should install them in the current environment before attempting to build the package
+            # we assume the presence of `wheel`, `build`, `setuptools>=61.0.0`
+            pip_output = get_pip_list_output(sys.executable)
+            necessary_install_requirements = [
+                req for req in setup_parsed.requires if parse_require(req).name not in pip_output.keys()
+            ]
+            run_logged(
+                [sys.executable, "-m", "pip", "install", *necessary_install_requirements],
+                cwd=setup_parsed.folder,
+                check=False,
+                should_stream_to_console=should_log_build_output,
+            )
+            run_logged(
+                [
+                    sys.executable,
+                    "-m",
+                    "build",
+                    f"-n{'s' if enable_sdist else ''}{'w' if enable_wheel else ''}",
+                    "-o",
+                    dist,
+                ],
+                cwd=setup_parsed.folder,
+                check=True,
+                should_stream_to_console=should_log_build_output,
+            )
     else:
         if enable_wheel:
             if setup_parsed.ext_modules:

eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py

@@ -703,6 +703,25 @@ def parse_pyproject(
     ext_modules = get_value_from_dict(toml_dict, "tool.setuptools.ext-modules", [])
     ext_modules = [Extension(**moduleArgDict) for moduleArgDict in ext_modules]
 
+    # Declaring ext-modules in [tool.setuptools.ext-modules] is still experimental in setuptools, and the
+    # abi3 / py_limited_api wheel tag cannot be expressed declaratively, so compiled-extension packages keep
+    # their Extension(...) definition in setup.py. When such a package also has a [project] table, pyproject
+    # wins over setup.py during parsing and the extension would otherwise go undetected, causing the build to
+    # be mis-routed to `python -m build` (pure-Python) instead of cibuildwheel. Fall back to setup.py here so
+    # the extension is still discovered. Guarded so a setup.py that cannot be parsed cannot regress packages
+    # that parse fine today.
+    if not ext_modules:
+        sibling_setup_py = os.path.join(package_directory, "setup.py")
+        if os.path.exists(sibling_setup_py):
+            try:
+                setup_py_result = parse_setup_py(sibling_setup_py)
+                ext_package = ext_package or setup_py_result[11]  # ext_package
+                ext_modules = setup_py_result[12]  # ext_modules
+            except Exception as e:  # pragma: no cover - defensive, preserves prior behavior
+                logging.warning(
+                    f"Found setup.py alongside {pyproject_filename} but could not parse it for ext_modules: {e}"
+                )
+
     # fmt: off
     return (
         name,                   # str

sdk/storage/ci.yml

@@ -39,6 +39,9 @@ extends:
     ServiceDirectory: storage
     TestProxy: true
     TestTimeoutInMinutes: 120
+    # Enable Compiled CodeQL because azure-storage-extensions has C code that 
+    # must be scanned
+    EnableCompiledCodeql: true
     Artifacts:
     - name: azure-storage-blob
       safeName: azurestorageblob