diff --git a/PERF.md b/PERF.md index 3a284a47b7..2264200706 100644 --- a/PERF.md +++ b/PERF.md @@ -253,6 +253,24 @@ What made dev about eight megabytes heavier than `main` on linux/amd64 by the CI size job) is not attributed here; #1694 finds it, cuts what is redundant, and lowers this number in the same commit as each cut. +It was reset a seventh time on 2026-10-05, when sec — sec-af, the +security auditor, copied in once at its tag `codeaf-absorb` — became the second +program built into codeaf (`internal/secaf`). Like senior-dev's, this one is a +decision: the programs codeaf hands a whole task to are built into every build. +Measured on darwin/amd64 with its own furrow artifact staged, with the flags +`make build` uses, the same tree with and without the program in the build's +list (`internal/delegate/builtin`), on Go 1.27.0: + +| platform | without | with | what sec cost | +| --- | --- | --- | --- | +| darwin/amd64, furrow staged | 67,051,312 | 69,564,304 | 2,512,992 | + +About 1.4 megabytes of it is sec-af's own code and the schema and YAML readers +it brings (`invopop/jsonschema`, `santhosh-tekuri/jsonschema`, `yaml/v4`); the +rest is the type and line tables that code carries. The budget rises by exactly +that cost, to 71,363,000 — this change's bill and nothing else. The tree without +it was 1,798,688 under the sixth reset's 68,850,000, and still is. + ## Adaptive run shutdown grace `Agent.Close` cancels adaptive runs and their name calls, then gives all accepted @@ -1482,6 +1500,15 @@ fixed cap is **57,218** bytes, exactly 94 above the previous measurement. The lean cap remains **49,590** bytes. The dated fixed waiver in `prefixWaivers` pays only that measured increase. +**sec, the second program (sec-af's security audit), adds its guide (2026-10-05).** A program +costs the fixed prefix one item of the hand-off paragraph's list — its guide, at +most 400 bytes — and nothing else: its manual page and the wording of the turn its +report wakes ride no request. sec's guide is 386 bytes, and the caps +rise by exactly what it measured: against dev on 2026-10-07 (baa2d7f0f) it costs +both prefixes 396 bytes, so the full cap is **57,614** bytes and the lean cap +**49,986**, both dated in `prefixWaivers`. (Against the dev of 2026-10-05 it cost +the lean prefix only 229, before the page carried a program's guide there too.) + ## Following through on a completion claim A turn may decline handoff once per request when its own continuation says no diff --git a/SIZE-BUDGET b/SIZE-BUDGET index 9c613a556f..ebe4066ad7 100644 --- a/SIZE-BUDGET +++ b/SIZE-BUDGET @@ -1 +1 @@ -68850000 +71363000 diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md index 312ba56c9c..fd9c7ca8cd 100644 --- a/THIRD-PARTY-NOTICES.md +++ b/THIRD-PARTY-NOTICES.md @@ -468,6 +468,74 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ``` +## github.com/bahlo/generic-list-go + +- Version: v0.2.0 +- Licence: BSD-3-Clause + +### LICENSE + +```text +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` + +## github.com/buger/jsonparser + +- Version: v1.1.2 +- Licence: MIT + +### LICENSE + +```text +MIT License + +Copyright (c) 2016 Leonid Bugaev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` + ## github.com/charmbracelet/bubbletea - Version: v1.3.10 @@ -1711,6 +1779,35 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. limitations under the License. ``` +## github.com/invopop/jsonschema + +- Version: v0.14.0 +- Licence: MIT + +### COPYING + +```text +Copyright (C) 2014 Alec Thomas + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` + ## github.com/lucasb-eyer/go-colorful - Version: v1.4.0 @@ -2128,109 +2225,505 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ``` -## github.com/remyoudompheng/bigfft +## github.com/pb33f/ordered-map/v2 -- Version: v0.0.0-20230129092748-24d4a6f8daec -- Licence: BSD-3-Clause +- Version: v2.3.1 +- Licence: Apache-2.0 ### LICENSE ```text -Copyright (c) 2012 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -## github.com/rivo/uniseg + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -- Version: v0.4.7 -- Licence: MIT + 1. Definitions. -### LICENSE.txt + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. -```text -MIT License + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. -Copyright (c) 2019 Oliver Kuederle + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -``` + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. -## github.com/segmentio/asm + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). -- Version: v1.1.3 -- Licence: MIT + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. -### LICENSE + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." -```text -MIT License + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. -Copyright (c) 2021 Segment + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -``` + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and -## github.com/segmentio/encoding + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and -- Version: v0.5.4 -- Licence: MIT + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` + +## github.com/remyoudompheng/bigfft + +- Version: v0.0.0-20230129092748-24d4a6f8daec +- Licence: BSD-3-Clause + +### LICENSE + +```text +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` + +## github.com/rivo/uniseg + +- Version: v0.4.7 +- Licence: MIT + +### LICENSE.txt + +```text +MIT License + +Copyright (c) 2019 Oliver Kuederle + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` + +## github.com/santhosh-tekuri/jsonschema/v5 + +- Version: v5.3.1 +- Licence: Apache-2.0 + +### LICENSE + +```text + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. +``` + +## github.com/segmentio/asm + +- Version: v1.1.3 +- Licence: MIT + +### LICENSE + +```text +MIT License + +Copyright (c) 2021 Segment + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` + +## github.com/segmentio/encoding + +- Version: v0.5.4 +- Licence: MIT ### LICENSE @@ -2355,6 +2848,84 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ``` +## go.yaml.in/yaml/v4 + +- Version: v4.0.0-rc.2 +- Licence: Apache-2.0 + +### LICENSE + +```text + +This project is covered by two different licenses: MIT and Apache. + +#### MIT License #### + +The following files were ported to Go from C files of libyaml, and thus +are still covered by their original MIT license, with the additional +copyright staring in 2011 when the project was ported over: + + apic.go emitterc.go parserc.go readerc.go scannerc.go + writerc.go yamlh.go yamlprivateh.go + +Copyright (c) 2006-2010 Kirill Simonov +Copyright (c) 2006-2011 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +### Apache License ### + +All the remaining project files are covered by the Apache license: + +Copyright (c) 2011-2019 Canonical Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +``` + +### NOTICE + +```text +Copyright 2011-2016 Canonical Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +``` + ## golang.org/x/crypto - Version: v0.54.0 diff --git a/cmd/codeaf/carried.go b/cmd/codeaf/carried.go index 0d3b4f1e50..49eadc4767 100644 --- a/cmd/codeaf/carried.go +++ b/cmd/codeaf/carried.go @@ -236,14 +236,21 @@ func runCarriedHost(ctx context.Context, inv *delegate.Invocation) error { strings.Join(foldSynopsis("codeaf "+inv.Program.Name+" "+carriedSynopsis), "\n"), inv.Program.Name) return exitCannotRun } + // THE PROGRAM SAYS WHICH OF ITS FLAGS NAMES ITS MODELS + // ([delegate.Delegate.ModelFlag]); codeaf resolves what was typed there. + modelFlag := strings.TrimSpace(inv.Program.ModelFlag) + asked := "" + if modelFlag != "" { + asked = strings.TrimSpace(inv.ExplicitFlags[modelFlag]) + } road, err := carriedModels() if err != nil { - if word := strings.TrimSpace(inv.ExplicitFlags["high"]); word != "" && errors.Is(err, config.ErrNoAPIKey) { + if word := asked; word != "" && errors.Is(err, config.ErrNoAPIKey) { return session.ProgramShellModelRefusal(word) } return err } - if word := strings.TrimSpace(inv.ExplicitFlags["high"]); word != "" && road.resolveModel != nil { + if word := asked; word != "" && road.resolveModel != nil { resolved, err := road.resolveModel(word) if err != nil { return err @@ -259,9 +266,9 @@ func runCarriedHost(ctx context.Context, inv *delegate.Invocation) error { } resolved = strings.Join(models, ",") } - inv.Line = carriedResolvedHigh(inv.Line, resolved) + inv.Line = carriedResolvedModel(inv.Line, modelFlag, resolved) } - if strings.TrimSpace(inv.ExplicitFlags["high"]) == "" && road.defaultSeat != nil { + if asked == "" && road.defaultSeat != nil { road.seat, err = road.defaultSeat() if err != nil { return err @@ -390,8 +397,8 @@ func runCarriedHost(ctx context.Context, inv *delegate.Invocation) error { Args: carriedInFolder(carriedChildLine(inv), inv, folder), // NO PROVIDER KEY IS INHERITED BY THE PROGRAM (delegate.ChildEnv): the engine // gets the loopback token it needs, and model commands lose that token. - Env: append(delegate.ChildEnv(api.API()), "SENIOR_DEV_IGNORED_AT_START="+folder.IgnoredFile(), - gitidentity.InputsEnv+"="+folder.InputsFile()), + Env: append(append(delegate.ChildEnv(api.API()), "SENIOR_DEV_IGNORED_AT_START="+folder.IgnoredFile(), + gitidentity.InputsEnv+"="+folder.InputsFile()), delegate.RecordsEnv(record)...), Dir: here, StderrPath: filepath.Join(record, carriedStderrName), Grace: grace, @@ -562,23 +569,23 @@ func carriedSeatedLine(inv *delegate.Invocation, seat string) []string { return append(line, inv.Line[at:]...) } -// carriedResolvedHigh replaces only the model flag's value on the person's +// carriedResolvedModel replaces only the model flag's value on the person's // line. Every other program flag and every word of the brief stays as typed. -func carriedResolvedHigh(line []string, model string) []string { +func carriedResolvedModel(line []string, flag, model string) []string { resolved := append([]string(nil), line...) for i, word := range resolved { if word == "--" { break } switch { - case word == "--high" || word == "-high": + case word == "--"+flag || word == "-"+flag: if i+1 < len(resolved) { resolved[i+1] = model } - case strings.HasPrefix(word, "--high="): - resolved[i] = "--high=" + model - case strings.HasPrefix(word, "-high="): - resolved[i] = "-high=" + model + case strings.HasPrefix(word, "--"+flag+"="): + resolved[i] = "--" + flag + "=" + model + case strings.HasPrefix(word, "-"+flag+"="): + resolved[i] = "-" + flag + "=" + model } } return resolved @@ -718,7 +725,11 @@ func (v *carriedView) begin() { if v.records != nil { return } - v.say("%s · working in %s · %s", v.inv.Program.Name, v.where(), v.inv.Ceilings.Summary()) + if ceilings := v.inv.Ceilings.Summary(); ceilings != "" { + v.say("%s · working in %s · %s", v.inv.Program.Name, v.where(), ceilings) + } else { + v.say("%s · working in %s", v.inv.Program.Name, v.where()) + } // A COPY IS CUT FROM A COMMIT, so what the person had not committed is not // in it, and a person at a shell is told so before the run spends a cent on // work that needed it — as a conversation's receipt tells them. @@ -1008,6 +1019,23 @@ func (v *carriedView) end(result delegate.Result, runErr error, limited bool, sp line += "; its last stage was " + result.Reading.LastStage } v.say("%s", line) + case !v.inv.Program.LandsTree(): + // A PROGRAM THAT ANSWERS IS READ FOR ITS ANSWER. Its ending's first + // line is what it came to, and the answer under it is the report the + // person ran it for — the account a conversation is handed — rather + // than what a program that edits code claimed and checked. + v.say("%s", carriedEnding(name, *terminal)) + if answer := strings.TrimSpace(terminal.Deliverable()); answer != "" && answer != strings.TrimSpace(terminal.Message) { + v.say("") + for _, line := range strings.Split(answer, "\n") { + if line = strings.TrimRight(line, " "); line == "" { + v.say("") + continue + } + v.say(" %s", line) + } + v.say("") + } default: v.say("%s", carriedEnding(name, *terminal)) if claim := terminal.Claim(); claim != "" { diff --git a/cmd/codeaf/carried_test.go b/cmd/codeaf/carried_test.go index a8fcfdd85f..6db957c59a 100644 --- a/cmd/codeaf/carried_test.go +++ b/cmd/codeaf/carried_test.go @@ -76,7 +76,7 @@ func TestTheFrontPageFitsWithTheProgramsThisBuildCarries(t *testing.T) { func TestSeniorDevShellFirstLineNamesItsEffectiveCeiling(t *testing.T) { program := fakeCarriedProgram() - program.Name = "senior-dev" + program.Name, program.Unattended = "senior-dev", delegate.SeniorDevCeilings for _, tc := range []struct { line []string want string diff --git a/docs/changes/unreleased/1781-sec.md b/docs/changes/unreleased/1781-sec.md new file mode 100644 index 0000000000..9ea277726c --- /dev/null +++ b/docs/changes/unreleased/1781-sec.md @@ -0,0 +1,28 @@ +--- +kind: added +title: sec, sec-af's security audit built into codeaf, audits a repository or its changes +pr: 1781 +surface: [chat, engine, docs, build] +invalidates: + - "senior-dev was the only program codeaf carries. There are two: sec (`/sec`, `codeaf sec`, `via: \"sec\"`) is the second, and it changes no files." + - "sec-af was a separate AgentField node with its own key and a coding-agent binary per agent call. It is copied into codeaf at sec-af's tag codeaf-absorb (47d57d7) as internal/secaf, runs only through codeaf, and is frozen in its own repository." + - "Every program's unattended ceilings were senior-dev's, chosen by its name in nine places. A program names its own (`Delegate.Unattended`); a program with none runs on what the conversation has left." + - "A program that lands text had no ending of its own and was sent looking for a branch. Its ending asks the chat for a summary and the program's own offer (`Delegate.FollowUp`) and starts nothing." + - "The model API answered 402 to any call its ceiling could not admit at that moment. A call held only by calls still in flight is answered 429 with Retry-After and X-Codeaf-Held; 402 is kept for a ceiling truly reached." + - "A shell run resolved a person's model only through a flag named `--high`. A program names its own model flag (`Delegate.ModelFlag`)." + - "A program the chat proposed was always handed the composed brief, which opens on `WHAT THE PERSON ASKED FOR`. A program whose brief is a few words it reads itself (`Delegate.Words`, sec's `changes since main thorough`) is handed those words alone, and a report program's row in the project's index points at its record folder, not at the repository." + - "A bare `/` always asked for a brief. A program with a default brief runs bare (`/sec` audits the whole repository), and its row shows its own arguments." + - "sec-af failed every audit that named compliance frameworks, and its PR mode read the diff from the wrong folder and only filtered findings after a whole-repository scan. Frameworks map and report; an audit of the changes reads them in the folder under audit and tells the hunters what changed." + - "The fixed and lean prefix caps were 57,218 and 49,590 bytes and SIZE-BUDGET was 68,850,000. They are 57,614, 49,986 and 71,363,000, raised by exactly what sec measured (PERF.md)." +--- + +`/sec` (or `changes`, `changes since `, `quick`, `thorough`) starts an +audit that reads the folder with four read-only tools, hunts for vulnerabilities, +tests each one against the code, and reports what stands with file, line and fix. +Its report goes to the task's record folder (`sec-report.md`, `.json`, `.sarif`) +and its account to the conversation, which offers to hand confirmed findings to +senior-dev. Every model call goes through the run's model API, priced and held to +its ceiling: $5 and four hours unless the conversation has less. +Its agents run on `internal/agentsession`, a read-only agent loop over the run's +model API that any program codeaf carries can use, and that names each program's +own work in its prompt. diff --git a/docs/design/delegate/PROTOCOL.md b/docs/design/delegate/PROTOCOL.md index 9325807ebc..1ea36ddb44 100644 --- a/docs/design/delegate/PROTOCOL.md +++ b/docs/design/delegate/PROTOCOL.md @@ -114,7 +114,13 @@ one package codeaf's funnel law lets spell a model route. Every call: 1. is refused before it is made when the run's dollar ceiling is reached, with HTTP 402 (a status senior-dev does not retry). A run a refusal ended is reported as ` reached the run's dollar ceiling of $X: …`, whatever - status the program itself wrote, and ends on the run's cost limit; + status the program itself wrote, and ends on the run's cost limit. A call + that would fit once calls already in flight have given back what they + reserved is not refused but HELD (2026-10-05): HTTP 429 with `Retry-After` + and `X-Codeaf-Held: ceiling`, and no turn is written, because it was not + made and will be asked again. A program that makes many calls at once + (sec) waits it out; one that makes one call at a time never + meets it; 2. goes through codeaf's own model funnel, with its router, retries, caching and billing, on the model the program asked for when one of the person's services can serve it, and otherwise on the run's work seat, which the turn @@ -261,6 +267,14 @@ record that named one, and its stage's word before any has. ## 8. Built in now for later programs +*sec-af became the second program on 2026-10-05: `sec`, in +`internal/secaf`, landing text (docs/design/security-audit/ABSORB.md). It is the +first program to make many calls at once and the first to land text, which is +why the held answer in §3, a program's own ceilings (`Unattended`), its follow-up +offer and its record folder (`CODEAF_RECORDS`) arrived with it. What follows is +as it was written on 2026-09-23.* + + pr-af and sec-af, looked at on 2026-09-23, would need: plain structured calls with `response_format`, many conversations at once (kept apart by thread), grandchildren inheriting the API's address and token, quiet stretches of up to diff --git a/docs/design/security-audit/ABSORB.md b/docs/design/security-audit/ABSORB.md new file mode 100644 index 0000000000..e116595236 --- /dev/null +++ b/docs/design/security-audit/ABSORB.md @@ -0,0 +1,84 @@ +# Absorbing sec-af into codeaf + +The security auditor was copied once from the sec-af repository and is frozen +there. This page records what was copied, what was left behind, and what was +changed on the way in, so a reader diffing `internal/secaf` against sec-af can +tell a deliberate change from drift. + +## Source + +- Repository: sec-af, Go module `github.com/Agent-Field/sec-af/go` +- Tag `codeaf-absorb`, commit `47d57d76ea170c44893d138862384476e3a617b8` +- Copied on 2026-10-05 +- `go/internal/*` became `internal/secaf/*`; `go/docs/DESIGN.md` is + `SEC-AF-PORT-DESIGN.md` and `go/README.md` is `SEC-AF-GO-README.md` here, + both unchanged. + +## Not copied + +- `cmd/` (the HTTP node binary), `Dockerfile`, `docker-entrypoint.sh`, + `agentfield-package.yaml`, `Makefile`, `scripts/` (the Python golden and + schema generators). + +## Deleted from the copy + +- `internal/node`: the AgentField node: agent construction from the + environment, control-plane registration and serving, git cloning into a + workspaces directory, and the `audit` reasoner's HTTP status mapping. What + the in-process program needs moved to `internal/secaf/audit` (below). +- `internal/audit`: an uncalled one-field stub ported only so every Python + module had a counterpart. The name now belongs to the audit entry. +- `config/ai.go`'s provider machinery: provider selection, the external + CLI binaries and their paths, `ProviderEnv`, the eager `XDG_DATA_HOME` + directory, and every `SEC_AF_*` / `HARNESS_*` / `AI_MODEL` variable read. + `AuditConfig.Provider` (unread) went with it. +- The prompt drift test that compared `prompts/files` with the Python tree: + sec-af is frozen, so there is nothing left to drift from. +- Tests that only covered the deleted parts: node construction and env + precedence, cloning and workspace fallbacks, provider env, the SDK router's + `/discover` payload and router tags. + +## Changed + +- Imports were rewritten to `github.com/Agent-Field/codeaf/internal/secaf/...`. +- The AgentField SDK's `agent` and `harness` packages are no longer imported. + Only `sdk/go/ai` remains, at codeaf's pinned version; no `ai` symbol needed + an adaptation. +- `appx` and the agent loop codeaf wrote for the audit live outside the copy, + at `internal/agentsession/appx` and `internal/agentsession`, because they are + codeaf's and not sec-af's, and a second carried program (`/review`, being built on + its own branch) is to run on them too. The loop names the work in its + system prompt from `Config.Work`; sec's is `a security audit`. +- `appx` declares its own `HarnessOptions{Cwd, ProjectDir}` and + `HarnessResult{Result, Parsed, IsError, ErrorMessage, NumTurns, DurationMS, + CostUSD}` in place of the SDK's types. `CostUSD` stays a `*float64` because + the cost trackers skip an unreported cost. +- `reasoners.RegisterAll` fills an in-process `Registry` (name to handler, + plus input schema) instead of an SDK router. The 33 names, their order, and + the input validation are the same. A refused body is a + `*reasoners.InputError` carrying 422, where it was an SDK `ExecuteError`. +- `audit.WithLocalCalls` answers `.call` in process. It keeps the JSON round + trip in both directions and returns a failed call as an opaque error, as the + control-plane hop did. +- `audit.Run` resolves the repository before it builds the orchestrator, and + passes it in through the new `orch.NewAt`. The node built the orchestrator + against `SEC_AF_REPO_PATH` or its working directory first. +- Only an existing local directory is audited. A URL, a missing path or a + file is refused with a 400 `*audit.Error`, and nothing is cloned. +- `phases.NodeID()` is the constant `"sec-af"`; `NODE_ID` is no longer read. +- `config.AIIntegrationConfig` keeps `AIModel` and the retry schedule. + `DefaultAIConfig()` has the node's retry defaults (3 retries, 2s initial and + 8s maximum backoff) and no model. With no model set, the AI gate passes no + `ai.WithModel`, so the App picks the model. +- The gates' `HarnessWrapper` (not on the live path) passes only cwd and + project_dir; its model, max-turns and budget overrides are gone. +- `harnessx.Extract` writes its diagnostic block to `harnessx.Diagnostics` + (default `io.Discard`) instead of stdout. The audit entry no longer prints + `AUDIT ERROR:` to stdout; the failure note carries the same text. +- `schemas.Verdict` was renamed `schemas.ExploitVerdict`, because codeaf + allows exactly one type named `Verdict` (the taxonomy law). +- Every `aforge`/`openaf` spelling was removed from Go sources; testdata keeps + its recorded values. +- `invopop/jsonschema` resolves to v0.14.0, the version codeaf's module graph + already selected (sec-af pinned v0.13.0). It is only the fallback schema + reflector, and every golden passes on it. diff --git a/docs/design/security-audit/SEC-AF-GO-README.md b/docs/design/security-audit/SEC-AF-GO-README.md new file mode 100644 index 0000000000..b80cba4e3b --- /dev/null +++ b/docs/design/security-audit/SEC-AF-GO-README.md @@ -0,0 +1,242 @@ +# SEC-AF — Go node + +A Go implementation of the SEC-AF security-audit node. It registers the same +reasoner surface under the same names as the Python node, exposes a +byte-compatible HTTP API, and reaches every sub-agent through the control plane, +so the control-plane DAG UI renders the same multi-node orchestration graph as +the Python node (see [Pipeline DAG on the control plane](#pipeline-dag-on-the-control-plane)). +The Python package under `src/sec_af/` is untouched; this implementation lives +entirely under `go/`. + +One binary: + +| Binary | Node ID | Default port | Role | +|----------|----------|--------------|---------------------------------------------| +| `sec-af` | `sec-af` | `8013` | Full audit pipeline (recon → hunt → prove → remediation) | + +Module path: `github.com/Agent-Field/sec-af/go`. + +## Install + +Installing the bare repository URL follows the root manifest's redirect to this +package and registers the Go node as `sec-af` on `:8013`: + +```bash +af install https://github.com/Agent-Field/sec-af +af run sec-af +af call sec-af.audit --in '{"repo_url":"https://github.com/dolevf/Damn-Vulnerable-GraphQL-Application"}' +``` + +To install the Python node deliberately, clone the repository and run +`af install ./sec-af`; local-path installs do not follow the redirect. `NODE_ID` +and `PORT` still override the Go defaults if you want a different id or port. + +## Pipeline DAG on the control plane + +An audit is not one execution. Every phase and every sub-agent runs as a +**tracked child execution** of the parent `audit`, and the control-plane UI +renders the run as that DAG: + +``` +audit +├── recon_phase +│ ├── run_architecture_mapper ┐ +│ ├── run_dependency_auditor ├ gather (3) +│ ├── run_config_scanner ┘ +│ ├── run_data_flow_mapper ┐ gather (2) — skipped when depth == quick +│ └── run_security_context_profiler ┘ +├── hunt_phase +│ ├── run__hunter × N semaphore max(1, min(max_concurrent_hunters=4, N)) +│ └── run_deduplicator only when ≥1 fingerprint-unique finding +├── prove_phase +│ └── run_verifier × K K = min(len(findings), prover cap) semaphore 3 +└── remediation_phase + └── run_remediation × M M = confirmed/likely findings without a remediation +``` + +Python draws that graph with `await router.call(f"{NODE_ID}.", ...)`; +the Go port makes the identical `Agent.Call` with the same target name and the +same kwargs, so the node/edge multiset is the same. The registered surface is 34 +reasoners: `audit` plus the 33 router reasoners, tagged +`["security","audit","red-team"]`. + +Two nodes that are registered but never `.call`ed — `run_cwe_expansion` and the +individual `run_tracer` / `run_sanitization_analyzer` / `run_exploit_hypothesizer` +/ `run_verdict_agent` reasoners — are part of the surface because Python +registers them; the verifier runs those four in process, exactly as Python does, +so they contribute no DAG node on a normal audit. + +## Depending on the AgentField Go SDK + +This module depends on the AgentField Go SDK +(`github.com/Agent-Field/agentfield/sdk/go`) via a **real, committed `require`** +resolved from `proxy.golang.org` — there is **no `replace` directive** and no +sibling checkout to lay out. `go build ./...` works out of the box against the +pinned SDK version in `go.mod`. + +- **CI / Docker.** `go mod download` pulls the SDK (and every other dependency) + straight from the module proxy. No `GOWORK=off`, no sparse clone. +- **Dev — optional Go workspace.** A gitignored `go.work` (spanning this module + and a local `agentfield/sdk/go` checkout) is the way to develop against + unreleased SDK changes; with it present, `go build ./...` picks up local SDK + edits live. It is never committed. + +Bumping the SDK is a deliberate, reviewable change: bump the `require` version +in `go.mod`, and move the Docker builder image tag together with it if the SDK's +own `go` directive ever advances past `go 1.21`. + +## Build & run locally + +From `go/`: + +```bash +make build # go build ./... +make vet # go vet ./... +make test # go test ./... +make fmtcheck # test -z "$(gofmt -l .)" +make check # build + vet + test + fmtcheck (the CI gate) +make fmt # gofmt -w . +make run # run the node (sec-af, :8013) +``` + +`make run` needs a control plane reachable at `AGENTFIELD_SERVER` (default +`http://localhost:8080`). The node reads all configuration from the environment +at startup. + +## Docker + +The image is a multi-stage build: `go mod download` + `go build` in a +`golang:1.23` stage, then a slim Debian runtime mirroring the Python image — +the pinned `opencode` CLI (`1.17.15`), the checksum-verified `aforge` binary, +a non-root `secaf` user (uid/gid 10001), and a `docker-entrypoint.sh` that +generates `opencode.json` from `HARNESS_MODEL` at container start. + +The build context is the **repo root** so the `go/` module is in context: + +```bash +# from the repo root +docker build -f go/Dockerfile -t sec-af-go:latest . +``` + +The tag is `sec-af-go`, not `sec-af`: the root README's `docker build -t sec-af .` +builds the **Python** image under `sec-af:latest`, and the two are different +artifacts (different entrypoint, different port). + +### Compose: opt-in add-on to the Python stack + +`docker-compose.go.yml` (at the repo root) is an **add-on**, not a standalone +stack. It defines only the Go node and joins the Python stack's compose network +as an external reference, sharing the control plane (`agentfield`) and the +`workspaces` volume the Python stack brings up. The Python `docker-compose.yml` +is left untouched. Start the Python stack first, then layer the Go node: + +```bash +docker compose up -d # Python stack (control plane + sec-af :8003) +docker compose -f docker-compose.go.yml up -d # adds sec-af-go :8013 +``` + +Adds: + +| Service | Port | Node id | Notes | +|-------------|--------|-------------|---------------------| +| `sec-af-go` | `8013` | `sec-af-go` | full audit pipeline | + +The control plane (`:8080`) and the `workspaces` volume come from the Python +stack — the Go add-on joins them via the external `sec-af_default` network and +`sec-af_workspaces` volume. This assumes the Python stack was brought up with +the default project name `sec-af` (the Python compose has no explicit `name:`, +so its project name is the checkout directory's basename); see the compose file +header for the `COMPOSE_PROJECT_NAME` override. Health: +`curl -f http://localhost:8013/health`. + +## Environment variables + +The node is configured entirely through the environment. + +| Variable | Purpose | +|-----------------------------|------------------------------------------------------------------| +| `OPENROUTER_API_KEY` | LLM provider key (OpenRouter) — required | +| `AGENTFIELD_URL` | Control-plane URL — checked **first** in this repo | +| `AGENTFIELD_SERVER` | Control-plane URL fallback (default `http://localhost:8080`) | +| `AGENTFIELD_API_KEY` | Control-plane API key (if the CP has auth enabled) | +| `AGENT_CALLBACK_URL` | Base URL the control plane uses to reach this node | +| `NODE_ID` | Node ID (default `sec-af`) | +| `PORT` | Listen port (default `8013`) | +| `HARNESS_PROVIDER` | Harness provider (default `aforge`; `opencode` to roll back). `SEC_AF_PROVIDER` wins when both are set | +| `AGENTFIELD_AFORGE_COMMAND` | AForge headless command — `exec` (default) or `do` | +| `HARNESS_MODEL` | Harness model (`SEC_AF_MODEL` wins when both are set) | +| `AI_MODEL` | Model for direct `.ai()` calls (`SEC_AF_AI_MODEL` wins) | +| `SEC_AF_AFORGE_BIN` | AForge executable override (falls back to `AFORGE_BIN`, then `aforge` on PATH) | +| `SEC_AF_OPENCODE_BIN` | OpenCode executable override (default `opencode`) | +| `SEC_AF_MAX_TURNS` | Harness turn cap (default `50`) | +| `SEC_AF_AI_MAX_RETRIES` | Retry count for `.ai()` gate calls (default `3`) | +| `SEC_AF_AI_INITIAL_BACKOFF_SECONDS` | First retry backoff in seconds (default `2.0`) | +| `SEC_AF_AI_MAX_BACKOFF_SECONDS` | Backoff ceiling in seconds (default `8.0`) | +| `SEC_AF_OPENCODE_SERVER` | OpenCode server URL (falls back to `OPENCODE_SERVER`; unset by default) | +| `SEC_AF_WORKSPACES_DIR` | Clone destination for remote repo URLs (default `/workspaces`; falls back to `~/.sec-af/workspaces` when not writable) | +| `SEC_AF_REPO_PATH` | Local checkout used when `repo_url` is neither a directory nor a URL | + +The image ships the released AForge CLI (fetched and checksum-verified at build +time from `https://agentfield.ai/downloads/aforge`, pinned by the +`AFORGE_VERSION` build arg) and runs `exec` by default. OpenCode remains +installed and can be selected with `HARNESS_PROVIDER=opencode` without +rebuilding. + +Note on model defaults — there are three, and they are all deliberate (each one +mirrors the Python node's corresponding artifact): + +| Where | Default | Why | +| --- | --- | --- | +| Code (`internal/config`, `src/sec_af/config.py`) | `minimax/minimax-m2.5` | Bare slug; the fallback when nothing is set. | +| Docker image (`go/Dockerfile`, root `Dockerfile`) | `openrouter/minimax/minimax-m2.5` | Same model, `openrouter/`-prefixed for the harness router. | +| Compose, `.env.example`, `agentfield-package.yaml` | `deepseek/deepseek-v4-flash-0731` | A different model — the one an installed or composed node runs. | + +`SEC_AF_MODEL` / `SEC_AF_AI_MODEL` win over `HARNESS_MODEL` / `AI_MODEL`, and +either env var wins over every default above. + +## Parity notes + +Behaviour is a 1:1 port of `src/sec_af/`, including its quirks. Four +differences are deliberate and are documented at their call sites: + +- **Callback URL.** Python computes `http://127.0.0.1:${PORT:-8004}` while its + `main()` listens on `${PORT:-8080}` — the two defaults disagree, so an unset + `PORT` makes the Python node advertise an address it is not listening on. The + Go node sets its public URL from `AGENT_CALLBACK_URL` when set and otherwise + lets the SDK derive `http://localhost:`. +- **`/health` body.** Python's route returns + `{"status":"healthy","version":"0.1.0"}`; the SDK's built-in route returns + `{"status":"ok"}` with the same 200. Every consumer here (the Dockerfile + healthcheck, the compose healthcheck, the manifest) only checks the status + code, so the SDK route is used as-is. The SDK also serves `/status`, which is + what the control plane's own health monitor polls. +- **AI credentials.** The Go SDK's `ai.Config` rejects an empty API key at + construction, so the AI client is attached only when `OPENROUTER_API_KEY` is + set. Python accepts the empty key and fails at call time instead; either way + an `.ai()` call without a key fails. +- **Control plane unreachable at boot.** This one is SDK-level, not something + the port chose, and it is the only one with an operational consequence. The + Python SDK's `ConnectionManager` treats registration as best-effort: it logs + `AgentField server unavailable - running in degraded mode`, keeps serving, and + retries every 10s, so the Python node survives a control plane that is not up + yet or that restarts under it. The Go SDK's `Agent.Serve` returns the + registration error (`client.RegisterNode` has no retry), and + `cmd/sec-af/main.go` treats that as fatal — the process binds its port, logs + `node.register.failed`, and exits. In practice the Go node therefore + restart-loops (compose sets `restart: unless-stopped`) until the control plane + answers, instead of degrading in place. Start the Python stack — which owns + the control plane — first, and expect the add-on container to retry until that + stack is healthy. + +## Deployment: `af install` + +Because the root package redirects git installs here, install the Go node with +the repository's bare URL: + +```bash +af install https://github.com/Agent-Field/sec-af +``` + +This resolves to `go/agentfield-package.yaml` (node id `sec-af`, default port +`8013`) and builds `./cmd/sec-af`. A prior Python `sec-af` installation is +replaced in place, retaining its node id, triggers, and node-scoped secrets. diff --git a/docs/design/security-audit/SEC-AF-PORT-DESIGN.md b/docs/design/security-audit/SEC-AF-PORT-DESIGN.md new file mode 100644 index 0000000000..92bcfd9eec --- /dev/null +++ b/docs/design/security-audit/SEC-AF-PORT-DESIGN.md @@ -0,0 +1,286 @@ +# SEC-AF Go port — design contract + +This document is the design contract the Go port under `go/` is written +against: parity rules, package layout, SDK mapping, DAG shape, packaging and the +testing contract. It mirrors the established Agent-Field porting pattern (pr-af +PRs #53/#54/#64, SWE-AF `go/`), adapted to the fact that this node builds its +reasoner DAG with `app.call(f"{NODE_ID}.")` through the control +plane. + +Reference material (read-only): + +- pr-af Go port — under `go/` + (afx.Bind/ToMap, harnessx.Run[T] with embedded pydantic schemas, node/ wiring, + register.go, Dockerfile/Makefile/compose, go/README.md, root README section) +- SWE-AF Go port — under `go/` + (app.Call DAG, envelope.UnwrapCallResult) +- AgentField Go SDK (v0.1.131 is the pinned tag) — + under `sdk/go` + (`agent/agent.go` Call/CallLocal/AI/Note, `agent/harness.go`, `agent/router.go`, + `harness/provider.go` Options, `harness/result.go`, `ai/request.go` WithSchema) +- AgentField Python SDK (to check what Python does) — the same repository under + `sdk/python/agentfield` + +The Python node this port reproduces lives in this repository: `src/sec_af` +(sources) and `tests/` (its test suite). + +## -1. Tooling facts + +- The generators (`go/scripts/gen_schemas.py`, `go/scripts/gen_golden.py`) need a + Python 3.11 interpreter with pydantic v2, `agentfield` and this repo's own + dependencies — the interpreter `af install` provisions for the Python node is + the convenient one. Run them as + `PYTHONPATH=/src go/scripts/gen_schemas.py`. +- Go toolchain: go1.25 or newer on PATH; the module targets the Go 1.21 language + level. `go mod tidy` resolves + `github.com/Agent-Field/agentfield/sdk/go v0.1.131` from the proxy. +- Never register a test node against a control plane you do not own: bring up an + isolated one (see §7). + +## 0. Non-negotiables + +1. **Python is byte-untouched.** Every diff lives under `go/`, plus + `docker-compose.go.yml`, the root `agentfield-package.yaml` redirect, one + root-README section, and `.github/workflows/go.yml`. Never edit `src/`, + `tests/`, `pyproject.toml`, `Dockerfile`, `docker-compose.yml`. +2. **1:1 parity is the goal.** Same reasoner names, same input parameter + names/defaults, same result JSON key sets (snake_case, pydantic + `model_dump()` shape), same prompts (byte-verbatim), same concurrency + shape (gather/semaphore), same notes (message + tags), same error mapping. + When Python does something odd, reproduce it and leave a comment + `// Python parity: ...`. Do not "improve" behavior. If a Python behavior is + non-deterministic (set iteration order) make it deterministic and comment it. +3. **Same DAG.** Every place Python does `router.call(f"{NODE_ID}.x", ...)` / + `app.call(...)`, Go does `app.Call(ctx, nodeID+".x", kwargsMap)` with the + SAME target name and the SAME kwargs keys. Never replace a Python `.call` + with a direct Go function call — that collapses the control-plane DAG. + Conversely never add a `.Call` where Python calls a function in-process. +4. **Gate: `cd go && go build ./... && go vet ./... && go test ./... && test -z "$(gofmt -l .)"`** + must be green for everything you touch. Tests are derived from the Python + tests and from the behaviors in this doc (validation contract), not from + the Go implementation. +5. Go 1.21 language level in `go.mod` (`go 1.21`), matching the SDK. Local + toolchain is go1.25 — fine, but do not use APIs newer than 1.21 (no + `slices`/`maps` std packages? — those ARE 1.21, ok; `min`/`max` builtins + are 1.21, ok; avoid `range over int` (1.22) and `iter` (1.23)). +6. No new third-party dependencies beyond: the SDK, `golang.org/x/sync`, + `github.com/invopop/jsonschema` and `github.com/santhosh-tekuri/jsonschema/v5` + (pulled by the SDK). Ask before adding anything else. + +## 1. Repository layout (`go/` at the repo root) + +``` +go/ +├── agentfield-package.yaml # name: sec-af, language: go +├── Dockerfile # multi-stage, aforge fetch + opencode, non-root user, static binary +├── docker-entrypoint.sh # writes opencode.json from HARNESS_MODEL at container start +├── Makefile # build/vet/test/check/fmt/run/docker-* +├── README.md # build/run/compose/install story (model on pr-af go/README.md) +├── .gitignore # bin/, go.work*, coverage +├── doc.go # package doc for the module root +├── go.mod / go.sum # module github.com/Agent-Field/sec-af/go ; go 1.21 ; sdk/go v0.1.131 +├── cmd/sec-af/main.go # the node binary +├── docs/DESIGN.md # this document +├── scripts/gen_schemas.py # pydantic model_json_schema() → internal/harnessx/testdata/schemas/*.json +├── scripts/gen_golden.py # (where useful) Python prompt-builder goldens → internal/.../testdata/*.txt +├── internal/ +│ ├── afx/ Bind[T], ToMap, Unwrap/AsMap (the _unwrap/_as_dict parity), DropNulls (model_dump(exclude_none=True)) +│ ├── pyfmt/ Round(x, ndigits) banker's rounding (Python round()), Repr(v) Python repr for list/dict/str/bool/None +│ │ (needed wherever a prompt f-string embeds a Python list/dict), FormatFloat (Python str(float)) +│ ├── appx/ App interface {Harness, AI, Note, Call} that *agent.Agent satisfies; fakes for tests +│ ├── config/ DepthProfile, BudgetConfig, AuditConfig, AIIntegrationConfig (env), ProviderEnv() +│ ├── schemas/ every pydantic model → Go struct (json tags = pydantic field names), enums → string types +│ ├── harnessx/ Run[T] + RegisterSchema + embedded pydantic schema fixtures; Extract (extract_harness_result parity) +│ ├── aix/ Structured[T]: Python `.ai(user=, schema=Model)` = app.AI(WithSystem?, WithSchema(strictified pydantic schema)) → parse T +│ ├── prompts/ embedded copies of the Python prompt .txt files + Load(relpath) + drift test vs the Python tree +│ ├── see §3 +│ ├── reasoners/ Name* constants + RegisterAll (router with the Python AgentRouter tags) + handler adapters +│ ├── phases/ the *_phase reasoners (Call-based DAG) +│ ├── orch/ orchestrator (generate_output, checkpoints, budget/cost bookkeeping, progress notes) +│ └── node/ BuildAgent (env → agent.Config), the top-level `audit` reasoner handler, Serve +└── test/functional/ (build tag `functional`) registration parity against a live control plane +``` + +Root additions: `docker-compose.go.yml` (Go node as an add-on to the Python +stack, distinct NODE_ID `sec-af-go` and port), root `agentfield-package.yaml` +gains the `superseded_by: https://github.com/Agent-Field/sec-af//go` block +(copy the comment block from pr-af's root manifest verbatim, adjusting names), +root README gains a "Go implementation" section, `.github/workflows/go.yml` +(build/vet/test/gofmt on push + PR, paths-filtered to `go/**`). + +Node identity / ports: + +| Python code default NODE_ID | Go default NODE_ID | Python port | Go default port | router tags | +|---|---|---|---|---| +| `sec-af` | `sec-af` | 8080 (manifest) / 8003 (compose) | **8013** | `["security","audit","red-team"]` | + +`NODE_ID` and `PORT` env override both (Python parity). `docker-compose.go.yml` +sets `NODE_ID=sec-af-go` so both stacks can share one control plane. + +## 2. SDK mapping (Python → Go) + +| Python (agentfield py SDK) | Go (sdk/go v0.1.131) | +|---|---| +| `Agent(node_id, version, description, agentfield_server, callback_url, api_key, harness_config=HarnessConfig(provider, model, max_turns, env, opencode_bin, aforge_bin, permission_mode="auto"), ai_config=AIConfig(model, api_key, api_base))` | `agent.New(agent.Config{NodeID, Version:"0.1.0", AgentFieldURL, Token, ListenAddress:":"+port, PublicURL: AGENT_CALLBACK_URL, CLIConfig:&agent.CLIConfig{AppDescription}, HarnessConfig:&agent.HarnessConfig{Provider, Model, MaxTurns, PermissionMode:"auto", Env: ProviderEnv(), BinPath: }, AIConfig: &ai.Config{Model: strip "openrouter/" prefix, APIKey, BaseURL:"https://openrouter.ai/api/v1"} ONLY when the key is non-empty})` — copy pr-af `node.BuildAgent` incl. the `aiModelForAPI` prefix-strip rationale. | +| `@app.reasoner()` (top-level) | `app.RegisterReasoner(name, handler, agent.WithInputSchema(raw))` — transcribe the Python signature into the input schema (see pr-af `reviewInputSchema`). | +| `router = AgentRouter(tags=[...])`, `@router.reasoner()`, `app.include_router(router)` | `r := agent.NewRouter(); r.RegisterReasoner(name, h)`; `app.IncludeRouter(r, agent.RouterOptions{Tags: tags})` (no Prefix). | +| `await router.call(f"{NODE_ID}.x", a=1, b=2)` | `app.Call(ctx, nodeID+".x", map[string]any{"a":1,"b":2})` — returns the reasoner's result map already unwrapped on success; error on failure. Keep `afx.Unwrap(raw, name)` that mirrors `_unwrap` (error dict → error; `"output"` / `"result"` keys → inner) and `afx.AsMap` (`_as_dict`) for parity; apply them to the returned map exactly where Python does. The ctx passed MUST be the handler's ctx (carries the execution context so the CP parents the child execution). | +| `await app.harness(prompt=p, schema=Model, cwd=c, project_dir=d)` | `harnessx.Run[Model](ctx, app, p, harness.Options{Cwd:c, ProjectDir:d})` — provider/model/max_turns/env/permission come from the agent default HarnessConfig (the Go SDK merges them). `chain_builder` calls harness with NO schema (`app.harness(prompt, cwd=repo_path)`) → `app.Harness(ctx, prompt, nil, nil, opts)` and read `Result.Result` text. | +| `extract_harness_result(result, Model, name)` | `harnessx.Extract[Model](res, name)`: IsError → print the same diagnostic line and return `fmt.Errorf("%s harness error: %s", name, res.ErrorMessage)`; Parsed → value; else TypeError-equivalent error `"%s did not return a valid %s"`. | +| `await app.ai(user=prompt, schema=Model)` / `router.ai(system=, user=, schema=)` | `aix.Structured[Model](ctx, app, system, user)` → `app.AI(ctx, user, ai.WithSystem(system) if system!="", ai.WithSchema(json.RawMessage(strictified schema)))` then `resp.JSON(&v)`. Strictify exactly like Python's `_strictify_openai_schema` (every object: `additionalProperties:false`, `required` = all property names, recursing into `$defs`/`properties`/`items`/`anyOf`). | +| `app.note(msg, tags=[...])` / `router.note(...)` | `app.Note(ctx, msg, tags...)` — same message string, same tag order. | +| `HTTPException(400, detail={"error": msg})` | `return nil, &agent.ExecuteError{StatusCode: 400, Message: msg}` | +| `HTTPException(500, detail={"error": "audit execution failed: ..."})` | `&agent.ExecuteError{StatusCode: 500, Message: "audit execution failed: "+err.Error()}` (after the same `app.Note("Audit pipeline failed: ...", "audit","error")`) | +| `asyncio.gather(*coros)` | `errgroup` / WaitGroup writing into a pre-indexed slice (order preserved). `return_exceptions=True` → per-index error slots. | +| `asyncio.Semaphore(n)` | `semaphore.NewWeighted(n)` from x/sync (or a buffered chan). | +| `asyncio.Queue` producer/consumer (hunt incremental dedup) | channel + consumer goroutine; preserve the note strings. | +| `model_dump()` | `json.Marshal(struct)` — all fields emitted, no `omitempty` (except where Python has `exclude_none=True`: use `afx.DropNulls` on the marshaled map). | +| `Model.model_validate(d)` / `Model(**d)` | `afx.Bind[Model](d)` (JSON round-trip; UnmarshalJSON seeds defaults). | +| `str(float)` in prompts | `pyfmt.FormatFloat` ; `round(x, n)` → `pyfmt.Round` (half-even, like pr-af's). | +| `datetime.now(UTC)` inside `model_dump()` (serialized by FastAPI `jsonable_encoder` → `datetime.isoformat()`) | VERIFIED: `2026-01-02T03:04:05.123456+00:00` (microseconds omitted when zero: `2026-01-02T03:04:05+00:00`). Implement `schemas.Timestamp` (time.Time wrapper) whose MarshalJSON emits exactly that; UnmarshalJSON accepts RFC3339 with or without fraction and `Z`. | + +Python round-trips every reasoner boundary through JSON (`model_dump()` → +control plane → `model_validate`). Go must tolerate the same inputs: numbers +arrive as float64 in `map[string]any`; `afx.Bind` handles that. + +## 2b. Shared test fake and Python-JSON parity helper + +- `internal/appx.Fake` (already written) is THE test double for every package: + scripted `HarnessFn`/`AIFn`/`CallFn` (helpers `appx.HarnessJSON`, `appx.AIJSON`), + recorded `Harnesses`/`AIs`/`Notes`/`Calls`, and `MaxConcurrentHarness()` / + `MaxConcurrentCalls()` for semaphore assertions. Do not write another fake. +- `pyfmt.Dumps(v any, indent int) string` reproduces Python `json.dumps(x, indent=n)` + applied to a pydantic `model_dump()` dict: walks Go values by reflection + (struct fields in declaration order honoring json tags, pointers, slices, + maps with SORTED keys — documented deviation, Python keeps insertion order — + float64 kinds rendered as Python float repr e.g. `1.0`, ints as ints, + `true/false/null`, strings escaped like Python's `ensure_ascii=True` (non-ASCII + → `\uXXXX`, and NO escaping of `<>&`), values implementing json.Marshaler + (e.g. `schemas.Timestamp`) rendered via their MarshalJSON. `pyfmt.Dumps(v, 0)` + /`DumpsCompact` = `json.dumps(x)` with `", "` and `": "` separators. Use it + wherever Python embeds `json.dumps(...)` output in a prompt, a checkpoint file, + or an output artifact that a test compares textually. + +## 2c. Foundation API facts (as actually landed — read the code, these are pointers) + +- `harnessx.Run[T](ctx, app, prompt, opts) (*T, *harness.Result, error)`, + `harnessx.Extract[T](res *harness.Result, dest *T, agentName string) (T, error)`, + `harnessx.RunExtract[T](ctx, app appx.Harnesser, prompt string, opts harness.Options, agentName string) (T, error)` + (the 3-arg Extract is deliberate: the Go SDK stores the dest pointer in Result.Parsed), + `harnessx.SchemaFor[T]() map[string]any` (fixture by Go type name, invopop fallback). +- `aix.Structured[T](ctx, app appx.AIer, system, user string) (T, error)`, `aix.Strictify`. +- `afx.Bind[T]`, `afx.ToMap`, `afx.Unwrap(raw any, name string) (any, error)`, `afx.AsMap(payload any, name string) (map[string]any, error)`, `afx.DropNulls(any) any`. sec-af has only the lenient `Unwrap` (its two Python copies — app.py's and reasoners/phases.py's — are identical). +- `pyfmt.Round(x, ndigits)`, `pyfmt.FormatFloat`, `pyfmt.Str`, `pyfmt.Repr` (maps sorted; use `pyfmt.Ordered`/`pyfmt.O(...)` for insertion-ordered dict repr), `pyfmt.KV`, `pyfmt.Dumps` (`pyfmt/pyjson.go`). +- `prompts.Load(rel) (string, error)`, `prompts.MustLoad(rel)`, `prompts.Names()`; files under `internal/prompts/files//prompts>`. +- `config`: sec-af `DepthProfile` + `NormalizeDepth`, `BudgetConfig`/`DefaultBudgetConfig`, `AuditConfig` + `FromInput(in any, repoPath) (AuditConfig, error)` (JSON-projects any AuditInput-shaped value; `FromInputFields` typed core), `AIIntegrationConfig` + `AIConfigFromEnv() (AIIntegrationConfig, error)` + `ProviderEnv() (map[string]string, error)` (node must fail boot on error, like Python). +- `schemas`: struct per pydantic class, `New()` constructors (these mint uuid4 ids; `UnmarshalJSON` seeds defaults but never mints ids), `Timestamp`, enums as string types with `Parse*`/`Valid`. The enums have NO strict `UnmarshalJSON`, so `afx.Bind` alone will accept an out-of-vocabulary value — `internal/phases/validate.go` is where pydantic's enum gate is reproduced, and every `.call` boundary must bind through it. + Name collisions: `schemas.DataFlowStep` = prove's (file/line/description/tainted); `schemas.ReconDataFlowStep` = recon's (file_path/line/component/operation); `Location/CvssV4Score/EpssScore/ReproductionStep` declared once (output.go); `HuntStrategy` has the LOGIC_BUGS value alias; `schemas.PolicyEvalResult` exists. +- Stale Python tests discovered (port the CODE behavior, note the stale assertion): `tests/test_scoring.py` has two reachability-default assertions the code contradicts (empty tags → externally_reachable 1.0). +- Live verification uses a mock `opencode` shim that resolves every prompt role and writes canned schema-valid JSON (see §7). Known nondeterminism: `run_verdict_agent` is a real `.ai()` call, so the remediation fan-out varies with the LLM. + +## 3. sec-af port map (Python module → Go package) + +| Python | Go package | Notes | +|---|---|---| +| `app.py` | `internal/node` (+ `cmd/sec-af`) | `audit` reasoner: build AuditInput (same defaults: scan_types `["sast","sca","secrets","config"]`, output_formats `["json"]`, exclude_paths `["tests/","vendor/","node_modules/",".git/"]`), `_resolve_repo` (local dir → abs; http(s)/git@ → clone into `SEC_AF_WORKSPACES_DIR` default `/workspaces`, PermissionError fallback `~/.sec-af/workspaces`, `git pull --ff-only` if exists, `git clone --depth 1`, env GIT_TERMINAL_PROMPT=0 GIT_ASKPASS=echo, timeouts 60/120s; else `SEC_AF_REPO_PATH` or cwd), orchestrator construction, `resume_from_checkpoint` branch, the 4 `app.call`s in order with the SAME kwargs, checkpoint writes, `agent_invocations = total_selected + len(strategies_run) + 3`, `_generate_output`, notes, error mapping. `/health` is served by the SDK already (`{"status":"healthy",...}`) — do not add a custom route unless the SDK's differs materially; document. | +| `reasoners/__init__.py` + `recon.py` + `hunt.py` + `prove.py` + `phases.py` | `internal/reasoners` (registration + thin adapters) and `internal/phases` (the `*_phase` + `run_cwe_expansion` bodies) | Register EXACTLY these 33 router reasoners (tags `security, audit, red-team`) in this order: `run_architecture_mapper, run_dependency_auditor, run_config_scanner, run_data_flow_mapper, run_security_context_profiler, run_injection_hunter, run_dos_hunter, run_ssrf_hunter, run_auth_hunter, run_xss_hunter, run_crypto_hunter, run_business_logic_hunter, run_logic_bugs_hunter, run_data_exposure_hunter, run_supply_chain_hunter, run_config_secrets_hunter, run_api_security_hunter, run_deduplicator, run_dep_reachability, run_verifier, run_tracer, run_sanitization_analyzer, run_exploit_hypothesizer, run_verdict_agent, run_remediation, run_remediation_agent, run_dast_verifier, run_cross_service_analyzer, run_cwe_expansion, recon_phase, hunt_phase, prove_phase, remediation_phase` plus the top-level `audit` (34 total). Each adapter: the "X starting" note with its exact tags, bind inputs, call the agent function, return `model_dump()` map. `_run_hunter`'s TypeError-cascade is Python duck-typing noise — Go calls each hunter with the single real signature `(ctx, app, repoPath, recon, depth, maxFilesWithoutSignal)`. `_recon_model` normalization (seed defaults then overlay) → Bind with default-seeding handles it; keep `security_context` default `{"auth_model":"unknown","auth_details":""}`. `_coerce_verifier_finding` (RawFinding or FindingForVerifier projection) must be ported. | +| `orchestrator.py` | `internal/orch` | Port the whole class: `AuditOrchestrator` fields, `run()` (streaming in-process path — keep for completeness, it is not reached by the API but `run_from_checkpoint` uses `_run_hunt/_run_prove` which are in-process), `run_from_checkpoint`, `_run_recon/_run_fast_recon/_run_deep_recon_async/_merge_recon`, `_run_hunt(_streaming)`, `_run_prove(_streaming)`, `_run_dast_verification`, `_generate_output` (severity threshold filter, CWE floor, exploitability score, `get_compliance_mappings_hybrid` with the AI gate, verdict/severity counts, noise reduction, AttackChain mapping, compliance gaps, SecurityAuditResult, sarif/json/report generation, compliance report files under checkpoint_dir), checkpoints (`checkpoint-.json` with `{"phase","created_at","data"}` and `indent=2`), `_prover_cap`, `_prioritize_findings`, `_assess_reachability_parallel` (semaphore min(5,n), fallback tag `requires_auth`), budget/cost helpers, `_PhaseHarnessProxy` (a wrapper that checks budget, counts invocations and registers cost — it exposes ONLY `harness`, exactly as the Python class does, so dedup's `hasattr(app,"ai")` gate is False and run_prove's verdict call raises AttributeError), `_emit_progress` (AuditProgress JSON note), `_track_drop`, `_verified_finding_fallback`, `merge_recon_findings_into_hunt`. | +| `harness.py` | `internal/harnessx` (+ `AIGateWrapper` in `internal/gates`) | `HarnessWrapper` is unused by the live path — port `AIGateWrapper` (invoke with retry on transient errors, `classify_severity`, `check_duplicate`, `select_strategy`, `assess_reachability`, the exact prompt strings incl. Python list repr of `default_candidates`) since the orchestrator uses it (`assess_reachability`, `get_compliance_mappings_hybrid`). `_is_transient_error` patterns + backoff (`min(initial*2**attempt, max)`) must match. Port `HarnessWrapper` too only if cheap (it is pure; `_with_phase_guidance`/`_schema_guidance` strings are testable) — lower priority. | +| `config.py` | `internal/config` | env names: `SEC_AF_PROVIDER`/`HARNESS_PROVIDER` (default `aforge`), `SEC_AF_MODEL`/`HARNESS_MODEL` (default `minimax/minimax-m2.5`), `SEC_AF_AI_MODEL`/`AI_MODEL`/`SEC_AF_MODEL` (default `minimax/minimax-m2.5`), `SEC_AF_MAX_TURNS` 50, `SEC_AF_AI_MAX_RETRIES` 3, `SEC_AF_AI_INITIAL_BACKOFF_SECONDS` 2.0, `SEC_AF_AI_MAX_BACKOFF_SECONDS` 8.0, `SEC_AF_OPENCODE_BIN` opencode, `SEC_AF_AFORGE_BIN`/`AFORGE_BIN` aforge, `SEC_AF_OPENCODE_SERVER`/`OPENCODE_SERVER`. `provider_env()`: the 6 keys + `AGENTFIELD_AFORGE_COMMAND` (default exec) + `XDG_DATA_HOME` (default `/opencode-shared-data`, mkdir). Malformed ints crash at boot (Python crashes at import) → return error from `FromEnv`. | +| `context.py` | `internal/context` (package name `recontext` to avoid clashing with std `context`) | `prune_recon_for_strategy`, `recon_context_generic`, the per-strategy projections, `get_framework_hints`/`get_language_hints` wiring (hints live in agents/hunt in Python — put them in `internal/agents/hunt/hints.go` and import). Outputs are embedded in prompts → byte-exact; golden test against Python via `scripts/gen_golden.py`. | +| `schemas/*.py` | `internal/schemas` | One Go file per Python module. Enums: `Severity, Confidence, FindingType, HuntStrategy, Verdict, EvidenceLevel, ...` as `type X string` + consts. Pydantic defaults ≠ Go zero → `UnmarshalJSON` default seeding (pr-af `schemas/defaults.go` pattern). Computed properties (`RawFinding.fingerprint` default, `id` default uuid/hash, `for_verifier()` projection) → methods; check how `id`/`fingerprint` defaults are generated in Python and reproduce (if `uuid4` → `uuid.NewString()` — accept nondeterminism; if hash → exact). `SecurityAuditResult.timestamp` → Timestamp type (§2). Field descriptions: keep as Go doc comments (they matter for the pydantic schema fixtures, which come from Python, so no need to reproduce in Go tags). | +| `schemas/gates.py` | `internal/schemas/gates.go` | `CWEExpansion, DuplicateCheck, ReachabilityGate, SeverityClassification, StrategySelection, ...` — used by aix. | +| `scoring.py` | `internal/scoring` | port + port `tests/test_scoring.py` fully. | +| `compliance/mapping.py` | `internal/compliance` | static tables byte-exact; `get_compliance_mappings`, `get_compliance_gaps`, `get_compliance_mappings_hybrid(cwe, frameworks, ai_gate)`; port `tests/test_compliance.py`. | +| `output/sarif.py, json_output.py, report.py, compliance_report.py` | `internal/output` | SARIF JSON must be byte-comparable modulo key order → build with ordered structs; port `tests/test_sarif.py`, `test_json_output.py`, `test_compliance_report.py`. Python `json.dumps(indent=2)` → Go `json.MarshalIndent(v,""," ")` (note Python puts a space after `:` and `,` — MarshalIndent matches; but Python escapes non-ASCII as `\uXXXX` by default (`ensure_ascii=True`) and Go escapes `<>&` as `<` — document; where a test compares strings, compare parsed JSON). | +| `diff_analysis.py`, `monitoring.py`, `policies.py` | `internal/diffanalysis`, `internal/monitoring`, `internal/policies` | pure modules with Python tests → port them + tests (monitoring/policies are not wired into the API; port for 1:1 completeness; `audit.py` stub can be skipped — note it). | +| `agents/_utils.py` | `internal/harnessx` (Extract) | see §2. | +| `agents/recon/*` | `internal/agents/recon` | 5 mappers (each: read prompt template, append the exact CONTEXT block, `tempfile.mkdtemp(prefix=f"secaf-{agent_name}-")` → `os.MkdirTemp("", "secaf-"+name+"-")`, harness with `cwd=tmp, project_dir=repo`, extract, parse `*Raw` → typed via `_parsers.py` port, `defer os.RemoveAll`), `_repo_metrics` (same SKIP_DIRS/CODE_EXTS, count lines of code files, count all files), `extract_recon_findings` (+ `tests/test_recon_findings.py`), `run_recon`, `run_fast_recon`, `run_deep_recon`. | +| `agents/hunt/*` | `internal/agents/hunt` | `_scan_enrich.py` (scan_locations prompt, enrich_finding prompt, `enrich_locations_parallel`, `assemble_finding`), 12 hunters (injection, dos, ssrf, auth, xss, crypto, business_logic, logic, data_exposure, supply_chain, config_secrets, api_security) — each hunter file is a thin wrapper with its strategy name/CWE baseline/prompt path: read them all; `_framework_hints.py`, `_language_hints.py` (static tables, byte-exact), `__init__.py` `run_hunt` + `run_hunt_streaming` + `_default_strategies` + dedup-by-fingerprint + `include_paths` handling (`tests/test_hunt_include_paths.py`, `test_hunt_crypto.py`, `test_strategy_selection.py`). | +| `agents/dedup.py` | `internal/agents/dedup` | `deduplicate_and_correlate` (fingerprint + semantic `.ai(DuplicateCheck)` pass + chain correlation harness call), `tests/test_dedup.py`. | +| `agents/prove/*` | `internal/agents/prove` | `verifier.py` (orchestrates tracer→sanitization→exploit→verdict in-process + `assembler`, `fallback(finding, reason, drop_reason, original_verdict)`), `tracer`, `sanitization`, `exploit`, `verdict` (uses `.ai(schema=VerdictDecision)` — aix), `chain_builder` (no-schema harness), `cross_service`, `dast_verifier`, `dep_reachability`, `sandbox` (subprocess w/ limits → `exec.CommandContext` + timeout), `__init__.py` `run_prove`/`run_prove_streaming`; `tests/test_prove_phase_demotion.py`. | +| `agents/remediation.py` | `internal/agents/remediation` | `generate_remediation` (VerifiedFinding → RemediationSuggestion) and `run_remediation` (RawFinding+verdict+rationale). | +| `prompts/**/*.txt` (in `src/sec_af/prompts`) | `internal/prompts/files/**` (go:embed) | byte-identical copies; `prompts.Load("hunt/injection.txt")`; drift test walks `../../../src/sec_af/prompts` when it exists and asserts byte equality for every file in both directions. | + +### The DAG the control plane must show + +``` +audit +├── recon_phase +│ ├── run_architecture_mapper ┐ +│ ├── run_dependency_auditor ├ gather (3) +│ ├── run_config_scanner ┘ +│ ├── run_data_flow_mapper ┐ gather (2) — skipped when depth == quick +│ └── run_security_context_profiler ┘ +├── hunt_phase +│ ├── run__hunter × N (semaphore max(1,min(max_concurrent_hunters=4, N)); N from _default_strategies) +│ └── run_deduplicator (only when ≥1 fingerprint-unique finding) +├── prove_phase +│ └── run_verifier × K (K = min(len(findings), prover cap: quick 10 / standard 30 / thorough 10000, max_provers); semaphore 3) +└── remediation_phase + └── run_remediation × M (M = confirmed/likely findings without remediation; semaphore 3) +``` + +`run_cwe_expansion` is registered but Python calls `expand_cwes_for_hunt` +in-process from `hunt_phase` (an `.ai()` call, not a `.call`) — reproduce +that: no DAG node for it. + +## 5. Testing contract + +- Port EVERY Python test file to a Go test in the owning package (same + assertions, same fixtures). Name tests after the Python ones so reviewers + can diff coverage (`TestScoring_...` ↔ `test_scoring.py::test_...`). +- Add golden tests for every prompt-building function whose output reaches + the LLM (`scripts/gen_golden.py` runs the Python builders with fixed inputs + and writes `testdata/golden/*.txt`; the Go test renders the same inputs and + compares byte-for-byte). Commit the generator AND the goldens. +- Schema fixtures: `scripts/gen_schemas.py` imports the pydantic models and + writes `model_json_schema()` JSON for every model that is passed to + `app.harness(schema=)` or `app.ai(schema=)`. Commit them under + `internal/harnessx/testdata/schemas/`. Add the pr-af drift test that checks + every embedded schema's `properties` keys ⊆ the Go struct's json tags and + vice-versa (required ones). +- Concurrency tests for each phase: a fake `appx.App` whose `Call` records + (target, kwargs) — assert the exact target names, kwargs keys, call counts, + order where Python orders, and the max observed concurrency ≤ the semaphore + limit. +- Node tests: registration parity (exact ordered name list + tags), `audit` / + `scan` input binding defaults, error mapping. +- Functional test (build tag) against a live CP is optional; the manual live + verification (§7) is mandatory and done by the integrator. + +## 6. Packaging + +Copy pr-af's `go/Dockerfile`, `go/docker-entrypoint.sh`, `go/Makefile`, +`docker-compose.go.yml`, `go/README.md`, root README section, root manifest +redirect, and adapt: binary name, user (`secaf`), ports, env var names +(`HARNESS_PROVIDER`, `HARNESS_MODEL`, `AI_MODEL`, `SEC_AF_*`, +`SEC_AF_WORKSPACES_DIR`), the aforge fetch +stage (take it from the repo's OWN Python Dockerfile — it is already the +checksum-verified download), the Python image's opencode config (the Go +entrypoint generates it from `HARNESS_MODEL`). The Go manifest's +`user_environment` block = the root manifest's block (same keys) — the Go node +reads the same env vars. `go.mod` requires +`github.com/Agent-Field/agentfield/sdk/go v0.1.131` (no replace). +CI: `.github/workflows/go.yml` — `actions/setup-go` with `go-version-file: +go/go.mod`, `working-directory: go`, steps `go build ./...`, `go vet ./...`, +`go test ./...`, `test -z "$(gofmt -l .)"`, `docker build -f go/Dockerfile .`. + +## 7. Live verification (integrator) + +1. Isolated control plane (the `af` binary, or a fresh build) on a free port + with `HOME` + `AGENTFIELD_HOME` pointed at a scratch dir — never a control + plane you do not own. +2. Python node (`pip install -e .` in a venv, or the installed package) and Go + node (`go run ./cmd/sec-af`) both registered (distinct NODE_IDs), same + `OPENROUTER_API_KEY`, same harness provider/model, same + `SEC_AF_WORKSPACES_DIR`. +3. Deterministic DAG comparison: a mock harness CLI (`HARNESS_PROVIDER=opencode` + + `SEC_AF_OPENCODE_BIN` pointing at a shim that recognizes the prompt's role + and writes canned schema-valid JSON to the output file; see pr-af + `go/test/mockcli`) → run `audit` on the same fixture repo through both nodes → pull `/api/v1/executions?...`/workflow + tree for each run and compare the node/edge multiset (parent→child + reasoner names, counts). Must be identical. +4. Real run (depth `quick`) of the Go node on a small public vulnerable repo + with the real key → succeeds, result JSON has the expected keys, DAG shape + matches the Python structure. diff --git a/go.mod b/go.mod index 9e8b2a6489..5a72a86af9 100644 --- a/go.mod +++ b/go.mod @@ -19,13 +19,16 @@ require ( github.com/gtank/ristretto255 v0.1.2 github.com/int128/listener v1.3.0 github.com/int128/oauth2cli v1.18.0 + github.com/invopop/jsonschema v0.14.0 github.com/modelcontextprotocol/go-sdk v1.7.0 github.com/rivo/uniseg v0.4.7 + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/yuin/goldmark v1.8.5 golang.org/x/crypto v0.54.0 golang.org/x/image v0.32.0 golang.org/x/net v0.57.0 golang.org/x/oauth2 v0.36.0 + golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 golang.org/x/text v0.40.0 gopkg.in/yaml.v3 v3.0.1 @@ -52,6 +55,8 @@ require ( github.com/aws/smithy-go v1.27.6 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymanbagabas/go-udiff v0.4.1 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/colorprofile v0.4.3 // indirect github.com/charmbracelet/x/cellbuf v0.0.15 // indirect @@ -84,6 +89,7 @@ require ( github.com/muesli/termenv v0.16.0 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/neilotoole/slogt v1.1.0 // indirect + github.com/pb33f/ordered-map/v2 v2.3.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/segmentio/asm v1.1.3 // indirect github.com/segmentio/encoding v0.5.4 // indirect @@ -97,7 +103,7 @@ require ( go.opentelemetry.io/otel/metric v1.45.0 // indirect go.opentelemetry.io/otel/trace v1.45.0 // indirect go.uber.org/atomic v1.11.0 // indirect - golang.org/x/sync v0.22.0 // indirect + go.yaml.in/yaml/v4 v4.0.0-rc.2 // indirect golang.org/x/time v0.15.0 // indirect modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect modernc.org/libc v1.41.0 // indirect diff --git a/go.sum b/go.sum index 43bae125c2..4066505585 100644 --- a/go.sum +++ b/go.sum @@ -52,6 +52,10 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymanbagabas/go-udiff v0.4.1 h1:OEIrQ8maEeDBXQDoGCbbTTXYJMYRCRO1fnodZ12Gv5o= github.com/aymanbagabas/go-udiff v0.4.1/go.mod h1:0L9PGwj20lrtmEMeyw4WKJ/TMyDtvAoK9bf2u/mNo3w= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= +github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw= @@ -131,6 +135,8 @@ github.com/int128/listener v1.3.0 h1:ZFePbpzFUt1i6hBSY15rzqo8tHZHJPPQkqCtgOAwS8g github.com/int128/listener v1.3.0/go.mod h1:zF9mx2wn+2J/7Idmxi5kgqrGgERr6vr8fK8KqENrRZ0= github.com/int128/oauth2cli v1.18.0 h1:ECW600WoYKh5Z4gv92yNVsJ22iqzPW7u7z0dfWxMzkU= github.com/int128/oauth2cli v1.18.0/go.mod h1:8wWAPwFYS91aX3KKACFP//v2oLdqnugK/ndfJ6oH3xE= +github.com/invopop/jsonschema v0.14.0 h1:MHQqLhvpNUZfw+hM3AZDYK7jxO8FZoQeQM77g8iyZjg= +github.com/invopop/jsonschema v0.14.0/go.mod h1:ygm6C2EaVNMBDPpaPlnOA2pFAxBnxGjFlMZABxm9n2I= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -164,6 +170,8 @@ github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdh github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/neilotoole/slogt v1.1.0 h1:c7qE92sq+V0yvCuaxph+RQ2jOKL61c4hqS1Bv9W7FZE= github.com/neilotoole/slogt v1.1.0/go.mod h1:RCrGXkPc/hYybNulqQrMHRtvlQ7F6NktNVLuLwk6V+w= +github.com/pb33f/ordered-map/v2 v2.3.1 h1:5319HDO0aw4DA4gzi+zv4FXU9UlSs3xGZ40wcP1nBjY= +github.com/pb33f/ordered-map/v2 v2.3.1/go.mod h1:qxFQgd0PkVUtOMCkTapqotNgzRhMPL7VvaHKbd1HnmQ= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= @@ -172,6 +180,8 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= @@ -201,6 +211,8 @@ go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOy go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.yaml.in/yaml/v4 v4.0.0-rc.2 h1:/FrI8D64VSr4HtGIlUtlFMGsm7H7pWTbj6vOLVZcA6s= +go.yaml.in/yaml/v4 v4.0.0-rc.2/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= diff --git a/internal/agentsession/agentsession_test.go b/internal/agentsession/agentsession_test.go new file mode 100644 index 0000000000..641f5da56a --- /dev/null +++ b/internal/agentsession/agentsession_test.go @@ -0,0 +1,581 @@ +package agentsession + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + "unicode/utf8" + + "github.com/Agent-Field/agentfield/sdk/go/ai" + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/delegate" + "github.com/Agent-Field/codeaf/internal/provider/modelapi" +) + +// fakeAPI is the run's model API played by a script: each call is answered by +// the next reply, and every body it was sent is kept. +type fakeAPI struct { + mu sync.Mutex + replies []func(body wireRequest) (int, any) + bodies []wireRequest + auth []string +} + +func (f *fakeAPI) serve(t *testing.T) *Client { return f.serveWithHeader(t, "", "") } + +// serveWithHeader is serve with one header on every answer that is not a 200. +func (f *fakeAPI) serveWithHeader(t *testing.T, header, value string) *Client { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + var body wireRequest + if err := json.Unmarshal(raw, &body); err != nil { + t.Errorf("the client sent a body that does not parse: %v", err) + } + f.mu.Lock() + f.bodies = append(f.bodies, body) + f.auth = append(f.auth, r.Header.Get("Authorization")) + if len(f.replies) == 0 { + f.mu.Unlock() + t.Errorf("an unscripted call reached the API: %s", raw) + w.WriteHeader(http.StatusInternalServerError) + return + } + reply := f.replies[0] + f.replies = f.replies[1:] + f.mu.Unlock() + status, payload := reply(body) + if header != "" && status != 200 { + w.Header().Set(header, value) + } + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(payload) + })) + t.Cleanup(server.Close) + client, err := NewClient(delegate.ModelAPI{BaseURL: server.URL + "/v1", Token: "tok"}, 2) + if err != nil { + t.Fatal(err) + } + client.pause = func(int) time.Duration { return time.Millisecond } + client.held = func() time.Duration { return time.Millisecond } + return client +} + +func (f *fakeAPI) then(reply func(body wireRequest) (int, any)) *fakeAPI { + f.replies = append(f.replies, reply) + return f +} + +func answer(text string, cost float64) func(wireRequest) (int, any) { + return func(wireRequest) (int, any) { + return 200, ai.Response{Choices: []ai.Choice{{Message: textMessage("assistant", text)}}, Usage: &ai.Usage{Cost: &cost}} + } +} + +func callTool(name, args string) func(wireRequest) (int, any) { + return func(wireRequest) (int, any) { + cost := 0.01 + return 200, ai.Response{Choices: []ai.Choice{{Message: ai.Message{Role: "assistant", + ToolCalls: []ai.ToolCall{{ID: "c1", Type: "function", Function: ai.ToolCallFunction{Name: name, Arguments: args}}}}}}, + Usage: &ai.Usage{Cost: &cost}} + } +} + +func repo(t *testing.T) string { + t.Helper() + root := t.TempDir() + write := func(rel, text string) { + path := filepath.Join(root, rel) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + write("app/views.py", "import os\n\ndef run(cmd):\n return os.popen(cmd).read()\n") + write("app/models.py", "class User:\n password = ''\n") + write("node_modules/lib/index.js", "os.popen('x')\n") + write("README.md", "# demo\n") + real, err := filepath.EvalSymlinks(root) + if err != nil { + t.Fatal(err) + } + return real +} + +var findingSchema = map[string]any{ + "type": "object", + "properties": map[string]any{"file": map[string]any{"type": "string"}, "line": map[string]any{"type": "integer"}}, + "required": []any{"file", "line"}, + "additionalProperties": false, +} + +// A SESSION READS, THEN ANSWERS IN ITS SCHEMA. The model's tool call is run +// against the repository, its result goes back on the same thread, and the +// answer that meets the schema is the session's answer, priced by every turn. +func TestASessionReadsTheRepositoryAndAnswersInItsSchema(t *testing.T) { + root := repo(t) + api := (&fakeAPI{}). + then(callTool("grep", `{"pattern":"os\\.popen"}`)). + then(func(body wireRequest) (int, any) { + tool := body.Messages[len(body.Messages)-1] + if tool.Role != "tool" || !strings.Contains(tool.Content[0].Text, "app/views.py:4:") { + t.Errorf("the search result handed back = %+v", tool) + } + if strings.Contains(tool.Content[0].Text, "node_modules") { + t.Errorf("a dependency folder was searched: %s", tool.Content[0].Text) + } + return answer(`{"file":"app/views.py","line":4}`, 0.02)(body) + }) + client := api.serve(t) + result, err := RunSession(t.Context(), client, SessionOrder{Policy: testPolicy, MaxTurns: 50, Model: "vendor/model", Thread: "hunt-1", Root: root, + Prompt: "find the command injection", Schema: findingSchema}) + if err != nil { + t.Fatal(err) + } + if result.Failed != "" || string(result.JSON) != `{"file":"app/views.py","line":4}` { + t.Fatalf("the session answered %+v", result) + } + if result.Turns != 2 || result.Tools != 1 || result.CostUSD < 0.0299 || result.CostUSD > 0.0301 { + t.Fatalf("turns %d, tools %d, cost %.4f", result.Turns, result.Tools, result.CostUSD) + } + for _, body := range api.bodies { + if body.PromptCacheKey != "hunt-1" || body.Model != "vendor/model" { + t.Fatalf("a call went out on thread %q for %q", body.PromptCacheKey, body.Model) + } + } + if api.auth[0] != "Bearer tok" { + t.Fatalf("the call carried %q", api.auth[0]) + } + if !strings.Contains(api.bodies[0].Messages[0].Content[0].Text, `"required":["file","line"]`) { + t.Fatalf("the system message does not carry the schema: %s", api.bodies[0].Messages[0].Content[0].Text) + } +} + +// AN ANSWER THAT MISSES ITS SCHEMA IS ASKED FOR AGAIN, twice at most and with +// no tools, and the problem is named so the model can fix it. +func TestAnAnswerThatMissesItsSchemaIsAskedForAgainTwice(t *testing.T) { + root := repo(t) + api := (&fakeAPI{}). + then(answer("I found it in views.py", 0)). + then(func(body wireRequest) (int, any) { + last := body.Messages[len(body.Messages)-1].Content[0].Text + if !strings.Contains(last, "holds no JSON object") || len(body.Tools) != 0 { + t.Errorf("the second ask = %q with %d tools", last, len(body.Tools)) + } + return answer("```json\n{\"file\":\"app/views.py\"}\n```", 0)(body) + }). + then(func(body wireRequest) (int, any) { + if last := body.Messages[len(body.Messages)-1].Content[0].Text; !strings.Contains(last, "line") { + t.Errorf("the third ask does not name the missing field: %q", last) + } + return answer(`Here: {"file":"app/views.py","line":4} done`, 0)(body) + }) + result, err := RunSession(t.Context(), api.serve(t), SessionOrder{Policy: testPolicy, MaxTurns: 50, Root: root, Prompt: "find it", Schema: findingSchema}) + if err != nil || result.Failed != "" || string(result.JSON) != `{"file":"app/views.py","line":4}` { + t.Fatalf("result %+v, err %v", result, err) + } + stubborn := (&fakeAPI{}).then(answer("no", 0)).then(answer("no", 0)).then(answer("no", 0)) + result, err = RunSession(t.Context(), stubborn.serve(t), SessionOrder{Policy: testPolicy, MaxTurns: 50, Root: root, Prompt: "find it", Schema: findingSchema}) + if err != nil || result.Failed == "" || result.Turns != 3 { + t.Fatalf("a model that never met the schema gave %+v, %v", result, err) + } +} + +// THE CEILING ENDS EVERY SESSION AT ONCE. The first 402 is the answer to every +// call after it, without asking a server that already gave it. +func TestTheCeilingIsHeardOnceAndAnsweredToEveryLaterCall(t *testing.T) { + api := (&fakeAPI{}).then(func(wireRequest) (int, any) { + return 402, ai.ErrorResponse{Error: ai.ErrorDetail{Message: "the run's dollar ceiling of $1.00 is reached"}} + }) + client := api.serve(t) + _, err := client.Complete(t.Context(), Request{Messages: []ai.Message{textMessage("user", "hi")}}) + if !errors.Is(err, ErrCeiling) { + t.Fatalf("a 402 answered %v", err) + } + if _, again := client.Complete(t.Context(), Request{Messages: []ai.Message{textMessage("user", "hi")}}); !errors.Is(again, ErrCeiling) { + t.Fatalf("the call after the ceiling answered %v", again) + } + if len(api.bodies) != 1 { + t.Fatalf("the API was asked %d times", len(api.bodies)) + } +} + +// A passing failure is sent again; a bad request is not. +func TestOnlyAPassingFailureIsSentAgain(t *testing.T) { + api := (&fakeAPI{}). + then(func(wireRequest) (int, any) { return 503, map[string]string{"error": "busy"} }). + then(answer("ok", 0)) + if response, err := api.serve(t).Complete(t.Context(), Request{Messages: []ai.Message{textMessage("user", "hi")}}); err != nil || response.Text() != "ok" { + t.Fatalf("a 503 then an answer gave %v, %v", response, err) + } + bad := (&fakeAPI{}).then(func(wireRequest) (int, any) { + return 400, ai.ErrorResponse{Error: ai.ErrorDetail{Message: "no messages"}} + }) + if _, err := bad.serve(t).Complete(t.Context(), Request{Messages: []ai.Message{textMessage("user", "hi")}}); err == nil || !strings.Contains(err.Error(), "no messages") { + t.Fatalf("a 400 gave %v", err) + } + if len(bad.bodies) != 1 { + t.Fatalf("a 400 was sent %d times", len(bad.bodies)) + } +} + +// NO TOOL LEADS OUT OF THE REPOSITORY, by name or by link. +func TestNoToolReadsOutsideTheRepository(t *testing.T) { + root := repo(t) + outside := t.TempDir() + if err := os.WriteFile(filepath.Join(outside, "secret"), []byte("key"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(root, "away")); err != nil { + t.Fatal(err) + } + tools := toolbox{root: root, limits: testPolicy.Tools} + for _, call := range []struct{ name, args string }{ + {"read_file", `{"path":"../secret"}`}, + {"read_file", `{"path":"` + filepath.Join(outside, "secret") + `"}`}, + {"read_file", `{"path":"away/secret"}`}, + {"list_dir", `{"path":"away"}`}, + {"grep", `{"pattern":"key","path":"away"}`}, + } { + got := tools.run(ai.ToolCall{Function: ai.ToolCallFunction{Name: call.name, Arguments: call.args}}) + if !strings.HasPrefix(got, "error:") || strings.Contains(got, "key\n") { + t.Errorf("%s %s answered %q", call.name, call.args, got) + } + } + if got := tools.run(ai.ToolCall{Function: ai.ToolCallFunction{Name: "read_file", Arguments: `{"path":"app/views.py","offset":4}`}}); !strings.Contains(got, " 4\t return os.popen(cmd).read()") { + t.Errorf("a read inside answered %q", got) + } + if got := tools.run(ai.ToolCall{Function: ai.ToolCallFunction{Name: "glob", Arguments: `{"pattern":"**/*.py"}`}}); got != "app/models.py\napp/views.py\n" { + t.Errorf("the glob answered %q", got) + } + if got := tools.run(ai.ToolCall{Function: ai.ToolCallFunction{Name: "grep", Arguments: `{"pattern":"PASSWORD","ignore_case":true,"glob":"*.py"}`}}); got != "app/models.py:2: password = ''\n" { + t.Errorf("the search answered %q", got) + } + if got := tools.run(ai.ToolCall{Function: ai.ToolCallFunction{Name: "bash", Arguments: `{}`}}); !strings.Contains(got, "there is no tool") { + t.Errorf("an unknown tool answered %q", got) + } +} + +// A session that is still reading on its last turn is told to answer and is +// offered no tools. +func TestTheLastTurnIsAnAnswer(t *testing.T) { + root := repo(t) + api := (&fakeAPI{}). + then(callTool("list_dir", `{}`)). + then(func(body wireRequest) (int, any) { + if len(body.Tools) != 0 || !strings.Contains(body.Messages[len(body.Messages)-1].Content[0].Text, "Stop reading now") { + t.Errorf("the last turn offered %d tools", len(body.Tools)) + } + return answer("plain words", 0)(body) + }) + result, err := RunSession(context.Background(), api.serve(t), SessionOrder{Policy: testPolicy, Root: root, Prompt: "look", MaxTurns: 2}) + if err != nil || result.Text != "plain words" || result.Failed != "" { + t.Fatalf("result %+v, err %v", result, err) + } +} + +// THE OLD HARNESS'S CONTRACT HOLDS: an answer that met its schema is decoded +// into the caller's value, a session that ran and gave none is a result with +// IsError and no Go error, and the session is named by the agent that ran it. +func TestHarnessKeepsTheOldHarnesssContract(t *testing.T) { + root := repo(t) + var heard []string + api := (&fakeAPI{}).then(answer(`{"file":"app/views.py","line":4}`, 0.5)). + then(answer("no", 0)).then(answer("no", 0)).then(answer("no", 0)) + app := mustNew(t, api.serve(t), Config{Root: root, SessionModel: "vendor/work", + Watch: &Watch{Session: func(label string, _ SessionResult, _ error) { heard = append(heard, label) }}}) + var dest struct { + File string `json:"file"` + Line int `json:"line"` + } + scratch := filepath.Join(t.TempDir(), "secaf-hunt-scan-123456789") + result, err := app.Harness(t.Context(), "find it", findingSchema, &dest, appxOptions(scratch, root)) + if err != nil || result.IsError || result.Parsed == nil || dest.File != "app/views.py" || dest.Line != 4 || *result.CostUSD != 0.5 { + t.Fatalf("result %+v, dest %+v, err %v", result, dest, err) + } + if api.bodies[0].PromptCacheKey != "hunt-scan-1" || api.bodies[0].Model != "vendor/work" { + t.Fatalf("the session ran on thread %q, model %q", api.bodies[0].PromptCacheKey, api.bodies[0].Model) + } + failed, err := app.Harness(t.Context(), "find it", findingSchema, &dest, appxOptions("/tmp/secaf-prove-tracer-42", root)) + if err != nil || !failed.IsError || failed.Parsed != nil || failed.ErrorMessage == "" { + t.Fatalf("a session with no answer gave %+v, %v", failed, err) + } + labelled := appxOptions(scratch, root) + labelled.Label = "DataFlowTracer" + api.then(answer(`{"file":"a","line":1}`, 0)) + if _, err := app.Harness(t.Context(), "trace", findingSchema, &dest, labelled); err != nil { + t.Fatal(err) + } + if strings.Join(heard, ",") != "hunt scan,prove tracer,data flow tracer" { + t.Fatalf("the sessions were heard as %v", heard) + } + if _, err := app.Harness(t.Context(), "x", nil, nil, appxOptions("", t.TempDir())); err == nil { + t.Fatal("a session over another folder was run") + } + if cost, sessions, _ := app.Spent(); sessions != 3 || cost != 0.5 { + t.Fatalf("spent %.2f over %d sessions", cost, sessions) + } +} + +// A served model that will not take a schema as a response format is asked +// once more with the schema in words. +func TestAStructuredCallFallsBackToTheSchemaInWords(t *testing.T) { + api := (&fakeAPI{}). + then(func(body wireRequest) (int, any) { + if body.ResponseFormat == nil || body.Model != "vendor/light" { + t.Errorf("the first ask had format %v on %q", body.ResponseFormat, body.Model) + } + return 400, ai.ErrorResponse{Error: ai.ErrorDetail{Message: "response_format is not supported"}} + }). + then(func(body wireRequest) (int, any) { + last := body.Messages[len(body.Messages)-1].Content[0].Text + if body.ResponseFormat != nil || !strings.Contains(last, `"verdict"`) { + t.Errorf("the second ask had format %v and said %q", body.ResponseFormat, last) + } + return answer(`{"verdict":"likely"}`, 0.1)(body) + }) + app := mustNew(t, api.serve(t), Config{Root: t.TempDir(), AIModel: "vendor/light"}) + response, err := app.AI(t.Context(), "decide", ai.WithSystem("you decide"), + ai.WithSchema(json.RawMessage(`{"type":"object","properties":{"verdict":{"type":"string"}}}`))) + if err != nil || response.Text() != `{"verdict":"likely"}` { + t.Fatalf("got %v, %v", response, err) + } + if api.bodies[0].Messages[0].Role != "system" { + t.Fatalf("the system words were not first: %+v", api.bodies[0].Messages) + } + if _, err := app.Call(t.Context(), "sec-af.run_verifier", nil); err == nil { + t.Fatal("the App answered a reasoner call itself") + } +} + +func appxOptions(cwd, project string) appx.HarnessOptions { + return appx.HarnessOptions{Cwd: cwd, ProjectDir: project} +} + +// A CALL THE CEILING HOLDS BEHIND CALLS IN FLIGHT WAITS ITS TURN, however many +// times it is held, and is not a failure; the ceiling itself is. +func TestAHeldCallWaitsItsTurn(t *testing.T) { + held := func(wireRequest) (int, any) { + return 429, ai.ErrorResponse{Error: ai.ErrorDetail{Message: "the run's dollar ceiling of $5.00 is held by calls in flight"}} + } + api := &fakeAPI{} + for range 6 { + api.then(held) + } + api.then(answer("ok", 0)) + client := api.serveWithHeader(t, modelapi.HeldHeader, "ceiling") + response, err := client.Complete(t.Context(), Request{Messages: []ai.Message{textMessage("user", "hi")}}) + if err != nil || response.Text() != "ok" || len(api.bodies) != 7 { + t.Fatalf("a held call answered %v, %v after %d asks", response, err, len(api.bodies)) + } +} + +// Every tool's parameters are a JSON Schema a strict server takes: an object +// whose `required` is a list, empty included. +func TestEveryToolSchemaIsStrictlyValid(t *testing.T) { + for _, tool := range (toolbox{limits: testPolicy.Tools}).definitions() { + encoded, _ := json.Marshal(tool.Function.Parameters) + var shape struct { + Type string `json:"type"` + Required json.RawMessage `json:"required"` + } + _ = json.Unmarshal(encoded, &shape) + if shape.Type != "object" || !strings.HasPrefix(string(shape.Required), "[") { + t.Errorf("%s's parameters are %s", tool.Function.Name, encoded) + } + if _, err := compileSchema(tool.Function.Parameters); err != nil { + t.Errorf("%s's parameters do not compile: %v", tool.Function.Name, err) + } + } +} + +// A label already in words keeps them; an identifier is split into words; an +// ordinary opening capital is lowered and an acronym kept. +func TestSessionLabelsReadAsAPersonWouldSayThem(t *testing.T) { + for _, tc := range []struct{ label, cwd, want string }{ + {"DataFlowTracer", "", "data flow tracer"}, + {"Data flow mapper", "", "data flow mapper"}, + {"DoS hunter · scan", "", "DoS hunter · scan"}, + {"auth hunter · src/UserView.py:12", "", "auth hunter · src/UserView.py:12"}, + {"", "/tmp/secaf-remediation-123", "remediation"}, + {"", "/repo", "agent"}, + } { + if got := sessionLabel(tc.label, tc.cwd); got != tc.want { + t.Errorf("%q (%q) reads %q, want %q", tc.label, tc.cwd, got, tc.want) + } + } +} + +// THE SHARED LOOP SAYS THE PROGRAM'S OWN WORK. It was sec's alone and told +// every agent it was one of a security audit; a review's reviewers read the +// code for the wrong thing when told that. +func TestASessionIsToldTheWorkItsProgramNames(t *testing.T) { + audit := sessionSystem(SessionOrder{Root: "/repo", Work: "a security audit"}) + if !strings.HasPrefix(audit, "You are one agent of a security audit of the repository at /repo. ") { + t.Fatalf("sec's sessions are told %q", firstLineOf(audit)) + } + review := sessionSystem(SessionOrder{Root: "/repo", Work: "a code review"}) + if !strings.HasPrefix(review, "You are one agent of a code review of the repository at /repo. ") || strings.Contains(review, "security") { + t.Fatalf("a review's sessions are told %q", firstLineOf(review)) + } + if plain := sessionSystem(SessionOrder{Root: "/repo"}); !strings.HasPrefix(plain, "You are one agent working on the repository at /repo. ") { + t.Fatalf("a session with no work named is told %q", firstLineOf(plain)) + } +} + +func firstLineOf(text string) string { + line, _, _ := strings.Cut(text, "\n") + return line +} + +// ONE AGENT'S FAILED SESSION IS NOT THE RUN'S. A model that keeps answering +// with no choices fails that session as a result the program can degrade +// around; the ceiling still ends the run as an error. +func TestAnEmptyReplyFailsOnlyItsSessionAndTheCeilingEndsTheRun(t *testing.T) { + root := repo(t) + empty := func(wireRequest) (int, any) { return 200, ai.Response{} } + api := (&fakeAPI{}).then(empty).then(empty).then(empty) + app := mustNew(t, api.serve(t), Config{Root: root}) + var dest struct { + File string `json:"file"` + } + failed, err := app.Harness(t.Context(), "find it", findingSchema, &dest, appxOptions("/tmp/secaf-hunt-scan-1", root)) + if err != nil || failed == nil || !failed.IsError || !strings.Contains(failed.ErrorMessage, "no choices") { + t.Fatalf("an empty reply gave %+v, %v; want a failed session and no error", failed, err) + } + refused := (&fakeAPI{}).then(func(wireRequest) (int, any) { + return http.StatusPaymentRequired, map[string]any{"error": map[string]any{"message": "the ceiling is reached"}} + }) + capped := mustNew(t, refused.serve(t), Config{Root: root}) + if _, err := capped.Harness(t.Context(), "find it", findingSchema, &dest, appxOptions("/tmp/secaf-hunt-scan-2", root)); !errors.Is(err, ErrCeiling) { + t.Fatalf("the ceiling gave %v; want it to end the run", err) + } +} + +// EACH AGENT IS BOUNDED BY ITS OWN FIGURES, and a figure the program leaves +// at zero is the App's default. +func TestEachAgentRunsUnderItsOwnLimits(t *testing.T) { + app := mustNew(t, nil, Config{MaxTurns: 50, SessionWall: 30 * time.Minute, Limits: func(opts appx.HarnessOptions) Limits { + if opts.Label == "scanner" { + return Limits{Turns: 75} + } + return Limits{} + }}) + if got := app.limitsFor(appx.HarnessOptions{Label: "scanner"}); got.Turns != 75 || got.Wall != 30*time.Minute { + t.Fatalf("the scanner runs under %+v", got) + } + if got := app.limitsFor(appx.HarnessOptions{Label: "other"}); got.Turns != 50 || got.Wall != 30*time.Minute { + t.Fatalf("an agent with no limits of its own runs under %+v", got) + } +} + +// A LONG LINE SAYS HOW TO REACH THE REST, AND GREP REACHES IT. A context file +// written as one line of JSON was cut at 2,000 bytes with no way to its rest, +// and the agents reading it ran to their turn cap. +func TestALongLineIsCutOnACharacterAndGrepShowsAMatchFarIntoIt(t *testing.T) { + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + long := strings.Repeat("a", 1999) + "é" + strings.Repeat("b", 28000) + `"needle":"found it"` + strings.Repeat("c", 500) + if err := os.WriteFile(filepath.Join(root, "context.json"), []byte(long+"\n"), 0o600); err != nil { + t.Fatal(err) + } + tools := toolbox{root: root, limits: testPolicy.Tools} + read := tools.run(ai.ToolCall{Function: ai.ToolCallFunction{Name: "read_file", Arguments: `{"path":"context.json"}`}}) + if !utf8.ValidString(read) || !strings.Contains(read, "é …[line cut: the first 2000 of its 30519 characters; grep for what you need in it") { + t.Fatalf("the long line read as %q", read[max(0, len(read)-200):]) + } + found := tools.run(ai.ToolCall{Function: ai.ToolCallFunction{Name: "grep", Arguments: `{"pattern":"needle"}`}}) + if !strings.HasPrefix(found, "context.json:1: … ") || !strings.Contains(found, `"needle":"found it"`) || len([]rune(found)) > 300 { + t.Fatalf("grep showed %q", found) + } +} + +// testPolicy is the tests' own figures: a program states every one, and so do +// the tests that stand in for one. +var testPolicy = Policy{ + FollowUps: 2, ContextChars: 400_000, + AnswerNow: "Stop reading now and give your answer from what you have found, in the form the system message asks for.", + Tools: ToolLimits{ReadLines: 400, ReadBytes: 48 << 10, ReadLineRunes: 2000, ListEntries: 400, GlobMatches: 300, + GrepMatches: 200, GrepFileBytes: 2 << 20, GrepLineRunes: 240}, +} + +// mustNew opens an App for a test, stating every figure the test left unset +// with the tests' own. +func mustNew(t *testing.T, client *Client, config Config) *App { + t.Helper() + if config.Sessions == 0 { + config.Sessions = 8 + } + if config.Calls == 0 { + config.Calls = 8 + } + if config.MaxTurns == 0 { + config.MaxTurns = 50 + } + if config.SessionWall == 0 { + config.SessionWall = 30 * time.Minute + } + if config.Policy == (Policy{}) { + config.Policy = testPolicy + } + app, err := New(client, config) + if err != nil { + t.Fatal(err) + } + return app +} + +// NOTHING HERE HAS A DEFAULT. A program that leaves a figure unset is refused +// where the App is opened and where a session is run, naming the figure, so +// no program runs on a number another program chose. +func TestEveryFigureIsTheProgramsAndAnUnsetOneIsRefused(t *testing.T) { + whole := Config{Sessions: 8, Calls: 8, MaxTurns: 50, SessionWall: time.Minute, Policy: testPolicy} + if _, err := New(nil, whole); err != nil { + t.Fatalf("a Config with every figure stated was refused: %v", err) + } + for name, unset := range map[string]func(*Config){ + "Sessions": func(c *Config) { c.Sessions = 0 }, + "Calls": func(c *Config) { c.Calls = 0 }, + "MaxTurns": func(c *Config) { c.MaxTurns = 0 }, + "SessionWall": func(c *Config) { c.SessionWall = 0 }, + "Policy.FollowUps": func(c *Config) { c.Policy.FollowUps = 0 }, + "Policy.ContextChars": func(c *Config) { c.Policy.ContextChars = 0 }, + "Policy.AnswerNow": func(c *Config) { c.Policy.AnswerNow = " " }, + "Policy.Tools.ReadLines": func(c *Config) { c.Policy.Tools.ReadLines = 0 }, + "Policy.Tools.ReadBytes": func(c *Config) { c.Policy.Tools.ReadBytes = 0 }, + "Policy.Tools.ReadLineRunes": func(c *Config) { c.Policy.Tools.ReadLineRunes = 0 }, + "Policy.Tools.ListEntries": func(c *Config) { c.Policy.Tools.ListEntries = 0 }, + "Policy.Tools.GlobMatches": func(c *Config) { c.Policy.Tools.GlobMatches = 0 }, + "Policy.Tools.GrepMatches": func(c *Config) { c.Policy.Tools.GrepMatches = 0 }, + "Policy.Tools.GrepFileBytes": func(c *Config) { c.Policy.Tools.GrepFileBytes = 0 }, + "Policy.Tools.GrepLineRunes": func(c *Config) { c.Policy.Tools.GrepLineRunes = 0 }, + } { + config := whole + unset(&config) + if _, err := New(nil, config); err == nil || !strings.Contains(err.Error(), "Config."+name+" is not set") { + t.Errorf("leaving %s unset gave %v", name, err) + } + } + if _, err := RunSession(t.Context(), nil, SessionOrder{Root: t.TempDir(), Prompt: "look", Policy: testPolicy}); err == nil || !strings.Contains(err.Error(), "SessionOrder.MaxTurns is not set") { + t.Fatalf("a session with no turns of its own ran: %v", err) + } + if _, err := RunSession(t.Context(), nil, SessionOrder{Root: t.TempDir(), Prompt: "look", MaxTurns: 3}); err == nil || !strings.Contains(err.Error(), "SessionOrder.Policy.FollowUps is not set") { + t.Fatalf("a session with no policy ran: %v", err) + } + if _, err := NewClient(delegate.ModelAPI{BaseURL: "http://127.0.0.1:1/v1", Token: "tok"}, -1); err == nil { + t.Fatal("a client with negative retries was opened") + } +} diff --git a/internal/agentsession/app.go b/internal/agentsession/app.go new file mode 100644 index 0000000000..69b8cde265 --- /dev/null +++ b/internal/agentsession/app.go @@ -0,0 +1,325 @@ +package agentsession + +// The App a program runs on: sec-af's four verbs answered by codeaf. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "regexp" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/Agent-Field/agentfield/sdk/go/ai" + "github.com/Agent-Field/codeaf/internal/agentsession/appx" +) + +// Config is how the App runs: where, on which models, and how many sessions +// at once. +type Config struct { + // Root is the repository the run reads, absolute. No session reads + // outside it, whatever a caller's options say. + Root string + // Work is what every session is one agent of, as a noun phrase, said in its + // system prompt ([SessionOrder.Work]): sec's "a security audit". + Work string + // SessionModel answers the agent sessions — the hunters, tracers and + // provers that read the code — and AIModel the single structured calls: + // the duplicate checks, the exploit reasoning, the compliance mapping. + // Empty leaves the choice to the model API, which answers on the run's + // work seat. + SessionModel, AIModel string + // Sessions is how many agent sessions run at once across the whole run, + // and Calls how many single calls. sec-af's own fan-outs multiply — four + // hunters each enriching five locations — and the old harness capped its + // processes at eight for the same reason. + Sessions, Calls int + // MaxTurns bounds one session's turns; sec-af's own default was fifty. + MaxTurns int + // SessionWall bounds one session's time; sec-af's harness waited thirty + // minutes. + SessionWall time.Duration + // Policy is how every session reads and answers ([Policy]). + Policy Policy + // Limits is each agent's own turns and time, by the options its session + // was started with; a zero field falls back to MaxTurns or SessionWall. + // Nil gives every agent those two. + // + // EACH AGENT IS BOUNDED BY WHAT ITS OWN WORK TAKES. One cap for every agent + // was too short for a scanner that walks the repository and far too long + // for a profiler that reads three files: /review's reviewers spent fifty turns + // at twenty seconds each where a dozen would have done, and sec's + // scanners were the ones cut at fifty. The program knows its agents, so the + // figures are its own. + Limits func(appx.HarnessOptions) Limits + // Watch hears every session, call and note as it ends, for the task's + // page. Nil hears nothing. + Watch *Watch +} + +// Limits is one agent session's bounds: the most turns it takes and the +// longest it runs. Zero is the App's own default for that bound. +type Limits struct { + Turns int + Wall time.Duration +} + +// limitsFor is the bounds one session runs under: the program's for this +// agent, each zero filled from the App's defaults. +func (a *App) limitsFor(opts appx.HarnessOptions) Limits { + var own Limits + if a.config.Limits != nil { + own = a.config.Limits(opts) + } + if own.Turns <= 0 { + own.Turns = a.config.MaxTurns + } + if own.Wall <= 0 { + own.Wall = a.config.SessionWall + } + return own +} + +// Watch is what the program's progress is told to. +type Watch struct { + // Session is one agent session that ended: its label (the agent that ran + // it, such as `hunt-scan`), how it went, and the error that stopped it + // before it could answer. + Session func(label string, result SessionResult, err error) + // Note is one of sec-af's progress notes, with its tags. + Note func(message string, tags []string) + // Call is one single structured call that ended: the title of the schema + // its answer took (`VerdictDecision`), and the error that stopped it. + Call func(schema string, err error) +} + +// App is [appx.App] on codeaf. +type App struct { + config Config + client *Client + sessions chan struct{} + calls chan struct{} + threads atomic.Int64 + mu sync.Mutex + cost float64 + count struct{ sessions, calls int } +} + +var _ appx.App = (*App)(nil) + +// New opens the App over the run's model API. It refuses a Config that +// leaves any figure unset: Sessions, Calls, MaxTurns, SessionWall and every +// field of Policy are the program's to state ([Policy] says why). +func New(client *Client, config Config) (*App, error) { + if field := config.missing(); field != "" { + return nil, unsetError("Config", field) + } + return &App{config: config, client: client, + sessions: make(chan struct{}, config.Sessions), calls: make(chan struct{}, config.Calls)}, nil +} + +// Spent is what the program's calls have cost so far, as the replies priced +// them, and how many sessions and single calls it made. The model API's own +// meter is what bills the run; this is the program's own account of itself for +// its report. +func (a *App) Spent() (cost float64, sessions, calls int) { + a.mu.Lock() + defer a.mu.Unlock() + return a.cost, a.count.sessions, a.count.calls +} + +func (a *App) spend(cost float64, session bool) { + a.mu.Lock() + defer a.mu.Unlock() + a.cost += cost + if session { + a.count.sessions++ + } else { + a.count.calls++ + } +} + +// Harness runs one agent session over the repository. A session that ran and +// gave no answer of its schema is a result with IsError, as the old harness +// reported one; a session that could not run is the error. +func (a *App) Harness(ctx context.Context, prompt string, schema map[string]any, dest any, opts appx.HarnessOptions) (*appx.HarnessResult, error) { + label := sessionLabel(opts.Label, opts.Cwd) + select { + case a.sessions <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + defer func() { <-a.sessions }() + root := a.config.Root + // THE REPOSITORY IS THE APP'S, NOT THE CALLER'S. Every agent names the + // folder it was handed; one that named another would be read nowhere. + if project := strings.TrimSpace(opts.ProjectDir); project != "" && filepath.Clean(project) != filepath.Clean(root) { + if real, err := filepath.EvalSymlinks(project); err != nil || !within(root, real) { + return nil, fmt.Errorf("%s asked to read %s, which is not the repository this run reads", label, project) + } + } + thread := fmt.Sprintf("%s-%d", strings.ReplaceAll(label, " ", "-"), a.threads.Add(1)) + limits := a.limitsFor(opts) + result, err := RunSession(ctx, a.client, SessionOrder{ + Model: a.config.SessionModel, Thread: thread, Root: root, Work: a.config.Work, Prompt: prompt, Schema: schema, + MaxTurns: limits.Turns, Wall: limits.Wall, Policy: a.config.Policy, + }) + a.spend(result.CostUSD, true) + if watch := a.config.Watch; watch != nil && watch.Session != nil { + watch.Session(label, result, err) + } + if err != nil && endsTheRun(ctx, err) { + return nil, err + } + cost := result.CostUSD + out := &appx.HarnessResult{Result: result.Text, NumTurns: result.Turns, + DurationMS: result.Duration.Milliseconds(), CostUSD: &cost} + // A SESSION THAT FAILED IS ONE AGENT'S FAILURE, NOT THE RUN'S. A model + // that kept answering with nothing (`the model's reply has no choices`, + // after the client's own retries) was returned as an error, and a program + // that reads an agent's error as fatal ended a 389-call review with no + // review. Only what ends every call — the ceiling, a refused key, a stop — + // is an error; anything else is this session's failed result, for the + // program to degrade around the way it does any agent that came back + // without an answer. + if err != nil { + out.IsError, out.ErrorMessage = true, err.Error() + return out, nil + } + if result.Failed != "" { + out.IsError, out.ErrorMessage = true, result.Failed + return out, nil + } + if schema != nil && dest != nil { + if err := json.Unmarshal(result.JSON, dest); err != nil { + out.IsError, out.ErrorMessage = true, "the answer met its schema but does not decode: "+err.Error() + return out, nil + } + out.Parsed = dest + } + return out, nil +} + +// AI makes one structured call. A served model that refuses a schema-bound +// answer is asked once more with the schema in words, which every model +// follows and sec-af's tolerant reading (internal/secaf/aix) parses. +func (a *App) AI(ctx context.Context, prompt string, opts ...ai.Option) (*ai.Response, error) { + request := &ai.Request{Messages: []ai.Message{textMessage("user", prompt)}} + for _, option := range opts { + if err := option(request); err != nil { + return nil, err + } + } + select { + case a.calls <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + defer func() { <-a.calls }() + model := strings.TrimPrefix(strings.TrimSpace(request.Model), "openrouter/") + if model == "" { + model = a.config.AIModel + } + call := Request{Model: model, Thread: fmt.Sprintf("call-%d", a.threads.Add(1)), + Messages: request.Messages, ResponseFormat: request.ResponseFormat} + response, err := a.client.Complete(ctx, call) + var refused *RefusedError + if err != nil && call.ResponseFormat != nil && !errors.As(err, &refused) && ctx.Err() == nil { + call.Messages = withSchemaInWords(call.Messages, call.ResponseFormat) + call.ResponseFormat = nil + response, err = a.client.Complete(ctx, call) + } + a.spend(costOf(response), false) + if watch := a.config.Watch; watch != nil && watch.Call != nil { + watch.Call(schemaTitle(request.ResponseFormat), err) + } + return response, err +} + +// schemaTitle is the title a response format's schema carries, "" for none. +func schemaTitle(format *ai.ResponseFormat) string { + if format == nil || format.JSONSchema == nil { + return "" + } + var schema struct { + Title string `json:"title"` + } + _ = json.Unmarshal(format.JSONSchema.Schema, &schema) + return schema.Title +} + +// Note is one of sec-af's progress notes. +func (a *App) Note(_ context.Context, message string, tags ...string) { + if watch := a.config.Watch; watch != nil && watch.Note != nil { + watch.Note(message, tags) + } +} + +// Call is never answered here: the program's calls between its own reasoners +// stay in the process (internal/secaf/audit's local calls), which wrap this +// App. A call that reached it is a reasoner nobody registered. +func (a *App) Call(_ context.Context, target string, _ map[string]any) (map[string]any, error) { + return nil, fmt.Errorf("no reasoner %s is carried in this run", target) +} + +// endsTheRun says an agent session's error ends the run's model work and not +// only this session: the ceiling or a refused key, which the client answers to +// every call after, or the caller's own context ending. A session's own wall +// ([Config.SessionWall]) is that session's failure, not the run's, so a +// deadline is read off the caller's context and never off the error. +func endsTheRun(ctx context.Context, err error) bool { + var refused *RefusedError + return ctx.Err() != nil || errors.Is(err, ErrCeiling) || errors.As(err, &refused) +} + +// withSchemaInWords puts a schema the served model would not take as a +// response format into the last user message, as words. +func withSchemaInWords(messages []ai.Message, format *ai.ResponseFormat) []ai.Message { + if format == nil || format.JSONSchema == nil { + return messages + } + out := append([]ai.Message(nil), messages...) + out = append(out, textMessage("user", "Reply with ONLY one JSON object that meets this JSON Schema — no prose and no code fence:\n"+string(format.JSONSchema.Schema))) + return out +} + +// tempSuffix is the random tail os.MkdirTemp puts on a scratch folder's name. +var tempSuffix = regexp.MustCompile(`-?\d+$`) + +// camelBoundary is where a CamelCase agent name's words meet. +var camelBoundary = regexp.MustCompile(`([a-z])([A-Z])`) + +// sessionLabel is the agent a session ran for, in a person's words: the label +// sec-af's call gave it (`DataFlowTracer` is `data flow tracer`), else what +// the scratch folder sec-af made for it says (`secaf-hunt-scan-123456` is +// `hunt scan`), else `agent`. +func sessionLabel(label, cwd string) string { + if label = strings.TrimSpace(label); label != "" { + // AN IDENTIFIER IS SPLIT INTO WORDS; A LABEL ALREADY IN WORDS KEEPS + // THEM, only its first letter lowered, because it may carry a path + // whose case is the file's own (`auth hunter · src/UserView.py:12`). + if !strings.Contains(label, " ") { + return strings.ToLower(camelBoundary.ReplaceAllString(label, "$1 $2")) + } + // Only a capital that opens an ordinary word is lowered: `Data flow + // mapper` is `data flow mapper`, and `DoS hunter` stays as it is. + first, _, _ := strings.Cut(label, " ") + if len(first) > 1 && first[0] >= 'A' && first[0] <= 'Z' && strings.ToLower(first[1:]) == first[1:] { + return strings.ToLower(label[:1]) + label[1:] + } + return label + } + name, scratch := strings.CutPrefix(filepath.Base(strings.TrimSpace(cwd)), "secaf-") + if !scratch { + return "agent" + } + name = strings.Trim(tempSuffix.ReplaceAllString(name, ""), "-") + if name == "" || strings.ContainsAny(name, `/\`) { + return "agent" + } + return strings.ReplaceAll(name, "-", " ") +} diff --git a/internal/agentsession/appx/appx.go b/internal/agentsession/appx/appx.go new file mode 100644 index 0000000000..a5b5b684b4 --- /dev/null +++ b/internal/agentsession/appx/appx.go @@ -0,0 +1,99 @@ +// Package appx declares the agent-capability seam every reasoner and phase of +// a carried program depends on: sec's first, which it was written for. The original Python code received the SDK +// `Agent` (or the `AgentRouter` proxying to it) and called `.harness(...)`, +// `.ai(...)`, `.note(...)` and `.call(...)` on it; the Go code receives an App. +// +// Inside codeaf there is no control plane and no external coding-agent CLI, so +// the seam no longer names any SDK agent type. The harness options and result +// are declared here, carrying only the fields the program actually sets and +// reads, and the codeaf side supplies the implementation: Harness runs +// codeaf's own agent loop, AI goes through codeaf's model API, and Call is +// answered in process by the reasoner registry (internal/secaf/audit). +package appx + +import ( + "context" + + "github.com/Agent-Field/agentfield/sdk/go/ai" +) + +// HarnessOptions is what one harness run is told about where it works. +// +// Cwd is the directory the agent runs in — for most agents a scratch +// directory the agent writes its structured output into — and ProjectDir is +// the repository the run reads, which the agent reads. Those two are the only +// options any agent in the program sets: the model, turn budget and provider were +// process-wide configuration of the external CLI this seam used to drive, and +// are now whatever the codeaf implementation of Harness decides. +type HarnessOptions struct { + Cwd string + ProjectDir string + // Label is the agent the run is for, in the words its errors already use + // ("Hunt location scanner", "DataFlowTracer"). It changes nothing about + // the run; codeaf names the session by it on the program's page. + Label string +} + +// HarnessResult is the outcome of one harness run, mirroring the fields a +// program reads off it. +// +// A run that HAPPENED and failed is reported with IsError and ErrorMessage and +// a nil Go error; a Go error from Harness means the run could not be made at +// all. The callers treat the two differently (harnessx.Extract, the gate +// retry loop), so an implementation must keep that split. +// +// Parsed is the dest pointer the caller handed to Harness once it holds a +// schema-valid object, and nil when the run produced nothing that validated; +// the callers then fall back to parsing Result, the agent's raw final text. +// +// CostUSD is a pointer because "no cost reported" is not "free": the cost +// trackers skip a nil and add a reported zero, so an implementation that cannot +// price a run leaves it nil rather than writing 0. +type HarnessResult struct { + Result string + Parsed any + IsError bool + ErrorMessage string + NumTurns int + DurationMS int64 + CostUSD *float64 +} + +// Harnesser is the `app.harness(...)` seam. +// +// schema is the JSON schema of the object the run must produce (nil for a +// free-text run), and dest is a pointer the implementation decodes that object +// into before returning it as HarnessResult.Parsed. +type Harnesser interface { + Harness(ctx context.Context, prompt string, schema map[string]any, dest any, opts HarnessOptions) (*HarnessResult, error) +} + +// AIer is the `app.ai(...)` seam. The option list and the response keep the +// SDK `ai` package's shapes, which codeaf already speaks everywhere it talks to +// a model. +type AIer interface { + AI(ctx context.Context, prompt string, opts ...ai.Option) (*ai.Response, error) +} + +// Noter is the `app.note(msg, tags=[...])` seam: a progress line for whoever +// is watching the program, never something the algorithm reads back. +type Noter interface { + Note(ctx context.Context, message string, tags ...string) +} + +// Caller is the `app.call(f"{NODE_ID}.x", **kwargs)` seam — the reasoner +// invocation that used to travel through the control plane as a tracked child +// execution. It returns the target reasoner's result map, already decoded the +// way a JSON hop decodes it, and an error on any failure. +type Caller interface { + Call(ctx context.Context, target string, input map[string]any) (map[string]any, error) +} + +// App is the union the reasoners, phases and orchestrator are written +// against. +type App interface { + Harnesser + AIer + Noter + Caller +} diff --git a/internal/agentsession/appx/fake.go b/internal/agentsession/appx/fake.go new file mode 100644 index 0000000000..3b4bc365ba --- /dev/null +++ b/internal/agentsession/appx/fake.go @@ -0,0 +1,206 @@ +package appx + +import ( + "context" + "encoding/json" + "fmt" + "sync" + + "github.com/Agent-Field/agentfield/sdk/go/ai" +) + +// Fake is a scripted, recording App for tests. Every seam is a plain function +// field; unset seams fail loudly (Harness/AI/Call return an error, Note is +// recorded). Fake also tracks the maximum number of concurrently in-flight +// Harness and Call invocations so phase tests can assert the semaphore bounds +// the Python code enforces (asyncio.Semaphore(n)). +// +// Fake is safe for concurrent use; all recorded slices are guarded by mu. +type Fake struct { + mu sync.Mutex + + // HarnessFn answers Harness. To mimic a successful structured run, marshal + // the canned value into dest and return &HarnessResult{Parsed: dest}. + // Use HarnessJSON for the common case. + HarnessFn func(ctx context.Context, prompt string, schema map[string]any, dest any, opts HarnessOptions) (*HarnessResult, error) + // AIFn answers AI. Use AIJSON for the common "return this JSON" case. + AIFn func(ctx context.Context, prompt string, opts ...ai.Option) (*ai.Response, error) + // CallFn answers Call. + CallFn func(ctx context.Context, target string, input map[string]any) (map[string]any, error) + + // Recorded invocations, in call order. + Harnesses []HarnessCall + AIs []AICall + Notes []NoteCall + Calls []CallCall + + inflightHarness, maxHarness int + inflightCall, maxCall int +} + +// HarnessCall is one recorded Harness invocation. +type HarnessCall struct { + Prompt string + Schema map[string]any + Opts HarnessOptions +} + +// AICall is one recorded AI invocation (the option list is opaque; tests that +// need the system prompt/schema should apply the options to an ai.Request). +type AICall struct { + Prompt string + Opts []ai.Option +} + +// NoteCall is one recorded Note invocation. +type NoteCall struct { + Message string + Tags []string +} + +// CallCall is one recorded Call invocation. +type CallCall struct { + Target string + Input map[string]any +} + +var _ App = (*Fake)(nil) + +// Harness implements Harnesser. +func (f *Fake) Harness(ctx context.Context, prompt string, schema map[string]any, dest any, opts HarnessOptions) (*HarnessResult, error) { + f.mu.Lock() + f.Harnesses = append(f.Harnesses, HarnessCall{Prompt: prompt, Schema: schema, Opts: opts}) + f.inflightHarness++ + if f.inflightHarness > f.maxHarness { + f.maxHarness = f.inflightHarness + } + fn := f.HarnessFn + f.mu.Unlock() + defer func() { + f.mu.Lock() + f.inflightHarness-- + f.mu.Unlock() + }() + if fn == nil { + return nil, fmt.Errorf("appx.Fake: Harness not scripted (prompt %q)", truncate(prompt, 80)) + } + return fn(ctx, prompt, schema, dest, opts) +} + +// AI implements AIer. +func (f *Fake) AI(ctx context.Context, prompt string, opts ...ai.Option) (*ai.Response, error) { + f.mu.Lock() + f.AIs = append(f.AIs, AICall{Prompt: prompt, Opts: opts}) + fn := f.AIFn + f.mu.Unlock() + if fn == nil { + return nil, fmt.Errorf("appx.Fake: AI not scripted (prompt %q)", truncate(prompt, 80)) + } + return fn(ctx, prompt, opts...) +} + +// Note implements Noter. +func (f *Fake) Note(ctx context.Context, message string, tags ...string) { + f.mu.Lock() + defer f.mu.Unlock() + f.Notes = append(f.Notes, NoteCall{Message: message, Tags: append([]string(nil), tags...)}) +} + +// Call implements Caller. +func (f *Fake) Call(ctx context.Context, target string, input map[string]any) (map[string]any, error) { + f.mu.Lock() + f.Calls = append(f.Calls, CallCall{Target: target, Input: input}) + f.inflightCall++ + if f.inflightCall > f.maxCall { + f.maxCall = f.inflightCall + } + fn := f.CallFn + f.mu.Unlock() + defer func() { + f.mu.Lock() + f.inflightCall-- + f.mu.Unlock() + }() + if fn == nil { + return nil, fmt.Errorf("appx.Fake: Call not scripted (target %q)", target) + } + return fn(ctx, target, input) +} + +// MaxConcurrentHarness returns the peak number of simultaneously in-flight +// Harness invocations observed so far. +func (f *Fake) MaxConcurrentHarness() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.maxHarness +} + +// MaxConcurrentCalls returns the peak number of simultaneously in-flight Call +// invocations observed so far. +func (f *Fake) MaxConcurrentCalls() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.maxCall +} + +// CallTargets returns the recorded Call targets in order. +func (f *Fake) CallTargets() []string { + f.mu.Lock() + defer f.mu.Unlock() + out := make([]string, 0, len(f.Calls)) + for _, c := range f.Calls { + out = append(out, c.Target) + } + return out +} + +// NoteMessages returns the recorded Note messages in order. +func (f *Fake) NoteMessages() []string { + f.mu.Lock() + defer f.mu.Unlock() + out := make([]string, 0, len(f.Notes)) + for _, n := range f.Notes { + out = append(out, n.Message) + } + return out +} + +// HarnessJSON builds a HarnessFn that answers every invocation by unmarshaling +// the JSON produced by pick(prompt) into dest and returning a successful +// Result whose Parsed is dest — exactly what the SDK runner does on a +// schema-valid run. pick returning an error yields a Result with IsError set +// and that message (the SDK's failure shape), NOT a transport error. +func HarnessJSON(pick func(prompt string, opts HarnessOptions) (json.RawMessage, error)) func(context.Context, string, map[string]any, any, HarnessOptions) (*HarnessResult, error) { + return func(_ context.Context, prompt string, _ map[string]any, dest any, opts HarnessOptions) (*HarnessResult, error) { + raw, err := pick(prompt, opts) + if err != nil { + return &HarnessResult{IsError: true, ErrorMessage: err.Error()}, nil + } + if dest == nil { + return &HarnessResult{Result: string(raw)}, nil + } + if err := json.Unmarshal(raw, dest); err != nil { + return &HarnessResult{IsError: true, ErrorMessage: "fake: unmarshal into dest: " + err.Error()}, nil + } + return &HarnessResult{Parsed: dest, Result: string(raw)}, nil + } +} + +// AIJSON builds an AIFn that answers every invocation with a response whose +// text content is the JSON produced by pick(prompt). +func AIJSON(pick func(prompt string) (json.RawMessage, error)) func(context.Context, string, ...ai.Option) (*ai.Response, error) { + return func(_ context.Context, prompt string, _ ...ai.Option) (*ai.Response, error) { + raw, err := pick(prompt) + if err != nil { + return nil, err + } + return &ai.Response{Choices: []ai.Choice{{Message: ai.Message{Role: "assistant", Content: []ai.ContentPart{{Type: "text", Text: string(raw)}}}}}}, nil + } +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "…" +} diff --git a/internal/agentsession/client.go b/internal/agentsession/client.go new file mode 100644 index 0000000000..d2ebad0410 --- /dev/null +++ b/internal/agentsession/client.go @@ -0,0 +1,280 @@ +// Package agentsession is what a carried program's agents run on inside +// codeaf: an implementation of [appx.App] over the run's model API, with a +// small read-only agent loop ([Session]) for every agent session. +// +// IT IS SHARED, AND IT IS CODEAF'S. It was written for sec-af, which was a node +// on a control plane: its plain model calls went to a router with a key it +// held, its agent sessions were a coding-agent binary it spawned once per call, +// and every call between its own reasoners was an HTTP round trip. Here all +// three are codeaf's: a model call goes to the run's model API +// (internal/provider/modelapi), the only road to a model a program codeaf +// carries has; an agent session is a loop in this process whose every turn is +// one call on that same road; and a call between reasoners stays in the +// process (internal/secaf/audit's local calls). Nothing in it is one +// program's, so it lives beside the programs that use it and not inside one +// of them: a change here reaches every program that runs on it. +// +// IT CHANGES NOTHING IN THE FOLDER IT READS. A program that lands text promises +// the person their repository back exactly as it was, so the tools an agent +// session is given read and search and never write, and there is no shell to +// write with. +package agentsession + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math/rand/v2" + "net/http" + "strings" + "sync/atomic" + "time" + + "github.com/Agent-Field/agentfield/sdk/go/ai" + "github.com/Agent-Field/codeaf/internal/delegate" + "github.com/Agent-Field/codeaf/internal/provider/modelapi" +) + +// Client is the run's model API as sec-af's calls reach it: one base URL, one +// token, and the ceiling's refusal remembered once it has been heard. +type Client struct { + base string + token string + http *http.Client + // retries is how many times a call that failed on the way is sent again: + // the program's own figure ([NewClient]). + retries int + // stopped is set by the first refusal that ends the run — the dollar + // ceiling, a key the service refused — so the dozens of calls still queued + // behind it learn the same answer without asking a server that already + // gave it. + stopped atomic.Pointer[error] + // pause is how long a retry waits, per attempt, and held how long a held + // call waits; a test shortens both. + pause func(attempt int) time.Duration + held func() time.Duration +} + +// NewClient opens the run's model API. It refuses an API with no address or no +// token, because a program with neither has no road to a model at all. +// +// retries is how many times a call that failed on the way is sent again, and +// it is the program's to state like every figure here ([Policy] says why); +// zero is no retry. The model API already walks codeaf's own ladder of +// services for every call, so a retry covers only what that cannot — a +// connection dropped on this machine, a moment the API was busy. +func NewClient(api delegate.ModelAPI, retries int) (*Client, error) { + if strings.TrimSpace(api.BaseURL) == "" || strings.TrimSpace(api.Token) == "" { + return nil, errors.New("a program codeaf carries runs only inside codeaf, which serves its models; this process was handed no model API") + } + if retries < 0 { + return nil, fmt.Errorf("agentsession: NewClient's retries is %d; it is how many times a failed call is sent again, zero or more", retries) + } + return &Client{ + base: api.BaseURL, token: api.Token, retries: retries, + // NO CLIENT TIMEOUT: a thinking model can be quiet for minutes, and the + // call's own context (the session's wall, the run's stop) is what ends + // it. + http: &http.Client{}, + pause: func(attempt int) time.Duration { return time.Duration(2< 0 { + wire.ToolChoice = "auto" + } + body, err := json.Marshal(wire) + if err != nil { + return nil, fmt.Errorf("encode the call: %w", err) + } + var last error + for attempt := 0; attempt <= c.retries; { + response, err, again := c.send(ctx, body) + if err == nil { + return response, nil + } + var refused *RefusedError + if errors.As(err, &refused) { + c.stopped.CompareAndSwap(nil, &err) + return nil, err + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + // A CALL HELD BEHIND CALLS IN FLIGHT WAITS ITS TURN, for as long as + // the session's own bounds allow, and is not counted as a failure: a + // program runs many calls at once and the ceiling admits as many as it + // can price (modelapi's held answer). + if errors.Is(err, errHeld) { + if !c.wait(ctx, c.holdPause()) { + return nil, ctx.Err() + } + continue + } + if !again { + return nil, err + } + last = err + attempt++ + if attempt > c.retries { + break + } + if !c.wait(ctx, c.pause(attempt)) { + return nil, ctx.Err() + } + } + return nil, last +} + +// errHeld is the model API asking a call to wait while calls in flight hold +// what is left of the run's ceiling. +var errHeld = errors.New("the run's ceiling is held by calls in flight") + +// holdPause is how long a held call waits before it asks again: about the two +// seconds the model API suggests, spread so that calls held together do not +// all ask again together. +func (c *Client) holdPause() time.Duration { + if c.held != nil { + return c.held() + } + return 2*time.Second + time.Duration(rand.Int64N(int64(2*time.Second))) +} + +// wait sleeps d unless ctx ends first, and answers whether it slept. +func (c *Client) wait(ctx context.Context, d time.Duration) bool { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +// send is one attempt: the reply, or the error and whether another attempt +// could answer differently. +func (c *Client) send(ctx context.Context, body []byte) (*ai.Response, error, bool) { + call, err := http.NewRequestWithContext(ctx, http.MethodPost, modelapi.ChatURL(c.base), bytes.NewReader(body)) + if err != nil { + return nil, err, false + } + call.Header.Set("Content-Type", "application/json") + call.Header.Set("Authorization", "Bearer "+c.token) + answer, err := c.http.Do(call) + if err != nil { + return nil, fmt.Errorf("reach the model API: %w", err), true + } + defer answer.Body.Close() + raw, err := io.ReadAll(answer.Body) + if err != nil { + return nil, fmt.Errorf("read the model's reply: %w", err), true + } + if answer.StatusCode >= 400 { + message := apiMessage(raw) + if answer.Header.Get(modelapi.HeldHeader) != "" { + return nil, fmt.Errorf("%w: %s", errHeld, message), true + } + switch answer.StatusCode { + case http.StatusPaymentRequired: + return nil, &RefusedError{Ceiling: true, Code: answer.StatusCode, Message: message}, false + case http.StatusUnauthorized, http.StatusForbidden: + return nil, &RefusedError{Code: answer.StatusCode, Message: message}, false + case http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout, http.StatusInternalServerError: + return nil, fmt.Errorf("the model API answered %d: %s", answer.StatusCode, message), true + } + return nil, fmt.Errorf("the model API answered %d: %s", answer.StatusCode, message), false + } + var response ai.Response + if err := json.Unmarshal(raw, &response); err != nil { + return nil, fmt.Errorf("the model's reply does not parse: %w", err), true + } + if len(response.Choices) == 0 { + return nil, errors.New("the model's reply has no choices"), true + } + return &response, nil, false +} + +// apiMessage is the one sentence an error body carries, or the body itself. +func apiMessage(raw []byte) string { + var shaped ai.ErrorResponse + if json.Unmarshal(raw, &shaped) == nil && strings.TrimSpace(shaped.Error.Message) != "" { + return strings.TrimSpace(shaped.Error.Message) + } + text := strings.TrimSpace(string(raw)) + if len(text) > 400 { + text = text[:400] + "…" + } + return text +} + +// costOf is a reply's metered price, zero when it carries none. +func costOf(response *ai.Response) float64 { + if response == nil || response.Usage == nil || response.Usage.Cost == nil { + return 0 + } + return *response.Usage.Cost +} diff --git a/internal/agentsession/policy.go b/internal/agentsession/policy.go new file mode 100644 index 0000000000..1d52083c7a --- /dev/null +++ b/internal/agentsession/policy.go @@ -0,0 +1,98 @@ +package agentsession + +// What every agent session of one program runs under, all of it the program's. +// +// THIS PACKAGE IS MACHINERY, AND EVERY NUMBER IN IT IS A PROGRAM'S. It began as +// sec's, and its figures were sec-af's: fifty turns, thirty minutes, eight +// sessions at once, two follow-ups, four hundred lines to a read. A second +// program running on it would have inherited every one of them without saying +// so, and a figure tuned for one program's agents would have quietly become the +// other's. So nothing here has a default: a program states each figure, a zero +// is refused where the App is opened ([New]) and where a session is run +// ([RunSession]), and a program that forgets one finds out at once rather +// than running on a figure it never chose. Two programs that choose the same +// figure today each say so in their own package. + +import ( + "fmt" + "strings" +) + +// Policy is how every agent session of a program reads and answers. +type Policy struct { + // FollowUps is how many times an answer that does not meet its schema is + // asked for again. + FollowUps int + // ContextChars is how much a session's transcript may hold before it is + // told to stop reading and answer. + ContextChars int + // AnswerNow is what a session is told when its turns or its room run out: + // to stop reading and answer with what it has. + AnswerNow string + // Tools is what one call of each read-only tool may answer. + Tools ToolLimits +} + +// ToolLimits is the most one call of each tool answers. +type ToolLimits struct { + // ReadLines and ReadBytes are the most lines and bytes one read_file + // answers; ReadLineRunes the most of one line it shows. + ReadLines, ReadBytes, ReadLineRunes int + // ListEntries is the most entries one list_dir answers, and GlobMatches + // the most paths one glob answers. + ListEntries, GlobMatches int + // GrepMatches is the most lines one grep answers, GrepFileBytes the + // largest file it reads, and GrepLineRunes how much of a matching line it + // quotes. + GrepMatches, GrepFileBytes, GrepLineRunes int +} + +// missing names the first figure a program left unset, or "" when it set +// every one. +func (p Policy) missing() string { + switch { + case p.FollowUps <= 0: + return "Policy.FollowUps" + case p.ContextChars <= 0: + return "Policy.ContextChars" + case strings.TrimSpace(p.AnswerNow) == "": + return "Policy.AnswerNow" + } + return p.Tools.missing() +} + +func (l ToolLimits) missing() string { + for _, field := range []struct { + name string + value int + }{ + {"ReadLines", l.ReadLines}, {"ReadBytes", l.ReadBytes}, {"ReadLineRunes", l.ReadLineRunes}, + {"ListEntries", l.ListEntries}, {"GlobMatches", l.GlobMatches}, + {"GrepMatches", l.GrepMatches}, {"GrepFileBytes", l.GrepFileBytes}, {"GrepLineRunes", l.GrepLineRunes}, + } { + if field.value <= 0 { + return "Policy.Tools." + field.name + } + } + return "" +} + +// missing names the first figure a program left unset in its Config, or "". +func (c Config) missing() string { + switch { + case c.Sessions <= 0: + return "Sessions" + case c.Calls <= 0: + return "Calls" + case c.MaxTurns <= 0: + return "MaxTurns" + case c.SessionWall <= 0: + return "SessionWall" + } + return c.Policy.missing() +} + +// unsetError is the refusal of a figure a program did not state. +func unsetError(where, field string) error { + return fmt.Errorf("agentsession: %s.%s is not set; every program states its own, and this package has no default", where, field) +} diff --git a/internal/agentsession/session.go b/internal/agentsession/session.go new file mode 100644 index 0000000000..1cba673c10 --- /dev/null +++ b/internal/agentsession/session.go @@ -0,0 +1,273 @@ +package agentsession + +// One agent session: what sec-af called a harness call. A coding-agent binary +// used to be spawned for each one and told to write its answer to a file in +// the repository; here it is a loop of turns on the run's model API with four +// read-only tools, ending in one JSON answer that is checked against the +// schema the agent was given. +// +// WHAT THE OLD HARNESS DID FOR THE AUDIT IS KEPT; HOW IT DID IT IS NOT. sec-af +// relied on three things from it — a session that can read the code, an +// answer of the schema it asked for, and a second chance when the answer did +// not parse ([Policy.FollowUps]) — and each is here. The output file in the person's +// repository, the binary and the key it held are gone, because the audit +// promised to change nothing in the folder and codeaf holds the keys. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/Agent-Field/agentfield/sdk/go/ai" + "github.com/santhosh-tekuri/jsonschema/v5" +) + +// SessionOrder is one session: who answers, where it reads, what it is asked, +// the shape of its answer, and its bounds. +type SessionOrder struct { + Model string + Thread string + // Root is the repository, absolute; every tool reads only below it. + Root string + // Work is what the session is one agent of, as a noun phrase: sec's + // "a security audit". Empty says only that it works on the repository. + Work string + Prompt string + // Schema is the JSON Schema the answer must meet; nil is a free-text + // answer, which sec-af's chain correlation asks for. + Schema map[string]any + MaxTurns int + // Wall is the session's own time bound; zero is none past the context's. + Wall time.Duration + // Policy is how the session reads and answers; every field is required. + Policy Policy +} + +// SessionResult is how a session ended: its answer, and what it cost. +type SessionResult struct { + // Text is the answer as the model wrote it. + Text string + // JSON is the answer's object when a schema was asked for and met. + JSON json.RawMessage + // Failed is why a session that ran did not give an answer of the shape + // asked for; empty for one that did. + Failed string + Turns int + Tools int + CostUSD float64 + Duration time.Duration +} + +// errNoAnswer is a session that ran out of turns without answering. +var errNoAnswer = errors.New("the session ended without an answer") + +// RunSession runs one session to its answer. The error is for a session that +// could not run at all — the model API refused, the run was stopped; a session +// that ran and gave no usable answer says so in Failed. +func RunSession(ctx context.Context, client *Client, order SessionOrder) (SessionResult, error) { + started := time.Now() + if order.Wall > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, order.Wall) + defer cancel() + } + maxTurns := order.MaxTurns + if maxTurns <= 0 { + return SessionResult{}, unsetError("SessionOrder", "MaxTurns") + } + if field := order.Policy.missing(); field != "" { + return SessionResult{}, unsetError("SessionOrder", field) + } + policy := order.Policy + var validate *jsonschema.Schema + if order.Schema != nil { + compiled, err := compileSchema(order.Schema) + if err != nil { + return SessionResult{}, fmt.Errorf("the answer's schema does not compile: %w", err) + } + validate = compiled + } + tools := toolbox{root: order.Root, limits: policy.Tools} + messages := []ai.Message{ + textMessage("system", sessionSystem(order)), + textMessage("user", order.Prompt), + } + result := SessionResult{} + defer func() { result.Duration = time.Since(started) }() + size := len(order.Prompt) + asked := 0 + answering := false + for result.Turns < maxTurns+policy.FollowUps+1 { + // THE LAST TURN, OR A FULL CONTEXT, IS AN ANSWER. A session that is + // still reading when its turns or its room run out is told to answer + // with what it has, and is offered no tools to do anything else. + if !answering && (result.Turns >= maxTurns-1 || size > policy.ContextChars) { + answering = true + messages = append(messages, textMessage("user", policy.AnswerNow)) + } + request := Request{Model: order.Model, Thread: order.Thread, Messages: messages} + if !answering { + request.Tools = tools.definitions() + } + response, err := client.Complete(ctx, request) + result.Turns++ + result.CostUSD += costOf(response) + if err != nil { + result.Failed = err.Error() + return result, err + } + reply := response.Choices[0].Message + reply.Role = "assistant" + messages = append(messages, reply) + if calls := reply.ToolCalls; len(calls) > 0 && !answering { + for _, call := range calls { + observation := tools.run(call) + result.Tools++ + size += len(observation) + len(call.Function.Arguments) + messages = append(messages, ai.Message{Role: "tool", ToolCallID: call.ID, + Content: []ai.ContentPart{{Type: "text", Text: observation}}}) + } + continue + } + text := strings.TrimSpace(response.Text()) + result.Text = text + if validate == nil { + if text == "" { + result.Failed = errNoAnswer.Error() + } + return result, nil + } + object, problem := checkAnswer(validate, text) + if problem == "" { + result.JSON, result.Failed = object, "" + return result, nil + } + result.Failed = problem + if asked == policy.FollowUps { + return result, nil + } + asked++ + answering = true + messages = append(messages, textMessage("user", "Your answer does not meet the required JSON Schema: "+problem+ + "\nReply with ONLY the corrected JSON object — no prose, no code fence.")) + } + if result.Failed == "" { + result.Failed = errNoAnswer.Error() + } + return result, nil +} + +// sessionSystem is the session's working method: where it is, what it can do, +// and the one shape its answer takes. +func sessionSystem(order SessionOrder) string { + var b strings.Builder + // THE WORK IS THE PROGRAM'S TO NAME. This loop is shared, and an agent of a + // code review told it was one of a security audit reads the code for the + // wrong thing. + if work := strings.TrimSpace(order.Work); work != "" { + fmt.Fprintf(&b, "You are one agent of %s of the repository at %s. ", work, order.Root) + } else { + fmt.Fprintf(&b, "You are one agent working on the repository at %s. ", order.Root) + } + b.WriteString("You can read it with the tools read_file, list_dir, glob and grep, and nothing else: you cannot change any file, and there is no shell. ") + b.WriteString("Paths are relative to the repository root. Do the task you are given and only that; read what you need to be sure, and stop when you are.\n\n") + if order.Schema == nil { + b.WriteString("When you are done, stop calling tools and reply with your answer as plain text.") + return b.String() + } + schema, _ := json.Marshal(order.Schema) + b.WriteString("When you are done, stop calling tools and reply with ONLY one JSON object that meets this JSON Schema — no prose before or after it and no code fence:\n") + b.Write(schema) + return b.String() +} + +// checkAnswer is the object a reply holds and the problem with it, empty when +// it meets the schema. A reply is read the forgiving way the old harness read +// one: the whole of it, else what a code fence holds, else the span from its +// first brace to its last. +func checkAnswer(schema *jsonschema.Schema, text string) (json.RawMessage, string) { + raw, ok := extractObject(text) + if !ok { + return nil, "the reply holds no JSON object" + } + var value any + if err := json.Unmarshal(raw, &value); err != nil { + return nil, "the JSON does not parse: " + err.Error() + } + if err := schema.Validate(value); err != nil { + return nil, schemaProblem(err) + } + return raw, "" +} + +// extractObject finds the JSON object in a reply. +func extractObject(text string) (json.RawMessage, bool) { + text = strings.TrimSpace(text) + if json.Valid([]byte(text)) && strings.HasPrefix(text, "{") { + return json.RawMessage(text), true + } + if open := strings.Index(text, "```"); open >= 0 { + inner := text[open+3:] + inner = strings.TrimPrefix(inner, "json") + if end := strings.Index(inner, "```"); end >= 0 { + if candidate := strings.TrimSpace(inner[:end]); json.Valid([]byte(candidate)) && strings.HasPrefix(candidate, "{") { + return json.RawMessage(candidate), true + } + } + } + first, last := strings.Index(text, "{"), strings.LastIndex(text, "}") + if first >= 0 && last > first { + if candidate := text[first : last+1]; json.Valid([]byte(candidate)) { + return json.RawMessage(candidate), true + } + } + return nil, false +} + +// schemaProblem is a validation failure in a few lines the model can fix from. +func schemaProblem(err error) string { + var invalid *jsonschema.ValidationError + if !errors.As(err, &invalid) { + return err.Error() + } + var lines []string + var walk func(e *jsonschema.ValidationError) + walk = func(e *jsonschema.ValidationError) { + if len(e.Causes) == 0 { + where := e.InstanceLocation + if where == "" { + where = "the answer" + } + lines = append(lines, where+": "+e.Message) + return + } + for _, cause := range e.Causes { + walk(cause) + } + } + walk(invalid) + if len(lines) > 8 { + lines = append(lines[:8], fmt.Sprintf("and %d more", len(lines)-8)) + } + return strings.Join(lines, "; ") +} + +// compileSchema compiles one schema for validation. +func compileSchema(schema map[string]any) (*jsonschema.Schema, error) { + raw, err := json.Marshal(schema) + if err != nil { + return nil, err + } + compiler := jsonschema.NewCompiler() + if err := compiler.AddResource("answer.json", strings.NewReader(string(raw))); err != nil { + return nil, err + } + return compiler.Compile("answer.json") +} + +func textMessage(role, text string) ai.Message { + return ai.Message{Role: role, Content: []ai.ContentPart{{Type: "text", Text: text}}} +} diff --git a/internal/agentsession/tools.go b/internal/agentsession/tools.go new file mode 100644 index 0000000000..625bedeaca --- /dev/null +++ b/internal/agentsession/tools.go @@ -0,0 +1,475 @@ +package agentsession + +// The tools an agent session is handed: four ways to read the repository and +// none to change it. +// +// EVERY PATH IS THE REPOSITORY'S. A path is read relative to the folder the run +// reads, and one that leads out of it — `..`, an absolute path elsewhere, a +// link whose target is elsewhere — is refused in a sentence the model can act +// on. sec-af's hunters chase data flow wherever it goes, and the person's home +// folder is not where it goes. +// +// EVERY ANSWER IS BOUNDED. A session runs for up to fifty turns, and a tool +// that hands back a whole minified bundle or every line matching `e` spends +// the session's context on noise; each tool says how much it left out and how +// to ask for the rest. + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + + "github.com/Agent-Field/agentfield/sdk/go/ai" +) + +// skippedDirs are folders a search walks past: version control, dependency +// trees and build output. A read or a listing still reaches them by name, +// because a hunter that has a reason to open a vendored file may. +var skippedDirs = map[string]bool{ + ".git": true, "node_modules": true, "vendor": true, ".venv": true, "venv": true, + "__pycache__": true, "dist": true, "build": true, "target": true, ".next": true, + ".tox": true, ".mypy_cache": true, ".pytest_cache": true, ".gradle": true, ".idea": true, +} + +// toolbox is the four tools over one repository. +type toolbox struct { + root string + limits ToolLimits +} + +// definitions is what the model is offered. +func (t toolbox) definitions() []ai.ToolDefinition { + tool := func(name, description string, properties map[string]any, required ...string) ai.ToolDefinition { + return ai.ToolDefinition{Type: "function", Function: ai.ToolFunction{ + Name: name, Description: description, + // AN EMPTY LIST, NEVER NULL: a strict server refuses a tool whose + // `required` is not an array, and a tool with no required argument + // would otherwise send null. + Parameters: map[string]any{"type": "object", "properties": properties, "required": append([]string{}, required...)}, + }} + } + text := func(description string) map[string]any { + return map[string]any{"type": "string", "description": description} + } + number := func(description string) map[string]any { + return map[string]any{"type": "integer", "description": description} + } + return []ai.ToolDefinition{ + tool("read_file", fmt.Sprintf("Read a file of the repository, with line numbers. At most %d lines per call; pass offset to read further.", t.limits.ReadLines), + map[string]any{"path": text("path relative to the repository root"), "offset": number("first line to read, 1-based (default 1)"), "limit": number(fmt.Sprintf("how many lines (default and most %d)", t.limits.ReadLines))}, "path"), + tool("list_dir", "List one folder of the repository: files, and folders with a trailing slash.", + map[string]any{"path": text("folder relative to the repository root (default: the root)")}), + tool("glob", "Find files whose path matches a pattern such as **/*.py or src/**/routes*.ts. Dependency and build folders are skipped.", + map[string]any{"pattern": text("the pattern, relative to the repository root")}, "pattern"), + tool("grep", "Search file contents with a regular expression (RE2 syntax). Answers path:line: text. Dependency and build folders are skipped.", + map[string]any{"pattern": text("the regular expression"), "path": text("folder or file to search (default: the whole repository)"), + "glob": text("only files whose path matches this pattern, such as *.go"), "ignore_case": map[string]any{"type": "boolean"}}, "pattern"), + } +} + +// run answers one tool call in the words the model reads: the result, or what +// was wrong with the call. +func (t toolbox) run(call ai.ToolCall) string { + var args struct { + Path string `json:"path"` + Offset int `json:"offset"` + Limit int `json:"limit"` + Pattern string `json:"pattern"` + Glob string `json:"glob"` + IgnoreCase bool `json:"ignore_case"` + } + if raw := strings.TrimSpace(call.Function.Arguments); raw != "" { + if err := json.Unmarshal([]byte(raw), &args); err != nil { + return "error: the arguments are not a JSON object: " + err.Error() + } + } + var ( + out string + err error + ) + switch call.Function.Name { + case "read_file": + out, err = t.read(args.Path, args.Offset, args.Limit) + case "list_dir": + out, err = t.list(args.Path) + case "glob": + out, err = t.glob(args.Pattern) + case "grep": + out, err = t.grep(args.Pattern, args.Path, args.Glob, args.IgnoreCase) + default: + return fmt.Sprintf("error: there is no tool %q; the tools are read_file, list_dir, glob and grep", call.Function.Name) + } + if err != nil { + return "error: " + err.Error() + } + return out +} + +// resolve is a path the model named, as an absolute path inside the +// repository, or the refusal. +func (t toolbox) resolve(name string) (string, error) { + name = strings.TrimSpace(name) + if name == "" || name == "." || name == "/" { + return t.root, nil + } + path := name + if !filepath.IsAbs(path) { + path = filepath.Join(t.root, path) + } + path = filepath.Clean(path) + if !within(t.root, path) { + return "", fmt.Errorf("%s is outside the repository; paths are relative to its root", name) + } + // A LINK IS FOLLOWED ONLY TO WHERE IT STAYS INSIDE. The name is checked + // above and the target here, so neither road leads out. + if real, err := filepath.EvalSymlinks(path); err == nil && !within(t.root, real) { + return "", fmt.Errorf("%s links outside the repository", name) + } + return path, nil +} + +// within says path is root or below it. +func within(root, path string) bool { + rel, err := filepath.Rel(root, path) + return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) && !filepath.IsAbs(rel) +} + +// rel is a path as the model reads it: relative to the root, slash-separated. +func (t toolbox) rel(path string) string { + if rel, err := filepath.Rel(t.root, path); err == nil { + return filepath.ToSlash(rel) + } + return path +} + +func (t toolbox) read(name string, offset, limit int) (string, error) { + if strings.TrimSpace(name) == "" { + return "", errors.New("name the file to read") + } + path, err := t.resolve(name) + if err != nil { + return "", err + } + info, err := os.Stat(path) + if err != nil { + return "", fmt.Errorf("%s: %s", name, plainFSError(err)) + } + if info.IsDir() { + return "", fmt.Errorf("%s is a folder; list it with list_dir", name) + } + file, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("%s: %s", name, plainFSError(err)) + } + defer file.Close() + if offset < 1 { + offset = 1 + } + if limit <= 0 || limit > t.limits.ReadLines { + limit = t.limits.ReadLines + } + var b strings.Builder + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 64<<10), 4<<20) + line, shown, more := 0, 0, false + for scanner.Scan() { + line++ + if line < offset { + continue + } + if shown >= limit || b.Len() >= t.limits.ReadBytes { + more = true + break + } + text := scanner.Text() + if line == 1 && bytes.IndexByte([]byte(text), 0) >= 0 { + return "", fmt.Errorf("%s is a binary file", name) + } + text = cutLongLine(text, t.limits.ReadLineRunes) + fmt.Fprintf(&b, "%6d\t%s\n", line, text) + shown++ + } + if err := scanner.Err(); err != nil { + return "", fmt.Errorf("%s: %v", name, err) + } + if shown == 0 { + if line == 0 { + return name + " is empty", nil + } + return fmt.Sprintf("%s has %d lines; there is nothing from line %d", name, line, offset), nil + } + if more { + fmt.Fprintf(&b, "[more lines follow; read again with offset %d]\n", offset+shown) + } + return b.String(), nil +} + +func (t toolbox) list(name string) (string, error) { + path, err := t.resolve(name) + if err != nil { + return "", err + } + entries, err := os.ReadDir(path) + if err != nil { + return "", fmt.Errorf("%s: %s", displayName(name), plainFSError(err)) + } + var b strings.Builder + for index, entry := range entries { + if index == t.limits.ListEntries { + fmt.Fprintf(&b, "[%d more entries not shown]\n", len(entries)-index) + break + } + b.WriteString(entry.Name()) + if entry.IsDir() { + b.WriteString("/") + } + b.WriteString("\n") + } + if b.Len() == 0 { + return displayName(name) + " is empty", nil + } + return b.String(), nil +} + +func (t toolbox) glob(pattern string) (string, error) { + pattern = strings.TrimPrefix(strings.TrimSpace(filepath.ToSlash(pattern)), "./") + if pattern == "" { + return "", errors.New("give a pattern, such as **/*.py") + } + match, err := globPattern(pattern) + if err != nil { + return "", err + } + var found []string + more := false + walkErr := t.walk(t.root, func(path string) bool { + rel := t.rel(path) + if match.MatchString(rel) { + if len(found) == t.limits.GlobMatches { + more = true + return false + } + found = append(found, rel) + } + return true + }) + if walkErr != nil { + return "", walkErr + } + if len(found) == 0 { + return "no file matches " + pattern, nil + } + sort.Strings(found) + out := strings.Join(found, "\n") + "\n" + if more { + out += fmt.Sprintf("[stopped at %d paths; narrow the pattern]\n", t.limits.GlobMatches) + } + return out, nil +} + +func (t toolbox) grep(expression, name, filter string, ignoreCase bool) (string, error) { + if strings.TrimSpace(expression) == "" { + return "", errors.New("give a pattern to search for") + } + if ignoreCase { + expression = "(?i)" + expression + } + search, err := regexp.Compile(expression) + if err != nil { + return "", fmt.Errorf("the pattern is not a regular expression this search reads (RE2): %v", err) + } + var only *regexp.Regexp + if filter = strings.TrimSpace(filepath.ToSlash(filter)); filter != "" { + if !strings.Contains(filter, "/") { + filter = "**/" + filter + } + if only, err = globPattern(filter); err != nil { + return "", err + } + } + start, err := t.resolve(name) + if err != nil { + return "", err + } + var b strings.Builder + count, more := 0, false + walkErr := t.walk(start, func(path string) bool { + if only != nil && !only.MatchString(t.rel(path)) { + return true + } + info, err := os.Stat(path) + if err != nil || info.Size() > int64(t.limits.GrepFileBytes) { + return true + } + data, err := os.ReadFile(path) + if err != nil || bytes.IndexByte(data[:min(len(data), 8000)], 0) >= 0 { + return true + } + for number, line := range strings.Split(string(data), "\n") { + if !search.MatchString(line) { + continue + } + if count == t.limits.GrepMatches { + more = true + return false + } + line = aroundMatch(strings.TrimRight(line, "\r"), search, t.limits.GrepLineRunes) + fmt.Fprintf(&b, "%s:%d: %s\n", t.rel(path), number+1, line) + count++ + } + return true + }) + if walkErr != nil { + return "", walkErr + } + if count == 0 { + return "no line matches", nil + } + if more { + fmt.Fprintf(&b, "[stopped at %d lines; narrow the pattern or the path]\n", t.limits.GrepMatches) + } + return b.String(), nil +} + +// walk visits every regular file under start, skipping the folders a search +// walks past and every link, until visit answers false. +func (t toolbox) walk(start string, visit func(path string) bool) error { + info, err := os.Stat(start) + if err != nil { + return fmt.Errorf("%s: %s", t.rel(start), plainFSError(err)) + } + if !info.IsDir() { + visit(start) + return nil + } + stop := errors.New("stop") + err = filepath.WalkDir(start, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return nil + } + if entry.IsDir() { + if path != start && skippedDirs[entry.Name()] { + return filepath.SkipDir + } + return nil + } + if !entry.Type().IsRegular() { + return nil + } + if !visit(path) { + return stop + } + return nil + }) + if errors.Is(err, stop) { + return nil + } + return err +} + +// globPattern turns a glob into a regular expression over slash paths: `**` +// crosses folders, `*` and `?` do not. +func globPattern(pattern string) (*regexp.Regexp, error) { + var b strings.Builder + b.WriteString("^") + for index := 0; index < len(pattern); index++ { + switch c := pattern[index]; c { + case '*': + if index+1 < len(pattern) && pattern[index+1] == '*' { + index++ + if index+1 < len(pattern) && pattern[index+1] == '/' { + index++ + b.WriteString("(?:.*/)?") + } else { + b.WriteString(".*") + } + } else { + b.WriteString("[^/]*") + } + case '?': + b.WriteString("[^/]") + default: + b.WriteString(regexp.QuoteMeta(string(c))) + } + } + b.WriteString("$") + compiled, err := regexp.Compile(b.String()) + if err != nil { + return nil, fmt.Errorf("the pattern %q does not read as a glob: %v", pattern, err) + } + return compiled, nil +} + +// plainFSError is a file error without the path the model already named. +func plainFSError(err error) string { + switch { + case errors.Is(err, fs.ErrNotExist): + return "no such file or folder" + case errors.Is(err, fs.ErrPermission): + return "permission denied" + } + var path *fs.PathError + if errors.As(err, &path) { + return path.Err.Error() + } + return err.Error() +} + +func displayName(name string) string { + if strings.TrimSpace(name) == "" { + return "the repository root" + } + return name +} + +// cutLongLine is one line as read_file shows it: whole, or its first +// [ToolLimits.ReadLineRunes] characters and what was left out. +// +// A LONG LINE SAYS HOW TO REACH THE REST OF IT. Minified code, generated data +// and a context file written as one line of JSON are single lines of tens of +// thousands of characters; the cut used to say only `[line cut]`, at a byte that +// could split a character, and an agent that needed what followed read on to +// its turn cap without finding it (/review's live review, 2026-10-06). grep shows +// the text around a match anywhere in a line ([aroundMatch]), so the cut points +// there. +func cutLongLine(text string, most int) string { + runes := []rune(text) + if len(runes) <= most { + return text + } + return fmt.Sprintf("%s …[line cut: the first %d of its %d characters; grep for what you need in it to see the text around each match]", + string(runes[:most]), most, len(runes)) +} + +// aroundMatch is one matching line as grep shows it: whole when it is short, +// else [ToolLimits.GrepLineRunes] characters around its first match, so a match far into +// a long line is shown with its own text rather than with the line's start. +func aroundMatch(line string, search *regexp.Regexp, most int) string { + runes := []rune(line) + if len(runes) <= most { + return line + } + at := 0 + if found := search.FindStringIndex(line); found != nil { + at = len([]rune(line[:found[0]])) + } + start := max(0, at-most/4) + end := min(len(runes), start+most) + start = max(0, end-most) + shown := string(runes[start:end]) + if start > 0 { + shown = "… " + shown + } + if end < len(runes) { + shown += " …" + } + return shown +} diff --git a/internal/config/settings.go b/internal/config/settings.go index 0fa12a6b9c..0fde5e2bb3 100644 --- a/internal/config/settings.go +++ b/internal/config/settings.go @@ -1102,6 +1102,11 @@ var OperatorEnvPins = []string{ // that can be told something while it works, and that program reads it // back; it is a path into one task's record folder, so plumbing. "CODEAF_INBOX", + // The run's own record folder (internal/delegate's EnvRecords), where a + // program that answers with a report writes it. codeaf sets it on every + // program's launch and the program reads it back; it is one run's folder, + // so plumbing. + "CODEAF_RECORDS", // The release check's one-launch opt-out and its two mirror addresses // (internal/update). They are plumbing rather than settings rows: the first // is a shell's decision not to make a launch request, while the other two diff --git a/internal/delegate/builtin/carried_unix.go b/internal/delegate/builtin/carried_unix.go index 52e54d097a..a1f0783298 100644 --- a/internal/delegate/builtin/carried_unix.go +++ b/internal/delegate/builtin/carried_unix.go @@ -4,9 +4,11 @@ package builtin import ( "github.com/Agent-Field/codeaf/internal/delegate" + "github.com/Agent-Field/codeaf/internal/secaf" "github.com/Agent-Field/codeaf/internal/seniordev" ) // carried is every program this build carries on a unix: senior-dev, whose -// engine lives in internal/seniordev. -var carried = []delegate.Delegate{seniordev.Program} +// engine lives in internal/seniordev, and sec, sec-af's auditor, +// in internal/secaf. +var carried = []delegate.Delegate{seniordev.Program, secaf.Program} diff --git a/internal/delegate/builtin/carried_windows.go b/internal/delegate/builtin/carried_windows.go index e2cf947657..5ca443a5ff 100644 --- a/internal/delegate/builtin/carried_windows.go +++ b/internal/delegate/builtin/carried_windows.go @@ -7,5 +7,7 @@ import "github.com/Agent-Field/codeaf/internal/delegate" // carried is empty on Windows. senior-dev's engine uses process groups, file // locks and a bash shell, none of which it has ever had a Windows form of, so // on Windows it is ABSENT — no row, no verb, no paragraph in the prompt — -// rather than present and failing every time it is asked. +// rather than present and failing every time it is asked. sec is +// absent with it: a program's run rides the same process-group launch +// (internal/delegate's launch.go), and neither has been run on Windows. var carried []delegate.Delegate diff --git a/internal/delegate/cli.go b/internal/delegate/cli.go index b614cf07c4..62495b0f41 100644 --- a/internal/delegate/cli.go +++ b/internal/delegate/cli.go @@ -112,16 +112,23 @@ func Parse(program Delegate, line []string, out io.Writer) (*Invocation, error) if err != nil { return nil, fmt.Errorf("%s %s: --dir: %w", program.Name, command.Name, err) } - ceilings := Ceilings{CostUSD: *cost, Hours: *hours} - if program.Name == "senior-dev" { - ceilings = ceilings.SeniorDevDefaults() + // A shell run nobody set a ceiling for still stops on its own, at the + // program's unattended ceilings ([Delegate.Unattended]). + ceilings := (Ceilings{CostUSD: *cost, Hours: *hours}).FilledFrom(program.Unattended) + args := fs.Args() + // A BARE RUN OF A PROGRAM WHOSE NAME SAYS THE WORK RUNS ITS DEFAULT BRIEF + // ([Delegate.DefaultBrief]), and the line a host hands its child says it, + // so the child reads back the same invocation. + if len(args) == 0 && command.Name == program.Default && strings.TrimSpace(program.DefaultBrief) != "" { + args = []string{program.DefaultBrief} + ordered = append(append([]string(nil), ordered...), "--", program.DefaultBrief) } return &Invocation{ Program: program, Command: command, Workspace: abs, Ceilings: ceilings, JSON: *asJSON, - Args: fs.Args(), + Args: args, Line: append(append([]string(nil), line[:len(line)-len(rest)]...), ordered...), ExplicitFlags: explicitFlags, body: body, diff --git a/internal/delegate/delegate.go b/internal/delegate/delegate.go index f94acc8d5c..6431e5124b 100644 --- a/internal/delegate/delegate.go +++ b/internal/delegate/delegate.go @@ -108,6 +108,65 @@ type Delegate struct { // conversation's `say`. A program that does not listen is refused a message // in so many words, never handed one nothing reads. Listens bool + // Unattended is the ceilings a run of this program starts under when + // nobody set smaller ones: a conversation with no limit, or a shell line + // with no --max-cost or --max-hours. A zero field is no ceiling of the + // program's own, which leaves that run on whatever the conversation has + // left. + // + // A PROGRAM NOBODY WATCHES MUST STOP ON ITS OWN, AND HOW LONG IT MAY GO IS + // THE PROGRAM'S TO KNOW. senior-dev's change and sec's whole-repository + // audit are different shapes of work: an audit spends its hours long + // before its dollars, which a change does not ([SeniorDevCeilings] and + // secaf.Unattended say each one's figures and why). So the figure lives + // beside the program rather than in a branch of codeaf that names it + // ([Ceilings.CappedBy], [Ceilings.FilledFrom]). + Unattended Ceilings + // FollowUp is one sentence the conversation is handed when a run of a + // program that lands text finishes: what to offer the person next, in the + // program's own terms. Empty offers nothing. It is read only on that turn, + // so it costs the conversation's fixed prefix nothing. + // + // THE OFFER IS THE PROGRAM'S, AND THE DECISION THE PERSON'S. A report that + // found something to fix suggests its own next step better than codeaf can, + // and the wake turn starts nothing ([LandsText]). + FollowUp string + // DefaultBrief is the brief a bare `/` or `codeaf ` runs. + // Empty is a program that needs one, and is refused without it. + // + // A BRIEF IS THE WORK, AND SOME WORK IS ALREADY SAID BY THE NAME. "Fix + // it" is no task for senior-dev, but an audit of the repository it is + // started in is exactly what a bare `/sec` means. + DefaultBrief string + // Words says the program's brief is a few words it reads itself, such as + // sec's `changes since main thorough`, and not a document for a model. + // Such a program is handed exactly the words it was asked with, from the + // chat's proposal as from a typed `/`; false hands it the composed + // brief every task gets. + // + // A COMPOSED BRIEF OPENS ON ITS OWN HEADINGS, NOT ON THE WORK. A + // proposal's brief is the person's message, the work, the deliverable and + // lines from the conversation under capitals, and a program that read its + // scope off the first line read `WHAT THE PERSON ASKED FOR` there: a + // proposed `whole repository thorough` ran at standard depth, and a + // proposed `changes` audited the whole repository. + Words bool + // Title is the title a typed `/ ` gives its task, from the + // brief as typed; nil takes the brief's own first words, as a task's + // title always has. A brief that is only a program's own scope words + // (`quick`, `changes`) says nothing as a title, so such a program names + // its run itself. + Title func(brief string) string + // Args is how the command row shows what follows the name, such as + // `[changes] [quick | thorough]`. Empty is ``. + Args string + // ModelFlag is the flag of the default command that names the models it + // works on (senior-dev's `high`, sec's `model`), without its + // dashes. A shell run resolves what the person typed there through the + // profile's own services before the program starts, and puts the + // profile's work seat on the line when they typed nothing. Empty is a + // program that takes no model. + ModelFlag string // CrewFlags is the flags the default command takes to use the models of // the conversation's crew ([Crew]), which codeaf puts on the line of every // run it starts from a conversation. Nil is a program that picks its own @@ -304,6 +363,9 @@ func (d Delegate) validateLineFlags() error { } } } + if flag := strings.TrimSpace(d.ModelFlag); flag != "" && !parses([]string{"--" + flag, "vendor/model"}) { + return fmt.Errorf("%s: the model flag --%s is not a flag its %s command takes", d.Name, flag, command.Name) + } if len(d.PlainFolder) > 0 && !parses(d.PlainFolder) { return fmt.Errorf("%s: the plain folder flags %q are not flags its %s command takes", d.Name, strings.Join(d.PlainFolder, " "), command.Name) } diff --git a/internal/delegate/host.go b/internal/delegate/host.go index 982f5cda8d..4116b60b59 100644 --- a/internal/delegate/host.go +++ b/internal/delegate/host.go @@ -23,6 +23,34 @@ const ( EnvModelToken = "CODEAF_MODEL_TOKEN" ) +// EnvRecords names the run's own record folder in a program's environment: the +// task's folder for a run the chat started, the run's folder under +// `~/.codeaf/v3/carried//` for a shell run. It is where a program that +// lands text leaves the files its answer points at, because the folder it +// works in is the person's and it promised to change nothing there. +const EnvRecords = "CODEAF_RECORDS" + +// Recorder is a host that names the run's record folder ([EnvRecords]). A +// program asks for it by type, as it asks for a [Listener], so a test's host +// that names none is still a Host. +type Recorder interface { + // Records is the run's record folder, absolute, or "" when codeaf named + // none. + Records() string +} + +// RecordsEnv is the environment entry that names a run's record folder to its +// program, nil for none. +func RecordsEnv(folder string) []string { + if strings.TrimSpace(folder) == "" { + return nil + } + return []string{EnvRecords + "=" + folder} +} + +// Records is the run's record folder codeaf named on the child's environment. +func (h *childHost) Records() string { return strings.TrimSpace(env.Get(EnvRecords)) } + // Host is what a running program asks codeaf for. Its body is handed one and // reports through it: the records go to codeaf, and the models come from it. type Host interface { @@ -47,6 +75,9 @@ type Ceilings struct { Hours float64 } +// senior-dev's own unattended ceilings ([Delegate.Unattended]). They are +// named here rather than in its package because the manual's truth gate and +// the session's tests read them without linking its engine. const ( // An autonomous senior-dev run with no person watching must stop on its own. DefaultSeniorDevCostUSD = 10.0 @@ -54,31 +85,58 @@ const ( DefaultSeniorDevHours = 3.0 ) -// SeniorDev fills absent conversation limits and caps larger ones at defaults. -func (c Ceilings) SeniorDev() Ceilings { - if c.CostUSD <= 0 || c.CostUSD > DefaultSeniorDevCostUSD || math.IsNaN(c.CostUSD) { - c.CostUSD = DefaultSeniorDevCostUSD +// SeniorDevCeilings is senior-dev's [Delegate.Unattended]. +var SeniorDevCeilings = Ceilings{CostUSD: DefaultSeniorDevCostUSD, Hours: DefaultSeniorDevHours} + +// IsZero says neither ceiling is set. +func (c Ceilings) IsZero() bool { return c.CostUSD == 0 && c.Hours == 0 } + +// CappedBy is a conversation's remaining limits held to a program's unattended +// ceilings: an absent or unreadable limit becomes the program's, and a larger +// one is cut to it. A field the program leaves at zero is passed through +// untouched, so a program with no ceiling of its own runs on what is left. +func (c Ceilings) CappedBy(unattended Ceilings) Ceilings { + if limit := unattended.CostUSD; limit > 0 && (c.CostUSD <= 0 || c.CostUSD > limit || math.IsNaN(c.CostUSD)) { + c.CostUSD = limit } - if c.Hours <= 0 || c.Hours > DefaultSeniorDevHours || math.IsNaN(c.Hours) { - c.Hours = DefaultSeniorDevHours + if limit := unattended.Hours; limit > 0 && (c.Hours <= 0 || c.Hours > limit || math.IsNaN(c.Hours)) { + c.Hours = limit } return c } -// SeniorDevDefaults fills omitted shell limits while preserving explicit flags. -func (c Ceilings) SeniorDevDefaults() Ceilings { +// FilledFrom is a shell line's ceilings with the program's unattended ones put +// where the line set none. A ceiling the person typed is kept as typed, larger +// or not: at a shell they are the one watching. +func (c Ceilings) FilledFrom(unattended Ceilings) Ceilings { if c.CostUSD <= 0 || math.IsNaN(c.CostUSD) || math.IsInf(c.CostUSD, 0) { - c.CostUSD = DefaultSeniorDevCostUSD + c.CostUSD = unattended.CostUSD } if c.Hours <= 0 || math.IsNaN(c.Hours) || math.IsInf(c.Hours, 0) { - c.Hours = DefaultSeniorDevHours + c.Hours = unattended.Hours } return c } -// Summary says the two ceilings as the person sees them at either start door. +// SeniorDev is [Ceilings.CappedBy] senior-dev's ceilings. +func (c Ceilings) SeniorDev() Ceilings { return c.CappedBy(SeniorDevCeilings) } + +// SeniorDevDefaults is [Ceilings.FilledFrom] senior-dev's ceilings. +func (c Ceilings) SeniorDevDefaults() Ceilings { return c.FilledFrom(SeniorDevCeilings) } + +// Summary says the two ceilings as the person sees them at either start door, +// and says nothing of a ceiling that is not set: none at all is "", never +// `up to $0.00 and 0h` (the emptiness law). func (c Ceilings) Summary() string { - return fmt.Sprintf("up to $%.2f and %s", c.CostUSD, c.TimeWord()) + switch { + case c.CostUSD > 0 && c.Hours > 0: + return fmt.Sprintf("up to $%.2f and %s", c.CostUSD, c.TimeWord()) + case c.CostUSD > 0: + return fmt.Sprintf("up to $%.2f", c.CostUSD) + case c.Hours > 0: + return "up to " + c.TimeWord() + } + return "" } // TimeWord spells the wall ceiling without padded zero units. diff --git a/internal/delegate/limits_test.go b/internal/delegate/limits_test.go index 98d287da15..cab8ce1642 100644 --- a/internal/delegate/limits_test.go +++ b/internal/delegate/limits_test.go @@ -42,7 +42,7 @@ func TestSeniorDevShellFlagsReplaceDefaults(t *testing.T) { func TestSeniorDevShellParserSetsFiniteDefaultsAndHonorsFlags(t *testing.T) { program := testProgram(nil) - program.Name = "senior-dev" + program.Name, program.Unattended = "senior-dev", SeniorDevCeilings for _, tc := range []struct { line []string want Ceilings @@ -62,7 +62,7 @@ func TestSeniorDevShellParserSetsFiniteDefaultsAndHonorsFlags(t *testing.T) { func TestSeniorDevRejectsNonFiniteCeilings(t *testing.T) { program := testProgram(nil) - program.Name = "senior-dev" + program.Name, program.Unattended = "senior-dev", SeniorDevCeilings for _, line := range [][]string{ {"--max-cost", "NaN", "repair"}, {"--max-hours", "+Inf", "repair"}, @@ -75,3 +75,83 @@ func TestSeniorDevRejectsNonFiniteCeilings(t *testing.T) { t.Fatalf("non-finite conversation limit produced %+v", got) } } + +// A program that names no ceilings of its own is held to none: the +// conversation's remaining limits pass through as they are, and a shell line +// with no flags runs unbounded, as every program but senior-dev did before the +// field existed. One that names its own is held to those and not to +// senior-dev's. +func TestUnattendedCeilingsAreTheProgramsOwn(t *testing.T) { + left := Ceilings{CostUSD: 30, Hours: 9} + if got := left.CappedBy(Ceilings{}); got != left { + t.Fatalf("a program with no ceilings capped %+v to %+v", left, got) + } + own := Ceilings{CostUSD: 5, Hours: 2} + if got := left.CappedBy(own); got != own { + t.Fatalf("%+v capped by %+v became %+v", left, own, got) + } + if got := (Ceilings{}).CappedBy(own); got != own { + t.Fatalf("an unlimited conversation ran under %+v, want %+v", got, own) + } + if got := (Ceilings{CostUSD: 1}).CappedBy(Ceilings{Hours: 2}); got != (Ceilings{CostUSD: 1, Hours: 2}) { + t.Fatalf("a program with only a wall ceiling gave %+v", got) + } + program := testProgram(nil) + inv, err := Parse(program, []string{"repair"}, &bytes.Buffer{}) + if err != nil { + t.Fatal(err) + } + if !inv.Ceilings.IsZero() { + t.Fatalf("a shell run of a program with no ceilings got %+v", inv.Ceilings) + } + program.Unattended = own + if inv, err = Parse(program, []string{"repair"}, &bytes.Buffer{}); err != nil { + t.Fatal(err) + } + if inv.Ceilings != own { + t.Fatalf("a shell run got %+v, want the program's own %+v", inv.Ceilings, own) + } +} + +// A BARE RUN OF A PROGRAM WITH A DEFAULT BRIEF RUNS THAT BRIEF, and the line a +// host hands its child carries it, so the child parses back the same +// invocation. A program with none still has nothing to run. +func TestABareRunTakesTheProgramsDefaultBrief(t *testing.T) { + program := testProgram(nil) + program.DefaultBrief = "the whole repository" + inv, err := Parse(program, []string{"--max-cost", "2"}, &bytes.Buffer{}) + if err != nil { + t.Fatal(err) + } + if inv.Brief() != "the whole repository" { + t.Fatalf("a bare run's brief = %q", inv.Brief()) + } + again, err := Parse(program, inv.Line, &bytes.Buffer{}) + if err != nil || again.Brief() != inv.Brief() || again.Ceilings != inv.Ceilings { + t.Fatalf("the line %q read back as %+v, %v", inv.Line, again, err) + } + if typed, _ := Parse(program, []string{"changes"}, &bytes.Buffer{}); typed.Brief() != "changes" { + t.Fatalf("a typed brief became %q", typed.Brief()) + } + program.DefaultBrief = "" + if bare, _ := Parse(program, nil, &bytes.Buffer{}); bare.Brief() != "" { + t.Fatalf("a program with no default brief ran %q", bare.Brief()) + } +} + +// No ceiling is said as no words, and one ceiling as that one alone. +func TestACeilingThatIsNotSetIsNotSaid(t *testing.T) { + for _, tc := range []struct { + in Ceilings + want string + }{ + {Ceilings{}, ""}, + {Ceilings{CostUSD: 5}, "up to $5.00"}, + {Ceilings{Hours: 2}, "up to 2h"}, + {Ceilings{CostUSD: 5, Hours: 2}, "up to $5.00 and 2h"}, + } { + if got := tc.in.Summary(); got != tc.want { + t.Errorf("%+v says %q, want %q", tc.in, got, tc.want) + } + } +} diff --git a/internal/manual/chat/commands.md b/internal/manual/chat/commands.md index 4955d8888d..f74939041d 100644 --- a/internal/manual/chat/commands.md +++ b/internal/manual/chat/commands.md @@ -282,6 +282,7 @@ Canonical word, the other words it answers to, its argument form, and what it do | `/subharness` | `/sub` | `` | opens that subharness's intake card straight away | | `/` | — | `` | one row per program this build carries: starts a task that program does on its own | | `/senior-dev` | — | `` | when this build or far engine carries senior-dev, its named row hands the whole task to that program; with no brief, it shows the required `` usage | +| `/sec` | — | `[changes [since ]] [quick \| thorough]` | when this build or far engine carries sec, audits the whole repository for security problems, or with `changes` only the branch's changes; with nothing after it, the whole repository. It changes no files | | `/skill` | `/skills` | — | opens the skill shelf under the message box; enter toggles a skill, and its chip stays attached across messages | | `/memory` | — | — | opens the memory panel | | `/memory` | `/memories` | `` | prints matching memories into the conversation | diff --git a/internal/manual/chat/delegates.md b/internal/manual/chat/delegates.md index be75e50a60..da59dc320e 100644 --- a/internal/manual/chat/delegates.md +++ b/internal/manual/chat/delegates.md @@ -1,13 +1,17 @@ # Programs codeaf carries -## What a program codeaf carries is — a delegate, another coding agent, an agent of its own for a whole task - -codeaf carries programs of its own that take one whole coding task and do it alone, for as -long as an hour or more. People call them delegates. You hand one a task the way codeaf -hands a task to its own worker: in a git repository it works in a private copy of its -own and leaves its work on a branch of its own when it ends; in a folder with no git -history it works in the folder itself. It runs under this conversation's dollar and time -limits, shows on the rail while it runs, and can be stopped. +## What a program codeaf carries is — a delegate, another coding agent, an agent of its own for a whole task, which programs there are + +codeaf carries programs of its own that take one whole task and do it alone, for as +long as an hour or more. People call them delegates. There are two: **senior-dev**, a +coding agent for one large, well-specified change, and **sec**, which audits +the repository or its changes for security problems and changes nothing (its own page +says how). You hand one a task the way codeaf hands a task to its own worker. A program +that edits code, as senior-dev does, works in a private copy of its own in a git +repository and leaves its work on a branch of its own when it ends; in a folder with no +git history it works in the folder itself. One that answers, as sec does, +reads the folder and leaves its report. Each runs under this conversation's dollar and +time limits, shows on the rail while it runs, and can be stopped. Each one is **built into codeaf**. There is nothing to install and nothing to set up, and none of them runs on its own outside codeaf. Each is a command in the chat, `/ @@ -28,7 +32,7 @@ did. **Every program's tasks wear its name as a badge**: `[]` after the task's title on the side list, the proposal card, the task's page, the `@` list, the sessions place and home, and its -initials (`[sd]` for senior-dev) where a list is narrow. A task codeaf's own worker does +initials (`[sd]` for senior-dev, `[s]` for sec) where a list is narrow. A task codeaf's own worker does wears none, and a program added to codeaf later gets its own badge from its name. The landed card's head shows the title and outcome without a program badge. @@ -66,8 +70,9 @@ than doing it itself or giving it to codeaf's own worker, even when it is one lo with nothing to run beside it. Each program's own line says what it is for: senior-dev's claims complex, multi-part coding work, such as fixing an issue in a mature codebase whose cause spans files, a feature with its tests, a rewrite across a package, or a -migration. The model proposes that work with `via` naming the program, and its card goes -up like any proposal's. +migration; sec's claims a security audit of the whole repository or of the +changes on the branch. The model proposes that work with `via` naming the program, and +its card goes up like any proposal's. **Naming the program is enough.** Say it in your message, by name or as its command ("fix issue 412 with senior-dev", "give this to /senior-dev", "senior dev should do @@ -147,15 +152,18 @@ everything it would stop and ask is already settled. The model is told the same it proposes one. A program that listens, as senior-dev does until it hands in, can still be told something while it works; it never waits for a reply. -**It has no step cap.** senior-dev has finite dollar and wall-clock ceilings even when -the conversation sets none; `/budget conversation` can lower the dollar ceiling, +**It has no step cap.** Each program has finite dollar and wall-clock ceilings of its own +even when the conversation sets none — senior-dev's are $10 and three hours, +sec's $5 and four hours; `/budget conversation` can lower the dollar ceiling, and shell flags set either ceiling directly. An open chat's `/budget conversation` change binds its next proposal, run and turn as soon as the setting receipt appears. Before forwarding a call, codeaf reserves the larger estimate from the requested model and its possible fallback seat when both have known prices, using input size and output cap; if either price is unknown, it uses the unpriced bound. It refuses a call whose -estimate would cross the ceiling. An +estimate would cross the ceiling, and a call that would fit once calls already in flight have +ended (a program that makes many calls at once, as sec does) is asked to wait for +them instead and goes on when one ends. An answer can cost more than its estimate. When a model has no known price, codeaf reserves half the dollar ceiling and limits concurrent calls once half the recorded spend is used; an unpriced service's actual charge cannot be measured here. The wall-clock ceiling still @@ -225,14 +233,27 @@ and keeps what was loose as a patch in the run's record folder. Its own notes folder. In a folder with no git history its work is simply there, and nothing is committed. -A program that only answers works in your folder in place and changes nothing. Its answer -arrives in the conversation the way a task's landing does. +A program that only answers, as sec does, works in your folder in place and +changes nothing. Its answer arrives in the conversation the way a task's landing does, and +the chat tells you what it found and offers the next step it names; the files its answer +points at are in the task's record folder. What it spent is in the conversation's total, in `/cost` and on the status line. Every model call it made went through codeaf and is priced like one of codeaf's own. A run stopped in the middle of a call is not over until that call's price has come in, for at most 70 seconds, so the call it was cut in is in those figures too. +## When a program finished but the chat said nothing — done with no summary, no answer after it ended, I pressed stop + +When a program ends, its ending wakes a turn in the conversation, and that turn tells you +what it did or found. **The ending is never lost to a missing reply:** if that turn does +not finish an answer, the conversation gives the program's own account in full in a line +of its own. When every model it was offered failed, the line opens +` ended, but the chat could not finish its answer to it, so here is what said:`; +when you stopped the answer yourself, it opens +` ended, and you stopped the chat's answer to it, so here is what said:`. +It is written once per run. Ask again in your own words and the chat answers from it. + ## Why is there no command for it — missing, not in this build, Windows, a hosted conversation A program's command exists only in a build that carries it. On Windows codeaf carries diff --git a/internal/manual/chat/models-and-cost.md b/internal/manual/chat/models-and-cost.md index fd70b150e5..41f01cabc5 100644 --- a/internal/manual/chat/models-and-cost.md +++ b/internal/manual/chat/models-and-cost.md @@ -3125,8 +3125,9 @@ tab. The `$` on the status line counts what the tasks are spending while they spend it, and `/cost` splits that figure into `conversation` and `tasks`. -senior-dev's run ceiling is separate from this per-task figure. Its own page names -the defaults and flags; its run still spends against the conversation and day. +senior-dev's and sec's run ceilings are separate from this per-task figure. +Each program's own page names the defaults and flags; its run still spends against the +conversation and day. `/budget task 20` is not a shape this command takes; the per-task figure lives in `/crew`. The **composer layer** can put a further figure on one errand; see the tasks page. diff --git a/internal/manual/chat/running-from-the-terminal.md b/internal/manual/chat/running-from-the-terminal.md index 4e32bbb4a2..3f5caed86d 100644 --- a/internal/manual/chat/running-from-the-terminal.md +++ b/internal/manual/chat/running-from-the-terminal.md @@ -299,7 +299,8 @@ other call. headings, in this order — **Talk to it**, **Hand it work**, **Hand it a whole task**, **Look at what happened**, **Housekeeping** and **Plan work by hand** — then five worked examples. **Hand it a whole task** lists the programs this build carries, such as -`codeaf senior-dev`; a build that carries none prints the other five. +`codeaf senior-dev` and `codeaf sec`; a build that carries none prints the +other five. **The environment table is not on that page**: it is `codeaf help env`, because it is a reference somebody consults and it used to be more than half of what `--help` printed. diff --git a/internal/manual/chat/sec.md b/internal/manual/chat/sec.md new file mode 100644 index 0000000000..5575be211d --- /dev/null +++ b/internal/manual/chat/sec.md @@ -0,0 +1,92 @@ +# sec — a security audit of your code + +## What sec is — a security review, a vulnerability scan, find security bugs in my code, sec-af + +sec is a program codeaf carries that audits code for security problems: injection, broken authentication, server-side request forgery, cross-site scripting, weak cryptography, exposed secrets and data, unsafe dependencies, denial of service, business-logic and API flaws. It reads the code, hunts for problems, then tests each one against the code to show it can really be exploited or to rule it out, and reports only what stands, each with its file, line and a suggested fix. + +It is sec-af, the security-audit program from AgentField, built into codeaf under the shorter name `sec`. There is nothing to install and no key to set: every model call it makes goes through codeaf, so it is priced into your spending and held to the run's ceiling like any program's (see *Programs codeaf carries*). It does not run on its own outside codeaf. + +**It changes nothing.** It reads your folder and writes nothing there: no branch, no commit, no file. Its answer comes back to the conversation, and its full report goes to the task's record folder. + +## How do I run a security audit — /sec, codeaf sec, audit the whole repository, ask the chat for a security review + +In the chat, type `/sec` on its own to audit the whole repository the conversation is in, or add words: + +``` +/sec +/sec changes +/sec changes since main +/sec whole repository thorough +``` + +It starts as a task at once, shows on the rail which of its phases it is in (`recon`, `hunt`, `prove`, `remediate`, `report`), and can be stopped. The turn goes on while it runs. + +**Asking in words works too.** "Do a security review of this repo" or "audit my changes for security problems" makes the chat propose the work with `via: "sec"`; you answer its card like any proposal's. The proposal's brief is sec's own words — `whole repository thorough`, `changes since main` — and those words are all sec is handed, exactly as if you had typed them after `/sec`. + +At a shell, `codeaf sec` audits the folder you are in, or the one `--dir` names. `codeaf sec --changes` audits the changes, `--base ` measures them from another commit, and `--depth quick|standard|thorough` sets how hard it looks. `--max-cost` and `--max-hours` set its ceilings. `codeaf sec run --help` lists every flag. + +## Audit only my changes — this branch, a pull request, uncommitted work, since main, a diff + +`/sec changes` (or `--changes` at a shell) audits the work on your branch instead of the whole repository: the commits since the branch left its base, **plus your uncommitted edits and files not yet tracked**, as the work stands now. The base is where your branch left the remote's default branch (`origin/HEAD`), else `origin/main`, `origin/master`, `origin/dev`, `main`, `master`, `dev` or `trunk`; on the default branch itself it is your last commit, so the changes are what you have not committed. `changes since ` (`--base `) names another base. + +Only code counts as a change: docs, config, lock files and data files (`.md`, `.txt`, `.yml`, `.yaml`, `.json`, `.toml`, `.cfg`, `.ini`, `.lock`) and anything under a top-level `tests/`, `test/`, `vendor/` or `node_modules/` are left out. The files near the change, those that name a changed file's module, come along, because a change is unsafe where its callers meet it. + +The hunters are shown the change and told to report only what it touches or makes reachable; the rest of the code is read for context. Findings outside the changed and nearby files are dropped, and at most 15 findings are tested unless `--max-provers` says otherwise. With nothing changed it ends at once: `sec finished: it had nothing to audit: nothing has changed since in a file this audit reads (code, not docs or config)`. Outside a git repository it is refused: `an audit of the changes needs a git repository with at least one commit; audit the whole repository instead`. + +## What sec does, step by step — why it takes so long, how it works, the phases, recon hunt prove remediate + +Its phases keep sec-af's own names, on the rail, as the headings of its task's page and in its notes there (`RECON phase complete`): + +1. **Recon** (`recon`): agents read the architecture, the dependencies, the configuration, and (except at `quick`) the data flows and security context. +2. **Hunt** (`hunt`): a hunter per kind of problem — injection, denial of service, request forgery, authentication, data exposure and secrets in configuration always; cross-site scripting and business logic at `standard` and `thorough`; cryptography, supply chain and API security when the code has them. Each finds candidate locations and describes each one; duplicates are merged. +3. **Prove** (`prove`): the most serious findings first, each goes through four agents — the data flow tracer follows the data from its source to the dangerous call, the sanitization analyzer looks for checks on the way, the exploit hypothesizer builds a concrete attack, and the deciding agent weighs the three (sec-af's `VerdictAgent`). Findings come out confirmed, likely, unclear or ruled out. `quick` tests at most 10, `standard` 30, `thorough` all. +4. **Remediate** (`remediate`): the remediation agent writes a suggested fix, with a patch, for each confirmed or likely finding. +5. **Report** (`report`). + +Up to eight agent sessions run at once. **Each kind of agent has its own turns and time**, from what a real audit measured: a location scanner up to 75 turns and 20 minutes, a context profiler 30 turns and 5 minutes, an agent nothing measured yet 50 turns and 30 minutes. At a shell, `--max-turns` and `--session-wall` set one figure for every agent instead. A standard audit runs a hundred or more sessions, which is why it takes minutes to hours. + +## What sec's task page shows — each agent's line, which hunter, where it looked, what it found, the same lines repeated + +The task's page shows one line per agent session — which agent, in which phase, and what it found — plus sec-af's own note as each phase starts and ends; `ctrl+y` shows the raw model calls. Under `hunt`, each hunter's scan says how many places it will look at (`injection hunter · scan` · `2 locations`, or `nothing`), and each of those places gets a line of its own with the finding it made and its severity (`injection hunter · app/views.py:4 · Command injection via os.popen` · `critical`). + +## What sec reports and where the full report is — confirmed, likely, unclear, ruled out, SARIF, JSON, Markdown + +When it finishes, the chat is handed a short account and tells you what it found, most serious first: how many problems are **confirmed** (shown exploitable from the code) or **likely**, by severity, then each one on a line — severity, how it came out, title, `file:line`, the weakness's CWE number and a one-line fix. How many stayed **unclear** or were **ruled out** is counted. A finding below `--severity` (default `low`) is left out. + +The full report is three files in the task's record folder, named at the end of the account: + +- `sec-report.md` — the readable report, in the same words as the account: confirmed and likely problems in full (where, the weakness, why, how the data gets there, the attack, the fix and its patch), then the unclear ones, what was ruled out, attack chains and how the run went +- `sec-report.json` — every finding in full, as data +- `sec-report.sarif` — SARIF 2.1.0, which code-scanning tools and editors read; it names its tool `sec`, with codeaf's build + +`--compliance owasp,pci-dss` (at a shell) maps the findings to compliance frameworks — OWASP, PCI-DSS, SOC2, HIPAA, ISO27001 — and adds `sec-compliance.md`. A shell run's record folder is under `~/.codeaf/v3/carried/sec/`, one folder per run. + +## An earlier audit's report — find it from another conversation, run it again, the last audit, start fresh + +A sec run in the chat keeps its report in that task's record folder, `~/.codeaf/v3/projects///tasks//`. Every conversation in the project can see the run in `tasks`: its row's outcome is the first line of its account, which says what it found — `Security audit of the whole repository: 14 problems, 9 confirmed and 5 likely (1 critical, 1 high, 12 medium).` — and its `artifact` is that record folder, where `sec-report.md` is. + +Running it again needs nothing cleared away. Each run starts from nothing in a new task folder of its own, reads the folder as it is on disk now, and never reads an earlier run's checkpoints or report. A run in the same conversation is the next task number; a new conversation starts its own. Comparing two runs is the chat's to do, from the two reports. + +## Does sec change my files — read only, no shell, will it edit or commit anything + +No. Its agents have four tools — read a file, list a folder, find files by pattern, search file contents — and nothing that writes; there is no shell, so it runs none of your code, tests or builds. Every path is held inside the folder it audits: a path or a link that leads out of it is refused. It makes no branch and no commit, and its working files (its checkpoints, its report) go to the task's record folder, never into your folder. The chat keeps working in the folder while it runs. + +## What a security audit costs and how long it takes — limit, ceiling, $5, four hours, quick, thorough + +A run nobody set a limit for stops at **$5 and four hours**, or sooner where this conversation has less left; `/budget conversation` can lower that, and at a shell `--max-cost` and `--max-hours` set either. The proposal card and the start line name the ceiling: `up to $5.00 and 4h`. + +What a run spends depends on the repository and the models: a `quick` audit of a small project is cents; a `standard` audit of a mid-sized one is commonly under a dollar or two on an inexpensive model, and a `thorough` one more. It keeps three minutes of its time ceiling back to write its report. **A run its time ceiling cuts still reports**, and says so at the top of its account and its report: `It reached its time ceiling of 4h during prove, so findings whose tests had not finished are marked unclear, and no fixes were written.` A large repository can need more than four hours at `standard`; give it `--max-hours` at a shell, or audit only the changes. A run that reaches its dollar ceiling before the report ends with `sec reached the run's dollar ceiling of $5.00: sec said it reached the run's dollar ceiling before it finished, so it has no report` (codeaf refused its next model call); one that reaches its time ceiling ends with `sec stopped on its own ceiling: it reached the run's time ceiling before it finished, so it has no report`. Either way the chat asks whether to spend more rather than running it again. + +**Which models.** From the chat, the conversation's working seat reads the code and its light seat makes the single quick calls (merging duplicates, the compliance mapping). A model you name for the hand-off reads the code in place of the working seat. At a shell, the profile's work seat does both unless `--model` (and `--light`) name others. + +## Fixing what sec found — hand the findings to senior-dev, fix the vulnerabilities + +When the audit finds confirmed or likely problems, the chat offers to hand them to senior-dev as one task that fixes them, and names which. It starts nothing until you say yes; then it proposes that task, and you answer its card as usual. You can also ask directly: "fix the two critical findings with senior-dev". + +## What sec cannot do — URLs, running the app, a CVE database, Windows, telling it something + +- **It audits a folder on this machine, never a URL.** To audit a repository you have not cloned, the chat clones it into a new folder first and audits that. +- **It runs nothing.** It does not start your app, send requests to it or run exploits; what it shows exploitable it shows from the code. +- **It looks up nothing online.** Known vulnerabilities in your dependencies come from what its models know, not from a live advisory database, so a very recent advisory can be missed. +- **It cannot be told anything while it runs.** It reads no messages; stop it and start it again with other words. +- **It is absent on Windows**, as every program codeaf carries is there. diff --git a/internal/manual/chat/tasks.md b/internal/manual/chat/tasks.md index 09e42d1b85..5ae28b65b9 100644 --- a/internal/manual/chat/tasks.md +++ b/internal/manual/chat/tasks.md @@ -2349,8 +2349,9 @@ $100.00 default. **An ordinary `/task`, or any task codeaf's own worker does, also has a money limit of its own: $5 unless you set another**, shown on the Spending tab's `per task` row and set with `/crew cap task <$>`. A call that would spend past it is not made, whichever way the ordinary task was started, and the task stops there. -**senior-dev has its own dollar and time ceilings for each run**, separate from that -`per task` row; see the senior-dev page for its defaults and how to change them. +**senior-dev and sec have their own dollar and time ceilings for each run**, +separate from that `per task` row; see each program's page for its defaults and how to +change them. Changing the engine's default changes the figure the composer layer opens on; the two are meant to be one number and are stated in both places on purpose. diff --git a/internal/manual/chat_test.go b/internal/manual/chat_test.go index be17600c79..b4e78a20cb 100644 --- a/internal/manual/chat_test.go +++ b/internal/manual/chat_test.go @@ -1066,6 +1066,20 @@ func TestTheChatManualAnswersTheQuestionsPeopleAsk(t *testing.T) { {"why can't the delegate ask me anything", "delegates"}, {"why is there no command for my delegate", "delegates"}, {"where does a delegate's work go, does it squash the commits", "delegates"}, + // sec, the auditing program codeaf carries, asked the way + // somebody wants a security review: by its job, not its name. + {"how do I run a security review of my repo", "sec"}, + {"can codeaf scan my code for vulnerabilities", "sec"}, + {"audit only my changes for security problems", "sec"}, + {"where is the security audit report and the sarif file", "sec"}, + {"will the security audit change or commit my files", "sec"}, + {"how much does a security audit cost", "sec"}, + {"fix the vulnerabilities the security audit found", "sec"}, + {"why does the sec task page show the same hunter lines over and over", "sec"}, + {"do I need to clear out the old security audit before running it again", "sec"}, + {"how many turns does each security audit agent get", "sec"}, + {"find the last security audit's report from another conversation", "sec"}, + {"the program finished but the chat said nothing, no summary", "delegates"}, // senior-dev, the program codeaf carries, asked the ways somebody meets // it: what the command does, whether it will stop to ask, where its // commits went, what it cost, its flags, and why a Windows build has none. @@ -1143,6 +1157,7 @@ func TestTheChatManualAnswersTheQuestionsPeopleAsk(t *testing.T) { {"why can't codeaf edit files while senior-dev is working", "senior-dev"}, {"my task was refused because senior-dev is working in the folder", "senior-dev"}, {"the delegate was refused because of uncommitted changes", "delegates"}, + {"I pressed stop while the chat was answering a program that ended", "delegates"}, {"the harness I just had built is not in /subharness", "subharnesses"}, {"how do I run a harness I had designed", "subharnesses"}, // The card codeaf raises by itself, asked the three ways somebody meets diff --git a/internal/programguide/sec.go b/internal/programguide/sec.go new file mode 100644 index 0000000000..7be0a85a45 --- /dev/null +++ b/internal/programguide/sec.go @@ -0,0 +1,9 @@ +package programguide + +// Sec is the guide sec gives the conversation. Its brief +// is read by the program itself (internal/secaf's scope reading), so the guide +// spells the brief's few words exactly as the program reads them. +const Sec = "For a security audit of code: the whole repository, or only the changes on this branch. " + + "It hunts for vulnerabilities, traces each from input to sink, sets aside what it cannot show exploitable, " + + "and reports the rest with file, line and fix; it changes no files. Its brief is `whole repository` or " + + "`changes` (`changes since ` for another base), optionally with `quick` or `thorough`." diff --git a/internal/provider/machinery.go b/internal/provider/machinery.go index 8a6737809a..f1d4b7a235 100644 --- a/internal/provider/machinery.go +++ b/internal/provider/machinery.go @@ -86,6 +86,25 @@ func machinerySymbol(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) && !unicode.IsSpace(r) && r != '_' } +// proseMarks are the symbols prose itself puts around a word: the separators +// of a path, the backtick of inline code, the asterisk of emphasis, and the +// punctuation of a sentence. A tool's name between two of them is a person's +// writing, not a grammar's token. +// +// IT WAS FOUND BY A PATH. On 2026-10-05 a conversation summarising sec's +// report — a table of findings, each at a backticked `file:line`, and the +// report's own path, `…/tasks/1/sec-report.md` — read as a tenth symbols and +// spelled the declared tool `tasks` between two slashes, and four models in a +// row were cut for writing it. A leak fences a name in brackets, bars and +// quotes (`｜_web_search>`, `"web_search"`), none of which is here. +const proseMarks = "/\\`*.,:;()'!?-" + +// fenceRune is a symbol that can wall a name in a tool grammar: any symbol +// but the marks prose uses ([proseMarks]). +func fenceRune(r rune) bool { + return machinerySymbol(r) && !strings.ContainsRune(proseMarks, r) +} + func symbolDensity(text string) float64 { total, symbols := 0, 0 for _, r := range text { @@ -118,7 +137,7 @@ func fencedInSymbols(text, name string) bool { // start too. before, sizeBefore := utf8.DecodeLastRuneInString(strings.TrimRight(text[:at], "_")) after, sizeAfter := utf8.DecodeRuneInString(strings.TrimLeft(text[at+len(name):], "_")) - if sizeBefore > 0 && sizeAfter > 0 && machinerySymbol(before) && machinerySymbol(after) { + if sizeBefore > 0 && sizeAfter > 0 && fenceRune(before) && fenceRune(after) { return true } from = at + len(name) diff --git a/internal/provider/machinery_test.go b/internal/provider/machinery_test.go index a8d1f2dde7..57377b3e37 100644 --- a/internal/provider/machinery_test.go +++ b/internal/provider/machinery_test.go @@ -193,3 +193,37 @@ func TestReplyGuardOffLetsTheLeakThrough(t *testing.T) { t.Fatal("no answer came back") } } + +// A TOOL'S NAME IN A PERSON'S WRITING IS NOT A LEAK, however dense the text +// around it: a path that runs through it, inline code, emphasis, a parenthesis. +// The fixture is the reply that was cut on 2026-10-05 — a table of findings +// and the report's path under `…/tasks/1/` — at a tenth symbols, spelling the +// declared tool `tasks` between two slashes. +func TestMachineryLeakLeavesAToolsNameInProseAlone(t *testing.T) { + request := &ai.Request{Tools: machineryTools("tasks", "read", "web_search")} + table := "Audit's done — full report at `/Users/a/.codeaf/v3/projects/-Users-a-Code-x/c33/tasks/1/sec-report.md`.\n\n" + + "| Severity | Finding | Where |\n|---|---|---|\n" + + "| Critical (likely) | `--merge-check` runs `/bin/sh -c` | `src/repository.rs:2344` |\n" + + "| High (confirmed) | SSH host into `ssh` (SSRF) | `src/remote.rs:131` |\n" + + "| Medium | `contents: write` token | `.github/workflows/release.yml:20` |\n" + if symbolDensity(table) < machineryDensityFloor { + t.Fatalf("the fixture is %.3f symbols, under the floor it has to be over to test anything", symbolDensity(table)) + } + for name, text := range map[string]string{ + "a path through the name": table, + "inline code": table + "Ask `tasks` for the rest.", + "emphasis": table + "See **tasks** above.", + "a parenthesis": table + "(read) it.", + } { + if MachineryLeak(request, machineryAnswer(text)) { + t.Errorf("%s was cut as machinery", name) + } + } + // And the leak's own shapes are still read as the leak. + if !MachineryLeak(request, machineryAnswer(leakedGrammar)) { + t.Fatal("the live leak was no longer read as machinery") + } + if !MachineryLeak(request, machineryAnswer(`[TOOL_REQUEST]{"name":"tasks","arguments":{}}[END_TOOL_REQUEST]`)) { + t.Fatal("a quoted tool name in bracketed grammar was no longer read as machinery") + } +} diff --git a/internal/provider/modelapi/server.go b/internal/provider/modelapi/server.go index fb5dc92fbd..e0f7c07d97 100644 --- a/internal/provider/modelapi/server.go +++ b/internal/provider/modelapi/server.go @@ -173,6 +173,9 @@ type Server struct { threads threads // refused counts the calls answered 402 at the ceiling. refused int + // held counts the calls asked to wait while others in flight held the + // ceiling ([held]). + held int // bankMu keeps charges in the order they were metered, one at a time, and // logMu keeps two turns from sharing one write of the log. @@ -498,40 +501,78 @@ func (r *record) close(fill func(turn *delegate.Turn)) delegate.Turn { return r.turn } +// admission is what the ceiling says to one call as it arrives. +type admission int + +const ( + // admitted is a call that fits: its estimate is reserved and it goes on. + admitted admission = iota + // refused is a call that could not fit even with nothing else in flight: + // the ceiling is reached, or its estimate alone would cross it. + refused + // held is a call that would fit once calls already in flight have ended + // and given back what they reserved. Asked again later it can go. + held +) + // open numbers one call, opens its turn with what the thread had not said // before, names the working the program handed back by the field its thread's // working last arrived on, and answers the run's spend at the moment the call // arrived — the figure its ceiling is asked against. -func (s *Server) open(request *call, thread, served, model string) (*record, float64, float64, bool) { +// +// A HELD CALL OPENS NO TURN. It was not made, and it will be asked again, so +// neither the log nor its thread's memory of what was last sent may count it: +// a thread that remembered a held request would record nothing new when the +// same request came back and was made. +func (s *Server) open(request *call, thread, served, model string) (*record, float64, float64, admission) { s.mu.Lock() defer s.mu.Unlock() - s.seq++ - entry := &record{turn: delegate.Turn{Seq: s.seq, Thread: thread, Started: time.Now(), Model: request.asked, Served: served}} - entry.turn.Sent, entry.turn.Restarted = s.threads.delta(thread, request.messages) - request.reasoning = s.threads.name(thread, request.reasoning) ceiling := s.config.Ceiling - if ceiling <= 0 { - s.inflight++ - return entry, s.spent, 0, true - } - reserve, known := s.estimate(request, model) - if !known { - // An unpriced local or custom model can still spend real money. Once - // half the ceiling is spent, admit at most one such call in flight. - reserve = ceiling / 2 - if left := ceiling - s.spent; reserve > left { - reserve = left + reserve := 0.0 + if ceiling > 0 { + var known bool + reserve, known = s.estimate(request, model) + if !known { + // An unpriced local or custom model can still spend real money. Once + // half the ceiling is spent, admit at most one such call in flight. + reserve = ceiling / 2 + if left := ceiling - s.spent; reserve > left { + reserve = left + } + } + switch { + case ceilingReached(ceiling, s.spent) || reserve > ceiling-s.spent+ceilingDust: + s.refused++ + return s.openTurn(request, thread, served), s.spent, 0, refused + case ceilingReached(ceiling, s.spent+s.reserved) || + (!known && s.spent >= ceiling/2 && s.inflight > 0) || + reserve > ceiling-s.spent-s.reserved+ceilingDust: + s.held++ + return nil, s.spent, 0, held } } - if ceilingReached(ceiling, s.spent+s.reserved) || - (!known && s.spent >= ceiling/2 && s.inflight > 0) || - reserve > ceiling-s.spent-s.reserved+ceilingDust { - s.refused++ - return entry, s.spent, 0, false - } + entry := s.openTurn(request, thread, served) s.reserved += reserve s.inflight++ - return entry, s.spent, reserve, true + return entry, s.spent, reserve, admitted +} + +// openTurn numbers one call that is being answered — made, or refused at the +// ceiling — and opens its turn. The caller holds s.mu. +func (s *Server) openTurn(request *call, thread, served string) *record { + s.seq++ + entry := &record{turn: delegate.Turn{Seq: s.seq, Thread: thread, Started: time.Now(), Model: request.asked, Served: served}} + entry.turn.Sent, entry.turn.Restarted = s.threads.delta(thread, request.messages) + request.reasoning = s.threads.name(thread, request.reasoning) + return entry +} + +// HeldAtCeiling is how many calls were asked to wait because calls in flight +// held what was left of the ceiling. +func (s *Server) HeldAtCeiling() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.held } // defaultOutputCap is the conservative output allowance used for a request @@ -645,9 +686,23 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, request *call) { if thread == "" { thread = delegate.MainThread } - entry, spent, reserved, admitted := s.open(request, thread, served, model) - - if !admitted { + entry, spent, reserved, verdict := s.open(request, thread, served, model) + + if verdict == held { + // 429 AND A TIME TO ASK AGAIN: what is left of the ceiling is held by + // calls still in flight, and once one ends this call fits. That IS + // weather — the one kind of "no" the ceiling gives that changes on its + // own — so it is said in the status every program's client already + // retries, with the header that says which weather it is. A program that + // makes one call at a time never meets it; one that makes many at once + // (sec) would otherwise be told its ceiling was reached at $0 + // spent. + w.Header().Set("Retry-After", "2") + w.Header().Set(HeldHeader, "ceiling") + writeError(w, http.StatusTooManyRequests, heldSentence(s.config.Ceiling, spent)) + return + } + if verdict == refused { // 402 AND NOTHING THAT READS AS PASSING: a program's client retries a // 408, a 409, a 429 and a 5xx as the weather, and a ceiling is not // weather — asked again it answers the same. @@ -1107,6 +1162,18 @@ func ceilingSentence(ceiling, spent float64) string { return "the run's dollar ceiling of " + dollars(ceiling) + " is reached (" + dollars(spent) + " spent), so codeaf made no call" } +// HeldHeader marks a 429 the ceiling gave because calls in flight hold what is +// left of it ([held]): a program that makes calls at once waits and asks again, +// for as long as its own bounds allow, where any other 429 is retried a few +// times at most. +const HeldHeader = "X-Codeaf-Held" + +// heldSentence is what a held call is told. +func heldSentence(ceiling, spent float64) string { + return "the run's dollar ceiling of " + dollars(ceiling) + " is held by calls in flight (" + dollars(spent) + + " spent); ask again when one has ended" +} + // ceilingDust is the most a ceiling may still have left and be reached: a // billionth of a dollar, far below any call's price and far above the float // rounding in a sum of prices. diff --git a/internal/provider/modelapi/server_test.go b/internal/provider/modelapi/server_test.go index baaa057baa..40597d2007 100644 --- a/internal/provider/modelapi/server_test.go +++ b/internal/provider/modelapi/server_test.go @@ -338,15 +338,18 @@ func TestConcurrentCallsReserveTheCeilingBeforeForwarding(t *testing.T) { close(answers) refused := 0 for got := range answers { - if got.status == http.StatusPaymentRequired { + // Refused at the ceiling (402) or held behind the calls in flight + // (429): either way the call was not made, and the answer names the + // ceiling. + if got.status == http.StatusPaymentRequired || got.status == http.StatusTooManyRequests { refused++ if !strings.Contains(string(got.payload), "the run's dollar ceiling of $1.00") { - t.Fatalf("402 did not name the ceiling: %s", got.payload) + t.Fatalf("%d did not name the ceiling: %s", got.status, got.payload) } } } - if spent := server.Spent(); spent > 1.40 || server.RefusedAtCeiling() == 0 { - t.Fatalf("eight calls spent $%.2f with %d refusals", spent, server.RefusedAtCeiling()) + if spent := server.Spent(); spent > 1.40 || server.RefusedAtCeiling()+server.HeldAtCeiling() == 0 { + t.Fatalf("eight calls spent $%.2f with %d refusals and %d held", spent, server.RefusedAtCeiling(), server.HeldAtCeiling()) } if refused == 0 { t.Fatal("all eight calls were forwarded") @@ -439,7 +442,9 @@ func TestPendingReceiptKeepsItsReservationUntilTheRealChargeArrives(t *testing.T if status, payload := post(t, api, api.Token, body); status != http.StatusOK { t.Fatalf("first call got %d: %s", status, payload) } - if status, payload := post(t, api, api.Token, body); status != http.StatusPaymentRequired { + // HELD, NOT REFUSED: the first call's reservation is all that stands in the + // way, and it is given back when its charge arrives. + if status, payload := post(t, api, api.Token, body); status != http.StatusTooManyRequests || !strings.Contains(string(payload), "is held by calls in flight") { t.Fatalf("pending receipt admitted a second call: %d %s", status, payload) } close(gate) @@ -1018,3 +1023,59 @@ func TestARunsCallsAskTheFunnelToSettleAnAnswerWithNoUsage(t *testing.T) { t.Fatal("the call's context does not ask the funnel to settle an answer with no usage block") } } + +// THE CEILING SAYS TWO KINDS OF NO. A call whose estimate alone crosses what is +// left is refused (402), and asked again it is refused again. A call that only +// waits on calls still in flight is held (429, Retry-After and the held +// header), opens no turn, and is made once they have ended. +func TestACallHeldByCallsInFlightIsAskedToWaitAndLaterMade(t *testing.T) { + release := make(chan struct{}) + var first atomic.Bool + calls := &script{reply: func(ctx context.Context, model string, _ []ai.Message, _ ai.Request) (*ai.Response, error) { + if first.CompareAndSwap(false, true) { + <-release + } + bill(ctx, model, 1, 1, 0, 0.10) + return saying(model, "ok"), nil + }} + server, api := open(t, modelapi.Config{CompleterFor: calls.completerFor, Ceiling: 1, + ModelPrice: func(string) (float64, float64, bool) { return 0, 0.60, true }, + }) + body := `{"model":"priced/model","messages":[{"role":"user","content":"hi"}],"max_completion_tokens":1}` + done := make(chan int, 1) + go func() { + status, _ := post(t, api, api.Token, body) + done <- status + }() + deadline := time.Now().Add(3 * time.Second) + for len(calls.calls()) == 0 && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + request, _ := http.NewRequest(http.MethodPost, modelapi.ChatURL(api.BaseURL), strings.NewReader(body)) + request.Header.Set("Authorization", "Bearer "+api.Token) + request.Header.Set("Content-Type", "application/json") + answer, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatal(err) + } + payload, _ := io.ReadAll(answer.Body) + answer.Body.Close() + if answer.StatusCode != http.StatusTooManyRequests || answer.Header.Get(modelapi.HeldHeader) != "ceiling" || + answer.Header.Get("Retry-After") == "" || !strings.Contains(string(payload), "is held by calls in flight") { + t.Fatalf("a call behind one in flight got %d %v: %s", answer.StatusCode, answer.Header, payload) + } + if server.HeldAtCeiling() != 1 || server.RefusedAtCeiling() != 0 { + t.Fatalf("held %d, refused %d", server.HeldAtCeiling(), server.RefusedAtCeiling()) + } + close(release) + if status := <-done; status != http.StatusOK { + t.Fatalf("the first call got %d", status) + } + if status, payload := post(t, api, api.Token, body); status != http.StatusOK { + t.Fatalf("the held call, asked again, got %d: %s", status, payload) + } + tooBig := `{"model":"priced/model","messages":[{"role":"user","content":"hi"}],"max_completion_tokens":2}` + if status, payload := post(t, api, api.Token, tooBig); status != http.StatusPaymentRequired { + t.Fatalf("a call whose estimate alone crosses what is left got %d: %s", status, payload) + } +} diff --git a/internal/run/delegateworker.go b/internal/run/delegateworker.go index c41483f500..7e347999fc 100644 --- a/internal/run/delegateworker.go +++ b/internal/run/delegateworker.go @@ -731,6 +731,10 @@ func (w *DelegateWorker) Run(ctx context.Context, task plandb.Task) (Report, err } childEnv := append(delegate.ChildEnv(api.API()), "SENIOR_DEV_IGNORED_AT_START="+w.setup.IgnoredFile, gitidentity.InputsEnv+"="+w.setup.InputsFile) + // THE TASK'S FOLDER IS THE PROGRAM'S RECORD FOLDER: a program that answers + // with a report leaves the report there, beside its conversation and its + // actions, rather than in the person's folder. + childEnv = append(childEnv, delegate.RecordsEnv(taskDir)...) // THE INBOX STARTS EMPTY ON EVERY LAUNCH. It lives in the task's folder, // which outlives a run, and a program reads it from its first line: a second // worker of the same task would otherwise be handed the last run's messages @@ -747,7 +751,10 @@ func (w *DelegateWorker) Run(ctx context.Context, task plandb.Task) (Report, err if brief == "" { brief = strings.TrimSpace(task.Title) } - if note := strings.TrimSpace(w.setup.BriefNote); note != "" { + // A PROGRAM WHOSE BRIEF IS WORDS READS NO NOTE ABOVE THEM: it reads its + // scope off the brief's first line (delegate.Delegate.Words), and the + // folder it works in is its --dir. + if note := strings.TrimSpace(w.setup.BriefNote); note != "" && !w.program.Words { brief = note + "\n\n" + brief } started = time.Now() diff --git a/internal/secaf/afx/bind.go b/internal/secaf/afx/bind.go new file mode 100644 index 0000000000..aa1b091326 --- /dev/null +++ b/internal/secaf/afx/bind.go @@ -0,0 +1,123 @@ +// Package afx holds the small ergonomics over the AgentField Go SDK that every +// reasoner handler, phase and orchestrator step in the SEC-AF port reuses: +// +// - Bind/ToMap — the map[string]any <-> typed struct boundary that stands in +// for pydantic's Model.model_validate / Model.model_dump; +// - Unwrap/AsMap — byte-exact ports of the private _unwrap/_as_dict helpers +// that SEC-AF applies to every `await router.call(...)` result; +// - DropNulls — the model_dump(exclude_none=True) filter; +// - WireNumbers — the int-vs-float distinction CPython's json.loads makes and +// Go's decoder does not (wire.go). +package afx + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" +) + +// Bind decodes a reasoner's untyped input map into a typed value T. +// +// Handlers registered with the SDK receive input as map[string]any. Bind +// round-trips that map through JSON (marshal then unmarshal into T), which +// mirrors how Python materializes a pydantic model from the request body: +// field-name matching is by the json struct tags (the exact snake_case pydantic +// field names), and any custom UnmarshalJSON on T runs — so a T whose +// UnmarshalJSON seeds non-zero pydantic defaults gets those defaults for keys +// absent from the input (the schemas package's default-seeding pattern). +// +// Plain encoding/json is deliberate: no json.Decoder/UseNumber. Numbers in the +// input map are already Go float64/int (they came from the SDK's own JSON +// decode, or from a Go caller), and re-marshaling then unmarshaling into the +// typed fields of T yields the correct concrete types without number-precision +// gymnastics. +// +// Bind is NOT the whole of `Model.model_validate(dict)`. Three pydantic +// behaviours have no encoding/json equivalent, and internal/phases' checked +// binders (BindRawFinding, BindVerifiedFinding, BindReconResult, …) supply all +// three on top of Bind — use those for anything arriving off a `.call` +// boundary: +// +// - a MISSING required field is a ValidationError in Python; here it keeps +// the Go zero value (or the seeded pydantic default); +// - an explicit NULL on a non-Optional field is a ValidationError in Python; +// here it is a no-op for scalars and ZEROES a slice/map/pointer, which also +// wipes the `[]` the schemas package seeded; +// - pydantic's LAX mode parses a string-encoded number into an int/float +// field (`start_line="10"` -> 10); json.Unmarshal answers +// UnmarshalTypeError. +func Bind[T any](input map[string]any) (T, error) { + var out T + b, err := json.Marshal(input) + if err != nil { + return out, fmt.Errorf("afx.Bind: marshal input: %w", err) + } + if err := json.Unmarshal(b, &out); err != nil { + return out, fmt.Errorf("afx.Bind: unmarshal into %T: %w", out, err) + } + return out, nil +} + +// ToMap is Bind's inverse: it renders a typed struct as the map[string]any +// shape the SDK's reasoner handlers (and Agent.Call) accept. Top-level +// exported fields become map entries keyed by their json tag, and the field +// VALUES stay typed — deliberately NOT a marshal->unmarshal round trip, which +// would decode nested values into plain Go maps and lose whatever their custom +// marshalers encode (ordered objects, the Timestamp wrapper's exact isoformat +// spelling). Keeping values typed lets Bind on the handler side — and the SDK's +// own workflow-event emitter — re-marshal them through the same custom +// marshalers, so ToMap -> Bind is lossless. +// +// The reasoner input structs are flat, fully json-tagged, and carry no +// omitempty (every key is emitted, so Bind-side default seeding never overrides +// a deliberately zero field); ToMap ignores omitempty accordingly. Anonymous +// EXPORTED struct fields without their own json tag are flattened the way +// encoding/json flattens them; an embedded field of UNEXPORTED type is skipped +// (encoding/json would promote its exported fields, but reflect refuses to read +// through an unexported field, and the port has no such struct). +func ToMap(v any) (map[string]any, error) { + rv := reflect.ValueOf(v) + for rv.Kind() == reflect.Pointer { + if rv.IsNil() { + return nil, fmt.Errorf("afx.ToMap: nil %T", v) + } + rv = rv.Elem() + } + if rv.Kind() != reflect.Struct { + return nil, fmt.Errorf("afx.ToMap: %T is not a struct", v) + } + out := make(map[string]any, rv.NumField()) + fillMap(out, rv) + return out, nil +} + +// fillMap writes rv's fields into out, recursing through untagged anonymous +// struct fields (encoding/json flattening). +func fillMap(out map[string]any, rv reflect.Value) { + rt := rv.Type() + for i := 0; i < rt.NumField(); i++ { + f := rt.Field(i) + if !f.IsExported() { + continue + } + name, _, _ := strings.Cut(f.Tag.Get("json"), ",") + if name == "-" { + continue + } + if name == "" { + if f.Anonymous { + fv := rv.Field(i) + for fv.Kind() == reflect.Pointer && !fv.IsNil() { + fv = fv.Elem() + } + if fv.Kind() == reflect.Struct { + fillMap(out, fv) + continue + } + } + name = f.Name + } + out[name] = rv.Field(i).Interface() + } +} diff --git a/internal/secaf/afx/bind_test.go b/internal/secaf/afx/bind_test.go new file mode 100644 index 0000000000..2d7befd508 --- /dev/null +++ b/internal/secaf/afx/bind_test.go @@ -0,0 +1,184 @@ +package afx + +import ( + "encoding/json" + "reflect" + "testing" +) + +// depthDefaulted models the schemas-package pattern: pydantic defaults that are +// not the Go zero value are seeded in UnmarshalJSON, so Bind on a map missing +// those keys still produces the Python default. +type depthDefaulted struct { + RepoPath string `json:"repo_path"` + Depth string `json:"depth"` + SeverityThreshold string `json:"severity_threshold"` + ScanTypes []string `json:"scan_types"` + MaxProvers *int `json:"max_provers"` +} + +func (d *depthDefaulted) UnmarshalJSON(b []byte) error { + type alias depthDefaulted + v := alias{ + Depth: "standard", + SeverityThreshold: "low", + ScanTypes: []string{"sast", "sca", "secrets", "config"}, + } + if err := json.Unmarshal(b, &v); err != nil { + return err + } + *d = depthDefaulted(v) + return nil +} + +// TestBindSeedsPydanticDefaults: Bind must run T's UnmarshalJSON so absent keys +// pick up the pydantic default rather than the Go zero value — the behaviour +// `AuditInput(**payload)` has in Python. +func TestBindSeedsPydanticDefaults(t *testing.T) { + got, err := Bind[depthDefaulted](map[string]any{"repo_path": "/tmp/repo"}) + if err != nil { + t.Fatalf("Bind: %v", err) + } + want := depthDefaulted{ + RepoPath: "/tmp/repo", + Depth: "standard", + SeverityThreshold: "low", + ScanTypes: []string{"sast", "sca", "secrets", "config"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("Bind = %#v, want %#v", got, want) + } +} + +// TestBindOverridesDefaults: keys present in the input win over the seed. +func TestBindOverridesDefaults(t *testing.T) { + got, err := Bind[depthDefaulted](map[string]any{ + "repo_path": "/tmp/repo", + "depth": "thorough", + "scan_types": []any{"sast"}, + "max_provers": 4, + }) + if err != nil { + t.Fatalf("Bind: %v", err) + } + if got.Depth != "thorough" { + t.Errorf("Depth = %q, want thorough", got.Depth) + } + if !reflect.DeepEqual(got.ScanTypes, []string{"sast"}) { + t.Errorf("ScanTypes = %#v", got.ScanTypes) + } + if got.MaxProvers == nil || *got.MaxProvers != 4 { + t.Errorf("MaxProvers = %#v, want 4", got.MaxProvers) + } +} + +// TestBindAcceptsJSONNumbers: reasoner inputs arrive over the control plane as +// JSON, so every number is a float64 by the time it reaches a handler. Bind +// must land those in int/float fields alike. +func TestBindAcceptsJSONNumbers(t *testing.T) { + type numeric struct { + Turns int `json:"turns"` + Cost float64 `json:"cost"` + } + got, err := Bind[numeric](map[string]any{"turns": float64(50), "cost": float64(1)}) + if err != nil { + t.Fatalf("Bind: %v", err) + } + if got.Turns != 50 || got.Cost != 1 { + t.Errorf("Bind = %#v, want {50 1}", got) + } +} + +// TestBindTypeMismatchIsAnError mirrors pydantic's ValidationError. +func TestBindTypeMismatchIsAnError(t *testing.T) { + type numeric struct { + Turns int `json:"turns"` + } + if _, err := Bind[numeric](map[string]any{"turns": "fifty"}); err == nil { + t.Fatal("Bind should reject a string in an int field") + } +} + +// TestToMapUsesJSONTagsAndKeepsValuesTyped. +func TestToMapUsesJSONTagsAndKeepsValuesTyped(t *testing.T) { + type inner struct { + N int `json:"n"` + } + type outer struct { + RepoPath string `json:"repo_path"` + Depth string `json:"depth"` + Nested inner `json:"nested"` + Tags []string `json:"tags"` + Skipped string `json:"-"` + unset string //nolint:unused // unexported fields are skipped + } + got, err := ToMap(outer{RepoPath: "/r", Depth: "quick", Nested: inner{N: 3}, Tags: []string{"a"}, Skipped: "x"}) + if err != nil { + t.Fatalf("ToMap: %v", err) + } + if got["repo_path"] != "/r" || got["depth"] != "quick" { + t.Errorf("ToMap = %#v", got) + } + if _, present := got["-"]; present { + t.Error(`ToMap emitted the json:"-" field`) + } + if _, present := got["Skipped"]; present { + t.Error(`ToMap emitted the json:"-" field under its Go name`) + } + if _, present := got["unset"]; present { + t.Error("ToMap emitted an unexported field") + } + // Values stay typed — not flattened into map[string]any. + if _, ok := got["nested"].(inner); !ok { + t.Errorf("ToMap flattened a nested struct: %T", got["nested"]) + } + if _, ok := got["tags"].([]string); !ok { + t.Errorf("ToMap changed a slice's type: %T", got["tags"]) + } +} + +// TestToMapFlattensEmbedded mirrors encoding/json's anonymous-field flattening. +func TestToMapFlattensEmbedded(t *testing.T) { + type Base struct { + A string `json:"a"` + } + type derived struct { + Base + B string `json:"b"` + } + got, err := ToMap(derived{Base: Base{A: "1"}, B: "2"}) + if err != nil { + t.Fatalf("ToMap: %v", err) + } + if got["a"] != "1" || got["b"] != "2" { + t.Errorf("ToMap = %#v, want flattened {a:1 b:2}", got) + } +} + +// TestToMapRejectsNonStructs. +func TestToMapRejectsNonStructs(t *testing.T) { + if _, err := ToMap(map[string]any{"a": 1}); err == nil { + t.Error("ToMap should reject a map") + } + var p *struct{ A int } + if _, err := ToMap(p); err == nil { + t.Error("ToMap should reject a nil pointer") + } +} + +// TestToMapThenBindRoundTrips: the phase code hands ToMap output to app.Call +// and the handler Binds it back. +func TestToMapThenBindRoundTrips(t *testing.T) { + in := depthDefaulted{RepoPath: "/r", Depth: "thorough", SeverityThreshold: "high", ScanTypes: []string{"sast"}} + m, err := ToMap(in) + if err != nil { + t.Fatalf("ToMap: %v", err) + } + back, err := Bind[depthDefaulted](m) + if err != nil { + t.Fatalf("Bind: %v", err) + } + if !reflect.DeepEqual(back, in) { + t.Errorf("round trip = %#v, want %#v", back, in) + } +} diff --git a/internal/secaf/afx/unwrap.go b/internal/secaf/afx/unwrap.go new file mode 100644 index 0000000000..ea84f6cbd0 --- /dev/null +++ b/internal/secaf/afx/unwrap.go @@ -0,0 +1,256 @@ +package afx + +import ( + "fmt" + "reflect" + + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" +) + +// Unwrap ports the private _unwrap helper that SEC-AF defines TWICE, verbatim +// and identically, in src/sec_af/app.py:59 and src/sec_af/reasoners/phases.py:34: +// +// def _unwrap(result: object, name: str) -> object: +// if isinstance(result, dict): +// if "error" in result and isinstance(result["error"], dict): +// message = result["error"].get("message") or result["error"].get("detail") or str(result["error"]) +// raise RuntimeError(f"{name} failed: {message}") +// if "output" in result: +// return result["output"] +// if "result" in result: +// return result["result"] +// return result +// +// It is applied to the return value of every `await router.call(f"{NODE_ID}.x", ...)` +// before _as_dict/model_validate. Both call sites are byte-identical in THIS +// repo, so there is a single Go function; cloudsecurity-af carries a stricter +// variant (it additionally fails on an "error_message" key and on +// status in ("failed", "error")) which deliberately does NOT exist here — do +// not add it. +// +// Parity details worth keeping in mind: +// +// - The error branch fires only when result["error"] is itself a dict. A +// string "error" value falls through to the "output"/"result" lookups. +// - `a or b or c` is PYTHON truthiness, not a nil check: an empty-string +// message falls through to "detail", and an empty detail falls through to +// str(the whole error dict). +// - The "output"/"result" lookups test key PRESENCE, not truthiness, so a +// present-but-null "output" unwraps to nil (and then trips AsMap). +// +// Python parity divergence: the str(error_dict) fallback renders a Python dict +// repr. Python dicts iterate in insertion order; a Go map cannot, so +// pyfmt.Str sorts the keys to stay deterministic. A numeric value inside that +// dict also renders as a float ({'code': 500.0}) rather than an int, because +// encoding/json decodes every JSON number to float64 — see PyTypeName for the +// same caveat. Both only affect the text of an error raised for a malformed +// error envelope that carried neither a "message" nor a "detail". +func Unwrap(raw any, name string) (any, error) { + m, ok := raw.(map[string]any) + if !ok { + return raw, nil + } + + if errVal, present := m["error"]; present { + if errMap, isDict := errVal.(map[string]any); isDict { + message := errMap["message"] + if !pyTruthy(message) { + message = errMap["detail"] + } + if !pyTruthy(message) { + return nil, fmt.Errorf("%s failed: %s", name, pyfmt.Str(errMap)) + } + return nil, fmt.Errorf("%s failed: %s", name, pyfmt.Str(message)) + } + } + if v, present := m["output"]; present { + return v, nil + } + if v, present := m["result"]; present { + return v, nil + } + return raw, nil +} + +// AsMap ports _as_dict (src/sec_af/app.py:71, src/sec_af/reasoners/phases.py:46): +// +// def _as_dict(payload: object, name: str) -> dict[str, Any]: +// if not isinstance(payload, dict): +// raise RuntimeError(f"{name} returned non-dict payload: {type(payload).__name__}") +// return payload +// +// The error text is reproduced exactly, including the Python type NAME of the +// offending payload (see PyTypeName). +// +// A NIL map[string]any is rejected as NoneType, not accepted as `{}`. That case +// is not hypothetical: the SDK's `agent.Call` returned `(nil, nil)` for a +// SUCCEEDED execution whose result was null, and the in-process Call that +// replaced it (internal/secaf/audit) keeps that shape on purpose. Boxed into an `any` that is a TYPED nil, so `payload.(map[string]any)` +// succeeds with ok=true and m=nil. Python's `Agent.call` hands back `None` for +// the same execution, and `_as_dict(None, name)` raises +// `RuntimeError(" returned non-dict payload: NoneType")` — so without +// this guard a null `.call` result would bind to a default-seeded model +// (BindHuntResult(nil) yields an EMPTY HuntResult with no error) and the audit +// would answer 200 with zero findings where Python answers 500. +// +// An EMPTY BUT NON-NIL map is Python's `{}` and is accepted: encoding/json +// produces a non-nil map for a `{}` body and leaves the map nil for `null`, so +// the two stay distinguishable. +func AsMap(payload any, name string) (map[string]any, error) { + m, ok := payload.(map[string]any) + if !ok { + return nil, fmt.Errorf("%s returned non-dict payload: %s", name, PyTypeName(payload)) + } + if m == nil { + return nil, fmt.Errorf("%s returned non-dict payload: NoneType", name) + } + return m, nil +} + +// PyTypeName renders `type(x).__name__` for a value that came off the wire as +// JSON, which is the only place SEC-AF's error strings expose a type name. +// +// Go Python +// map[string]any dict +// []any / array list +// string str +// float32/64 float +// int kinds int +// bool bool +// nil NoneType +// +// Python parity divergence: Go's encoding/json decodes EVERY JSON number to +// float64, so a bare integral payload reports "float" where CPython's +// json.loads would have produced an int and reported "int". Unreachable in +// practice — the payloads _as_dict guards are reasoner result envelopes — and +// it only changes the text of an error raised on a malformed payload. +func PyTypeName(v any) string { + switch v.(type) { + case nil: + return "NoneType" + case bool: + return "bool" + case string: + return "str" + case float32, float64: + return "float" + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: + return "int" + } + rv := reflect.ValueOf(v) + switch rv.Kind() { + case reflect.Map: + return "dict" + case reflect.Slice, reflect.Array: + return "list" + case reflect.String: + return "str" + case reflect.Bool: + return "bool" + case reflect.Float32, reflect.Float64: + return "float" + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return "int" + case reflect.Pointer, reflect.Interface: + if rv.IsNil() { + return "NoneType" + } + return PyTypeName(rv.Elem().Interface()) + } + return reflect.TypeOf(v).Name() +} + +// DropNulls reproduces pydantic's model_dump(exclude_none=True) over a decoded +// JSON value: every mapping entry whose value is None disappears, recursively, +// everywhere in the tree. +// +// Two deliberate scoping rules, both matching pydantic: +// +// - Only MAPPING entries are dropped. A None element inside a list survives, +// because exclude_none is a field-level filter, not a value filter — +// model_dump of `list[str | None]` containing None keeps the None. +// - An empty container is not None and survives ([] and {} are kept). +// +// A nil Go POINTER counts as None alongside the untyped nil interface, so the +// function is correct whether it is handed a json.Unmarshal result (where None +// is always the untyped nil) or a hand-built map. A nil SLICE or MAP does NOT +// count: those stand for empty pydantic collections (`Field(default_factory=list)`), +// which model_dump emits as [] / {} rather than dropping. +// +// The input is never mutated; fresh maps and slices are returned. +func DropNulls(v any) any { + switch x := v.(type) { + case map[string]any: + out := make(map[string]any, len(x)) + for k, val := range x { + if isPyNone(val) { + continue + } + out[k] = DropNulls(val) + } + return out + case []any: + out := make([]any, len(x)) + for i, e := range x { + out[i] = DropNulls(e) + } + return out + default: + return v + } +} + +// isPyNone reports whether val is Python's None: the untyped nil interface or a +// nil pointer. See DropNulls for why nil slices/maps are excluded. +func isPyNone(val any) bool { + if val == nil { + return true + } + rv := reflect.ValueOf(val) + return rv.Kind() == reflect.Pointer && rv.IsNil() +} + +// pyTruthy reproduces Python's `bool(x)` for the value kinds a decoded JSON +// document can hold, which is what the `a or b or c` chain inside _unwrap +// actually tests. Falsy: None, False, 0, 0.0, "", [], {}. Everything else is +// truthy. +func pyTruthy(v any) bool { + switch x := v.(type) { + case nil: + return false + case bool: + return x + case string: + return x != "" + case float64: + return x != 0 + case float32: + return x != 0 + case int: + return x != 0 + case int64: + return x != 0 + } + rv := reflect.ValueOf(v) + switch rv.Kind() { + case reflect.String: + return rv.Len() != 0 + case reflect.Slice, reflect.Array, reflect.Map: + return rv.Len() != 0 + case reflect.Bool: + return rv.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return rv.Int() != 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return rv.Uint() != 0 + case reflect.Float32, reflect.Float64: + return rv.Float() != 0 + case reflect.Pointer, reflect.Interface: + if rv.IsNil() { + return false + } + return pyTruthy(rv.Elem().Interface()) + } + return true +} diff --git a/internal/secaf/afx/unwrap_test.go b/internal/secaf/afx/unwrap_test.go new file mode 100644 index 0000000000..58356b12ef --- /dev/null +++ b/internal/secaf/afx/unwrap_test.go @@ -0,0 +1,272 @@ +package afx + +import ( + "reflect" + "testing" +) + +// Ground truth for every expectation below was produced by running the exact +// Python bodies of _unwrap / _as_dict (src/sec_af/app.py:59,71 and +// src/sec_af/reasoners/phases.py:34,46) under +// ~/.agentfield/packages/sec-af/venv/bin/python and printing str(exc). + +// TestUnwrapErrorMessages pins the RuntimeError text of the error branch, +// including the `message or detail or str(error)` truthiness cascade. +func TestUnwrapErrorMessages(t *testing.T) { + cases := []struct { + name string + raw map[string]any + target string + wantErr string + }{ + { + // python: _unwrap({"error": {"message": "boom"}}, "recon_phase") + name: "message wins", + raw: map[string]any{"error": map[string]any{"message": "boom"}}, + target: "recon_phase", + wantErr: "recon_phase failed: boom", + }, + { + // python: _unwrap({"error": {"detail": "detail-msg"}}, "hunt_phase") + name: "detail when message absent", + raw: map[string]any{"error": map[string]any{"detail": "detail-msg"}}, + target: "hunt_phase", + wantErr: "hunt_phase failed: detail-msg", + }, + { + // python: _unwrap({"error": {"message": "", "detail": "fallback"}}, "prove_phase") + // The empty message is FALSY, so `or` falls through to detail. + name: "empty message falls through to detail", + raw: map[string]any{"error": map[string]any{"message": "", "detail": "fallback"}}, + target: "prove_phase", + wantErr: "prove_phase failed: fallback", + }, + { + // python: _unwrap({"error": {"code": 500}}, "run_verifier") + name: "neither key -> str(error dict)", + raw: map[string]any{"error": map[string]any{"code": 500}}, + target: "run_verifier", + wantErr: "run_verifier failed: {'code': 500}", + }, + { + // python: _unwrap({"error": {"message": {"a": 1}}}, "run_deduplicator") + // A non-string message still goes through str(). + name: "dict message renders as a python dict repr", + raw: map[string]any{"error": map[string]any{"message": map[string]any{"a": 1}}}, + target: "run_deduplicator", + wantErr: "run_deduplicator failed: {'a': 1}", + }, + { + // python: _unwrap({"error": {"message": None, "detail": None}}, "x") + // -> "x failed: {'message': None, 'detail': None}" + // Go parity divergence: a Go map has no insertion order, so + // pyfmt.Str sorts the keys. Same content, deterministic order. + name: "null message and detail -> sorted dict repr", + raw: map[string]any{"error": map[string]any{"message": nil, "detail": nil}}, + target: "x", + wantErr: "x failed: {'detail': None, 'message': None}", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, err := Unwrap(c.raw, c.target) + if err == nil { + t.Fatalf("Unwrap returned (%v, nil), want an error", got) + } + if err.Error() != c.wantErr { + t.Errorf("Unwrap error = %q, python = %q", err.Error(), c.wantErr) + } + }) + } +} + +// TestUnwrapSuccessPaths pins the non-error branches, exactly as the Python +// helper resolves them. +func TestUnwrapSuccessPaths(t *testing.T) { + cases := []struct { + name string + raw any + want any + }{ + // python: _unwrap({"output": {"a": 1}}, "x") -> {'a': 1} + {"output key", map[string]any{"output": map[string]any{"a": 1}}, map[string]any{"a": 1}}, + // python: _unwrap({"result": {"b": 2}}, "x") -> {'b': 2} + {"result key", map[string]any{"result": map[string]any{"b": 2}}, map[string]any{"b": 2}}, + // python: _unwrap({"output": None}, "x") -> None + // PRESENCE, not truthiness: a null output unwraps to None (and then + // trips _as_dict), it does not fall through to "result". + {"present-but-null output", map[string]any{"output": nil, "result": map[string]any{"b": 2}}, nil}, + // python: _unwrap({"output": {...}, "result": {...}}, "x") -> the output + {"output beats result", map[string]any{"output": map[string]any{"a": 1}, "result": map[string]any{"b": 2}}, map[string]any{"a": 1}}, + // python: _unwrap({"error": "plain-string", "output": {"a": 1}}, "x") -> {'a': 1} + // The error branch requires error to be a DICT. + {"string error is not the error branch", map[string]any{"error": "plain-string", "output": map[string]any{"a": 1}}, map[string]any{"a": 1}}, + // python: _unwrap({"a": 1}, "x") -> {'a': 1} (returned unchanged) + {"plain map passes through", map[string]any{"a": 1}, map[string]any{"a": 1}}, + // python: _unwrap([1, 2], "x") -> [1, 2] (non-dict returned unchanged) + {"non-map passes through", []any{1, 2}, []any{1, 2}}, + // python: _unwrap("txt", "x") -> 'txt' + {"string passes through", "txt", "txt"}, + {"nil passes through", nil, nil}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, err := Unwrap(c.raw, "x") + if err != nil { + t.Fatalf("Unwrap: unexpected error %v", err) + } + if !reflect.DeepEqual(got, c.want) { + t.Errorf("Unwrap = %#v, want %#v", got, c.want) + } + }) + } +} + +// TestAsMap pins the _as_dict error text, including the Python type name of the +// offending payload. +func TestAsMap(t *testing.T) { + if got, err := AsMap(map[string]any{"a": 1}, "run_verifier"); err != nil { + t.Fatalf("AsMap on a dict: %v", err) + } else if !reflect.DeepEqual(got, map[string]any{"a": 1}) { + t.Errorf("AsMap returned %#v", got) + } + + cases := []struct { + in any + wantErr string + }{ + // python: _as_dict(None, "run_verifier") + {nil, "run_verifier returned non-dict payload: NoneType"}, + // The shape agent.Call actually produces for a SUCCEEDED execution + // whose result is `null`: a TYPED nil map boxed in an interface, which + // `payload.(map[string]any)` accepts with ok=true. Python's Agent.call + // returns None there, so _as_dict must reject it as NoneType — an + // empty-but-non-nil map (Python's `{}`) is a different value and is + // accepted above. + {map[string]any(nil), "run_verifier returned non-dict payload: NoneType"}, + // python: _as_dict([1, 2], "run_verifier") + {[]any{1, 2}, "run_verifier returned non-dict payload: list"}, + // python: _as_dict("s", "run_verifier") + {"s", "run_verifier returned non-dict payload: str"}, + // python: _as_dict(5, "run_verifier") + {5, "run_verifier returned non-dict payload: int"}, + // python: _as_dict(5.0, "run_verifier") + {5.0, "run_verifier returned non-dict payload: float"}, + // python: _as_dict(True, "run_verifier") + {true, "run_verifier returned non-dict payload: bool"}, + } + for _, c := range cases { + _, err := AsMap(c.in, "run_verifier") + if err == nil { + t.Errorf("AsMap(%#v): want an error", c.in) + continue + } + if err.Error() != c.wantErr { + t.Errorf("AsMap(%#v) error = %q, python = %q", c.in, err.Error(), c.wantErr) + } + } +} + +// TestPyTypeName covers the mapping table directly, including the documented +// float64 divergence for JSON integers. +func TestPyTypeName(t *testing.T) { + cases := []struct { + in any + want string + }{ + {nil, "NoneType"}, + {map[string]any{}, "dict"}, + {map[string]string{}, "dict"}, + {[]any{}, "list"}, + {[]string{}, "list"}, + {"s", "str"}, + {true, "bool"}, + {1, "int"}, + {int64(1), "int"}, + {1.5, "float"}, + // Python parity divergence: encoding/json produces float64 for every + // JSON number, so an integral JSON value reports "float" here where + // CPython's json.loads would have made it an int. + {float64(5), "float"}, + } + for _, c := range cases { + if got := PyTypeName(c.in); got != c.want { + t.Errorf("PyTypeName(%#v) = %q, want %q", c.in, got, c.want) + } + } + + var p *int + if got := PyTypeName(p); got != "NoneType" { + t.Errorf("PyTypeName((*int)(nil)) = %q, want NoneType", got) + } +} + +// TestDropNulls covers model_dump(exclude_none=True) semantics. +func TestDropNulls(t *testing.T) { + in := map[string]any{ + "kept": "v", + "dropped": nil, + "zero": 0, + "empty_str": "", + "empty_list": []any{}, + "empty_map": map[string]any{}, + "nested": map[string]any{ + "a": nil, + "b": 1, + }, + "list_of_maps": []any{ + map[string]any{"x": nil, "y": 2}, + nil, // a None list ELEMENT survives: exclude_none is field-level + }, + } + want := map[string]any{ + "kept": "v", + "zero": 0, + "empty_str": "", + "empty_list": []any{}, + "empty_map": map[string]any{}, + "nested": map[string]any{ + "b": 1, + }, + "list_of_maps": []any{ + map[string]any{"y": 2}, + nil, + }, + } + + got := DropNulls(in) + if !reflect.DeepEqual(got, want) { + t.Errorf("DropNulls =\n %#v\nwant\n %#v", got, want) + } + + // The input must be untouched. + if _, still := in["dropped"]; !still { + t.Error("DropNulls mutated its input") + } + if _, still := in["nested"].(map[string]any)["a"]; !still { + t.Error("DropNulls mutated a nested input map") + } +} + +// TestDropNullsTypedNilPointer: a nil Go pointer is Python's None too. +func TestDropNullsTypedNilPointer(t *testing.T) { + var p *string + got := DropNulls(map[string]any{"p": p, "q": "x"}).(map[string]any) + if _, present := got["p"]; present { + t.Error("DropNulls kept a nil pointer entry") + } + if got["q"] != "x" { + t.Errorf("DropNulls dropped a non-nil entry: %#v", got) + } +} + +// TestDropNullsScalarPassthrough: a non-container input is returned unchanged. +func TestDropNullsScalarPassthrough(t *testing.T) { + for _, v := range []any{"s", 1, 1.5, true, nil} { + if got := DropNulls(v); !reflect.DeepEqual(got, v) { + t.Errorf("DropNulls(%#v) = %#v", v, got) + } + } +} diff --git a/internal/secaf/afx/wire.go b/internal/secaf/afx/wire.go new file mode 100644 index 0000000000..bc2a6202ea --- /dev/null +++ b/internal/secaf/afx/wire.go @@ -0,0 +1,56 @@ +package afx + +import ( + "bytes" + "encoding/json" +) + +// WireNumbers restores the int-vs-float distinction CPython's json.loads makes +// and Go's encoding/json does not. +// +// # Why it is needed +// +// SEC-AF's Python side reads a `.call` result with `json.loads`, which turns an +// integer LITERAL into an `int` and anything carrying '.', 'e' or 'E' into a +// `float`. Go's decoder — the one inside the SDK's Call — turns EVERY JSON +// number into a float64, so the distinction is gone by the time a handler sees +// the payload. +// +// For a value that is immediately bound into a typed struct that costs nothing: +// the target field's type decides the spelling. It costs something for a value +// that is stored UNTYPED and re-serialised later, because the writer then has +// only the float64 to go on and spells `2` as `2.0`. The audit result's +// `metadata["prove_drop_summary"]` is exactly that: app.py:205-208 copies the +// prove_phase payload's `drop_summary` straight into +// `SecurityAuditResult.metadata` (a `dict[str, object]`), and every number in +// it — `demoted_total`, each `by_reason` count — is an int in Python. +// +// # What it does +// +// It re-decodes the value through encoding/json with UseNumber, leaving every +// numeric leaf as a json.Number that carries its literal spelling. The marshal +// step is what recovers the int-ness: encoding/json writes an integral float64 +// as "2", not "2.0", so the re-decode yields json.Number("2"). +// +// DOCUMENTED RESIDUAL, one case. A payload that spells an integral value with +// an explicit fraction or exponent — `{"demoted_total": 2.0}` — is a float in +// CPython and prints "2.0", where this round trip flattens it to "2". The +// distinction is unrecoverable once the SDK has decoded the body, and no +// producer of `drop_summary` emits that spelling: reasoners/phases.py's +// `_track_drop` builds the counts with `+= 1`. +// +// On any marshal/decode failure the input is returned unchanged — a metadata +// entry spelled the Go way beats losing the entry. +func WireNumbers(v any) any { + b, err := json.Marshal(v) + if err != nil { + return v + } + dec := json.NewDecoder(bytes.NewReader(b)) + dec.UseNumber() + var out any + if err := dec.Decode(&out); err != nil { + return v + } + return out +} diff --git a/internal/secaf/afx/wire_test.go b/internal/secaf/afx/wire_test.go new file mode 100644 index 0000000000..436970b583 --- /dev/null +++ b/internal/secaf/afx/wire_test.go @@ -0,0 +1,73 @@ +package afx + +// Tests for WireNumbers. +// +// Validation contract (behaviour, from CPython's json.loads): an integer +// LITERAL decodes to an `int` and prints without a fraction, a fractional or +// exponent literal decodes to a `float`. Go's decoder makes both float64, which +// a later re-serialisation of an UNTYPED value spells "2.0" — the shape +// SecurityAuditResult.metadata carries (see internal/secaf/audit/run.go). + +import ( + "encoding/json" + "testing" +) + +func TestWireNumbersRestoresIntegerSpelling(t *testing.T) { + // The exact shape reasoners/phases.py's _track_drop produces, after the SDK + // has decoded it: every number a float64. + decoded := map[string]any{ + "demoted_total": float64(2), + "by_reason": map[string]any{"verifier_error": float64(2)}, + "findings": []any{map[string]any{"score": float64(0.5), "rank": float64(3)}}, + } + + raw, err := json.Marshal(WireNumbers(decoded)) + if err != nil { + t.Fatalf("marshal: %v", err) + } + want := `{"by_reason":{"verifier_error":2},"demoted_total":2,` + + `"findings":[{"rank":3,"score":0.5}]}` + if string(raw) != want { + t.Errorf("re-serialised\n got: %s\nwant: %s", raw, want) + } + + for _, value := range WireNumbers(decoded).(map[string]any) { + if _, isFloat := value.(float64); isFloat { + t.Errorf("a float64 survived: %#v", value) + } + } + if _, isNumber := WireNumbers(decoded).(map[string]any)["demoted_total"].(json.Number); !isNumber { + t.Error("demoted_total is not a json.Number") + } +} + +// TestWireNumbersKeepsNonNumbers proves it is a spelling fix and nothing else. +func TestWireNumbersKeepsNonNumbers(t *testing.T) { + for _, value := range []any{nil, "text", true, []any{}, map[string]any{}} { + raw, err := json.Marshal(WireNumbers(value)) + if err != nil { + t.Fatalf("marshal %#v: %v", value, err) + } + want, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal %#v: %v", value, err) + } + if string(raw) != string(want) { + t.Errorf("%#v round-tripped to %s, want %s", value, raw, want) + } + } +} + +// TestWireNumbersReturnsTheInputOnFailure: a value encoding/json cannot marshal +// must not be lost. +func TestWireNumbersReturnsTheInputOnFailure(t *testing.T) { + unmarshalable := map[string]any{"fn": func() {}} + got, ok := WireNumbers(unmarshalable).(map[string]any) + if !ok || len(got) != 1 { + t.Fatalf("WireNumbers = %#v, want the input unchanged", got) + } + if _, present := got["fn"]; !present { + t.Error("the unmarshalable entry was dropped") + } +} diff --git a/internal/secaf/agents/dedup/dedup.go b/internal/secaf/agents/dedup/dedup.go new file mode 100644 index 0000000000..a97c083091 --- /dev/null +++ b/internal/secaf/agents/dedup/dedup.go @@ -0,0 +1,645 @@ +// Package dedup ports src/sec_af/agents/dedup.py — the HUNT-phase +// deduplicator and chain correlator that runs between the hunters and the +// PROVE phase. +// +// The Python module does three things in sequence: +// +// 1. fingerprint dedup — collapse findings that share +// sha256("::")[:16], merging the loser's extra data into +// the winner (_merge_duplicate); +// 2. a semantic dedup pass — for every same-file/same-CWE pair, ask the LLM +// `.ai(schema=DuplicateCheck)` whether the two are the same root cause, +// all pairs in parallel (_deduplicate_with_ai); +// 3. chain correlation — seed candidate chains from a hardcoded CWE-pair +// table (_fallback_correlate) and hand them, with the finding list, to the +// harness for expansion (deduplicate_and_correlate). +// +// Everything in step 2 and 3 is best-effort: Python swallows every exception +// (a failed AI check is "not a duplicate", a failed harness call means "no +// chains" and falls back to the seeds), and the Go port reproduces that +// exactly rather than propagating errors. +package dedup + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "os" + "sort" + "strconv" + "strings" + "sync" + "time" + "unicode/utf8" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/aix" + "github.com/Agent-Field/codeaf/internal/secaf/harnessx" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// severityScore ports dedup.py _SEVERITY_SCORE. A severity outside the table +// scores 0, matching Python's `.get(sev, 0)` — and a Go map lookup for a +// missing key already yields 0, so the two agree without a helper. +var severityScore = map[schemas.Severity]int{ + schemas.SeverityCritical: 5, + schemas.SeverityHigh: 4, + schemas.SeverityMedium: 3, + schemas.SeverityLow: 2, + schemas.SeverityInfo: 1, +} + +// confidenceScore ports dedup.py _CONFIDENCE_SCORE. +var confidenceScore = map[schemas.Confidence]int{ + schemas.ConfidenceHigh: 3, + schemas.ConfidenceMedium: 2, + schemas.ConfidenceLow: 1, +} + +// chainPatterns ports dedup.py _CHAIN_PATTERNS: ordered (first, second) CWE +// pairs that seed heuristic attack chains. Order is load-bearing — the seed +// chains are emitted in this order and reach the prompt. +var chainPatterns = [][2]string{ + {"CWE-918", "CWE-798"}, + {"CWE-862", "CWE-285"}, + {"CWE-89", "CWE-200"}, + {"CWE-16", "CWE-798"}, +} + +// aiDuplicateCheckTimeout ports the `timeout_seconds: float = 60.0` default of +// _ai_check_duplicate's asyncio.wait_for. +const aiDuplicateCheckTimeout = 60 * time.Second + +// chainCorrelationTimeout ports the `timeout=600.0` on the chain-correlation +// harness call in deduplicate_and_correlate. +const chainCorrelationTimeout = 600 * time.Second + +// ComputeFingerprint ports src/sec_af/agents/dedup.py compute_fingerprint: +// +// key = f"{finding.file_path}:{finding.start_line}:{finding.cwe_id}" +// return hashlib.sha256(key.encode("utf-8")).hexdigest()[:16] +// +// The 16-character prefix is of the lowercase hex digest, i.e. the first 8 +// bytes of the digest. +func ComputeFingerprint(finding schemas.RawFinding) string { + key := finding.FilePath + ":" + strconv.Itoa(finding.StartLine) + ":" + finding.CweID + sum := sha256.Sum256([]byte(key)) + return hex.EncodeToString(sum[:])[:16] +} + +// confidenceValue ports _confidence_value. +func confidenceValue(finding *schemas.RawFinding) int { + return confidenceScore[finding.Confidence] +} + +// severityConfidenceLess reports whether a sorts BEFORE b under Python's +// +// final.sort(key=_severity_confidence_sort_key, reverse=True) +// +// i.e. the tuple (severity_score, confidence_score) compared lexicographically, +// descending. Callers must use a STABLE sort: Python's list.sort is stable and +// `reverse=True` is documented to preserve the original order of equal +// elements (it reverses the comparison, not the result). +func severityConfidenceLess(a, b *schemas.RawFinding) bool { + as, bs := severityScore[a.EstimatedSeverity], severityScore[b.EstimatedSeverity] + if as != bs { + return as > bs + } + return confidenceScore[a.Confidence] > confidenceScore[b.Confidence] +} + +// mergeDuplicate ports _merge_duplicate. +// +// Python parity — MUTATION IS THE POINT. The Python helper mutates the winning +// pydantic model in place and returns it; the caller's `deduped` list holds the +// very same object, so the merge is visible to every later step even though the +// caller only ever reassigns a dict entry. The Go port therefore threads +// *schemas.RawFinding throughout (and DeduplicateAndCorrelate takes pointers +// into the caller's slice) so the same aliasing holds. +func mergeDuplicate(existing, incoming *schemas.RawFinding) *schemas.RawFinding { + winner := existing + loser := incoming + if confidenceValue(incoming) > confidenceValue(existing) { + winner = incoming + loser = existing + } + + // Python parity: len() on a str counts CODE POINTS, not bytes. + if utf8.RuneCountInString(loser.Description) > utf8.RuneCountInString(winner.Description) { + winner.Description = loser.Description + } + + winner.RelatedFiles = sortedUnion(winner.RelatedFiles, loser.RelatedFiles) + + // Python parity: `data_flow` is `list[DataFlowStep] | None`, so the test is + // None-vs-not-None. A nil Go slice is the None; an EMPTY non-nil slice is + // Python's `[]`, which is not None and therefore does NOT get overwritten. + if winner.DataFlow == nil && loser.DataFlow != nil { + winner.DataFlow = loser.DataFlow + } + + // Python: `winner.fingerprint = winner.fingerprint or compute_fingerprint(winner)`. + if winner.Fingerprint == "" { + winner.Fingerprint = ComputeFingerprint(*winner) + } + return winner +} + +// sortedUnion ports `sorted(set(a) | set(b))`. Go's sort.Strings orders by +// bytes, which for UTF-8 is the same order as Python's code-point comparison. +func sortedUnion(a, b []string) []string { + seen := make(map[string]struct{}, len(a)+len(b)) + out := make([]string, 0, len(a)+len(b)) + for _, s := range a { + if _, dup := seen[s]; !dup { + seen[s] = struct{}{} + out = append(out, s) + } + } + for _, s := range b { + if _, dup := seen[s]; !dup { + seen[s] = struct{}{} + out = append(out, s) + } + } + sort.Strings(out) + return out +} + +// buildDuplicateCheckPrompt is the prompt _ai_check_duplicate hands to +// `app.ai(user=..., schema=DuplicateCheck)`. Extracted as a pure function so a +// golden test can compare it byte-for-byte against the Python f-string. +func buildDuplicateCheckPrompt(candidate, existing *schemas.RawFinding) string { + var b strings.Builder + b.WriteString("Determine if these two security findings are duplicates (same root cause).\n\n") + b.WriteString("Finding A:\n") + b.WriteString("- Title: " + candidate.Title + "\n") + b.WriteString("- CWE: " + candidate.CweID + " (" + candidate.CweName + ")\n") + b.WriteString("- File: " + candidate.FilePath + ":" + strconv.Itoa(candidate.StartLine) + "\n") + b.WriteString("- Description: " + runeSlice(candidate.Description, 200) + "\n\n") + b.WriteString("Finding B:\n") + b.WriteString("- Title: " + existing.Title + "\n") + b.WriteString("- CWE: " + existing.CweID + " (" + existing.CweName + ")\n") + b.WriteString("- File: " + existing.FilePath + ":" + strconv.Itoa(existing.StartLine) + "\n") + b.WriteString("- Description: " + runeSlice(existing.Description, 200)) + return b.String() +} + +// aiCheckDuplicate ports _ai_check_duplicate. +// +// Python parity: the whole call is wrapped in `except Exception: return False`, +// so a transport failure, a timeout, a malformed response — anything at all — +// means "not a duplicate". The Go port swallows the error identically and +// returns a bool, not (bool, error). +func aiCheckDuplicate(ctx context.Context, app appx.AIer, candidate, existing *schemas.RawFinding, timeout time.Duration) bool { + prompt := buildDuplicateCheckPrompt(candidate, existing) + + // Ports `asyncio.wait_for(app.ai(...), timeout=timeout_seconds)`. + callCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + result, err := aix.Structured[schemas.DuplicateCheck](callCtx, app, "", prompt) + if err != nil { + return false + } + return result.IsDuplicate +} + +// deduplicateWithAI ports _deduplicate_with_ai. +// +// findings are pointers into the caller's storage; the fingerprint seeding and +// every merge mutate them in place, exactly as the Python code mutates the +// caller's pydantic models. +// +// app is deliberately `any`: Python types this parameter `object` and probes +// `hasattr(app, "ai") and callable(...)` at runtime, so a harness-only app +// skips the semantic pass entirely. The Go equivalent is an optional-interface +// assertion to appx.AIer. +func deduplicateWithAI(ctx context.Context, findings []*schemas.RawFinding, app any) []*schemas.RawFinding { + // Python: `by_fingerprint: dict[str, RawFinding]`. A Python dict preserves + // insertion order and `deduped = list(by_fingerprint.values())` depends on + // it, so the Go port carries an explicit key order alongside the map. + byFingerprint := make(map[string]*schemas.RawFinding, len(findings)) + fpOrder := make([]string, 0, len(findings)) + for _, finding := range findings { + if finding.Fingerprint == "" { + finding.Fingerprint = ComputeFingerprint(*finding) + } + existing, ok := byFingerprint[finding.Fingerprint] + if !ok { + byFingerprint[finding.Fingerprint] = finding + fpOrder = append(fpOrder, finding.Fingerprint) + continue + } + // Python reassigns the SAME key, which keeps its original position. + byFingerprint[finding.Fingerprint] = mergeDuplicate(existing, finding) + } + + deduped := make([]*schemas.RawFinding, 0, len(fpOrder)) + for _, fp := range fpOrder { + deduped = append(deduped, byFingerprint[fp]) + } + + // Python: `by_file: dict[str, list[RawFinding]] = defaultdict(list)`, again + // iterated in insertion order. + byFile := make(map[string][]*schemas.RawFinding, len(deduped)) + fileOrder := make([]string, 0, len(deduped)) + for _, finding := range deduped { + if _, ok := byFile[finding.FilePath]; !ok { + fileOrder = append(fileOrder, finding.FilePath) + } + byFile[finding.FilePath] = append(byFile[finding.FilePath], finding) + } + + toRemove := make(map[string]struct{}) + + // Python: `has_ai = hasattr(app, "ai") and callable(getattr(app, "ai", None))`. + aiApp, hasAI := app.(appx.AIer) + if hasAI { + // Collect all candidate pairs up front so we can check them in parallel. + type pair struct{ candidate, existing *schemas.RawFinding } + var pairs []pair + for _, file := range fileOrder { + fileFindings := byFile[file] + if len(fileFindings) < 2 { + continue + } + for i, candidate := range fileFindings { + for _, existing := range fileFindings[i+1:] { + if candidate.CweID == existing.CweID { + pairs = append(pairs, pair{candidate, existing}) + } + } + } + } + + if len(pairs) > 0 { + // Ports `await asyncio.gather(*[_ai_check_duplicate(...) for ...])`: + // UNBOUNDED fan-out (no semaphore in Python), results collected + // positionally. _ai_check_duplicate never raises, so there is no + // error channel to model and a plain WaitGroup suffices — the + // gather's return_exceptions default is unreachable here. + results := make([]bool, len(pairs)) + var wg sync.WaitGroup + wg.Add(len(pairs)) + for i, p := range pairs { + go func(i int, p pair) { + defer wg.Done() + results[i] = aiCheckDuplicate(ctx, aiApp, p.candidate, p.existing, aiDuplicateCheckTimeout) + }(i, p) + } + wg.Wait() + + for i, p := range pairs { + if !results[i] { + continue + } + candidate, existing := p.candidate, p.existing + // Skip if either side was already removed by an earlier pair result. + if _, gone := toRemove[candidate.Fingerprint]; gone { + continue + } + if _, gone := toRemove[existing.Fingerprint]; gone { + continue + } + if confidenceValue(candidate) >= confidenceValue(existing) { + toRemove[existing.Fingerprint] = struct{}{} + // Python parity: the by_fingerprint reassignment cannot + // change what survives — `final` is filtered from `deduped`, + // which was snapshotted before this loop — but the merge it + // wraps mutates the shared finding, and that DOES survive. + byFingerprint[candidate.Fingerprint] = mergeDuplicate(candidate, existing) + } else { + toRemove[candidate.Fingerprint] = struct{}{} + byFingerprint[existing.Fingerprint] = mergeDuplicate(existing, candidate) + } + } + } + } + + final := make([]*schemas.RawFinding, 0, len(deduped)) + for _, f := range deduped { + if _, gone := toRemove[f.Fingerprint]; gone { + continue + } + final = append(final, f) + } + sort.SliceStable(final, func(i, j int) bool { return severityConfidenceLess(final[i], final[j]) }) + return final +} + +// fallbackCorrelate ports _fallback_correlate: the hardcoded CWE-pair seed +// chains. +func fallbackCorrelate(findings []*schemas.RawFinding) []schemas.PotentialChain { + // Python: `defaultdict(list)` keyed by the UPPERCASED cwe_id; only lookups + // happen afterwards, so insertion order is irrelevant here. + byCWE := make(map[string][]*schemas.RawFinding, len(findings)) + for _, finding := range findings { + up := strings.ToUpper(finding.CweID) + byCWE[up] = append(byCWE[up], finding) + } + + chains := make([]schemas.PotentialChain, 0, len(chainPatterns)) + for _, pattern := range chainPatterns { + firstCWE, secondCWE := pattern[0], pattern[1] + firstCandidates := byCWE[firstCWE] + secondCandidates := byCWE[secondCWE] + if len(firstCandidates) == 0 || len(secondCandidates) == 0 { + continue + } + first := firstCandidates[0] + second := secondCandidates[0] + + // Python: `max(a, b, key=lambda s: _SEVERITY_SCORE.get(s, 0))` returns + // the FIRST argument on a tie. + severity := first.EstimatedSeverity + if severityScore[second.EstimatedSeverity] > severityScore[first.EstimatedSeverity] { + severity = second.EstimatedSeverity + } + + chain := schemas.NewPotentialChain() // mints chain_id, like PotentialChain(...) + chain.Title = "Potential attack chain: " + firstCWE + " -> " + secondCWE + chain.FindingIDs = []string{first.ID, second.ID} + chain.CombinedImpact = "Combined exploitation path discovered by correlation heuristics; verify chain during PROVE phase." + chain.EstimatedSeverity = severity + chains = append(chains, chain) + } + return chains +} + +// seedChainContext ports _seed_chain_context: the block of prompt text that +// tells the harness which heuristic chains to validate and expand. +func seedChainContext(seedChains []schemas.PotentialChain, findings []*schemas.RawFinding) string { + // Python: `{finding.id: finding for finding in findings}` — a later finding + // with the same id wins. + byID := make(map[string]*schemas.RawFinding, len(findings)) + for _, finding := range findings { + byID[finding.ID] = finding + } + + lines := []string{"Seed chain candidates (validate and expand these):"} + if len(seedChains) == 0 { + lines = append(lines, "- No heuristic seed chains were detected from hardcoded CWE pairs.") + } else { + for _, chain := range seedChains { + orderedLabels := make([]string, 0, len(chain.FindingIDs)) + for _, findingID := range chain.FindingIDs { + finding, ok := byID[findingID] + if !ok { + continue + } + orderedLabels = append(orderedLabels, finding.CweName) + } + label := chain.Title + if len(orderedLabels) > 0 { + label = strings.Join(orderedLabels, " -> ") + } + lines = append(lines, "- Potential chain: "+label+" (findings "+strings.Join(chain.FindingIDs, ", ")+")") + } + } + lines = append(lines, "Look for additional multi-step attack chains beyond these seeds.") + return strings.Join(lines, "\n") +} + +// buildChainCorrelationPrompt is the prompt deduplicate_and_correlate hands to +// `app.harness(..., schema=ChainCorrelationResult, ...)`. Extracted as a pure +// function so a golden test can compare it byte-for-byte against Python. +func buildChainCorrelationPrompt(findings []*schemas.RawFinding, seedContext string) string { + summaryLines := make([]string, 0, len(findings)) + for _, f := range findings { + summaryLines = append(summaryLines, + "- id="+f.ID+" cwe="+f.CweID+" file="+f.FilePath+":"+strconv.Itoa(f.StartLine)+ + " title="+f.Title+" severity="+string(f.EstimatedSeverity)) + } + findingsSummary := strings.Join(summaryLines, "\n") + + return "You are SEC-AF's chain correlator.\n" + + "Identify multi-step attack chains across the findings below.\n" + + "A chain means one vulnerability enables exploitation of another.\n" + + "Also flag any remaining duplicate IDs that should be dropped.\n\n" + + "Findings:\n" + findingsSummary + "\n\n" + + seedContext +} + +// splitPipe ports _split_pipe: split on "|" with at most expected-1 splits, +// strip each part, then right-pad with "" to exactly expected entries. +func splitPipe(s string, expected int) []string { + parts := strings.SplitN(s, "|", expected) + out := make([]string, 0, expected) + for _, p := range parts { + // Python's str.strip() removes whitespace as defined by str.isspace(); + // strings.TrimSpace uses unicode.IsSpace. The two agree on every + // character that appears in LLM output. + out = append(out, strings.TrimSpace(p)) + } + for len(out) < expected { + out = append(out, "") + } + return out +} + +// parseChainFromStr ports _parse_chain_from_str. The bool reports Python's +// "not None" — a chain with fewer than two RESOLVABLE finding ids is dropped. +func parseChainFromStr(entry string, availableIDs map[string]struct{}) (schemas.PotentialChain, bool) { + parts := splitPipe(entry, 4) + title := parts[0] + + validIDs := []string{} + for _, fid := range strings.Split(parts[1], ",") { + fid = strings.TrimSpace(fid) + if fid == "" { + continue + } + if _, ok := availableIDs[fid]; ok { + validIDs = append(validIDs, fid) + } + } + if len(validIDs) < 2 { + return schemas.PotentialChain{}, false + } + + impact := parts[2] + if impact == "" { + impact = "Combined exploitation path" + } + + severityStr := strings.TrimSpace(strings.ToLower(parts[3])) + severity := schemas.SeverityHigh // Python: severity_map.get(severity_str, Severity.HIGH) + switch severityStr { + case "critical": + severity = schemas.SeverityCritical + case "high": + severity = schemas.SeverityHigh + case "medium": + severity = schemas.SeverityMedium + case "low": + severity = schemas.SeverityLow + } + + chain := schemas.NewPotentialChain() // mints chain_id + chain.Title = title + chain.FindingIDs = validIDs + chain.CombinedImpact = impact + chain.EstimatedSeverity = severity + return chain, true +} + +// extractChainCorrelation ports _extract_chain_correlation. +// +// Python inspects the HarnessResult: an already-typed ChainCorrelationResult +// passes straight through, a `.parsed` of the right type is returned, a +// `.parsed` dict is model_validate'd, and anything else is None. The Go SDK +// sets Result.Parsed to the very pointer harnessx.Run allocated when — and only +// when — the output validated, so a non-nil Parsed is exactly Python's +// `isinstance(parsed, ChainCorrelationResult)`; the dict branch is unreachable +// (see harnessx.Extract's note) and is deliberately not ported. +func extractChainCorrelation(res *appx.HarnessResult, dest *schemas.ChainCorrelationResult) *schemas.ChainCorrelationResult { + if res == nil || res.Parsed == nil || dest == nil { + return nil + } + return dest +} + +// DeduplicateAndCorrelate ports src/sec_af/agents/dedup.py deduplicate_and_correlate. +// +// async def deduplicate_and_correlate(findings, recon, app, repo_path) -> HuntResult +// +// Python parity notes: +// +// - `recon` is accepted and never read. It is kept in the signature because +// every call site passes it and the reasoner adapter mirrors the Python +// argument list. +// - The findings are MUTATED in place (fingerprint seeding, merges). Python +// mutates the caller's pydantic objects; the Go port takes pointers into +// the caller's slice so the same thing happens. +// - The chain-correlation harness call is wrapped in a bare `except +// Exception: chains = []`, so ANY failure — transport error, timeout, +// is_error result, unparseable output — silently yields the seed chains. +// The only error this function can return is a tempdir-creation failure, +// which in Python happens OUTSIDE the try and therefore propagates. +func DeduplicateAndCorrelate( + ctx context.Context, + findings []schemas.RawFinding, + recon schemas.ReconResult, + app appx.Harnesser, + repoPath string, +) (schemas.HuntResult, error) { + _ = recon // Python parity: unused. + + ptrs := make([]*schemas.RawFinding, len(findings)) + for i := range findings { + ptrs[i] = &findings[i] + } + + deduplicated := deduplicateWithAI(ctx, ptrs, app) + + var chains []schemas.PotentialChain + seedChains := fallbackCorrelate(deduplicated) + seedContext := seedChainContext(seedChains, deduplicated) + + if len(deduplicated) > 0 { + prompt := buildChainCorrelationPrompt(deduplicated, seedContext) + + harnessCwd, err := os.MkdirTemp("", "secaf-dedup-") + if err != nil { + return schemas.HuntResult{}, err + } + func() { + // Ports Python's `finally: shutil.rmtree(harness_cwd, ignore_errors=True)`. + defer os.RemoveAll(harnessCwd) + + callCtx, cancel := context.WithTimeout(ctx, chainCorrelationTimeout) + defer cancel() + + dest, res, runErr := harnessx.Run[schemas.ChainCorrelationResult]( + callCtx, app, prompt, + appx.HarnessOptions{Cwd: harnessCwd, ProjectDir: repoPath, Label: "Chain correlator"}, + ) + if runErr != nil { + // Python: the `await` raises, the except clause resets chains. + chains = nil + return + } + parsed := extractChainCorrelation(res, dest) + if parsed == nil { + return + } + availableIDs := make(map[string]struct{}, len(deduplicated)) + for _, f := range deduplicated { + availableIDs[f.ID] = struct{}{} + } + for _, chainStr := range parsed.Chains { + if chain, ok := parseChainFromStr(chainStr, availableIDs); ok { + chains = append(chains, chain) + } + } + if len(parsed.DuplicateIDs) > 0 { + dropSet := make(map[string]struct{}, len(parsed.DuplicateIDs)) + for _, id := range parsed.DuplicateIDs { + dropSet[id] = struct{}{} + } + kept := make([]*schemas.RawFinding, 0, len(deduplicated)) + for _, f := range deduplicated { + if _, dropped := dropSet[f.ID]; dropped { + continue + } + kept = append(kept, f) + } + deduplicated = kept + } + }() + } + + if len(chains) == 0 { + chains = seedChains + } + + sort.SliceStable(deduplicated, func(i, j int) bool { + return severityConfidenceLess(deduplicated[i], deduplicated[j]) + }) + + result := schemas.NewHuntResult() + result.Findings = make([]schemas.RawFinding, 0, len(deduplicated)) + for _, f := range deduplicated { + result.Findings = append(result.Findings, *f) + } + if chains == nil { + chains = []schemas.PotentialChain{} + } + result.Chains = chains + result.TotalRaw = len(findings) + result.DeduplicatedCount = len(deduplicated) + result.ChainCount = len(chains) + return result, nil +} + +// Deduplicator ports the src/sec_af/agents/dedup.py Deduplicator class — a thin object binding an +// app and a repo path so callers can invoke the free function without repeating +// them. Nothing in the live path uses it; it is part of the module's __all__ and +// is ported for completeness. +type Deduplicator struct { + app appx.Harnesser + repoPath string +} + +// NewDeduplicator ports Deduplicator.__init__. +func NewDeduplicator(app appx.Harnesser, repoPath string) *Deduplicator { + return &Deduplicator{app: app, repoPath: repoPath} +} + +// Run ports Deduplicator.run. +func (d *Deduplicator) Run(ctx context.Context, findings []schemas.RawFinding, recon schemas.ReconResult) (schemas.HuntResult, error) { + return DeduplicateAndCorrelate(ctx, findings, recon, d.app, d.repoPath) +} + +// runeSlice reproduces Python's s[:n], which counts code points, not bytes. +func runeSlice(s string, n int) string { + if n < 0 { + n = 0 + } + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) +} diff --git a/internal/secaf/agents/dedup/dedup_test.go b/internal/secaf/agents/dedup/dedup_test.go new file mode 100644 index 0000000000..d05b2a6d08 --- /dev/null +++ b/internal/secaf/agents/dedup/dedup_test.go @@ -0,0 +1,775 @@ +package dedup + +import ( + "context" + "encoding/json" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/Agent-Field/agentfield/sdk/go/ai" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// --------------------------------------------------------------------------- +// test doubles +// --------------------------------------------------------------------------- + +// harnessOnly exposes ONLY the Harness seam of the wrapped Fake. +// +// It is the Go equivalent of tests/test_dedup.py's `_HarnessApp`, which defines +// `async def harness(...)` and nothing else: dedup._deduplicate_with_ai probes +// `hasattr(app, "ai")` and skips the whole semantic pass when it is absent. +// *appx.Fake implements appx.AIer, so handing it over directly would take the +// OTHER branch — this wrapper is what makes the Python test's shape reachable. +type harnessOnly struct{ f *appx.Fake } + +func (h harnessOnly) Harness(ctx context.Context, prompt string, schema map[string]any, dest any, opts appx.HarnessOptions) (*appx.HarnessResult, error) { + return h.f.Harness(ctx, prompt, schema, dest, opts) +} + +// chainFake returns a Fake whose harness answers every call with resp, plus the +// harness-only wrapper around it. +func chainFake(t *testing.T, resp schemas.ChainCorrelationResult) (*appx.Fake, harnessOnly) { + t.Helper() + raw, err := json.Marshal(resp) + if err != nil { + t.Fatalf("marshal ChainCorrelationResult: %v", err) + } + f := &appx.Fake{HarnessFn: appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return raw, nil + })} + return f, harnessOnly{f} +} + +// finding ports tests/test_dedup.py::_finding. +func finding(id, cweID, cweName string, severity schemas.Severity) schemas.RawFinding { + f := schemas.NewRawFinding() + f.ID = id + f.HunterStrategy = "injection" + f.Title = "Finding " + id + f.Description = "Description for " + id + f.FindingType = schemas.FindingTypeSast + f.CweID = cweID + f.CweName = cweName + f.FilePath = "src/" + strings.ToLower(id) + ".py" + f.StartLine = 10 + f.EndLine = 10 + f.CodeSnippet = "dangerous_call(user_input)" + f.EstimatedSeverity = severity + f.Confidence = schemas.ConfidenceHigh + f.RelatedFiles = []string{} + f.Fingerprint = "fp-" + id + return f +} + +func emptyChains() schemas.ChainCorrelationResult { return schemas.NewChainCorrelationResult() } + +// --------------------------------------------------------------------------- +// ports of tests/test_dedup.py +// --------------------------------------------------------------------------- + +// Ports test_dedup_prompt_includes_seed_chain_candidates. +func TestDedup_PromptIncludesSeedChainCandidates(t *testing.T) { + findings := []schemas.RawFinding{ + finding("F1", "CWE-918", "Server-Side Request Forgery (SSRF)", schemas.SeverityHigh), + finding("F3", "CWE-798", "Use of Hard-coded Credentials", schemas.SeverityHigh), + } + fake, app := chainFake(t, emptyChains()) + + if _, err := DeduplicateAndCorrelate(context.Background(), findings, schemas.ReconResult{}, app, "."); err != nil { + t.Fatalf("DeduplicateAndCorrelate: %v", err) + } + + if len(fake.Harnesses) != 1 { + t.Fatalf("want exactly 1 harness call, got %d", len(fake.Harnesses)) + } + prompt := fake.Harnesses[0].Prompt + for _, want := range []string{ + "Seed chain candidates (validate and expand these):", + "Potential chain: Server-Side Request Forgery (SSRF) -> Use of Hard-coded Credentials (findings F1, F3)", + "Look for additional multi-step attack chains beyond these seeds.", + } { + if !strings.Contains(prompt, want) { + t.Errorf("prompt missing %q\n--- prompt ---\n%s", want, prompt) + } + } +} + +// Ports test_seed_chains_are_used_when_ai_returns_no_chains. +func TestDedup_SeedChainsUsedWhenAIReturnsNoChains(t *testing.T) { + findings := []schemas.RawFinding{ + finding("F1", "CWE-89", "SQL Injection", schemas.SeverityCritical), + finding("F2", "CWE-200", "Exposure of Sensitive Information", schemas.SeverityHigh), + } + _, app := chainFake(t, emptyChains()) + + result, err := DeduplicateAndCorrelate(context.Background(), findings, schemas.ReconResult{}, app, ".") + if err != nil { + t.Fatalf("DeduplicateAndCorrelate: %v", err) + } + if len(result.Chains) != 1 { + t.Fatalf("want 1 chain, got %d", len(result.Chains)) + } + if got := result.Chains[0].FindingIDs; len(got) != 2 || got[0] != "F1" || got[1] != "F2" { + t.Errorf("finding_ids = %v, want [F1 F2]", got) + } + if got, want := result.Chains[0].Title, "Potential attack chain: CWE-89 -> CWE-200"; got != want { + t.Errorf("title = %q, want %q", got, want) + } + if got, want := result.ChainCount, 1; got != want { + t.Errorf("chain_count = %d, want %d", got, want) + } +} + +// Ports test_ai_discovered_chains_take_priority_over_seed_chains. +func TestDedup_AIDiscoveredChainsTakePriorityOverSeedChains(t *testing.T) { + findings := []schemas.RawFinding{ + finding("F1", "CWE-918", "Server-Side Request Forgery (SSRF)", schemas.SeverityHigh), + finding("F2", "CWE-200", "Exposure of Sensitive Information", schemas.SeverityMedium), + finding("F3", "CWE-798", "Use of Hard-coded Credentials", schemas.SeverityHigh), + } + resp := schemas.NewChainCorrelationResult() + resp.Chains = []string{ + "AI-discovered chain: SSRF to data exfiltration|F1,F2,F3|SSRF reaches metadata; stolen secret enables privileged API access and data exfiltration.|critical", + } + _, app := chainFake(t, resp) + + result, err := DeduplicateAndCorrelate(context.Background(), findings, schemas.ReconResult{}, app, ".") + if err != nil { + t.Fatalf("DeduplicateAndCorrelate: %v", err) + } + if len(result.Chains) != 1 { + t.Fatalf("want 1 chain, got %d", len(result.Chains)) + } + if got, want := result.Chains[0].Title, "AI-discovered chain: SSRF to data exfiltration"; got != want { + t.Errorf("title = %q, want %q", got, want) + } + if got := result.Chains[0].FindingIDs; len(got) != 3 || got[0] != "F1" || got[1] != "F2" || got[2] != "F3" { + t.Errorf("finding_ids = %v, want [F1 F2 F3]", got) + } + if got, want := result.Chains[0].EstimatedSeverity, schemas.SeverityCritical; got != want { + t.Errorf("severity = %q, want %q", got, want) + } +} + +// --------------------------------------------------------------------------- +// compute_fingerprint +// --------------------------------------------------------------------------- + +func TestComputeFingerprint(t *testing.T) { + f := schemas.RawFinding{FilePath: "src/f1.py", StartLine: 10, CweID: "CWE-89"} + // Ground truth from the venv interpreter: + // hashlib.sha256(b"src/f1.py:10:CWE-89").hexdigest()[:16] + if got, want := ComputeFingerprint(f), "afd9dad9b5320992"; got != want { + t.Errorf("ComputeFingerprint = %q, want %q", got, want) + } + if got := len(ComputeFingerprint(f)); got != 16 { + t.Errorf("fingerprint length = %d, want 16", got) + } +} + +// A finding that arrives with an empty fingerprint gets one computed, and the +// CALLER's slice sees it (Python mutates the pydantic models in place). +func TestDedup_SeedsFingerprintsInPlace(t *testing.T) { + findings := []schemas.RawFinding{finding("F1", "CWE-89", "SQL Injection", schemas.SeverityHigh)} + findings[0].Fingerprint = "" + _, app := chainFake(t, emptyChains()) + + if _, err := DeduplicateAndCorrelate(context.Background(), findings, schemas.ReconResult{}, app, "."); err != nil { + t.Fatalf("DeduplicateAndCorrelate: %v", err) + } + want := ComputeFingerprint(findings[0]) + if findings[0].Fingerprint != want { + t.Errorf("caller's finding fingerprint = %q, want %q", findings[0].Fingerprint, want) + } +} + +// --------------------------------------------------------------------------- +// fingerprint collapse + _merge_duplicate +// --------------------------------------------------------------------------- + +func TestDedup_CollapsesSharedFingerprintAndMerges(t *testing.T) { + a := finding("A", "CWE-89", "SQL Injection", schemas.SeverityHigh) + a.Fingerprint = "same" + a.Description = "short" + a.RelatedFiles = []string{"z.py", "b.py"} + a.Confidence = schemas.ConfidenceMedium + + b := finding("B", "CWE-89", "SQL Injection", schemas.SeverityHigh) + b.Fingerprint = "same" + b.Description = "a much much longer description" + b.RelatedFiles = []string{"a.py", "b.py"} + b.Confidence = schemas.ConfidenceLow // lower, so A stays the winner + b.DataFlow = []schemas.ReconDataFlowStep{{FilePath: "src/x.py", Line: 3, Component: "c", Operation: "op"}} + + findings := []schemas.RawFinding{a, b} + _, app := chainFake(t, emptyChains()) + + result, err := DeduplicateAndCorrelate(context.Background(), findings, schemas.ReconResult{}, app, ".") + if err != nil { + t.Fatalf("DeduplicateAndCorrelate: %v", err) + } + if len(result.Findings) != 1 { + t.Fatalf("want 1 finding after fingerprint collapse, got %d", len(result.Findings)) + } + got := result.Findings[0] + if got.ID != "A" { + t.Errorf("winner id = %q, want A (existing wins unless incoming has HIGHER confidence)", got.ID) + } + if got.Description != "a much much longer description" { + t.Errorf("description = %q, want the loser's longer text", got.Description) + } + if want := []string{"a.py", "b.py", "z.py"}; !equalStrings(got.RelatedFiles, want) { + t.Errorf("related_files = %v, want %v (sorted set union)", got.RelatedFiles, want) + } + if len(got.DataFlow) != 1 || got.DataFlow[0].FilePath != "src/x.py" { + t.Errorf("data_flow = %v, want the loser's flow adopted when the winner had none", got.DataFlow) + } + if result.TotalRaw != 2 { + t.Errorf("total_raw = %d, want 2 (pre-dedup count)", result.TotalRaw) + } + if result.DeduplicatedCount != 1 { + t.Errorf("deduplicated_count = %d, want 1", result.DeduplicatedCount) + } +} + +func TestDedup_HigherConfidenceIncomingWins(t *testing.T) { + a := finding("A", "CWE-89", "SQL Injection", schemas.SeverityHigh) + a.Fingerprint = "same" + a.Confidence = schemas.ConfidenceLow + b := finding("B", "CWE-89", "SQL Injection", schemas.SeverityHigh) + b.Fingerprint = "same" + b.Confidence = schemas.ConfidenceHigh + + _, app := chainFake(t, emptyChains()) + result, err := DeduplicateAndCorrelate(context.Background(), []schemas.RawFinding{a, b}, schemas.ReconResult{}, app, ".") + if err != nil { + t.Fatalf("DeduplicateAndCorrelate: %v", err) + } + if len(result.Findings) != 1 || result.Findings[0].ID != "B" { + t.Fatalf("want the higher-confidence incoming finding to win, got %+v", result.Findings) + } +} + +// A winner whose data_flow is an EMPTY (non-nil) list is not None in Python, so +// the loser's flow must NOT overwrite it. +func TestDedup_EmptyDataFlowIsNotNone(t *testing.T) { + winner := schemas.NewRawFinding() + winner.DataFlow = []schemas.ReconDataFlowStep{} + winner.Confidence = schemas.ConfidenceHigh + loser := schemas.NewRawFinding() + loser.DataFlow = []schemas.ReconDataFlowStep{{FilePath: "a", Line: 1}} + loser.Confidence = schemas.ConfidenceLow + + got := mergeDuplicate(&winner, &loser) + if len(got.DataFlow) != 0 { + t.Errorf("data_flow = %v, want the winner's empty list preserved", got.DataFlow) + } +} + +// --------------------------------------------------------------------------- +// the .ai(DuplicateCheck) semantic pass +// --------------------------------------------------------------------------- + +func dupCheckFake(isDuplicate bool, hook func()) *appx.Fake { + return &appx.Fake{AIFn: func(_ context.Context, _ string, _ ...ai.Option) (*ai.Response, error) { + if hook != nil { + hook() + } + body := `{"is_duplicate":false,"duplicate_of":null,"reason":"different"}` + if isDuplicate { + body = `{"is_duplicate":true,"duplicate_of":"x","reason":"same root cause"}` + } + return &ai.Response{Choices: []ai.Choice{{Message: ai.Message{ + Role: "assistant", + Content: []ai.ContentPart{{Type: "text", Text: body}}, + }}}}, nil + }} +} + +// sameFile builds n findings in one file with one CWE and distinct fingerprints, +// so every pair is an AI-check candidate. +func sameFile(n int) []schemas.RawFinding { + out := make([]schemas.RawFinding, 0, n) + for i := 0; i < n; i++ { + f := finding(string(rune('A'+i)), "CWE-89", "SQL Injection", schemas.SeverityHigh) + f.FilePath = "src/shared.py" + f.StartLine = 10 + i + f.Fingerprint = "fp-" + string(rune('A'+i)) + out = append(out, f) + } + return out +} + +func TestDedup_AIPassRemovesSemanticDuplicates(t *testing.T) { + fake := dupCheckFake(true, nil) + fake.HarnessFn = appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return json.RawMessage(`{"chains":[],"duplicate_ids":[]}`), nil + }) + + result, err := DeduplicateAndCorrelate(context.Background(), sameFile(3), schemas.ReconResult{}, fake, ".") + if err != nil { + t.Fatalf("DeduplicateAndCorrelate: %v", err) + } + if len(fake.AIs) != 3 { + t.Errorf("want 3 pair checks (C(3,2)), got %d", len(fake.AIs)) + } + if len(result.Findings) != 1 || result.Findings[0].ID != "A" { + t.Fatalf("want only the first finding to survive, got %+v", result.Findings) + } +} + +// Python fans every pair out with asyncio.gather and NO semaphore. The barrier +// below deadlocks (and the test fails on the timeout) if the Go port checks the +// pairs one at a time. +func TestDedup_AIPairChecksRunConcurrently(t *testing.T) { + const pairs = 3 + var mu sync.Mutex + arrived := 0 + release := make(chan struct{}) + timedOut := make(chan struct{}) + + fake := dupCheckFake(false, func() { + mu.Lock() + arrived++ + full := arrived == pairs + mu.Unlock() + if full { + close(release) + } + select { + case <-release: + case <-timedOut: + } + }) + fake.HarnessFn = appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return json.RawMessage(`{"chains":[],"duplicate_ids":[]}`), nil + }) + + done := make(chan struct{}) + go func() { + defer close(done) + if _, err := DeduplicateAndCorrelate(context.Background(), sameFile(3), schemas.ReconResult{}, fake, "."); err != nil { + t.Errorf("DeduplicateAndCorrelate: %v", err) + } + }() + + select { + case <-done: + case <-time.After(10 * time.Second): + close(timedOut) + <-done + t.Fatal("pair checks did not run concurrently: only " + itoa(arrived) + " of 3 were in flight at once") + } +} + +// The `except Exception: return False` in _ai_check_duplicate means a failing +// gate keeps BOTH findings. +func TestDedup_AIErrorMeansNotDuplicate(t *testing.T) { + fake := &appx.Fake{ + AIFn: func(context.Context, string, ...ai.Option) (*ai.Response, error) { + return nil, context.DeadlineExceeded + }, + HarnessFn: appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return json.RawMessage(`{"chains":[],"duplicate_ids":[]}`), nil + }), + } + result, err := DeduplicateAndCorrelate(context.Background(), sameFile(2), schemas.ReconResult{}, fake, ".") + if err != nil { + t.Fatalf("DeduplicateAndCorrelate: %v", err) + } + if len(result.Findings) != 2 { + t.Errorf("want both findings kept when the gate fails, got %d", len(result.Findings)) + } +} + +// The hasattr(app, "ai") probe: a harness-only app skips the semantic pass. +func TestDedup_HarnessOnlyAppSkipsAIPass(t *testing.T) { + fake, app := chainFake(t, emptyChains()) + result, err := DeduplicateAndCorrelate(context.Background(), sameFile(3), schemas.ReconResult{}, app, ".") + if err != nil { + t.Fatalf("DeduplicateAndCorrelate: %v", err) + } + if len(fake.AIs) != 0 { + t.Errorf("want no .ai calls for a harness-only app, got %d", len(fake.AIs)) + } + if len(result.Findings) != 3 { + t.Errorf("want all 3 findings kept, got %d", len(result.Findings)) + } +} + +func TestDedup_DuplicateCheckPromptShape(t *testing.T) { + a := finding("A", "CWE-89", "SQL Injection", schemas.SeverityHigh) + b := finding("B", "CWE-79", "XSS", schemas.SeverityLow) + a.Description = strings.Repeat("x", 250) + + prompt := buildDuplicateCheckPrompt(&a, &b) + if !strings.HasPrefix(prompt, "Determine if these two security findings are duplicates (same root cause).\n\nFinding A:\n") { + t.Errorf("unexpected prompt head:\n%s", prompt) + } + if !strings.Contains(prompt, "- CWE: CWE-89 (SQL Injection)\n") { + t.Errorf("missing candidate CWE line:\n%s", prompt) + } + if !strings.Contains(prompt, "- File: src/a.py:10\n") { + t.Errorf("missing candidate file line:\n%s", prompt) + } + if strings.Contains(prompt, strings.Repeat("x", 201)) { + t.Error("description was not truncated to 200 characters") + } + if !strings.Contains(prompt, strings.Repeat("x", 200)) { + t.Error("description was truncated to fewer than 200 characters") + } + if strings.HasSuffix(prompt, "\n") { + t.Error("prompt must not end with a newline (Python's f-string does not)") + } +} + +// --------------------------------------------------------------------------- +// ordering +// --------------------------------------------------------------------------- + +func TestDedup_SortsBySeverityThenConfidenceDescendingStable(t *testing.T) { + mk := func(id string, sev schemas.Severity, conf schemas.Confidence) schemas.RawFinding { + f := finding(id, "CWE-1", "one", sev) + f.Confidence = conf + return f + } + findings := []schemas.RawFinding{ + mk("L", schemas.SeverityLow, schemas.ConfidenceHigh), + mk("C1", schemas.SeverityCritical, schemas.ConfidenceLow), + mk("H", schemas.SeverityHigh, schemas.ConfidenceHigh), + mk("C2", schemas.SeverityCritical, schemas.ConfidenceLow), // ties C1 -> stable + mk("CH", schemas.SeverityCritical, schemas.ConfidenceHigh), + } + _, app := chainFake(t, emptyChains()) + result, err := DeduplicateAndCorrelate(context.Background(), findings, schemas.ReconResult{}, app, ".") + if err != nil { + t.Fatalf("DeduplicateAndCorrelate: %v", err) + } + got := make([]string, 0, len(result.Findings)) + for _, f := range result.Findings { + got = append(got, f.ID) + } + want := []string{"CH", "C1", "C2", "H", "L"} + if !equalStrings(got, want) { + t.Errorf("order = %v, want %v", got, want) + } +} + +// --------------------------------------------------------------------------- +// harness interaction +// --------------------------------------------------------------------------- + +func TestDedup_HarnessOptionsAndTempDirLifecycle(t *testing.T) { + var seenCwd string + var existedDuringCall bool + fake := &appx.Fake{HarnessFn: func(_ context.Context, _ string, _ map[string]any, dest any, opts appx.HarnessOptions) (*appx.HarnessResult, error) { + seenCwd = opts.Cwd + if st, err := os.Stat(opts.Cwd); err == nil && st.IsDir() { + existedDuringCall = true + } + if err := json.Unmarshal([]byte(`{"chains":[],"duplicate_ids":[]}`), dest); err != nil { + return nil, err + } + return &appx.HarnessResult{Parsed: dest}, nil + }} + + findings := []schemas.RawFinding{finding("F1", "CWE-89", "SQL Injection", schemas.SeverityHigh)} + if _, err := DeduplicateAndCorrelate(context.Background(), findings, schemas.ReconResult{}, harnessOnly{fake}, "/repo/root"); err != nil { + t.Fatalf("DeduplicateAndCorrelate: %v", err) + } + if !existedDuringCall { + t.Error("harness cwd did not exist during the call") + } + if base := lastPathSegment(seenCwd); !strings.HasPrefix(base, "secaf-dedup-") { + t.Errorf("harness cwd %q does not use the secaf-dedup- prefix", seenCwd) + } + if _, err := os.Stat(seenCwd); !os.IsNotExist(err) { + t.Errorf("harness cwd %q was not removed after the call (err=%v)", seenCwd, err) + } + if got, want := fake.Harnesses[0].Opts.ProjectDir, "/repo/root"; got != want { + t.Errorf("project_dir = %q, want %q", got, want) + } +} + +// A harness that fails (is_error) leaves chains empty, so the seed chains win — +// Python's bare `except Exception: chains = []` plus `if not chains:`. +func TestDedup_HarnessFailureFallsBackToSeedChains(t *testing.T) { + fake := &appx.Fake{HarnessFn: appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return nil, errString("provider exploded") + })} + findings := []schemas.RawFinding{ + finding("F1", "CWE-89", "SQL Injection", schemas.SeverityCritical), + finding("F2", "CWE-200", "Exposure of Sensitive Information", schemas.SeverityHigh), + } + result, err := DeduplicateAndCorrelate(context.Background(), findings, schemas.ReconResult{}, harnessOnly{fake}, ".") + if err != nil { + t.Fatalf("DeduplicateAndCorrelate must swallow harness failures, got %v", err) + } + if len(result.Chains) != 1 || result.Chains[0].Title != "Potential attack chain: CWE-89 -> CWE-200" { + t.Errorf("want the seed chain, got %+v", result.Chains) + } +} + +// A transport error out of app.Harness is the `await` raising in Python. +func TestDedup_HarnessTransportErrorFallsBackToSeedChains(t *testing.T) { + fake := &appx.Fake{HarnessFn: func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return nil, errString("connection refused") + }} + findings := []schemas.RawFinding{ + finding("F1", "CWE-89", "SQL Injection", schemas.SeverityCritical), + finding("F2", "CWE-200", "Exposure of Sensitive Information", schemas.SeverityHigh), + } + result, err := DeduplicateAndCorrelate(context.Background(), findings, schemas.ReconResult{}, harnessOnly{fake}, ".") + if err != nil { + t.Fatalf("want the error swallowed, got %v", err) + } + if len(result.Chains) != 1 { + t.Errorf("want the seed chain, got %+v", result.Chains) + } +} + +func TestDedup_DuplicateIDsFromHarnessDropFindings(t *testing.T) { + resp := schemas.NewChainCorrelationResult() + resp.DuplicateIDs = []string{"F2"} + _, app := chainFake(t, resp) + + findings := []schemas.RawFinding{ + finding("F1", "CWE-89", "SQL Injection", schemas.SeverityCritical), + finding("F2", "CWE-200", "Exposure of Sensitive Information", schemas.SeverityHigh), + } + result, err := DeduplicateAndCorrelate(context.Background(), findings, schemas.ReconResult{}, app, ".") + if err != nil { + t.Fatalf("DeduplicateAndCorrelate: %v", err) + } + if len(result.Findings) != 1 || result.Findings[0].ID != "F1" { + t.Fatalf("want F2 dropped, got %+v", result.Findings) + } + if result.DeduplicatedCount != 1 { + t.Errorf("deduplicated_count = %d, want 1", result.DeduplicatedCount) + } + if result.TotalRaw != 2 { + t.Errorf("total_raw = %d, want 2", result.TotalRaw) + } + // The seed chain referenced F2, and Python does NOT re-derive chains after + // the drop, so the (now dangling) seed chain survives. + if len(result.Chains) != 1 { + t.Errorf("chains = %+v, want the seed chain retained", result.Chains) + } +} + +// No findings means no harness call at all (Python guards with `if deduplicated:`). +func TestDedup_NoFindingsSkipsHarness(t *testing.T) { + fake, app := chainFake(t, emptyChains()) + result, err := DeduplicateAndCorrelate(context.Background(), nil, schemas.ReconResult{}, app, ".") + if err != nil { + t.Fatalf("DeduplicateAndCorrelate: %v", err) + } + if len(fake.Harnesses) != 0 { + t.Errorf("want no harness call for an empty finding list, got %d", len(fake.Harnesses)) + } + if result.Findings == nil || len(result.Findings) != 0 { + t.Errorf("findings = %v, want an empty (non-nil) list", result.Findings) + } + if result.Chains == nil || len(result.Chains) != 0 { + t.Errorf("chains = %v, want an empty (non-nil) list", result.Chains) + } + if result.StrategiesRun == nil { + t.Error("strategies_run must be [] (pydantic default_factory=list), not null") + } +} + +// --------------------------------------------------------------------------- +// _parse_chain_from_str / _split_pipe / _seed_chain_context +// --------------------------------------------------------------------------- + +func TestSplitPipe(t *testing.T) { + cases := []struct { + in string + expected int + want []string + }{ + {"a|b|c|d|e", 4, []string{"a", "b", "c", "d|e"}}, + {"only-title", 4, []string{"only-title", "", "", ""}}, + {" a | b ", 4, []string{"a", "b", "", ""}}, + {"", 2, []string{"", ""}}, + } + for _, c := range cases { + got := splitPipe(c.in, c.expected) + if !equalStrings(got, c.want) { + t.Errorf("splitPipe(%q, %d) = %v, want %v", c.in, c.expected, got, c.want) + } + } +} + +func TestParseChainFromStr(t *testing.T) { + available := map[string]struct{}{"F1": {}, "F2": {}, "F3": {}} + + if _, ok := parseChainFromStr("t|F1|impact|high", available); ok { + t.Error("want a chain with a single valid id to be dropped") + } + if _, ok := parseChainFromStr("t|F1,NOPE|impact|high", available); ok { + t.Error("want unknown ids filtered out before the <2 check") + } + + chain, ok := parseChainFromStr("Title|F1, F2 ,,F3||bogus", available) + if !ok { + t.Fatal("want the chain parsed") + } + if chain.Title != "Title" { + t.Errorf("title = %q", chain.Title) + } + if !equalStrings(chain.FindingIDs, []string{"F1", "F2", "F3"}) { + t.Errorf("finding_ids = %v", chain.FindingIDs) + } + if chain.CombinedImpact != "Combined exploitation path" { + t.Errorf("impact = %q, want the empty-string default", chain.CombinedImpact) + } + if chain.EstimatedSeverity != schemas.SeverityHigh { + t.Errorf("severity = %q, want the HIGH default for an unknown label", chain.EstimatedSeverity) + } + if chain.ChainID == "" { + t.Error("chain_id must be minted (pydantic default_factory=uuid4)") + } + + for label, want := range map[string]schemas.Severity{ + "critical": schemas.SeverityCritical, + " HIGH ": schemas.SeverityHigh, + "Medium": schemas.SeverityMedium, + "low": schemas.SeverityLow, + "info": schemas.SeverityHigh, // not in Python's severity_map -> default + } { + c, ok := parseChainFromStr("t|F1,F2|i|"+label, available) + if !ok { + t.Fatalf("chain %q did not parse", label) + } + if c.EstimatedSeverity != want { + t.Errorf("severity for %q = %q, want %q", label, c.EstimatedSeverity, want) + } + } +} + +func TestSeedChainContext_UnresolvableIDsFallBackToTitle(t *testing.T) { + f := finding("F1", "CWE-89", "SQL Injection", schemas.SeverityHigh) + chain := schemas.NewPotentialChain() + chain.Title = "Fallback title" + chain.FindingIDs = []string{"NOPE", "ALSO-NOPE"} + + got := seedChainContext([]schemas.PotentialChain{chain}, []*schemas.RawFinding{&f}) + want := "Seed chain candidates (validate and expand these):\n" + + "- Potential chain: Fallback title (findings NOPE, ALSO-NOPE)\n" + + "Look for additional multi-step attack chains beyond these seeds." + if got != want { + t.Errorf("seedChainContext =\n%q\nwant\n%q", got, want) + } +} + +// _fallback_correlate upper-cases the CWE id before matching, and `max(...)` +// keeps the FIRST argument on a severity tie. +func TestFallbackCorrelate_UpperCasesAndTieKeepsFirst(t *testing.T) { + a := finding("A", "cwe-918", "SSRF", schemas.SeverityHigh) + b := finding("B", "CWE-798", "Creds", schemas.SeverityHigh) + chains := fallbackCorrelate([]*schemas.RawFinding{&a, &b}) + if len(chains) != 1 { + t.Fatalf("want 1 chain, got %d", len(chains)) + } + if chains[0].Title != "Potential attack chain: CWE-918 -> CWE-798" { + t.Errorf("title = %q", chains[0].Title) + } + if chains[0].EstimatedSeverity != schemas.SeverityHigh { + t.Errorf("severity = %q", chains[0].EstimatedSeverity) + } + if chains[0].CombinedImpact != "Combined exploitation path discovered by correlation heuristics; verify chain during PROVE phase." { + t.Errorf("impact = %q", chains[0].CombinedImpact) + } +} + +// Every pattern in _CHAIN_PATTERNS fires, in table order. +func TestFallbackCorrelate_AllPatternsInOrder(t *testing.T) { + cwes := []string{"CWE-918", "CWE-798", "CWE-862", "CWE-285", "CWE-89", "CWE-200", "CWE-16"} + ptrs := make([]*schemas.RawFinding, 0, len(cwes)) + for i, cwe := range cwes { + f := finding(string(rune('A'+i)), cwe, cwe, schemas.SeverityMedium) + ptrs = append(ptrs, &f) + } + chains := fallbackCorrelate(ptrs) + want := []string{ + "Potential attack chain: CWE-918 -> CWE-798", + "Potential attack chain: CWE-862 -> CWE-285", + "Potential attack chain: CWE-89 -> CWE-200", + "Potential attack chain: CWE-16 -> CWE-798", + } + got := make([]string, 0, len(chains)) + for _, c := range chains { + got = append(got, c.Title) + } + if !equalStrings(got, want) { + t.Errorf("chains = %v, want %v", got, want) + } +} + +// --------------------------------------------------------------------------- +// Deduplicator +// --------------------------------------------------------------------------- + +func TestDeduplicator_Run(t *testing.T) { + fake, app := chainFake(t, emptyChains()) + d := NewDeduplicator(app, "/repo") + findings := []schemas.RawFinding{finding("F1", "CWE-89", "SQL Injection", schemas.SeverityHigh)} + result, err := d.Run(context.Background(), findings, schemas.ReconResult{}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if len(result.Findings) != 1 { + t.Fatalf("findings = %+v", result.Findings) + } + if got, want := fake.Harnesses[0].Opts.ProjectDir, "/repo"; got != want { + t.Errorf("project_dir = %q, want %q", got, want) + } +} + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- + +type errString string + +func (e errString) Error() string { return string(e) } + +func equalStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func lastPathSegment(p string) string { + if i := strings.LastIndexByte(p, '/'); i >= 0 { + return p[i+1:] + } + return p +} + +func itoa(n int) string { + if n == 0 { + return "0" + } + var buf [20]byte + i := len(buf) + for n > 0 { + i-- + buf[i] = byte('0' + n%10) + n /= 10 + } + return string(buf[i:]) +} diff --git a/internal/secaf/agents/dedup/golden_test.go b/internal/secaf/agents/dedup/golden_test.go new file mode 100644 index 0000000000..67c2eb5b6e --- /dev/null +++ b/internal/secaf/agents/dedup/golden_test.go @@ -0,0 +1,193 @@ +package dedup + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// The goldens in testdata/golden are written by go/scripts/gen_golden.py +// (section "S6"), which captures what the REAL Python functions in +// src/sec_af/agents/dedup.py hand to `app.ai(user=...)` and +// `app.harness(prompt=...)` for the fixtures rebuilt below. Regenerate with: +// +// PYTHONPATH=src ~/.agentfield/packages/sec-af/venv/bin/python go/scripts/gen_golden.py +// +// NOTE (integration): the "S6" section named above is NO LONGER PRESENT in +// go/scripts/gen_golden.py — it was lost when several agents rewrote that file +// concurrently during the port. Running the script does NOT refresh these +// files. The committed goldens ARE the ones that section produced from the real +// Python functions, and this test still guards them; but if the Python prompt +// builder changes, re-derive them by hand from the fixtures below (or restore +// the section) rather than trusting the script. See the COVERAGE GAP comment in +// gen_golden.py. +// +// A diff here means the Go prompt builder and the Python one have drifted, which +// is a real behavioural change: the prompt is the LLM's entire instruction. + +const goldenFixtureRepo = "/fixtures/demo-repo" + +func golden(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join("testdata", "golden", name+".txt")) + if err != nil { + t.Fatalf("read golden %s: %v", name, err) + } + return string(b) +} + +func assertGolden(t *testing.T, name, got string) { + t.Helper() + want := golden(t, name) + if got == want { + return + } + t.Errorf("prompt does not match golden %s.txt\n--- got (%d bytes) ---\n%s\n--- want (%d bytes) ---\n%s", + name, len(got), got, len(want), want) +} + +// goldenFinding mirrors gen_golden.py's _s6_raw_finding defaults. +func goldenFinding(mut func(f *schemas.RawFinding)) schemas.RawFinding { + f := schemas.NewRawFinding() + f.ID = "F1" + f.HunterStrategy = "injection" + f.Title = "SQL injection in user lookup" + f.Description = "User-controlled `user_id` flows into a raw SQL string." + f.FindingType = schemas.FindingTypeSast + f.CweID = "CWE-89" + f.CweName = "SQL Injection" + f.FilePath = "src/db/users.py" + f.StartLine = 42 + f.EndLine = 44 + f.CodeSnippet = `cur.execute("SELECT * FROM users WHERE id = " + user_id)` + f.EstimatedSeverity = schemas.SeverityCritical + f.Confidence = schemas.ConfidenceHigh + f.RelatedFiles = []string{} + f.Fingerprint = "fp-F1" + if mut != nil { + mut(&f) + } + return f +} + +func TestGolden_DuplicateCheckPrompt(t *testing.T) { + candidate := goldenFinding(func(f *schemas.RawFinding) { + f.ID = "F1" + f.Title = "SQL injection in user lookup" + f.Description = strings.Repeat("Tainted `user_id` reaches cur.execute without parameterization. ", 5) + f.Fingerprint = "fp-F1" + }) + existing := goldenFinding(func(f *schemas.RawFinding) { + f.ID = "F2" + f.Title = "Unsanitized SQL string concatenation" + f.Description = "Same sink, different wording — café ☕." + f.FilePath = "src/db/users.py" + f.StartLine = 43 + f.Fingerprint = "fp-F2" + }) + + assertGolden(t, "duplicate_check_prompt", buildDuplicateCheckPrompt(&candidate, &existing)) +} + +// capturePrompt runs DeduplicateAndCorrelate through a harness-only recorder — +// the same shape gen_golden.py's _S6HarnessApp has (no `.ai` attribute, so the +// semantic pass is skipped) — and returns the prompt it was handed. +func capturePrompt(t *testing.T, findings []schemas.RawFinding) string { + t.Helper() + fake := &appx.Fake{HarnessFn: appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return nil, errString("captured") + })} + if _, err := DeduplicateAndCorrelate(context.Background(), findings, schemas.ReconResult{}, harnessOnly{fake}, goldenFixtureRepo); err != nil { + t.Fatalf("DeduplicateAndCorrelate: %v", err) + } + if len(fake.Harnesses) != 1 { + t.Fatalf("want 1 harness call, got %d", len(fake.Harnesses)) + } + return fake.Harnesses[0].Prompt +} + +func TestGolden_ChainCorrelationPromptSeeded(t *testing.T) { + findings := []schemas.RawFinding{ + goldenFinding(func(f *schemas.RawFinding) { + f.ID = "F1" + f.CweID = "CWE-918" + f.CweName = "Server-Side Request Forgery (SSRF)" + f.Title = "SSRF in webhook fetcher" + f.FilePath = "src/net/webhook.py" + f.StartLine = 17 + f.EstimatedSeverity = schemas.SeverityHigh + f.Confidence = schemas.ConfidenceHigh + f.Fingerprint = "fp-F1" + }), + goldenFinding(func(f *schemas.RawFinding) { + f.ID = "F2" + f.CweID = "CWE-89" + f.CweName = "SQL Injection" + f.Title = "SQL injection in user lookup" + f.FilePath = "src/db/users.py" + f.StartLine = 42 + f.EstimatedSeverity = schemas.SeverityCritical + f.Confidence = schemas.ConfidenceMedium + f.Fingerprint = "fp-F2" + }), + goldenFinding(func(f *schemas.RawFinding) { + f.ID = "F3" + // Lower-case on purpose: _fallback_correlate upper-cases before + // matching, but the findings summary echoes the raw value. + f.CweID = "cwe-798" + f.CweName = "Use of Hard-coded Credentials" + f.Title = "Hard-coded AWS key" + f.FilePath = "src/config/settings.py" + f.StartLine = 8 + f.EstimatedSeverity = schemas.SeverityMedium + f.Confidence = schemas.ConfidenceLow + f.Fingerprint = "fp-F3" + }), + goldenFinding(func(f *schemas.RawFinding) { + f.ID = "F4" + f.CweID = "CWE-200" + f.CweName = "Exposure of Sensitive Information" + f.Title = "Stack trace leaked to client" + f.FilePath = "src/api/errors.py" + f.StartLine = 55 + f.EstimatedSeverity = schemas.SeverityLow + f.Confidence = schemas.ConfidenceHigh + f.Fingerprint = "fp-F4" + }), + } + assertGolden(t, "chain_correlation_prompt_seeded", capturePrompt(t, findings)) +} + +func TestGolden_ChainCorrelationPromptUnseeded(t *testing.T) { + findings := []schemas.RawFinding{ + goldenFinding(func(f *schemas.RawFinding) { + f.ID = "G1" + f.CweID = "CWE-79" + f.CweName = "Cross-site Scripting" + f.Title = "Reflected XSS in search" + f.FilePath = "src/web/search.py" + f.StartLine = 12 + f.EstimatedSeverity = schemas.SeverityMedium + f.Confidence = schemas.ConfidenceMedium + f.Fingerprint = "fp-G1" + }), + goldenFinding(func(f *schemas.RawFinding) { + f.ID = "G2" + f.CweID = "CWE-327" + f.CweName = "Use of a Broken or Risky Cryptographic Algorithm" + f.Title = "MD5 used for password hashing" + f.FilePath = "src/auth/hash.py" + f.StartLine = 9 + f.EstimatedSeverity = schemas.SeverityHigh + f.Confidence = schemas.ConfidenceLow + f.Fingerprint = "fp-G2" + }), + } + assertGolden(t, "chain_correlation_prompt_unseeded", capturePrompt(t, findings)) +} diff --git a/internal/secaf/agents/dedup/testdata/golden/chain_correlation_prompt_seeded.txt b/internal/secaf/agents/dedup/testdata/golden/chain_correlation_prompt_seeded.txt new file mode 100644 index 0000000000..193b360f5e --- /dev/null +++ b/internal/secaf/agents/dedup/testdata/golden/chain_correlation_prompt_seeded.txt @@ -0,0 +1,15 @@ +You are SEC-AF's chain correlator. +Identify multi-step attack chains across the findings below. +A chain means one vulnerability enables exploitation of another. +Also flag any remaining duplicate IDs that should be dropped. + +Findings: +- id=F2 cwe=CWE-89 file=src/db/users.py:42 title=SQL injection in user lookup severity=critical +- id=F1 cwe=CWE-918 file=src/net/webhook.py:17 title=SSRF in webhook fetcher severity=high +- id=F3 cwe=cwe-798 file=src/config/settings.py:8 title=Hard-coded AWS key severity=medium +- id=F4 cwe=CWE-200 file=src/api/errors.py:55 title=Stack trace leaked to client severity=low + +Seed chain candidates (validate and expand these): +- Potential chain: Server-Side Request Forgery (SSRF) -> Use of Hard-coded Credentials (findings F1, F3) +- Potential chain: SQL Injection -> Exposure of Sensitive Information (findings F2, F4) +Look for additional multi-step attack chains beyond these seeds. \ No newline at end of file diff --git a/internal/secaf/agents/dedup/testdata/golden/chain_correlation_prompt_unseeded.txt b/internal/secaf/agents/dedup/testdata/golden/chain_correlation_prompt_unseeded.txt new file mode 100644 index 0000000000..d1e46b657d --- /dev/null +++ b/internal/secaf/agents/dedup/testdata/golden/chain_correlation_prompt_unseeded.txt @@ -0,0 +1,12 @@ +You are SEC-AF's chain correlator. +Identify multi-step attack chains across the findings below. +A chain means one vulnerability enables exploitation of another. +Also flag any remaining duplicate IDs that should be dropped. + +Findings: +- id=G2 cwe=CWE-327 file=src/auth/hash.py:9 title=MD5 used for password hashing severity=high +- id=G1 cwe=CWE-79 file=src/web/search.py:12 title=Reflected XSS in search severity=medium + +Seed chain candidates (validate and expand these): +- No heuristic seed chains were detected from hardcoded CWE pairs. +Look for additional multi-step attack chains beyond these seeds. \ No newline at end of file diff --git a/internal/secaf/agents/dedup/testdata/golden/duplicate_check_prompt.txt b/internal/secaf/agents/dedup/testdata/golden/duplicate_check_prompt.txt new file mode 100644 index 0000000000..ae7ad6b4d3 --- /dev/null +++ b/internal/secaf/agents/dedup/testdata/golden/duplicate_check_prompt.txt @@ -0,0 +1,13 @@ +Determine if these two security findings are duplicates (same root cause). + +Finding A: +- Title: SQL injection in user lookup +- CWE: CWE-89 (SQL Injection) +- File: src/db/users.py:42 +- Description: Tainted `user_id` reaches cur.execute without parameterization. Tainted `user_id` reaches cur.execute without parameterization. Tainted `user_id` reaches cur.execute without parameterization. Tainted + +Finding B: +- Title: Unsanitized SQL string concatenation +- CWE: CWE-89 (SQL Injection) +- File: src/db/users.py:43 +- Description: Same sink, different wording — café ☕. \ No newline at end of file diff --git a/internal/secaf/agents/hunt/api_security.go b/internal/secaf/agents/hunt/api_security.go new file mode 100644 index 0000000000..8ff2294c10 --- /dev/null +++ b/internal/secaf/agents/hunt/api_security.go @@ -0,0 +1,75 @@ +package hunt + +// Ports src/sec_af/agents/hunt/api_security.py. + +import ( + "context" + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/recontext" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +const apiSecurityPromptPath = "hunt/api_security.txt" + +// apiSecurityScanPrompt builds the exact prompt run_api_security_hunter sends. +// +// Python parity: the only hunter whose block closes with an explicit +// "write the JSON output file using your Write tool" instruction (the recon +// mappers use the same wording). earlyStop is where the argument cascade lands +// the depth string for this hunter (package doc). +func apiSecurityScanPrompt(repoPath string, recon schemas.ReconResult, earlyStop string) (scanPrompt, reconContext string) { + reconContext = recontext.ReconContextForAPISecurity(recon) + template := prompts.MustLoad(apiSecurityPromptPath) + template = strings.ReplaceAll(template, "{{RECON_CONTEXT}}", reconContext) + template = strings.ReplaceAll(template, "{{LANGUAGE_HINTS}}", recontext.LanguageHintsForContext(recon)) + template = strings.ReplaceAll(template, "{{FRAMEWORK_HINTS}}", recontext.FrameworkHintsForContext(recon)) + + scanPrompt = template + + "\n\nCONTEXT:\n" + + "- Repository path: " + repoPath + "\n" + + "- Focus only on API-relevant code paths and endpoint handlers surfaced by RECON.\n" + + "- Early stop rule: if you inspect " + + earlyStop + " files without credible API issues, " + + "stop and return empty findings.\n" + + "- Read the handler files first, then generate findings.\n" + + "- After gathering evidence, write the JSON output file using your Write tool." + return scanPrompt, reconContext +} + +func runAPISecurityHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, earlyStop string, +) (schemas.HuntResult, error) { + // Python parity: `if not recon.architecture.api_surface`. Unlike crypto's + // and supply_chain's gates, THIS early exit names the strategy — it is + // identical to the "scanner found nothing" return. + if len(recon.Architecture.APISurface) == 0 { + empty := schemas.NewHuntResult() + empty.StrategiesRun = []string{string(schemas.HuntStrategyAPISecurity)} + return empty, nil + } + scanPrompt, reconContext := apiSecurityScanPrompt(repoPath, recon, earlyStop) + return runHunterBody(ctx, app, repoPath, hunterSpec{ + ScanPrompt: scanPrompt, + ReconContext: reconContext, + // Python parity: the only hunter tagging its findings "api". + FindingType: "api", + Strategy: string(schemas.HuntStrategyAPISecurity), + EmptyStrategiesRun: []string{string(schemas.HuntStrategyAPISecurity)}, + }) +} + +// RunAPISecurityHunter ports api_security.py run_api_security_hunter: +// +// async def run_api_security_hunter(app, repo_path, recon, +// max_files_without_signal: int = 30) -> HuntResult +func RunAPISecurityHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, maxFilesWithoutSignal int, +) (schemas.HuntResult, error) { + return runAPISecurityHunter(ctx, app, repoPath, recon, strconv.Itoa(maxFilesWithoutSignal)) +} diff --git a/internal/secaf/agents/hunt/auth.go b/internal/secaf/agents/hunt/auth.go new file mode 100644 index 0000000000..c4e7cfb93b --- /dev/null +++ b/internal/secaf/agents/hunt/auth.go @@ -0,0 +1,94 @@ +package hunt + +// Ports src/sec_af/agents/hunt/auth.py. + +import ( + "context" + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/recontext" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +const authPromptPath = "hunt/auth.txt" + +// authTargetCWEs ports auth.py `_TARGET_CWES`. Order is load-bearing: the list +// is joined with ", " into the prompt's {{TARGET_CWES}} slot. +var authTargetCWEs = []string{"CWE-287", "CWE-306", "CWE-862", "CWE-863", "CWE-352"} + +// authDepthLabel ports auth.py _depth_label: +// +// normalized = depth.lower().strip() +// return normalized if normalized in {"quick", "standard", "thorough"} else "standard" +// +// Python parity: lower() runs BEFORE strip(), which is immaterial (neither +// changes the other's result), but the STRIP is what distinguishes this from +// config.NormalizeDepth — " THOROUGH " normalizes to "thorough" here and to +// "standard" there. auth.py is the only module that trims. +func authDepthLabel(depth string) string { + normalized := strings.TrimSpace(strings.ToLower(depth)) + switch normalized { + case "quick", "standard", "thorough": + return normalized + default: + return "standard" + } +} + +// authBuildPrompt ports auth.py _build_prompt — the six template substitutions. +// It returns the recon context alongside the prompt because run_auth_hunter +// recomputes the same string for the enrichment step. +func authBuildPrompt(template, repoPath string, recon schemas.ReconResult, depth string) (string, string) { + reconContext := recontext.ReconContextForAuth(recon) + out := strings.ReplaceAll(template, "{{REPO_PATH}}", repoPath) + out = strings.ReplaceAll(out, "{{DEPTH}}", authDepthLabel(depth)) + out = strings.ReplaceAll(out, "{{TARGET_CWES}}", strings.Join(authTargetCWEs, ", ")) + out = strings.ReplaceAll(out, "{{RECON_CONTEXT}}", reconContext) + out = strings.ReplaceAll(out, "{{LANGUAGE_HINTS}}", recontext.LanguageHintsForContext(recon)) + out = strings.ReplaceAll(out, "{{FRAMEWORK_HINTS}}", recontext.FrameworkHintsForContext(recon)) + return out, reconContext +} + +// authScanPrompt builds the exact prompt run_auth_hunter sends. +// +// Python parity: auth is the ONLY hunter whose appended block is headed +// "EXECUTION CONSTRAINTS:" rather than "CONTEXT:", it is the only one that ends +// with a trailing newline, and it carries no repository-path or depth line — +// both already reached the prompt through the template's {{REPO_PATH}} and +// {{DEPTH}} markers. run_auth_hunter also passes the ALREADY-normalized label +// into _build_prompt, which normalizes again; authDepthLabel is idempotent, so +// the port folds the double normalization into one call. +func authScanPrompt(repoPath string, recon schemas.ReconResult, depth, earlyStop string) (scanPrompt, reconContext string) { + built, reconContext := authBuildPrompt(prompts.MustLoad(authPromptPath), repoPath, recon, authDepthLabel(depth)) + scanPrompt = built + + "\n\nEXECUTION CONSTRAINTS:\n" + + "- Early stop rule: if you inspect " + + earlyStop + " files without credible auth issues, " + + "stop and return empty findings.\n" + return scanPrompt, reconContext +} + +func runAuthHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, depth, earlyStop string, +) (schemas.HuntResult, error) { + scanPrompt, reconContext := authScanPrompt(repoPath, recon, depth, earlyStop) + return runHunterBody(ctx, app, repoPath, hunterSpec{ + ScanPrompt: scanPrompt, + ReconContext: reconContext, + FindingType: "sast", + Strategy: "auth", + EmptyStrategiesRun: nil, // Python: bare HuntResult() + }) +} + +// RunAuthHunter ports auth.py run_auth_hunter. +func RunAuthHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, depth string, maxFilesWithoutSignal int, +) (schemas.HuntResult, error) { + return runAuthHunter(ctx, app, repoPath, recon, depth, strconv.Itoa(maxFilesWithoutSignal)) +} diff --git a/internal/secaf/agents/hunt/business_logic.go b/internal/secaf/agents/hunt/business_logic.go new file mode 100644 index 0000000000..e26a6f3885 --- /dev/null +++ b/internal/secaf/agents/hunt/business_logic.go @@ -0,0 +1,105 @@ +package hunt + +// Ports src/sec_af/agents/hunt/business_logic.py. + +import ( + "context" + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/config" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/recontext" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +const businessLogicPromptPath = "hunt/business_logic.txt" + +// IsBusinessLogicHunterEnabled ports business_logic.py +// is_business_logic_hunter_enabled: +// +// profile = _normalize_depth(depth) +// return profile in {DepthProfile.STANDARD, DepthProfile.THOROUGH} +// +// Python parity: the normalization is the LENIENT one, so an unrecognised depth +// becomes STANDARD and the hunter runs. Only "quick" (in any case) disables it. +// business_logic.py declares its own `_normalize_depth` accepting +// `str | DepthProfile`; config.NormalizeDepth covers both arms because Go's +// DepthProfile IS a string type. +func IsBusinessLogicHunterEnabled(depth string) bool { + profile := config.NormalizeDepth(depth) + return profile == config.DepthStandard || profile == config.DepthThorough +} + +// businessLogicScanPrompt builds the exact prompt run_business_logic_hunter +// sends. +// +// Python parity: unlike injection/dos/ssrf/xss, the depth line interpolates the +// NORMALIZED profile (`_normalize_depth(depth).value`), not the raw argument — +// so a direct caller passing "Thorough" gets "thorough" here and "Thorough" +// there. The optional depth_prompt tail is appended AFTER the block, on its own +// line, and only when non-empty. +func businessLogicScanPrompt( + repoPath string, recon schemas.ReconResult, depth, earlyStop, depthPrompt string, +) (scanPrompt, reconContext string) { + reconContext = businessLogicContextBlock(recon) + template := prompts.MustLoad(businessLogicPromptPath) + template = strings.ReplaceAll(template, "{{RECON_CONTEXT_JSON}}", reconContext) + template = strings.ReplaceAll(template, "{{LANGUAGE_HINTS}}", recontext.LanguageHintsForContext(recon)) + template = strings.ReplaceAll(template, "{{FRAMEWORK_HINTS}}", recontext.FrameworkHintsForContext(recon)) + + scanPrompt = template + + "\n\nCONTEXT:\n" + + "- Repository path: " + repoPath + "\n" + + "- Depth profile: " + string(config.NormalizeDepth(depth)) + "\n" + + "- Early stop rule: if you inspect " + earlyStop + + " files without credible business-logic signal, stop and return empty findings.\n" + + "- Strategy: business_logic\n" + + "- Focus CWEs: CWE-840, CWE-841, CWE-362, CWE-367, CWE-639.\n" + + "- Reason about intended business behavior versus exploitable implementation behavior.\n" + + "- Take multiple turns, trace complete workflows, and return final JSON only when complete." + if depthPrompt != "" { + scanPrompt += "\n- Additional depth guidance: " + depthPrompt + } + return scanPrompt, reconContext +} + +func runBusinessLogicHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, depth, earlyStop, depthPrompt string, +) (schemas.HuntResult, error) { + // Python parity: `return HuntResult(findings=[], strategies_run=[])`, which + // is the bare default shape. + if !IsBusinessLogicHunterEnabled(depth) { + return schemas.NewHuntResult(), nil + } + scanPrompt, reconContext := businessLogicScanPrompt(repoPath, recon, depth, earlyStop, depthPrompt) + return runHunterBody(ctx, app, repoPath, hunterSpec{ + ScanPrompt: scanPrompt, + ReconContext: reconContext, + // Python parity: business_logic is one of only three hunters whose + // finding_type is not "sast" — it tags its findings "logic". + FindingType: "logic", + Strategy: string(schemas.HuntStrategyBusinessLogic), + EmptyStrategiesRun: []string{string(schemas.HuntStrategyBusinessLogic)}, + }) +} + +// RunBusinessLogicHunter ports business_logic.py run_business_logic_hunter: +// +// async def run_business_logic_hunter(app, repo_path, recon_result, depth, +// max_files_without_signal: int = 30, +// depth_prompt: str = "") -> HuntResult +// +// depthPrompt is the Go spelling of the `depth_prompt: str = ""` default; pass +// "" for it. __init__.py computes a depth_prompt but never manages to deliver +// it (package doc), so the live pipeline always passes "". +func RunBusinessLogicHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, depth string, maxFilesWithoutSignal int, depthPrompt string, +) (schemas.HuntResult, error) { + return runBusinessLogicHunter( + ctx, app, repoPath, recon, depth, strconv.Itoa(maxFilesWithoutSignal), depthPrompt, + ) +} diff --git a/internal/secaf/agents/hunt/config_secrets.go b/internal/secaf/agents/hunt/config_secrets.go new file mode 100644 index 0000000000..0c2ff1763c --- /dev/null +++ b/internal/secaf/agents/hunt/config_secrets.go @@ -0,0 +1,69 @@ +package hunt + +// Ports src/sec_af/agents/hunt/config_secrets.py. + +import ( + "context" + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/recontext" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +const configSecretsPromptPath = "hunt/config_secrets.txt" + +// configSecretsScanPrompt builds the exact prompt run_config_secrets_hunter +// sends. +// +// Python parity: the block ends with a trailing newline and — unlike every +// other hunter — has NO closing "write final JSON" line. earlyStop is where the +// argument cascade lands the depth string for this hunter (package doc). +func configSecretsScanPrompt(repoPath string, recon schemas.ReconResult, earlyStop string) (scanPrompt, reconContext string) { + reconContext = recontext.ReconContextForConfigSecrets(recon) + template := prompts.MustLoad(configSecretsPromptPath) + template = strings.ReplaceAll(template, "{{RECON_CONTEXT}}", reconContext) + template = strings.ReplaceAll(template, "{{LANGUAGE_HINTS}}", recontext.LanguageHintsForContext(recon)) + template = strings.ReplaceAll(template, "{{FRAMEWORK_HINTS}}", recontext.FrameworkHintsForContext(recon)) + + scanPrompt = template + + "\n\nCONTEXT:\n" + + "- Repository path: " + repoPath + "\n" + + "- Hunt strategy: config_secrets (CWE-798, CWE-259, CWE-321, CWE-16).\n" + + "- Early stop rule: if you inspect " + + earlyStop + " files without credible secrets/config issues, " + + "stop and return empty findings.\n" + + "- Use RECON ConfigReport and SecurityContext to prioritize likely real findings.\n" + + "- Take multiple turns: inspect files, validate exploitability signal, then build findings.\n" + return scanPrompt, reconContext +} + +func runConfigSecretsHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, earlyStop string, +) (schemas.HuntResult, error) { + scanPrompt, reconContext := configSecretsScanPrompt(repoPath, recon, earlyStop) + return runHunterBody(ctx, app, repoPath, hunterSpec{ + ScanPrompt: scanPrompt, + ReconContext: reconContext, + // Python parity: the only hunter tagging its findings "config". + FindingType: "config", + Strategy: "config_secrets", + // Python parity: bare HuntResult() here, even though the sibling + // secrets-adjacent hunters name their strategy. + EmptyStrategiesRun: nil, + }) +} + +// RunConfigSecretsHunter ports config_secrets.py run_config_secrets_hunter: +// +// async def run_config_secrets_hunter(app, repo_path, recon, +// max_files_without_signal: int = 30) -> HuntResult +func RunConfigSecretsHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, maxFilesWithoutSignal int, +) (schemas.HuntResult, error) { + return runConfigSecretsHunter(ctx, app, repoPath, recon, strconv.Itoa(maxFilesWithoutSignal)) +} diff --git a/internal/secaf/agents/hunt/context_blocks.go b/internal/secaf/agents/hunt/context_blocks.go new file mode 100644 index 0000000000..96a08550aa --- /dev/null +++ b/internal/secaf/agents/hunt/context_blocks.go @@ -0,0 +1,94 @@ +package hunt + +// The inline JSON recon-context blocks four hunters build for themselves +// instead of calling into src/sec_af/context.py. + +import ( + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// headEntryPoints ports Python's `architecture.entry_points[:n]`. +// +// The result is always NON-NIL, because pyfmt.Dumps renders a nil Go slice as +// `null` while a Python list slice of an empty list is `[]`. Every other +// element of the block is a struct, whose rendering needs no such care. +func headEntryPoints(items []schemas.EntryPoint, n int) []schemas.EntryPoint { + if n > len(items) { + n = len(items) + } + out := make([]schemas.EntryPoint, 0, n) + return append(out, items[:n]...) +} + +// headAPIEndpoints ports `architecture.api_surface[:n]`. See headEntryPoints. +func headAPIEndpoints(items []schemas.APIEndpoint, n int) []schemas.APIEndpoint { + if n > len(items) { + n = len(items) + } + out := make([]schemas.APIEndpoint, 0, n) + return append(out, items[:n]...) +} + +// headDataFlows ports `data_flows.flows[:n]`. See headEntryPoints. +func headDataFlows(items []schemas.DataFlow, n int) []schemas.DataFlow { + if n > len(items) { + n = len(items) + } + out := make([]schemas.DataFlow, 0, n) + return append(out, items[:n]...) +} + +// entryFlowContextBlock ports the `_recon_context_block` helper that +// agents/hunt/dos.py, ssrf.py and xss.py each declare — three byte-identical +// copies of: +// +// entry_points = [entry.model_dump() for entry in recon_result.architecture.entry_points[:10]] +// data_flows = [flow.model_dump() for flow in recon_result.data_flows.flows[:10]] +// context = { +// "app_type": recon_result.architecture.app_type, +// "auth_model": recon_result.security_context.auth_model, +// "frameworks": recon_result.frameworks, +// "languages": recon_result.languages, +// "entry_points": entry_points, +// "data_flows": data_flows, +// } +// return json.dumps(context, indent=2) +// +// The dict's key order is INSERTION order, which is not alphabetical, so the +// port builds a pyfmt.Ordered rather than a Go map (see DESIGN.md §2b) — and +// pyfmt.Dumps, not encoding/json, because the text reaches the LLM verbatim and +// CPython does not escape `<`, `>` or `&` while Go's encoder does. +// +// `app_type` is `str | None`: a nil *string renders as `null`, exactly as +// model_dump() -> json.dumps does for None. +func entryFlowContextBlock(recon schemas.ReconResult) string { + return pyfmt.Dumps(pyfmt.O( + "app_type", recon.Architecture.AppType, + "auth_model", recon.SecurityContext.AuthModel, + "frameworks", recon.Frameworks, + "languages", recon.Languages, + "entry_points", headEntryPoints(recon.Architecture.EntryPoints, 10), + "data_flows", headDataFlows(recon.DataFlows.Flows, 10), + ), 2) +} + +// businessLogicContextBlock ports agents/hunt/business_logic.py +// `_recon_context_block` — a WIDER projection than entryFlowContextBlock, with +// different limits (15 entry points, 20 endpoints, 20 flows), the api_surface +// added, and a different key order: +// +// {"app_type", "frameworks", "languages", "auth_model", "auth_details", +// "entry_points", "api_surface", "data_flows"} +func businessLogicContextBlock(recon schemas.ReconResult) string { + return pyfmt.Dumps(pyfmt.O( + "app_type", recon.Architecture.AppType, + "frameworks", recon.Frameworks, + "languages", recon.Languages, + "auth_model", recon.SecurityContext.AuthModel, + "auth_details", recon.SecurityContext.AuthDetails, + "entry_points", headEntryPoints(recon.Architecture.EntryPoints, 15), + "api_surface", headAPIEndpoints(recon.Architecture.APISurface, 20), + "data_flows", headDataFlows(recon.DataFlows.Flows, 20), + ), 2) +} diff --git a/internal/secaf/agents/hunt/crypto.go b/internal/secaf/agents/hunt/crypto.go new file mode 100644 index 0000000000..e1aaddd51e --- /dev/null +++ b/internal/secaf/agents/hunt/crypto.go @@ -0,0 +1,155 @@ +package hunt + +// Ports src/sec_af/agents/hunt/crypto.py. + +import ( + "context" + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/recontext" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +const cryptoPromptPath = "hunt/crypto.txt" + +// cryptoSecurityCriticalTerms ports crypto.py `_SECURITY_CRITICAL_TERMS`. +// Order is load-bearing only in that membership is tested with `any(...)`; the +// matched CONTEXTS are emitted in recon order, not term order. +var cryptoSecurityCriticalTerms = []string{ + "password", "passwd", "credential", "auth", "token", "session", + "encrypt", "decrypt", "signature", "sign", "verify", "jwt", "tls", "ssl", "key", +} + +// cryptoNonSecurityTerms ports crypto.py `_NON_SECURITY_TERMS`. +var cryptoNonSecurityTerms = []string{ + "checksum", "etag", "cache", "fingerprint", "dedup", "integrity", +} + +// cryptoUsageContexts ports crypto.py _usage_contexts: +// +// return [usage.usage_context for usage in recon.security_context.crypto_usage if usage.usage_context] +// +// Python parity: the guard is a TRUTHINESS test on `str | None`, so both None +// AND the empty string are dropped — a nil *string and a "" both skip here. +func cryptoUsageContexts(recon schemas.ReconResult) []string { + out := make([]string, 0, len(recon.SecurityContext.CryptoUsage)) + for _, usage := range recon.SecurityContext.CryptoUsage { + if usage.UsageContext == nil || *usage.UsageContext == "" { + continue + } + out = append(out, *usage.UsageContext) + } + return out +} + +// filterContextsByTerms ports crypto.py _filter_contexts_by_terms — keep every +// context whose LOWERCASED form contains any of the terms, in input order. +// +// Python parity: the terms are already lowercase and are matched as plain +// substrings, so "auth token cache" matches BOTH tables ("auth" and "cache") +// and appears in both candidate lists. +func filterContextsByTerms(contexts, terms []string) []string { + filtered := make([]string, 0, len(contexts)) + for _, context := range contexts { + lowered := strings.ToLower(context) + for _, term := range terms { + if strings.Contains(lowered, term) { + filtered = append(filtered, context) + break + } + } + } + return filtered +} + +// ShouldRunCryptoHunter ports crypto.py should_run_crypto_hunter: +// +// return bool(recon.security_context.crypto_usage) +// +// Note this checks the LIST, not the usage contexts: a CryptoUsage with a nil +// usage_context still opens the gate (and then contributes to neither candidate +// list). +func ShouldRunCryptoHunter(recon schemas.ReconResult) bool { + return len(recon.SecurityContext.CryptoUsage) > 0 +} + +// cryptoCandidateList ports the two +// `", ".join(candidates) if candidates else "none"` expressions. +func cryptoCandidateList(candidates []string) string { + if len(candidates) == 0 { + return "none" + } + return strings.Join(candidates, ", ") +} + +// cryptoScanPrompt builds the exact prompt run_crypto_hunter sends. +// +// Python parity: crypto's CONTEXT block has NO depth line — it carries +// "- Hunt strategy: crypto" instead — and its early-stop sentence ends with +// "without credible crypto misuse". earlyStop is where the argument cascade +// lands the depth string for this hunter (see the package doc). +func cryptoScanPrompt(repoPath string, recon schemas.ReconResult, earlyStop string) (scanPrompt, reconContext string) { + reconContext = recontext.ReconContextForCrypto(recon) + usageContexts := cryptoUsageContexts(recon) + securityCritical := filterContextsByTerms(usageContexts, cryptoSecurityCriticalTerms) + nonSecurity := filterContextsByTerms(usageContexts, cryptoNonSecurityTerms) + + template := prompts.MustLoad(cryptoPromptPath) + template = strings.ReplaceAll(template, "{{RECON_CONTEXT}}", reconContext) + template = strings.ReplaceAll(template, "{{LANGUAGE_HINTS}}", recontext.LanguageHintsForContext(recon)) + template = strings.ReplaceAll(template, "{{FRAMEWORK_HINTS}}", recontext.FrameworkHintsForContext(recon)) + + scanPrompt = template + + "\n\nCONTEXT:\n" + + "- Repository path: " + repoPath + "\n" + + "- Hunt strategy: crypto\n" + + "- Early stop rule: if you inspect " + earlyStop + + " files without credible crypto misuse, stop and return empty findings.\n" + + "- Focus CWEs: CWE-326, CWE-327, CWE-328, CWE-330, CWE-916, CWE-259, CWE-321, CWE-798\n" + + "- Security-critical usage candidates: " + cryptoCandidateList(securityCritical) + "\n" + + "- Non-security usage candidates: " + cryptoCandidateList(nonSecurity) + "\n" + + "- Prioritize weak crypto findings only when used in security-sensitive contexts; avoid checksum/cache-only noise.\n" + + "- Take multiple turns to explore relevant files before finalizing findings.\n" + + "- Write final JSON only when analysis is complete." + return scanPrompt, reconContext +} + +func runCryptoHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, earlyStop string, +) (schemas.HuntResult, error) { + // Python parity: the gate returns a BARE HuntResult() — strategies_run + // stays [] — while the "scanner found nothing" return below names the + // strategy. The two early exits are deliberately different shapes. + if !ShouldRunCryptoHunter(recon) { + return schemas.NewHuntResult(), nil + } + scanPrompt, reconContext := cryptoScanPrompt(repoPath, recon, earlyStop) + return runHunterBody(ctx, app, repoPath, hunterSpec{ + ScanPrompt: scanPrompt, + ReconContext: reconContext, + FindingType: "sast", + Strategy: string(schemas.HuntStrategyCrypto), + EmptyStrategiesRun: []string{string(schemas.HuntStrategyCrypto)}, + }) +} + +// RunCryptoHunter ports crypto.py run_crypto_hunter: +// +// async def run_crypto_hunter(app, repo_path, recon, +// max_files_without_signal: int = 30) -> HuntResult +// +// Note the third parameter is named `recon`, not `recon_result`, and there is +// no `depth` at all — which is exactly why __init__.py's argument cascade falls +// through to its POSITIONAL shape for this hunter and lands the depth string in +// max_files_without_signal. This exported entry point is the honest one, used +// by src/sec_af/reasoners/hunt.py. +func RunCryptoHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, maxFilesWithoutSignal int, +) (schemas.HuntResult, error) { + return runCryptoHunter(ctx, app, repoPath, recon, strconv.Itoa(maxFilesWithoutSignal)) +} diff --git a/internal/secaf/agents/hunt/data_exposure.go b/internal/secaf/agents/hunt/data_exposure.go new file mode 100644 index 0000000000..c826443979 --- /dev/null +++ b/internal/secaf/agents/hunt/data_exposure.go @@ -0,0 +1,66 @@ +package hunt + +// Ports src/sec_af/agents/hunt/data_exposure.py. + +import ( + "context" + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/recontext" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +const dataExposurePromptPath = "hunt/data_exposure.txt" + +// dataExposureScanPrompt builds the exact prompt run_data_exposure_hunter +// sends. +// +// Python parity: no depth line ("- Strategy: data_exposure" instead), and the +// early-stop sentence ends "without credible exposure risk". earlyStop is where +// the argument cascade lands the depth string for this hunter (package doc). +func dataExposureScanPrompt(repoPath string, recon schemas.ReconResult, earlyStop string) (scanPrompt, reconContext string) { + reconContext = recontext.ReconContextForDataExposure(recon) + template := prompts.MustLoad(dataExposurePromptPath) + template = strings.ReplaceAll(template, "{{RECON_CONTEXT}}", reconContext) + template = strings.ReplaceAll(template, "{{LANGUAGE_HINTS}}", recontext.LanguageHintsForContext(recon)) + template = strings.ReplaceAll(template, "{{FRAMEWORK_HINTS}}", recontext.FrameworkHintsForContext(recon)) + + scanPrompt = template + + "\n\nCONTEXT:\n" + + "- Repository path: " + repoPath + "\n" + + "- Strategy: data_exposure\n" + + "- Early stop rule: if you inspect " + + earlyStop + " files without credible exposure risk, " + + "stop and return empty findings.\n" + + "- Use multiple turns: inspect files first, then produce findings.\n" + + "- Return final JSON only when analysis is complete." + return scanPrompt, reconContext +} + +func runDataExposureHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, earlyStop string, +) (schemas.HuntResult, error) { + scanPrompt, reconContext := dataExposureScanPrompt(repoPath, recon, earlyStop) + return runHunterBody(ctx, app, repoPath, hunterSpec{ + ScanPrompt: scanPrompt, + ReconContext: reconContext, + FindingType: "sast", + Strategy: "data_exposure", + EmptyStrategiesRun: []string{"data_exposure"}, + }) +} + +// RunDataExposureHunter ports data_exposure.py run_data_exposure_hunter: +// +// async def run_data_exposure_hunter(app, repo_path, recon, +// max_files_without_signal: int = 30) -> HuntResult +func RunDataExposureHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, maxFilesWithoutSignal int, +) (schemas.HuntResult, error) { + return runDataExposureHunter(ctx, app, repoPath, recon, strconv.Itoa(maxFilesWithoutSignal)) +} diff --git a/internal/secaf/agents/hunt/doc.go b/internal/secaf/agents/hunt/doc.go new file mode 100644 index 0000000000..5c2f845c19 --- /dev/null +++ b/internal/secaf/agents/hunt/doc.go @@ -0,0 +1,72 @@ +// Package hunt ports src/sec_af/agents/hunt — the HUNT phase: twelve hunter +// modules, the two-step scan/enrich harness pipeline they share, and the +// run_hunt / run_hunt_streaming orchestration in that package's __init__.py. +// +// # Shape of a hunter +// +// Every hunter is the same four steps (agents/hunt/_scan_enrich.py): +// +// 1. build a scan prompt: load its own prompt template, substitute a recon +// context (and, for most, the language/framework hint blocks), then append +// a CONTEXT block whose exact wording differs per hunter; +// 2. ScanLocations — one harness call with the ScanLocationsResult schema, +// which returns bare VulnLocations. An empty list short-circuits; +// 3. EnrichLocationsParallel — one harness call per location, at most five at +// a time, each producing an EnrichedFinding; +// 4. AssembleFinding — zip the two lists and coerce them into RawFindings. +// +// # Two Python quirks reproduced here +// +// ARGUMENT CASCADE. `_run_single_hunter` does not know the hunters' signatures. +// It tries five different call shapes in order, moving on whenever Python +// raises TypeError, and the FIRST shape that binds wins: +// +// shape 1 runner(app=, repo_path=, recon_result=, depth=, depth_prompt=, +// max_files_without_signal=, include_paths=) +// shape 2 runner(app=, repo_path=, recon_result=, depth=) +// shape 3 runner(app=, repo_path=, recon_result=, max_files_without_signal=) +// shape 4 runner(app, repo_path, recon_result, depth.value) [positional] +// shape 5 runner(app, repo_path, recon_result) [positional] +// +// No hunter accepts `include_paths` or all of shape 1's keywords, so shape 1 +// never binds. Shape 3 never binds either — the hunters that omit `depth` name +// their third parameter `recon`, not `recon_result`. What is left: +// +// - injection, xss, dos, ssrf, auth, business_logic declare +// `(app, repo_path, recon_result, depth, max_files_without_signal=30, ...)` +// and bind at shape 2, so max_files_without_signal keeps its own default of +// 30 and business_logic's depth_prompt keeps its default of ""; +// - crypto, data_exposure, supply_chain, config_secrets, api_security declare +// `(app, repo_path, recon, max_files_without_signal=30)` and bind at shape +// 4 — POSITIONALLY. The fourth positional argument is `depth.value`, so +// these five hunters receive the DEPTH STRING in their +// max_files_without_signal slot. Python never type-checks it, and the value +// is only ever interpolated into the prompt, so their early-stop rule reads +// "if you inspect standard files without credible crypto misuse". +// +// That is not a bug this port fixes; it is what reaches the LLM today, and the +// goldens under testdata/golden pin it (see prompt_crypto_standard.txt). It +// also makes run_hunt's early_stop_file_threshold parameter DEAD: it is only +// ever passed in shapes 1 and 3, neither of which binds. The Go table in +// hunt.go encodes the settled result of the cascade directly, one closure per +// strategy, rather than re-deriving it at run time. +// +// MISSING-HUNTER STUBS. `_load_hunter` swallows ImportError and substitutes an +// async no-op that returns []. That is not dead code: each hunter module +// imports sec_af.context, and sec_af.context imports back into +// sec_af.agents.hunt._framework_hints, so whichever of the two packages is +// imported FIRST decides the outcome. Import sec_af.agents.hunt first (which is +// what the live node does, app.py -> orchestrator.py -> `.agents.hunt`) and all +// eleven hunters load; import sec_af.context first and all eleven silently +// become no-ops. The Go port has no such hazard — the table is a package-level +// slice of direct references — so the stubs are not ported, only recorded here. +// go/scripts/gen_golden.py forces the correct order and asserts it. +// +// # Package boundaries +// +// The language/framework hint tables live in internal/recontext, not here, +// even though Python declares them in agents/hunt/_framework_hints.py and +// _language_hints.py: internal/recontext is what every hunter needs for its +// recon context, so hosting the tables there is what breaks the import cycle +// Go would otherwise reject. See internal/recontext/hints.go. +package hunt diff --git a/internal/secaf/agents/hunt/dos.go b/internal/secaf/agents/hunt/dos.go new file mode 100644 index 0000000000..cec09fc393 --- /dev/null +++ b/internal/secaf/agents/hunt/dos.go @@ -0,0 +1,62 @@ +package hunt + +// Ports src/sec_af/agents/hunt/dos.py. + +import ( + "context" + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/recontext" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +const dosPromptPath = "hunt/dos.txt" + +// dosScanPrompt builds the exact prompt run_dos_hunter sends, and returns the +// recon context it embedded (reused verbatim for enrichment). +// +// Python parity: dos.py substitutes `{{RECON_CONTEXT_JSON}}` — the inline JSON +// block, not one of context.py's prose builders. +func dosScanPrompt(repoPath string, recon schemas.ReconResult, depth, earlyStop string) (scanPrompt, reconContext string) { + reconContext = entryFlowContextBlock(recon) + template := prompts.MustLoad(dosPromptPath) + template = strings.ReplaceAll(template, "{{RECON_CONTEXT_JSON}}", reconContext) + template = strings.ReplaceAll(template, "{{LANGUAGE_HINTS}}", recontext.LanguageHintsForContext(recon)) + template = strings.ReplaceAll(template, "{{FRAMEWORK_HINTS}}", recontext.FrameworkHintsForContext(recon)) + + scanPrompt = template + + "\n\nCONTEXT:\n" + + "- Repository path: " + repoPath + "\n" + + "- Depth profile: " + depth + "\n" + + "- Early stop rule: if you inspect " + earlyStop + + " files without credible signal, stop and return empty findings.\n" + + "- Focus on RECON entry points and data flows where unbounded work can be attacker-controlled.\n" + + "- Explore code paths that can trigger excessive CPU, memory, I/O, or external-service consumption.\n" + + "- Take multiple turns to build findings incrementally and write final JSON only when complete." + return scanPrompt, reconContext +} + +func runDosHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, depth, earlyStop string, +) (schemas.HuntResult, error) { + scanPrompt, reconContext := dosScanPrompt(repoPath, recon, depth, earlyStop) + return runHunterBody(ctx, app, repoPath, hunterSpec{ + ScanPrompt: scanPrompt, + ReconContext: reconContext, + FindingType: "sast", + Strategy: "dos", + EmptyStrategiesRun: nil, // Python: bare HuntResult() + }) +} + +// RunDosHunter ports dos.py run_dos_hunter. +func RunDosHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, depth string, maxFilesWithoutSignal int, +) (schemas.HuntResult, error) { + return runDosHunter(ctx, app, repoPath, recon, depth, strconv.Itoa(maxFilesWithoutSignal)) +} diff --git a/internal/secaf/agents/hunt/golden_test.go b/internal/secaf/agents/hunt/golden_test.go new file mode 100644 index 0000000000..6ca5f1a699 --- /dev/null +++ b/internal/secaf/agents/hunt/golden_test.go @@ -0,0 +1,317 @@ +package hunt + +// Shared helpers for the golden-fixture tests in this package. +// +// Every fixture under testdata/golden is produced by go/scripts/gen_golden.py +// running the REAL Python code from src/sec_af/agents/hunt. Regenerate with: +// +// PYTHONPATH=src ~/.agentfield/packages/sec-af/venv/bin/python go/scripts/gen_golden.py +// +// A test failing here means the Go port and the Python source disagree about +// bytes that reach the LLM (prompts) or the wire (assembled findings) — not +// that a fixture needs refreshing. Refresh only after a deliberate Python +// change. + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +const ( + goldenDir = "testdata/golden" + testdataDir = "testdata" +) + +// fixtureRepo is gen_golden.py's _S4_FIXTURE_REPO — a path that deliberately +// does not exist on disk, so nothing in these tests touches the filesystem +// beyond the harness scratch dirs. +const fixtureRepo = "/fixtures/demo-repo" + +// goldenText reads a *.txt fixture verbatim. +func goldenText(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join(goldenDir, name+".txt")) + if err != nil { + t.Fatalf("read golden %s.txt: %v", name, err) + } + return string(b) +} + +// goldenJSON decodes a *.json fixture into dest. +func goldenJSON(t *testing.T, name string, dest any) { + t.Helper() + b, err := os.ReadFile(filepath.Join(goldenDir, name+".json")) + if err != nil { + t.Fatalf("read golden %s.json: %v", name, err) + } + if err := json.Unmarshal(b, dest); err != nil { + t.Fatalf("decode golden %s.json: %v", name, err) + } +} + +// loadRecon binds one of the two committed ReconResult fixtures. +// +// recon_fixture.json — the rich shared fixture (a copy of internal/recontext's) +// recon_small.json — the small-but-complete one gen_golden.py emits +func loadRecon(t *testing.T, name string) schemas.ReconResult { + t.Helper() + b, err := os.ReadFile(filepath.Join(testdataDir, name+".json")) + if err != nil { + t.Fatalf("read fixture %s.json: %v", name, err) + } + var recon schemas.ReconResult + if err := json.Unmarshal(b, &recon); err != nil { + t.Fatalf("decode fixture %s.json: %v", name, err) + } + return recon +} + +// emptyRecon reproduces gen_golden.py's _s4_recon_empty: every pydantic default +// plus the one non-default SecurityContext the Python fixture sets. +func emptyRecon() schemas.ReconResult { + recon := schemas.NewReconResult() + recon.SecurityContext.AuthModel = "session" + recon.SecurityContext.AuthDetails = "cookie" + return recon +} + +// assertTextEqual compares a rendered string against a golden and reports the +// first differing line, which is far more useful than a 20 KB diff. +func assertTextEqual(t *testing.T, name, got, want string) { + t.Helper() + if got == want { + return + } + gotLines := strings.Split(got, "\n") + wantLines := strings.Split(want, "\n") + for i := 0; i < len(gotLines) || i < len(wantLines); i++ { + var g, w string + if i < len(gotLines) { + g = gotLines[i] + } + if i < len(wantLines) { + w = wantLines[i] + } + if g != w { + t.Fatalf("%s: first difference at line %d\n go: %q\n python: %q\n(got %d lines / %d bytes, want %d lines / %d bytes)", + name, i+1, g, w, len(gotLines), len(got), len(wantLines), len(want)) + } + } + t.Fatalf("%s: strings differ but no differing line found (got %d bytes, want %d bytes)", name, len(got), len(want)) +} + +// sha256Hex is the digest gen_golden.py pins prompts by when the full text +// would be redundant. +func sha256Hex(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +} + +// jsonTree marshals v and decodes the result into the untyped tree shape the +// golden fixtures decode to, so the two compare without either side's Go types +// leaking in. +func jsonTree(t *testing.T, v any) any { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatalf("marshal %T: %v", v, err) + } + var tree any + if err := json.Unmarshal(b, &tree); err != nil { + t.Fatalf("unmarshal %T: %v", v, err) + } + return tree +} + +// scrubIDs replaces every string under an "id" or "fingerprint" key with the +// placeholder gen_golden.py writes. RawFinding mints both as fresh uuid4s +// (pydantic default_factory / schemas.NewRawFinding), so they are +// nondeterministic by construction. +func scrubIDs(v any) any { + switch t := v.(type) { + case map[string]any: + out := make(map[string]any, len(t)) + for k, val := range t { + if k == "id" || k == "fingerprint" { + if _, isStr := val.(string); isStr { + out[k] = "" + continue + } + } + out[k] = scrubIDs(val) + } + return out + case []any: + out := make([]any, len(t)) + for i, val := range t { + out[i] = scrubIDs(val) + } + return out + default: + return v + } +} + +// diffJSON renders got/want for a readable failure message. +func diffJSON(got, want any) string { + g, _ := json.MarshalIndent(got, "", " ") + w, _ := json.MarshalIndent(want, "", " ") + return fmt.Sprintf("\n--- go ---\n%s\n--- python ---\n%s", g, w) +} + +// --------------------------------------------------------------------------- +// harness fakes +// --------------------------------------------------------------------------- + +// schemaTitle is how the fakes tell a scan call from an enrich call — the same +// discriminator gen_golden.py's _S4App uses (`schema.__name__`), since the +// embedded pydantic fixtures carry their class name as "title". +func schemaTitle(schema map[string]any) string { + if schema == nil { + return "" + } + title, _ := schema["title"].(string) + return title +} + +// cannedLocations reproduces gen_golden.py's _s4_locations: a multi-line +// snippet (two lines, so end_line == start_line + 1) and a single-line one. +func cannedLocations() []schemas.VulnLocation { + return []schemas.VulnLocation{ + { + FilePath: "app/api/users.py", + StartLine: 42, + CodeSnippet: "query = f\"SELECT * FROM users WHERE id = {user_id}\"\ncursor.execute(query)", + PatternType: "sql_injection", + }, + { + FilePath: "app/utils/hash.py", + StartLine: 7, + CodeSnippet: "digest = hashlib.md5(password).hexdigest()", + PatternType: "weak_hash", + }, + } +} + +// cannedEnriched reproduces gen_golden.py's _s4_enriched: a well-formed +// enrichment and a coercion torture case (unknown severity, unknown confidence, +// whitespace-only data-flow summary). +func cannedEnriched() []schemas.EnrichedFinding { + return []schemas.EnrichedFinding{ + { + Title: "SQL injection in user lookup", + Description: "user_id flows unescaped into an f-string query.", + CweID: "CWE-89", + Severity: "HIGH", + Confidence: "high", + DataFlowSummary: " request.args['id'] -> query -> cursor.execute ", + }, + { + Title: "Weak hash for password storage", + Description: "MD5 used to derive a credential digest.", + CweID: "CWE-327", + Severity: "catastrophic", + Confidence: "certain", + DataFlowSummary: " ", + }, + } +} + +// huntFake is the Go twin of gen_golden.py's _S4App: it answers app.harness by +// the schema requested, hands back canned values and records every prompt. +// +// One deliberate difference from the Python fake. _S4App pairs the i-th ENRICH +// CALL with enriched[i], which is well defined there because asyncio runs the +// gathered coroutines to completion in creation order when nothing really +// suspends. Go runs the enrichment fan-out on real goroutines, so call order is +// nondeterministic; the fake therefore pairs by LOCATION, parsing the location +// block out of the prompt. That reproduces exactly the pairing Python observes +// (locations[i] <-> enriched[i]) without depending on scheduling. +type huntFake struct { + *appx.Fake + locations []schemas.VulnLocation + enriched []schemas.EnrichedFinding +} + +// newHuntFake builds a fake that returns locations from step 1 and, for step 2, +// the enrichment belonging to the location the prompt names. +func newHuntFake(locations []schemas.VulnLocation, enriched []schemas.EnrichedFinding) *huntFake { + f := &huntFake{Fake: &appx.Fake{}, locations: locations, enriched: enriched} + f.HarnessFn = func(_ context.Context, prompt string, schema map[string]any, dest any, _ appx.HarnessOptions) (*appx.HarnessResult, error) { + switch schemaTitle(schema) { + case "ScanLocationsResult": + result := dest.(*schemas.ScanLocationsResult) + *result = schemas.NewScanLocationsResult() + result.Locations = append(result.Locations, f.locations...) + return &appx.HarnessResult{Parsed: dest}, nil + case "EnrichedFinding": + index, ok := f.locationIndex(prompt) + if !ok { + return nil, fmt.Errorf("huntFake: enrich prompt names no known location") + } + result := dest.(*schemas.EnrichedFinding) + *result = f.enriched[index%len(f.enriched)] + return &appx.HarnessResult{Parsed: dest}, nil + default: + return nil, fmt.Errorf("huntFake: unexpected harness schema %q", schemaTitle(schema)) + } + } + return f +} + +// locationBlock renders the three prompt lines enrich_finding.txt fills from a +// VulnLocation. The triple is unique per location in every fixture, so it is a +// sound key. +func locationBlock(location schemas.VulnLocation) string { + return "- File path: " + location.FilePath + "\n" + + "- Start line: " + strconv.Itoa(location.StartLine) + "\n" + + "- Pattern type: " + location.PatternType + "\n" +} + +func (f *huntFake) locationIndex(prompt string) (int, bool) { + for i, location := range f.locations { + if strings.Contains(prompt, locationBlock(location)) { + return i, true + } + } + return 0, false +} + +// scanPrompts returns the recorded step-1 prompts, in call order. +func (f *huntFake) scanPrompts() []string { return f.promptsFor("ScanLocationsResult") } + +// enrichPrompts returns the recorded step-2 prompts. Their ORDER is the +// goroutine completion order and is not meaningful; tests that care about a +// specific location's prompt build it with EnrichPrompt instead. +func (f *huntFake) enrichPrompts() []string { return f.promptsFor("EnrichedFinding") } + +func (f *huntFake) promptsFor(title string) []string { + var out []string + for _, call := range f.Harnesses { + if schemaTitle(call.Schema) == title { + out = append(out, call.Prompt) + } + } + return out +} + +// onlyScanPrompt fails unless exactly one step-1 prompt was recorded. +func (f *huntFake) onlyScanPrompt(t *testing.T) string { + t.Helper() + prompts := f.scanPrompts() + if len(prompts) != 1 { + t.Fatalf("want exactly 1 scan prompt, got %d", len(prompts)) + } + return prompts[0] +} diff --git a/internal/secaf/agents/hunt/hunt.go b/internal/secaf/agents/hunt/hunt.go new file mode 100644 index 0000000000..1e603c873c --- /dev/null +++ b/internal/secaf/agents/hunt/hunt.go @@ -0,0 +1,484 @@ +package hunt + +// Ports the orchestration half of src/sec_af/agents/hunt/__init__.py: +// _STRATEGY_RUNNERS, _QUICK_STRATEGIES, _select_strategies, _extract_findings, +// _run_single_hunter, run_hunt and run_hunt_streaming. + +import ( + "context" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/sync/semaphore" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/agents/dedup" + "github.com/Agent-Field/codeaf/internal/secaf/config" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// The two keyword defaults run_hunt / run_hunt_streaming declare. Go has no +// default arguments, so callers name them explicitly. +const ( + // DefaultMaxConcurrentHunters ports `max_concurrent_hunters: int = 4`. + DefaultMaxConcurrentHunters = 4 + // DefaultEarlyStopFileThreshold ports `early_stop_file_threshold: int = 30`. + // + // It has NO observable effect: _run_single_hunter only ever passes it in + // the two call shapes that never bind (package doc). It is kept so the Go + // signature matches Python's and so a future fix to the cascade has an + // obvious home. + DefaultEarlyStopFileThreshold = 30 +) + +// deduplicateAndCorrelate is the seam both run_hunt variants use to reach +// src/sec_af/agents/dedup.py deduplicate_and_correlate. It is a variable rather +// than a direct call for one reason: tests/test_hunt_include_paths.py +// monkeypatches exactly that module attribute +// (`monkeypatch.setattr(hunt_module, "deduplicate_and_correlate", ...)`), and +// the ported test needs the same substitution point. Production code never +// reassigns it. +var deduplicateAndCorrelate = dedup.DeduplicateAndCorrelate + +// hunterInvocation is the SETTLED form of _run_single_hunter's argument +// cascade for one strategy — the call Python actually ends up making once its +// five-shape TypeError probe finishes. Encoding the outcome in the table (one +// closure per strategy) is what the package doc describes; re-deriving it at +// run time would mean reproducing Python's argument binding, which Go cannot do +// and which would be far less legible. +type hunterInvocation func( + ctx context.Context, + app appx.Harnesser, + repoPath string, + recon schemas.ReconResult, + depth config.DepthProfile, +) (schemas.HuntResult, error) + +// strategyEntry is one row of the ordered strategy table. +type strategyEntry struct { + Strategy schemas.HuntStrategy + Run hunterInvocation +} + +// strategyRunners ports _STRATEGY_RUNNERS. +// +// Python's dict preserves insertion order and `_select_strategies` returns +// `list(_STRATEGY_RUNNERS)` — the KEYS in that order — so the order below is +// observable: it is the order hunters are launched in, and the order +// `HuntResult.strategies_run` reports. +// +// Each closure spells out the effective argument list for its hunter: +// +// - the six hunters that declare `depth` bind at the cascade's keyword shape, +// so they get the depth and their OWN max_files_without_signal default (30); +// - the five that do not declare `depth` bind POSITIONALLY, so `depth.value` +// lands in their max_files_without_signal slot and reaches the prompt as +// text. `string(depth)` below is that argument, and it is deliberate. +// +// Python parity: _load_hunter's ImportError fallback (a stub returning []) is +// not represented. See the package doc for why it can fire in Python and why it +// cannot here. +var strategyRunners = []strategyEntry{ + {schemas.HuntStrategyInjection, func(ctx context.Context, app appx.Harnesser, repoPath string, recon schemas.ReconResult, depth config.DepthProfile) (schemas.HuntResult, error) { + return runInjectionHunter(ctx, app, repoPath, recon, string(depth), "30") + }}, + {schemas.HuntStrategyXSS, func(ctx context.Context, app appx.Harnesser, repoPath string, recon schemas.ReconResult, depth config.DepthProfile) (schemas.HuntResult, error) { + return runXSSHunter(ctx, app, repoPath, recon, string(depth), "30") + }}, + {schemas.HuntStrategyDos, func(ctx context.Context, app appx.Harnesser, repoPath string, recon schemas.ReconResult, depth config.DepthProfile) (schemas.HuntResult, error) { + return runDosHunter(ctx, app, repoPath, recon, string(depth), "30") + }}, + {schemas.HuntStrategySSRF, func(ctx context.Context, app appx.Harnesser, repoPath string, recon schemas.ReconResult, depth config.DepthProfile) (schemas.HuntResult, error) { + return runSSRFHunter(ctx, app, repoPath, recon, string(depth), "30") + }}, + {schemas.HuntStrategyAuth, func(ctx context.Context, app appx.Harnesser, repoPath string, recon schemas.ReconResult, depth config.DepthProfile) (schemas.HuntResult, error) { + return runAuthHunter(ctx, app, repoPath, recon, string(depth), "30") + }}, + {schemas.HuntStrategyCrypto, func(ctx context.Context, app appx.Harnesser, repoPath string, recon schemas.ReconResult, depth config.DepthProfile) (schemas.HuntResult, error) { + // Positional bind: depth.value lands in max_files_without_signal. + return runCryptoHunter(ctx, app, repoPath, recon, string(depth)) + }}, + {schemas.HuntStrategyBusinessLogic, func(ctx context.Context, app appx.Harnesser, repoPath string, recon schemas.ReconResult, depth config.DepthProfile) (schemas.HuntResult, error) { + // depth_prompt stays "" — the cascade never delivers the one + // _run_single_hunter computes. + return runBusinessLogicHunter(ctx, app, repoPath, recon, string(depth), "30", "") + }}, + {schemas.HuntStrategyDataExposure, func(ctx context.Context, app appx.Harnesser, repoPath string, recon schemas.ReconResult, depth config.DepthProfile) (schemas.HuntResult, error) { + return runDataExposureHunter(ctx, app, repoPath, recon, string(depth)) + }}, + {schemas.HuntStrategySupplyChain, func(ctx context.Context, app appx.Harnesser, repoPath string, recon schemas.ReconResult, depth config.DepthProfile) (schemas.HuntResult, error) { + return runSupplyChainHunter(ctx, app, repoPath, recon, string(depth)) + }}, + {schemas.HuntStrategyConfigSecrets, func(ctx context.Context, app appx.Harnesser, repoPath string, recon schemas.ReconResult, depth config.DepthProfile) (schemas.HuntResult, error) { + return runConfigSecretsHunter(ctx, app, repoPath, recon, string(depth)) + }}, + {schemas.HuntStrategyAPISecurity, func(ctx context.Context, app appx.Harnesser, repoPath string, recon schemas.ReconResult, depth config.DepthProfile) (schemas.HuntResult, error) { + return runAPISecurityHunter(ctx, app, repoPath, recon, string(depth)) + }}, +} + +// strategyRunnerIndex is the `_STRATEGY_RUNNERS[strategy]` lookup. +var strategyRunnerIndex = func() map[schemas.HuntStrategy]hunterInvocation { + out := make(map[schemas.HuntStrategy]hunterInvocation, len(strategyRunners)) + for _, entry := range strategyRunners { + out[entry.Strategy] = entry.Run + } + return out +}() + +// quickStrategies ports _QUICK_STRATEGIES — the five-strategy tuple the QUICK +// profile runs, in its own order (note it is NOT a prefix of the table: xss and +// crypto are skipped, data_exposure is promoted). +var quickStrategies = []schemas.HuntStrategy{ + schemas.HuntStrategyInjection, + schemas.HuntStrategyDos, + schemas.HuntStrategySSRF, + schemas.HuntStrategyAuth, + schemas.HuntStrategyDataExposure, +} + +// QuickStrategies returns a copy of _QUICK_STRATEGIES. +func QuickStrategies() []schemas.HuntStrategy { + return append([]schemas.HuntStrategy(nil), quickStrategies...) +} + +// AllStrategies returns the table's keys in order — `list(_STRATEGY_RUNNERS)`. +func AllStrategies() []schemas.HuntStrategy { + out := make([]schemas.HuntStrategy, 0, len(strategyRunners)) + for _, entry := range strategyRunners { + out = append(out, entry.Strategy) + } + return out +} + +// SelectStrategies ports _select_strategies: +// +// if depth == DepthProfile.QUICK: return list(_QUICK_STRATEGIES) +// return list(_STRATEGY_RUNNERS) +// +// Anything that is not QUICK — including a depth the lenient normalizer already +// turned into STANDARD — runs the full eleven. +func SelectStrategies(depth config.DepthProfile) []schemas.HuntStrategy { + if depth == config.DepthQuick { + return QuickStrategies() + } + return AllStrategies() +} + +// extractFindings ports _extract_findings. +// +// Python probes four shapes in turn (a HuntResult, a bare list of RawFindings, +// a `.parsed` attribute holding either, and a `.findings` attribute) because +// the runner is typed `Callable[..., Awaitable[object]]` and could be the +// _missing_hunter stub, which returns a list. Go's hunterInvocation returns a +// concrete HuntResult, so only the first branch is reachable — the rest are +// Python duck-typing and are deliberately not ported. +func extractFindings(result schemas.HuntResult) []schemas.RawFinding { + return result.Findings +} + +// runSingleHunter ports _run_single_hunter, minus the argument cascade the +// table already resolved (package doc). +// +// Python parity: the `depth_prompt` this function computes for THOROUGH — +// "Use deep, multi-turn analysis. Trace cross-file flows and hunt secondary +// pivots." — only appears in the cascade's first call shape, which never binds, +// so it never reaches a hunter. It is not recreated here; the string lives on +// in ThoroughDepthPrompt for the callers that pass it deliberately. +func runSingleHunter( + ctx context.Context, + run hunterInvocation, + app appx.Harnesser, + repoPath string, + recon schemas.ReconResult, + depth config.DepthProfile, +) ([]schemas.RawFinding, error) { + result, err := run(ctx, app, repoPath, recon, depth) + if err != nil { + return nil, err + } + return extractFindings(result), nil +} + +// ThoroughDepthPrompt is the extra guidance _run_single_hunter builds for the +// THOROUGH profile. It is dead in the in-process path (see runSingleHunter) but +// is the exact string a deliberate caller — RunBusinessLogicHunter's +// depthPrompt argument, or src/sec_af/reasoners/hunt.py — should pass. +const ThoroughDepthPrompt = "Use deep, multi-turn analysis. Trace cross-file flows and hunt secondary pivots." + +// concurrencyLimit ports `max(1, min(max_concurrent_hunters, len(strategies)))`. +func concurrencyLimit(maxConcurrentHunters, strategyCount int) int { + limit := maxConcurrentHunters + if strategyCount < limit { + limit = strategyCount + } + if limit < 1 { + limit = 1 + } + return limit +} + +// normalizeIncludePaths ports +// `{path.strip() for path in include_paths if path and path.strip()}` — a SET, +// so only membership matters and the order is irrelevant. +func normalizeIncludePaths(includePaths []string) map[string]struct{} { + out := make(map[string]struct{}, len(includePaths)) + for _, path := range includePaths { + trimmed := strings.TrimSpace(path) + if path == "" || trimmed == "" { + continue + } + out[trimmed] = struct{}{} + } + return out +} + +// applyIncludePaths ports the post-dedup filter both run_hunt variants apply: +// +// if include_paths: +// normalized = {...} +// deduplicated.findings = [f for f in deduplicated.findings if f.file_path in normalized] +// +// Python parity: this is an EXACT string match on file_path, not a prefix or +// glob match, and it runs AFTER dedup and chain correlation — so a chain may +// reference a finding that the filter just dropped, and `total_raw` still counts +// the filtered-out findings. +func applyIncludePaths(findings []schemas.RawFinding, includePaths []string) []schemas.RawFinding { + if len(includePaths) == 0 { + return findings + } + normalized := normalizeIncludePaths(includePaths) + kept := make([]schemas.RawFinding, 0, len(findings)) + for _, finding := range findings { + if _, ok := normalized[finding.FilePath]; ok { + kept = append(kept, finding) + } + } + return kept +} + +// strategyValues ports `[strategy.value for strategy in strategies]`. +func strategyValues(strategies []schemas.HuntStrategy) []string { + out := make([]string, 0, len(strategies)) + for _, strategy := range strategies { + out = append(out, string(strategy)) + } + return out +} + +// RunHunt ports src/sec_af/agents/hunt/__init__.py run_hunt: +// +// async def run_hunt(app, repo_path, recon_result, depth, +// max_concurrent_hunters=4, early_stop_file_threshold=30, +// include_paths=None) -> HuntResult +// +// Sequence: normalize the depth leniently, select the strategies, run them all +// under a semaphore, flatten the per-hunter findings in STRATEGY order, hand +// everything to dedup, apply the include-paths filter, then overwrite the four +// counters and the duration. +// +// Concurrency parity: +// +// - `asyncio.gather(..., return_exceptions=True)` means a hunter that raises +// is SILENTLY DROPPED — its findings are simply absent and nothing is +// logged. The Go port reproduces that: a per-index error slot that nothing +// reads. The only error RunHunt can return is dedup's. +// - Results are flattened in strategy order regardless of completion order, +// so the port writes into a pre-sized slice rather than appending from the +// goroutines. +// - The semaphore is held for the WHOLE hunter, which includes that hunter's +// own five-way enrichment fan-out — the two limits multiply. +// +// Python parity: earlyStopFileThreshold is accepted and unused. See the package +// doc; DefaultEarlyStopFileThreshold is the value every caller passes. +func RunHunt( + ctx context.Context, + app appx.Harnesser, + repoPath string, + recon schemas.ReconResult, + depth string, + maxConcurrentHunters int, + earlyStopFileThreshold int, + includePaths []string, +) (schemas.HuntResult, error) { + _ = earlyStopFileThreshold // Python parity: never reaches a hunter. + + started := time.Now() + profile := config.NormalizeDepth(depth) + strategies := SelectStrategies(profile) + + sem := semaphore.NewWeighted(int64(concurrencyLimit(maxConcurrentHunters, len(strategies)))) + + perStrategy := make([][]schemas.RawFinding, len(strategies)) + var wg sync.WaitGroup + for i, strategy := range strategies { + i, strategy := i, strategy + wg.Add(1) + go func() { + defer wg.Done() + if err := sem.Acquire(ctx, 1); err != nil { + // Python: a cancelled task is an exception gather swallows. + return + } + defer sem.Release(1) + findings, err := runSingleHunter( + ctx, strategyRunnerIndex[strategy], app, repoPath, recon, profile, + ) + if err != nil { + // Python: `return_exceptions=True` then `if isinstance(result, + // Exception): continue`. + return + } + perStrategy[i] = findings + }() + } + wg.Wait() + + allFindings := []schemas.RawFinding{} + for _, findings := range perStrategy { + allFindings = append(allFindings, findings...) + } + + deduplicated, err := deduplicateAndCorrelate(ctx, allFindings, recon, app, repoPath) + if err != nil { + return schemas.HuntResult{}, err + } + deduplicated.Findings = applyIncludePaths(deduplicated.Findings, includePaths) + + deduplicated.TotalRaw = len(allFindings) + deduplicated.DeduplicatedCount = len(deduplicated.Findings) + deduplicated.ChainCount = len(deduplicated.Chains) + deduplicated.StrategiesRun = strategyValues(strategies) + deduplicated.HuntDurationSeconds = time.Since(started).Seconds() + return deduplicated, nil +} + +// RunHuntStreaming ports run_hunt_streaming — run_hunt plus an incremental +// fingerprint dedup that publishes each hunter's NEW findings as soon as it +// finishes, so the PROVE phase can start before HUNT is done. +// +// async def run_hunt_streaming(app, repo_path, recon_result, findings_queue, depth, +// max_concurrent_hunters=4, early_stop_file_threshold=30, +// include_paths=None) -> HuntResult +// +// Channel semantics: Python's `asyncio.Queue[list[RawFinding] | None]` carries +// batches and is terminated by a None sentinel. The Go port sends the same +// batches on findings and CLOSES it in place of the sentinel — a closed channel +// is Go's end-of-stream, and it cannot be mistaken for a real (empty) batch the +// way a nil slice could. The channel is always closed, including when a hunter +// fails. +// +// Two deliberate differences from Python, both forced by channels: +// +// - Python's default Queue is UNBOUNDED, so `put` never blocks. A Go channel +// is not: pass a buffer of at least len(SelectStrategies(depth)) if the +// consumer is not draining concurrently, or the producers will stall. +// - The send is guarded by ctx.Done() so a consumer that goes away cannot +// wedge the hunt. Python has no equivalent because it cannot block here. +// +// The fingerprint pass differs from run_hunt's in a way that matters: it seeds +// `finding.fingerprint` with `f"{file_path}:{start_line}:{cwe_id}"` when the +// hunter left it empty, which is NOT the sha256 form dedup.ComputeFingerprint +// produces. Findings therefore reach DeduplicateAndCorrelate already +// fingerprinted, and that function's own seeding never fires for them. +func RunHuntStreaming( + ctx context.Context, + app appx.Harnesser, + repoPath string, + recon schemas.ReconResult, + findings chan<- []schemas.RawFinding, + depth string, + maxConcurrentHunters int, + earlyStopFileThreshold int, + includePaths []string, +) (schemas.HuntResult, error) { + _ = earlyStopFileThreshold // Python parity: never reaches a hunter. + + started := time.Now() + profile := config.NormalizeDepth(depth) + strategies := SelectStrategies(profile) + + sem := semaphore.NewWeighted(int64(concurrencyLimit(maxConcurrentHunters, len(strategies)))) + + var ( + mu sync.Mutex + // allRawFindings ports `all_raw_findings`; only its length is read. + allRawCount int + // fingerprintDeduped ports the dict, plus the key order Python's dict + // preserves and `list(...values())` depends on. + fingerprintDeduped = map[string]*schemas.RawFinding{} + fingerprintOrder []string + ) + + var wg sync.WaitGroup + for _, strategy := range strategies { + strategy := strategy + wg.Add(1) + go func() { + defer wg.Done() + + if err := sem.Acquire(ctx, 1); err != nil { + return + } + hunterFindings, err := runSingleHunter( + ctx, strategyRunnerIndex[strategy], app, repoPath, recon, profile, + ) + sem.Release(1) // Python releases before taking the dedup lock. + if err != nil { + // Python: gather(return_exceptions=True) swallows it, and the + // task never reaches its queue put. + return + } + + var newFindings []schemas.RawFinding + mu.Lock() + allRawCount += len(hunterFindings) + for i := range hunterFindings { + finding := &hunterFindings[i] + if finding.Fingerprint == "" { + // Python: `finding.fingerprint or f"{file_path}:{start_line}:{cwe_id}"`. + finding.Fingerprint = finding.FilePath + ":" + + strconv.Itoa(finding.StartLine) + ":" + finding.CweID + } + if _, seen := fingerprintDeduped[finding.Fingerprint]; seen { + continue + } + fingerprintDeduped[finding.Fingerprint] = finding + fingerprintOrder = append(fingerprintOrder, finding.Fingerprint) + newFindings = append(newFindings, *finding) + } + mu.Unlock() + + if len(newFindings) > 0 { + select { + case findings <- newFindings: + case <-ctx.Done(): + } + } + }() + } + wg.Wait() + close(findings) // Python: `await findings_queue.put(None)`. + + unique := make([]schemas.RawFinding, 0, len(fingerprintOrder)) + for _, fingerprint := range fingerprintOrder { + unique = append(unique, *fingerprintDeduped[fingerprint]) + } + + deduplicated, err := deduplicateAndCorrelate(ctx, unique, recon, app, repoPath) + if err != nil { + return schemas.HuntResult{}, err + } + deduplicated.Findings = applyIncludePaths(deduplicated.Findings, includePaths) + + // Python parity: total_raw counts EVERY finding the hunters produced, not + // the fingerprint-unique list that was handed to dedup. + deduplicated.TotalRaw = allRawCount + deduplicated.DeduplicatedCount = len(deduplicated.Findings) + deduplicated.ChainCount = len(deduplicated.Chains) + deduplicated.StrategiesRun = strategyValues(strategies) + deduplicated.HuntDurationSeconds = time.Since(started).Seconds() + return deduplicated, nil +} diff --git a/internal/secaf/agents/hunt/hunt_test.go b/internal/secaf/agents/hunt/hunt_test.go new file mode 100644 index 0000000000..391e127049 --- /dev/null +++ b/internal/secaf/agents/hunt/hunt_test.go @@ -0,0 +1,684 @@ +package hunt + +// Tests for the orchestration half of src/sec_af/agents/hunt/__init__.py. +// +// Validation contract (behaviour, not implementation): +// +// - QUICK selects exactly the five _QUICK_STRATEGIES, in their own order; +// every other depth — including an unrecognised one — selects all eleven in +// table order; +// - each selected hunter receives the arguments Python's TypeError cascade +// settles on, which is observable in its scan prompt's early-stop line; +// - hunters run at most max(1, min(max_concurrent_hunters, len(strategies))) +// at a time, and a hunter that fails is silently dropped rather than +// failing the phase; +// - findings are flattened in STRATEGY order, handed to dedup, then filtered +// by include_paths on an exact file_path match, and the four counters plus +// strategies_run are overwritten afterwards; +// - early_stop_file_threshold changes nothing; +// - the streaming variant publishes each hunter's fingerprint-NEW findings as +// a batch, seeds a missing fingerprint as "::", terminates +// the stream once, and reports total_raw over ALL findings rather than the +// unique ones. + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strconv" + "strings" + "sync/atomic" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/config" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// --------------------------------------------------------------------------- +// strategy selection +// --------------------------------------------------------------------------- + +// TestSelectStrategiesMatchesPython pins _normalize_depth + _select_strategies +// over the same depth spellings gen_golden.py ran. +func TestSelectStrategiesMatchesPython(t *testing.T) { + var selected map[string][]string + goldenJSON(t, "select_strategies", &selected) + if len(selected) == 0 { + t.Fatal("select_strategies golden is empty") + } + for depth, want := range selected { + got := strategyValues(SelectStrategies(config.NormalizeDepth(depth))) + if !reflect.DeepEqual(got, want) { + t.Errorf("SelectStrategies(NormalizeDepth(%q)) = %v, want %v", depth, got, want) + } + } + + var normalized map[string]string + goldenJSON(t, "normalize_depth", &normalized) + for depth, want := range normalized { + if got := string(config.NormalizeDepth(depth)); got != want { + t.Errorf("NormalizeDepth(%q) = %q, want %q", depth, got, want) + } + } +} + +// TestStrategyTablesMatchPython pins the two ordered tables themselves. +func TestStrategyTablesMatchPython(t *testing.T) { + var quick []string + goldenJSON(t, "quick_strategies", &quick) + if got := strategyValues(QuickStrategies()); !reflect.DeepEqual(got, quick) { + t.Errorf("_QUICK_STRATEGIES = %v, want %v", got, quick) + } + + var all []string + goldenJSON(t, "strategy_runner_order", &all) + if got := strategyValues(AllStrategies()); !reflect.DeepEqual(got, all) { + t.Errorf("_STRATEGY_RUNNERS order = %v, want %v", got, all) + } + if len(strategyRunnerIndex) != len(all) { + t.Errorf("the lookup map has %d entries, the table has %d", len(strategyRunnerIndex), len(all)) + } + for _, strategy := range AllStrategies() { + if strategyRunnerIndex[strategy] == nil { + t.Errorf("no runner registered for %q", strategy) + } + } +} + +// TestQuickStrategiesIsACopy guards the accessors against a caller mutating the +// package tables. +func TestQuickStrategiesIsACopy(t *testing.T) { + first := QuickStrategies() + first[0] = "clobbered" + if QuickStrategies()[0] != schemas.HuntStrategyInjection { + t.Fatal("QuickStrategies handed out the package slice") + } + all := AllStrategies() + all[0] = "clobbered" + if AllStrategies()[0] != schemas.HuntStrategyInjection { + t.Fatal("AllStrategies handed out the package slice") + } +} + +// --------------------------------------------------------------------------- +// the argument cascade, as seen in the prompts +// --------------------------------------------------------------------------- + +// TestCascadePromptsMatchPython runs every strategy the way run_hunt does, at +// every depth, and compares the resulting scan prompt with the one Python +// produced through the real _run_single_hunter cascade. +// +// This is the test that pins the quirk: for crypto, data_exposure, +// supply_chain, config_secrets and api_security the early-stop line reads +// "if you inspect standard files", because the cascade binds POSITIONALLY and +// depth.value lands in max_files_without_signal. +func TestCascadePromptsMatchPython(t *testing.T) { + var cascade map[string]struct { + Fixture string `json:"fixture"` + Hunters map[string]map[string]any `json:"hunters"` + } + goldenJSON(t, "cascade_binding", &cascade) + if len(cascade) != 3 { + t.Fatalf("cascade_binding golden covers %d depths, want 3", len(cascade)) + } + + fixtures := map[string]schemas.ReconResult{ + "rich": loadRecon(t, "recon_fixture"), + "small": loadRecon(t, "recon_small"), + } + + for depth, entry := range cascade { + depth, entry := depth, entry + t.Run(depth, func(t *testing.T) { + recon, ok := fixtures[entry.Fixture] + if !ok { + t.Fatalf("unknown fixture %q", entry.Fixture) + } + profile := config.NormalizeDepth(depth) + selected := strategyValues(SelectStrategies(profile)) + if len(selected) != len(entry.Hunters) { + t.Fatalf("depth %s selects %d strategies, the golden recorded %d", depth, len(selected), len(entry.Hunters)) + } + + for _, strategy := range SelectStrategies(profile) { + strategy := strategy + t.Run(string(strategy), func(t *testing.T) { + bound, ok := entry.Hunters[string(strategy)] + if !ok { + t.Fatalf("no cascade_binding entry for %s/%s", depth, strategy) + } + fake := newHuntFake(nil, cannedEnriched()) + if _, err := strategyRunnerIndex[strategy]( + context.Background(), fake, fixtureRepo, recon, profile, + ); err != nil { + t.Fatalf("%s: %v", strategy, err) + } + got := fake.onlyScanPrompt(t) + assertTextEqual(t, "prompt_"+string(strategy)+"_"+depth, got, + goldenText(t, "prompt_"+string(strategy)+"_"+depth)) + + // And state the quirk directly, so a reader of the test does + // not have to diff two 15 KB prompts to see it. + want := renderBound(t, bound["max_files_without_signal"]) + if !strings.Contains(got, "if you inspect "+want+" ") { + t.Fatalf("%s/%s: early-stop line does not interpolate %q", depth, strategy, want) + } + }) + } + }) + } +} + +// renderBound formats the max_files_without_signal value the golden recorded — +// a JSON number for the six hunters that take a depth, a JSON string (the depth +// itself) for the five that do not. +func renderBound(t *testing.T, v any) string { + t.Helper() + switch value := v.(type) { + case string: + return value + case float64: + return strconv.Itoa(int(value)) + default: + t.Fatalf("unexpected max_files_without_signal %T (%v)", v, v) + return "" + } +} + +// TestCascadeNeverPassesTheEarlyStopThreshold states, in one place, what the +// binding table means: every hunter is reached with either its own 30 default +// or the depth string, and never with run_hunt's early_stop_file_threshold. +func TestCascadeNeverPassesTheEarlyStopThreshold(t *testing.T) { + var cascade map[string]struct { + Hunters map[string]map[string]any `json:"hunters"` + } + goldenJSON(t, "cascade_binding", &cascade) + for depth, entry := range cascade { + for strategy, bound := range entry.Hunters { + got := renderBound(t, bound["max_files_without_signal"]) + if got != "30" && got != depth { + t.Errorf("%s/%s bound max_files_without_signal to %q, want \"30\" or %q", depth, strategy, got, depth) + } + if _, ok := bound["include_paths"]; ok { + t.Errorf("%s/%s received include_paths, which no hunter accepts", depth, strategy) + } + if depthPrompt, ok := bound["depth_prompt"]; ok && depthPrompt != "" { + t.Errorf("%s/%s received depth_prompt %q, want the empty default", depth, strategy, depthPrompt) + } + } + } +} + +// --------------------------------------------------------------------------- +// run_hunt +// --------------------------------------------------------------------------- + +// withStrategyTable temporarily replaces the strategy table, which is how the +// Python tests reach run_hunt: +// +// monkeypatch.setattr(hunt_module, "_select_strategies", lambda _d: [INJECTION]) +// monkeypatch.setitem(hunt_module._STRATEGY_RUNNERS, HuntStrategy.INJECTION, fake_runner) +func withStrategyTable(t *testing.T, entries ...strategyEntry) { + t.Helper() + savedRunners := strategyRunners + savedIndex := strategyRunnerIndex + savedQuick := quickStrategies + + strategyRunners = entries + strategyRunnerIndex = make(map[schemas.HuntStrategy]hunterInvocation, len(entries)) + quickStrategies = nil + for _, entry := range entries { + strategyRunnerIndex[entry.Strategy] = entry.Run + quickStrategies = append(quickStrategies, entry.Strategy) + } + t.Cleanup(func() { + strategyRunners = savedRunners + strategyRunnerIndex = savedIndex + quickStrategies = savedQuick + }) +} + +// withDedup temporarily replaces the dedup seam, the way +// tests/test_hunt_include_paths.py monkeypatches deduplicate_and_correlate. +func withDedup(t *testing.T, fn func(ctx context.Context, findings []schemas.RawFinding, recon schemas.ReconResult, app appx.Harnesser, repoPath string) (schemas.HuntResult, error)) { + t.Helper() + saved := deduplicateAndCorrelate + deduplicateAndCorrelate = fn + t.Cleanup(func() { deduplicateAndCorrelate = saved }) +} + +// passthroughDedup is the Python test's +// `HuntResult(findings=list(all_findings))` stub. +func passthroughDedup(_ context.Context, findings []schemas.RawFinding, _ schemas.ReconResult, _ appx.Harnesser, _ string) (schemas.HuntResult, error) { + result := schemas.NewHuntResult() + result.Findings = append(result.Findings, findings...) + return result, nil +} + +// testFinding ports the `_finding(file_path, suffix)` helper in +// tests/test_hunt_include_paths.py. +func testFinding(filePath, suffix string) schemas.RawFinding { + finding := schemas.NewRawFinding() + finding.ID = "id-" + suffix + finding.HunterStrategy = "injection" + finding.Title = "title-" + suffix + finding.Description = "desc" + finding.FindingType = schemas.FindingTypeSast + finding.CweID = "CWE-89" + finding.CweName = "SQL Injection" + finding.FilePath = filePath + finding.StartLine = 1 + finding.EndLine = 1 + finding.CodeSnippet = "query" + finding.EstimatedSeverity = schemas.SeverityHigh + finding.Confidence = schemas.ConfidenceMedium + finding.Fingerprint = "fp-" + suffix + return finding +} + +// includePathsRecon ports the `_recon_result()` helper in the same file. +func includePathsRecon() schemas.ReconResult { + return emptyRecon() +} + +// TestRunHuntFiltersFindingsByIncludePaths ports +// tests/test_hunt_include_paths.py::test_run_hunt_filters_findings_by_include_paths. +// +// The Python test additionally asserts that its fake runner was handed +// include_paths, which only holds because that fake accepts **kwargs and so +// binds the cascade's FIRST call shape. No real hunter does — the Go +// hunterInvocation signature has no include_paths parameter at all, and +// TestCascadeNeverPassesTheEarlyStopThreshold pins that. What the assertion is +// really about is the observable behaviour below: include_paths filters the +// findings AFTER dedup, while total_raw still counts what the hunters produced. +func TestRunHuntFiltersFindingsByIncludePaths(t *testing.T) { + withStrategyTable(t, strategyEntry{ + Strategy: schemas.HuntStrategyInjection, + Run: func(context.Context, appx.Harnesser, string, schemas.ReconResult, config.DepthProfile) (schemas.HuntResult, error) { + result := schemas.NewHuntResult() + result.Findings = []schemas.RawFinding{ + testFinding("src/keep.py", "keep"), + testFinding("src/drop.py", "drop"), + } + return result, nil + }, + }) + withDedup(t, passthroughDedup) + + result, err := RunHunt( + context.Background(), &appx.Fake{}, ".", includePathsRecon(), "standard", + DefaultMaxConcurrentHunters, DefaultEarlyStopFileThreshold, []string{"src/keep.py"}, + ) + if err != nil { + t.Fatalf("RunHunt: %v", err) + } + + var paths []string + for _, finding := range result.Findings { + paths = append(paths, finding.FilePath) + } + if !reflect.DeepEqual(paths, []string{"src/keep.py"}) { + t.Errorf("findings = %v, want [src/keep.py]", paths) + } + if result.TotalRaw != 2 { + t.Errorf("total_raw = %d, want 2", result.TotalRaw) + } + if result.DeduplicatedCount != 1 { + t.Errorf("deduplicated_count = %d, want 1", result.DeduplicatedCount) + } + if !reflect.DeepEqual(result.StrategiesRun, []string{"injection"}) { + t.Errorf("strategies_run = %v, want [injection]", result.StrategiesRun) + } +} + +// TestNormalizeIncludePaths pins the set construction: blanks dropped, entries +// trimmed, matching exact on file_path. +func TestNormalizeIncludePaths(t *testing.T) { + got := normalizeIncludePaths([]string{" a.py ", "", " ", "b.py", "a.py"}) + want := map[string]struct{}{"a.py": {}, "b.py": {}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("normalizeIncludePaths = %v, want %v", got, want) + } + + findings := []schemas.RawFinding{ + {FilePath: "a.py"}, {FilePath: "src/a.py"}, {FilePath: "b.py"}, {FilePath: " a.py "}, + } + kept := applyIncludePaths(findings, []string{" a.py "}) + if len(kept) != 1 || kept[0].FilePath != "a.py" { + t.Fatalf("applyIncludePaths kept %v, want exactly the exact-match a.py", kept) + } + // An empty include list disables the filter entirely. + if got := applyIncludePaths(findings, nil); len(got) != len(findings) { + t.Fatalf("a nil include list must not filter (kept %d of %d)", len(got), len(findings)) + } + if got := applyIncludePaths(findings, []string{}); len(got) != len(findings) { + t.Fatalf("an empty include list must not filter (kept %d of %d)", len(got), len(findings)) + } + // A list of only-blank entries IS truthy in Python, so it filters + // everything out. + if got := applyIncludePaths(findings, []string{"", " "}); len(got) != 0 { + t.Fatalf("an all-blank include list must drop every finding (kept %d)", len(got)) + } +} + +// TestRunHuntFlattensInStrategyOrder asserts the findings arrive in table +// order, not completion order. +func TestRunHuntFlattensInStrategyOrder(t *testing.T) { + makeRunner := func(name string, delay int) hunterInvocation { + return func(context.Context, appx.Harnesser, string, schemas.ReconResult, config.DepthProfile) (schemas.HuntResult, error) { + // Burn some time so the slow hunters finish out of order. + for i := 0; i < delay*10000; i++ { + _ = i + } + result := schemas.NewHuntResult() + result.Findings = []schemas.RawFinding{testFinding(name+".py", name)} + return result, nil + } + } + withStrategyTable(t, + strategyEntry{schemas.HuntStrategyInjection, makeRunner("injection", 30)}, + strategyEntry{schemas.HuntStrategyXSS, makeRunner("xss", 20)}, + strategyEntry{schemas.HuntStrategyDos, makeRunner("dos", 1)}, + ) + withDedup(t, passthroughDedup) + + result, err := RunHunt( + context.Background(), &appx.Fake{}, ".", emptyRecon(), "standard", + DefaultMaxConcurrentHunters, DefaultEarlyStopFileThreshold, nil, + ) + if err != nil { + t.Fatalf("RunHunt: %v", err) + } + var paths []string + for _, finding := range result.Findings { + paths = append(paths, finding.FilePath) + } + if want := []string{"injection.py", "xss.py", "dos.py"}; !reflect.DeepEqual(paths, want) { + t.Fatalf("findings = %v, want %v (strategy order)", paths, want) + } +} + +// TestRunHuntSwallowsHunterErrors pins `gather(..., return_exceptions=True)`: +// a failing hunter contributes nothing and the phase still succeeds. +func TestRunHuntSwallowsHunterErrors(t *testing.T) { + withStrategyTable(t, + strategyEntry{schemas.HuntStrategyInjection, func(context.Context, appx.Harnesser, string, schemas.ReconResult, config.DepthProfile) (schemas.HuntResult, error) { + return schemas.HuntResult{}, fmt.Errorf("Hunt location scanner harness error: boom") + }}, + strategyEntry{schemas.HuntStrategyXSS, func(context.Context, appx.Harnesser, string, schemas.ReconResult, config.DepthProfile) (schemas.HuntResult, error) { + result := schemas.NewHuntResult() + result.Findings = []schemas.RawFinding{testFinding("ok.py", "ok")} + return result, nil + }}, + ) + withDedup(t, passthroughDedup) + + result, err := RunHunt( + context.Background(), &appx.Fake{}, ".", emptyRecon(), "standard", + DefaultMaxConcurrentHunters, DefaultEarlyStopFileThreshold, nil, + ) + if err != nil { + t.Fatalf("a failing hunter must not fail the phase: %v", err) + } + if len(result.Findings) != 1 || result.Findings[0].FilePath != "ok.py" { + t.Fatalf("findings = %v, want only ok.py", result.Findings) + } + if result.TotalRaw != 1 { + t.Errorf("total_raw = %d, want 1", result.TotalRaw) + } + // strategies_run still names EVERY selected strategy, including the one + // that blew up — Python builds it from `strategies`, not from the results. + if want := []string{"injection", "xss"}; !reflect.DeepEqual(result.StrategiesRun, want) { + t.Errorf("strategies_run = %v, want %v", result.StrategiesRun, want) + } +} + +// TestRunHuntPropagatesDedupError pins the one error run_hunt can return. +func TestRunHuntPropagatesDedupError(t *testing.T) { + withStrategyTable(t, strategyEntry{schemas.HuntStrategyInjection, + func(context.Context, appx.Harnesser, string, schemas.ReconResult, config.DepthProfile) (schemas.HuntResult, error) { + return schemas.NewHuntResult(), nil + }}) + withDedup(t, func(context.Context, []schemas.RawFinding, schemas.ReconResult, appx.Harnesser, string) (schemas.HuntResult, error) { + return schemas.HuntResult{}, fmt.Errorf("mkdtemp: no space left on device") + }) + + if _, err := RunHunt( + context.Background(), &appx.Fake{}, ".", emptyRecon(), "standard", + DefaultMaxConcurrentHunters, DefaultEarlyStopFileThreshold, nil, + ); err == nil { + t.Fatal("want the dedup error to propagate") + } +} + +// TestConcurrencyLimit pins `max(1, min(max_concurrent_hunters, len(strategies)))`. +func TestConcurrencyLimit(t *testing.T) { + cases := []struct{ maxHunters, strategies, want int }{ + {4, 11, 4}, {4, 3, 3}, {1, 11, 1}, {0, 11, 1}, {-5, 11, 1}, {4, 0, 1}, {100, 5, 5}, + } + for _, tc := range cases { + if got := concurrencyLimit(tc.maxHunters, tc.strategies); got != tc.want { + t.Errorf("concurrencyLimit(%d, %d) = %d, want %d", tc.maxHunters, tc.strategies, got, tc.want) + } + } +} + +// TestRunHuntConcurrencyBound asserts hunters really are gated by the semaphore. +func TestRunHuntConcurrencyBound(t *testing.T) { + for _, maxHunters := range []int{1, 2, 4} { + maxHunters := maxHunters + t.Run(strconv.Itoa(maxHunters), func(t *testing.T) { + var inflight, peak int32 + runner := func(context.Context, appx.Harnesser, string, schemas.ReconResult, config.DepthProfile) (schemas.HuntResult, error) { + now := atomic.AddInt32(&inflight, 1) + for { + old := atomic.LoadInt32(&peak) + if now <= old || atomic.CompareAndSwapInt32(&peak, old, now) { + break + } + } + for i := 0; i < 20000; i++ { + _ = i + } + atomic.AddInt32(&inflight, -1) + return schemas.NewHuntResult(), nil + } + entries := make([]strategyEntry, 0, len(AllStrategies())) + for _, strategy := range AllStrategies() { + entries = append(entries, strategyEntry{strategy, runner}) + } + withStrategyTable(t, entries...) + withDedup(t, passthroughDedup) + + if _, err := RunHunt( + context.Background(), &appx.Fake{}, ".", emptyRecon(), "standard", + maxHunters, DefaultEarlyStopFileThreshold, nil, + ); err != nil { + t.Fatalf("RunHunt: %v", err) + } + if got := int(atomic.LoadInt32(&peak)); got > maxHunters { + t.Fatalf("peak hunter concurrency %d exceeds the bound %d", got, maxHunters) + } + }) + } +} + +// TestRunHuntEarlyStopThresholdIsInert pins the dead parameter: changing it +// changes nothing, because the cascade never delivers it. +func TestRunHuntEarlyStopThresholdIsInert(t *testing.T) { + var seen []string + withStrategyTable(t, strategyEntry{schemas.HuntStrategyCrypto, + func(ctx context.Context, app appx.Harnesser, repoPath string, recon schemas.ReconResult, depth config.DepthProfile) (schemas.HuntResult, error) { + return runCryptoHunter(ctx, app, repoPath, recon, string(depth)) + }}) + withDedup(t, passthroughDedup) + + recon := emptyRecon() + recon.SecurityContext.CryptoUsage = []schemas.CryptoUsage{{Algorithm: "MD5", UsageContext: strptr("password hashing")}} + + for _, threshold := range []int{30, 999} { + fake := newHuntFake(nil, cannedEnriched()) + if _, err := RunHunt( + context.Background(), fake, fixtureRepo, recon, "thorough", + DefaultMaxConcurrentHunters, threshold, nil, + ); err != nil { + t.Fatalf("RunHunt: %v", err) + } + seen = append(seen, fake.onlyScanPrompt(t)) + } + if seen[0] != seen[1] { + t.Fatal("early_stop_file_threshold changed the prompt; it must be inert") + } + if !strings.Contains(seen[0], "if you inspect thorough files") { + t.Fatalf("crypto's early-stop line should carry the depth string, got:\n%s", + lastLines(seen[0], 8)) + } +} + +func lastLines(s string, n int) string { + lines := strings.Split(strings.TrimRight(s, "\n"), "\n") + if len(lines) > n { + lines = lines[len(lines)-n:] + } + return strings.Join(lines, "\n") +} + +// --------------------------------------------------------------------------- +// run_hunt_streaming +// --------------------------------------------------------------------------- + +// TestRunHuntStreamingPublishesAndCloses pins the stream contract: one batch per +// hunter that found something NEW, the channel closed exactly once at the end, +// and the final HuntResult built from the fingerprint-unique findings. +func TestRunHuntStreamingPublishesAndCloses(t *testing.T) { + shared := testFinding("shared.py", "shared") // same fingerprint from two hunters + withStrategyTable(t, + strategyEntry{schemas.HuntStrategyInjection, func(context.Context, appx.Harnesser, string, schemas.ReconResult, config.DepthProfile) (schemas.HuntResult, error) { + result := schemas.NewHuntResult() + result.Findings = []schemas.RawFinding{testFinding("a.py", "a"), shared} + return result, nil + }}, + strategyEntry{schemas.HuntStrategyXSS, func(context.Context, appx.Harnesser, string, schemas.ReconResult, config.DepthProfile) (schemas.HuntResult, error) { + result := schemas.NewHuntResult() + result.Findings = []schemas.RawFinding{shared, testFinding("b.py", "b")} + return result, nil + }}, + ) + withDedup(t, passthroughDedup) + + // Buffer for every strategy, matching Python's unbounded asyncio.Queue. + stream := make(chan []schemas.RawFinding, len(AllStrategies())) + result, err := RunHuntStreaming( + context.Background(), &appx.Fake{}, ".", emptyRecon(), stream, "standard", + DefaultMaxConcurrentHunters, DefaultEarlyStopFileThreshold, nil, + ) + if err != nil { + t.Fatalf("RunHuntStreaming: %v", err) + } + + var streamed []string + batches := 0 + for batch := range stream { + batches++ + for _, finding := range batch { + streamed = append(streamed, finding.FilePath) + } + } + if batches != 2 { + t.Errorf("streamed %d batches, want 2 (one per hunter with new findings)", batches) + } + if len(streamed) != 3 { + t.Errorf("streamed %d findings, want 3 (the duplicate is published once)", len(streamed)) + } + + // total_raw counts all four hunter outputs, not the three unique ones. + if result.TotalRaw != 4 { + t.Errorf("total_raw = %d, want 4", result.TotalRaw) + } + if result.DeduplicatedCount != 3 { + t.Errorf("deduplicated_count = %d, want 3", result.DeduplicatedCount) + } + if want := []string{"injection", "xss"}; !reflect.DeepEqual(result.StrategiesRun, want) { + t.Errorf("strategies_run = %v, want %v", result.StrategiesRun, want) + } +} + +// TestRunHuntStreamingSeedsFingerprints pins the seeding rule, which is NOT the +// sha256 form dedup.ComputeFingerprint uses. +func TestRunHuntStreamingSeedsFingerprints(t *testing.T) { + withStrategyTable(t, strategyEntry{schemas.HuntStrategyInjection, + func(context.Context, appx.Harnesser, string, schemas.ReconResult, config.DepthProfile) (schemas.HuntResult, error) { + finding := testFinding("app/x.py", "x") + finding.Fingerprint = "" // hunter left it empty + finding.StartLine = 17 + finding.CweID = "CWE-89" + result := schemas.NewHuntResult() + result.Findings = []schemas.RawFinding{finding} + return result, nil + }}) + + var handed []schemas.RawFinding + withDedup(t, func(_ context.Context, findings []schemas.RawFinding, _ schemas.ReconResult, _ appx.Harnesser, _ string) (schemas.HuntResult, error) { + handed = append(handed, findings...) + return passthroughDedup(context.Background(), findings, schemas.ReconResult{}, nil, "") + }) + + stream := make(chan []schemas.RawFinding, 4) + if _, err := RunHuntStreaming( + context.Background(), &appx.Fake{}, ".", emptyRecon(), stream, "standard", + DefaultMaxConcurrentHunters, DefaultEarlyStopFileThreshold, nil, + ); err != nil { + t.Fatalf("RunHuntStreaming: %v", err) + } + batch := <-stream + if got, want := batch[0].Fingerprint, "app/x.py:17:CWE-89"; got != want { + t.Errorf("streamed fingerprint = %q, want %q", got, want) + } + if len(handed) != 1 || handed[0].Fingerprint != "app/x.py:17:CWE-89" { + t.Errorf("dedup received %#v, want the seeded fingerprint", handed) + } +} + +// TestRunHuntStreamingClosesOnHunterFailure asserts the stream terminates even +// when every hunter blows up, so a consumer ranging over it cannot hang. +func TestRunHuntStreamingClosesOnHunterFailure(t *testing.T) { + withStrategyTable(t, strategyEntry{schemas.HuntStrategyInjection, + func(context.Context, appx.Harnesser, string, schemas.ReconResult, config.DepthProfile) (schemas.HuntResult, error) { + return schemas.HuntResult{}, fmt.Errorf("boom") + }}) + withDedup(t, passthroughDedup) + + stream := make(chan []schemas.RawFinding, 4) + result, err := RunHuntStreaming( + context.Background(), &appx.Fake{}, ".", emptyRecon(), stream, "standard", + DefaultMaxConcurrentHunters, DefaultEarlyStopFileThreshold, nil, + ) + if err != nil { + t.Fatalf("RunHuntStreaming: %v", err) + } + count := 0 + for range stream { + count++ + } + if count != 0 { + t.Errorf("streamed %d batches, want 0", count) + } + if result.TotalRaw != 0 || len(result.Findings) != 0 { + t.Errorf("result = %+v, want an empty one", result) + } +} + +// TestHuntResultJSONShape guards the wire contract the reasoner adapter will +// serialize: the exact pydantic key set, in declaration order. +func TestHuntResultJSONShape(t *testing.T) { + b, err := json.Marshal(schemas.NewHuntResult()) + if err != nil { + t.Fatalf("marshal: %v", err) + } + want := `{"findings":[],"chains":[],"total_raw":0,"deduplicated_count":0,"chain_count":0,"strategies_run":[],"hunt_duration_seconds":0}` + if string(b) != want { + t.Fatalf("HuntResult JSON = %s, want %s", b, want) + } +} diff --git a/internal/secaf/agents/hunt/hunters_test.go b/internal/secaf/agents/hunt/hunters_test.go new file mode 100644 index 0000000000..a405f73477 --- /dev/null +++ b/internal/secaf/agents/hunt/hunters_test.go @@ -0,0 +1,703 @@ +package hunt + +// Tests for the twelve hunter modules under src/sec_af/agents/hunt. +// +// Validation contract (behaviour, not implementation): +// +// - each hunter's SCAN PROMPT is byte-identical to the Python module's, for +// the direct call AND for the call __init__.py's argument cascade makes; +// - each hunter enriches with its own finding_type/strategy pair and its own +// recon context, and the assembled HuntResult (findings, counters, +// strategies_run) matches Python's model_dump(); +// - the four gated hunters (crypto, supply_chain, api_security, +// business_logic/logic) make NO harness call when their gate is closed, and +// return the exact empty shape Python returns — which is not the same shape +// for all four; +// - a scan that finds no locations returns without enriching, again in each +// hunter's own empty shape; +// - crypto partitions its usage contexts into security-critical and +// non-security candidates by substring, in recon order, with a context +// landing in both lists when it matches both tables; +// - run_logic_hunter is indistinguishable from run_business_logic_hunter. + +import ( + "context" + "reflect" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// hunterCase describes one hunter for the table-driven tests. Direct calls the +// exported entry point the way src/sec_af/reasoners/hunt.py does — depth +// "standard" where the hunter accepts one, max_files_without_signal at its 30 +// default — and Prompt builds the same call's scan prompt plus the recon +// context it embeds. +type hunterCase struct { + Name string + FindingType string + Strategy string + Direct func(ctx context.Context, app appx.Harnesser, repoPath string, recon schemas.ReconResult) (schemas.HuntResult, error) + Prompt func(repoPath string, recon schemas.ReconResult) (string, string) +} + +func hunterCases() []hunterCase { + return []hunterCase{ + {"injection", "sast", "injection", + func(ctx context.Context, app appx.Harnesser, repo string, r schemas.ReconResult) (schemas.HuntResult, error) { + return RunInjectionHunter(ctx, app, repo, r, "standard", 30) + }, + func(repo string, r schemas.ReconResult) (string, string) { + return injectionScanPrompt(repo, r, "standard", "30") + }}, + {"xss", "sast", "xss", + func(ctx context.Context, app appx.Harnesser, repo string, r schemas.ReconResult) (schemas.HuntResult, error) { + return RunXSSHunter(ctx, app, repo, r, "standard", 30) + }, + func(repo string, r schemas.ReconResult) (string, string) { + return xssScanPrompt(repo, r, "standard", "30") + }}, + {"dos", "sast", "dos", + func(ctx context.Context, app appx.Harnesser, repo string, r schemas.ReconResult) (schemas.HuntResult, error) { + return RunDosHunter(ctx, app, repo, r, "standard", 30) + }, + func(repo string, r schemas.ReconResult) (string, string) { + return dosScanPrompt(repo, r, "standard", "30") + }}, + {"ssrf", "sast", "ssrf", + func(ctx context.Context, app appx.Harnesser, repo string, r schemas.ReconResult) (schemas.HuntResult, error) { + return RunSSRFHunter(ctx, app, repo, r, "standard", 30) + }, + func(repo string, r schemas.ReconResult) (string, string) { + return ssrfScanPrompt(repo, r, "standard", "30") + }}, + {"auth", "sast", "auth", + func(ctx context.Context, app appx.Harnesser, repo string, r schemas.ReconResult) (schemas.HuntResult, error) { + return RunAuthHunter(ctx, app, repo, r, "standard", 30) + }, + func(repo string, r schemas.ReconResult) (string, string) { + return authScanPrompt(repo, r, "standard", "30") + }}, + {"crypto", "sast", "crypto", + func(ctx context.Context, app appx.Harnesser, repo string, r schemas.ReconResult) (schemas.HuntResult, error) { + return RunCryptoHunter(ctx, app, repo, r, 30) + }, + func(repo string, r schemas.ReconResult) (string, string) { + return cryptoScanPrompt(repo, r, "30") + }}, + {"business_logic", "logic", "business_logic", + func(ctx context.Context, app appx.Harnesser, repo string, r schemas.ReconResult) (schemas.HuntResult, error) { + return RunBusinessLogicHunter(ctx, app, repo, r, "standard", 30, "") + }, + func(repo string, r schemas.ReconResult) (string, string) { + return businessLogicScanPrompt(repo, r, "standard", "30", "") + }}, + {"logic", "logic", "business_logic", + func(ctx context.Context, app appx.Harnesser, repo string, r schemas.ReconResult) (schemas.HuntResult, error) { + return RunLogicHunter(ctx, app, repo, r, "standard", 30, "") + }, + func(repo string, r schemas.ReconResult) (string, string) { + return businessLogicScanPrompt(repo, r, "standard", "30", "") + }}, + {"data_exposure", "sast", "data_exposure", + func(ctx context.Context, app appx.Harnesser, repo string, r schemas.ReconResult) (schemas.HuntResult, error) { + return RunDataExposureHunter(ctx, app, repo, r, 30) + }, + func(repo string, r schemas.ReconResult) (string, string) { + return dataExposureScanPrompt(repo, r, "30") + }}, + {"supply_chain", "sca", "supply_chain", + func(ctx context.Context, app appx.Harnesser, repo string, r schemas.ReconResult) (schemas.HuntResult, error) { + return RunSupplyChainHunter(ctx, app, repo, r, 30) + }, + func(repo string, r schemas.ReconResult) (string, string) { + return supplyChainScanPrompt(repo, r, "30") + }}, + {"config_secrets", "config", "config_secrets", + func(ctx context.Context, app appx.Harnesser, repo string, r schemas.ReconResult) (schemas.HuntResult, error) { + return RunConfigSecretsHunter(ctx, app, repo, r, 30) + }, + func(repo string, r schemas.ReconResult) (string, string) { + return configSecretsScanPrompt(repo, r, "30") + }}, + {"api_security", "api", "api_security", + func(ctx context.Context, app appx.Harnesser, repo string, r schemas.ReconResult) (schemas.HuntResult, error) { + return RunAPISecurityHunter(ctx, app, repo, r, 30) + }, + func(repo string, r schemas.ReconResult) (string, string) { + return apiSecurityScanPrompt(repo, r, "30") + }}, + } +} + +// TestDirectPromptDigestsMatchPython pins every hunter's DIRECT-call scan +// prompt, and the two enrichment prompts it would produce for the canned +// locations, by SHA-256 against the Python functions. +// +// The digest form is deliberate: the enrichment prompt embeds the same recon +// context as the scan prompt, so storing twenty-four more full texts would add +// a quarter-megabyte of testdata and no coverage. A digest mismatch is still an +// exact-bytes failure; the five full-text goldens below make the common +// failures readable. +func TestDirectPromptDigestsMatchPython(t *testing.T) { + var golden struct { + Scan map[string]string `json:"scan"` + Enrich map[string][]string `json:"enrich"` + } + goldenJSON(t, "direct_prompt_sha256", &golden) + recon := loadRecon(t, "recon_fixture") + + for _, hc := range hunterCases() { + hc := hc + t.Run(hc.Name, func(t *testing.T) { + scanPrompt, reconContext := hc.Prompt(fixtureRepo, recon) + if got, want := sha256Hex(ScanPrompt(scanPrompt)), golden.Scan[hc.Name]; got != want { + // The digest compared here is of the FULL step-1 prompt (the + // scan_locations wrapper around the hunter's own text), which + // is what Python's _S4App recorded. + t.Fatalf("scan prompt digest = %s, want %s", got, want) + } + wantEnrich := golden.Enrich[hc.Name] + if len(wantEnrich) != len(cannedLocations()) { + t.Fatalf("golden has %d enrich digests, want %d", len(wantEnrich), len(cannedLocations())) + } + for i, location := range cannedLocations() { + got := sha256Hex(EnrichPrompt(location, hc.FindingType, hc.Strategy, reconContext)) + if got != wantEnrich[i] { + t.Fatalf("enrich prompt %d digest = %s, want %s", i, got, wantEnrich[i]) + } + } + }) + } +} + +// TestDirectPromptTextMatchesPython compares the full text for the five hunters +// whose direct-call prompt differs from the one the cascade produces (their +// early-stop value is 30 here and the depth string there). +// +// Every prompt golden in this package is the prompt as app.harness SAW it — +// i.e. the hunter's own text already wrapped in the shared scan_locations +// template — because that is where gen_golden.py's fake recorded it. +func TestDirectPromptTextMatchesPython(t *testing.T) { + recon := loadRecon(t, "recon_fixture") + byName := map[string]hunterCase{} + for _, hc := range hunterCases() { + byName[hc.Name] = hc + } + for _, name := range []string{"crypto", "data_exposure", "supply_chain", "config_secrets", "api_security"} { + name := name + t.Run(name, func(t *testing.T) { + scanPrompt, _ := byName[name].Prompt(fixtureRepo, recon) + assertTextEqual(t, "direct_prompt_"+name, ScanPrompt(scanPrompt), goldenText(t, "direct_prompt_"+name)) + }) + } +} + +// TestLogicHunterIsBusinessLogicHunter pins logic.py's delegation: same prompt +// (down to the byte), same finding_type, same strategies_run. +func TestLogicHunterIsBusinessLogicHunter(t *testing.T) { + recon := loadRecon(t, "recon_fixture") + + logicPrompt, logicContext := businessLogicScanPrompt(fixtureRepo, recon, "standard", "30", "") + assertTextEqual(t, "logic == business_logic prompt", ScanPrompt(logicPrompt), + goldenText(t, "prompt_business_logic_standard")) + + logicFake := newHuntFake(cannedLocations(), cannedEnriched()) + logicResult, err := RunLogicHunter(context.Background(), logicFake, fixtureRepo, recon, "standard", 30, "") + if err != nil { + t.Fatalf("RunLogicHunter: %v", err) + } + blFake := newHuntFake(cannedLocations(), cannedEnriched()) + blResult, err := RunBusinessLogicHunter(context.Background(), blFake, fixtureRepo, recon, "standard", 30, "") + if err != nil { + t.Fatalf("RunBusinessLogicHunter: %v", err) + } + if !reflect.DeepEqual(scrubIDs(jsonTree(t, logicResult)), scrubIDs(jsonTree(t, blResult))) { + t.Fatalf("run_logic_hunter and run_business_logic_hunter disagree%s", + diffJSON(scrubIDs(jsonTree(t, logicResult)), scrubIDs(jsonTree(t, blResult)))) + } + if got := logicResult.StrategiesRun; !reflect.DeepEqual(got, []string{"business_logic"}) { + t.Fatalf("strategies_run = %v, want [business_logic] — logic never names itself", got) + } + if logicContext == "" { + t.Fatal("recon context is empty") + } +} + +// TestHunterResultsMatchPython runs every hunter end to end over the canned +// two-location scan and compares the whole HuntResult against Python's +// model_dump(). +func TestHunterResultsMatchPython(t *testing.T) { + var golden map[string]map[string]any + goldenJSON(t, "hunter_results", &golden) + recon := loadRecon(t, "recon_fixture") + + for _, hc := range hunterCases() { + hc := hc + t.Run(hc.Name, func(t *testing.T) { + want, ok := golden[hc.Name] + if !ok { + t.Fatalf("no hunter_results golden for %s", hc.Name) + } + fake := newHuntFake(cannedLocations(), cannedEnriched()) + got, err := hc.Direct(context.Background(), fake, fixtureRepo, recon) + if err != nil { + t.Fatalf("%s: %v", hc.Name, err) + } + if len(fake.scanPrompts()) != 1 { + t.Fatalf("want 1 scan call, got %d", len(fake.scanPrompts())) + } + if len(fake.enrichPrompts()) != len(cannedLocations()) { + t.Fatalf("want %d enrich calls, got %d", len(cannedLocations()), len(fake.enrichPrompts())) + } + gotTree := scrubIDs(jsonTree(t, got)) + wantTree := scrubIDs(jsonTree(t, want)) + if !reflect.DeepEqual(gotTree, wantTree) { + t.Fatalf("%s HuntResult mismatch%s", hc.Name, diffJSON(gotTree, wantTree)) + } + }) + } +} + +// TestHunterEmptyLocationsMatchPython pins the "scanner found nothing" return +// for every hunter — one scan call, no enrichment, and each hunter's own empty +// shape (six bare, five naming their strategy). +func TestHunterEmptyLocationsMatchPython(t *testing.T) { + var golden map[string]struct { + ScanCalls int `json:"scan_calls"` + EnrichCalls int `json:"enrich_calls"` + Want map[string]any `json:"want"` + } + goldenJSON(t, "hunter_empty_locations", &golden) + recon := loadRecon(t, "recon_fixture") + + for _, hc := range hunterCases() { + hc := hc + t.Run(hc.Name, func(t *testing.T) { + want, ok := golden[hc.Name] + if !ok { + t.Fatalf("no hunter_empty_locations golden for %s", hc.Name) + } + fake := newHuntFake(nil, cannedEnriched()) + got, err := hc.Direct(context.Background(), fake, fixtureRepo, recon) + if err != nil { + t.Fatalf("%s: %v", hc.Name, err) + } + if len(fake.scanPrompts()) != want.ScanCalls { + t.Errorf("scan calls = %d, want %d", len(fake.scanPrompts()), want.ScanCalls) + } + if len(fake.enrichPrompts()) != want.EnrichCalls { + t.Errorf("enrich calls = %d, want %d", len(fake.enrichPrompts()), want.EnrichCalls) + } + gotTree := scrubIDs(jsonTree(t, got)) + wantTree := scrubIDs(jsonTree(t, want.Want)) + if !reflect.DeepEqual(gotTree, wantTree) { + t.Fatalf("%s empty HuntResult mismatch%s", hc.Name, diffJSON(gotTree, wantTree)) + } + }) + } +} + +// TestHunterSkipsMatchPython pins the four gates: no harness call at all, and +// the exact empty shape each gate returns. +func TestHunterSkipsMatchPython(t *testing.T) { + var golden map[string]struct { + HarnessCalls int `json:"harness_calls"` + Want map[string]any `json:"want"` + } + goldenJSON(t, "hunter_skips", &golden) + + rich := loadRecon(t, "recon_fixture") + + // gen_golden.py's no_crypto: the rich recon with an empty crypto_usage. + noCrypto := loadRecon(t, "recon_fixture") + noCrypto.SecurityContext = schemas.SecurityContext{ + AuthModel: "jwt", + AuthDetails: "x", + CryptoUsage: []schemas.CryptoUsage{}, + FrameworkSecurity: append([]string(nil), rich.SecurityContext.FrameworkSecurity...), + SecurityHeaders: append([]string(nil), rich.SecurityContext.SecurityHeaders...), + DeploymentSignals: append([]string(nil), rich.SecurityContext.DeploymentSignals...), + } + + // gen_golden.py's no_deps: direct_count back to 0. + noDeps := loadRecon(t, "recon_fixture") + noDeps.Dependencies = schemas.NewDependencyReport() + noDeps.Dependencies.DirectCount = 0 + noDeps.Dependencies.TransitiveCount = 9 + + // gen_golden.py's no_api: an empty api_surface. + noAPI := loadRecon(t, "recon_fixture") + noAPI.Architecture.APISurface = []schemas.APIEndpoint{} + + cases := map[string]func(ctx context.Context, app appx.Harnesser) (schemas.HuntResult, error){ + "crypto_no_usage": func(ctx context.Context, app appx.Harnesser) (schemas.HuntResult, error) { + return RunCryptoHunter(ctx, app, fixtureRepo, noCrypto, 30) + }, + "crypto_empty_recon": func(ctx context.Context, app appx.Harnesser) (schemas.HuntResult, error) { + return RunCryptoHunter(ctx, app, fixtureRepo, emptyRecon(), 30) + }, + "supply_chain_no_direct_deps": func(ctx context.Context, app appx.Harnesser) (schemas.HuntResult, error) { + return RunSupplyChainHunter(ctx, app, fixtureRepo, noDeps, 30) + }, + "api_security_no_surface": func(ctx context.Context, app appx.Harnesser) (schemas.HuntResult, error) { + return RunAPISecurityHunter(ctx, app, fixtureRepo, noAPI, 30) + }, + "business_logic_quick": func(ctx context.Context, app appx.Harnesser) (schemas.HuntResult, error) { + return RunBusinessLogicHunter(ctx, app, fixtureRepo, rich, "quick", 30, "") + }, + "logic_quick": func(ctx context.Context, app appx.Harnesser) (schemas.HuntResult, error) { + return RunLogicHunter(ctx, app, fixtureRepo, rich, "quick", 30, "") + }, + } + if len(cases) != len(golden) { + t.Fatalf("golden has %d skip cases, the test covers %d", len(golden), len(cases)) + } + + for name, run := range cases { + name, run := name, run + t.Run(name, func(t *testing.T) { + want := golden[name] + fake := newHuntFake(cannedLocations(), cannedEnriched()) + got, err := run(context.Background(), fake) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if len(fake.Harnesses) != want.HarnessCalls { + t.Errorf("harness calls = %d, want %d", len(fake.Harnesses), want.HarnessCalls) + } + gotTree := scrubIDs(jsonTree(t, got)) + wantTree := scrubIDs(jsonTree(t, want.Want)) + if !reflect.DeepEqual(gotTree, wantTree) { + t.Fatalf("%s HuntResult mismatch%s", name, diffJSON(gotTree, wantTree)) + } + }) + } +} + +// TestReconContextBlocksMatchPython pins the two inline JSON context builders +// (which are hunt's own code, not context.py's) against json.dumps(..., indent=2) +// of the same model_dump, for a rich and an all-defaults recon. +func TestReconContextBlocksMatchPython(t *testing.T) { + rich := loadRecon(t, "recon_fixture") + empty := emptyRecon() + + // dos, ssrf and xss declare three byte-identical copies of the helper. + for _, name := range []string{"dos", "ssrf", "xss"} { + assertTextEqual(t, "recon_context_block_"+name, + entryFlowContextBlock(rich), goldenText(t, "recon_context_block_"+name)) + } + assertTextEqual(t, "recon_context_block_dos_empty", + entryFlowContextBlock(empty), goldenText(t, "recon_context_block_dos_empty")) + + assertTextEqual(t, "recon_context_block_business_logic", + businessLogicContextBlock(rich), goldenText(t, "recon_context_block_business_logic")) + assertTextEqual(t, "recon_context_block_business_logic_empty", + businessLogicContextBlock(empty), goldenText(t, "recon_context_block_business_logic_empty")) +} + +// TestReconContextBlockTruncation states the slice limits, which differ between +// the two builders (10 entry points / 10 flows vs 15 / 20 endpoints / 20 flows). +// +// The rich fixture holds 17 entry points, 20 endpoints and 20 flows — enough +// for the entry-point limits but exactly at the endpoint/flow ceilings — so the +// counts are asserted against a synthetic recon carrying 30 of each. +func TestReconContextBlockTruncation(t *testing.T) { + recon := emptyRecon() + for i := 0; i < 30; i++ { + recon.Architecture.EntryPoints = append(recon.Architecture.EntryPoints, + schemas.EntryPoint{Kind: "http", Identifier: "e", FilePath: "f.py", Line: i}) + recon.Architecture.APISurface = append(recon.Architecture.APISurface, + schemas.APIEndpoint{Method: "GET", Path: "/p", Handler: "h", FilePath: "f.py", Line: i}) + recon.DataFlows.Flows = append(recon.DataFlows.Flows, + schemas.DataFlow{Source: "s", Sink: "k", Path: []schemas.ReconDataFlowStep{}, Files: []string{}}) + } + + entryFlow := entryFlowContextBlock(recon) + if got := strings.Count(entryFlow, `"kind":`); got != 10 { + t.Errorf("entryFlowContextBlock kept %d entry points, want 10", got) + } + if got := strings.Count(entryFlow, `"sanitized":`); got != 10 { + t.Errorf("entryFlowContextBlock kept %d data flows, want 10", got) + } + if strings.Contains(entryFlow, `"api_surface"`) { + t.Error("entryFlowContextBlock must not carry api_surface") + } + + businessLogic := businessLogicContextBlock(recon) + if got := strings.Count(businessLogic, `"kind":`); got != 15 { + t.Errorf("businessLogicContextBlock kept %d entry points, want 15", got) + } + if got := strings.Count(businessLogic, `"handler":`); got != 20 { + t.Errorf("businessLogicContextBlock kept %d api endpoints, want 20", got) + } + if got := strings.Count(businessLogic, `"sanitized":`); got != 20 { + t.Errorf("businessLogicContextBlock kept %d data flows, want 20", got) + } + if !strings.Contains(businessLogic, `"auth_details":`) { + t.Error("businessLogicContextBlock must carry auth_details") + } +} + +// --------------------------------------------------------------------------- +// crypto +// --------------------------------------------------------------------------- + +func strptr(s string) *string { return &s } +func boolptr(b bool) *bool { return &b } + +// cryptoUsageCases mirrors gen_golden.py's crypto usage-context table. A nil +// entry is Python's None usage_context. +func cryptoUsageCases() map[string][]*string { + return map[string][]*string{ + "mixed": { + strptr("password hashing"), + strptr("file integrity checksum"), + strptr("etag generation for cache"), + strptr("TLS session key derivation"), + strptr("unrelated purpose"), + nil, + strptr(""), + }, + "none": {strptr("unrelated purpose"), strptr("widget rendering")}, + "both_terms": {strptr("auth token cache")}, + } +} + +func cryptoUsageRecon(contexts []*string) schemas.ReconResult { + recon := emptyRecon() + usage := make([]schemas.CryptoUsage, 0, len(contexts)) + for _, c := range contexts { + usage = append(usage, schemas.CryptoUsage{Algorithm: "MD5", UsageContext: c, IsWeak: boolptr(true)}) + } + recon.SecurityContext.CryptoUsage = usage + return recon +} + +// TestCryptoUsagePartitionMatchesPython pins _usage_contexts, +// _filter_contexts_by_terms and should_run_crypto_hunter, plus the prompt each +// partition produces. +func TestCryptoUsagePartitionMatchesPython(t *testing.T) { + var golden map[string]struct { + UsageContexts []string `json:"usage_contexts"` + SecurityCritical []string `json:"security_critical"` + NonSecurity []string `json:"non_security"` + ShouldRun bool `json:"should_run"` + } + goldenJSON(t, "crypto_usage_partition", &golden) + + for name, contexts := range cryptoUsageCases() { + name, contexts := name, contexts + t.Run(name, func(t *testing.T) { + want, ok := golden[name] + if !ok { + t.Fatalf("no crypto_usage_partition golden for %s", name) + } + recon := cryptoUsageRecon(contexts) + + if got := cryptoUsageContexts(recon); !reflect.DeepEqual(got, want.UsageContexts) { + t.Errorf("usage contexts = %#v, want %#v", got, want.UsageContexts) + } + gotCritical := filterContextsByTerms(cryptoUsageContexts(recon), cryptoSecurityCriticalTerms) + if !reflect.DeepEqual(gotCritical, want.SecurityCritical) { + t.Errorf("security-critical = %#v, want %#v", gotCritical, want.SecurityCritical) + } + gotNon := filterContextsByTerms(cryptoUsageContexts(recon), cryptoNonSecurityTerms) + if !reflect.DeepEqual(gotNon, want.NonSecurity) { + t.Errorf("non-security = %#v, want %#v", gotNon, want.NonSecurity) + } + if got := ShouldRunCryptoHunter(recon); got != want.ShouldRun { + t.Errorf("should_run = %v, want %v", got, want.ShouldRun) + } + + prompt, _ := cryptoScanPrompt(fixtureRepo, recon, "30") + assertTextEqual(t, "crypto_prompt_"+name, ScanPrompt(prompt), goldenText(t, "crypto_prompt_"+name)) + }) + } +} + +// TestCryptoTermTables pins the two substring tables verbatim. +func TestCryptoTermTables(t *testing.T) { + var golden struct { + SecurityCritical []string `json:"security_critical"` + NonSecurity []string `json:"non_security"` + } + goldenJSON(t, "crypto_term_tables", &golden) + if !reflect.DeepEqual(cryptoSecurityCriticalTerms, golden.SecurityCritical) { + t.Errorf("_SECURITY_CRITICAL_TERMS = %#v, want %#v", cryptoSecurityCriticalTerms, golden.SecurityCritical) + } + if !reflect.DeepEqual(cryptoNonSecurityTerms, golden.NonSecurity) { + t.Errorf("_NON_SECURITY_TERMS = %#v, want %#v", cryptoNonSecurityTerms, golden.NonSecurity) + } +} + +// TestCryptoCandidateListFallback pins the `else "none"` spelling both +// candidate lines use when nothing matched. +func TestCryptoCandidateListFallback(t *testing.T) { + if got := cryptoCandidateList(nil); got != "none" { + t.Errorf("empty candidates render as %q, want %q", got, "none") + } + if got := cryptoCandidateList([]string{"a", "b"}); got != "a, b" { + t.Errorf("candidates render as %q, want %q", got, "a, b") + } +} + +// --------------------------------------------------------------------------- +// auth / business_logic knobs +// --------------------------------------------------------------------------- + +// TestAuthDepthLabelMatchesPython pins _depth_label, including the strip() that +// makes it differ from config.NormalizeDepth. +func TestAuthDepthLabelMatchesPython(t *testing.T) { + var golden map[string]string + goldenJSON(t, "auth_depth_label", &golden) + if len(golden) == 0 { + t.Fatal("auth_depth_label golden is empty") + } + for in, want := range golden { + if got := authDepthLabel(in); got != want { + t.Errorf("authDepthLabel(%q) = %q, want %q", in, got, want) + } + } + // The strip() is what separates this from the lenient normalizer used + // everywhere else in the pipeline. + if authDepthLabel(" THOROUGH ") != "thorough" { + t.Error("authDepthLabel must trim before matching") + } +} + +// TestAuthTargetCWEs pins the CWE list joined into {{TARGET_CWES}}. +func TestAuthTargetCWEs(t *testing.T) { + var golden []string + goldenJSON(t, "auth_target_cwes", &golden) + if !reflect.DeepEqual(authTargetCWEs, golden) { + t.Fatalf("_TARGET_CWES = %#v, want %#v", authTargetCWEs, golden) + } +} + +// TestBusinessLogicEnabledMatchesPython pins the depth gate. +func TestBusinessLogicEnabledMatchesPython(t *testing.T) { + var golden map[string]bool + goldenJSON(t, "business_logic_enabled", &golden) + if len(golden) == 0 { + t.Fatal("business_logic_enabled golden is empty") + } + for in, want := range golden { + if got := IsBusinessLogicHunterEnabled(in); got != want { + t.Errorf("IsBusinessLogicHunterEnabled(%q) = %v, want %v", in, got, want) + } + if got := IsLogicHunterEnabled(in); got != want { + t.Errorf("IsLogicHunterEnabled(%q) = %v, want %v", in, got, want) + } + } +} + +// TestBusinessLogicDepthPromptMatchesPython pins the optional +// "- Additional depth guidance: ..." tail, which only a deliberate caller can +// trigger (the argument cascade never delivers it). +func TestBusinessLogicDepthPromptMatchesPython(t *testing.T) { + recon := loadRecon(t, "recon_small") + got, _ := businessLogicScanPrompt(fixtureRepo, recon, "thorough", "30", ThoroughDepthPrompt) + assertTextEqual(t, "business_logic_prompt_with_depth_prompt", ScanPrompt(got), + goldenText(t, "business_logic_prompt_with_depth_prompt")) + + if !strings.HasSuffix(got, "\n- Additional depth guidance: "+ThoroughDepthPrompt) { + t.Fatal("the depth-guidance tail must be appended last, on its own line") + } + without, _ := businessLogicScanPrompt(fixtureRepo, recon, "thorough", "30", "") + if strings.Contains(without, "Additional depth guidance") { + t.Fatal("an empty depth_prompt must add nothing") + } +} + +// --------------------------------------------------------------------------- +// direct ports of tests/test_hunt_crypto.py +// --------------------------------------------------------------------------- + +// cryptoTestRecon ports tests/test_hunt_crypto.py::_recon_with_crypto_usage — +// an otherwise-default recon carrying one non-security and one security-critical +// crypto usage. +func cryptoTestRecon() schemas.ReconResult { + recon := emptyRecon() + recon.SecurityContext.AuthModel = "session" + recon.SecurityContext.AuthDetails = "cookie" + recon.SecurityContext.CryptoUsage = []schemas.CryptoUsage{ + {Algorithm: "MD5", UsageContext: strptr("file integrity checksum"), IsWeak: boolptr(true)}, + {Algorithm: "SHA1", UsageContext: strptr("password hashing"), IsWeak: boolptr(true)}, + } + return recon +} + +// TestCryptoHunterPromptIncludesContextAwareRiskGating ports +// tests/test_hunt_crypto.py::test_crypto_hunter_prompt_includes_context_aware_risk_gating. +func TestCryptoHunterPromptIncludesContextAwareRiskGating(t *testing.T) { + fake := newHuntFake(nil, cannedEnriched()) + if _, err := RunCryptoHunter(context.Background(), fake, ".", cryptoTestRecon(), 30); err != nil { + t.Fatalf("RunCryptoHunter: %v", err) + } + prompt := fake.onlyScanPrompt(t) + + for _, want := range []string{ + "- Focus CWEs: CWE-326, CWE-327, CWE-328, CWE-330, CWE-916, CWE-259, CWE-321, CWE-798", + "Prioritize weak crypto findings only when used in security-sensitive contexts", + "file integrity checksum", + "password hashing", + } { + if !strings.Contains(prompt, want) { + t.Errorf("crypto prompt is missing %q", want) + } + } + // And the gating itself: the checksum context is a NON-security candidate + // while the password one is security-critical. + if !strings.Contains(prompt, "- Security-critical usage candidates: password hashing\n") { + t.Error("security-critical candidates line is wrong") + } + if !strings.Contains(prompt, "- Non-security usage candidates: file integrity checksum\n") { + t.Error("non-security candidates line is wrong") + } +} + +// TestCryptoHunterSkipsWhenReconHasNoCryptoUsage ports +// tests/test_hunt_crypto.py::test_crypto_hunter_skips_when_recon_has_no_crypto_usage: +// the bare HuntResult() shape and, crucially, NO harness call at all (the +// Python test asserts `app.prompt == ""`). +func TestCryptoHunterSkipsWhenReconHasNoCryptoUsage(t *testing.T) { + recon := emptyRecon() + recon.SecurityContext.CryptoUsage = []schemas.CryptoUsage{} + + fake := newHuntFake(cannedLocations(), cannedEnriched()) + got, err := RunCryptoHunter(context.Background(), fake, ".", recon, 30) + if err != nil { + t.Fatalf("RunCryptoHunter: %v", err) + } + if !reflect.DeepEqual(got, schemas.NewHuntResult()) { + t.Errorf("result = %+v, want the bare HuntResult()", got) + } + if len(fake.Harnesses) != 0 { + t.Errorf("made %d harness calls, want 0", len(fake.Harnesses)) + } +} + +// TestHuntPromptTemplatesAreEmbedded guards every template this package loads +// with prompts.MustLoad — which panics on a missing name — plus logic.txt, +// which ships in the Python tree and is deliberately loaded by nobody +// (logic.py forwards to business_logic.py without reading it). +func TestHuntPromptTemplatesAreEmbedded(t *testing.T) { + loaded := []string{ + scanPromptPath, enrichPromptPath, + injectionPromptPath, xssPromptPath, dosPromptPath, ssrfPromptPath, authPromptPath, + cryptoPromptPath, businessLogicPromptPath, dataExposurePromptPath, + supplyChainPromptPath, configSecretsPromptPath, apiSecurityPromptPath, + } + for _, rel := range loaded { + if body, err := prompts.Load(rel); err != nil || body == "" { + t.Errorf("prompts.Load(%q) = %d bytes, err %v", rel, len(body), err) + } + } + if body, err := prompts.Load("hunt/logic.txt"); err != nil || body == "" { + t.Errorf("hunt/logic.txt must stay embedded even though no code path reads it: %v", err) + } +} diff --git a/internal/secaf/agents/hunt/injection.go b/internal/secaf/agents/hunt/injection.go new file mode 100644 index 0000000000..2992727b78 --- /dev/null +++ b/internal/secaf/agents/hunt/injection.go @@ -0,0 +1,72 @@ +package hunt + +// Ports src/sec_af/agents/hunt/injection.py. + +import ( + "context" + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/recontext" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +const injectionPromptPath = "hunt/injection.txt" + +// injectionScanPrompt builds the exact prompt run_injection_hunter sends, and +// returns the recon context it embedded — the hunter reuses that same string +// for the enrichment step, so it is computed once here. +// +// earlyStop is the pre-rendered text for the early-stop rule. Python +// interpolates `max_files_without_signal` there with an f-string and never +// checks its type; see the package doc for why that matters. +func injectionScanPrompt(repoPath string, recon schemas.ReconResult, depth, earlyStop string) (scanPrompt, reconContext string) { + reconContext = recontext.ReconContextForInjection(recon) + template := prompts.MustLoad(injectionPromptPath) + template = strings.ReplaceAll(template, "{{RECON_CONTEXT}}", reconContext) + template = strings.ReplaceAll(template, "{{LANGUAGE_HINTS}}", recontext.LanguageHintsForContext(recon)) + template = strings.ReplaceAll(template, "{{FRAMEWORK_HINTS}}", recontext.FrameworkHintsForContext(recon)) + + scanPrompt = template + + "\n\nCONTEXT:\n" + + "- Repository path: " + repoPath + "\n" + + "- Depth profile: " + depth + "\n" + + "- Early stop rule: if you inspect " + + earlyStop + " files without credible signal, " + + "stop and return empty findings.\n" + + "- Focus on RECON entry points and data flows as primary source-to-sink paths.\n" + + "- Explore the codebase, trace data flows from sources to sinks, and identify injection points.\n" + + "- Take multiple turns to build findings incrementally and write final JSON only when complete." + return scanPrompt, reconContext +} + +// runInjectionHunter is the shared body; earlyStop is pre-rendered text so the +// hunt table can pass the value Python's argument cascade actually delivers. +func runInjectionHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, depth, earlyStop string, +) (schemas.HuntResult, error) { + scanPrompt, reconContext := injectionScanPrompt(repoPath, recon, depth, earlyStop) + return runHunterBody(ctx, app, repoPath, hunterSpec{ + ScanPrompt: scanPrompt, + ReconContext: reconContext, + FindingType: "sast", + Strategy: "injection", + // Python parity: `return HuntResult()` — strategies_run keeps its [] + // default rather than naming the strategy that just ran. + EmptyStrategiesRun: nil, + }) +} + +// RunInjectionHunter ports injection.py run_injection_hunter: +// +// async def run_injection_hunter(app, repo_path, recon_result, depth, +// max_files_without_signal: int = 30) -> HuntResult +func RunInjectionHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, depth string, maxFilesWithoutSignal int, +) (schemas.HuntResult, error) { + return runInjectionHunter(ctx, app, repoPath, recon, depth, strconv.Itoa(maxFilesWithoutSignal)) +} diff --git a/internal/secaf/agents/hunt/logic.go b/internal/secaf/agents/hunt/logic.go new file mode 100644 index 0000000000..239655ef55 --- /dev/null +++ b/internal/secaf/agents/hunt/logic.go @@ -0,0 +1,46 @@ +package hunt + +// Ports src/sec_af/agents/hunt/logic.py — a thin alias module. +// +// logic.py exists only so the LOGIC_BUGS spelling of the strategy has a hunter +// of its own; both of its functions forward verbatim to business_logic.py, and +// nothing in the module reads its own prompts/hunt/logic.txt template (that +// file is loaded by no code path in the repository). logic.py also imports +// extract_harness_result and never calls it. +// +// It is NOT in _STRATEGY_RUNNERS: schemas.HuntStrategy.LOGIC_BUGS aliases +// BUSINESS_LOGIC, so the table's business_logic entry already covers it and +// __init__.py never reaches run_logic_hunter. src/sec_af/reasoners/hunt.py +// registers it as its own reasoner, which is where this entry point is used. + +import ( + "context" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// IsLogicHunterEnabled ports logic.py is_logic_hunter_enabled — a straight +// delegation to is_business_logic_hunter_enabled. +func IsLogicHunterEnabled(depth string) bool { + return IsBusinessLogicHunterEnabled(depth) +} + +// RunLogicHunter ports logic.py run_logic_hunter: +// +// async def run_logic_hunter(app, repo_path, recon, depth, +// max_files_without_signal=30, depth_prompt="") -> HuntResult: +// return await run_business_logic_hunter( +// app=app, repo_path=repo_path, recon_result=recon, +// depth=depth, max_files_without_signal=max_files_without_signal, +// depth_prompt=depth_prompt) +// +// Every argument is forwarded unchanged, so the prompt, the findings and the +// strategies_run value ("business_logic", never "logic") are byte-identical to +// RunBusinessLogicHunter's for the same inputs. +func RunLogicHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, depth string, maxFilesWithoutSignal int, depthPrompt string, +) (schemas.HuntResult, error) { + return RunBusinessLogicHunter(ctx, app, repoPath, recon, depth, maxFilesWithoutSignal, depthPrompt) +} diff --git a/internal/secaf/agents/hunt/scan_enrich.go b/internal/secaf/agents/hunt/scan_enrich.go new file mode 100644 index 0000000000..9d5393359e --- /dev/null +++ b/internal/secaf/agents/hunt/scan_enrich.go @@ -0,0 +1,421 @@ +package hunt + +// Ports src/sec_af/agents/hunt/_scan_enrich.py — the two-step scan/enrich +// harness pipeline and the RawFinding assembler that all twelve hunters share. + +import ( + "context" + "os" + "strconv" + "strings" + + "golang.org/x/sync/errgroup" + "golang.org/x/sync/semaphore" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/focus" + "github.com/Agent-Field/codeaf/internal/secaf/harnessx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// The two shared templates, as the embed-relative names internal/prompts uses. +// Python spells them as module-level Paths computed off __file__: +// +// PROMPTS_DIR = Path(__file__).resolve().parents[2] / "prompts" / "hunt" +// SCAN_PROMPT_PATH = PROMPTS_DIR / "scan_locations.txt" +// ENRICH_PROMPT_PATH = PROMPTS_DIR / "enrich_finding.txt" +const ( + scanPromptPath = "hunt/scan_locations.txt" + enrichPromptPath = "hunt/enrich_finding.txt" +) + +// The agent names extract_harness_result prints and embeds in its errors. +const ( + scanExtractName = "Hunt location scanner" + enrichExtractName = "Hunt finding enricher" +) + +// DefaultEnrichConcurrency ports enrich_locations_parallel's +// `max_concurrent: int = 5` default. +const DefaultEnrichConcurrency = 5 + +// toFindingType ports _to_finding_type: +// +// try: return FindingType(value.lower()) +// except ValueError: return FindingType.SAST +// +// Python parity: the lower() happens BEFORE the lookup, so "SAST" resolves and +// "logic " (trailing space) does not. +func toFindingType(value string) schemas.FindingType { + if v, err := schemas.ParseFindingType(strings.ToLower(value)); err == nil { + return v + } + return schemas.FindingTypeSast +} + +// toSeverity ports _to_severity — same shape, MEDIUM fallback. +func toSeverity(value string) schemas.Severity { + if v, err := schemas.ParseSeverity(strings.ToLower(value)); err == nil { + return v + } + return schemas.SeverityMedium +} + +// toConfidence ports _to_confidence — same shape, MEDIUM fallback. +func toConfidence(value string) schemas.Confidence { + if v, err := schemas.ParseConfidence(strings.ToLower(value)); err == nil { + return v + } + return schemas.ConfidenceMedium +} + +// ScanPrompt builds the Step 1 prompt: the shared scan_locations template with +// the hunter's own prompt substituted for {{HUNTER_PROMPT}}. +// +// Extracted as a pure function so a golden test can compare it byte-for-byte +// against the Python builder (testdata/golden/scan_locations_prompt.txt). +func ScanPrompt(hunterPrompt string) string { + return strings.ReplaceAll(prompts.MustLoad(scanPromptPath), "{{HUNTER_PROMPT}}", hunterPrompt) +} + +// ScanLocations ports _scan_enrich.py scan_locations: +// +// async def scan_locations(app, prompt, repo_path) -> list[VulnLocation]: +// scan_template = SCAN_PROMPT_PATH.read_text(encoding="utf-8") +// scan_prompt = scan_template.replace("{{HUNTER_PROMPT}}", prompt) +// harness_cwd = tempfile.mkdtemp(prefix="secaf-hunt-scan-") +// try: +// result = await app.harness(prompt=scan_prompt, schema=ScanLocationsResult, +// cwd=harness_cwd, project_dir=repo_path) +// parsed = extract_harness_result(result, ScanLocationsResult, "Hunt location scanner") +// return parsed.locations +// finally: +// shutil.rmtree(harness_cwd, ignore_errors=True) +// +// Python parity: `str.replace` replaces EVERY occurrence, so a template with +// two {{HUNTER_PROMPT}} markers would get both filled — strings.ReplaceAll, not +// strings.Replace(…, 1). The harness runs with Cwd on a private scratch dir and +// ProjectDir on the repository, so the coding agent explores the repo but +// writes its JSON output outside it; `shutil.rmtree(..., ignore_errors=True)` +// maps to a deferred os.RemoveAll whose error is deliberately dropped. +func ScanLocations(ctx context.Context, app appx.Harnesser, prompt, repoPath string) ([]schemas.VulnLocation, error) { + return scanLocationsFor(ctx, app, prompt, repoPath, "") +} + +// hunterLabel is the session label a hunter's own sessions carry on the +// audit's page: `injection hunter · scan`, `auth hunter · app/views.py:4`. +// Every hunter's scan was `Hunt location scanner` and every enrichment `Hunt +// finding enricher`, so a hunt of eleven strategies read as the same two lines +// over and over, saying neither which hunter nor where. +func hunterLabel(strategy, what string) string { + strategy = strings.TrimSpace(strategy) + if strategy == "" { + return "" + } + name, known := hunterNames[strategy] + if !known { + name = strings.ReplaceAll(strategy, "_", " ") + } + return name + " hunter · " + what +} + +// hunterNames is the strategies whose name is not its id with spaces: the +// ones written in capitals wherever a person meets them. +var hunterNames = map[string]string{"dos": "DoS", "ssrf": "SSRF", "xss": "XSS", "api_security": "API security"} + +// scanLocationsFor is ScanLocations for one hunter, its session named for it. +func scanLocationsFor(ctx context.Context, app appx.Harnesser, prompt, repoPath, strategy string) ([]schemas.VulnLocation, error) { + scanPrompt := ScanPrompt(prompt) + // AN AUDIT OF THE CHANGES TELLS THE SCAN WHAT CHANGED (internal/secaf/focus). + // sec-af's own PR mode only filtered the findings after a whole-repository + // scan; the scan is now told where to look, and the filter stays behind it. + // An audit of the whole repository carries no focus and its prompt is + // exactly sec-af's. + if change, ok := focus.From(ctx); ok { + scanPrompt += change.Section() + } + + harnessCwd, err := os.MkdirTemp("", "secaf-hunt-scan-") + if err != nil { + return nil, err + } + defer os.RemoveAll(harnessCwd) + + parsed, err := harnessx.RunExtract[schemas.ScanLocationsResult]( + ctx, app, scanPrompt, + appx.HarnessOptions{Cwd: harnessCwd, ProjectDir: repoPath, Label: hunterLabel(strategy, "scan")}, + scanExtractName, + ) + if err != nil { + return nil, err + } + return parsed.Locations, nil +} + +// EnrichPrompt builds the Step 2 prompt for one location. +// +// Ports the seven chained `.replace(...)` calls in enrich_location. Order is +// irrelevant to the output here (no substituted value contains another marker +// in practice) but is kept identical to Python's anyway. +func EnrichPrompt(location schemas.VulnLocation, findingType, strategy, reconContext string) string { + prompt := prompts.MustLoad(enrichPromptPath) + prompt = strings.ReplaceAll(prompt, "{{FINDING_TYPE}}", findingType) + prompt = strings.ReplaceAll(prompt, "{{STRATEGY}}", strategy) + prompt = strings.ReplaceAll(prompt, "{{RECON_CONTEXT}}", reconContext) + prompt = strings.ReplaceAll(prompt, "{{FILE_PATH}}", location.FilePath) + // Python interpolates `str(location.start_line)`, an int, so no thousands + // separators and a leading '-' for negatives — strconv.Itoa matches. + prompt = strings.ReplaceAll(prompt, "{{START_LINE}}", strconv.Itoa(location.StartLine)) + prompt = strings.ReplaceAll(prompt, "{{CODE_SNIPPET}}", location.CodeSnippet) + prompt = strings.ReplaceAll(prompt, "{{PATTERN_TYPE}}", location.PatternType) + return prompt +} + +// EnrichLocation ports _scan_enrich.py enrich_location. +// +// Python parity: the temp-dir prefix embeds the strategy — +// `tempfile.mkdtemp(prefix=f"secaf-hunt-enrich-{strategy}-")` — so a strategy +// containing a path separator would change where the scratch dir lands. Go's +// os.MkdirTemp REJECTS a pattern containing a separator (it returns +// ErrPatternHasSeparator) where Python's mkdtemp would happily build a nested +// path; every strategy this port passes is a bare HuntStrategy value, so the +// two agree in practice. The error is returned rather than swallowed. +func EnrichLocation( + ctx context.Context, + app appx.Harnesser, + location schemas.VulnLocation, + findingType, strategy, reconContext, repoPath string, +) (schemas.EnrichedFinding, error) { + enrichPrompt := EnrichPrompt(location, findingType, strategy, reconContext) + + harnessCwd, err := os.MkdirTemp("", "secaf-hunt-enrich-"+strategy+"-") + if err != nil { + return schemas.EnrichedFinding{}, err + } + defer os.RemoveAll(harnessCwd) + + return harnessx.RunExtract[schemas.EnrichedFinding]( + ctx, app, enrichPrompt, + appx.HarnessOptions{Cwd: harnessCwd, ProjectDir: repoPath, + Label: hunterLabel(strategy, location.FilePath+":"+strconv.Itoa(location.StartLine))}, + enrichExtractName, + ) +} + +// EnrichLocationsParallel ports _scan_enrich.py enrich_locations_parallel: +// +// if not locations: return [] +// semaphore = asyncio.Semaphore(max(1, max_concurrent)) +// async def _run(location): async with semaphore: return await enrich_location(...) +// return await asyncio.gather(*[_run(location) for location in locations]) +// +// Concurrency parity: +// +// - the bound is `max(1, max_concurrent)`, so a zero or negative +// max_concurrent still admits one enrichment at a time rather than +// deadlocking; +// - results are index-aligned with locations. asyncio.gather preserves the +// input order regardless of completion order, so the port writes into a +// pre-sized slice rather than appending; +// - errgroup.Group is used WITHOUT WithContext so a failing enrichment does +// not cancel its siblings — matching gather(return_exceptions=False), which +// never cancels the other awaitables either. Wait() returns the first error +// by completion time, the same one gather surfaces. +// - DIFFERENCE: Wait() blocks until every goroutine has finished, while +// `await gather(...)` resumes the caller as soon as the first exception +// fires and leaves the rest running detached. The returned value is +// identical; only the moment of return differs, and every caller's next act +// on error is to propagate it. +// +// On error the returned slice is nil: Python's caller never reads the partial +// results either, because the exception propagates out of the `await`. +// +// A cancelled ctx surfaces through semaphore.Acquire as ctx.Err(), which is the +// closest Go analogue of the CancelledError asyncio would raise inside the +// gathered coroutines. +func EnrichLocationsParallel( + ctx context.Context, + app appx.Harnesser, + locations []schemas.VulnLocation, + findingType, strategy, reconContext, repoPath string, + maxConcurrent int, +) ([]schemas.EnrichedFinding, error) { + if len(locations) == 0 { + // Python parity: the early return is `[]`, not None. + return []schemas.EnrichedFinding{}, nil + } + + limit := maxConcurrent + if limit < 1 { + limit = 1 + } + sem := semaphore.NewWeighted(int64(limit)) + + out := make([]schemas.EnrichedFinding, len(locations)) + var g errgroup.Group + for i := range locations { + i := i + g.Go(func() error { + if err := sem.Acquire(ctx, 1); err != nil { + return err + } + defer sem.Release(1) + enriched, err := EnrichLocation( + ctx, app, locations[i], findingType, strategy, reconContext, repoPath, + ) + if err != nil { + return err + } + out[i] = enriched + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, err + } + return out, nil +} + +// AssembleFinding ports _scan_enrich.py assemble_finding: +// +// snippet_line_count = max(1, location.code_snippet.count("\n") + 1) +// data_flow = None +// summary = enriched.data_flow_summary.strip() +// if summary: +// data_flow = [DataFlowStep(file_path=location.file_path, line=location.start_line, +// component=strategy, operation=summary)] +// return RawFinding(..., end_line=location.start_line + snippet_line_count - 1, ...) +// +// Python parity notes: +// +// - END LINE. `count("\n") + 1` counts LINES, so a snippet with no newline is +// one line and end_line == start_line; a snippet with a TRAILING newline +// ("one\ntwo\n") counts as three, so end_line overshoots by one. That is +// reproduced, not corrected. The `max(1, ...)` can never bite (count is +// never negative) and is kept only to mirror the source. +// - DATA FLOW. An empty-after-strip summary leaves data_flow at None, which +// model_dump()s as `null` — hence a nil slice here, not an empty one. The +// step's `component` is the STRATEGY, not a code component, and its `line` +// is the location's start_line. +// - `strip()` vs strings.TrimSpace: Python strips every character whose +// str.isspace() is true, which includes U+001C..U+001F (the file/group/ +// record/unit separators); Go's unicode.IsSpace does not. Both strip the +// ASCII whitespace, NEL and NBSP that real LLM output contains. +// - CWE NAME. Python passes `cwe_name=enriched.cwe_id` — the ID, not a name. +// Deliberate duplication, reproduced. +// - The finding's `id` and `fingerprint` are pydantic +// `default_factory=lambda: str(uuid4())`, so both are FRESH RANDOM UUIDs +// here; the fingerprint is overwritten with a real content fingerprint +// later, by dedup. schemas.NewRawFinding mints both the same way. +func AssembleFinding( + location schemas.VulnLocation, + enriched schemas.EnrichedFinding, + findingType, strategy string, +) schemas.RawFinding { + snippetLineCount := strings.Count(location.CodeSnippet, "\n") + 1 + if snippetLineCount < 1 { + snippetLineCount = 1 + } + + var dataFlow []schemas.ReconDataFlowStep + if summary := strings.TrimSpace(enriched.DataFlowSummary); summary != "" { + dataFlow = []schemas.ReconDataFlowStep{{ + FilePath: location.FilePath, + Line: location.StartLine, + Component: strategy, + Operation: summary, + }} + } + + finding := schemas.NewRawFinding() + finding.HunterStrategy = strategy + finding.Title = enriched.Title + finding.Description = enriched.Description + finding.FindingType = toFindingType(findingType) + finding.CweID = enriched.CweID + finding.CweName = enriched.CweID + finding.FilePath = location.FilePath + finding.StartLine = location.StartLine + finding.EndLine = location.StartLine + snippetLineCount - 1 + finding.CodeSnippet = location.CodeSnippet + finding.EstimatedSeverity = toSeverity(enriched.Severity) + finding.Confidence = toConfidence(enriched.Confidence) + finding.DataFlow = dataFlow + return finding +} + +// hunterSpec is everything the shared hunter body needs after a hunter module +// has built its own scan prompt. Every hunter's tail is byte-identical apart +// from these five values, so it is written once here rather than eleven times. +type hunterSpec struct { + // ScanPrompt is the hunter-specific prompt handed to ScanLocations. + ScanPrompt string + // ReconContext is what the enrichment step embeds; it is the SAME string + // the hunter substituted into its own template, not a re-derivation. + ReconContext string + // FindingType is the literal Python passes as `finding_type=` ("sast", + // "sca", "config", "api", "logic"). + FindingType string + // Strategy is the literal Python passes as `strategy=`. + Strategy string + // EmptyStrategiesRun is `strategies_run` on the "scanner found nothing" + // early return. Six hunters return a bare HuntResult() there (so the field + // keeps its `[]` default) and five return HuntResult(strategies_run=[s]); + // nil means the bare form. + EmptyStrategiesRun []string +} + +// runHunterBody is the shared tail of every hunter: scan, short-circuit on an +// empty location list, enrich in parallel, zip and assemble, and report the +// counters. +// +// Python parity: the zip() over (locations, enriched_findings) is safe because +// enrich_locations_parallel returns exactly one result per location, in order — +// a shorter enriched list would silently TRUNCATE the findings in Python, which +// is why the Go port keeps the two index-aligned rather than appending. +func runHunterBody( + ctx context.Context, + app appx.Harnesser, + repoPath string, + spec hunterSpec, +) (schemas.HuntResult, error) { + locations, err := scanLocationsFor(ctx, app, spec.ScanPrompt, repoPath, spec.Strategy) + if err != nil { + return schemas.HuntResult{}, err + } + if len(locations) == 0 { + empty := schemas.NewHuntResult() + if spec.EmptyStrategiesRun != nil { + empty.StrategiesRun = spec.EmptyStrategiesRun + } + return empty, nil + } + + enrichedFindings, err := EnrichLocationsParallel( + ctx, app, locations, spec.FindingType, spec.Strategy, + spec.ReconContext, repoPath, DefaultEnrichConcurrency, + ) + if err != nil { + return schemas.HuntResult{}, err + } + + n := len(locations) + if len(enrichedFindings) < n { + n = len(enrichedFindings) + } + findings := make([]schemas.RawFinding, 0, n) + for i := 0; i < n; i++ { + findings = append(findings, AssembleFinding( + locations[i], enrichedFindings[i], spec.FindingType, spec.Strategy, + )) + } + + result := schemas.NewHuntResult() + result.Findings = findings + result.TotalRaw = len(findings) + result.DeduplicatedCount = len(findings) + result.ChainCount = 0 + result.StrategiesRun = []string{spec.Strategy} + return result, nil +} diff --git a/internal/secaf/agents/hunt/scan_enrich_test.go b/internal/secaf/agents/hunt/scan_enrich_test.go new file mode 100644 index 0000000000..1ec2711e0d --- /dev/null +++ b/internal/secaf/agents/hunt/scan_enrich_test.go @@ -0,0 +1,419 @@ +package hunt + +// Tests for src/sec_af/agents/hunt/_scan_enrich.py. +// +// Validation contract (behaviour, not implementation): +// +// - the Step 1 prompt is the scan_locations template with the hunter's prompt +// substituted, byte-for-byte; +// - the Step 2 prompt is the enrich_finding template with all seven markers +// substituted, byte-for-byte; +// - both harness calls run with Cwd on a fresh scratch directory that is +// removed afterwards and ProjectDir on the repository; +// - a harness that reports an error surfaces as +// " harness error: " and no findings; +// - enrichment runs at most max(1, maxConcurrent) at a time, returns one +// result per location IN LOCATION ORDER, and an empty location list makes +// no harness call at all; +// - AssembleFinding derives end_line from the snippet's line count, drops a +// whitespace-only data-flow summary, coerces unknown severity/confidence/ +// finding_type to their documented fallbacks, and copies cwe_id into +// cwe_name. + +import ( + "context" + "os" + "reflect" + "strings" + "sync/atomic" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// TestScanPromptMatchesPython pins ScanPrompt against scan_locations() run in +// Python over the same hunter prompt. +func TestScanPromptMatchesPython(t *testing.T) { + var input struct { + HunterPrompt string `json:"hunter_prompt"` + } + goldenJSON(t, "scan_locations_input", &input) + assertTextEqual(t, "scan_locations_prompt", ScanPrompt(input.HunterPrompt), goldenText(t, "scan_locations_prompt")) +} + +// TestEnrichPromptMatchesPython pins EnrichPrompt against enrich_location() run +// in Python over the same location and context. +func TestEnrichPromptMatchesPython(t *testing.T) { + var input struct { + Location schemas.VulnLocation `json:"location"` + FindingType string `json:"finding_type"` + Strategy string `json:"strategy"` + ReconContext string `json:"recon_context"` + } + goldenJSON(t, "enrich_location_input", &input) + got := EnrichPrompt(input.Location, input.FindingType, input.Strategy, input.ReconContext) + assertTextEqual(t, "enrich_location_prompt", got, goldenText(t, "enrich_location_prompt")) +} + +// TestAssembleFindingMatchesPython replays every case gen_golden.py ran through +// the real assemble_finding: end-line arithmetic, the data-flow gate, and the +// three enum coercions. +func TestAssembleFindingMatchesPython(t *testing.T) { + var cases []struct { + Name string `json:"name"` + Location schemas.VulnLocation `json:"location"` + Enriched schemas.EnrichedFinding `json:"enriched"` + FindingType string `json:"finding_type"` + Strategy string `json:"strategy"` + Want map[string]any `json:"want"` + } + goldenJSON(t, "assemble_finding", &cases) + if len(cases) == 0 { + t.Fatal("assemble_finding golden is empty") + } + for _, tc := range cases { + tc := tc + t.Run(tc.Name, func(t *testing.T) { + got := AssembleFinding(tc.Location, tc.Enriched, tc.FindingType, tc.Strategy) + gotTree := scrubIDs(jsonTree(t, got)) + wantTree := scrubIDs(jsonTree(t, tc.Want)) + if !reflect.DeepEqual(gotTree, wantTree) { + t.Fatalf("assemble_finding %s mismatch%s", tc.Name, diffJSON(gotTree, wantTree)) + } + }) + } +} + +// TestEnumCoercionFallbacks covers _to_finding_type / _to_severity / +// _to_confidence directly, including the lower() that runs before the lookup +// and the fallbacks an out-of-vocabulary value lands on. +func TestEnumCoercionFallbacks(t *testing.T) { + findingTypes := map[string]schemas.FindingType{ + "sast": schemas.FindingTypeSast, "SAST": schemas.FindingTypeSast, + "sca": schemas.FindingTypeSca, "Config": schemas.FindingTypeConfig, + "logic": schemas.FindingTypeLogic, "api": schemas.FindingTypeAPI, + "secrets": schemas.FindingTypeSecrets, + // Fallbacks: unknown, empty, and a value that only differs by padding. + "not_a_type": schemas.FindingTypeSast, "": schemas.FindingTypeSast, + " sast ": schemas.FindingTypeSast, + } + for in, want := range findingTypes { + if got := toFindingType(in); got != want { + t.Errorf("toFindingType(%q) = %q, want %q", in, got, want) + } + } + + severities := map[string]schemas.Severity{ + "critical": schemas.SeverityCritical, "HIGH": schemas.SeverityHigh, + "Medium": schemas.SeverityMedium, "low": schemas.SeverityLow, + "info": schemas.SeverityInfo, + // Fallback is MEDIUM, not INFO. + "catastrophic": schemas.SeverityMedium, "": schemas.SeverityMedium, + " high ": schemas.SeverityMedium, + } + for in, want := range severities { + if got := toSeverity(in); got != want { + t.Errorf("toSeverity(%q) = %q, want %q", in, got, want) + } + } + + confidences := map[string]schemas.Confidence{ + "high": schemas.ConfidenceHigh, "MEDIUM": schemas.ConfidenceMedium, + "Low": schemas.ConfidenceLow, + // Fallback is MEDIUM. + "certain": schemas.ConfidenceMedium, "": schemas.ConfidenceMedium, + } + for in, want := range confidences { + if got := toConfidence(in); got != want { + t.Errorf("toConfidence(%q) = %q, want %q", in, got, want) + } + } +} + +// TestAssembleFindingEndLineArithmetic states the end-line rule on its own, +// including the trailing-newline overshoot the port reproduces. +func TestAssembleFindingEndLineArithmetic(t *testing.T) { + cases := []struct { + snippet string + startLine int + wantEnd int + }{ + {"", 5, 5}, // no newline -> one line + {"one", 5, 5}, // no newline -> one line + {"one\ntwo", 5, 6}, // two lines + {"one\ntwo\n", 5, 7}, // TRAILING newline counts a third line + {"\n\n\n", 1, 4}, // three newlines -> four lines + {"a\r\nb", 10, 11}, // CRLF: only the \n counts + {"one\ntwo", 0, 1}, // start_line 0 is not special-cased + {"one\ntwo", -3, -2}, // nor is a negative one + } + for _, tc := range cases { + location := schemas.VulnLocation{FilePath: "f", StartLine: tc.startLine, CodeSnippet: tc.snippet} + got := AssembleFinding(location, schemas.EnrichedFinding{}, "sast", "injection") + if got.EndLine != tc.wantEnd { + t.Errorf("snippet %q start %d: end_line = %d, want %d", tc.snippet, tc.startLine, got.EndLine, tc.wantEnd) + } + } +} + +// TestAssembleFindingDataFlowGate states the data-flow rule: the summary is +// stripped, an empty result leaves data_flow nil (JSON null), and a surviving +// summary produces exactly one step whose component is the STRATEGY. +func TestAssembleFindingDataFlowGate(t *testing.T) { + location := schemas.VulnLocation{FilePath: "app/x.py", StartLine: 12, CodeSnippet: "x"} + + for _, blank := range []string{"", " ", "\n\t \n"} { + got := AssembleFinding(location, schemas.EnrichedFinding{DataFlowSummary: blank}, "sast", "ssrf") + if got.DataFlow != nil { + t.Errorf("summary %q: want nil data_flow, got %#v", blank, got.DataFlow) + } + } + + got := AssembleFinding( + location, schemas.EnrichedFinding{DataFlowSummary: " src -> sink "}, "sast", "ssrf", + ) + want := []schemas.ReconDataFlowStep{{ + FilePath: "app/x.py", Line: 12, Component: "ssrf", Operation: "src -> sink", + }} + if !reflect.DeepEqual(got.DataFlow, want) { + t.Errorf("data_flow = %#v, want %#v", got.DataFlow, want) + } +} + +// TestAssembleFindingCopiesCweIDIntoName pins the deliberate duplication +// `cwe_name=enriched.cwe_id`. +func TestAssembleFindingCopiesCweIDIntoName(t *testing.T) { + got := AssembleFinding( + schemas.VulnLocation{}, schemas.EnrichedFinding{CweID: "CWE-89"}, "sast", "injection", + ) + if got.CweID != "CWE-89" || got.CweName != "CWE-89" { + t.Fatalf("cwe_id/cwe_name = %q/%q, want CWE-89/CWE-89", got.CweID, got.CweName) + } +} + +// TestScanLocationsHarnessOptions asserts the scratch-directory contract: Cwd +// is a fresh directory that is NOT the repository, ProjectDir is, and the +// directory is gone once the call returns. +func TestScanLocationsHarnessOptions(t *testing.T) { + fake := newHuntFake(cannedLocations(), cannedEnriched()) + if _, err := ScanLocations(context.Background(), fake, "HUNTER", fixtureRepo); err != nil { + t.Fatalf("ScanLocations: %v", err) + } + if len(fake.Harnesses) != 1 { + t.Fatalf("want 1 harness call, got %d", len(fake.Harnesses)) + } + opts := fake.Harnesses[0].Opts + if opts.ProjectDir != fixtureRepo { + t.Errorf("ProjectDir = %q, want %q", opts.ProjectDir, fixtureRepo) + } + if opts.Cwd == "" || opts.Cwd == fixtureRepo { + t.Fatalf("Cwd = %q, want a private scratch directory", opts.Cwd) + } + if !strings.Contains(opts.Cwd, "secaf-hunt-scan-") { + t.Errorf("Cwd = %q, want the secaf-hunt-scan- prefix", opts.Cwd) + } + if _, err := os.Stat(opts.Cwd); !os.IsNotExist(err) { + t.Errorf("scratch dir %q still exists after the call (stat err %v)", opts.Cwd, err) + } +} + +// TestEnrichLocationHarnessOptions is the same contract for step 2, whose +// prefix embeds the strategy. +func TestEnrichLocationHarnessOptions(t *testing.T) { + fake := newHuntFake(cannedLocations(), cannedEnriched()) + _, err := EnrichLocation( + context.Background(), fake, cannedLocations()[0], "sast", "supply_chain", "CTX", fixtureRepo, + ) + if err != nil { + t.Fatalf("EnrichLocation: %v", err) + } + opts := fake.Harnesses[0].Opts + if !strings.Contains(opts.Cwd, "secaf-hunt-enrich-supply_chain-") { + t.Errorf("Cwd = %q, want the secaf-hunt-enrich-supply_chain- prefix", opts.Cwd) + } + if opts.ProjectDir != fixtureRepo { + t.Errorf("ProjectDir = %q, want %q", opts.ProjectDir, fixtureRepo) + } + if _, err := os.Stat(opts.Cwd); !os.IsNotExist(err) { + t.Errorf("scratch dir %q still exists after the call", opts.Cwd) + } +} + +// TestScanLocationsHarnessErrorMessage pins extract_harness_result's error +// text for the scan agent name. +func TestScanLocationsHarnessErrorMessage(t *testing.T) { + fake := &appx.Fake{HarnessFn: func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{IsError: true, ErrorMessage: "provider exited 1"}, nil + }} + _, err := ScanLocations(context.Background(), fake, "HUNTER", fixtureRepo) + if err == nil { + t.Fatal("want an error") + } + if got, want := err.Error(), "Hunt location scanner harness error: provider exited 1"; got != want { + t.Fatalf("error = %q, want %q", got, want) + } +} + +// TestEnrichLocationHarnessErrorMessage is the same for the enrich agent name. +func TestEnrichLocationHarnessErrorMessage(t *testing.T) { + fake := &appx.Fake{HarnessFn: func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{IsError: true, ErrorMessage: "timeout"}, nil + }} + _, err := EnrichLocation( + context.Background(), fake, schemas.VulnLocation{}, "sast", "auth", "CTX", fixtureRepo, + ) + if err == nil { + t.Fatal("want an error") + } + if got, want := err.Error(), "Hunt finding enricher harness error: timeout"; got != want { + t.Fatalf("error = %q, want %q", got, want) + } +} + +// TestEnrichLocationsParallelEmpty pins the `if not locations: return []` +// short-circuit — no harness call, and an EMPTY (not nil) slice. +func TestEnrichLocationsParallelEmpty(t *testing.T) { + fake := newHuntFake(nil, cannedEnriched()) + got, err := EnrichLocationsParallel( + context.Background(), fake, nil, "sast", "injection", "CTX", fixtureRepo, DefaultEnrichConcurrency, + ) + if err != nil { + t.Fatalf("EnrichLocationsParallel: %v", err) + } + if got == nil { + t.Fatal("want an empty slice, got nil") + } + if len(got) != 0 { + t.Fatalf("want 0 results, got %d", len(got)) + } + if len(fake.Harnesses) != 0 { + t.Fatalf("want 0 harness calls, got %d", len(fake.Harnesses)) + } +} + +// TestEnrichLocationsParallelPreservesOrder asserts result[i] belongs to +// location[i] regardless of completion order — the invariant assemble_finding's +// zip() depends on. +func TestEnrichLocationsParallelPreservesOrder(t *testing.T) { + locations := make([]schemas.VulnLocation, 8) + for i := range locations { + locations[i] = schemas.VulnLocation{ + FilePath: "f" + string(rune('a'+i)) + ".py", + StartLine: i, + PatternType: "p", + } + } + fake := &appx.Fake{HarnessFn: func(_ context.Context, prompt string, _ map[string]any, dest any, _ appx.HarnessOptions) (*appx.HarnessResult, error) { + // Answer with the file path the prompt names, so a mis-ordered result + // is immediately visible. + out := dest.(*schemas.EnrichedFinding) + for _, location := range locations { + if strings.Contains(prompt, locationBlock(location)) { + out.Title = location.FilePath + } + } + return &appx.HarnessResult{Parsed: dest}, nil + }} + + got, err := EnrichLocationsParallel( + context.Background(), fake, locations, "sast", "injection", "CTX", fixtureRepo, 4, + ) + if err != nil { + t.Fatalf("EnrichLocationsParallel: %v", err) + } + if len(got) != len(locations) { + t.Fatalf("want %d results, got %d", len(locations), len(got)) + } + for i, location := range locations { + if got[i].Title != location.FilePath { + t.Errorf("result[%d].Title = %q, want %q", i, got[i].Title, location.FilePath) + } + } +} + +// TestEnrichLocationsParallelConcurrencyBound asserts the semaphore: at most +// max(1, maxConcurrent) enrichments are ever in flight, and a non-positive +// bound still admits one rather than deadlocking. +func TestEnrichLocationsParallelConcurrencyBound(t *testing.T) { + for _, limit := range []int{1, 2, 5, 0, -3} { + limit := limit + t.Run(strings.TrimSpace(itoaSigned(limit)), func(t *testing.T) { + want := limit + if want < 1 { + want = 1 + } + locations := make([]schemas.VulnLocation, 12) + for i := range locations { + locations[i] = schemas.VulnLocation{FilePath: "f.py", StartLine: i} + } + var inflight, peak int32 + fake := &appx.Fake{HarnessFn: func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + now := atomic.AddInt32(&inflight, 1) + for { + old := atomic.LoadInt32(&peak) + if now <= old || atomic.CompareAndSwapInt32(&peak, old, now) { + break + } + } + // Give the scheduler a chance to over-admit if the bound is broken. + for i := 0; i < 200; i++ { + _ = i + } + atomic.AddInt32(&inflight, -1) + return &appx.HarnessResult{Parsed: new(schemas.EnrichedFinding)}, nil + }} + if _, err := EnrichLocationsParallel( + context.Background(), fake, locations, "sast", "injection", "CTX", fixtureRepo, limit, + ); err != nil { + t.Fatalf("EnrichLocationsParallel: %v", err) + } + if got := int(atomic.LoadInt32(&peak)); got > want { + t.Fatalf("peak concurrency %d exceeds the bound %d", got, want) + } + if got := fake.MaxConcurrentHarness(); got > want { + t.Fatalf("appx.Fake peak concurrency %d exceeds the bound %d", got, want) + } + }) + } +} + +// TestEnrichLocationsParallelPropagatesError asserts a failing enrichment +// surfaces as an error and a nil result, the way Python's gather re-raises. +func TestEnrichLocationsParallelPropagatesError(t *testing.T) { + locations := []schemas.VulnLocation{{FilePath: "a.py"}, {FilePath: "b.py"}} + fake := &appx.Fake{HarnessFn: func(_ context.Context, prompt string, _ map[string]any, dest any, _ appx.HarnessOptions) (*appx.HarnessResult, error) { + if strings.Contains(prompt, "- File path: b.py\n") { + return &appx.HarnessResult{IsError: true, ErrorMessage: "boom"}, nil + } + return &appx.HarnessResult{Parsed: dest}, nil + }} + got, err := EnrichLocationsParallel( + context.Background(), fake, locations, "sast", "injection", "CTX", fixtureRepo, 5, + ) + if err == nil { + t.Fatal("want an error") + } + if got != nil { + t.Fatalf("want a nil result alongside the error, got %#v", got) + } + if !strings.Contains(err.Error(), "Hunt finding enricher harness error: boom") { + t.Fatalf("error = %v, want the enricher harness error", err) + } +} + +// itoaSigned keeps the sub-test names readable for negative bounds. +func itoaSigned(n int) string { + if n < 0 { + return "neg" + itoaSigned(-n) + } + digits := "" + if n == 0 { + return "0" + } + for n > 0 { + digits = string(rune('0'+n%10)) + digits + n /= 10 + } + return digits +} diff --git a/internal/secaf/agents/hunt/ssrf.go b/internal/secaf/agents/hunt/ssrf.go new file mode 100644 index 0000000000..b9ffa1aa26 --- /dev/null +++ b/internal/secaf/agents/hunt/ssrf.go @@ -0,0 +1,63 @@ +package hunt + +// Ports src/sec_af/agents/hunt/ssrf.py. + +import ( + "context" + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/recontext" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +const ssrfPromptPath = "hunt/ssrf.txt" + +// ssrfScanPrompt builds the exact prompt run_ssrf_hunter sends. +// +// Python parity: ssrf.py's CONTEXT block is injection.py's with "identify +// injection points" swapped for "identify SSRF points", but the early-stop +// sentence is written as ONE f-string here where injection.py splits it across +// three concatenated literals — the resulting bytes are identical. +func ssrfScanPrompt(repoPath string, recon schemas.ReconResult, depth, earlyStop string) (scanPrompt, reconContext string) { + reconContext = entryFlowContextBlock(recon) + template := prompts.MustLoad(ssrfPromptPath) + template = strings.ReplaceAll(template, "{{RECON_CONTEXT_JSON}}", reconContext) + template = strings.ReplaceAll(template, "{{LANGUAGE_HINTS}}", recontext.LanguageHintsForContext(recon)) + template = strings.ReplaceAll(template, "{{FRAMEWORK_HINTS}}", recontext.FrameworkHintsForContext(recon)) + + scanPrompt = template + + "\n\nCONTEXT:\n" + + "- Repository path: " + repoPath + "\n" + + "- Depth profile: " + depth + "\n" + + "- Early stop rule: if you inspect " + earlyStop + + " files without credible signal, stop and return empty findings.\n" + + "- Focus on RECON entry points and data flows as primary source-to-sink paths.\n" + + "- Explore the codebase, trace data flows from sources to sinks, and identify SSRF points.\n" + + "- Take multiple turns to build findings incrementally and write final JSON only when complete." + return scanPrompt, reconContext +} + +func runSSRFHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, depth, earlyStop string, +) (schemas.HuntResult, error) { + scanPrompt, reconContext := ssrfScanPrompt(repoPath, recon, depth, earlyStop) + return runHunterBody(ctx, app, repoPath, hunterSpec{ + ScanPrompt: scanPrompt, + ReconContext: reconContext, + FindingType: "sast", + Strategy: "ssrf", + EmptyStrategiesRun: nil, // Python: bare HuntResult() + }) +} + +// RunSSRFHunter ports ssrf.py run_ssrf_hunter. +func RunSSRFHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, depth string, maxFilesWithoutSignal int, +) (schemas.HuntResult, error) { + return runSSRFHunter(ctx, app, repoPath, recon, depth, strconv.Itoa(maxFilesWithoutSignal)) +} diff --git a/internal/secaf/agents/hunt/supply_chain.go b/internal/secaf/agents/hunt/supply_chain.go new file mode 100644 index 0000000000..9736f301d3 --- /dev/null +++ b/internal/secaf/agents/hunt/supply_chain.go @@ -0,0 +1,87 @@ +package hunt + +// Ports src/sec_af/agents/hunt/supply_chain.py. + +import ( + "context" + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/recontext" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +const supplyChainPromptPath = "hunt/supply_chain.txt" + +// ShouldRunSupplyChainHunter ports supply_chain.py +// should_run_supply_chain_hunter: +// +// return recon.dependencies.direct_count > 0 +// +// Python parity: the gate reads direct_count only. A repository whose SBOM is +// populated but whose direct_count was never set still skips the hunter. +func ShouldRunSupplyChainHunter(recon schemas.ReconResult) bool { + return recon.Dependencies.DirectCount > 0 +} + +// supplyChainScanPrompt builds the exact prompt run_supply_chain_hunter sends. +// +// Python parity: the early-stop sentence says "manifests/files", not "files", +// and the whole block ends with a trailing newline (only auth.py does the +// same). earlyStop is where the argument cascade lands the depth string for +// this hunter (package doc). +func supplyChainScanPrompt(repoPath string, recon schemas.ReconResult, earlyStop string) (scanPrompt, reconContext string) { + reconContext = recontext.ReconContextForSupplyChain(recon) + template := prompts.MustLoad(supplyChainPromptPath) + template = strings.ReplaceAll(template, "{{RECON_CONTEXT}}", reconContext) + template = strings.ReplaceAll(template, "{{LANGUAGE_HINTS}}", recontext.LanguageHintsForContext(recon)) + template = strings.ReplaceAll(template, "{{FRAMEWORK_HINTS}}", recontext.FrameworkHintsForContext(recon)) + + scanPrompt = template + + "\n\nCONTEXT:\n" + + "- Repository path: " + repoPath + "\n" + + "- Hunt strategy: supply_chain (CWE-1104, CWE-829).\n" + + "- Early stop rule: if you inspect " + + earlyStop + " manifests/files without credible dependency risk, " + + "stop and return empty findings.\n" + + "- Focus manifests/lockfiles (package.json, requirements.txt, go.mod, Pipfile, " + + "poetry.lock, package-lock.json, yarn.lock, pnpm-lock.yaml, Cargo.toml).\n" + + "- Take multiple turns: inspect manifests/lockfiles, validate dependency risks, " + + "then produce final structured findings.\n" + + "- Write final JSON only when analysis is complete.\n" + return scanPrompt, reconContext +} + +func runSupplyChainHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, earlyStop string, +) (schemas.HuntResult, error) { + // Python parity: `_empty_supply_chain_result()` spells out + // HuntResult(findings=[], chains=[], strategies_run=[]), which is the bare + // default shape. + if !ShouldRunSupplyChainHunter(recon) { + return schemas.NewHuntResult(), nil + } + scanPrompt, reconContext := supplyChainScanPrompt(repoPath, recon, earlyStop) + return runHunterBody(ctx, app, repoPath, hunterSpec{ + ScanPrompt: scanPrompt, + ReconContext: reconContext, + // Python parity: the only hunter tagging its findings "sca". + FindingType: "sca", + Strategy: "supply_chain", + EmptyStrategiesRun: []string{"supply_chain"}, + }) +} + +// RunSupplyChainHunter ports supply_chain.py run_supply_chain_hunter: +// +// async def run_supply_chain_hunter(app, repo_path, recon, +// max_files_without_signal: int = 30) -> HuntResult +func RunSupplyChainHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, maxFilesWithoutSignal int, +) (schemas.HuntResult, error) { + return runSupplyChainHunter(ctx, app, repoPath, recon, strconv.Itoa(maxFilesWithoutSignal)) +} diff --git a/internal/secaf/agents/hunt/testdata/golden/assemble_finding.json b/internal/secaf/agents/hunt/testdata/golden/assemble_finding.json new file mode 100644 index 0000000000..7e9f0f1a16 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/assemble_finding.json @@ -0,0 +1,179 @@ +[ + { + "name": "multiline_snippet_sast", + "location": { + "file_path": "app/api/users.py", + "start_line": 42, + "code_snippet": "query = f\"SELECT * FROM users WHERE id = {user_id}\"\ncursor.execute(query)", + "pattern_type": "sql_injection" + }, + "enriched": { + "title": "SQL injection in user lookup", + "description": "user_id flows unescaped into an f-string query.", + "cwe_id": "CWE-89", + "severity": "HIGH", + "confidence": "high", + "data_flow_summary": " request.args['id'] -> query -> cursor.execute " + }, + "finding_type": "sast", + "strategy": "injection", + "want": { + "id": "", + "hunter_strategy": "injection", + "title": "SQL injection in user lookup", + "description": "user_id flows unescaped into an f-string query.", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "CWE-89", + "owasp_category": null, + "file_path": "app/api/users.py", + "start_line": 42, + "end_line": 43, + "function_name": null, + "code_snippet": "query = f\"SELECT * FROM users WHERE id = {user_id}\"\ncursor.execute(query)", + "estimated_severity": "high", + "confidence": "high", + "data_flow": [ + { + "file_path": "app/api/users.py", + "line": 42, + "component": "injection", + "operation": "request.args['id'] -> query -> cursor.execute" + } + ], + "related_files": [], + "fingerprint": "" + } + }, + { + "name": "single_line_bad_enums", + "location": { + "file_path": "app/utils/hash.py", + "start_line": 7, + "code_snippet": "digest = hashlib.md5(password).hexdigest()", + "pattern_type": "weak_hash" + }, + "enriched": { + "title": "Weak hash for password storage", + "description": "MD5 used to derive a credential digest.", + "cwe_id": "CWE-327", + "severity": "catastrophic", + "confidence": "certain", + "data_flow_summary": " " + }, + "finding_type": "sast", + "strategy": "crypto", + "want": { + "id": "", + "hunter_strategy": "crypto", + "title": "Weak hash for password storage", + "description": "MD5 used to derive a credential digest.", + "finding_type": "sast", + "cwe_id": "CWE-327", + "cwe_name": "CWE-327", + "owasp_category": null, + "file_path": "app/utils/hash.py", + "start_line": 7, + "end_line": 7, + "function_name": null, + "code_snippet": "digest = hashlib.md5(password).hexdigest()", + "estimated_severity": "medium", + "confidence": "medium", + "data_flow": null, + "related_files": [], + "fingerprint": "" + } + }, + { + "name": "empty_snippet_unknown_type", + "location": { + "file_path": "a.py", + "start_line": 0, + "code_snippet": "", + "pattern_type": "" + }, + "enriched": { + "title": "t", + "description": "d", + "cwe_id": "", + "severity": "", + "confidence": "", + "data_flow_summary": "x" + }, + "finding_type": "not_a_type", + "strategy": "logic", + "want": { + "id": "", + "hunter_strategy": "logic", + "title": "t", + "description": "d", + "finding_type": "sast", + "cwe_id": "", + "cwe_name": "", + "owasp_category": null, + "file_path": "a.py", + "start_line": 0, + "end_line": 0, + "function_name": null, + "code_snippet": "", + "estimated_severity": "medium", + "confidence": "medium", + "data_flow": [ + { + "file_path": "a.py", + "line": 0, + "component": "logic", + "operation": "x" + } + ], + "related_files": [], + "fingerprint": "" + } + }, + { + "name": "trailing_newline_snippet_uppercase_type", + "location": { + "file_path": "b/c.js", + "start_line": 10, + "code_snippet": "one\ntwo\n", + "pattern_type": "p" + }, + "enriched": { + "title": "t", + "description": "d", + "cwe_id": "CWE-1", + "severity": "INFO", + "confidence": "LOW", + "data_flow_summary": "\n flow \n" + }, + "finding_type": "SCA", + "strategy": "supply_chain", + "want": { + "id": "", + "hunter_strategy": "supply_chain", + "title": "t", + "description": "d", + "finding_type": "sca", + "cwe_id": "CWE-1", + "cwe_name": "CWE-1", + "owasp_category": null, + "file_path": "b/c.js", + "start_line": 10, + "end_line": 12, + "function_name": null, + "code_snippet": "one\ntwo\n", + "estimated_severity": "info", + "confidence": "low", + "data_flow": [ + { + "file_path": "b/c.js", + "line": 10, + "component": "supply_chain", + "operation": "flow" + } + ], + "related_files": [], + "fingerprint": "" + } + } +] diff --git a/internal/secaf/agents/hunt/testdata/golden/auth_depth_label.json b/internal/secaf/agents/hunt/testdata/golden/auth_depth_label.json new file mode 100644 index 0000000000..d43562406a --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/auth_depth_label.json @@ -0,0 +1,8 @@ +{ + "quick": "quick", + "Standard": "standard", + " THOROUGH ": "thorough", + "bogus": "standard", + "": "standard", + " ": "standard" +} diff --git a/internal/secaf/agents/hunt/testdata/golden/auth_target_cwes.json b/internal/secaf/agents/hunt/testdata/golden/auth_target_cwes.json new file mode 100644 index 0000000000..8676f53d14 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/auth_target_cwes.json @@ -0,0 +1,7 @@ +[ + "CWE-287", + "CWE-306", + "CWE-862", + "CWE-863", + "CWE-352" +] diff --git a/internal/secaf/agents/hunt/testdata/golden/business_logic_enabled.json b/internal/secaf/agents/hunt/testdata/golden/business_logic_enabled.json new file mode 100644 index 0000000000..4b46a5cab5 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/business_logic_enabled.json @@ -0,0 +1,8 @@ +{ + "quick": false, + "QUICK": false, + "standard": true, + "thorough": true, + "bogus": true, + "": true +} diff --git a/internal/secaf/agents/hunt/testdata/golden/business_logic_prompt_with_depth_prompt.txt b/internal/secaf/agents/hunt/testdata/golden/business_logic_prompt_with_depth_prompt.txt new file mode 100644 index 0000000000..f76f228691 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/business_logic_prompt_with_depth_prompt.txt @@ -0,0 +1,164 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are the SEC-AF Business Logic Hunter focused on high-impact workflow abuse and invariant violations. + +CONTEXT: +You are in HUNT phase. Analyze this repository using the provided recon context and return HuntResult JSON. + +RECON_CONTEXT: +{ + "app_type": "web_api", + "frameworks": [ + "django" + ], + "languages": [ + "python" + ], + "auth_model": "jwt", + "auth_details": "HS256 access tokens", + "entry_points": [ + { + "kind": "http", + "identifier": "POST /login", + "file_path": "app/api/auth.py", + "line": 12, + "method": "POST", + "route": "/login", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "seed", + "file_path": "app/cli.py", + "line": 3, + "method": null, + "route": null, + "auth_required": null + } + ], + "api_surface": [ + { + "method": "POST", + "path": "/login", + "handler": "login", + "file_path": "app/api/auth.py", + "line": 12, + "auth_required": false, + "rate_limited": false + }, + { + "method": "GET", + "path": "/users/{id}", + "handler": "get_user", + "file_path": "app/api/users.py", + "line": 40, + "auth_required": true, + "rate_limited": true + } + ], + "data_flows": [ + { + "source": "request.json", + "path": [ + { + "file_path": "app/api/users.py", + "line": 41, + "component": "handler", + "operation": "read id" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/api/users.py" + ] + } + ] +} + +OBJECTIVE: +Find business-logic vulnerabilities by reasoning about intended behavior versus actual implementation behavior. + +FOCUS CWES: +- CWE-840: Business Logic Errors +- CWE-841: Improper Enforcement of Behavioral Workflow +- CWE-362: Race Condition +- CWE-367: Time-of-check Time-of-use (TOCTOU) Race Condition +- CWE-639: Authorization Bypass Through User-Controlled Key (IDOR patterns) + +HUNT FOR THESE PATTERNS: +- Race conditions in concurrent requests, especially check-then-act updates without robust locking/transactions +- State machine violations in multi-step workflows (skipping mandatory steps or reordering transitions) +- Missing validation of amounts, ownership, limits, or permissions beyond basic authentication +- Workflow step bypasses (for example, directly invoking step 3 from step 1) +- Integer overflow/underflow or sign bugs in balances, pricing, credits, quantities, and counters +- TOCTOU vulnerabilities where validation/check and use/mutation are separable and exploitable +- IDOR beyond missing auth (predictable identifiers, insecure ownership reassignment, mass assignment to owner/account fields) +- Price manipulation (client-controlled cart totals, discounts, tax, shipping, or final payable amounts) + +DO NOT FLAG: +- Properly locked/serialized concurrent operations +- Explicitly validated state transitions with strict transition guards +- Server-side amount/price calculation that ignores client-submitted totals +- Rate-limited operations where the suspected abuse depends on rapid repetition + +ANALYSIS GUIDELINES: +- Start from recon entry points and data flows, then trace complete end-to-end business workflows. +- Compare behavior across layers (controller/API, domain/service, persistence, queue/worker). +- Validate whether business invariants are enforced at mutation boundaries. +- Distinguish real exploit paths from expected eventual consistency or benign races. +- Include only evidence-backed findings with concrete exploit narrative. + +OUTPUT REQUIREMENTS: +- Return JSON that strictly matches HuntResult. +- Set finding_type to "logic" for business-logic findings. +- Use cwe_id and cwe_name that match the specific issue. +- Populate file_path, start_line, end_line, code_snippet, and related_files whenever possible. +- Generate stable fingerprints from vulnerability identity and location. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +CONSTRAINTS: +- Take multiple turns: investigate workflow intent, validate exploitability, then produce findings. +- Prefer high-confidence findings over speculative broad coverage. +- No markdown, no prose outside JSON, no code fences. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Depth profile: thorough +- Early stop rule: if you inspect 30 files without credible business-logic signal, stop and return empty findings. +- Strategy: business_logic +- Focus CWEs: CWE-840, CWE-841, CWE-362, CWE-367, CWE-639. +- Reason about intended business behavior versus exploitable implementation behavior. +- Take multiple turns, trace complete workflows, and return final JSON only when complete. +- Additional depth guidance: Use deep, multi-turn analysis. Trace cross-file flows and hunt secondary pivots. diff --git a/internal/secaf/agents/hunt/testdata/golden/cascade_binding.json b/internal/secaf/agents/hunt/testdata/golden/cascade_binding.json new file mode 100644 index 0000000000..29cd50dd47 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/cascade_binding.json @@ -0,0 +1,170 @@ +{ + "quick": { + "fixture": "small", + "hunters": { + "auth": { + "repo_path": "/fixtures/demo-repo", + "recon_result": "", + "depth": "quick", + "max_files_without_signal": 30 + }, + "data_exposure": { + "repo_path": "/fixtures/demo-repo", + "recon": "", + "max_files_without_signal": "quick" + }, + "dos": { + "repo_path": "/fixtures/demo-repo", + "recon_result": "", + "depth": "quick", + "max_files_without_signal": 30 + }, + "injection": { + "repo_path": "/fixtures/demo-repo", + "recon_result": "", + "depth": "quick", + "max_files_without_signal": 30 + }, + "ssrf": { + "repo_path": "/fixtures/demo-repo", + "recon_result": "", + "depth": "quick", + "max_files_without_signal": 30 + } + } + }, + "standard": { + "fixture": "rich", + "hunters": { + "api_security": { + "repo_path": "/fixtures/demo-repo", + "recon": "", + "max_files_without_signal": "standard" + }, + "auth": { + "repo_path": "/fixtures/demo-repo", + "recon_result": "", + "depth": "standard", + "max_files_without_signal": 30 + }, + "business_logic": { + "repo_path": "/fixtures/demo-repo", + "recon_result": "", + "depth": "standard", + "max_files_without_signal": 30, + "depth_prompt": "" + }, + "config_secrets": { + "repo_path": "/fixtures/demo-repo", + "recon": "", + "max_files_without_signal": "standard" + }, + "crypto": { + "repo_path": "/fixtures/demo-repo", + "recon": "", + "max_files_without_signal": "standard" + }, + "data_exposure": { + "repo_path": "/fixtures/demo-repo", + "recon": "", + "max_files_without_signal": "standard" + }, + "dos": { + "repo_path": "/fixtures/demo-repo", + "recon_result": "", + "depth": "standard", + "max_files_without_signal": 30 + }, + "injection": { + "repo_path": "/fixtures/demo-repo", + "recon_result": "", + "depth": "standard", + "max_files_without_signal": 30 + }, + "ssrf": { + "repo_path": "/fixtures/demo-repo", + "recon_result": "", + "depth": "standard", + "max_files_without_signal": 30 + }, + "supply_chain": { + "repo_path": "/fixtures/demo-repo", + "recon": "", + "max_files_without_signal": "standard" + }, + "xss": { + "repo_path": "/fixtures/demo-repo", + "recon_result": "", + "depth": "standard", + "max_files_without_signal": 30 + } + } + }, + "thorough": { + "fixture": "small", + "hunters": { + "api_security": { + "repo_path": "/fixtures/demo-repo", + "recon": "", + "max_files_without_signal": "thorough" + }, + "auth": { + "repo_path": "/fixtures/demo-repo", + "recon_result": "", + "depth": "thorough", + "max_files_without_signal": 30 + }, + "business_logic": { + "repo_path": "/fixtures/demo-repo", + "recon_result": "", + "depth": "thorough", + "max_files_without_signal": 30, + "depth_prompt": "" + }, + "config_secrets": { + "repo_path": "/fixtures/demo-repo", + "recon": "", + "max_files_without_signal": "thorough" + }, + "crypto": { + "repo_path": "/fixtures/demo-repo", + "recon": "", + "max_files_without_signal": "thorough" + }, + "data_exposure": { + "repo_path": "/fixtures/demo-repo", + "recon": "", + "max_files_without_signal": "thorough" + }, + "dos": { + "repo_path": "/fixtures/demo-repo", + "recon_result": "", + "depth": "thorough", + "max_files_without_signal": 30 + }, + "injection": { + "repo_path": "/fixtures/demo-repo", + "recon_result": "", + "depth": "thorough", + "max_files_without_signal": 30 + }, + "ssrf": { + "repo_path": "/fixtures/demo-repo", + "recon_result": "", + "depth": "thorough", + "max_files_without_signal": 30 + }, + "supply_chain": { + "repo_path": "/fixtures/demo-repo", + "recon": "", + "max_files_without_signal": "thorough" + }, + "xss": { + "repo_path": "/fixtures/demo-repo", + "recon_result": "", + "depth": "thorough", + "max_files_without_signal": 30 + } + } + } +} diff --git a/internal/secaf/agents/hunt/testdata/golden/crypto_prompt_both_terms.txt b/internal/secaf/agents/hunt/testdata/golden/crypto_prompt_both_terms.txt new file mode 100644 index 0000000000..bb2a794f05 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/crypto_prompt_both_terms.txt @@ -0,0 +1,93 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are the SEC-AF Cryptography Hunter specializing in weak cryptography and predictable randomness. + +CONTEXT: +You are in SEC-AF HUNT phase and must return HuntResult JSON. +Use the RECON context summary below. +Focus on whether detected crypto patterns are used in security-sensitive contexts. + +RECON_CONTEXT: +Cryptography-focused recon summary. + +Crypto usage entries: 1 total. + +Algorithms and key handling (weak entries first): 1 total, showing top 1: +- algorithm=MD5, key_size=None, mode=None, context=auth token cache, is_weak=True + +Potential secret/key findings from config scan: none identified in recon. + +Deployment/TLS/security header signals: none identified in recon. + +TASK: +Find potential vulnerabilities related to weak cryptography. + +CWE COVERAGE (REQUIRED): +- CWE-326 Inadequate Encryption Strength +- CWE-327 Broken or Risky Cryptographic Algorithm +- CWE-328 Reversible One-Way Hash +- CWE-330 Insufficiently Random Values +- CWE-916 Weak Password Hashing +- CWE-259 Hard-Coded Password +- CWE-321 Hard-Coded Cryptographic Key +- CWE-798 Hardcoded Credentials + +WHAT TO HUNT: +- Weak algorithms: MD5, SHA1 for security decisions, DES, 3DES, RC4, ECB mode. +- Inadequate key lengths: RSA < 2048, AES < 128, weak DH parameters. +- Reversible password handling: base64/encoding used as if it were hashing. +- Predictable randomness: Math.random(), non-crypto random module usage for tokens/secrets, predictable seeds. +- Weak password storage: unsalted/fast hashes, low-iteration PBKDF2, custom insecure hashing. +- Hardcoded cryptographic keys, secrets, or static IV/nonce values. +- Security-critical usage contexts: password hashing, encryption, token/session generation, signing and verification, auth flows. +- Non-critical usage contexts: checksums, ETags, cache keys, dedup/fingerprinting. + +DO NOT FLAG: +- Crypto operations in tests or fixtures. +- MD5/SHA1 used only for non-security checksums/caching/ETags/cache keys. +- bcrypt/argon2/scrypt for password hashing (these are GOOD). +- Crypto libraries used correctly with sufficient key lengths and secure modes. +- Framework secure defaults (e.g., Django PBKDF2, bcrypt, argon2). + +OUTPUT: +- Return JSON matching HuntResult. +- findings must contain RawFinding entries with evidence-backed descriptions and correct CWE mapping. +- hunter_strategy should be "crypto". + +No language-specific hints available for detected languages. +No framework-specific hints available for detected frameworks. + +CONSTRAINTS: +- Take multiple turns: first discover crypto-relevant files, then evaluate exploitability context. +- Prefer concrete evidence (file path, line range, code snippet, why risky in context). +- No markdown/code fences; JSON output only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Hunt strategy: crypto +- Early stop rule: if you inspect 30 files without credible crypto misuse, stop and return empty findings. +- Focus CWEs: CWE-326, CWE-327, CWE-328, CWE-330, CWE-916, CWE-259, CWE-321, CWE-798 +- Security-critical usage candidates: auth token cache +- Non-security usage candidates: auth token cache +- Prioritize weak crypto findings only when used in security-sensitive contexts; avoid checksum/cache-only noise. +- Take multiple turns to explore relevant files before finalizing findings. +- Write final JSON only when analysis is complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/crypto_prompt_mixed.txt b/internal/secaf/agents/hunt/testdata/golden/crypto_prompt_mixed.txt new file mode 100644 index 0000000000..350c818d8a --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/crypto_prompt_mixed.txt @@ -0,0 +1,99 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are the SEC-AF Cryptography Hunter specializing in weak cryptography and predictable randomness. + +CONTEXT: +You are in SEC-AF HUNT phase and must return HuntResult JSON. +Use the RECON context summary below. +Focus on whether detected crypto patterns are used in security-sensitive contexts. + +RECON_CONTEXT: +Cryptography-focused recon summary. + +Crypto usage entries: 7 total. + +Algorithms and key handling (weak entries first): 7 total, showing top 7: +- algorithm=MD5, key_size=None, mode=None, context=password hashing, is_weak=True +- algorithm=MD5, key_size=None, mode=None, context=file integrity checksum, is_weak=True +- algorithm=MD5, key_size=None, mode=None, context=etag generation for cache, is_weak=True +- algorithm=MD5, key_size=None, mode=None, context=TLS session key derivation, is_weak=True +- algorithm=MD5, key_size=None, mode=None, context=unrelated purpose, is_weak=True +- algorithm=MD5, key_size=None, mode=None, context=unspecified, is_weak=True +- algorithm=MD5, key_size=None, mode=None, context=unspecified, is_weak=True + +Potential secret/key findings from config scan: none identified in recon. + +Deployment/TLS/security header signals: none identified in recon. + +TASK: +Find potential vulnerabilities related to weak cryptography. + +CWE COVERAGE (REQUIRED): +- CWE-326 Inadequate Encryption Strength +- CWE-327 Broken or Risky Cryptographic Algorithm +- CWE-328 Reversible One-Way Hash +- CWE-330 Insufficiently Random Values +- CWE-916 Weak Password Hashing +- CWE-259 Hard-Coded Password +- CWE-321 Hard-Coded Cryptographic Key +- CWE-798 Hardcoded Credentials + +WHAT TO HUNT: +- Weak algorithms: MD5, SHA1 for security decisions, DES, 3DES, RC4, ECB mode. +- Inadequate key lengths: RSA < 2048, AES < 128, weak DH parameters. +- Reversible password handling: base64/encoding used as if it were hashing. +- Predictable randomness: Math.random(), non-crypto random module usage for tokens/secrets, predictable seeds. +- Weak password storage: unsalted/fast hashes, low-iteration PBKDF2, custom insecure hashing. +- Hardcoded cryptographic keys, secrets, or static IV/nonce values. +- Security-critical usage contexts: password hashing, encryption, token/session generation, signing and verification, auth flows. +- Non-critical usage contexts: checksums, ETags, cache keys, dedup/fingerprinting. + +DO NOT FLAG: +- Crypto operations in tests or fixtures. +- MD5/SHA1 used only for non-security checksums/caching/ETags/cache keys. +- bcrypt/argon2/scrypt for password hashing (these are GOOD). +- Crypto libraries used correctly with sufficient key lengths and secure modes. +- Framework secure defaults (e.g., Django PBKDF2, bcrypt, argon2). + +OUTPUT: +- Return JSON matching HuntResult. +- findings must contain RawFinding entries with evidence-backed descriptions and correct CWE mapping. +- hunter_strategy should be "crypto". + +No language-specific hints available for detected languages. +No framework-specific hints available for detected frameworks. + +CONSTRAINTS: +- Take multiple turns: first discover crypto-relevant files, then evaluate exploitability context. +- Prefer concrete evidence (file path, line range, code snippet, why risky in context). +- No markdown/code fences; JSON output only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Hunt strategy: crypto +- Early stop rule: if you inspect 30 files without credible crypto misuse, stop and return empty findings. +- Focus CWEs: CWE-326, CWE-327, CWE-328, CWE-330, CWE-916, CWE-259, CWE-321, CWE-798 +- Security-critical usage candidates: password hashing, TLS session key derivation +- Non-security usage candidates: file integrity checksum, etag generation for cache +- Prioritize weak crypto findings only when used in security-sensitive contexts; avoid checksum/cache-only noise. +- Take multiple turns to explore relevant files before finalizing findings. +- Write final JSON only when analysis is complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/crypto_prompt_none.txt b/internal/secaf/agents/hunt/testdata/golden/crypto_prompt_none.txt new file mode 100644 index 0000000000..2d086e9613 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/crypto_prompt_none.txt @@ -0,0 +1,94 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are the SEC-AF Cryptography Hunter specializing in weak cryptography and predictable randomness. + +CONTEXT: +You are in SEC-AF HUNT phase and must return HuntResult JSON. +Use the RECON context summary below. +Focus on whether detected crypto patterns are used in security-sensitive contexts. + +RECON_CONTEXT: +Cryptography-focused recon summary. + +Crypto usage entries: 2 total. + +Algorithms and key handling (weak entries first): 2 total, showing top 2: +- algorithm=MD5, key_size=None, mode=None, context=unrelated purpose, is_weak=True +- algorithm=MD5, key_size=None, mode=None, context=widget rendering, is_weak=True + +Potential secret/key findings from config scan: none identified in recon. + +Deployment/TLS/security header signals: none identified in recon. + +TASK: +Find potential vulnerabilities related to weak cryptography. + +CWE COVERAGE (REQUIRED): +- CWE-326 Inadequate Encryption Strength +- CWE-327 Broken or Risky Cryptographic Algorithm +- CWE-328 Reversible One-Way Hash +- CWE-330 Insufficiently Random Values +- CWE-916 Weak Password Hashing +- CWE-259 Hard-Coded Password +- CWE-321 Hard-Coded Cryptographic Key +- CWE-798 Hardcoded Credentials + +WHAT TO HUNT: +- Weak algorithms: MD5, SHA1 for security decisions, DES, 3DES, RC4, ECB mode. +- Inadequate key lengths: RSA < 2048, AES < 128, weak DH parameters. +- Reversible password handling: base64/encoding used as if it were hashing. +- Predictable randomness: Math.random(), non-crypto random module usage for tokens/secrets, predictable seeds. +- Weak password storage: unsalted/fast hashes, low-iteration PBKDF2, custom insecure hashing. +- Hardcoded cryptographic keys, secrets, or static IV/nonce values. +- Security-critical usage contexts: password hashing, encryption, token/session generation, signing and verification, auth flows. +- Non-critical usage contexts: checksums, ETags, cache keys, dedup/fingerprinting. + +DO NOT FLAG: +- Crypto operations in tests or fixtures. +- MD5/SHA1 used only for non-security checksums/caching/ETags/cache keys. +- bcrypt/argon2/scrypt for password hashing (these are GOOD). +- Crypto libraries used correctly with sufficient key lengths and secure modes. +- Framework secure defaults (e.g., Django PBKDF2, bcrypt, argon2). + +OUTPUT: +- Return JSON matching HuntResult. +- findings must contain RawFinding entries with evidence-backed descriptions and correct CWE mapping. +- hunter_strategy should be "crypto". + +No language-specific hints available for detected languages. +No framework-specific hints available for detected frameworks. + +CONSTRAINTS: +- Take multiple turns: first discover crypto-relevant files, then evaluate exploitability context. +- Prefer concrete evidence (file path, line range, code snippet, why risky in context). +- No markdown/code fences; JSON output only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Hunt strategy: crypto +- Early stop rule: if you inspect 30 files without credible crypto misuse, stop and return empty findings. +- Focus CWEs: CWE-326, CWE-327, CWE-328, CWE-330, CWE-916, CWE-259, CWE-321, CWE-798 +- Security-critical usage candidates: none +- Non-security usage candidates: none +- Prioritize weak crypto findings only when used in security-sensitive contexts; avoid checksum/cache-only noise. +- Take multiple turns to explore relevant files before finalizing findings. +- Write final JSON only when analysis is complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/crypto_term_tables.json b/internal/secaf/agents/hunt/testdata/golden/crypto_term_tables.json new file mode 100644 index 0000000000..e6f62e28f2 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/crypto_term_tables.json @@ -0,0 +1,27 @@ +{ + "security_critical": [ + "password", + "passwd", + "credential", + "auth", + "token", + "session", + "encrypt", + "decrypt", + "signature", + "sign", + "verify", + "jwt", + "tls", + "ssl", + "key" + ], + "non_security": [ + "checksum", + "etag", + "cache", + "fingerprint", + "dedup", + "integrity" + ] +} diff --git a/internal/secaf/agents/hunt/testdata/golden/crypto_usage_partition.json b/internal/secaf/agents/hunt/testdata/golden/crypto_usage_partition.json new file mode 100644 index 0000000000..f7cc04ae5f --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/crypto_usage_partition.json @@ -0,0 +1,41 @@ +{ + "mixed": { + "usage_contexts": [ + "password hashing", + "file integrity checksum", + "etag generation for cache", + "TLS session key derivation", + "unrelated purpose" + ], + "security_critical": [ + "password hashing", + "TLS session key derivation" + ], + "non_security": [ + "file integrity checksum", + "etag generation for cache" + ], + "should_run": true + }, + "none": { + "usage_contexts": [ + "unrelated purpose", + "widget rendering" + ], + "security_critical": [], + "non_security": [], + "should_run": true + }, + "both_terms": { + "usage_contexts": [ + "auth token cache" + ], + "security_critical": [ + "auth token cache" + ], + "non_security": [ + "auth token cache" + ], + "should_run": true + } +} diff --git a/internal/secaf/agents/hunt/testdata/golden/direct_prompt_api_security.txt b/internal/secaf/agents/hunt/testdata/golden/direct_prompt_api_security.txt new file mode 100644 index 0000000000..b29bcfb4e7 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/direct_prompt_api_security.txt @@ -0,0 +1,182 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are an API security hunter specializing in authorization bypasses, origin validation flaws, SSRF, and open redirect vulnerabilities. + +API_SECURITY_CONTEXT: +API security-focused recon summary. + +API endpoints prioritized by missing auth/rate-limits: 20 total, showing top 15: +- POST /api/v1/thing/1 -> ThingController.action1 (app/api/thing_1.py:103, auth_required=False, rate_limited=False) +- DELETE /api/v1/thing/3 -> ThingController.action3 (app/api/thing_3.py:109, auth_required=False, rate_limited=False) +- PUT /api/v1/thing/7 -> ThingController.action7 (app/api/thing_7.py:121, auth_required=False, rate_limited=False) +- PATCH /api/v1/thing/9 -> ThingController.action9 (app/api/thing_9.py:127, auth_required=False, rate_limited=False) +- DELETE /api/v1/thing/13 -> ThingController.action13 (app/api/thing_13.py:139, auth_required=False, rate_limited=False) +- GET /api/v1/thing/15 -> ThingController.action15 (app/api/thing_15.py:145, auth_required=False, rate_limited=False) +- PATCH /api/v1/thing/19 -> ThingController.action19 (app/api/thing_19.py:157, auth_required=False, rate_limited=False) +- GET /api/v1/thing/0 -> ThingController.action0 (app/api/thing_0.py:100, auth_required=True, rate_limited=None) +- PUT /api/v1/thing/2 -> ThingController.action2 (app/api/thing_2.py:106, auth_required=None, rate_limited=True) +- PATCH /api/v1/thing/4 -> ThingController.action4 (app/api/thing_4.py:112, auth_required=True, rate_limited=None) +- GET /api/v1/thing/5 -> ThingController.action5 (app/api/thing_5.py:115, auth_required=None, rate_limited=True) +- POST /api/v1/thing/6 -> ThingController.action6 (app/api/thing_6.py:118, auth_required=True, rate_limited=None) +- DELETE /api/v1/thing/8 -> ThingController.action8 (app/api/thing_8.py:124, auth_required=None, rate_limited=True) +- GET /api/v1/thing/10 -> ThingController.action10 (app/api/thing_10.py:130, auth_required=True, rate_limited=None) +- POST /api/v1/thing/11 -> ThingController.action11 (app/api/thing_11.py:133, auth_required=None, rate_limited=True) + +HTTP/API entry points: 11 total, showing top 10: +- http handler_0 (app/entry/e0.py:10, auth_required=True) +- HTTP /v1/resource/1 (app/entry/e1.py:17, auth_required=False) +- api handler_3 (app/entry/e3.py:31, auth_required=True) +- graphql /v1/resource/4 (app/entry/e4.py:38, auth_required=False) +- rpc /v1/resource/5 (app/entry/e5.py:45, auth_required=None) +- route handler_6 (app/entry/e6.py:52, auth_required=True) +- http handler_9 (app/entry/e9.py:73, auth_required=True) +- api /v1/resource/10 (app/entry/e10.py:80, auth_required=False) +- http handler_12 (app/entry/e12.py:94, auth_required=True) +- route /v1/resource/13 (app/entry/e13.py:101, auth_required=False) + +Trust boundaries relevant to API calls: 12 total, showing top 10: +- boundary_0: internet -> app; enforcement=none +- boundary_1: dmz -> db; enforcement=waf_1, mtls_1 +- boundary_2: vpc -> cache; enforcement=waf_2, mtls_2 +- boundary_3: worker -> queue; enforcement=waf_3, mtls_3 +- boundary_4: internet -> app; enforcement=none +- boundary_5: dmz -> db; enforcement=waf_5, mtls_5 +- boundary_6: vpc -> cache; enforcement=waf_6, mtls_6 +- boundary_7: worker -> queue; enforcement=waf_7, mtls_7 +- boundary_8: internet -> app; enforcement=none +- boundary_9: dmz -> db; enforcement=waf_9, mtls_9 + +Framework/deployment API security signals: 13 total, showing top 10: +- django.middleware.csrf.CsrfViewMiddleware +- helmet defaults +- SecurityMiddleware +- rack-protection +- spring-security filter chain +- kubernetes ingress with TLS termination +- docker-compose exposes 5432 +- no WAF in front of /api +- secrets mounted from vault +- readiness probe on /healthz + +TASK: +Analyze API endpoints and supporting middleware/controllers for: +- CWE-285 Improper Authorization (BOLA/IDOR and missing ownership checks) +- CWE-346 Origin Validation Error (CORS misconfiguration, weak origin validation) +- CWE-918 SSRF (user-controlled URLs reaching server-side fetch clients) +- CWE-601 Open Redirect (untrusted redirect targets) + +Also hunt for: +- Missing/weak rate limiting on authentication and sensitive endpoints +- Mass assignment where request payloads are mapped directly to models/ORM updates +- Missing input validation (unbounded strings, unsafe coercion, negative or out-of-range values) + +WORKFLOW (follow these steps in order): +1. Read the API endpoint files and route handlers identified in the API_SECURITY_CONTEXT above. +2. For each endpoint, trace user input from request parameters/body to where it is used (database queries, redirects, outbound HTTP calls, etc.). +3. Check for missing authorization, missing input validation, CORS misconfigurations, and SSRF vectors. +4. For each confirmed vulnerability, record the file path, line number, and relevant code snippet. +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +SCOPE: +- Prioritize internet-facing API routes and handlers from api_surface. +- Trace request parameters and body fields to authorization checks, outbound HTTP clients, redirect responses, and persistence layers. +- Correlate evidence across route definitions, middleware, validators, and shared utilities. +- Exclude speculative findings without source evidence. + +OUTPUT: +- Populate findings with finding_type="api" and hunter_strategy="api_security". +- Include endpoint-specific evidence (method/path, file_path, lines, and relevant snippets). +- Include attack chains only when there is concrete multi-step linkage. +- Empty arrays are acceptable if no vulnerabilities are confirmed. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: JAVASCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec, new Function + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, parameterized pg queries + DO NOT FLAG: React JSX expressions (auto-escaped), Angular template bindings (sanitized by default) + Framework notes: Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous. + +Language: GO + Key sinks: fmt.Sprintf into SQL, exec.Command with user input, template.HTML(), os.Exec + Safe patterns (skip these): database/sql with ? placeholders, html/template (auto-escapes), exec.Command with separate args + DO NOT FLAG: GORM parameterized queries, html/template default escaping, database/sql prepared statements + Framework notes: Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML. + +Language: RUBY + Key sinks: eval, system, exec, send, public_send, ERB.new with user input + Safe patterns (skip these): ActiveRecord parameterized queries, Rack::Utils.escape_html, sanitize helper in Rails + DO NOT FLAG: ActiveRecord where with hash conditions, Rails CSRF protection, Rails html_safe on constants + Framework notes: Rails auto-escapes ERB templates. raw/html_safe bypasses escaping. Check for mass assignment. + +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +Framework: NEXTJS + Security features: React JSX auto-escapes output by default, API routes can share centralized auth middleware, Server/client boundaries reduce accidental secret exposure + DO NOT FLAG: React JSX escaped rendering, getServerSideProps/getServerSession with proper auth checks, Typed route handlers with validated schema guards + Watch for: dangerouslySetInnerHTML, unprotected API routes under /api, secret leakage to client bundles, rewrites/redirects from untrusted user input + Common vulns: XSS through dangerous HTML rendering, IDOR/auth bypass in API handlers, Sensitive env/config exposure in client-side code + +Framework: SPRING + Security features: Spring Security provides auth, CSRF, and filter chain defaults, JPA/Hibernate prepared parameter binding by default, Bean validation can enforce request constraints + DO NOT FLAG: PreparedStatement/JPA named parameter usage, Spring Security CSRF/auth filters clearly active, Thymeleaf auto-escaped output + Watch for: @PreAuthorize missing on sensitive methods, JdbcTemplate/raw Statement with concatenation, csrf().disable() on browser session flows, Actuator endpoints exposed without auth + Common vulns: Authz bypass from weak method-level security, SQL injection in raw JDBC queries, Sensitive management endpoint exposure + +Framework: REACT + Security features: JSX escapes strings before DOM rendering, Component model encourages explicit data flow, Framework discourages direct DOM mutation + DO NOT FLAG: Standard JSX expression rendering, textContent assignment for untrusted text, sanitized HTML via trusted DOMPurify policy + Watch for: dangerouslySetInnerHTML, untrusted URL assignment to href/src, eval/new Function in client code, token/secret exposure in bundles + Common vulns: DOM XSS through unsafe HTML injection, Open redirect via unvalidated navigation targets, Sensitive data exposure in frontend artifacts + +Framework: VUE + Security features: Mustache template interpolation escapes HTML, Component props/events provide explicit boundaries, Router guards can enforce auth flows + DO NOT FLAG: escaped template interpolation {{ value }}, validated route guards protecting private routes, sanitized content rendered through safe components + Watch for: v-html with untrusted data, dynamic component/template compilation, unsafe URL bindings in href/src, client-side auth checks without server enforcement + Common vulns: XSS through v-html and unsafe render paths, Auth bypass from client-only route protection, Open redirect patterns in router navigation + +CONSTRAINTS: +- Prefer confirmed code evidence over heuristics. +- Base all findings on actual file content you read. Do not speculate. +- Do not include markdown, prose, or code fences in the output file. JSON only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Focus only on API-relevant code paths and endpoint handlers surfaced by RECON. +- Early stop rule: if you inspect 30 files without credible API issues, stop and return empty findings. +- Read the handler files first, then generate findings. +- After gathering evidence, write the JSON output file using your Write tool. diff --git a/internal/secaf/agents/hunt/testdata/golden/direct_prompt_config_secrets.txt b/internal/secaf/agents/hunt/testdata/golden/direct_prompt_config_secrets.txt new file mode 100644 index 0000000000..fa0ffedf9c --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/direct_prompt_config_secrets.txt @@ -0,0 +1,178 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are the SEC-AF Config/Secrets Hunter for HUNT phase. + +CONTEXT: +You receive RECON context summary and must produce HuntResult JSON for strategy config_secrets. + +RECON_CONTEXT: +Config and secrets-focused recon summary. + +Detected secret-like findings: 17 total, showing top 15: +- aws_access_key at config/env_0.yaml:3; confidence=high; is_test_value=False +- github_token at config/env_1.yaml:4; confidence=medium; is_test_value=True +- private_key at config/env_2.yaml:5; confidence=low; is_test_value=None +- slack_webhook at config/env_3.yaml:6; confidence=high; is_test_value=False +- generic_api_key at config/env_4.yaml:7; confidence=medium; is_test_value=True +- aws_access_key at config/env_5.yaml:8; confidence=low; is_test_value=None +- github_token at config/env_6.yaml:9; confidence=high; is_test_value=False +- private_key at config/env_7.yaml:10; confidence=medium; is_test_value=True +- slack_webhook at config/env_8.yaml:11; confidence=low; is_test_value=None +- generic_api_key at config/env_9.yaml:12; confidence=high; is_test_value=False +- aws_access_key at config/env_10.yaml:13; confidence=medium; is_test_value=True +- github_token at config/env_11.yaml:14; confidence=low; is_test_value=None +- private_key at config/env_12.yaml:15; confidence=high; is_test_value=False +- slack_webhook at config/env_13.yaml:16; confidence=medium; is_test_value=True +- generic_api_key at config/env_14.yaml:17; confidence=low; is_test_value=None + +Configuration weaknesses from recon: 16 total, showing top 15: +- logging at config/app_0.ini:0; risk=high; key=LOG_LEVEL +- tls at config/app_1.ini:13; risk=critical; key=SSL_VERIFY +- cors at config/app_2.ini:14; risk=high; key=ALLOW_ORIGIN +- headers at config/app_3.ini:0; risk=medium; key=X_FRAME_OPTIONS +- debug at config/app_4.ini:16; risk=critical; key=DJANGO_DEBUG +- storage at config/app_5.ini:0; risk=critical; key=BUCKET_ACL +- auth at config/app_6.ini:18; risk=medium; key=SESSION_TIMEOUT +- http at config/app_7.ini:19; risk=high; key=REDIRECT_HTTPS +- secrets at config/app_8.ini:20; risk=medium; key=n/a +- trace at config/app_9.ini:21; risk=low; key=OTEL_TRACE_ALL +- network at config/app_10.ini:0; risk=medium; key=BIND_ADDR +- errors at config/app_11.ini:23; risk=high; key=SHOW_STACKTRACE +- cache at config/app_12.ini:24; risk=low; key=CACHE_TTL +- db at config/app_13.ini:25; risk=critical; key=SSLMODE +- queue at config/app_14.ini:26; risk=low; key=PREFETCH + +Security/deployment context affecting config risk: 13 total, showing top 10: +- kubernetes ingress with TLS termination +- docker-compose exposes 5432 +- no WAF in front of /api +- secrets mounted from vault +- readiness probe on /healthz +- single replica for the worker +- TLS 1.2 minimum +- internal service mesh mTLS +- django.middleware.csrf.CsrfViewMiddleware +- helmet defaults + +OBJECTIVE: +Find real security findings for: +- CWE-798 Hardcoded Credentials +- CWE-259 Hard-Coded Password +- CWE-321 Hard-Coded Cryptographic Key +- CWE-16 Configuration weaknesses + +HUNT FOR: +- API keys, passwords, bearer tokens, database credentials, cloud credentials, signing secrets, encryption keys hardcoded in source. +- Insecure defaults and configuration issues such as DEBUG enabled in production paths, permissive CORS (`*`), insecure cookies, disabled TLS verification, missing security headers, exposed admin/debug endpoints. + +DO NOT FLAG: +- `.env.example`, `config.example.*`, template/sample config files. +- Test fixtures and explicit fake values used for tests. +- Documentation snippets. +- Secure environment-variable usage patterns (for example `os.getenv(...)`, `process.env.*`, `${ENV_VAR}`). + +OUTPUT REQUIREMENTS: +- Return strict JSON matching HuntResult. +- Set `hunter_strategy` to `config_secrets` on every finding. +- Use `finding_type` as `secrets` for hardcoded secrets and `config` for configuration issues. +- Include precise file path, line range, and code snippet evidence. +- Include related files when context spans multiple files. +- Add reviewer metadata in `description` using this suffix format for each finding: + `[is_test_file=;is_example=]` + +QUALITY BAR: +- Prioritize high-confidence, exploitable findings over noisy pattern matches. +- Distinguish real credentials from placeholders by context, naming, entropy, and usage. +- If uncertain, lower confidence instead of over-claiming. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: JAVASCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec, new Function + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, parameterized pg queries + DO NOT FLAG: React JSX expressions (auto-escaped), Angular template bindings (sanitized by default) + Framework notes: Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous. + +Language: GO + Key sinks: fmt.Sprintf into SQL, exec.Command with user input, template.HTML(), os.Exec + Safe patterns (skip these): database/sql with ? placeholders, html/template (auto-escapes), exec.Command with separate args + DO NOT FLAG: GORM parameterized queries, html/template default escaping, database/sql prepared statements + Framework notes: Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML. + +Language: RUBY + Key sinks: eval, system, exec, send, public_send, ERB.new with user input + Safe patterns (skip these): ActiveRecord parameterized queries, Rack::Utils.escape_html, sanitize helper in Rails + DO NOT FLAG: ActiveRecord where with hash conditions, Rails CSRF protection, Rails html_safe on constants + Framework notes: Rails auto-escapes ERB templates. raw/html_safe bypasses escaping. Check for mass assignment. + +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +Framework: NEXTJS + Security features: React JSX auto-escapes output by default, API routes can share centralized auth middleware, Server/client boundaries reduce accidental secret exposure + DO NOT FLAG: React JSX escaped rendering, getServerSideProps/getServerSession with proper auth checks, Typed route handlers with validated schema guards + Watch for: dangerouslySetInnerHTML, unprotected API routes under /api, secret leakage to client bundles, rewrites/redirects from untrusted user input + Common vulns: XSS through dangerous HTML rendering, IDOR/auth bypass in API handlers, Sensitive env/config exposure in client-side code + +Framework: SPRING + Security features: Spring Security provides auth, CSRF, and filter chain defaults, JPA/Hibernate prepared parameter binding by default, Bean validation can enforce request constraints + DO NOT FLAG: PreparedStatement/JPA named parameter usage, Spring Security CSRF/auth filters clearly active, Thymeleaf auto-escaped output + Watch for: @PreAuthorize missing on sensitive methods, JdbcTemplate/raw Statement with concatenation, csrf().disable() on browser session flows, Actuator endpoints exposed without auth + Common vulns: Authz bypass from weak method-level security, SQL injection in raw JDBC queries, Sensitive management endpoint exposure + +Framework: REACT + Security features: JSX escapes strings before DOM rendering, Component model encourages explicit data flow, Framework discourages direct DOM mutation + DO NOT FLAG: Standard JSX expression rendering, textContent assignment for untrusted text, sanitized HTML via trusted DOMPurify policy + Watch for: dangerouslySetInnerHTML, untrusted URL assignment to href/src, eval/new Function in client code, token/secret exposure in bundles + Common vulns: DOM XSS through unsafe HTML injection, Open redirect via unvalidated navigation targets, Sensitive data exposure in frontend artifacts + +Framework: VUE + Security features: Mustache template interpolation escapes HTML, Component props/events provide explicit boundaries, Router guards can enforce auth flows + DO NOT FLAG: escaped template interpolation {{ value }}, validated route guards protecting private routes, sanitized content rendered through safe components + Watch for: v-html with untrusted data, dynamic component/template compilation, unsafe URL bindings in href/src, client-side auth checks without server enforcement + Common vulns: XSS through v-html and unsafe render paths, Auth bypass from client-only route protection, Open redirect patterns in router navigation + +CONSTRAINTS: +- Take multiple turns: first explore files, then validate candidate findings, then output final JSON. +- No markdown or code fences. JSON only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Hunt strategy: config_secrets (CWE-798, CWE-259, CWE-321, CWE-16). +- Early stop rule: if you inspect 30 files without credible secrets/config issues, stop and return empty findings. +- Use RECON ConfigReport and SecurityContext to prioritize likely real findings. +- Take multiple turns: inspect files, validate exploitability signal, then build findings. + diff --git a/internal/secaf/agents/hunt/testdata/golden/direct_prompt_crypto.txt b/internal/secaf/agents/hunt/testdata/golden/direct_prompt_crypto.txt new file mode 100644 index 0000000000..381f8bbf3c --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/direct_prompt_crypto.txt @@ -0,0 +1,185 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are the SEC-AF Cryptography Hunter specializing in weak cryptography and predictable randomness. + +CONTEXT: +You are in SEC-AF HUNT phase and must return HuntResult JSON. +Use the RECON context summary below. +Focus on whether detected crypto patterns are used in security-sensitive contexts. + +RECON_CONTEXT: +Cryptography-focused recon summary. + +Crypto usage entries: 16 total. + +Algorithms and key handling (weak entries first): 16 total, showing top 15: +- algorithm=MD5, key_size=None, mode=None, context=password hashing, is_weak=True +- algorithm=RSA, key_size=1024, mode=None, context=token signing, is_weak=True +- algorithm=DES, key_size=56, mode=CBC, context=legacy export, is_weak=True +- algorithm=HMAC-SHA1, key_size=160, mode=None, context=webhook signature, is_weak=True +- algorithm=RC4, key_size=128, mode=None, context=unspecified, is_weak=True +- algorithm=SHA-1, key_size=None, mode=None, context=checksum, is_weak=True +- algorithm=AES, key_size=128, mode=ECB, context=legacy blob, is_weak=True +- algorithm=3DES, key_size=168, mode=CBC, context=legacy tape, is_weak=True +- algorithm=AES, key_size=256, mode=GCM, context=at-rest encryption, is_weak=False +- algorithm=SHA-256, key_size=None, mode=None, context=unspecified, is_weak=False +- algorithm=ChaCha20, key_size=256, mode=Poly1305, context=transport, is_weak=False +- algorithm=bcrypt, key_size=None, mode=None, context=password hashing, is_weak=False +- algorithm=ECDSA, key_size=256, mode=None, context=JWT signing, is_weak=None +- algorithm=PBKDF2, key_size=None, mode=None, context=key derivation, is_weak=False +- algorithm=Ed25519, key_size=256, mode=None, context=package signing, is_weak=False + +Potential secret/key findings from config scan: 17 total, showing top 10: +- aws_access_key at config/env_0.yaml:3 (confidence=high) +- github_token at config/env_1.yaml:4 (confidence=medium) +- private_key at config/env_2.yaml:5 (confidence=low) +- slack_webhook at config/env_3.yaml:6 (confidence=high) +- generic_api_key at config/env_4.yaml:7 (confidence=medium) +- aws_access_key at config/env_5.yaml:8 (confidence=low) +- github_token at config/env_6.yaml:9 (confidence=high) +- private_key at config/env_7.yaml:10 (confidence=medium) +- slack_webhook at config/env_8.yaml:11 (confidence=low) +- generic_api_key at config/env_9.yaml:12 (confidence=high) + +Deployment/TLS/security header signals: 14 total, showing top 10: +- kubernetes ingress with TLS termination +- docker-compose exposes 5432 +- no WAF in front of /api +- secrets mounted from vault +- readiness probe on /healthz +- single replica for the worker +- TLS 1.2 minimum +- internal service mesh mTLS +- Content-Security-Policy: default-src 'self' +- X-Content-Type-Options: nosniff + +TASK: +Find potential vulnerabilities related to weak cryptography. + +CWE COVERAGE (REQUIRED): +- CWE-326 Inadequate Encryption Strength +- CWE-327 Broken or Risky Cryptographic Algorithm +- CWE-328 Reversible One-Way Hash +- CWE-330 Insufficiently Random Values +- CWE-916 Weak Password Hashing +- CWE-259 Hard-Coded Password +- CWE-321 Hard-Coded Cryptographic Key +- CWE-798 Hardcoded Credentials + +WHAT TO HUNT: +- Weak algorithms: MD5, SHA1 for security decisions, DES, 3DES, RC4, ECB mode. +- Inadequate key lengths: RSA < 2048, AES < 128, weak DH parameters. +- Reversible password handling: base64/encoding used as if it were hashing. +- Predictable randomness: Math.random(), non-crypto random module usage for tokens/secrets, predictable seeds. +- Weak password storage: unsalted/fast hashes, low-iteration PBKDF2, custom insecure hashing. +- Hardcoded cryptographic keys, secrets, or static IV/nonce values. +- Security-critical usage contexts: password hashing, encryption, token/session generation, signing and verification, auth flows. +- Non-critical usage contexts: checksums, ETags, cache keys, dedup/fingerprinting. + +DO NOT FLAG: +- Crypto operations in tests or fixtures. +- MD5/SHA1 used only for non-security checksums/caching/ETags/cache keys. +- bcrypt/argon2/scrypt for password hashing (these are GOOD). +- Crypto libraries used correctly with sufficient key lengths and secure modes. +- Framework secure defaults (e.g., Django PBKDF2, bcrypt, argon2). + +OUTPUT: +- Return JSON matching HuntResult. +- findings must contain RawFinding entries with evidence-backed descriptions and correct CWE mapping. +- hunter_strategy should be "crypto". + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: JAVASCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec, new Function + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, parameterized pg queries + DO NOT FLAG: React JSX expressions (auto-escaped), Angular template bindings (sanitized by default) + Framework notes: Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous. + +Language: GO + Key sinks: fmt.Sprintf into SQL, exec.Command with user input, template.HTML(), os.Exec + Safe patterns (skip these): database/sql with ? placeholders, html/template (auto-escapes), exec.Command with separate args + DO NOT FLAG: GORM parameterized queries, html/template default escaping, database/sql prepared statements + Framework notes: Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML. + +Language: RUBY + Key sinks: eval, system, exec, send, public_send, ERB.new with user input + Safe patterns (skip these): ActiveRecord parameterized queries, Rack::Utils.escape_html, sanitize helper in Rails + DO NOT FLAG: ActiveRecord where with hash conditions, Rails CSRF protection, Rails html_safe on constants + Framework notes: Rails auto-escapes ERB templates. raw/html_safe bypasses escaping. Check for mass assignment. + +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +Framework: NEXTJS + Security features: React JSX auto-escapes output by default, API routes can share centralized auth middleware, Server/client boundaries reduce accidental secret exposure + DO NOT FLAG: React JSX escaped rendering, getServerSideProps/getServerSession with proper auth checks, Typed route handlers with validated schema guards + Watch for: dangerouslySetInnerHTML, unprotected API routes under /api, secret leakage to client bundles, rewrites/redirects from untrusted user input + Common vulns: XSS through dangerous HTML rendering, IDOR/auth bypass in API handlers, Sensitive env/config exposure in client-side code + +Framework: SPRING + Security features: Spring Security provides auth, CSRF, and filter chain defaults, JPA/Hibernate prepared parameter binding by default, Bean validation can enforce request constraints + DO NOT FLAG: PreparedStatement/JPA named parameter usage, Spring Security CSRF/auth filters clearly active, Thymeleaf auto-escaped output + Watch for: @PreAuthorize missing on sensitive methods, JdbcTemplate/raw Statement with concatenation, csrf().disable() on browser session flows, Actuator endpoints exposed without auth + Common vulns: Authz bypass from weak method-level security, SQL injection in raw JDBC queries, Sensitive management endpoint exposure + +Framework: REACT + Security features: JSX escapes strings before DOM rendering, Component model encourages explicit data flow, Framework discourages direct DOM mutation + DO NOT FLAG: Standard JSX expression rendering, textContent assignment for untrusted text, sanitized HTML via trusted DOMPurify policy + Watch for: dangerouslySetInnerHTML, untrusted URL assignment to href/src, eval/new Function in client code, token/secret exposure in bundles + Common vulns: DOM XSS through unsafe HTML injection, Open redirect via unvalidated navigation targets, Sensitive data exposure in frontend artifacts + +Framework: VUE + Security features: Mustache template interpolation escapes HTML, Component props/events provide explicit boundaries, Router guards can enforce auth flows + DO NOT FLAG: escaped template interpolation {{ value }}, validated route guards protecting private routes, sanitized content rendered through safe components + Watch for: v-html with untrusted data, dynamic component/template compilation, unsafe URL bindings in href/src, client-side auth checks without server enforcement + Common vulns: XSS through v-html and unsafe render paths, Auth bypass from client-only route protection, Open redirect patterns in router navigation + +CONSTRAINTS: +- Take multiple turns: first discover crypto-relevant files, then evaluate exploitability context. +- Prefer concrete evidence (file path, line range, code snippet, why risky in context). +- No markdown/code fences; JSON output only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Hunt strategy: crypto +- Early stop rule: if you inspect 30 files without credible crypto misuse, stop and return empty findings. +- Focus CWEs: CWE-326, CWE-327, CWE-328, CWE-330, CWE-916, CWE-259, CWE-321, CWE-798 +- Security-critical usage candidates: at-rest encryption, password hashing, token signing, webhook signature, password hashing, JWT signing, key derivation, package signing, password hashing +- Non-security usage candidates: checksum +- Prioritize weak crypto findings only when used in security-sensitive contexts; avoid checksum/cache-only noise. +- Take multiple turns to explore relevant files before finalizing findings. +- Write final JSON only when analysis is complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/direct_prompt_data_exposure.txt b/internal/secaf/agents/hunt/testdata/golden/direct_prompt_data_exposure.txt new file mode 100644 index 0000000000..e35874ce23 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/direct_prompt_data_exposure.txt @@ -0,0 +1,174 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are a data exposure security analyst for SEC-AF HUNT Phase. + +CONTEXT: +You are analyzing a real repository for Data Exposure Hunter. +Use the RECON context summary below to focus your investigation. + +RECON_CONTEXT: +Data exposure-focused recon summary. + +Data flows touching likely sensitive domains: 8 total, showing top 8: +- request.cookies['session'] -> redis.set; sanitized=True; files=app/auth/session_store.py +- form['password'] -> logger.info; sanitized=False; files=app/auth/service.py, app/obs/telemetry.go +- header['Authorization'] -> jwt.decode; sanitized=True; files=app/common/jwt_tools.py +- query['email'] -> smtp.send; sanitized=False; files=app/notify/mailer.rb +- body['card_number'] -> stripe.Charge.create; sanitized=True; files=app/billing/payments.go, app/billing/core.py +- session['role'] -> permission_check; sanitized=True; files=app/auth/permissions.py, app/auth/rbac.py +- token -> cache.set; sanitized=True; files=app/common/cache.py +- request.headers['X-User-Phone'] -> audit_log; sanitized=False; files=app/obs/audit.py + +Logging/exposure-related misconfig signals: 7 total, showing top 7: +- logging at config/app_0.ini:0; risk=high; key=LOG_LEVEL +- tls at config/app_1.ini:13; risk=critical; key=SSL_VERIFY +- debug at config/app_4.ini:16; risk=critical; key=DJANGO_DEBUG +- http at config/app_7.ini:19; risk=high; key=REDIRECT_HTTPS +- trace at config/app_9.ini:21; risk=low; key=OTEL_TRACE_ALL +- errors at config/app_11.ini:23; risk=high; key=SHOW_STACKTRACE +- exposure at config/app_15.ini:0; risk=medium; key=ADMIN_PATH + +Entry points and API surface with exposure risk: 20 total, showing top 10: +- GET /api/v1/thing/0 (app/api/thing_0.py:100, auth_required=True) +- POST /api/v1/thing/1 (app/api/thing_1.py:103, auth_required=False) +- PUT /api/v1/thing/2 (app/api/thing_2.py:106, auth_required=None) +- DELETE /api/v1/thing/3 (app/api/thing_3.py:109, auth_required=False) +- PATCH /api/v1/thing/4 (app/api/thing_4.py:112, auth_required=True) +- GET /api/v1/thing/5 (app/api/thing_5.py:115, auth_required=None) +- POST /api/v1/thing/6 (app/api/thing_6.py:118, auth_required=True) +- PUT /api/v1/thing/7 (app/api/thing_7.py:121, auth_required=False) +- DELETE /api/v1/thing/8 (app/api/thing_8.py:124, auth_required=None) +- PATCH /api/v1/thing/9 (app/api/thing_9.py:127, auth_required=False) + +TASK: +Find potential data exposure vulnerabilities and return HuntResult JSON. + +WORKFLOW (follow these steps in order): +1. Read the source files referenced in the RECON context above to understand the codebase. +2. Search for logging calls, error handlers, data persistence, HTTP clients, and configuration settings in the source code. +3. Read the identified files to confirm vulnerabilities with actual code evidence. +4. For each confirmed vulnerability, record the file path, line number, and relevant code snippet. +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +COVERAGE: +- CWE-200: information exposure (debug endpoints, verbose responses, server/internal metadata leakage) +- CWE-209: detailed error message leakage (stack traces, SQL errors, file paths, internal host/IP details) +- CWE-532: sensitive data in logs (PII, credentials, tokens, secrets, auth headers, session IDs) +- CWE-312: cleartext storage of sensitive data (passwords, tokens, PII persisted without encryption) +- CWE-319: cleartext transmission of sensitive data (HTTP transport for auth/session/PII or disabled TLS verification) + +FOCUS AREAS: +- Logging calls, logger middleware, and request/response logging filters +- Error handlers, exception formatters, debug/traceback output, API error payloads +- Persistence layer, model fields, data-at-rest handling, local file writes/caches/backups +- HTTP clients, webhook integrations, transport settings, TLS/SSL flags +- Configuration and deployment toggles that affect production exposure + +REQUIRED OUTPUT: +- Return JSON that strictly matches HuntResult. +- Populate findings with concrete evidence; chains may be empty. +- For each finding, fill all required RawFinding fields including: + - hunter_strategy = "data_exposure" + - cwe_id and cwe_name aligned to this strategy + - file_path, start_line, end_line, code_snippet + - estimated_severity and confidence + - fingerprint stable for file_path + start_line + cwe_id +- Distinguish production impact vs development-only behavior in description and confidence. +- Empty arrays are acceptable if no vulnerabilities are confirmed. + +DO NOT FLAG: +- Logging of non-sensitive operational data (request IDs, timestamps, non-sensitive metrics) +- Error verbosity that is clearly limited to development/local debug mode +- HTTPS URLs or TLS-enabled transmissions + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: JAVASCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec, new Function + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, parameterized pg queries + DO NOT FLAG: React JSX expressions (auto-escaped), Angular template bindings (sanitized by default) + Framework notes: Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous. + +Language: GO + Key sinks: fmt.Sprintf into SQL, exec.Command with user input, template.HTML(), os.Exec + Safe patterns (skip these): database/sql with ? placeholders, html/template (auto-escapes), exec.Command with separate args + DO NOT FLAG: GORM parameterized queries, html/template default escaping, database/sql prepared statements + Framework notes: Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML. + +Language: RUBY + Key sinks: eval, system, exec, send, public_send, ERB.new with user input + Safe patterns (skip these): ActiveRecord parameterized queries, Rack::Utils.escape_html, sanitize helper in Rails + DO NOT FLAG: ActiveRecord where with hash conditions, Rails CSRF protection, Rails html_safe on constants + Framework notes: Rails auto-escapes ERB templates. raw/html_safe bypasses escaping. Check for mass assignment. + +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +Framework: NEXTJS + Security features: React JSX auto-escapes output by default, API routes can share centralized auth middleware, Server/client boundaries reduce accidental secret exposure + DO NOT FLAG: React JSX escaped rendering, getServerSideProps/getServerSession with proper auth checks, Typed route handlers with validated schema guards + Watch for: dangerouslySetInnerHTML, unprotected API routes under /api, secret leakage to client bundles, rewrites/redirects from untrusted user input + Common vulns: XSS through dangerous HTML rendering, IDOR/auth bypass in API handlers, Sensitive env/config exposure in client-side code + +Framework: SPRING + Security features: Spring Security provides auth, CSRF, and filter chain defaults, JPA/Hibernate prepared parameter binding by default, Bean validation can enforce request constraints + DO NOT FLAG: PreparedStatement/JPA named parameter usage, Spring Security CSRF/auth filters clearly active, Thymeleaf auto-escaped output + Watch for: @PreAuthorize missing on sensitive methods, JdbcTemplate/raw Statement with concatenation, csrf().disable() on browser session flows, Actuator endpoints exposed without auth + Common vulns: Authz bypass from weak method-level security, SQL injection in raw JDBC queries, Sensitive management endpoint exposure + +Framework: REACT + Security features: JSX escapes strings before DOM rendering, Component model encourages explicit data flow, Framework discourages direct DOM mutation + DO NOT FLAG: Standard JSX expression rendering, textContent assignment for untrusted text, sanitized HTML via trusted DOMPurify policy + Watch for: dangerouslySetInnerHTML, untrusted URL assignment to href/src, eval/new Function in client code, token/secret exposure in bundles + Common vulns: DOM XSS through unsafe HTML injection, Open redirect via unvalidated navigation targets, Sensitive data exposure in frontend artifacts + +Framework: VUE + Security features: Mustache template interpolation escapes HTML, Component props/events provide explicit boundaries, Router guards can enforce auth flows + DO NOT FLAG: escaped template interpolation {{ value }}, validated route guards protecting private routes, sanitized content rendered through safe components + Watch for: v-html with untrusted data, dynamic component/template compilation, unsafe URL bindings in href/src, client-side auth checks without server enforcement + Common vulns: XSS through v-html and unsafe render paths, Auth bypass from client-only route protection, Open redirect patterns in router navigation + +CONSTRAINTS: +- Prefer evidence over assumptions; if uncertain, lower confidence. +- No markdown or code fences; JSON output only. +- Base all findings on actual file content you read. Do not speculate. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Strategy: data_exposure +- Early stop rule: if you inspect 30 files without credible exposure risk, stop and return empty findings. +- Use multiple turns: inspect files first, then produce findings. +- Return final JSON only when analysis is complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/direct_prompt_sha256.json b/internal/secaf/agents/hunt/testdata/golden/direct_prompt_sha256.json new file mode 100644 index 0000000000..57c5e02649 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/direct_prompt_sha256.json @@ -0,0 +1,66 @@ +{ + "scan": { + "injection": "cf6d0f6d8415e5067bb8ff4d84beceef1f9013e3ac3e879d29d949d5668656eb", + "xss": "5a0a69fbc2252506bfa1fd6b51dfced7182463f3e1bcad8be9b95a106088b82e", + "dos": "43a7d16d16a4c28f75ef8ee35d31e9d830edef8c2882d1d69341fec29ba2dd2e", + "ssrf": "565502128e548af4ac41afbbc3b3229c6a02f777d01c6b65fd9ce6b5d565a91c", + "auth": "03d72e8334604df7be4aa1b441848f8ebe79067b5753811304a4bfcf5e77dd37", + "crypto": "c012d99195f420df2d380335fdb14ddf43e49600a0510d0e299155520a6fe2d9", + "business_logic": "001712d5e7602460e52df7c48512185d08375b5476462d41da55a60eda43ece8", + "logic": "001712d5e7602460e52df7c48512185d08375b5476462d41da55a60eda43ece8", + "data_exposure": "4a16b73daa97aa7d3b0448e97453da6231d9a0d81b3a08aa02622c05476ba6e5", + "supply_chain": "d7497cb488eb0c9edd915da49c4dcb9b8b69cf6faa54fe20b5bbd3589591ed9f", + "config_secrets": "104a6110e168058d19a47c04406f17ecdd5dabe3f685e03583672dc4ffeac2bd", + "api_security": "4170958685e8225165e479f95e92ada12e3851fa04d04b31d9d36e51b9459e6e" + }, + "enrich": { + "injection": [ + "760b133668a3a13d6a73b0309bb923afc721c7dc69f88100a12b7afcbe42c8bd", + "c7a31ca70d9eec5b43bc9c0f9e3121e45a5992ff54cea6bab5cff81b84e8d8e7" + ], + "xss": [ + "efcec35e1150526419a36b39a4526137291b850b4da50e2450d63640e8165839", + "6e14997215bf24ee8adff3917d2a6eaec760b6f17b3a46373970e77718b4b59c" + ], + "dos": [ + "f398b344e907bda92b5738e90d47b329c9e9e6d4150d9c66b31cf72e055ba76e", + "b7cf07efc7a02f23e0dba97955a5835b1e59d2fb15b27bebdd3dd7d19fe5dccd" + ], + "ssrf": [ + "47657a189efca7944a414b5102e4956710e1c6e2e48fd09195dfee77c1689010", + "b8349ba85ea6c34d7909e8cc9b8583a7118171e03a1f5d880bd799f957416a5e" + ], + "auth": [ + "48b130cb17467b5c680f8e0eb15bbcd766b1676ad8d5f0407caad11a4bd016c6", + "9b6d972be51353fba2cb51e18f783145a720ae64eefff1cb5409755f8717ba01" + ], + "crypto": [ + "108ce0d993396356401ee622a0e7bb343f225201f33ed4fcb90f0d77b7f77588", + "ac4da4c1863fa9d7b7fad321e131a8279850c744c8b4d74031cfce08f0405ebf" + ], + "business_logic": [ + "7a027cf03d9d5cd8ef93225791970061a36f6e66998a663004049be0285ca639", + "e1bd25b5ecb309de78774aa5764ea3daa1059215a214bd948aebc6fdcb27646f" + ], + "logic": [ + "7a027cf03d9d5cd8ef93225791970061a36f6e66998a663004049be0285ca639", + "e1bd25b5ecb309de78774aa5764ea3daa1059215a214bd948aebc6fdcb27646f" + ], + "data_exposure": [ + "696eef5f90cd79632826c2c082bbb53fee6dcd295cc2fdb909911182dd5329ba", + "3c519a45da85098689781f78f225effb8504c2c1d6391f1e26de31c3964a3d48" + ], + "supply_chain": [ + "3c93052ed54c8d765a528dec27067ae43a6718e4c611408e857d6cdd0c914920", + "989cd6965263ea82ba5bf8599856cb8e381348299d8caf0546b6100e79ddafac" + ], + "config_secrets": [ + "22da7a4fd1d5cf3d8e29c7e25a2dec9955c3ccd8be30c5696876126582a9abca", + "4b930ca980b38041664d226f1e9398c366267f5631d7c2959298938440a58f5a" + ], + "api_security": [ + "0d6f7a125e6dad5680e9587ddc690e557a6ac7ca1d30ecc3dcc9c71fb550f126", + "f49ff086a9ded47ddf6409f92156f08d66b3d46a57b03d30d6ac508e384d48f0" + ] + } +} diff --git a/internal/secaf/agents/hunt/testdata/golden/direct_prompt_supply_chain.txt b/internal/secaf/agents/hunt/testdata/golden/direct_prompt_supply_chain.txt new file mode 100644 index 0000000000..9e7b5ffb93 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/direct_prompt_supply_chain.txt @@ -0,0 +1,169 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are SEC-AF's Supply Chain Hunter specializing in dependency risk and package ecosystem abuse. + +CONTEXT: +You are in HUNT Phase and must output a HuntResult JSON object for supply-chain findings. +Use the RECON context summary below as primary evidence. + +RECON_CONTEXT: +Supply-chain-focused recon summary. + +Dependency inventory: direct=7, transitive=143, SBOM entries=14. + +Known CVE exposure (reachable/high severity first): 18 total, showing top 15: +- CVE-2023-0014 in log4j 2.14.0 (fixed=2.17.1, cvss=10.0, epss=0.97, direct=False, reachable=True) +- CVE-2023-0003 in lodash 4.17.19 (fixed=4.17.21, cvss=9.8, epss=0.9, direct=True, reachable=True) +- CVE-2023-0001 in django 4.2.1 (fixed=4.2.5, cvss=9.8, epss=0.5, direct=True, reachable=True) +- CVE-2023-0013 in pyyaml 5.3 (fixed=5.4, cvss=9.8, epss=0.5, direct=False, reachable=True) +- CVE-2023-0008 in nokogiri 1.13.6 (fixed=1.13.10, cvss=8.8, epss=0.3, direct=False, reachable=True) +- CVE-2023-0018 in curl 7.68.0 (fixed=7.88.0, cvss=8.1, epss=None, direct=True, reachable=True) +- CVE-2023-0009 in golang.org/x/net 0.7.0 (fixed=0.17.0, cvss=7.5, epss=0.44, direct=False, reachable=True) +- CVE-2023-0012 in requests 2.31.0 (fixed=2.32.0, cvss=6.5, epss=0.07, direct=True, reachable=True) +- CVE-2023-0005 in axios 0.21.1 (fixed=0.21.2, cvss=5.3, epss=0.02, direct=False, reachable=True) +- CVE-2023-0017 in libxml2 2.9.10 (fixed=unknown, cvss=None, epss=0.25, direct=False, reachable=True) +- CVE-2023-0010 in rails 7.0.4 (fixed=7.0.7, cvss=9.1, epss=0.6, direct=True, reachable=False) +- CVE-2023-0016 in zlib 1.2.11 (fixed=1.2.12, cvss=8.2, epss=0.04, direct=False, reachable=False) +- CVE-2023-0002 in urllib3 1.26.5 (fixed=unknown, cvss=7.5, epss=0.1, direct=False, reachable=False) +- CVE-2023-0015 in openssl 1.1.1k (fixed=1.1.1t, cvss=7.4, epss=0.12, direct=False, reachable=None) +- CVE-2023-0006 in jinja2 3.1.2 (fixed=unknown, cvss=6.1, epss=None, direct=False, reachable=False) + +Outdated dependencies: 12 total, showing top 10: +- pkg-0: 1.0.0 -> 2.0.0 (direct=True) +- pkg-1: 1.1.0 -> 2.1.0 (direct=False) +- pkg-2: 1.2.0 -> 2.2.0 (direct=True) +- pkg-3: 1.3.0 -> 2.3.0 (direct=False) +- pkg-4: 1.4.0 -> 2.4.0 (direct=True) +- pkg-5: 1.5.0 -> 2.5.0 (direct=False) +- pkg-6: 1.6.0 -> 2.6.0 (direct=True) +- pkg-7: 1.7.0 -> 2.7.0 (direct=False) +- pkg-8: 1.8.0 -> 2.8.0 (direct=True) +- pkg-9: 1.9.0 -> 2.9.0 (direct=False) + +Primary dependency ecosystems in this repo: 12 total, showing top 10: +- go: github.com/gin-gonic/gin@1.9.0 +- go: golang.org/x/net@0.7.0 +- npm: axios@0.21.1 +- npm: express@4.17.1 +- npm: lodash@4.17.19 +- npm: minimist@1.2.5 +- pypi: django@4.2.1 +- pypi: jinja2@3.1.2 +- pypi: requests@2.31.0 +- pypi: urllib3@1.26.5 + +TASK: +Identify supply-chain vulnerabilities with emphasis on CWE-1104 (unmaintained third-party components) +and CWE-829 (inclusion of functionality from untrusted control sphere). + +SCOPE: +- Analyze dependency manifests and lockfiles across ecosystems (npm/yarn/pnpm, Python, Go, Rust, etc.). +- Use RECON dependency context (SBOM, known CVEs, outdated packages) as primary evidence. +- Detect known vulnerable versions and include CVE IDs when available. +- Flag unmaintained/abandoned packages and stale dependencies with security impact. +- Flag typosquatting and look-alike package names likely to be malicious. +- Flag untrusted package sources, private-registry confusion, and dependency confusion patterns. +- Flag risky version pinning posture: missing lockfile, wildcard ranges, loose semver constraints. +- Flag lockfile drift/integrity issues where manifest and lock data are inconsistent. + +OUTPUT: +- Return JSON that strictly matches HuntResult. +- Use finding_type "sca" for dependency findings. +- Every finding must include concrete package evidence (package name, version, file path, and rationale). +- Include CWE mapping: CWE-1104 or CWE-829 for each finding. +- Include CVE identifiers in description/code_snippet when available from recon evidence. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: JAVASCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec, new Function + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, parameterized pg queries + DO NOT FLAG: React JSX expressions (auto-escaped), Angular template bindings (sanitized by default) + Framework notes: Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous. + +Language: GO + Key sinks: fmt.Sprintf into SQL, exec.Command with user input, template.HTML(), os.Exec + Safe patterns (skip these): database/sql with ? placeholders, html/template (auto-escapes), exec.Command with separate args + DO NOT FLAG: GORM parameterized queries, html/template default escaping, database/sql prepared statements + Framework notes: Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML. + +Language: RUBY + Key sinks: eval, system, exec, send, public_send, ERB.new with user input + Safe patterns (skip these): ActiveRecord parameterized queries, Rack::Utils.escape_html, sanitize helper in Rails + DO NOT FLAG: ActiveRecord where with hash conditions, Rails CSRF protection, Rails html_safe on constants + Framework notes: Rails auto-escapes ERB templates. raw/html_safe bypasses escaping. Check for mass assignment. + +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +Framework: NEXTJS + Security features: React JSX auto-escapes output by default, API routes can share centralized auth middleware, Server/client boundaries reduce accidental secret exposure + DO NOT FLAG: React JSX escaped rendering, getServerSideProps/getServerSession with proper auth checks, Typed route handlers with validated schema guards + Watch for: dangerouslySetInnerHTML, unprotected API routes under /api, secret leakage to client bundles, rewrites/redirects from untrusted user input + Common vulns: XSS through dangerous HTML rendering, IDOR/auth bypass in API handlers, Sensitive env/config exposure in client-side code + +Framework: SPRING + Security features: Spring Security provides auth, CSRF, and filter chain defaults, JPA/Hibernate prepared parameter binding by default, Bean validation can enforce request constraints + DO NOT FLAG: PreparedStatement/JPA named parameter usage, Spring Security CSRF/auth filters clearly active, Thymeleaf auto-escaped output + Watch for: @PreAuthorize missing on sensitive methods, JdbcTemplate/raw Statement with concatenation, csrf().disable() on browser session flows, Actuator endpoints exposed without auth + Common vulns: Authz bypass from weak method-level security, SQL injection in raw JDBC queries, Sensitive management endpoint exposure + +Framework: REACT + Security features: JSX escapes strings before DOM rendering, Component model encourages explicit data flow, Framework discourages direct DOM mutation + DO NOT FLAG: Standard JSX expression rendering, textContent assignment for untrusted text, sanitized HTML via trusted DOMPurify policy + Watch for: dangerouslySetInnerHTML, untrusted URL assignment to href/src, eval/new Function in client code, token/secret exposure in bundles + Common vulns: DOM XSS through unsafe HTML injection, Open redirect via unvalidated navigation targets, Sensitive data exposure in frontend artifacts + +Framework: VUE + Security features: Mustache template interpolation escapes HTML, Component props/events provide explicit boundaries, Router guards can enforce auth flows + DO NOT FLAG: escaped template interpolation {{ value }}, validated route guards protecting private routes, sanitized content rendered through safe components + Watch for: v-html with untrusted data, dynamic component/template compilation, unsafe URL bindings in href/src, client-side auth checks without server enforcement + Common vulns: XSS through v-html and unsafe render paths, Auth bypass from client-only route protection, Open redirect patterns in router navigation + +CONSTRAINTS: +- Take multiple turns: inspect relevant files first, then build findings incrementally. +- Do not invent package versions, CVEs, or exploitability claims. +- Prefer high-confidence findings with explicit evidence over speculative findings. +- No markdown/code fences; JSON output only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Hunt strategy: supply_chain (CWE-1104, CWE-829). +- Early stop rule: if you inspect 30 manifests/files without credible dependency risk, stop and return empty findings. +- Focus manifests/lockfiles (package.json, requirements.txt, go.mod, Pipfile, poetry.lock, package-lock.json, yarn.lock, pnpm-lock.yaml, Cargo.toml). +- Take multiple turns: inspect manifests/lockfiles, validate dependency risks, then produce final structured findings. +- Write final JSON only when analysis is complete. + diff --git a/internal/secaf/agents/hunt/testdata/golden/enrich_location_input.json b/internal/secaf/agents/hunt/testdata/golden/enrich_location_input.json new file mode 100644 index 0000000000..5700fdc174 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/enrich_location_input.json @@ -0,0 +1,11 @@ +{ + "location": { + "file_path": "app/api/users.py", + "start_line": 42, + "code_snippet": "query = f\"SELECT * FROM users WHERE id = {user_id}\"\ncursor.execute(query)", + "pattern_type": "sql_injection" + }, + "finding_type": "sast", + "strategy": "injection", + "recon_context": "RECON CONTEXT LINE 1\nRECON CONTEXT LINE 2" +} diff --git a/internal/secaf/agents/hunt/testdata/golden/enrich_location_prompt.txt b/internal/secaf/agents/hunt/testdata/golden/enrich_location_prompt.txt new file mode 100644 index 0000000000..d6c634b047 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/enrich_location_prompt.txt @@ -0,0 +1,42 @@ +ROLE: +You are Step 2 FindingEnricher for SEC-AF HUNT phase. + +TASK: +Enrich one scanned vulnerability location into one complete finding. + +CONTEXT: +- Finding type: sast +- Hunter strategy: injection + +RECON CONTEXT: +RECON CONTEXT LINE 1 +RECON CONTEXT LINE 2 + +LOCATION: +- File path: app/api/users.py +- Start line: 42 +- Pattern type: sql_injection +- Code snippet: +query = f"SELECT * FROM users WHERE id = {user_id}" +cursor.execute(query) + +WORKFLOW (follow these steps in order): +1. Read the file at the file path above, focusing on the code around the start line. +2. Analyze the code for the specific vulnerability pattern indicated by the finding type and strategy. +3. Determine the CWE ID, severity, and confidence based on the code evidence. +4. Write the final JSON output file with exactly these 6 fields. + +OUTPUT: +- Return a SINGLE JSON object (not an array, not wrapped in another object). +- The JSON object must have exactly these 6 string fields: + - title: A short descriptive title for this finding + - description: Detailed explanation of the vulnerability and its impact + - cwe_id: The CWE identifier (e.g. "CWE-89") + - severity: One of: critical, high, medium, low, info + - confidence: One of: high, medium, low + - data_flow_summary: A plain string describing the data flow (source to sink), or empty string if not applicable + +CONSTRAINTS: +- Ground every statement in the provided code location and recon context. +- Do NOT wrap the output in {"findings": [...]} or any other wrapper — return a flat object. +- Do not include markdown or prose outside JSON. diff --git a/internal/secaf/agents/hunt/testdata/golden/hunter_empty_locations.json b/internal/secaf/agents/hunt/testdata/golden/hunter_empty_locations.json new file mode 100644 index 0000000000..61a016d962 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/hunter_empty_locations.json @@ -0,0 +1,170 @@ +{ + "injection": { + "scan_calls": 1, + "enrich_calls": 0, + "want": { + "findings": [], + "chains": [], + "total_raw": 0, + "deduplicated_count": 0, + "chain_count": 0, + "strategies_run": [], + "hunt_duration_seconds": 0.0 + } + }, + "xss": { + "scan_calls": 1, + "enrich_calls": 0, + "want": { + "findings": [], + "chains": [], + "total_raw": 0, + "deduplicated_count": 0, + "chain_count": 0, + "strategies_run": [], + "hunt_duration_seconds": 0.0 + } + }, + "dos": { + "scan_calls": 1, + "enrich_calls": 0, + "want": { + "findings": [], + "chains": [], + "total_raw": 0, + "deduplicated_count": 0, + "chain_count": 0, + "strategies_run": [], + "hunt_duration_seconds": 0.0 + } + }, + "ssrf": { + "scan_calls": 1, + "enrich_calls": 0, + "want": { + "findings": [], + "chains": [], + "total_raw": 0, + "deduplicated_count": 0, + "chain_count": 0, + "strategies_run": [], + "hunt_duration_seconds": 0.0 + } + }, + "auth": { + "scan_calls": 1, + "enrich_calls": 0, + "want": { + "findings": [], + "chains": [], + "total_raw": 0, + "deduplicated_count": 0, + "chain_count": 0, + "strategies_run": [], + "hunt_duration_seconds": 0.0 + } + }, + "crypto": { + "scan_calls": 1, + "enrich_calls": 0, + "want": { + "findings": [], + "chains": [], + "total_raw": 0, + "deduplicated_count": 0, + "chain_count": 0, + "strategies_run": [ + "crypto" + ], + "hunt_duration_seconds": 0.0 + } + }, + "business_logic": { + "scan_calls": 1, + "enrich_calls": 0, + "want": { + "findings": [], + "chains": [], + "total_raw": 0, + "deduplicated_count": 0, + "chain_count": 0, + "strategies_run": [ + "business_logic" + ], + "hunt_duration_seconds": 0.0 + } + }, + "logic": { + "scan_calls": 1, + "enrich_calls": 0, + "want": { + "findings": [], + "chains": [], + "total_raw": 0, + "deduplicated_count": 0, + "chain_count": 0, + "strategies_run": [ + "business_logic" + ], + "hunt_duration_seconds": 0.0 + } + }, + "data_exposure": { + "scan_calls": 1, + "enrich_calls": 0, + "want": { + "findings": [], + "chains": [], + "total_raw": 0, + "deduplicated_count": 0, + "chain_count": 0, + "strategies_run": [ + "data_exposure" + ], + "hunt_duration_seconds": 0.0 + } + }, + "supply_chain": { + "scan_calls": 1, + "enrich_calls": 0, + "want": { + "findings": [], + "chains": [], + "total_raw": 0, + "deduplicated_count": 0, + "chain_count": 0, + "strategies_run": [ + "supply_chain" + ], + "hunt_duration_seconds": 0.0 + } + }, + "config_secrets": { + "scan_calls": 1, + "enrich_calls": 0, + "want": { + "findings": [], + "chains": [], + "total_raw": 0, + "deduplicated_count": 0, + "chain_count": 0, + "strategies_run": [], + "hunt_duration_seconds": 0.0 + } + }, + "api_security": { + "scan_calls": 1, + "enrich_calls": 0, + "want": { + "findings": [], + "chains": [], + "total_raw": 0, + "deduplicated_count": 0, + "chain_count": 0, + "strategies_run": [ + "api_security" + ], + "hunt_duration_seconds": 0.0 + } + } +} diff --git a/internal/secaf/agents/hunt/testdata/golden/hunter_results.json b/internal/secaf/agents/hunt/testdata/golden/hunter_results.json new file mode 100644 index 0000000000..effccc97bf --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/hunter_results.json @@ -0,0 +1,710 @@ +{ + "injection": { + "findings": [ + { + "id": "", + "hunter_strategy": "injection", + "title": "SQL injection in user lookup", + "description": "user_id flows unescaped into an f-string query.", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "CWE-89", + "owasp_category": null, + "file_path": "app/api/users.py", + "start_line": 42, + "end_line": 43, + "function_name": null, + "code_snippet": "query = f\"SELECT * FROM users WHERE id = {user_id}\"\ncursor.execute(query)", + "estimated_severity": "high", + "confidence": "high", + "data_flow": [ + { + "file_path": "app/api/users.py", + "line": 42, + "component": "injection", + "operation": "request.args['id'] -> query -> cursor.execute" + } + ], + "related_files": [], + "fingerprint": "" + }, + { + "id": "", + "hunter_strategy": "injection", + "title": "Weak hash for password storage", + "description": "MD5 used to derive a credential digest.", + "finding_type": "sast", + "cwe_id": "CWE-327", + "cwe_name": "CWE-327", + "owasp_category": null, + "file_path": "app/utils/hash.py", + "start_line": 7, + "end_line": 7, + "function_name": null, + "code_snippet": "digest = hashlib.md5(password).hexdigest()", + "estimated_severity": "medium", + "confidence": "medium", + "data_flow": null, + "related_files": [], + "fingerprint": "" + } + ], + "chains": [], + "total_raw": 2, + "deduplicated_count": 2, + "chain_count": 0, + "strategies_run": [ + "injection" + ], + "hunt_duration_seconds": 0.0 + }, + "xss": { + "findings": [ + { + "id": "", + "hunter_strategy": "xss", + "title": "SQL injection in user lookup", + "description": "user_id flows unescaped into an f-string query.", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "CWE-89", + "owasp_category": null, + "file_path": "app/api/users.py", + "start_line": 42, + "end_line": 43, + "function_name": null, + "code_snippet": "query = f\"SELECT * FROM users WHERE id = {user_id}\"\ncursor.execute(query)", + "estimated_severity": "high", + "confidence": "high", + "data_flow": [ + { + "file_path": "app/api/users.py", + "line": 42, + "component": "xss", + "operation": "request.args['id'] -> query -> cursor.execute" + } + ], + "related_files": [], + "fingerprint": "" + }, + { + "id": "", + "hunter_strategy": "xss", + "title": "Weak hash for password storage", + "description": "MD5 used to derive a credential digest.", + "finding_type": "sast", + "cwe_id": "CWE-327", + "cwe_name": "CWE-327", + "owasp_category": null, + "file_path": "app/utils/hash.py", + "start_line": 7, + "end_line": 7, + "function_name": null, + "code_snippet": "digest = hashlib.md5(password).hexdigest()", + "estimated_severity": "medium", + "confidence": "medium", + "data_flow": null, + "related_files": [], + "fingerprint": "" + } + ], + "chains": [], + "total_raw": 2, + "deduplicated_count": 2, + "chain_count": 0, + "strategies_run": [ + "xss" + ], + "hunt_duration_seconds": 0.0 + }, + "dos": { + "findings": [ + { + "id": "", + "hunter_strategy": "dos", + "title": "SQL injection in user lookup", + "description": "user_id flows unescaped into an f-string query.", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "CWE-89", + "owasp_category": null, + "file_path": "app/api/users.py", + "start_line": 42, + "end_line": 43, + "function_name": null, + "code_snippet": "query = f\"SELECT * FROM users WHERE id = {user_id}\"\ncursor.execute(query)", + "estimated_severity": "high", + "confidence": "high", + "data_flow": [ + { + "file_path": "app/api/users.py", + "line": 42, + "component": "dos", + "operation": "request.args['id'] -> query -> cursor.execute" + } + ], + "related_files": [], + "fingerprint": "" + }, + { + "id": "", + "hunter_strategy": "dos", + "title": "Weak hash for password storage", + "description": "MD5 used to derive a credential digest.", + "finding_type": "sast", + "cwe_id": "CWE-327", + "cwe_name": "CWE-327", + "owasp_category": null, + "file_path": "app/utils/hash.py", + "start_line": 7, + "end_line": 7, + "function_name": null, + "code_snippet": "digest = hashlib.md5(password).hexdigest()", + "estimated_severity": "medium", + "confidence": "medium", + "data_flow": null, + "related_files": [], + "fingerprint": "" + } + ], + "chains": [], + "total_raw": 2, + "deduplicated_count": 2, + "chain_count": 0, + "strategies_run": [ + "dos" + ], + "hunt_duration_seconds": 0.0 + }, + "ssrf": { + "findings": [ + { + "id": "", + "hunter_strategy": "ssrf", + "title": "SQL injection in user lookup", + "description": "user_id flows unescaped into an f-string query.", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "CWE-89", + "owasp_category": null, + "file_path": "app/api/users.py", + "start_line": 42, + "end_line": 43, + "function_name": null, + "code_snippet": "query = f\"SELECT * FROM users WHERE id = {user_id}\"\ncursor.execute(query)", + "estimated_severity": "high", + "confidence": "high", + "data_flow": [ + { + "file_path": "app/api/users.py", + "line": 42, + "component": "ssrf", + "operation": "request.args['id'] -> query -> cursor.execute" + } + ], + "related_files": [], + "fingerprint": "" + }, + { + "id": "", + "hunter_strategy": "ssrf", + "title": "Weak hash for password storage", + "description": "MD5 used to derive a credential digest.", + "finding_type": "sast", + "cwe_id": "CWE-327", + "cwe_name": "CWE-327", + "owasp_category": null, + "file_path": "app/utils/hash.py", + "start_line": 7, + "end_line": 7, + "function_name": null, + "code_snippet": "digest = hashlib.md5(password).hexdigest()", + "estimated_severity": "medium", + "confidence": "medium", + "data_flow": null, + "related_files": [], + "fingerprint": "" + } + ], + "chains": [], + "total_raw": 2, + "deduplicated_count": 2, + "chain_count": 0, + "strategies_run": [ + "ssrf" + ], + "hunt_duration_seconds": 0.0 + }, + "auth": { + "findings": [ + { + "id": "", + "hunter_strategy": "auth", + "title": "SQL injection in user lookup", + "description": "user_id flows unescaped into an f-string query.", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "CWE-89", + "owasp_category": null, + "file_path": "app/api/users.py", + "start_line": 42, + "end_line": 43, + "function_name": null, + "code_snippet": "query = f\"SELECT * FROM users WHERE id = {user_id}\"\ncursor.execute(query)", + "estimated_severity": "high", + "confidence": "high", + "data_flow": [ + { + "file_path": "app/api/users.py", + "line": 42, + "component": "auth", + "operation": "request.args['id'] -> query -> cursor.execute" + } + ], + "related_files": [], + "fingerprint": "" + }, + { + "id": "", + "hunter_strategy": "auth", + "title": "Weak hash for password storage", + "description": "MD5 used to derive a credential digest.", + "finding_type": "sast", + "cwe_id": "CWE-327", + "cwe_name": "CWE-327", + "owasp_category": null, + "file_path": "app/utils/hash.py", + "start_line": 7, + "end_line": 7, + "function_name": null, + "code_snippet": "digest = hashlib.md5(password).hexdigest()", + "estimated_severity": "medium", + "confidence": "medium", + "data_flow": null, + "related_files": [], + "fingerprint": "" + } + ], + "chains": [], + "total_raw": 2, + "deduplicated_count": 2, + "chain_count": 0, + "strategies_run": [ + "auth" + ], + "hunt_duration_seconds": 0.0 + }, + "crypto": { + "findings": [ + { + "id": "", + "hunter_strategy": "crypto", + "title": "SQL injection in user lookup", + "description": "user_id flows unescaped into an f-string query.", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "CWE-89", + "owasp_category": null, + "file_path": "app/api/users.py", + "start_line": 42, + "end_line": 43, + "function_name": null, + "code_snippet": "query = f\"SELECT * FROM users WHERE id = {user_id}\"\ncursor.execute(query)", + "estimated_severity": "high", + "confidence": "high", + "data_flow": [ + { + "file_path": "app/api/users.py", + "line": 42, + "component": "crypto", + "operation": "request.args['id'] -> query -> cursor.execute" + } + ], + "related_files": [], + "fingerprint": "" + }, + { + "id": "", + "hunter_strategy": "crypto", + "title": "Weak hash for password storage", + "description": "MD5 used to derive a credential digest.", + "finding_type": "sast", + "cwe_id": "CWE-327", + "cwe_name": "CWE-327", + "owasp_category": null, + "file_path": "app/utils/hash.py", + "start_line": 7, + "end_line": 7, + "function_name": null, + "code_snippet": "digest = hashlib.md5(password).hexdigest()", + "estimated_severity": "medium", + "confidence": "medium", + "data_flow": null, + "related_files": [], + "fingerprint": "" + } + ], + "chains": [], + "total_raw": 2, + "deduplicated_count": 2, + "chain_count": 0, + "strategies_run": [ + "crypto" + ], + "hunt_duration_seconds": 0.0 + }, + "business_logic": { + "findings": [ + { + "id": "", + "hunter_strategy": "business_logic", + "title": "SQL injection in user lookup", + "description": "user_id flows unescaped into an f-string query.", + "finding_type": "logic", + "cwe_id": "CWE-89", + "cwe_name": "CWE-89", + "owasp_category": null, + "file_path": "app/api/users.py", + "start_line": 42, + "end_line": 43, + "function_name": null, + "code_snippet": "query = f\"SELECT * FROM users WHERE id = {user_id}\"\ncursor.execute(query)", + "estimated_severity": "high", + "confidence": "high", + "data_flow": [ + { + "file_path": "app/api/users.py", + "line": 42, + "component": "business_logic", + "operation": "request.args['id'] -> query -> cursor.execute" + } + ], + "related_files": [], + "fingerprint": "" + }, + { + "id": "", + "hunter_strategy": "business_logic", + "title": "Weak hash for password storage", + "description": "MD5 used to derive a credential digest.", + "finding_type": "logic", + "cwe_id": "CWE-327", + "cwe_name": "CWE-327", + "owasp_category": null, + "file_path": "app/utils/hash.py", + "start_line": 7, + "end_line": 7, + "function_name": null, + "code_snippet": "digest = hashlib.md5(password).hexdigest()", + "estimated_severity": "medium", + "confidence": "medium", + "data_flow": null, + "related_files": [], + "fingerprint": "" + } + ], + "chains": [], + "total_raw": 2, + "deduplicated_count": 2, + "chain_count": 0, + "strategies_run": [ + "business_logic" + ], + "hunt_duration_seconds": 0.0 + }, + "logic": { + "findings": [ + { + "id": "", + "hunter_strategy": "business_logic", + "title": "SQL injection in user lookup", + "description": "user_id flows unescaped into an f-string query.", + "finding_type": "logic", + "cwe_id": "CWE-89", + "cwe_name": "CWE-89", + "owasp_category": null, + "file_path": "app/api/users.py", + "start_line": 42, + "end_line": 43, + "function_name": null, + "code_snippet": "query = f\"SELECT * FROM users WHERE id = {user_id}\"\ncursor.execute(query)", + "estimated_severity": "high", + "confidence": "high", + "data_flow": [ + { + "file_path": "app/api/users.py", + "line": 42, + "component": "business_logic", + "operation": "request.args['id'] -> query -> cursor.execute" + } + ], + "related_files": [], + "fingerprint": "" + }, + { + "id": "", + "hunter_strategy": "business_logic", + "title": "Weak hash for password storage", + "description": "MD5 used to derive a credential digest.", + "finding_type": "logic", + "cwe_id": "CWE-327", + "cwe_name": "CWE-327", + "owasp_category": null, + "file_path": "app/utils/hash.py", + "start_line": 7, + "end_line": 7, + "function_name": null, + "code_snippet": "digest = hashlib.md5(password).hexdigest()", + "estimated_severity": "medium", + "confidence": "medium", + "data_flow": null, + "related_files": [], + "fingerprint": "" + } + ], + "chains": [], + "total_raw": 2, + "deduplicated_count": 2, + "chain_count": 0, + "strategies_run": [ + "business_logic" + ], + "hunt_duration_seconds": 0.0 + }, + "data_exposure": { + "findings": [ + { + "id": "", + "hunter_strategy": "data_exposure", + "title": "SQL injection in user lookup", + "description": "user_id flows unescaped into an f-string query.", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "CWE-89", + "owasp_category": null, + "file_path": "app/api/users.py", + "start_line": 42, + "end_line": 43, + "function_name": null, + "code_snippet": "query = f\"SELECT * FROM users WHERE id = {user_id}\"\ncursor.execute(query)", + "estimated_severity": "high", + "confidence": "high", + "data_flow": [ + { + "file_path": "app/api/users.py", + "line": 42, + "component": "data_exposure", + "operation": "request.args['id'] -> query -> cursor.execute" + } + ], + "related_files": [], + "fingerprint": "" + }, + { + "id": "", + "hunter_strategy": "data_exposure", + "title": "Weak hash for password storage", + "description": "MD5 used to derive a credential digest.", + "finding_type": "sast", + "cwe_id": "CWE-327", + "cwe_name": "CWE-327", + "owasp_category": null, + "file_path": "app/utils/hash.py", + "start_line": 7, + "end_line": 7, + "function_name": null, + "code_snippet": "digest = hashlib.md5(password).hexdigest()", + "estimated_severity": "medium", + "confidence": "medium", + "data_flow": null, + "related_files": [], + "fingerprint": "" + } + ], + "chains": [], + "total_raw": 2, + "deduplicated_count": 2, + "chain_count": 0, + "strategies_run": [ + "data_exposure" + ], + "hunt_duration_seconds": 0.0 + }, + "supply_chain": { + "findings": [ + { + "id": "", + "hunter_strategy": "supply_chain", + "title": "SQL injection in user lookup", + "description": "user_id flows unescaped into an f-string query.", + "finding_type": "sca", + "cwe_id": "CWE-89", + "cwe_name": "CWE-89", + "owasp_category": null, + "file_path": "app/api/users.py", + "start_line": 42, + "end_line": 43, + "function_name": null, + "code_snippet": "query = f\"SELECT * FROM users WHERE id = {user_id}\"\ncursor.execute(query)", + "estimated_severity": "high", + "confidence": "high", + "data_flow": [ + { + "file_path": "app/api/users.py", + "line": 42, + "component": "supply_chain", + "operation": "request.args['id'] -> query -> cursor.execute" + } + ], + "related_files": [], + "fingerprint": "" + }, + { + "id": "", + "hunter_strategy": "supply_chain", + "title": "Weak hash for password storage", + "description": "MD5 used to derive a credential digest.", + "finding_type": "sca", + "cwe_id": "CWE-327", + "cwe_name": "CWE-327", + "owasp_category": null, + "file_path": "app/utils/hash.py", + "start_line": 7, + "end_line": 7, + "function_name": null, + "code_snippet": "digest = hashlib.md5(password).hexdigest()", + "estimated_severity": "medium", + "confidence": "medium", + "data_flow": null, + "related_files": [], + "fingerprint": "" + } + ], + "chains": [], + "total_raw": 2, + "deduplicated_count": 2, + "chain_count": 0, + "strategies_run": [ + "supply_chain" + ], + "hunt_duration_seconds": 0.0 + }, + "config_secrets": { + "findings": [ + { + "id": "", + "hunter_strategy": "config_secrets", + "title": "SQL injection in user lookup", + "description": "user_id flows unescaped into an f-string query.", + "finding_type": "config", + "cwe_id": "CWE-89", + "cwe_name": "CWE-89", + "owasp_category": null, + "file_path": "app/api/users.py", + "start_line": 42, + "end_line": 43, + "function_name": null, + "code_snippet": "query = f\"SELECT * FROM users WHERE id = {user_id}\"\ncursor.execute(query)", + "estimated_severity": "high", + "confidence": "high", + "data_flow": [ + { + "file_path": "app/api/users.py", + "line": 42, + "component": "config_secrets", + "operation": "request.args['id'] -> query -> cursor.execute" + } + ], + "related_files": [], + "fingerprint": "" + }, + { + "id": "", + "hunter_strategy": "config_secrets", + "title": "Weak hash for password storage", + "description": "MD5 used to derive a credential digest.", + "finding_type": "config", + "cwe_id": "CWE-327", + "cwe_name": "CWE-327", + "owasp_category": null, + "file_path": "app/utils/hash.py", + "start_line": 7, + "end_line": 7, + "function_name": null, + "code_snippet": "digest = hashlib.md5(password).hexdigest()", + "estimated_severity": "medium", + "confidence": "medium", + "data_flow": null, + "related_files": [], + "fingerprint": "" + } + ], + "chains": [], + "total_raw": 2, + "deduplicated_count": 2, + "chain_count": 0, + "strategies_run": [ + "config_secrets" + ], + "hunt_duration_seconds": 0.0 + }, + "api_security": { + "findings": [ + { + "id": "", + "hunter_strategy": "api_security", + "title": "SQL injection in user lookup", + "description": "user_id flows unescaped into an f-string query.", + "finding_type": "api", + "cwe_id": "CWE-89", + "cwe_name": "CWE-89", + "owasp_category": null, + "file_path": "app/api/users.py", + "start_line": 42, + "end_line": 43, + "function_name": null, + "code_snippet": "query = f\"SELECT * FROM users WHERE id = {user_id}\"\ncursor.execute(query)", + "estimated_severity": "high", + "confidence": "high", + "data_flow": [ + { + "file_path": "app/api/users.py", + "line": 42, + "component": "api_security", + "operation": "request.args['id'] -> query -> cursor.execute" + } + ], + "related_files": [], + "fingerprint": "" + }, + { + "id": "", + "hunter_strategy": "api_security", + "title": "Weak hash for password storage", + "description": "MD5 used to derive a credential digest.", + "finding_type": "api", + "cwe_id": "CWE-327", + "cwe_name": "CWE-327", + "owasp_category": null, + "file_path": "app/utils/hash.py", + "start_line": 7, + "end_line": 7, + "function_name": null, + "code_snippet": "digest = hashlib.md5(password).hexdigest()", + "estimated_severity": "medium", + "confidence": "medium", + "data_flow": null, + "related_files": [], + "fingerprint": "" + } + ], + "chains": [], + "total_raw": 2, + "deduplicated_count": 2, + "chain_count": 0, + "strategies_run": [ + "api_security" + ], + "hunt_duration_seconds": 0.0 + } +} diff --git a/internal/secaf/agents/hunt/testdata/golden/hunter_skips.json b/internal/secaf/agents/hunt/testdata/golden/hunter_skips.json new file mode 100644 index 0000000000..5267f42197 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/hunter_skips.json @@ -0,0 +1,76 @@ +{ + "crypto_no_usage": { + "harness_calls": 0, + "want": { + "findings": [], + "chains": [], + "total_raw": 0, + "deduplicated_count": 0, + "chain_count": 0, + "strategies_run": [], + "hunt_duration_seconds": 0.0 + } + }, + "crypto_empty_recon": { + "harness_calls": 0, + "want": { + "findings": [], + "chains": [], + "total_raw": 0, + "deduplicated_count": 0, + "chain_count": 0, + "strategies_run": [], + "hunt_duration_seconds": 0.0 + } + }, + "supply_chain_no_direct_deps": { + "harness_calls": 0, + "want": { + "findings": [], + "chains": [], + "total_raw": 0, + "deduplicated_count": 0, + "chain_count": 0, + "strategies_run": [], + "hunt_duration_seconds": 0.0 + } + }, + "api_security_no_surface": { + "harness_calls": 0, + "want": { + "findings": [], + "chains": [], + "total_raw": 0, + "deduplicated_count": 0, + "chain_count": 0, + "strategies_run": [ + "api_security" + ], + "hunt_duration_seconds": 0.0 + } + }, + "business_logic_quick": { + "harness_calls": 0, + "want": { + "findings": [], + "chains": [], + "total_raw": 0, + "deduplicated_count": 0, + "chain_count": 0, + "strategies_run": [], + "hunt_duration_seconds": 0.0 + } + }, + "logic_quick": { + "harness_calls": 0, + "want": { + "findings": [], + "chains": [], + "total_raw": 0, + "deduplicated_count": 0, + "chain_count": 0, + "strategies_run": [], + "hunt_duration_seconds": 0.0 + } + } +} diff --git a/internal/secaf/agents/hunt/testdata/golden/normalize_depth.json b/internal/secaf/agents/hunt/testdata/golden/normalize_depth.json new file mode 100644 index 0000000000..ac37cdb0b9 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/normalize_depth.json @@ -0,0 +1,9 @@ +{ + "quick": "quick", + "QUICK": "quick", + "Standard": "standard", + "thorough": "thorough", + "bogus": "standard", + "": "standard", + " quick ": "standard" +} diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_api_security_standard.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_api_security_standard.txt new file mode 100644 index 0000000000..bffee9dc68 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_api_security_standard.txt @@ -0,0 +1,182 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are an API security hunter specializing in authorization bypasses, origin validation flaws, SSRF, and open redirect vulnerabilities. + +API_SECURITY_CONTEXT: +API security-focused recon summary. + +API endpoints prioritized by missing auth/rate-limits: 20 total, showing top 15: +- POST /api/v1/thing/1 -> ThingController.action1 (app/api/thing_1.py:103, auth_required=False, rate_limited=False) +- DELETE /api/v1/thing/3 -> ThingController.action3 (app/api/thing_3.py:109, auth_required=False, rate_limited=False) +- PUT /api/v1/thing/7 -> ThingController.action7 (app/api/thing_7.py:121, auth_required=False, rate_limited=False) +- PATCH /api/v1/thing/9 -> ThingController.action9 (app/api/thing_9.py:127, auth_required=False, rate_limited=False) +- DELETE /api/v1/thing/13 -> ThingController.action13 (app/api/thing_13.py:139, auth_required=False, rate_limited=False) +- GET /api/v1/thing/15 -> ThingController.action15 (app/api/thing_15.py:145, auth_required=False, rate_limited=False) +- PATCH /api/v1/thing/19 -> ThingController.action19 (app/api/thing_19.py:157, auth_required=False, rate_limited=False) +- GET /api/v1/thing/0 -> ThingController.action0 (app/api/thing_0.py:100, auth_required=True, rate_limited=None) +- PUT /api/v1/thing/2 -> ThingController.action2 (app/api/thing_2.py:106, auth_required=None, rate_limited=True) +- PATCH /api/v1/thing/4 -> ThingController.action4 (app/api/thing_4.py:112, auth_required=True, rate_limited=None) +- GET /api/v1/thing/5 -> ThingController.action5 (app/api/thing_5.py:115, auth_required=None, rate_limited=True) +- POST /api/v1/thing/6 -> ThingController.action6 (app/api/thing_6.py:118, auth_required=True, rate_limited=None) +- DELETE /api/v1/thing/8 -> ThingController.action8 (app/api/thing_8.py:124, auth_required=None, rate_limited=True) +- GET /api/v1/thing/10 -> ThingController.action10 (app/api/thing_10.py:130, auth_required=True, rate_limited=None) +- POST /api/v1/thing/11 -> ThingController.action11 (app/api/thing_11.py:133, auth_required=None, rate_limited=True) + +HTTP/API entry points: 11 total, showing top 10: +- http handler_0 (app/entry/e0.py:10, auth_required=True) +- HTTP /v1/resource/1 (app/entry/e1.py:17, auth_required=False) +- api handler_3 (app/entry/e3.py:31, auth_required=True) +- graphql /v1/resource/4 (app/entry/e4.py:38, auth_required=False) +- rpc /v1/resource/5 (app/entry/e5.py:45, auth_required=None) +- route handler_6 (app/entry/e6.py:52, auth_required=True) +- http handler_9 (app/entry/e9.py:73, auth_required=True) +- api /v1/resource/10 (app/entry/e10.py:80, auth_required=False) +- http handler_12 (app/entry/e12.py:94, auth_required=True) +- route /v1/resource/13 (app/entry/e13.py:101, auth_required=False) + +Trust boundaries relevant to API calls: 12 total, showing top 10: +- boundary_0: internet -> app; enforcement=none +- boundary_1: dmz -> db; enforcement=waf_1, mtls_1 +- boundary_2: vpc -> cache; enforcement=waf_2, mtls_2 +- boundary_3: worker -> queue; enforcement=waf_3, mtls_3 +- boundary_4: internet -> app; enforcement=none +- boundary_5: dmz -> db; enforcement=waf_5, mtls_5 +- boundary_6: vpc -> cache; enforcement=waf_6, mtls_6 +- boundary_7: worker -> queue; enforcement=waf_7, mtls_7 +- boundary_8: internet -> app; enforcement=none +- boundary_9: dmz -> db; enforcement=waf_9, mtls_9 + +Framework/deployment API security signals: 13 total, showing top 10: +- django.middleware.csrf.CsrfViewMiddleware +- helmet defaults +- SecurityMiddleware +- rack-protection +- spring-security filter chain +- kubernetes ingress with TLS termination +- docker-compose exposes 5432 +- no WAF in front of /api +- secrets mounted from vault +- readiness probe on /healthz + +TASK: +Analyze API endpoints and supporting middleware/controllers for: +- CWE-285 Improper Authorization (BOLA/IDOR and missing ownership checks) +- CWE-346 Origin Validation Error (CORS misconfiguration, weak origin validation) +- CWE-918 SSRF (user-controlled URLs reaching server-side fetch clients) +- CWE-601 Open Redirect (untrusted redirect targets) + +Also hunt for: +- Missing/weak rate limiting on authentication and sensitive endpoints +- Mass assignment where request payloads are mapped directly to models/ORM updates +- Missing input validation (unbounded strings, unsafe coercion, negative or out-of-range values) + +WORKFLOW (follow these steps in order): +1. Read the API endpoint files and route handlers identified in the API_SECURITY_CONTEXT above. +2. For each endpoint, trace user input from request parameters/body to where it is used (database queries, redirects, outbound HTTP calls, etc.). +3. Check for missing authorization, missing input validation, CORS misconfigurations, and SSRF vectors. +4. For each confirmed vulnerability, record the file path, line number, and relevant code snippet. +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +SCOPE: +- Prioritize internet-facing API routes and handlers from api_surface. +- Trace request parameters and body fields to authorization checks, outbound HTTP clients, redirect responses, and persistence layers. +- Correlate evidence across route definitions, middleware, validators, and shared utilities. +- Exclude speculative findings without source evidence. + +OUTPUT: +- Populate findings with finding_type="api" and hunter_strategy="api_security". +- Include endpoint-specific evidence (method/path, file_path, lines, and relevant snippets). +- Include attack chains only when there is concrete multi-step linkage. +- Empty arrays are acceptable if no vulnerabilities are confirmed. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: JAVASCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec, new Function + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, parameterized pg queries + DO NOT FLAG: React JSX expressions (auto-escaped), Angular template bindings (sanitized by default) + Framework notes: Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous. + +Language: GO + Key sinks: fmt.Sprintf into SQL, exec.Command with user input, template.HTML(), os.Exec + Safe patterns (skip these): database/sql with ? placeholders, html/template (auto-escapes), exec.Command with separate args + DO NOT FLAG: GORM parameterized queries, html/template default escaping, database/sql prepared statements + Framework notes: Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML. + +Language: RUBY + Key sinks: eval, system, exec, send, public_send, ERB.new with user input + Safe patterns (skip these): ActiveRecord parameterized queries, Rack::Utils.escape_html, sanitize helper in Rails + DO NOT FLAG: ActiveRecord where with hash conditions, Rails CSRF protection, Rails html_safe on constants + Framework notes: Rails auto-escapes ERB templates. raw/html_safe bypasses escaping. Check for mass assignment. + +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +Framework: NEXTJS + Security features: React JSX auto-escapes output by default, API routes can share centralized auth middleware, Server/client boundaries reduce accidental secret exposure + DO NOT FLAG: React JSX escaped rendering, getServerSideProps/getServerSession with proper auth checks, Typed route handlers with validated schema guards + Watch for: dangerouslySetInnerHTML, unprotected API routes under /api, secret leakage to client bundles, rewrites/redirects from untrusted user input + Common vulns: XSS through dangerous HTML rendering, IDOR/auth bypass in API handlers, Sensitive env/config exposure in client-side code + +Framework: SPRING + Security features: Spring Security provides auth, CSRF, and filter chain defaults, JPA/Hibernate prepared parameter binding by default, Bean validation can enforce request constraints + DO NOT FLAG: PreparedStatement/JPA named parameter usage, Spring Security CSRF/auth filters clearly active, Thymeleaf auto-escaped output + Watch for: @PreAuthorize missing on sensitive methods, JdbcTemplate/raw Statement with concatenation, csrf().disable() on browser session flows, Actuator endpoints exposed without auth + Common vulns: Authz bypass from weak method-level security, SQL injection in raw JDBC queries, Sensitive management endpoint exposure + +Framework: REACT + Security features: JSX escapes strings before DOM rendering, Component model encourages explicit data flow, Framework discourages direct DOM mutation + DO NOT FLAG: Standard JSX expression rendering, textContent assignment for untrusted text, sanitized HTML via trusted DOMPurify policy + Watch for: dangerouslySetInnerHTML, untrusted URL assignment to href/src, eval/new Function in client code, token/secret exposure in bundles + Common vulns: DOM XSS through unsafe HTML injection, Open redirect via unvalidated navigation targets, Sensitive data exposure in frontend artifacts + +Framework: VUE + Security features: Mustache template interpolation escapes HTML, Component props/events provide explicit boundaries, Router guards can enforce auth flows + DO NOT FLAG: escaped template interpolation {{ value }}, validated route guards protecting private routes, sanitized content rendered through safe components + Watch for: v-html with untrusted data, dynamic component/template compilation, unsafe URL bindings in href/src, client-side auth checks without server enforcement + Common vulns: XSS through v-html and unsafe render paths, Auth bypass from client-only route protection, Open redirect patterns in router navigation + +CONSTRAINTS: +- Prefer confirmed code evidence over heuristics. +- Base all findings on actual file content you read. Do not speculate. +- Do not include markdown, prose, or code fences in the output file. JSON only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Focus only on API-relevant code paths and endpoint handlers surfaced by RECON. +- Early stop rule: if you inspect standard files without credible API issues, stop and return empty findings. +- Read the handler files first, then generate findings. +- After gathering evidence, write the JSON output file using your Write tool. diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_api_security_thorough.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_api_security_thorough.txt new file mode 100644 index 0000000000..c5b0f3ee4e --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_api_security_thorough.txt @@ -0,0 +1,95 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are an API security hunter specializing in authorization bypasses, origin validation flaws, SSRF, and open redirect vulnerabilities. + +API_SECURITY_CONTEXT: +API security-focused recon summary. + +API endpoints prioritized by missing auth/rate-limits: 2 total, showing top 2: +- POST /login -> login (app/api/auth.py:12, auth_required=False, rate_limited=False) +- GET /users/{id} -> get_user (app/api/users.py:40, auth_required=True, rate_limited=True) + +HTTP/API entry points: 1 total, showing top 1: +- http /login (app/api/auth.py:12, auth_required=False) + +Trust boundaries relevant to API calls: 1 total, showing top 1: +- edge: internet -> app; enforcement=waf + +Framework/deployment API security signals: 2 total, showing top 2: +- django-csrf +- docker + +TASK: +Analyze API endpoints and supporting middleware/controllers for: +- CWE-285 Improper Authorization (BOLA/IDOR and missing ownership checks) +- CWE-346 Origin Validation Error (CORS misconfiguration, weak origin validation) +- CWE-918 SSRF (user-controlled URLs reaching server-side fetch clients) +- CWE-601 Open Redirect (untrusted redirect targets) + +Also hunt for: +- Missing/weak rate limiting on authentication and sensitive endpoints +- Mass assignment where request payloads are mapped directly to models/ORM updates +- Missing input validation (unbounded strings, unsafe coercion, negative or out-of-range values) + +WORKFLOW (follow these steps in order): +1. Read the API endpoint files and route handlers identified in the API_SECURITY_CONTEXT above. +2. For each endpoint, trace user input from request parameters/body to where it is used (database queries, redirects, outbound HTTP calls, etc.). +3. Check for missing authorization, missing input validation, CORS misconfigurations, and SSRF vectors. +4. For each confirmed vulnerability, record the file path, line number, and relevant code snippet. +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +SCOPE: +- Prioritize internet-facing API routes and handlers from api_surface. +- Trace request parameters and body fields to authorization checks, outbound HTTP clients, redirect responses, and persistence layers. +- Correlate evidence across route definitions, middleware, validators, and shared utilities. +- Exclude speculative findings without source evidence. + +OUTPUT: +- Populate findings with finding_type="api" and hunter_strategy="api_security". +- Include endpoint-specific evidence (method/path, file_path, lines, and relevant snippets). +- Include attack chains only when there is concrete multi-step linkage. +- Empty arrays are acceptable if no vulnerabilities are confirmed. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +CONSTRAINTS: +- Prefer confirmed code evidence over heuristics. +- Base all findings on actual file content you read. Do not speculate. +- Do not include markdown, prose, or code fences in the output file. JSON only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Focus only on API-relevant code paths and endpoint handlers surfaced by RECON. +- Early stop rule: if you inspect thorough files without credible API issues, stop and return empty findings. +- Read the handler files first, then generate findings. +- After gathering evidence, write the JSON output file using your Write tool. diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_auth_quick.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_auth_quick.txt new file mode 100644 index 0000000000..7f85225541 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_auth_quick.txt @@ -0,0 +1,107 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are SEC-AF's Auth/AuthZ Hunter for HUNT phase. + +MISSION: +Find potential authentication and authorization vulnerabilities with high signal and low false positives. + +REPOSITORY: +/fixtures/demo-repo + +DEPTH PROFILE: +quick + +TARGET CWES: +CWE-287, CWE-306, CWE-862, CWE-863, CWE-352 + +RECON CONTEXT SUMMARY: +Authentication/authorization-focused recon summary. + +Auth model: jwt. Details: HS256 access tokens. + +Auth/session/RBAC modules and middleware candidates: none identified in recon. + +API endpoints to validate for auth/authz coverage: 2 total, showing top 2: +- POST /login -> login (app/api/auth.py:12, auth_required=False, rate_limited=False) +- GET /users/{id} -> get_user (app/api/users.py:40, auth_required=True, rate_limited=True) + +Auth/session-relevant data flows: none identified in recon. + +Security headers and framework security signals: 2 total, showing top 2: +- Content-Security-Policy +- django-csrf + +WORKFLOW (follow these steps in order): +1. Read the source files referenced in the RECON context above to understand the codebase. +2. Search for authentication checks, authorization guards, session handling, and permission validation in the source code. +3. Read the identified files to confirm vulnerabilities with actual code evidence. +4. For each confirmed vulnerability, record the file path, line number, and relevant code snippet. +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +ANALYSIS REQUIREMENTS: +1. Prioritize concrete code locations showing missing, bypassed, or inconsistent auth/authz checks. +3. Focus on these CWE classes: + - CWE-287 Improper Authentication + - CWE-306 Missing Authentication for Critical Function + - CWE-862 Missing Authorization + - CWE-863 Incorrect Authorization + - CWE-352 Cross-Site Request Forgery +4. Use recon context to locate auth middleware, request guards, session handling, and RBAC/permission checks. +5. For each finding, provide stable dedup fingerprint and direct code evidence. + +AUTH-MODEL SPECIFIC CHECKS: +- If auth_model is "jwt": verify signature validation, accepted algorithms, key handling, claim validation, and privileged-token paths. +- If auth_model is "session": verify session fixation controls, cookie security flags, CSRF protections, and logout/session invalidation paths. +- If auth_model is "api_key": verify key exposure risks, scope restrictions, rotation controls, and protection of privileged endpoints. +- If auth_model is "none": treat exposed endpoints and critical operations as potentially unauthenticated unless explicit compensating controls exist. + +DO NOT FLAG: +- CSRF issues in API-only/stateless auth flows. +- Endpoints covered by router/global auth middleware where enforcement is clearly present. +- Endpoints explicitly marked as test/demo fixtures when not active in production paths. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +OUTPUT FORMAT: +- Return JSON only matching HuntResult. +- Populate `findings` with RawFinding entries for this strategy. +- Keep `chains` empty unless you discover a concrete auth chain in this strategy. +- Set `strategies_run` to ["auth"]. +- Do not return markdown. +- Do not include explanations outside JSON. +- Empty arrays are acceptable if no vulnerabilities are confirmed. +- Base all findings on actual file content you read. Do not speculate. + + +EXECUTION CONSTRAINTS: +- Early stop rule: if you inspect 30 files without credible auth issues, stop and return empty findings. + diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_auth_standard.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_auth_standard.txt new file mode 100644 index 0000000000..2cb4633278 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_auth_standard.txt @@ -0,0 +1,189 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are SEC-AF's Auth/AuthZ Hunter for HUNT phase. + +MISSION: +Find potential authentication and authorization vulnerabilities with high signal and low false positives. + +REPOSITORY: +/fixtures/demo-repo + +DEPTH PROFILE: +standard + +TARGET CWES: +CWE-287, CWE-306, CWE-862, CWE-863, CWE-352 + +RECON CONTEXT SUMMARY: +Authentication/authorization-focused recon summary. + +Auth model: jwt. Details: HS256 JWTs issued by app/auth/service.py, refresh tokens in Redis. + +Auth/session/RBAC modules and middleware candidates: 8 total, showing top 8: +- app/auth/service.py (python) - Session and JWT issuance +- app/auth/rbac.py (python) +- app/web/middleware/csrf.js (javascript) - CSRF middleware +- app/auth/session_store.py (python) - Redis-backed sessions +- app/auth/permissions.py (python) - Role → permission table +- app/common/guard.py (python) - Assorted guard helpers +- db/migrate/2024_add_roles.rb (ruby) - Adds role column +- app/common/jwt_tools.py (python) + +API endpoints to validate for auth/authz coverage: 20 total, showing top 15: +- POST /api/v1/thing/1 -> ThingController.action1 (app/api/thing_1.py:103, auth_required=False, rate_limited=False) +- DELETE /api/v1/thing/3 -> ThingController.action3 (app/api/thing_3.py:109, auth_required=False, rate_limited=False) +- PUT /api/v1/thing/7 -> ThingController.action7 (app/api/thing_7.py:121, auth_required=False, rate_limited=False) +- PATCH /api/v1/thing/9 -> ThingController.action9 (app/api/thing_9.py:127, auth_required=False, rate_limited=False) +- DELETE /api/v1/thing/13 -> ThingController.action13 (app/api/thing_13.py:139, auth_required=False, rate_limited=False) +- GET /api/v1/thing/15 -> ThingController.action15 (app/api/thing_15.py:145, auth_required=False, rate_limited=False) +- PATCH /api/v1/thing/19 -> ThingController.action19 (app/api/thing_19.py:157, auth_required=False, rate_limited=False) +- GET /api/v1/thing/0 -> ThingController.action0 (app/api/thing_0.py:100, auth_required=True, rate_limited=None) +- PUT /api/v1/thing/2 -> ThingController.action2 (app/api/thing_2.py:106, auth_required=None, rate_limited=True) +- PATCH /api/v1/thing/4 -> ThingController.action4 (app/api/thing_4.py:112, auth_required=True, rate_limited=None) +- GET /api/v1/thing/5 -> ThingController.action5 (app/api/thing_5.py:115, auth_required=None, rate_limited=True) +- POST /api/v1/thing/6 -> ThingController.action6 (app/api/thing_6.py:118, auth_required=True, rate_limited=None) +- DELETE /api/v1/thing/8 -> ThingController.action8 (app/api/thing_8.py:124, auth_required=None, rate_limited=True) +- GET /api/v1/thing/10 -> ThingController.action10 (app/api/thing_10.py:130, auth_required=True, rate_limited=None) +- POST /api/v1/thing/11 -> ThingController.action11 (app/api/thing_11.py:133, auth_required=None, rate_limited=True) + +Auth/session-relevant data flows: 5 total, showing top 5: +- request.cookies['session'] -> redis.set (files=app/auth/session_store.py, sanitized=True) +- form['password'] -> logger.info (files=app/auth/service.py, app/obs/telemetry.go, sanitized=False) +- header['Authorization'] -> jwt.decode (files=app/common/jwt_tools.py, sanitized=True) +- session['role'] -> permission_check (files=app/auth/permissions.py, app/auth/rbac.py, sanitized=True) +- token -> cache.set (files=app/common/cache.py, sanitized=True) + +Security headers and framework security signals: 11 total, showing top 10: +- Content-Security-Policy: default-src 'self' +- X-Content-Type-Options: nosniff +- Strict-Transport-Security: max-age=31536000 +- X-Frame-Options: DENY +- Referrer-Policy: no-referrer +- Permissions-Policy: geolocation=() +- django.middleware.csrf.CsrfViewMiddleware +- helmet defaults +- SecurityMiddleware +- rack-protection + +WORKFLOW (follow these steps in order): +1. Read the source files referenced in the RECON context above to understand the codebase. +2. Search for authentication checks, authorization guards, session handling, and permission validation in the source code. +3. Read the identified files to confirm vulnerabilities with actual code evidence. +4. For each confirmed vulnerability, record the file path, line number, and relevant code snippet. +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +ANALYSIS REQUIREMENTS: +1. Prioritize concrete code locations showing missing, bypassed, or inconsistent auth/authz checks. +3. Focus on these CWE classes: + - CWE-287 Improper Authentication + - CWE-306 Missing Authentication for Critical Function + - CWE-862 Missing Authorization + - CWE-863 Incorrect Authorization + - CWE-352 Cross-Site Request Forgery +4. Use recon context to locate auth middleware, request guards, session handling, and RBAC/permission checks. +5. For each finding, provide stable dedup fingerprint and direct code evidence. + +AUTH-MODEL SPECIFIC CHECKS: +- If auth_model is "jwt": verify signature validation, accepted algorithms, key handling, claim validation, and privileged-token paths. +- If auth_model is "session": verify session fixation controls, cookie security flags, CSRF protections, and logout/session invalidation paths. +- If auth_model is "api_key": verify key exposure risks, scope restrictions, rotation controls, and protection of privileged endpoints. +- If auth_model is "none": treat exposed endpoints and critical operations as potentially unauthenticated unless explicit compensating controls exist. + +DO NOT FLAG: +- CSRF issues in API-only/stateless auth flows. +- Endpoints covered by router/global auth middleware where enforcement is clearly present. +- Endpoints explicitly marked as test/demo fixtures when not active in production paths. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: JAVASCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec, new Function + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, parameterized pg queries + DO NOT FLAG: React JSX expressions (auto-escaped), Angular template bindings (sanitized by default) + Framework notes: Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous. + +Language: GO + Key sinks: fmt.Sprintf into SQL, exec.Command with user input, template.HTML(), os.Exec + Safe patterns (skip these): database/sql with ? placeholders, html/template (auto-escapes), exec.Command with separate args + DO NOT FLAG: GORM parameterized queries, html/template default escaping, database/sql prepared statements + Framework notes: Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML. + +Language: RUBY + Key sinks: eval, system, exec, send, public_send, ERB.new with user input + Safe patterns (skip these): ActiveRecord parameterized queries, Rack::Utils.escape_html, sanitize helper in Rails + DO NOT FLAG: ActiveRecord where with hash conditions, Rails CSRF protection, Rails html_safe on constants + Framework notes: Rails auto-escapes ERB templates. raw/html_safe bypasses escaping. Check for mass assignment. + +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +Framework: NEXTJS + Security features: React JSX auto-escapes output by default, API routes can share centralized auth middleware, Server/client boundaries reduce accidental secret exposure + DO NOT FLAG: React JSX escaped rendering, getServerSideProps/getServerSession with proper auth checks, Typed route handlers with validated schema guards + Watch for: dangerouslySetInnerHTML, unprotected API routes under /api, secret leakage to client bundles, rewrites/redirects from untrusted user input + Common vulns: XSS through dangerous HTML rendering, IDOR/auth bypass in API handlers, Sensitive env/config exposure in client-side code + +Framework: SPRING + Security features: Spring Security provides auth, CSRF, and filter chain defaults, JPA/Hibernate prepared parameter binding by default, Bean validation can enforce request constraints + DO NOT FLAG: PreparedStatement/JPA named parameter usage, Spring Security CSRF/auth filters clearly active, Thymeleaf auto-escaped output + Watch for: @PreAuthorize missing on sensitive methods, JdbcTemplate/raw Statement with concatenation, csrf().disable() on browser session flows, Actuator endpoints exposed without auth + Common vulns: Authz bypass from weak method-level security, SQL injection in raw JDBC queries, Sensitive management endpoint exposure + +Framework: REACT + Security features: JSX escapes strings before DOM rendering, Component model encourages explicit data flow, Framework discourages direct DOM mutation + DO NOT FLAG: Standard JSX expression rendering, textContent assignment for untrusted text, sanitized HTML via trusted DOMPurify policy + Watch for: dangerouslySetInnerHTML, untrusted URL assignment to href/src, eval/new Function in client code, token/secret exposure in bundles + Common vulns: DOM XSS through unsafe HTML injection, Open redirect via unvalidated navigation targets, Sensitive data exposure in frontend artifacts + +Framework: VUE + Security features: Mustache template interpolation escapes HTML, Component props/events provide explicit boundaries, Router guards can enforce auth flows + DO NOT FLAG: escaped template interpolation {{ value }}, validated route guards protecting private routes, sanitized content rendered through safe components + Watch for: v-html with untrusted data, dynamic component/template compilation, unsafe URL bindings in href/src, client-side auth checks without server enforcement + Common vulns: XSS through v-html and unsafe render paths, Auth bypass from client-only route protection, Open redirect patterns in router navigation + +OUTPUT FORMAT: +- Return JSON only matching HuntResult. +- Populate `findings` with RawFinding entries for this strategy. +- Keep `chains` empty unless you discover a concrete auth chain in this strategy. +- Set `strategies_run` to ["auth"]. +- Do not return markdown. +- Do not include explanations outside JSON. +- Empty arrays are acceptable if no vulnerabilities are confirmed. +- Base all findings on actual file content you read. Do not speculate. + + +EXECUTION CONSTRAINTS: +- Early stop rule: if you inspect 30 files without credible auth issues, stop and return empty findings. + diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_auth_thorough.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_auth_thorough.txt new file mode 100644 index 0000000000..afb1041875 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_auth_thorough.txt @@ -0,0 +1,107 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are SEC-AF's Auth/AuthZ Hunter for HUNT phase. + +MISSION: +Find potential authentication and authorization vulnerabilities with high signal and low false positives. + +REPOSITORY: +/fixtures/demo-repo + +DEPTH PROFILE: +thorough + +TARGET CWES: +CWE-287, CWE-306, CWE-862, CWE-863, CWE-352 + +RECON CONTEXT SUMMARY: +Authentication/authorization-focused recon summary. + +Auth model: jwt. Details: HS256 access tokens. + +Auth/session/RBAC modules and middleware candidates: none identified in recon. + +API endpoints to validate for auth/authz coverage: 2 total, showing top 2: +- POST /login -> login (app/api/auth.py:12, auth_required=False, rate_limited=False) +- GET /users/{id} -> get_user (app/api/users.py:40, auth_required=True, rate_limited=True) + +Auth/session-relevant data flows: none identified in recon. + +Security headers and framework security signals: 2 total, showing top 2: +- Content-Security-Policy +- django-csrf + +WORKFLOW (follow these steps in order): +1. Read the source files referenced in the RECON context above to understand the codebase. +2. Search for authentication checks, authorization guards, session handling, and permission validation in the source code. +3. Read the identified files to confirm vulnerabilities with actual code evidence. +4. For each confirmed vulnerability, record the file path, line number, and relevant code snippet. +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +ANALYSIS REQUIREMENTS: +1. Prioritize concrete code locations showing missing, bypassed, or inconsistent auth/authz checks. +3. Focus on these CWE classes: + - CWE-287 Improper Authentication + - CWE-306 Missing Authentication for Critical Function + - CWE-862 Missing Authorization + - CWE-863 Incorrect Authorization + - CWE-352 Cross-Site Request Forgery +4. Use recon context to locate auth middleware, request guards, session handling, and RBAC/permission checks. +5. For each finding, provide stable dedup fingerprint and direct code evidence. + +AUTH-MODEL SPECIFIC CHECKS: +- If auth_model is "jwt": verify signature validation, accepted algorithms, key handling, claim validation, and privileged-token paths. +- If auth_model is "session": verify session fixation controls, cookie security flags, CSRF protections, and logout/session invalidation paths. +- If auth_model is "api_key": verify key exposure risks, scope restrictions, rotation controls, and protection of privileged endpoints. +- If auth_model is "none": treat exposed endpoints and critical operations as potentially unauthenticated unless explicit compensating controls exist. + +DO NOT FLAG: +- CSRF issues in API-only/stateless auth flows. +- Endpoints covered by router/global auth middleware where enforcement is clearly present. +- Endpoints explicitly marked as test/demo fixtures when not active in production paths. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +OUTPUT FORMAT: +- Return JSON only matching HuntResult. +- Populate `findings` with RawFinding entries for this strategy. +- Keep `chains` empty unless you discover a concrete auth chain in this strategy. +- Set `strategies_run` to ["auth"]. +- Do not return markdown. +- Do not include explanations outside JSON. +- Empty arrays are acceptable if no vulnerabilities are confirmed. +- Base all findings on actual file content you read. Do not speculate. + + +EXECUTION CONSTRAINTS: +- Early stop rule: if you inspect 30 files without credible auth issues, stop and return empty findings. + diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_business_logic_standard.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_business_logic_standard.txt new file mode 100644 index 0000000000..99d9848ddd --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_business_logic_standard.txt @@ -0,0 +1,938 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are the SEC-AF Business Logic Hunter focused on high-impact workflow abuse and invariant violations. + +CONTEXT: +You are in HUNT phase. Analyze this repository using the provided recon context and return HuntResult JSON. + +RECON_CONTEXT: +{ + "app_type": "web_api", + "frameworks": [ + "Django", + "next.js", + "Spring Boot", + " React ", + "unknown-fw", + "NEXT", + "Vue" + ], + "languages": [ + "Python", + "JavaScript", + "Go", + "Ruby", + "Rust", + "python" + ], + "auth_model": "jwt", + "auth_details": "HS256 JWTs issued by app/auth/service.py, refresh tokens in Redis", + "entry_points": [ + { + "kind": "http", + "identifier": "handler_0", + "file_path": "app/entry/e0.py", + "line": 10, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "HTTP", + "identifier": "handler_1", + "file_path": "app/entry/e1.py", + "line": 17, + "method": "POST", + "route": "/v1/resource/1", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_2", + "file_path": "app/entry/e2.py", + "line": 24, + "method": null, + "route": "/v1/resource/2", + "auth_required": null + }, + { + "kind": "api", + "identifier": "handler_3", + "file_path": "app/entry/e3.py", + "line": 31, + "method": "PUT", + "route": null, + "auth_required": true + }, + { + "kind": "graphql", + "identifier": "handler_4", + "file_path": "app/entry/e4.py", + "line": 38, + "method": "GET", + "route": "/v1/resource/4", + "auth_required": false + }, + { + "kind": "rpc", + "identifier": "handler_5", + "file_path": "app/entry/e5.py", + "line": 45, + "method": "POST", + "route": "/v1/resource/5", + "auth_required": null + }, + { + "kind": "route", + "identifier": "handler_6", + "file_path": "app/entry/e6.py", + "line": 52, + "method": null, + "route": null, + "auth_required": true + }, + { + "kind": "cron", + "identifier": "handler_7", + "file_path": "app/entry/e7.py", + "line": 59, + "method": "PUT", + "route": "/v1/resource/7", + "auth_required": false + }, + { + "kind": "queue", + "identifier": "handler_8", + "file_path": "app/entry/e8.py", + "line": 66, + "method": "GET", + "route": "/v1/resource/8", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_9", + "file_path": "app/entry/e9.py", + "line": 73, + "method": "POST", + "route": null, + "auth_required": true + }, + { + "kind": "api", + "identifier": "handler_10", + "file_path": "app/entry/e10.py", + "line": 80, + "method": null, + "route": "/v1/resource/10", + "auth_required": false + }, + { + "kind": "grpc", + "identifier": "handler_11", + "file_path": "app/entry/e11.py", + "line": 87, + "method": "PUT", + "route": "/v1/resource/11", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_12", + "file_path": "app/entry/e12.py", + "line": 94, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "route", + "identifier": "handler_13", + "file_path": "app/entry/e13.py", + "line": 101, + "method": "POST", + "route": "/v1/resource/13", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_14", + "file_path": "app/entry/e14.py", + "line": 108, + "method": null, + "route": "/v1/resource/14", + "auth_required": null + } + ], + "api_surface": [ + { + "method": "GET", + "path": "/api/v1/thing/0", + "handler": "ThingController.action0", + "file_path": "app/api/thing_0.py", + "line": 100, + "auth_required": true, + "rate_limited": null + }, + { + "method": "POST", + "path": "/api/v1/thing/1", + "handler": "ThingController.action1", + "file_path": "app/api/thing_1.py", + "line": 103, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PUT", + "path": "/api/v1/thing/2", + "handler": "ThingController.action2", + "file_path": "app/api/thing_2.py", + "line": 106, + "auth_required": null, + "rate_limited": true + }, + { + "method": "DELETE", + "path": "/api/v1/thing/3", + "handler": "ThingController.action3", + "file_path": "app/api/thing_3.py", + "line": 109, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PATCH", + "path": "/api/v1/thing/4", + "handler": "ThingController.action4", + "file_path": "app/api/thing_4.py", + "line": 112, + "auth_required": true, + "rate_limited": null + }, + { + "method": "GET", + "path": "/api/v1/thing/5", + "handler": "ThingController.action5", + "file_path": "app/api/thing_5.py", + "line": 115, + "auth_required": null, + "rate_limited": true + }, + { + "method": "POST", + "path": "/api/v1/thing/6", + "handler": "ThingController.action6", + "file_path": "app/api/thing_6.py", + "line": 118, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PUT", + "path": "/api/v1/thing/7", + "handler": "ThingController.action7", + "file_path": "app/api/thing_7.py", + "line": 121, + "auth_required": false, + "rate_limited": false + }, + { + "method": "DELETE", + "path": "/api/v1/thing/8", + "handler": "ThingController.action8", + "file_path": "app/api/thing_8.py", + "line": 124, + "auth_required": null, + "rate_limited": true + }, + { + "method": "PATCH", + "path": "/api/v1/thing/9", + "handler": "ThingController.action9", + "file_path": "app/api/thing_9.py", + "line": 127, + "auth_required": false, + "rate_limited": false + }, + { + "method": "GET", + "path": "/api/v1/thing/10", + "handler": "ThingController.action10", + "file_path": "app/api/thing_10.py", + "line": 130, + "auth_required": true, + "rate_limited": null + }, + { + "method": "POST", + "path": "/api/v1/thing/11", + "handler": "ThingController.action11", + "file_path": "app/api/thing_11.py", + "line": 133, + "auth_required": null, + "rate_limited": true + }, + { + "method": "PUT", + "path": "/api/v1/thing/12", + "handler": "ThingController.action12", + "file_path": "app/api/thing_12.py", + "line": 136, + "auth_required": true, + "rate_limited": null + }, + { + "method": "DELETE", + "path": "/api/v1/thing/13", + "handler": "ThingController.action13", + "file_path": "app/api/thing_13.py", + "line": 139, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PATCH", + "path": "/api/v1/thing/14", + "handler": "ThingController.action14", + "file_path": "app/api/thing_14.py", + "line": 142, + "auth_required": null, + "rate_limited": true + }, + { + "method": "GET", + "path": "/api/v1/thing/15", + "handler": "ThingController.action15", + "file_path": "app/api/thing_15.py", + "line": 145, + "auth_required": false, + "rate_limited": false + }, + { + "method": "POST", + "path": "/api/v1/thing/16", + "handler": "ThingController.action16", + "file_path": "app/api/thing_16.py", + "line": 148, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PUT", + "path": "/api/v1/thing/17", + "handler": "ThingController.action17", + "file_path": "app/api/thing_17.py", + "line": 151, + "auth_required": null, + "rate_limited": true + }, + { + "method": "DELETE", + "path": "/api/v1/thing/18", + "handler": "ThingController.action18", + "file_path": "app/api/thing_18.py", + "line": 154, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PATCH", + "path": "/api/v1/thing/19", + "handler": "ThingController.action19", + "file_path": "app/api/thing_19.py", + "line": 157, + "auth_required": false, + "rate_limited": false + } + ], + "data_flows": [ + { + "source": "request.args['q']", + "path": [ + { + "file_path": "app/search/index.ts", + "line": 20, + "component": "step_0_a", + "operation": "read" + }, + { + "file_path": "app/util/b.py", + "line": 40, + "component": "step_0_b", + "operation": "write" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/search/index.ts", + "app/db/raw.py", + "app/util/a.py", + "app/util/b.py" + ] + }, + { + "source": "request.cookies['session']", + "path": [ + { + "file_path": "app/auth/session_store.py", + "line": 21, + "component": "step_1_a", + "operation": "read" + }, + { + "file_path": "app/auth/session_store.py", + "line": 41, + "component": "step_1_b", + "operation": "write" + } + ], + "sink": "redis.set", + "sanitized": true, + "files": [ + "app/auth/session_store.py" + ] + }, + { + "source": "request.json['url']", + "path": [ + { + "file_path": "app/integrations/webhooks.go", + "line": 22, + "component": "step_2_a", + "operation": "read" + }, + { + "file_path": "app/net/client.go", + "line": 42, + "component": "step_2_b", + "operation": "write" + } + ], + "sink": "requests.get", + "sanitized": false, + "files": [ + "app/integrations/webhooks.go", + "app/net/client.go" + ] + }, + { + "source": "form['password']", + "path": [ + { + "file_path": "app/auth/service.py", + "line": 23, + "component": "step_3_a", + "operation": "read" + }, + { + "file_path": "app/obs/telemetry.go", + "line": 43, + "component": "step_3_b", + "operation": "write" + } + ], + "sink": "logger.info", + "sanitized": false, + "files": [ + "app/auth/service.py", + "app/obs/telemetry.go" + ] + }, + { + "source": "header['Authorization']", + "path": [ + { + "file_path": "app/common/jwt_tools.py", + "line": 24, + "component": "step_4_a", + "operation": "read" + }, + { + "file_path": "app/common/jwt_tools.py", + "line": 44, + "component": "step_4_b", + "operation": "write" + } + ], + "sink": "jwt.decode", + "sanitized": true, + "files": [ + "app/common/jwt_tools.py" + ] + }, + { + "source": "query['email']", + "path": [ + { + "file_path": "app/notify/mailer.rb", + "line": 25, + "component": "step_5_a", + "operation": "read" + }, + { + "file_path": "app/notify/mailer.rb", + "line": 45, + "component": "step_5_b", + "operation": "write" + } + ], + "sink": "smtp.send", + "sanitized": false, + "files": [ + "app/notify/mailer.rb" + ] + }, + { + "source": "body['card_number']", + "path": [ + { + "file_path": "app/billing/payments.go", + "line": 26, + "component": "step_6_a", + "operation": "read" + }, + { + "file_path": "app/billing/core.py", + "line": 46, + "component": "step_6_b", + "operation": "write" + } + ], + "sink": "stripe.Charge.create", + "sanitized": true, + "files": [ + "app/billing/payments.go", + "app/billing/core.py" + ] + }, + { + "source": "path_param['id']", + "path": [ + { + "file_path": "app/api/thing_1.py", + "line": 27, + "component": "step_7_a", + "operation": "read" + }, + { + "file_path": "app/api/thing_1.py", + "line": 47, + "component": "step_7_b", + "operation": "write" + } + ], + "sink": "orm.filter", + "sanitized": true, + "files": [ + "app/api/thing_1.py" + ] + }, + { + "source": "upload.filename", + "path": [ + { + "file_path": "app/media/uploads.py", + "line": 28, + "component": "step_8_a", + "operation": "read" + }, + { + "file_path": "app/x/w.py", + "line": 48, + "component": "step_8_b", + "operation": "write" + } + ], + "sink": "open", + "sanitized": false, + "files": [ + "app/media/uploads.py", + "app/common/cache.py", + "app/common/guard.py", + "app/x/y.py", + "app/x/z.py", + "app/x/w.py" + ] + }, + { + "source": "env['DEBUG']", + "path": [ + { + "file_path": "app/web/render.py", + "line": 29, + "component": "step_9_a", + "operation": "read" + }, + { + "file_path": "app/web/render.py", + "line": 49, + "component": "step_9_b", + "operation": "write" + } + ], + "sink": "template.render", + "sanitized": false, + "files": [ + "app/web/render.py" + ] + }, + { + "source": "request.body", + "path": [ + { + "file_path": "app/admin/views.py", + "line": 30, + "component": "step_10_a", + "operation": "read" + }, + { + "file_path": "app/admin/views.py", + "line": 50, + "component": "step_10_b", + "operation": "write" + } + ], + "sink": "eval", + "sanitized": false, + "files": [ + "app/admin/views.py" + ] + }, + { + "source": "websocket.msg", + "path": [ + { + "file_path": "app/ws/handler.py", + "line": 31, + "component": "step_11_a", + "operation": "read" + }, + { + "file_path": "app/ws/exec.py", + "line": 51, + "component": "step_11_b", + "operation": "write" + } + ], + "sink": "subprocess.run", + "sanitized": false, + "files": [ + "app/ws/handler.py", + "app/ws/exec.py" + ] + }, + { + "source": "cli.argv", + "path": [ + { + "file_path": "tools/run.py", + "line": 32, + "component": "step_12_a", + "operation": "read" + }, + { + "file_path": "tools/run.py", + "line": 52, + "component": "step_12_b", + "operation": "write" + } + ], + "sink": "os.system", + "sanitized": false, + "files": [ + "tools/run.py" + ] + }, + { + "source": "queue.payload", + "path": [ + { + "file_path": "app/worker/task.py", + "line": 33, + "component": "step_13_a", + "operation": "read" + }, + { + "file_path": "app/worker/task.py", + "line": 53, + "component": "step_13_b", + "operation": "write" + } + ], + "sink": "pickle.loads", + "sanitized": false, + "files": [ + "app/worker/task.py" + ] + }, + { + "source": "request.args['redirect']", + "path": [ + { + "file_path": "app/web/redirect.py", + "line": 34, + "component": "step_14_a", + "operation": "read" + }, + { + "file_path": "app/web/redirect.py", + "line": 54, + "component": "step_14_b", + "operation": "write" + } + ], + "sink": "HttpResponseRedirect", + "sanitized": false, + "files": [ + "app/web/redirect.py" + ] + }, + { + "source": "session['role']", + "path": [ + { + "file_path": "app/auth/permissions.py", + "line": 35, + "component": "step_15_a", + "operation": "read" + }, + { + "file_path": "app/auth/rbac.py", + "line": 55, + "component": "step_15_b", + "operation": "write" + } + ], + "sink": "permission_check", + "sanitized": true, + "files": [ + "app/auth/permissions.py", + "app/auth/rbac.py" + ] + }, + { + "source": "request.files['avatar']", + "path": [ + { + "file_path": "app/media/uploads.py", + "line": 36, + "component": "step_16_a", + "operation": "read" + }, + { + "file_path": "app/media/uploads.py", + "line": 56, + "component": "step_16_b", + "operation": "write" + } + ], + "sink": "s3.put_object", + "sanitized": true, + "files": [ + "app/media/uploads.py" + ] + }, + { + "source": "token", + "path": [ + { + "file_path": "app/common/cache.py", + "line": 37, + "component": "step_17_a", + "operation": "read" + }, + { + "file_path": "app/common/cache.py", + "line": 57, + "component": "step_17_b", + "operation": "write" + } + ], + "sink": "cache.set", + "sanitized": true, + "files": [ + "app/common/cache.py" + ] + }, + { + "source": "request.headers['X-User-Phone']", + "path": [ + { + "file_path": "app/obs/audit.py", + "line": 38, + "component": "step_18_a", + "operation": "read" + }, + { + "file_path": "app/obs/audit.py", + "line": 58, + "component": "step_18_b", + "operation": "write" + } + ], + "sink": "audit_log", + "sanitized": false, + "files": [ + "app/obs/audit.py" + ] + }, + { + "source": "graphql.variables", + "path": [ + { + "file_path": "app/api/graphql.ts", + "line": 39, + "component": "step_19_a", + "operation": "read" + }, + { + "file_path": "app/api/graphql.ts", + "line": 59, + "component": "step_19_b", + "operation": "write" + } + ], + "sink": "db.query", + "sanitized": false, + "files": [ + "app/api/graphql.ts" + ] + } + ] +} + +OBJECTIVE: +Find business-logic vulnerabilities by reasoning about intended behavior versus actual implementation behavior. + +FOCUS CWES: +- CWE-840: Business Logic Errors +- CWE-841: Improper Enforcement of Behavioral Workflow +- CWE-362: Race Condition +- CWE-367: Time-of-check Time-of-use (TOCTOU) Race Condition +- CWE-639: Authorization Bypass Through User-Controlled Key (IDOR patterns) + +HUNT FOR THESE PATTERNS: +- Race conditions in concurrent requests, especially check-then-act updates without robust locking/transactions +- State machine violations in multi-step workflows (skipping mandatory steps or reordering transitions) +- Missing validation of amounts, ownership, limits, or permissions beyond basic authentication +- Workflow step bypasses (for example, directly invoking step 3 from step 1) +- Integer overflow/underflow or sign bugs in balances, pricing, credits, quantities, and counters +- TOCTOU vulnerabilities where validation/check and use/mutation are separable and exploitable +- IDOR beyond missing auth (predictable identifiers, insecure ownership reassignment, mass assignment to owner/account fields) +- Price manipulation (client-controlled cart totals, discounts, tax, shipping, or final payable amounts) + +DO NOT FLAG: +- Properly locked/serialized concurrent operations +- Explicitly validated state transitions with strict transition guards +- Server-side amount/price calculation that ignores client-submitted totals +- Rate-limited operations where the suspected abuse depends on rapid repetition + +ANALYSIS GUIDELINES: +- Start from recon entry points and data flows, then trace complete end-to-end business workflows. +- Compare behavior across layers (controller/API, domain/service, persistence, queue/worker). +- Validate whether business invariants are enforced at mutation boundaries. +- Distinguish real exploit paths from expected eventual consistency or benign races. +- Include only evidence-backed findings with concrete exploit narrative. + +OUTPUT REQUIREMENTS: +- Return JSON that strictly matches HuntResult. +- Set finding_type to "logic" for business-logic findings. +- Use cwe_id and cwe_name that match the specific issue. +- Populate file_path, start_line, end_line, code_snippet, and related_files whenever possible. +- Generate stable fingerprints from vulnerability identity and location. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: JAVASCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec, new Function + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, parameterized pg queries + DO NOT FLAG: React JSX expressions (auto-escaped), Angular template bindings (sanitized by default) + Framework notes: Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous. + +Language: GO + Key sinks: fmt.Sprintf into SQL, exec.Command with user input, template.HTML(), os.Exec + Safe patterns (skip these): database/sql with ? placeholders, html/template (auto-escapes), exec.Command with separate args + DO NOT FLAG: GORM parameterized queries, html/template default escaping, database/sql prepared statements + Framework notes: Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML. + +Language: RUBY + Key sinks: eval, system, exec, send, public_send, ERB.new with user input + Safe patterns (skip these): ActiveRecord parameterized queries, Rack::Utils.escape_html, sanitize helper in Rails + DO NOT FLAG: ActiveRecord where with hash conditions, Rails CSRF protection, Rails html_safe on constants + Framework notes: Rails auto-escapes ERB templates. raw/html_safe bypasses escaping. Check for mass assignment. + +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +Framework: NEXTJS + Security features: React JSX auto-escapes output by default, API routes can share centralized auth middleware, Server/client boundaries reduce accidental secret exposure + DO NOT FLAG: React JSX escaped rendering, getServerSideProps/getServerSession with proper auth checks, Typed route handlers with validated schema guards + Watch for: dangerouslySetInnerHTML, unprotected API routes under /api, secret leakage to client bundles, rewrites/redirects from untrusted user input + Common vulns: XSS through dangerous HTML rendering, IDOR/auth bypass in API handlers, Sensitive env/config exposure in client-side code + +Framework: SPRING + Security features: Spring Security provides auth, CSRF, and filter chain defaults, JPA/Hibernate prepared parameter binding by default, Bean validation can enforce request constraints + DO NOT FLAG: PreparedStatement/JPA named parameter usage, Spring Security CSRF/auth filters clearly active, Thymeleaf auto-escaped output + Watch for: @PreAuthorize missing on sensitive methods, JdbcTemplate/raw Statement with concatenation, csrf().disable() on browser session flows, Actuator endpoints exposed without auth + Common vulns: Authz bypass from weak method-level security, SQL injection in raw JDBC queries, Sensitive management endpoint exposure + +Framework: REACT + Security features: JSX escapes strings before DOM rendering, Component model encourages explicit data flow, Framework discourages direct DOM mutation + DO NOT FLAG: Standard JSX expression rendering, textContent assignment for untrusted text, sanitized HTML via trusted DOMPurify policy + Watch for: dangerouslySetInnerHTML, untrusted URL assignment to href/src, eval/new Function in client code, token/secret exposure in bundles + Common vulns: DOM XSS through unsafe HTML injection, Open redirect via unvalidated navigation targets, Sensitive data exposure in frontend artifacts + +Framework: VUE + Security features: Mustache template interpolation escapes HTML, Component props/events provide explicit boundaries, Router guards can enforce auth flows + DO NOT FLAG: escaped template interpolation {{ value }}, validated route guards protecting private routes, sanitized content rendered through safe components + Watch for: v-html with untrusted data, dynamic component/template compilation, unsafe URL bindings in href/src, client-side auth checks without server enforcement + Common vulns: XSS through v-html and unsafe render paths, Auth bypass from client-only route protection, Open redirect patterns in router navigation + +CONSTRAINTS: +- Take multiple turns: investigate workflow intent, validate exploitability, then produce findings. +- Prefer high-confidence findings over speculative broad coverage. +- No markdown, no prose outside JSON, no code fences. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Depth profile: standard +- Early stop rule: if you inspect 30 files without credible business-logic signal, stop and return empty findings. +- Strategy: business_logic +- Focus CWEs: CWE-840, CWE-841, CWE-362, CWE-367, CWE-639. +- Reason about intended business behavior versus exploitable implementation behavior. +- Take multiple turns, trace complete workflows, and return final JSON only when complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_business_logic_thorough.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_business_logic_thorough.txt new file mode 100644 index 0000000000..ef49bc8018 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_business_logic_thorough.txt @@ -0,0 +1,163 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are the SEC-AF Business Logic Hunter focused on high-impact workflow abuse and invariant violations. + +CONTEXT: +You are in HUNT phase. Analyze this repository using the provided recon context and return HuntResult JSON. + +RECON_CONTEXT: +{ + "app_type": "web_api", + "frameworks": [ + "django" + ], + "languages": [ + "python" + ], + "auth_model": "jwt", + "auth_details": "HS256 access tokens", + "entry_points": [ + { + "kind": "http", + "identifier": "POST /login", + "file_path": "app/api/auth.py", + "line": 12, + "method": "POST", + "route": "/login", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "seed", + "file_path": "app/cli.py", + "line": 3, + "method": null, + "route": null, + "auth_required": null + } + ], + "api_surface": [ + { + "method": "POST", + "path": "/login", + "handler": "login", + "file_path": "app/api/auth.py", + "line": 12, + "auth_required": false, + "rate_limited": false + }, + { + "method": "GET", + "path": "/users/{id}", + "handler": "get_user", + "file_path": "app/api/users.py", + "line": 40, + "auth_required": true, + "rate_limited": true + } + ], + "data_flows": [ + { + "source": "request.json", + "path": [ + { + "file_path": "app/api/users.py", + "line": 41, + "component": "handler", + "operation": "read id" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/api/users.py" + ] + } + ] +} + +OBJECTIVE: +Find business-logic vulnerabilities by reasoning about intended behavior versus actual implementation behavior. + +FOCUS CWES: +- CWE-840: Business Logic Errors +- CWE-841: Improper Enforcement of Behavioral Workflow +- CWE-362: Race Condition +- CWE-367: Time-of-check Time-of-use (TOCTOU) Race Condition +- CWE-639: Authorization Bypass Through User-Controlled Key (IDOR patterns) + +HUNT FOR THESE PATTERNS: +- Race conditions in concurrent requests, especially check-then-act updates without robust locking/transactions +- State machine violations in multi-step workflows (skipping mandatory steps or reordering transitions) +- Missing validation of amounts, ownership, limits, or permissions beyond basic authentication +- Workflow step bypasses (for example, directly invoking step 3 from step 1) +- Integer overflow/underflow or sign bugs in balances, pricing, credits, quantities, and counters +- TOCTOU vulnerabilities where validation/check and use/mutation are separable and exploitable +- IDOR beyond missing auth (predictable identifiers, insecure ownership reassignment, mass assignment to owner/account fields) +- Price manipulation (client-controlled cart totals, discounts, tax, shipping, or final payable amounts) + +DO NOT FLAG: +- Properly locked/serialized concurrent operations +- Explicitly validated state transitions with strict transition guards +- Server-side amount/price calculation that ignores client-submitted totals +- Rate-limited operations where the suspected abuse depends on rapid repetition + +ANALYSIS GUIDELINES: +- Start from recon entry points and data flows, then trace complete end-to-end business workflows. +- Compare behavior across layers (controller/API, domain/service, persistence, queue/worker). +- Validate whether business invariants are enforced at mutation boundaries. +- Distinguish real exploit paths from expected eventual consistency or benign races. +- Include only evidence-backed findings with concrete exploit narrative. + +OUTPUT REQUIREMENTS: +- Return JSON that strictly matches HuntResult. +- Set finding_type to "logic" for business-logic findings. +- Use cwe_id and cwe_name that match the specific issue. +- Populate file_path, start_line, end_line, code_snippet, and related_files whenever possible. +- Generate stable fingerprints from vulnerability identity and location. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +CONSTRAINTS: +- Take multiple turns: investigate workflow intent, validate exploitability, then produce findings. +- Prefer high-confidence findings over speculative broad coverage. +- No markdown, no prose outside JSON, no code fences. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Depth profile: thorough +- Early stop rule: if you inspect 30 files without credible business-logic signal, stop and return empty findings. +- Strategy: business_logic +- Focus CWEs: CWE-840, CWE-841, CWE-362, CWE-367, CWE-639. +- Reason about intended business behavior versus exploitable implementation behavior. +- Take multiple turns, trace complete workflows, and return final JSON only when complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_config_secrets_standard.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_config_secrets_standard.txt new file mode 100644 index 0000000000..623f08773f --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_config_secrets_standard.txt @@ -0,0 +1,178 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are the SEC-AF Config/Secrets Hunter for HUNT phase. + +CONTEXT: +You receive RECON context summary and must produce HuntResult JSON for strategy config_secrets. + +RECON_CONTEXT: +Config and secrets-focused recon summary. + +Detected secret-like findings: 17 total, showing top 15: +- aws_access_key at config/env_0.yaml:3; confidence=high; is_test_value=False +- github_token at config/env_1.yaml:4; confidence=medium; is_test_value=True +- private_key at config/env_2.yaml:5; confidence=low; is_test_value=None +- slack_webhook at config/env_3.yaml:6; confidence=high; is_test_value=False +- generic_api_key at config/env_4.yaml:7; confidence=medium; is_test_value=True +- aws_access_key at config/env_5.yaml:8; confidence=low; is_test_value=None +- github_token at config/env_6.yaml:9; confidence=high; is_test_value=False +- private_key at config/env_7.yaml:10; confidence=medium; is_test_value=True +- slack_webhook at config/env_8.yaml:11; confidence=low; is_test_value=None +- generic_api_key at config/env_9.yaml:12; confidence=high; is_test_value=False +- aws_access_key at config/env_10.yaml:13; confidence=medium; is_test_value=True +- github_token at config/env_11.yaml:14; confidence=low; is_test_value=None +- private_key at config/env_12.yaml:15; confidence=high; is_test_value=False +- slack_webhook at config/env_13.yaml:16; confidence=medium; is_test_value=True +- generic_api_key at config/env_14.yaml:17; confidence=low; is_test_value=None + +Configuration weaknesses from recon: 16 total, showing top 15: +- logging at config/app_0.ini:0; risk=high; key=LOG_LEVEL +- tls at config/app_1.ini:13; risk=critical; key=SSL_VERIFY +- cors at config/app_2.ini:14; risk=high; key=ALLOW_ORIGIN +- headers at config/app_3.ini:0; risk=medium; key=X_FRAME_OPTIONS +- debug at config/app_4.ini:16; risk=critical; key=DJANGO_DEBUG +- storage at config/app_5.ini:0; risk=critical; key=BUCKET_ACL +- auth at config/app_6.ini:18; risk=medium; key=SESSION_TIMEOUT +- http at config/app_7.ini:19; risk=high; key=REDIRECT_HTTPS +- secrets at config/app_8.ini:20; risk=medium; key=n/a +- trace at config/app_9.ini:21; risk=low; key=OTEL_TRACE_ALL +- network at config/app_10.ini:0; risk=medium; key=BIND_ADDR +- errors at config/app_11.ini:23; risk=high; key=SHOW_STACKTRACE +- cache at config/app_12.ini:24; risk=low; key=CACHE_TTL +- db at config/app_13.ini:25; risk=critical; key=SSLMODE +- queue at config/app_14.ini:26; risk=low; key=PREFETCH + +Security/deployment context affecting config risk: 13 total, showing top 10: +- kubernetes ingress with TLS termination +- docker-compose exposes 5432 +- no WAF in front of /api +- secrets mounted from vault +- readiness probe on /healthz +- single replica for the worker +- TLS 1.2 minimum +- internal service mesh mTLS +- django.middleware.csrf.CsrfViewMiddleware +- helmet defaults + +OBJECTIVE: +Find real security findings for: +- CWE-798 Hardcoded Credentials +- CWE-259 Hard-Coded Password +- CWE-321 Hard-Coded Cryptographic Key +- CWE-16 Configuration weaknesses + +HUNT FOR: +- API keys, passwords, bearer tokens, database credentials, cloud credentials, signing secrets, encryption keys hardcoded in source. +- Insecure defaults and configuration issues such as DEBUG enabled in production paths, permissive CORS (`*`), insecure cookies, disabled TLS verification, missing security headers, exposed admin/debug endpoints. + +DO NOT FLAG: +- `.env.example`, `config.example.*`, template/sample config files. +- Test fixtures and explicit fake values used for tests. +- Documentation snippets. +- Secure environment-variable usage patterns (for example `os.getenv(...)`, `process.env.*`, `${ENV_VAR}`). + +OUTPUT REQUIREMENTS: +- Return strict JSON matching HuntResult. +- Set `hunter_strategy` to `config_secrets` on every finding. +- Use `finding_type` as `secrets` for hardcoded secrets and `config` for configuration issues. +- Include precise file path, line range, and code snippet evidence. +- Include related files when context spans multiple files. +- Add reviewer metadata in `description` using this suffix format for each finding: + `[is_test_file=;is_example=]` + +QUALITY BAR: +- Prioritize high-confidence, exploitable findings over noisy pattern matches. +- Distinguish real credentials from placeholders by context, naming, entropy, and usage. +- If uncertain, lower confidence instead of over-claiming. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: JAVASCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec, new Function + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, parameterized pg queries + DO NOT FLAG: React JSX expressions (auto-escaped), Angular template bindings (sanitized by default) + Framework notes: Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous. + +Language: GO + Key sinks: fmt.Sprintf into SQL, exec.Command with user input, template.HTML(), os.Exec + Safe patterns (skip these): database/sql with ? placeholders, html/template (auto-escapes), exec.Command with separate args + DO NOT FLAG: GORM parameterized queries, html/template default escaping, database/sql prepared statements + Framework notes: Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML. + +Language: RUBY + Key sinks: eval, system, exec, send, public_send, ERB.new with user input + Safe patterns (skip these): ActiveRecord parameterized queries, Rack::Utils.escape_html, sanitize helper in Rails + DO NOT FLAG: ActiveRecord where with hash conditions, Rails CSRF protection, Rails html_safe on constants + Framework notes: Rails auto-escapes ERB templates. raw/html_safe bypasses escaping. Check for mass assignment. + +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +Framework: NEXTJS + Security features: React JSX auto-escapes output by default, API routes can share centralized auth middleware, Server/client boundaries reduce accidental secret exposure + DO NOT FLAG: React JSX escaped rendering, getServerSideProps/getServerSession with proper auth checks, Typed route handlers with validated schema guards + Watch for: dangerouslySetInnerHTML, unprotected API routes under /api, secret leakage to client bundles, rewrites/redirects from untrusted user input + Common vulns: XSS through dangerous HTML rendering, IDOR/auth bypass in API handlers, Sensitive env/config exposure in client-side code + +Framework: SPRING + Security features: Spring Security provides auth, CSRF, and filter chain defaults, JPA/Hibernate prepared parameter binding by default, Bean validation can enforce request constraints + DO NOT FLAG: PreparedStatement/JPA named parameter usage, Spring Security CSRF/auth filters clearly active, Thymeleaf auto-escaped output + Watch for: @PreAuthorize missing on sensitive methods, JdbcTemplate/raw Statement with concatenation, csrf().disable() on browser session flows, Actuator endpoints exposed without auth + Common vulns: Authz bypass from weak method-level security, SQL injection in raw JDBC queries, Sensitive management endpoint exposure + +Framework: REACT + Security features: JSX escapes strings before DOM rendering, Component model encourages explicit data flow, Framework discourages direct DOM mutation + DO NOT FLAG: Standard JSX expression rendering, textContent assignment for untrusted text, sanitized HTML via trusted DOMPurify policy + Watch for: dangerouslySetInnerHTML, untrusted URL assignment to href/src, eval/new Function in client code, token/secret exposure in bundles + Common vulns: DOM XSS through unsafe HTML injection, Open redirect via unvalidated navigation targets, Sensitive data exposure in frontend artifacts + +Framework: VUE + Security features: Mustache template interpolation escapes HTML, Component props/events provide explicit boundaries, Router guards can enforce auth flows + DO NOT FLAG: escaped template interpolation {{ value }}, validated route guards protecting private routes, sanitized content rendered through safe components + Watch for: v-html with untrusted data, dynamic component/template compilation, unsafe URL bindings in href/src, client-side auth checks without server enforcement + Common vulns: XSS through v-html and unsafe render paths, Auth bypass from client-only route protection, Open redirect patterns in router navigation + +CONSTRAINTS: +- Take multiple turns: first explore files, then validate candidate findings, then output final JSON. +- No markdown or code fences. JSON only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Hunt strategy: config_secrets (CWE-798, CWE-259, CWE-321, CWE-16). +- Early stop rule: if you inspect standard files without credible secrets/config issues, stop and return empty findings. +- Use RECON ConfigReport and SecurityContext to prioritize likely real findings. +- Take multiple turns: inspect files, validate exploitability signal, then build findings. + diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_config_secrets_thorough.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_config_secrets_thorough.txt new file mode 100644 index 0000000000..e234ec5492 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_config_secrets_thorough.txt @@ -0,0 +1,94 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are the SEC-AF Config/Secrets Hunter for HUNT phase. + +CONTEXT: +You receive RECON context summary and must produce HuntResult JSON for strategy config_secrets. + +RECON_CONTEXT: +Config and secrets-focused recon summary. + +Detected secret-like findings: 1 total, showing top 1: +- api_key at .env:2; confidence=high; is_test_value=False + +Configuration weaknesses from recon: 1 total, showing top 1: +- debug at settings.py:9; risk=high; key=DEBUG + +Security/deployment context affecting config risk: 2 total, showing top 2: +- docker +- django-csrf + +OBJECTIVE: +Find real security findings for: +- CWE-798 Hardcoded Credentials +- CWE-259 Hard-Coded Password +- CWE-321 Hard-Coded Cryptographic Key +- CWE-16 Configuration weaknesses + +HUNT FOR: +- API keys, passwords, bearer tokens, database credentials, cloud credentials, signing secrets, encryption keys hardcoded in source. +- Insecure defaults and configuration issues such as DEBUG enabled in production paths, permissive CORS (`*`), insecure cookies, disabled TLS verification, missing security headers, exposed admin/debug endpoints. + +DO NOT FLAG: +- `.env.example`, `config.example.*`, template/sample config files. +- Test fixtures and explicit fake values used for tests. +- Documentation snippets. +- Secure environment-variable usage patterns (for example `os.getenv(...)`, `process.env.*`, `${ENV_VAR}`). + +OUTPUT REQUIREMENTS: +- Return strict JSON matching HuntResult. +- Set `hunter_strategy` to `config_secrets` on every finding. +- Use `finding_type` as `secrets` for hardcoded secrets and `config` for configuration issues. +- Include precise file path, line range, and code snippet evidence. +- Include related files when context spans multiple files. +- Add reviewer metadata in `description` using this suffix format for each finding: + `[is_test_file=;is_example=]` + +QUALITY BAR: +- Prioritize high-confidence, exploitable findings over noisy pattern matches. +- Distinguish real credentials from placeholders by context, naming, entropy, and usage. +- If uncertain, lower confidence instead of over-claiming. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +CONSTRAINTS: +- Take multiple turns: first explore files, then validate candidate findings, then output final JSON. +- No markdown or code fences. JSON only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Hunt strategy: config_secrets (CWE-798, CWE-259, CWE-321, CWE-16). +- Early stop rule: if you inspect thorough files without credible secrets/config issues, stop and return empty findings. +- Use RECON ConfigReport and SecurityContext to prioritize likely real findings. +- Take multiple turns: inspect files, validate exploitability signal, then build findings. + diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_crypto_standard.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_crypto_standard.txt new file mode 100644 index 0000000000..19f0f0a7f6 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_crypto_standard.txt @@ -0,0 +1,185 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are the SEC-AF Cryptography Hunter specializing in weak cryptography and predictable randomness. + +CONTEXT: +You are in SEC-AF HUNT phase and must return HuntResult JSON. +Use the RECON context summary below. +Focus on whether detected crypto patterns are used in security-sensitive contexts. + +RECON_CONTEXT: +Cryptography-focused recon summary. + +Crypto usage entries: 16 total. + +Algorithms and key handling (weak entries first): 16 total, showing top 15: +- algorithm=MD5, key_size=None, mode=None, context=password hashing, is_weak=True +- algorithm=RSA, key_size=1024, mode=None, context=token signing, is_weak=True +- algorithm=DES, key_size=56, mode=CBC, context=legacy export, is_weak=True +- algorithm=HMAC-SHA1, key_size=160, mode=None, context=webhook signature, is_weak=True +- algorithm=RC4, key_size=128, mode=None, context=unspecified, is_weak=True +- algorithm=SHA-1, key_size=None, mode=None, context=checksum, is_weak=True +- algorithm=AES, key_size=128, mode=ECB, context=legacy blob, is_weak=True +- algorithm=3DES, key_size=168, mode=CBC, context=legacy tape, is_weak=True +- algorithm=AES, key_size=256, mode=GCM, context=at-rest encryption, is_weak=False +- algorithm=SHA-256, key_size=None, mode=None, context=unspecified, is_weak=False +- algorithm=ChaCha20, key_size=256, mode=Poly1305, context=transport, is_weak=False +- algorithm=bcrypt, key_size=None, mode=None, context=password hashing, is_weak=False +- algorithm=ECDSA, key_size=256, mode=None, context=JWT signing, is_weak=None +- algorithm=PBKDF2, key_size=None, mode=None, context=key derivation, is_weak=False +- algorithm=Ed25519, key_size=256, mode=None, context=package signing, is_weak=False + +Potential secret/key findings from config scan: 17 total, showing top 10: +- aws_access_key at config/env_0.yaml:3 (confidence=high) +- github_token at config/env_1.yaml:4 (confidence=medium) +- private_key at config/env_2.yaml:5 (confidence=low) +- slack_webhook at config/env_3.yaml:6 (confidence=high) +- generic_api_key at config/env_4.yaml:7 (confidence=medium) +- aws_access_key at config/env_5.yaml:8 (confidence=low) +- github_token at config/env_6.yaml:9 (confidence=high) +- private_key at config/env_7.yaml:10 (confidence=medium) +- slack_webhook at config/env_8.yaml:11 (confidence=low) +- generic_api_key at config/env_9.yaml:12 (confidence=high) + +Deployment/TLS/security header signals: 14 total, showing top 10: +- kubernetes ingress with TLS termination +- docker-compose exposes 5432 +- no WAF in front of /api +- secrets mounted from vault +- readiness probe on /healthz +- single replica for the worker +- TLS 1.2 minimum +- internal service mesh mTLS +- Content-Security-Policy: default-src 'self' +- X-Content-Type-Options: nosniff + +TASK: +Find potential vulnerabilities related to weak cryptography. + +CWE COVERAGE (REQUIRED): +- CWE-326 Inadequate Encryption Strength +- CWE-327 Broken or Risky Cryptographic Algorithm +- CWE-328 Reversible One-Way Hash +- CWE-330 Insufficiently Random Values +- CWE-916 Weak Password Hashing +- CWE-259 Hard-Coded Password +- CWE-321 Hard-Coded Cryptographic Key +- CWE-798 Hardcoded Credentials + +WHAT TO HUNT: +- Weak algorithms: MD5, SHA1 for security decisions, DES, 3DES, RC4, ECB mode. +- Inadequate key lengths: RSA < 2048, AES < 128, weak DH parameters. +- Reversible password handling: base64/encoding used as if it were hashing. +- Predictable randomness: Math.random(), non-crypto random module usage for tokens/secrets, predictable seeds. +- Weak password storage: unsalted/fast hashes, low-iteration PBKDF2, custom insecure hashing. +- Hardcoded cryptographic keys, secrets, or static IV/nonce values. +- Security-critical usage contexts: password hashing, encryption, token/session generation, signing and verification, auth flows. +- Non-critical usage contexts: checksums, ETags, cache keys, dedup/fingerprinting. + +DO NOT FLAG: +- Crypto operations in tests or fixtures. +- MD5/SHA1 used only for non-security checksums/caching/ETags/cache keys. +- bcrypt/argon2/scrypt for password hashing (these are GOOD). +- Crypto libraries used correctly with sufficient key lengths and secure modes. +- Framework secure defaults (e.g., Django PBKDF2, bcrypt, argon2). + +OUTPUT: +- Return JSON matching HuntResult. +- findings must contain RawFinding entries with evidence-backed descriptions and correct CWE mapping. +- hunter_strategy should be "crypto". + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: JAVASCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec, new Function + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, parameterized pg queries + DO NOT FLAG: React JSX expressions (auto-escaped), Angular template bindings (sanitized by default) + Framework notes: Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous. + +Language: GO + Key sinks: fmt.Sprintf into SQL, exec.Command with user input, template.HTML(), os.Exec + Safe patterns (skip these): database/sql with ? placeholders, html/template (auto-escapes), exec.Command with separate args + DO NOT FLAG: GORM parameterized queries, html/template default escaping, database/sql prepared statements + Framework notes: Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML. + +Language: RUBY + Key sinks: eval, system, exec, send, public_send, ERB.new with user input + Safe patterns (skip these): ActiveRecord parameterized queries, Rack::Utils.escape_html, sanitize helper in Rails + DO NOT FLAG: ActiveRecord where with hash conditions, Rails CSRF protection, Rails html_safe on constants + Framework notes: Rails auto-escapes ERB templates. raw/html_safe bypasses escaping. Check for mass assignment. + +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +Framework: NEXTJS + Security features: React JSX auto-escapes output by default, API routes can share centralized auth middleware, Server/client boundaries reduce accidental secret exposure + DO NOT FLAG: React JSX escaped rendering, getServerSideProps/getServerSession with proper auth checks, Typed route handlers with validated schema guards + Watch for: dangerouslySetInnerHTML, unprotected API routes under /api, secret leakage to client bundles, rewrites/redirects from untrusted user input + Common vulns: XSS through dangerous HTML rendering, IDOR/auth bypass in API handlers, Sensitive env/config exposure in client-side code + +Framework: SPRING + Security features: Spring Security provides auth, CSRF, and filter chain defaults, JPA/Hibernate prepared parameter binding by default, Bean validation can enforce request constraints + DO NOT FLAG: PreparedStatement/JPA named parameter usage, Spring Security CSRF/auth filters clearly active, Thymeleaf auto-escaped output + Watch for: @PreAuthorize missing on sensitive methods, JdbcTemplate/raw Statement with concatenation, csrf().disable() on browser session flows, Actuator endpoints exposed without auth + Common vulns: Authz bypass from weak method-level security, SQL injection in raw JDBC queries, Sensitive management endpoint exposure + +Framework: REACT + Security features: JSX escapes strings before DOM rendering, Component model encourages explicit data flow, Framework discourages direct DOM mutation + DO NOT FLAG: Standard JSX expression rendering, textContent assignment for untrusted text, sanitized HTML via trusted DOMPurify policy + Watch for: dangerouslySetInnerHTML, untrusted URL assignment to href/src, eval/new Function in client code, token/secret exposure in bundles + Common vulns: DOM XSS through unsafe HTML injection, Open redirect via unvalidated navigation targets, Sensitive data exposure in frontend artifacts + +Framework: VUE + Security features: Mustache template interpolation escapes HTML, Component props/events provide explicit boundaries, Router guards can enforce auth flows + DO NOT FLAG: escaped template interpolation {{ value }}, validated route guards protecting private routes, sanitized content rendered through safe components + Watch for: v-html with untrusted data, dynamic component/template compilation, unsafe URL bindings in href/src, client-side auth checks without server enforcement + Common vulns: XSS through v-html and unsafe render paths, Auth bypass from client-only route protection, Open redirect patterns in router navigation + +CONSTRAINTS: +- Take multiple turns: first discover crypto-relevant files, then evaluate exploitability context. +- Prefer concrete evidence (file path, line range, code snippet, why risky in context). +- No markdown/code fences; JSON output only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Hunt strategy: crypto +- Early stop rule: if you inspect standard files without credible crypto misuse, stop and return empty findings. +- Focus CWEs: CWE-326, CWE-327, CWE-328, CWE-330, CWE-916, CWE-259, CWE-321, CWE-798 +- Security-critical usage candidates: at-rest encryption, password hashing, token signing, webhook signature, password hashing, JWT signing, key derivation, package signing, password hashing +- Non-security usage candidates: checksum +- Prioritize weak crypto findings only when used in security-sensitive contexts; avoid checksum/cache-only noise. +- Take multiple turns to explore relevant files before finalizing findings. +- Write final JSON only when analysis is complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_crypto_thorough.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_crypto_thorough.txt new file mode 100644 index 0000000000..725890c0b8 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_crypto_thorough.txt @@ -0,0 +1,107 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are the SEC-AF Cryptography Hunter specializing in weak cryptography and predictable randomness. + +CONTEXT: +You are in SEC-AF HUNT phase and must return HuntResult JSON. +Use the RECON context summary below. +Focus on whether detected crypto patterns are used in security-sensitive contexts. + +RECON_CONTEXT: +Cryptography-focused recon summary. + +Crypto usage entries: 2 total. + +Algorithms and key handling (weak entries first): 2 total, showing top 2: +- algorithm=MD5, key_size=None, mode=None, context=password hashing, is_weak=True +- algorithm=SHA256, key_size=None, mode=None, context=etag cache key, is_weak=False + +Potential secret/key findings from config scan: 1 total, showing top 1: +- api_key at .env:2 (confidence=high) + +Deployment/TLS/security header signals: 2 total, showing top 2: +- docker +- Content-Security-Policy + +TASK: +Find potential vulnerabilities related to weak cryptography. + +CWE COVERAGE (REQUIRED): +- CWE-326 Inadequate Encryption Strength +- CWE-327 Broken or Risky Cryptographic Algorithm +- CWE-328 Reversible One-Way Hash +- CWE-330 Insufficiently Random Values +- CWE-916 Weak Password Hashing +- CWE-259 Hard-Coded Password +- CWE-321 Hard-Coded Cryptographic Key +- CWE-798 Hardcoded Credentials + +WHAT TO HUNT: +- Weak algorithms: MD5, SHA1 for security decisions, DES, 3DES, RC4, ECB mode. +- Inadequate key lengths: RSA < 2048, AES < 128, weak DH parameters. +- Reversible password handling: base64/encoding used as if it were hashing. +- Predictable randomness: Math.random(), non-crypto random module usage for tokens/secrets, predictable seeds. +- Weak password storage: unsalted/fast hashes, low-iteration PBKDF2, custom insecure hashing. +- Hardcoded cryptographic keys, secrets, or static IV/nonce values. +- Security-critical usage contexts: password hashing, encryption, token/session generation, signing and verification, auth flows. +- Non-critical usage contexts: checksums, ETags, cache keys, dedup/fingerprinting. + +DO NOT FLAG: +- Crypto operations in tests or fixtures. +- MD5/SHA1 used only for non-security checksums/caching/ETags/cache keys. +- bcrypt/argon2/scrypt for password hashing (these are GOOD). +- Crypto libraries used correctly with sufficient key lengths and secure modes. +- Framework secure defaults (e.g., Django PBKDF2, bcrypt, argon2). + +OUTPUT: +- Return JSON matching HuntResult. +- findings must contain RawFinding entries with evidence-backed descriptions and correct CWE mapping. +- hunter_strategy should be "crypto". + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +CONSTRAINTS: +- Take multiple turns: first discover crypto-relevant files, then evaluate exploitability context. +- Prefer concrete evidence (file path, line range, code snippet, why risky in context). +- No markdown/code fences; JSON output only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Hunt strategy: crypto +- Early stop rule: if you inspect thorough files without credible crypto misuse, stop and return empty findings. +- Focus CWEs: CWE-326, CWE-327, CWE-328, CWE-330, CWE-916, CWE-259, CWE-321, CWE-798 +- Security-critical usage candidates: password hashing, etag cache key +- Non-security usage candidates: etag cache key +- Prioritize weak crypto findings only when used in security-sensitive contexts; avoid checksum/cache-only noise. +- Take multiple turns to explore relevant files before finalizing findings. +- Write final JSON only when analysis is complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_data_exposure_quick.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_data_exposure_quick.txt new file mode 100644 index 0000000000..316364e551 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_data_exposure_quick.txt @@ -0,0 +1,105 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are a data exposure security analyst for SEC-AF HUNT Phase. + +CONTEXT: +You are analyzing a real repository for Data Exposure Hunter. +Use the RECON context summary below to focus your investigation. + +RECON_CONTEXT: +Data exposure-focused recon summary. + +Data flows touching likely sensitive domains: 1 total, showing top 1: +- request.json -> cursor.execute; sanitized=False; files=app/api/users.py + +Logging/exposure-related misconfig signals: 1 total, showing top 1: +- debug at settings.py:9; risk=high; key=DEBUG + +Entry points and API surface with exposure risk: 2 total, showing top 2: +- POST /login (app/api/auth.py:12, auth_required=False) +- GET /users/{id} (app/api/users.py:40, auth_required=True) + +TASK: +Find potential data exposure vulnerabilities and return HuntResult JSON. + +WORKFLOW (follow these steps in order): +1. Read the source files referenced in the RECON context above to understand the codebase. +2. Search for logging calls, error handlers, data persistence, HTTP clients, and configuration settings in the source code. +3. Read the identified files to confirm vulnerabilities with actual code evidence. +4. For each confirmed vulnerability, record the file path, line number, and relevant code snippet. +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +COVERAGE: +- CWE-200: information exposure (debug endpoints, verbose responses, server/internal metadata leakage) +- CWE-209: detailed error message leakage (stack traces, SQL errors, file paths, internal host/IP details) +- CWE-532: sensitive data in logs (PII, credentials, tokens, secrets, auth headers, session IDs) +- CWE-312: cleartext storage of sensitive data (passwords, tokens, PII persisted without encryption) +- CWE-319: cleartext transmission of sensitive data (HTTP transport for auth/session/PII or disabled TLS verification) + +FOCUS AREAS: +- Logging calls, logger middleware, and request/response logging filters +- Error handlers, exception formatters, debug/traceback output, API error payloads +- Persistence layer, model fields, data-at-rest handling, local file writes/caches/backups +- HTTP clients, webhook integrations, transport settings, TLS/SSL flags +- Configuration and deployment toggles that affect production exposure + +REQUIRED OUTPUT: +- Return JSON that strictly matches HuntResult. +- Populate findings with concrete evidence; chains may be empty. +- For each finding, fill all required RawFinding fields including: + - hunter_strategy = "data_exposure" + - cwe_id and cwe_name aligned to this strategy + - file_path, start_line, end_line, code_snippet + - estimated_severity and confidence + - fingerprint stable for file_path + start_line + cwe_id +- Distinguish production impact vs development-only behavior in description and confidence. +- Empty arrays are acceptable if no vulnerabilities are confirmed. + +DO NOT FLAG: +- Logging of non-sensitive operational data (request IDs, timestamps, non-sensitive metrics) +- Error verbosity that is clearly limited to development/local debug mode +- HTTPS URLs or TLS-enabled transmissions + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +CONSTRAINTS: +- Prefer evidence over assumptions; if uncertain, lower confidence. +- No markdown or code fences; JSON output only. +- Base all findings on actual file content you read. Do not speculate. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Strategy: data_exposure +- Early stop rule: if you inspect quick files without credible exposure risk, stop and return empty findings. +- Use multiple turns: inspect files first, then produce findings. +- Return final JSON only when analysis is complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_data_exposure_standard.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_data_exposure_standard.txt new file mode 100644 index 0000000000..8bb45aa26b --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_data_exposure_standard.txt @@ -0,0 +1,174 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are a data exposure security analyst for SEC-AF HUNT Phase. + +CONTEXT: +You are analyzing a real repository for Data Exposure Hunter. +Use the RECON context summary below to focus your investigation. + +RECON_CONTEXT: +Data exposure-focused recon summary. + +Data flows touching likely sensitive domains: 8 total, showing top 8: +- request.cookies['session'] -> redis.set; sanitized=True; files=app/auth/session_store.py +- form['password'] -> logger.info; sanitized=False; files=app/auth/service.py, app/obs/telemetry.go +- header['Authorization'] -> jwt.decode; sanitized=True; files=app/common/jwt_tools.py +- query['email'] -> smtp.send; sanitized=False; files=app/notify/mailer.rb +- body['card_number'] -> stripe.Charge.create; sanitized=True; files=app/billing/payments.go, app/billing/core.py +- session['role'] -> permission_check; sanitized=True; files=app/auth/permissions.py, app/auth/rbac.py +- token -> cache.set; sanitized=True; files=app/common/cache.py +- request.headers['X-User-Phone'] -> audit_log; sanitized=False; files=app/obs/audit.py + +Logging/exposure-related misconfig signals: 7 total, showing top 7: +- logging at config/app_0.ini:0; risk=high; key=LOG_LEVEL +- tls at config/app_1.ini:13; risk=critical; key=SSL_VERIFY +- debug at config/app_4.ini:16; risk=critical; key=DJANGO_DEBUG +- http at config/app_7.ini:19; risk=high; key=REDIRECT_HTTPS +- trace at config/app_9.ini:21; risk=low; key=OTEL_TRACE_ALL +- errors at config/app_11.ini:23; risk=high; key=SHOW_STACKTRACE +- exposure at config/app_15.ini:0; risk=medium; key=ADMIN_PATH + +Entry points and API surface with exposure risk: 20 total, showing top 10: +- GET /api/v1/thing/0 (app/api/thing_0.py:100, auth_required=True) +- POST /api/v1/thing/1 (app/api/thing_1.py:103, auth_required=False) +- PUT /api/v1/thing/2 (app/api/thing_2.py:106, auth_required=None) +- DELETE /api/v1/thing/3 (app/api/thing_3.py:109, auth_required=False) +- PATCH /api/v1/thing/4 (app/api/thing_4.py:112, auth_required=True) +- GET /api/v1/thing/5 (app/api/thing_5.py:115, auth_required=None) +- POST /api/v1/thing/6 (app/api/thing_6.py:118, auth_required=True) +- PUT /api/v1/thing/7 (app/api/thing_7.py:121, auth_required=False) +- DELETE /api/v1/thing/8 (app/api/thing_8.py:124, auth_required=None) +- PATCH /api/v1/thing/9 (app/api/thing_9.py:127, auth_required=False) + +TASK: +Find potential data exposure vulnerabilities and return HuntResult JSON. + +WORKFLOW (follow these steps in order): +1. Read the source files referenced in the RECON context above to understand the codebase. +2. Search for logging calls, error handlers, data persistence, HTTP clients, and configuration settings in the source code. +3. Read the identified files to confirm vulnerabilities with actual code evidence. +4. For each confirmed vulnerability, record the file path, line number, and relevant code snippet. +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +COVERAGE: +- CWE-200: information exposure (debug endpoints, verbose responses, server/internal metadata leakage) +- CWE-209: detailed error message leakage (stack traces, SQL errors, file paths, internal host/IP details) +- CWE-532: sensitive data in logs (PII, credentials, tokens, secrets, auth headers, session IDs) +- CWE-312: cleartext storage of sensitive data (passwords, tokens, PII persisted without encryption) +- CWE-319: cleartext transmission of sensitive data (HTTP transport for auth/session/PII or disabled TLS verification) + +FOCUS AREAS: +- Logging calls, logger middleware, and request/response logging filters +- Error handlers, exception formatters, debug/traceback output, API error payloads +- Persistence layer, model fields, data-at-rest handling, local file writes/caches/backups +- HTTP clients, webhook integrations, transport settings, TLS/SSL flags +- Configuration and deployment toggles that affect production exposure + +REQUIRED OUTPUT: +- Return JSON that strictly matches HuntResult. +- Populate findings with concrete evidence; chains may be empty. +- For each finding, fill all required RawFinding fields including: + - hunter_strategy = "data_exposure" + - cwe_id and cwe_name aligned to this strategy + - file_path, start_line, end_line, code_snippet + - estimated_severity and confidence + - fingerprint stable for file_path + start_line + cwe_id +- Distinguish production impact vs development-only behavior in description and confidence. +- Empty arrays are acceptable if no vulnerabilities are confirmed. + +DO NOT FLAG: +- Logging of non-sensitive operational data (request IDs, timestamps, non-sensitive metrics) +- Error verbosity that is clearly limited to development/local debug mode +- HTTPS URLs or TLS-enabled transmissions + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: JAVASCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec, new Function + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, parameterized pg queries + DO NOT FLAG: React JSX expressions (auto-escaped), Angular template bindings (sanitized by default) + Framework notes: Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous. + +Language: GO + Key sinks: fmt.Sprintf into SQL, exec.Command with user input, template.HTML(), os.Exec + Safe patterns (skip these): database/sql with ? placeholders, html/template (auto-escapes), exec.Command with separate args + DO NOT FLAG: GORM parameterized queries, html/template default escaping, database/sql prepared statements + Framework notes: Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML. + +Language: RUBY + Key sinks: eval, system, exec, send, public_send, ERB.new with user input + Safe patterns (skip these): ActiveRecord parameterized queries, Rack::Utils.escape_html, sanitize helper in Rails + DO NOT FLAG: ActiveRecord where with hash conditions, Rails CSRF protection, Rails html_safe on constants + Framework notes: Rails auto-escapes ERB templates. raw/html_safe bypasses escaping. Check for mass assignment. + +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +Framework: NEXTJS + Security features: React JSX auto-escapes output by default, API routes can share centralized auth middleware, Server/client boundaries reduce accidental secret exposure + DO NOT FLAG: React JSX escaped rendering, getServerSideProps/getServerSession with proper auth checks, Typed route handlers with validated schema guards + Watch for: dangerouslySetInnerHTML, unprotected API routes under /api, secret leakage to client bundles, rewrites/redirects from untrusted user input + Common vulns: XSS through dangerous HTML rendering, IDOR/auth bypass in API handlers, Sensitive env/config exposure in client-side code + +Framework: SPRING + Security features: Spring Security provides auth, CSRF, and filter chain defaults, JPA/Hibernate prepared parameter binding by default, Bean validation can enforce request constraints + DO NOT FLAG: PreparedStatement/JPA named parameter usage, Spring Security CSRF/auth filters clearly active, Thymeleaf auto-escaped output + Watch for: @PreAuthorize missing on sensitive methods, JdbcTemplate/raw Statement with concatenation, csrf().disable() on browser session flows, Actuator endpoints exposed without auth + Common vulns: Authz bypass from weak method-level security, SQL injection in raw JDBC queries, Sensitive management endpoint exposure + +Framework: REACT + Security features: JSX escapes strings before DOM rendering, Component model encourages explicit data flow, Framework discourages direct DOM mutation + DO NOT FLAG: Standard JSX expression rendering, textContent assignment for untrusted text, sanitized HTML via trusted DOMPurify policy + Watch for: dangerouslySetInnerHTML, untrusted URL assignment to href/src, eval/new Function in client code, token/secret exposure in bundles + Common vulns: DOM XSS through unsafe HTML injection, Open redirect via unvalidated navigation targets, Sensitive data exposure in frontend artifacts + +Framework: VUE + Security features: Mustache template interpolation escapes HTML, Component props/events provide explicit boundaries, Router guards can enforce auth flows + DO NOT FLAG: escaped template interpolation {{ value }}, validated route guards protecting private routes, sanitized content rendered through safe components + Watch for: v-html with untrusted data, dynamic component/template compilation, unsafe URL bindings in href/src, client-side auth checks without server enforcement + Common vulns: XSS through v-html and unsafe render paths, Auth bypass from client-only route protection, Open redirect patterns in router navigation + +CONSTRAINTS: +- Prefer evidence over assumptions; if uncertain, lower confidence. +- No markdown or code fences; JSON output only. +- Base all findings on actual file content you read. Do not speculate. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Strategy: data_exposure +- Early stop rule: if you inspect standard files without credible exposure risk, stop and return empty findings. +- Use multiple turns: inspect files first, then produce findings. +- Return final JSON only when analysis is complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_data_exposure_thorough.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_data_exposure_thorough.txt new file mode 100644 index 0000000000..4762c10e3f --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_data_exposure_thorough.txt @@ -0,0 +1,105 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are a data exposure security analyst for SEC-AF HUNT Phase. + +CONTEXT: +You are analyzing a real repository for Data Exposure Hunter. +Use the RECON context summary below to focus your investigation. + +RECON_CONTEXT: +Data exposure-focused recon summary. + +Data flows touching likely sensitive domains: 1 total, showing top 1: +- request.json -> cursor.execute; sanitized=False; files=app/api/users.py + +Logging/exposure-related misconfig signals: 1 total, showing top 1: +- debug at settings.py:9; risk=high; key=DEBUG + +Entry points and API surface with exposure risk: 2 total, showing top 2: +- POST /login (app/api/auth.py:12, auth_required=False) +- GET /users/{id} (app/api/users.py:40, auth_required=True) + +TASK: +Find potential data exposure vulnerabilities and return HuntResult JSON. + +WORKFLOW (follow these steps in order): +1. Read the source files referenced in the RECON context above to understand the codebase. +2. Search for logging calls, error handlers, data persistence, HTTP clients, and configuration settings in the source code. +3. Read the identified files to confirm vulnerabilities with actual code evidence. +4. For each confirmed vulnerability, record the file path, line number, and relevant code snippet. +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +COVERAGE: +- CWE-200: information exposure (debug endpoints, verbose responses, server/internal metadata leakage) +- CWE-209: detailed error message leakage (stack traces, SQL errors, file paths, internal host/IP details) +- CWE-532: sensitive data in logs (PII, credentials, tokens, secrets, auth headers, session IDs) +- CWE-312: cleartext storage of sensitive data (passwords, tokens, PII persisted without encryption) +- CWE-319: cleartext transmission of sensitive data (HTTP transport for auth/session/PII or disabled TLS verification) + +FOCUS AREAS: +- Logging calls, logger middleware, and request/response logging filters +- Error handlers, exception formatters, debug/traceback output, API error payloads +- Persistence layer, model fields, data-at-rest handling, local file writes/caches/backups +- HTTP clients, webhook integrations, transport settings, TLS/SSL flags +- Configuration and deployment toggles that affect production exposure + +REQUIRED OUTPUT: +- Return JSON that strictly matches HuntResult. +- Populate findings with concrete evidence; chains may be empty. +- For each finding, fill all required RawFinding fields including: + - hunter_strategy = "data_exposure" + - cwe_id and cwe_name aligned to this strategy + - file_path, start_line, end_line, code_snippet + - estimated_severity and confidence + - fingerprint stable for file_path + start_line + cwe_id +- Distinguish production impact vs development-only behavior in description and confidence. +- Empty arrays are acceptable if no vulnerabilities are confirmed. + +DO NOT FLAG: +- Logging of non-sensitive operational data (request IDs, timestamps, non-sensitive metrics) +- Error verbosity that is clearly limited to development/local debug mode +- HTTPS URLs or TLS-enabled transmissions + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +CONSTRAINTS: +- Prefer evidence over assumptions; if uncertain, lower confidence. +- No markdown or code fences; JSON output only. +- Base all findings on actual file content you read. Do not speculate. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Strategy: data_exposure +- Early stop rule: if you inspect thorough files without credible exposure risk, stop and return empty findings. +- Use multiple turns: inspect files first, then produce findings. +- Return final JSON only when analysis is complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_dos_quick.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_dos_quick.txt new file mode 100644 index 0000000000..88ff9e26b6 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_dos_quick.txt @@ -0,0 +1,143 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are an expert denial-of-service and resource exhaustion analyst for SEC-AF HUNT phase. + +CONTEXT: +Use the RECON context below to prioritize analysis by real entry points and known data flows. + +{ + "app_type": "web_api", + "auth_model": "jwt", + "frameworks": [ + "django" + ], + "languages": [ + "python" + ], + "entry_points": [ + { + "kind": "http", + "identifier": "POST /login", + "file_path": "app/api/auth.py", + "line": 12, + "method": "POST", + "route": "/login", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "seed", + "file_path": "app/cli.py", + "line": 3, + "method": null, + "route": null, + "auth_required": null + } + ], + "data_flows": [ + { + "source": "request.json", + "path": [ + { + "file_path": "app/api/users.py", + "line": 41, + "component": "handler", + "operation": "read id" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/api/users.py" + ] + } + ] +} + +TASK: +Systematically hunt for DoS and resource exhaustion vulnerabilities in this codebase. + +WORKFLOW (follow these steps in order): +1. Read the source files referenced in the RECON context above to understand the codebase. +2. Search for unbounded loops, regex patterns, resource allocation, pagination gaps, and expensive operations in the source code. +3. Read the identified files to confirm vulnerabilities with actual code evidence. +4. For each confirmed vulnerability, record the file path, line number, and relevant code snippet. +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +SCOPE: +- Prioritize attacker-reachable code paths from documented entry points and API surfaces. +- Trace request handling, query building, parsing, recursion, and expensive operations for unbounded work. +- Cover these classes: + - CWE-400 Uncontrolled Resource Consumption + - CWE-770 Allocation of Resources Without Limits or Throttling + - CWE-1333 ReDoS (Inefficient Regular Expression Complexity) + - CWE-835 Loop with Unreachable Exit Condition (Infinite Loop) +- Hunt for: + - Missing pagination or hard limits on list/index endpoints + - Unbounded DB queries/scans and expensive operations without request-level limits + - Regex patterns vulnerable to catastrophic backtracking on user-controlled input + - Missing rate limiting on expensive operations + - Recursive structures without depth limits + - Unbounded file upload or payload size handling + +OUTPUT: +- Return JSON that strictly matches HuntResult. +- Populate HuntResult.findings with RawFinding entries focused on DoS/resource exhaustion issues. +- For each finding, include title, description, cwe_id, cwe_name, file_path, start_line, end_line, + code_snippet, estimated_severity, confidence, fingerprint, and related_files. +- Include data_flow when you can trace attacker control to expensive resource consumption. +- Set hunter_strategy to "dos" for each finding. +- Keep HuntResult metadata coherent (total_raw, deduplicated_count, chain_count, strategies_run, + hunt_duration_seconds). +- Empty arrays are acceptable if no vulnerabilities are confirmed. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +CONSTRAINTS: +- Prefer high confidence only when unbounded or weakly bounded behavior is proven in code. +- Do not speculate about operational outages without code evidence. +- Do not include markdown, prose outside JSON, or code fences in final output. +- Base all findings on actual file content you read. Do not speculate. +- DO NOT FLAG: + - Rate limiting enforced at gateway or middleware + - Endpoints that already enforce pagination or explicit query limits + - Recursive code with validated depth bounds + - Resource limits clearly enforced by container or infrastructure controls + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Depth profile: quick +- Early stop rule: if you inspect 30 files without credible signal, stop and return empty findings. +- Focus on RECON entry points and data flows where unbounded work can be attacker-controlled. +- Explore code paths that can trigger excessive CPU, memory, I/O, or external-service consumption. +- Take multiple turns to build findings incrementally and write final JSON only when complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_dos_standard.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_dos_standard.txt new file mode 100644 index 0000000000..803524cb01 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_dos_standard.txt @@ -0,0 +1,489 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are an expert denial-of-service and resource exhaustion analyst for SEC-AF HUNT phase. + +CONTEXT: +Use the RECON context below to prioritize analysis by real entry points and known data flows. + +{ + "app_type": "web_api", + "auth_model": "jwt", + "frameworks": [ + "Django", + "next.js", + "Spring Boot", + " React ", + "unknown-fw", + "NEXT", + "Vue" + ], + "languages": [ + "Python", + "JavaScript", + "Go", + "Ruby", + "Rust", + "python" + ], + "entry_points": [ + { + "kind": "http", + "identifier": "handler_0", + "file_path": "app/entry/e0.py", + "line": 10, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "HTTP", + "identifier": "handler_1", + "file_path": "app/entry/e1.py", + "line": 17, + "method": "POST", + "route": "/v1/resource/1", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_2", + "file_path": "app/entry/e2.py", + "line": 24, + "method": null, + "route": "/v1/resource/2", + "auth_required": null + }, + { + "kind": "api", + "identifier": "handler_3", + "file_path": "app/entry/e3.py", + "line": 31, + "method": "PUT", + "route": null, + "auth_required": true + }, + { + "kind": "graphql", + "identifier": "handler_4", + "file_path": "app/entry/e4.py", + "line": 38, + "method": "GET", + "route": "/v1/resource/4", + "auth_required": false + }, + { + "kind": "rpc", + "identifier": "handler_5", + "file_path": "app/entry/e5.py", + "line": 45, + "method": "POST", + "route": "/v1/resource/5", + "auth_required": null + }, + { + "kind": "route", + "identifier": "handler_6", + "file_path": "app/entry/e6.py", + "line": 52, + "method": null, + "route": null, + "auth_required": true + }, + { + "kind": "cron", + "identifier": "handler_7", + "file_path": "app/entry/e7.py", + "line": 59, + "method": "PUT", + "route": "/v1/resource/7", + "auth_required": false + }, + { + "kind": "queue", + "identifier": "handler_8", + "file_path": "app/entry/e8.py", + "line": 66, + "method": "GET", + "route": "/v1/resource/8", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_9", + "file_path": "app/entry/e9.py", + "line": 73, + "method": "POST", + "route": null, + "auth_required": true + } + ], + "data_flows": [ + { + "source": "request.args['q']", + "path": [ + { + "file_path": "app/search/index.ts", + "line": 20, + "component": "step_0_a", + "operation": "read" + }, + { + "file_path": "app/util/b.py", + "line": 40, + "component": "step_0_b", + "operation": "write" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/search/index.ts", + "app/db/raw.py", + "app/util/a.py", + "app/util/b.py" + ] + }, + { + "source": "request.cookies['session']", + "path": [ + { + "file_path": "app/auth/session_store.py", + "line": 21, + "component": "step_1_a", + "operation": "read" + }, + { + "file_path": "app/auth/session_store.py", + "line": 41, + "component": "step_1_b", + "operation": "write" + } + ], + "sink": "redis.set", + "sanitized": true, + "files": [ + "app/auth/session_store.py" + ] + }, + { + "source": "request.json['url']", + "path": [ + { + "file_path": "app/integrations/webhooks.go", + "line": 22, + "component": "step_2_a", + "operation": "read" + }, + { + "file_path": "app/net/client.go", + "line": 42, + "component": "step_2_b", + "operation": "write" + } + ], + "sink": "requests.get", + "sanitized": false, + "files": [ + "app/integrations/webhooks.go", + "app/net/client.go" + ] + }, + { + "source": "form['password']", + "path": [ + { + "file_path": "app/auth/service.py", + "line": 23, + "component": "step_3_a", + "operation": "read" + }, + { + "file_path": "app/obs/telemetry.go", + "line": 43, + "component": "step_3_b", + "operation": "write" + } + ], + "sink": "logger.info", + "sanitized": false, + "files": [ + "app/auth/service.py", + "app/obs/telemetry.go" + ] + }, + { + "source": "header['Authorization']", + "path": [ + { + "file_path": "app/common/jwt_tools.py", + "line": 24, + "component": "step_4_a", + "operation": "read" + }, + { + "file_path": "app/common/jwt_tools.py", + "line": 44, + "component": "step_4_b", + "operation": "write" + } + ], + "sink": "jwt.decode", + "sanitized": true, + "files": [ + "app/common/jwt_tools.py" + ] + }, + { + "source": "query['email']", + "path": [ + { + "file_path": "app/notify/mailer.rb", + "line": 25, + "component": "step_5_a", + "operation": "read" + }, + { + "file_path": "app/notify/mailer.rb", + "line": 45, + "component": "step_5_b", + "operation": "write" + } + ], + "sink": "smtp.send", + "sanitized": false, + "files": [ + "app/notify/mailer.rb" + ] + }, + { + "source": "body['card_number']", + "path": [ + { + "file_path": "app/billing/payments.go", + "line": 26, + "component": "step_6_a", + "operation": "read" + }, + { + "file_path": "app/billing/core.py", + "line": 46, + "component": "step_6_b", + "operation": "write" + } + ], + "sink": "stripe.Charge.create", + "sanitized": true, + "files": [ + "app/billing/payments.go", + "app/billing/core.py" + ] + }, + { + "source": "path_param['id']", + "path": [ + { + "file_path": "app/api/thing_1.py", + "line": 27, + "component": "step_7_a", + "operation": "read" + }, + { + "file_path": "app/api/thing_1.py", + "line": 47, + "component": "step_7_b", + "operation": "write" + } + ], + "sink": "orm.filter", + "sanitized": true, + "files": [ + "app/api/thing_1.py" + ] + }, + { + "source": "upload.filename", + "path": [ + { + "file_path": "app/media/uploads.py", + "line": 28, + "component": "step_8_a", + "operation": "read" + }, + { + "file_path": "app/x/w.py", + "line": 48, + "component": "step_8_b", + "operation": "write" + } + ], + "sink": "open", + "sanitized": false, + "files": [ + "app/media/uploads.py", + "app/common/cache.py", + "app/common/guard.py", + "app/x/y.py", + "app/x/z.py", + "app/x/w.py" + ] + }, + { + "source": "env['DEBUG']", + "path": [ + { + "file_path": "app/web/render.py", + "line": 29, + "component": "step_9_a", + "operation": "read" + }, + { + "file_path": "app/web/render.py", + "line": 49, + "component": "step_9_b", + "operation": "write" + } + ], + "sink": "template.render", + "sanitized": false, + "files": [ + "app/web/render.py" + ] + } + ] +} + +TASK: +Systematically hunt for DoS and resource exhaustion vulnerabilities in this codebase. + +WORKFLOW (follow these steps in order): +1. Read the source files referenced in the RECON context above to understand the codebase. +2. Search for unbounded loops, regex patterns, resource allocation, pagination gaps, and expensive operations in the source code. +3. Read the identified files to confirm vulnerabilities with actual code evidence. +4. For each confirmed vulnerability, record the file path, line number, and relevant code snippet. +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +SCOPE: +- Prioritize attacker-reachable code paths from documented entry points and API surfaces. +- Trace request handling, query building, parsing, recursion, and expensive operations for unbounded work. +- Cover these classes: + - CWE-400 Uncontrolled Resource Consumption + - CWE-770 Allocation of Resources Without Limits or Throttling + - CWE-1333 ReDoS (Inefficient Regular Expression Complexity) + - CWE-835 Loop with Unreachable Exit Condition (Infinite Loop) +- Hunt for: + - Missing pagination or hard limits on list/index endpoints + - Unbounded DB queries/scans and expensive operations without request-level limits + - Regex patterns vulnerable to catastrophic backtracking on user-controlled input + - Missing rate limiting on expensive operations + - Recursive structures without depth limits + - Unbounded file upload or payload size handling + +OUTPUT: +- Return JSON that strictly matches HuntResult. +- Populate HuntResult.findings with RawFinding entries focused on DoS/resource exhaustion issues. +- For each finding, include title, description, cwe_id, cwe_name, file_path, start_line, end_line, + code_snippet, estimated_severity, confidence, fingerprint, and related_files. +- Include data_flow when you can trace attacker control to expensive resource consumption. +- Set hunter_strategy to "dos" for each finding. +- Keep HuntResult metadata coherent (total_raw, deduplicated_count, chain_count, strategies_run, + hunt_duration_seconds). +- Empty arrays are acceptable if no vulnerabilities are confirmed. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: JAVASCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec, new Function + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, parameterized pg queries + DO NOT FLAG: React JSX expressions (auto-escaped), Angular template bindings (sanitized by default) + Framework notes: Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous. + +Language: GO + Key sinks: fmt.Sprintf into SQL, exec.Command with user input, template.HTML(), os.Exec + Safe patterns (skip these): database/sql with ? placeholders, html/template (auto-escapes), exec.Command with separate args + DO NOT FLAG: GORM parameterized queries, html/template default escaping, database/sql prepared statements + Framework notes: Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML. + +Language: RUBY + Key sinks: eval, system, exec, send, public_send, ERB.new with user input + Safe patterns (skip these): ActiveRecord parameterized queries, Rack::Utils.escape_html, sanitize helper in Rails + DO NOT FLAG: ActiveRecord where with hash conditions, Rails CSRF protection, Rails html_safe on constants + Framework notes: Rails auto-escapes ERB templates. raw/html_safe bypasses escaping. Check for mass assignment. + +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +Framework: NEXTJS + Security features: React JSX auto-escapes output by default, API routes can share centralized auth middleware, Server/client boundaries reduce accidental secret exposure + DO NOT FLAG: React JSX escaped rendering, getServerSideProps/getServerSession with proper auth checks, Typed route handlers with validated schema guards + Watch for: dangerouslySetInnerHTML, unprotected API routes under /api, secret leakage to client bundles, rewrites/redirects from untrusted user input + Common vulns: XSS through dangerous HTML rendering, IDOR/auth bypass in API handlers, Sensitive env/config exposure in client-side code + +Framework: SPRING + Security features: Spring Security provides auth, CSRF, and filter chain defaults, JPA/Hibernate prepared parameter binding by default, Bean validation can enforce request constraints + DO NOT FLAG: PreparedStatement/JPA named parameter usage, Spring Security CSRF/auth filters clearly active, Thymeleaf auto-escaped output + Watch for: @PreAuthorize missing on sensitive methods, JdbcTemplate/raw Statement with concatenation, csrf().disable() on browser session flows, Actuator endpoints exposed without auth + Common vulns: Authz bypass from weak method-level security, SQL injection in raw JDBC queries, Sensitive management endpoint exposure + +Framework: REACT + Security features: JSX escapes strings before DOM rendering, Component model encourages explicit data flow, Framework discourages direct DOM mutation + DO NOT FLAG: Standard JSX expression rendering, textContent assignment for untrusted text, sanitized HTML via trusted DOMPurify policy + Watch for: dangerouslySetInnerHTML, untrusted URL assignment to href/src, eval/new Function in client code, token/secret exposure in bundles + Common vulns: DOM XSS through unsafe HTML injection, Open redirect via unvalidated navigation targets, Sensitive data exposure in frontend artifacts + +Framework: VUE + Security features: Mustache template interpolation escapes HTML, Component props/events provide explicit boundaries, Router guards can enforce auth flows + DO NOT FLAG: escaped template interpolation {{ value }}, validated route guards protecting private routes, sanitized content rendered through safe components + Watch for: v-html with untrusted data, dynamic component/template compilation, unsafe URL bindings in href/src, client-side auth checks without server enforcement + Common vulns: XSS through v-html and unsafe render paths, Auth bypass from client-only route protection, Open redirect patterns in router navigation + +CONSTRAINTS: +- Prefer high confidence only when unbounded or weakly bounded behavior is proven in code. +- Do not speculate about operational outages without code evidence. +- Do not include markdown, prose outside JSON, or code fences in final output. +- Base all findings on actual file content you read. Do not speculate. +- DO NOT FLAG: + - Rate limiting enforced at gateway or middleware + - Endpoints that already enforce pagination or explicit query limits + - Recursive code with validated depth bounds + - Resource limits clearly enforced by container or infrastructure controls + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Depth profile: standard +- Early stop rule: if you inspect 30 files without credible signal, stop and return empty findings. +- Focus on RECON entry points and data flows where unbounded work can be attacker-controlled. +- Explore code paths that can trigger excessive CPU, memory, I/O, or external-service consumption. +- Take multiple turns to build findings incrementally and write final JSON only when complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_dos_thorough.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_dos_thorough.txt new file mode 100644 index 0000000000..969f659240 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_dos_thorough.txt @@ -0,0 +1,143 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are an expert denial-of-service and resource exhaustion analyst for SEC-AF HUNT phase. + +CONTEXT: +Use the RECON context below to prioritize analysis by real entry points and known data flows. + +{ + "app_type": "web_api", + "auth_model": "jwt", + "frameworks": [ + "django" + ], + "languages": [ + "python" + ], + "entry_points": [ + { + "kind": "http", + "identifier": "POST /login", + "file_path": "app/api/auth.py", + "line": 12, + "method": "POST", + "route": "/login", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "seed", + "file_path": "app/cli.py", + "line": 3, + "method": null, + "route": null, + "auth_required": null + } + ], + "data_flows": [ + { + "source": "request.json", + "path": [ + { + "file_path": "app/api/users.py", + "line": 41, + "component": "handler", + "operation": "read id" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/api/users.py" + ] + } + ] +} + +TASK: +Systematically hunt for DoS and resource exhaustion vulnerabilities in this codebase. + +WORKFLOW (follow these steps in order): +1. Read the source files referenced in the RECON context above to understand the codebase. +2. Search for unbounded loops, regex patterns, resource allocation, pagination gaps, and expensive operations in the source code. +3. Read the identified files to confirm vulnerabilities with actual code evidence. +4. For each confirmed vulnerability, record the file path, line number, and relevant code snippet. +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +SCOPE: +- Prioritize attacker-reachable code paths from documented entry points and API surfaces. +- Trace request handling, query building, parsing, recursion, and expensive operations for unbounded work. +- Cover these classes: + - CWE-400 Uncontrolled Resource Consumption + - CWE-770 Allocation of Resources Without Limits or Throttling + - CWE-1333 ReDoS (Inefficient Regular Expression Complexity) + - CWE-835 Loop with Unreachable Exit Condition (Infinite Loop) +- Hunt for: + - Missing pagination or hard limits on list/index endpoints + - Unbounded DB queries/scans and expensive operations without request-level limits + - Regex patterns vulnerable to catastrophic backtracking on user-controlled input + - Missing rate limiting on expensive operations + - Recursive structures without depth limits + - Unbounded file upload or payload size handling + +OUTPUT: +- Return JSON that strictly matches HuntResult. +- Populate HuntResult.findings with RawFinding entries focused on DoS/resource exhaustion issues. +- For each finding, include title, description, cwe_id, cwe_name, file_path, start_line, end_line, + code_snippet, estimated_severity, confidence, fingerprint, and related_files. +- Include data_flow when you can trace attacker control to expensive resource consumption. +- Set hunter_strategy to "dos" for each finding. +- Keep HuntResult metadata coherent (total_raw, deduplicated_count, chain_count, strategies_run, + hunt_duration_seconds). +- Empty arrays are acceptable if no vulnerabilities are confirmed. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +CONSTRAINTS: +- Prefer high confidence only when unbounded or weakly bounded behavior is proven in code. +- Do not speculate about operational outages without code evidence. +- Do not include markdown, prose outside JSON, or code fences in final output. +- Base all findings on actual file content you read. Do not speculate. +- DO NOT FLAG: + - Rate limiting enforced at gateway or middleware + - Endpoints that already enforce pagination or explicit query limits + - Recursive code with validated depth bounds + - Resource limits clearly enforced by container or infrastructure controls + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Depth profile: thorough +- Early stop rule: if you inspect 30 files without credible signal, stop and return empty findings. +- Focus on RECON entry points and data flows where unbounded work can be attacker-controlled. +- Explore code paths that can trigger excessive CPU, memory, I/O, or external-service consumption. +- Take multiple turns to build findings incrementally and write final JSON only when complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_injection_quick.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_injection_quick.txt new file mode 100644 index 0000000000..b18af63301 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_injection_quick.txt @@ -0,0 +1,106 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are an expert injection vulnerability analyst for SEC-AF HUNT phase. + +CONTEXT: +Use the RECON context below to prioritize analysis by real entry points and known data flows. + +Injection-focused recon summary. + +Codebase profile: 56 files, 1234 LOC, languages=python, frameworks=django. + +Entry points likely to receive untrusted input: 2 total, showing top 2: +- http /login (app/api/auth.py:12) +- cli seed (app/cli.py:3) + +High-value sinks: 1 total, showing top 1: +- sql at app/api/users.py:42 (get_user) + +Source-to-sink flow candidates (unsanitized first): 1 total, showing top 1: +- request.json -> cursor.execute; sanitized=False; files=app/api/users.py + +Known sanitization points: none identified in recon. + +TASK: +Systematically hunt for injection vulnerabilities in this codebase. + +WORKFLOW (follow these steps in order): +1. Read the source files referenced in the RECON context above to understand the codebase. +2. Search for SQL queries, command execution, template rendering, and other injection sinks in the source code. +3. Read the identified files to confirm vulnerabilities with actual code evidence. +4. For each confirmed vulnerability, record the file path, line number, and relevant code snippet. +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +SCOPE: +- Prioritize source-to-sink paths that originate from documented entry points. +- Trace data flow across files and layers to identify missing validation, sanitization, or safe encoding. +- Cover these classes: + - CWE-78 OS Command Injection + - CWE-79 Cross-Site Scripting (XSS) + - CWE-89 SQL Injection + - CWE-90 LDAP Injection + - CWE-91 XML/XPath Injection + - CWE-94 Code Injection + - CWE-917 Expression Language / Template Injection + +OUTPUT: +- Return JSON that strictly matches HuntResult. +- Populate HuntResult.findings with RawFinding entries focused on injection issues. +- For each finding, include title, description, cwe_id, cwe_name, file_path, start_line, end_line, + code_snippet, estimated_severity, confidence, fingerprint, and related_files. +- Include data_flow when you can trace user-controlled source to vulnerable sink. +- Set hunter_strategy to "injection" for each finding. +- Keep HuntResult metadata coherent (total_raw, deduplicated_count, chain_count, strategies_run, + hunt_duration_seconds). +- Empty arrays are acceptable if no vulnerabilities are confirmed. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +CONSTRAINTS: +- Prefer high confidence only when source-to-sink flow is clearly traced. +- Do not speculate about exploitability without code evidence. +- Do not include markdown, prose outside JSON, or code fences in final output. +- Base all findings on actual file content you read. Do not speculate. +- DO NOT FLAG: + - Parameterized queries (for example cursor.execute("SELECT ... WHERE id = %s", [user_id])) + - ORM operations that safely bind parameters (Django ORM, SQLAlchemy bound params, Prisma) + - Auto-escaped output contexts (Django templates default escaping, React JSX, Vue template escaping) + - Test fixtures, synthetic payload examples, and comments that only demonstrate attack strings + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Depth profile: quick +- Early stop rule: if you inspect 30 files without credible signal, stop and return empty findings. +- Focus on RECON entry points and data flows as primary source-to-sink paths. +- Explore the codebase, trace data flows from sources to sinks, and identify injection points. +- Take multiple turns to build findings incrementally and write final JSON only when complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_injection_standard.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_injection_standard.txt new file mode 100644 index 0000000000..9748c4b0f2 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_injection_standard.txt @@ -0,0 +1,203 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are an expert injection vulnerability analyst for SEC-AF HUNT phase. + +CONTEXT: +Use the RECON context below to prioritize analysis by real entry points and known data flows. + +Injection-focused recon summary. + +Codebase profile: 2417 files, 184213 LOC, languages=Python, JavaScript, Go, Ruby, Rust, python, frameworks=Django, next.js, Spring Boot, React , unknown-fw, NEXT, Vue. + +Entry points likely to receive untrusted input: 17 total, showing top 15: +- http handler_0 (app/entry/e0.py:10) +- HTTP /v1/resource/1 (app/entry/e1.py:17) +- cli /v1/resource/2 (app/entry/e2.py:24) +- api handler_3 (app/entry/e3.py:31) +- graphql /v1/resource/4 (app/entry/e4.py:38) +- rpc /v1/resource/5 (app/entry/e5.py:45) +- route handler_6 (app/entry/e6.py:52) +- cron /v1/resource/7 (app/entry/e7.py:59) +- queue /v1/resource/8 (app/entry/e8.py:66) +- http handler_9 (app/entry/e9.py:73) +- api /v1/resource/10 (app/entry/e10.py:80) +- grpc /v1/resource/11 (app/entry/e11.py:87) +- http handler_12 (app/entry/e12.py:94) +- route /v1/resource/13 (app/entry/e13.py:101) +- cli /v1/resource/14 (app/entry/e14.py:108) + +High-value sinks: 18 total, showing top 15: +- sql at app/sink/k0.py:200 +- command at app/sink/k1.py:202 (sink_fn_1) +- template at app/sink/k2.py:204 (sink_fn_2) +- file at app/sink/k3.py:206 (sink_fn_3) +- network at app/sink/k4.py:208 +- deserialization at app/sink/k5.py:210 (sink_fn_5) +- sql at app/sink/k6.py:212 (sink_fn_6) +- command at app/sink/k7.py:214 (sink_fn_7) +- template at app/sink/k8.py:216 +- file at app/sink/k9.py:218 (sink_fn_9) +- network at app/sink/k10.py:220 (sink_fn_10) +- deserialization at app/sink/k11.py:222 (sink_fn_11) +- sql at app/sink/k12.py:224 +- command at app/sink/k13.py:226 (sink_fn_13) +- template at app/sink/k14.py:228 (sink_fn_14) + +Source-to-sink flow candidates (unsanitized first): 13 total, showing top 13: +- request.args['q'] -> cursor.execute; sanitized=False; files=app/search/index.ts, app/db/raw.py, app/util/a.py +- request.json['url'] -> requests.get; sanitized=False; files=app/integrations/webhooks.go, app/net/client.go +- form['password'] -> logger.info; sanitized=False; files=app/auth/service.py, app/obs/telemetry.go +- query['email'] -> smtp.send; sanitized=False; files=app/notify/mailer.rb +- upload.filename -> open; sanitized=False; files=app/media/uploads.py, app/common/cache.py, app/common/guard.py +- env['DEBUG'] -> template.render; sanitized=False; files=app/web/render.py +- request.body -> eval; sanitized=False; files=app/admin/views.py +- websocket.msg -> subprocess.run; sanitized=False; files=app/ws/handler.py, app/ws/exec.py +- cli.argv -> os.system; sanitized=False; files=tools/run.py +- queue.payload -> pickle.loads; sanitized=False; files=app/worker/task.py +- request.args['redirect'] -> HttpResponseRedirect; sanitized=False; files=app/web/redirect.py +- request.headers['X-User-Phone'] -> audit_log; sanitized=False; files=app/obs/audit.py +- graphql.variables -> db.query; sanitized=False; files=app/api/graphql.ts + +Known sanitization points: 12 total, showing top 10: +- app/sanitize/s0.py:5 type=escape protects=unspecified +- app/sanitize/s1.py:9 type=parameterize protects=CWE-80, CWE-90 +- app/sanitize/s2.py:13 type=allowlist protects=CWE-81, CWE-91 +- app/sanitize/s3.py:17 type=encode protects=unspecified +- app/sanitize/s4.py:21 type=escape protects=CWE-83, CWE-93 +- app/sanitize/s5.py:25 type=parameterize protects=CWE-84, CWE-94 +- app/sanitize/s6.py:29 type=allowlist protects=unspecified +- app/sanitize/s7.py:33 type=encode protects=CWE-86, CWE-96 +- app/sanitize/s8.py:37 type=escape protects=CWE-87, CWE-97 +- app/sanitize/s9.py:41 type=parameterize protects=unspecified + +TASK: +Systematically hunt for injection vulnerabilities in this codebase. + +WORKFLOW (follow these steps in order): +1. Read the source files referenced in the RECON context above to understand the codebase. +2. Search for SQL queries, command execution, template rendering, and other injection sinks in the source code. +3. Read the identified files to confirm vulnerabilities with actual code evidence. +4. For each confirmed vulnerability, record the file path, line number, and relevant code snippet. +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +SCOPE: +- Prioritize source-to-sink paths that originate from documented entry points. +- Trace data flow across files and layers to identify missing validation, sanitization, or safe encoding. +- Cover these classes: + - CWE-78 OS Command Injection + - CWE-79 Cross-Site Scripting (XSS) + - CWE-89 SQL Injection + - CWE-90 LDAP Injection + - CWE-91 XML/XPath Injection + - CWE-94 Code Injection + - CWE-917 Expression Language / Template Injection + +OUTPUT: +- Return JSON that strictly matches HuntResult. +- Populate HuntResult.findings with RawFinding entries focused on injection issues. +- For each finding, include title, description, cwe_id, cwe_name, file_path, start_line, end_line, + code_snippet, estimated_severity, confidence, fingerprint, and related_files. +- Include data_flow when you can trace user-controlled source to vulnerable sink. +- Set hunter_strategy to "injection" for each finding. +- Keep HuntResult metadata coherent (total_raw, deduplicated_count, chain_count, strategies_run, + hunt_duration_seconds). +- Empty arrays are acceptable if no vulnerabilities are confirmed. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: JAVASCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec, new Function + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, parameterized pg queries + DO NOT FLAG: React JSX expressions (auto-escaped), Angular template bindings (sanitized by default) + Framework notes: Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous. + +Language: GO + Key sinks: fmt.Sprintf into SQL, exec.Command with user input, template.HTML(), os.Exec + Safe patterns (skip these): database/sql with ? placeholders, html/template (auto-escapes), exec.Command with separate args + DO NOT FLAG: GORM parameterized queries, html/template default escaping, database/sql prepared statements + Framework notes: Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML. + +Language: RUBY + Key sinks: eval, system, exec, send, public_send, ERB.new with user input + Safe patterns (skip these): ActiveRecord parameterized queries, Rack::Utils.escape_html, sanitize helper in Rails + DO NOT FLAG: ActiveRecord where with hash conditions, Rails CSRF protection, Rails html_safe on constants + Framework notes: Rails auto-escapes ERB templates. raw/html_safe bypasses escaping. Check for mass assignment. + +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +Framework: NEXTJS + Security features: React JSX auto-escapes output by default, API routes can share centralized auth middleware, Server/client boundaries reduce accidental secret exposure + DO NOT FLAG: React JSX escaped rendering, getServerSideProps/getServerSession with proper auth checks, Typed route handlers with validated schema guards + Watch for: dangerouslySetInnerHTML, unprotected API routes under /api, secret leakage to client bundles, rewrites/redirects from untrusted user input + Common vulns: XSS through dangerous HTML rendering, IDOR/auth bypass in API handlers, Sensitive env/config exposure in client-side code + +Framework: SPRING + Security features: Spring Security provides auth, CSRF, and filter chain defaults, JPA/Hibernate prepared parameter binding by default, Bean validation can enforce request constraints + DO NOT FLAG: PreparedStatement/JPA named parameter usage, Spring Security CSRF/auth filters clearly active, Thymeleaf auto-escaped output + Watch for: @PreAuthorize missing on sensitive methods, JdbcTemplate/raw Statement with concatenation, csrf().disable() on browser session flows, Actuator endpoints exposed without auth + Common vulns: Authz bypass from weak method-level security, SQL injection in raw JDBC queries, Sensitive management endpoint exposure + +Framework: REACT + Security features: JSX escapes strings before DOM rendering, Component model encourages explicit data flow, Framework discourages direct DOM mutation + DO NOT FLAG: Standard JSX expression rendering, textContent assignment for untrusted text, sanitized HTML via trusted DOMPurify policy + Watch for: dangerouslySetInnerHTML, untrusted URL assignment to href/src, eval/new Function in client code, token/secret exposure in bundles + Common vulns: DOM XSS through unsafe HTML injection, Open redirect via unvalidated navigation targets, Sensitive data exposure in frontend artifacts + +Framework: VUE + Security features: Mustache template interpolation escapes HTML, Component props/events provide explicit boundaries, Router guards can enforce auth flows + DO NOT FLAG: escaped template interpolation {{ value }}, validated route guards protecting private routes, sanitized content rendered through safe components + Watch for: v-html with untrusted data, dynamic component/template compilation, unsafe URL bindings in href/src, client-side auth checks without server enforcement + Common vulns: XSS through v-html and unsafe render paths, Auth bypass from client-only route protection, Open redirect patterns in router navigation + +CONSTRAINTS: +- Prefer high confidence only when source-to-sink flow is clearly traced. +- Do not speculate about exploitability without code evidence. +- Do not include markdown, prose outside JSON, or code fences in final output. +- Base all findings on actual file content you read. Do not speculate. +- DO NOT FLAG: + - Parameterized queries (for example cursor.execute("SELECT ... WHERE id = %s", [user_id])) + - ORM operations that safely bind parameters (Django ORM, SQLAlchemy bound params, Prisma) + - Auto-escaped output contexts (Django templates default escaping, React JSX, Vue template escaping) + - Test fixtures, synthetic payload examples, and comments that only demonstrate attack strings + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Depth profile: standard +- Early stop rule: if you inspect 30 files without credible signal, stop and return empty findings. +- Focus on RECON entry points and data flows as primary source-to-sink paths. +- Explore the codebase, trace data flows from sources to sinks, and identify injection points. +- Take multiple turns to build findings incrementally and write final JSON only when complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_injection_thorough.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_injection_thorough.txt new file mode 100644 index 0000000000..752cfcfdfa --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_injection_thorough.txt @@ -0,0 +1,106 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are an expert injection vulnerability analyst for SEC-AF HUNT phase. + +CONTEXT: +Use the RECON context below to prioritize analysis by real entry points and known data flows. + +Injection-focused recon summary. + +Codebase profile: 56 files, 1234 LOC, languages=python, frameworks=django. + +Entry points likely to receive untrusted input: 2 total, showing top 2: +- http /login (app/api/auth.py:12) +- cli seed (app/cli.py:3) + +High-value sinks: 1 total, showing top 1: +- sql at app/api/users.py:42 (get_user) + +Source-to-sink flow candidates (unsanitized first): 1 total, showing top 1: +- request.json -> cursor.execute; sanitized=False; files=app/api/users.py + +Known sanitization points: none identified in recon. + +TASK: +Systematically hunt for injection vulnerabilities in this codebase. + +WORKFLOW (follow these steps in order): +1. Read the source files referenced in the RECON context above to understand the codebase. +2. Search for SQL queries, command execution, template rendering, and other injection sinks in the source code. +3. Read the identified files to confirm vulnerabilities with actual code evidence. +4. For each confirmed vulnerability, record the file path, line number, and relevant code snippet. +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +SCOPE: +- Prioritize source-to-sink paths that originate from documented entry points. +- Trace data flow across files and layers to identify missing validation, sanitization, or safe encoding. +- Cover these classes: + - CWE-78 OS Command Injection + - CWE-79 Cross-Site Scripting (XSS) + - CWE-89 SQL Injection + - CWE-90 LDAP Injection + - CWE-91 XML/XPath Injection + - CWE-94 Code Injection + - CWE-917 Expression Language / Template Injection + +OUTPUT: +- Return JSON that strictly matches HuntResult. +- Populate HuntResult.findings with RawFinding entries focused on injection issues. +- For each finding, include title, description, cwe_id, cwe_name, file_path, start_line, end_line, + code_snippet, estimated_severity, confidence, fingerprint, and related_files. +- Include data_flow when you can trace user-controlled source to vulnerable sink. +- Set hunter_strategy to "injection" for each finding. +- Keep HuntResult metadata coherent (total_raw, deduplicated_count, chain_count, strategies_run, + hunt_duration_seconds). +- Empty arrays are acceptable if no vulnerabilities are confirmed. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +CONSTRAINTS: +- Prefer high confidence only when source-to-sink flow is clearly traced. +- Do not speculate about exploitability without code evidence. +- Do not include markdown, prose outside JSON, or code fences in final output. +- Base all findings on actual file content you read. Do not speculate. +- DO NOT FLAG: + - Parameterized queries (for example cursor.execute("SELECT ... WHERE id = %s", [user_id])) + - ORM operations that safely bind parameters (Django ORM, SQLAlchemy bound params, Prisma) + - Auto-escaped output contexts (Django templates default escaping, React JSX, Vue template escaping) + - Test fixtures, synthetic payload examples, and comments that only demonstrate attack strings + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Depth profile: thorough +- Early stop rule: if you inspect 30 files without credible signal, stop and return empty findings. +- Focus on RECON entry points and data flows as primary source-to-sink paths. +- Explore the codebase, trace data flows from sources to sinks, and identify injection points. +- Take multiple turns to build findings incrementally and write final JSON only when complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_ssrf_quick.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_ssrf_quick.txt new file mode 100644 index 0000000000..7614e34491 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_ssrf_quick.txt @@ -0,0 +1,139 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are an expert SSRF vulnerability analyst for SEC-AF HUNT phase. + +CONTEXT: +Use the RECON context below to prioritize analysis by real entry points and known data flows. + +{ + "app_type": "web_api", + "auth_model": "jwt", + "frameworks": [ + "django" + ], + "languages": [ + "python" + ], + "entry_points": [ + { + "kind": "http", + "identifier": "POST /login", + "file_path": "app/api/auth.py", + "line": 12, + "method": "POST", + "route": "/login", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "seed", + "file_path": "app/cli.py", + "line": 3, + "method": null, + "route": null, + "auth_required": null + } + ], + "data_flows": [ + { + "source": "request.json", + "path": [ + { + "file_path": "app/api/users.py", + "line": 41, + "component": "handler", + "operation": "read id" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/api/users.py" + ] + } + ] +} + +TASK: +Systematically hunt for server-side request forgery vulnerabilities in this codebase. + +WORKFLOW (follow these steps in order): +1. Read the source files referenced in the RECON context above to understand the codebase. +2. Search for HTTP clients, URL handling, webhook integrations, and outbound network calls in the source code. +3. Read the identified files to confirm vulnerabilities with actual code evidence. +4. For each confirmed vulnerability, record the file path, line number, and relevant code snippet. +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +SCOPE: +- Prioritize source-to-sink paths that originate from documented entry points. +- Trace data flow across files and layers to identify user-controlled URL input reaching outbound network sinks. +- Cover these classes: + - CWE-918 Server-Side Request Forgery (SSRF) + - User-controlled URLs passed to HTTP clients + - URL parsing bypasses (protocol smuggling, IP encoding tricks) + - Cloud metadata endpoint access (169.254.169.254 for AWS, metadata.google.internal for GCP) + - Internal service discovery via SSRF + - Webhook URL validation gaps + - PDF/image renderers fetching external URLs + +OUTPUT: +- Return JSON that strictly matches HuntResult. +- Populate HuntResult.findings with RawFinding entries focused on SSRF issues. +- For each finding, include title, description, cwe_id, cwe_name, file_path, start_line, end_line, + code_snippet, estimated_severity, confidence, fingerprint, and related_files. +- Include data_flow when you can trace user-controlled source to vulnerable sink. +- Set hunter_strategy to "ssrf" for each finding. +- Keep HuntResult metadata coherent (total_raw, deduplicated_count, chain_count, strategies_run, + hunt_duration_seconds). +- Empty arrays are acceptable if no vulnerabilities are confirmed. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +CONSTRAINTS: +- Prefer high confidence only when source-to-sink flow is clearly traced. +- Do not speculate about exploitability without code evidence. +- Do not include markdown, prose outside JSON, or code fences in final output. +- Base all findings on actual file content you read. Do not speculate. +- DO NOT FLAG: + - URLs from trusted config/env variables + - Allowlisted domains with strict validation + - URLs validated against strict regex patterns + - Internal service calls with no user input + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Depth profile: quick +- Early stop rule: if you inspect 30 files without credible signal, stop and return empty findings. +- Focus on RECON entry points and data flows as primary source-to-sink paths. +- Explore the codebase, trace data flows from sources to sinks, and identify SSRF points. +- Take multiple turns to build findings incrementally and write final JSON only when complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_ssrf_standard.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_ssrf_standard.txt new file mode 100644 index 0000000000..a49ca0addf --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_ssrf_standard.txt @@ -0,0 +1,485 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are an expert SSRF vulnerability analyst for SEC-AF HUNT phase. + +CONTEXT: +Use the RECON context below to prioritize analysis by real entry points and known data flows. + +{ + "app_type": "web_api", + "auth_model": "jwt", + "frameworks": [ + "Django", + "next.js", + "Spring Boot", + " React ", + "unknown-fw", + "NEXT", + "Vue" + ], + "languages": [ + "Python", + "JavaScript", + "Go", + "Ruby", + "Rust", + "python" + ], + "entry_points": [ + { + "kind": "http", + "identifier": "handler_0", + "file_path": "app/entry/e0.py", + "line": 10, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "HTTP", + "identifier": "handler_1", + "file_path": "app/entry/e1.py", + "line": 17, + "method": "POST", + "route": "/v1/resource/1", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_2", + "file_path": "app/entry/e2.py", + "line": 24, + "method": null, + "route": "/v1/resource/2", + "auth_required": null + }, + { + "kind": "api", + "identifier": "handler_3", + "file_path": "app/entry/e3.py", + "line": 31, + "method": "PUT", + "route": null, + "auth_required": true + }, + { + "kind": "graphql", + "identifier": "handler_4", + "file_path": "app/entry/e4.py", + "line": 38, + "method": "GET", + "route": "/v1/resource/4", + "auth_required": false + }, + { + "kind": "rpc", + "identifier": "handler_5", + "file_path": "app/entry/e5.py", + "line": 45, + "method": "POST", + "route": "/v1/resource/5", + "auth_required": null + }, + { + "kind": "route", + "identifier": "handler_6", + "file_path": "app/entry/e6.py", + "line": 52, + "method": null, + "route": null, + "auth_required": true + }, + { + "kind": "cron", + "identifier": "handler_7", + "file_path": "app/entry/e7.py", + "line": 59, + "method": "PUT", + "route": "/v1/resource/7", + "auth_required": false + }, + { + "kind": "queue", + "identifier": "handler_8", + "file_path": "app/entry/e8.py", + "line": 66, + "method": "GET", + "route": "/v1/resource/8", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_9", + "file_path": "app/entry/e9.py", + "line": 73, + "method": "POST", + "route": null, + "auth_required": true + } + ], + "data_flows": [ + { + "source": "request.args['q']", + "path": [ + { + "file_path": "app/search/index.ts", + "line": 20, + "component": "step_0_a", + "operation": "read" + }, + { + "file_path": "app/util/b.py", + "line": 40, + "component": "step_0_b", + "operation": "write" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/search/index.ts", + "app/db/raw.py", + "app/util/a.py", + "app/util/b.py" + ] + }, + { + "source": "request.cookies['session']", + "path": [ + { + "file_path": "app/auth/session_store.py", + "line": 21, + "component": "step_1_a", + "operation": "read" + }, + { + "file_path": "app/auth/session_store.py", + "line": 41, + "component": "step_1_b", + "operation": "write" + } + ], + "sink": "redis.set", + "sanitized": true, + "files": [ + "app/auth/session_store.py" + ] + }, + { + "source": "request.json['url']", + "path": [ + { + "file_path": "app/integrations/webhooks.go", + "line": 22, + "component": "step_2_a", + "operation": "read" + }, + { + "file_path": "app/net/client.go", + "line": 42, + "component": "step_2_b", + "operation": "write" + } + ], + "sink": "requests.get", + "sanitized": false, + "files": [ + "app/integrations/webhooks.go", + "app/net/client.go" + ] + }, + { + "source": "form['password']", + "path": [ + { + "file_path": "app/auth/service.py", + "line": 23, + "component": "step_3_a", + "operation": "read" + }, + { + "file_path": "app/obs/telemetry.go", + "line": 43, + "component": "step_3_b", + "operation": "write" + } + ], + "sink": "logger.info", + "sanitized": false, + "files": [ + "app/auth/service.py", + "app/obs/telemetry.go" + ] + }, + { + "source": "header['Authorization']", + "path": [ + { + "file_path": "app/common/jwt_tools.py", + "line": 24, + "component": "step_4_a", + "operation": "read" + }, + { + "file_path": "app/common/jwt_tools.py", + "line": 44, + "component": "step_4_b", + "operation": "write" + } + ], + "sink": "jwt.decode", + "sanitized": true, + "files": [ + "app/common/jwt_tools.py" + ] + }, + { + "source": "query['email']", + "path": [ + { + "file_path": "app/notify/mailer.rb", + "line": 25, + "component": "step_5_a", + "operation": "read" + }, + { + "file_path": "app/notify/mailer.rb", + "line": 45, + "component": "step_5_b", + "operation": "write" + } + ], + "sink": "smtp.send", + "sanitized": false, + "files": [ + "app/notify/mailer.rb" + ] + }, + { + "source": "body['card_number']", + "path": [ + { + "file_path": "app/billing/payments.go", + "line": 26, + "component": "step_6_a", + "operation": "read" + }, + { + "file_path": "app/billing/core.py", + "line": 46, + "component": "step_6_b", + "operation": "write" + } + ], + "sink": "stripe.Charge.create", + "sanitized": true, + "files": [ + "app/billing/payments.go", + "app/billing/core.py" + ] + }, + { + "source": "path_param['id']", + "path": [ + { + "file_path": "app/api/thing_1.py", + "line": 27, + "component": "step_7_a", + "operation": "read" + }, + { + "file_path": "app/api/thing_1.py", + "line": 47, + "component": "step_7_b", + "operation": "write" + } + ], + "sink": "orm.filter", + "sanitized": true, + "files": [ + "app/api/thing_1.py" + ] + }, + { + "source": "upload.filename", + "path": [ + { + "file_path": "app/media/uploads.py", + "line": 28, + "component": "step_8_a", + "operation": "read" + }, + { + "file_path": "app/x/w.py", + "line": 48, + "component": "step_8_b", + "operation": "write" + } + ], + "sink": "open", + "sanitized": false, + "files": [ + "app/media/uploads.py", + "app/common/cache.py", + "app/common/guard.py", + "app/x/y.py", + "app/x/z.py", + "app/x/w.py" + ] + }, + { + "source": "env['DEBUG']", + "path": [ + { + "file_path": "app/web/render.py", + "line": 29, + "component": "step_9_a", + "operation": "read" + }, + { + "file_path": "app/web/render.py", + "line": 49, + "component": "step_9_b", + "operation": "write" + } + ], + "sink": "template.render", + "sanitized": false, + "files": [ + "app/web/render.py" + ] + } + ] +} + +TASK: +Systematically hunt for server-side request forgery vulnerabilities in this codebase. + +WORKFLOW (follow these steps in order): +1. Read the source files referenced in the RECON context above to understand the codebase. +2. Search for HTTP clients, URL handling, webhook integrations, and outbound network calls in the source code. +3. Read the identified files to confirm vulnerabilities with actual code evidence. +4. For each confirmed vulnerability, record the file path, line number, and relevant code snippet. +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +SCOPE: +- Prioritize source-to-sink paths that originate from documented entry points. +- Trace data flow across files and layers to identify user-controlled URL input reaching outbound network sinks. +- Cover these classes: + - CWE-918 Server-Side Request Forgery (SSRF) + - User-controlled URLs passed to HTTP clients + - URL parsing bypasses (protocol smuggling, IP encoding tricks) + - Cloud metadata endpoint access (169.254.169.254 for AWS, metadata.google.internal for GCP) + - Internal service discovery via SSRF + - Webhook URL validation gaps + - PDF/image renderers fetching external URLs + +OUTPUT: +- Return JSON that strictly matches HuntResult. +- Populate HuntResult.findings with RawFinding entries focused on SSRF issues. +- For each finding, include title, description, cwe_id, cwe_name, file_path, start_line, end_line, + code_snippet, estimated_severity, confidence, fingerprint, and related_files. +- Include data_flow when you can trace user-controlled source to vulnerable sink. +- Set hunter_strategy to "ssrf" for each finding. +- Keep HuntResult metadata coherent (total_raw, deduplicated_count, chain_count, strategies_run, + hunt_duration_seconds). +- Empty arrays are acceptable if no vulnerabilities are confirmed. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: JAVASCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec, new Function + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, parameterized pg queries + DO NOT FLAG: React JSX expressions (auto-escaped), Angular template bindings (sanitized by default) + Framework notes: Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous. + +Language: GO + Key sinks: fmt.Sprintf into SQL, exec.Command with user input, template.HTML(), os.Exec + Safe patterns (skip these): database/sql with ? placeholders, html/template (auto-escapes), exec.Command with separate args + DO NOT FLAG: GORM parameterized queries, html/template default escaping, database/sql prepared statements + Framework notes: Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML. + +Language: RUBY + Key sinks: eval, system, exec, send, public_send, ERB.new with user input + Safe patterns (skip these): ActiveRecord parameterized queries, Rack::Utils.escape_html, sanitize helper in Rails + DO NOT FLAG: ActiveRecord where with hash conditions, Rails CSRF protection, Rails html_safe on constants + Framework notes: Rails auto-escapes ERB templates. raw/html_safe bypasses escaping. Check for mass assignment. + +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +Framework: NEXTJS + Security features: React JSX auto-escapes output by default, API routes can share centralized auth middleware, Server/client boundaries reduce accidental secret exposure + DO NOT FLAG: React JSX escaped rendering, getServerSideProps/getServerSession with proper auth checks, Typed route handlers with validated schema guards + Watch for: dangerouslySetInnerHTML, unprotected API routes under /api, secret leakage to client bundles, rewrites/redirects from untrusted user input + Common vulns: XSS through dangerous HTML rendering, IDOR/auth bypass in API handlers, Sensitive env/config exposure in client-side code + +Framework: SPRING + Security features: Spring Security provides auth, CSRF, and filter chain defaults, JPA/Hibernate prepared parameter binding by default, Bean validation can enforce request constraints + DO NOT FLAG: PreparedStatement/JPA named parameter usage, Spring Security CSRF/auth filters clearly active, Thymeleaf auto-escaped output + Watch for: @PreAuthorize missing on sensitive methods, JdbcTemplate/raw Statement with concatenation, csrf().disable() on browser session flows, Actuator endpoints exposed without auth + Common vulns: Authz bypass from weak method-level security, SQL injection in raw JDBC queries, Sensitive management endpoint exposure + +Framework: REACT + Security features: JSX escapes strings before DOM rendering, Component model encourages explicit data flow, Framework discourages direct DOM mutation + DO NOT FLAG: Standard JSX expression rendering, textContent assignment for untrusted text, sanitized HTML via trusted DOMPurify policy + Watch for: dangerouslySetInnerHTML, untrusted URL assignment to href/src, eval/new Function in client code, token/secret exposure in bundles + Common vulns: DOM XSS through unsafe HTML injection, Open redirect via unvalidated navigation targets, Sensitive data exposure in frontend artifacts + +Framework: VUE + Security features: Mustache template interpolation escapes HTML, Component props/events provide explicit boundaries, Router guards can enforce auth flows + DO NOT FLAG: escaped template interpolation {{ value }}, validated route guards protecting private routes, sanitized content rendered through safe components + Watch for: v-html with untrusted data, dynamic component/template compilation, unsafe URL bindings in href/src, client-side auth checks without server enforcement + Common vulns: XSS through v-html and unsafe render paths, Auth bypass from client-only route protection, Open redirect patterns in router navigation + +CONSTRAINTS: +- Prefer high confidence only when source-to-sink flow is clearly traced. +- Do not speculate about exploitability without code evidence. +- Do not include markdown, prose outside JSON, or code fences in final output. +- Base all findings on actual file content you read. Do not speculate. +- DO NOT FLAG: + - URLs from trusted config/env variables + - Allowlisted domains with strict validation + - URLs validated against strict regex patterns + - Internal service calls with no user input + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Depth profile: standard +- Early stop rule: if you inspect 30 files without credible signal, stop and return empty findings. +- Focus on RECON entry points and data flows as primary source-to-sink paths. +- Explore the codebase, trace data flows from sources to sinks, and identify SSRF points. +- Take multiple turns to build findings incrementally and write final JSON only when complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_ssrf_thorough.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_ssrf_thorough.txt new file mode 100644 index 0000000000..51192092de --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_ssrf_thorough.txt @@ -0,0 +1,139 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are an expert SSRF vulnerability analyst for SEC-AF HUNT phase. + +CONTEXT: +Use the RECON context below to prioritize analysis by real entry points and known data flows. + +{ + "app_type": "web_api", + "auth_model": "jwt", + "frameworks": [ + "django" + ], + "languages": [ + "python" + ], + "entry_points": [ + { + "kind": "http", + "identifier": "POST /login", + "file_path": "app/api/auth.py", + "line": 12, + "method": "POST", + "route": "/login", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "seed", + "file_path": "app/cli.py", + "line": 3, + "method": null, + "route": null, + "auth_required": null + } + ], + "data_flows": [ + { + "source": "request.json", + "path": [ + { + "file_path": "app/api/users.py", + "line": 41, + "component": "handler", + "operation": "read id" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/api/users.py" + ] + } + ] +} + +TASK: +Systematically hunt for server-side request forgery vulnerabilities in this codebase. + +WORKFLOW (follow these steps in order): +1. Read the source files referenced in the RECON context above to understand the codebase. +2. Search for HTTP clients, URL handling, webhook integrations, and outbound network calls in the source code. +3. Read the identified files to confirm vulnerabilities with actual code evidence. +4. For each confirmed vulnerability, record the file path, line number, and relevant code snippet. +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +SCOPE: +- Prioritize source-to-sink paths that originate from documented entry points. +- Trace data flow across files and layers to identify user-controlled URL input reaching outbound network sinks. +- Cover these classes: + - CWE-918 Server-Side Request Forgery (SSRF) + - User-controlled URLs passed to HTTP clients + - URL parsing bypasses (protocol smuggling, IP encoding tricks) + - Cloud metadata endpoint access (169.254.169.254 for AWS, metadata.google.internal for GCP) + - Internal service discovery via SSRF + - Webhook URL validation gaps + - PDF/image renderers fetching external URLs + +OUTPUT: +- Return JSON that strictly matches HuntResult. +- Populate HuntResult.findings with RawFinding entries focused on SSRF issues. +- For each finding, include title, description, cwe_id, cwe_name, file_path, start_line, end_line, + code_snippet, estimated_severity, confidence, fingerprint, and related_files. +- Include data_flow when you can trace user-controlled source to vulnerable sink. +- Set hunter_strategy to "ssrf" for each finding. +- Keep HuntResult metadata coherent (total_raw, deduplicated_count, chain_count, strategies_run, + hunt_duration_seconds). +- Empty arrays are acceptable if no vulnerabilities are confirmed. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +CONSTRAINTS: +- Prefer high confidence only when source-to-sink flow is clearly traced. +- Do not speculate about exploitability without code evidence. +- Do not include markdown, prose outside JSON, or code fences in final output. +- Base all findings on actual file content you read. Do not speculate. +- DO NOT FLAG: + - URLs from trusted config/env variables + - Allowlisted domains with strict validation + - URLs validated against strict regex patterns + - Internal service calls with no user input + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Depth profile: thorough +- Early stop rule: if you inspect 30 files without credible signal, stop and return empty findings. +- Focus on RECON entry points and data flows as primary source-to-sink paths. +- Explore the codebase, trace data flows from sources to sinks, and identify SSRF points. +- Take multiple turns to build findings incrementally and write final JSON only when complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_supply_chain_standard.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_supply_chain_standard.txt new file mode 100644 index 0000000000..6b2be8e01e --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_supply_chain_standard.txt @@ -0,0 +1,169 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are SEC-AF's Supply Chain Hunter specializing in dependency risk and package ecosystem abuse. + +CONTEXT: +You are in HUNT Phase and must output a HuntResult JSON object for supply-chain findings. +Use the RECON context summary below as primary evidence. + +RECON_CONTEXT: +Supply-chain-focused recon summary. + +Dependency inventory: direct=7, transitive=143, SBOM entries=14. + +Known CVE exposure (reachable/high severity first): 18 total, showing top 15: +- CVE-2023-0014 in log4j 2.14.0 (fixed=2.17.1, cvss=10.0, epss=0.97, direct=False, reachable=True) +- CVE-2023-0003 in lodash 4.17.19 (fixed=4.17.21, cvss=9.8, epss=0.9, direct=True, reachable=True) +- CVE-2023-0001 in django 4.2.1 (fixed=4.2.5, cvss=9.8, epss=0.5, direct=True, reachable=True) +- CVE-2023-0013 in pyyaml 5.3 (fixed=5.4, cvss=9.8, epss=0.5, direct=False, reachable=True) +- CVE-2023-0008 in nokogiri 1.13.6 (fixed=1.13.10, cvss=8.8, epss=0.3, direct=False, reachable=True) +- CVE-2023-0018 in curl 7.68.0 (fixed=7.88.0, cvss=8.1, epss=None, direct=True, reachable=True) +- CVE-2023-0009 in golang.org/x/net 0.7.0 (fixed=0.17.0, cvss=7.5, epss=0.44, direct=False, reachable=True) +- CVE-2023-0012 in requests 2.31.0 (fixed=2.32.0, cvss=6.5, epss=0.07, direct=True, reachable=True) +- CVE-2023-0005 in axios 0.21.1 (fixed=0.21.2, cvss=5.3, epss=0.02, direct=False, reachable=True) +- CVE-2023-0017 in libxml2 2.9.10 (fixed=unknown, cvss=None, epss=0.25, direct=False, reachable=True) +- CVE-2023-0010 in rails 7.0.4 (fixed=7.0.7, cvss=9.1, epss=0.6, direct=True, reachable=False) +- CVE-2023-0016 in zlib 1.2.11 (fixed=1.2.12, cvss=8.2, epss=0.04, direct=False, reachable=False) +- CVE-2023-0002 in urllib3 1.26.5 (fixed=unknown, cvss=7.5, epss=0.1, direct=False, reachable=False) +- CVE-2023-0015 in openssl 1.1.1k (fixed=1.1.1t, cvss=7.4, epss=0.12, direct=False, reachable=None) +- CVE-2023-0006 in jinja2 3.1.2 (fixed=unknown, cvss=6.1, epss=None, direct=False, reachable=False) + +Outdated dependencies: 12 total, showing top 10: +- pkg-0: 1.0.0 -> 2.0.0 (direct=True) +- pkg-1: 1.1.0 -> 2.1.0 (direct=False) +- pkg-2: 1.2.0 -> 2.2.0 (direct=True) +- pkg-3: 1.3.0 -> 2.3.0 (direct=False) +- pkg-4: 1.4.0 -> 2.4.0 (direct=True) +- pkg-5: 1.5.0 -> 2.5.0 (direct=False) +- pkg-6: 1.6.0 -> 2.6.0 (direct=True) +- pkg-7: 1.7.0 -> 2.7.0 (direct=False) +- pkg-8: 1.8.0 -> 2.8.0 (direct=True) +- pkg-9: 1.9.0 -> 2.9.0 (direct=False) + +Primary dependency ecosystems in this repo: 12 total, showing top 10: +- go: github.com/gin-gonic/gin@1.9.0 +- go: golang.org/x/net@0.7.0 +- npm: axios@0.21.1 +- npm: express@4.17.1 +- npm: lodash@4.17.19 +- npm: minimist@1.2.5 +- pypi: django@4.2.1 +- pypi: jinja2@3.1.2 +- pypi: requests@2.31.0 +- pypi: urllib3@1.26.5 + +TASK: +Identify supply-chain vulnerabilities with emphasis on CWE-1104 (unmaintained third-party components) +and CWE-829 (inclusion of functionality from untrusted control sphere). + +SCOPE: +- Analyze dependency manifests and lockfiles across ecosystems (npm/yarn/pnpm, Python, Go, Rust, etc.). +- Use RECON dependency context (SBOM, known CVEs, outdated packages) as primary evidence. +- Detect known vulnerable versions and include CVE IDs when available. +- Flag unmaintained/abandoned packages and stale dependencies with security impact. +- Flag typosquatting and look-alike package names likely to be malicious. +- Flag untrusted package sources, private-registry confusion, and dependency confusion patterns. +- Flag risky version pinning posture: missing lockfile, wildcard ranges, loose semver constraints. +- Flag lockfile drift/integrity issues where manifest and lock data are inconsistent. + +OUTPUT: +- Return JSON that strictly matches HuntResult. +- Use finding_type "sca" for dependency findings. +- Every finding must include concrete package evidence (package name, version, file path, and rationale). +- Include CWE mapping: CWE-1104 or CWE-829 for each finding. +- Include CVE identifiers in description/code_snippet when available from recon evidence. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: JAVASCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec, new Function + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, parameterized pg queries + DO NOT FLAG: React JSX expressions (auto-escaped), Angular template bindings (sanitized by default) + Framework notes: Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous. + +Language: GO + Key sinks: fmt.Sprintf into SQL, exec.Command with user input, template.HTML(), os.Exec + Safe patterns (skip these): database/sql with ? placeholders, html/template (auto-escapes), exec.Command with separate args + DO NOT FLAG: GORM parameterized queries, html/template default escaping, database/sql prepared statements + Framework notes: Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML. + +Language: RUBY + Key sinks: eval, system, exec, send, public_send, ERB.new with user input + Safe patterns (skip these): ActiveRecord parameterized queries, Rack::Utils.escape_html, sanitize helper in Rails + DO NOT FLAG: ActiveRecord where with hash conditions, Rails CSRF protection, Rails html_safe on constants + Framework notes: Rails auto-escapes ERB templates. raw/html_safe bypasses escaping. Check for mass assignment. + +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +Framework: NEXTJS + Security features: React JSX auto-escapes output by default, API routes can share centralized auth middleware, Server/client boundaries reduce accidental secret exposure + DO NOT FLAG: React JSX escaped rendering, getServerSideProps/getServerSession with proper auth checks, Typed route handlers with validated schema guards + Watch for: dangerouslySetInnerHTML, unprotected API routes under /api, secret leakage to client bundles, rewrites/redirects from untrusted user input + Common vulns: XSS through dangerous HTML rendering, IDOR/auth bypass in API handlers, Sensitive env/config exposure in client-side code + +Framework: SPRING + Security features: Spring Security provides auth, CSRF, and filter chain defaults, JPA/Hibernate prepared parameter binding by default, Bean validation can enforce request constraints + DO NOT FLAG: PreparedStatement/JPA named parameter usage, Spring Security CSRF/auth filters clearly active, Thymeleaf auto-escaped output + Watch for: @PreAuthorize missing on sensitive methods, JdbcTemplate/raw Statement with concatenation, csrf().disable() on browser session flows, Actuator endpoints exposed without auth + Common vulns: Authz bypass from weak method-level security, SQL injection in raw JDBC queries, Sensitive management endpoint exposure + +Framework: REACT + Security features: JSX escapes strings before DOM rendering, Component model encourages explicit data flow, Framework discourages direct DOM mutation + DO NOT FLAG: Standard JSX expression rendering, textContent assignment for untrusted text, sanitized HTML via trusted DOMPurify policy + Watch for: dangerouslySetInnerHTML, untrusted URL assignment to href/src, eval/new Function in client code, token/secret exposure in bundles + Common vulns: DOM XSS through unsafe HTML injection, Open redirect via unvalidated navigation targets, Sensitive data exposure in frontend artifacts + +Framework: VUE + Security features: Mustache template interpolation escapes HTML, Component props/events provide explicit boundaries, Router guards can enforce auth flows + DO NOT FLAG: escaped template interpolation {{ value }}, validated route guards protecting private routes, sanitized content rendered through safe components + Watch for: v-html with untrusted data, dynamic component/template compilation, unsafe URL bindings in href/src, client-side auth checks without server enforcement + Common vulns: XSS through v-html and unsafe render paths, Auth bypass from client-only route protection, Open redirect patterns in router navigation + +CONSTRAINTS: +- Take multiple turns: inspect relevant files first, then build findings incrementally. +- Do not invent package versions, CVEs, or exploitability claims. +- Prefer high-confidence findings with explicit evidence over speculative findings. +- No markdown/code fences; JSON output only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Hunt strategy: supply_chain (CWE-1104, CWE-829). +- Early stop rule: if you inspect standard manifests/files without credible dependency risk, stop and return empty findings. +- Focus manifests/lockfiles (package.json, requirements.txt, go.mod, Pipfile, poetry.lock, package-lock.json, yarn.lock, pnpm-lock.yaml, Cargo.toml). +- Take multiple turns: inspect manifests/lockfiles, validate dependency risks, then produce final structured findings. +- Write final JSON only when analysis is complete. + diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_supply_chain_thorough.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_supply_chain_thorough.txt new file mode 100644 index 0000000000..bc3542a1df --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_supply_chain_thorough.txt @@ -0,0 +1,88 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are SEC-AF's Supply Chain Hunter specializing in dependency risk and package ecosystem abuse. + +CONTEXT: +You are in HUNT Phase and must output a HuntResult JSON object for supply-chain findings. +Use the RECON context summary below as primary evidence. + +RECON_CONTEXT: +Supply-chain-focused recon summary. + +Dependency inventory: direct=3, transitive=9, SBOM entries=1. + +Known CVE exposure (reachable/high severity first): 1 total, showing top 1: +- CVE-2024-0001 in django 4.2.1 (fixed=4.2.11, cvss=7.5, epss=0.42, direct=True, reachable=True) + +Outdated dependencies: none identified in recon. + +Primary dependency ecosystems in this repo: 1 total, showing top 1: +- pypi: django@4.2.1 + +TASK: +Identify supply-chain vulnerabilities with emphasis on CWE-1104 (unmaintained third-party components) +and CWE-829 (inclusion of functionality from untrusted control sphere). + +SCOPE: +- Analyze dependency manifests and lockfiles across ecosystems (npm/yarn/pnpm, Python, Go, Rust, etc.). +- Use RECON dependency context (SBOM, known CVEs, outdated packages) as primary evidence. +- Detect known vulnerable versions and include CVE IDs when available. +- Flag unmaintained/abandoned packages and stale dependencies with security impact. +- Flag typosquatting and look-alike package names likely to be malicious. +- Flag untrusted package sources, private-registry confusion, and dependency confusion patterns. +- Flag risky version pinning posture: missing lockfile, wildcard ranges, loose semver constraints. +- Flag lockfile drift/integrity issues where manifest and lock data are inconsistent. + +OUTPUT: +- Return JSON that strictly matches HuntResult. +- Use finding_type "sca" for dependency findings. +- Every finding must include concrete package evidence (package name, version, file path, and rationale). +- Include CWE mapping: CWE-1104 or CWE-829 for each finding. +- Include CVE identifiers in description/code_snippet when available from recon evidence. + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +CONSTRAINTS: +- Take multiple turns: inspect relevant files first, then build findings incrementally. +- Do not invent package versions, CVEs, or exploitability claims. +- Prefer high-confidence findings with explicit evidence over speculative findings. +- No markdown/code fences; JSON output only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Hunt strategy: supply_chain (CWE-1104, CWE-829). +- Early stop rule: if you inspect thorough manifests/files without credible dependency risk, stop and return empty findings. +- Focus manifests/lockfiles (package.json, requirements.txt, go.mod, Pipfile, poetry.lock, package-lock.json, yarn.lock, pnpm-lock.yaml, Cargo.toml). +- Take multiple turns: inspect manifests/lockfiles, validate dependency risks, then produce final structured findings. +- Write final JSON only when analysis is complete. + diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_xss_standard.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_xss_standard.txt new file mode 100644 index 0000000000..6c5f971703 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_xss_standard.txt @@ -0,0 +1,485 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are an expert XSS and client-side injection analyst for SEC-AF HUNT phase. + +CONTEXT: +Use the RECON context below to prioritize analysis by real entry points and known data flows. + +{ + "app_type": "web_api", + "auth_model": "jwt", + "frameworks": [ + "Django", + "next.js", + "Spring Boot", + " React ", + "unknown-fw", + "NEXT", + "Vue" + ], + "languages": [ + "Python", + "JavaScript", + "Go", + "Ruby", + "Rust", + "python" + ], + "entry_points": [ + { + "kind": "http", + "identifier": "handler_0", + "file_path": "app/entry/e0.py", + "line": 10, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "HTTP", + "identifier": "handler_1", + "file_path": "app/entry/e1.py", + "line": 17, + "method": "POST", + "route": "/v1/resource/1", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_2", + "file_path": "app/entry/e2.py", + "line": 24, + "method": null, + "route": "/v1/resource/2", + "auth_required": null + }, + { + "kind": "api", + "identifier": "handler_3", + "file_path": "app/entry/e3.py", + "line": 31, + "method": "PUT", + "route": null, + "auth_required": true + }, + { + "kind": "graphql", + "identifier": "handler_4", + "file_path": "app/entry/e4.py", + "line": 38, + "method": "GET", + "route": "/v1/resource/4", + "auth_required": false + }, + { + "kind": "rpc", + "identifier": "handler_5", + "file_path": "app/entry/e5.py", + "line": 45, + "method": "POST", + "route": "/v1/resource/5", + "auth_required": null + }, + { + "kind": "route", + "identifier": "handler_6", + "file_path": "app/entry/e6.py", + "line": 52, + "method": null, + "route": null, + "auth_required": true + }, + { + "kind": "cron", + "identifier": "handler_7", + "file_path": "app/entry/e7.py", + "line": 59, + "method": "PUT", + "route": "/v1/resource/7", + "auth_required": false + }, + { + "kind": "queue", + "identifier": "handler_8", + "file_path": "app/entry/e8.py", + "line": 66, + "method": "GET", + "route": "/v1/resource/8", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_9", + "file_path": "app/entry/e9.py", + "line": 73, + "method": "POST", + "route": null, + "auth_required": true + } + ], + "data_flows": [ + { + "source": "request.args['q']", + "path": [ + { + "file_path": "app/search/index.ts", + "line": 20, + "component": "step_0_a", + "operation": "read" + }, + { + "file_path": "app/util/b.py", + "line": 40, + "component": "step_0_b", + "operation": "write" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/search/index.ts", + "app/db/raw.py", + "app/util/a.py", + "app/util/b.py" + ] + }, + { + "source": "request.cookies['session']", + "path": [ + { + "file_path": "app/auth/session_store.py", + "line": 21, + "component": "step_1_a", + "operation": "read" + }, + { + "file_path": "app/auth/session_store.py", + "line": 41, + "component": "step_1_b", + "operation": "write" + } + ], + "sink": "redis.set", + "sanitized": true, + "files": [ + "app/auth/session_store.py" + ] + }, + { + "source": "request.json['url']", + "path": [ + { + "file_path": "app/integrations/webhooks.go", + "line": 22, + "component": "step_2_a", + "operation": "read" + }, + { + "file_path": "app/net/client.go", + "line": 42, + "component": "step_2_b", + "operation": "write" + } + ], + "sink": "requests.get", + "sanitized": false, + "files": [ + "app/integrations/webhooks.go", + "app/net/client.go" + ] + }, + { + "source": "form['password']", + "path": [ + { + "file_path": "app/auth/service.py", + "line": 23, + "component": "step_3_a", + "operation": "read" + }, + { + "file_path": "app/obs/telemetry.go", + "line": 43, + "component": "step_3_b", + "operation": "write" + } + ], + "sink": "logger.info", + "sanitized": false, + "files": [ + "app/auth/service.py", + "app/obs/telemetry.go" + ] + }, + { + "source": "header['Authorization']", + "path": [ + { + "file_path": "app/common/jwt_tools.py", + "line": 24, + "component": "step_4_a", + "operation": "read" + }, + { + "file_path": "app/common/jwt_tools.py", + "line": 44, + "component": "step_4_b", + "operation": "write" + } + ], + "sink": "jwt.decode", + "sanitized": true, + "files": [ + "app/common/jwt_tools.py" + ] + }, + { + "source": "query['email']", + "path": [ + { + "file_path": "app/notify/mailer.rb", + "line": 25, + "component": "step_5_a", + "operation": "read" + }, + { + "file_path": "app/notify/mailer.rb", + "line": 45, + "component": "step_5_b", + "operation": "write" + } + ], + "sink": "smtp.send", + "sanitized": false, + "files": [ + "app/notify/mailer.rb" + ] + }, + { + "source": "body['card_number']", + "path": [ + { + "file_path": "app/billing/payments.go", + "line": 26, + "component": "step_6_a", + "operation": "read" + }, + { + "file_path": "app/billing/core.py", + "line": 46, + "component": "step_6_b", + "operation": "write" + } + ], + "sink": "stripe.Charge.create", + "sanitized": true, + "files": [ + "app/billing/payments.go", + "app/billing/core.py" + ] + }, + { + "source": "path_param['id']", + "path": [ + { + "file_path": "app/api/thing_1.py", + "line": 27, + "component": "step_7_a", + "operation": "read" + }, + { + "file_path": "app/api/thing_1.py", + "line": 47, + "component": "step_7_b", + "operation": "write" + } + ], + "sink": "orm.filter", + "sanitized": true, + "files": [ + "app/api/thing_1.py" + ] + }, + { + "source": "upload.filename", + "path": [ + { + "file_path": "app/media/uploads.py", + "line": 28, + "component": "step_8_a", + "operation": "read" + }, + { + "file_path": "app/x/w.py", + "line": 48, + "component": "step_8_b", + "operation": "write" + } + ], + "sink": "open", + "sanitized": false, + "files": [ + "app/media/uploads.py", + "app/common/cache.py", + "app/common/guard.py", + "app/x/y.py", + "app/x/z.py", + "app/x/w.py" + ] + }, + { + "source": "env['DEBUG']", + "path": [ + { + "file_path": "app/web/render.py", + "line": 29, + "component": "step_9_a", + "operation": "read" + }, + { + "file_path": "app/web/render.py", + "line": 49, + "component": "step_9_b", + "operation": "write" + } + ], + "sink": "template.render", + "sanitized": false, + "files": [ + "app/web/render.py" + ] + } + ] +} + +TASK: +Systematically hunt for XSS and client-side injection vulnerabilities in this codebase. + +SCOPE: +- Prioritize source-to-sink paths that originate from documented entry points. +- Trace data flow across files and layers to identify missing validation, sanitization, context-aware escaping, or safe encoding. +- Cover these classes: + - CWE-79 Cross-Site Scripting (XSS) + - CWE-80 Improper Neutralization of Script-Related HTML Tags in a Web Page (Basic XSS) + - CWE-87 Improper Neutralization of Alternate XSS Syntax + - CWE-116 Improper Encoding or Escaping of Output +- Hunt for: + - Stored XSS (database/content store to rendered output) + - Reflected XSS (request input to response output) + - DOM XSS (client-side JS flows into dangerous DOM sinks) + - HTML injection that enables script execution paths + - Template injection in client rendering paths that can lead to executable markup/script + +OUTPUT: +- Return JSON that strictly matches HuntResult. +- Populate HuntResult.findings with RawFinding entries focused on XSS/client-side injection issues. +- For each finding, include title, description, cwe_id, cwe_name, file_path, start_line, end_line, + code_snippet, estimated_severity, confidence, fingerprint, and related_files. +- Include data_flow when you can trace user-controlled source to vulnerable rendering sink. +- Set hunter_strategy to "xss" for each finding. +- Keep HuntResult metadata coherent (total_raw, deduplicated_count, chain_count, strategies_run, + hunt_duration_seconds). + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: JAVASCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec, new Function + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, parameterized pg queries + DO NOT FLAG: React JSX expressions (auto-escaped), Angular template bindings (sanitized by default) + Framework notes: Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous. + +Language: GO + Key sinks: fmt.Sprintf into SQL, exec.Command with user input, template.HTML(), os.Exec + Safe patterns (skip these): database/sql with ? placeholders, html/template (auto-escapes), exec.Command with separate args + DO NOT FLAG: GORM parameterized queries, html/template default escaping, database/sql prepared statements + Framework notes: Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML. + +Language: RUBY + Key sinks: eval, system, exec, send, public_send, ERB.new with user input + Safe patterns (skip these): ActiveRecord parameterized queries, Rack::Utils.escape_html, sanitize helper in Rails + DO NOT FLAG: ActiveRecord where with hash conditions, Rails CSRF protection, Rails html_safe on constants + Framework notes: Rails auto-escapes ERB templates. raw/html_safe bypasses escaping. Check for mass assignment. + +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +Framework: NEXTJS + Security features: React JSX auto-escapes output by default, API routes can share centralized auth middleware, Server/client boundaries reduce accidental secret exposure + DO NOT FLAG: React JSX escaped rendering, getServerSideProps/getServerSession with proper auth checks, Typed route handlers with validated schema guards + Watch for: dangerouslySetInnerHTML, unprotected API routes under /api, secret leakage to client bundles, rewrites/redirects from untrusted user input + Common vulns: XSS through dangerous HTML rendering, IDOR/auth bypass in API handlers, Sensitive env/config exposure in client-side code + +Framework: SPRING + Security features: Spring Security provides auth, CSRF, and filter chain defaults, JPA/Hibernate prepared parameter binding by default, Bean validation can enforce request constraints + DO NOT FLAG: PreparedStatement/JPA named parameter usage, Spring Security CSRF/auth filters clearly active, Thymeleaf auto-escaped output + Watch for: @PreAuthorize missing on sensitive methods, JdbcTemplate/raw Statement with concatenation, csrf().disable() on browser session flows, Actuator endpoints exposed without auth + Common vulns: Authz bypass from weak method-level security, SQL injection in raw JDBC queries, Sensitive management endpoint exposure + +Framework: REACT + Security features: JSX escapes strings before DOM rendering, Component model encourages explicit data flow, Framework discourages direct DOM mutation + DO NOT FLAG: Standard JSX expression rendering, textContent assignment for untrusted text, sanitized HTML via trusted DOMPurify policy + Watch for: dangerouslySetInnerHTML, untrusted URL assignment to href/src, eval/new Function in client code, token/secret exposure in bundles + Common vulns: DOM XSS through unsafe HTML injection, Open redirect via unvalidated navigation targets, Sensitive data exposure in frontend artifacts + +Framework: VUE + Security features: Mustache template interpolation escapes HTML, Component props/events provide explicit boundaries, Router guards can enforce auth flows + DO NOT FLAG: escaped template interpolation {{ value }}, validated route guards protecting private routes, sanitized content rendered through safe components + Watch for: v-html with untrusted data, dynamic component/template compilation, unsafe URL bindings in href/src, client-side auth checks without server enforcement + Common vulns: XSS through v-html and unsafe render paths, Auth bypass from client-only route protection, Open redirect patterns in router navigation + +CONSTRAINTS: +- Take multiple turns: explore the codebase, trace data flows from sources to rendering sinks, + identify candidate XSS/client-side injection paths, then construct final findings. +- Prefer high confidence only when source-to-sink flow is clearly traced. +- Do not speculate about exploitability without code evidence. +- Do not include markdown, prose outside JSON, or code fences in final output. +- DO NOT FLAG: + - React JSX auto-escaping when no dangerous bypass APIs are used + - Django template default escaping when no unsafe escape bypass is present + - Vue template escaping when no unsafe render/bypass pattern is present + - Angular built-in sanitization when not explicitly bypassed + - Output that is sanitized through DOMPurify + - Properly escaped template variables in server or client rendering + - Test fixtures, synthetic payload examples, and comments that only demonstrate attack strings + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Depth profile: standard +- Early stop rule: if you inspect 30 files without credible signal, stop and return empty findings. +- Focus on RECON entry points and data flows as primary source-to-sink paths. +- Explore the codebase, trace user-controlled data into rendering sinks, and identify XSS/client-side injection points. +- Target CWEs: CWE-79, CWE-80, CWE-87, CWE-116. +- Take multiple turns to build findings incrementally and write final JSON only when complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/prompt_xss_thorough.txt b/internal/secaf/agents/hunt/testdata/golden/prompt_xss_thorough.txt new file mode 100644 index 0000000000..4387bd2c9e --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/prompt_xss_thorough.txt @@ -0,0 +1,139 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +ROLE: +You are an expert XSS and client-side injection analyst for SEC-AF HUNT phase. + +CONTEXT: +Use the RECON context below to prioritize analysis by real entry points and known data flows. + +{ + "app_type": "web_api", + "auth_model": "jwt", + "frameworks": [ + "django" + ], + "languages": [ + "python" + ], + "entry_points": [ + { + "kind": "http", + "identifier": "POST /login", + "file_path": "app/api/auth.py", + "line": 12, + "method": "POST", + "route": "/login", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "seed", + "file_path": "app/cli.py", + "line": 3, + "method": null, + "route": null, + "auth_required": null + } + ], + "data_flows": [ + { + "source": "request.json", + "path": [ + { + "file_path": "app/api/users.py", + "line": 41, + "component": "handler", + "operation": "read id" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/api/users.py" + ] + } + ] +} + +TASK: +Systematically hunt for XSS and client-side injection vulnerabilities in this codebase. + +SCOPE: +- Prioritize source-to-sink paths that originate from documented entry points. +- Trace data flow across files and layers to identify missing validation, sanitization, context-aware escaping, or safe encoding. +- Cover these classes: + - CWE-79 Cross-Site Scripting (XSS) + - CWE-80 Improper Neutralization of Script-Related HTML Tags in a Web Page (Basic XSS) + - CWE-87 Improper Neutralization of Alternate XSS Syntax + - CWE-116 Improper Encoding or Escaping of Output +- Hunt for: + - Stored XSS (database/content store to rendered output) + - Reflected XSS (request input to response output) + - DOM XSS (client-side JS flows into dangerous DOM sinks) + - HTML injection that enables script execution paths + - Template injection in client rendering paths that can lead to executable markup/script + +OUTPUT: +- Return JSON that strictly matches HuntResult. +- Populate HuntResult.findings with RawFinding entries focused on XSS/client-side injection issues. +- For each finding, include title, description, cwe_id, cwe_name, file_path, start_line, end_line, + code_snippet, estimated_severity, confidence, fingerprint, and related_files. +- Include data_flow when you can trace user-controlled source to vulnerable rendering sink. +- Set hunter_strategy to "xss" for each finding. +- Keep HuntResult metadata coherent (total_raw, deduplicated_count, chain_count, strategies_run, + hunt_duration_seconds). + +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +CONSTRAINTS: +- Take multiple turns: explore the codebase, trace data flows from sources to rendering sinks, + identify candidate XSS/client-side injection paths, then construct final findings. +- Prefer high confidence only when source-to-sink flow is clearly traced. +- Do not speculate about exploitability without code evidence. +- Do not include markdown, prose outside JSON, or code fences in final output. +- DO NOT FLAG: + - React JSX auto-escaping when no dangerous bypass APIs are used + - Django template default escaping when no unsafe escape bypass is present + - Vue template escaping when no unsafe render/bypass pattern is present + - Angular built-in sanitization when not explicitly bypassed + - Output that is sanitized through DOMPurify + - Properly escaped template variables in server or client rendering + - Test fixtures, synthetic payload examples, and comments that only demonstrate attack strings + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Depth profile: thorough +- Early stop rule: if you inspect 30 files without credible signal, stop and return empty findings. +- Focus on RECON entry points and data flows as primary source-to-sink paths. +- Explore the codebase, trace user-controlled data into rendering sinks, and identify XSS/client-side injection points. +- Target CWEs: CWE-79, CWE-80, CWE-87, CWE-116. +- Take multiple turns to build findings incrementally and write final JSON only when complete. diff --git a/internal/secaf/agents/hunt/testdata/golden/quick_strategies.json b/internal/secaf/agents/hunt/testdata/golden/quick_strategies.json new file mode 100644 index 0000000000..de7c8ee56d --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/quick_strategies.json @@ -0,0 +1,7 @@ +[ + "injection", + "dos", + "ssrf", + "auth", + "data_exposure" +] diff --git a/internal/secaf/agents/hunt/testdata/golden/recon_context_block_business_logic.txt b/internal/secaf/agents/hunt/testdata/golden/recon_context_block_business_logic.txt new file mode 100644 index 0000000000..428b15429b --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/recon_context_block_business_logic.txt @@ -0,0 +1,796 @@ +{ + "app_type": "web_api", + "frameworks": [ + "Django", + "next.js", + "Spring Boot", + " React ", + "unknown-fw", + "NEXT", + "Vue" + ], + "languages": [ + "Python", + "JavaScript", + "Go", + "Ruby", + "Rust", + "python" + ], + "auth_model": "jwt", + "auth_details": "HS256 JWTs issued by app/auth/service.py, refresh tokens in Redis", + "entry_points": [ + { + "kind": "http", + "identifier": "handler_0", + "file_path": "app/entry/e0.py", + "line": 10, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "HTTP", + "identifier": "handler_1", + "file_path": "app/entry/e1.py", + "line": 17, + "method": "POST", + "route": "/v1/resource/1", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_2", + "file_path": "app/entry/e2.py", + "line": 24, + "method": null, + "route": "/v1/resource/2", + "auth_required": null + }, + { + "kind": "api", + "identifier": "handler_3", + "file_path": "app/entry/e3.py", + "line": 31, + "method": "PUT", + "route": null, + "auth_required": true + }, + { + "kind": "graphql", + "identifier": "handler_4", + "file_path": "app/entry/e4.py", + "line": 38, + "method": "GET", + "route": "/v1/resource/4", + "auth_required": false + }, + { + "kind": "rpc", + "identifier": "handler_5", + "file_path": "app/entry/e5.py", + "line": 45, + "method": "POST", + "route": "/v1/resource/5", + "auth_required": null + }, + { + "kind": "route", + "identifier": "handler_6", + "file_path": "app/entry/e6.py", + "line": 52, + "method": null, + "route": null, + "auth_required": true + }, + { + "kind": "cron", + "identifier": "handler_7", + "file_path": "app/entry/e7.py", + "line": 59, + "method": "PUT", + "route": "/v1/resource/7", + "auth_required": false + }, + { + "kind": "queue", + "identifier": "handler_8", + "file_path": "app/entry/e8.py", + "line": 66, + "method": "GET", + "route": "/v1/resource/8", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_9", + "file_path": "app/entry/e9.py", + "line": 73, + "method": "POST", + "route": null, + "auth_required": true + }, + { + "kind": "api", + "identifier": "handler_10", + "file_path": "app/entry/e10.py", + "line": 80, + "method": null, + "route": "/v1/resource/10", + "auth_required": false + }, + { + "kind": "grpc", + "identifier": "handler_11", + "file_path": "app/entry/e11.py", + "line": 87, + "method": "PUT", + "route": "/v1/resource/11", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_12", + "file_path": "app/entry/e12.py", + "line": 94, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "route", + "identifier": "handler_13", + "file_path": "app/entry/e13.py", + "line": 101, + "method": "POST", + "route": "/v1/resource/13", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_14", + "file_path": "app/entry/e14.py", + "line": 108, + "method": null, + "route": "/v1/resource/14", + "auth_required": null + } + ], + "api_surface": [ + { + "method": "GET", + "path": "/api/v1/thing/0", + "handler": "ThingController.action0", + "file_path": "app/api/thing_0.py", + "line": 100, + "auth_required": true, + "rate_limited": null + }, + { + "method": "POST", + "path": "/api/v1/thing/1", + "handler": "ThingController.action1", + "file_path": "app/api/thing_1.py", + "line": 103, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PUT", + "path": "/api/v1/thing/2", + "handler": "ThingController.action2", + "file_path": "app/api/thing_2.py", + "line": 106, + "auth_required": null, + "rate_limited": true + }, + { + "method": "DELETE", + "path": "/api/v1/thing/3", + "handler": "ThingController.action3", + "file_path": "app/api/thing_3.py", + "line": 109, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PATCH", + "path": "/api/v1/thing/4", + "handler": "ThingController.action4", + "file_path": "app/api/thing_4.py", + "line": 112, + "auth_required": true, + "rate_limited": null + }, + { + "method": "GET", + "path": "/api/v1/thing/5", + "handler": "ThingController.action5", + "file_path": "app/api/thing_5.py", + "line": 115, + "auth_required": null, + "rate_limited": true + }, + { + "method": "POST", + "path": "/api/v1/thing/6", + "handler": "ThingController.action6", + "file_path": "app/api/thing_6.py", + "line": 118, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PUT", + "path": "/api/v1/thing/7", + "handler": "ThingController.action7", + "file_path": "app/api/thing_7.py", + "line": 121, + "auth_required": false, + "rate_limited": false + }, + { + "method": "DELETE", + "path": "/api/v1/thing/8", + "handler": "ThingController.action8", + "file_path": "app/api/thing_8.py", + "line": 124, + "auth_required": null, + "rate_limited": true + }, + { + "method": "PATCH", + "path": "/api/v1/thing/9", + "handler": "ThingController.action9", + "file_path": "app/api/thing_9.py", + "line": 127, + "auth_required": false, + "rate_limited": false + }, + { + "method": "GET", + "path": "/api/v1/thing/10", + "handler": "ThingController.action10", + "file_path": "app/api/thing_10.py", + "line": 130, + "auth_required": true, + "rate_limited": null + }, + { + "method": "POST", + "path": "/api/v1/thing/11", + "handler": "ThingController.action11", + "file_path": "app/api/thing_11.py", + "line": 133, + "auth_required": null, + "rate_limited": true + }, + { + "method": "PUT", + "path": "/api/v1/thing/12", + "handler": "ThingController.action12", + "file_path": "app/api/thing_12.py", + "line": 136, + "auth_required": true, + "rate_limited": null + }, + { + "method": "DELETE", + "path": "/api/v1/thing/13", + "handler": "ThingController.action13", + "file_path": "app/api/thing_13.py", + "line": 139, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PATCH", + "path": "/api/v1/thing/14", + "handler": "ThingController.action14", + "file_path": "app/api/thing_14.py", + "line": 142, + "auth_required": null, + "rate_limited": true + }, + { + "method": "GET", + "path": "/api/v1/thing/15", + "handler": "ThingController.action15", + "file_path": "app/api/thing_15.py", + "line": 145, + "auth_required": false, + "rate_limited": false + }, + { + "method": "POST", + "path": "/api/v1/thing/16", + "handler": "ThingController.action16", + "file_path": "app/api/thing_16.py", + "line": 148, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PUT", + "path": "/api/v1/thing/17", + "handler": "ThingController.action17", + "file_path": "app/api/thing_17.py", + "line": 151, + "auth_required": null, + "rate_limited": true + }, + { + "method": "DELETE", + "path": "/api/v1/thing/18", + "handler": "ThingController.action18", + "file_path": "app/api/thing_18.py", + "line": 154, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PATCH", + "path": "/api/v1/thing/19", + "handler": "ThingController.action19", + "file_path": "app/api/thing_19.py", + "line": 157, + "auth_required": false, + "rate_limited": false + } + ], + "data_flows": [ + { + "source": "request.args['q']", + "path": [ + { + "file_path": "app/search/index.ts", + "line": 20, + "component": "step_0_a", + "operation": "read" + }, + { + "file_path": "app/util/b.py", + "line": 40, + "component": "step_0_b", + "operation": "write" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/search/index.ts", + "app/db/raw.py", + "app/util/a.py", + "app/util/b.py" + ] + }, + { + "source": "request.cookies['session']", + "path": [ + { + "file_path": "app/auth/session_store.py", + "line": 21, + "component": "step_1_a", + "operation": "read" + }, + { + "file_path": "app/auth/session_store.py", + "line": 41, + "component": "step_1_b", + "operation": "write" + } + ], + "sink": "redis.set", + "sanitized": true, + "files": [ + "app/auth/session_store.py" + ] + }, + { + "source": "request.json['url']", + "path": [ + { + "file_path": "app/integrations/webhooks.go", + "line": 22, + "component": "step_2_a", + "operation": "read" + }, + { + "file_path": "app/net/client.go", + "line": 42, + "component": "step_2_b", + "operation": "write" + } + ], + "sink": "requests.get", + "sanitized": false, + "files": [ + "app/integrations/webhooks.go", + "app/net/client.go" + ] + }, + { + "source": "form['password']", + "path": [ + { + "file_path": "app/auth/service.py", + "line": 23, + "component": "step_3_a", + "operation": "read" + }, + { + "file_path": "app/obs/telemetry.go", + "line": 43, + "component": "step_3_b", + "operation": "write" + } + ], + "sink": "logger.info", + "sanitized": false, + "files": [ + "app/auth/service.py", + "app/obs/telemetry.go" + ] + }, + { + "source": "header['Authorization']", + "path": [ + { + "file_path": "app/common/jwt_tools.py", + "line": 24, + "component": "step_4_a", + "operation": "read" + }, + { + "file_path": "app/common/jwt_tools.py", + "line": 44, + "component": "step_4_b", + "operation": "write" + } + ], + "sink": "jwt.decode", + "sanitized": true, + "files": [ + "app/common/jwt_tools.py" + ] + }, + { + "source": "query['email']", + "path": [ + { + "file_path": "app/notify/mailer.rb", + "line": 25, + "component": "step_5_a", + "operation": "read" + }, + { + "file_path": "app/notify/mailer.rb", + "line": 45, + "component": "step_5_b", + "operation": "write" + } + ], + "sink": "smtp.send", + "sanitized": false, + "files": [ + "app/notify/mailer.rb" + ] + }, + { + "source": "body['card_number']", + "path": [ + { + "file_path": "app/billing/payments.go", + "line": 26, + "component": "step_6_a", + "operation": "read" + }, + { + "file_path": "app/billing/core.py", + "line": 46, + "component": "step_6_b", + "operation": "write" + } + ], + "sink": "stripe.Charge.create", + "sanitized": true, + "files": [ + "app/billing/payments.go", + "app/billing/core.py" + ] + }, + { + "source": "path_param['id']", + "path": [ + { + "file_path": "app/api/thing_1.py", + "line": 27, + "component": "step_7_a", + "operation": "read" + }, + { + "file_path": "app/api/thing_1.py", + "line": 47, + "component": "step_7_b", + "operation": "write" + } + ], + "sink": "orm.filter", + "sanitized": true, + "files": [ + "app/api/thing_1.py" + ] + }, + { + "source": "upload.filename", + "path": [ + { + "file_path": "app/media/uploads.py", + "line": 28, + "component": "step_8_a", + "operation": "read" + }, + { + "file_path": "app/x/w.py", + "line": 48, + "component": "step_8_b", + "operation": "write" + } + ], + "sink": "open", + "sanitized": false, + "files": [ + "app/media/uploads.py", + "app/common/cache.py", + "app/common/guard.py", + "app/x/y.py", + "app/x/z.py", + "app/x/w.py" + ] + }, + { + "source": "env['DEBUG']", + "path": [ + { + "file_path": "app/web/render.py", + "line": 29, + "component": "step_9_a", + "operation": "read" + }, + { + "file_path": "app/web/render.py", + "line": 49, + "component": "step_9_b", + "operation": "write" + } + ], + "sink": "template.render", + "sanitized": false, + "files": [ + "app/web/render.py" + ] + }, + { + "source": "request.body", + "path": [ + { + "file_path": "app/admin/views.py", + "line": 30, + "component": "step_10_a", + "operation": "read" + }, + { + "file_path": "app/admin/views.py", + "line": 50, + "component": "step_10_b", + "operation": "write" + } + ], + "sink": "eval", + "sanitized": false, + "files": [ + "app/admin/views.py" + ] + }, + { + "source": "websocket.msg", + "path": [ + { + "file_path": "app/ws/handler.py", + "line": 31, + "component": "step_11_a", + "operation": "read" + }, + { + "file_path": "app/ws/exec.py", + "line": 51, + "component": "step_11_b", + "operation": "write" + } + ], + "sink": "subprocess.run", + "sanitized": false, + "files": [ + "app/ws/handler.py", + "app/ws/exec.py" + ] + }, + { + "source": "cli.argv", + "path": [ + { + "file_path": "tools/run.py", + "line": 32, + "component": "step_12_a", + "operation": "read" + }, + { + "file_path": "tools/run.py", + "line": 52, + "component": "step_12_b", + "operation": "write" + } + ], + "sink": "os.system", + "sanitized": false, + "files": [ + "tools/run.py" + ] + }, + { + "source": "queue.payload", + "path": [ + { + "file_path": "app/worker/task.py", + "line": 33, + "component": "step_13_a", + "operation": "read" + }, + { + "file_path": "app/worker/task.py", + "line": 53, + "component": "step_13_b", + "operation": "write" + } + ], + "sink": "pickle.loads", + "sanitized": false, + "files": [ + "app/worker/task.py" + ] + }, + { + "source": "request.args['redirect']", + "path": [ + { + "file_path": "app/web/redirect.py", + "line": 34, + "component": "step_14_a", + "operation": "read" + }, + { + "file_path": "app/web/redirect.py", + "line": 54, + "component": "step_14_b", + "operation": "write" + } + ], + "sink": "HttpResponseRedirect", + "sanitized": false, + "files": [ + "app/web/redirect.py" + ] + }, + { + "source": "session['role']", + "path": [ + { + "file_path": "app/auth/permissions.py", + "line": 35, + "component": "step_15_a", + "operation": "read" + }, + { + "file_path": "app/auth/rbac.py", + "line": 55, + "component": "step_15_b", + "operation": "write" + } + ], + "sink": "permission_check", + "sanitized": true, + "files": [ + "app/auth/permissions.py", + "app/auth/rbac.py" + ] + }, + { + "source": "request.files['avatar']", + "path": [ + { + "file_path": "app/media/uploads.py", + "line": 36, + "component": "step_16_a", + "operation": "read" + }, + { + "file_path": "app/media/uploads.py", + "line": 56, + "component": "step_16_b", + "operation": "write" + } + ], + "sink": "s3.put_object", + "sanitized": true, + "files": [ + "app/media/uploads.py" + ] + }, + { + "source": "token", + "path": [ + { + "file_path": "app/common/cache.py", + "line": 37, + "component": "step_17_a", + "operation": "read" + }, + { + "file_path": "app/common/cache.py", + "line": 57, + "component": "step_17_b", + "operation": "write" + } + ], + "sink": "cache.set", + "sanitized": true, + "files": [ + "app/common/cache.py" + ] + }, + { + "source": "request.headers['X-User-Phone']", + "path": [ + { + "file_path": "app/obs/audit.py", + "line": 38, + "component": "step_18_a", + "operation": "read" + }, + { + "file_path": "app/obs/audit.py", + "line": 58, + "component": "step_18_b", + "operation": "write" + } + ], + "sink": "audit_log", + "sanitized": false, + "files": [ + "app/obs/audit.py" + ] + }, + { + "source": "graphql.variables", + "path": [ + { + "file_path": "app/api/graphql.ts", + "line": 39, + "component": "step_19_a", + "operation": "read" + }, + { + "file_path": "app/api/graphql.ts", + "line": 59, + "component": "step_19_b", + "operation": "write" + } + ], + "sink": "db.query", + "sanitized": false, + "files": [ + "app/api/graphql.ts" + ] + } + ] +} \ No newline at end of file diff --git a/internal/secaf/agents/hunt/testdata/golden/recon_context_block_business_logic_empty.txt b/internal/secaf/agents/hunt/testdata/golden/recon_context_block_business_logic_empty.txt new file mode 100644 index 0000000000..21f92b5ded --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/recon_context_block_business_logic_empty.txt @@ -0,0 +1,10 @@ +{ + "app_type": null, + "frameworks": [], + "languages": [], + "auth_model": "session", + "auth_details": "cookie", + "entry_points": [], + "api_surface": [], + "data_flows": [] +} \ No newline at end of file diff --git a/internal/secaf/agents/hunt/testdata/golden/recon_context_block_dos.txt b/internal/secaf/agents/hunt/testdata/golden/recon_context_block_dos.txt new file mode 100644 index 0000000000..a5c108b3a2 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/recon_context_block_dos.txt @@ -0,0 +1,346 @@ +{ + "app_type": "web_api", + "auth_model": "jwt", + "frameworks": [ + "Django", + "next.js", + "Spring Boot", + " React ", + "unknown-fw", + "NEXT", + "Vue" + ], + "languages": [ + "Python", + "JavaScript", + "Go", + "Ruby", + "Rust", + "python" + ], + "entry_points": [ + { + "kind": "http", + "identifier": "handler_0", + "file_path": "app/entry/e0.py", + "line": 10, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "HTTP", + "identifier": "handler_1", + "file_path": "app/entry/e1.py", + "line": 17, + "method": "POST", + "route": "/v1/resource/1", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_2", + "file_path": "app/entry/e2.py", + "line": 24, + "method": null, + "route": "/v1/resource/2", + "auth_required": null + }, + { + "kind": "api", + "identifier": "handler_3", + "file_path": "app/entry/e3.py", + "line": 31, + "method": "PUT", + "route": null, + "auth_required": true + }, + { + "kind": "graphql", + "identifier": "handler_4", + "file_path": "app/entry/e4.py", + "line": 38, + "method": "GET", + "route": "/v1/resource/4", + "auth_required": false + }, + { + "kind": "rpc", + "identifier": "handler_5", + "file_path": "app/entry/e5.py", + "line": 45, + "method": "POST", + "route": "/v1/resource/5", + "auth_required": null + }, + { + "kind": "route", + "identifier": "handler_6", + "file_path": "app/entry/e6.py", + "line": 52, + "method": null, + "route": null, + "auth_required": true + }, + { + "kind": "cron", + "identifier": "handler_7", + "file_path": "app/entry/e7.py", + "line": 59, + "method": "PUT", + "route": "/v1/resource/7", + "auth_required": false + }, + { + "kind": "queue", + "identifier": "handler_8", + "file_path": "app/entry/e8.py", + "line": 66, + "method": "GET", + "route": "/v1/resource/8", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_9", + "file_path": "app/entry/e9.py", + "line": 73, + "method": "POST", + "route": null, + "auth_required": true + } + ], + "data_flows": [ + { + "source": "request.args['q']", + "path": [ + { + "file_path": "app/search/index.ts", + "line": 20, + "component": "step_0_a", + "operation": "read" + }, + { + "file_path": "app/util/b.py", + "line": 40, + "component": "step_0_b", + "operation": "write" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/search/index.ts", + "app/db/raw.py", + "app/util/a.py", + "app/util/b.py" + ] + }, + { + "source": "request.cookies['session']", + "path": [ + { + "file_path": "app/auth/session_store.py", + "line": 21, + "component": "step_1_a", + "operation": "read" + }, + { + "file_path": "app/auth/session_store.py", + "line": 41, + "component": "step_1_b", + "operation": "write" + } + ], + "sink": "redis.set", + "sanitized": true, + "files": [ + "app/auth/session_store.py" + ] + }, + { + "source": "request.json['url']", + "path": [ + { + "file_path": "app/integrations/webhooks.go", + "line": 22, + "component": "step_2_a", + "operation": "read" + }, + { + "file_path": "app/net/client.go", + "line": 42, + "component": "step_2_b", + "operation": "write" + } + ], + "sink": "requests.get", + "sanitized": false, + "files": [ + "app/integrations/webhooks.go", + "app/net/client.go" + ] + }, + { + "source": "form['password']", + "path": [ + { + "file_path": "app/auth/service.py", + "line": 23, + "component": "step_3_a", + "operation": "read" + }, + { + "file_path": "app/obs/telemetry.go", + "line": 43, + "component": "step_3_b", + "operation": "write" + } + ], + "sink": "logger.info", + "sanitized": false, + "files": [ + "app/auth/service.py", + "app/obs/telemetry.go" + ] + }, + { + "source": "header['Authorization']", + "path": [ + { + "file_path": "app/common/jwt_tools.py", + "line": 24, + "component": "step_4_a", + "operation": "read" + }, + { + "file_path": "app/common/jwt_tools.py", + "line": 44, + "component": "step_4_b", + "operation": "write" + } + ], + "sink": "jwt.decode", + "sanitized": true, + "files": [ + "app/common/jwt_tools.py" + ] + }, + { + "source": "query['email']", + "path": [ + { + "file_path": "app/notify/mailer.rb", + "line": 25, + "component": "step_5_a", + "operation": "read" + }, + { + "file_path": "app/notify/mailer.rb", + "line": 45, + "component": "step_5_b", + "operation": "write" + } + ], + "sink": "smtp.send", + "sanitized": false, + "files": [ + "app/notify/mailer.rb" + ] + }, + { + "source": "body['card_number']", + "path": [ + { + "file_path": "app/billing/payments.go", + "line": 26, + "component": "step_6_a", + "operation": "read" + }, + { + "file_path": "app/billing/core.py", + "line": 46, + "component": "step_6_b", + "operation": "write" + } + ], + "sink": "stripe.Charge.create", + "sanitized": true, + "files": [ + "app/billing/payments.go", + "app/billing/core.py" + ] + }, + { + "source": "path_param['id']", + "path": [ + { + "file_path": "app/api/thing_1.py", + "line": 27, + "component": "step_7_a", + "operation": "read" + }, + { + "file_path": "app/api/thing_1.py", + "line": 47, + "component": "step_7_b", + "operation": "write" + } + ], + "sink": "orm.filter", + "sanitized": true, + "files": [ + "app/api/thing_1.py" + ] + }, + { + "source": "upload.filename", + "path": [ + { + "file_path": "app/media/uploads.py", + "line": 28, + "component": "step_8_a", + "operation": "read" + }, + { + "file_path": "app/x/w.py", + "line": 48, + "component": "step_8_b", + "operation": "write" + } + ], + "sink": "open", + "sanitized": false, + "files": [ + "app/media/uploads.py", + "app/common/cache.py", + "app/common/guard.py", + "app/x/y.py", + "app/x/z.py", + "app/x/w.py" + ] + }, + { + "source": "env['DEBUG']", + "path": [ + { + "file_path": "app/web/render.py", + "line": 29, + "component": "step_9_a", + "operation": "read" + }, + { + "file_path": "app/web/render.py", + "line": 49, + "component": "step_9_b", + "operation": "write" + } + ], + "sink": "template.render", + "sanitized": false, + "files": [ + "app/web/render.py" + ] + } + ] +} \ No newline at end of file diff --git a/internal/secaf/agents/hunt/testdata/golden/recon_context_block_dos_empty.txt b/internal/secaf/agents/hunt/testdata/golden/recon_context_block_dos_empty.txt new file mode 100644 index 0000000000..bf01954297 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/recon_context_block_dos_empty.txt @@ -0,0 +1,8 @@ +{ + "app_type": null, + "auth_model": "session", + "frameworks": [], + "languages": [], + "entry_points": [], + "data_flows": [] +} \ No newline at end of file diff --git a/internal/secaf/agents/hunt/testdata/golden/recon_context_block_ssrf.txt b/internal/secaf/agents/hunt/testdata/golden/recon_context_block_ssrf.txt new file mode 100644 index 0000000000..a5c108b3a2 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/recon_context_block_ssrf.txt @@ -0,0 +1,346 @@ +{ + "app_type": "web_api", + "auth_model": "jwt", + "frameworks": [ + "Django", + "next.js", + "Spring Boot", + " React ", + "unknown-fw", + "NEXT", + "Vue" + ], + "languages": [ + "Python", + "JavaScript", + "Go", + "Ruby", + "Rust", + "python" + ], + "entry_points": [ + { + "kind": "http", + "identifier": "handler_0", + "file_path": "app/entry/e0.py", + "line": 10, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "HTTP", + "identifier": "handler_1", + "file_path": "app/entry/e1.py", + "line": 17, + "method": "POST", + "route": "/v1/resource/1", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_2", + "file_path": "app/entry/e2.py", + "line": 24, + "method": null, + "route": "/v1/resource/2", + "auth_required": null + }, + { + "kind": "api", + "identifier": "handler_3", + "file_path": "app/entry/e3.py", + "line": 31, + "method": "PUT", + "route": null, + "auth_required": true + }, + { + "kind": "graphql", + "identifier": "handler_4", + "file_path": "app/entry/e4.py", + "line": 38, + "method": "GET", + "route": "/v1/resource/4", + "auth_required": false + }, + { + "kind": "rpc", + "identifier": "handler_5", + "file_path": "app/entry/e5.py", + "line": 45, + "method": "POST", + "route": "/v1/resource/5", + "auth_required": null + }, + { + "kind": "route", + "identifier": "handler_6", + "file_path": "app/entry/e6.py", + "line": 52, + "method": null, + "route": null, + "auth_required": true + }, + { + "kind": "cron", + "identifier": "handler_7", + "file_path": "app/entry/e7.py", + "line": 59, + "method": "PUT", + "route": "/v1/resource/7", + "auth_required": false + }, + { + "kind": "queue", + "identifier": "handler_8", + "file_path": "app/entry/e8.py", + "line": 66, + "method": "GET", + "route": "/v1/resource/8", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_9", + "file_path": "app/entry/e9.py", + "line": 73, + "method": "POST", + "route": null, + "auth_required": true + } + ], + "data_flows": [ + { + "source": "request.args['q']", + "path": [ + { + "file_path": "app/search/index.ts", + "line": 20, + "component": "step_0_a", + "operation": "read" + }, + { + "file_path": "app/util/b.py", + "line": 40, + "component": "step_0_b", + "operation": "write" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/search/index.ts", + "app/db/raw.py", + "app/util/a.py", + "app/util/b.py" + ] + }, + { + "source": "request.cookies['session']", + "path": [ + { + "file_path": "app/auth/session_store.py", + "line": 21, + "component": "step_1_a", + "operation": "read" + }, + { + "file_path": "app/auth/session_store.py", + "line": 41, + "component": "step_1_b", + "operation": "write" + } + ], + "sink": "redis.set", + "sanitized": true, + "files": [ + "app/auth/session_store.py" + ] + }, + { + "source": "request.json['url']", + "path": [ + { + "file_path": "app/integrations/webhooks.go", + "line": 22, + "component": "step_2_a", + "operation": "read" + }, + { + "file_path": "app/net/client.go", + "line": 42, + "component": "step_2_b", + "operation": "write" + } + ], + "sink": "requests.get", + "sanitized": false, + "files": [ + "app/integrations/webhooks.go", + "app/net/client.go" + ] + }, + { + "source": "form['password']", + "path": [ + { + "file_path": "app/auth/service.py", + "line": 23, + "component": "step_3_a", + "operation": "read" + }, + { + "file_path": "app/obs/telemetry.go", + "line": 43, + "component": "step_3_b", + "operation": "write" + } + ], + "sink": "logger.info", + "sanitized": false, + "files": [ + "app/auth/service.py", + "app/obs/telemetry.go" + ] + }, + { + "source": "header['Authorization']", + "path": [ + { + "file_path": "app/common/jwt_tools.py", + "line": 24, + "component": "step_4_a", + "operation": "read" + }, + { + "file_path": "app/common/jwt_tools.py", + "line": 44, + "component": "step_4_b", + "operation": "write" + } + ], + "sink": "jwt.decode", + "sanitized": true, + "files": [ + "app/common/jwt_tools.py" + ] + }, + { + "source": "query['email']", + "path": [ + { + "file_path": "app/notify/mailer.rb", + "line": 25, + "component": "step_5_a", + "operation": "read" + }, + { + "file_path": "app/notify/mailer.rb", + "line": 45, + "component": "step_5_b", + "operation": "write" + } + ], + "sink": "smtp.send", + "sanitized": false, + "files": [ + "app/notify/mailer.rb" + ] + }, + { + "source": "body['card_number']", + "path": [ + { + "file_path": "app/billing/payments.go", + "line": 26, + "component": "step_6_a", + "operation": "read" + }, + { + "file_path": "app/billing/core.py", + "line": 46, + "component": "step_6_b", + "operation": "write" + } + ], + "sink": "stripe.Charge.create", + "sanitized": true, + "files": [ + "app/billing/payments.go", + "app/billing/core.py" + ] + }, + { + "source": "path_param['id']", + "path": [ + { + "file_path": "app/api/thing_1.py", + "line": 27, + "component": "step_7_a", + "operation": "read" + }, + { + "file_path": "app/api/thing_1.py", + "line": 47, + "component": "step_7_b", + "operation": "write" + } + ], + "sink": "orm.filter", + "sanitized": true, + "files": [ + "app/api/thing_1.py" + ] + }, + { + "source": "upload.filename", + "path": [ + { + "file_path": "app/media/uploads.py", + "line": 28, + "component": "step_8_a", + "operation": "read" + }, + { + "file_path": "app/x/w.py", + "line": 48, + "component": "step_8_b", + "operation": "write" + } + ], + "sink": "open", + "sanitized": false, + "files": [ + "app/media/uploads.py", + "app/common/cache.py", + "app/common/guard.py", + "app/x/y.py", + "app/x/z.py", + "app/x/w.py" + ] + }, + { + "source": "env['DEBUG']", + "path": [ + { + "file_path": "app/web/render.py", + "line": 29, + "component": "step_9_a", + "operation": "read" + }, + { + "file_path": "app/web/render.py", + "line": 49, + "component": "step_9_b", + "operation": "write" + } + ], + "sink": "template.render", + "sanitized": false, + "files": [ + "app/web/render.py" + ] + } + ] +} \ No newline at end of file diff --git a/internal/secaf/agents/hunt/testdata/golden/recon_context_block_xss.txt b/internal/secaf/agents/hunt/testdata/golden/recon_context_block_xss.txt new file mode 100644 index 0000000000..a5c108b3a2 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/recon_context_block_xss.txt @@ -0,0 +1,346 @@ +{ + "app_type": "web_api", + "auth_model": "jwt", + "frameworks": [ + "Django", + "next.js", + "Spring Boot", + " React ", + "unknown-fw", + "NEXT", + "Vue" + ], + "languages": [ + "Python", + "JavaScript", + "Go", + "Ruby", + "Rust", + "python" + ], + "entry_points": [ + { + "kind": "http", + "identifier": "handler_0", + "file_path": "app/entry/e0.py", + "line": 10, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "HTTP", + "identifier": "handler_1", + "file_path": "app/entry/e1.py", + "line": 17, + "method": "POST", + "route": "/v1/resource/1", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_2", + "file_path": "app/entry/e2.py", + "line": 24, + "method": null, + "route": "/v1/resource/2", + "auth_required": null + }, + { + "kind": "api", + "identifier": "handler_3", + "file_path": "app/entry/e3.py", + "line": 31, + "method": "PUT", + "route": null, + "auth_required": true + }, + { + "kind": "graphql", + "identifier": "handler_4", + "file_path": "app/entry/e4.py", + "line": 38, + "method": "GET", + "route": "/v1/resource/4", + "auth_required": false + }, + { + "kind": "rpc", + "identifier": "handler_5", + "file_path": "app/entry/e5.py", + "line": 45, + "method": "POST", + "route": "/v1/resource/5", + "auth_required": null + }, + { + "kind": "route", + "identifier": "handler_6", + "file_path": "app/entry/e6.py", + "line": 52, + "method": null, + "route": null, + "auth_required": true + }, + { + "kind": "cron", + "identifier": "handler_7", + "file_path": "app/entry/e7.py", + "line": 59, + "method": "PUT", + "route": "/v1/resource/7", + "auth_required": false + }, + { + "kind": "queue", + "identifier": "handler_8", + "file_path": "app/entry/e8.py", + "line": 66, + "method": "GET", + "route": "/v1/resource/8", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_9", + "file_path": "app/entry/e9.py", + "line": 73, + "method": "POST", + "route": null, + "auth_required": true + } + ], + "data_flows": [ + { + "source": "request.args['q']", + "path": [ + { + "file_path": "app/search/index.ts", + "line": 20, + "component": "step_0_a", + "operation": "read" + }, + { + "file_path": "app/util/b.py", + "line": 40, + "component": "step_0_b", + "operation": "write" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/search/index.ts", + "app/db/raw.py", + "app/util/a.py", + "app/util/b.py" + ] + }, + { + "source": "request.cookies['session']", + "path": [ + { + "file_path": "app/auth/session_store.py", + "line": 21, + "component": "step_1_a", + "operation": "read" + }, + { + "file_path": "app/auth/session_store.py", + "line": 41, + "component": "step_1_b", + "operation": "write" + } + ], + "sink": "redis.set", + "sanitized": true, + "files": [ + "app/auth/session_store.py" + ] + }, + { + "source": "request.json['url']", + "path": [ + { + "file_path": "app/integrations/webhooks.go", + "line": 22, + "component": "step_2_a", + "operation": "read" + }, + { + "file_path": "app/net/client.go", + "line": 42, + "component": "step_2_b", + "operation": "write" + } + ], + "sink": "requests.get", + "sanitized": false, + "files": [ + "app/integrations/webhooks.go", + "app/net/client.go" + ] + }, + { + "source": "form['password']", + "path": [ + { + "file_path": "app/auth/service.py", + "line": 23, + "component": "step_3_a", + "operation": "read" + }, + { + "file_path": "app/obs/telemetry.go", + "line": 43, + "component": "step_3_b", + "operation": "write" + } + ], + "sink": "logger.info", + "sanitized": false, + "files": [ + "app/auth/service.py", + "app/obs/telemetry.go" + ] + }, + { + "source": "header['Authorization']", + "path": [ + { + "file_path": "app/common/jwt_tools.py", + "line": 24, + "component": "step_4_a", + "operation": "read" + }, + { + "file_path": "app/common/jwt_tools.py", + "line": 44, + "component": "step_4_b", + "operation": "write" + } + ], + "sink": "jwt.decode", + "sanitized": true, + "files": [ + "app/common/jwt_tools.py" + ] + }, + { + "source": "query['email']", + "path": [ + { + "file_path": "app/notify/mailer.rb", + "line": 25, + "component": "step_5_a", + "operation": "read" + }, + { + "file_path": "app/notify/mailer.rb", + "line": 45, + "component": "step_5_b", + "operation": "write" + } + ], + "sink": "smtp.send", + "sanitized": false, + "files": [ + "app/notify/mailer.rb" + ] + }, + { + "source": "body['card_number']", + "path": [ + { + "file_path": "app/billing/payments.go", + "line": 26, + "component": "step_6_a", + "operation": "read" + }, + { + "file_path": "app/billing/core.py", + "line": 46, + "component": "step_6_b", + "operation": "write" + } + ], + "sink": "stripe.Charge.create", + "sanitized": true, + "files": [ + "app/billing/payments.go", + "app/billing/core.py" + ] + }, + { + "source": "path_param['id']", + "path": [ + { + "file_path": "app/api/thing_1.py", + "line": 27, + "component": "step_7_a", + "operation": "read" + }, + { + "file_path": "app/api/thing_1.py", + "line": 47, + "component": "step_7_b", + "operation": "write" + } + ], + "sink": "orm.filter", + "sanitized": true, + "files": [ + "app/api/thing_1.py" + ] + }, + { + "source": "upload.filename", + "path": [ + { + "file_path": "app/media/uploads.py", + "line": 28, + "component": "step_8_a", + "operation": "read" + }, + { + "file_path": "app/x/w.py", + "line": 48, + "component": "step_8_b", + "operation": "write" + } + ], + "sink": "open", + "sanitized": false, + "files": [ + "app/media/uploads.py", + "app/common/cache.py", + "app/common/guard.py", + "app/x/y.py", + "app/x/z.py", + "app/x/w.py" + ] + }, + { + "source": "env['DEBUG']", + "path": [ + { + "file_path": "app/web/render.py", + "line": 29, + "component": "step_9_a", + "operation": "read" + }, + { + "file_path": "app/web/render.py", + "line": 49, + "component": "step_9_b", + "operation": "write" + } + ], + "sink": "template.render", + "sanitized": false, + "files": [ + "app/web/render.py" + ] + } + ] +} \ No newline at end of file diff --git a/internal/secaf/agents/hunt/testdata/golden/scan_locations_input.json b/internal/secaf/agents/hunt/testdata/golden/scan_locations_input.json new file mode 100644 index 0000000000..4c2e802898 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/scan_locations_input.json @@ -0,0 +1,3 @@ +{ + "hunter_prompt": "HUNTER PROMPT BODY\nwith {{braces}} and & caf\u00e9 \u2192\n" +} diff --git a/internal/secaf/agents/hunt/testdata/golden/scan_locations_prompt.txt b/internal/secaf/agents/hunt/testdata/golden/scan_locations_prompt.txt new file mode 100644 index 0000000000..828e313a68 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/scan_locations_prompt.txt @@ -0,0 +1,22 @@ +ROLE: +You are Step 1 LocationScanner for SEC-AF HUNT phase. + +TASK: +Find only plausible vulnerable code locations. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate the locations array with entries only. +- Each location must include file_path, start_line, code_snippet, and pattern_type. +- Keep results focused and deduplicated by location. + +CONSTRAINTS: +- Scan broadly and prioritize source-to-sink candidates. +- Do not enrich with severity, confidence, exploitability, or remediation details. +- If no credible locations exist, return an empty locations list. +- Do not include markdown or prose outside JSON. + +HUNTER CONTEXT: +HUNTER PROMPT BODY +with {{braces}} and & café → + diff --git a/internal/secaf/agents/hunt/testdata/golden/select_strategies.json b/internal/secaf/agents/hunt/testdata/golden/select_strategies.json new file mode 100644 index 0000000000..d699735d20 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/select_strategies.json @@ -0,0 +1,94 @@ +{ + "quick": [ + "injection", + "dos", + "ssrf", + "auth", + "data_exposure" + ], + "QUICK": [ + "injection", + "dos", + "ssrf", + "auth", + "data_exposure" + ], + " quick": [ + "injection", + "xss", + "dos", + "ssrf", + "auth", + "crypto", + "business_logic", + "data_exposure", + "supply_chain", + "config_secrets", + "api_security" + ], + "standard": [ + "injection", + "xss", + "dos", + "ssrf", + "auth", + "crypto", + "business_logic", + "data_exposure", + "supply_chain", + "config_secrets", + "api_security" + ], + "thorough": [ + "injection", + "xss", + "dos", + "ssrf", + "auth", + "crypto", + "business_logic", + "data_exposure", + "supply_chain", + "config_secrets", + "api_security" + ], + "Thorough": [ + "injection", + "xss", + "dos", + "ssrf", + "auth", + "crypto", + "business_logic", + "data_exposure", + "supply_chain", + "config_secrets", + "api_security" + ], + "bogus": [ + "injection", + "xss", + "dos", + "ssrf", + "auth", + "crypto", + "business_logic", + "data_exposure", + "supply_chain", + "config_secrets", + "api_security" + ], + "": [ + "injection", + "xss", + "dos", + "ssrf", + "auth", + "crypto", + "business_logic", + "data_exposure", + "supply_chain", + "config_secrets", + "api_security" + ] +} diff --git a/internal/secaf/agents/hunt/testdata/golden/strategy_runner_order.json b/internal/secaf/agents/hunt/testdata/golden/strategy_runner_order.json new file mode 100644 index 0000000000..3f55a88cbe --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/golden/strategy_runner_order.json @@ -0,0 +1,13 @@ +[ + "injection", + "xss", + "dos", + "ssrf", + "auth", + "crypto", + "business_logic", + "data_exposure", + "supply_chain", + "config_secrets", + "api_security" +] diff --git a/internal/secaf/agents/hunt/testdata/recon_fixture.json b/internal/secaf/agents/hunt/testdata/recon_fixture.json new file mode 100644 index 0000000000..bf916bb218 --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/recon_fixture.json @@ -0,0 +1,2189 @@ +{ + "architecture": { + "app_type": "web_api", + "modules": [ + { + "name": "auth_service", + "path": "app/auth/service.py", + "language": "python", + "description": "Session and JWT issuance", + "dependencies": [ + "jwt", + "redis" + ] + }, + { + "name": "rbac", + "path": "app/auth/rbac.py", + "language": "python", + "description": null, + "dependencies": [] + }, + { + "name": "billing", + "path": "app/billing/core.py", + "language": "python", + "description": "Invoice state machine — handles refunds", + "dependencies": [ + "stripe" + ] + }, + { + "name": "csrf_guard", + "path": "app/web/middleware/csrf.js", + "language": "javascript", + "description": "CSRF middleware", + "dependencies": [] + }, + { + "name": "payments", + "path": "app/billing/payments.go", + "language": "go", + "description": "Charge orchestration", + "dependencies": [ + "stripe-go" + ] + }, + { + "name": "reporting", + "path": "app/reports/render.py", + "language": "python", + "description": "PDF and CSV export", + "dependencies": [ + "weasyprint" + ] + }, + { + "name": "search", + "path": "app/search/index.ts", + "language": "typescript", + "description": "Elasticsearch query builder", + "dependencies": [ + "@elastic/elasticsearch" + ] + }, + { + "name": "session_store", + "path": "app/auth/session_store.py", + "language": "python", + "description": "Redis-backed sessions", + "dependencies": [ + "redis" + ] + }, + { + "name": "notifications", + "path": "app/notify/mailer.rb", + "language": "ruby", + "description": null, + "dependencies": [ + "mail" + ] + }, + { + "name": "permission_matrix", + "path": "app/auth/permissions.py", + "language": "python", + "description": "Role → permission table", + "dependencies": [] + }, + { + "name": "uploads", + "path": "app/media/uploads.py", + "language": "python", + "description": "S3 multipart upload helper", + "dependencies": [ + "boto3" + ] + }, + { + "name": "webhooks", + "path": "app/integrations/webhooks.go", + "language": "go", + "description": "Outbound webhook dispatcher", + "dependencies": [] + }, + { + "name": "graph_api", + "path": "app/api/graphql.ts", + "language": "typescript", + "description": "GraphQL resolvers — naïve depth limit", + "dependencies": [ + "graphql" + ] + }, + { + "name": "admin_panel", + "path": "app/admin/views.py", + "language": "python", + "description": "Django admin overrides", + "dependencies": [ + "django" + ] + }, + { + "name": "guard_utils", + "path": "app/common/guard.py", + "language": "python", + "description": "Assorted guard helpers", + "dependencies": [] + }, + { + "name": "cache", + "path": "app/common/cache.py", + "language": "python", + "description": "Memoization wrappers", + "dependencies": [] + }, + { + "name": "migrations", + "path": "db/migrate/2024_add_roles.rb", + "language": "ruby", + "description": "Adds role column", + "dependencies": [] + }, + { + "name": "jwt_tools", + "path": "app/common/jwt_tools.py", + "language": "python", + "description": null, + "dependencies": [ + "pyjwt" + ] + }, + { + "name": "telemetry", + "path": "app/obs/telemetry.go", + "language": "go", + "description": "OTel exporter", + "dependencies": [] + } + ], + "entry_points": [ + { + "kind": "http", + "identifier": "handler_0", + "file_path": "app/entry/e0.py", + "line": 10, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "HTTP", + "identifier": "handler_1", + "file_path": "app/entry/e1.py", + "line": 17, + "method": "POST", + "route": "/v1/resource/1", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_2", + "file_path": "app/entry/e2.py", + "line": 24, + "method": null, + "route": "/v1/resource/2", + "auth_required": null + }, + { + "kind": "api", + "identifier": "handler_3", + "file_path": "app/entry/e3.py", + "line": 31, + "method": "PUT", + "route": null, + "auth_required": true + }, + { + "kind": "graphql", + "identifier": "handler_4", + "file_path": "app/entry/e4.py", + "line": 38, + "method": "GET", + "route": "/v1/resource/4", + "auth_required": false + }, + { + "kind": "rpc", + "identifier": "handler_5", + "file_path": "app/entry/e5.py", + "line": 45, + "method": "POST", + "route": "/v1/resource/5", + "auth_required": null + }, + { + "kind": "route", + "identifier": "handler_6", + "file_path": "app/entry/e6.py", + "line": 52, + "method": null, + "route": null, + "auth_required": true + }, + { + "kind": "cron", + "identifier": "handler_7", + "file_path": "app/entry/e7.py", + "line": 59, + "method": "PUT", + "route": "/v1/resource/7", + "auth_required": false + }, + { + "kind": "queue", + "identifier": "handler_8", + "file_path": "app/entry/e8.py", + "line": 66, + "method": "GET", + "route": "/v1/resource/8", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_9", + "file_path": "app/entry/e9.py", + "line": 73, + "method": "POST", + "route": null, + "auth_required": true + }, + { + "kind": "api", + "identifier": "handler_10", + "file_path": "app/entry/e10.py", + "line": 80, + "method": null, + "route": "/v1/resource/10", + "auth_required": false + }, + { + "kind": "grpc", + "identifier": "handler_11", + "file_path": "app/entry/e11.py", + "line": 87, + "method": "PUT", + "route": "/v1/resource/11", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_12", + "file_path": "app/entry/e12.py", + "line": 94, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "route", + "identifier": "handler_13", + "file_path": "app/entry/e13.py", + "line": 101, + "method": "POST", + "route": "/v1/resource/13", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_14", + "file_path": "app/entry/e14.py", + "line": 108, + "method": null, + "route": "/v1/resource/14", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_15", + "file_path": "app/entry/e15.py", + "line": 115, + "method": "PUT", + "route": null, + "auth_required": true + }, + { + "kind": "websocket", + "identifier": "handler_16", + "file_path": "app/entry/e16.py", + "line": 122, + "method": "GET", + "route": "/v1/resource/16", + "auth_required": false + } + ], + "trust_boundaries": [ + { + "name": "boundary_0", + "source_zone": "internet", + "target_zone": "app", + "description": "Transition 0", + "enforcement": [] + }, + { + "name": "boundary_1", + "source_zone": "dmz", + "target_zone": "db", + "description": "Transition 1", + "enforcement": [ + "waf_1", + "mtls_1" + ] + }, + { + "name": "boundary_2", + "source_zone": "vpc", + "target_zone": "cache", + "description": "Transition 2", + "enforcement": [ + "waf_2", + "mtls_2" + ] + }, + { + "name": "boundary_3", + "source_zone": "worker", + "target_zone": "queue", + "description": "Transition 3", + "enforcement": [ + "waf_3", + "mtls_3" + ] + }, + { + "name": "boundary_4", + "source_zone": "internet", + "target_zone": "app", + "description": "Transition 4", + "enforcement": [] + }, + { + "name": "boundary_5", + "source_zone": "dmz", + "target_zone": "db", + "description": "Transition 5", + "enforcement": [ + "waf_5", + "mtls_5" + ] + }, + { + "name": "boundary_6", + "source_zone": "vpc", + "target_zone": "cache", + "description": "Transition 6", + "enforcement": [ + "waf_6", + "mtls_6" + ] + }, + { + "name": "boundary_7", + "source_zone": "worker", + "target_zone": "queue", + "description": "Transition 7", + "enforcement": [ + "waf_7", + "mtls_7" + ] + }, + { + "name": "boundary_8", + "source_zone": "internet", + "target_zone": "app", + "description": "Transition 8", + "enforcement": [] + }, + { + "name": "boundary_9", + "source_zone": "dmz", + "target_zone": "db", + "description": "Transition 9", + "enforcement": [ + "waf_9", + "mtls_9" + ] + }, + { + "name": "boundary_10", + "source_zone": "vpc", + "target_zone": "cache", + "description": "Transition 10", + "enforcement": [ + "waf_10", + "mtls_10" + ] + }, + { + "name": "boundary_11", + "source_zone": "worker", + "target_zone": "queue", + "description": "Transition 11", + "enforcement": [ + "waf_11", + "mtls_11" + ] + } + ], + "services": [ + { + "name": "postgres", + "service_type": "database", + "endpoint": "postgres://db:5432", + "purpose": "primary store", + "auth_mechanism": "password" + }, + { + "name": "redis", + "service_type": "cache", + "endpoint": null, + "purpose": "sessions", + "auth_mechanism": null + }, + { + "name": "stripe", + "service_type": "payments", + "endpoint": "https://api.stripe.com", + "purpose": null, + "auth_mechanism": "api_key" + }, + { + "name": "s3", + "service_type": "object_store", + "endpoint": "https://s3.amazonaws.com", + "purpose": "uploads", + "auth_mechanism": "iam" + }, + { + "name": "smtp", + "service_type": "mail", + "endpoint": null, + "purpose": "transactional email", + "auth_mechanism": null + } + ], + "api_surface": [ + { + "method": "GET", + "path": "/api/v1/thing/0", + "handler": "ThingController.action0", + "file_path": "app/api/thing_0.py", + "line": 100, + "auth_required": true, + "rate_limited": null + }, + { + "method": "POST", + "path": "/api/v1/thing/1", + "handler": "ThingController.action1", + "file_path": "app/api/thing_1.py", + "line": 103, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PUT", + "path": "/api/v1/thing/2", + "handler": "ThingController.action2", + "file_path": "app/api/thing_2.py", + "line": 106, + "auth_required": null, + "rate_limited": true + }, + { + "method": "DELETE", + "path": "/api/v1/thing/3", + "handler": "ThingController.action3", + "file_path": "app/api/thing_3.py", + "line": 109, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PATCH", + "path": "/api/v1/thing/4", + "handler": "ThingController.action4", + "file_path": "app/api/thing_4.py", + "line": 112, + "auth_required": true, + "rate_limited": null + }, + { + "method": "GET", + "path": "/api/v1/thing/5", + "handler": "ThingController.action5", + "file_path": "app/api/thing_5.py", + "line": 115, + "auth_required": null, + "rate_limited": true + }, + { + "method": "POST", + "path": "/api/v1/thing/6", + "handler": "ThingController.action6", + "file_path": "app/api/thing_6.py", + "line": 118, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PUT", + "path": "/api/v1/thing/7", + "handler": "ThingController.action7", + "file_path": "app/api/thing_7.py", + "line": 121, + "auth_required": false, + "rate_limited": false + }, + { + "method": "DELETE", + "path": "/api/v1/thing/8", + "handler": "ThingController.action8", + "file_path": "app/api/thing_8.py", + "line": 124, + "auth_required": null, + "rate_limited": true + }, + { + "method": "PATCH", + "path": "/api/v1/thing/9", + "handler": "ThingController.action9", + "file_path": "app/api/thing_9.py", + "line": 127, + "auth_required": false, + "rate_limited": false + }, + { + "method": "GET", + "path": "/api/v1/thing/10", + "handler": "ThingController.action10", + "file_path": "app/api/thing_10.py", + "line": 130, + "auth_required": true, + "rate_limited": null + }, + { + "method": "POST", + "path": "/api/v1/thing/11", + "handler": "ThingController.action11", + "file_path": "app/api/thing_11.py", + "line": 133, + "auth_required": null, + "rate_limited": true + }, + { + "method": "PUT", + "path": "/api/v1/thing/12", + "handler": "ThingController.action12", + "file_path": "app/api/thing_12.py", + "line": 136, + "auth_required": true, + "rate_limited": null + }, + { + "method": "DELETE", + "path": "/api/v1/thing/13", + "handler": "ThingController.action13", + "file_path": "app/api/thing_13.py", + "line": 139, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PATCH", + "path": "/api/v1/thing/14", + "handler": "ThingController.action14", + "file_path": "app/api/thing_14.py", + "line": 142, + "auth_required": null, + "rate_limited": true + }, + { + "method": "GET", + "path": "/api/v1/thing/15", + "handler": "ThingController.action15", + "file_path": "app/api/thing_15.py", + "line": 145, + "auth_required": false, + "rate_limited": false + }, + { + "method": "POST", + "path": "/api/v1/thing/16", + "handler": "ThingController.action16", + "file_path": "app/api/thing_16.py", + "line": 148, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PUT", + "path": "/api/v1/thing/17", + "handler": "ThingController.action17", + "file_path": "app/api/thing_17.py", + "line": 151, + "auth_required": null, + "rate_limited": true + }, + { + "method": "DELETE", + "path": "/api/v1/thing/18", + "handler": "ThingController.action18", + "file_path": "app/api/thing_18.py", + "line": 154, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PATCH", + "path": "/api/v1/thing/19", + "handler": "ThingController.action19", + "file_path": "app/api/thing_19.py", + "line": 157, + "auth_required": false, + "rate_limited": false + } + ] + }, + "data_flows": { + "flows": [ + { + "source": "request.args['q']", + "path": [ + { + "file_path": "app/search/index.ts", + "line": 20, + "component": "step_0_a", + "operation": "read" + }, + { + "file_path": "app/util/b.py", + "line": 40, + "component": "step_0_b", + "operation": "write" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/search/index.ts", + "app/db/raw.py", + "app/util/a.py", + "app/util/b.py" + ] + }, + { + "source": "request.cookies['session']", + "path": [ + { + "file_path": "app/auth/session_store.py", + "line": 21, + "component": "step_1_a", + "operation": "read" + }, + { + "file_path": "app/auth/session_store.py", + "line": 41, + "component": "step_1_b", + "operation": "write" + } + ], + "sink": "redis.set", + "sanitized": true, + "files": [ + "app/auth/session_store.py" + ] + }, + { + "source": "request.json['url']", + "path": [ + { + "file_path": "app/integrations/webhooks.go", + "line": 22, + "component": "step_2_a", + "operation": "read" + }, + { + "file_path": "app/net/client.go", + "line": 42, + "component": "step_2_b", + "operation": "write" + } + ], + "sink": "requests.get", + "sanitized": false, + "files": [ + "app/integrations/webhooks.go", + "app/net/client.go" + ] + }, + { + "source": "form['password']", + "path": [ + { + "file_path": "app/auth/service.py", + "line": 23, + "component": "step_3_a", + "operation": "read" + }, + { + "file_path": "app/obs/telemetry.go", + "line": 43, + "component": "step_3_b", + "operation": "write" + } + ], + "sink": "logger.info", + "sanitized": false, + "files": [ + "app/auth/service.py", + "app/obs/telemetry.go" + ] + }, + { + "source": "header['Authorization']", + "path": [ + { + "file_path": "app/common/jwt_tools.py", + "line": 24, + "component": "step_4_a", + "operation": "read" + }, + { + "file_path": "app/common/jwt_tools.py", + "line": 44, + "component": "step_4_b", + "operation": "write" + } + ], + "sink": "jwt.decode", + "sanitized": true, + "files": [ + "app/common/jwt_tools.py" + ] + }, + { + "source": "query['email']", + "path": [ + { + "file_path": "app/notify/mailer.rb", + "line": 25, + "component": "step_5_a", + "operation": "read" + }, + { + "file_path": "app/notify/mailer.rb", + "line": 45, + "component": "step_5_b", + "operation": "write" + } + ], + "sink": "smtp.send", + "sanitized": false, + "files": [ + "app/notify/mailer.rb" + ] + }, + { + "source": "body['card_number']", + "path": [ + { + "file_path": "app/billing/payments.go", + "line": 26, + "component": "step_6_a", + "operation": "read" + }, + { + "file_path": "app/billing/core.py", + "line": 46, + "component": "step_6_b", + "operation": "write" + } + ], + "sink": "stripe.Charge.create", + "sanitized": true, + "files": [ + "app/billing/payments.go", + "app/billing/core.py" + ] + }, + { + "source": "path_param['id']", + "path": [ + { + "file_path": "app/api/thing_1.py", + "line": 27, + "component": "step_7_a", + "operation": "read" + }, + { + "file_path": "app/api/thing_1.py", + "line": 47, + "component": "step_7_b", + "operation": "write" + } + ], + "sink": "orm.filter", + "sanitized": true, + "files": [ + "app/api/thing_1.py" + ] + }, + { + "source": "upload.filename", + "path": [ + { + "file_path": "app/media/uploads.py", + "line": 28, + "component": "step_8_a", + "operation": "read" + }, + { + "file_path": "app/x/w.py", + "line": 48, + "component": "step_8_b", + "operation": "write" + } + ], + "sink": "open", + "sanitized": false, + "files": [ + "app/media/uploads.py", + "app/common/cache.py", + "app/common/guard.py", + "app/x/y.py", + "app/x/z.py", + "app/x/w.py" + ] + }, + { + "source": "env['DEBUG']", + "path": [ + { + "file_path": "app/web/render.py", + "line": 29, + "component": "step_9_a", + "operation": "read" + }, + { + "file_path": "app/web/render.py", + "line": 49, + "component": "step_9_b", + "operation": "write" + } + ], + "sink": "template.render", + "sanitized": false, + "files": [ + "app/web/render.py" + ] + }, + { + "source": "request.body", + "path": [ + { + "file_path": "app/admin/views.py", + "line": 30, + "component": "step_10_a", + "operation": "read" + }, + { + "file_path": "app/admin/views.py", + "line": 50, + "component": "step_10_b", + "operation": "write" + } + ], + "sink": "eval", + "sanitized": false, + "files": [ + "app/admin/views.py" + ] + }, + { + "source": "websocket.msg", + "path": [ + { + "file_path": "app/ws/handler.py", + "line": 31, + "component": "step_11_a", + "operation": "read" + }, + { + "file_path": "app/ws/exec.py", + "line": 51, + "component": "step_11_b", + "operation": "write" + } + ], + "sink": "subprocess.run", + "sanitized": false, + "files": [ + "app/ws/handler.py", + "app/ws/exec.py" + ] + }, + { + "source": "cli.argv", + "path": [ + { + "file_path": "tools/run.py", + "line": 32, + "component": "step_12_a", + "operation": "read" + }, + { + "file_path": "tools/run.py", + "line": 52, + "component": "step_12_b", + "operation": "write" + } + ], + "sink": "os.system", + "sanitized": false, + "files": [ + "tools/run.py" + ] + }, + { + "source": "queue.payload", + "path": [ + { + "file_path": "app/worker/task.py", + "line": 33, + "component": "step_13_a", + "operation": "read" + }, + { + "file_path": "app/worker/task.py", + "line": 53, + "component": "step_13_b", + "operation": "write" + } + ], + "sink": "pickle.loads", + "sanitized": false, + "files": [ + "app/worker/task.py" + ] + }, + { + "source": "request.args['redirect']", + "path": [ + { + "file_path": "app/web/redirect.py", + "line": 34, + "component": "step_14_a", + "operation": "read" + }, + { + "file_path": "app/web/redirect.py", + "line": 54, + "component": "step_14_b", + "operation": "write" + } + ], + "sink": "HttpResponseRedirect", + "sanitized": false, + "files": [ + "app/web/redirect.py" + ] + }, + { + "source": "session['role']", + "path": [ + { + "file_path": "app/auth/permissions.py", + "line": 35, + "component": "step_15_a", + "operation": "read" + }, + { + "file_path": "app/auth/rbac.py", + "line": 55, + "component": "step_15_b", + "operation": "write" + } + ], + "sink": "permission_check", + "sanitized": true, + "files": [ + "app/auth/permissions.py", + "app/auth/rbac.py" + ] + }, + { + "source": "request.files['avatar']", + "path": [ + { + "file_path": "app/media/uploads.py", + "line": 36, + "component": "step_16_a", + "operation": "read" + }, + { + "file_path": "app/media/uploads.py", + "line": 56, + "component": "step_16_b", + "operation": "write" + } + ], + "sink": "s3.put_object", + "sanitized": true, + "files": [ + "app/media/uploads.py" + ] + }, + { + "source": "token", + "path": [ + { + "file_path": "app/common/cache.py", + "line": 37, + "component": "step_17_a", + "operation": "read" + }, + { + "file_path": "app/common/cache.py", + "line": 57, + "component": "step_17_b", + "operation": "write" + } + ], + "sink": "cache.set", + "sanitized": true, + "files": [ + "app/common/cache.py" + ] + }, + { + "source": "request.headers['X-User-Phone']", + "path": [ + { + "file_path": "app/obs/audit.py", + "line": 38, + "component": "step_18_a", + "operation": "read" + }, + { + "file_path": "app/obs/audit.py", + "line": 58, + "component": "step_18_b", + "operation": "write" + } + ], + "sink": "audit_log", + "sanitized": false, + "files": [ + "app/obs/audit.py" + ] + }, + { + "source": "graphql.variables", + "path": [ + { + "file_path": "app/api/graphql.ts", + "line": 39, + "component": "step_19_a", + "operation": "read" + }, + { + "file_path": "app/api/graphql.ts", + "line": 59, + "component": "step_19_b", + "operation": "write" + } + ], + "sink": "db.query", + "sanitized": false, + "files": [ + "app/api/graphql.ts" + ] + } + ], + "sanitization_points": [ + { + "file_path": "app/sanitize/s0.py", + "line": 5, + "function_name": null, + "sanitization_type": "escape", + "protects_against": [] + }, + { + "file_path": "app/sanitize/s1.py", + "line": 9, + "function_name": "clean_1", + "sanitization_type": "parameterize", + "protects_against": [ + "CWE-80", + "CWE-90" + ] + }, + { + "file_path": "app/sanitize/s2.py", + "line": 13, + "function_name": "clean_2", + "sanitization_type": "allowlist", + "protects_against": [ + "CWE-81", + "CWE-91" + ] + }, + { + "file_path": "app/sanitize/s3.py", + "line": 17, + "function_name": "clean_3", + "sanitization_type": "encode", + "protects_against": [] + }, + { + "file_path": "app/sanitize/s4.py", + "line": 21, + "function_name": "clean_4", + "sanitization_type": "escape", + "protects_against": [ + "CWE-83", + "CWE-93" + ] + }, + { + "file_path": "app/sanitize/s5.py", + "line": 25, + "function_name": null, + "sanitization_type": "parameterize", + "protects_against": [ + "CWE-84", + "CWE-94" + ] + }, + { + "file_path": "app/sanitize/s6.py", + "line": 29, + "function_name": "clean_6", + "sanitization_type": "allowlist", + "protects_against": [] + }, + { + "file_path": "app/sanitize/s7.py", + "line": 33, + "function_name": "clean_7", + "sanitization_type": "encode", + "protects_against": [ + "CWE-86", + "CWE-96" + ] + }, + { + "file_path": "app/sanitize/s8.py", + "line": 37, + "function_name": "clean_8", + "sanitization_type": "escape", + "protects_against": [ + "CWE-87", + "CWE-97" + ] + }, + { + "file_path": "app/sanitize/s9.py", + "line": 41, + "function_name": "clean_9", + "sanitization_type": "parameterize", + "protects_against": [] + }, + { + "file_path": "app/sanitize/s10.py", + "line": 45, + "function_name": null, + "sanitization_type": "allowlist", + "protects_against": [ + "CWE-89", + "CWE-99" + ] + }, + { + "file_path": "app/sanitize/s11.py", + "line": 49, + "function_name": "clean_11", + "sanitization_type": "encode", + "protects_against": [ + "CWE-90", + "CWE-100" + ] + } + ], + "sinks": [ + { + "sink_type": "sql", + "file_path": "app/sink/k0.py", + "line": 200, + "function_name": null, + "exploitability_notes": null + }, + { + "sink_type": "command", + "file_path": "app/sink/k1.py", + "line": 202, + "function_name": "sink_fn_1", + "exploitability_notes": "note 1" + }, + { + "sink_type": "template", + "file_path": "app/sink/k2.py", + "line": 204, + "function_name": "sink_fn_2", + "exploitability_notes": "note 2" + }, + { + "sink_type": "file", + "file_path": "app/sink/k3.py", + "line": 206, + "function_name": "sink_fn_3", + "exploitability_notes": null + }, + { + "sink_type": "network", + "file_path": "app/sink/k4.py", + "line": 208, + "function_name": null, + "exploitability_notes": "note 4" + }, + { + "sink_type": "deserialization", + "file_path": "app/sink/k5.py", + "line": 210, + "function_name": "sink_fn_5", + "exploitability_notes": "note 5" + }, + { + "sink_type": "sql", + "file_path": "app/sink/k6.py", + "line": 212, + "function_name": "sink_fn_6", + "exploitability_notes": null + }, + { + "sink_type": "command", + "file_path": "app/sink/k7.py", + "line": 214, + "function_name": "sink_fn_7", + "exploitability_notes": "note 7" + }, + { + "sink_type": "template", + "file_path": "app/sink/k8.py", + "line": 216, + "function_name": null, + "exploitability_notes": "note 8" + }, + { + "sink_type": "file", + "file_path": "app/sink/k9.py", + "line": 218, + "function_name": "sink_fn_9", + "exploitability_notes": null + }, + { + "sink_type": "network", + "file_path": "app/sink/k10.py", + "line": 220, + "function_name": "sink_fn_10", + "exploitability_notes": "note 10" + }, + { + "sink_type": "deserialization", + "file_path": "app/sink/k11.py", + "line": 222, + "function_name": "sink_fn_11", + "exploitability_notes": "note 11" + }, + { + "sink_type": "sql", + "file_path": "app/sink/k12.py", + "line": 224, + "function_name": null, + "exploitability_notes": null + }, + { + "sink_type": "command", + "file_path": "app/sink/k13.py", + "line": 226, + "function_name": "sink_fn_13", + "exploitability_notes": "note 13" + }, + { + "sink_type": "template", + "file_path": "app/sink/k14.py", + "line": 228, + "function_name": "sink_fn_14", + "exploitability_notes": "note 14" + }, + { + "sink_type": "file", + "file_path": "app/sink/k15.py", + "line": 230, + "function_name": "sink_fn_15", + "exploitability_notes": null + }, + { + "sink_type": "network", + "file_path": "app/sink/k16.py", + "line": 232, + "function_name": null, + "exploitability_notes": "note 16" + }, + { + "sink_type": "deserialization", + "file_path": "app/sink/k17.py", + "line": 234, + "function_name": "sink_fn_17", + "exploitability_notes": "note 17" + } + ] + }, + "dependencies": { + "sbom": [ + { + "name": "django", + "version": "4.2.1", + "ecosystem": "pypi", + "direct": true, + "license": "BSD-3-Clause" + }, + { + "name": "requests", + "version": "2.31.0", + "ecosystem": "pypi", + "direct": true, + "license": "Apache-2.0" + }, + { + "name": "urllib3", + "version": "1.26.5", + "ecosystem": "pypi", + "direct": false, + "license": "MIT" + }, + { + "name": "jinja2", + "version": "3.1.2", + "ecosystem": "pypi", + "direct": false, + "license": null + }, + { + "name": "lodash", + "version": "4.17.19", + "ecosystem": "npm", + "direct": true, + "license": "MIT" + }, + { + "name": "express", + "version": "4.17.1", + "ecosystem": "npm", + "direct": true, + "license": "MIT" + }, + { + "name": "minimist", + "version": "1.2.5", + "ecosystem": "npm", + "direct": false, + "license": "MIT" + }, + { + "name": "axios", + "version": "0.21.1", + "ecosystem": "npm", + "direct": false, + "license": "MIT" + }, + { + "name": "github.com/gin-gonic/gin", + "version": "1.9.0", + "ecosystem": "go", + "direct": true, + "license": "MIT" + }, + { + "name": "golang.org/x/net", + "version": "0.7.0", + "ecosystem": "go", + "direct": false, + "license": "BSD-3-Clause" + }, + { + "name": "rails", + "version": "7.0.4", + "ecosystem": "rubygems", + "direct": true, + "license": "MIT" + }, + { + "name": "nokogiri", + "version": "1.13.6", + "ecosystem": "rubygems", + "direct": false, + "license": "MIT" + }, + { + "name": "django", + "version": "4.2.1", + "ecosystem": "pypi", + "direct": true, + "license": "BSD-3-Clause" + }, + { + "name": "requests", + "version": "2.31.0", + "ecosystem": "pypi", + "direct": true, + "license": "Apache-2.0" + } + ], + "known_cves": [ + { + "cve_id": "CVE-2023-0001", + "package": "django", + "installed_version": "4.2.1", + "fixed_version": "4.2.5", + "cvss_v4_score": 9.8, + "epss_score": 0.5, + "direct": true, + "reachable": true + }, + { + "cve_id": "CVE-2023-0002", + "package": "urllib3", + "installed_version": "1.26.5", + "fixed_version": null, + "cvss_v4_score": 7.5, + "epss_score": 0.1, + "direct": false, + "reachable": false + }, + { + "cve_id": "CVE-2023-0003", + "package": "lodash", + "installed_version": "4.17.19", + "fixed_version": "4.17.21", + "cvss_v4_score": 9.8, + "epss_score": 0.9, + "direct": true, + "reachable": true + }, + { + "cve_id": "CVE-2023-0004", + "package": "minimist", + "installed_version": "1.2.5", + "fixed_version": "1.2.6", + "cvss_v4_score": null, + "epss_score": null, + "direct": false, + "reachable": null + }, + { + "cve_id": "CVE-2023-0005", + "package": "axios", + "installed_version": "0.21.1", + "fixed_version": "0.21.2", + "cvss_v4_score": 5.3, + "epss_score": 0.02, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0006", + "package": "jinja2", + "installed_version": "3.1.2", + "fixed_version": null, + "cvss_v4_score": 6.1, + "epss_score": null, + "direct": false, + "reachable": false + }, + { + "cve_id": "CVE-2023-0007", + "package": "express", + "installed_version": "4.17.1", + "fixed_version": "4.18.0", + "cvss_v4_score": 4.3, + "epss_score": 0.005, + "direct": true, + "reachable": null + }, + { + "cve_id": "CVE-2023-0008", + "package": "nokogiri", + "installed_version": "1.13.6", + "fixed_version": "1.13.10", + "cvss_v4_score": 8.8, + "epss_score": 0.3, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0009", + "package": "golang.org/x/net", + "installed_version": "0.7.0", + "fixed_version": "0.17.0", + "cvss_v4_score": 7.5, + "epss_score": 0.44, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0010", + "package": "rails", + "installed_version": "7.0.4", + "fixed_version": "7.0.7", + "cvss_v4_score": 9.1, + "epss_score": 0.6, + "direct": true, + "reachable": false + }, + { + "cve_id": "CVE-2023-0011", + "package": "gin", + "installed_version": "1.9.0", + "fixed_version": null, + "cvss_v4_score": 3.7, + "epss_score": 0.001, + "direct": true, + "reachable": null + }, + { + "cve_id": "CVE-2023-0012", + "package": "requests", + "installed_version": "2.31.0", + "fixed_version": "2.32.0", + "cvss_v4_score": 6.5, + "epss_score": 0.07, + "direct": true, + "reachable": true + }, + { + "cve_id": "CVE-2023-0013", + "package": "pyyaml", + "installed_version": "5.3", + "fixed_version": "5.4", + "cvss_v4_score": 9.8, + "epss_score": 0.5, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0014", + "package": "log4j", + "installed_version": "2.14.0", + "fixed_version": "2.17.1", + "cvss_v4_score": 10.0, + "epss_score": 0.97, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0015", + "package": "openssl", + "installed_version": "1.1.1k", + "fixed_version": "1.1.1t", + "cvss_v4_score": 7.4, + "epss_score": 0.12, + "direct": false, + "reachable": null + }, + { + "cve_id": "CVE-2023-0016", + "package": "zlib", + "installed_version": "1.2.11", + "fixed_version": "1.2.12", + "cvss_v4_score": 8.2, + "epss_score": 0.04, + "direct": false, + "reachable": false + }, + { + "cve_id": "CVE-2023-0017", + "package": "libxml2", + "installed_version": "2.9.10", + "fixed_version": null, + "cvss_v4_score": null, + "epss_score": 0.25, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0018", + "package": "curl", + "installed_version": "7.68.0", + "fixed_version": "7.88.0", + "cvss_v4_score": 8.1, + "epss_score": null, + "direct": true, + "reachable": true + } + ], + "outdated": [ + { + "package": "pkg-0", + "current_version": "1.0.0", + "latest_version": "2.0.0", + "direct": true + }, + { + "package": "pkg-1", + "current_version": "1.1.0", + "latest_version": "2.1.0", + "direct": false + }, + { + "package": "pkg-2", + "current_version": "1.2.0", + "latest_version": "2.2.0", + "direct": true + }, + { + "package": "pkg-3", + "current_version": "1.3.0", + "latest_version": "2.3.0", + "direct": false + }, + { + "package": "pkg-4", + "current_version": "1.4.0", + "latest_version": "2.4.0", + "direct": true + }, + { + "package": "pkg-5", + "current_version": "1.5.0", + "latest_version": "2.5.0", + "direct": false + }, + { + "package": "pkg-6", + "current_version": "1.6.0", + "latest_version": "2.6.0", + "direct": true + }, + { + "package": "pkg-7", + "current_version": "1.7.0", + "latest_version": "2.7.0", + "direct": false + }, + { + "package": "pkg-8", + "current_version": "1.8.0", + "latest_version": "2.8.0", + "direct": true + }, + { + "package": "pkg-9", + "current_version": "1.9.0", + "latest_version": "2.9.0", + "direct": false + }, + { + "package": "pkg-10", + "current_version": "1.10.0", + "latest_version": "2.10.0", + "direct": true + }, + { + "package": "pkg-11", + "current_version": "1.11.0", + "latest_version": "2.11.0", + "direct": false + } + ], + "direct_count": 7, + "transitive_count": 143 + }, + "config": { + "secrets": [ + { + "id": "secret-00", + "secret_type": "aws_access_key", + "file_path": "config/env_0.yaml", + "line": 3, + "match": "AKIA****0000", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-01", + "secret_type": "github_token", + "file_path": "config/env_1.yaml", + "line": 4, + "match": "AKIA****0001", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-02", + "secret_type": "private_key", + "file_path": "config/env_2.yaml", + "line": 5, + "match": "AKIA****0002", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-03", + "secret_type": "slack_webhook", + "file_path": "config/env_3.yaml", + "line": 6, + "match": "AKIA****0003", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-04", + "secret_type": "generic_api_key", + "file_path": "config/env_4.yaml", + "line": 7, + "match": "AKIA****0004", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-05", + "secret_type": "aws_access_key", + "file_path": "config/env_5.yaml", + "line": 8, + "match": "AKIA****0005", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-06", + "secret_type": "github_token", + "file_path": "config/env_6.yaml", + "line": 9, + "match": "AKIA****0006", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-07", + "secret_type": "private_key", + "file_path": "config/env_7.yaml", + "line": 10, + "match": "AKIA****0007", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-08", + "secret_type": "slack_webhook", + "file_path": "config/env_8.yaml", + "line": 11, + "match": "AKIA****0008", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-09", + "secret_type": "generic_api_key", + "file_path": "config/env_9.yaml", + "line": 12, + "match": "AKIA****0009", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-10", + "secret_type": "aws_access_key", + "file_path": "config/env_10.yaml", + "line": 13, + "match": "AKIA****0010", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-11", + "secret_type": "github_token", + "file_path": "config/env_11.yaml", + "line": 14, + "match": "AKIA****0011", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-12", + "secret_type": "private_key", + "file_path": "config/env_12.yaml", + "line": 15, + "match": "AKIA****0012", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-13", + "secret_type": "slack_webhook", + "file_path": "config/env_13.yaml", + "line": 16, + "match": "AKIA****0013", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-14", + "secret_type": "generic_api_key", + "file_path": "config/env_14.yaml", + "line": 17, + "match": "AKIA****0014", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-15", + "secret_type": "aws_access_key", + "file_path": "config/env_15.yaml", + "line": 18, + "match": "AKIA****0015", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-16", + "secret_type": "github_token", + "file_path": "config/env_16.yaml", + "line": 19, + "match": "AKIA****0016", + "confidence": "medium", + "is_test_value": true + } + ], + "misconfigs": [ + { + "id": "misconfig-00", + "category": "logging", + "file_path": "config/app_0.ini", + "line": null, + "key": "LOG_LEVEL", + "value": "DEBUG", + "risk": "high", + "remediation": "Debug logging in production" + }, + { + "id": "misconfig-01", + "category": "tls", + "file_path": "config/app_1.ini", + "line": 13, + "key": "SSL_VERIFY", + "value": "false", + "risk": "critical", + "remediation": "TLS verification disabled" + }, + { + "id": "misconfig-02", + "category": "cors", + "file_path": "config/app_2.ini", + "line": 14, + "key": "ALLOW_ORIGIN", + "value": "*", + "risk": "high", + "remediation": null + }, + { + "id": "misconfig-03", + "category": "headers", + "file_path": "config/app_3.ini", + "line": 0, + "key": "X_FRAME_OPTIONS", + "value": null, + "risk": "medium", + "remediation": "Missing clickjacking header" + }, + { + "id": "misconfig-04", + "category": "debug", + "file_path": "config/app_4.ini", + "line": 16, + "key": "DJANGO_DEBUG", + "value": "True", + "risk": "critical", + "remediation": "Debug mode enabled" + }, + { + "id": "misconfig-05", + "category": "storage", + "file_path": "config/app_5.ini", + "line": null, + "key": "BUCKET_ACL", + "value": "public-read", + "risk": "critical", + "remediation": "Public bucket" + }, + { + "id": "misconfig-06", + "category": "auth", + "file_path": "config/app_6.ini", + "line": 18, + "key": "SESSION_TIMEOUT", + "value": "999999", + "risk": "medium", + "remediation": null + }, + { + "id": "misconfig-07", + "category": "http", + "file_path": "config/app_7.ini", + "line": 19, + "key": "REDIRECT_HTTPS", + "value": "false", + "risk": "high", + "remediation": "Plain HTTP allowed" + }, + { + "id": "misconfig-08", + "category": "secrets", + "file_path": "config/app_8.ini", + "line": 20, + "key": null, + "value": "inline", + "risk": "medium", + "remediation": "Inline secret" + }, + { + "id": "misconfig-09", + "category": "trace", + "file_path": "config/app_9.ini", + "line": 21, + "key": "OTEL_TRACE_ALL", + "value": "true", + "risk": "low", + "remediation": "Verbose tracing" + }, + { + "id": "misconfig-10", + "category": "network", + "file_path": "config/app_10.ini", + "line": null, + "key": "BIND_ADDR", + "value": "0.0.0.0", + "risk": "medium", + "remediation": null + }, + { + "id": "misconfig-11", + "category": "errors", + "file_path": "config/app_11.ini", + "line": 23, + "key": "SHOW_STACKTRACE", + "value": "true", + "risk": "high", + "remediation": "Error stacktraces exposed" + }, + { + "id": "misconfig-12", + "category": "cache", + "file_path": "config/app_12.ini", + "line": 24, + "key": "CACHE_TTL", + "value": "0", + "risk": "low", + "remediation": null + }, + { + "id": "misconfig-13", + "category": "db", + "file_path": "config/app_13.ini", + "line": 25, + "key": "SSLMODE", + "value": "disable", + "risk": "critical", + "remediation": "Database TLS off" + }, + { + "id": "misconfig-14", + "category": "queue", + "file_path": "config/app_14.ini", + "line": 26, + "key": "PREFETCH", + "value": "1000", + "risk": "low", + "remediation": null + }, + { + "id": "misconfig-15", + "category": "exposure", + "file_path": "config/app_15.ini", + "line": null, + "key": "ADMIN_PATH", + "value": "/admin", + "risk": "medium", + "remediation": "Admin surface exposed" + } + ] + }, + "security_context": { + "auth_model": "jwt", + "auth_details": "HS256 JWTs issued by app/auth/service.py, refresh tokens in Redis", + "crypto_usage": [ + { + "algorithm": "AES", + "key_size": 256, + "mode": "GCM", + "usage_context": "at-rest encryption", + "is_weak": false + }, + { + "algorithm": "MD5", + "key_size": null, + "mode": null, + "usage_context": "password hashing", + "is_weak": true + }, + { + "algorithm": "RSA", + "key_size": 1024, + "mode": null, + "usage_context": "token signing", + "is_weak": true + }, + { + "algorithm": "SHA-256", + "key_size": null, + "mode": null, + "usage_context": null, + "is_weak": false + }, + { + "algorithm": "DES", + "key_size": 56, + "mode": "CBC", + "usage_context": "legacy export", + "is_weak": true + }, + { + "algorithm": "HMAC-SHA1", + "key_size": 160, + "mode": null, + "usage_context": "webhook signature", + "is_weak": true + }, + { + "algorithm": "ChaCha20", + "key_size": 256, + "mode": "Poly1305", + "usage_context": "transport", + "is_weak": false + }, + { + "algorithm": "bcrypt", + "key_size": null, + "mode": null, + "usage_context": "password hashing", + "is_weak": false + }, + { + "algorithm": "RC4", + "key_size": 128, + "mode": null, + "usage_context": null, + "is_weak": true + }, + { + "algorithm": "ECDSA", + "key_size": 256, + "mode": null, + "usage_context": "JWT signing", + "is_weak": null + }, + { + "algorithm": "PBKDF2", + "key_size": null, + "mode": null, + "usage_context": "key derivation", + "is_weak": false + }, + { + "algorithm": "SHA-1", + "key_size": null, + "mode": null, + "usage_context": "checksum", + "is_weak": true + }, + { + "algorithm": "AES", + "key_size": 128, + "mode": "ECB", + "usage_context": "legacy blob", + "is_weak": true + }, + { + "algorithm": "Ed25519", + "key_size": 256, + "mode": null, + "usage_context": "package signing", + "is_weak": false + }, + { + "algorithm": "3DES", + "key_size": 168, + "mode": "CBC", + "usage_context": "legacy tape", + "is_weak": true + }, + { + "algorithm": "Argon2id", + "key_size": null, + "mode": null, + "usage_context": "password hashing", + "is_weak": null + } + ], + "framework_security": [ + "django.middleware.csrf.CsrfViewMiddleware", + "", + "helmet defaults", + "SecurityMiddleware", + "rack-protection", + "spring-security filter chain" + ], + "security_headers": [ + "Content-Security-Policy: default-src 'self'", + "X-Content-Type-Options: nosniff", + "Strict-Transport-Security: max-age=31536000", + "X-Frame-Options: DENY", + "Referrer-Policy: no-referrer", + "", + "Permissions-Policy: geolocation=()" + ], + "deployment_signals": [ + "kubernetes ingress with TLS termination", + "docker-compose exposes 5432", + "no WAF in front of /api", + "secrets mounted from vault", + "readiness probe on /healthz", + "single replica for the worker", + "TLS 1.2 minimum", + "internal service mesh mTLS" + ] + }, + "languages": [ + "Python", + "JavaScript", + "Go", + "Ruby", + "Rust", + "python" + ], + "frameworks": [ + "Django", + "next.js", + "Spring Boot", + " React ", + "unknown-fw", + "NEXT", + "Vue" + ], + "lines_of_code": 184213, + "file_count": 2417, + "recon_duration_seconds": 42.5 +} diff --git a/internal/secaf/agents/hunt/testdata/recon_small.json b/internal/secaf/agents/hunt/testdata/recon_small.json new file mode 100644 index 0000000000..33f644c9da --- /dev/null +++ b/internal/secaf/agents/hunt/testdata/recon_small.json @@ -0,0 +1,195 @@ +{ + "architecture": { + "app_type": "web_api", + "modules": [ + { + "name": "api", + "path": "app/api/", + "language": "Python", + "description": "HTTP layer", + "dependencies": [ + "db" + ] + } + ], + "entry_points": [ + { + "kind": "http", + "identifier": "POST /login", + "file_path": "app/api/auth.py", + "line": 12, + "method": "POST", + "route": "/login", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "seed", + "file_path": "app/cli.py", + "line": 3, + "method": null, + "route": null, + "auth_required": null + } + ], + "trust_boundaries": [ + { + "name": "edge", + "source_zone": "internet", + "target_zone": "app", + "description": "TLS terminator", + "enforcement": [ + "waf" + ] + } + ], + "services": [ + { + "name": "postgres", + "service_type": "database", + "endpoint": "db:5432", + "purpose": "primary store", + "auth_mechanism": "password" + } + ], + "api_surface": [ + { + "method": "POST", + "path": "/login", + "handler": "login", + "file_path": "app/api/auth.py", + "line": 12, + "auth_required": false, + "rate_limited": false + }, + { + "method": "GET", + "path": "/users/{id}", + "handler": "get_user", + "file_path": "app/api/users.py", + "line": 40, + "auth_required": true, + "rate_limited": true + } + ] + }, + "data_flows": { + "flows": [ + { + "source": "request.json", + "path": [ + { + "file_path": "app/api/users.py", + "line": 41, + "component": "handler", + "operation": "read id" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/api/users.py" + ] + } + ], + "sanitization_points": [], + "sinks": [ + { + "sink_type": "sql", + "file_path": "app/api/users.py", + "line": 42, + "function_name": "get_user", + "exploitability_notes": "f-string query" + } + ] + }, + "dependencies": { + "sbom": [ + { + "name": "django", + "version": "4.2.1", + "ecosystem": "pypi", + "direct": true, + "license": "BSD-3" + } + ], + "known_cves": [ + { + "cve_id": "CVE-2024-0001", + "package": "django", + "installed_version": "4.2.1", + "fixed_version": "4.2.11", + "cvss_v4_score": 7.5, + "epss_score": 0.42, + "direct": true, + "reachable": true + } + ], + "outdated": [], + "direct_count": 3, + "transitive_count": 9 + }, + "config": { + "secrets": [ + { + "id": "", + "secret_type": "api_key", + "file_path": ".env", + "line": 2, + "match": "AKIA...", + "confidence": "high", + "is_test_value": false + } + ], + "misconfigs": [ + { + "id": "", + "category": "debug", + "file_path": "settings.py", + "line": 9, + "key": "DEBUG", + "value": "True", + "risk": "high", + "remediation": "disable in production" + } + ] + }, + "security_context": { + "auth_model": "jwt", + "auth_details": "HS256 access tokens", + "crypto_usage": [ + { + "algorithm": "MD5", + "key_size": null, + "mode": null, + "usage_context": "password hashing", + "is_weak": true + }, + { + "algorithm": "SHA256", + "key_size": null, + "mode": null, + "usage_context": "etag cache key", + "is_weak": false + } + ], + "framework_security": [ + "django-csrf" + ], + "security_headers": [ + "Content-Security-Policy" + ], + "deployment_signals": [ + "docker" + ] + }, + "languages": [ + "python" + ], + "frameworks": [ + "django" + ], + "lines_of_code": 1234, + "file_count": 56, + "recon_duration_seconds": 0.0 +} diff --git a/internal/secaf/agents/hunt/xss.go b/internal/secaf/agents/hunt/xss.go new file mode 100644 index 0000000000..ced94b31d3 --- /dev/null +++ b/internal/secaf/agents/hunt/xss.go @@ -0,0 +1,62 @@ +package hunt + +// Ports src/sec_af/agents/hunt/xss.py. + +import ( + "context" + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/recontext" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +const xssPromptPath = "hunt/xss.txt" + +// xssScanPrompt builds the exact prompt run_xss_hunter sends. +// +// Python parity: xss.py is the only one of the three inline-JSON hunters whose +// CONTEXT block carries an explicit "- Target CWEs:" line. +func xssScanPrompt(repoPath string, recon schemas.ReconResult, depth, earlyStop string) (scanPrompt, reconContext string) { + reconContext = entryFlowContextBlock(recon) + template := prompts.MustLoad(xssPromptPath) + template = strings.ReplaceAll(template, "{{RECON_CONTEXT_JSON}}", reconContext) + template = strings.ReplaceAll(template, "{{LANGUAGE_HINTS}}", recontext.LanguageHintsForContext(recon)) + template = strings.ReplaceAll(template, "{{FRAMEWORK_HINTS}}", recontext.FrameworkHintsForContext(recon)) + + scanPrompt = template + + "\n\nCONTEXT:\n" + + "- Repository path: " + repoPath + "\n" + + "- Depth profile: " + depth + "\n" + + "- Early stop rule: if you inspect " + earlyStop + + " files without credible signal, stop and return empty findings.\n" + + "- Focus on RECON entry points and data flows as primary source-to-sink paths.\n" + + "- Explore the codebase, trace user-controlled data into rendering sinks, and identify XSS/client-side injection points.\n" + + "- Target CWEs: CWE-79, CWE-80, CWE-87, CWE-116.\n" + + "- Take multiple turns to build findings incrementally and write final JSON only when complete." + return scanPrompt, reconContext +} + +func runXSSHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, depth, earlyStop string, +) (schemas.HuntResult, error) { + scanPrompt, reconContext := xssScanPrompt(repoPath, recon, depth, earlyStop) + return runHunterBody(ctx, app, repoPath, hunterSpec{ + ScanPrompt: scanPrompt, + ReconContext: reconContext, + FindingType: "sast", + Strategy: "xss", + EmptyStrategiesRun: nil, // Python: bare HuntResult() + }) +} + +// RunXSSHunter ports xss.py run_xss_hunter. +func RunXSSHunter( + ctx context.Context, app appx.Harnesser, repoPath string, + recon schemas.ReconResult, depth string, maxFilesWithoutSignal int, +) (schemas.HuntResult, error) { + return runXSSHunter(ctx, app, repoPath, recon, depth, strconv.Itoa(maxFilesWithoutSignal)) +} diff --git a/internal/secaf/agents/prove/agents_test.go b/internal/secaf/agents/prove/agents_test.go new file mode 100644 index 0000000000..22da949384 --- /dev/null +++ b/internal/secaf/agents/prove/agents_test.go @@ -0,0 +1,318 @@ +package prove + +// Tests for the six schema-driven prove sub-agents (tracer, sanitization, +// exploit, cross-service, DAST, dependency reachability) that share the +// `tempfile.mkdtemp -> app.harness -> extract_harness_result -> rmtree` shape. +// +// Validation contract: +// - each agent runs with Cwd set to a fresh PRIVATE temp dir named +// `secaf--*` and ProjectDir set to the repository, and the temp +// dir is removed whether the run succeeds or fails; +// - each agent asks for the pydantic schema of its destination model; +// - a harness failure surfaces as ` harness error: `; +// - a run with no parsed output surfaces as +// ` did not return a valid `. + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// runAgent invokes one sub-agent against a scripted fake and returns the fake. +type agentCase struct { + name string + tempPrefix string + extractName string + modelName string + canned func() (json.RawMessage, error) + run func(context.Context, appx.Harnesser) error +} + +func proveAgentCases() []agentCase { + repo := fixtureRepo + return []agentCase{ + { + name: "tracer", tempPrefix: "secaf-prove-tracer-", + extractName: "DataFlowTracer", modelName: "DataFlowTrace", + canned: func() (json.RawMessage, error) { return json.Marshal(traceRich()) }, + run: func(ctx context.Context, app appx.Harnesser) error { + _, err := RunTracer(ctx, app, repo, findingRich(), "quick") + return err + }, + }, + { + name: "sanitization", tempPrefix: "secaf-prove-sanitization-", + extractName: "SanitizationAnalyzer", modelName: "SanitizationResult", + canned: func() (json.RawMessage, error) { return json.Marshal(sanitizationRich()) }, + run: func(ctx context.Context, app appx.Harnesser) error { + _, err := RunSanitizationAnalyzer(ctx, app, repo, findingRich(), traceRich(), "quick") + return err + }, + }, + { + name: "exploit", tempPrefix: "secaf-prove-exploit-", + extractName: "ExploitHypothesizer", modelName: "ExploitHypothesis", + canned: func() (json.RawMessage, error) { return json.Marshal(exploitRich()) }, + run: func(ctx context.Context, app appx.Harnesser) error { + _, err := RunExploitHypothesizer(ctx, app, repo, findingRich(), traceRich(), sanitizationRich(), "quick") + return err + }, + }, + { + name: "cross_service", tempPrefix: "secaf-prove-cross-service-", + extractName: "CrossServiceAnalyzer", modelName: "CrossServiceFinding", + canned: func() (json.RawMessage, error) { + return json.Marshal(schemas.CrossServiceFinding{ + ChainDescription: "d", ServicesInvolved: []string{"a"}, EntryPoint: "e", Impact: "i", + }) + }, + run: func(ctx context.Context, app appx.Harnesser) error { + _, err := RunCrossServiceAnalyzer(ctx, app, repo, []string{"a"}, "summary", "quick") + return err + }, + }, + { + name: "dast", tempPrefix: "secaf-prove-dast-", + extractName: "DastVerifier", modelName: "DastVerificationResult", + canned: func() (json.RawMessage, error) { + return json.Marshal(schemas.DastVerificationResult{ + PayloadSent: "p", ResponseSummary: "r", ExploitConfirmed: true, SafetyNotes: "s", + }) + }, + run: func(ctx context.Context, app appx.Harnesser) error { + _, err := RunDastVerifier(ctx, app, repo, findingRich(), "payload", "quick") + return err + }, + }, + { + name: "dep_reachability", tempPrefix: "secaf-prove-dep-reachability-", + extractName: "DependencyReachabilityAnalyzer", modelName: "ReachabilityProof", + canned: func() (json.RawMessage, error) { + return json.Marshal(schemas.ReachabilityProof{ + VulnerableFunction: "f", CallChain: []string{"a"}, Reachable: true, Direct: false, + }) + }, + run: func(ctx context.Context, app appx.Harnesser) error { + _, err := RunDepReachability(ctx, app, repo, map[string]any{"cve": "CVE-1"}, "quick") + return err + }, + }, + } +} + +// TestAgentTempDirsAreIsolatedAndRemoved pins +// `tempfile.mkdtemp(prefix=f"secaf-{agent_name}-")` / +// `shutil.rmtree(harness_cwd, ignore_errors=True)`, plus project_dir=repo_path. +func TestAgentTempDirsAreIsolatedAndRemoved(t *testing.T) { + for _, tc := range proveAgentCases() { + app := &appx.Fake{HarnessFn: appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return tc.canned() + })} + if err := tc.run(context.Background(), app); err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + if len(app.Harnesses) != 1 { + t.Fatalf("%s: want 1 harness call, got %d", tc.name, len(app.Harnesses)) + } + h := app.Harnesses[0] + if h.Opts.Cwd == "" { + t.Fatalf("%s: ran with no Cwd", tc.name) + } + if base := filepath.Base(h.Opts.Cwd); !strings.HasPrefix(base, tc.tempPrefix) { + t.Errorf("%s: Cwd base = %q, want prefix %q", tc.name, base, tc.tempPrefix) + } + if h.Opts.Cwd == fixtureRepo { + t.Errorf("%s: the harness must run in a scratch dir, not the repository", tc.name) + } + if h.Opts.ProjectDir != fixtureRepo { + t.Errorf("%s: ProjectDir = %q, want %q", tc.name, h.Opts.ProjectDir, fixtureRepo) + } + if _, err := os.Stat(h.Opts.Cwd); !os.IsNotExist(err) { + t.Errorf("%s: temp dir %q still exists after the agent returned (err=%v)", tc.name, h.Opts.Cwd, err) + } + } +} + +// TestAgentTempDirRemovedOnFailure pins the `finally:` — the scratch dir goes +// away even when extract_harness_result raises. +func TestAgentTempDirRemovedOnFailure(t *testing.T) { + for _, tc := range proveAgentCases() { + app := &appx.Fake{HarnessFn: appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return nil, errors.New("provider exploded") + })} + if err := tc.run(context.Background(), app); err == nil { + t.Fatalf("%s: want an error", tc.name) + } + cwd := app.Harnesses[0].Opts.Cwd + if _, err := os.Stat(cwd); !os.IsNotExist(err) { + t.Errorf("%s: temp dir %q survived a failure (err=%v)", tc.name, cwd, err) + } + } +} + +// TestAgentHarnessErrorMessages pins extract_harness_result's two error +// spellings, including the AGENT NAME each module passes (which differs from +// the temp-dir name). +func TestAgentHarnessErrorMessages(t *testing.T) { + for _, tc := range proveAgentCases() { + app := &appx.Fake{HarnessFn: appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return nil, errors.New("provider exploded") + })} + err := tc.run(context.Background(), app) + want := tc.extractName + " harness error: provider exploded" + if err == nil || err.Error() != want { + t.Errorf("%s: error = %v, want %q", tc.name, err, want) + } + + // A run that produced no parsed value is the TypeError branch. + app = &appx.Fake{HarnessFn: func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{Result: "not json"}, nil + }} + err = tc.run(context.Background(), app) + want = tc.extractName + " did not return a valid " + tc.modelName + if err == nil || err.Error() != want { + t.Errorf("%s: error = %v, want %q", tc.name, err, want) + } + } +} + +// TestAgentSchemasComeFromPydanticFixtures pins that each agent asks for the +// committed pydantic schema of its destination model rather than a Go +// reflection of it — an invopop schema would mark every field required and +// reject valid Python output. +func TestAgentSchemasComeFromPydanticFixtures(t *testing.T) { + for _, tc := range proveAgentCases() { + app := &appx.Fake{HarnessFn: appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return tc.canned() + })} + if err := tc.run(context.Background(), app); err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + schema := app.Harnesses[0].Schema + if schema == nil { + t.Fatalf("%s: no schema was passed", tc.name) + } + if title, _ := schema["title"].(string); title != tc.modelName { + t.Errorf("%s: schema title = %v, want %q (the pydantic fixture)", tc.name, schema["title"], tc.modelName) + } + } +} + +// TestFindingDataFlowRendering pins `_finding_data_flow`'s two branches and its +// literal key order (a Go map would sort them). +func TestFindingDataFlowRendering(t *testing.T) { + if got := findingDataFlow(findingBare()); got != "[]" { + t.Errorf("no data flow must render as []; got %q", got) + } + empty := findingRich() + empty.DataFlow = []schemas.ReconDataFlowStep{} + if got := findingDataFlow(empty); got != "[]" { + t.Errorf("an EMPTY data flow is falsy in Python too; got %q", got) + } + got := findingDataFlow(findingRich()) + want := `[ + { + "file_path": "src/routes.py", + "line": 10, + "component": "handler", + "operation": "read request.args" + }, + { + "file_path": "src/users.py", + "line": 42, + "component": "db", + "operation": "execute" + } +]` + if got != want { + t.Errorf("findingDataFlow =\n%s\nwant\n%s", got, want) + } +} + +// TestRelatedFilesJSONNilIsEmptyList pins that a nil Go slice renders as +// Python's `[]` — RawFinding.related_files can never be None. +func TestRelatedFilesJSONNilIsEmptyList(t *testing.T) { + if got := relatedFilesJSON(nil); got != "[]" { + t.Errorf("relatedFilesJSON(nil) = %q, want []", got) + } +} + +// TestTraceContextRendering pins the shared `_trace_context` helper, including +// its "no concrete trace steps" placeholder and yes/no rendering. +func TestTraceContextRendering(t *testing.T) { + got := traceContext(traceBare()) + want := "Source: unknown\nSink: unknown\nSink reached: no\nTrace steps:\n- (no concrete trace steps)" + if got != want { + t.Errorf("traceContext(empty) =\n%q\nwant\n%q", got, want) + } + got = traceContext(traceRich()) + want = "Source: request.args['id']\nSink: cursor.execute(query)\nSink reached: yes\n" + + "Trace steps:\n- src/routes.py:10 read request.args\n- src/users.py:42 execute" + if got != want { + t.Errorf("traceContext(rich) =\n%q\nwant\n%q", got, want) + } +} + +// TestSanitizationContextTriState pins that `sufficient` distinguishes None +// (unknown) from False (no) — the reason SanitizationResult.Sufficient is a +// pointer. +func TestSanitizationContextTriState(t *testing.T) { + for _, tc := range []struct { + sufficient *bool + want string + }{ + {nil, "Sanitization sufficient: unknown"}, + {boolp(false), "Sanitization sufficient: no"}, + {boolp(true), "Sanitization sufficient: yes"}, + } { + got := sanitizationContext(schemas.SanitizationResult{Sufficient: tc.sufficient}) + if !strings.Contains(got, tc.want) { + t.Errorf("sanitizationContext(%v) =\n%s\nwant it to contain %q", tc.sufficient, got, tc.want) + } + } + // `type` / `bypass_method` use Python's `or`, so BOTH None and "" -> "none". + for _, value := range []*string{nil, str("")} { + got := sanitizationContext(schemas.SanitizationResult{Type: value, BypassMethod: value}) + if !strings.Contains(got, "Sanitization type: none") || !strings.Contains(got, "Bypass method: none") { + t.Errorf("a falsy optional must render as 'none'; got\n%s", got) + } + } +} + +// TestVerdictContextRendersPythonBools pins verdict.py's raw f-string +// interpolation: booleans print as True/False and a None `sufficient` prints as +// None, unlike the yes/no/unknown mapping every other block uses. +func TestVerdictContextRendersPythonBools(t *testing.T) { + got := verdictBuildContext(traceRich(), sanitizationRich(), exploitRich()) + for _, want := range []string{ + "- sink_reached: True", + "- found: True", + "- sufficient: False", + } { + if !strings.Contains(got, want) { + t.Errorf("verdict context missing %q; got\n%s", want, got) + } + } + got = verdictBuildContext(traceBare(), sanitizationBare(), exploitBare()) + for _, want := range []string{ + "- sink_reached: False", + "- steps:\n- (none)", + "- found: False", + "- type: none", + "- sufficient: None", + "- bypass_method: none", + "- payload: none", + } { + if !strings.Contains(got, want) { + t.Errorf("verdict context missing %q; got\n%s", want, got) + } + } +} diff --git a/internal/secaf/agents/prove/assembler.go b/internal/secaf/agents/prove/assembler.go new file mode 100644 index 0000000000..b8f19ad6bc --- /dev/null +++ b/internal/secaf/agents/prove/assembler.go @@ -0,0 +1,181 @@ +package prove + +// Ports src/sec_af/agents/prove/assembler.py — the pure function that folds the +// four sub-agent outputs into one schemas.VerifiedFinding. + +import ( + "strconv" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// verdictMap ports assembler.py `_VERDICT_MAP`. Anything not in it falls back +// to INCONCLUSIVE, which is how a model that invents a verdict word (e.g. +// "unverified") is absorbed rather than raising. +var verdictMap = map[string]schemas.ExploitVerdict{ + "confirmed": schemas.VerdictConfirmed, + "likely": schemas.VerdictLikely, + "inconclusive": schemas.VerdictInconclusive, + "not_exploitable": schemas.VerdictNotExploitable, +} + +// toEvidenceLevel ports `_to_evidence_level`: +// +// bounded = max(1, min(6, level)) +// return EvidenceLevel(bounded) +// +// The clamp is what makes the IntEnum construction total: VerdictDecision +// declares evidence_level as a bare int, so a model answering 0 or 9 must not +// blow up. +func toEvidenceLevel(level int) schemas.EvidenceLevel { + bounded := level + if bounded > 6 { + bounded = 6 + } + if bounded < 1 { + bounded = 1 + } + return schemas.EvidenceLevel(bounded) +} + +// toDataFlowSteps ports `_to_data_flow_steps`: +// +// for index, step in enumerate(trace.steps, start=1): +// rows.append(DataFlowStep(file=f"trace_step_{index}", line=index, +// description=step, tainted=True)) +// +// The synthetic file/line are deliberate: the tracer returns free-text steps, +// and the evidence artifact wants a positional handle for each one. +// +// The result is always a non-nil slice, matching Python's `rows: list = []` — +// so `Proof.data_flow_trace` marshals as `[]`, never `null`. +func toDataFlowSteps(trace schemas.DataFlowTrace) []schemas.DataFlowStep { + rows := make([]schemas.DataFlowStep, 0, len(trace.Steps)) + for i, step := range trace.Steps { + index := i + 1 + rows = append(rows, schemas.DataFlowStep{ + File: "trace_step_" + strconv.Itoa(index), + Line: index, + Description: step, + Tainted: true, + }) + } + return rows +} + +// reproductionSteps ports `_reproduction_steps`: nothing to reproduce when the +// verdict is NOT_EXPLOITABLE, otherwise a fixed two-step recipe whose second +// step carries the exploit hypothesis/payload/outcome verbatim. +// +// Python parity: `command=exploit.payload` passes the Optional straight +// through, so a None payload yields a step with `command: null`. +func reproductionSteps(verdict schemas.ExploitVerdict, exploit schemas.ExploitHypothesis) []schemas.ReproductionStep { + if verdict == schemas.VerdictNotExploitable { + return []schemas.ReproductionStep{} + } + step1Desc := "Trace attacker-controlled input from source to sink in target code path." + step1Out := "Input reaches a sensitive sink." + return []schemas.ReproductionStep{ + {Step: 1, Description: step1Desc, Command: nil, ExpectedOutput: &step1Out}, + {Step: 2, Description: exploit.Hypothesis, Command: exploit.Payload, ExpectedOutput: &exploit.ExpectedOutcome}, + } +} + +// AssembleVerifiedFinding ports assembler.py assemble_verified_finding. +// +// Python parity notes: +// +// - `bypass_possible=bool(sanitization.bypass_method)` is Python truthiness on +// an `str | None`: None AND the empty string are both False. The field is +// `bool | None` in the schema but assembler always supplies a real bool, so +// the Go pointer is always non-nil here. +// - `data_flow_trace` and `data_flow_evidence.steps` are the SAME list object +// in Python; Go copies the slice header into both, which is equivalent for +// everything downstream (nobody mutates it). +// - `tags=[]`, `exploitability_score=0.0` and `sarif_security_severity=0.0` +// are literal here — `_apply_metadata` overwrites the two scores afterwards. +// - `related_locations` / `compliance` are not passed, so they take pydantic's +// `default_factory=list`; Go seeds them explicitly so model_dump() parity +// holds (`[]`, not `null`). +func AssembleVerifiedFinding( + finding schemas.RawFinding, + dataFlowTrace schemas.DataFlowTrace, + sanitization schemas.SanitizationResult, + exploit schemas.ExploitHypothesis, + verdictDecision schemas.VerdictDecision, +) schemas.VerifiedFinding { + verdict, ok := verdictMap[verdictDecision.Verdict] + if !ok { + verdict = schemas.VerdictInconclusive + } + evidenceLevel := toEvidenceLevel(verdictDecision.EvidenceLevel) + dataFlowSteps := toDataFlowSteps(dataFlowTrace) + + source := dataFlowTrace.Source + sink := dataFlowTrace.Sink + bypassPossible := sanitization.BypassMethod != nil && *sanitization.BypassMethod != "" + + proof := schemas.Proof{ + ExploitHypothesis: exploit.Hypothesis, + VerificationMethod: "composite_subagent_chain:" + string(finding.FindingType), + EvidenceLevel: evidenceLevel, + DataFlowTrace: dataFlowSteps, + DataFlowEvidence: &schemas.DataFlowEvidence{ + Steps: dataFlowSteps, + Source: &source, + Sink: &sink, + SinkReached: dataFlowTrace.SinkReached, + }, + SanitizationAnalysis: &schemas.SanitizationAnalysis{ + SanitizationFound: sanitization.Found, + SanitizationType: sanitization.Type, + SanitizationSufficient: sanitization.Sufficient, + BypassPossible: &bypassPossible, + BypassMethod: sanitization.BypassMethod, + }, + ExploitPayload: exploit.Payload, + ExpectedOutcome: &exploit.ExpectedOutcome, + } + + return schemas.VerifiedFinding{ + ID: finding.ID, + Fingerprint: finding.Fingerprint, + Title: finding.Title, + Description: finding.Description, + FindingType: finding.FindingType, + CweID: finding.CweID, + CweName: finding.CweName, + OwaspCategory: finding.OwaspCategory, + Verdict: verdict, + EvidenceLevel: evidenceLevel, + Rationale: verdictDecision.Rationale, + Severity: finding.EstimatedSeverity, + Tags: []string{}, + ExploitabilityScore: 0.0, + Proof: &proof, + Location: locationOf(finding), + RelatedLocations: []schemas.Location{}, + Compliance: []schemas.ComplianceMapping{}, + ReproductionSteps: reproductionSteps(verdict, exploit), + SarifRuleID: sarifRuleID(finding.FindingType, finding.CweName), + SarifSecuritySeverity: 0.0, + DropReason: nil, + } +} + +// locationOf builds the Location both assembler.py and verifier.py construct +// from a RawFinding. `start_column` / `end_column` are left at pydantic's None. +// +// Python parity: `code_snippet` is a REQUIRED str on RawFinding but an +// Optional on Location, so an empty snippet crosses over as "" rather than +// null — the pointer is always non-nil. +func locationOf(finding schemas.RawFinding) schemas.Location { + snippet := finding.CodeSnippet + return schemas.Location{ + FilePath: finding.FilePath, + StartLine: finding.StartLine, + EndLine: finding.EndLine, + FunctionName: finding.FunctionName, + CodeSnippet: &snippet, + } +} diff --git a/internal/secaf/agents/prove/assembler_test.go b/internal/secaf/agents/prove/assembler_test.go new file mode 100644 index 0000000000..2c8e999624 --- /dev/null +++ b/internal/secaf/agents/prove/assembler_test.go @@ -0,0 +1,231 @@ +package prove + +// Tests for assembler.go and the two other pure helpers verifier.py exposes +// (Fallback, sarifRuleID), all pinned against the Python goldens. +// +// Validation contract (behaviour, not implementation): +// - assembling four sub-agent outputs yields exactly the VerifiedFinding +// Python's assemble_verified_finding produces, field for field; +// - an unrecognised verdict word degrades to "inconclusive"; +// - an out-of-range evidence level is clamped into 1..6; +// - NOT_EXPLOITABLE suppresses the reproduction steps, every other verdict +// emits the two-step recipe carrying the exploit payload; +// - Fallback demotes to inconclusive/static_match with a zero score, adds +// "low_confidence" only when a drop reason is given, and appends the +// original verdict to the rationale only when one is given. + +import ( + "reflect" + "testing" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +func TestAssembleVerifiedFindingGolden(t *testing.T) { + var want map[string]any + goldenJSON(t, "assemble", &want) + + rich, bare := findingRich(), findingBare() + cases := map[string]schemas.VerifiedFinding{ + "confirmed": AssembleVerifiedFinding(rich, traceRich(), sanitizationRich(), exploitRich(), + verdictDecision("confirmed", 5)), + "not_exploitable": AssembleVerifiedFinding(rich, traceRich(), sanitizationRich(), exploitRich(), + verdictDecision("not_exploitable", 1)), + "unknown_verdict": AssembleVerifiedFinding(bare, traceBare(), sanitizationBare(), exploitBare(), + verdictDecision("unverified", 9)), + "clamped_low": AssembleVerifiedFinding(bare, traceBare(), sanitizationBare(), exploitBare(), + verdictDecision("likely", 0)), + } + if len(cases) != len(want) { + t.Fatalf("case count drift: go has %d, golden has %d", len(cases), len(want)) + } + for name, got := range cases { + if !reflect.DeepEqual(jsonTree(t, got), want[name]) { + t.Errorf("assemble[%s] mismatch:\n got: %#v\nwant: %#v", name, jsonTree(t, got), want[name]) + } + } +} + +// TestAssembleVerdictMapping pins the _VERDICT_MAP lookup and its default. +func TestAssembleVerdictMapping(t *testing.T) { + for in, want := range map[string]schemas.ExploitVerdict{ + "confirmed": schemas.VerdictConfirmed, + "likely": schemas.VerdictLikely, + "inconclusive": schemas.VerdictInconclusive, + "not_exploitable": schemas.VerdictNotExploitable, + "unverified": schemas.VerdictInconclusive, // unknown -> default + "": schemas.VerdictInconclusive, + "CONFIRMED": schemas.VerdictInconclusive, // the map is case-SENSITIVE + } { + got := AssembleVerifiedFinding(findingBare(), traceBare(), sanitizationBare(), exploitBare(), + verdictDecision(in, 3)).Verdict + if got != want { + t.Errorf("verdict %q -> %q, want %q", in, got, want) + } + } +} + +// TestToEvidenceLevelClamp pins `max(1, min(6, level))`. +func TestToEvidenceLevelClamp(t *testing.T) { + for in, want := range map[int]schemas.EvidenceLevel{ + -5: 1, 0: 1, 1: 1, 3: 3, 6: 6, 7: 6, 99: 6, + } { + if got := toEvidenceLevel(in); got != want { + t.Errorf("toEvidenceLevel(%d) = %d, want %d", in, got, want) + } + } +} + +// TestToDataFlowStepsNumbering pins the 1-based enumerate and the synthetic +// file/line handles. +func TestToDataFlowStepsNumbering(t *testing.T) { + got := toDataFlowSteps(schemas.DataFlowTrace{Steps: []string{"a", "b", "c"}}) + want := []schemas.DataFlowStep{ + {File: "trace_step_1", Line: 1, Description: "a", Tainted: true}, + {File: "trace_step_2", Line: 2, Description: "b", Tainted: true}, + {File: "trace_step_3", Line: 3, Description: "c", Tainted: true}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("toDataFlowSteps = %+v, want %+v", got, want) + } + // An empty trace yields an empty (non-nil) list, so Proof.data_flow_trace + // marshals as [] rather than null. + if empty := toDataFlowSteps(schemas.DataFlowTrace{}); empty == nil || len(empty) != 0 { + t.Errorf("empty trace must give a non-nil empty slice, got %#v", empty) + } +} + +// TestReproductionStepsCarryExploit pins _reproduction_steps' two branches. +func TestReproductionStepsCarryExploit(t *testing.T) { + if got := reproductionSteps(schemas.VerdictNotExploitable, exploitRich()); len(got) != 0 { + t.Errorf("NOT_EXPLOITABLE must suppress reproduction steps, got %d", len(got)) + } + got := reproductionSteps(schemas.VerdictLikely, exploitRich()) + if len(got) != 2 { + t.Fatalf("want 2 reproduction steps, got %d", len(got)) + } + if got[1].Description != exploitRich().Hypothesis { + t.Errorf("step 2 description = %q, want the exploit hypothesis", got[1].Description) + } + if got[1].Command == nil || *got[1].Command != "1 OR 1=1" { + t.Errorf("step 2 command must be the exploit payload, got %v", got[1].Command) + } + // A None payload passes straight through as null. + noPayload := reproductionSteps(schemas.VerdictLikely, exploitBare()) + if noPayload[1].Command != nil { + t.Errorf("a nil payload must stay nil, got %v", noPayload[1].Command) + } +} + +// TestBypassPossibleTruthiness pins `bool(sanitization.bypass_method)`. +func TestBypassPossibleTruthiness(t *testing.T) { + for _, tc := range []struct { + name string + method *string + want bool + }{ + {"nil", nil, false}, + {"empty", str(""), false}, + {"set", str("x"), true}, + } { + got := AssembleVerifiedFinding(findingBare(), traceBare(), + schemas.SanitizationResult{BypassMethod: tc.method}, exploitBare(), + verdictDecision("likely", 2)) + if got.Proof == nil || got.Proof.SanitizationAnalysis == nil { + t.Fatalf("%s: proof/sanitization analysis missing", tc.name) + } + bp := got.Proof.SanitizationAnalysis.BypassPossible + if bp == nil || *bp != tc.want { + t.Errorf("%s: bypass_possible = %v, want %v", tc.name, bp, tc.want) + } + } +} + +func TestFallbackGolden(t *testing.T) { + var want map[string]any + goldenJSON(t, "fallback", &want) + + rich, bare := findingRich(), findingBare() + cases := map[string]schemas.VerifiedFinding{ + "plain": Fallback(rich, "harness blew up", nil, nil), + "with_drop_reason": Fallback(rich, "boom", StrPtr("verifier_error"), nil), + "demoted": Fallback(bare, + "Verifier returned unverified verdict; demoted for manual review", + StrPtr("verdict_unverified"), StrPtr("unverified")), + "original_verdict_only": Fallback(bare, "why", nil, StrPtr("unverified")), + } + if len(cases) != len(want) { + t.Fatalf("case count drift: go has %d, golden has %d", len(cases), len(want)) + } + for name, got := range cases { + if !reflect.DeepEqual(jsonTree(t, got), want[name]) { + t.Errorf("fallback[%s] mismatch:\n got: %#v\nwant: %#v", name, jsonTree(t, got), want[name]) + } + } +} + +// TestFallbackDemotionContract restates what tests/test_prove_phase_demotion.py +// asserts about a demoted finding, at the level this package owns. +func TestFallbackDemotionContract(t *testing.T) { + got := Fallback(findingRich(), "Verifier returned unverified verdict; demoted for manual review", + StrPtr("verdict_unverified"), StrPtr("unverified")) + + if got.Verdict != schemas.VerdictInconclusive { + t.Errorf("verdict = %q, want inconclusive", got.Verdict) + } + if got.EvidenceLevel != schemas.EvidenceLevelStaticMatch { + t.Errorf("evidence_level = %d, want 1 (STATIC_MATCH)", got.EvidenceLevel) + } + if got.DropReason == nil || *got.DropReason != "verdict_unverified" { + t.Errorf("drop_reason = %v, want verdict_unverified", got.DropReason) + } + if len(got.Tags) != 1 || got.Tags[0] != "low_confidence" { + t.Errorf("tags = %v, want [low_confidence]", got.Tags) + } + wantRationale := "Verification incomplete: Verifier returned unverified verdict; " + + "demoted for manual review (original verdict: unverified)" + if got.Rationale != wantRationale { + t.Errorf("rationale = %q, want %q", got.Rationale, wantRationale) + } + if got.ExploitabilityScore != 0 || got.SarifSecuritySeverity != 0 { + t.Errorf("a demoted finding must score 0, got %v/%v", got.ExploitabilityScore, got.SarifSecuritySeverity) + } + if got.Proof != nil { + t.Error("a demoted finding carries no proof") + } +} + +// TestFallbackEmptyOptionalsAreFalsy pins that "" behaves like None for both +// keyword arguments, which is what Python's truthiness tests do. +func TestFallbackEmptyOptionalsAreFalsy(t *testing.T) { + got := Fallback(findingBare(), "reason", StrPtr(""), StrPtr("")) + if len(got.Tags) != 0 { + t.Errorf("an empty drop_reason is falsy, so tags must stay empty; got %v", got.Tags) + } + if got.Rationale != "Verification incomplete: reason" { + t.Errorf("an empty original_verdict must not be appended; got %q", got.Rationale) + } + // The field itself still carries the pointer Python would store. + if got.DropReason == nil || *got.DropReason != "" { + t.Errorf("drop_reason field must hold the value passed, got %v", got.DropReason) + } +} + +func TestSarifRuleIDSlug(t *testing.T) { + for _, tc := range []struct { + cweName string + want string + }{ + {"SQL Injection", "sec-af/sast/sql-injection"}, + {"Improper Neutralization/Escaping of Special Elements", + "sec-af/sast/improper-neutralization-escaping-of-special-elements"}, + {"Broken Access/Control Check", "sec-af/sast/broken-access-control-check"}, + {"", "sec-af/sast/"}, + // Only SPACE and SLASH are replaced — an underscore survives. + {"Weak_Crypto", "sec-af/sast/weak_crypto"}, + } { + if got := sarifRuleID(schemas.FindingTypeSast, tc.cweName); got != tc.want { + t.Errorf("sarifRuleID(%q) = %q, want %q", tc.cweName, got, tc.want) + } + } +} diff --git a/internal/secaf/agents/prove/chain_builder.go b/internal/secaf/agents/prove/chain_builder.go new file mode 100644 index 0000000000..c5aa843595 --- /dev/null +++ b/internal/secaf/agents/prove/chain_builder.go @@ -0,0 +1,445 @@ +package prove + +// Ports src/sec_af/agents/prove/chain_builder.py — the ONE prove agent that +// calls the harness WITHOUT a schema and parses the raw text itself. + +import ( + "context" + "encoding/json" + "errors" + "sort" + "strings" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// chainBuilderPromptPath mirrors chain_builder.py's module-level PROMPT_PATH. +const chainBuilderPromptPath = "prove/chain_builder.txt" + +// ErrChainTagsNotASet reproduces the AttributeError Python raises inside +// `_apply_validated_chain`. +// +// PYTHON PARITY — THIS IS A REAL BUG IN THE PYTHON SOURCE, REPRODUCED ON +// PURPOSE. `_apply_validated_chain` finishes each matched finding with +// +// finding.tags.add("attack_chain") +// +// but `VerifiedFinding.tags` is declared `list[str]`, and a list has no `.add`. +// The call therefore raises `AttributeError: 'list' object has no attribute +// 'add'` the moment a validated chain names a finding that is actually present +// — and, unlike the harness call above it, `_apply_validated_chain` runs +// OUTSIDE run_chain_builder's `try/except`, so the exception escapes +// run_chain_builder and run_prove to the orchestrator. +// +// Verified against the real package (the repro script printed +// `f"{type(e).__name__} {e}"`, hence the class name in the transcript): +// +// $ PYTHONPATH=src ~/.agentfield/packages/sec-af/venv/bin/python -c '...' +// apply raised: AttributeError 'list' object has no attribute 'add' +// +// The MESSAGE, however, is `str(exc)` alone — `'list' object has no attribute +// 'add'`, with no `AttributeError: ` prefix — and that is what this error text +// must be, because app.py:229-230 interpolates it into both the note +// (`f"Audit pipeline failed: {exc}"`) and the 500 body +// (`f"audit execution failed: {exc}"`), which internal/secaf/audit/run.go builds +// from err.Error(). The sibling reproductions spell it the same way +// (orch.AttributeError, phases' select_strategy gate). +// +// The partial mutations performed before the raise (chain_id, chain_step and +// enables on the FIRST matching finding) are observable and are reproduced too. +// Do not "fix" this without fixing the Python side first; the two nodes must +// fail identically. +var ErrChainTagsNotASet = errors.New("'list' object has no attribute 'add'") + +// chainBuilderBuildPrompt ports chain_builder.py `_build_prompt`. +// +// findings_by_id = {finding.id: finding.model_dump() for finding in findings} +// chains_payload = [chain.model_dump() for chain in potential_chains] +// prompt = template.replace("{{DEPTH}}", depth) +// prompt = prompt.replace("{{CHAINS_JSON}}", json.dumps(chains_payload, indent=2)) +// prompt = prompt.replace("{{FINDINGS_JSON}}", json.dumps(findings_by_id, indent=2)) +// +// Python parity notes: +// +// - {{DEPTH}} is substituted FIRST, so a literal "{{CHAINS_JSON}}" in the +// depth string would be replaced afterwards (and a "{{DEPTH}}" inside the +// JSON payloads would not). +// - `findings_by_id` is a dict comprehension: duplicate ids keep the FIRST +// key position but the LAST value, and json.dumps renders insertion order. +// A Go map cannot express that, so the payload is built as a pyfmt.Ordered +// with the same first-seen ordering — see findingsByID. +func chainBuilderBuildPrompt( + template string, + potentialChains []schemas.PotentialChain, + findings []schemas.VerifiedFinding, + depth string, +) string { + _, ordered := findingsByID(findings) + + payload := make(pyfmt.Ordered, len(ordered)) + for i, entry := range ordered { + payload[i] = pyfmt.KV{Key: entry.id, Value: *entry.finding} + } + + chainsPayload := potentialChains + if chainsPayload == nil { + chainsPayload = []schemas.PotentialChain{} + } + + prompt := template + prompt = strings.ReplaceAll(prompt, "{{DEPTH}}", depth) + prompt = strings.ReplaceAll(prompt, "{{CHAINS_JSON}}", pyfmt.Dumps(chainsPayload, 2)) + prompt = strings.ReplaceAll(prompt, "{{FINDINGS_JSON}}", pyfmt.Dumps(payload, 2)) + return prompt +} + +// ChainBuilderPrompt builds the exact prompt RunChainBuilder sends. There is no +// trailing CONTEXT block — chain_builder.py sends the substituted template +// alone. Exported for the golden test. +func ChainBuilderPrompt(potentialChains []schemas.PotentialChain, findings []schemas.VerifiedFinding, depth string) string { + return chainBuilderBuildPrompt(prompts.MustLoad(chainBuilderPromptPath), potentialChains, findings, depth) +} + +// indexedFinding pairs an id with the (mutable) finding that id resolves to. +type indexedFinding struct { + id string + finding *schemas.VerifiedFinding +} + +// findingsByID reproduces `{finding.id: finding for finding in findings}`. +// +// Python dict semantics for a duplicate key: the key keeps its ORIGINAL +// position and the value is OVERWRITTEN. Both matter — the prompt payload and +// run_chain_builder's return value are both built by iterating this mapping, so +// a duplicate id shortens the returned list and keeps the LAST finding at the +// FIRST occurrence's position. +// +// The returned pointers alias entries of a private copy of `findings`, so +// mutating them cannot disturb the caller's slice until the copy is handed +// back. +func findingsByID(findings []schemas.VerifiedFinding) (map[string]*schemas.VerifiedFinding, []indexedFinding) { + store := make([]schemas.VerifiedFinding, len(findings)) + copy(store, findings) + + byID := make(map[string]*schemas.VerifiedFinding, len(store)) + ordered := make([]indexedFinding, 0, len(store)) + pos := make(map[string]int, len(store)) + for i := range store { + id := store[i].ID + if at, seen := pos[id]; seen { + ordered[at].finding = &store[i] + byID[id] = &store[i] + continue + } + pos[id] = len(ordered) + ordered = append(ordered, indexedFinding{id: id, finding: &store[i]}) + byID[id] = &store[i] + } + return byID, ordered +} + +// chainAnalysisPayload is chain_builder.py's ChainAnalysisPayload TypedDict. +// The nested chain/step entries stay untyped (map[string]any) because Python's +// `_parse_payload` validates ONLY that `payload["chains"]` is a list — every +// key access below it is a bare subscript that raises on a malformed shape, and +// the port reproduces that. +type chainAnalysisPayload struct { + chains []any +} + +// parseChainPayload ports `_parse_payload`. +// +// parsed = getattr(result, "parsed", None) +// if isinstance(parsed, dict): payload = parsed +// elif isinstance(result, dict): payload = result +// elif isinstance(parsed, str): payload = json.loads(parsed) # None -> None +// else: +// text = getattr(result, "text", None) +// payload = json.loads(text) if isinstance(text, str) else return None +// chains = payload.get("chains") +// return None if not isinstance(chains, list) else payload +// +// Go collapses the duck-typed ladder because the SDK's appx.HarnessResult is a +// concrete struct: `app.Harness(..., nil, nil, ...)` never populates Parsed +// (there is no schema and no destination), and `result` is never a bare map, so +// Python takes the `text` branch every time — `json.loads(result.text)`, i.e. +// json.Unmarshal of Result.Result. +// +// Python parity on the failure modes, all of which end at `payload = None` +// because run_chain_builder wraps this call in `except Exception`: +// +// - empty or non-JSON text -> JSONDecodeError -> caught here, nil returned; +// - JSON that is not an object (e.g. `[1,2]`) -> Python's `payload.get` +// raises AttributeError, caught by run_chain_builder -> nil; +// - `chains` missing or not a list -> None. +func parseChainPayload(res *appx.HarnessResult) *chainAnalysisPayload { + if res == nil { + return nil + } + var decoded any + if err := json.Unmarshal([]byte(res.Result), &decoded); err != nil { + return nil + } + obj, isObject := decoded.(map[string]any) + if !isObject { + // Python: AttributeError from `.get` on a list/str/number, swallowed + // by run_chain_builder's except. + return nil + } + chains, isList := obj["chains"].([]any) + if !isList { + return nil + } + return &chainAnalysisPayload{chains: chains} +} + +// applyValidatedChain ports `_apply_validated_chain`: +// +// if not chain["validated"] or not chain["steps"]: +// return +// ordered = sorted(chain["steps"], key=lambda step: step["step_number"]) +// ordered_ids = [step["finding_id"] for step in ordered] +// for index, step in enumerate(ordered): +// finding = findings_by_id.get(step["finding_id"]) +// if finding is None: continue +// finding.chain_id = chain["chain_id"] +// finding.chain_step = step["step_number"] +// if index + 1 < len(ordered_ids): +// finding.enables = [ordered_ids[index + 1]] +// finding.tags.add("attack_chain") # <- AttributeError, see ErrChainTagsNotASet +// +// The error is returned rather than raised; RunChainBuilder propagates it, as +// Python's uncaught AttributeError does. +// +// Python parity details: +// +// - `not chain["validated"]` / `not chain["steps"]` are TRUTHINESS tests, so +// a chain with validated=false, validated missing-but-null, or an empty +// steps list is a no-op. +// - `sorted` is STABLE, so equal step_numbers keep their input order. +// - the loop `continue`s past unknown finding ids, so a chain naming only +// absent findings mutates nothing AND raises nothing — including when the +// chain has no `chain_id` at all, because that key is only read once a +// step has matched (see the check inside the loop). +// - a malformed step (no "step_number" / "finding_id", or a non-numeric +// step_number) raises KeyError/TypeError in Python; the Go port returns a +// descriptive error from the same place. +// +// ERROR TEXT. `_apply_validated_chain` runs OUTSIDE run_chain_builder's +// try/except, so whatever it raises escapes to app.py:229-230, which +// interpolates `str(exc)` — NOT `repr(exc)` — into both the note +// (`f"Audit pipeline failed: {exc}"`) and the 500 body +// (`f"audit execution failed: {exc}"`). `str(exc)` never carries the exception +// CLASS NAME, so neither may these sentinels; the same rule already governs +// ErrChainTagsNotASet above. For the five MISSING-KEY cases the Go text is then +// byte-exact with Python, because `str(KeyError('validated'))` is `'validated'` +// — quotes included (VERIFIED on the pinned interpreter, all five keys). The +// TypeError cases keep a descriptive English text (Python's own wording depends +// on where in `sorted()` the subscript fails, e.g. +// `string indices must be integers, not 'str'`); they are a deliberate +// substitution, but they carry no class-name prefix either. +func applyValidatedChain(byID map[string]*schemas.VerifiedFinding, chain map[string]any) error { + validated, ok := chain["validated"] + if !ok { + return errors.New("'validated'") + } + rawSteps, ok := chain["steps"] + if !ok { + return errors.New("'steps'") + } + if !pyTruthy(validated) || !pyTruthy(rawSteps) { + return nil + } + steps, isList := rawSteps.([]any) + if !isList { + return errors.New("chain['steps'] is not a list") + } + + // One slice of (step_number, step) pairs so sorting keeps the key and the + // row together — Python's `sorted(steps, key=...)` moves whole elements. + type orderedStep struct { + number float64 + step map[string]any + } + ordered := make([]orderedStep, len(steps)) + for i, s := range steps { + step, isMap := s.(map[string]any) + if !isMap { + return errors.New("chain step is not an object") + } + n, hasNumber := step["step_number"] + if !hasNumber { + return errors.New("'step_number'") + } + f, isNumber := toFloat(n) + if !isNumber { + return errors.New("chain step 'step_number' is not a number") + } + ordered[i] = orderedStep{number: f, step: step} + } + // Python's sorted() is stable, so equal step_numbers keep input order. + sort.SliceStable(ordered, func(i, j int) bool { return ordered[i].number < ordered[j].number }) + + orderedIDs := make([]string, len(ordered)) + for i, entry := range ordered { + id, hasID := entry.step["finding_id"] + if !hasID { + return errors.New("'finding_id'") + } + s, isStr := id.(string) + if !isStr { + return errors.New("chain step 'finding_id' is not a string") + } + orderedIDs[i] = s + } + + for index, entry := range ordered { + finding := byID[orderedIDs[index]] + if finding == nil { + continue + } + // Python parity: `chain["chain_id"]` is subscripted INSIDE the loop + // (chain_builder.py:99), AFTER `if finding is None: continue`. A chain + // that names only unknown finding ids therefore never touches the key — + // a chain missing `chain_id` entirely is a silent no-op in Python, not + // a KeyError. Hoisting this check above the loop (as an earlier draft + // did) failed the whole prove phase on a payload Python ignores. + // VERIFIED on the pinned interpreter: _apply_validated_chain({}, { + // "validated": True, + // "steps": [{"step_number": 1, "finding_id": "hallucinated"}]}) -> None. + chainID, hasChainID := chain["chain_id"] + if !hasChainID { + return errors.New("'chain_id'") + } + chainIDStr, isChainIDStr := chainID.(string) + if !isChainIDStr { + return errors.New("chain['chain_id'] is not a string") + } + id := chainIDStr + // Python assigns `step["step_number"]` verbatim into an `int | None` + // field; pydantic accepts an integral float there, so the Go port + // truncates the decoded JSON number the same way. + stepNumber := int(entry.number) + finding.ChainID = &id + finding.ChainStep = &stepNumber + if index+1 < len(orderedIDs) { + finding.Enables = []string{orderedIDs[index+1]} + } + return ErrChainTagsNotASet + } + return nil +} + +// RunChainBuilder ports chain_builder.py run_chain_builder. +// +// if not potential_chains or not findings: +// return findings +// prompt = _build_prompt(...) +// try: +// result = await app.harness(prompt=prompt, cwd=repo_path) +// payload = _parse_payload(result) +// except Exception: +// payload = None +// if payload is None: +// return findings +// findings_by_id = {f.id: f for f in findings} +// for chain in payload["chains"]: +// _apply_validated_chain(findings_by_id, chain) +// return list(findings_by_id.values()) +// +// Python parity notes: +// +// - the harness call passes NO schema and cwd=repo_path (NOT a scratch temp +// dir and no project_dir) — the only prove agent that lets the coding agent +// work directly inside the repository. +// - the try/except swallows EVERY exception from the harness call and from +// parsing, so a transport error, a provider failure and unparseable output +// are all "no chains" and the input findings come back untouched. +// - `_apply_validated_chain` runs OUTSIDE that guard, so its AttributeError +// escapes — see ErrChainTagsNotASet. On that error the partially-mutated +// findings are returned alongside it so a caller that logs and continues +// sees exactly what Python's traceback would have left behind. +func RunChainBuilder( + ctx context.Context, + app appx.Harnesser, + repoPath string, + potentialChains []schemas.PotentialChain, + findings []schemas.VerifiedFinding, + depth string, +) ([]schemas.VerifiedFinding, error) { + if len(potentialChains) == 0 || len(findings) == 0 { + return findings, nil + } + + prompt := ChainBuilderPrompt(potentialChains, findings, depth) + + var payload *chainAnalysisPayload + res, err := app.Harness(ctx, prompt, nil, nil, appx.HarnessOptions{Cwd: repoPath}) + if err == nil { + payload = parseChainPayload(res) + } + if payload == nil { + return findings, nil + } + + byID, ordered := findingsByID(findings) + out := func() []schemas.VerifiedFinding { + result := make([]schemas.VerifiedFinding, len(ordered)) + for i, entry := range ordered { + result[i] = *entry.finding + } + return result + } + + for _, raw := range payload.chains { + chain, isMap := raw.(map[string]any) + if !isMap { + // Python: `chain["validated"]` on a non-dict raises TypeError. + return out(), errors.New("chain entry is not an object") + } + if err := applyValidatedChain(byID, chain); err != nil { + return out(), err + } + } + return out(), nil +} + +// pyTruthy is Python's `bool(x)` for the JSON value kinds a decoded payload can +// hold: None, false, 0, "", [] and {} are falsy; everything else is truthy. +func pyTruthy(v any) bool { + switch x := v.(type) { + case nil: + return false + case bool: + return x + case float64: + return x != 0 + case json.Number: + f, err := x.Float64() + return err == nil && f != 0 + case string: + return x != "" + case []any: + return len(x) > 0 + case map[string]any: + return len(x) > 0 + } + return true +} + +// toFloat accepts the numeric shapes a decoded JSON value can take. +func toFloat(v any) (float64, bool) { + switch x := v.(type) { + case float64: + return x, true + case int: + return float64(x), true + case json.Number: + f, err := x.Float64() + return f, err == nil + } + return 0, false +} diff --git a/internal/secaf/agents/prove/chain_builder_test.go b/internal/secaf/agents/prove/chain_builder_test.go new file mode 100644 index 0000000000..68930c7c1c --- /dev/null +++ b/internal/secaf/agents/prove/chain_builder_test.go @@ -0,0 +1,511 @@ +package prove + +// Tests for chain_builder.go. +// +// Validation contract: +// - no chains or no findings short-circuits and returns the input untouched; +// - the harness is called with NO schema and cwd= (the only prove agent +// that runs inside the repository); +// - a transport failure, a provider failure, unparseable output, output that +// is not a JSON object, or output whose "chains" is not a list all mean +// "no chains" and return the input untouched; +// - a chain with validated=false or empty steps mutates nothing; +// - a validated chain whose steps name no known finding mutates nothing and +// raises nothing; +// - a validated chain naming a known finding sets chain_id / chain_step / +// enables on the FIRST matching step and then fails with the Python +// AttributeError (VerifiedFinding.tags is a list, not a set). + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// chainGoldenCase mirrors one entry of apply_validated_chain.json. +type chainGoldenCase struct { + Chain map[string]any `json:"chain"` + Error *string `json:"error"` + Findings []struct { + ID string `json:"id"` + ChainID *string `json:"chain_id"` + ChainStep *int `json:"chain_step"` + Enables []string `json:"enables"` + Tags []string `json:"tags"` + } `json:"findings"` +} + +func TestApplyValidatedChainGolden(t *testing.T) { + var golden map[string]chainGoldenCase + goldenJSON(t, "apply_validated_chain", &golden) + + // Each case's finding set, keyed the same way gen_golden_prove builds it. + inputs := map[string][]schemas.VerifiedFinding{ + "not_validated": {verified("v1", "fp-v1", 1.0, 1), verified("v2", "fp-v2", 1.0, 1)}, + "no_steps": {verified("v1", "fp-v1", 1.0, 1)}, + "no_matching_finding": {verified("v1", "fp-v1", 1.0, 1)}, + "matching_finding_raises": {verified("v1", "fp-v1", 1.0, 1), verified("v2", "fp-v2", 1.0, 1)}, + "second_step_matches": {verified("v1", "fp-v1", 1.0, 1)}, + } + if len(inputs) != len(golden) { + t.Fatalf("case count drift: go has %d, golden has %d", len(inputs), len(golden)) + } + + for name, want := range golden { + findings, ok := inputs[name] + if !ok { + t.Fatalf("no Go input for golden case %q", name) + } + byID, ordered := findingsByID(findings) + err := applyValidatedChain(byID, want.Chain) + + if want.Error == nil { + if err != nil { + t.Errorf("%s: unexpected error %v", name, err) + } + } else { + if err == nil { + t.Errorf("%s: want error %q, got nil", name, *want.Error) + } else if err.Error() != *want.Error { + t.Errorf("%s: error = %q, want %q", name, err.Error(), *want.Error) + } + } + + if len(ordered) != len(want.Findings) { + t.Fatalf("%s: finding count %d, want %d", name, len(ordered), len(want.Findings)) + } + for i, entry := range ordered { + w := want.Findings[i] + got := entry.finding + if got.ID != w.ID { + t.Errorf("%s[%d]: id = %q, want %q", name, i, got.ID, w.ID) + } + if !reflect.DeepEqual(got.ChainID, w.ChainID) { + t.Errorf("%s[%d]: chain_id = %v, want %v", name, i, deref(got.ChainID), deref(w.ChainID)) + } + if !reflect.DeepEqual(got.ChainStep, w.ChainStep) { + t.Errorf("%s[%d]: chain_step = %v, want %v", name, i, got.ChainStep, w.ChainStep) + } + if !reflect.DeepEqual(got.Enables, w.Enables) { + t.Errorf("%s[%d]: enables = %v, want %v", name, i, got.Enables, w.Enables) + } + // Python never reaches the tags mutation: it raises on it. + if len(got.Tags) != len(w.Tags) { + t.Errorf("%s[%d]: tags = %v, want %v", name, i, got.Tags, w.Tags) + } + } + } +} + +func deref(p *string) any { + if p == nil { + return nil + } + return *p +} + +// TestApplyValidatedChainStepOrdering pins that the steps are sorted by +// step_number before ids are collected, so `enables` points at the NEXT step in +// exploitation order rather than the next element of the raw list. +func TestApplyValidatedChainStepOrdering(t *testing.T) { + findings := []schemas.VerifiedFinding{verified("a", "fp-a", 1, 1), verified("b", "fp-b", 1, 1)} + byID, ordered := findingsByID(findings) + chain := map[string]any{ + "chain_id": "c1", + "validated": true, + "steps": []any{ + map[string]any{"step_number": float64(2), "finding_id": "b"}, + map[string]any{"step_number": float64(1), "finding_id": "a"}, + }, + } + if err := applyValidatedChain(byID, chain); !errors.Is(err, ErrChainTagsNotASet) { + t.Fatalf("want the AttributeError parity error, got %v", err) + } + a := ordered[0].finding + if a.ChainStep == nil || *a.ChainStep != 1 { + t.Errorf("first finding chain_step = %v, want 1", a.ChainStep) + } + if !reflect.DeepEqual(a.Enables, []string{"b"}) { + t.Errorf("enables = %v, want [b] (the step AFTER sorting)", a.Enables) + } +} + +// TestFindingsByIDDuplicateSemantics pins Python dict-comprehension semantics: +// a duplicate id keeps the FIRST position and the LAST value, so the returned +// list is shorter than the input. +func TestFindingsByIDDuplicateSemantics(t *testing.T) { + first := verified("dup", "fp-1", 1, 1) + first.Title = "first" + other := verified("other", "fp-2", 1, 1) + last := verified("dup", "fp-3", 1, 1) + last.Title = "last" + + _, ordered := findingsByID([]schemas.VerifiedFinding{first, other, last}) + if len(ordered) != 2 { + t.Fatalf("want 2 entries after de-duplication, got %d", len(ordered)) + } + if ordered[0].id != "dup" || ordered[1].id != "other" { + t.Errorf("order = %q,%q, want dup,other (first-seen positions)", ordered[0].id, ordered[1].id) + } + if ordered[0].finding.Title != "last" { + t.Errorf("duplicate id must keep the LAST value, got %q", ordered[0].finding.Title) + } +} + +func TestRunChainBuilderShortCircuits(t *testing.T) { + app := &appx.Fake{} + findings := []schemas.VerifiedFinding{verified("v1", "fp-v1", 1, 1)} + + got, err := RunChainBuilder(context.Background(), app, fixtureRepo, nil, findings, "quick") + if err != nil || !reflect.DeepEqual(got, findings) { + t.Errorf("no chains must return the input untouched, got %v / %v", got, err) + } + got, err = RunChainBuilder(context.Background(), app, fixtureRepo, chainBuilderFixtureChains(), nil, "quick") + if err != nil || got != nil { + t.Errorf("no findings must return the input untouched, got %v / %v", got, err) + } + if len(app.Harnesses) != 0 { + t.Error("a short circuit must not call the harness") + } +} + +// TestRunChainBuilderHarnessOptions pins the schema-less, in-repo harness call. +func TestRunChainBuilderHarnessOptions(t *testing.T) { + app := &appx.Fake{ + HarnessFn: func(_ context.Context, _ string, schema map[string]any, dest any, opts appx.HarnessOptions) (*appx.HarnessResult, error) { + if schema != nil { + t.Errorf("chain_builder passes NO schema, got %v", schema) + } + if dest != nil { + t.Errorf("chain_builder passes NO destination, got %v", dest) + } + if opts.Cwd != fixtureRepo { + t.Errorf("cwd = %q, want the repository path", opts.Cwd) + } + if opts.ProjectDir != "" { + t.Errorf("chain_builder passes no project_dir, got %q", opts.ProjectDir) + } + return &appx.HarnessResult{Result: `{"chains": []}`}, nil + }, + } + findings := []schemas.VerifiedFinding{verified("v1", "fp-v1", 1, 1)} + if _, err := RunChainBuilder(context.Background(), app, fixtureRepo, + chainBuilderFixtureChains(), findings, "standard"); err != nil { + t.Fatalf("RunChainBuilder: %v", err) + } + if len(app.Harnesses) != 1 { + t.Errorf("want exactly one harness call, got %d", len(app.Harnesses)) + } +} + +// TestRunChainBuilderSwallowsFailures pins the blanket `except Exception` +// around the harness call and the payload parse. +func TestRunChainBuilderSwallowsFailures(t *testing.T) { + findings := []schemas.VerifiedFinding{verified("v1", "fp-v1", 1, 1)} + for _, tc := range []struct { + name string + fn func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) + }{ + {"transport error", func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return nil, errors.New("connection refused") + }}, + {"provider failure", func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{IsError: true, ErrorMessage: "boom"}, nil + }}, + {"empty text", func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{Result: ""}, nil + }}, + {"not json", func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{Result: "I could not find any chains."}, nil + }}, + {"json but not an object", func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{Result: `[1, 2]`}, nil + }}, + {"chains missing", func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{Result: `{"other": 1}`}, nil + }}, + {"chains not a list", func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{Result: `{"chains": {"a": 1}}`}, nil + }}, + } { + app := &appx.Fake{HarnessFn: tc.fn} + got, err := RunChainBuilder(context.Background(), app, fixtureRepo, + chainBuilderFixtureChains(), findings, "standard") + if err != nil { + t.Errorf("%s: want no error, got %v", tc.name, err) + } + if !reflect.DeepEqual(got, findings) { + t.Errorf("%s: findings must come back untouched, got %v", tc.name, got) + } + } +} + +// TestRunChainBuilderNoOpChain pins the end-to-end "chains present but nothing +// validated" path: findings are returned in dict order with no mutation and no +// error. +func TestRunChainBuilderNoOpChain(t *testing.T) { + payload, _ := json.Marshal(map[string]any{ + "chains": []any{map[string]any{ + "chain_id": "c1", "title": "t", "validated": false, "rationale": "r", + "steps": []any{map[string]any{"step_number": 1, "finding_id": "v1"}}, + }}, + }) + app := &appx.Fake{HarnessFn: appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return payload, nil + })} + findings := []schemas.VerifiedFinding{verified("v1", "fp-v1", 1, 1), verified("v2", "fp-v2", 1, 1)} + + got, err := RunChainBuilder(context.Background(), app, fixtureRepo, + chainBuilderFixtureChains(), findings, "standard") + if err != nil { + t.Fatalf("RunChainBuilder: %v", err) + } + if !reflect.DeepEqual(got, findings) { + t.Errorf("a non-validated chain must mutate nothing, got %v", got) + } +} + +// TestRunChainBuilderUnknownFindingIDsIsASilentNoOp pins that `chain_id` is +// read only AFTER a step matched a known finding (chain_builder.py:96-99), so a +// model-authored chain that names only hallucinated ids — and carries no +// `chain_id` at all — mutates nothing and raises nothing. +// +// VERIFIED on the pinned interpreter: +// +// _apply_validated_chain({}, {"validated": True, "steps": [ +// {"step_number": 1, "finding_id": "hallucinated-a"}, +// {"step_number": 2, "finding_id": "hallucinated-b"}]}) -> None +// +// The chain-builder harness runs with NO schema (`app.harness(prompt, cwd=...)`, +// chain_builder.py:120), so every key in the payload is model-authored and this +// shape is reachable. +func TestRunChainBuilderUnknownFindingIDsIsASilentNoOp(t *testing.T) { + payload, _ := json.Marshal(map[string]any{ + "chains": []any{map[string]any{ + // No "chain_id" key at all. + "validated": true, + "steps": []any{ + map[string]any{"step_number": 1, "finding_id": "hallucinated-a"}, + map[string]any{"step_number": 2, "finding_id": "hallucinated-b"}, + }, + }}, + }) + app := &appx.Fake{HarnessFn: appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return payload, nil + })} + findings := []schemas.VerifiedFinding{verified("v1", "fp-v1", 1, 1), verified("v2", "fp-v2", 1, 1)} + + got, err := RunChainBuilder(context.Background(), app, fixtureRepo, + chainBuilderFixtureChains(), findings, "standard") + if err != nil { + t.Fatalf("a chain naming only unknown ids must not fail the phase: %v", err) + } + if !reflect.DeepEqual(got, findings) { + t.Errorf("findings must come back untouched, got %v", got) + } +} + +// TestRunChainBuilderMissingChainIDOnAMatchedStep is the other half: once a +// step DOES match, Python evaluates `chain["chain_id"]` and raises KeyError. +func TestRunChainBuilderMissingChainIDOnAMatchedStep(t *testing.T) { + payload, _ := json.Marshal(map[string]any{ + "chains": []any{map[string]any{ + "validated": true, + "steps": []any{map[string]any{"step_number": 1, "finding_id": "v1"}}, + }}, + }) + app := &appx.Fake{HarnessFn: appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return payload, nil + })} + findings := []schemas.VerifiedFinding{verified("v1", "fp-v1", 1, 1)} + + _, err := RunChainBuilder(context.Background(), app, fixtureRepo, + chainBuilderFixtureChains(), findings, "standard") + if err == nil || err.Error() != "'chain_id'" { + t.Fatalf("err = %v, want 'chain_id'", err) + } +} + +// TestRunChainBuilderPropagatesTagsBug pins the reproduced Python AttributeError +// and the partial mutation that precedes it. See ErrChainTagsNotASet. +func TestRunChainBuilderPropagatesTagsBug(t *testing.T) { + payload, _ := json.Marshal(map[string]any{ + "chains": []any{map[string]any{ + "chain_id": "c1", "title": "t", "validated": true, "rationale": "r", + "steps": []any{ + map[string]any{"step_number": 1, "finding_id": "v1", "description": "d", "enables": "e"}, + map[string]any{"step_number": 2, "finding_id": "v2", "description": "d", "enables": "e"}, + }, + }}, + }) + app := &appx.Fake{HarnessFn: appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return payload, nil + })} + findings := []schemas.VerifiedFinding{verified("v1", "fp-v1", 1, 1), verified("v2", "fp-v2", 1, 1)} + + got, err := RunChainBuilder(context.Background(), app, fixtureRepo, + chainBuilderFixtureChains(), findings, "standard") + if !errors.Is(err, ErrChainTagsNotASet) { + t.Fatalf("want ErrChainTagsNotASet, got %v", err) + } + if len(got) != 2 { + t.Fatalf("the partially mutated findings must still come back, got %d", len(got)) + } + if got[0].ChainID == nil || *got[0].ChainID != "c1" { + t.Errorf("the first matching finding must have been assigned the chain id, got %v", got[0].ChainID) + } + if !reflect.DeepEqual(got[0].Enables, []string{"v2"}) { + t.Errorf("enables = %v, want [v2]", got[0].Enables) + } + if got[1].ChainID != nil { + t.Error("the raise happens on the FIRST match, so later steps are untouched") + } + if len(got[0].Tags) != 0 { + t.Error("Python raises ON the tag mutation, so no tag is ever added") + } + // The caller's slice must be untouched — findingsByID works on a copy. + if findings[0].ChainID != nil { + t.Error("RunChainBuilder must not mutate the caller's slice in place") + } +} + +// TestRunProvePropagatesChainBuilderError pins that the AttributeError escapes +// run_prove, exactly as it does in Python, with the metadata pass already +// applied to the partially mutated findings. +func TestRunProvePropagatesChainBuilderError(t *testing.T) { + chainPayload, _ := json.Marshal(map[string]any{ + "chains": []any{map[string]any{ + "chain_id": "c1", "validated": true, + "steps": []any{map[string]any{"step_number": 1, "finding_id": "raw-1"}}, + }}, + }) + app := &appx.Fake{ + HarnessFn: appx.HarnessJSON(func(prompt string, _ appx.HarnessOptions) (json.RawMessage, error) { + switch { + case containsRole(prompt, "DataFlowTracer"): + return json.Marshal(traceRich()) + case containsRole(prompt, "SanitizationAnalyzer"): + return json.Marshal(sanitizationRich()) + case containsRole(prompt, "ExploitHypothesizer"): + return json.Marshal(exploitRich()) + } + return chainPayload, nil + }), + AIFn: appx.AIJSON(func(string) (json.RawMessage, error) { + return json.Marshal(map[string]any{"verdict": "confirmed", "evidence_level": 5, "rationale": "r", "confidence": "high"}) + }), + } + hunt := schemas.HuntResult{ + Findings: []schemas.RawFinding{findingRich()}, + Chains: chainBuilderFixtureChains(), + } + got, err := RunProve(context.Background(), app, fixtureRepo, hunt, "standard", 3) + if !errors.Is(err, ErrChainTagsNotASet) { + t.Fatalf("want ErrChainTagsNotASet to escape RunProve, got %v", err) + } + if len(got) != 1 { + t.Fatalf("want the partially mutated finding back, got %d", len(got)) + } + if got[0].ChainID == nil || *got[0].ChainID != "c1" { + t.Errorf("chain_id = %v, want c1", got[0].ChainID) + } + // Python parity: run_chain_builder RAISES, so the SECOND _apply_metadata + // pass never runs — the score is still the one the first pass computed + // (severity critical x evidence 5 x reachability, no chain bonus). + if got[0].ExploitabilityScore != 9.0 { + t.Errorf("the post-chain metadata pass must NOT have run; score = %v, want 9", got[0].ExploitabilityScore) + } +} + +// containsRole matches a prove sub-agent's ROLE line. +func containsRole(prompt, role string) bool { + return strings.Contains(prompt, "You are "+role) +} + +// TestApplyValidatedChainMissingKeyErrorsCarryNoClassName pins the exception +// TEXT for all five subscripts `_apply_validated_chain` performs +// (chain_builder.py:88-101: chain["validated"], chain["steps"], +// step["step_number"], step["finding_id"], chain["chain_id"]). +// +// Validation contract (behaviour, measured on the pinned interpreter by +// calling the real `_apply_validated_chain` and printing `str(exc)`): +// +// {} -> 'validated' +// {"validated": True} -> 'steps' +// {"validated": True, "steps": [{}]} -> 'step_number' +// {... "steps": [{"step_number": 1}]} -> 'finding_id' +// {... "steps": [{"step_number": 1, "finding_id": "a"}]} -> 'chain_id' +// +// `str(KeyError('validated'))` is `'validated'` — the repr of the key, WITHOUT +// a `KeyError: ` prefix. The distinction is user-visible because +// `_apply_validated_chain` runs outside run_chain_builder's try/except, so the +// text lands verbatim in the audit's "Audit pipeline failed: {exc}" note and in +// its 500 body "audit execution failed: {exc}" (internal/secaf/audit/run.go). +func TestApplyValidatedChainMissingKeyErrorsCarryNoClassName(t *testing.T) { + for _, tc := range []struct { + name string + byID map[string]*schemas.VerifiedFinding + chain map[string]any + want string + }{ + {"validated", nil, map[string]any{}, "'validated'"}, + {"steps", nil, map[string]any{"validated": true}, "'steps'"}, + {"step_number", nil, + map[string]any{"validated": true, "steps": []any{map[string]any{}}}, + "'step_number'"}, + {"finding_id", nil, + map[string]any{"validated": true, "steps": []any{map[string]any{"step_number": 1.0}}}, + "'finding_id'"}, + {"chain_id", func() map[string]*schemas.VerifiedFinding { + f := verified("a", "fp-a", 1, 1) + return map[string]*schemas.VerifiedFinding{"a": &f} + }(), + map[string]any{"validated": true, "steps": []any{ + map[string]any{"step_number": 1.0, "finding_id": "a"}}}, + "'chain_id'"}, + } { + t.Run(tc.name, func(t *testing.T) { + err := applyValidatedChain(tc.byID, tc.chain) + if err == nil { + t.Fatalf("applyValidatedChain accepted %v, want an error", tc.chain) + } + if err.Error() != tc.want { + t.Errorf("err = %q, want %q (str(exc) never carries the class name)", err.Error(), tc.want) + } + }) + } +} + +// TestChainBuilderErrorsNeverNameTheirExceptionClass is the whole-file rule: +// nothing this file raises may spell a CPython exception class, because every +// one of these errors reaches the operator through `str(exc)` interpolation. +func TestChainBuilderErrorsNeverNameTheirExceptionClass(t *testing.T) { + errs := []error{ + ErrChainTagsNotASet, + applyValidatedChain(nil, map[string]any{}), + applyValidatedChain(nil, map[string]any{"validated": true}), + applyValidatedChain(nil, map[string]any{"validated": true, "steps": "notalist"}), + applyValidatedChain(nil, map[string]any{"validated": true, "steps": []any{1.0}}), + applyValidatedChain(nil, map[string]any{"validated": true, "steps": []any{map[string]any{}}}), + applyValidatedChain(nil, map[string]any{"validated": true, + "steps": []any{map[string]any{"step_number": "nope"}}}), + applyValidatedChain(nil, map[string]any{"validated": true, + "steps": []any{map[string]any{"step_number": 1.0}}}), + } + for _, err := range errs { + if err == nil { + t.Fatal("expected every probe to produce an error") + } + for _, class := range []string{"KeyError:", "TypeError:", "AttributeError:", "ValueError:"} { + if strings.Contains(err.Error(), class) { + t.Errorf("%q names an exception class; str(exc) never does", err.Error()) + } + } + } +} diff --git a/internal/secaf/agents/prove/cross_service.go b/internal/secaf/agents/prove/cross_service.go new file mode 100644 index 0000000000..05add31b0a --- /dev/null +++ b/internal/secaf/agents/prove/cross_service.go @@ -0,0 +1,78 @@ +package prove + +// Ports src/sec_af/agents/prove/cross_service.py. + +import ( + "context" + "os" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/harnessx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// crossServicePromptPath mirrors cross_service.py's module-level PROMPT_PATH. +const crossServicePromptPath = "prove/cross_service.txt" + +const ( + crossServiceAgentName = "prove-cross-service" + crossServiceExtractName = "CrossServiceAnalyzer" +) + +// crossServiceBuildPrompt ports cross_service.py `_build_prompt`. +// +// `services` is a plain `list[str]` rendered with json.dumps(indent=2); a nil +// Go slice stands for Python's empty list and must render as `[]`. +func crossServiceBuildPrompt(template string, services []string, findingsSummary, depth string) string { + if services == nil { + services = []string{} + } + return applyReplacements(template, []replacement{ + {"{{SERVICES}}", pyfmt.Dumps(services, 2)}, + {"{{FINDINGS_SUMMARY}}", findingsSummary}, + {"{{DEPTH}}", depth}, + }) +} + +// CrossServicePrompt builds the exact prompt RunCrossServiceAnalyzer sends. +// Exported for the golden test. +func CrossServicePrompt(services []string, findingsSummary, repoPath, depth string) string { + return crossServiceBuildPrompt(prompts.MustLoad(crossServicePromptPath), services, findingsSummary, depth) + + "\n\nCONTEXT:\n" + + "- Repository path: " + repoPath + "\n" + + "- Use the repository path above for cross-service inspection." +} + +// RunCrossServiceAnalyzer ports cross_service.py run_cross_service_analyzer. +// +// result = await app.harness(prompt=prompt, schema=CrossServiceFinding, +// cwd=harness_cwd, project_dir=repo_path) +// return extract_harness_result(result, CrossServiceFinding, "CrossServiceAnalyzer") +// +// Same temp-dir contract as RunTracer. This agent is registered as the +// `run_cross_service_analyzer` reasoner but is NOT part of the audit DAG — the +// orchestrator never calls it (multi-repo scanning is driven externally). +func RunCrossServiceAnalyzer( + ctx context.Context, + app appx.Harnesser, + repoPath string, + services []string, + findingsSummary string, + depth string, +) (schemas.CrossServiceFinding, error) { + prompt := CrossServicePrompt(services, findingsSummary, repoPath, depth) + + harnessCwd, err := os.MkdirTemp("", "secaf-"+crossServiceAgentName+"-") + if err != nil { + return schemas.CrossServiceFinding{}, err + } + defer os.RemoveAll(harnessCwd) + + return harnessx.RunExtract[schemas.CrossServiceFinding]( + ctx, app, prompt, + appx.HarnessOptions{Cwd: harnessCwd, ProjectDir: repoPath}, + crossServiceExtractName, + ) +} diff --git a/internal/secaf/agents/prove/dast_verifier.go b/internal/secaf/agents/prove/dast_verifier.go new file mode 100644 index 0000000000..213dbe3ee3 --- /dev/null +++ b/internal/secaf/agents/prove/dast_verifier.go @@ -0,0 +1,79 @@ +package prove + +// Ports src/sec_af/agents/prove/dast_verifier.py. + +import ( + "context" + "os" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/harnessx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// dastPromptPath mirrors dast_verifier.py's module-level PROMPT_PATH. +const dastPromptPath = "prove/dast_verifier.txt" + +const ( + dastAgentName = "prove-dast" + dastExtractName = "DastVerifier" +) + +// dastBuildPrompt ports dast_verifier.py `_build_prompt`. +// +// Python parity: this builder is the SHORTEST of the prove set — no +// {{CWE_NAME}}, no {{CODE_SNIPPET}}, no {{FINDING_TYPE}}, no {{RELATED_FILES}}. +// Those markers do not appear in dast_verifier.txt either, so nothing is left +// unsubstituted. {{EXPLOIT_PAYLOAD}} is inserted BEFORE {{DEPTH}}, so a literal +// "{{DEPTH}}" inside the payload IS substituted. +func dastBuildPrompt(template string, finding schemas.RawFinding, exploitPayload, depth string) string { + return applyReplacements(template, []replacement{ + {"{{TITLE}}", finding.Title}, + {"{{DESCRIPTION}}", finding.Description}, + {"{{CWE_ID}}", finding.CweID}, + {"{{FILE_PATH}}", finding.FilePath}, + {"{{EXPLOIT_PAYLOAD}}", exploitPayload}, + {"{{DEPTH}}", depth}, + }) +} + +// DastPrompt builds the exact prompt RunDastVerifier sends. Exported for the +// golden test. +func DastPrompt(finding schemas.RawFinding, exploitPayload, repoPath, depth string) string { + return dastBuildPrompt(prompts.MustLoad(dastPromptPath), finding, exploitPayload, depth) + + "\n\nCONTEXT:\n" + + "- Repository path: " + repoPath + "\n" + + "- Use the repository path above for file inspection during DAST-style verification." +} + +// RunDastVerifier ports dast_verifier.py run_dast_verifier. +// +// result = await app.harness(prompt=prompt, schema=DastVerificationResult, +// cwd=harness_cwd, project_dir=repo_path) +// return extract_harness_result(result, DastVerificationResult, "DastVerifier") +// +// Same temp-dir contract as RunTracer. The orchestrator reaches this through +// `_run_dast_verification`, gated on AuditInput.enable_dast. +func RunDastVerifier( + ctx context.Context, + app appx.Harnesser, + repoPath string, + finding schemas.RawFinding, + exploitPayload string, + depth string, +) (schemas.DastVerificationResult, error) { + prompt := DastPrompt(finding, exploitPayload, repoPath, depth) + + harnessCwd, err := os.MkdirTemp("", "secaf-"+dastAgentName+"-") + if err != nil { + return schemas.DastVerificationResult{}, err + } + defer os.RemoveAll(harnessCwd) + + return harnessx.RunExtract[schemas.DastVerificationResult]( + ctx, app, prompt, + appx.HarnessOptions{Cwd: harnessCwd, ProjectDir: repoPath}, + dastExtractName, + ) +} diff --git a/internal/secaf/agents/prove/dep_reachability.go b/internal/secaf/agents/prove/dep_reachability.go new file mode 100644 index 0000000000..5c6da8211e --- /dev/null +++ b/internal/secaf/agents/prove/dep_reachability.go @@ -0,0 +1,165 @@ +package prove + +// Ports src/sec_af/agents/prove/dep_reachability.py. + +import ( + "bytes" + "context" + "encoding/json" + "os" + "strings" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/harnessx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// depReachabilityPromptPath mirrors dep_reachability.py's PROMPT_PATH. +const depReachabilityPromptPath = "prove/dep_reachability.txt" + +const ( + depReachabilityAgentName = "prove-dep-reachability" + depReachabilityExtractName = "DependencyReachabilityAnalyzer" +) + +// depReachabilityBuildPrompt ports dep_reachability.py `_build_prompt`: +// +// "{{CVE}}": str(finding.get("cve", "")), +// "{{PACKAGE}}": str(finding.get("package", "")), +// "{{VULNERABLE_FUNCTION}}": str(finding.get("vulnerable_function", "")), +// "{{VERSION}}": str(finding.get("version", "")), +// "{{EVIDENCE}}": json.dumps(finding.get("evidence", {}), indent=2), +// "{{DEPTH}}": depth, +// +// This is the only prove builder whose input is an untyped dict rather than a +// pydantic model, so it inherits two Python-vs-Go JSON asymmetries. Both are +// handled here rather than left to bite the prompt bytes: +// +// 1. INT vs FLOAT. Python's json.loads makes `2` an int (str -> "2", +// json.dumps -> "2") and `2.0` a float (-> "2.0"). Go's encoding/json makes +// BOTH a float64, which pyfmt would render "2.0". normalizeJSONNumbers +// re-decodes the map with json.Number so an integral literal keeps its +// integer spelling — recovering Python's behaviour for every value that +// arrived as a JSON integer. The `2.0` spelling is unrecoverable: Go +// collapsed it before this function ever saw the map, and it renders as +// "2". Documented divergence, unreachable in practice (evidence hints are +// counts and flags). +// +// 2. KEY ORDER. json.dumps follows the dict's insertion order; a Go +// map[string]any has none, so `evidence` renders with SORTED keys +// (DESIGN.md §2b). For an `evidence` dict whose keys are already sorted the +// two are byte-identical, which is what the golden fixture pins. +// +// Truthiness note: `finding.get(key, "")` returns the DEFAULT only when the key +// is ABSENT. A key present with value None yields `str(None)` == "None", not +// "" — reproduced by looking the key up with the comma-ok form. +func depReachabilityBuildPrompt(template string, finding map[string]any, depth string) string { + normalized := normalizeJSONNumbers(finding) + + get := func(key string) any { + if v, ok := normalized[key]; ok { + return v + } + return "" // Python's `.get(key, "")` + } + evidence := any(map[string]any{}) + if v, ok := normalized["evidence"]; ok { + evidence = v + } + + return applyReplacements(template, []replacement{ + {"{{CVE}}", pyStrDynamic(get("cve"))}, + {"{{PACKAGE}}", pyStrDynamic(get("package"))}, + {"{{VULNERABLE_FUNCTION}}", pyStrDynamic(get("vulnerable_function"))}, + {"{{VERSION}}", pyStrDynamic(get("version"))}, + {"{{EVIDENCE}}", pyfmt.Dumps(evidence, 2)}, + {"{{DEPTH}}", depth}, + }) +} + +// DepReachabilityPrompt builds the exact prompt RunDepReachability sends. +// Exported for the golden test. +func DepReachabilityPrompt(finding map[string]any, repoPath, depth string) string { + return depReachabilityBuildPrompt(prompts.MustLoad(depReachabilityPromptPath), finding, depth) + + "\n\nCONTEXT:\n" + + "- Repository path: " + repoPath + "\n" + + "- Use the repository path above for file inspection during dependency reachability analysis." +} + +// RunDepReachability ports dep_reachability.py run_dep_reachability. +// +// result = await app.harness(prompt=prompt, schema=ReachabilityProof, +// cwd=harness_cwd, project_dir=repo_path) +// return extract_harness_result(result, ReachabilityProof, "DependencyReachabilityAnalyzer") +// +// Same temp-dir contract as RunTracer. +func RunDepReachability( + ctx context.Context, + app appx.Harnesser, + repoPath string, + finding map[string]any, + depth string, +) (schemas.ReachabilityProof, error) { + prompt := DepReachabilityPrompt(finding, repoPath, depth) + + harnessCwd, err := os.MkdirTemp("", "secaf-"+depReachabilityAgentName+"-") + if err != nil { + return schemas.ReachabilityProof{}, err + } + defer os.RemoveAll(harnessCwd) + + return harnessx.RunExtract[schemas.ReachabilityProof]( + ctx, app, prompt, + appx.HarnessOptions{Cwd: harnessCwd, ProjectDir: repoPath}, + depReachabilityExtractName, + ) +} + +// normalizeJSONNumbers round-trips the map through JSON with UseNumber so every +// numeric leaf becomes a json.Number carrying its literal spelling. See +// depReachabilityBuildPrompt's doc for why. +// +// On any marshal/decode failure (a value encoding/json cannot represent) the +// input is returned unchanged — a prompt with a "2.0" where Python wrote "2" +// beats no prompt at all. +func normalizeJSONNumbers(finding map[string]any) map[string]any { + if finding == nil { + return map[string]any{} + } + b, err := json.Marshal(finding) + if err != nil { + return finding + } + dec := json.NewDecoder(bytes.NewReader(b)) + dec.UseNumber() + var out map[string]any + if err := dec.Decode(&out); err != nil { + return finding + } + return out +} + +// pyStrDynamic is `str(v)` for a value decoded out of JSON. +// +// It exists because json.Number is a Go STRING type: handing it to pyfmt.Str +// would take the reflect.String branch and quote it. Python's json.loads turns +// a number literal without '.', 'e' or 'E' into an int (str -> the digits) and +// anything else into a float (str -> repr(float)), which is exactly the split +// reproduced here. Every other kind delegates to pyfmt.Str. +func pyStrDynamic(v any) string { + n, ok := v.(json.Number) + if !ok { + return pyfmt.Str(v) + } + s := n.String() + if !strings.ContainsAny(s, ".eE") { + return s // Python int + } + f, err := n.Float64() + if err != nil { + return s + } + return pyfmt.FormatFloat(f) // Python float, repr()-style +} diff --git a/internal/secaf/agents/prove/doc.go b/internal/secaf/agents/prove/doc.go new file mode 100644 index 0000000000..7e6657284a --- /dev/null +++ b/internal/secaf/agents/prove/doc.go @@ -0,0 +1,36 @@ +// Package prove ports src/sec_af/agents/prove — the PROVE phase. +// +// PROVE takes the provisional findings HUNT produced and decides, per finding, +// whether the vulnerability is real: it traces the data flow, analyses the +// sanitization on that path, hypothesises an exploit, asks a judgment model for +// a verdict, and assembles the evidence into a schemas.VerifiedFinding. +// +// Module map (Python file -> Go file): +// +// __init__.py prove.go _priority_sort, _apply_metadata, +// _run_parallel_verification, run_prove, +// run_prove_streaming +// verifier.py verifier.go run_verifier (the in-process +// tracer -> sanitization -> exploit -> +// verdict chain), fallback +// assembler.py assembler.go assemble_verified_finding +// tracer.py tracer.go run_tracer +// sanitization.py sanitization.go run_sanitization_analyzer +// exploit.py exploit.go run_exploit_hypothesizer +// verdict.py verdict.go run_verdict_agent (.ai, not .harness) +// chain_builder.py chain_builder.go run_chain_builder (schema-less harness) +// cross_service.py cross_service.go run_cross_service_analyzer +// dast_verifier.py dast_verifier.go run_dast_verifier +// dep_reachability.py dep_reachability.go run_dep_reachability +// sandbox.py sandbox.go run_sandboxed +// +// DAG note (DESIGN.md §3): every function here is called IN PROCESS. The +// control-plane `.call(...)` fan-out that produces the DAG's `run_verifier` +// nodes lives in internal/phases (Python: reasoners/phases.py prove_phase); +// this package is what those reasoners execute once routed. Nothing in here +// may grow an app.Call — that would double the DAG. +// +// Every exported function takes ctx first and the narrowest appx sub-interface +// it needs (Harnesser, AIer, or the union HarnessAIer), because Python passes +// the SDK agent/router and only ever touches `.harness(...)` / `.ai(...)`. +package prove diff --git a/internal/secaf/agents/prove/exploit.go b/internal/secaf/agents/prove/exploit.go new file mode 100644 index 0000000000..462a94c0d5 --- /dev/null +++ b/internal/secaf/agents/prove/exploit.go @@ -0,0 +1,121 @@ +package prove + +// Ports src/sec_af/agents/prove/exploit.py. + +import ( + "context" + "os" + "strconv" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/harnessx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// exploitPromptPath mirrors exploit.py's module-level PROMPT_PATH. +const exploitPromptPath = "prove/exploit.txt" + +const ( + exploitAgentName = "prove-exploit" + exploitExtractName = "ExploitHypothesizer" +) + +// sanitizationContext ports exploit.py `_sanitization_context`: +// +// found = "yes" if sanitization.found else "no" +// sufficient = "unknown" if sanitization.sufficient is None else ("yes" if sanitization.sufficient else "no") +// return (f"Sanitization found: {found}\n" +// f"Sanitization type: {sanitization.type or 'none'}\n" +// f"Sanitization sufficient: {sufficient}\n" +// f"Bypass method: {sanitization.bypass_method or 'none'}") +// +// Python parity: `sufficient` distinguishes None (unknown) from False (no), +// which is why SanitizationResult.Sufficient is a *bool and not a bool; `type` +// and `bypass_method` use `or`, so an EMPTY string also renders as "none". +func sanitizationContext(sanitization schemas.SanitizationResult) string { + found := "no" + if sanitization.Found { + found = "yes" + } + sufficient := "unknown" + if sanitization.Sufficient != nil { + sufficient = "no" + if *sanitization.Sufficient { + sufficient = "yes" + } + } + return "Sanitization found: " + found + "\n" + + "Sanitization type: " + pyOr(sanitization.Type, "none") + "\n" + + "Sanitization sufficient: " + sufficient + "\n" + + "Bypass method: " + pyOr(sanitization.BypassMethod, "none") +} + +// exploitBuildPrompt ports exploit.py `_build_prompt`. +func exploitBuildPrompt( + template string, + finding schemas.RawFinding, + dataFlowTrace schemas.DataFlowTrace, + sanitization schemas.SanitizationResult, + depth string, +) string { + return applyReplacements(template, []replacement{ + {"{{TITLE}}", finding.Title}, + {"{{DESCRIPTION}}", finding.Description}, + {"{{CWE_ID}}", finding.CweID}, + {"{{CWE_NAME}}", finding.CweName}, + {"{{FILE_PATH}}", finding.FilePath}, + {"{{START_LINE}}", strconv.Itoa(finding.StartLine)}, + {"{{CODE_SNIPPET}}", finding.CodeSnippet}, + {"{{FINDING_TYPE}}", string(finding.FindingType)}, + {"{{RELATED_FILES}}", relatedFilesJSON(finding.RelatedFiles)}, + {"{{TRACE_CONTEXT}}", traceContext(dataFlowTrace)}, + {"{{SANITIZATION_CONTEXT}}", sanitizationContext(sanitization)}, + {"{{DEPTH}}", depth}, + }) +} + +// ExploitPrompt builds the exact prompt RunExploitHypothesizer sends. Exported +// for the golden test. +func ExploitPrompt( + finding schemas.RawFinding, + dataFlowTrace schemas.DataFlowTrace, + sanitization schemas.SanitizationResult, + repoPath, depth string, +) string { + return exploitBuildPrompt(prompts.MustLoad(exploitPromptPath), finding, dataFlowTrace, sanitization, depth) + + "\n\nCONTEXT:\n" + + "- Repository path: " + repoPath + "\n" + + "- Use the repository path above for file inspection during exploit construction." +} + +// RunExploitHypothesizer ports exploit.py run_exploit_hypothesizer. +// +// result = await app.harness(prompt=prompt, schema=ExploitHypothesis, +// cwd=harness_cwd, project_dir=repo_path) +// return extract_harness_result(result, ExploitHypothesis, "ExploitHypothesizer") +// +// Same temp-dir contract as RunTracer. +func RunExploitHypothesizer( + ctx context.Context, + app appx.Harnesser, + repoPath string, + finding schemas.RawFinding, + dataFlowTrace schemas.DataFlowTrace, + sanitization schemas.SanitizationResult, + depth string, +) (schemas.ExploitHypothesis, error) { + prompt := ExploitPrompt(finding, dataFlowTrace, sanitization, repoPath, depth) + + harnessCwd, err := os.MkdirTemp("", "secaf-"+exploitAgentName+"-") + if err != nil { + return schemas.ExploitHypothesis{}, err + } + defer os.RemoveAll(harnessCwd) + + return harnessx.RunExtract[schemas.ExploitHypothesis]( + ctx, app, prompt, + appx.HarnessOptions{Cwd: harnessCwd, ProjectDir: repoPath}, + exploitExtractName, + ) +} diff --git a/internal/secaf/agents/prove/golden_test.go b/internal/secaf/agents/prove/golden_test.go new file mode 100644 index 0000000000..e29df43008 --- /dev/null +++ b/internal/secaf/agents/prove/golden_test.go @@ -0,0 +1,208 @@ +package prove + +// Shared helpers and fixtures for the golden tests in this package. +// +// Every fixture under testdata/golden is produced by go/scripts/gen_golden_prove.py +// running the REAL Python code from src/sec_af/agents/prove. Regenerate with: +// +// PYTHONPATH=src ~/.agentfield/packages/sec-af/venv/bin/python go/scripts/gen_golden.py +// +// A test failing here means the Go port and the Python source disagree about +// bytes that reach the LLM (prompts) or about the shape of a model that crosses +// the wire — not that a fixture needs refreshing. Refresh only after a +// deliberate Python change. +// +// The Go literals below repeat the Python literals in gen_golden_prove.py +// EXACTLY. Keeping the two in sync by hand is the point: a drift in either is a +// test failure rather than a silent divergence. + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +const goldenDir = "testdata/golden" + +// fixtureRepo matches gen_golden_prove.FIXTURE_REPO. +const fixtureRepo = "/fixtures/demo-repo" + +func goldenText(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join(goldenDir, name+".txt")) + if err != nil { + t.Fatalf("read golden %s.txt: %v", name, err) + } + return string(b) +} + +func goldenJSON(t *testing.T, name string, dest any) { + t.Helper() + b, err := os.ReadFile(filepath.Join(goldenDir, name+".json")) + if err != nil { + t.Fatalf("read golden %s.json: %v", name, err) + } + if err := json.Unmarshal(b, dest); err != nil { + t.Fatalf("decode golden %s.json: %v", name, err) + } +} + +// jsonTree marshals v and decodes the result into the untyped shape the golden +// fixtures decode to, so the two can be compared with reflect.DeepEqual without +// either side's Go types leaking into the comparison. +func jsonTree(t *testing.T, v any) any { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatalf("marshal %T: %v", v, err) + } + var tree any + if err := json.Unmarshal(b, &tree); err != nil { + t.Fatalf("unmarshal %T: %v", v, err) + } + return tree +} + +func str(s string) *string { return &s } +func boolp(b bool) *bool { return &b } + +// findingRich mirrors gen_golden_prove._finding_rich, including the two +// substitution-order traps ("{{TITLE}}" in the description must survive, +// "{{DEPTH}}" in the snippet must be replaced). +func findingRich() schemas.RawFinding { + return schemas.RawFinding{ + ID: "raw-1", + HunterStrategy: "injection", + Title: "Potential SQL injection in user lookup", + Description: "Request parameter reaches a formatted SQL string. Marker: {{TITLE}}", + FindingType: schemas.FindingTypeSast, + CweID: "CWE-89", + CweName: "Improper Neutralization/Escaping of Special Elements", + OwaspCategory: str("A03:2021 - Injection"), + FilePath: "src/users.py", + StartLine: 42, + EndLine: 44, + FunctionName: str("get_user"), + CodeSnippet: `cursor.execute("SELECT * FROM users WHERE id = " + user_id) # depth={{DEPTH}}`, + EstimatedSeverity: schemas.SeverityHigh, + Confidence: schemas.ConfidenceHigh, + DataFlow: []schemas.ReconDataFlowStep{ + {FilePath: "src/routes.py", Line: 10, Component: "handler", Operation: "read request.args"}, + {FilePath: "src/users.py", Line: 42, Component: "db", Operation: "execute"}, + }, + RelatedFiles: []string{"src/routes.py", "src/café &.py"}, + Fingerprint: "fp-1", + } +} + +// findingBare mirrors gen_golden_prove._finding_bare — every optional at its +// pydantic default. +func findingBare() schemas.RawFinding { + return schemas.RawFinding{ + ID: "raw-2", + HunterStrategy: "crypto", + Title: "Weak hash", + Description: "MD5 used for password hashing", + FindingType: schemas.FindingTypeSast, + CweID: "CWE-327", + CweName: "Broken Crypto", + FilePath: "src/hash.py", + StartLine: 7, + EndLine: 7, + CodeSnippet: "hashlib.md5(pw).hexdigest()", + EstimatedSeverity: schemas.SeverityMedium, + Confidence: schemas.ConfidenceLow, + RelatedFiles: []string{}, + Fingerprint: "fp-2", + } +} + +func traceRich() schemas.DataFlowTrace { + return schemas.DataFlowTrace{ + Source: "request.args['id']", + Sink: "cursor.execute(query)", + Steps: []string{"src/routes.py:10 read request.args", "src/users.py:42 execute"}, + SinkReached: true, + } +} + +func traceBare() schemas.DataFlowTrace { + return schemas.DataFlowTrace{Source: "unknown", Sink: "unknown", Steps: []string{}, SinkReached: false} +} + +func sanitizationRich() schemas.SanitizationResult { + return schemas.SanitizationResult{ + Found: true, + Type: str("parameterized query"), + Sufficient: boolp(false), + BypassMethod: str("second-order injection through the audit log"), + } +} + +func sanitizationBare() schemas.SanitizationResult { + return schemas.SanitizationResult{Found: false} +} + +func exploitRich() schemas.ExploitHypothesis { + return schemas.ExploitHypothesis{ + Hypothesis: "Attacker supplies id=1 OR 1=1 to dump the table", + Payload: str("1 OR 1=1"), + ExpectedOutcome: "Full users table returned", + } +} + +func exploitBare() schemas.ExploitHypothesis { + return schemas.ExploitHypothesis{Hypothesis: "unknown", ExpectedOutcome: "unknown"} +} + +func verdictDecision(verdict string, level int) schemas.VerdictDecision { + return schemas.VerdictDecision{ + Verdict: verdict, + EvidenceLevel: level, + Rationale: "rationale for " + verdict, + Confidence: "high", + } +} + +// verified mirrors gen_golden_prove._verified. +func verified(id, fingerprint string, score float64, level int, tags ...string) schemas.VerifiedFinding { + if tags == nil { + tags = []string{} + } + return schemas.VerifiedFinding{ + ID: id, + Fingerprint: fingerprint, + Title: "finding " + id, + Description: "d", + FindingType: schemas.FindingTypeSast, + CweID: "CWE-89", + CweName: "SQL Injection", + Verdict: schemas.VerdictConfirmed, + EvidenceLevel: schemas.EvidenceLevel(level), + Rationale: "r", + Severity: schemas.SeverityHigh, + ExploitabilityScore: score, + Location: schemas.Location{FilePath: "src/users.py", StartLine: 1, EndLine: 2}, + Tags: tags, + RelatedLocations: []schemas.Location{}, + Compliance: []schemas.ComplianceMapping{}, + ReproductionSteps: []schemas.ReproductionStep{}, + SarifRuleID: "sec-af/sast/sql-injection", + SarifSecuritySeverity: score, + } +} + +// chainBuilderFixtureChains mirrors the `chains` list gen_golden_prove builds +// for the chain-builder prompt fixture. +func chainBuilderFixtureChains() []schemas.PotentialChain { + return []schemas.PotentialChain{{ + ChainID: "chain-1", + Title: "SSRF to internal API", + FindingIDs: []string{"v1", "v2"}, + CombinedImpact: "Internal service access", + EstimatedSeverity: schemas.SeverityCritical, + }} +} diff --git a/internal/secaf/agents/prove/prompts_test.go b/internal/secaf/agents/prove/prompts_test.go new file mode 100644 index 0000000000..b2af329542 --- /dev/null +++ b/internal/secaf/agents/prove/prompts_test.go @@ -0,0 +1,218 @@ +package prove + +// Golden prompt tests: every string this package hands to an LLM, compared +// byte-for-byte against the output of the real Python builder. +// +// Contract items pinned here (see DESIGN.md §5): +// - each prompt equals the Python builder's output for the same inputs; +// - the trailing CONTEXT block is present (and absent for verdict/chain); +// - substitution ORDER is preserved: "{{TITLE}}" placed inside a value +// substituted AFTER it survives, "{{DEPTH}}" placed inside a value +// substituted BEFORE it is itself replaced. + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +func TestTracerPromptGolden(t *testing.T) { + got := TracerPrompt(findingRich(), fixtureRepo, "thorough") + if want := goldenText(t, "tracer_prompt_A"); got != want { + t.Errorf("tracer_prompt_A mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } + got = TracerPrompt(findingBare(), fixtureRepo, "quick") + if want := goldenText(t, "tracer_prompt_B"); got != want { + t.Errorf("tracer_prompt_B mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +// TestTracerPromptSubstitutionOrder states the ordering contract explicitly, so +// a regression names the cause rather than dumping two 1.5 KB prompts. +func TestTracerPromptSubstitutionOrder(t *testing.T) { + got := TracerPrompt(findingRich(), fixtureRepo, "thorough") + if !strings.Contains(got, "Marker: {{TITLE}}") { + t.Error("{{TITLE}} inside the description must survive: TITLE is substituted before DESCRIPTION") + } + if strings.Contains(got, "depth={{DEPTH}}") { + t.Error("{{DEPTH}} inside the code snippet must be replaced: DEPTH is substituted after CODE_SNIPPET") + } + if !strings.Contains(got, "depth=thorough") { + t.Error("{{DEPTH}} inside the code snippet should have become the depth string") + } +} + +func TestSanitizationPromptGolden(t *testing.T) { + got := SanitizationPrompt(findingRich(), traceRich(), fixtureRepo, "thorough") + if want := goldenText(t, "sanitization_prompt_A"); got != want { + t.Errorf("sanitization_prompt_A mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } + got = SanitizationPrompt(findingBare(), traceBare(), fixtureRepo, "quick") + if want := goldenText(t, "sanitization_prompt_B"); got != want { + t.Errorf("sanitization_prompt_B mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +func TestExploitPromptGolden(t *testing.T) { + got := ExploitPrompt(findingRich(), traceRich(), sanitizationRich(), fixtureRepo, "thorough") + if want := goldenText(t, "exploit_prompt_A"); got != want { + t.Errorf("exploit_prompt_A mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } + got = ExploitPrompt(findingBare(), traceBare(), sanitizationBare(), fixtureRepo, "quick") + if want := goldenText(t, "exploit_prompt_B"); got != want { + t.Errorf("exploit_prompt_B mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +func TestVerdictPromptGolden(t *testing.T) { + got := VerdictPrompt(findingRich(), traceRich(), sanitizationRich(), exploitRich()) + if want := goldenText(t, "verdict_prompt_A"); got != want { + t.Errorf("verdict_prompt_A mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } + got = VerdictPrompt(findingBare(), traceBare(), sanitizationBare(), exploitBare()) + if want := goldenText(t, "verdict_prompt_B"); got != want { + t.Errorf("verdict_prompt_B mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +// TestVerdictPromptHasNoContextBlock and TestVerdictPromptKeepsDepthMarker pin +// the two things verdict.py does differently from every other prove agent. +func TestVerdictPromptHasNoContextBlock(t *testing.T) { + got := VerdictPrompt(findingRich(), traceRich(), sanitizationRich(), exploitRich()) + if strings.Contains(got, "- Repository path: ") { + t.Error("verdict.py appends no CONTEXT block — VerdictAgent needs no file access") + } +} + +func TestVerdictPromptKeepsDepthMarker(t *testing.T) { + got := VerdictPrompt(findingRich(), traceRich(), sanitizationRich(), exploitRich()) + if !strings.Contains(got, "depth={{DEPTH}}") { + t.Error("verdict._build_prompt has no {{DEPTH}} entry, so the marker must survive verbatim") + } +} + +func TestDepReachabilityPromptGolden(t *testing.T) { + var inputA map[string]any + goldenJSON(t, "dep_reachability_input_A", &inputA) + got := DepReachabilityPrompt(inputA, fixtureRepo, "thorough") + if want := goldenText(t, "dep_reachability_prompt_A"); got != want { + t.Errorf("dep_reachability_prompt_A mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } + + // An EMPTY dict: every `.get(key, "")` falls back and `evidence` is `{}`. + got = DepReachabilityPrompt(map[string]any{}, fixtureRepo, "quick") + if want := goldenText(t, "dep_reachability_prompt_B"); got != want { + t.Errorf("dep_reachability_prompt_B mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } + + // Non-string scalars exercise `str(...)`: an int keeps its integer + // spelling, a float gets repr(float), True/None stringify Python-style, + // and a key present with a null value is "None" rather than "". + var inputC map[string]any + goldenJSON(t, "dep_reachability_input_C", &inputC) + got = DepReachabilityPrompt(inputC, fixtureRepo, "standard") + if want := goldenText(t, "dep_reachability_prompt_C"); got != want { + t.Errorf("dep_reachability_prompt_C mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +// TestDepReachabilityPromptNilFinding covers the Go-only nil-map case: Python +// always receives a dict, and a nil Go map must behave like the empty one. +func TestDepReachabilityPromptNilFinding(t *testing.T) { + got := DepReachabilityPrompt(nil, fixtureRepo, "quick") + if want := goldenText(t, "dep_reachability_prompt_B"); got != want { + t.Errorf("nil finding must render like the empty dict:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +// TestDepReachabilityIntegerSpelling pins the json.Number normalization: the +// SDK hands the handler a map whose numbers are float64, and an integral value +// must still render the way Python's int does ("2", not "2.0"). +func TestDepReachabilityIntegerSpelling(t *testing.T) { + got := DepReachabilityPrompt(map[string]any{ + "cve": float64(1234), + "evidence": map[string]any{"count": float64(2), "ratio": 0.5}, + }, fixtureRepo, "quick") + if !strings.Contains(got, "- CVE: 1234\n") { + t.Errorf("integral float64 must render as a Python int; got:\n%s", got) + } + if !strings.Contains(got, `"count": 2,`) { + t.Errorf("integral float64 inside evidence must render as a Python int; got:\n%s", got) + } + if !strings.Contains(got, `"ratio": 0.5`) { + t.Errorf("a real float must keep its fraction; got:\n%s", got) + } +} + +// TestDepReachabilityEvidenceKeyOrder documents the ONE accepted divergence +// from Python in this package: json.dumps follows dict insertion order, a Go +// map has none, so evidence keys come out sorted (DESIGN.md §2b). +func TestDepReachabilityEvidenceKeyOrder(t *testing.T) { + got := DepReachabilityPrompt(map[string]any{ + "evidence": map[string]any{"zeta": 1, "alpha": 2}, + }, fixtureRepo, "quick") + if !strings.Contains(got, "{\n \"alpha\": 2,\n \"zeta\": 1\n}") { + t.Errorf("evidence must render with SORTED keys; got:\n%s", got) + } +} + +func TestDastPromptGolden(t *testing.T) { + got := DastPrompt(findingRich(), "1 OR 1=1 -- {{DEPTH}}", fixtureRepo, "thorough") + if want := goldenText(t, "dast_prompt_A"); got != want { + t.Errorf("dast_prompt_A mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } + got = DastPrompt(findingBare(), "", fixtureRepo, "quick") + if want := goldenText(t, "dast_prompt_B"); got != want { + t.Errorf("dast_prompt_B mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +func TestCrossServicePromptGolden(t *testing.T) { + got := CrossServicePrompt( + []string{"gateway", "billing-café", "db<&>"}, + "- gateway: SSRF\n- billing: IDOR", + fixtureRepo, "thorough") + if want := goldenText(t, "cross_service_prompt_A"); got != want { + t.Errorf("cross_service_prompt_A mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } + got = CrossServicePrompt([]string{}, "", fixtureRepo, "quick") + if want := goldenText(t, "cross_service_prompt_B"); got != want { + t.Errorf("cross_service_prompt_B mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } + // A nil slice stands for Python's empty list, never for null. + if got := CrossServicePrompt(nil, "", fixtureRepo, "quick"); got != goldenText(t, "cross_service_prompt_B") { + t.Error("a nil services slice must render as [] like Python's empty list") + } +} + +func TestChainBuilderPromptGolden(t *testing.T) { + var input struct { + Depth string `json:"depth"` + Chains []json.RawMessage `json:"chains"` + Findings []json.RawMessage `json:"findings"` + } + goldenJSON(t, "chain_builder_prompt_input", &input) + if input.Depth != "standard" || len(input.Chains) != 1 || len(input.Findings) != 2 { + t.Fatalf("unexpected chain_builder_prompt_input shape: %+v", input) + } + + chains := chainBuilderFixtureChains() + findings := []schemas.VerifiedFinding{verified("v1", "fp-v1", 7.5, 4), verified("v2", "fp-v2", 3.0, 2)} + + got := ChainBuilderPrompt(chains, findings, "standard") + if want := goldenText(t, "chain_builder_prompt"); got != want { + t.Errorf("chain_builder_prompt mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +// TestChainBuilderPromptHasNoContextBlock: like verdict.py, chain_builder.py +// sends the substituted template alone. +func TestChainBuilderPromptHasNoContextBlock(t *testing.T) { + got := ChainBuilderPrompt(chainBuilderFixtureChains(), + []schemas.VerifiedFinding{verified("v1", "fp-v1", 7.5, 4)}, "standard") + if strings.Contains(got, "- Repository path: ") { + t.Error("chain_builder.py appends no CONTEXT block") + } +} diff --git a/internal/secaf/agents/prove/prove.go b/internal/secaf/agents/prove/prove.go new file mode 100644 index 0000000000..d3174d9de7 --- /dev/null +++ b/internal/secaf/agents/prove/prove.go @@ -0,0 +1,403 @@ +package prove + +// Ports src/sec_af/agents/prove/__init__.py — the PROVE phase driver that +// prioritizes findings, fans verification out under a semaphore, decorates each +// result with scoring/compliance metadata, and (when HUNT proposed chains) +// hands the set to the chain builder. + +import ( + "context" + "sort" + "strings" + "sync" + + "golang.org/x/sync/semaphore" + + "github.com/Agent-Field/codeaf/internal/secaf/compliance" + "github.com/Agent-Field/codeaf/internal/secaf/config" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" + "github.com/Agent-Field/codeaf/internal/secaf/scoring" +) + +// DefaultMaxConcurrentProvers is the `max_concurrent_provers: int = 3` default +// shared by run_prove and run_prove_streaming. +const DefaultMaxConcurrentProvers = 3 + +// DefaultProverCap is run_prove_streaming's `prover_cap: int = 30` default. +const DefaultProverCap = 30 + +// severityRank / confidenceRank port `_SEVERITY_RANK` and `_CONFIDENCE_RANK`. +// Both are looked up with `.get(key, 0)`, so an unknown value ranks below INFO +// / LOW rather than raising. +var severityRank = map[schemas.Severity]int{ + schemas.SeverityCritical: 5, + schemas.SeverityHigh: 4, + schemas.SeverityMedium: 3, + schemas.SeverityLow: 2, + schemas.SeverityInfo: 1, +} + +var confidenceRank = map[schemas.Confidence]int{ + schemas.ConfidenceHigh: 3, + schemas.ConfidenceMedium: 2, + schemas.ConfidenceLow: 1, +} + +// PrioritySort ports `_priority_sort`: +// +// return sorted(findings, +// key=lambda f: (_SEVERITY_RANK.get(f.estimated_severity, 0), +// _CONFIDENCE_RANK.get(f.confidence, 0)), +// reverse=True) +// +// Python parity: `sorted(..., reverse=True)` returns a NEW list and is STABLE — +// reversing does not reverse ties, so findings with the same +// (severity, confidence) keep their input order. sort.SliceStable over a copy +// with a strict greater-than comparator is exactly that; sort.Slice would be +// free to permute ties. +func PrioritySort(findings []schemas.RawFinding) []schemas.RawFinding { + out := make([]schemas.RawFinding, len(findings)) + copy(out, findings) + sort.SliceStable(out, func(i, j int) bool { + si, sj := severityRank[out[i].EstimatedSeverity], severityRank[out[j].EstimatedSeverity] + if si != sj { + return si > sj + } + return confidenceRank[out[i].Confidence] > confidenceRank[out[j].Confidence] + }) + return out +} + +// ApplyMetadata ports `_apply_metadata`: +// +// finding.severity = apply_cwe_severity_floor(finding.cwe_id, finding.severity) +// finding.compliance = get_compliance_mappings(finding.cwe_id) +// finding.exploitability_score = compute_exploitability_score(finding) +// finding.sarif_security_severity = finding.exploitability_score +// if not finding.sarif_rule_id: +// finding.sarif_rule_id = f"sec-af/{finding.finding_type.value}/{cwe_slug}" +// return finding +// +// ORDER IS LOAD-BEARING: the severity floor is applied BEFORE the exploitability +// score is computed, so a floored severity feeds the score. The rule-id backfill +// runs last and only when the id is falsy (empty string). +// +// Python mutates the model in place and returns it; Go takes and returns a +// value. Every call site rebuilds its list from the return value, so the two are +// observably identical. +// +// `get_compliance_mappings(finding.cwe_id)` is called with the default +// `frameworks=None`, which the Go signature spells as a nil slice. +func ApplyMetadata(finding schemas.VerifiedFinding) schemas.VerifiedFinding { + finding.Severity = scoring.ApplyCWESeverityFloor(finding.CweID, finding.Severity) + finding.Compliance = compliance.GetComplianceMappings(finding.CweID, nil) + finding.ExploitabilityScore = scoring.ComputeExploitabilityScore(finding) + finding.SarifSecuritySeverity = finding.ExploitabilityScore + if finding.SarifRuleID == "" { + finding.SarifRuleID = sarifRuleID(finding.FindingType, finding.CweName) + } + return finding +} + +// demoteOnError ports the `except BaseException` body shared by +// `_run_parallel_verification._verify` and `run_prove_streaming._verify_one`: +// +// message = str(exc) +// lowered = message.lower() +// if "unverified" in lowered and "verdict" in lowered: +// return verifier_fallback(finding, +// "Verifier returned unverified verdict; demoted for manual review", +// drop_reason="verdict_unverified", original_verdict="unverified") +// drop_reason = "schema_parse_failure" if "validationerror" in lowered else "verifier_error" +// return verifier_fallback(finding, message, drop_reason=drop_reason) +// +// Python parity: the "validationerror" probe is a substring test with NO space, +// so a pydantic ValidationError — whose str() begins "1 validation error for +// ..." — does NOT match it here. The branch is effectively unreachable through +// this path (reasoners/phases.py classifies parse failures separately, which is +// what tests/test_prove_phase_demotion.py exercises). It is ported verbatim +// rather than "fixed": changing it would change which findings get the +// schema_parse_failure drop reason. +func demoteOnError(finding schemas.RawFinding, err error) schemas.VerifiedFinding { + message := err.Error() + lowered := strings.ToLower(message) + if strings.Contains(lowered, "unverified") && strings.Contains(lowered, "verdict") { + return Fallback(finding, + "Verifier returned unverified verdict; demoted for manual review", + StrPtr("verdict_unverified"), StrPtr("unverified")) + } + dropReason := "verifier_error" + if strings.Contains(lowered, "validationerror") { + dropReason = "schema_parse_failure" + } + return Fallback(finding, message, StrPtr(dropReason), nil) +} + +// runParallelVerification ports `_run_parallel_verification`: +// +// if not findings: return [] +// concurrency_limit = max(1, min(max_concurrent_provers, len(findings))) +// semaphore = asyncio.Semaphore(concurrency_limit) +// async def _verify(finding): async with semaphore: try: ... except: ... +// return await asyncio.gather(*[_verify(f) for f in findings]) +// +// Concurrency parity: +// +// - the limit is `max(1, min(max_concurrent_provers, len(findings)))`, so a +// zero or negative max_concurrent_provers still admits one prover at a time +// and the semaphore is never wider than the work; +// - RESULT ORDER follows INPUT order (asyncio.gather preserves it), so results +// go into a pre-indexed slice, not an append-as-they-finish list; +// - `_verify` catches BaseException and always returns a VerifiedFinding, so +// the plain `gather` (no return_exceptions) never sees an exception. Go uses +// a WaitGroup for the same reason — there is no error to propagate. +// - a semaphore acquire that fails (only possible once ctx is done) is the +// closest analogue of asyncio cancelling a queued task, and is funnelled +// through demoteOnError so the finding is still reported. +func runParallelVerification( + ctx context.Context, + app HarnessAIer, + repoPath string, + findings []schemas.RawFinding, + depth string, + maxConcurrentProvers int, +) []schemas.VerifiedFinding { + if len(findings) == 0 { + return []schemas.VerifiedFinding{} + } + + limit := maxConcurrentProvers + if limit > len(findings) { + limit = len(findings) + } + if limit < 1 { + limit = 1 + } + sem := semaphore.NewWeighted(int64(limit)) + + out := make([]schemas.VerifiedFinding, len(findings)) + var wg sync.WaitGroup + for i := range findings { + wg.Add(1) + go func(idx int, finding schemas.RawFinding) { + defer wg.Done() + if err := sem.Acquire(ctx, 1); err != nil { + out[idx] = demoteOnError(finding, err) + return + } + defer sem.Release(1) + verified, err := RunVerifier(ctx, app, repoPath, finding, depth) + if err != nil { + out[idx] = demoteOnError(finding, err) + return + } + out[idx] = verified + }(i, findings[i]) + } + wg.Wait() + return out +} + +// sortByScore ports the tail both run_prove and run_prove_streaming share: +// +// verified.sort(key=lambda f: (f.exploitability_score, f.evidence_level), reverse=True) +// +// In-place, stable, descending on the pair. EvidenceLevel is an IntEnum, so the +// second key compares as an integer. +func sortByScore(findings []schemas.VerifiedFinding) { + sort.SliceStable(findings, func(i, j int) bool { + if findings[i].ExploitabilityScore != findings[j].ExploitabilityScore { + return findings[i].ExploitabilityScore > findings[j].ExploitabilityScore + } + return findings[i].EvidenceLevel > findings[j].EvidenceLevel + }) +} + +// RunProve ports `run_prove`. +// +// profile = _normalize_depth(depth) +// prioritized = _priority_sort(hunt_result.findings) +// verified = await _run_parallel_verification(app, repo_path, prioritized, profile.value, max_concurrent_provers) +// verified = [_apply_metadata(f) for f in verified] +// if hunt_result.chains: +// verified = await run_chain_builder(app=..., repo_path=..., potential_chains=hunt_result.chains, +// findings=verified, depth=profile.value) +// verified = [_apply_metadata(f) for f in verified] +// verified.sort(key=lambda f: (f.exploitability_score, f.evidence_level), reverse=True) +// return verified +// +// Python parity notes: +// +// - the depth string handed to the sub-agents is the NORMALIZED profile value, +// so an unrecognised depth reaches the prompts as "standard"; +// - `_apply_metadata` runs TWICE when chains are present — the second pass +// re-scores the findings the chain builder tagged with a chain_id, which is +// what turns a chained finding's exploitability score into the 2x +// chain-bonus form; +// - run_chain_builder's AttributeError (see ErrChainTagsNotASet) propagates +// out of run_prove, exactly as it does in Python. The partially-mutated +// findings are returned alongside the error. +func RunProve( + ctx context.Context, + app HarnessAIer, + repoPath string, + huntResult schemas.HuntResult, + depth string, + maxConcurrentProvers int, +) ([]schemas.VerifiedFinding, error) { + profile := config.NormalizeDepth(depth) + prioritized := PrioritySort(huntResult.Findings) + + verified := runParallelVerification(ctx, app, repoPath, prioritized, profile.String(), maxConcurrentProvers) + for i := range verified { + verified[i] = ApplyMetadata(verified[i]) + } + + if len(huntResult.Chains) > 0 { + chained, err := RunChainBuilder(ctx, app, repoPath, huntResult.Chains, verified, profile.String()) + if err != nil { + // Python parity: run_chain_builder RAISES here, so neither the + // second _apply_metadata pass nor the final sort ever runs. The + // partially-mutated findings are handed back purely so a Go caller + // can log them; Python has no return value at all on this path. + return chained, err + } + verified = chained + for i := range verified { + verified[i] = ApplyMetadata(verified[i]) + } + } + + sortByScore(verified) + return verified, nil +} + +// RunProveStreaming ports `run_prove_streaming` — the incremental variant the +// orchestrator drives while HUNT is still producing findings. +// +// profile = _normalize_depth(depth) +// semaphore = asyncio.Semaphore(max(1, max_concurrent_provers)) +// while True: +// batch = await findings_queue.get() +// if batch is None: break +// for finding in batch: +// if proved_count >= prover_cap: break +// pending_tasks.append(asyncio.create_task(_verify_one(finding))) +// proved_count += 1 +// if proved_count >= prover_cap: +// while True: # drain to the sentinel +// remaining = await findings_queue.get() +// if remaining is None: break +// break +// if pending_tasks: +// results = await asyncio.gather(*pending_tasks, return_exceptions=True) +// for result in results: +// if isinstance(result, BaseException): continue +// verified.append(_apply_metadata(result)) +// verified.sort(key=lambda f: (f.exploitability_score, f.evidence_level), reverse=True) +// +// The Go channel stands in for the asyncio.Queue; a NIL batch is the `None` +// sentinel. An EMPTY-but-non-nil batch is a real (empty) batch and does NOT +// terminate the loop, matching Python where `[]` is not None. +// +// Python parity notes: +// +// - tasks START as soon as they are created (asyncio.create_task schedules +// immediately), bounded only by the semaphore — so verification of batch N +// overlaps arrival of batch N+1. The Go port launches a goroutine per +// finding at the same moment for the same reason. +// - the semaphore here is `max(1, max_concurrent_provers)` — NOT clamped to +// the finding count as in _run_parallel_verification, because the total is +// not known up front. +// - once the cap is hit the producer is drained to its sentinel rather than +// abandoned, which is what keeps the producing side from blocking forever +// on a full queue. +// - `_verify_one` swallows every exception into a Fallback, so the +// `isinstance(result, BaseException): continue` filter never actually drops +// a finding. Ported as a nil-error check for the same reason as +// runParallelVerification. +// - the results keep TASK CREATION order before the final sort, which is the +// queue arrival order. +// +// Unlike RunProve there is no chain-builder pass here, so no error can escape; +// the signature returns only the findings. +func RunProveStreaming( + ctx context.Context, + app HarnessAIer, + repoPath string, + findingsQueue <-chan []schemas.RawFinding, + depth string, + maxConcurrentProvers int, + proverCap int, +) []schemas.VerifiedFinding { + profile := config.NormalizeDepth(depth) + + limit := maxConcurrentProvers + if limit < 1 { + limit = 1 + } + sem := semaphore.NewWeighted(int64(limit)) + + var ( + wg sync.WaitGroup + mu sync.Mutex + pending = map[int]schemas.VerifiedFinding{} + provedCount int + ) + + // Results land in a MAP keyed by task-creation index rather than a slice: + // slots are handed out while goroutines are already running, and appending + // to a shared slice would reallocate the backing array out from under a + // concurrent write. The map is materialized in slot order after Wait. + verifyOne := func(slot int, finding schemas.RawFinding) { + defer wg.Done() + result := func() schemas.VerifiedFinding { + if err := sem.Acquire(ctx, 1); err != nil { + return demoteOnError(finding, err) + } + defer sem.Release(1) + verified, err := RunVerifier(ctx, app, repoPath, finding, profile.String()) + if err != nil { + return demoteOnError(finding, err) + } + return verified + }() + mu.Lock() + pending[slot] = result + mu.Unlock() + } + + // Python loops on `await queue.get()` forever and relies on the None + // sentinel. Go's range additionally ends when the producer CLOSES the + // channel, which Python would hang on; that is a strictly safer superset. + for batch := range findingsQueue { + if batch == nil { + break + } + for _, finding := range batch { + if provedCount >= proverCap { + break + } + wg.Add(1) + go verifyOne(provedCount, finding) + provedCount++ + } + if provedCount >= proverCap { + // Drain the producer to its sentinel, then stop consuming. + for remaining := range findingsQueue { + if remaining == nil { + break + } + } + break + } + } + wg.Wait() + + verified := make([]schemas.VerifiedFinding, 0, len(pending)) + for slot := 0; slot < provedCount; slot++ { + verified = append(verified, ApplyMetadata(pending[slot])) + } + sortByScore(verified) + return verified +} diff --git a/internal/secaf/agents/prove/prove_test.go b/internal/secaf/agents/prove/prove_test.go new file mode 100644 index 0000000000..beb69a8b6f --- /dev/null +++ b/internal/secaf/agents/prove/prove_test.go @@ -0,0 +1,758 @@ +package prove + +// Tests for prove.go — the PROVE driver. +// +// Validation contract: +// - PrioritySort orders by (severity rank, confidence rank) DESCENDING and is +// stable for ties; the input slice is not mutated; +// - ApplyMetadata applies the CWE severity floor BEFORE scoring, fills the +// compliance mappings, mirrors the score into sarif_security_severity, and +// backfills the rule id only when it is empty; +// - RunProve verifies every finding, applies metadata, and returns the set +// sorted by (exploitability_score, evidence_level) descending; +// - RunProve's verification fan-out never exceeds +// max(1, min(max_concurrent_provers, len(findings))) in flight; +// - a failing sub-agent demotes that finding via Fallback instead of failing +// the phase; +// - RunProveStreaming consumes batches until the nil sentinel, stops +// scheduling at prover_cap, drains the producer afterwards, and bounds +// concurrency at max(1, max_concurrent_provers). + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/Agent-Field/agentfield/sdk/go/ai" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// cannedApp mirrors gen_golden_prove._CannedApp: it recognises each prove +// sub-agent by its ROLE line and answers with a fixed model. +func cannedApp(verdict string, level int) *appx.Fake { + return &appx.Fake{ + HarnessFn: appx.HarnessJSON(func(prompt string, _ appx.HarnessOptions) (json.RawMessage, error) { + switch { + case strings.Contains(prompt, "You are DataFlowTracer"): + return json.Marshal(traceRich()) + case strings.Contains(prompt, "You are SanitizationAnalyzer"): + return json.Marshal(sanitizationRich()) + case strings.Contains(prompt, "You are ExploitHypothesizer"): + return json.Marshal(exploitRich()) + } + return nil, errors.New("unexpected harness prompt") + }), + AIFn: appx.AIJSON(func(string) (json.RawMessage, error) { + return json.Marshal(map[string]any{ + "verdict": verdict, + "evidence_level": level, + "rationale": "canned rationale", + "confidence": "high", + }) + }), + } +} + +func TestPrioritySortGolden(t *testing.T) { + var golden struct { + Input []struct { + ID string `json:"id"` + EstimatedSeverity string `json:"estimated_severity"` + Confidence string `json:"confidence"` + } `json:"input"` + WantIDs []string `json:"want_ids"` + } + goldenJSON(t, "priority_sort", &golden) + + input := make([]schemas.RawFinding, len(golden.Input)) + for i, row := range golden.Input { + input[i] = schemas.RawFinding{ + ID: row.ID, + EstimatedSeverity: schemas.Severity(row.EstimatedSeverity), + Confidence: schemas.Confidence(row.Confidence), + } + } + + got := PrioritySort(input) + gotIDs := make([]string, len(got)) + for i, f := range got { + gotIDs[i] = f.ID + } + if !reflect.DeepEqual(gotIDs, golden.WantIDs) { + t.Errorf("PrioritySort order = %v, want %v", gotIDs, golden.WantIDs) + } + + // `sorted()` returns a NEW list; the caller's slice keeps its order. + if input[0].ID != golden.Input[0].ID { + t.Error("PrioritySort must not reorder its input slice") + } +} + +// TestPrioritySortStability pins that reverse=True does NOT reverse ties: "b" +// and "e" share (critical, low) and keep their input order. +func TestPrioritySortStability(t *testing.T) { + in := []schemas.RawFinding{ + {ID: "b", EstimatedSeverity: schemas.SeverityCritical, Confidence: schemas.ConfidenceLow}, + {ID: "e", EstimatedSeverity: schemas.SeverityCritical, Confidence: schemas.ConfidenceLow}, + } + got := PrioritySort(in) + if got[0].ID != "b" || got[1].ID != "e" { + t.Errorf("ties must keep input order, got %s,%s", got[0].ID, got[1].ID) + } +} + +// TestPrioritySortUnknownRanksLast pins `.get(key, 0)` — an out-of-vocabulary +// severity or confidence ranks below every declared member. +func TestPrioritySortUnknownRanksLast(t *testing.T) { + in := []schemas.RawFinding{ + {ID: "unknown", EstimatedSeverity: "bogus", Confidence: "bogus"}, + {ID: "info", EstimatedSeverity: schemas.SeverityInfo, Confidence: schemas.ConfidenceLow}, + } + got := PrioritySort(in) + if got[0].ID != "info" || got[1].ID != "unknown" { + t.Errorf("unknown values must sort last, got %s,%s", got[0].ID, got[1].ID) + } +} + +func TestApplyMetadataGolden(t *testing.T) { + var want map[string]any + goldenJSON(t, "apply_metadata", &want) + + mints := verified("m2", "fp-m2", 0.0, 6) + mints.SarifRuleID = "" + mints.CweName = "Broken Access/Control Check" + + cases := map[string]schemas.VerifiedFinding{ + "keeps_rule_id": ApplyMetadata(verified("m1", "fp-m1", 0.0, 3)), + "mints_rule_id": ApplyMetadata(mints), + "with_reachability_tag": ApplyMetadata(verified("m3", "fp-m3", 0.0, 6, "externally_reachable")), + } + if len(cases) != len(want) { + t.Fatalf("case count drift: go has %d, golden has %d", len(cases), len(want)) + } + for name, got := range cases { + if !reflect.DeepEqual(jsonTree(t, got), want[name]) { + t.Errorf("apply_metadata[%s] mismatch:\n got: %#v\nwant: %#v", name, jsonTree(t, got), want[name]) + } + } +} + +// TestApplyMetadataFloorsBeforeScoring pins the ORDER inside _apply_metadata: +// CWE-89 floors "low" up to "critical", and the score must reflect the FLOORED +// severity, not the original one. +func TestApplyMetadataFloorsBeforeScoring(t *testing.T) { + f := verified("x", "fp-x", 0.0, 6) + f.Severity = schemas.SeverityLow + got := ApplyMetadata(f) + if got.Severity != schemas.SeverityCritical { + t.Fatalf("CWE-89 must floor severity to critical, got %q", got.Severity) + } + if got.ExploitabilityScore != 10.0 { + t.Errorf("score must use the floored severity (10.0), got %v", got.ExploitabilityScore) + } + if got.SarifSecuritySeverity != got.ExploitabilityScore { + t.Errorf("sarif_security_severity must mirror the score, got %v", got.SarifSecuritySeverity) + } + if len(got.Compliance) == 0 { + t.Error("CWE-89 has static compliance mappings; they must be attached") + } +} + +func TestRunProveGolden(t *testing.T) { + var golden struct { + RepoPath string `json:"repo_path"` + Depth string `json:"depth"` + Want []json.RawMessage `json:"want"` + } + goldenJSON(t, "run_prove", &golden) + + hunt := schemas.HuntResult{ + Findings: []schemas.RawFinding{findingBare(), findingRich()}, + Chains: []schemas.PotentialChain{}, + TotalRaw: 2, + DeduplicatedCount: 2, + StrategiesRun: []string{"injection"}, + } + got, err := RunProve(context.Background(), cannedApp("confirmed", 5), golden.RepoPath, hunt, golden.Depth, 3) + if err != nil { + t.Fatalf("RunProve: %v", err) + } + if len(got) != len(golden.Want) { + t.Fatalf("got %d findings, want %d", len(got), len(golden.Want)) + } + for i := range got { + var want any + if err := json.Unmarshal(golden.Want[i], &want); err != nil { + t.Fatalf("decode want[%d]: %v", i, err) + } + if !reflect.DeepEqual(jsonTree(t, got[i]), want) { + t.Errorf("run_prove[%d] mismatch:\n got: %#v\nwant: %#v", i, jsonTree(t, got[i]), want) + } + } +} + +// TestRunProveNormalizesDepth pins that the sub-agent prompts receive the +// NORMALIZED profile value — an unrecognised depth becomes "standard". +func TestRunProveNormalizesDepth(t *testing.T) { + app := cannedApp("likely", 3) + hunt := schemas.HuntResult{Findings: []schemas.RawFinding{findingBare()}} + if _, err := RunProve(context.Background(), app, fixtureRepo, hunt, "NoSuchDepth", 1); err != nil { + t.Fatalf("RunProve: %v", err) + } + for _, call := range app.Harnesses { + if strings.Contains(call.Prompt, "- Analysis depth: NoSuchDepth") { + t.Fatal("an unrecognised depth must be normalized to standard before it reaches a prompt") + } + } + if len(app.Harnesses) == 0 { + t.Fatal("expected harness calls") + } + if !strings.Contains(app.Harnesses[0].Prompt, "- Analysis depth: standard") { + t.Errorf("prompt should carry the normalized depth; got:\n%s", app.Harnesses[0].Prompt) + } +} + +// TestRunProveEmptyHunt pins that an empty finding list short-circuits to an +// empty (non-nil) result with no harness traffic. +func TestRunProveEmptyHunt(t *testing.T) { + app := &appx.Fake{} + got, err := RunProve(context.Background(), app, fixtureRepo, schemas.HuntResult{}, "quick", 3) + if err != nil { + t.Fatalf("RunProve: %v", err) + } + if got == nil || len(got) != 0 { + t.Errorf("want an empty non-nil slice, got %#v", got) + } + if len(app.Harnesses) != 0 || len(app.AIs) != 0 { + t.Error("no findings must mean no sub-agent calls") + } +} + +// TestRunProveDemotesVerifierFailure pins that a sub-agent failure demotes ONE +// finding rather than failing the phase — the behaviour orchestrator.py and +// reasoners/phases.py both depend on. +func TestRunProveDemotesVerifierFailure(t *testing.T) { + app := &appx.Fake{ + HarnessFn: appx.HarnessJSON(func(prompt string, _ appx.HarnessOptions) (json.RawMessage, error) { + if strings.Contains(prompt, "You are ExploitHypothesizer") { + return nil, errors.New("provider exploded") + } + if strings.Contains(prompt, "You are DataFlowTracer") { + return json.Marshal(traceRich()) + } + return json.Marshal(sanitizationRich()) + }), + } + hunt := schemas.HuntResult{Findings: []schemas.RawFinding{findingRich()}} + got, err := RunProve(context.Background(), app, fixtureRepo, hunt, "quick", 3) + if err != nil { + t.Fatalf("RunProve must absorb a sub-agent failure, got %v", err) + } + if len(got) != 1 { + t.Fatalf("want 1 demoted finding, got %d", len(got)) + } + if got[0].Verdict != schemas.VerdictInconclusive { + t.Errorf("verdict = %q, want inconclusive", got[0].Verdict) + } + if got[0].DropReason == nil || *got[0].DropReason != "verifier_error" { + t.Errorf("drop_reason = %v, want verifier_error", got[0].DropReason) + } + if !strings.Contains(got[0].Rationale, "ExploitHypothesizer harness error: provider exploded") { + t.Errorf("rationale must carry the failure message, got %q", got[0].Rationale) + } + if len(got[0].Tags) != 1 || got[0].Tags[0] != "low_confidence" { + t.Errorf("tags = %v, want [low_confidence]", got[0].Tags) + } +} + +// TestDemoteOnErrorClassification pins the three drop-reason branches of the +// shared `except BaseException` body, including the "validationerror" probe +// that Python spells WITHOUT a space. +func TestDemoteOnErrorClassification(t *testing.T) { + f := findingBare() + for _, tc := range []struct { + name string + err error + dropReason string + rationalePart string + }{ + {"unverified verdict", errors.New("Verdict unverified for finding"), "verdict_unverified", + "Verifier returned unverified verdict; demoted for manual review (original verdict: unverified)"}, + {"validationerror substring", errors.New("pydantic ValidationError: bad"), "schema_parse_failure", + "pydantic ValidationError: bad"}, + {"anything else", errors.New("boom"), "verifier_error", "boom"}, + // A real pydantic message says "1 validation error for X" — with a + // SPACE — so it does NOT hit the schema_parse_failure branch here. + {"spaced validation error", errors.New("1 validation error for VerifiedFinding"), "verifier_error", + "1 validation error for VerifiedFinding"}, + } { + got := demoteOnError(f, tc.err) + if got.DropReason == nil || *got.DropReason != tc.dropReason { + t.Errorf("%s: drop_reason = %v, want %q", tc.name, got.DropReason, tc.dropReason) + } + if !strings.Contains(got.Rationale, tc.rationalePart) { + t.Errorf("%s: rationale = %q, want it to contain %q", tc.name, got.Rationale, tc.rationalePart) + } + } +} + +// TestRunProveConcurrencyBound pins the semaphore: +// max(1, min(max_concurrent_provers, len(findings))). Each verification issues +// three harness calls, two of which (tracer + sanitization) run concurrently +// WITHIN a verification, so the observable ceiling is 2*limit. +func TestRunProveConcurrencyBound(t *testing.T) { + findings := make([]schemas.RawFinding, 8) + for i := range findings { + f := findingBare() + f.ID = "f" + string(rune('a'+i)) + f.Fingerprint = f.ID + findings[i] = f + } + + for _, limit := range []int{1, 2, 3} { + app := cannedApp("confirmed", 4) + if _, err := RunProve(context.Background(), app, fixtureRepo, + schemas.HuntResult{Findings: findings}, "quick", limit); err != nil { + t.Fatalf("RunProve: %v", err) + } + if got := app.MaxConcurrentHarness(); got > 2*limit { + t.Errorf("limit %d: peak concurrent harness calls = %d, want <= %d", limit, got, 2*limit) + } + } + + // A zero/negative limit still admits one prover — `max(1, ...)`. + app := cannedApp("confirmed", 4) + if _, err := RunProve(context.Background(), app, fixtureRepo, + schemas.HuntResult{Findings: findings}, "quick", 0); err != nil { + t.Fatalf("RunProve: %v", err) + } + if got := app.MaxConcurrentHarness(); got > 2 { + t.Errorf("a zero limit must behave like 1 prover; peak = %d", got) + } +} + +// TestRunVerifierRunsTracerAndSanitizationConcurrently pins the +// asyncio.gather(tracer, sanitization) shape: both are in flight at once. +func TestRunVerifierRunsTracerAndSanitizationConcurrently(t *testing.T) { + release := make(chan struct{}) + seen := make(chan string, 2) + app := &appx.Fake{ + HarnessFn: func(_ context.Context, prompt string, _ map[string]any, dest any, _ appx.HarnessOptions) (*appx.HarnessResult, error) { + switch { + case strings.Contains(prompt, "You are DataFlowTracer"): + seen <- "tracer" + <-release + b, _ := json.Marshal(traceRich()) + _ = json.Unmarshal(b, dest) + case strings.Contains(prompt, "You are SanitizationAnalyzer"): + seen <- "sanitization" + <-release + b, _ := json.Marshal(sanitizationRich()) + _ = json.Unmarshal(b, dest) + default: + b, _ := json.Marshal(exploitRich()) + _ = json.Unmarshal(b, dest) + } + return &appx.HarnessResult{Parsed: dest}, nil + }, + AIFn: appx.AIJSON(func(string) (json.RawMessage, error) { + return json.Marshal(map[string]any{"verdict": "likely", "evidence_level": 2, "rationale": "r", "confidence": "low"}) + }), + } + + done := make(chan error, 1) + go func() { + _, err := RunVerifier(context.Background(), app, fixtureRepo, findingRich(), "quick") + done <- err + }() + + // Both stage-1 agents must arrive before either is allowed to finish. + first, second := <-seen, <-seen + if first == second { + t.Fatalf("expected tracer and sanitization concurrently, saw %q twice", first) + } + close(release) + if err := <-done; err != nil { + t.Fatalf("RunVerifier: %v", err) + } +} + +func TestRunVerifierGolden(t *testing.T) { + var want map[string]any + goldenJSON(t, "run_verifier", &want) + + notExploitable, err := RunVerifier(context.Background(), cannedApp("not_exploitable", 1), fixtureRepo, findingRich(), "quick") + if err != nil { + t.Fatalf("RunVerifier(not_exploitable): %v", err) + } + likely, err := RunVerifier(context.Background(), cannedApp("likely", 3), fixtureRepo, findingBare(), "quick") + if err != nil { + t.Fatalf("RunVerifier(likely): %v", err) + } + for name, got := range map[string]schemas.VerifiedFinding{ + "not_exploitable": notExploitable, + "likely": likely, + } { + if !reflect.DeepEqual(jsonTree(t, got), want[name]) { + t.Errorf("run_verifier[%s] mismatch:\n got: %#v\nwant: %#v", name, jsonTree(t, got), want[name]) + } + } +} + +// TestRunVerifierFallsBackOnTracerFailure pins the return_exceptions=True +// contract: a failed tracer falls back to the SEED trace built from the +// hunter's data_flow, and a failed sanitization falls back to the explicit +// "nothing found" result — neither aborts the verification. +func TestRunVerifierFallsBackOnStageOneFailures(t *testing.T) { + var exploitPrompt string + app := &appx.Fake{ + HarnessFn: appx.HarnessJSON(func(prompt string, _ appx.HarnessOptions) (json.RawMessage, error) { + switch { + case strings.Contains(prompt, "You are DataFlowTracer"), + strings.Contains(prompt, "You are SanitizationAnalyzer"): + return nil, errors.New("stage one down") + } + exploitPrompt = prompt + return json.Marshal(exploitRich()) + }), + AIFn: appx.AIJSON(func(string) (json.RawMessage, error) { + return json.Marshal(map[string]any{"verdict": "likely", "evidence_level": 2, "rationale": "r", "confidence": "low"}) + }), + } + got, err := RunVerifier(context.Background(), app, fixtureRepo, findingRich(), "quick") + if err != nil { + t.Fatalf("stage-one failures must not abort verification: %v", err) + } + // The seed trace's source is ":" and its steps come from + // the hunter's data_flow. + if got.Proof == nil || got.Proof.DataFlowEvidence == nil { + t.Fatal("proof/data flow evidence missing") + } + if src := got.Proof.DataFlowEvidence.Source; src == nil || *src != "src/users.py:42" { + t.Errorf("seed trace source = %v, want src/users.py:42", src) + } + if got.Proof.DataFlowEvidence.SinkReached { + t.Error("the seed trace must never claim the sink was reached") + } + if len(got.Proof.DataFlowTrace) != 2 { + t.Errorf("seed trace should carry the hunter's 2 data-flow steps, got %d", len(got.Proof.DataFlowTrace)) + } + if !strings.Contains(exploitPrompt, "Sanitization found: no") { + t.Error("a failed sanitization analysis must reach the exploit prompt as 'no'") + } + if !strings.Contains(exploitPrompt, "Sanitization sufficient: unknown") { + t.Error("the sanitization fallback leaves `sufficient` None, which renders as 'unknown'") + } +} + +// TestSeedTraceSinkFallback pins `finding.function_name or finding.file_path`. +func TestSeedTraceSinkFallback(t *testing.T) { + f := findingBare() + if got := seedTraceFor(f).Sink; got != "src/hash.py" { + t.Errorf("a nil function_name must fall back to the file path, got %q", got) + } + f.FunctionName = str("") + if got := seedTraceFor(f).Sink; got != "src/hash.py" { + t.Errorf("an EMPTY function_name is falsy and must fall back too, got %q", got) + } + f.FunctionName = str("do_hash") + if got := seedTraceFor(f).Sink; got != "do_hash" { + t.Errorf("sink = %q, want do_hash", got) + } + // No data flow means an empty (non-nil) step list. + if steps := seedTraceFor(findingBare()).Steps; steps == nil || len(steps) != 0 { + t.Errorf("want empty non-nil steps, got %#v", steps) + } +} + +// TestVerdictAgentUsesAINotHarness pins that the verdict stage goes through +// `.ai()` — one structured request, no harness session and no temp dir. +func TestVerdictAgentUsesAINotHarness(t *testing.T) { + app := &appx.Fake{ + AIFn: appx.AIJSON(func(string) (json.RawMessage, error) { + return json.Marshal(map[string]any{"verdict": "confirmed", "evidence_level": 6, "rationale": "r", "confidence": "high"}) + }), + } + got, err := RunVerdictAgent(context.Background(), app, ".", findingRich(), traceRich(), sanitizationRich(), exploitRich()) + if err != nil { + t.Fatalf("RunVerdictAgent: %v", err) + } + if got.Verdict != "confirmed" || got.EvidenceLevel != 6 { + t.Errorf("unexpected decision %+v", got) + } + if len(app.Harnesses) != 0 { + t.Error("the verdict stage must not open a harness session") + } + if len(app.AIs) != 1 { + t.Fatalf("want exactly one .ai() call, got %d", len(app.AIs)) + } + // Python passes no `system=`, so the request carries no system message — + // ai.WithSystem is the only option that prepends one. + req := &ai.Request{} + for _, opt := range app.AIs[0].Opts { + _ = opt(req) + } + for _, msg := range req.Messages { + if msg.Role == "system" { + t.Errorf("verdict.py passes no system prompt, got %+v", msg) + } + } + if req.ResponseFormat == nil { + t.Fatal("the request must carry the strictified VerdictDecision schema") + } + if req.ResponseFormat.JSONSchema == nil || !req.ResponseFormat.JSONSchema.Strict { + t.Errorf("the schema must be sent in OpenAI strict mode, got %+v", req.ResponseFormat) + } + // It must be the committed PYDANTIC schema (title == the class name), then + // strictified: every object gets additionalProperties:false and a required + // list naming all of its properties. + var sent map[string]any + if err := json.Unmarshal(req.ResponseFormat.JSONSchema.Schema, &sent); err != nil { + t.Fatalf("decode sent schema: %v", err) + } + if title, _ := sent["title"].(string); title != "VerdictDecision" { + t.Errorf("schema title = %v, want VerdictDecision", sent["title"]) + } + if extra, ok := sent["additionalProperties"].(bool); !ok || extra { + t.Errorf("strict mode requires additionalProperties:false, got %v", sent["additionalProperties"]) + } + required, _ := sent["required"].([]any) + if len(required) != 4 { + t.Errorf("strict mode requires all 4 properties, got %v", required) + } +} + +// --- run_prove_streaming ---------------------------------------------------- + +func TestRunProveStreamingConsumesUntilSentinel(t *testing.T) { + app := cannedApp("confirmed", 4) + queue := make(chan []schemas.RawFinding, 4) + queue <- []schemas.RawFinding{findingBare()} + queue <- []schemas.RawFinding{} // an EMPTY batch is not the sentinel + queue <- []schemas.RawFinding{findingRich()} + queue <- nil // sentinel + close(queue) + + got := RunProveStreaming(context.Background(), app, fixtureRepo, queue, "standard", 3, 30) + if len(got) != 2 { + t.Fatalf("want 2 verified findings, got %d", len(got)) + } + // Sorted by (exploitability_score, evidence_level) descending: the rich + // finding is CWE-89 (critical floor) so it outscores the CWE-327 one. + if got[0].ID != "raw-1" || got[1].ID != "raw-2" { + t.Errorf("order = %s,%s, want raw-1,raw-2", got[0].ID, got[1].ID) + } + if got[0].ExploitabilityScore < got[1].ExploitabilityScore { + t.Error("results must be sorted by exploitability score, descending") + } +} + +func TestRunProveStreamingRespectsProverCap(t *testing.T) { + app := cannedApp("confirmed", 4) + queue := make(chan []schemas.RawFinding, 4) + batch := make([]schemas.RawFinding, 5) + for i := range batch { + f := findingBare() + f.ID = "f" + string(rune('a'+i)) + f.Fingerprint = f.ID + batch[i] = f + } + queue <- batch + queue <- batch // never scheduled: the cap is already reached + queue <- nil + close(queue) + + got := RunProveStreaming(context.Background(), app, fixtureRepo, queue, "quick", 2, 3) + if len(got) != 3 { + t.Fatalf("prover_cap=3 must stop after 3 verifications, got %d", len(got)) + } + // Each verification issues exactly three harness calls. + if len(app.Harnesses) != 9 { + t.Errorf("want 9 harness calls (3 findings x 3 sub-agents), got %d", len(app.Harnesses)) + } + if got := app.MaxConcurrentHarness(); got > 4 { + t.Errorf("max_concurrent_provers=2 caps in-flight harness calls at 4, got %d", got) + } +} + +// TestRunProveStreamingDrainsProducer pins that once the cap is hit the queue +// is drained to its sentinel rather than abandoned — otherwise a producer +// blocked on an unbuffered channel would never finish. +func TestRunProveStreamingDrainsProducer(t *testing.T) { + app := cannedApp("confirmed", 4) + queue := make(chan []schemas.RawFinding) // UNBUFFERED + producerDone := make(chan struct{}) + go func() { + defer close(producerDone) + for i := 0; i < 4; i++ { + queue <- []schemas.RawFinding{findingBare()} + } + queue <- nil + }() + + got := RunProveStreaming(context.Background(), app, fixtureRepo, queue, "quick", 2, 1) + if len(got) != 1 { + t.Fatalf("prover_cap=1 must verify exactly one finding, got %d", len(got)) + } + select { + case <-producerDone: + default: + t.Fatal("the producer must have been drained to its sentinel") + } +} + +func TestRunProveStreamingEmptyQueue(t *testing.T) { + app := &appx.Fake{} + queue := make(chan []schemas.RawFinding, 1) + queue <- nil + close(queue) + + got := RunProveStreaming(context.Background(), app, fixtureRepo, queue, "quick", 3, 30) + if got == nil || len(got) != 0 { + t.Errorf("want an empty non-nil slice, got %#v", got) + } + if len(app.Harnesses) != 0 { + t.Error("an immediate sentinel must schedule no work") + } +} + +// TestRunProveStreamingDemotesFailures pins that a failing verification is +// demoted and KEPT, not dropped — the `except BaseException` inside +// `_verify_one` means gather's exception filter never actually fires. +func TestRunProveStreamingDemotesFailures(t *testing.T) { + app := &appx.Fake{ + HarnessFn: appx.HarnessJSON(func(prompt string, _ appx.HarnessOptions) (json.RawMessage, error) { + if strings.Contains(prompt, "You are ExploitHypothesizer") { + return nil, errors.New("nope") + } + if strings.Contains(prompt, "You are DataFlowTracer") { + return json.Marshal(traceRich()) + } + return json.Marshal(sanitizationRich()) + }), + } + queue := make(chan []schemas.RawFinding, 2) + queue <- []schemas.RawFinding{findingBare()} + queue <- nil + close(queue) + + got := RunProveStreaming(context.Background(), app, fixtureRepo, queue, "quick", 3, 30) + if len(got) != 1 { + t.Fatalf("a failed verification must still be reported, got %d findings", len(got)) + } + if got[0].DropReason == nil || *got[0].DropReason != "verifier_error" { + t.Errorf("drop_reason = %v, want verifier_error", got[0].DropReason) + } +} + +// TestRunProveSemaphoreAdmitsExactlyTheLimit is the strong form of the +// concurrency contract: `asyncio.Semaphore(max(1, min(n, len(findings))))` must +// admit EXACTLY that many verifications at once — no more (an over-wide +// semaphore would stampede the provider) and no fewer (a serialized phase would +// take N times as long). +// +// Every tracer call parks until the test releases it, so the number that have +// arrived after a grace period IS the semaphore width. A too-narrow semaphore +// never fills the wave and the test fails on its deadline. +func TestRunProveSemaphoreAdmitsExactlyTheLimit(t *testing.T) { + const findingsN, limit = 6, 3 + + var ( + mu sync.Mutex + arrived int + inFlight int + peak int + ) + release := make(chan struct{}) + timedOut := make(chan struct{}) + + app := &appx.Fake{ + HarnessFn: func(_ context.Context, prompt string, _ map[string]any, dest any, _ appx.HarnessOptions) (*appx.HarnessResult, error) { + var canned any = exploitRich() + switch { + case strings.Contains(prompt, "You are DataFlowTracer"): + mu.Lock() + arrived++ + inFlight++ + if inFlight > peak { + peak = inFlight + } + mu.Unlock() + select { + case <-release: + case <-timedOut: + } + mu.Lock() + inFlight-- + mu.Unlock() + canned = traceRich() + case strings.Contains(prompt, "You are SanitizationAnalyzer"): + canned = sanitizationRich() + } + b, _ := json.Marshal(canned) + if err := json.Unmarshal(b, dest); err != nil { + return nil, err + } + return &appx.HarnessResult{Parsed: dest}, nil + }, + AIFn: appx.AIJSON(func(string) (json.RawMessage, error) { + return json.Marshal(map[string]any{"verdict": "likely", "evidence_level": 2, "rationale": "r", "confidence": "low"}) + }), + } + + findings := make([]schemas.RawFinding, findingsN) + for i := range findings { + f := findingBare() + f.ID = "f" + strconv.Itoa(i) + f.Fingerprint = f.ID + findings[i] = f + } + + done := make(chan error, 1) + go func() { + _, err := RunProve(context.Background(), app, fixtureRepo, + schemas.HuntResult{Findings: findings}, "quick", limit) + done <- err + }() + + // Wait for the first wave to fill, then give a too-wide semaphore a chance + // to let a fourth verification through. + deadline := time.Now().Add(10 * time.Second) + for { + mu.Lock() + n := arrived + mu.Unlock() + if n >= limit { + break + } + if time.Now().After(deadline) { + close(timedOut) + <-done + t.Fatalf("only %d of %d verifications were in flight: the semaphore is too narrow", n, limit) + } + time.Sleep(time.Millisecond) + } + time.Sleep(150 * time.Millisecond) + mu.Lock() + over := arrived + mu.Unlock() + close(release) + + if err := <-done; err != nil { + t.Fatalf("RunProve: %v", err) + } + if over != limit { + t.Errorf("%d verifications entered the first wave, want exactly %d", over, limit) + } + if peak != limit { + t.Errorf("peak concurrent verifications = %d, want exactly %d", peak, limit) + } +} diff --git a/internal/secaf/agents/prove/sandbox.go b/internal/secaf/agents/prove/sandbox.go new file mode 100644 index 0000000000..43cc197b27 --- /dev/null +++ b/internal/secaf/agents/prove/sandbox.go @@ -0,0 +1,174 @@ +package prove + +// Ports src/sec_af/agents/prove/sandbox.py: +// +// """Sandbox execution helper for DAST-like verification. +// +// Provides a safe execution context for running exploit payloads +// against target applications. Currently uses subprocess isolation +// with strict timeouts and resource limits. Future: Docker containers. +// """ +// +// Nothing in the Python tree calls run_sandboxed — it is ported for 1:1 +// completeness so the Go node has the same surface when DAST verification grows +// a real execution path. + +import ( + "bytes" + "context" + "errors" + "os" + "os/exec" + "syscall" + "time" +) + +// DefaultSandboxTimeout is sandbox.py `_DEFAULT_TIMEOUT`, in seconds. +const DefaultSandboxTimeout = 10 + +// MaxSandboxOutputBytes is sandbox.py `_MAX_OUTPUT_BYTES`. Python slices the +// raw BYTES before decoding (`stdout_bytes[:8192].decode("utf-8", errors="replace")`), +// so a multi-byte character straddling the cut is replaced with U+FFFD rather +// than dropped — which Go's []byte -> string conversion of an invalid tail +// reproduces on read. +const MaxSandboxOutputBytes = 8192 + +// SandboxResult ports sandbox.py's frozen dataclass of the same name. +type SandboxResult struct { + Stdout string + Stderr string + ExitCode int + TimedOut bool +} + +// RunSandboxed ports sandbox.py run_sandboxed. +// +// async def run_sandboxed(command, *, timeout=_DEFAULT_TIMEOUT, cwd=None) -> SandboxResult: +// try: +// proc = await asyncio.create_subprocess_exec(*command, stdout=PIPE, stderr=PIPE, cwd=cwd) +// out, err = await asyncio.wait_for(proc.communicate(), timeout=timeout) +// return SandboxResult(out[:8192].decode(...), err[:8192].decode(...), proc.returncode or 0, False) +// except asyncio.TimeoutError: +// proc.kill() +// return SandboxResult("", "Execution timed out", -1, True) +// except Exception as exc: +// return SandboxResult("", str(exc), -1, False) +// +// Signature: Go has no keyword arguments, so the two keyword-only parameters +// become positional. `timeout` is in SECONDS (pass DefaultSandboxTimeout for +// Python's default) and `cwd == ""` is Python's `cwd=None` (inherit the +// parent's working directory). +// +// emptyCommandStderr is `str(TypeError)` for +// `asyncio.create_subprocess_exec(*[])` on CPython 3.11 — the message the +// blanket handler records when `command` is empty. +const emptyCommandStderr = "create_subprocess_exec() missing 1 required positional argument: 'program'" + +// Python parity notes: +// +// - NEVER returns an error. Every failure — a missing binary, an empty +// command, a permission denial — lands in Stderr with ExitCode -1, which is +// what Python's blanket `except Exception` does. +// - `proc.returncode or 0` keeps a nonzero code and maps 0/None to 0. For a +// child killed by a SIGNAL, `returncode` is the NEGATIVE signal number +// (VERIFIED on the pinned interpreter: `sh -c "kill -9 $$"` -> -9, +// `kill -15 $$` -> -15). Go's `ProcessState.ExitCode()` discards that and +// answers -1 for ANY signalled child, so the signal is read back off the +// wait status below. Reading it matters beyond the port's own deadline — +// which the timeout branch already owns — because the OOM killer, a +// container runtime stopping the pod, a supervisor's SIGTERM or a payload +// killing itself all land here, and -1 is also the sentinel the exception +// branch uses. +// - ctx is honoured on top of the timeout: a cancelled ctx kills the child +// just as the timeout does, but reports timed_out=false because Python has +// no equivalent of a caller-cancelled await here (its CancelledError would +// propagate, and BaseException is not caught by `except Exception`). +// Documented divergence; nothing in the port passes a cancellable ctx. +func RunSandboxed(ctx context.Context, command []string, timeout int, cwd string) SandboxResult { + if len(command) == 0 { + // Python parity: `asyncio.create_subprocess_exec(*[])` raises a + // TypeError before any process starts, and the blanket + // `except Exception as exc` stores `str(exc)`. VERIFIED on the pinned + // interpreter: + // + // run_sandboxed([]) -> SandboxResult(stdout='', exit_code=-1, timed_out=False, + // stderr="create_subprocess_exec() missing 1 required positional argument: 'program'") + return SandboxResult{Stderr: emptyCommandStderr, ExitCode: -1} + } + + runCtx, cancel := context.WithTimeout(ctx, time.Duration(timeout)*time.Second) + defer cancel() + + cmd := exec.CommandContext(runCtx, command[0], command[1:]...) + cmd.Dir = cwd + // WaitDelay bounds how long Run keeps waiting for the output pipes AFTER + // the deadline killed the child. Without it a grandchild that inherited the + // pipe (`sh -c "sleep 60"`) would hold Run open for the full 60s, while + // Python's `asyncio.wait_for(proc.communicate())` returns the moment the + // timeout fires. The delay restores that timing. + cmd.WaitDelay = 200 * time.Millisecond + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + if errors.Is(runCtx.Err(), context.DeadlineExceeded) { + // CommandContext already killed the child, which is Python's + // `proc.kill()` inside the TimeoutError handler. + return SandboxResult{Stderr: "Execution timed out", ExitCode: -1, TimedOut: true} + } + if err != nil { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + // Start failure (binary missing, cwd unreadable, ctx cancelled): + // Python's generic `except Exception` branch. + return SandboxResult{Stderr: err.Error(), ExitCode: -1} + } + // A nonzero exit is NOT an exception in Python — communicate() returns + // normally and returncode carries the value. + return SandboxResult{ + Stdout: truncateBytes(stdout.Bytes(), MaxSandboxOutputBytes), + Stderr: truncateBytes(stderr.Bytes(), MaxSandboxOutputBytes), + ExitCode: pyReturnCode(exitErr.ProcessState), + } + } + return SandboxResult{ + Stdout: truncateBytes(stdout.Bytes(), MaxSandboxOutputBytes), + Stderr: truncateBytes(stderr.Bytes(), MaxSandboxOutputBytes), + ExitCode: 0, + } +} + +// signaledStatus is the part of syscall.WaitStatus this file needs. Asserting +// against an interface rather than the concrete type keeps the file free of +// build tags: on a platform whose wait status carries no signal (Windows) the +// assertion simply fails and ExitCode() stands. +type signaledStatus interface { + Signaled() bool + Signal() syscall.Signal +} + +// pyReturnCode is `proc.returncode`: the exit status for a normal exit, and the +// NEGATIVE signal number for a child killed by a signal (-9 for SIGKILL, -15 +// for SIGTERM), which is what asyncio's Process reports and what +// `exit_code=proc.returncode or 0` therefore carries through. +func pyReturnCode(state *os.ProcessState) int { + if ws, ok := state.Sys().(signaledStatus); ok && ws.Signaled() { + return -int(ws.Signal()) + } + return state.ExitCode() +} + +// truncateBytes is Python's `b[:n].decode("utf-8", errors="replace")`: the cut +// is by BYTE, and an invalid UTF-8 tail left by the cut decodes to U+FFFD. +// +// Known divergence: CPython's replace handler emits one U+FFFD per undecodable +// BYTE, while bytes.ToValidUTF8 emits one per invalid RUN. It only shows up on +// a cut that lands mid-character in already-truncated output, and the value is +// diagnostic text, so the simpler form is kept. +func truncateBytes(b []byte, n int) string { + if len(b) > n { + b = b[:n] + } + return string(bytes.ToValidUTF8(b, []byte("�"))) +} diff --git a/internal/secaf/agents/prove/sandbox_test.go b/internal/secaf/agents/prove/sandbox_test.go new file mode 100644 index 0000000000..21aa4e65c1 --- /dev/null +++ b/internal/secaf/agents/prove/sandbox_test.go @@ -0,0 +1,170 @@ +package prove + +// Tests for sandbox.go. +// +// Validation contract (sandbox.py run_sandboxed): +// - a successful command returns its stdout/stderr and exit code 0, and never +// an error; +// - a nonzero exit is NOT a failure: the code is reported and timed_out stays +// false; +// - a child killed by a SIGNAL reports `proc.returncode`, which is the +// NEGATIVE signal number (VERIFIED on the pinned interpreter: `kill -9 $$` +// -> exit_code -9, `kill -15 $$` -> -15, both with timed_out False); +// - a command that outlives the timeout is killed and reported as +// ("", "Execution timed out", -1, true); +// - a command that cannot start at all reports the error text with exit +// code -1 and timed_out false; +// - output is truncated at 8192 BYTES; +// - cwd is honoured. + +import ( + "context" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func skipWithoutShell(t *testing.T) { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("sandbox tests need a POSIX shell") + } +} + +func TestRunSandboxedSuccess(t *testing.T) { + skipWithoutShell(t) + got := RunSandboxed(context.Background(), []string{"sh", "-c", "printf out; printf err 1>&2"}, + DefaultSandboxTimeout, "") + if got.Stdout != "out" { + t.Errorf("stdout = %q, want %q", got.Stdout, "out") + } + if got.Stderr != "err" { + t.Errorf("stderr = %q, want %q", got.Stderr, "err") + } + if got.ExitCode != 0 || got.TimedOut { + t.Errorf("exit_code = %d, timed_out = %v, want 0/false", got.ExitCode, got.TimedOut) + } +} + +// TestRunSandboxedNonZeroExitIsNotAnError pins that Python's communicate() +// returns normally for a failing command — only the return code changes. +func TestRunSandboxedNonZeroExitIsNotAnError(t *testing.T) { + skipWithoutShell(t) + got := RunSandboxed(context.Background(), []string{"sh", "-c", "printf partial; exit 3"}, + DefaultSandboxTimeout, "") + if got.ExitCode != 3 { + t.Errorf("exit_code = %d, want 3", got.ExitCode) + } + if got.Stdout != "partial" { + t.Errorf("stdout must still be captured, got %q", got.Stdout) + } + if got.TimedOut { + t.Error("a nonzero exit is not a timeout") + } +} + +// TestRunSandboxedSignalledChildReportsTheNegativeSignal covers the case Go's +// ProcessState.ExitCode() flattens to -1 — which is also the sentinel the +// start-failure and timeout branches use, so a caller could not tell "killed by +// SIGKILL" from "never started". Python keeps them apart. +func TestRunSandboxedSignalledChildReportsTheNegativeSignal(t *testing.T) { + skipWithoutShell(t) + for _, tc := range []struct { + signal string + want int + }{ + {"9", -9}, + {"15", -15}, + {"6", -6}, + } { + got := RunSandboxed(context.Background(), + []string{"sh", "-c", "kill -" + tc.signal + " $$"}, DefaultSandboxTimeout, "") + if got.ExitCode != tc.want { + t.Errorf("kill -%s: exit_code = %d, want %d", tc.signal, got.ExitCode, tc.want) + } + if got.TimedOut { + t.Errorf("kill -%s: timed_out must stay false", tc.signal) + } + } +} + +func TestRunSandboxedTimeout(t *testing.T) { + skipWithoutShell(t) + got := RunSandboxed(context.Background(), []string{"sh", "-c", "sleep 5"}, 1, "") + if !got.TimedOut { + t.Error("want timed_out = true") + } + if got.Stderr != "Execution timed out" { + t.Errorf("stderr = %q, want %q", got.Stderr, "Execution timed out") + } + if got.ExitCode != -1 { + t.Errorf("exit_code = %d, want -1", got.ExitCode) + } + if got.Stdout != "" { + t.Errorf("a timed-out run reports no stdout, got %q", got.Stdout) + } +} + +func TestRunSandboxedStartFailure(t *testing.T) { + got := RunSandboxed(context.Background(), + []string{"definitely-not-a-real-binary-secaf"}, DefaultSandboxTimeout, "") + if got.ExitCode != -1 || got.TimedOut { + t.Errorf("exit_code = %d, timed_out = %v, want -1/false", got.ExitCode, got.TimedOut) + } + if got.Stderr == "" { + t.Error("a start failure must report the exception text in stderr") + } +} + +// TestRunSandboxedEmptyCommand pins the whole result, TEXT INCLUDED. Python's +// `asyncio.create_subprocess_exec(*[])` raises before any process starts and +// the blanket `except Exception as exc` stores `str(exc)`. VERIFIED on the +// pinned interpreter (PYTHONPATH=src ~/.agentfield/packages/sec-af/venv/bin/python): +// +// run_sandboxed([]) -> SandboxResult(stdout='', exit_code=-1, timed_out=False, +// stderr="create_subprocess_exec() missing 1 required positional argument: 'program'") +func TestRunSandboxedEmptyCommand(t *testing.T) { + got := RunSandboxed(context.Background(), nil, DefaultSandboxTimeout, "") + want := SandboxResult{ + Stdout: "", + Stderr: "create_subprocess_exec() missing 1 required positional argument: 'program'", + ExitCode: -1, + TimedOut: false, + } + if got != want { + t.Errorf("RunSandboxed(nil) = %+v, want %+v", got, want) + } +} + +func TestRunSandboxedTruncatesOutput(t *testing.T) { + skipWithoutShell(t) + // 20000 'a' bytes, well past the 8192-byte cap. + got := RunSandboxed(context.Background(), + []string{"sh", "-c", "printf 'a%.0s' $(seq 1 20000)"}, DefaultSandboxTimeout, "") + if got.ExitCode != 0 { + t.Fatalf("exit_code = %d, stderr=%q", got.ExitCode, got.Stderr) + } + if len(got.Stdout) != MaxSandboxOutputBytes { + t.Errorf("stdout length = %d, want %d", len(got.Stdout), MaxSandboxOutputBytes) + } + if strings.Trim(got.Stdout, "a") != "" { + t.Error("truncation must keep the leading bytes verbatim") + } +} + +func TestRunSandboxedHonoursCwd(t *testing.T) { + skipWithoutShell(t) + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "marker"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + got := RunSandboxed(context.Background(), []string{"sh", "-c", "ls"}, DefaultSandboxTimeout, dir) + if got.ExitCode != 0 { + t.Fatalf("exit_code = %d, stderr = %q", got.ExitCode, got.Stderr) + } + if !strings.Contains(got.Stdout, "marker") { + t.Errorf("the command must run inside cwd; got %q", got.Stdout) + } +} diff --git a/internal/secaf/agents/prove/sanitization.go b/internal/secaf/agents/prove/sanitization.go new file mode 100644 index 0000000000..dc5341b509 --- /dev/null +++ b/internal/secaf/agents/prove/sanitization.go @@ -0,0 +1,83 @@ +package prove + +// Ports src/sec_af/agents/prove/sanitization.py. + +import ( + "context" + "os" + "strconv" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/harnessx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// sanitizationPromptPath mirrors sanitization.py's module-level PROMPT_PATH. +const sanitizationPromptPath = "prove/sanitization.txt" + +const ( + sanitizationAgentName = "prove-sanitization" + sanitizationExtractName = "SanitizationAnalyzer" +) + +// sanitizationBuildPrompt ports sanitization.py `_build_prompt`. +// +// Note the two differences from tracer.py's builder: {{TRACE_CONTEXT}} replaces +// {{DATA_FLOW_JSON}}, and it sits one slot earlier — before {{DEPTH}}, so a +// literal "{{DEPTH}}" inside the trace context WOULD be substituted. Order is +// preserved exactly. +func sanitizationBuildPrompt(template string, finding schemas.RawFinding, trace schemas.DataFlowTrace, depth string) string { + return applyReplacements(template, []replacement{ + {"{{TITLE}}", finding.Title}, + {"{{DESCRIPTION}}", finding.Description}, + {"{{CWE_ID}}", finding.CweID}, + {"{{CWE_NAME}}", finding.CweName}, + {"{{FILE_PATH}}", finding.FilePath}, + {"{{START_LINE}}", strconv.Itoa(finding.StartLine)}, + {"{{CODE_SNIPPET}}", finding.CodeSnippet}, + {"{{FINDING_TYPE}}", string(finding.FindingType)}, + {"{{RELATED_FILES}}", relatedFilesJSON(finding.RelatedFiles)}, + {"{{TRACE_CONTEXT}}", traceContext(trace)}, + {"{{DEPTH}}", depth}, + }) +} + +// SanitizationPrompt builds the exact prompt RunSanitizationAnalyzer sends. +// Exported for the golden test. +func SanitizationPrompt(finding schemas.RawFinding, trace schemas.DataFlowTrace, repoPath, depth string) string { + return sanitizationBuildPrompt(prompts.MustLoad(sanitizationPromptPath), finding, trace, depth) + + "\n\nCONTEXT:\n" + + "- Repository path: " + repoPath + "\n" + + "- Use the repository path above for file inspection during sanitization analysis." +} + +// RunSanitizationAnalyzer ports sanitization.py run_sanitization_analyzer. +// +// result = await app.harness(prompt=prompt, schema=SanitizationResult, +// cwd=harness_cwd, project_dir=repo_path) +// return extract_harness_result(result, SanitizationResult, "SanitizationAnalyzer") +// +// Same temp-dir contract as RunTracer. +func RunSanitizationAnalyzer( + ctx context.Context, + app appx.Harnesser, + repoPath string, + finding schemas.RawFinding, + dataFlowTrace schemas.DataFlowTrace, + depth string, +) (schemas.SanitizationResult, error) { + prompt := SanitizationPrompt(finding, dataFlowTrace, repoPath, depth) + + harnessCwd, err := os.MkdirTemp("", "secaf-"+sanitizationAgentName+"-") + if err != nil { + return schemas.SanitizationResult{}, err + } + defer os.RemoveAll(harnessCwd) + + return harnessx.RunExtract[schemas.SanitizationResult]( + ctx, app, prompt, + appx.HarnessOptions{Cwd: harnessCwd, ProjectDir: repoPath}, + sanitizationExtractName, + ) +} diff --git a/internal/secaf/agents/prove/shared.go b/internal/secaf/agents/prove/shared.go new file mode 100644 index 0000000000..57b3d63bef --- /dev/null +++ b/internal/secaf/agents/prove/shared.go @@ -0,0 +1,132 @@ +package prove + +// Helpers shared by more than one prove module, plus the Python idioms the +// prompt builders lean on. Each one names the Python source it reproduces. + +import ( + "strings" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// HarnessAIer is the capability set verifier.run_verifier needs: the four +// sub-agents split between `.harness(...)` (tracer, sanitization, exploit) and +// `.ai(...)` (verdict). Python declares exactly this union as its +// `HarnessCapable` Protocol in verifier.py. +type HarnessAIer interface { + appx.Harnesser + appx.AIer +} + +// replacement is one entry of Python's `replacements` dict inside a +// `_build_prompt`. ORDER IS LOAD-BEARING: Python iterates the dict in insertion +// order and applies `prompt.replace(needle, value)` cumulatively, so a marker +// that appears INSIDE an already-substituted value is itself substituted by a +// later entry, while a marker inside a value substituted later survives. The Go +// port keeps a slice for exactly that reason — a map would scramble it. +type replacement struct { + needle string + value string +} + +// applyReplacements is the shared tail of every `_build_prompt`: +// +// prompt = template +// for needle, value in replacements.items(): +// prompt = prompt.replace(needle, value) +// return prompt +// +// strings.ReplaceAll matches Python's str.replace, which replaces EVERY +// occurrence (dep_reachability.txt names {{PACKAGE}} and +// {{VULNERABLE_FUNCTION}} twice each). +func applyReplacements(template string, reps []replacement) string { + prompt := template + for _, r := range reps { + prompt = strings.ReplaceAll(prompt, r.needle, r.value) + } + return prompt +} + +// traceContext ports the `_trace_context` helper that sanitization.py and +// exploit.py declare byte-identically: +// +// def _trace_context(trace: DataFlowTrace) -> str: +// steps = "\n".join(f"- {step}" for step in trace.steps) if trace.steps else "- (no concrete trace steps)" +// sink_reached = "yes" if trace.sink_reached else "no" +// return f"Source: {trace.source}\nSink: {trace.sink}\nSink reached: {sink_reached}\nTrace steps:\n{steps}" +// +// verdict.py builds a DIFFERENT block (`_build_context`) and does not share +// this one; see verdict.go. +func traceContext(trace schemas.DataFlowTrace) string { + steps := "- (no concrete trace steps)" + if len(trace.Steps) > 0 { + parts := make([]string, len(trace.Steps)) + for i, step := range trace.Steps { + parts[i] = "- " + step + } + steps = strings.Join(parts, "\n") + } + sinkReached := "no" + if trace.SinkReached { + sinkReached = "yes" + } + return "Source: " + trace.Source + "\nSink: " + trace.Sink + + "\nSink reached: " + sinkReached + "\nTrace steps:\n" + steps +} + +// relatedFilesJSON is `json.dumps(finding.related_files, indent=2)`. +// +// Python parity: `related_files` is `list[str] = Field(default_factory=list)`, +// so pydantic can never make it None — a NIL Go slice therefore stands for +// Python's `[]` and must render as `[]`, not `null` (which is what +// pyfmt.Dumps does for a nil slice by design). +func relatedFilesJSON(files []string) string { + if files == nil { + files = []string{} + } + return pyfmt.Dumps(files, 2) +} + +// pyOr ports Python's `value or fallback` for an optional string: both None +// and the empty string are falsy, so both take the fallback. +func pyOr(value *string, fallback string) string { + if value == nil || *value == "" { + return fallback + } + return *value +} + +// pyStrOptBool renders `f"{x}"` for a `bool | None`: True, False or None. +// pyfmt.Str cannot be handed the *bool directly — a typed nil inside an +// interface is not the untyped nil its type switch tests for. +func pyStrOptBool(v *bool) string { + if v == nil { + return "None" + } + return pyfmt.Str(*v) +} + +// sarifRuleID ports the `_sarif_rule_id` helper that verifier.py, assembler.py +// and __init__.py._apply_metadata each declare identically: +// +// cwe_slug = finding.cwe_name.lower().replace(" ", "-").replace("/", "-") +// return f"sec-af/{finding.finding_type.value}/{cwe_slug}" +// +// Python parity: `str.lower()` is Unicode-aware full case folding for the +// simple cases; strings.ToLower matches for every character SEC-AF sees. Only +// SPACE and SLASH are replaced — an underscore, tab or newline in a CWE name +// survives verbatim, exactly as in Python. +func sarifRuleID(findingType schemas.FindingType, cweName string) string { + slug := strings.ReplaceAll(strings.ReplaceAll(strings.ToLower(cweName), " ", "-"), "/", "-") + return "sec-af/" + string(findingType) + "/" + slug +} + +// StrPtr returns a pointer to s. +// +// Several ported functions take `*string` parameters because their Python +// counterparts declare keyword-only arguments defaulting to None (Fallback's +// dropReason / originalVerdict). Go has no literal address-of for a constant, +// so callers need this one-liner; it lives here rather than in each caller. +func StrPtr(s string) *string { return &s } diff --git a/internal/secaf/agents/prove/testdata/golden/apply_metadata.json b/internal/secaf/agents/prove/testdata/golden/apply_metadata.json new file mode 100644 index 0000000000..d9d18d17f7 --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/apply_metadata.json @@ -0,0 +1,196 @@ +{ + "keeps_rule_id": { + "id": "m1", + "fingerprint": "fp-m1", + "title": "finding m1", + "description": "d", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "SQL Injection", + "owasp_category": null, + "tags": [], + "verdict": "confirmed", + "evidence_level": 3, + "rationale": "r", + "severity": "critical", + "cvss_v4": null, + "epss": null, + "exploitability_score": 5.0, + "proof": null, + "location": { + "file_path": "src/users.py", + "start_line": 1, + "end_line": 2, + "start_column": null, + "end_column": null, + "function_name": null, + "code_snippet": null + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [ + { + "framework": "PCI-DSS", + "control_id": "Req 6.2.4", + "control_name": "Custom software addresses common coding vulnerabilities" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A03:2021", + "control_name": "Injection" + }, + { + "framework": "HIPAA", + "control_id": "\u00a7164.312(a)(1)", + "control_name": "Access control" + }, + { + "framework": "ISO27001", + "control_id": "A.8.28", + "control_name": "Secure coding" + } + ], + "reproduction_steps": [], + "remediation": null, + "sarif_rule_id": "sec-af/sast/sql-injection", + "sarif_security_severity": 5.0, + "drop_reason": null + }, + "mints_rule_id": { + "id": "m2", + "fingerprint": "fp-m2", + "title": "finding m2", + "description": "d", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "Broken Access/Control Check", + "owasp_category": null, + "tags": [], + "verdict": "confirmed", + "evidence_level": 6, + "rationale": "r", + "severity": "critical", + "cvss_v4": null, + "epss": null, + "exploitability_score": 10.0, + "proof": null, + "location": { + "file_path": "src/users.py", + "start_line": 1, + "end_line": 2, + "start_column": null, + "end_column": null, + "function_name": null, + "code_snippet": null + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [ + { + "framework": "PCI-DSS", + "control_id": "Req 6.2.4", + "control_name": "Custom software addresses common coding vulnerabilities" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A03:2021", + "control_name": "Injection" + }, + { + "framework": "HIPAA", + "control_id": "\u00a7164.312(a)(1)", + "control_name": "Access control" + }, + { + "framework": "ISO27001", + "control_id": "A.8.28", + "control_name": "Secure coding" + } + ], + "reproduction_steps": [], + "remediation": null, + "sarif_rule_id": "sec-af/sast/broken-access-control-check", + "sarif_security_severity": 10.0, + "drop_reason": null + }, + "with_reachability_tag": { + "id": "m3", + "fingerprint": "fp-m3", + "title": "finding m3", + "description": "d", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "SQL Injection", + "owasp_category": null, + "tags": [ + "externally_reachable" + ], + "verdict": "confirmed", + "evidence_level": 6, + "rationale": "r", + "severity": "critical", + "cvss_v4": null, + "epss": null, + "exploitability_score": 10.0, + "proof": null, + "location": { + "file_path": "src/users.py", + "start_line": 1, + "end_line": 2, + "start_column": null, + "end_column": null, + "function_name": null, + "code_snippet": null + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [ + { + "framework": "PCI-DSS", + "control_id": "Req 6.2.4", + "control_name": "Custom software addresses common coding vulnerabilities" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A03:2021", + "control_name": "Injection" + }, + { + "framework": "HIPAA", + "control_id": "\u00a7164.312(a)(1)", + "control_name": "Access control" + }, + { + "framework": "ISO27001", + "control_id": "A.8.28", + "control_name": "Secure coding" + } + ], + "reproduction_steps": [], + "remediation": null, + "sarif_rule_id": "sec-af/sast/sql-injection", + "sarif_security_severity": 10.0, + "drop_reason": null + } +} diff --git a/internal/secaf/agents/prove/testdata/golden/apply_validated_chain.json b/internal/secaf/agents/prove/testdata/golden/apply_validated_chain.json new file mode 100644 index 0000000000..1a71ac8bbf --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/apply_validated_chain.json @@ -0,0 +1,159 @@ +{ + "not_validated": { + "chain": { + "chain_id": "c1", + "title": "t", + "validated": false, + "rationale": "r", + "steps": [ + { + "step_number": 1, + "finding_id": "v1", + "description": "d", + "enables": "e" + } + ] + }, + "error": null, + "findings": [ + { + "id": "v1", + "chain_id": null, + "chain_step": null, + "enables": null, + "tags": [] + }, + { + "id": "v2", + "chain_id": null, + "chain_step": null, + "enables": null, + "tags": [] + } + ] + }, + "no_steps": { + "chain": { + "chain_id": "c1", + "title": "t", + "validated": true, + "rationale": "r", + "steps": [] + }, + "error": null, + "findings": [ + { + "id": "v1", + "chain_id": null, + "chain_step": null, + "enables": null, + "tags": [] + } + ] + }, + "no_matching_finding": { + "chain": { + "chain_id": "c1", + "title": "t", + "validated": true, + "rationale": "r", + "steps": [ + { + "step_number": 2, + "finding_id": "zz", + "description": "d", + "enables": "e" + }, + { + "step_number": 1, + "finding_id": "yy", + "description": "d", + "enables": "e" + } + ] + }, + "error": null, + "findings": [ + { + "id": "v1", + "chain_id": null, + "chain_step": null, + "enables": null, + "tags": [] + } + ] + }, + "matching_finding_raises": { + "chain": { + "chain_id": "c1", + "title": "t", + "validated": true, + "rationale": "r", + "steps": [ + { + "step_number": 2, + "finding_id": "v2", + "description": "d", + "enables": "e" + }, + { + "step_number": 1, + "finding_id": "v1", + "description": "d", + "enables": "e" + } + ] + }, + "error": "'list' object has no attribute 'add'", + "findings": [ + { + "id": "v1", + "chain_id": "c1", + "chain_step": 1, + "enables": [ + "v2" + ], + "tags": [] + }, + { + "id": "v2", + "chain_id": null, + "chain_step": null, + "enables": null, + "tags": [] + } + ] + }, + "second_step_matches": { + "chain": { + "chain_id": "c9", + "title": "t", + "validated": true, + "rationale": "r", + "steps": [ + { + "step_number": 1, + "finding_id": "missing", + "description": "d", + "enables": "e" + }, + { + "step_number": 2, + "finding_id": "v1", + "description": "d", + "enables": "e" + } + ] + }, + "error": "'list' object has no attribute 'add'", + "findings": [ + { + "id": "v1", + "chain_id": "c9", + "chain_step": 2, + "enables": null, + "tags": [] + } + ] + } +} diff --git a/internal/secaf/agents/prove/testdata/golden/assemble.json b/internal/secaf/agents/prove/testdata/golden/assemble.json new file mode 100644 index 0000000000..315985f0dc --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/assemble.json @@ -0,0 +1,353 @@ +{ + "confirmed": { + "id": "raw-1", + "fingerprint": "fp-1", + "title": "Potential SQL injection in user lookup", + "description": "Request parameter reaches a formatted SQL string. Marker: {{TITLE}}", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "Improper Neutralization/Escaping of Special Elements", + "owasp_category": "A03:2021 - Injection", + "tags": [], + "verdict": "confirmed", + "evidence_level": 5, + "rationale": "rationale for confirmed", + "severity": "high", + "cvss_v4": null, + "epss": null, + "exploitability_score": 0.0, + "proof": { + "exploit_hypothesis": "Attacker supplies id=1 OR 1=1 to dump the table", + "verification_method": "composite_subagent_chain:sast", + "evidence_level": 5, + "data_flow_trace": [ + { + "file": "trace_step_1", + "line": 1, + "description": "src/routes.py:10 read request.args", + "tainted": true + }, + { + "file": "trace_step_2", + "line": 2, + "description": "src/users.py:42 execute", + "tainted": true + } + ], + "data_flow_evidence": { + "steps": [ + { + "file": "trace_step_1", + "line": 1, + "description": "src/routes.py:10 read request.args", + "tainted": true + }, + { + "file": "trace_step_2", + "line": 2, + "description": "src/users.py:42 execute", + "tainted": true + } + ], + "source": "request.args['id']", + "sink": "cursor.execute(query)", + "sink_reached": true + }, + "sanitization_analysis": { + "sanitization_found": true, + "sanitization_type": "parameterized query", + "sanitization_sufficient": false, + "bypass_possible": true, + "bypass_method": "second-order injection through the audit log" + }, + "vulnerable_code": null, + "exploit_payload": "1 OR 1=1", + "expected_outcome": "Full users table returned", + "poc_code": null, + "poc_execution_output": null, + "http_request": null, + "http_response": null, + "reachability": null, + "chain_steps": null + }, + "location": { + "file_path": "src/users.py", + "start_line": 42, + "end_line": 44, + "start_column": null, + "end_column": null, + "function_name": "get_user", + "code_snippet": "cursor.execute(\"SELECT * FROM users WHERE id = \" + user_id) # depth={{DEPTH}}" + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [], + "reproduction_steps": [ + { + "step": 1, + "description": "Trace attacker-controlled input from source to sink in target code path.", + "command": null, + "expected_output": "Input reaches a sensitive sink." + }, + { + "step": 2, + "description": "Attacker supplies id=1 OR 1=1 to dump the table", + "command": "1 OR 1=1", + "expected_output": "Full users table returned" + } + ], + "remediation": null, + "sarif_rule_id": "sec-af/sast/improper-neutralization-escaping-of-special-elements", + "sarif_security_severity": 0.0, + "drop_reason": null + }, + "not_exploitable": { + "id": "raw-1", + "fingerprint": "fp-1", + "title": "Potential SQL injection in user lookup", + "description": "Request parameter reaches a formatted SQL string. Marker: {{TITLE}}", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "Improper Neutralization/Escaping of Special Elements", + "owasp_category": "A03:2021 - Injection", + "tags": [], + "verdict": "not_exploitable", + "evidence_level": 1, + "rationale": "rationale for not_exploitable", + "severity": "high", + "cvss_v4": null, + "epss": null, + "exploitability_score": 0.0, + "proof": { + "exploit_hypothesis": "Attacker supplies id=1 OR 1=1 to dump the table", + "verification_method": "composite_subagent_chain:sast", + "evidence_level": 1, + "data_flow_trace": [ + { + "file": "trace_step_1", + "line": 1, + "description": "src/routes.py:10 read request.args", + "tainted": true + }, + { + "file": "trace_step_2", + "line": 2, + "description": "src/users.py:42 execute", + "tainted": true + } + ], + "data_flow_evidence": { + "steps": [ + { + "file": "trace_step_1", + "line": 1, + "description": "src/routes.py:10 read request.args", + "tainted": true + }, + { + "file": "trace_step_2", + "line": 2, + "description": "src/users.py:42 execute", + "tainted": true + } + ], + "source": "request.args['id']", + "sink": "cursor.execute(query)", + "sink_reached": true + }, + "sanitization_analysis": { + "sanitization_found": true, + "sanitization_type": "parameterized query", + "sanitization_sufficient": false, + "bypass_possible": true, + "bypass_method": "second-order injection through the audit log" + }, + "vulnerable_code": null, + "exploit_payload": "1 OR 1=1", + "expected_outcome": "Full users table returned", + "poc_code": null, + "poc_execution_output": null, + "http_request": null, + "http_response": null, + "reachability": null, + "chain_steps": null + }, + "location": { + "file_path": "src/users.py", + "start_line": 42, + "end_line": 44, + "start_column": null, + "end_column": null, + "function_name": "get_user", + "code_snippet": "cursor.execute(\"SELECT * FROM users WHERE id = \" + user_id) # depth={{DEPTH}}" + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [], + "reproduction_steps": [], + "remediation": null, + "sarif_rule_id": "sec-af/sast/improper-neutralization-escaping-of-special-elements", + "sarif_security_severity": 0.0, + "drop_reason": null + }, + "unknown_verdict": { + "id": "raw-2", + "fingerprint": "fp-2", + "title": "Weak hash", + "description": "MD5 used for password hashing", + "finding_type": "sast", + "cwe_id": "CWE-327", + "cwe_name": "Broken Crypto", + "owasp_category": null, + "tags": [], + "verdict": "inconclusive", + "evidence_level": 6, + "rationale": "rationale for unverified", + "severity": "medium", + "cvss_v4": null, + "epss": null, + "exploitability_score": 0.0, + "proof": { + "exploit_hypothesis": "unknown", + "verification_method": "composite_subagent_chain:sast", + "evidence_level": 6, + "data_flow_trace": [], + "data_flow_evidence": { + "steps": [], + "source": "unknown", + "sink": "unknown", + "sink_reached": false + }, + "sanitization_analysis": { + "sanitization_found": false, + "sanitization_type": null, + "sanitization_sufficient": null, + "bypass_possible": false, + "bypass_method": null + }, + "vulnerable_code": null, + "exploit_payload": null, + "expected_outcome": "unknown", + "poc_code": null, + "poc_execution_output": null, + "http_request": null, + "http_response": null, + "reachability": null, + "chain_steps": null + }, + "location": { + "file_path": "src/hash.py", + "start_line": 7, + "end_line": 7, + "start_column": null, + "end_column": null, + "function_name": null, + "code_snippet": "hashlib.md5(pw).hexdigest()" + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [], + "reproduction_steps": [ + { + "step": 1, + "description": "Trace attacker-controlled input from source to sink in target code path.", + "command": null, + "expected_output": "Input reaches a sensitive sink." + }, + { + "step": 2, + "description": "unknown", + "command": null, + "expected_output": "unknown" + } + ], + "remediation": null, + "sarif_rule_id": "sec-af/sast/broken-crypto", + "sarif_security_severity": 0.0, + "drop_reason": null + }, + "clamped_low": { + "id": "raw-2", + "fingerprint": "fp-2", + "title": "Weak hash", + "description": "MD5 used for password hashing", + "finding_type": "sast", + "cwe_id": "CWE-327", + "cwe_name": "Broken Crypto", + "owasp_category": null, + "tags": [], + "verdict": "likely", + "evidence_level": 1, + "rationale": "rationale for likely", + "severity": "medium", + "cvss_v4": null, + "epss": null, + "exploitability_score": 0.0, + "proof": { + "exploit_hypothesis": "unknown", + "verification_method": "composite_subagent_chain:sast", + "evidence_level": 1, + "data_flow_trace": [], + "data_flow_evidence": { + "steps": [], + "source": "unknown", + "sink": "unknown", + "sink_reached": false + }, + "sanitization_analysis": { + "sanitization_found": false, + "sanitization_type": null, + "sanitization_sufficient": null, + "bypass_possible": false, + "bypass_method": null + }, + "vulnerable_code": null, + "exploit_payload": null, + "expected_outcome": "unknown", + "poc_code": null, + "poc_execution_output": null, + "http_request": null, + "http_response": null, + "reachability": null, + "chain_steps": null + }, + "location": { + "file_path": "src/hash.py", + "start_line": 7, + "end_line": 7, + "start_column": null, + "end_column": null, + "function_name": null, + "code_snippet": "hashlib.md5(pw).hexdigest()" + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [], + "reproduction_steps": [ + { + "step": 1, + "description": "Trace attacker-controlled input from source to sink in target code path.", + "command": null, + "expected_output": "Input reaches a sensitive sink." + }, + { + "step": 2, + "description": "unknown", + "command": null, + "expected_output": "unknown" + } + ], + "remediation": null, + "sarif_rule_id": "sec-af/sast/broken-crypto", + "sarif_security_severity": 0.0, + "drop_reason": null + } +} diff --git a/internal/secaf/agents/prove/testdata/golden/chain_builder_prompt.txt b/internal/secaf/agents/prove/testdata/golden/chain_builder_prompt.txt new file mode 100644 index 0000000000..0362899d8a --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/chain_builder_prompt.txt @@ -0,0 +1,129 @@ +ROLE: +You are a security attack-chain verifier for SEC-AF PROVE phase. + +CONTEXT: +- Analysis depth: standard +- Seed chain candidates from HUNT (validate and expand these): [ + { + "chain_id": "chain-1", + "title": "SSRF to internal API", + "finding_ids": [ + "v1", + "v2" + ], + "combined_impact": "Internal service access", + "estimated_severity": "critical" + } +] +- Individually verified findings: { + "v1": { + "id": "v1", + "fingerprint": "fp-v1", + "title": "finding v1", + "description": "d", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "SQL Injection", + "owasp_category": null, + "tags": [], + "verdict": "confirmed", + "evidence_level": 4, + "rationale": "r", + "severity": "high", + "cvss_v4": null, + "epss": null, + "exploitability_score": 7.5, + "proof": null, + "location": { + "file_path": "src/users.py", + "start_line": 1, + "end_line": 2, + "start_column": null, + "end_column": null, + "function_name": null, + "code_snippet": null + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [], + "reproduction_steps": [], + "remediation": null, + "sarif_rule_id": "sec-af/sast/sql-injection", + "sarif_security_severity": 7.5, + "drop_reason": null + }, + "v2": { + "id": "v2", + "fingerprint": "fp-v2", + "title": "finding v2", + "description": "d", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "SQL Injection", + "owasp_category": null, + "tags": [], + "verdict": "confirmed", + "evidence_level": 2, + "rationale": "r", + "severity": "high", + "cvss_v4": null, + "epss": null, + "exploitability_score": 3.0, + "proof": null, + "location": { + "file_path": "src/users.py", + "start_line": 1, + "end_line": 2, + "start_column": null, + "end_column": null, + "function_name": null, + "code_snippet": null + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [], + "reproduction_steps": [], + "remediation": null, + "sarif_rule_id": "sec-af/sast/sql-injection", + "sarif_security_severity": 3.0, + "drop_reason": null + } +} + +TASK: +Use the HUNT chain candidates as seeds, but do not assume they are complete. +Validate whether each step is exploitable and whether step N enables step N+1. +Look for additional multi-step attack chains beyond the provided seeds when verified findings support them. +A chain is multi-step when exploitation of one finding enables exploitation of another. + +OUTPUT: +- Return JSON matching ChainAnalysisResult with this shape: + { + "chains": [ + { + "chain_id": "...", + "title": "...", + "validated": true | false, + "rationale": "...", + "steps": [ + { + "step_number": 1, + "finding_id": "...", + "description": "...", + "enables": "..." + } + ] + } + ] + } + +CONSTRAINTS: +- Mark validated=true only when all links are plausible with evidence from verified findings. +- Keep step order aligned with exploitation progression. +- If not validated, include rationale and return empty steps. +- You may include newly discovered chains that were not in the seed list. +- Do not return markdown or prose outside JSON. diff --git a/internal/secaf/agents/prove/testdata/golden/chain_builder_prompt_input.json b/internal/secaf/agents/prove/testdata/golden/chain_builder_prompt_input.json new file mode 100644 index 0000000000..b287109fa5 --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/chain_builder_prompt_input.json @@ -0,0 +1,93 @@ +{ + "depth": "standard", + "chains": [ + { + "chain_id": "chain-1", + "title": "SSRF to internal API", + "finding_ids": [ + "v1", + "v2" + ], + "combined_impact": "Internal service access", + "estimated_severity": "critical" + } + ], + "findings": [ + { + "id": "v1", + "fingerprint": "fp-v1", + "title": "finding v1", + "description": "d", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "SQL Injection", + "owasp_category": null, + "tags": [], + "verdict": "confirmed", + "evidence_level": 4, + "rationale": "r", + "severity": "high", + "cvss_v4": null, + "epss": null, + "exploitability_score": 7.5, + "proof": null, + "location": { + "file_path": "src/users.py", + "start_line": 1, + "end_line": 2, + "start_column": null, + "end_column": null, + "function_name": null, + "code_snippet": null + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [], + "reproduction_steps": [], + "remediation": null, + "sarif_rule_id": "sec-af/sast/sql-injection", + "sarif_security_severity": 7.5, + "drop_reason": null + }, + { + "id": "v2", + "fingerprint": "fp-v2", + "title": "finding v2", + "description": "d", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "SQL Injection", + "owasp_category": null, + "tags": [], + "verdict": "confirmed", + "evidence_level": 2, + "rationale": "r", + "severity": "high", + "cvss_v4": null, + "epss": null, + "exploitability_score": 3.0, + "proof": null, + "location": { + "file_path": "src/users.py", + "start_line": 1, + "end_line": 2, + "start_column": null, + "end_column": null, + "function_name": null, + "code_snippet": null + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [], + "reproduction_steps": [], + "remediation": null, + "sarif_rule_id": "sec-af/sast/sql-injection", + "sarif_security_severity": 3.0, + "drop_reason": null + } + ] +} diff --git a/internal/secaf/agents/prove/testdata/golden/cross_service_prompt_A.txt b/internal/secaf/agents/prove/testdata/golden/cross_service_prompt_A.txt new file mode 100644 index 0000000000..c2c200db42 --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/cross_service_prompt_A.txt @@ -0,0 +1,44 @@ +ROLE: +You are a cross-service security analyst for SEC-AF. You identify attack chains that span multiple microservices. + +CONTEXT: +Multiple services in a microservice architecture have been individually scanned. +Your task is to identify attack paths that cross service boundaries. + +Services: +[ + "gateway", + "billing-caf\u00e9", + "db<&>" +] + +Individual findings summary: +- gateway: SSRF +- billing: IDOR + +Depth: thorough + +TASK: +Analyze cross-service attack vectors: +1. Identify API call chains between services +2. Find trust boundary violations where one service trusts another's input +3. Trace data flow across service boundaries (e.g., public API -> internal service -> database) +4. Identify privilege escalation paths across services + +OUTPUT: +Return a JSON object matching CrossServiceFinding schema: +- chain_description: Full description of the cross-service attack path +- services_involved: List of service names in the chain +- entry_point: Where an attacker would begin (public-facing endpoint) +- impact: Business impact if exploited + +CONSTRAINTS: +- Focus on realistic cross-service paths, not theoretical +- Consider API authentication between services +- Consider data validation at service boundaries +- Do not return markdown or prose outside the JSON object + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Use the repository path above for cross-service inspection. \ No newline at end of file diff --git a/internal/secaf/agents/prove/testdata/golden/cross_service_prompt_B.txt b/internal/secaf/agents/prove/testdata/golden/cross_service_prompt_B.txt new file mode 100644 index 0000000000..56d84eb160 --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/cross_service_prompt_B.txt @@ -0,0 +1,39 @@ +ROLE: +You are a cross-service security analyst for SEC-AF. You identify attack chains that span multiple microservices. + +CONTEXT: +Multiple services in a microservice architecture have been individually scanned. +Your task is to identify attack paths that cross service boundaries. + +Services: +[] + +Individual findings summary: + + +Depth: quick + +TASK: +Analyze cross-service attack vectors: +1. Identify API call chains between services +2. Find trust boundary violations where one service trusts another's input +3. Trace data flow across service boundaries (e.g., public API -> internal service -> database) +4. Identify privilege escalation paths across services + +OUTPUT: +Return a JSON object matching CrossServiceFinding schema: +- chain_description: Full description of the cross-service attack path +- services_involved: List of service names in the chain +- entry_point: Where an attacker would begin (public-facing endpoint) +- impact: Business impact if exploited + +CONSTRAINTS: +- Focus on realistic cross-service paths, not theoretical +- Consider API authentication between services +- Consider data validation at service boundaries +- Do not return markdown or prose outside the JSON object + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Use the repository path above for cross-service inspection. \ No newline at end of file diff --git a/internal/secaf/agents/prove/testdata/golden/dast_prompt_A.txt b/internal/secaf/agents/prove/testdata/golden/dast_prompt_A.txt new file mode 100644 index 0000000000..2cd32d3460 --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/dast_prompt_A.txt @@ -0,0 +1,38 @@ +ROLE: +You are a DAST (Dynamic Application Security Testing) verification specialist for SEC-AF. + +CONTEXT: +A static analysis finding has been identified and an exploit hypothesis generated. +Your task is to design and evaluate a runtime verification approach for this finding. + +Finding: Potential SQL injection in user lookup +Description: Request parameter reaches a formatted SQL string. Marker: {{TITLE}} +CWE: CWE-89 +File: src/users.py +Proposed payload: 1 OR 1=1 -- thorough +Depth: thorough + +TASK: +Analyze whether the proposed exploit payload would succeed at runtime: +1. Evaluate the payload against the application's runtime behavior +2. Consider framework protections, middleware, and runtime sanitization +3. Determine if the exploit would be blocked, modified, or succeed +4. Document safety measures that would be needed for actual execution + +OUTPUT: +Return a JSON object matching DastVerificationResult schema with: +- payload_sent: The specific payload analyzed +- response_summary: Expected application behavior +- exploit_confirmed: Whether exploit would succeed at runtime +- safety_notes: Safety measures for actual verification + +CONSTRAINTS: +- This is SIMULATED verification -- analyze expected runtime behavior, do not execute +- Consider all runtime protections (WAF, framework middleware, input validation) +- Be conservative: only confirm if runtime success is highly likely +- Do not return markdown or prose outside the JSON object + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Use the repository path above for file inspection during DAST-style verification. \ No newline at end of file diff --git a/internal/secaf/agents/prove/testdata/golden/dast_prompt_B.txt b/internal/secaf/agents/prove/testdata/golden/dast_prompt_B.txt new file mode 100644 index 0000000000..38655a3de6 --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/dast_prompt_B.txt @@ -0,0 +1,38 @@ +ROLE: +You are a DAST (Dynamic Application Security Testing) verification specialist for SEC-AF. + +CONTEXT: +A static analysis finding has been identified and an exploit hypothesis generated. +Your task is to design and evaluate a runtime verification approach for this finding. + +Finding: Weak hash +Description: MD5 used for password hashing +CWE: CWE-327 +File: src/hash.py +Proposed payload: +Depth: quick + +TASK: +Analyze whether the proposed exploit payload would succeed at runtime: +1. Evaluate the payload against the application's runtime behavior +2. Consider framework protections, middleware, and runtime sanitization +3. Determine if the exploit would be blocked, modified, or succeed +4. Document safety measures that would be needed for actual execution + +OUTPUT: +Return a JSON object matching DastVerificationResult schema with: +- payload_sent: The specific payload analyzed +- response_summary: Expected application behavior +- exploit_confirmed: Whether exploit would succeed at runtime +- safety_notes: Safety measures for actual verification + +CONSTRAINTS: +- This is SIMULATED verification -- analyze expected runtime behavior, do not execute +- Consider all runtime protections (WAF, framework middleware, input validation) +- Be conservative: only confirm if runtime success is highly likely +- Do not return markdown or prose outside the JSON object + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Use the repository path above for file inspection during DAST-style verification. \ No newline at end of file diff --git a/internal/secaf/agents/prove/testdata/golden/dep_reachability_input_A.json b/internal/secaf/agents/prove/testdata/golden/dep_reachability_input_A.json new file mode 100644 index 0000000000..a40868025e --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/dep_reachability_input_A.json @@ -0,0 +1,19 @@ +{ + "cve": "CVE-2021-44228", + "package": "log4j-core", + "vulnerable_function": "JndiLookup.lookup", + "version": "2.14.1", + "evidence": { + "direct": true, + "manifest": "pom.xml", + "nested": { + "a": 1, + "b": [ + true, + null + ] + }, + "score": 9.8, + "transitive_depth": 2 + } +} diff --git a/internal/secaf/agents/prove/testdata/golden/dep_reachability_input_C.json b/internal/secaf/agents/prove/testdata/golden/dep_reachability_input_C.json new file mode 100644 index 0000000000..bf2210c752 --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/dep_reachability_input_C.json @@ -0,0 +1,10 @@ +{ + "cve": 1234, + "package": null, + "vulnerable_function": 3.5, + "version": true, + "evidence": [ + 1, + "two" + ] +} diff --git a/internal/secaf/agents/prove/testdata/golden/dep_reachability_prompt_A.txt b/internal/secaf/agents/prove/testdata/golden/dep_reachability_prompt_A.txt new file mode 100644 index 0000000000..34cd4c7ef8 --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/dep_reachability_prompt_A.txt @@ -0,0 +1,43 @@ +ROLE: +You are DependencyReachabilityAnalyzer, a focused security analysis sub-agent. + +CONTEXT: +- Analysis depth: thorough +- CVE: CVE-2021-44228 +- Package: log4j-core +- Version: 2.14.1 +- Vulnerable function: JndiLookup.lookup +- Additional evidence/hints: { + "direct": true, + "manifest": "pom.xml", + "nested": { + "a": 1, + "b": [ + true, + null + ] + }, + "score": 9.8, + "transitive_depth": 2 +} + +TASK: +1) Inspect the repository to determine whether application code imports/calls into the vulnerable dependency path. +2) Build a concrete import/call chain from app entry points to JndiLookup.lookup when possible. +3) Decide if the vulnerable function is reachable in runtime paths. +4) Determine whether log4j-core is a direct dependency or transitive dependency. + +OUTPUT: +- Return JSON that strictly matches ReachabilityProof. +- Fill vulnerable_function, call_chain, reachable, and direct. +- call_chain must be ordered app-to-dependency evidence and can be empty when unreachable. + +CONSTRAINTS: +- Take multiple turns for file inspection before final JSON. +- Be evidence-driven; avoid assumptions when the chain is not present. +- Do not return markdown, prose wrappers, or code fences. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Use the repository path above for file inspection during dependency reachability analysis. \ No newline at end of file diff --git a/internal/secaf/agents/prove/testdata/golden/dep_reachability_prompt_B.txt b/internal/secaf/agents/prove/testdata/golden/dep_reachability_prompt_B.txt new file mode 100644 index 0000000000..bd5e8bf356 --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/dep_reachability_prompt_B.txt @@ -0,0 +1,31 @@ +ROLE: +You are DependencyReachabilityAnalyzer, a focused security analysis sub-agent. + +CONTEXT: +- Analysis depth: quick +- CVE: +- Package: +- Version: +- Vulnerable function: +- Additional evidence/hints: {} + +TASK: +1) Inspect the repository to determine whether application code imports/calls into the vulnerable dependency path. +2) Build a concrete import/call chain from app entry points to when possible. +3) Decide if the vulnerable function is reachable in runtime paths. +4) Determine whether is a direct dependency or transitive dependency. + +OUTPUT: +- Return JSON that strictly matches ReachabilityProof. +- Fill vulnerable_function, call_chain, reachable, and direct. +- call_chain must be ordered app-to-dependency evidence and can be empty when unreachable. + +CONSTRAINTS: +- Take multiple turns for file inspection before final JSON. +- Be evidence-driven; avoid assumptions when the chain is not present. +- Do not return markdown, prose wrappers, or code fences. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Use the repository path above for file inspection during dependency reachability analysis. \ No newline at end of file diff --git a/internal/secaf/agents/prove/testdata/golden/dep_reachability_prompt_C.txt b/internal/secaf/agents/prove/testdata/golden/dep_reachability_prompt_C.txt new file mode 100644 index 0000000000..7ef8527343 --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/dep_reachability_prompt_C.txt @@ -0,0 +1,34 @@ +ROLE: +You are DependencyReachabilityAnalyzer, a focused security analysis sub-agent. + +CONTEXT: +- Analysis depth: standard +- CVE: 1234 +- Package: None +- Version: True +- Vulnerable function: 3.5 +- Additional evidence/hints: [ + 1, + "two" +] + +TASK: +1) Inspect the repository to determine whether application code imports/calls into the vulnerable dependency path. +2) Build a concrete import/call chain from app entry points to 3.5 when possible. +3) Decide if the vulnerable function is reachable in runtime paths. +4) Determine whether None is a direct dependency or transitive dependency. + +OUTPUT: +- Return JSON that strictly matches ReachabilityProof. +- Fill vulnerable_function, call_chain, reachable, and direct. +- call_chain must be ordered app-to-dependency evidence and can be empty when unreachable. + +CONSTRAINTS: +- Take multiple turns for file inspection before final JSON. +- Be evidence-driven; avoid assumptions when the chain is not present. +- Do not return markdown, prose wrappers, or code fences. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Use the repository path above for file inspection during dependency reachability analysis. \ No newline at end of file diff --git a/internal/secaf/agents/prove/testdata/golden/exploit_prompt_A.txt b/internal/secaf/agents/prove/testdata/golden/exploit_prompt_A.txt new file mode 100644 index 0000000000..502a2287fd --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/exploit_prompt_A.txt @@ -0,0 +1,51 @@ +ROLE: +You are ExploitHypothesizer, a focused security analysis sub-agent. + +CONTEXT: +- Finding type: sast +- Analysis depth: thorough +- Related files: [ + "src/routes.py", + "src/caf\u00e9 &.py" +] +- Data flow context: +Source: request.args['id'] +Sink: cursor.execute(query) +Sink reached: yes +Trace steps: +- src/routes.py:10 read request.args +- src/users.py:42 execute +- Sanitization context: +Sanitization found: yes +Sanitization type: parameterized query +Sanitization sufficient: no +Bypass method: second-order injection through the audit log + +TASK: +1) Read code at src/users.py around line 42 and inspect supporting files as needed. +2) Build the most plausible exploit scenario for this finding. +3) Propose concrete payload if feasible. +4) State expected observable outcome on success. + +SCOPE: +FINDING: +- Title: Potential SQL injection in user lookup +- Description: Request parameter reaches a formatted SQL string. Marker: {{TITLE}} +- CWE: CWE-89 - Improper Neutralization/Escaping of Special Elements +- File: src/users.py:42 +- Code snippet: +cursor.execute("SELECT * FROM users WHERE id = " + user_id) # depth=thorough + +OUTPUT: +- Return JSON that strictly matches ExploitHypothesis. +- Fill hypothesis, payload, and expected_outcome. + +CONSTRAINTS: +- Take multiple turns for file inspection before final JSON. +- Keep exploit logic consistent with trace and sanitization context. +- Do not return markdown, prose wrappers, or code fences. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Use the repository path above for file inspection during exploit construction. \ No newline at end of file diff --git a/internal/secaf/agents/prove/testdata/golden/exploit_prompt_B.txt b/internal/secaf/agents/prove/testdata/golden/exploit_prompt_B.txt new file mode 100644 index 0000000000..6891193e32 --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/exploit_prompt_B.txt @@ -0,0 +1,47 @@ +ROLE: +You are ExploitHypothesizer, a focused security analysis sub-agent. + +CONTEXT: +- Finding type: sast +- Analysis depth: quick +- Related files: [] +- Data flow context: +Source: unknown +Sink: unknown +Sink reached: no +Trace steps: +- (no concrete trace steps) +- Sanitization context: +Sanitization found: no +Sanitization type: none +Sanitization sufficient: unknown +Bypass method: none + +TASK: +1) Read code at src/hash.py around line 7 and inspect supporting files as needed. +2) Build the most plausible exploit scenario for this finding. +3) Propose concrete payload if feasible. +4) State expected observable outcome on success. + +SCOPE: +FINDING: +- Title: Weak hash +- Description: MD5 used for password hashing +- CWE: CWE-327 - Broken Crypto +- File: src/hash.py:7 +- Code snippet: +hashlib.md5(pw).hexdigest() + +OUTPUT: +- Return JSON that strictly matches ExploitHypothesis. +- Fill hypothesis, payload, and expected_outcome. + +CONSTRAINTS: +- Take multiple turns for file inspection before final JSON. +- Keep exploit logic consistent with trace and sanitization context. +- Do not return markdown, prose wrappers, or code fences. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Use the repository path above for file inspection during exploit construction. \ No newline at end of file diff --git a/internal/secaf/agents/prove/testdata/golden/fallback.json b/internal/secaf/agents/prove/testdata/golden/fallback.json new file mode 100644 index 0000000000..e67859e079 --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/fallback.json @@ -0,0 +1,158 @@ +{ + "plain": { + "id": "raw-1", + "fingerprint": "fp-1", + "title": "Potential SQL injection in user lookup", + "description": "Request parameter reaches a formatted SQL string. Marker: {{TITLE}}", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "Improper Neutralization/Escaping of Special Elements", + "owasp_category": "A03:2021 - Injection", + "tags": [], + "verdict": "inconclusive", + "evidence_level": 1, + "rationale": "Verification incomplete: harness blew up", + "severity": "high", + "cvss_v4": null, + "epss": null, + "exploitability_score": 0.0, + "proof": null, + "location": { + "file_path": "src/users.py", + "start_line": 42, + "end_line": 44, + "start_column": null, + "end_column": null, + "function_name": "get_user", + "code_snippet": "cursor.execute(\"SELECT * FROM users WHERE id = \" + user_id) # depth={{DEPTH}}" + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [], + "reproduction_steps": [], + "remediation": null, + "sarif_rule_id": "sec-af/sast/improper-neutralization-escaping-of-special-elements", + "sarif_security_severity": 0.0, + "drop_reason": null + }, + "with_drop_reason": { + "id": "raw-1", + "fingerprint": "fp-1", + "title": "Potential SQL injection in user lookup", + "description": "Request parameter reaches a formatted SQL string. Marker: {{TITLE}}", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "Improper Neutralization/Escaping of Special Elements", + "owasp_category": "A03:2021 - Injection", + "tags": [ + "low_confidence" + ], + "verdict": "inconclusive", + "evidence_level": 1, + "rationale": "Verification incomplete: boom", + "severity": "high", + "cvss_v4": null, + "epss": null, + "exploitability_score": 0.0, + "proof": null, + "location": { + "file_path": "src/users.py", + "start_line": 42, + "end_line": 44, + "start_column": null, + "end_column": null, + "function_name": "get_user", + "code_snippet": "cursor.execute(\"SELECT * FROM users WHERE id = \" + user_id) # depth={{DEPTH}}" + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [], + "reproduction_steps": [], + "remediation": null, + "sarif_rule_id": "sec-af/sast/improper-neutralization-escaping-of-special-elements", + "sarif_security_severity": 0.0, + "drop_reason": "verifier_error" + }, + "demoted": { + "id": "raw-2", + "fingerprint": "fp-2", + "title": "Weak hash", + "description": "MD5 used for password hashing", + "finding_type": "sast", + "cwe_id": "CWE-327", + "cwe_name": "Broken Crypto", + "owasp_category": null, + "tags": [ + "low_confidence" + ], + "verdict": "inconclusive", + "evidence_level": 1, + "rationale": "Verification incomplete: Verifier returned unverified verdict; demoted for manual review (original verdict: unverified)", + "severity": "medium", + "cvss_v4": null, + "epss": null, + "exploitability_score": 0.0, + "proof": null, + "location": { + "file_path": "src/hash.py", + "start_line": 7, + "end_line": 7, + "start_column": null, + "end_column": null, + "function_name": null, + "code_snippet": "hashlib.md5(pw).hexdigest()" + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [], + "reproduction_steps": [], + "remediation": null, + "sarif_rule_id": "sec-af/sast/broken-crypto", + "sarif_security_severity": 0.0, + "drop_reason": "verdict_unverified" + }, + "original_verdict_only": { + "id": "raw-2", + "fingerprint": "fp-2", + "title": "Weak hash", + "description": "MD5 used for password hashing", + "finding_type": "sast", + "cwe_id": "CWE-327", + "cwe_name": "Broken Crypto", + "owasp_category": null, + "tags": [], + "verdict": "inconclusive", + "evidence_level": 1, + "rationale": "Verification incomplete: why (original verdict: unverified)", + "severity": "medium", + "cvss_v4": null, + "epss": null, + "exploitability_score": 0.0, + "proof": null, + "location": { + "file_path": "src/hash.py", + "start_line": 7, + "end_line": 7, + "start_column": null, + "end_column": null, + "function_name": null, + "code_snippet": "hashlib.md5(pw).hexdigest()" + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [], + "reproduction_steps": [], + "remediation": null, + "sarif_rule_id": "sec-af/sast/broken-crypto", + "sarif_security_severity": 0.0, + "drop_reason": null + } +} diff --git a/internal/secaf/agents/prove/testdata/golden/priority_sort.json b/internal/secaf/agents/prove/testdata/golden/priority_sort.json new file mode 100644 index 0000000000..69fb933966 --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/priority_sort.json @@ -0,0 +1,42 @@ +{ + "input": [ + { + "id": "a", + "estimated_severity": "low", + "confidence": "high" + }, + { + "id": "b", + "estimated_severity": "critical", + "confidence": "low" + }, + { + "id": "c", + "estimated_severity": "critical", + "confidence": "high" + }, + { + "id": "d", + "estimated_severity": "info", + "confidence": "medium" + }, + { + "id": "e", + "estimated_severity": "critical", + "confidence": "low" + }, + { + "id": "f", + "estimated_severity": "medium", + "confidence": "medium" + } + ], + "want_ids": [ + "c", + "b", + "e", + "f", + "a", + "d" + ] +} diff --git a/internal/secaf/agents/prove/testdata/golden/run_prove.json b/internal/secaf/agents/prove/testdata/golden/run_prove.json new file mode 100644 index 0000000000..33bfac07d4 --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/run_prove.json @@ -0,0 +1,362 @@ +{ + "repo_path": "/fixtures/demo-repo", + "depth": "standard", + "hunt_result": { + "findings": [ + { + "id": "raw-2", + "hunter_strategy": "crypto", + "title": "Weak hash", + "description": "MD5 used for password hashing", + "finding_type": "sast", + "cwe_id": "CWE-327", + "cwe_name": "Broken Crypto", + "owasp_category": null, + "file_path": "src/hash.py", + "start_line": 7, + "end_line": 7, + "function_name": null, + "code_snippet": "hashlib.md5(pw).hexdigest()", + "estimated_severity": "medium", + "confidence": "low", + "data_flow": null, + "related_files": [], + "fingerprint": "fp-2" + }, + { + "id": "raw-1", + "hunter_strategy": "injection", + "title": "Potential SQL injection in user lookup", + "description": "Request parameter reaches a formatted SQL string. Marker: {{TITLE}}", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "Improper Neutralization/Escaping of Special Elements", + "owasp_category": "A03:2021 - Injection", + "file_path": "src/users.py", + "start_line": 42, + "end_line": 44, + "function_name": "get_user", + "code_snippet": "cursor.execute(\"SELECT * FROM users WHERE id = \" + user_id) # depth={{DEPTH}}", + "estimated_severity": "high", + "confidence": "high", + "data_flow": [ + { + "file_path": "src/routes.py", + "line": 10, + "component": "handler", + "operation": "read request.args" + }, + { + "file_path": "src/users.py", + "line": 42, + "component": "db", + "operation": "execute" + } + ], + "related_files": [ + "src/routes.py", + "src/caf\u00e9 &.py" + ], + "fingerprint": "fp-1" + } + ], + "chains": [], + "total_raw": 2, + "deduplicated_count": 2, + "chain_count": 0, + "strategies_run": [ + "injection" + ], + "hunt_duration_seconds": 0.0 + }, + "canned": { + "tracer": { + "source": "request.args['id']", + "sink": "cursor.execute(query)", + "steps": [ + "src/routes.py:10 read request.args", + "src/users.py:42 execute" + ], + "sink_reached": true + }, + "sanitization": { + "found": true, + "type": "parameterized query", + "sufficient": false, + "bypass_method": "second-order injection through the audit log" + }, + "exploit": { + "hypothesis": "Attacker supplies id=1 OR 1=1 to dump the table", + "payload": "1 OR 1=1", + "expected_outcome": "Full users table returned" + }, + "verdict": { + "verdict": "confirmed", + "evidence_level": 5, + "rationale": "canned rationale", + "confidence": "high" + } + }, + "want": [ + { + "id": "raw-1", + "fingerprint": "fp-1", + "title": "Potential SQL injection in user lookup", + "description": "Request parameter reaches a formatted SQL string. Marker: {{TITLE}}", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "Improper Neutralization/Escaping of Special Elements", + "owasp_category": "A03:2021 - Injection", + "tags": [], + "verdict": "confirmed", + "evidence_level": 5, + "rationale": "canned rationale", + "severity": "critical", + "cvss_v4": null, + "epss": null, + "exploitability_score": 9.0, + "proof": { + "exploit_hypothesis": "Attacker supplies id=1 OR 1=1 to dump the table", + "verification_method": "composite_subagent_chain:sast", + "evidence_level": 5, + "data_flow_trace": [ + { + "file": "trace_step_1", + "line": 1, + "description": "src/routes.py:10 read request.args", + "tainted": true + }, + { + "file": "trace_step_2", + "line": 2, + "description": "src/users.py:42 execute", + "tainted": true + } + ], + "data_flow_evidence": { + "steps": [ + { + "file": "trace_step_1", + "line": 1, + "description": "src/routes.py:10 read request.args", + "tainted": true + }, + { + "file": "trace_step_2", + "line": 2, + "description": "src/users.py:42 execute", + "tainted": true + } + ], + "source": "request.args['id']", + "sink": "cursor.execute(query)", + "sink_reached": true + }, + "sanitization_analysis": { + "sanitization_found": true, + "sanitization_type": "parameterized query", + "sanitization_sufficient": false, + "bypass_possible": true, + "bypass_method": "second-order injection through the audit log" + }, + "vulnerable_code": null, + "exploit_payload": "1 OR 1=1", + "expected_outcome": "Full users table returned", + "poc_code": null, + "poc_execution_output": null, + "http_request": null, + "http_response": null, + "reachability": null, + "chain_steps": null + }, + "location": { + "file_path": "src/users.py", + "start_line": 42, + "end_line": 44, + "start_column": null, + "end_column": null, + "function_name": "get_user", + "code_snippet": "cursor.execute(\"SELECT * FROM users WHERE id = \" + user_id) # depth={{DEPTH}}" + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [ + { + "framework": "PCI-DSS", + "control_id": "Req 6.2.4", + "control_name": "Custom software addresses common coding vulnerabilities" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A03:2021", + "control_name": "Injection" + }, + { + "framework": "HIPAA", + "control_id": "\u00a7164.312(a)(1)", + "control_name": "Access control" + }, + { + "framework": "ISO27001", + "control_id": "A.8.28", + "control_name": "Secure coding" + } + ], + "reproduction_steps": [ + { + "step": 1, + "description": "Trace attacker-controlled input from source to sink in target code path.", + "command": null, + "expected_output": "Input reaches a sensitive sink." + }, + { + "step": 2, + "description": "Attacker supplies id=1 OR 1=1 to dump the table", + "command": "1 OR 1=1", + "expected_output": "Full users table returned" + } + ], + "remediation": null, + "sarif_rule_id": "sec-af/sast/improper-neutralization-escaping-of-special-elements", + "sarif_security_severity": 9.0, + "drop_reason": null + }, + { + "id": "raw-2", + "fingerprint": "fp-2", + "title": "Weak hash", + "description": "MD5 used for password hashing", + "finding_type": "sast", + "cwe_id": "CWE-327", + "cwe_name": "Broken Crypto", + "owasp_category": null, + "tags": [], + "verdict": "confirmed", + "evidence_level": 5, + "rationale": "canned rationale", + "severity": "medium", + "cvss_v4": null, + "epss": null, + "exploitability_score": 4.5, + "proof": { + "exploit_hypothesis": "Attacker supplies id=1 OR 1=1 to dump the table", + "verification_method": "composite_subagent_chain:sast", + "evidence_level": 5, + "data_flow_trace": [ + { + "file": "trace_step_1", + "line": 1, + "description": "src/routes.py:10 read request.args", + "tainted": true + }, + { + "file": "trace_step_2", + "line": 2, + "description": "src/users.py:42 execute", + "tainted": true + } + ], + "data_flow_evidence": { + "steps": [ + { + "file": "trace_step_1", + "line": 1, + "description": "src/routes.py:10 read request.args", + "tainted": true + }, + { + "file": "trace_step_2", + "line": 2, + "description": "src/users.py:42 execute", + "tainted": true + } + ], + "source": "request.args['id']", + "sink": "cursor.execute(query)", + "sink_reached": true + }, + "sanitization_analysis": { + "sanitization_found": true, + "sanitization_type": "parameterized query", + "sanitization_sufficient": false, + "bypass_possible": true, + "bypass_method": "second-order injection through the audit log" + }, + "vulnerable_code": null, + "exploit_payload": "1 OR 1=1", + "expected_outcome": "Full users table returned", + "poc_code": null, + "poc_execution_output": null, + "http_request": null, + "http_response": null, + "reachability": null, + "chain_steps": null + }, + "location": { + "file_path": "src/hash.py", + "start_line": 7, + "end_line": 7, + "start_column": null, + "end_column": null, + "function_name": null, + "code_snippet": "hashlib.md5(pw).hexdigest()" + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [ + { + "framework": "PCI-DSS", + "control_id": "Req 3", + "control_name": "Protect stored account data" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A02:2021", + "control_name": "Cryptographic Failures" + }, + { + "framework": "HIPAA", + "control_id": "\u00a7164.312(a)(2)(iv)", + "control_name": "Encryption and decryption" + }, + { + "framework": "ISO27001", + "control_id": "A.8.24", + "control_name": "Use of cryptography" + } + ], + "reproduction_steps": [ + { + "step": 1, + "description": "Trace attacker-controlled input from source to sink in target code path.", + "command": null, + "expected_output": "Input reaches a sensitive sink." + }, + { + "step": 2, + "description": "Attacker supplies id=1 OR 1=1 to dump the table", + "command": "1 OR 1=1", + "expected_output": "Full users table returned" + } + ], + "remediation": null, + "sarif_rule_id": "sec-af/sast/broken-crypto", + "sarif_security_severity": 4.5, + "drop_reason": null + } + ] +} diff --git a/internal/secaf/agents/prove/testdata/golden/run_verifier.json b/internal/secaf/agents/prove/testdata/golden/run_verifier.json new file mode 100644 index 0000000000..0bb0af10a1 --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/run_verifier.json @@ -0,0 +1,197 @@ +{ + "not_exploitable": { + "id": "raw-1", + "fingerprint": "fp-1", + "title": "Potential SQL injection in user lookup", + "description": "Request parameter reaches a formatted SQL string. Marker: {{TITLE}}", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "Improper Neutralization/Escaping of Special Elements", + "owasp_category": "A03:2021 - Injection", + "tags": [], + "verdict": "not_exploitable", + "evidence_level": 1, + "rationale": "canned rationale", + "severity": "high", + "cvss_v4": null, + "epss": null, + "exploitability_score": 0.0, + "proof": { + "exploit_hypothesis": "Attacker supplies id=1 OR 1=1 to dump the table", + "verification_method": "composite_subagent_chain:sast", + "evidence_level": 1, + "data_flow_trace": [ + { + "file": "trace_step_1", + "line": 1, + "description": "src/routes.py:10 read request.args", + "tainted": true + }, + { + "file": "trace_step_2", + "line": 2, + "description": "src/users.py:42 execute", + "tainted": true + } + ], + "data_flow_evidence": { + "steps": [ + { + "file": "trace_step_1", + "line": 1, + "description": "src/routes.py:10 read request.args", + "tainted": true + }, + { + "file": "trace_step_2", + "line": 2, + "description": "src/users.py:42 execute", + "tainted": true + } + ], + "source": "request.args['id']", + "sink": "cursor.execute(query)", + "sink_reached": true + }, + "sanitization_analysis": { + "sanitization_found": true, + "sanitization_type": "parameterized query", + "sanitization_sufficient": false, + "bypass_possible": true, + "bypass_method": "second-order injection through the audit log" + }, + "vulnerable_code": null, + "exploit_payload": "1 OR 1=1", + "expected_outcome": "Full users table returned", + "poc_code": null, + "poc_execution_output": null, + "http_request": null, + "http_response": null, + "reachability": null, + "chain_steps": null + }, + "location": { + "file_path": "src/users.py", + "start_line": 42, + "end_line": 44, + "start_column": null, + "end_column": null, + "function_name": "get_user", + "code_snippet": "cursor.execute(\"SELECT * FROM users WHERE id = \" + user_id) # depth={{DEPTH}}" + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [], + "reproduction_steps": [], + "remediation": null, + "sarif_rule_id": "sec-af/sast/improper-neutralization-escaping-of-special-elements", + "sarif_security_severity": 0.0, + "drop_reason": null + }, + "likely": { + "id": "raw-2", + "fingerprint": "fp-2", + "title": "Weak hash", + "description": "MD5 used for password hashing", + "finding_type": "sast", + "cwe_id": "CWE-327", + "cwe_name": "Broken Crypto", + "owasp_category": null, + "tags": [], + "verdict": "likely", + "evidence_level": 3, + "rationale": "canned rationale", + "severity": "medium", + "cvss_v4": null, + "epss": null, + "exploitability_score": 0.0, + "proof": { + "exploit_hypothesis": "Attacker supplies id=1 OR 1=1 to dump the table", + "verification_method": "composite_subagent_chain:sast", + "evidence_level": 3, + "data_flow_trace": [ + { + "file": "trace_step_1", + "line": 1, + "description": "src/routes.py:10 read request.args", + "tainted": true + }, + { + "file": "trace_step_2", + "line": 2, + "description": "src/users.py:42 execute", + "tainted": true + } + ], + "data_flow_evidence": { + "steps": [ + { + "file": "trace_step_1", + "line": 1, + "description": "src/routes.py:10 read request.args", + "tainted": true + }, + { + "file": "trace_step_2", + "line": 2, + "description": "src/users.py:42 execute", + "tainted": true + } + ], + "source": "request.args['id']", + "sink": "cursor.execute(query)", + "sink_reached": true + }, + "sanitization_analysis": { + "sanitization_found": true, + "sanitization_type": "parameterized query", + "sanitization_sufficient": false, + "bypass_possible": true, + "bypass_method": "second-order injection through the audit log" + }, + "vulnerable_code": null, + "exploit_payload": "1 OR 1=1", + "expected_outcome": "Full users table returned", + "poc_code": null, + "poc_execution_output": null, + "http_request": null, + "http_response": null, + "reachability": null, + "chain_steps": null + }, + "location": { + "file_path": "src/hash.py", + "start_line": 7, + "end_line": 7, + "start_column": null, + "end_column": null, + "function_name": null, + "code_snippet": "hashlib.md5(pw).hexdigest()" + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [], + "reproduction_steps": [ + { + "step": 1, + "description": "Trace attacker-controlled input from source to sink in target code path.", + "command": null, + "expected_output": "Input reaches a sensitive sink." + }, + { + "step": 2, + "description": "Attacker supplies id=1 OR 1=1 to dump the table", + "command": "1 OR 1=1", + "expected_output": "Full users table returned" + } + ], + "remediation": null, + "sarif_rule_id": "sec-af/sast/broken-crypto", + "sarif_security_severity": 0.0, + "drop_reason": null + } +} diff --git a/internal/secaf/agents/prove/testdata/golden/sanitization_prompt_A.txt b/internal/secaf/agents/prove/testdata/golden/sanitization_prompt_A.txt new file mode 100644 index 0000000000..6d83b89d9a --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/sanitization_prompt_A.txt @@ -0,0 +1,46 @@ +ROLE: +You are SanitizationAnalyzer, a focused security analysis sub-agent. + +CONTEXT: +- Finding type: sast +- Analysis depth: thorough +- Related files: [ + "src/routes.py", + "src/caf\u00e9 &.py" +] +- Data flow context from tracer/hints: +Source: request.args['id'] +Sink: cursor.execute(query) +Sink reached: yes +Trace steps: +- src/routes.py:10 read request.args +- src/users.py:42 execute + +TASK: +1) Read code at src/users.py around line 42 and inspect related files as needed. +2) Locate validation/sanitization/encoding controls on the source-to-sink path. +3) Assess whether controls are sufficient against realistic attacker input. +4) If bypassable, describe practical bypass method. + +SCOPE: +FINDING: +- Title: Potential SQL injection in user lookup +- Description: Request parameter reaches a formatted SQL string. Marker: {{TITLE}} +- CWE: CWE-89 - Improper Neutralization/Escaping of Special Elements +- File: src/users.py:42 +- Code snippet: +cursor.execute("SELECT * FROM users WHERE id = " + user_id) # depth=thorough + +OUTPUT: +- Return JSON that strictly matches SanitizationResult. +- Fill found, type, sufficient, and bypass_method. + +CONSTRAINTS: +- Take multiple turns for file inspection before final JSON. +- Be explicit about uncertainty using null values where needed. +- Do not return markdown, prose wrappers, or code fences. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Use the repository path above for file inspection during sanitization analysis. \ No newline at end of file diff --git a/internal/secaf/agents/prove/testdata/golden/sanitization_prompt_B.txt b/internal/secaf/agents/prove/testdata/golden/sanitization_prompt_B.txt new file mode 100644 index 0000000000..e77dee4d5e --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/sanitization_prompt_B.txt @@ -0,0 +1,42 @@ +ROLE: +You are SanitizationAnalyzer, a focused security analysis sub-agent. + +CONTEXT: +- Finding type: sast +- Analysis depth: quick +- Related files: [] +- Data flow context from tracer/hints: +Source: unknown +Sink: unknown +Sink reached: no +Trace steps: +- (no concrete trace steps) + +TASK: +1) Read code at src/hash.py around line 7 and inspect related files as needed. +2) Locate validation/sanitization/encoding controls on the source-to-sink path. +3) Assess whether controls are sufficient against realistic attacker input. +4) If bypassable, describe practical bypass method. + +SCOPE: +FINDING: +- Title: Weak hash +- Description: MD5 used for password hashing +- CWE: CWE-327 - Broken Crypto +- File: src/hash.py:7 +- Code snippet: +hashlib.md5(pw).hexdigest() + +OUTPUT: +- Return JSON that strictly matches SanitizationResult. +- Fill found, type, sufficient, and bypass_method. + +CONSTRAINTS: +- Take multiple turns for file inspection before final JSON. +- Be explicit about uncertainty using null values where needed. +- Do not return markdown, prose wrappers, or code fences. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Use the repository path above for file inspection during sanitization analysis. \ No newline at end of file diff --git a/internal/secaf/agents/prove/testdata/golden/tracer_prompt_A.txt b/internal/secaf/agents/prove/testdata/golden/tracer_prompt_A.txt new file mode 100644 index 0000000000..34e15eefa1 --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/tracer_prompt_A.txt @@ -0,0 +1,54 @@ +ROLE: +You are DataFlowTracer, a focused security analysis sub-agent. + +CONTEXT: +- Finding type: sast +- Analysis depth: thorough +- Related files: [ + "src/routes.py", + "src/caf\u00e9 &.py" +] +- Candidate flow hints from HUNT: [ + { + "file_path": "src/routes.py", + "line": 10, + "component": "handler", + "operation": "read request.args" + }, + { + "file_path": "src/users.py", + "line": 42, + "component": "db", + "operation": "execute" + } +] + +TASK: +1) Read code at src/users.py around line 42 and inspect related files as needed. +2) Identify the attacker-controlled source and security-sensitive sink. +3) Trace source-to-sink flow with ordered path evidence. +4) Decide whether tainted input actually reaches the sink. + +SCOPE: +FINDING: +- Title: Potential SQL injection in user lookup +- Description: Request parameter reaches a formatted SQL string. Marker: {{TITLE}} +- CWE: CWE-89 - Improper Neutralization/Escaping of Special Elements +- File: src/users.py:42 +- Code snippet: +cursor.execute("SELECT * FROM users WHERE id = " + user_id) # depth=thorough + +OUTPUT: +- Return JSON that strictly matches DataFlowTrace. +- Fill source, sink, steps, and sink_reached. +- Keep steps concise, ordered, and concrete (file:line path style). + +CONSTRAINTS: +- Take multiple turns for file inspection before final JSON. +- Be evidence-driven and avoid speculation. +- Do not return markdown, prose wrappers, or code fences. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Use the repository path above for file inspection during source-to-sink tracing. \ No newline at end of file diff --git a/internal/secaf/agents/prove/testdata/golden/tracer_prompt_B.txt b/internal/secaf/agents/prove/testdata/golden/tracer_prompt_B.txt new file mode 100644 index 0000000000..a5689c7086 --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/tracer_prompt_B.txt @@ -0,0 +1,38 @@ +ROLE: +You are DataFlowTracer, a focused security analysis sub-agent. + +CONTEXT: +- Finding type: sast +- Analysis depth: quick +- Related files: [] +- Candidate flow hints from HUNT: [] + +TASK: +1) Read code at src/hash.py around line 7 and inspect related files as needed. +2) Identify the attacker-controlled source and security-sensitive sink. +3) Trace source-to-sink flow with ordered path evidence. +4) Decide whether tainted input actually reaches the sink. + +SCOPE: +FINDING: +- Title: Weak hash +- Description: MD5 used for password hashing +- CWE: CWE-327 - Broken Crypto +- File: src/hash.py:7 +- Code snippet: +hashlib.md5(pw).hexdigest() + +OUTPUT: +- Return JSON that strictly matches DataFlowTrace. +- Fill source, sink, steps, and sink_reached. +- Keep steps concise, ordered, and concrete (file:line path style). + +CONSTRAINTS: +- Take multiple turns for file inspection before final JSON. +- Be evidence-driven and avoid speculation. +- Do not return markdown, prose wrappers, or code fences. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Use the repository path above for file inspection during source-to-sink tracing. \ No newline at end of file diff --git a/internal/secaf/agents/prove/testdata/golden/verdict_prompt_A.txt b/internal/secaf/agents/prove/testdata/golden/verdict_prompt_A.txt new file mode 100644 index 0000000000..2091877c07 --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/verdict_prompt_A.txt @@ -0,0 +1,54 @@ +ROLE: +You are VerdictAgent, a focused judgment sub-agent. + +CONTEXT: +- Finding type: sast +- Related files: [ + "src/routes.py", + "src/caf\u00e9 &.py" +] +- Consolidated sub-agent outputs: +Tracer output: +- source: request.args['id'] +- sink: cursor.execute(query) +- sink_reached: True +- steps: +- src/routes.py:10 read request.args +- src/users.py:42 execute + +Sanitization output: +- found: True +- type: parameterized query +- sufficient: False +- bypass_method: second-order injection through the audit log + +Exploit output: +- hypothesis: Attacker supplies id=1 OR 1=1 to dump the table +- payload: 1 OR 1=1 +- expected_outcome: Full users table returned + +TASK: +1) Evaluate tracer, sanitization, and exploit outputs together. +2) Decide final exploitability verdict. +3) Assign evidence level on 1-6 scale. +4) Provide concise rationale and confidence level. + +SCOPE: +FINDING: +- Title: Potential SQL injection in user lookup +- Description: Request parameter reaches a formatted SQL string. Marker: {{TITLE}} +- CWE: CWE-89 - Improper Neutralization/Escaping of Special Elements +- File: src/users.py:42 +- Code snippet: +cursor.execute("SELECT * FROM users WHERE id = " + user_id) # depth={{DEPTH}} + +OUTPUT: +- Return JSON that strictly matches VerdictDecision. +- verdict MUST be one of: confirmed, likely, inconclusive, not_exploitable. +- evidence_level MUST be integer 1..6. +- confidence MUST be one of: high, medium, low. + +CONSTRAINTS: +- Prioritize consistency with provided sub-agent evidence. +- Keep rationale concise and evidence-grounded. +- Do not return markdown, prose wrappers, or code fences. diff --git a/internal/secaf/agents/prove/testdata/golden/verdict_prompt_B.txt b/internal/secaf/agents/prove/testdata/golden/verdict_prompt_B.txt new file mode 100644 index 0000000000..bb7fd2820b --- /dev/null +++ b/internal/secaf/agents/prove/testdata/golden/verdict_prompt_B.txt @@ -0,0 +1,50 @@ +ROLE: +You are VerdictAgent, a focused judgment sub-agent. + +CONTEXT: +- Finding type: sast +- Related files: [] +- Consolidated sub-agent outputs: +Tracer output: +- source: unknown +- sink: unknown +- sink_reached: False +- steps: +- (none) + +Sanitization output: +- found: False +- type: none +- sufficient: None +- bypass_method: none + +Exploit output: +- hypothesis: unknown +- payload: none +- expected_outcome: unknown + +TASK: +1) Evaluate tracer, sanitization, and exploit outputs together. +2) Decide final exploitability verdict. +3) Assign evidence level on 1-6 scale. +4) Provide concise rationale and confidence level. + +SCOPE: +FINDING: +- Title: Weak hash +- Description: MD5 used for password hashing +- CWE: CWE-327 - Broken Crypto +- File: src/hash.py:7 +- Code snippet: +hashlib.md5(pw).hexdigest() + +OUTPUT: +- Return JSON that strictly matches VerdictDecision. +- verdict MUST be one of: confirmed, likely, inconclusive, not_exploitable. +- evidence_level MUST be integer 1..6. +- confidence MUST be one of: high, medium, low. + +CONSTRAINTS: +- Prioritize consistency with provided sub-agent evidence. +- Keep rationale concise and evidence-grounded. +- Do not return markdown, prose wrappers, or code fences. diff --git a/internal/secaf/agents/prove/tracer.go b/internal/secaf/agents/prove/tracer.go new file mode 100644 index 0000000000..bb3a822ef0 --- /dev/null +++ b/internal/secaf/agents/prove/tracer.go @@ -0,0 +1,117 @@ +package prove + +// Ports src/sec_af/agents/prove/tracer.py. + +import ( + "context" + "os" + "strconv" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/harnessx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// tracerPromptPath is Python's module-level +// `PROMPT_PATH = Path(__file__).resolve().parents[2] / "prompts" / "prove" / "tracer.txt"`, +// expressed as the embed-relative name internal/prompts uses. +const tracerPromptPath = "prove/tracer.txt" + +// Agent identity strings. Python spells the same sub-agent two ways and both +// are observable: agentName goes into the temp-dir prefix +// `secaf--`, extractName is what extract_harness_result prints and +// embeds in the error ("DataFlowTracer harness error: ..."). +const ( + tracerAgentName = "prove-tracer" + tracerExtractName = "DataFlowTracer" +) + +// findingDataFlow ports `_finding_data_flow`: +// +// if not finding.data_flow: +// return "[]" +// rows = [{"file_path": s.file_path, "line": s.line, +// "component": s.component, "operation": s.operation} for s in finding.data_flow] +// return json.dumps(rows, indent=2) +// +// Python parity: `if not finding.data_flow` is falsy for BOTH None and an empty +// list, which len(...)==0 covers for a Go slice. The row dicts are built with +// literal keys, so their json.dumps order is the literal order — reproduced +// with pyfmt.O (an insertion-ordered mapping); a Go map would sort them into +// component/file_path/line/operation. +func findingDataFlow(finding schemas.RawFinding) string { + if len(finding.DataFlow) == 0 { + return "[]" + } + rows := make([]pyfmt.Ordered, len(finding.DataFlow)) + for i, step := range finding.DataFlow { + rows[i] = pyfmt.O( + "file_path", step.FilePath, + "line", step.Line, + "component", step.Component, + "operation", step.Operation, + ) + } + return pyfmt.Dumps(rows, 2) +} + +// tracerBuildPrompt ports tracer.py `_build_prompt`. See replacement's doc for +// why the entries are an ORDERED slice. +func tracerBuildPrompt(template string, finding schemas.RawFinding, depth string) string { + return applyReplacements(template, []replacement{ + {"{{TITLE}}", finding.Title}, + {"{{DESCRIPTION}}", finding.Description}, + {"{{CWE_ID}}", finding.CweID}, + {"{{CWE_NAME}}", finding.CweName}, + {"{{FILE_PATH}}", finding.FilePath}, + {"{{START_LINE}}", strconv.Itoa(finding.StartLine)}, + {"{{CODE_SNIPPET}}", finding.CodeSnippet}, + {"{{FINDING_TYPE}}", string(finding.FindingType)}, + {"{{RELATED_FILES}}", relatedFilesJSON(finding.RelatedFiles)}, + {"{{DATA_FLOW_JSON}}", findingDataFlow(finding)}, + {"{{DEPTH}}", depth}, + }) +} + +// TracerPrompt builds the exact prompt RunTracer sends, including the CONTEXT +// block Python appends after substitution. Exported for the golden test. +func TracerPrompt(finding schemas.RawFinding, repoPath, depth string) string { + return tracerBuildPrompt(prompts.MustLoad(tracerPromptPath), finding, depth) + + "\n\nCONTEXT:\n" + + "- Repository path: " + repoPath + "\n" + + "- Use the repository path above for file inspection during source-to-sink tracing." +} + +// RunTracer ports tracer.py run_tracer. +// +// async def run_tracer(app, repo_path, finding, depth) -> DataFlowTrace: +// prompt = _build_prompt(...) + "\n\nCONTEXT:\n..." +// harness_cwd = tempfile.mkdtemp(prefix="secaf-prove-tracer-") +// try: +// result = await app.harness(prompt=prompt, schema=DataFlowTrace, +// cwd=harness_cwd, project_dir=repo_path) +// return extract_harness_result(result, DataFlowTrace, "DataFlowTracer") +// finally: +// shutil.rmtree(harness_cwd, ignore_errors=True) +// +// Python parity: the harness runs with Cwd set to a PRIVATE scratch directory +// and ProjectDir set to the repository, so the coding agent explores the repo +// but writes its JSON output outside it. `shutil.rmtree(..., ignore_errors=True)` +// maps to a deferred os.RemoveAll whose error is deliberately dropped. +func RunTracer(ctx context.Context, app appx.Harnesser, repoPath string, finding schemas.RawFinding, depth string) (schemas.DataFlowTrace, error) { + prompt := TracerPrompt(finding, repoPath, depth) + + harnessCwd, err := os.MkdirTemp("", "secaf-"+tracerAgentName+"-") + if err != nil { + return schemas.DataFlowTrace{}, err + } + defer os.RemoveAll(harnessCwd) + + return harnessx.RunExtract[schemas.DataFlowTrace]( + ctx, app, prompt, + appx.HarnessOptions{Cwd: harnessCwd, ProjectDir: repoPath}, + tracerExtractName, + ) +} diff --git a/internal/secaf/agents/prove/verdict.go b/internal/secaf/agents/prove/verdict.go new file mode 100644 index 0000000000..c151a85817 --- /dev/null +++ b/internal/secaf/agents/prove/verdict.go @@ -0,0 +1,151 @@ +package prove + +// Ports src/sec_af/agents/prove/verdict.py. + +import ( + "context" + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/aix" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// verdictPromptPath mirrors verdict.py's module-level PROMPT_PATH. +const verdictPromptPath = "prove/verdict.txt" + +// verdictExtractName is the name Python's `_extract_ai_result` puts in its +// TypeError ("VerdictAgent .ai() did not return a valid VerdictDecision: ..."). +// There is no temp dir here: VerdictAgent is a pure judgment task and uses +// `.ai()` (one structured LLM request) rather than `.harness()` (a multi-turn +// coding-agent session), so it never touches the filesystem. +const verdictExtractName = "VerdictAgent" + +// verdictBuildContext ports verdict.py `_build_context` — the +// {{SUBAGENT_CONTEXT}} block. +// +// trace_steps = "\n".join(f"- {step}" for step in data_flow.steps) if data_flow.steps else "- (none)" +// return ("Tracer output:\n" +// f"- source: {data_flow.source}\n" +// f"- sink: {data_flow.sink}\n" +// f"- sink_reached: {data_flow.sink_reached}\n" +// f"- steps:\n{trace_steps}\n\n" +// "Sanitization output:\n" +// f"- found: {sanitization.found}\n" +// f"- type: {sanitization.type or 'none'}\n" +// f"- sufficient: {sanitization.sufficient}\n" +// f"- bypass_method: {sanitization.bypass_method or 'none'}\n\n" +// "Exploit output:\n" +// f"- hypothesis: {exploit.hypothesis}\n" +// f"- payload: {exploit.payload or 'none'}\n" +// f"- expected_outcome: {exploit.expected_outcome}") +// +// This is deliberately NOT shared with `_trace_context` (shared.go): the empty +// placeholder is "- (none)" rather than "- (no concrete trace steps)", and the +// booleans are interpolated as Python bools ("True"/"False") rather than +// yes/no. `sufficient` is interpolated RAW, so None prints as "None" — the one +// place a tri-state reaches the model unmapped. +func verdictBuildContext( + dataFlow schemas.DataFlowTrace, + sanitization schemas.SanitizationResult, + exploit schemas.ExploitHypothesis, +) string { + traceSteps := "- (none)" + if len(dataFlow.Steps) > 0 { + parts := make([]string, len(dataFlow.Steps)) + for i, step := range dataFlow.Steps { + parts[i] = "- " + step + } + traceSteps = strings.Join(parts, "\n") + } + var b strings.Builder + b.WriteString("Tracer output:\n") + b.WriteString("- source: " + dataFlow.Source + "\n") + b.WriteString("- sink: " + dataFlow.Sink + "\n") + b.WriteString("- sink_reached: " + pyfmt.Str(dataFlow.SinkReached) + "\n") + b.WriteString("- steps:\n" + traceSteps + "\n\n") + b.WriteString("Sanitization output:\n") + b.WriteString("- found: " + pyfmt.Str(sanitization.Found) + "\n") + b.WriteString("- type: " + pyOr(sanitization.Type, "none") + "\n") + b.WriteString("- sufficient: " + pyStrOptBool(sanitization.Sufficient) + "\n") + b.WriteString("- bypass_method: " + pyOr(sanitization.BypassMethod, "none") + "\n\n") + b.WriteString("Exploit output:\n") + b.WriteString("- hypothesis: " + exploit.Hypothesis + "\n") + b.WriteString("- payload: " + pyOr(exploit.Payload, "none") + "\n") + b.WriteString("- expected_outcome: " + exploit.ExpectedOutcome) + return b.String() +} + +// verdictBuildPrompt ports verdict.py `_build_prompt`. +// +// Python parity: this builder has NO {{DEPTH}} entry — verdict.txt does not +// name one and run_verdict_agent takes no depth argument — so a literal +// "{{DEPTH}}" anywhere in the finding survives into the prompt. +func verdictBuildPrompt( + template string, + finding schemas.RawFinding, + dataFlow schemas.DataFlowTrace, + sanitization schemas.SanitizationResult, + exploit schemas.ExploitHypothesis, +) string { + return applyReplacements(template, []replacement{ + {"{{TITLE}}", finding.Title}, + {"{{DESCRIPTION}}", finding.Description}, + {"{{CWE_ID}}", finding.CweID}, + {"{{CWE_NAME}}", finding.CweName}, + {"{{FILE_PATH}}", finding.FilePath}, + {"{{START_LINE}}", strconv.Itoa(finding.StartLine)}, + {"{{CODE_SNIPPET}}", finding.CodeSnippet}, + {"{{FINDING_TYPE}}", string(finding.FindingType)}, + {"{{RELATED_FILES}}", relatedFilesJSON(finding.RelatedFiles)}, + {"{{SUBAGENT_CONTEXT}}", verdictBuildContext(dataFlow, sanitization, exploit)}, + }) +} + +// VerdictPrompt builds the exact prompt RunVerdictAgent sends. Unlike every +// other prove agent there is NO trailing CONTEXT block — verdict.py sends the +// substituted template alone. Exported for the golden test. +func VerdictPrompt( + finding schemas.RawFinding, + dataFlow schemas.DataFlowTrace, + sanitization schemas.SanitizationResult, + exploit schemas.ExploitHypothesis, +) string { + return verdictBuildPrompt(prompts.MustLoad(verdictPromptPath), finding, dataFlow, sanitization, exploit) +} + +// RunVerdictAgent ports verdict.py run_verdict_agent. +// +// result = await app.ai(user=prompt, schema=VerdictDecision) +// return _extract_ai_result(result, "VerdictAgent") +// +// Python parity notes: +// +// - `repo_path` is accepted and IGNORED (the comment in verdict.py explains +// why: a pure judgment task needs no file access). The parameter is kept so +// the Go call sites read like the Python ones and so reasoners/phases.py's +// `repo_path="."` has somewhere to go. +// - `system=` is not passed, so aix.Structured gets the empty system prompt +// and adds no system message — Python's `system=None`. +// - `_extract_ai_result`'s three duck-typed branches (already a +// VerdictDecision / a dict / `.parsed`) all collapse into aix.Structured's +// single "unmarshal the response text into T", because the Go SDK's +// ai.Response only ever carries text. A response that will not parse yields +// an error whose message names VerdictDecision, which is what the Python +// TypeError does. +func RunVerdictAgent( + ctx context.Context, + app appx.AIer, + repoPath string, + finding schemas.RawFinding, + dataFlow schemas.DataFlowTrace, + sanitization schemas.SanitizationResult, + exploit schemas.ExploitHypothesis, +) (schemas.VerdictDecision, error) { + _ = repoPath // Python parity: accepted, never used. + prompt := VerdictPrompt(finding, dataFlow, sanitization, exploit) + return aix.Structured[schemas.VerdictDecision](ctx, app, "", prompt) +} diff --git a/internal/secaf/agents/prove/verifier.go b/internal/secaf/agents/prove/verifier.go new file mode 100644 index 0000000000..99871935d2 --- /dev/null +++ b/internal/secaf/agents/prove/verifier.go @@ -0,0 +1,187 @@ +package prove + +// Ports src/sec_af/agents/prove/verifier.py — the per-finding verification +// chain and the demotion fallback the PROVE phase reaches for when it fails. + +import ( + "context" + "strconv" + "sync" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// Fallback ports verifier.py `fallback`: +// +// def fallback(finding, reason, *, drop_reason=None, original_verdict=None) -> VerifiedFinding: +// rationale = f"Verification incomplete: {reason}" +// if original_verdict: +// rationale = f"{rationale} (original verdict: {original_verdict})" +// tags = ["low_confidence"] if drop_reason else [] +// return VerifiedFinding(... verdict=INCONCLUSIVE, evidence_level=STATIC_MATCH ...) +// +// This is the demotion path: a finding whose verification could not complete +// stays in the report as INCONCLUSIVE / STATIC_MATCH with a zero score, rather +// than being dropped silently. reasoners/phases.py and orchestrator.py both +// call it, and tests/test_prove_phase_demotion.py asserts its observable shape +// (verdict "inconclusive", drop_reason preserved, "low_confidence" in tags). +// +// SIGNATURE PARITY. Python's two keyword-only arguments default to None, and +// both defaults are load-bearing: +// +// - `drop_reason` drives BOTH the drop_reason field and the "low_confidence" +// tag — passing None yields an EMPTY tag list, not a tag list containing "". +// - `original_verdict` is appended to the rationale only when TRUTHY, so an +// empty string behaves like None. +// +// Go spells "or None" as a nil *string; StrPtr (shared.go) is the call-site +// helper. Passing (nil, nil) is Python's `fallback(finding, reason)`. +func Fallback(finding schemas.RawFinding, reason string, dropReason, originalVerdict *string) schemas.VerifiedFinding { + rationale := "Verification incomplete: " + reason + if originalVerdict != nil && *originalVerdict != "" { + rationale = rationale + " (original verdict: " + *originalVerdict + ")" + } + tags := []string{} + if dropReason != nil && *dropReason != "" { + tags = []string{"low_confidence"} + } + return schemas.VerifiedFinding{ + ID: finding.ID, + Fingerprint: finding.Fingerprint, + Title: finding.Title, + Description: finding.Description, + FindingType: finding.FindingType, + CweID: finding.CweID, + CweName: finding.CweName, + OwaspCategory: finding.OwaspCategory, + Verdict: schemas.VerdictInconclusive, + EvidenceLevel: schemas.EvidenceLevelStaticMatch, + Rationale: rationale, + Severity: finding.EstimatedSeverity, + Tags: tags, + ExploitabilityScore: 0.0, + Location: locationOf(finding), + RelatedLocations: []schemas.Location{}, + Compliance: []schemas.ComplianceMapping{}, + ReproductionSteps: []schemas.ReproductionStep{}, + SarifRuleID: sarifRuleID(finding.FindingType, finding.CweName), + SarifSecuritySeverity: 0.0, + DropReason: dropReason, + } +} + +// RunVerifier ports verifier.py run_verifier — the four-stage in-process chain. +// +// seed_trace = DataFlowTrace(source=f"{file}:{line}", sink=function or file, +// steps=[...] if finding.data_flow else [], sink_reached=False) +// results = await asyncio.gather(run_tracer(...), run_sanitization_analyzer(...), +// return_exceptions=True) +// data_flow_trace = seed_trace if tracer failed else tracer_result +// sanitization = SanitizationResult(found=False, sufficient=None, bypass_method=None) +// if sanitization failed else sanitization_result +// exploit = await run_exploit_hypothesizer(...) # errors PROPAGATE +// verdict = await run_verdict_agent(...) # errors PROPAGATE +// verified = assemble_verified_finding(...) +// # backfill sarif_rule_id and reproduction_steps +// +// Concurrency parity: stages 1 and 2 run CONCURRENTLY with +// `return_exceptions=True`, so a failure in one does not cancel or fail the +// other — each has its own documented fallback value. Go uses a WaitGroup with +// two indexed result slots rather than errgroup precisely because no error must +// escape. Stages 3 and 4 are sequential awaits whose errors DO propagate; the +// caller (_run_parallel_verification) turns them into a demoted Fallback. +// +// Python parity on the sanitization seed: the note that the seed trace's +// `sink_reached` is False even when the hunter reported a flow is deliberate — +// only the tracer may claim the sink is reached. +func RunVerifier(ctx context.Context, app HarnessAIer, repoPath string, finding schemas.RawFinding, depth string) (schemas.VerifiedFinding, error) { + seedTrace := seedTraceFor(finding) + + var ( + wg sync.WaitGroup + tracerRes schemas.DataFlowTrace + tracerErr error + sanitizeRes schemas.SanitizationResult + sanitizeErr error + ) + wg.Add(2) + go func() { + defer wg.Done() + tracerRes, tracerErr = RunTracer(ctx, app, repoPath, finding, depth) + }() + go func() { + defer wg.Done() + sanitizeRes, sanitizeErr = RunSanitizationAnalyzer(ctx, app, repoPath, finding, seedTrace, depth) + }() + wg.Wait() + + dataFlowTrace := tracerRes + if tracerErr != nil { + dataFlowTrace = seedTrace + } + + sanitization := sanitizeRes + if sanitizeErr != nil { + // Python parity: the failure fallback is constructed explicitly with + // found=False and BOTH optionals None — not SanitizationResult()'s + // defaults, though they happen to be identical. + sanitization = schemas.SanitizationResult{Found: false, Sufficient: nil, BypassMethod: nil} + } + + exploit, err := RunExploitHypothesizer(ctx, app, repoPath, finding, dataFlowTrace, sanitization, depth) + if err != nil { + return schemas.VerifiedFinding{}, err + } + verdict, err := RunVerdictAgent(ctx, app, repoPath, finding, dataFlowTrace, sanitization, exploit) + if err != nil { + return schemas.VerifiedFinding{}, err + } + + verified := AssembleVerifiedFinding(finding, dataFlowTrace, sanitization, exploit, verdict) + + // Backfills. AssembleVerifiedFinding always sets a rule id, so the first + // branch is unreachable through this path; it is ported anyway because + // Python guards it and a future assembler change would rely on it. + if verified.SarifRuleID == "" { + verified.SarifRuleID = sarifRuleID(finding.FindingType, finding.CweName) + } + if len(verified.ReproductionSteps) == 0 && verified.Verdict != schemas.VerdictNotExploitable { + desc1 := "Review vulnerable code location and trace data flow to sink." + out1 := "Flow reaches sensitive sink without sufficient mitigation." + desc2 := "Craft payload from exploit_hypothesis and execute against target path." + out2 := "Observed effect aligns with expected exploit outcome." + verified.ReproductionSteps = []schemas.ReproductionStep{ + {Step: 1, Description: desc1, Command: nil, ExpectedOutput: &out1}, + {Step: 2, Description: desc2, Command: nil, ExpectedOutput: &out2}, + } + } + return verified, nil +} + +// seedTraceFor builds run_verifier's `seed_trace`: +// +// DataFlowTrace( +// source=f"{finding.file_path}:{finding.start_line}", +// sink=finding.function_name or finding.file_path, +// steps=[f"{s.file_path}:{s.line} {s.operation}" for s in finding.data_flow] if finding.data_flow else [], +// sink_reached=False, +// ) +// +// Python parity: `finding.function_name or finding.file_path` is truthiness, so +// an EMPTY function name also falls back to the file path. +func seedTraceFor(finding schemas.RawFinding) schemas.DataFlowTrace { + sink := finding.FilePath + if finding.FunctionName != nil && *finding.FunctionName != "" { + sink = *finding.FunctionName + } + steps := []string{} + for _, step := range finding.DataFlow { + steps = append(steps, step.FilePath+":"+strconv.Itoa(step.Line)+" "+step.Operation) + } + return schemas.DataFlowTrace{ + Source: finding.FilePath + ":" + strconv.Itoa(finding.StartLine), + Sink: sink, + Steps: steps, + SinkReached: false, + } +} diff --git a/internal/secaf/agents/recon/architecture.go b/internal/secaf/agents/recon/architecture.go new file mode 100644 index 0000000000..0b18fb4dac --- /dev/null +++ b/internal/secaf/agents/recon/architecture.go @@ -0,0 +1,117 @@ +package recon + +// Ports src/sec_af/agents/recon/architecture.py. + +import ( + "context" + "os" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/harnessx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// architecturePromptPath is Python's module-level +// `PROMPT_PATH = Path(__file__).resolve().parents[2] / "prompts" / "recon" / "architecture.txt"`, +// expressed as the embed-relative name internal/prompts uses. +const architecturePromptPath = "recon/architecture.txt" + +// Agent identity strings. Python spells the SAME agent two different ways and +// both are observable, so both are pinned here: +// +// - agentName goes into the temp-dir prefix `secaf--`; +// - extractName is what extract_harness_result prints and puts in the error +// ("Architecture mapper harness error: ..."). +const ( + architectureAgentName = "recon-architecture" + architectureExtractName = "Architecture mapper" +) + +// fileListingContextSuffix is the CONTEXT block the three repo-only mappers +// (architecture, dependencies, config scanner) append to their template +// verbatim. Python builds it with an f-string per module; the three copies are +// byte-identical apart from the interpolated path, so the port shares one +// builder. Any change here changes what reaches the LLM — it is golden-tested +// against the real Python builders (testdata/golden/*_prompt.txt). +func fileListingContextSuffix(repoPath string) string { + return "\n\nCONTEXT:\n" + + "- Repository path: " + repoPath + "\n" + + "- Start by listing files in the repository path above.\n" + + "- After gathering evidence, write the JSON output file using your Write tool." +} + +// architecturePrompt builds the exact prompt run_architecture_mapper sends. +func architecturePrompt(repoPath string) string { + return prompts.MustLoad(architecturePromptPath) + fileListingContextSuffix(repoPath) +} + +// RunArchitectureMapper ports architecture.py run_architecture_mapper. +// +// async def run_architecture_mapper(app, repo_path) -> ArchitectureMap: +// prompt = PROMPT_PATH.read_text(...) + "\n\nCONTEXT:\n" + ... +// harness_cwd = tempfile.mkdtemp(prefix="secaf-recon-architecture-") +// try: +// result = await app.harness(prompt=prompt, schema=ArchitectureMapRaw, +// cwd=harness_cwd, project_dir=repo_path) +// raw = extract_harness_result(result, ArchitectureMapRaw, "Architecture mapper") +// return parse_architecture_raw(raw) +// finally: +// shutil.rmtree(harness_cwd, ignore_errors=True) +// +// Python parity: the harness runs with Cwd set to a PRIVATE scratch directory +// and ProjectDir set to the repository, so the coding agent explores the repo +// but writes its JSON output outside it. `shutil.rmtree(..., ignore_errors=True)` +// maps to a deferred os.RemoveAll whose error is deliberately dropped. +// +// On any failure the zero ArchitectureMap is returned alongside the error; +// Python raises, so no caller reads the value in that case. +func RunArchitectureMapper(ctx context.Context, app appx.Harnesser, repoPath string) (schemas.ArchitectureMap, error) { + prompt := architecturePrompt(repoPath) + + harnessCwd, err := os.MkdirTemp("", "secaf-"+architectureAgentName+"-") + if err != nil { + return schemas.ArchitectureMap{}, err + } + defer os.RemoveAll(harnessCwd) + + raw, err := harnessx.RunExtract[schemas.ArchitectureMapRaw]( + ctx, app, prompt, + appx.HarnessOptions{Cwd: harnessCwd, ProjectDir: repoPath}, + architectureExtractName, + ) + if err != nil { + return schemas.ArchitectureMap{}, err + } + return ParseArchitectureRaw(raw), nil +} + +// ArchitectureContextBlock ports architecture.py architecture_context_block: +// +// def architecture_context_block(architecture: ArchitectureMap) -> str: +// return json.dumps(architecture.model_dump(), indent=2) +// +// The result is substituted for `{{ARCHITECTURE_MAP_JSON}}` in the data-flow +// and security-context prompt templates, so it reaches the LLM verbatim and +// must be byte-identical to CPython's json.dumps — which Go's encoding/json is +// NOT (it escapes `<`, `>` and `&`, leaves non-ASCII unescaped where CPython's +// ensure_ascii=True writes \uXXXX, and renders floats with Go's rules rather +// than repr()). pyfmt.Dumps is the shared encoder that closes all of those; see +// DESIGN.md §2b. +// +// Struct fields are emitted in DECLARATION order, which is the order the +// pydantic class declares them and therefore the order model_dump() inserts +// them, so no key-order fixup is needed. +// +// Python parity caveat inherited from pyfmt.Dumps: a NIL Go slice renders as +// `null`, while a pydantic `Field(default_factory=list)` always dumps as `[]`. +// Every ArchitectureMap that reaches this function came from +// ParseArchitectureRaw (which builds non-nil slices) or from JSON via +// ArchitectureMap.UnmarshalJSON (which seeds the `[]` defaults), so the nil +// case is not reachable through the pipeline — construct with +// schemas.NewArchitectureMap() rather than a bare literal if you ever call this +// by hand. +func ArchitectureContextBlock(architecture schemas.ArchitectureMap) string { + return pyfmt.Dumps(architecture, 2) +} diff --git a/internal/secaf/agents/recon/config_scanner.go b/internal/secaf/agents/recon/config_scanner.go new file mode 100644 index 0000000000..06c3370971 --- /dev/null +++ b/internal/secaf/agents/recon/config_scanner.go @@ -0,0 +1,51 @@ +package recon + +// Ports src/sec_af/agents/recon/config_scanner.py. + +import ( + "context" + "os" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/harnessx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +const ( + configScannerPromptPath = "recon/config_scanner.txt" + configScannerAgentName = "recon-config-scanner" + configScannerExtractName = "Config scanner" +) + +// configScannerPrompt builds the exact prompt run_config_scanner sends. It +// shares the CONTEXT suffix with the architecture and dependency mappers (see +// fileListingContextSuffix). +func configScannerPrompt(repoPath string) string { + return prompts.MustLoad(configScannerPromptPath) + fileListingContextSuffix(repoPath) +} + +// RunConfigScanner ports config_scanner.py run_config_scanner: harness for a +// ConfigReportRaw, then parse_config_report_raw. +// +// Structurally identical to RunArchitectureMapper — see its doc comment for the +// temp-dir / Cwd / ProjectDir contract, which is the same here. +func RunConfigScanner(ctx context.Context, app appx.Harnesser, repoPath string) (schemas.ConfigReport, error) { + prompt := configScannerPrompt(repoPath) + + harnessCwd, err := os.MkdirTemp("", "secaf-"+configScannerAgentName+"-") + if err != nil { + return schemas.ConfigReport{}, err + } + defer os.RemoveAll(harnessCwd) + + raw, err := harnessx.RunExtract[schemas.ConfigReportRaw]( + ctx, app, prompt, + appx.HarnessOptions{Cwd: harnessCwd, ProjectDir: repoPath}, + configScannerExtractName, + ) + if err != nil { + return schemas.ConfigReport{}, err + } + return ParseConfigReportRaw(raw), nil +} diff --git a/internal/secaf/agents/recon/data_flow.go b/internal/secaf/agents/recon/data_flow.go new file mode 100644 index 0000000000..ef01e1b0be --- /dev/null +++ b/internal/secaf/agents/recon/data_flow.go @@ -0,0 +1,76 @@ +package recon + +// Ports src/sec_af/agents/recon/data_flow.py. + +import ( + "context" + "os" + "strings" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/harnessx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +const ( + dataFlowPromptPath = "recon/data_flow.txt" + dataFlowAgentName = "recon-data-flow" + dataFlowExtractName = "Data flow mapper" + + // architectureMapPlaceholder is the token the two architecture-aware + // templates carry; Python replaces it with architecture_context_block(...) + // before appending the CONTEXT block. + architectureMapPlaceholder = "{{ARCHITECTURE_MAP_JSON}}" +) + +// explorationContextSuffix is the CONTEXT block the two architecture-aware +// mappers (data flow, security context) append. It differs from the +// file-listing suffix the other three use: those tell the agent to start by +// listing files, these tell it to explore across multiple turns first. +func explorationContextSuffix(repoPath string) string { + return "\n\nCONTEXT:\n" + + "- Repository path: " + repoPath + "\n" + + "- Take multiple turns to explore the codebase first, then build your analysis.\n" + + "- Write final JSON only when analysis is complete." +} + +// dataFlowPrompt builds the exact prompt run_data_flow_mapper sends. +// +// Python parity: `str.replace(old, new)` with no count replaces EVERY +// occurrence of the placeholder, and the substitution happens BEFORE the +// CONTEXT block is appended — so an architecture map that itself contained the +// placeholder text would not be re-substituted. strings.ReplaceAll matches on +// both counts. +func dataFlowPrompt(repoPath string, architecture schemas.ArchitectureMap) string { + template := prompts.MustLoad(dataFlowPromptPath) + return strings.ReplaceAll(template, architectureMapPlaceholder, ArchitectureContextBlock(architecture)) + + explorationContextSuffix(repoPath) +} + +// RunDataFlowMapper ports data_flow.py run_data_flow_mapper: substitute the +// architecture map into the template, harness for a DataFlowMapRaw, then +// parse_data_flow_raw. +// +// The architecture argument is the ArchitectureMap the architecture mapper +// produced earlier in the same RECON phase; run_recon only reaches this mapper +// after that gather has completed. +func RunDataFlowMapper(ctx context.Context, app appx.Harnesser, repoPath string, architecture schemas.ArchitectureMap) (schemas.DataFlowMap, error) { + prompt := dataFlowPrompt(repoPath, architecture) + + harnessCwd, err := os.MkdirTemp("", "secaf-"+dataFlowAgentName+"-") + if err != nil { + return schemas.DataFlowMap{}, err + } + defer os.RemoveAll(harnessCwd) + + raw, err := harnessx.RunExtract[schemas.DataFlowMapRaw]( + ctx, app, prompt, + appx.HarnessOptions{Cwd: harnessCwd, ProjectDir: repoPath}, + dataFlowExtractName, + ) + if err != nil { + return schemas.DataFlowMap{}, err + } + return ParseDataFlowRaw(raw), nil +} diff --git a/internal/secaf/agents/recon/dependencies.go b/internal/secaf/agents/recon/dependencies.go new file mode 100644 index 0000000000..af94395155 --- /dev/null +++ b/internal/secaf/agents/recon/dependencies.go @@ -0,0 +1,51 @@ +package recon + +// Ports src/sec_af/agents/recon/dependencies.py. + +import ( + "context" + "os" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/harnessx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +const ( + dependenciesPromptPath = "recon/dependencies.txt" + dependenciesAgentName = "recon-dependencies" + dependenciesExtractName = "Dependency auditor" +) + +// dependenciesPrompt builds the exact prompt run_dependency_auditor sends. +// It shares the CONTEXT suffix with the architecture and config-scanner +// mappers (see fileListingContextSuffix). +func dependenciesPrompt(repoPath string) string { + return prompts.MustLoad(dependenciesPromptPath) + fileListingContextSuffix(repoPath) +} + +// RunDependencyAuditor ports dependencies.py run_dependency_auditor: harness +// for a DependencyReportRaw, then parse_dependency_report_raw. +// +// Structurally identical to RunArchitectureMapper — see its doc comment for the +// temp-dir / Cwd / ProjectDir contract, which is the same here. +func RunDependencyAuditor(ctx context.Context, app appx.Harnesser, repoPath string) (schemas.DependencyReport, error) { + prompt := dependenciesPrompt(repoPath) + + harnessCwd, err := os.MkdirTemp("", "secaf-"+dependenciesAgentName+"-") + if err != nil { + return schemas.DependencyReport{}, err + } + defer os.RemoveAll(harnessCwd) + + raw, err := harnessx.RunExtract[schemas.DependencyReportRaw]( + ctx, app, prompt, + appx.HarnessOptions{Cwd: harnessCwd, ProjectDir: repoPath}, + dependenciesExtractName, + ) + if err != nil { + return schemas.DependencyReport{}, err + } + return ParseDependencyReportRaw(raw), nil +} diff --git a/internal/secaf/agents/recon/doc.go b/internal/secaf/agents/recon/doc.go new file mode 100644 index 0000000000..c26f0d6c95 --- /dev/null +++ b/internal/secaf/agents/recon/doc.go @@ -0,0 +1,33 @@ +// Package recon ports src/sec_af/agents/recon — the RECON phase of the SEC-AF +// pipeline. +// +// RECON runs five independent harness "mappers" over a repository and folds +// their flat, pipe-delimited output into the structured schemas the HUNT and +// PROVE phases consume: +// +// run_architecture_mapper -> schemas.ArchitectureMap +// run_dependency_auditor -> schemas.DependencyReport +// run_config_scanner -> schemas.ConfigReport +// run_data_flow_mapper -> schemas.DataFlowMap (needs the architecture) +// run_security_context_profiler -> schemas.SecurityContext (needs the architecture) +// +// Python module -> Go file: +// +// agents/recon/architecture.py -> architecture.go +// agents/recon/dependencies.py -> dependencies.go +// agents/recon/config_scanner.py -> config_scanner.go +// agents/recon/data_flow.py -> data_flow.go +// agents/recon/security_context.py -> security_context.go +// agents/recon/_parsers.py -> parsers.go +// agents/recon/__init__.py -> recon.go (run_recon / run_fast_recon / +// run_deep_recon), metrics.go (_repo_metrics), +// findings.go (extract_recon_findings) +// +// Every mapper follows one shape, so the port does too: build the prompt +// (template + a literal CONTEXT suffix), make a private temp dir named +// `secaf--*`, run the harness with `Cwd` = that temp dir and +// `ProjectDir` = the repository, extract the `*Raw` model, parse it, and remove +// the temp dir. The prompt strings are byte-verbatim; the golden fixtures under +// testdata/golden are produced from the REAL Python builders by +// go/scripts/gen_golden.py. +package recon diff --git a/internal/secaf/agents/recon/findings.go b/internal/secaf/agents/recon/findings.go new file mode 100644 index 0000000000..cf3407085b --- /dev/null +++ b/internal/secaf/agents/recon/findings.go @@ -0,0 +1,230 @@ +package recon + +// Ports the RawFinding-extraction half of src/sec_af/agents/recon/__init__.py: +// _safe_line, _to_recon_finding, _extract_from_config, _extract_weak_tls, +// _extract_structured_security_items and extract_recon_findings. +// +// RECON is not a hunter, but two of its mappers already produce concrete +// vulnerabilities (hardcoded secrets, insecure config, weak transport crypto). +// These helpers lift those into the same RawFinding currency the HUNT phase +// speaks so the orchestrator can merge them into the hunt result. + +import ( + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// safeLine ports `_safe_line(value, default=1)` — "use the reported line only +// when it is a positive int, otherwise 1". The two call sites differ in the +// static type of `value` (SecretFinding.line is `int`, MisconfigFinding.line is +// `int | None`), so the port has one helper per shape. +func safeLine(value int) int { + if value > 0 { + return value + } + return 1 +} + +// safeLinePtr is safeLine for an `int | None` field: nil is not an int, so +// Python's `isinstance(value, int)` guard fails and the default wins. +func safeLinePtr(value *int) int { + if value != nil && *value > 0 { + return *value + } + return 1 +} + +// toReconFinding ports _to_recon_finding — the single RawFinding factory every +// RECON extraction goes through. +// +// Python parity: end_line is always start_line (RECON reports points, not +// ranges), confidence is always HIGH, hunter_strategy is always the literal +// "recon", and id/fingerprint get fresh uuid4s from schemas.NewRawFinding — so +// two extractions of the same input are never deeply equal. +func toReconFinding( + title, description string, + findingType schemas.FindingType, + cweID, cweName, owaspCategory string, + severity schemas.Severity, + filePath string, + startLine int, + codeSnippet string, +) schemas.RawFinding { + f := schemas.NewRawFinding() + f.HunterStrategy = "recon" + f.Title = title + f.Description = description + f.FindingType = findingType + f.CweID = cweID + f.CweName = cweName + owasp := owaspCategory + f.OwaspCategory = &owasp + f.FilePath = filePath + f.StartLine = startLine + f.EndLine = startLine + f.CodeSnippet = codeSnippet + f.EstimatedSeverity = severity + f.Confidence = schemas.ConfidenceHigh + return f +} + +// extractFromConfig ports _extract_from_config: every secret becomes a +// CWE-798 HIGH finding and every misconfiguration a CWE-16 MEDIUM one. +// +// Python parity: the misconfig snippet is `", ".join(item for item in [key, +// value] if item)` — a TRUTHINESS filter, so an empty-string key or value is +// dropped just like None — and falls back to the risk text when both are +// absent. +func extractFromConfig(config schemas.ConfigReport) []schemas.RawFinding { + findings := []schemas.RawFinding{} + + for _, secret := range config.Secrets { + line := safeLine(secret.Line) + location := secret.FilePath + ":" + strconv.Itoa(line) + findings = append(findings, toReconFinding( + "Hardcoded secret in "+secret.FilePath, + "Detected "+secret.SecretType+" secret at "+location+". "+ + "Data flow summary: hardcoded credential from source file can be reused by an attacker.", + schemas.FindingTypeSecrets, + "CWE-798", + "Use of Hard-coded Credentials", + "A07:2021", + schemas.SeverityHigh, + secret.FilePath, + line, + secret.Match, + )) + } + + for _, misconfig := range config.Misconfigs { + line := safeLinePtr(misconfig.Line) + var details []string + if misconfig.Key != nil && *misconfig.Key != "" { + details = append(details, *misconfig.Key) + } + if misconfig.Value != nil && *misconfig.Value != "" { + details = append(details, *misconfig.Value) + } + snippet := strings.Join(details, ", ") + if snippet == "" { + snippet = misconfig.Risk + } + findings = append(findings, toReconFinding( + "Insecure configuration in "+misconfig.FilePath, + "Detected "+misconfig.Category+" with risk: "+misconfig.Risk+". "+ + "Data flow summary: insecure runtime configuration weakens application security controls.", + schemas.FindingTypeConfig, + "CWE-16", + "Configuration", + "A05:2021", + schemas.SeverityMedium, + misconfig.FilePath, + line, + snippet, + )) + } + + return findings +} + +// extractWeakTLS ports _extract_weak_tls: a CWE-327 MEDIUM finding per crypto +// usage that is BOTH flagged weak and transport-related. +// +// Python parity, all three of which are load-bearing: +// +// - `if usage.is_weak is not True: continue` is an identity test, so only an +// explicit true qualifies — `None` (unknown) does not. +// - `algorithm.strip() if usage.algorithm else "unknown"` substitutes the +// literal "unknown" for an EMPTY algorithm, and that substitute is then +// what the "tls"/"ssl" substring test runs against. +// - the transport test is an OR across three lowercased haystacks: "tls" or +// "ssl" in the algorithm, or "tls" in the usage context. "ssl" in the usage +// context alone does NOT qualify. +// +// file_path is the literal "security_context" and start_line is 1, because a +// SecurityContext entry has no source location. +func extractWeakTLS(context schemas.SecurityContext) []schemas.RawFinding { + findings := []schemas.RawFinding{} + + for _, usage := range context.CryptoUsage { + if usage.IsWeak == nil || !*usage.IsWeak { + continue + } + algorithm := "unknown" + if usage.Algorithm != "" { + algorithm = pyStrip(usage.Algorithm) + } + usageContext := "security context" + if usage.UsageContext != nil && *usage.UsageContext != "" { + usageContext = *usage.UsageContext + } + lowerAlgorithm := strings.ToLower(algorithm) + if !strings.Contains(lowerAlgorithm, "tls") && + !strings.Contains(lowerAlgorithm, "ssl") && + !strings.Contains(strings.ToLower(usageContext), "tls") { + continue + } + findings = append(findings, toReconFinding( + "Weak TLS configuration: "+algorithm, + "Detected weak transport crypto usage in "+usageContext+". "+ + "Data flow summary: clients may negotiate weak encryption for in-transit data.", + schemas.FindingTypeConfig, + "CWE-327", + "Use of a Broken or Risky Cryptographic Algorithm", + "A02:2021", + schemas.SeverityMedium, + "security_context", + 1, + "algorithm="+algorithm+"; context="+usageContext, + )) + } + + return findings +} + +// extractStructuredSecurityItems ports _extract_structured_security_items, +// which in Python iterates four categories — hardcoded_secrets, +// dangerous_configs, weak_tls, exposed_endpoints — pulled off the +// SecurityContext with `getattr(context, category, None)`. +// +// Python parity: it always returns []. schemas/recon.py's SecurityContext +// declares NONE of those four attributes and sets no model_config, so pydantic +// v2's default `extra="ignore"` drops them at validation time and +// `getattr(context, category, None)` returns None for every category. The very +// next line is `if not isinstance(items_obj, list): continue`, so every +// iteration short-circuits. This was verified against the pinned interpreter: +// validating a SecurityContext with all four keys present still yields None for +// each getattr, and tests/test_recon_findings.py's expected count of 3 +// (1 secret + 1 misconfig + 1 weak TLS) confirms this contributes nothing. +// +// Go's SecurityContext is a static struct with the same six fields, so the +// dead branch is not merely unused — it is unrepresentable. The function is +// kept so the call order in ExtractReconFindings still reads 1:1 against the +// Python, and so this analysis has somewhere to live. `_safe_path`, which only +// that dead branch calls, is deliberately not ported. +// +// The one way Python could reach the live branch is an UNVALIDATED assignment +// (`recon.security_context = `); pydantic v2 +// leaves attribute assignment unchecked by default. No SEC-AF code does that. +func extractStructuredSecurityItems(_ schemas.SecurityContext) []schemas.RawFinding { + return []schemas.RawFinding{} +} + +// ExtractReconFindings ports extract_recon_findings: +// +// findings.extend(_extract_from_config(recon.config)) +// findings.extend(_extract_structured_security_items(recon.security_context)) +// findings.extend(_extract_weak_tls(recon.security_context)) +// +// Order matters — the orchestrator prepends this list to the hunt findings, so +// config secrets lead, then misconfigurations, then weak TLS. Always non-nil so +// it serializes as `[]`. +func ExtractReconFindings(recon schemas.ReconResult) []schemas.RawFinding { + findings := []schemas.RawFinding{} + findings = append(findings, extractFromConfig(recon.Config)...) + findings = append(findings, extractStructuredSecurityItems(recon.SecurityContext)...) + findings = append(findings, extractWeakTLS(recon.SecurityContext)...) + return findings +} diff --git a/internal/secaf/agents/recon/findings_test.go b/internal/secaf/agents/recon/findings_test.go new file mode 100644 index 0000000000..75d9691dd0 --- /dev/null +++ b/internal/secaf/agents/recon/findings_test.go @@ -0,0 +1,393 @@ +package recon + +// Ports tests/test_recon_findings.py::test_extract_recon_findings_builds_raw_findings_from_recon_detections +// plus the behaviors that test exercises only implicitly. +// +// The file's OTHER test — test_merge_recon_findings_prepends_and_updates_counts +// — exercises sec_af.orchestrator.merge_recon_findings_into_hunt, which lives +// in internal/orch and is ported there, not here. +// +// Validation contract for extract_recon_findings: +// +// - Every ConfigReport secret becomes one CWE-798 / A07:2021 HIGH SECRETS +// finding whose snippet is the matched text. +// - Every ConfigReport misconfiguration becomes one CWE-16 / A05:2021 MEDIUM +// CONFIG finding whose snippet is "key, value" (empty parts dropped), +// falling back to the risk text. +// - Every crypto usage flagged weak AND transport-related becomes one +// CWE-327 / A02:2021 MEDIUM CONFIG finding located at "security_context:1". +// - Every finding carries hunter_strategy "recon", confidence HIGH and +// end_line == start_line. +// - The four "structured security items" categories contribute nothing. +// - Order is: config secrets, config misconfigurations, weak TLS. + +import ( + "testing" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// minimalReconResult ports the test module's _minimal_recon_result(): a +// ReconResult validated from a sparse dict, so every absent nested model falls +// back to its pydantic defaults. +func minimalReconResult() schemas.ReconResult { + recon := schemas.NewReconResult() + recon.SecurityContext.AuthModel = "jwt" + recon.SecurityContext.AuthDetails = "Bearer token" + weak := true + usageContext := "legacy tls terminator" + recon.SecurityContext.CryptoUsage = []schemas.CryptoUsage{ + {Algorithm: "TLSv1.0", UsageContext: &usageContext, IsWeak: &weak}, + } + return recon +} + +// TestExtractReconFindingsBuildsRawFindingsFromReconDetections ports +// test_extract_recon_findings_builds_raw_findings_from_recon_detections +// assertion for assertion. +func TestExtractReconFindingsBuildsRawFindingsFromReconDetections(t *testing.T) { + str := func(s string) *string { return &s } + intp := func(i int) *int { return &i } + boolp := func(b bool) *bool { return &b } + + recon := minimalReconResult() + + secret := schemas.NewSecretFinding() + secret.SecretType = "api_key" + secret.FilePath = "src/config.py" + secret.Line = 7 + secret.Match = `API_KEY = "sk-live-123"` + secret.Confidence = "high" + + misconfig := schemas.NewMisconfigFinding() + misconfig.Category = "dangerous_config" + misconfig.FilePath = "deploy/prod.yaml" + misconfig.Line = intp(22) + misconfig.Key = str("DEBUG") + misconfig.Value = str("true") + misconfig.Risk = "Debug mode enabled in production" + + recon.Config = schemas.ConfigReport{ + Secrets: []schemas.SecretFinding{secret}, + Misconfigs: []schemas.MisconfigFinding{misconfig}, + } + + recon.SecurityContext = schemas.NewSecurityContext() + recon.SecurityContext.AuthModel = "jwt" + recon.SecurityContext.AuthDetails = "Bearer token" + recon.SecurityContext.CryptoUsage = []schemas.CryptoUsage{ + {Algorithm: "TLSv1.0", UsageContext: str("public edge"), IsWeak: boolp(true)}, + } + + findings := ExtractReconFindings(recon) + + // assert len(findings) == 3 + if len(findings) != 3 { + t.Fatalf("len(findings) = %d, want 3", len(findings)) + } + + // assert {f.hunter_strategy for f in findings} == {"recon"} + for i, f := range findings { + if f.HunterStrategy != "recon" { + t.Errorf("findings[%d].hunter_strategy = %q, want %q", i, f.HunterStrategy, "recon") + } + // assert {f.confidence for f in findings} == {Confidence.HIGH} + if f.Confidence != schemas.ConfidenceHigh { + t.Errorf("findings[%d].confidence = %q, want %q", i, f.Confidence, schemas.ConfidenceHigh) + } + // finding_type is one of {SECRETS, CONFIG} + if f.FindingType != schemas.FindingTypeSecrets && f.FindingType != schemas.FindingTypeConfig { + t.Errorf("findings[%d].finding_type = %q, want secrets or config", i, f.FindingType) + } + if f.EndLine != f.StartLine { + t.Errorf("findings[%d] end_line %d != start_line %d", i, f.EndLine, f.StartLine) + } + } + + types := map[schemas.FindingType]bool{} + for _, f := range findings { + types[f.FindingType] = true + } + if !types[schemas.FindingTypeSecrets] || !types[schemas.FindingTypeConfig] || len(types) != 2 { + t.Errorf("finding types = %v, want exactly {secrets, config}", types) + } + + // assert any(f.cwe_id == "CWE-798" and f.estimated_severity == Severity.HIGH ...) + if !anyFinding(findings, func(f schemas.RawFinding) bool { + return f.CweID == "CWE-798" && f.EstimatedSeverity == schemas.SeverityHigh + }) { + t.Error("no CWE-798 HIGH finding") + } + // assert any(f.cwe_id == "CWE-16" and f.estimated_severity == Severity.MEDIUM ...) + if !anyFinding(findings, func(f schemas.RawFinding) bool { + return f.CweID == "CWE-16" && f.EstimatedSeverity == schemas.SeverityMedium + }) { + t.Error("no CWE-16 MEDIUM finding") + } + // assert any(f.cwe_id == "CWE-327" and f.file_path == "security_context" ...) + if !anyFinding(findings, func(f schemas.RawFinding) bool { + return f.CweID == "CWE-327" && f.FilePath == "security_context" + }) { + t.Error("no CWE-327 finding located at security_context") + } +} + +// TestExtractReconFindingsFieldsAndOrder pins the exact strings the extraction +// builds — the descriptions and snippets are what the HUNT/PROVE prompts see — +// and the config-secrets / config-misconfigs / weak-TLS ordering. +func TestExtractReconFindingsFieldsAndOrder(t *testing.T) { + str := func(s string) *string { return &s } + intp := func(i int) *int { return &i } + boolp := func(b bool) *bool { return &b } + + recon := schemas.NewReconResult() + + secret := schemas.NewSecretFinding() + secret.SecretType = "api_key" + secret.FilePath = "src/config.py" + secret.Line = 7 + secret.Match = `API_KEY = "sk-live-123"` + secret.Confidence = "high" + + misconfig := schemas.NewMisconfigFinding() + misconfig.Category = "dangerous_config" + misconfig.FilePath = "deploy/prod.yaml" + misconfig.Line = intp(22) + misconfig.Key = str("DEBUG") + misconfig.Value = str("true") + misconfig.Risk = "Debug mode enabled in production" + + recon.Config = schemas.ConfigReport{ + Secrets: []schemas.SecretFinding{secret}, + Misconfigs: []schemas.MisconfigFinding{misconfig}, + } + recon.SecurityContext.CryptoUsage = []schemas.CryptoUsage{ + {Algorithm: "TLSv1.0", UsageContext: str("public edge"), IsWeak: boolp(true)}, + } + + got := ExtractReconFindings(recon) + if len(got) != 3 { + t.Fatalf("len = %d, want 3", len(got)) + } + + // [0] the secret + wantTitle := "Hardcoded secret in src/config.py" + wantDesc := "Detected api_key secret at src/config.py:7. " + + "Data flow summary: hardcoded credential from source file can be reused by an attacker." + if got[0].Title != wantTitle { + t.Errorf("findings[0].title = %q, want %q", got[0].Title, wantTitle) + } + if got[0].Description != wantDesc { + t.Errorf("findings[0].description = %q, want %q", got[0].Description, wantDesc) + } + if got[0].CodeSnippet != `API_KEY = "sk-live-123"` { + t.Errorf("findings[0].code_snippet = %q", got[0].CodeSnippet) + } + if got[0].OwaspCategory == nil || *got[0].OwaspCategory != "A07:2021" { + t.Errorf("findings[0].owasp_category = %v, want A07:2021", got[0].OwaspCategory) + } + if got[0].StartLine != 7 || got[0].EndLine != 7 { + t.Errorf("findings[0] lines = (%d, %d), want (7, 7)", got[0].StartLine, got[0].EndLine) + } + + // [1] the misconfiguration — snippet is "key, value" + wantMisDesc := "Detected dangerous_config with risk: Debug mode enabled in production. " + + "Data flow summary: insecure runtime configuration weakens application security controls." + if got[1].Title != "Insecure configuration in deploy/prod.yaml" { + t.Errorf("findings[1].title = %q", got[1].Title) + } + if got[1].Description != wantMisDesc { + t.Errorf("findings[1].description = %q, want %q", got[1].Description, wantMisDesc) + } + if got[1].CodeSnippet != "DEBUG, true" { + t.Errorf("findings[1].code_snippet = %q, want %q", got[1].CodeSnippet, "DEBUG, true") + } + + // [2] weak TLS + if got[2].Title != "Weak TLS configuration: TLSv1.0" { + t.Errorf("findings[2].title = %q", got[2].Title) + } + wantTLSDesc := "Detected weak transport crypto usage in public edge. " + + "Data flow summary: clients may negotiate weak encryption for in-transit data." + if got[2].Description != wantTLSDesc { + t.Errorf("findings[2].description = %q, want %q", got[2].Description, wantTLSDesc) + } + if got[2].CodeSnippet != "algorithm=TLSv1.0; context=public edge" { + t.Errorf("findings[2].code_snippet = %q", got[2].CodeSnippet) + } + if got[2].StartLine != 1 || got[2].EndLine != 1 { + t.Errorf("findings[2] lines = (%d, %d), want (1, 1)", got[2].StartLine, got[2].EndLine) + } +} + +// TestSafeLineFallbacks pins _safe_line: a non-positive or absent line becomes +// 1, so a finding always points somewhere. +func TestSafeLineFallbacks(t *testing.T) { + recon := schemas.NewReconResult() + + zeroLineSecret := schemas.NewSecretFinding() + zeroLineSecret.SecretType = "token" + zeroLineSecret.FilePath = "a.py" + zeroLineSecret.Line = 0 + zeroLineSecret.Match = "tok" + zeroLineSecret.Confidence = "low" + + noLineMisconfig := schemas.NewMisconfigFinding() + noLineMisconfig.Category = "cors" + noLineMisconfig.FilePath = "nginx.conf" + noLineMisconfig.Line = nil + noLineMisconfig.Risk = "Wildcard origin" + + recon.Config = schemas.ConfigReport{ + Secrets: []schemas.SecretFinding{zeroLineSecret}, + Misconfigs: []schemas.MisconfigFinding{noLineMisconfig}, + } + + got := ExtractReconFindings(recon) + if len(got) != 2 { + t.Fatalf("len = %d, want 2", len(got)) + } + if got[0].StartLine != 1 { + t.Errorf("secret with line 0 -> start_line %d, want 1", got[0].StartLine) + } + if got[1].StartLine != 1 { + t.Errorf("misconfig with line nil -> start_line %d, want 1", got[1].StartLine) + } + // Both key and value absent -> the snippet falls back to the risk text. + if got[1].CodeSnippet != "Wildcard origin" { + t.Errorf("misconfig snippet = %q, want the risk text", got[1].CodeSnippet) + } +} + +// TestExtractWeakTLSGate pins the three-clause filter: only an EXPLICIT +// is_weak==true qualifies, and the usage must look transport-related through +// the algorithm ("tls"/"ssl") or the usage context ("tls" only). +func TestExtractWeakTLSGate(t *testing.T) { + str := func(s string) *string { return &s } + boolp := func(b bool) *bool { return &b } + + cases := []struct { + name string + usage schemas.CryptoUsage + want bool + }{ + {"weak_tls_algorithm", schemas.CryptoUsage{Algorithm: "TLSv1.0", UsageContext: str("edge"), IsWeak: boolp(true)}, true}, + {"weak_ssl_algorithm", schemas.CryptoUsage{Algorithm: "SSLv3", UsageContext: str("edge"), IsWeak: boolp(true)}, true}, + {"weak_tls_context", schemas.CryptoUsage{Algorithm: "RC4", UsageContext: str("legacy TLS terminator"), IsWeak: boolp(true)}, true}, + {"weak_ssl_context_only", schemas.CryptoUsage{Algorithm: "RC4", UsageContext: str("ssl offload"), IsWeak: boolp(true)}, false}, + {"weak_but_not_transport", schemas.CryptoUsage{Algorithm: "MD5", UsageContext: str("password hashing"), IsWeak: boolp(true)}, false}, + {"transport_but_not_weak", schemas.CryptoUsage{Algorithm: "TLSv1.3", UsageContext: str("edge"), IsWeak: boolp(false)}, false}, + {"transport_weak_unknown", schemas.CryptoUsage{Algorithm: "TLSv1.0", UsageContext: str("edge"), IsWeak: nil}, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx := schemas.NewSecurityContext() + ctx.CryptoUsage = []schemas.CryptoUsage{tc.usage} + got := extractWeakTLS(ctx) + if (len(got) == 1) != tc.want { + t.Errorf("extractWeakTLS produced %d findings, want %v", len(got), tc.want) + } + }) + } +} + +// TestExtractWeakTLSUnknownSubstitutions pins that an EMPTY algorithm becomes +// the literal "unknown" (which is then what the transport test sees, so such a +// usage only qualifies via its context) and an empty/absent usage context +// becomes "security context". +func TestExtractWeakTLSUnknownSubstitutions(t *testing.T) { + boolp := func(b bool) *bool { return &b } + str := func(s string) *string { return &s } + + ctx := schemas.NewSecurityContext() + ctx.CryptoUsage = []schemas.CryptoUsage{ + {Algorithm: "", UsageContext: str("tls handshake"), IsWeak: boolp(true)}, + {Algorithm: "TLSv1.0", UsageContext: nil, IsWeak: boolp(true)}, + } + got := extractWeakTLS(ctx) + if len(got) != 2 { + t.Fatalf("len = %d, want 2", len(got)) + } + if got[0].Title != "Weak TLS configuration: unknown" { + t.Errorf("empty algorithm -> title %q, want the 'unknown' substitution", got[0].Title) + } + if got[1].CodeSnippet != "algorithm=TLSv1.0; context=security context" { + t.Errorf("absent usage context -> snippet %q", got[1].CodeSnippet) + } +} + +// TestExtractWeakTLSDoesNotFireOnEmptyAlgorithmAlone pins the consequence of +// the "unknown" substitution: an empty algorithm with a non-transport context +// is filtered out, because "unknown" contains neither "tls" nor "ssl". +func TestExtractWeakTLSDoesNotFireOnEmptyAlgorithmAlone(t *testing.T) { + boolp := func(b bool) *bool { return &b } + ctx := schemas.NewSecurityContext() + ctx.CryptoUsage = []schemas.CryptoUsage{{Algorithm: "", IsWeak: boolp(true)}} + if got := extractWeakTLS(ctx); len(got) != 0 { + t.Errorf("extractWeakTLS = %d findings, want 0", len(got)) + } +} + +// TestExtractStructuredSecurityItemsIsAlwaysEmpty pins the dead-in-Python +// branch. SecurityContext declares none of the four categories and pydantic's +// default extra="ignore" drops them, so getattr returns None and the loop never +// runs. See the function's doc comment. +func TestExtractStructuredSecurityItemsIsAlwaysEmpty(t *testing.T) { + ctx := schemas.NewSecurityContext() + ctx.AuthModel = "jwt" + ctx.FrameworkSecurity = []string{"whatever"} + if got := extractStructuredSecurityItems(ctx); len(got) != 0 { + t.Errorf("extractStructuredSecurityItems = %d findings, want 0", len(got)) + } +} + +// TestExtractReconFindingsEmptyIsNotNil pins that the result serializes as `[]` +// rather than `null`, matching Python's list return. +func TestExtractReconFindingsEmptyIsNotNil(t *testing.T) { + got := ExtractReconFindings(schemas.NewReconResult()) + if got == nil { + t.Fatal("ExtractReconFindings returned nil, want an empty slice") + } + if len(got) != 0 { + t.Errorf("len = %d, want 0", len(got)) + } +} + +// TestReconFindingsGetDistinctIDs pins that every finding carries its own uuid4 +// id and fingerprint (pydantic default_factory parity), which the deduplicator +// relies on. +func TestReconFindingsGetDistinctIDs(t *testing.T) { + recon := schemas.NewReconResult() + secrets := make([]schemas.SecretFinding, 0, 3) + for i := 0; i < 3; i++ { + s := schemas.NewSecretFinding() + s.SecretType = "api_key" + s.FilePath = "a.py" + s.Line = i + 1 + s.Match = "m" + s.Confidence = "high" + secrets = append(secrets, s) + } + recon.Config = schemas.ConfigReport{Secrets: secrets} + + got := ExtractReconFindings(recon) + seen := map[string]bool{} + for _, f := range got { + if f.ID == "" || f.Fingerprint == "" { + t.Fatalf("finding has empty id/fingerprint: %+v", f) + } + if seen[f.ID] { + t.Errorf("duplicate finding id %q", f.ID) + } + seen[f.ID] = true + } +} + +func anyFinding(findings []schemas.RawFinding, pred func(schemas.RawFinding) bool) bool { + for _, f := range findings { + if pred(f) { + return true + } + } + return false +} diff --git a/internal/secaf/agents/recon/golden_test.go b/internal/secaf/agents/recon/golden_test.go new file mode 100644 index 0000000000..86b9f9927b --- /dev/null +++ b/internal/secaf/agents/recon/golden_test.go @@ -0,0 +1,96 @@ +package recon + +// Shared helpers for the golden-fixture tests in this package. +// +// Every fixture under testdata/golden is produced by go/scripts/gen_golden.py +// running the REAL Python code from src/sec_af/agents/recon. Regenerate with: +// +// PYTHONPATH=src ~/.agentfield/packages/sec-af/venv/bin/python go/scripts/gen_golden.py +// +// A test failing here means the Go port and the Python source disagree about +// bytes that reach the LLM (prompts) or the wire (parsed models) — not that a +// fixture needs refreshing. Refresh only after a deliberate Python change. + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +const goldenDir = "testdata/golden" + +// goldenText reads a *.txt fixture verbatim. +func goldenText(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join(goldenDir, name+".txt")) + if err != nil { + t.Fatalf("read golden %s.txt: %v", name, err) + } + return string(b) +} + +// goldenJSON decodes a *.json fixture into dest. +func goldenJSON(t *testing.T, name string, dest any) { + t.Helper() + b, err := os.ReadFile(filepath.Join(goldenDir, name+".json")) + if err != nil { + t.Fatalf("read golden %s.json: %v", name, err) + } + if err := json.Unmarshal(b, dest); err != nil { + t.Fatalf("decode golden %s.json: %v", name, err) + } +} + +// jsonTree marshals v and decodes the result into the untyped tree shape the +// golden fixtures decode to, so the two can be compared with reflect.DeepEqual +// without either side's Go types leaking into the comparison. +func jsonTree(t *testing.T, v any) any { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatalf("marshal %T: %v", v, err) + } + var tree any + if err := json.Unmarshal(b, &tree); err != nil { + t.Fatalf("unmarshal %T: %v", v, err) + } + return tree +} + +// scrubIDs replaces every string value under an "id" key with the placeholder +// gen_golden.py writes. SecretFinding and MisconfigFinding mint a fresh uuid4 +// per parse (pydantic default_factory / schemas.New*), so the ids are +// nondeterministic by construction and cannot be compared. +func scrubIDs(v any) any { + switch t := v.(type) { + case map[string]any: + out := make(map[string]any, len(t)) + for k, val := range t { + if k == "id" { + if _, isStr := val.(string); isStr { + out[k] = "" + continue + } + } + out[k] = scrubIDs(val) + } + return out + case []any: + out := make([]any, len(t)) + for i, val := range t { + out[i] = scrubIDs(val) + } + return out + default: + return v + } +} + +// diffJSON renders got/want for a readable failure message. +func diffJSON(t *testing.T, got, want any) string { + t.Helper() + g, _ := json.MarshalIndent(got, "", " ") + w, _ := json.MarshalIndent(want, "", " ") + return "\n--- got ---\n" + string(g) + "\n--- want (python) ---\n" + string(w) +} diff --git a/internal/secaf/agents/recon/metrics.go b/internal/secaf/agents/recon/metrics.go new file mode 100644 index 0000000000..550dd829c0 --- /dev/null +++ b/internal/secaf/agents/recon/metrics.go @@ -0,0 +1,265 @@ +package recon + +// Ports the `_SKIP_DIRS` / `_CODE_EXTS` tables and `_repo_metrics` from +// src/sec_af/agents/recon/__init__.py. + +import ( + "io" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// skipDirs ports _SKIP_DIRS. Membership is tested against EVERY path component +// (see RepoMetrics), not just the immediate parent. +var skipDirs = map[string]bool{ + ".git": true, + ".hg": true, + ".svn": true, + "node_modules": true, + "vendor": true, + ".venv": true, + "venv": true, + "__pycache__": true, +} + +// codeExts ports _CODE_EXTS — the extensions whose LINES are counted. Files +// outside this set still count toward the file total. +var codeExts = map[string]bool{ + ".py": true, ".go": true, ".js": true, ".jsx": true, ".ts": true, ".tsx": true, + ".java": true, ".kt": true, ".swift": true, ".rs": true, ".c": true, ".cc": true, + ".cpp": true, ".h": true, ".hpp": true, ".cs": true, ".rb": true, ".php": true, + ".scala": true, ".sql": true, ".sh": true, ".yaml": true, ".yml": true, + ".json": true, ".toml": true, +} + +// RepoMetrics ports _repo_metrics: +// +// def _repo_metrics(repo_path: str) -> tuple[int, int]: +// root = Path(repo_path) +// if not root.exists(): +// return 0, 0 +// file_count = 0 +// line_count = 0 +// for path in root.rglob("*"): +// if any(part in _SKIP_DIRS for part in path.parts): +// continue +// if not path.is_file(): +// continue +// file_count += 1 +// if path.suffix.lower() not in _CODE_EXTS: +// continue +// try: +// with path.open("r", encoding="utf-8", errors="ignore") as handle: +// for _ in handle: +// line_count += 1 +// except OSError: +// continue +// return line_count, file_count +// +// Returns (linesOfCode, fileCount) — Python's tuple order, which is the +// opposite of what the names suggest, so callers must not swap them. +// +// Python parity, in the order the quirks bite: +// +// - The skip test runs over `path.parts`, the components of the FULL path +// including everything in repo_path itself. A repository that happens to +// live under a directory named `vendor` or `venv` therefore reports (0, 0). +// That is reproduced here (rootHasSkippedPart short-circuits), not fixed. +// - `is_file()` follows symlinks: a symlink to a regular file IS counted, a +// broken symlink and a symlink to a directory are not, and neither are +// sockets/fifos/devices. +// - pathlib does not descend THROUGH a symlinked directory, and neither does +// filepath.WalkDir, so a symlink loop cannot hang either implementation. +// - `Path.suffix` is NOT filepath.Ext: a leading dot does not start a suffix +// (".gitignore" has none), a trailing dot does not end one ("x." has none), +// and only the last component counts ("a.tar.gz" is ".gz"). pySuffix +// implements pathlib's rule. +// - Line counting is Python TEXT-mode iteration with universal newlines: LF, +// CRLF and a bare CR each terminate a line, and a final line with no +// terminator still counts. `errors="ignore"` cannot change the count — +// UTF-8 continuation bytes are 0x80..0xBF, so a dropped invalid byte can +// never have been a 0x0A/0x0D. +// - Both loops swallow their errors: an unreadable directory contributes +// nothing (pathlib catches PermissionError), and a file that fails to open +// stays counted in file_count but adds no lines (`except OSError: continue` +// runs AFTER the increment). +// +// Deviation: Python's `Path("")` is `Path(".")`, which exists, so an empty +// repo_path would scan the process working directory. Go's os.Stat("") fails, +// so this returns (0, 0). No caller passes an empty path — app.py resolves it +// to an absolute directory first. +func RepoMetrics(repoPath string) (int, int) { + if _, err := os.Stat(repoPath); err != nil { + return 0, 0 + } + if rootHasSkippedPart(repoPath) { + return 0, 0 + } + + lineCount, fileCount := 0, 0 + _ = filepath.WalkDir(repoPath, func(path string, d fs.DirEntry, err error) error { + if err != nil { + // pathlib swallows PermissionError while iterating; an unreadable + // directory simply yields nothing. + if d != nil && d.IsDir() { + return fs.SkipDir + } + return nil + } + if path == repoPath { + // rglob("*") never yields the root itself. + return nil + } + if d.IsDir() { + if skipDirs[d.Name()] { + // Every path under a skipped directory carries that component, + // so Python's per-path filter rejects all of them. Pruning here + // is observationally identical and avoids walking node_modules. + return fs.SkipDir + } + return nil + } + if skipDirs[d.Name()] { + // A FILE whose own name is in the table is filtered too — the + // Python test is over components, not just directories. + return nil + } + + if !isRegularFollowingSymlinks(path, d) { + return nil + } + fileCount++ + + if !codeExts[strings.ToLower(pySuffix(d.Name()))] { + return nil + } + n, err := countPythonLines(path) + if err != nil { + return nil // except OSError: continue + } + lineCount += n + return nil + }) + + return lineCount, fileCount +} + +// rootHasSkippedPart reports whether repo_path's OWN components include a +// skipped directory name, which makes Python's `any(part in _SKIP_DIRS for part +// in path.parts)` true for every descendant. +func rootHasSkippedPart(repoPath string) bool { + for _, part := range pathParts(repoPath) { + if skipDirs[part] { + return true + } + } + return false +} + +// pathParts reproduces pathlib.PurePosixPath(p).parts: the POSIX root (if any) +// as its own leading element, then the non-empty components with "." segments +// dropped and ".." kept. +// +// "/a/b/c" -> ["/", "a", "b", "c"] "./a/b" -> ["a", "b"] +// "/a//b/" -> ["/", "a", "b"] "../a" -> ["..", "a"] +// "a" -> ["a"] "/" -> ["/"] +// +// The exactly-two-leading-slashes case is POSIX's implementation-defined root +// and pathlib keeps it as the literal "//" element. +func pathParts(p string) []string { + var parts []string + rest := p + if strings.HasPrefix(p, "/") { + if strings.HasPrefix(p, "//") && !strings.HasPrefix(p, "///") { + parts = append(parts, "//") + } else { + parts = append(parts, "/") + } + rest = strings.TrimLeft(p, "/") + } + for _, seg := range strings.Split(rest, "/") { + if seg == "" || seg == "." { + continue + } + parts = append(parts, seg) + } + return parts +} + +// pySuffix reproduces pathlib.PurePath(name).suffix — NOT filepath.Ext. +// +// "a.py" -> ".py" ".gitignore" -> "" "x." -> "" "a.tar.gz" -> ".gz" +func pySuffix(name string) string { + i := strings.LastIndexByte(name, '.') + if i > 0 && i < len(name)-1 { + return name[i:] + } + return "" +} + +// isRegularFollowingSymlinks implements Path.is_file(): true for a regular +// file, and for a symlink that RESOLVES to one. Directories, broken symlinks, +// sockets, fifos and devices are all false. +func isRegularFollowingSymlinks(path string, d fs.DirEntry) bool { + if d.Type()&os.ModeSymlink != 0 { + st, err := os.Stat(path) // follows the link + return err == nil && st.Mode().IsRegular() + } + return d.Type().IsRegular() +} + +// countPythonLines counts lines the way iterating a Python text file opened +// with the default newline=None (universal newlines) does: LF, CRLF and a lone +// CR each end a line, and trailing content with no terminator counts as one +// more. An empty file has zero lines. +// +// Streamed in 64 KiB chunks rather than read whole so a pathologically large +// file cannot blow up the process the way Python's line-at-a-time loop never +// would. +func countPythonLines(path string) (int, error) { + f, err := os.Open(path) + if err != nil { + return 0, err + } + defer f.Close() + + buf := make([]byte, 64*1024) + lines := 0 + pending := false // saw content since the last terminator + sawCR := false // previous byte was CR; a following LF joins it + for { + n, readErr := f.Read(buf) + for i := 0; i < n; i++ { + c := buf[i] + if sawCR { + sawCR = false + if c == '\n' { + continue // CRLF: the CR already counted the line + } + } + switch c { + case '\n': + lines++ + pending = false + case '\r': + lines++ + pending = false + sawCR = true + default: + pending = true + } + } + if readErr != nil { + if readErr == io.EOF { + break + } + return 0, readErr + } + } + if pending { + lines++ + } + return lines, nil +} diff --git a/internal/secaf/agents/recon/metrics_test.go b/internal/secaf/agents/recon/metrics_test.go new file mode 100644 index 0000000000..56f3b89a7a --- /dev/null +++ b/internal/secaf/agents/recon/metrics_test.go @@ -0,0 +1,263 @@ +package recon + +// Validation contract for _repo_metrics: +// +// - A missing repository reports (0, 0). +// - Any path whose components include a _SKIP_DIRS name is invisible — +// including when the component comes from repo_path itself. +// - file_count counts EVERY surviving regular file (following symlinks), +// whatever its extension; line_count adds lines only for _CODE_EXTS files. +// - The extension test uses pathlib's suffix rule, not filepath.Ext. +// - Lines are counted the way Python text-mode iteration counts them: LF, +// CRLF and a lone CR each end a line, a final unterminated line still +// counts, and an empty file has none. + +import ( + "encoding/base64" + "os" + "path/filepath" + "sort" + "testing" +) + +// repoMetricsGolden mirrors testdata/golden/repo_metrics.json. +type repoMetricsGolden struct { + Files map[string]string `json:"files"` // relpath -> base64 content + Symlinks map[string]string `json:"symlinks"` // relpath -> target + LinesOfCode int `json:"lines_of_code"` + FileCount int `json:"file_count"` +} + +// materialize writes the golden's tree under root. Directories are created on +// demand, exactly as gen_golden.py does on the Python side. +func materialize(t *testing.T, root string, g repoMetricsGolden) { + t.Helper() + // Sorted so the (rare) case of a symlink target that must exist first is + // deterministic; content files are all written before any link. + rels := make([]string, 0, len(g.Files)) + for rel := range g.Files { + rels = append(rels, rel) + } + sort.Strings(rels) + for _, rel := range rels { + data, err := base64.StdEncoding.DecodeString(g.Files[rel]) + if err != nil { + t.Fatalf("decode %s: %v", rel, err) + } + path := filepath.Join(root, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir for %s: %v", rel, err) + } + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatalf("write %s: %v", rel, err) + } + } + + links := make([]string, 0, len(g.Symlinks)) + for rel := range g.Symlinks { + links = append(links, rel) + } + sort.Strings(links) + for _, rel := range links { + path := filepath.Join(root, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir for symlink %s: %v", rel, err) + } + if err := os.Symlink(g.Symlinks[rel], path); err != nil { + t.Fatalf("symlink %s -> %s: %v", rel, g.Symlinks[rel], err) + } + } +} + +// TestRepoMetricsGolden runs RepoMetrics over the exact tree gen_golden.py fed +// to the real Python _repo_metrics and asserts the same tuple. +func TestRepoMetricsGolden(t *testing.T) { + var g repoMetricsGolden + goldenJSON(t, "repo_metrics", &g) + if len(g.Files) == 0 { + t.Fatal("golden tree is empty") + } + + root := t.TempDir() + materialize(t, root, g) + + lines, files := RepoMetrics(root) + if lines != g.LinesOfCode || files != g.FileCount { + t.Errorf("RepoMetrics = (lines %d, files %d), want (lines %d, files %d) from Python", + lines, files, g.LinesOfCode, g.FileCount) + } +} + +// TestRepoMetricsMissingRoot ports `if not root.exists(): return 0, 0`. +func TestRepoMetricsMissingRoot(t *testing.T) { + missing := filepath.Join(t.TempDir(), "does-not-exist") + if lines, files := RepoMetrics(missing); lines != 0 || files != 0 { + t.Errorf("RepoMetrics(missing) = (%d, %d), want (0, 0)", lines, files) + } +} + +// TestRepoMetricsSkipsWhenRepoPathItselfIsSkipped pins the Python quirk that +// the _SKIP_DIRS test runs over path.parts — the components of the FULL path — +// so a repository living under a directory named "vendor" is entirely +// invisible. This is a faithful port of a bug, not a bug in the port. +func TestRepoMetricsSkipsWhenRepoPathItselfIsSkipped(t *testing.T) { + base := t.TempDir() + for _, skipped := range []string{"vendor", "venv", ".git", "node_modules"} { + repo := filepath.Join(base, skipped, "myrepo") + if err := os.MkdirAll(repo, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(repo, "main.py"), []byte("a\nb\n"), 0o644); err != nil { + t.Fatal(err) + } + if lines, files := RepoMetrics(repo); lines != 0 || files != 0 { + t.Errorf("RepoMetrics under %q = (%d, %d), want (0, 0)", skipped, lines, files) + } + } +} + +// TestCountPythonLines pins the universal-newline line count against CPython +// ground truth, produced by iterating each byte string through +// `open(path, "r", encoding="utf-8", errors="ignore")` under the pinned +// interpreter (~/.agentfield/packages/sec-af/venv/bin/python, 3.11.12). +func TestCountPythonLines(t *testing.T) { + cases := []struct { + name string + data []byte + want int + }{ + {"lf_terminated", []byte("a\nb\nc\n"), 3}, + {"lf_unterminated", []byte("a\nb\nc"), 3}, + {"empty", []byte(""), 0}, + {"single_newline", []byte("\n"), 1}, + {"crlf", []byte("a\r\nb\r\n"), 2}, + {"cr_only_terminated", []byte("a\rb\r"), 2}, + {"cr_only_unterminated", []byte("a\rb"), 2}, + {"invalid_utf8", []byte("\xff\xfe bad bytes\nsecond\n"), 2}, + {"blank_lines", []byte("a\n\n\nb"), 4}, + {"cr_at_eof_only", []byte("\r"), 1}, + {"crlf_then_content", []byte("a\r\nb"), 2}, + } + dir := t.TempDir() + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(dir, tc.name) + if err := os.WriteFile(path, tc.data, 0o644); err != nil { + t.Fatal(err) + } + got, err := countPythonLines(path) + if err != nil { + t.Fatalf("countPythonLines: %v", err) + } + if got != tc.want { + t.Errorf("countPythonLines(%q) = %d, want %d", tc.data, got, tc.want) + } + }) + } +} + +// TestCountPythonLinesAcrossChunkBoundary pins that the streaming reader's +// CR/LF state survives a buffer boundary: a CRLF split across two 64 KiB reads +// must still count as ONE line terminator. +func TestCountPythonLinesAcrossChunkBoundary(t *testing.T) { + const chunk = 64 * 1024 + data := make([]byte, 0, chunk+8) + for len(data) < chunk-1 { + data = append(data, 'x') + } + data = append(data, '\r', '\n', 'y', '\n') // the CR is the last byte of chunk 1 + path := filepath.Join(t.TempDir(), "boundary.py") + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } + got, err := countPythonLines(path) + if err != nil { + t.Fatal(err) + } + if got != 2 { + t.Errorf("countPythonLines across a chunk boundary = %d, want 2", got) + } +} + +// TestPySuffix pins pathlib's suffix rule, which filepath.Ext does not share. +// Ground truth from `Path(name).suffix` under the pinned interpreter. +func TestPySuffix(t *testing.T) { + cases := map[string]string{ + "a.py": ".py", + ".gitignore": "", + "foo.": "", + "foo.tar.gz": ".gz", + "Makefile": "", + "a.PY": ".PY", + ".config.yaml": ".yaml", + "x.YML": ".YML", + "": "", + ".": "", + } + for name, want := range cases { + if got := pySuffix(name); got != want { + t.Errorf("pySuffix(%q) = %q, want %q", name, got, want) + } + } +} + +// TestPathParts pins pathlib.PurePosixPath(p).parts, which drives the skip-dir +// test. Ground truth from `Path(p).parts` under the pinned interpreter. +func TestPathParts(t *testing.T) { + cases := []struct { + in string + want []string + }{ + {"/a/b/c", []string{"/", "a", "b", "c"}}, + {"a/b", []string{"a", "b"}}, + {"./a/b", []string{"a", "b"}}, + {"/a//b/", []string{"/", "a", "b"}}, + {"../a", []string{"..", "a"}}, + {"/", []string{"/"}}, + {"a", []string{"a"}}, + {"//a", []string{"//", "a"}}, + {"///a", []string{"/", "a"}}, + } + for _, tc := range cases { + got := pathParts(tc.in) + if len(got) != len(tc.want) { + t.Errorf("pathParts(%q) = %q, want %q", tc.in, got, tc.want) + continue + } + for i := range got { + if got[i] != tc.want[i] { + t.Errorf("pathParts(%q) = %q, want %q", tc.in, got, tc.want) + break + } + } + } +} + +// TestRepoMetricsCountsNonCodeFilesButNotTheirLines pins the split between the +// two counters: every surviving regular file bumps file_count, only _CODE_EXTS +// files contribute lines. +func TestRepoMetricsCountsNonCodeFilesButNotTheirLines(t *testing.T) { + root := t.TempDir() + write := func(rel, content string) { + t.Helper() + path := filepath.Join(root, rel) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + write("a.py", "1\n2\n3\n") // 3 lines, code + write("README.md", "x\ny\n") // 0 lines, not code + write("LICENSE", "long text\n") // 0 lines, no suffix at all + write("b/c.go", "package b\n") // 1 line, code + + lines, files := RepoMetrics(root) + if files != 4 { + t.Errorf("file_count = %d, want 4", files) + } + if lines != 4 { + t.Errorf("lines_of_code = %d, want 4 (3 from a.py + 1 from b/c.go)", lines) + } +} diff --git a/internal/secaf/agents/recon/parsers.go b/internal/secaf/agents/recon/parsers.go new file mode 100644 index 0000000000..4955a68ea8 --- /dev/null +++ b/internal/secaf/agents/recon/parsers.go @@ -0,0 +1,590 @@ +package recon + +// Ports src/sec_af/agents/recon/_parsers.py in full. +// +// Every RECON mapper asks the harness for a FLAT model (`ArchitectureMapRaw`, +// `DataFlowMapRaw`, ...) whose fields are `list[str]` of pipe-delimited rows, +// because LLMs produce flat rows far more reliably than nested JSON. The +// functions here turn those rows into the structured schemas the rest of the +// pipeline consumes. They are total: a malformed row never errors, it degrades +// into empty strings / zero lines / nil optionals, exactly as in Python. + +import ( + "strconv" + "strings" + "unicode" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// --------------------------------------------------------------------------- +// CPython string/number primitives +// --------------------------------------------------------------------------- + +// pyStrip reproduces Python's str.strip() with no argument. +// +// Python parity: CPython strips every code point for which Py_UNICODE_ISSPACE +// is true. That set is Go's unicode.IsSpace PLUS the four C0 information +// separators U+001C..U+001F (FS, GS, RS, US), which Go does not consider +// whitespace. The extra clause below closes that gap so a row separated with an +// exotic control character strips identically in both runtimes. +func pyStrip(s string) string { + return strings.TrimFunc(s, func(r rune) bool { + return unicode.IsSpace(r) || (r >= 0x1C && r <= 0x1F) + }) +} + +// splitPipe ports _split_pipe: +// +// def _split_pipe(s: str, expected: int) -> list[str]: +// parts = [p.strip() for p in s.split("|", maxsplit=expected - 1)] +// while len(parts) < expected: +// parts.append("") +// return parts +// +// Python parity: `maxsplit=expected-1` caps the result at `expected` fields, so +// a row with MORE pipes than the format allows keeps the surplus (pipes and +// all) in the LAST field — "a | b | c | d | e" with expected=4 yields +// ["a", "b", "c", "d | e"], not a dropped tail. Go's strings.SplitN(s, "|", n) +// has exactly that cap semantics for n >= 1, and every call site passes +// expected in 4..7. Short rows are right-padded with "". +func splitPipe(s string, expected int) []string { + parts := strings.SplitN(s, "|", expected) + out := make([]string, 0, expected) + for _, p := range parts { + out = append(out, pyStrip(p)) + } + for len(out) < expected { + out = append(out, "") + } + return out +} + +// parseBool ports _parse_bool: "true"/"yes"/"1" -> true, "false"/"no"/"0" -> +// false, anything else -> None (nil). +// +// Python parity: Python lowercases FIRST and strips SECOND +// (`s.lower().strip()`); the order is immaterial for this alphabet but is +// preserved anyway. Go's strings.ToLower is simple per-rune lowering while +// CPython's str.lower() applies full Unicode case mappings; the two differ only +// for code points whose lowering expands (e.g. U+0130), none of which can occur +// in the six accepted literals. +func parseBool(s string) *bool { + switch pyStrip(strings.ToLower(s)) { + case "true", "yes", "1": + t := true + return &t + case "false", "no", "0": + f := false + return &f + } + return nil +} + +// parseInt ports _parse_int(s, default=0) — `int(s.strip())` with the given +// fallback on ValueError/TypeError. +// +// Python parity: CPython's int() accepts an optional sign, ASCII digits, and +// underscores used as digit separators ("1_0" is 10; "_1", "1_" and "1__0" are +// all errors). Documented deviations, both unreachable for the pipe-delimited +// line numbers this parses: non-ASCII decimal digits (CPython accepts them, +// this does not) and values beyond int64 (CPython has arbitrary precision, this +// returns the fallback). +func parseInt(s string, def int) int { + t, ok := stripUnderscores(pyStrip(s)) + if !ok { + return def + } + n, err := strconv.Atoi(t) + if err != nil { + return def + } + return n +} + +// parseFloat ports _parse_float — `float(s.strip())`, None on failure. +// +// Python parity: the underscore rule is the same as int()'s. Two guards keep Go +// from being MORE permissive than CPython: hexadecimal float literals +// ("0x1p-2") are rejected here because float() rejects them while +// strconv.ParseFloat accepts them. The special spellings CPython does accept — +// "inf", "-inf", "infinity", "nan", any case — are accepted by ParseFloat too, +// so they need no special handling. +func parseFloat(s string) *float64 { + t, ok := stripUnderscores(pyStrip(s)) + if !ok { + return nil + } + mantissa := strings.TrimLeft(t, "+-") + if len(mantissa) > 1 && mantissa[0] == '0' && (mantissa[1] == 'x' || mantissa[1] == 'X') { + return nil // CPython: float("0x1p-2") raises ValueError + } + f, err := strconv.ParseFloat(t, 64) + if err != nil { + return nil + } + return &f +} + +// stripUnderscores validates CPython's numeric-literal underscore rule — every +// "_" must sit BETWEEN two ASCII digits — and returns the string with the +// underscores removed. ok is false when the rule is violated, which is the +// ValueError CPython raises. +func stripUnderscores(s string) (string, bool) { + if !strings.ContainsRune(s, '_') { + return s, true + } + b := make([]byte, 0, len(s)) + for i := 0; i < len(s); i++ { + if s[i] != '_' { + b = append(b, s[i]) + continue + } + if i == 0 || i == len(s)-1 || !isASCIIDigit(s[i-1]) || !isASCIIDigit(s[i+1]) { + return "", false + } + } + return string(b), true +} + +func isASCIIDigit(c byte) bool { return c >= '0' && c <= '9' } + +// parseFileLine ports _parse_file_line: +// +// s = s.strip() +// if ":" in s: +// idx = s.rfind(":") +// path = s[:idx] +// line = _parse_int(s[idx + 1 :], 0) +// if line > 0: +// return path, line +// return s, 0 +// +// Python parity: three details survive verbatim. (1) The LAST colon splits, so +// "a:b:12" is path "a:b" line 12 and a Windows-style "C:/x.py:9" still parses. +// (2) A non-positive line number makes the whole string the path, colon +// included ("src/x.py:0" -> path "src/x.py:0", line 0). (3) Only the WHOLE +// string is stripped, never the split halves, so " a.py : 4 " yields the path +// "a.py " with its trailing space intact. +func parseFileLine(s string) (string, int) { + t := pyStrip(s) + if idx := strings.LastIndexByte(t, ':'); idx >= 0 { + path := t[:idx] + line := parseInt(t[idx+1:], 0) + if line > 0 { + return path, line + } + } + return t, 0 +} + +// isNA ports _is_na — the "the model had nothing to say here" sentinel test +// applied to optional fields before they become nil. +func isNA(s string) bool { + switch pyStrip(strings.ToLower(s)) { + case "", "na", "n/a", "none", "unknown": + return true + } + return false +} + +// --------------------------------------------------------------------------- +// small optional-field helpers +// --------------------------------------------------------------------------- + +// strOrNil ports the `parts[i] or None` idiom: "" becomes nil, anything else a +// pointer to the value. +func strOrNil(s string) *string { + if s == "" { + return nil + } + return &s +} + +// nilIfNA ports the `None if _is_na(parts[i]) else parts[i]` idiom. +func nilIfNA(s string) *string { + if isNA(s) { + return nil + } + return &s +} + +// boolOrFalse ports the `_parse_bool(x) or False` idiom, where Python's `or` +// collapses BOTH None and False to False. +func boolOrFalse(b *bool) bool { return b != nil && *b } + +// splitCSV ports the `[p.strip() for p in field.split(",") if p.strip()]` +// idiom: comma-separated, stripped, empties dropped. Always non-nil so it +// serializes as `[]` like a pydantic default_factory=list field. +func splitCSV(s string) []string { + out := []string{} + for _, p := range strings.Split(s, ",") { + if v := pyStrip(p); v != "" { + out = append(out, v) + } + } + return out +} + +// --------------------------------------------------------------------------- +// parse_architecture_raw +// --------------------------------------------------------------------------- + +// ParseArchitectureRaw ports _parsers.parse_architecture_raw. +// +// Row formats (from the ArchitectureMapRaw field descriptions): +// +// modules name | path | language | description +// entry_points kind | route_or_id | file_path:line | auth_required +// trust_bounds name | source_zone | target_zone | description +// services name | type | endpoint | auth_mechanism +// api_endpoints method | path | handler | file_path:line | auth_required | rate_limited +// +// Python parity: EntryPoint.route is derived, not parsed — the identifier is +// reused as the route IFF it contains a "/", so "POST /api/login" becomes a +// route while "migrate" does not. EntryPoint.method, Module.dependencies, +// TrustBoundary.enforcement and Service.purpose are never populated here; they +// keep their pydantic defaults. +func ParseArchitectureRaw(raw schemas.ArchitectureMapRaw) schemas.ArchitectureMap { + modules := make([]schemas.Module, 0, len(raw.Modules)) + for _, entry := range raw.Modules { + parts := splitPipe(entry, 4) + m := schemas.NewModule() + m.Name = parts[0] + m.Path = parts[1] + m.Language = parts[2] + m.Description = strOrNil(parts[3]) + modules = append(modules, m) + } + + entryPoints := make([]schemas.EntryPoint, 0, len(raw.EntryPoints)) + for _, entry := range raw.EntryPoints { + parts := splitPipe(entry, 4) + filePath, line := parseFileLine(parts[2]) + ident := parts[1] + var route *string + if strings.Contains(ident, "/") { + r := ident + route = &r + } + entryPoints = append(entryPoints, schemas.EntryPoint{ + Kind: parts[0], + Identifier: ident, + FilePath: filePath, + Line: line, + Route: route, + AuthRequired: parseBool(parts[3]), + }) + } + + trustBoundaries := make([]schemas.TrustBoundary, 0, len(raw.TrustBoundaries)) + for _, entry := range raw.TrustBoundaries { + parts := splitPipe(entry, 4) + tb := schemas.NewTrustBoundary() + tb.Name = parts[0] + tb.SourceZone = parts[1] + tb.TargetZone = parts[2] + tb.Description = parts[3] + trustBoundaries = append(trustBoundaries, tb) + } + + services := make([]schemas.Service, 0, len(raw.Services)) + for _, entry := range raw.Services { + parts := splitPipe(entry, 4) + services = append(services, schemas.Service{ + Name: parts[0], + ServiceType: parts[1], + Endpoint: nilIfNA(parts[2]), + AuthMechanism: nilIfNA(parts[3]), + }) + } + + apiEndpoints := make([]schemas.APIEndpoint, 0, len(raw.APIEndpoints)) + for _, entry := range raw.APIEndpoints { + parts := splitPipe(entry, 6) + filePath, line := parseFileLine(parts[3]) + apiEndpoints = append(apiEndpoints, schemas.APIEndpoint{ + Method: parts[0], + Path: parts[1], + Handler: parts[2], + FilePath: filePath, + Line: line, + AuthRequired: parseBool(parts[4]), + RateLimited: parseBool(parts[5]), + }) + } + + // Python parity: ArchitectureMapRaw.app_type is a plain `str` (default + // "unknown") while ArchitectureMap.app_type is `str | None`, so the pointer + // is ALWAYS non-nil after a parse — never null, even for an empty string. + appType := raw.AppType + return schemas.ArchitectureMap{ + AppType: &appType, + Modules: modules, + EntryPoints: entryPoints, + TrustBoundaries: trustBoundaries, + Services: services, + APISurface: apiEndpoints, + } +} + +// --------------------------------------------------------------------------- +// parse_data_flow_raw +// --------------------------------------------------------------------------- + +// ParseDataFlowRaw ports _parsers.parse_data_flow_raw. +// +// Row formats: +// +// flows source | sink | sanitized | file1, file2, ... +// sanitization_points file_path:line | function_name | type | protects_against +// sinks sink_type | file_path:line | function_name | notes +// +// Python parity: `sanitized=_parse_bool(parts[2]) or False` collapses an +// UNPARSEABLE value to false, so an ambiguous row is treated as unsanitized — +// the conservative direction for a security tool. DataFlow.path is never +// populated here and keeps its `[]` default. +func ParseDataFlowRaw(raw schemas.DataFlowMapRaw) schemas.DataFlowMap { + flows := make([]schemas.DataFlow, 0, len(raw.Flows)) + for _, entry := range raw.Flows { + parts := splitPipe(entry, 4) + f := schemas.NewDataFlow() + f.Source = parts[0] + f.Sink = parts[1] + f.Sanitized = boolOrFalse(parseBool(parts[2])) + f.Files = splitCSV(parts[3]) + flows = append(flows, f) + } + + sanitizationPoints := make([]schemas.SanitizationPoint, 0, len(raw.SanitizationPoints)) + for _, entry := range raw.SanitizationPoints { + parts := splitPipe(entry, 4) + filePath, line := parseFileLine(parts[0]) + sp := schemas.NewSanitizationPoint() + sp.FilePath = filePath + sp.Line = line + sp.FunctionName = strOrNil(parts[1]) + sp.SanitizationType = parts[2] + sp.ProtectsAgainst = splitCSV(parts[3]) + sanitizationPoints = append(sanitizationPoints, sp) + } + + sinks := make([]schemas.Sink, 0, len(raw.Sinks)) + for _, entry := range raw.Sinks { + parts := splitPipe(entry, 4) + filePath, line := parseFileLine(parts[1]) + sinks = append(sinks, schemas.Sink{ + SinkType: parts[0], + FilePath: filePath, + Line: line, + FunctionName: strOrNil(parts[2]), + ExploitabilityNotes: strOrNil(parts[3]), + }) + } + + return schemas.DataFlowMap{Flows: flows, SanitizationPoints: sanitizationPoints, Sinks: sinks} +} + +// --------------------------------------------------------------------------- +// parse_dependency_report_raw +// --------------------------------------------------------------------------- + +// ParseDependencyReportRaw ports _parsers.parse_dependency_report_raw. +// +// Row formats: +// +// sbom name | version | ecosystem | direct | license +// known_cves cve_id | package | installed_version | fixed_version | cvss_score | direct | reachable +// outdated package | current_version | latest_version | direct +// +// Python parity: direct_count / transitive_count are derived ONLY from the sbom +// rows (`if is_direct: direct_count += 1 else: transitive_count += 1`), so +// every sbom row lands in exactly one bucket and an unparseable `direct` field +// counts as transitive. KnownCVE.epss_score is never populated here. +func ParseDependencyReportRaw(raw schemas.DependencyReportRaw) schemas.DependencyReport { + sbom := make([]schemas.Dependency, 0, len(raw.Sbom)) + directCount, transitiveCount := 0, 0 + for _, entry := range raw.Sbom { + parts := splitPipe(entry, 5) + isDirect := boolOrFalse(parseBool(parts[3])) + if isDirect { + directCount++ + } else { + transitiveCount++ + } + sbom = append(sbom, schemas.Dependency{ + Name: parts[0], + Version: parts[1], + Ecosystem: parts[2], + Direct: isDirect, + License: nilIfNA(parts[4]), + }) + } + + knownCves := make([]schemas.KnownCVE, 0, len(raw.KnownCves)) + for _, entry := range raw.KnownCves { + parts := splitPipe(entry, 7) + knownCves = append(knownCves, schemas.KnownCVE{ + CveID: parts[0], + Package: parts[1], + InstalledVersion: parts[2], + FixedVersion: nilIfNA(parts[3]), + CvssV4Score: parseFloat(parts[4]), + Direct: boolOrFalse(parseBool(parts[5])), + Reachable: parseBool(parts[6]), + }) + } + + outdated := make([]schemas.OutdatedDep, 0, len(raw.Outdated)) + for _, entry := range raw.Outdated { + parts := splitPipe(entry, 4) + outdated = append(outdated, schemas.OutdatedDep{ + Package: parts[0], + CurrentVersion: parts[1], + LatestVersion: parts[2], + Direct: boolOrFalse(parseBool(parts[3])), + }) + } + + return schemas.DependencyReport{ + Sbom: sbom, + KnownCves: knownCves, + Outdated: outdated, + DirectCount: directCount, + TransitiveCount: transitiveCount, + } +} + +// --------------------------------------------------------------------------- +// parse_config_report_raw +// --------------------------------------------------------------------------- + +// ParseConfigReportRaw ports _parsers.parse_config_report_raw. +// +// Row formats: +// +// secrets type | file_path:line | match_preview | confidence | is_test +// misconfigs category | file_path:line | key | risk | remediation +// +// Python parity: SecretFinding.confidence falls back to the literal "medium" +// when the row leaves it empty, while MisconfigFinding.line is `int | None` and +// becomes nil (not 0) when no positive line was parsed — the two models spell +// "unknown" differently and the port keeps both spellings. +// MisconfigFinding.value is never populated here. +// +// Both models mint a fresh uuid4 `id` per row (schemas.NewSecretFinding / +// NewMisconfigFinding), so repeated parses of identical input are NOT equal — +// same as Python. +func ParseConfigReportRaw(raw schemas.ConfigReportRaw) schemas.ConfigReport { + secrets := make([]schemas.SecretFinding, 0, len(raw.Secrets)) + for _, entry := range raw.Secrets { + parts := splitPipe(entry, 5) + filePath, line := parseFileLine(parts[1]) + s := schemas.NewSecretFinding() + s.SecretType = parts[0] + s.FilePath = filePath + s.Line = line + s.Match = parts[2] + s.Confidence = parts[3] + if s.Confidence == "" { + s.Confidence = "medium" + } + s.IsTestValue = parseBool(parts[4]) + secrets = append(secrets, s) + } + + misconfigs := make([]schemas.MisconfigFinding, 0, len(raw.Misconfigs)) + for _, entry := range raw.Misconfigs { + parts := splitPipe(entry, 5) + filePath, line := parseFileLine(parts[1]) + m := schemas.NewMisconfigFinding() + m.Category = parts[0] + m.FilePath = filePath + if line > 0 { + l := line + m.Line = &l + } + m.Key = nilIfNA(parts[2]) + m.Risk = parts[3] + m.Remediation = nilIfNA(parts[4]) + misconfigs = append(misconfigs, m) + } + + return schemas.ConfigReport{Secrets: secrets, Misconfigs: misconfigs} +} + +// --------------------------------------------------------------------------- +// parse_security_context_raw +// --------------------------------------------------------------------------- + +// headerTerms / deployTerms port _parsers._HEADER_TERMS / _DEPLOY_TERMS — the +// substring tables that bucket a free-form security signal into one of +// SecurityContext's three signal lists. +var ( + headerTerms = []string{"header", "csp", "hsts", "x-frame", "x-content-type", "cors"} + deployTerms = []string{"deploy", "docker", "kubernetes", "cloud", "ssl", "tls", "https", "container", "k8s"} +) + +// ParseSecurityContextRaw ports _parsers.parse_security_context_raw. +// +// crypto_usage rows are "algorithm | key_size | mode | usage_context | is_weak". +// security_signals are unstructured one-liners routed into +// security_headers / deployment_signals / framework_security by substring +// match, first-match-wins in that order (headers beat deployment beats the +// framework catch-all). Order WITHIN each bucket is the model's emission order. +// +// Python parity: key_size uses `_parse_int(parts[1]) if not _is_na(parts[1]) +// else None`, so a non-numeric-but-not-NA value like "notanint" yields 0 rather +// than nil — the `_parse_int` default leaks through. +func ParseSecurityContextRaw(raw schemas.SecurityContextRaw) schemas.SecurityContext { + cryptoUsage := make([]schemas.CryptoUsage, 0, len(raw.CryptoUsage)) + for _, entry := range raw.CryptoUsage { + parts := splitPipe(entry, 5) + var keySize *int + if !isNA(parts[1]) { + k := parseInt(parts[1], 0) + keySize = &k + } + cryptoUsage = append(cryptoUsage, schemas.CryptoUsage{ + Algorithm: parts[0], + KeySize: keySize, + Mode: nilIfNA(parts[2]), + UsageContext: nilIfNA(parts[3]), + IsWeak: parseBool(parts[4]), + }) + } + + frameworkSecurity := []string{} + securityHeaders := []string{} + deploymentSignals := []string{} + for _, signal := range raw.SecuritySignals { + lowered := strings.ToLower(signal) + switch { + case containsAny(lowered, headerTerms): + securityHeaders = append(securityHeaders, signal) + case containsAny(lowered, deployTerms): + deploymentSignals = append(deploymentSignals, signal) + default: + frameworkSecurity = append(frameworkSecurity, signal) + } + } + + return schemas.SecurityContext{ + AuthModel: raw.AuthModel, + AuthDetails: raw.AuthDetails, + CryptoUsage: cryptoUsage, + FrameworkSecurity: frameworkSecurity, + SecurityHeaders: securityHeaders, + DeploymentSignals: deploymentSignals, + } +} + +// containsAny ports `any(term in lowered for term in TERMS)`. +func containsAny(s string, terms []string) bool { + for _, t := range terms { + if strings.Contains(s, t) { + return true + } + } + return false +} diff --git a/internal/secaf/agents/recon/parsers_test.go b/internal/secaf/agents/recon/parsers_test.go new file mode 100644 index 0000000000..a6c5f6b2b6 --- /dev/null +++ b/internal/secaf/agents/recon/parsers_test.go @@ -0,0 +1,288 @@ +package recon + +// Validation contract for src/sec_af/agents/recon/_parsers.py, checked against +// fixtures produced by the REAL Python helpers (go/scripts/gen_golden.py): +// +// - splitPipe caps the field count, keeping surplus pipes in the last field, +// strips each field, and right-pads short rows with "". +// - parseBool/parseInt/parseFloat/parseFileLine/isNA reproduce CPython's +// coercions exactly, including the underscore digit separator, the +// rfind(":") split and the "non-positive line means no line" rule. +// - Each parse_*_raw turns a flat *Raw model into the structured model with +// the same field values Python produces, byte for byte after JSON encoding. + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// primitivesGolden mirrors testdata/golden/parse_primitives.json. +type primitivesGolden struct { + SplitPipe []struct { + S string `json:"s"` + Expected int `json:"expected"` + Want []string `json:"want"` + } `json:"split_pipe"` + ParseBool []struct { + S string `json:"s"` + Want *bool `json:"want"` + } `json:"parse_bool"` + ParseInt []struct { + S string `json:"s"` + Want int `json:"want"` + } `json:"parse_int"` + ParseIntDefault9 []struct { + S string `json:"s"` + Want int `json:"want"` + } `json:"parse_int_default9"` + ParseFloat []struct { + S string `json:"s"` + Want *string `json:"want"` // Python repr(), or null + } `json:"parse_float"` + ParseFileLine []struct { + S string `json:"s"` + Path string `json:"path"` + Line int `json:"line"` + } `json:"parse_file_line"` + IsNA []struct { + S string `json:"s"` + Want bool `json:"want"` + } `json:"is_na"` +} + +// TestParsePrimitives ports the behavior of _split_pipe, _parse_bool, +// _parse_int, _parse_float, _parse_file_line and _is_na. +func TestParsePrimitives(t *testing.T) { + var g primitivesGolden + goldenJSON(t, "parse_primitives", &g) + + t.Run("split_pipe", func(t *testing.T) { + if len(g.SplitPipe) == 0 { + t.Fatal("golden has no split_pipe cases") + } + for _, c := range g.SplitPipe { + got := splitPipe(c.S, c.Expected) + if !reflect.DeepEqual(got, c.Want) { + t.Errorf("splitPipe(%q, %d) = %q, want %q", c.S, c.Expected, got, c.Want) + } + } + }) + + t.Run("parse_bool", func(t *testing.T) { + for _, c := range g.ParseBool { + got := parseBool(c.S) + if !equalBoolPtr(got, c.Want) { + t.Errorf("parseBool(%q) = %s, want %s", c.S, fmtBoolPtr(got), fmtBoolPtr(c.Want)) + } + } + }) + + t.Run("parse_int", func(t *testing.T) { + for _, c := range g.ParseInt { + if got := parseInt(c.S, 0); got != c.Want { + t.Errorf("parseInt(%q, 0) = %d, want %d", c.S, got, c.Want) + } + } + for _, c := range g.ParseIntDefault9 { + if got := parseInt(c.S, 9); got != c.Want { + t.Errorf("parseInt(%q, 9) = %d, want %d", c.S, got, c.Want) + } + } + }) + + t.Run("parse_float", func(t *testing.T) { + for _, c := range g.ParseFloat { + got := parseFloat(c.S) + switch { + case c.Want == nil && got != nil: + t.Errorf("parseFloat(%q) = %v, want nil", c.S, *got) + case c.Want != nil && got == nil: + t.Errorf("parseFloat(%q) = nil, want %s", c.S, *c.Want) + case c.Want != nil && got != nil: + // Compared through pyfmt.FormatFloat (an exact port of Python's + // str(float)) so inf/nan compare by spelling instead of by an + // equality operator that NaN fails. + if spelled := pyfmt.FormatFloat(*got); spelled != *c.Want { + t.Errorf("parseFloat(%q) = %s, want %s", c.S, spelled, *c.Want) + } + } + } + }) + + t.Run("parse_file_line", func(t *testing.T) { + for _, c := range g.ParseFileLine { + path, line := parseFileLine(c.S) + if path != c.Path || line != c.Line { + t.Errorf("parseFileLine(%q) = (%q, %d), want (%q, %d)", c.S, path, line, c.Path, c.Line) + } + } + }) + + t.Run("is_na", func(t *testing.T) { + for _, c := range g.IsNA { + if got := isNA(c.S); got != c.Want { + t.Errorf("isNA(%q) = %v, want %v", c.S, got, c.Want) + } + } + }) +} + +// parseGolden is the {input, want} envelope every parse_.json uses. +type parseGolden struct { + Input json.RawMessage `json:"input"` + Want any `json:"want"` +} + +// runParserGolden decodes the golden's flat *Raw input into T, runs parse, and +// compares the JSON shape of the result with Python's model_dump(). +func runParserGolden[Raw any, Out any](t *testing.T, name string, parse func(Raw) Out) { + t.Helper() + var g parseGolden + goldenJSON(t, name, &g) + + var raw Raw + if err := json.Unmarshal(g.Input, &raw); err != nil { + t.Fatalf("%s: decode input into %T: %v", name, raw, err) + } + + got := scrubIDs(jsonTree(t, parse(raw))) + if !reflect.DeepEqual(got, g.Want) { + t.Errorf("%s mismatch with Python%s", name, diffJSON(t, got, g.Want)) + } +} + +// TestParseArchitectureRaw ports parse_architecture_raw. +func TestParseArchitectureRaw(t *testing.T) { + runParserGolden(t, "parse_architecture", ParseArchitectureRaw) +} + +// TestParseDataFlowRaw ports parse_data_flow_raw. +func TestParseDataFlowRaw(t *testing.T) { + runParserGolden(t, "parse_data_flow", ParseDataFlowRaw) +} + +// TestParseDependencyReportRaw ports parse_dependency_report_raw. +func TestParseDependencyReportRaw(t *testing.T) { + runParserGolden(t, "parse_dependency_report", ParseDependencyReportRaw) +} + +// TestParseConfigReportRaw ports parse_config_report_raw. +func TestParseConfigReportRaw(t *testing.T) { + runParserGolden(t, "parse_config_report", ParseConfigReportRaw) +} + +// TestParseSecurityContextRaw ports parse_security_context_raw. +func TestParseSecurityContextRaw(t *testing.T) { + runParserGolden(t, "parse_security_context", ParseSecurityContextRaw) +} + +// TestParsersEmitEmptyListsNotNull pins that every list a parser produces is +// non-nil, so it serializes as `[]` like a pydantic default_factory=list field +// and never as `null`. The orchestrator round-trips these models through the +// control plane as JSON, so a null where Python sends [] would break the +// receiving model_validate. +func TestParsersEmitEmptyListsNotNull(t *testing.T) { + arch := ParseArchitectureRaw(schemas.NewArchitectureMapRaw()) + df := ParseDataFlowRaw(schemas.NewDataFlowMapRaw()) + dep := ParseDependencyReportRaw(schemas.NewDependencyReportRaw()) + cfg := ParseConfigReportRaw(schemas.NewConfigReportRaw()) + sec := ParseSecurityContextRaw(schemas.NewSecurityContextRaw()) + + for _, tc := range []struct { + name string + v any + want string + }{ + {"ArchitectureMap", arch, `{"app_type":"unknown","modules":[],"entry_points":[],"trust_boundaries":[],"services":[],"api_surface":[]}`}, + {"DataFlowMap", df, `{"flows":[],"sanitization_points":[],"sinks":[]}`}, + {"DependencyReport", dep, `{"sbom":[],"known_cves":[],"outdated":[],"direct_count":0,"transitive_count":0}`}, + {"ConfigReport", cfg, `{"secrets":[],"misconfigs":[]}`}, + {"SecurityContext", sec, `{"auth_model":"","auth_details":"","crypto_usage":[],"framework_security":[],"security_headers":[],"deployment_signals":[]}`}, + } { + b, err := json.Marshal(tc.v) + if err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + if string(b) != tc.want { + t.Errorf("%s = %s, want %s", tc.name, b, tc.want) + } + } +} + +// TestParseArchitectureRawRouteDerivation pins the EntryPoint.route rule, which +// is derived rather than parsed: the identifier doubles as the route only when +// it contains a "/". +func TestParseArchitectureRawRouteDerivation(t *testing.T) { + raw := schemas.NewArchitectureMapRaw() + raw.EntryPoints = []string{ + "http | POST /api/login | src/routes.py:42 | false", + "cli | migrate | src/cli.py:8 | true", + } + got := ParseArchitectureRaw(raw) + + if got.EntryPoints[0].Route == nil || *got.EntryPoints[0].Route != "POST /api/login" { + t.Errorf("route for an identifier containing '/' = %v, want the identifier", got.EntryPoints[0].Route) + } + if got.EntryPoints[1].Route != nil { + t.Errorf("route for an identifier without '/' = %q, want nil", *got.EntryPoints[1].Route) + } + // method is never populated by the parser; it keeps the pydantic default. + if got.EntryPoints[0].Method != nil { + t.Errorf("method = %q, want nil (never parsed)", *got.EntryPoints[0].Method) + } +} + +// TestParseDependencyReportRawCounts pins that direct/transitive counts come +// only from the sbom rows and that an unparseable `direct` flag counts as +// transitive. +func TestParseDependencyReportRawCounts(t *testing.T) { + raw := schemas.NewDependencyReportRaw() + raw.Sbom = []string{ + "a | 1 | pypi | true | MIT", + "b | 1 | pypi | false | MIT", + "c | 1 | pypi | garbage | MIT", + } + // known_cves rows carry their own `direct` flag but must not move the + // counters. + raw.KnownCves = []string{"CVE-1 | a | 1 | 2 | 9.8 | true | true"} + + got := ParseDependencyReportRaw(raw) + if got.DirectCount != 1 || got.TransitiveCount != 2 { + t.Errorf("counts = (direct %d, transitive %d), want (1, 2)", got.DirectCount, got.TransitiveCount) + } +} + +// TestSecuritySignalBucketingIsFirstMatchWins pins the ordering of the two +// substring tables: a signal matching BOTH a header term and a deployment term +// lands in security_headers, because the header test runs first. +func TestSecuritySignalBucketingIsFirstMatchWins(t *testing.T) { + raw := schemas.NewSecurityContextRaw() + raw.SecuritySignals = []string{"HSTS header set by the Docker ingress"} + + got := ParseSecurityContextRaw(raw) + if len(got.SecurityHeaders) != 1 || len(got.DeploymentSignals) != 0 || len(got.FrameworkSecurity) != 0 { + t.Errorf("buckets = headers %v, deployment %v, framework %v; want the signal in headers only", + got.SecurityHeaders, got.DeploymentSignals, got.FrameworkSecurity) + } +} + +func equalBoolPtr(a, b *bool) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return *a == *b +} + +func fmtBoolPtr(b *bool) string { + if b == nil { + return "nil" + } + if *b { + return "true" + } + return "false" +} diff --git a/internal/secaf/agents/recon/prompts_test.go b/internal/secaf/agents/recon/prompts_test.go new file mode 100644 index 0000000000..fc8109d058 --- /dev/null +++ b/internal/secaf/agents/recon/prompts_test.go @@ -0,0 +1,177 @@ +package recon + +// Validation contract for the five RECON prompt builders and for +// architecture_context_block: +// +// - Each mapper's prompt is its template followed by a literal CONTEXT block +// naming the repository path. The three repo-only mappers share one suffix +// ("start by listing files"); the two architecture-aware mappers share a +// different one ("take multiple turns to explore"). +// - The architecture-aware templates have {{ARCHITECTURE_MAP_JSON}} replaced +// by json.dumps(architecture.model_dump(), indent=2) BEFORE the suffix is +// appended. +// - Every byte of all of the above reaches the LLM, so all of it is compared +// against fixtures captured from the real Python builders. + +import ( + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// fixtureRepo is the stable repository path gen_golden.py interpolates. +const fixtureRepo = "/fixtures/demo-repo" + +// archRich mirrors gen_golden.py's arch_rich(): every optional field +// populated, plus the characters where CPython's json.dumps and Go's +// encoding/json disagree — `<`, `>` and `&` (Go escapes them, CPython does +// not) and non-ASCII (CPython escapes as \uXXXX, Go does not). +func archRich() schemas.ArchitectureMap { + str := func(s string) *string { return &s } + b := func(v bool) *bool { return &v } + + return schemas.ArchitectureMap{ + AppType: str("web_api"), + Modules: []schemas.Module{ + {Name: "auth", Path: "src/auth/", Language: "Python", Description: str("Sessions & tokens"), Dependencies: []string{"db", "cache"}}, + {Name: "ui", Path: "web/", Language: "TypeScript", Description: nil, Dependencies: []string{}}, + }, + EntryPoints: []schemas.EntryPoint{ + {Kind: "http", Identifier: "POST /api/login", FilePath: "src/routes.py", Line: 42, Method: str("POST"), Route: str("/api/login"), AuthRequired: b(false)}, + {Kind: "cli", Identifier: "migrate", FilePath: "src/cli.py", Line: 8}, + }, + TrustBoundaries: []schemas.TrustBoundary{ + {Name: "API Gateway", SourceZone: "external", TargetZone: "internal", Description: "Rate limiting auth — café → app", Enforcement: []string{"waf"}}, + }, + Services: []schemas.Service{ + {Name: "PostgreSQL", ServiceType: "database", Endpoint: str("localhost:5432"), Purpose: str("primary store"), AuthMechanism: str("password")}, + }, + APISurface: []schemas.APIEndpoint{ + {Method: "GET", Path: "/api/users", Handler: "get_users", FilePath: "src/api.py", Line: 15, AuthRequired: b(true), RateLimited: b(false)}, + }, + } +} + +// archEmpty mirrors gen_golden.py's arch_empty() — `ArchitectureMap()`, i.e. +// every pydantic default. +func archEmpty() schemas.ArchitectureMap { return schemas.NewArchitectureMap() } + +// TestArchitectureContextBlock pins architecture_context_block against +// CPython's json.dumps(model_dump(), indent=2). +func TestArchitectureContextBlock(t *testing.T) { + for _, tc := range []struct { + name string + arch schemas.ArchitectureMap + golden string + }{ + {"rich", archRich(), "architecture_context_block_A"}, + {"empty", archEmpty(), "architecture_context_block_B"}, + } { + t.Run(tc.name, func(t *testing.T) { + got := ArchitectureContextBlock(tc.arch) + want := goldenText(t, tc.golden) + if got != want { + t.Errorf("ArchitectureContextBlock mismatch\n--- got ---\n%s\n--- want (python) ---\n%s", got, want) + } + }) + } +} + +// TestArchitectureContextBlockNilSliceRendering pins the one documented place +// where pyfmt.Dumps and CPython diverge, so a future reader knows it is +// deliberate: a nil Go slice renders as `null`, whereas pydantic's +// `Field(default_factory=list)` always dumps as `[]`. +// +// The divergence is unreachable through the pipeline — every ArchitectureMap +// here comes from ParseArchitectureRaw (non-nil slices) or from JSON via +// ArchitectureMap.UnmarshalJSON (seeds the `[]` defaults) — so the test asserts +// BOTH: the seeded value matches Python's ArchitectureMap(), and the bare Go +// literal does not. +func TestArchitectureContextBlockNilSliceRendering(t *testing.T) { + pythonDefaults := goldenText(t, "architecture_context_block_B") + + if got := ArchitectureContextBlock(schemas.NewArchitectureMap()); got != pythonDefaults { + t.Errorf("NewArchitectureMap() block mismatch\n--- got ---\n%s\n--- want (python ArchitectureMap()) ---\n%s", + got, pythonDefaults) + } + + bare := ArchitectureContextBlock(schemas.ArchitectureMap{}) + if bare == pythonDefaults { + t.Fatal("a bare ArchitectureMap{} now renders like the pydantic defaults; " + + "pyfmt.Dumps' nil-slice rule changed and this test's premise is stale") + } + if !strings.Contains(bare, `"modules": null`) { + t.Errorf("bare ArchitectureMap{} should render nil slices as null, got:\n%s", bare) + } +} + +// TestReconPrompts pins every prompt string the five mappers hand to +// app.harness. +func TestReconPrompts(t *testing.T) { + for _, tc := range []struct { + name string + got string + golden string + }{ + {"architecture", architecturePrompt(fixtureRepo), "architecture_prompt"}, + {"dependencies", dependenciesPrompt(fixtureRepo), "dependencies_prompt"}, + {"config_scanner", configScannerPrompt(fixtureRepo), "config_scanner_prompt"}, + {"data_flow/rich", dataFlowPrompt(fixtureRepo, archRich()), "data_flow_prompt_A"}, + {"data_flow/empty", dataFlowPrompt(fixtureRepo, archEmpty()), "data_flow_prompt_B"}, + {"security_context/rich", securityContextPrompt(fixtureRepo, archRich()), "security_context_prompt_A"}, + {"security_context/empty", securityContextPrompt(fixtureRepo, archEmpty()), "security_context_prompt_B"}, + } { + t.Run(tc.name, func(t *testing.T) { + want := goldenText(t, tc.golden) + if tc.got != want { + t.Errorf("prompt mismatch (%d bytes vs %d)\n--- got ---\n%s\n--- want (python) ---\n%s", + len(tc.got), len(want), tc.got, want) + } + }) + } +} + +// TestPromptsSubstitutePlaceholder pins that no {{ARCHITECTURE_MAP_JSON}} +// token survives into a prompt, and that the substituted block is really the +// context block. +func TestPromptsSubstitutePlaceholder(t *testing.T) { + arch := archRich() + block := ArchitectureContextBlock(arch) + for _, tc := range []struct { + name string + prompt string + }{ + {"data_flow", dataFlowPrompt(fixtureRepo, arch)}, + {"security_context", securityContextPrompt(fixtureRepo, arch)}, + } { + t.Run(tc.name, func(t *testing.T) { + if strings.Contains(tc.prompt, architectureMapPlaceholder) { + t.Errorf("prompt still contains %s", architectureMapPlaceholder) + } + if !strings.Contains(tc.prompt, block) { + t.Error("prompt does not contain the architecture context block") + } + }) + } +} + +// TestContextSuffixes pins the two literal CONTEXT blocks, which are the only +// part of a RECON prompt the Go code composes rather than loads. +func TestContextSuffixes(t *testing.T) { + wantListing := "\n\nCONTEXT:\n" + + "- Repository path: /repo\n" + + "- Start by listing files in the repository path above.\n" + + "- After gathering evidence, write the JSON output file using your Write tool." + if got := fileListingContextSuffix("/repo"); got != wantListing { + t.Errorf("fileListingContextSuffix = %q, want %q", got, wantListing) + } + + wantExploration := "\n\nCONTEXT:\n" + + "- Repository path: /repo\n" + + "- Take multiple turns to explore the codebase first, then build your analysis.\n" + + "- Write final JSON only when analysis is complete." + if got := explorationContextSuffix("/repo"); got != wantExploration { + t.Errorf("explorationContextSuffix = %q, want %q", got, wantExploration) + } +} diff --git a/internal/secaf/agents/recon/recon.go b/internal/secaf/agents/recon/recon.go new file mode 100644 index 0000000000..61e42299fb --- /dev/null +++ b/internal/secaf/agents/recon/recon.go @@ -0,0 +1,253 @@ +package recon + +// Ports the orchestration half of src/sec_af/agents/recon/__init__.py: +// _normalize_depth, _quick_defaults, run_recon, run_fast_recon, run_deep_recon. + +import ( + "context" + "sort" + "strings" + "time" + + "golang.org/x/sync/errgroup" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/config" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// QuickDefaults ports _quick_defaults — the placeholder pair the QUICK depth +// profile substitutes for the two expensive architecture-aware mappers. +// +// Python parity: `SecurityContext(auth_model="unknown", auth_details="unknown")` +// uses the literal "unknown" for BOTH fields. That differs from the +// `{"auth_model": "unknown", "auth_details": ""}` default that +// reasoners/phases.py `_recon_model` seeds; do not conflate them. +func QuickDefaults() (schemas.DataFlowMap, schemas.SecurityContext) { + dataFlows := schemas.NewDataFlowMap() + securityContext := schemas.NewSecurityContext() + securityContext.AuthModel = "unknown" + securityContext.AuthDetails = "unknown" + return dataFlows, securityContext +} + +// runBaseMappers runs the three repo-only mappers concurrently, which is +// Python's +// +// architecture, dependencies, config = await asyncio.gather( +// run_architecture_mapper(app, repo_path), +// run_dependency_auditor(app, repo_path), +// run_config_scanner(app, repo_path), +// ) +// +// Concurrency parity, and its one deliberate difference: +// +// - errgroup.Group is used WITHOUT WithContext, so a failing mapper does not +// cancel its siblings — matching asyncio.gather(return_exceptions=False), +// which never cancels the other awaitables either. +// - Wait() returns the FIRST error by completion time (errgroup guards the +// slot with a sync.Once), which is the same error asyncio.gather surfaces. +// - DIFFERENCE: Wait() blocks until all three goroutines have finished, while +// `await gather(...)` resumes the caller as soon as the first exception +// fires and leaves the rest running detached. The value returned is +// identical; only the moment of return differs, and the caller's next act +// on error is to propagate a 500 either way. +func runBaseMappers(ctx context.Context, app appx.Harnesser, repoPath string) ( + schemas.ArchitectureMap, schemas.DependencyReport, schemas.ConfigReport, error, +) { + var ( + architecture schemas.ArchitectureMap + dependencies schemas.DependencyReport + configReport schemas.ConfigReport + ) + var g errgroup.Group + g.Go(func() error { + var err error + architecture, err = RunArchitectureMapper(ctx, app, repoPath) + return err + }) + g.Go(func() error { + var err error + dependencies, err = RunDependencyAuditor(ctx, app, repoPath) + return err + }) + g.Go(func() error { + var err error + configReport, err = RunConfigScanner(ctx, app, repoPath) + return err + }) + if err := g.Wait(); err != nil { + return architecture, dependencies, configReport, err + } + return architecture, dependencies, configReport, nil +} + +// RunDeepRecon ports run_deep_recon — the second gather, over the two mappers +// that need the architecture map as input. +// +// data_flows, security_context = await asyncio.gather( +// run_data_flow_mapper(app, repo_path, architecture), +// run_security_context_profiler(app, repo_path, architecture), +// ) +// +// Same errgroup contract as runBaseMappers. +func RunDeepRecon(ctx context.Context, app appx.Harnesser, repoPath string, architecture schemas.ArchitectureMap) ( + schemas.DataFlowMap, schemas.SecurityContext, error, +) { + var ( + dataFlows schemas.DataFlowMap + securityContext schemas.SecurityContext + ) + var g errgroup.Group + g.Go(func() error { + var err error + dataFlows, err = RunDataFlowMapper(ctx, app, repoPath, architecture) + return err + }) + g.Go(func() error { + var err error + securityContext, err = RunSecurityContextProfiler(ctx, app, repoPath, architecture) + return err + }) + if err := g.Wait(); err != nil { + return dataFlows, securityContext, err + } + return dataFlows, securityContext, nil +} + +// RunRecon ports run_recon — the full RECON phase. +// +// started = time.monotonic() +// profile = _normalize_depth(depth) +// architecture, dependencies, config = await gather(3 mappers) +// if profile == QUICK: data_flows, security_context = _quick_defaults() +// else: data_flows, security_context = await gather(2 mappers) +// languages = sorted({m.language.lower() for m in architecture.modules if m.language}) +// frameworks = sorted({s for s in security_context.framework_security if s}) +// lines_of_code, file_count = _repo_metrics(repo_path) +// return ReconResult(..., recon_duration_seconds=time.monotonic() - started) +// +// Python parity: +// +// - depth is normalized LENIENTLY (config.NormalizeDepth): anything that is +// not quick/standard/thorough silently becomes standard, so only the exact +// string "quick" (any case) skips the deep mappers. +// - languages are lowercased and deduplicated; frameworks keep their original +// case. Both are sorted, so the Python `set` iteration order — the only +// nondeterminism in this function — is erased in both runtimes. +// - recon_duration_seconds is set HERE and only here; run_fast_recon leaves it +// at its 0.0 default. +func RunRecon(ctx context.Context, app appx.Harnesser, repoPath string, depth string) (schemas.ReconResult, error) { + started := time.Now() + profile := config.NormalizeDepth(depth) + + architecture, dependencies, configReport, err := runBaseMappers(ctx, app, repoPath) + if err != nil { + return schemas.ReconResult{}, err + } + + var ( + dataFlows schemas.DataFlowMap + securityContext schemas.SecurityContext + ) + if profile == config.DepthQuick { + dataFlows, securityContext = QuickDefaults() + } else { + dataFlows, securityContext, err = RunDeepRecon(ctx, app, repoPath, architecture) + if err != nil { + return schemas.ReconResult{}, err + } + } + + linesOfCode, fileCount := RepoMetrics(repoPath) + + return schemas.ReconResult{ + Architecture: architecture, + DataFlows: dataFlows, + Dependencies: dependencies, + Config: configReport, + SecurityContext: securityContext, + Languages: moduleLanguages(architecture), + Frameworks: sortedNonEmpty(securityContext.FrameworkSecurity), + LinesOfCode: linesOfCode, + FileCount: fileCount, + ReconDurationSeconds: time.Since(started).Seconds(), + }, nil +} + +// RunFastRecon ports run_fast_recon — the three cheap mappers plus the QUICK +// placeholders, used by the orchestrator's streaming path. +// +// Python parity: it differs from RunRecon(depth="quick") in exactly two ways, +// both of which look like oversights but are reproduced faithfully — +// `frameworks` is hard-coded to the empty list instead of being derived from +// the (placeholder) security context, and `recon_duration_seconds` is never +// set, so it keeps its 0.0 pydantic default. +func RunFastRecon(ctx context.Context, app appx.Harnesser, repoPath string) (schemas.ReconResult, error) { + architecture, dependencies, configReport, err := runBaseMappers(ctx, app, repoPath) + if err != nil { + return schemas.ReconResult{}, err + } + + dataFlows, securityContext := QuickDefaults() + linesOfCode, fileCount := RepoMetrics(repoPath) + + return schemas.ReconResult{ + Architecture: architecture, + DataFlows: dataFlows, + Dependencies: dependencies, + Config: configReport, + SecurityContext: securityContext, + Languages: moduleLanguages(architecture), + Frameworks: []string{}, + LinesOfCode: linesOfCode, + FileCount: fileCount, + }, nil +} + +// moduleLanguages ports +// `sorted({module.language.lower() for module in architecture.modules if getattr(module, "language", None)})`. +// +// Python parity: the `if getattr(...)` guard is a truthiness test, so a module +// whose language is the empty string is dropped BEFORE lowering — an empty +// string never appears in the result. +func moduleLanguages(architecture schemas.ArchitectureMap) []string { + seen := make(map[string]struct{}, len(architecture.Modules)) + out := []string{} + for _, module := range architecture.Modules { + if module.Language == "" { + continue + } + lang := strings.ToLower(module.Language) + if _, dup := seen[lang]; dup { + continue + } + seen[lang] = struct{}{} + out = append(out, lang) + } + sort.Strings(out) + return out +} + +// sortedNonEmpty ports `sorted({item for item in values if item})` — drop empty +// strings, deduplicate, sort. +// +// sort.Strings orders by byte, and Python's sorted() orders strings by code +// point; UTF-8 preserves code-point order under byte comparison, so the two +// agree for every input. +func sortedNonEmpty(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := []string{} + for _, v := range values { + if v == "" { + continue + } + if _, dup := seen[v]; dup { + continue + } + seen[v] = struct{}{} + out = append(out, v) + } + sort.Strings(out) + return out +} diff --git a/internal/secaf/agents/recon/recon_test.go b/internal/secaf/agents/recon/recon_test.go new file mode 100644 index 0000000000..1fee00572e --- /dev/null +++ b/internal/secaf/agents/recon/recon_test.go @@ -0,0 +1,702 @@ +package recon + +// Validation contract for run_recon / run_fast_recon / run_deep_recon and the +// five mapper wrappers: +// +// - RECON issues exactly five harness calls at standard/thorough depth and +// exactly three at quick depth; there are no .call() reasoner invocations +// inside this package (the DAG fan-out lives in reasoners/phases). +// - The first three mappers run CONCURRENTLY (one asyncio.gather), and so do +// the two architecture-aware ones (a second gather). +// - Each mapper runs with Cwd set to a fresh private temp dir named +// `secaf--*` that is removed afterwards, and ProjectDir set to +// the repository. +// - Each mapper's harness schema is the pydantic schema of its *Raw model. +// - A mapper failure surfaces as an error naming the agent the way +// extract_harness_result names it. +// - depth normalization is lenient: only "quick" (any case) skips the deep +// mappers; anything unrecognized behaves like "standard". +// - languages are lowercased/deduplicated/sorted from the architecture's +// modules; frameworks are deduplicated/sorted from the security context — +// except in run_fast_recon, which hard-codes them empty. +// - recon_duration_seconds is set by run_recon only. + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// mapperOf identifies which RECON mapper a captured prompt belongs to, by the +// template it starts with. Returns "" for anything unrecognized. +func mapperOf(prompt string) string { + for _, m := range []struct{ name, path string }{ + {"architecture", architecturePromptPath}, + {"dependencies", dependenciesPromptPath}, + {"config_scanner", configScannerPromptPath}, + {"data_flow", dataFlowPromptPath}, + {"security_context", securityContextPromptPath}, + } { + template := prompts.MustLoad(m.path) + // The two architecture-aware templates carry a placeholder that is + // substituted before the prompt is sent, so compare on the prefix up to + // the placeholder. + if head, _, found := strings.Cut(template, architectureMapPlaceholder); found { + if strings.HasPrefix(prompt, head) { + return m.name + } + continue + } + if strings.HasPrefix(prompt, template) { + return m.name + } + } + return "" +} + +// reconGolden mirrors testdata/golden/run_recon.json — the end-to-end phase +// fixture: one canned flat harness payload per mapper, plus the ReconResult +// the REAL Python run_recon / run_fast_recon build from it. +type reconGolden struct { + RepoPath string `json:"repo_path"` + Canned map[string]json.RawMessage `json:"canned"` + Standard map[string]any `json:"standard"` + Quick map[string]any `json:"quick"` + Fast map[string]any `json:"fast"` +} + +// loadReconGolden reads the fixture once per test. +func loadReconGolden(t *testing.T) reconGolden { + t.Helper() + var g reconGolden + goldenJSON(t, "run_recon", &g) + if len(g.Canned) != 5 { + t.Fatalf("golden has %d canned mapper payloads, want 5", len(g.Canned)) + } + return g +} + +// barrier releases all its waiters once n of them have arrived, or lets them +// through individually after timeout. It turns "these calls overlap" into a +// deterministic assertion instead of a sleep-and-hope race. +type barrier struct { + mu sync.Mutex + n int + count int + ch chan struct{} +} + +func newBarrier(n int) *barrier { return &barrier{n: n, ch: make(chan struct{})} } + +func (b *barrier) wait(timeout time.Duration) { + b.mu.Lock() + b.count++ + if b.count == b.n { + close(b.ch) + } + b.mu.Unlock() + select { + case <-b.ch: + case <-time.After(timeout): + } +} + +// newReconFake builds a Fake that answers every mapper with cannedRaw. When +// bar is non-nil each harness call parks on it first, so a test can prove the +// calls really are in flight together. +func newReconFake(t *testing.T, bar *barrier) *appx.Fake { + t.Helper() + canned := loadReconGolden(t).Canned + f := &appx.Fake{} + f.HarnessFn = func(_ context.Context, prompt string, _ map[string]any, dest any, _ appx.HarnessOptions) (*appx.HarnessResult, error) { + if bar != nil { + bar.wait(3 * time.Second) + } + name := mapperOf(prompt) + raw, ok := canned[name] + if !ok { + t.Errorf("harness called with an unrecognized prompt (first 120 bytes): %.120q", prompt) + return &appx.HarnessResult{IsError: true, ErrorMessage: "unrecognized prompt"}, nil + } + if err := json.Unmarshal(raw, dest); err != nil { + t.Fatalf("canned %s output does not fit its *Raw model: %v", name, err) + } + return &appx.HarnessResult{Parsed: dest, Result: string(raw)}, nil + } + return f +} + +// harnessMappers lists the mappers a Fake was asked to run, in call order. +func harnessMappers(f *appx.Fake) []string { + out := make([]string, 0, len(f.Harnesses)) + for _, h := range f.Harnesses { + out = append(out, mapperOf(h.Prompt)) + } + return out +} + +func sortedCopy(in []string) []string { + out := append([]string(nil), in...) + for i := range out { + for j := i + 1; j < len(out); j++ { + if out[j] < out[i] { + out[i], out[j] = out[j], out[i] + } + } + } + return out +} + +// TestRunReconStandardRunsAllFiveMappers pins the standard-depth shape: three +// mappers, then two more that consume the architecture. +func TestRunReconStandardRunsAllFiveMappers(t *testing.T) { + f := newReconFake(t, nil) + got, err := RunRecon(context.Background(), f, "/repo", "standard") + if err != nil { + t.Fatalf("RunRecon: %v", err) + } + + mappers := harnessMappers(f) + if len(mappers) != 5 { + t.Fatalf("harness calls = %v, want 5", mappers) + } + // The first gather's three may complete in any order; the last two are the + // architecture-aware pair and cannot start before it finishes. + first := sortedCopy(mappers[:3]) + wantFirst := []string{"architecture", "config_scanner", "dependencies"} + for i := range wantFirst { + if first[i] != wantFirst[i] { + t.Fatalf("first gather = %v, want %v", first, wantFirst) + } + } + last := sortedCopy(mappers[3:]) + if last[0] != "data_flow" || last[1] != "security_context" { + t.Fatalf("second gather = %v, want [data_flow security_context]", last) + } + + // Every mapper sees the repository as its project dir. + for i, h := range f.Harnesses { + if h.Opts.ProjectDir != "/repo" { + t.Errorf("harness[%d].ProjectDir = %q, want /repo", i, h.Opts.ProjectDir) + } + } + + // The architecture-aware prompts carry the architecture the first gather + // produced, not an empty one. + block := ArchitectureContextBlock(got.Architecture) + for _, h := range f.Harnesses[3:] { + if !strings.Contains(h.Prompt, block) { + t.Error("a deep mapper prompt does not embed the parsed architecture map") + } + } + + // No .call() reasoner invocations happen inside this package. + if len(f.Calls) != 0 { + t.Errorf("Call targets = %v, want none (the DAG fan-out lives in phases)", f.CallTargets()) + } +} + +// TestRunReconQuickSkipsDeepMappers ports the `if profile == DepthProfile.QUICK` +// branch: the two expensive mappers are replaced by _quick_defaults(). +func TestRunReconQuickSkipsDeepMappers(t *testing.T) { + f := newReconFake(t, nil) + got, err := RunRecon(context.Background(), f, "/repo", "quick") + if err != nil { + t.Fatalf("RunRecon: %v", err) + } + + mappers := sortedCopy(harnessMappers(f)) + want := []string{"architecture", "config_scanner", "dependencies"} + if len(mappers) != 3 { + t.Fatalf("harness calls = %v, want exactly the three cheap mappers", mappers) + } + for i := range want { + if mappers[i] != want[i] { + t.Fatalf("harness calls = %v, want %v", mappers, want) + } + } + + wantFlows, wantContext := QuickDefaults() + if !equalJSON(t, got.DataFlows, wantFlows) { + t.Errorf("data_flows = %s, want the quick default", mustJSON(t, got.DataFlows)) + } + if !equalJSON(t, got.SecurityContext, wantContext) { + t.Errorf("security_context = %s, want the quick default", mustJSON(t, got.SecurityContext)) + } + // Both auth fields are the literal "unknown" — NOT the {"unknown", ""} + // pair reasoners/phases.py seeds. + if got.SecurityContext.AuthModel != "unknown" || got.SecurityContext.AuthDetails != "unknown" { + t.Errorf("quick defaults auth = (%q, %q), want (unknown, unknown)", + got.SecurityContext.AuthModel, got.SecurityContext.AuthDetails) + } + // frameworks derive from the (empty) placeholder context. + if len(got.Frameworks) != 0 { + t.Errorf("frameworks = %v, want empty", got.Frameworks) + } +} + +// TestRunReconDepthNormalization ports _normalize_depth: lowercase first, and +// anything unrecognized becomes standard. +func TestRunReconDepthNormalization(t *testing.T) { + cases := []struct { + depth string + wantCalls int + }{ + {"quick", 3}, + {"QUICK", 3}, + {" quick", 5}, // not stripped: Python only lowercases + {"standard", 5}, + {"thorough", 5}, + {"", 5}, + {"nonsense", 5}, + } + for _, tc := range cases { + t.Run("depth="+tc.depth, func(t *testing.T) { + f := newReconFake(t, nil) + if _, err := RunRecon(context.Background(), f, "/repo", tc.depth); err != nil { + t.Fatalf("RunRecon: %v", err) + } + if len(f.Harnesses) != tc.wantCalls { + t.Errorf("depth %q -> %d harness calls, want %d", tc.depth, len(f.Harnesses), tc.wantCalls) + } + }) + } +} + +// TestRunReconFirstGatherIsConcurrent pins asyncio.gather over the three cheap +// mappers. +func TestRunReconFirstGatherIsConcurrent(t *testing.T) { + f := newReconFake(t, newBarrier(3)) + if _, err := RunRecon(context.Background(), f, "/repo", "quick"); err != nil { + t.Fatalf("RunRecon: %v", err) + } + if got := f.MaxConcurrentHarness(); got != 3 { + t.Errorf("max concurrent harness calls = %d, want 3 (one gather over three mappers)", got) + } +} + +// TestRunDeepReconIsConcurrent pins the second asyncio.gather. +func TestRunDeepReconIsConcurrent(t *testing.T) { + f := newReconFake(t, newBarrier(2)) + flows, secCtx, err := RunDeepRecon(context.Background(), f, "/repo", schemas.NewArchitectureMap()) + if err != nil { + t.Fatalf("RunDeepRecon: %v", err) + } + if got := f.MaxConcurrentHarness(); got != 2 { + t.Errorf("max concurrent harness calls = %d, want 2", got) + } + if len(flows.Flows) != 1 { + t.Errorf("flows = %d, want 1 (parsed from the canned data-flow output)", len(flows.Flows)) + } + if secCtx.AuthModel != "jwt" { + t.Errorf("auth_model = %q, want jwt", secCtx.AuthModel) + } +} + +// TestMapperTempDirsAreIsolatedAndRemoved pins the +// `tempfile.mkdtemp(prefix=f"secaf-{agent_name}-")` / +// `shutil.rmtree(..., ignore_errors=True)` pair, including the exact agent +// names that appear in the prefix. +func TestMapperTempDirsAreIsolatedAndRemoved(t *testing.T) { + f := newReconFake(t, nil) + if _, err := RunRecon(context.Background(), f, "/repo", "standard"); err != nil { + t.Fatalf("RunRecon: %v", err) + } + + wantPrefix := map[string]string{ + "architecture": "secaf-recon-architecture-", + "dependencies": "secaf-recon-dependencies-", + "config_scanner": "secaf-recon-config-scanner-", + "data_flow": "secaf-recon-data-flow-", + "security_context": "secaf-recon-security-context-", + } + + seen := map[string]bool{} + for i, h := range f.Harnesses { + name := mapperOf(h.Prompt) + if h.Opts.Cwd == "" { + t.Fatalf("harness[%d] (%s) ran with no Cwd", i, name) + } + base := filepath.Base(h.Opts.Cwd) + if !strings.HasPrefix(base, wantPrefix[name]) { + t.Errorf("harness[%d] (%s) Cwd base = %q, want prefix %q", i, name, base, wantPrefix[name]) + } + if seen[h.Opts.Cwd] { + t.Errorf("two mappers shared the temp dir %q", h.Opts.Cwd) + } + seen[h.Opts.Cwd] = true + if _, err := os.Stat(h.Opts.Cwd); !os.IsNotExist(err) { + t.Errorf("temp dir %q still exists after the mapper returned (err=%v)", h.Opts.Cwd, err) + } + } +} + +// TestMapperHarnessSchemas pins that each mapper asks for its own *Raw model's +// pydantic schema — the Go-type-name -> fixture contract harnessx relies on. +func TestMapperHarnessSchemas(t *testing.T) { + f := newReconFake(t, nil) + if _, err := RunRecon(context.Background(), f, "/repo", "standard"); err != nil { + t.Fatalf("RunRecon: %v", err) + } + + wantKeys := map[string][]string{ + "architecture": {"app_type", "modules", "entry_points", "trust_boundaries", "services", "api_endpoints"}, + "dependencies": {"sbom", "known_cves", "outdated"}, + "config_scanner": {"secrets", "misconfigs"}, + "data_flow": {"flows", "sanitization_points", "sinks"}, + "security_context": {"auth_model", "auth_details", "crypto_usage", "security_signals"}, + } + for i, h := range f.Harnesses { + name := mapperOf(h.Prompt) + props, _ := h.Schema["properties"].(map[string]any) + if props == nil { + t.Errorf("harness[%d] (%s) schema has no properties: %v", i, name, h.Schema) + continue + } + for _, key := range wantKeys[name] { + if _, ok := props[key]; !ok { + t.Errorf("harness[%d] (%s) schema is missing property %q", i, name, key) + } + } + if len(props) != len(wantKeys[name]) { + t.Errorf("harness[%d] (%s) schema has %d properties, want %d", i, name, len(props), len(wantKeys[name])) + } + } +} + +// TestMapperErrorNames pins the agent names extract_harness_result puts in the +// error string, which is what an operator sees when a mapper fails. +func TestMapperErrorNames(t *testing.T) { + cases := []struct { + name string + run func(context.Context, appx.Harnesser) error + want string + }{ + {"architecture", func(ctx context.Context, a appx.Harnesser) error { + _, err := RunArchitectureMapper(ctx, a, "/repo") + return err + }, "Architecture mapper harness error: boom"}, + {"dependencies", func(ctx context.Context, a appx.Harnesser) error { + _, err := RunDependencyAuditor(ctx, a, "/repo") + return err + }, "Dependency auditor harness error: boom"}, + {"config_scanner", func(ctx context.Context, a appx.Harnesser) error { + _, err := RunConfigScanner(ctx, a, "/repo") + return err + }, "Config scanner harness error: boom"}, + {"data_flow", func(ctx context.Context, a appx.Harnesser) error { + _, err := RunDataFlowMapper(ctx, a, "/repo", schemas.NewArchitectureMap()) + return err + }, "Data flow mapper harness error: boom"}, + {"security_context", func(ctx context.Context, a appx.Harnesser) error { + _, err := RunSecurityContextProfiler(ctx, a, "/repo", schemas.NewArchitectureMap()) + return err + }, "Security context profiler harness error: boom"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := &appx.Fake{HarnessFn: func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{IsError: true, ErrorMessage: "boom"}, nil + }} + err := tc.run(context.Background(), f) + if err == nil || err.Error() != tc.want { + t.Errorf("error = %v, want %q", err, tc.want) + } + // The temp dir is still removed on the failure path (Python's + // `finally`). + if len(f.Harnesses) == 1 { + if _, statErr := os.Stat(f.Harnesses[0].Opts.Cwd); !os.IsNotExist(statErr) { + t.Errorf("temp dir survived the failure path: %v", statErr) + } + } + }) + } +} + +// TestRunReconPropagatesMapperError pins that a failure in either gather aborts +// the phase with that error. +func TestRunReconPropagatesMapperError(t *testing.T) { + for _, tc := range []struct { + name string + failOn string + depth string + want string + }{ + {"first gather", "dependencies", "quick", "Dependency auditor harness error: nope"}, + {"second gather", "data_flow", "standard", "Data flow mapper harness error: nope"}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newReconFake(t, nil) + inner := f.HarnessFn + f.HarnessFn = func(ctx context.Context, prompt string, schema map[string]any, dest any, opts appx.HarnessOptions) (*appx.HarnessResult, error) { + if mapperOf(prompt) == tc.failOn { + return &appx.HarnessResult{IsError: true, ErrorMessage: "nope"}, nil + } + return inner(ctx, prompt, schema, dest, opts) + } + _, err := RunRecon(context.Background(), f, "/repo", tc.depth) + if err == nil || err.Error() != tc.want { + t.Errorf("error = %v, want %q", err, tc.want) + } + }) + } +} + +// TestRunReconPropagatesTransportError pins that a transport-level failure from +// the SDK (a non-nil error, not a Result with IsError) also aborts. +func TestRunReconPropagatesTransportError(t *testing.T) { + sentinel := errors.New("provider unreachable") + f := &appx.Fake{HarnessFn: func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return nil, sentinel + }} + if _, err := RunRecon(context.Background(), f, "/repo", "quick"); !errors.Is(err, sentinel) { + t.Errorf("error = %v, want the transport error", err) + } +} + +// TestRunReconLanguagesAndFrameworks pins the two derived, deduplicated, +// sorted lists. +func TestRunReconLanguagesAndFrameworks(t *testing.T) { + f := newReconFake(t, nil) + got, err := RunRecon(context.Background(), f, "/repo", "standard") + if err != nil { + t.Fatalf("RunRecon: %v", err) + } + + // The canned architecture has Python, TypeScript, python (a case-duplicate) + // and a module with an EMPTY language, which the truthiness guard drops. + wantLanguages := []string{"python", "typescript"} + if len(got.Languages) != len(wantLanguages) { + t.Fatalf("languages = %v, want %v", got.Languages, wantLanguages) + } + for i := range wantLanguages { + if got.Languages[i] != wantLanguages[i] { + t.Fatalf("languages = %v, want %v", got.Languages, wantLanguages) + } + } + + // The canned security signals contain "Uses Flask-Login" twice; only the + // framework bucket feeds frameworks, and the set collapses the duplicate. + wantFrameworks := []string{"Uses Flask-Login"} + if len(got.Frameworks) != len(wantFrameworks) || got.Frameworks[0] != wantFrameworks[0] { + t.Errorf("frameworks = %v, want %v", got.Frameworks, wantFrameworks) + } +} + +// TestModuleLanguagesAndSortedNonEmpty pins the two set-derivations directly, +// including the sort and the empty-string drop. +func TestModuleLanguagesAndSortedNonEmpty(t *testing.T) { + arch := schemas.NewArchitectureMap() + arch.Modules = []schemas.Module{ + {Name: "b", Language: "Zig"}, + {Name: "a", Language: "GO"}, + {Name: "c", Language: "go"}, + {Name: "d", Language: ""}, + } + got := moduleLanguages(arch) + want := []string{"go", "zig"} + if len(got) != 2 || got[0] != want[0] || got[1] != want[1] { + t.Errorf("moduleLanguages = %v, want %v", got, want) + } + + gotFw := sortedNonEmpty([]string{"beta", "alpha", "", "beta"}) + wantFw := []string{"alpha", "beta"} + if len(gotFw) != 2 || gotFw[0] != wantFw[0] || gotFw[1] != wantFw[1] { + t.Errorf("sortedNonEmpty = %v, want %v", gotFw, wantFw) + } + + if got := moduleLanguages(schemas.NewArchitectureMap()); got == nil { + t.Error("moduleLanguages returned nil, want an empty slice (Python sorted() gives [])") + } + if got := sortedNonEmpty(nil); got == nil { + t.Error("sortedNonEmpty returned nil, want an empty slice") + } +} + +// TestRunReconSetsDurationAndMetrics pins that run_recon times itself and folds +// in _repo_metrics. +func TestRunReconSetsDurationAndMetrics(t *testing.T) { + repo := t.TempDir() + if err := os.WriteFile(filepath.Join(repo, "main.py"), []byte("a\nb\nc\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(repo, "README.md"), []byte("x\n"), 0o644); err != nil { + t.Fatal(err) + } + + f := newReconFake(t, nil) + got, err := RunRecon(context.Background(), f, repo, "quick") + if err != nil { + t.Fatalf("RunRecon: %v", err) + } + if got.ReconDurationSeconds <= 0 { + t.Errorf("recon_duration_seconds = %v, want > 0", got.ReconDurationSeconds) + } + if got.LinesOfCode != 3 || got.FileCount != 2 { + t.Errorf("metrics = (lines %d, files %d), want (3, 2)", got.LinesOfCode, got.FileCount) + } +} + +// TestRunFastRecon pins the two deliberate differences from +// RunRecon(depth="quick"): frameworks is hard-coded empty and the duration is +// never set. +func TestRunFastRecon(t *testing.T) { + f := newReconFake(t, nil) + got, err := RunFastRecon(context.Background(), f, "/repo") + if err != nil { + t.Fatalf("RunFastRecon: %v", err) + } + if len(f.Harnesses) != 3 { + t.Errorf("harness calls = %v, want the three cheap mappers", harnessMappers(f)) + } + if got.ReconDurationSeconds != 0 { + t.Errorf("recon_duration_seconds = %v, want 0 (run_fast_recon never sets it)", got.ReconDurationSeconds) + } + if got.Frameworks == nil || len(got.Frameworks) != 0 { + t.Errorf("frameworks = %v, want an empty (non-nil) list", got.Frameworks) + } + if got.SecurityContext.AuthModel != "unknown" { + t.Errorf("security_context.auth_model = %q, want the quick default", got.SecurityContext.AuthModel) + } + // Languages ARE still derived, unlike frameworks. + if len(got.Languages) != 2 { + t.Errorf("languages = %v, want the two derived from the architecture", got.Languages) + } +} + +// TestRunFastReconIsConcurrent pins that run_fast_recon uses the same one +// gather over three mappers. +func TestRunFastReconIsConcurrent(t *testing.T) { + f := newReconFake(t, newBarrier(3)) + if _, err := RunFastRecon(context.Background(), f, "/repo"); err != nil { + t.Fatalf("RunFastRecon: %v", err) + } + if got := f.MaxConcurrentHarness(); got != 3 { + t.Errorf("max concurrent harness calls = %d, want 3", got) + } +} + +// TestQuickDefaults pins _quick_defaults exactly. +func TestQuickDefaults(t *testing.T) { + flows, ctx := QuickDefaults() + if got := mustJSON(t, flows); got != `{"flows":[],"sanitization_points":[],"sinks":[]}` { + t.Errorf("quick DataFlowMap = %s", got) + } + want := `{"auth_model":"unknown","auth_details":"unknown","crypto_usage":[],` + + `"framework_security":[],"security_headers":[],"deployment_signals":[]}` + if got := mustJSON(t, ctx); got != want { + t.Errorf("quick SecurityContext = %s, want %s", got, want) + } +} + +// TestRunReconResultIsFullyPopulated pins that every ReconResult field the +// pipeline downstream reads is set, and that the JSON round-trips (the +// orchestrator sends this over the control plane). +func TestRunReconResultIsFullyPopulated(t *testing.T) { + f := newReconFake(t, nil) + got, err := RunRecon(context.Background(), f, "/repo", "thorough") + if err != nil { + t.Fatalf("RunRecon: %v", err) + } + if got.Architecture.AppType == nil || *got.Architecture.AppType != "web_api" { + t.Errorf("architecture.app_type = %v", got.Architecture.AppType) + } + if len(got.Dependencies.Sbom) != 1 || got.Dependencies.DirectCount != 1 { + t.Errorf("dependencies = %+v", got.Dependencies) + } + if len(got.Config.Secrets) != 1 || len(got.Config.Misconfigs) != 1 { + t.Errorf("config = %+v", got.Config) + } + if len(got.DataFlows.Flows) != 1 { + t.Errorf("data_flows = %+v", got.DataFlows) + } + if got.SecurityContext.AuthModel != "jwt" { + t.Errorf("security_context.auth_model = %q", got.SecurityContext.AuthModel) + } + + var round schemas.ReconResult + if err := json.Unmarshal([]byte(mustJSON(t, got)), &round); err != nil { + t.Fatalf("ReconResult does not round-trip through JSON: %v", err) + } + if mustJSON(t, round) != mustJSON(t, got) { + t.Errorf("ReconResult JSON round-trip is lossy:\n%s\n%s", mustJSON(t, round), mustJSON(t, got)) + } +} + +func mustJSON(t *testing.T, v any) string { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatalf("marshal: %v", err) + } + return string(b) +} + +func equalJSON(t *testing.T, a, b any) bool { + t.Helper() + return mustJSON(t, a) == mustJSON(t, b) +} + +// TestRunReconMatchesPythonEndToEnd is the strongest parity check in this +// package: it feeds the Go phase the SAME canned harness output the REAL +// Python run_recon / run_fast_recon were fed by go/scripts/gen_golden.py, and +// compares the whole ReconResult. +// +// The two nondeterministic parts are normalized on both sides — uuid4 `id` +// fields (SecretFinding / MisconfigFinding mint one per parse) and +// recon_duration_seconds (a wall-clock measurement). +func TestRunReconMatchesPythonEndToEnd(t *testing.T) { + g := loadReconGolden(t) + + cases := []struct { + name string + run func(context.Context, appx.Harnesser) (schemas.ReconResult, error) + want map[string]any + }{ + {"standard", func(ctx context.Context, a appx.Harnesser) (schemas.ReconResult, error) { + return RunRecon(ctx, a, g.RepoPath, "standard") + }, g.Standard}, + {"quick", func(ctx context.Context, a appx.Harnesser) (schemas.ReconResult, error) { + return RunRecon(ctx, a, g.RepoPath, "quick") + }, g.Quick}, + {"fast", func(ctx context.Context, a appx.Harnesser) (schemas.ReconResult, error) { + return RunFastRecon(ctx, a, g.RepoPath) + }, g.Fast}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if len(tc.want) == 0 { + t.Fatalf("golden has no %q result", tc.name) + } + f := newReconFake(t, nil) + got, err := tc.run(context.Background(), f) + if err != nil { + t.Fatalf("run: %v", err) + } + // The fixture repo path does not exist, so both runtimes report + // (0, 0) metrics — assert that rather than letting it pass silently. + if got.LinesOfCode != 0 || got.FileCount != 0 { + t.Fatalf("fixture repo unexpectedly exists on this machine: metrics = (%d, %d)", + got.LinesOfCode, got.FileCount) + } + got.ReconDurationSeconds = 0 + + tree, _ := scrubIDs(jsonTree(t, got)).(map[string]any) + if !reflect.DeepEqual(tree, tc.want) { + t.Errorf("ReconResult differs from Python%s", diffJSON(t, tree, tc.want)) + } + }) + } +} diff --git a/internal/secaf/agents/recon/security_context.go b/internal/secaf/agents/recon/security_context.go new file mode 100644 index 0000000000..af8534a10c --- /dev/null +++ b/internal/secaf/agents/recon/security_context.go @@ -0,0 +1,53 @@ +package recon + +// Ports src/sec_af/agents/recon/security_context.py. + +import ( + "context" + "os" + "strings" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/harnessx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +const ( + securityContextPromptPath = "recon/security_context.txt" + securityContextAgentName = "recon-security-context" + securityContextExtractName = "Security context profiler" +) + +// securityContextPrompt builds the exact prompt run_security_context_profiler +// sends. Same two-step shape as the data-flow prompt: placeholder substitution +// first, exploration CONTEXT suffix second. +func securityContextPrompt(repoPath string, architecture schemas.ArchitectureMap) string { + template := prompts.MustLoad(securityContextPromptPath) + return strings.ReplaceAll(template, architectureMapPlaceholder, ArchitectureContextBlock(architecture)) + + explorationContextSuffix(repoPath) +} + +// RunSecurityContextProfiler ports security_context.py +// run_security_context_profiler: substitute the architecture map, harness for a +// SecurityContextRaw, then parse_security_context_raw (which is where the flat +// `security_signals` list gets bucketed into headers / deployment / framework). +func RunSecurityContextProfiler(ctx context.Context, app appx.Harnesser, repoPath string, architecture schemas.ArchitectureMap) (schemas.SecurityContext, error) { + prompt := securityContextPrompt(repoPath, architecture) + + harnessCwd, err := os.MkdirTemp("", "secaf-"+securityContextAgentName+"-") + if err != nil { + return schemas.SecurityContext{}, err + } + defer os.RemoveAll(harnessCwd) + + raw, err := harnessx.RunExtract[schemas.SecurityContextRaw]( + ctx, app, prompt, + appx.HarnessOptions{Cwd: harnessCwd, ProjectDir: repoPath}, + securityContextExtractName, + ) + if err != nil { + return schemas.SecurityContext{}, err + } + return ParseSecurityContextRaw(raw), nil +} diff --git a/internal/secaf/agents/recon/testdata/golden/architecture_context_block_A.txt b/internal/secaf/agents/recon/testdata/golden/architecture_context_block_A.txt new file mode 100644 index 0000000000..13831c075d --- /dev/null +++ b/internal/secaf/agents/recon/testdata/golden/architecture_context_block_A.txt @@ -0,0 +1,73 @@ +{ + "app_type": "web_api", + "modules": [ + { + "name": "auth", + "path": "src/auth/", + "language": "Python", + "description": "Sessions & tokens", + "dependencies": [ + "db", + "cache" + ] + }, + { + "name": "ui", + "path": "web/", + "language": "TypeScript", + "description": null, + "dependencies": [] + } + ], + "entry_points": [ + { + "kind": "http", + "identifier": "POST /api/login", + "file_path": "src/routes.py", + "line": 42, + "method": "POST", + "route": "/api/login", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "migrate", + "file_path": "src/cli.py", + "line": 8, + "method": null, + "route": null, + "auth_required": null + } + ], + "trust_boundaries": [ + { + "name": "API Gateway", + "source_zone": "external", + "target_zone": "internal", + "description": "Rate limiting auth \u2014 caf\u00e9 \u2192 app", + "enforcement": [ + "waf" + ] + } + ], + "services": [ + { + "name": "PostgreSQL", + "service_type": "database", + "endpoint": "localhost:5432", + "purpose": "primary store", + "auth_mechanism": "password" + } + ], + "api_surface": [ + { + "method": "GET", + "path": "/api/users", + "handler": "get_users", + "file_path": "src/api.py", + "line": 15, + "auth_required": true, + "rate_limited": false + } + ] +} \ No newline at end of file diff --git a/internal/secaf/agents/recon/testdata/golden/architecture_context_block_B.txt b/internal/secaf/agents/recon/testdata/golden/architecture_context_block_B.txt new file mode 100644 index 0000000000..84c806693d --- /dev/null +++ b/internal/secaf/agents/recon/testdata/golden/architecture_context_block_B.txt @@ -0,0 +1,8 @@ +{ + "app_type": null, + "modules": [], + "entry_points": [], + "trust_boundaries": [], + "services": [], + "api_surface": [] +} \ No newline at end of file diff --git a/internal/secaf/agents/recon/testdata/golden/architecture_prompt.txt b/internal/secaf/agents/recon/testdata/golden/architecture_prompt.txt new file mode 100644 index 0000000000..56b778957e --- /dev/null +++ b/internal/secaf/agents/recon/testdata/golden/architecture_prompt.txt @@ -0,0 +1,34 @@ +ROLE: +You are a software architecture analyst specializing in security-relevant codebase mapping. + +TASK: +Map architecture, modules, entry points, trust boundaries, service integrations, and full API surface of the repository. + +WORKFLOW (follow these steps in order): +1. List the top-level directory to understand project structure. +2. Read key files: package.json, requirements.txt, setup.py, go.mod, Dockerfile, docker-compose.yml, or similar config files to identify the tech stack. +3. Find entry points by searching for route definitions, HTTP handlers, CLI commands, and main/app files. +4. Read the identified entry point files to extract routes, methods, handlers, and line numbers. +5. Identify trust boundaries (e.g., public vs. internal, user input vs. database) and service dependencies. +6. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +SCOPE: +- Analyze source code and runtime configuration files. +- Include HTTP routes, CLI entry points, worker/event handlers, and RPC endpoints. +- Exclude tests, generated artifacts, and dependency vendor folders unless they expose runtime entry points. + +OUTPUT: +- Populate modules, entry_points, trust_boundaries, services, and api_surface arrays. +- Use file paths and line numbers from the actual files you read. +- Keep all booleans explicit where known, otherwise null. +- Empty arrays are acceptable if no items are found for a category. + +CONSTRAINTS: +- Base all findings on actual file content you read. Do not speculate. +- Do not include markdown, prose, or code fences in the output file. JSON only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Start by listing files in the repository path above. +- After gathering evidence, write the JSON output file using your Write tool. \ No newline at end of file diff --git a/internal/secaf/agents/recon/testdata/golden/config_scanner_prompt.txt b/internal/secaf/agents/recon/testdata/golden/config_scanner_prompt.txt new file mode 100644 index 0000000000..404746d0ab --- /dev/null +++ b/internal/secaf/agents/recon/testdata/golden/config_scanner_prompt.txt @@ -0,0 +1,32 @@ +ROLE: +You are an application security configuration analyst focused on secrets and misconfigurations. + +TASK: +Find hardcoded secrets and insecure configuration defaults in the repository. + +WORKFLOW (follow these steps in order): +1. List the top-level directory to understand project structure. +2. Search for environment files (.env, .env.example, .env.local), config files (config.py, settings.py, application.yml, etc.), and infrastructure files (Dockerfile, docker-compose.yml, nginx.conf, etc.). +3. Read each config/env file and look for: hardcoded API keys, passwords, tokens, database URLs, debug flags set to true, permissive CORS origins, weak crypto settings. +4. Search source code for patterns like "password", "secret", "api_key", "token", "DEBUG = True", "CORS_ALLOW_ALL". +5. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +SCOPE: +- Inspect environment files, infrastructure configs, framework configs, deployment descriptors, and startup scripts. +- Detect exposed secrets, permissive CORS, debug mode in production paths, weak crypto settings, and missing security controls. + +OUTPUT: +- Populate secrets and misconfigs arrays with concrete file path and line number evidence. +- Provide practical risk descriptions and remediation text when possible. +- Empty arrays are acceptable if no issues are found. + +CONSTRAINTS: +- Do not treat obvious test placeholders as real secrets unless uncertainty remains. +- Base all findings on actual file content you read. Do not speculate. +- Do not include markdown, prose, or code fences in the output file. JSON only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Start by listing files in the repository path above. +- After gathering evidence, write the JSON output file using your Write tool. \ No newline at end of file diff --git a/internal/secaf/agents/recon/testdata/golden/data_flow_prompt_A.txt b/internal/secaf/agents/recon/testdata/golden/data_flow_prompt_A.txt new file mode 100644 index 0000000000..e26cd7def5 --- /dev/null +++ b/internal/secaf/agents/recon/testdata/golden/data_flow_prompt_A.txt @@ -0,0 +1,105 @@ +ROLE: +You are a data-flow security analyst focused on tracing attacker-controlled input through code. + +CONTEXT: +You are in SEC-AF RECON Phase and must produce the required JSON output. +Architecture context is provided below. + +ARCHITECTURE_MAP: +{ + "app_type": "web_api", + "modules": [ + { + "name": "auth", + "path": "src/auth/", + "language": "Python", + "description": "Sessions & tokens", + "dependencies": [ + "db", + "cache" + ] + }, + { + "name": "ui", + "path": "web/", + "language": "TypeScript", + "description": null, + "dependencies": [] + } + ], + "entry_points": [ + { + "kind": "http", + "identifier": "POST /api/login", + "file_path": "src/routes.py", + "line": 42, + "method": "POST", + "route": "/api/login", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "migrate", + "file_path": "src/cli.py", + "line": 8, + "method": null, + "route": null, + "auth_required": null + } + ], + "trust_boundaries": [ + { + "name": "API Gateway", + "source_zone": "external", + "target_zone": "internal", + "description": "Rate limiting auth \u2014 caf\u00e9 \u2192 app", + "enforcement": [ + "waf" + ] + } + ], + "services": [ + { + "name": "PostgreSQL", + "service_type": "database", + "endpoint": "localhost:5432", + "purpose": "primary store", + "auth_mechanism": "password" + } + ], + "api_surface": [ + { + "method": "GET", + "path": "/api/users", + "handler": "get_users", + "file_path": "src/api.py", + "line": 15, + "auth_required": true, + "rate_limited": false + } + ] +} + +TASK: +Trace user-controlled inputs from sources through transformations to security-critical sinks. + +SCOPE: +- Start from known entry points and input parsers. +- Follow processing layers, validation, and sanitization functions. +- Identify sinks such as SQL execution, command execution, template rendering, eval/deserialization, and external calls. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate flows, sanitization_points, and sinks. +- For each flow include source, ordered path, sink, sanitized, and involved files. + +CONSTRAINTS: +- Take multiple turns to explore the codebase first, then build your analysis. +- Only mark sanitized=true when evidence is clear. +- No markdown/code fences; JSON output only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Take multiple turns to explore the codebase first, then build your analysis. +- Write final JSON only when analysis is complete. \ No newline at end of file diff --git a/internal/secaf/agents/recon/testdata/golden/data_flow_prompt_B.txt b/internal/secaf/agents/recon/testdata/golden/data_flow_prompt_B.txt new file mode 100644 index 0000000000..06d471490d --- /dev/null +++ b/internal/secaf/agents/recon/testdata/golden/data_flow_prompt_B.txt @@ -0,0 +1,40 @@ +ROLE: +You are a data-flow security analyst focused on tracing attacker-controlled input through code. + +CONTEXT: +You are in SEC-AF RECON Phase and must produce the required JSON output. +Architecture context is provided below. + +ARCHITECTURE_MAP: +{ + "app_type": null, + "modules": [], + "entry_points": [], + "trust_boundaries": [], + "services": [], + "api_surface": [] +} + +TASK: +Trace user-controlled inputs from sources through transformations to security-critical sinks. + +SCOPE: +- Start from known entry points and input parsers. +- Follow processing layers, validation, and sanitization functions. +- Identify sinks such as SQL execution, command execution, template rendering, eval/deserialization, and external calls. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Populate flows, sanitization_points, and sinks. +- For each flow include source, ordered path, sink, sanitized, and involved files. + +CONSTRAINTS: +- Take multiple turns to explore the codebase first, then build your analysis. +- Only mark sanitized=true when evidence is clear. +- No markdown/code fences; JSON output only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Take multiple turns to explore the codebase first, then build your analysis. +- Write final JSON only when analysis is complete. \ No newline at end of file diff --git a/internal/secaf/agents/recon/testdata/golden/dependencies_prompt.txt b/internal/secaf/agents/recon/testdata/golden/dependencies_prompt.txt new file mode 100644 index 0000000000..dfba5fbebd --- /dev/null +++ b/internal/secaf/agents/recon/testdata/golden/dependencies_prompt.txt @@ -0,0 +1,34 @@ +ROLE: +You are a software supply-chain security analyst focused on dependency risk. + +TASK: +Build an SBOM, identify known CVE exposure, and report outdated dependencies. + +WORKFLOW (follow these steps in order): +1. List the top-level directory to find dependency manifests. +2. Read dependency files: package.json, package-lock.json, yarn.lock, requirements.txt, Pipfile.lock, go.mod, go.sum, Gemfile.lock, pom.xml, build.gradle, Cargo.lock, etc. +3. For each dependency found, record: name, version, ecosystem (npm/pip/go/etc.), and whether it is direct or transitive. +4. Check for known CVEs affecting the identified packages and versions based on your knowledge. +5. Identify outdated dependencies where the installed version significantly lags behind known latest versions. +6. Once you have gathered enough evidence, write the final JSON output file as specified in the OUTPUT REQUIREMENTS below. + +SCOPE: +- Parse dependency manifests and lock files across ecosystems. +- Distinguish direct vs transitive dependencies. +- Record known CVEs with available severity/reachability hints. + +OUTPUT: +- Populate sbom, known_cves, outdated, direct_count, and transitive_count. +- Include package ecosystem and version information whenever available. +- Empty arrays are acceptable if no CVEs or outdated deps are found. + +CONSTRAINTS: +- Do not invent CVEs or versions. Only report what you can verify from the dependency files. +- Base all findings on actual file content you read. Do not speculate. +- Do not include markdown, prose, or code fences in the output file. JSON only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Start by listing files in the repository path above. +- After gathering evidence, write the JSON output file using your Write tool. \ No newline at end of file diff --git a/internal/secaf/agents/recon/testdata/golden/parse_architecture.json b/internal/secaf/agents/recon/testdata/golden/parse_architecture.json new file mode 100644 index 0000000000..800451ccc0 --- /dev/null +++ b/internal/secaf/agents/recon/testdata/golden/parse_architecture.json @@ -0,0 +1,160 @@ +{ + "input": { + "app_type": "web_api", + "modules": [ + "auth | src/auth/ | Python | Authentication and session management", + "ui|web/|TypeScript|", + "orphan", + " a | b | c | d | e " + ], + "entry_points": [ + "http | POST /api/login | src/routes.py:42 | false", + "cli | migrate | src/cli.py | yes", + "event | queue:jobs | src/worker.py:0 | maybe" + ], + "trust_boundaries": [ + "API Gateway | external | internal | Rate limiting and auth", + "edge|dmz" + ], + "services": [ + "PostgreSQL | database | localhost:5432 | password", + "Stripe | payments | n/a | NONE", + "Redis | cache | | unknown" + ], + "api_endpoints": [ + "GET | /api/users | get_users | src/api.py:15 | true | false", + "POST | /api/users | create | src/api.py | 1 | 0", + "PUT | /x | h | a:b:12 | | " + ] + }, + "want": { + "app_type": "web_api", + "modules": [ + { + "name": "auth", + "path": "src/auth/", + "language": "Python", + "description": "Authentication and session management", + "dependencies": [] + }, + { + "name": "ui", + "path": "web/", + "language": "TypeScript", + "description": null, + "dependencies": [] + }, + { + "name": "orphan", + "path": "", + "language": "", + "description": null, + "dependencies": [] + }, + { + "name": "a", + "path": "b", + "language": "c", + "description": "d | e", + "dependencies": [] + } + ], + "entry_points": [ + { + "kind": "http", + "identifier": "POST /api/login", + "file_path": "src/routes.py", + "line": 42, + "method": null, + "route": "POST /api/login", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "migrate", + "file_path": "src/cli.py", + "line": 0, + "method": null, + "route": null, + "auth_required": true + }, + { + "kind": "event", + "identifier": "queue:jobs", + "file_path": "src/worker.py:0", + "line": 0, + "method": null, + "route": null, + "auth_required": null + } + ], + "trust_boundaries": [ + { + "name": "API Gateway", + "source_zone": "external", + "target_zone": "internal", + "description": "Rate limiting and auth", + "enforcement": [] + }, + { + "name": "edge", + "source_zone": "dmz", + "target_zone": "", + "description": "", + "enforcement": [] + } + ], + "services": [ + { + "name": "PostgreSQL", + "service_type": "database", + "endpoint": "localhost:5432", + "purpose": null, + "auth_mechanism": "password" + }, + { + "name": "Stripe", + "service_type": "payments", + "endpoint": null, + "purpose": null, + "auth_mechanism": null + }, + { + "name": "Redis", + "service_type": "cache", + "endpoint": null, + "purpose": null, + "auth_mechanism": null + } + ], + "api_surface": [ + { + "method": "GET", + "path": "/api/users", + "handler": "get_users", + "file_path": "src/api.py", + "line": 15, + "auth_required": true, + "rate_limited": false + }, + { + "method": "POST", + "path": "/api/users", + "handler": "create", + "file_path": "src/api.py", + "line": 0, + "auth_required": true, + "rate_limited": false + }, + { + "method": "PUT", + "path": "/x", + "handler": "h", + "file_path": "a:b", + "line": 12, + "auth_required": null, + "rate_limited": null + } + ] + } +} diff --git a/internal/secaf/agents/recon/testdata/golden/parse_config_report.json b/internal/secaf/agents/recon/testdata/golden/parse_config_report.json new file mode 100644 index 0000000000..43c3a3f33e --- /dev/null +++ b/internal/secaf/agents/recon/testdata/golden/parse_config_report.json @@ -0,0 +1,56 @@ +{ + "input": { + "secrets": [ + "api_key | src/config.py:7 | API_KEY = \"sk-live-123\" | high | false", + "password | src/settings.py | pw=hunter2 | | " + ], + "misconfigs": [ + "dangerous_config | deploy/prod.yaml:22 | DEBUG | Debug mode enabled in production | Set DEBUG=false", + "cors | deploy/nginx.conf | N/A | Wildcard origin | unknown" + ] + }, + "want": { + "secrets": [ + { + "id": "", + "secret_type": "api_key", + "file_path": "src/config.py", + "line": 7, + "match": "API_KEY = \"sk-live-123\"", + "confidence": "high", + "is_test_value": false + }, + { + "id": "", + "secret_type": "password", + "file_path": "src/settings.py", + "line": 0, + "match": "pw=hunter2", + "confidence": "medium", + "is_test_value": null + } + ], + "misconfigs": [ + { + "id": "", + "category": "dangerous_config", + "file_path": "deploy/prod.yaml", + "line": 22, + "key": "DEBUG", + "value": null, + "risk": "Debug mode enabled in production", + "remediation": "Set DEBUG=false" + }, + { + "id": "", + "category": "cors", + "file_path": "deploy/nginx.conf", + "line": null, + "key": null, + "value": null, + "risk": "Wildcard origin", + "remediation": null + } + ] + } +} diff --git a/internal/secaf/agents/recon/testdata/golden/parse_data_flow.json b/internal/secaf/agents/recon/testdata/golden/parse_data_flow.json new file mode 100644 index 0000000000..5bd1f368f7 --- /dev/null +++ b/internal/secaf/agents/recon/testdata/golden/parse_data_flow.json @@ -0,0 +1,83 @@ +{ + "input": { + "flows": [ + "request.body | sql.execute | false | src/db.py, src/routes.py", + "argv | os.system | TRUE | ", + "env | log | garbage | a, , b ," + ], + "sanitization_points": [ + "src/valid.py:12 | sanitize | escape | sqli, xss", + "src/valid.py | | strip | " + ], + "sinks": [ + "sql | src/db.py:88 | execute | user-controlled query string", + "exec | src/run.py | | " + ] + }, + "want": { + "flows": [ + { + "source": "request.body", + "path": [], + "sink": "sql.execute", + "sanitized": false, + "files": [ + "src/db.py", + "src/routes.py" + ] + }, + { + "source": "argv", + "path": [], + "sink": "os.system", + "sanitized": true, + "files": [] + }, + { + "source": "env", + "path": [], + "sink": "log", + "sanitized": false, + "files": [ + "a", + "b" + ] + } + ], + "sanitization_points": [ + { + "file_path": "src/valid.py", + "line": 12, + "function_name": "sanitize", + "sanitization_type": "escape", + "protects_against": [ + "sqli", + "xss" + ] + }, + { + "file_path": "src/valid.py", + "line": 0, + "function_name": null, + "sanitization_type": "strip", + "protects_against": [] + } + ], + "sinks": [ + { + "sink_type": "sql", + "file_path": "src/db.py", + "line": 88, + "function_name": "execute", + "exploitability_notes": "user-controlled query string" + }, + { + "sink_type": "exec", + "file_path": "src/run.py", + "line": 0, + "function_name": null, + "exploitability_notes": null + } + ] + } +} diff --git a/internal/secaf/agents/recon/testdata/golden/parse_dependency_report.json b/internal/secaf/agents/recon/testdata/golden/parse_dependency_report.json new file mode 100644 index 0000000000..07312c9f36 --- /dev/null +++ b/internal/secaf/agents/recon/testdata/golden/parse_dependency_report.json @@ -0,0 +1,80 @@ +{ + "input": { + "sbom": [ + "django | 3.2.1 | pypi | true | BSD-3-Clause", + "urllib3 | 1.26.5 | pypi | false | n/a", + "left-pad | 1.0.0 | npm | notabool | " + ], + "known_cves": [ + "CVE-2021-1 | django | 3.2.1 | 3.2.13 | 9.8 | true | true", + "CVE-2021-2 | urllib3 | 1.26.5 | none | notafloat | 0 | " + ], + "outdated": [ + "django | 3.2.1 | 5.0.0 | true", + "requests | 2.0 | 2.31 | 0" + ] + }, + "want": { + "sbom": [ + { + "name": "django", + "version": "3.2.1", + "ecosystem": "pypi", + "direct": true, + "license": "BSD-3-Clause" + }, + { + "name": "urllib3", + "version": "1.26.5", + "ecosystem": "pypi", + "direct": false, + "license": null + }, + { + "name": "left-pad", + "version": "1.0.0", + "ecosystem": "npm", + "direct": false, + "license": null + } + ], + "known_cves": [ + { + "cve_id": "CVE-2021-1", + "package": "django", + "installed_version": "3.2.1", + "fixed_version": "3.2.13", + "cvss_v4_score": 9.8, + "epss_score": null, + "direct": true, + "reachable": true + }, + { + "cve_id": "CVE-2021-2", + "package": "urllib3", + "installed_version": "1.26.5", + "fixed_version": null, + "cvss_v4_score": null, + "epss_score": null, + "direct": false, + "reachable": null + } + ], + "outdated": [ + { + "package": "django", + "current_version": "3.2.1", + "latest_version": "5.0.0", + "direct": true + }, + { + "package": "requests", + "current_version": "2.0", + "latest_version": "2.31", + "direct": false + } + ], + "direct_count": 1, + "transitive_count": 2 + } +} diff --git a/internal/secaf/agents/recon/testdata/golden/parse_primitives.json b/internal/secaf/agents/recon/testdata/golden/parse_primitives.json new file mode 100644 index 0000000000..91462fe7b2 --- /dev/null +++ b/internal/secaf/agents/recon/testdata/golden/parse_primitives.json @@ -0,0 +1,361 @@ +{ + "split_pipe": [ + { + "s": "a | b | c | d", + "expected": 4, + "want": [ + "a", + "b", + "c", + "d" + ] + }, + { + "s": "a|b", + "expected": 4, + "want": [ + "a", + "b", + "", + "" + ] + }, + { + "s": "", + "expected": 4, + "want": [ + "", + "", + "", + "" + ] + }, + { + "s": " a | b | c | d | e ", + "expected": 4, + "want": [ + "a", + "b", + "c", + "d | e" + ] + }, + { + "s": "a|b|c|d|e|f|g", + "expected": 6, + "want": [ + "a", + "b", + "c", + "d", + "e", + "f|g" + ] + }, + { + "s": "only", + "expected": 1, + "want": [ + "only" + ] + }, + { + "s": "a|b|c", + "expected": 1, + "want": [ + "a|b|c" + ] + }, + { + "s": "|||", + "expected": 4, + "want": [ + "", + "", + "", + "" + ] + }, + { + "s": " x ", + "expected": 2, + "want": [ + "x", + "" + ] + } + ], + "parse_bool": [ + { + "s": "true", + "want": true + }, + { + "s": "TRUE", + "want": true + }, + { + "s": " True ", + "want": true + }, + { + "s": "yes", + "want": true + }, + { + "s": "1", + "want": true + }, + { + "s": "false", + "want": false + }, + { + "s": "No", + "want": false + }, + { + "s": "0", + "want": false + }, + { + "s": "", + "want": null + }, + { + "s": "maybe", + "want": null + }, + { + "s": "n/a", + "want": null + }, + { + "s": " 01 ", + "want": null + } + ], + "parse_int": [ + { + "s": "0", + "want": 0 + }, + { + "s": "12", + "want": 12 + }, + { + "s": " 42 ", + "want": 42 + }, + { + "s": "-7", + "want": -7 + }, + { + "s": "+3", + "want": 3 + }, + { + "s": "1_0", + "want": 10 + }, + { + "s": "abc", + "want": 0 + }, + { + "s": "", + "want": 0 + }, + { + "s": "3.5", + "want": 0 + }, + { + "s": "0x10", + "want": 0 + }, + { + "s": " ", + "want": 0 + } + ], + "parse_int_default9": [ + { + "s": "0", + "want": 0 + }, + { + "s": "12", + "want": 12 + }, + { + "s": " 42 ", + "want": 42 + }, + { + "s": "-7", + "want": -7 + }, + { + "s": "+3", + "want": 3 + }, + { + "s": "1_0", + "want": 10 + }, + { + "s": "abc", + "want": 9 + }, + { + "s": "", + "want": 9 + }, + { + "s": "3.5", + "want": 9 + }, + { + "s": "0x10", + "want": 9 + }, + { + "s": " ", + "want": 9 + } + ], + "parse_float": [ + { + "s": "1.5", + "want": "1.5" + }, + { + "s": " 9.8 ", + "want": "9.8" + }, + { + "s": "0", + "want": "0.0" + }, + { + "s": "-2", + "want": "-2.0" + }, + { + "s": "1e3", + "want": "1000.0" + }, + { + "s": "abc", + "want": null + }, + { + "s": "", + "want": null + }, + { + "s": "inf", + "want": "inf" + }, + { + "s": "nan", + "want": "nan" + }, + { + "s": "1_0.5", + "want": "10.5" + }, + { + "s": "+.5", + "want": "0.5" + } + ], + "parse_file_line": [ + { + "s": "src/api.py:15", + "path": "src/api.py", + "line": 15 + }, + { + "s": "src/api.py", + "path": "src/api.py", + "line": 0 + }, + { + "s": "a:b:12", + "path": "a:b", + "line": 12 + }, + { + "s": "src/x.py:0", + "path": "src/x.py:0", + "line": 0 + }, + { + "s": "src/x.py:-3", + "path": "src/x.py:-3", + "line": 0 + }, + { + "s": ":", + "path": ":", + "line": 0 + }, + { + "s": "", + "path": "", + "line": 0 + }, + { + "s": " a.py : 4 ", + "path": "a.py ", + "line": 4 + }, + { + "s": "C:/x.py:9", + "path": "C:/x.py", + "line": 9 + } + ], + "is_na": [ + { + "s": "", + "want": true + }, + { + "s": " ", + "want": true + }, + { + "s": "na", + "want": true + }, + { + "s": "N/A", + "want": true + }, + { + "s": "None", + "want": true + }, + { + "s": "UNKNOWN", + "want": true + }, + { + "s": "unknown ", + "want": true + }, + { + "s": "value", + "want": false + }, + { + "s": "0", + "want": false + } + ] +} diff --git a/internal/secaf/agents/recon/testdata/golden/parse_security_context.json b/internal/secaf/agents/recon/testdata/golden/parse_security_context.json new file mode 100644 index 0000000000..2c237c9b02 --- /dev/null +++ b/internal/secaf/agents/recon/testdata/golden/parse_security_context.json @@ -0,0 +1,58 @@ +{ + "input": { + "auth_model": "jwt", + "auth_details": "Bearer token validated by middleware", + "crypto_usage": [ + "AES | 256 | GCM | data encryption | false", + "TLSv1.0 | n/a | none | legacy tls terminator | true", + "MD5 | notanint | | | TRUE" + ], + "security_signals": [ + "CSRF protection enabled", + "HSTS header present", + "Runs in Docker", + "CSP configured", + "Uses Kubernetes secrets", + "Input validation via pydantic" + ] + }, + "want": { + "auth_model": "jwt", + "auth_details": "Bearer token validated by middleware", + "crypto_usage": [ + { + "algorithm": "AES", + "key_size": 256, + "mode": "GCM", + "usage_context": "data encryption", + "is_weak": false + }, + { + "algorithm": "TLSv1.0", + "key_size": null, + "mode": null, + "usage_context": "legacy tls terminator", + "is_weak": true + }, + { + "algorithm": "MD5", + "key_size": 0, + "mode": null, + "usage_context": null, + "is_weak": true + } + ], + "framework_security": [ + "CSRF protection enabled", + "Input validation via pydantic" + ], + "security_headers": [ + "HSTS header present", + "CSP configured" + ], + "deployment_signals": [ + "Runs in Docker", + "Uses Kubernetes secrets" + ] + } +} diff --git a/internal/secaf/agents/recon/testdata/golden/repo_metrics.json b/internal/secaf/agents/recon/testdata/golden/repo_metrics.json new file mode 100644 index 0000000000..5a859d32f2 --- /dev/null +++ b/internal/secaf/agents/recon/testdata/golden/repo_metrics.json @@ -0,0 +1,36 @@ +{ + "files": { + "main.py": "aW1wb3J0IG9zCnByaW50KDEpCg==", + "trailing_none.go": "cGFja2FnZSBtYWluCmZ1bmMgbWFpbigpIHt9", + "crlf.ts": "Y29uc3QgYSA9IDE7DQpjb25zdCBiID0gMjsNCg==", + "cr_only.rb": "cHV0cyAxDXB1dHMgMg0=", + "empty.py": "", + "just_newline.sql": "Cg==", + "invalid_utf8.js": "dmFyIGEgPSAn//4nOwp2YXIgYiA9IDI7Cg==", + "UPPER.PY": "YQpiCg==", + "archive.tar.gz": "bm90IHJlYWxseSBnemlwCg==", + ".gitignore": "bm9kZV9tb2R1bGVzCi52ZW52Cg==", + "Makefile": "YWxsOgoJZ28gYnVpbGQK", + "trailingdot.": "eAo=", + "README.md": "IyBkb2NzCm5vdCBjb2RlCg==", + "pkg/lib.go": "cGFja2FnZSBwa2cKCmZ1bmMgRigpIHt9Cg==", + "pkg/deep/nested/util.rs": "Zm4gbWFpbigpIHt9Cg==", + "conf/app.yaml": "YTogMQpiOiAyCmM6IDMK", + "conf/app.YML": "eDogMQo=", + ".git/config": "W2NvcmVdCg==", + "node_modules/left-pad/index.js": "bW9kdWxlLmV4cG9ydHMgPSAxOwo=", + "vendor/dep/dep.go": "cGFja2FnZSBkZXAK", + ".venv/lib/site.py": "cGFzcwo=", + "venv/lib/site.py": "cGFzcwo=", + "__pycache__/main.cpython-311.pyc": "AAE=", + "src/.hg/store.py": "cGFzcwo=", + "src/.svn/entries.py": "cGFzcwo=" + }, + "symlinks": { + "link_to_main.py": "main.py", + "broken_link.py": "does_not_exist.py", + "link_to_pkg": "pkg" + }, + "lines_of_code": 23, + "file_count": 18 +} diff --git a/internal/secaf/agents/recon/testdata/golden/run_recon.json b/internal/secaf/agents/recon/testdata/golden/run_recon.json new file mode 100644 index 0000000000..1b489b52ee --- /dev/null +++ b/internal/secaf/agents/recon/testdata/golden/run_recon.json @@ -0,0 +1,576 @@ +{ + "repo_path": "/fixtures/demo-repo", + "canned": { + "architecture": { + "app_type": "web_api", + "modules": [ + "auth | src/auth | Python | sessions", + "ui | web | TypeScript | ", + "api | src/api | python | dup-language", + "legacy | old | | no language" + ], + "entry_points": [ + "http | POST /login | src/routes.py:42 | false" + ], + "trust_boundaries": [ + "edge | external | internal | tls" + ], + "services": [ + "pg | database | localhost:5432 | password" + ], + "api_endpoints": [ + "GET | /users | list | src/api.py:15 | true | false" + ] + }, + "dependencies": { + "sbom": [ + "django | 3.2 | pypi | true | BSD" + ], + "known_cves": [ + "CVE-1 | django | 3.2 | 3.3 | 9.8 | true | true" + ], + "outdated": [ + "django | 3.2 | 5.0 | true" + ] + }, + "config_scanner": { + "secrets": [ + "api_key | src/config.py:7 | KEY=1 | high | false" + ], + "misconfigs": [ + "debug | deploy/prod.yaml:22 | DEBUG | on in prod | turn it off" + ] + }, + "data_flow": { + "flows": [ + "request.body | sql.execute | false | src/db.py" + ], + "sanitization_points": [ + "src/valid.py:12 | clean | escape | sqli" + ], + "sinks": [ + "sql | src/db.py:88 | execute | tainted" + ] + }, + "security_context": { + "auth_model": "jwt", + "auth_details": "bearer", + "crypto_usage": [ + "AES | 256 | GCM | data | false" + ], + "security_signals": [ + "Uses Flask-Login", + "HSTS header present", + "Runs in Docker", + "Uses Flask-Login" + ] + } + }, + "standard": { + "architecture": { + "app_type": "web_api", + "modules": [ + { + "name": "auth", + "path": "src/auth", + "language": "Python", + "description": "sessions", + "dependencies": [] + }, + { + "name": "ui", + "path": "web", + "language": "TypeScript", + "description": null, + "dependencies": [] + }, + { + "name": "api", + "path": "src/api", + "language": "python", + "description": "dup-language", + "dependencies": [] + }, + { + "name": "legacy", + "path": "old", + "language": "", + "description": "no language", + "dependencies": [] + } + ], + "entry_points": [ + { + "kind": "http", + "identifier": "POST /login", + "file_path": "src/routes.py", + "line": 42, + "method": null, + "route": "POST /login", + "auth_required": false + } + ], + "trust_boundaries": [ + { + "name": "edge", + "source_zone": "external", + "target_zone": "internal", + "description": "tls", + "enforcement": [] + } + ], + "services": [ + { + "name": "pg", + "service_type": "database", + "endpoint": "localhost:5432", + "purpose": null, + "auth_mechanism": "password" + } + ], + "api_surface": [ + { + "method": "GET", + "path": "/users", + "handler": "list", + "file_path": "src/api.py", + "line": 15, + "auth_required": true, + "rate_limited": false + } + ] + }, + "data_flows": { + "flows": [ + { + "source": "request.body", + "path": [], + "sink": "sql.execute", + "sanitized": false, + "files": [ + "src/db.py" + ] + } + ], + "sanitization_points": [ + { + "file_path": "src/valid.py", + "line": 12, + "function_name": "clean", + "sanitization_type": "escape", + "protects_against": [ + "sqli" + ] + } + ], + "sinks": [ + { + "sink_type": "sql", + "file_path": "src/db.py", + "line": 88, + "function_name": "execute", + "exploitability_notes": "tainted" + } + ] + }, + "dependencies": { + "sbom": [ + { + "name": "django", + "version": "3.2", + "ecosystem": "pypi", + "direct": true, + "license": "BSD" + } + ], + "known_cves": [ + { + "cve_id": "CVE-1", + "package": "django", + "installed_version": "3.2", + "fixed_version": "3.3", + "cvss_v4_score": 9.8, + "epss_score": null, + "direct": true, + "reachable": true + } + ], + "outdated": [ + { + "package": "django", + "current_version": "3.2", + "latest_version": "5.0", + "direct": true + } + ], + "direct_count": 1, + "transitive_count": 0 + }, + "config": { + "secrets": [ + { + "id": "", + "secret_type": "api_key", + "file_path": "src/config.py", + "line": 7, + "match": "KEY=1", + "confidence": "high", + "is_test_value": false + } + ], + "misconfigs": [ + { + "id": "", + "category": "debug", + "file_path": "deploy/prod.yaml", + "line": 22, + "key": "DEBUG", + "value": null, + "risk": "on in prod", + "remediation": "turn it off" + } + ] + }, + "security_context": { + "auth_model": "jwt", + "auth_details": "bearer", + "crypto_usage": [ + { + "algorithm": "AES", + "key_size": 256, + "mode": "GCM", + "usage_context": "data", + "is_weak": false + } + ], + "framework_security": [ + "Uses Flask-Login", + "Uses Flask-Login" + ], + "security_headers": [ + "HSTS header present" + ], + "deployment_signals": [ + "Runs in Docker" + ] + }, + "languages": [ + "python", + "typescript" + ], + "frameworks": [ + "Uses Flask-Login" + ], + "lines_of_code": 0, + "file_count": 0, + "recon_duration_seconds": 0.0 + }, + "quick": { + "architecture": { + "app_type": "web_api", + "modules": [ + { + "name": "auth", + "path": "src/auth", + "language": "Python", + "description": "sessions", + "dependencies": [] + }, + { + "name": "ui", + "path": "web", + "language": "TypeScript", + "description": null, + "dependencies": [] + }, + { + "name": "api", + "path": "src/api", + "language": "python", + "description": "dup-language", + "dependencies": [] + }, + { + "name": "legacy", + "path": "old", + "language": "", + "description": "no language", + "dependencies": [] + } + ], + "entry_points": [ + { + "kind": "http", + "identifier": "POST /login", + "file_path": "src/routes.py", + "line": 42, + "method": null, + "route": "POST /login", + "auth_required": false + } + ], + "trust_boundaries": [ + { + "name": "edge", + "source_zone": "external", + "target_zone": "internal", + "description": "tls", + "enforcement": [] + } + ], + "services": [ + { + "name": "pg", + "service_type": "database", + "endpoint": "localhost:5432", + "purpose": null, + "auth_mechanism": "password" + } + ], + "api_surface": [ + { + "method": "GET", + "path": "/users", + "handler": "list", + "file_path": "src/api.py", + "line": 15, + "auth_required": true, + "rate_limited": false + } + ] + }, + "data_flows": { + "flows": [], + "sanitization_points": [], + "sinks": [] + }, + "dependencies": { + "sbom": [ + { + "name": "django", + "version": "3.2", + "ecosystem": "pypi", + "direct": true, + "license": "BSD" + } + ], + "known_cves": [ + { + "cve_id": "CVE-1", + "package": "django", + "installed_version": "3.2", + "fixed_version": "3.3", + "cvss_v4_score": 9.8, + "epss_score": null, + "direct": true, + "reachable": true + } + ], + "outdated": [ + { + "package": "django", + "current_version": "3.2", + "latest_version": "5.0", + "direct": true + } + ], + "direct_count": 1, + "transitive_count": 0 + }, + "config": { + "secrets": [ + { + "id": "", + "secret_type": "api_key", + "file_path": "src/config.py", + "line": 7, + "match": "KEY=1", + "confidence": "high", + "is_test_value": false + } + ], + "misconfigs": [ + { + "id": "", + "category": "debug", + "file_path": "deploy/prod.yaml", + "line": 22, + "key": "DEBUG", + "value": null, + "risk": "on in prod", + "remediation": "turn it off" + } + ] + }, + "security_context": { + "auth_model": "unknown", + "auth_details": "unknown", + "crypto_usage": [], + "framework_security": [], + "security_headers": [], + "deployment_signals": [] + }, + "languages": [ + "python", + "typescript" + ], + "frameworks": [], + "lines_of_code": 0, + "file_count": 0, + "recon_duration_seconds": 0.0 + }, + "fast": { + "architecture": { + "app_type": "web_api", + "modules": [ + { + "name": "auth", + "path": "src/auth", + "language": "Python", + "description": "sessions", + "dependencies": [] + }, + { + "name": "ui", + "path": "web", + "language": "TypeScript", + "description": null, + "dependencies": [] + }, + { + "name": "api", + "path": "src/api", + "language": "python", + "description": "dup-language", + "dependencies": [] + }, + { + "name": "legacy", + "path": "old", + "language": "", + "description": "no language", + "dependencies": [] + } + ], + "entry_points": [ + { + "kind": "http", + "identifier": "POST /login", + "file_path": "src/routes.py", + "line": 42, + "method": null, + "route": "POST /login", + "auth_required": false + } + ], + "trust_boundaries": [ + { + "name": "edge", + "source_zone": "external", + "target_zone": "internal", + "description": "tls", + "enforcement": [] + } + ], + "services": [ + { + "name": "pg", + "service_type": "database", + "endpoint": "localhost:5432", + "purpose": null, + "auth_mechanism": "password" + } + ], + "api_surface": [ + { + "method": "GET", + "path": "/users", + "handler": "list", + "file_path": "src/api.py", + "line": 15, + "auth_required": true, + "rate_limited": false + } + ] + }, + "data_flows": { + "flows": [], + "sanitization_points": [], + "sinks": [] + }, + "dependencies": { + "sbom": [ + { + "name": "django", + "version": "3.2", + "ecosystem": "pypi", + "direct": true, + "license": "BSD" + } + ], + "known_cves": [ + { + "cve_id": "CVE-1", + "package": "django", + "installed_version": "3.2", + "fixed_version": "3.3", + "cvss_v4_score": 9.8, + "epss_score": null, + "direct": true, + "reachable": true + } + ], + "outdated": [ + { + "package": "django", + "current_version": "3.2", + "latest_version": "5.0", + "direct": true + } + ], + "direct_count": 1, + "transitive_count": 0 + }, + "config": { + "secrets": [ + { + "id": "", + "secret_type": "api_key", + "file_path": "src/config.py", + "line": 7, + "match": "KEY=1", + "confidence": "high", + "is_test_value": false + } + ], + "misconfigs": [ + { + "id": "", + "category": "debug", + "file_path": "deploy/prod.yaml", + "line": 22, + "key": "DEBUG", + "value": null, + "risk": "on in prod", + "remediation": "turn it off" + } + ] + }, + "security_context": { + "auth_model": "unknown", + "auth_details": "unknown", + "crypto_usage": [], + "framework_security": [], + "security_headers": [], + "deployment_signals": [] + }, + "languages": [ + "python", + "typescript" + ], + "frameworks": [], + "lines_of_code": 0, + "file_count": 0, + "recon_duration_seconds": 0.0 + } +} diff --git a/internal/secaf/agents/recon/testdata/golden/security_context_prompt_A.txt b/internal/secaf/agents/recon/testdata/golden/security_context_prompt_A.txt new file mode 100644 index 0000000000..4c13d5d91b --- /dev/null +++ b/internal/secaf/agents/recon/testdata/golden/security_context_prompt_A.txt @@ -0,0 +1,105 @@ +ROLE: +You are a security context profiler focused on authentication, crypto posture, and deployment security signals. + +CONTEXT: +You are in SEC-AF RECON Phase and must produce the required JSON output. +Architecture context is provided below. + +ARCHITECTURE_MAP: +{ + "app_type": "web_api", + "modules": [ + { + "name": "auth", + "path": "src/auth/", + "language": "Python", + "description": "Sessions & tokens", + "dependencies": [ + "db", + "cache" + ] + }, + { + "name": "ui", + "path": "web/", + "language": "TypeScript", + "description": null, + "dependencies": [] + } + ], + "entry_points": [ + { + "kind": "http", + "identifier": "POST /api/login", + "file_path": "src/routes.py", + "line": 42, + "method": "POST", + "route": "/api/login", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "migrate", + "file_path": "src/cli.py", + "line": 8, + "method": null, + "route": null, + "auth_required": null + } + ], + "trust_boundaries": [ + { + "name": "API Gateway", + "source_zone": "external", + "target_zone": "internal", + "description": "Rate limiting auth \u2014 caf\u00e9 \u2192 app", + "enforcement": [ + "waf" + ] + } + ], + "services": [ + { + "name": "PostgreSQL", + "service_type": "database", + "endpoint": "localhost:5432", + "purpose": "primary store", + "auth_mechanism": "password" + } + ], + "api_surface": [ + { + "method": "GET", + "path": "/api/users", + "handler": "get_users", + "file_path": "src/api.py", + "line": 15, + "auth_required": true, + "rate_limited": false + } + ] +} + +TASK: +Profile the security model of this codebase: auth approach, crypto usage, framework protections, security headers, deployment signals. + +SCOPE: +- Identify auth model (JWT/session/OAuth2/API key/none/unknown) and evidence. +- Extract crypto algorithms, key sizes, modes, and weak usage where present. +- Capture framework-level protections and missing/used security headers. +- Infer deployment signals from infrastructure and runtime config. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Fill auth_model and auth_details, plus crypto_usage, framework_security, security_headers, deployment_signals. + +CONSTRAINTS: +- Take multiple turns to explore the codebase first, then build your analysis. +- Prefer explicit evidence over assumptions; if uncertain, state unknown in fields. +- No markdown/code fences; JSON output only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Take multiple turns to explore the codebase first, then build your analysis. +- Write final JSON only when analysis is complete. \ No newline at end of file diff --git a/internal/secaf/agents/recon/testdata/golden/security_context_prompt_B.txt b/internal/secaf/agents/recon/testdata/golden/security_context_prompt_B.txt new file mode 100644 index 0000000000..bb880a96c2 --- /dev/null +++ b/internal/secaf/agents/recon/testdata/golden/security_context_prompt_B.txt @@ -0,0 +1,40 @@ +ROLE: +You are a security context profiler focused on authentication, crypto posture, and deployment security signals. + +CONTEXT: +You are in SEC-AF RECON Phase and must produce the required JSON output. +Architecture context is provided below. + +ARCHITECTURE_MAP: +{ + "app_type": null, + "modules": [], + "entry_points": [], + "trust_boundaries": [], + "services": [], + "api_surface": [] +} + +TASK: +Profile the security model of this codebase: auth approach, crypto usage, framework protections, security headers, deployment signals. + +SCOPE: +- Identify auth model (JWT/session/OAuth2/API key/none/unknown) and evidence. +- Extract crypto algorithms, key sizes, modes, and weak usage where present. +- Capture framework-level protections and missing/used security headers. +- Infer deployment signals from infrastructure and runtime config. + +OUTPUT: +- Return JSON that strictly matches the schema provided below. +- Fill auth_model and auth_details, plus crypto_usage, framework_security, security_headers, deployment_signals. + +CONSTRAINTS: +- Take multiple turns to explore the codebase first, then build your analysis. +- Prefer explicit evidence over assumptions; if uncertain, state unknown in fields. +- No markdown/code fences; JSON output only. + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Take multiple turns to explore the codebase first, then build your analysis. +- Write final JSON only when analysis is complete. \ No newline at end of file diff --git a/internal/secaf/agents/remediation/remediation.go b/internal/secaf/agents/remediation/remediation.go new file mode 100644 index 0000000000..897b294be2 --- /dev/null +++ b/internal/secaf/agents/remediation/remediation.go @@ -0,0 +1,249 @@ +// Package remediation ports src/sec_af/agents/remediation.py — the agent that +// turns a finding into a concrete patch by running the coding harness against +// the repository. +// +// The Python module exposes two entry points that build the SAME prompt +// template (src/sec_af/prompts/remediation.txt) from two DIFFERENT finding +// shapes: +// +// - run_remediation(app, repo_path, finding: RawFinding, verdict, rationale) +// — the HUNT-shaped finding plus an externally supplied verdict/rationale +// (reasoners/prove.py run_remediation_agent); +// - generate_remediation(app, repo_path, finding: VerifiedFinding) — the +// PROVE-shaped finding, whose location/proof/verdict/rationale are read off +// the model itself (reasoners/prove.py run_remediation). +// +// Both then run the harness in a throwaway cwd with project_dir=repo_path and +// funnel the result through extract_harness_result(..., "RemediationAgent"). +package remediation + +import ( + "context" + "os" + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/harnessx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// promptRel is the Go form of remediation.py's +// +// PROMPT_PATH = Path(__file__).resolve().parents[1] / "prompts" / "remediation.txt" +// +// i.e. src/sec_af/prompts/remediation.txt, embedded by internal/prompts. +const promptRel = "remediation.txt" + +// agentName is the label extract_harness_result is called with. Python passes +// the literal "RemediationAgent" from BOTH entry points (note it differs from +// the "remediation" used for the tempdir prefix). +const agentName = "RemediationAgent" + +// replacement is one {{PLACEHOLDER}} -> value substitution. +// +// Order matters and is preserved: Python iterates a dict literal, which is +// insertion-ordered, and applies str.replace one key at a time, so a value that +// happens to contain a later placeholder WOULD itself be substituted. Reusing +// the same ordered list keeps that (admittedly pathological) behaviour. +type replacement struct { + needle string + value string +} + +func applyReplacements(template string, reps []replacement) string { + prompt := template + for _, r := range reps { + prompt = strings.ReplaceAll(prompt, r.needle, r.value) + } + return prompt +} + +// contextSuffix ports the identical trailer both entry points append: +// +// "\n\nCONTEXT:\n" + f"- Repository path: {repo_path}\n" +// + "- Use the repository path to inspect the actual source code for accurate patch generation." +func contextSuffix(repoPath string) string { + return "\n\nCONTEXT:\n" + + "- Repository path: " + repoPath + "\n" + + "- Use the repository path to inspect the actual source code for accurate patch generation." +} + +// buildPrompt ports src/sec_af/agents/remediation.py _build_prompt (the RawFinding form). +func buildPrompt(template string, finding schemas.RawFinding, verdict, rationale string) string { + return applyReplacements(template, []replacement{ + {"{{TITLE}}", finding.Title}, + {"{{DESCRIPTION}}", finding.Description}, + {"{{CWE_ID}}", finding.CweID}, + {"{{CWE_NAME}}", finding.CweName}, + {"{{FILE_PATH}}", finding.FilePath}, + {"{{START_LINE}}", strconv.Itoa(finding.StartLine)}, + {"{{CODE_SNIPPET}}", finding.CodeSnippet}, + {"{{FINDING_TYPE}}", string(finding.FindingType)}, + {"{{VERDICT}}", verdict}, + {"{{RATIONALE}}", rationale}, + {"{{RELATED_FILES}}", jsonDumpsStrings(finding.RelatedFiles, 2)}, + }) +} + +// buildVerifiedPrompt ports the inline replacement table inside +// generate_remediation — the VerifiedFinding adaptation. +// +// Python reaches every value through getattr with a default, because it types +// the parameter `Any` and wants to tolerate a RawFinding too. VerifiedFinding +// is a concrete Go struct, so the getattr chain collapses to direct field reads; +// the two places where the Python defaults are observable are commented below. +func buildVerifiedPrompt(template string, finding schemas.VerifiedFinding) string { + // Python: `location = getattr(finding, "location", None)`, then + // `getattr(location, "file_path", "") if location else getattr(finding, "file_path", "")`. + // VerifiedFinding.location is a REQUIRED pydantic model, and a BaseModel + // instance is always truthy, so the location branch always wins for the + // VerifiedFinding this function is called with. + filePath := finding.Location.FilePath + startLine := finding.Location.StartLine + + // Python: `(proof.vulnerable_code or "") if proof else ""` — a missing proof + // and a proof with a null vulnerable_code both yield "". + codeSnippet := "" + if finding.Proof != nil && finding.Proof.VulnerableCode != nil { + codeSnippet = *finding.Proof.VulnerableCode + } + + // Python: `[loc.file_path for loc in related_locs] if related_locs else []`. + relatedFiles := make([]string, 0, len(finding.RelatedLocations)) + for _, loc := range finding.RelatedLocations { + relatedFiles = append(relatedFiles, loc.FilePath) + } + + // Python: `str(verdict_val.value) if hasattr(verdict_val, "value") else str(verdict_val)`, + // over `getattr(finding, "verdict", "confirmed")`. The "confirmed" default + // only fires for an object with no verdict attribute at all, which a + // VerifiedFinding never is; the Go enum already IS its value. + verdictStr := string(finding.Verdict) + + // Python: `str(finding_type_val.value) if hasattr(...) else str(finding_type_val or "")`. + findingTypeStr := string(finding.FindingType) + + return applyReplacements(template, []replacement{ + {"{{TITLE}}", finding.Title}, + {"{{DESCRIPTION}}", finding.Description}, + {"{{CWE_ID}}", finding.CweID}, + {"{{CWE_NAME}}", finding.CweName}, + {"{{FILE_PATH}}", filePath}, + {"{{START_LINE}}", strconv.Itoa(startLine)}, + {"{{CODE_SNIPPET}}", codeSnippet}, + {"{{FINDING_TYPE}}", findingTypeStr}, + {"{{VERDICT}}", verdictStr}, + {"{{RATIONALE}}", finding.Rationale}, + {"{{RELATED_FILES}}", jsonDumpsStrings(relatedFiles, 2)}, + }) +} + +// RunRemediation ports src/sec_af/agents/remediation.py run_remediation. +// +// async def run_remediation(app, repo_path, finding: RawFinding, verdict, rationale) -> RemediationSuggestion +// +// The tempdir prefix is Python's f"secaf-{agent_name}-" with agent_name = +// "remediation", i.e. "secaf-remediation-" — the same prefix generate_remediation +// hardcodes. +func RunRemediation( + ctx context.Context, + app appx.Harnesser, + repoPath string, + finding schemas.RawFinding, + verdict string, + rationale string, +) (schemas.RemediationSuggestion, error) { + template, err := prompts.Load(promptRel) + if err != nil { + // Python: PROMPT_PATH.read_text() raises OSError out of the reasoner. + return schemas.RemediationSuggestion{}, err + } + prompt := buildPrompt(template, finding, verdict, rationale) + contextSuffix(repoPath) + + harnessCwd, err := os.MkdirTemp("", "secaf-remediation-") + if err != nil { + return schemas.RemediationSuggestion{}, err + } + // Ports `finally: shutil.rmtree(harness_cwd, ignore_errors=True)`. + defer os.RemoveAll(harnessCwd) + + return harnessx.RunExtract[schemas.RemediationSuggestion]( + ctx, app, prompt, + appx.HarnessOptions{Cwd: harnessCwd, ProjectDir: repoPath}, + agentName, + ) +} + +// GenerateRemediation ports src/sec_af/agents/remediation.py generate_remediation. +// +// async def generate_remediation(app, repo_path, finding: Any) -> RemediationSuggestion +// """Adapts VerifiedFinding (location-based) to the prompt template expected by run_remediation.""" +// +// It is NOT a wrapper around RunRemediation in Python (the replacement table is +// duplicated inline) and is not one here either, so the two prompt builders can +// drift independently exactly as the Python ones can. +func GenerateRemediation( + ctx context.Context, + app appx.Harnesser, + repoPath string, + finding schemas.VerifiedFinding, +) (schemas.RemediationSuggestion, error) { + template, err := prompts.Load(promptRel) + if err != nil { + return schemas.RemediationSuggestion{}, err + } + prompt := buildVerifiedPrompt(template, finding) + contextSuffix(repoPath) + + harnessCwd, err := os.MkdirTemp("", "secaf-remediation-") + if err != nil { + return schemas.RemediationSuggestion{}, err + } + defer os.RemoveAll(harnessCwd) + + return harnessx.RunExtract[schemas.RemediationSuggestion]( + ctx, app, prompt, + appx.HarnessOptions{Cwd: harnessCwd, ProjectDir: repoPath}, + agentName, + ) +} + +// jsonDumpsStrings reproduces Python's `json.dumps(list_of_str, indent=indent)` +// for the one shape remediation.py needs: a flat list of strings. +// +// encoding/json cannot be used directly, because the two libraries disagree on +// escaping in both directions: +// +// - Python defaults to ensure_ascii=True, so every code point >= 0x7f becomes +// \uXXXX (a surrogate PAIR above the BMP); Go emits it as raw UTF-8. +// - Go's Marshal HTML-escapes <, > and & into <, >, &; Python +// leaves them alone. +// +// The related_files values are file paths, so in practice both quirks are +// invisible — but the prompt reaches the LLM byte-for-byte and the golden tests +// compare bytes, so the port matches Python exactly rather than nearly. +// +// A nil slice renders as "[]": Python's related_files is +// `Field(default_factory=list)` and can never be None, so Go's nil is the empty +// list, not JSON null. +// +// The encoding itself is pyfmt.Dumps, the port's single CPython json.dumps +// implementation (DESIGN §2b) — this wrapper only pins the list-vs-None +// question below. It replaces an earlier package-local copy of the same +// escaping rules, written before pyfmt.Dumps existed; TestJSONDumpsStrings +// (whose expectations were taken from the venv interpreter) is unchanged and +// still passes, which is what proves the two encoders agree. +func jsonDumpsStrings(values []string, indent int) string { + // pyfmt.Dumps renders a NIL Go slice as `null` (encoding/json parity). The + // two Python fields this feeds — RawFinding.related_files and the list + // built from VerifiedFinding.related_locations — are pydantic + // `Field(default_factory=list)` and a list comprehension respectively, so + // Python has `[]` there and never None. Normalizing is what keeps the + // prompt bytes equal. + if values == nil { + values = []string{} + } + return pyfmt.Dumps(values, indent) +} diff --git a/internal/secaf/agents/remediation/remediation_test.go b/internal/secaf/agents/remediation/remediation_test.go new file mode 100644 index 0000000000..cc58c8cba9 --- /dev/null +++ b/internal/secaf/agents/remediation/remediation_test.go @@ -0,0 +1,384 @@ +package remediation + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/prompts" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// --------------------------------------------------------------------------- +// fixtures / helpers +// --------------------------------------------------------------------------- + +const fixtureRepo = "/fixtures/demo-repo" + +func str(s string) *string { return &s } + +// rawFinding mirrors gen_golden.py's _s6_raw_finding defaults. +func rawFinding(mut func(f *schemas.RawFinding)) schemas.RawFinding { + f := schemas.NewRawFinding() + f.ID = "F1" + f.HunterStrategy = "injection" + f.Title = "SQL injection in user lookup" + f.Description = "User-controlled `user_id` flows into a raw SQL string." + f.FindingType = schemas.FindingTypeSast + f.CweID = "CWE-89" + f.CweName = "SQL Injection" + f.FilePath = "src/db/users.py" + f.StartLine = 42 + f.EndLine = 44 + f.CodeSnippet = `cur.execute("SELECT * FROM users WHERE id = " + user_id)` + f.EstimatedSeverity = schemas.SeverityCritical + f.Confidence = schemas.ConfidenceHigh + f.RelatedFiles = []string{} + f.Fingerprint = "fp-F1" + if mut != nil { + mut(&f) + } + return f +} + +// verifiedFinding mirrors gen_golden.py's _s6_verified_finding. +func verifiedFinding(withProof bool) schemas.VerifiedFinding { + v := schemas.NewVerifiedFinding() + v.ID = "V1" + v.Fingerprint = "fp-V1" + v.Title = "SQL injection in user lookup" + v.Description = "User-controlled `user_id` flows into a raw SQL string." + v.FindingType = schemas.FindingTypeSast + v.CweID = "CWE-89" + v.CweName = "SQL Injection" + v.Verdict = schemas.VerdictConfirmed + v.EvidenceLevel = schemas.EvidenceLevelExploitScenarioValidated + v.Rationale = "Tainted parameter reaches the sink with no sanitization." + v.Severity = schemas.SeverityCritical + v.ExploitabilityScore = 8.5 + v.Location = schemas.Location{FilePath: "src/db/users.py", StartLine: 42, EndLine: 44} + v.SarifRuleID = "secaf/cwe-89" + v.SarifSecuritySeverity = 9.0 + if withProof { + v.Proof = &schemas.Proof{ + ExploitHypothesis: "Attacker supplies `1 OR 1=1` to dump the users table.", + VerificationMethod: "static", + EvidenceLevel: schemas.EvidenceLevelExploitScenarioValidated, + VulnerableCode: str(`cur.execute("SELECT * FROM users WHERE id = " + user_id)`), + } + v.RelatedLocations = []schemas.Location{ + {FilePath: "src/api/routes.py", StartLine: 10, EndLine: 12}, + {FilePath: `src/db/naïve_"cache".py`, StartLine: 3, EndLine: 3}, + } + } + return v +} + +// suggestionFake answers every harness call with a canned RemediationSuggestion. +func suggestionFake() *appx.Fake { + return &appx.Fake{HarnessFn: appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return json.RawMessage(`{"fix_description":"Use a parameterized query.","patch_diff":"--- a/x\n+++ b/x\n","confidence":"high"}`), nil + })} +} + +func golden(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join("testdata", "golden", name+".txt")) + if err != nil { + t.Fatalf("read golden %s: %v", name, err) + } + return string(b) +} + +func assertGolden(t *testing.T, name, got string) { + t.Helper() + if want := golden(t, name); got != want { + t.Errorf("prompt does not match golden %s.txt\n--- got (%d bytes) ---\n%s\n--- want (%d bytes) ---\n%s", + name, len(got), got, len(want), want) + } +} + +// capture runs fn and returns the single prompt the harness was handed. +func capture(t *testing.T, fake *appx.Fake, fn func() error) string { + t.Helper() + if err := fn(); err != nil { + t.Fatalf("call: %v", err) + } + if len(fake.Harnesses) != 1 { + t.Fatalf("want 1 harness call, got %d", len(fake.Harnesses)) + } + return fake.Harnesses[0].Prompt +} + +// --------------------------------------------------------------------------- +// golden prompts +// --------------------------------------------------------------------------- +// +// Regenerate with: +// +// PYTHONPATH=src ~/.agentfield/packages/sec-af/venv/bin/python go/scripts/gen_golden.py +// +// (section "S6" of that script drives the real Python run_remediation / +// generate_remediation with these exact fixtures).// +// NOTE (integration): the "S6" section named above is NO LONGER PRESENT in +// go/scripts/gen_golden.py — it was lost when several agents rewrote that file +// concurrently during the port. Running the script does NOT refresh these +// files. The committed goldens ARE the ones that section produced from the real +// Python functions, and this test still guards them; but if the Python prompt +// builder changes, re-derive them by hand from the fixtures below (or restore +// the section) rather than trusting the script. See the COVERAGE GAP comment in +// gen_golden.py. + +func TestGolden_RunRemediationPrompt(t *testing.T) { + fake := suggestionFake() + finding := rawFinding(func(f *schemas.RawFinding) { + f.RelatedFiles = []string{"src/api/routes.py", `src/db/naïve_"cache".py`, "src/&co.py"} + }) + got := capture(t, fake, func() error { + _, err := RunRemediation(context.Background(), fake, fixtureRepo, finding, + "confirmed", "Tainted parameter reaches the sink with no sanitization.") + return err + }) + assertGolden(t, "run_prompt", got) +} + +func TestGolden_RunRemediationPromptEmptyRelatedFiles(t *testing.T) { + fake := suggestionFake() + got := capture(t, fake, func() error { + _, err := RunRemediation(context.Background(), fake, fixtureRepo, rawFinding(nil), "likely", "") + return err + }) + assertGolden(t, "run_prompt_empty", got) +} + +func TestGolden_GenerateRemediationPrompt(t *testing.T) { + fake := suggestionFake() + got := capture(t, fake, func() error { + _, err := GenerateRemediation(context.Background(), fake, fixtureRepo, verifiedFinding(true)) + return err + }) + assertGolden(t, "generate_prompt", got) +} + +func TestGolden_GenerateRemediationPromptMinimal(t *testing.T) { + fake := suggestionFake() + got := capture(t, fake, func() error { + _, err := GenerateRemediation(context.Background(), fake, fixtureRepo, verifiedFinding(false)) + return err + }) + assertGolden(t, "generate_prompt_min", got) +} + +// --------------------------------------------------------------------------- +// prompt construction +// --------------------------------------------------------------------------- + +func TestBuildPrompt_SubstitutesEveryPlaceholder(t *testing.T) { + template := prompts.MustLoad(promptRel) + got := buildPrompt(template, rawFinding(nil), "confirmed", "because") + if strings.Contains(got, "{{") { + t.Errorf("unsubstituted placeholder left in prompt:\n%s", got) + } + for _, want := range []string{ + "Finding: SQL injection in user lookup", + "CWE: CWE-89 (SQL Injection)", + "Type: sast", + "File: src/db/users.py:42", + "Verdict: confirmed", + "Rationale: because", + } { + if !strings.Contains(got, want) { + t.Errorf("prompt missing %q", want) + } + } +} + +func TestBuildVerifiedPrompt_ReadsLocationProofAndRelatedLocations(t *testing.T) { + template := prompts.MustLoad(promptRel) + got := buildVerifiedPrompt(template, verifiedFinding(true)) + if strings.Contains(got, "{{") { + t.Errorf("unsubstituted placeholder left in prompt:\n%s", got) + } + for _, want := range []string{ + "File: src/db/users.py:42", + "Verdict: confirmed", + "Rationale: Tainted parameter reaches the sink with no sanitization.", + `cur.execute("SELECT * FROM users WHERE id = " + user_id)`, + `"src/api/routes.py"`, + } { + if !strings.Contains(got, want) { + t.Errorf("prompt missing %q", want) + } + } +} + +// Python: `(proof.vulnerable_code or "") if proof else ""`. +func TestBuildVerifiedPrompt_NilProofYieldsEmptySnippet(t *testing.T) { + template := prompts.MustLoad(promptRel) + got := buildVerifiedPrompt(template, verifiedFinding(false)) + if !strings.Contains(got, "Vulnerable code:\n\n") { + t.Errorf("want an empty vulnerable-code block, got:\n%s", got) + } + if !strings.Contains(got, "Related files:\n[]\n") { + t.Errorf("want an empty related-files list, got:\n%s", got) + } +} + +// A proof present but with a null vulnerable_code is Python's `or ""` branch. +func TestBuildVerifiedPrompt_NullVulnerableCodeYieldsEmptySnippet(t *testing.T) { + v := verifiedFinding(true) + v.Proof.VulnerableCode = nil + got := buildVerifiedPrompt(prompts.MustLoad(promptRel), v) + if !strings.Contains(got, "Vulnerable code:\n\n") { + t.Errorf("want an empty vulnerable-code block, got:\n%s", got) + } +} + +func TestContextSuffix(t *testing.T) { + want := "\n\nCONTEXT:\n" + + "- Repository path: /repo\n" + + "- Use the repository path to inspect the actual source code for accurate patch generation." + if got := contextSuffix("/repo"); got != want { + t.Errorf("contextSuffix = %q, want %q", got, want) + } +} + +// --------------------------------------------------------------------------- +// harness interaction +// --------------------------------------------------------------------------- + +func assertTempDirLifecycle(t *testing.T, run func(app appx.Harnesser) error) { + t.Helper() + var seenCwd string + var existedDuringCall bool + fake := &appx.Fake{HarnessFn: func(_ context.Context, _ string, _ map[string]any, dest any, opts appx.HarnessOptions) (*appx.HarnessResult, error) { + seenCwd = opts.Cwd + if st, err := os.Stat(opts.Cwd); err == nil && st.IsDir() { + existedDuringCall = true + } + if err := json.Unmarshal([]byte(`{"fix_description":"f","patch_diff":"d","confidence":"high"}`), dest); err != nil { + return nil, err + } + return &appx.HarnessResult{Parsed: dest}, nil + }} + if err := run(fake); err != nil { + t.Fatalf("call: %v", err) + } + if !existedDuringCall { + t.Error("harness cwd did not exist during the call") + } + if base := filepath.Base(seenCwd); !strings.HasPrefix(base, "secaf-remediation-") { + t.Errorf("harness cwd %q does not use the secaf-remediation- prefix", seenCwd) + } + if _, err := os.Stat(seenCwd); !os.IsNotExist(err) { + t.Errorf("harness cwd %q was not removed after the call (err=%v)", seenCwd, err) + } + if got, want := fake.Harnesses[0].Opts.ProjectDir, fixtureRepo; got != want { + t.Errorf("project_dir = %q, want %q", got, want) + } +} + +func TestRunRemediation_TempDirAndOptions(t *testing.T) { + assertTempDirLifecycle(t, func(app appx.Harnesser) error { + _, err := RunRemediation(context.Background(), app, fixtureRepo, rawFinding(nil), "confirmed", "r") + return err + }) +} + +func TestGenerateRemediation_TempDirAndOptions(t *testing.T) { + assertTempDirLifecycle(t, func(app appx.Harnesser) error { + _, err := GenerateRemediation(context.Background(), app, fixtureRepo, verifiedFinding(true)) + return err + }) +} + +func TestRunRemediation_ReturnsParsedSuggestion(t *testing.T) { + fake := suggestionFake() + got, err := RunRemediation(context.Background(), fake, fixtureRepo, rawFinding(nil), "confirmed", "r") + if err != nil { + t.Fatalf("RunRemediation: %v", err) + } + if got.FixDescription != "Use a parameterized query." || got.Confidence != "high" { + t.Errorf("suggestion = %+v", got) + } +} + +// extract_harness_result(..., "RemediationAgent") turns an is_error result into +// `RuntimeError(f"{agent_name} harness error: {error_message}")`. +func TestRemediation_HarnessErrorMapsToRemediationAgentError(t *testing.T) { + fake := &appx.Fake{HarnessFn: func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{IsError: true, ErrorMessage: "provider exploded"}, nil + }} + for name, call := range map[string]func() error{ + "RunRemediation": func() error { + _, err := RunRemediation(context.Background(), fake, fixtureRepo, rawFinding(nil), "confirmed", "r") + return err + }, + "GenerateRemediation": func() error { + _, err := GenerateRemediation(context.Background(), fake, fixtureRepo, verifiedFinding(true)) + return err + }, + } { + err := call() + if err == nil { + t.Fatalf("%s: want an error", name) + } + if got, want := err.Error(), "RemediationAgent harness error: provider exploded"; got != want { + t.Errorf("%s: error = %q, want %q", name, got, want) + } + } +} + +// A result that neither errored nor parsed is Python's TypeError branch. +func TestRemediation_UnparsedResultMapsToTypeError(t *testing.T) { + fake := &appx.Fake{HarnessFn: func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{Result: "not json"}, nil + }} + _, err := RunRemediation(context.Background(), fake, fixtureRepo, rawFinding(nil), "confirmed", "r") + if err == nil { + t.Fatal("want an error") + } + if got, want := err.Error(), "RemediationAgent did not return a valid RemediationSuggestion"; got != want { + t.Errorf("error = %q, want %q", got, want) + } +} + +// --------------------------------------------------------------------------- +// json.dumps parity +// --------------------------------------------------------------------------- + +// Ground truth captured from the sec-af venv interpreter: +// +// json.dumps(value, indent=2) +func TestJSONDumpsStrings(t *testing.T) { + cases := []struct { + in []string + want string + }{ + {nil, "[]"}, + {[]string{}, "[]"}, + {[]string{"a/b.py"}, "[\n \"a/b.py\"\n]"}, + {[]string{"a/b.py", "c.py"}, "[\n \"a/b.py\",\n \"c.py\"\n]"}, + // ensure_ascii=True: non-ASCII escaped, <>& left alone (Go's Marshal + // does the exact opposite on both counts). + {[]string{"café"}, "[\n \"caf\\u00e9\"\n]"}, + {[]string{"&y"}, "[\n \"&y\"\n]"}, + {[]string{"/slash"}, "[\n \"/slash\"\n]"}, + {[]string{`a"b\c`}, "[\n \"a\\\"b\\\\c\"\n]"}, + {[]string{"\n\t"}, "[\n \"\\n\\t\"\n]"}, + {[]string{"\x00"}, "[\n \"\\u0000\"\n]"}, + {[]string{"\x1f"}, "[\n \"\\u001f\"\n]"}, + {[]string{"\x7f"}, "[\n \"\\u007f\"\n]"}, + // Non-BMP: Python emits a surrogate PAIR. + {[]string{"😀"}, "[\n \"\\ud83d\\ude00\"\n]"}, + } + for _, c := range cases { + if got := jsonDumpsStrings(c.in, 2); got != c.want { + t.Errorf("jsonDumpsStrings(%q) = %q, want %q", c.in, got, c.want) + } + } +} diff --git a/internal/secaf/agents/remediation/testdata/golden/generate_prompt.txt b/internal/secaf/agents/remediation/testdata/golden/generate_prompt.txt new file mode 100644 index 0000000000..e9289c10f1 --- /dev/null +++ b/internal/secaf/agents/remediation/testdata/golden/generate_prompt.txt @@ -0,0 +1,48 @@ +ROLE: +You are a security remediation specialist for SEC-AF. You generate precise, production-ready code fixes. + +CONTEXT: +A security vulnerability has been identified and verified: + +Finding: SQL injection in user lookup +Description: User-controlled `user_id` flows into a raw SQL string. +CWE: CWE-89 (SQL Injection) +Type: sast +File: src/db/users.py:42 +Verdict: confirmed +Rationale: Tainted parameter reaches the sink with no sanitization. + +Vulnerable code: +cur.execute("SELECT * FROM users WHERE id = " + user_id) + +Related files: +[ + "src/api/routes.py", + "src/db/na\u00efve_\"cache\".py" +] + +TASK: +Generate a remediation suggestion with a concrete code patch: +1. Read the actual source file to understand full context +2. Determine the minimal fix that addresses the vulnerability +3. Generate a unified diff patch showing exact changes needed +4. Ensure the fix preserves existing functionality + +OUTPUT: +Return a JSON object matching RemediationSuggestion schema: +- fix_description: Clear explanation of what the fix does and why +- patch_diff: Unified diff format (--- a/file, +++ b/file, @@ lines) showing changes +- confidence: "high" if fix is straightforward, "medium" if context-dependent, "low" if complex + +CONSTRAINTS: +- Fix must be MINIMAL - change only what's needed to remediate +- Patch must be valid unified diff format +- Do not refactor surrounding code +- Preserve existing APIs and function signatures +- Use framework-idiomatic solutions (e.g., parameterized queries, not manual escaping) +- Do not return markdown or prose outside the JSON object + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Use the repository path to inspect the actual source code for accurate patch generation. \ No newline at end of file diff --git a/internal/secaf/agents/remediation/testdata/golden/generate_prompt_min.txt b/internal/secaf/agents/remediation/testdata/golden/generate_prompt_min.txt new file mode 100644 index 0000000000..7e00e57155 --- /dev/null +++ b/internal/secaf/agents/remediation/testdata/golden/generate_prompt_min.txt @@ -0,0 +1,45 @@ +ROLE: +You are a security remediation specialist for SEC-AF. You generate precise, production-ready code fixes. + +CONTEXT: +A security vulnerability has been identified and verified: + +Finding: SQL injection in user lookup +Description: User-controlled `user_id` flows into a raw SQL string. +CWE: CWE-89 (SQL Injection) +Type: sast +File: src/db/users.py:42 +Verdict: confirmed +Rationale: Tainted parameter reaches the sink with no sanitization. + +Vulnerable code: + + +Related files: +[] + +TASK: +Generate a remediation suggestion with a concrete code patch: +1. Read the actual source file to understand full context +2. Determine the minimal fix that addresses the vulnerability +3. Generate a unified diff patch showing exact changes needed +4. Ensure the fix preserves existing functionality + +OUTPUT: +Return a JSON object matching RemediationSuggestion schema: +- fix_description: Clear explanation of what the fix does and why +- patch_diff: Unified diff format (--- a/file, +++ b/file, @@ lines) showing changes +- confidence: "high" if fix is straightforward, "medium" if context-dependent, "low" if complex + +CONSTRAINTS: +- Fix must be MINIMAL - change only what's needed to remediate +- Patch must be valid unified diff format +- Do not refactor surrounding code +- Preserve existing APIs and function signatures +- Use framework-idiomatic solutions (e.g., parameterized queries, not manual escaping) +- Do not return markdown or prose outside the JSON object + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Use the repository path to inspect the actual source code for accurate patch generation. \ No newline at end of file diff --git a/internal/secaf/agents/remediation/testdata/golden/run_prompt.txt b/internal/secaf/agents/remediation/testdata/golden/run_prompt.txt new file mode 100644 index 0000000000..b5facb6017 --- /dev/null +++ b/internal/secaf/agents/remediation/testdata/golden/run_prompt.txt @@ -0,0 +1,49 @@ +ROLE: +You are a security remediation specialist for SEC-AF. You generate precise, production-ready code fixes. + +CONTEXT: +A security vulnerability has been identified and verified: + +Finding: SQL injection in user lookup +Description: User-controlled `user_id` flows into a raw SQL string. +CWE: CWE-89 (SQL Injection) +Type: sast +File: src/db/users.py:42 +Verdict: confirmed +Rationale: Tainted parameter reaches the sink with no sanitization. + +Vulnerable code: +cur.execute("SELECT * FROM users WHERE id = " + user_id) + +Related files: +[ + "src/api/routes.py", + "src/db/na\u00efve_\"cache\".py", + "src/&co.py" +] + +TASK: +Generate a remediation suggestion with a concrete code patch: +1. Read the actual source file to understand full context +2. Determine the minimal fix that addresses the vulnerability +3. Generate a unified diff patch showing exact changes needed +4. Ensure the fix preserves existing functionality + +OUTPUT: +Return a JSON object matching RemediationSuggestion schema: +- fix_description: Clear explanation of what the fix does and why +- patch_diff: Unified diff format (--- a/file, +++ b/file, @@ lines) showing changes +- confidence: "high" if fix is straightforward, "medium" if context-dependent, "low" if complex + +CONSTRAINTS: +- Fix must be MINIMAL - change only what's needed to remediate +- Patch must be valid unified diff format +- Do not refactor surrounding code +- Preserve existing APIs and function signatures +- Use framework-idiomatic solutions (e.g., parameterized queries, not manual escaping) +- Do not return markdown or prose outside the JSON object + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Use the repository path to inspect the actual source code for accurate patch generation. \ No newline at end of file diff --git a/internal/secaf/agents/remediation/testdata/golden/run_prompt_empty.txt b/internal/secaf/agents/remediation/testdata/golden/run_prompt_empty.txt new file mode 100644 index 0000000000..53548d1821 --- /dev/null +++ b/internal/secaf/agents/remediation/testdata/golden/run_prompt_empty.txt @@ -0,0 +1,45 @@ +ROLE: +You are a security remediation specialist for SEC-AF. You generate precise, production-ready code fixes. + +CONTEXT: +A security vulnerability has been identified and verified: + +Finding: SQL injection in user lookup +Description: User-controlled `user_id` flows into a raw SQL string. +CWE: CWE-89 (SQL Injection) +Type: sast +File: src/db/users.py:42 +Verdict: likely +Rationale: + +Vulnerable code: +cur.execute("SELECT * FROM users WHERE id = " + user_id) + +Related files: +[] + +TASK: +Generate a remediation suggestion with a concrete code patch: +1. Read the actual source file to understand full context +2. Determine the minimal fix that addresses the vulnerability +3. Generate a unified diff patch showing exact changes needed +4. Ensure the fix preserves existing functionality + +OUTPUT: +Return a JSON object matching RemediationSuggestion schema: +- fix_description: Clear explanation of what the fix does and why +- patch_diff: Unified diff format (--- a/file, +++ b/file, @@ lines) showing changes +- confidence: "high" if fix is straightforward, "medium" if context-dependent, "low" if complex + +CONSTRAINTS: +- Fix must be MINIMAL - change only what's needed to remediate +- Patch must be valid unified diff format +- Do not refactor surrounding code +- Preserve existing APIs and function signatures +- Use framework-idiomatic solutions (e.g., parameterized queries, not manual escaping) +- Do not return markdown or prose outside the JSON object + + +CONTEXT: +- Repository path: /fixtures/demo-repo +- Use the repository path to inspect the actual source code for accurate patch generation. \ No newline at end of file diff --git a/internal/secaf/aix/aix.go b/internal/secaf/aix/aix.go new file mode 100644 index 0000000000..c65f7fcdf4 --- /dev/null +++ b/internal/secaf/aix/aix.go @@ -0,0 +1,388 @@ +// Package aix is the Go form of SEC-AF's `.ai(...)` gate calls — the single +// structured LLM request, as opposed to the multi-turn coding-agent session +// that internal/harnessx drives. +// +// Python shape (src/sec_af/agents/prove/verdict.py:99, reasoners/phases.py:125, +// harness.py:420, agents/dedup.py:113, compliance/mapping.py:410): +// +// result = await app.ai(user=prompt, schema=VerdictDecision) +// result = await app.ai(system=system, user=prompt, schema=Model, model=...) +// +// The Python SDK turns `schema=Model` into an OpenAI structured-output request +// (agentfield/agent_ai.py:803): +// +// "schema": _strictify_openai_schema(schema.model_json_schema()) +// +// so the Go port must (a) start from the SAME pydantic schema — the committed +// fixture harnessx already embeds — and (b) apply the SAME strictification, +// because OpenAI's strict mode rejects a schema that omits +// additionalProperties:false or under-populates required. +package aix + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "reflect" + "sort" + "strings" + + "github.com/Agent-Field/agentfield/sdk/go/ai" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/harnessx" +) + +// Structured performs `await app.ai(system=system, user=user, schema=T)` and +// returns the parsed T. +// +// system == "" means Python's `system=None`: no system message is added, which +// is what every SEC-AF call site except AIGateWrapper.invoke does. +// +// Decoding is TOLERANT and RETRIED exactly as the Python SDK's is — see +// parseStructured and maxParseRetries. +func Structured[T any](ctx context.Context, app appx.AIer, system, user string) (T, error) { + return StructuredOpts[T](ctx, app, system, user) +} + +// StructuredOpts is Structured with extra SDK options appended after the schema +// option — the seam AIGateWrapper.invoke needs, since it also passes +// `model=self.config.ai_model` (src/sec_af/harness.go:420-425). Keeping it +// separate leaves Structured's signature exactly the one the design doc +// specifies. +func StructuredOpts[T any](ctx context.Context, app appx.AIer, system, user string, extra ...ai.Option) (T, error) { + var v T + typeName := reflect.TypeOf((*T)(nil)).Elem().Name() + + raw, err := json.Marshal(StrictifyOrdered[T](harnessx.SchemaFor[T]())) + if err != nil { + return v, fmt.Errorf("aix.Structured[%s]: marshal schema: %w", typeName, err) + } + + opts := make([]ai.Option, 0, 2+len(extra)) + if system != "" { + opts = append(opts, ai.WithSystem(system)) + } + // json.RawMessage takes the SDK's pass-through branch: the bytes become + // response_format.json_schema.schema verbatim, with strict:true — the same + // request body the Python SDK builds. + opts = append(opts, ai.WithSchema(json.RawMessage(raw))) + opts = append(opts, extra...) + + // Python parity (agentfield/agent_ai.py:1061-1088): the SDK retries the + // WHOLE call — request AND parse — up to `max_parse_retries = 2` more + // times when the body cannot be decoded, i.e. 3 attempts in total. A + // TRANSPORT error is not retried here: Python's loop only catches the + // `ValueError("Could not parse structured response: ...")` that + // `_execute_and_parse` raises, and the SDK's own rate-limit/fallback + // retries sit below this layer. + var lastErr error + for attempt := 0; attempt <= maxParseRetries; attempt++ { + resp, err := app.AI(ctx, user, opts...) + if err != nil { + return v, fmt.Errorf("aix.Structured[%s]: %w", typeName, err) + } + if resp == nil { + // Go-only guard. Python would raise AttributeError inside + // detect_multimodal_response, which the `except ValueError` retry + // loop does not catch either — so, as here, no retry. + return v, fmt.Errorf("aix.Structured[%s]: nil response", typeName) + } + parsed, perr := parseStructured[T](resp.Text()) + if perr == nil { + return parsed, nil + } + lastErr = fmt.Errorf("aix.Structured[%s]: %w", typeName, perr) + } + return v, lastErr +} + +// maxParseRetries ports `max_parse_retries = 2` (agent_ai.py:960): two RETRIES +// on top of the first attempt, so three AI requests at most. +const maxParseRetries = 2 + +// parseStructured ports the tolerant structured-output decode the Python SDK +// performs after every `.ai(schema=...)` completion (agent_ai.py:1032-1060): +// +// try: +// json_data = json.loads(str(text)); return schema(**json_data) +// except (json.JSONDecodeError, ValueError, ValidationError): +// json_match = re.search(r"\{.*\}", str(text), re.DOTALL) +// if json_match: +// try: return schema(**json.loads(json_match.group())) +// except (...): pass +// raise ValueError(f"Could not parse structured response: {text}") +// +// i.e. a straight decode first, then a greedy first-`{`..last-`}` extraction, +// then give up. This is what lets a model that wraps its JSON in a ```json +// fence — or prefixes it with prose — still succeed; observed live with +// kimi-k2.5 on the run_verifier gate, where the strict Go decode failed the +// execution and the Python node did not. +// +// The returned error text reproduces Python's message verbatim, including the +// full (untruncated) body, because that string is what an operator reads out +// of a failed execution on either node. +// +// Known, pre-existing divergence: `schema(**data)` is pydantic validation and +// rejects a payload missing a required field, while json.Unmarshal happily +// leaves it zero. Go therefore accepts a few bodies Python would retry on. +func parseStructured[T any](text string) (T, error) { + var direct T + if err := json.Unmarshal([]byte(text), &direct); err == nil { + return direct, nil + } + if candidate, ok := extractJSONObject(text); ok { + // A fresh destination: a failed Unmarshal may have already written + // part of `direct`. + var extracted T + if err := json.Unmarshal([]byte(candidate), &extracted); err == nil { + return extracted, nil + } + } + var zero T + return zero, fmt.Errorf("Could not parse structured response: %s", text) //nolint:staticcheck // Python's message, verbatim +} + +// extractJSONObject is Go's `re.search(r"\{.*\}", text, re.DOTALL)`. +// +// The pattern is GREEDY and unanchored: the engine takes the earliest `{` it +// can start from, then backtracks `.*` to the LAST `}` in the string. If no +// `}` follows the first `{` there is no later `{` that could do better either, +// so the whole search fails — which is exactly the two-index form below. +// Indexing by byte is safe: `{` and `}` are ASCII and can never occur inside a +// UTF-8 continuation sequence. +func extractJSONObject(s string) (string, bool) { + start := strings.Index(s, "{") + if start < 0 { + return "", false + } + end := strings.LastIndex(s, "}") + if end <= start { + return "", false + } + return s[start : end+1], true +} + +// Strictify ports the Python SDK's _strictify_openai_schema +// (`agentfield.agent_ai._strictify_openai_schema`, sdk/python/agentfield/agent_ai.py): +// +// def walk(node): +// if isinstance(node, dict): +// node = {key: walk(value) for key, value in node.items()} +// props = node.get("properties") +// if isinstance(props, dict) and (node.get("type") == "object" or "type" not in node): +// node["additionalProperties"] = False +// node["required"] = list(props.keys()) +// return node +// if isinstance(node, list): +// return [walk(item) for item in node] +// return node +// +// i.e. EVERY dict that has a "properties" object and is either explicitly +// type:"object" or carries no "type" at all gets additionalProperties:false and +// a required list naming all of its properties — recursing first, so $defs, +// nested properties, items and anyOf branches are all covered. Forcing every +// property into required is OpenAI's documented strict-mode requirement; a +// genuinely optional field is expressed as nullable, not as absent-from-required. +// +// Two Go-specific notes: +// +// - The input is never mutated; a fresh tree is returned. +// - `required` is emitted in SORTED key order, because a bare +// `map[string]any` carries no insertion order to reproduce Python's +// `list(props.keys())`. The LIVE path does not go through here: it calls +// StrictifyOrdered[T], which recovers pydantic's declaration order from the +// Go struct. This entry point stays for the schema transform itself (and +// for the goldens, which are generated from the committed sort_keys=True +// fixtures, so their own property order is sorted). +func Strictify(schema map[string]any) map[string]any { + out, _ := strictifyNode(schema).(map[string]any) + return out +} + +func strictifyNode(node any) any { + switch x := node.(type) { + case map[string]any: + out := make(map[string]any, len(x)+2) + for k, v := range x { + out[k] = strictifyNode(v) + } + props, isObject := out["properties"].(map[string]any) + if isObject { + typ, hasType := out["type"] + // Python: `node.get("type") == "object" or "type" not in node`. + // A list-valued "type" (e.g. ["object","null"]) satisfies neither + // arm, in Python and here alike. + if (hasType && typ == "object") || !hasType { + out["additionalProperties"] = false + keys := make([]string, 0, len(props)) + for k := range props { + keys = append(keys, k) + } + sort.Strings(keys) + required := make([]any, len(keys)) + for i, k := range keys { + required[i] = k + } + out["required"] = required + } + } + return out + case []any: + out := make([]any, len(x)) + for i, e := range x { + out[i] = strictifyNode(e) + } + return out + default: + return node + } +} + +// StrictifyOrdered is Strictify plus pydantic's DECLARATION order. +// +// Python strictifies the LIVE `Model.model_json_schema()`, whose `properties` +// is an insertion-ordered dict in pydantic field-declaration order; +// `node["required"] = list(props.keys())` (agent_ai.py:319) therefore inherits +// that order, and litellm serialises the dict as-is. Go decodes the committed +// fixture into a `map[string]any`, which has no order at all, and +// `json.Marshal` sorts every map key — so without this the bytes in +// `response_format.json_schema.schema` differ from Python's for every +// `.ai(schema=...)` call. +// +// The order is recovered from the Go struct, which the port keeps in pydantic +// declaration order (harnessx.FieldOrders). It is applied to the ROOT object +// and to every `$defs` entry whose name matches a struct reachable from T — +// which is every object node pydantic emits, because a nested BaseModel always +// becomes a `$defs` entry. Property names with no matching Go field are +// appended in sorted order rather than dropped, so a drifted fixture degrades +// to a visible ordering difference instead of losing schema content. +// +// Residual: the key order WITHIN each node (title/type/description/...) is +// still sorted rather than pydantic's, because go/scripts/gen_schemas.py writes +// the fixtures with sort_keys=True and that order is gone before Go sees it. +// The finding this closes is about `properties` and `required`; those are the +// two the pydantic document orders meaningfully. +func StrictifyOrdered[T any](schema map[string]any) any { + return strictifyOrderedNode(schema, harnessx.FieldOrdersFor[T](), + reflect.TypeOf((*T)(nil)).Elem().Name()) +} + +// strictifyOrderedNode is strictifyNode with an ORDER for the node's own +// properties. typeName names the pydantic class this node describes ("" when +// the node is not a model root), which is how the $defs entries are matched. +func strictifyOrderedNode(node any, orders map[string][]string, typeName string) any { + switch x := node.(type) { + case map[string]any: + out := make(map[string]any, len(x)+2) + for k, v := range x { + switch k { + case "$defs": + defs, isMap := v.(map[string]any) + if !isMap { + out[k] = strictifyOrderedNode(v, orders, "") + continue + } + newDefs := make(map[string]any, len(defs)) + for defName, defSchema := range defs { + newDefs[defName] = strictifyOrderedNode(defSchema, orders, defName) + } + out[k] = newDefs + case "properties": + props, isMap := v.(map[string]any) + if !isMap { + // Python leaves a non-dict "properties" alone (and the + // strictify branch below never fires for it). + out[k] = strictifyOrderedNode(v, orders, "") + continue + } + strict := make(map[string]any, len(props)) + for name, sub := range props { + strict[name] = strictifyOrderedNode(sub, orders, "") + } + out[k] = strict + default: + out[k] = strictifyOrderedNode(v, orders, "") + } + } + props, isObject := out["properties"].(map[string]any) + if isObject { + typ, hasType := out["type"] + if (hasType && typ == "object") || !hasType { + names := orderedPropertyNames(props, orders[typeName]) + out["additionalProperties"] = false + ordered := make(jsonObject, len(names)) + required := make([]any, len(names)) + for i, name := range names { + ordered[i] = jsonField{Key: name, Value: props[name]} + required[i] = name + } + out["properties"] = ordered + out["required"] = required + } + } + return out + case []any: + out := make([]any, len(x)) + for i, e := range x { + out[i] = strictifyOrderedNode(e, orders, "") + } + return out + default: + return node + } +} + +// orderedPropertyNames lists props following `order`, appending anything the +// order does not mention in sorted order. +func orderedPropertyNames(props map[string]any, order []string) []string { + names := make([]string, 0, len(props)) + emitted := make(map[string]struct{}, len(props)) + for _, name := range order { + if _, ok := props[name]; ok { + names = append(names, name) + emitted[name] = struct{}{} + } + } + leftovers := make([]string, 0) + for name := range props { + if _, done := emitted[name]; !done { + leftovers = append(leftovers, name) + } + } + sort.Strings(leftovers) + return append(names, leftovers...) +} + +// jsonObject is a JSON object that marshals its members in the order given +// rather than in the sorted order encoding/json imposes on a map. +type jsonObject []jsonField + +type jsonField struct { + Key string + Value any +} + +func (o jsonObject) MarshalJSON() ([]byte, error) { + var b bytes.Buffer + b.WriteByte('{') + for i, f := range o { + if i > 0 { + b.WriteByte(',') + } + key, err := json.Marshal(f.Key) + if err != nil { + return nil, err + } + b.Write(key) + b.WriteByte(':') + value, err := json.Marshal(f.Value) + if err != nil { + return nil, err + } + b.Write(value) + } + b.WriteByte('}') + return b.Bytes(), nil +} diff --git a/internal/secaf/aix/aix_test.go b/internal/secaf/aix/aix_test.go new file mode 100644 index 0000000000..87b8346186 --- /dev/null +++ b/internal/secaf/aix/aix_test.go @@ -0,0 +1,428 @@ +package aix + +import ( + "context" + "encoding/json" + "errors" + "os" + "reflect" + "strings" + "testing" + + "github.com/Agent-Field/agentfield/sdk/go/ai" + + "github.com/Agent-Field/codeaf/internal/secaf/harnessx" +) + +// --------------------------------------------------------------------------- +// Strictify goldens +// +// testdata/*.json is produced by testdata/gen_strictify_golden.py, which runs +// the REAL Python SDK function this file ports +// (`agentfield.agent_ai._strictify_openai_schema`) over the committed pydantic +// fixtures and over a hand-written edge-case document. Regenerate with any +// Python that has the `agentfield` package installed: +// +// go/internal/aix/testdata/gen_strictify_golden.py +// --------------------------------------------------------------------------- + +func loadJSON(t *testing.T, path string, dest any) { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + if err := json.Unmarshal(b, dest); err != nil { + t.Fatalf("decode %s: %v", path, err) + } +} + +// TestStrictifyMatchesPythonOnEveryFixture is the parity gate: for every schema +// SEC-AF actually strictifies at runtime, Go's output must equal what the Python +// SDK produced from the same document — same additionalProperties placement, +// same required lists (compared element-wise, so ORDER is checked too), same +// untouched nodes. +func TestStrictifyMatchesPythonOnEveryFixture(t *testing.T) { + var want map[string]any + loadJSON(t, "testdata/strictified_fixtures.json", &want) + + names := harnessx.FixtureNames() + if len(names) == 0 { + t.Fatal("harnessx embedded no fixtures") + } + if len(want) != len(names) { + t.Fatalf("golden has %d schemas, harnessx embeds %d — regenerate testdata/strictified_fixtures.json", + len(want), len(names)) + } + + for _, name := range names { + in, err := harnessx.LoadFixture(name) + if err != nil { + t.Errorf("%s: %v", name, err) + continue + } + got := Strictify(in) + if !reflect.DeepEqual(got, want[name]) { + gb, _ := json.MarshalIndent(got, "", " ") + wb, _ := json.MarshalIndent(want[name], "", " ") + t.Errorf("%s: Strictify diverged from python\n got: %s\nwant: %s", name, gb, wb) + } + } +} + +// TestStrictifyEdgeCases covers the walk branches the real fixtures do not +// reach: a properties-bearing node with NO "type" (strictified), a LIST-valued +// "type" (untouched), a non-dict "properties" value (untouched), an anyOf +// branch, nested items, and a stale required/additionalProperties pair that must +// be overwritten. +func TestStrictifyEdgeCases(t *testing.T) { + var in, want map[string]any + loadJSON(t, "testdata/edgecases_input.json", &in) + loadJSON(t, "testdata/edgecases_strict.json", &want) + + got := Strictify(in) + if !reflect.DeepEqual(got, want) { + gb, _ := json.MarshalIndent(got, "", " ") + wb, _ := json.MarshalIndent(want, "", " ") + t.Errorf("Strictify(edgecases) diverged from python\n got: %s\nwant: %s", gb, wb) + } +} + +// TestStrictifyDoesNotMutateInput — Python's walk rebuilds every dict, so the +// caller's schema is untouched. harnessx caches and shares the fixture map +// across goroutines, so mutating it would be a data race AND would corrupt the +// schema the harness path sends. +func TestStrictifyDoesNotMutateInput(t *testing.T) { + in := map[string]any{ + "type": "object", + "properties": map[string]any{ + "a": map[string]any{"type": "string"}, + }, + } + _ = Strictify(in) + if _, present := in["additionalProperties"]; present { + t.Error("Strictify mutated the input map") + } + if _, present := in["required"]; present { + t.Error("Strictify added required to the input map") + } +} + +// TestStrictifySharedFixtureIsNotMutated guards the specific aliasing hazard: +// Structured strictifies harnessx's CACHED fixture map. +func TestStrictifySharedFixtureIsNotMutated(t *testing.T) { + before, err := harnessx.LoadFixture("CWEExpansion") + if err != nil { + t.Fatalf("LoadFixture: %v", err) + } + cached := harnessx.SchemaFor[CWEExpansion]() + _ = Strictify(cached) + after := harnessx.SchemaFor[CWEExpansion]() + if !reflect.DeepEqual(after, before) { + t.Error("Strictify mutated the cached harnessx fixture") + } +} + +// --------------------------------------------------------------------------- +// Structured +// --------------------------------------------------------------------------- + +// CWEExpansion mirrors the pydantic gate model of the same name; the fixture is +// resolved by this Go type's NAME. +type CWEExpansion struct { + AdditionalCWEs []string `json:"additional_cwes"` + Rationale string `json:"rationale"` +} + +type fakeAI struct { + gotPrompt string + gotReq *ai.Request + calls int + + content string + err error +} + +func (f *fakeAI) AI(ctx context.Context, prompt string, opts ...ai.Option) (*ai.Response, error) { + f.calls++ + f.gotPrompt = prompt + + req := &ai.Request{} + for _, o := range opts { + if err := o(req); err != nil { + return nil, err + } + } + f.gotReq = req + + if f.err != nil { + return nil, f.err + } + return &ai.Response{ + Choices: []ai.Choice{{ + Message: ai.Message{ + Role: "assistant", + Content: []ai.ContentPart{{Type: "text", Text: f.content}}, + }, + }}, + }, nil +} + +func TestStructuredSendsStrictifiedPydanticSchema(t *testing.T) { + f := &fakeAI{content: `{"additional_cwes":["CWE-918"],"rationale":"ssrf"}`} + + got, err := Structured[CWEExpansion](context.Background(), f, "", "suggest CWEs") + if err != nil { + t.Fatalf("Structured: %v", err) + } + want := CWEExpansion{AdditionalCWEs: []string{"CWE-918"}, Rationale: "ssrf"} + if !reflect.DeepEqual(got, want) { + t.Errorf("Structured = %#v, want %#v", got, want) + } + + if f.gotPrompt != "suggest CWEs" { + t.Errorf("prompt = %q, want the `user=` argument", f.gotPrompt) + } + // system == "" is Python's system=None: NO system message is prepended. + if len(f.gotReq.Messages) != 0 { + t.Errorf("messages = %#v, want none for system==\"\"", f.gotReq.Messages) + } + rf := f.gotReq.ResponseFormat + if rf == nil || rf.Type != "json_schema" || rf.JSONSchema == nil { + t.Fatalf("response_format = %#v, want a json_schema block", rf) + } + if !rf.JSONSchema.Strict { + t.Error("json_schema.strict = false, want true (the SDK sets it for the RawMessage branch)") + } + + var sent map[string]any + if err := json.Unmarshal(rf.JSONSchema.Schema, &sent); err != nil { + t.Fatalf("sent schema is not JSON: %v", err) + } + if sent["title"] != "CWEExpansion" { + t.Errorf("sent schema title = %v, want the pydantic fixture's", sent["title"]) + } + if sent["additionalProperties"] != false { + t.Errorf("sent schema additionalProperties = %v, want false (strictified)", sent["additionalProperties"]) + } + if !reflect.DeepEqual(sent["required"], []any{"additional_cwes", "rationale"}) { + t.Errorf("sent schema required = %v, want every property", sent["required"]) + } +} + +func TestStructuredAddsSystemMessageWhenPresent(t *testing.T) { + f := &fakeAI{content: `{"additional_cwes":[],"rationale":""}`} + if _, err := Structured[CWEExpansion](context.Background(), f, "You are a gate.", "u"); err != nil { + t.Fatalf("Structured: %v", err) + } + if len(f.gotReq.Messages) != 1 || f.gotReq.Messages[0].Role != "system" { + t.Fatalf("messages = %#v, want one system message", f.gotReq.Messages) + } + if got := f.gotReq.Messages[0].Content[0].Text; got != "You are a gate." { + t.Errorf("system content = %q", got) + } +} + +func TestStructuredExtraOptionsAreApplied(t *testing.T) { + f := &fakeAI{content: `{"additional_cwes":[],"rationale":""}`} + if _, err := StructuredOpts[CWEExpansion](context.Background(), f, "", "u", ai.WithModel("minimax/minimax-m2.5")); err != nil { + t.Fatalf("StructuredOpts: %v", err) + } + if f.gotReq.Model != "minimax/minimax-m2.5" { + t.Errorf("model = %q, want the extra option applied (AIGateWrapper passes model=)", f.gotReq.Model) + } +} + +func TestStructuredPropagatesTransportError(t *testing.T) { + want := errors.New("429 rate limited") + f := &fakeAI{err: want} + _, err := Structured[CWEExpansion](context.Background(), f, "", "u") + if !errors.Is(err, want) { + t.Errorf("Structured error = %v, want the SDK error wrapped", err) + } + if err == nil || !strings.Contains(err.Error(), "CWEExpansion") { + t.Errorf("Structured error = %v, want the destination type named", err) + } + // Python parity: the `for attempt in range(max_parse_retries + 1)` loop + // catches only the parse ValueError, so a transport failure is NOT retried. + if f.calls != 1 { + t.Errorf("AI calls = %d, want 1 (a transport error must not be retried)", f.calls) + } +} + +// An empty completion is not a special case in Python: `json.loads("")` raises, +// the `\{.*\}` salvage finds nothing, and the SAME +// ValueError("Could not parse structured response: ") is raised and retried. +// Verified against agent_ai.py's `if schema:` branch. +func TestStructuredEmptyContentIsARetriedParseFailure(t *testing.T) { + f := &fakeAI{content: ""} + _, err := Structured[CWEExpansion](context.Background(), f, "", "u") + if err == nil { + t.Fatal("Structured should reject an empty completion") + } + if !strings.Contains(err.Error(), "Could not parse structured response") { + t.Errorf("Structured error = %v, want Python's parse-failure message", err) + } + if !strings.Contains(err.Error(), "CWEExpansion") { + t.Errorf("Structured error = %v, want the destination type named", err) + } + if f.calls != 3 { + t.Errorf("AI calls = %d, want 3 (1 + max_parse_retries)", f.calls) + } +} + +func TestStructuredMalformedJSONIsAnError(t *testing.T) { + f := &fakeAI{content: "I think the answer is CWE-918."} + _, err := Structured[CWEExpansion](context.Background(), f, "", "u") + if err == nil { + t.Fatal("Structured should reject non-JSON content") + } + if !strings.Contains(err.Error(), "CWEExpansion") || !strings.Contains(err.Error(), "I think the answer") { + t.Errorf("Structured error = %v, want the type and the offending content", err) + } + if f.calls != 3 { + t.Errorf("AI calls = %d, want 3 (1 + max_parse_retries)", f.calls) + } +} + +// --------------------------------------------------------------------------- +// F1 — tolerant structured-output parsing + parse retries +// +// Ports the Python SDK's post-completion decode (agent_ai.py:1032-1088), which +// SEC-AF's Python node inherits for free from `.ai(schema=...)`. Two live +// run_verifier executions on the Go node failed where Python's did not, because +// kimi-k2.5 wrapped its json_schema reply in a ```json fence. +// --------------------------------------------------------------------------- + +// scriptedAI answers each successive AI call with the next body in Bodies (the +// last body repeats once the script runs out), so a test can express "garbage, +// garbage, then valid". +type scriptedAI struct { + Bodies []string + calls int +} + +func (s *scriptedAI) AI(_ context.Context, _ string, _ ...ai.Option) (*ai.Response, error) { + body := s.Bodies[len(s.Bodies)-1] + if s.calls < len(s.Bodies) { + body = s.Bodies[s.calls] + } + s.calls++ + return &ai.Response{Choices: []ai.Choice{{ + Message: ai.Message{Role: "assistant", Content: []ai.ContentPart{{Type: "text", Text: body}}}, + }}}, nil +} + +// (a) and (b): the salvage step recovers a fenced body and a prose-wrapped one, +// on the FIRST attempt — no retry is needed or spent. +func TestStructuredSalvagesNonStrictBodies(t *testing.T) { + want := CWEExpansion{AdditionalCWEs: []string{"CWE-918"}, Rationale: "ssrf"} + const payload = `{"additional_cwes":["CWE-918"],"rationale":"ssrf"}` + + for _, tc := range []struct { + name string + body string + }{ + {"markdown fence", "```json\n" + payload + "\n```"}, + {"fence without a language", "```\n" + payload + "\n```"}, + {"prose before and after", "Sure! Here is the JSON: " + payload + " Hope that helps."}, + {"leading reasoning line", "Let me think.\n\n" + payload}, + } { + t.Run(tc.name, func(t *testing.T) { + f := &scriptedAI{Bodies: []string{tc.body}} + got, err := Structured[CWEExpansion](context.Background(), f, "", "u") + if err != nil { + t.Fatalf("Structured: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("Structured = %#v, want %#v", got, want) + } + if f.calls != 1 { + t.Errorf("AI calls = %d, want 1 (the salvage step must not cost a retry)", f.calls) + } + }) + } +} + +// (c): garbage twice, then a valid body — Python re-issues the whole request on +// a parse failure, so the third attempt succeeds and exactly 3 calls are made. +func TestStructuredRetriesTheRequestOnParseFailure(t *testing.T) { + f := &scriptedAI{Bodies: []string{ + "I cannot help with that.", + "still not json", + `{"additional_cwes":["CWE-79"],"rationale":"xss"}`, + }} + got, err := Structured[CWEExpansion](context.Background(), f, "", "u") + if err != nil { + t.Fatalf("Structured: %v", err) + } + want := CWEExpansion{AdditionalCWEs: []string{"CWE-79"}, Rationale: "xss"} + if !reflect.DeepEqual(got, want) { + t.Errorf("Structured = %#v, want %#v", got, want) + } + if f.calls != 3 { + t.Errorf("AI calls = %d, want 3", f.calls) + } +} + +// (d): garbage on every attempt — 3 calls, then Python's final message. +func TestStructuredGivesUpAfterThreeAttempts(t *testing.T) { + f := &scriptedAI{Bodies: []string{"nope", "nope", "nope", "nope"}} + _, err := Structured[CWEExpansion](context.Background(), f, "", "u") + if err == nil { + t.Fatal("Structured should fail when no attempt parses") + } + const want = "aix.Structured[CWEExpansion]: Could not parse structured response: nope" + if err.Error() != want { + t.Errorf("Structured error = %q, want %q", err.Error(), want) + } + if f.calls != 3 { + t.Errorf("AI calls = %d, want exactly 3 (1 + max_parse_retries), not %d", f.calls, f.calls) + } +} + +// A body that the salvage step recovers but that is STILL not decodable burns a +// retry like any other parse failure. +func TestStructuredSalvagedButUndecodableStillRetries(t *testing.T) { + f := &scriptedAI{Bodies: []string{ + "here you go: {not: valid, json}", + `{"additional_cwes":[],"rationale":"ok"}`, + }} + if _, err := Structured[CWEExpansion](context.Background(), f, "", "u"); err != nil { + t.Fatalf("Structured: %v", err) + } + if f.calls != 2 { + t.Errorf("AI calls = %d, want 2", f.calls) + } +} + +// extractJSONObject must equal CPython's `re.search(r"\{.*\}", s, re.DOTALL)`. +// Every want below was produced by running that expression under +// ~/.agentfield/packages/sec-af/venv/bin/python. +func TestExtractJSONObjectMatchesTheGreedyPythonRegex(t *testing.T) { + for _, tc := range []struct { + in string + want string // "" means: no match + }{ + {"```json\n{\"a\": 1, \"b\": [1,2]}\n```", "{\"a\": 1, \"b\": [1,2]}"}, + {"Sure! Here is the JSON: {\"a\": 1} hope that helps.", "{\"a\": 1}"}, + {"no braces at all", ""}, + {"} weird {\"a\":1}", "{\"a\":1}"}, + {"{\"a\":1} then } more", "{\"a\":1} then }"}, + {"{\"a\":1", ""}, + {"", ""}, + {"{\"a\": {\"b\": 2}} trailing {oops", "{\"a\": {\"b\": 2}}"}, + {"prefix {\"a\":1} middle {\"b\":2} suffix", "{\"a\":1} middle {\"b\":2}"}, + } { + got, ok := extractJSONObject(tc.in) + if tc.want == "" { + if ok { + t.Errorf("extractJSONObject(%q) = %q, want no match", tc.in, got) + } + continue + } + if !ok || got != tc.want { + t.Errorf("extractJSONObject(%q) = %q/%v, want %q", tc.in, got, ok, tc.want) + } + } +} diff --git a/internal/secaf/aix/order_test.go b/internal/secaf/aix/order_test.go new file mode 100644 index 0000000000..1eb3a8d563 --- /dev/null +++ b/internal/secaf/aix/order_test.go @@ -0,0 +1,232 @@ +package aix + +// Tests for the ORDER of the schema document Structured sends as +// response_format.json_schema.schema. +// +// Validation contract, taken from the Python SDK's live path +// (sdk/python/agentfield/agent_ai.py:803 -> +// `_strictify_openai_schema(schema.model_json_schema())`), not from the Go code: +// +// - Python strictifies `Model.model_json_schema()`, whose `properties` is an +// insertion-ordered dict in pydantic field-DECLARATION order, and sets +// `node["required"] = list(props.keys())` — so both render in declaration +// order, and litellm serialises the dict as-is. VERIFIED on the pinned +// interpreter, e.g. VerdictDecision -> ["verdict", "evidence_level", +// "rationale", "confidence"] for `model_fields`, `properties` AND +// `required`, and ComplianceGate's `$defs.ComplianceSuggestion` -> +// ["framework", "control_id", "control_name"]. +// - Go reads the committed fixture, which go/scripts/gen_schemas.py wrote with +// sort_keys=True, into a map — so without StrictifyOrdered the marshalled +// bytes carry `properties` and `required` in SORTED order for every +// `.ai(schema=...)` call the node makes. +// +// The declaration order is asserted against internal/schemas/testdata/model_keys.json, +// which go/scripts/gen_model_keys.py generates from the live pydantic models +// (`keys` is `model_dump()` order, which is `model_fields` order, which is the +// `properties` order verified above). + +import ( + "bytes" + "encoding/json" + "os" + "reflect" + "testing" + + "github.com/Agent-Field/codeaf/internal/secaf/harnessx" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// pydanticKeys returns a model's field-declaration order from the generated +// pydantic fixture. +func pydanticKeys(t *testing.T, module, class string) []string { + t.Helper() + raw, err := os.ReadFile("../schemas/testdata/model_keys.json") + if err != nil { + t.Fatalf("read model_keys.json: %v", err) + } + var doc struct { + Models []struct { + PythonModule string `json:"python_module"` + PythonClass string `json:"python_class"` + Keys []string `json:"keys"` + } `json:"models"` + } + if err := json.Unmarshal(raw, &doc); err != nil { + t.Fatalf("decode model_keys.json: %v", err) + } + for _, m := range doc.Models { + if m.PythonModule == module && m.PythonClass == class { + return m.Keys + } + } + t.Fatalf("%s.%s not in model_keys.json", module, class) + return nil +} + +// objectKeys returns a JSON object's member names in the order they appear in +// the bytes — which is the whole point of the assertion, so decoding into a map +// (unordered) would defeat it. +func objectKeys(t *testing.T, raw json.RawMessage) []string { + t.Helper() + dec := json.NewDecoder(bytes.NewReader(raw)) + tok, err := dec.Token() + if err != nil { + t.Fatalf("read object: %v", err) + } + if delim, ok := tok.(json.Delim); !ok || delim != '{' { + t.Fatalf("not an object: %v", tok) + } + var keys []string + for dec.More() { + key, err := dec.Token() + if err != nil { + t.Fatalf("read key: %v", err) + } + name, ok := key.(string) + if !ok { + t.Fatalf("key is not a string: %v", key) + } + keys = append(keys, name) + var skip json.RawMessage + if err := dec.Decode(&skip); err != nil { + t.Fatalf("skip value of %q: %v", name, err) + } + } + return keys +} + +// strictifiedNode is one object of the emitted document, decoded far enough to +// look at its properties order and required array. +type strictifiedNode struct { + Properties json.RawMessage `json:"properties"` + Required []string `json:"required"` + Defs map[string]json.RawMessage +} + +func emit[T any](t *testing.T) json.RawMessage { + t.Helper() + raw, err := json.Marshal(StrictifyOrdered[T](harnessx.SchemaFor[T]())) + if err != nil { + t.Fatalf("marshal: %v", err) + } + return raw +} + +func decodeNode(t *testing.T, raw json.RawMessage) strictifiedNode { + t.Helper() + var node strictifiedNode + if err := json.Unmarshal(raw, &node); err != nil { + t.Fatalf("decode node: %v", err) + } + var withDefs struct { + Defs map[string]json.RawMessage `json:"$defs"` + } + if err := json.Unmarshal(raw, &withDefs); err != nil { + t.Fatalf("decode $defs: %v", err) + } + node.Defs = withDefs.Defs + return node +} + +func assertDeclarationOrder(t *testing.T, label string, raw json.RawMessage, want []string) { + t.Helper() + node := decodeNode(t, raw) + if got := objectKeys(t, node.Properties); !reflect.DeepEqual(got, want) { + t.Errorf("%s properties order\n got: %v\nwant: %v", label, got, want) + } + if !reflect.DeepEqual(node.Required, want) { + t.Errorf("%s required order\n got: %v\nwant: %v", label, node.Required, want) + } +} + +// TestStructuredSchemaCarriesPydanticDeclarationOrder covers every model +// SEC-AF passes to `.ai(schema=...)`. +func TestStructuredSchemaCarriesPydanticDeclarationOrder(t *testing.T) { + const gates = "sec_af.schemas.gates" + + t.Run("DuplicateCheck", func(t *testing.T) { + assertDeclarationOrder(t, "DuplicateCheck", emit[schemas.DuplicateCheck](t), + pydanticKeys(t, gates, "DuplicateCheck")) + }) + t.Run("StrategySelection", func(t *testing.T) { + assertDeclarationOrder(t, "StrategySelection", emit[schemas.StrategySelection](t), + pydanticKeys(t, gates, "StrategySelection")) + }) + t.Run("CWEExpansion", func(t *testing.T) { + assertDeclarationOrder(t, "CWEExpansion", emit[schemas.CWEExpansion](t), + pydanticKeys(t, gates, "CWEExpansion")) + }) + t.Run("ReachabilityGate", func(t *testing.T) { + assertDeclarationOrder(t, "ReachabilityGate", emit[schemas.ReachabilityGate](t), + pydanticKeys(t, gates, "ReachabilityGate")) + }) + t.Run("VerdictDecision", func(t *testing.T) { + assertDeclarationOrder(t, "VerdictDecision", emit[schemas.VerdictDecision](t), + pydanticKeys(t, "sec_af.schemas.prove", "VerdictDecision")) + }) + // ComplianceGate is the only gate schema with a $defs entry, so it is the + // one that proves the order reaches nested models too. + t.Run("ComplianceGate", func(t *testing.T) { + raw := emit[schemas.ComplianceGate](t) + assertDeclarationOrder(t, "ComplianceGate", raw, pydanticKeys(t, gates, "ComplianceGate")) + + node := decodeNode(t, raw) + sub, ok := node.Defs["ComplianceSuggestion"] + if !ok { + t.Fatalf("$defs = %v, want a ComplianceSuggestion entry", node.Defs) + } + assertDeclarationOrder(t, "$defs.ComplianceSuggestion", sub, + pydanticKeys(t, gates, "ComplianceSuggestion")) + }) +} + +// TestStrictifyOrderedKeepsTheStrictifyTransform: the ordering wrapper must not +// change WHAT the document says — same additionalProperties placement, same +// required SET, same untouched nodes — only the order it says it in. Comparing +// against Strictify (whose parity with the Python SDK the goldens already pin) +// keeps the two from drifting apart. +func TestStrictifyOrderedKeepsTheStrictifyTransform(t *testing.T) { + for _, name := range harnessx.FixtureNames() { + in, err := harnessx.LoadFixture(name) + if err != nil { + t.Errorf("%s: %v", name, err) + continue + } + // The Go type is irrelevant to the transform; only the order changes, + // and re-decoding erases it. + gotRaw, err := json.Marshal(StrictifyOrdered[struct{}](in)) + if err != nil { + t.Fatalf("%s: marshal ordered: %v", name, err) + } + wantRaw, err := json.Marshal(Strictify(in)) + if err != nil { + t.Fatalf("%s: marshal strictify: %v", name, err) + } + var got, want any + if err := json.Unmarshal(gotRaw, &got); err != nil { + t.Fatalf("%s: %v", name, err) + } + if err := json.Unmarshal(wantRaw, &want); err != nil { + t.Fatalf("%s: %v", name, err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("%s: StrictifyOrdered changed the document, not just the order\n got: %s\nwant: %s", + name, gotRaw, wantRaw) + } + } +} + +// TestStrictifyOrderedDoesNotMutateTheCachedFixture: Structured runs this over +// harnessx's SHARED cached map, so a mutation would be both a data race and a +// corrupted harness schema. +func TestStrictifyOrderedDoesNotMutateTheCachedFixture(t *testing.T) { + before, err := harnessx.LoadFixture("ComplianceGate") + if err != nil { + t.Fatalf("LoadFixture: %v", err) + } + _ = StrictifyOrdered[schemas.ComplianceGate](harnessx.SchemaFor[schemas.ComplianceGate]()) + after := harnessx.SchemaFor[schemas.ComplianceGate]() + if !reflect.DeepEqual(after, before) { + t.Error("StrictifyOrdered mutated the cached harnessx fixture") + } +} diff --git a/internal/secaf/aix/testdata/edgecases_input.json b/internal/secaf/aix/testdata/edgecases_input.json new file mode 100644 index 0000000000..5c155792c6 --- /dev/null +++ b/internal/secaf/aix/testdata/edgecases_input.json @@ -0,0 +1,84 @@ +{ + "$defs": { + "AlreadyStrict": { + "additionalProperties": true, + "properties": { + "kept": { + "type": "string" + } + }, + "required": [ + "stale" + ], + "type": "object" + }, + "Leaf": { + "properties": { + "v": { + "type": "number" + } + }, + "type": "object" + }, + "ListTypeNode": { + "properties": { + "z": { + "type": "string" + } + }, + "type": [ + "object", + "null" + ] + }, + "NoTypeNode": { + "properties": { + "a": { + "type": "integer" + }, + "b": { + "type": "string" + } + } + } + }, + "properties": { + "decoy": { + "properties": "not-a-dict", + "type": "object" + }, + "list_of_objects": { + "items": { + "properties": { + "inner": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "nested": { + "$ref": "#/$defs/Leaf" + }, + "nullable_object": { + "anyOf": [ + { + "properties": { + "x": { + "type": "string" + } + }, + "type": "object" + }, + { + "type": "null" + } + ] + }, + "plain": { + "type": "string" + } + }, + "type": "object" +} diff --git a/internal/secaf/aix/testdata/edgecases_strict.json b/internal/secaf/aix/testdata/edgecases_strict.json new file mode 100644 index 0000000000..c5757907d0 --- /dev/null +++ b/internal/secaf/aix/testdata/edgecases_strict.json @@ -0,0 +1,109 @@ +{ + "$defs": { + "AlreadyStrict": { + "additionalProperties": false, + "properties": { + "kept": { + "type": "string" + } + }, + "required": [ + "kept" + ], + "type": "object" + }, + "Leaf": { + "additionalProperties": false, + "properties": { + "v": { + "type": "number" + } + }, + "required": [ + "v" + ], + "type": "object" + }, + "ListTypeNode": { + "properties": { + "z": { + "type": "string" + } + }, + "type": [ + "object", + "null" + ] + }, + "NoTypeNode": { + "additionalProperties": false, + "properties": { + "a": { + "type": "integer" + }, + "b": { + "type": "string" + } + }, + "required": [ + "a", + "b" + ] + } + }, + "additionalProperties": false, + "properties": { + "decoy": { + "properties": "not-a-dict", + "type": "object" + }, + "list_of_objects": { + "items": { + "additionalProperties": false, + "properties": { + "inner": { + "type": "string" + } + }, + "required": [ + "inner" + ], + "type": "object" + }, + "type": "array" + }, + "nested": { + "$ref": "#/$defs/Leaf" + }, + "nullable_object": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "x": { + "type": "string" + } + }, + "required": [ + "x" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "plain": { + "type": "string" + } + }, + "required": [ + "decoy", + "list_of_objects", + "nested", + "nullable_object", + "plain" + ], + "type": "object" +} diff --git a/internal/secaf/aix/testdata/gen_strictify_golden.py b/internal/secaf/aix/testdata/gen_strictify_golden.py new file mode 100644 index 0000000000..9466702bca --- /dev/null +++ b/internal/secaf/aix/testdata/gen_strictify_golden.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Golden generator for aix.Strictify. + +Runs the REAL Python SDK function that aix.Strictify ports — +``agentfield.agent_ai._strictify_openai_schema`` — over + + 1. every committed pydantic schema fixture under + ``go/internal/harnessx/testdata/schemas/`` (the exact documents + ``aix.Structured`` strictifies at runtime), writing the results as one + ``{"": }`` object, and + 2. a hand-written edge-case document that exercises the branches the real + fixtures do not: a properties-bearing node with NO "type" key (must be + strictified), a node with a LIST-valued "type" (must NOT be), an "anyOf" + branch, a nested "items", and a pre-existing wrong "required"/ + "additionalProperties" pair that the walk must overwrite. + +Run from the repo root: + + ~/.agentfield/packages/sec-af/venv/bin/python \ + go/internal/aix/testdata/gen_strictify_golden.py + +Deterministic: rerunning writes identical bytes unless the SDK function or a +schema fixture changed. +""" + +from __future__ import annotations + +import json +import os + +from agentfield.agent_ai import _strictify_openai_schema + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_GO = os.path.dirname(os.path.dirname(os.path.dirname(_HERE))) +_FIXTURES = os.path.join(_GO, "internal", "harnessx", "testdata", "schemas") + +# Exercises every branch of the walk. Key order is irrelevant to the assertion +# (the Go test compares decoded documents), but the required lists it produces +# are order-sensitive, so this file is written sorted to match Go's sorted +# `required` — see aix.Strictify's doc comment. +EDGE_CASES = { + "$defs": { + "NoTypeNode": { + # No "type" key at all -> `"type" not in node` -> strictified. + "properties": { + "b": {"type": "string"}, + "a": {"type": "integer"}, + }, + }, + "ListTypeNode": { + # type is a LIST -> neither `== "object"` nor `not in` -> untouched. + "type": ["object", "null"], + "properties": {"z": {"type": "string"}}, + }, + "AlreadyStrict": { + "type": "object", + "additionalProperties": True, + "required": ["stale"], + "properties": {"kept": {"type": "string"}}, + }, + "Leaf": { + "type": "object", + "properties": {"v": {"type": "number"}}, + }, + }, + "type": "object", + "properties": { + "nested": {"$ref": "#/$defs/Leaf"}, + "list_of_objects": { + "type": "array", + "items": { + "type": "object", + "properties": {"inner": {"type": "string"}}, + }, + }, + "nullable_object": { + "anyOf": [ + {"type": "object", "properties": {"x": {"type": "string"}}}, + {"type": "null"}, + ] + }, + "plain": {"type": "string"}, + # A non-dict "properties" value must not trip the isinstance check. + "decoy": {"type": "object", "properties": "not-a-dict"}, + }, +} + + +def main() -> None: + strictified: dict[str, object] = {} + for entry in sorted(os.listdir(_FIXTURES)): + if not entry.endswith(".json"): + continue + with open(os.path.join(_FIXTURES, entry), encoding="utf-8") as f: + schema = json.load(f) + strictified[entry[: -len(".json")]] = _strictify_openai_schema(schema) + + out = os.path.join(_HERE, "strictified_fixtures.json") + with open(out, "w", encoding="utf-8") as f: + f.write(json.dumps(strictified, indent=2, sort_keys=True) + "\n") + print(f"wrote {os.path.basename(out)} ({len(strictified)} schemas)") + + # Round-trip through sorted JSON FIRST so the dict Python walks has the same + # key order the Go test will read off disk. `list(props.keys())` is + # insertion-ordered, so without this the golden `required` lists would carry + # the literal's declaration order while Go produced sorted ones. + edge_input = json.loads(json.dumps(EDGE_CASES, sort_keys=True)) + with open(os.path.join(_HERE, "edgecases_input.json"), "w", encoding="utf-8") as f: + f.write(json.dumps(edge_input, indent=2, sort_keys=True) + "\n") + with open(os.path.join(_HERE, "edgecases_strict.json"), "w", encoding="utf-8") as f: + f.write(json.dumps(_strictify_openai_schema(edge_input), indent=2, sort_keys=True) + "\n") + print("wrote edgecases_input.json / edgecases_strict.json") + + +if __name__ == "__main__": + main() diff --git a/internal/secaf/aix/testdata/strictified_fixtures.json b/internal/secaf/aix/testdata/strictified_fixtures.json new file mode 100644 index 0000000000..2662b38487 --- /dev/null +++ b/internal/secaf/aix/testdata/strictified_fixtures.json @@ -0,0 +1,850 @@ +{ + "ArchitectureMapRaw": { + "additionalProperties": false, + "description": "Flat harness output for architecture mapper. All list[str], no nesting.", + "properties": { + "api_endpoints": { + "description": "One string per API endpoint. Format: 'method | path | handler | file_path:line | auth_required | rate_limited'. Example: 'GET | /api/users | get_users | src/api.py:15 | true | false'", + "items": { + "type": "string" + }, + "title": "Api Endpoints", + "type": "array" + }, + "app_type": { + "default": "unknown", + "description": "Application type: web_api, cli_tool, library, microservice, monolith", + "title": "App Type", + "type": "string" + }, + "entry_points": { + "description": "One string per entry point. Format: 'kind | route_or_id | file_path:line | auth_required'. Example: 'http | POST /api/login | src/routes.py:42 | false'", + "items": { + "type": "string" + }, + "title": "Entry Points", + "type": "array" + }, + "modules": { + "description": "One string per module. Format: 'name | path | language | description'. Example: 'auth | src/auth/ | python | Authentication and session management'", + "items": { + "type": "string" + }, + "title": "Modules", + "type": "array" + }, + "services": { + "description": "One string per external service. Format: 'name | type | endpoint | auth_mechanism'. Example: 'PostgreSQL | database | localhost:5432 | password'", + "items": { + "type": "string" + }, + "title": "Services", + "type": "array" + }, + "trust_boundaries": { + "description": "One string per boundary. Format: 'name | source_zone | target_zone | description'. Example: 'API Gateway | external | internal | Rate limiting and auth'", + "items": { + "type": "string" + }, + "title": "Trust Boundaries", + "type": "array" + } + }, + "required": [ + "api_endpoints", + "app_type", + "entry_points", + "modules", + "services", + "trust_boundaries" + ], + "title": "ArchitectureMapRaw", + "type": "object" + }, + "CWEExpansion": { + "additionalProperties": false, + "description": "AI-suggested CWE additions based on recon context.", + "properties": { + "additional_cwes": { + "description": "CWE IDs to add beyond baseline, e.g. ['CWE-918', 'CWE-611'].", + "items": { + "type": "string" + }, + "title": "Additional Cwes", + "type": "array" + }, + "rationale": { + "title": "Rationale", + "type": "string" + } + }, + "required": [ + "additional_cwes", + "rationale" + ], + "title": "CWEExpansion", + "type": "object" + }, + "ChainCorrelationResult": { + "additionalProperties": false, + "description": "Flat harness schema for chain correlation. LLM identifies chains only.", + "properties": { + "chains": { + "description": "Multi-step attack chains found. Format per entry: 'title | finding_id1,finding_id2,... | combined_impact | severity'. Example: 'SSRF to Internal API | f1,f2 | Access internal services | high'", + "items": { + "type": "string" + }, + "title": "Chains", + "type": "array" + }, + "duplicate_ids": { + "description": "Finding IDs that are duplicates missed by programmatic dedup (to drop)", + "items": { + "type": "string" + }, + "title": "Duplicate Ids", + "type": "array" + } + }, + "required": [ + "chains", + "duplicate_ids" + ], + "title": "ChainCorrelationResult", + "type": "object" + }, + "ComplianceGate": { + "$defs": { + "ComplianceSuggestion": { + "additionalProperties": false, + "properties": { + "control_id": { + "title": "Control Id", + "type": "string" + }, + "control_name": { + "title": "Control Name", + "type": "string" + }, + "framework": { + "title": "Framework", + "type": "string" + } + }, + "required": [ + "control_id", + "control_name", + "framework" + ], + "title": "ComplianceSuggestion", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "confidence": { + "title": "Confidence", + "type": "string" + }, + "mappings": { + "items": { + "$ref": "#/$defs/ComplianceSuggestion" + }, + "title": "Mappings", + "type": "array" + } + }, + "required": [ + "confidence", + "mappings" + ], + "title": "ComplianceGate", + "type": "object" + }, + "ConfigReportRaw": { + "additionalProperties": false, + "description": "Flat harness output for config scanner. All list[str], no nesting.", + "properties": { + "misconfigs": { + "description": "One string per misconfiguration. Format: 'category | file_path:line | key | risk | remediation'. Example: 'debug_mode | config.py:15 | DEBUG=True | Exposes stack traces | Set DEBUG=False'", + "items": { + "type": "string" + }, + "title": "Misconfigs", + "type": "array" + }, + "secrets": { + "description": "One string per secret finding. Format: 'type | file_path:line | match_preview | confidence | is_test(true/false)'. Example: 'aws_access_key | .env:3 | AKIA... | high | false'", + "items": { + "type": "string" + }, + "title": "Secrets", + "type": "array" + } + }, + "required": [ + "misconfigs", + "secrets" + ], + "title": "ConfigReportRaw", + "type": "object" + }, + "CrossServiceFinding": { + "additionalProperties": false, + "description": "Flat schema for cross-service attack chain analysis. 4 fields.", + "properties": { + "chain_description": { + "description": "Description of the cross-service attack path", + "title": "Chain Description", + "type": "string" + }, + "entry_point": { + "description": "Public-facing entry point where attack begins", + "title": "Entry Point", + "type": "string" + }, + "impact": { + "description": "Impact if the cross-service chain is exploited", + "title": "Impact", + "type": "string" + }, + "services_involved": { + "description": "Service names in the attack chain", + "items": { + "type": "string" + }, + "title": "Services Involved", + "type": "array" + } + }, + "required": [ + "chain_description", + "entry_point", + "impact", + "services_involved" + ], + "title": "CrossServiceFinding", + "type": "object" + }, + "DastVerificationResult": { + "additionalProperties": false, + "description": "Flat schema for DAST-like runtime verification. 4 fields.", + "properties": { + "exploit_confirmed": { + "description": "Whether the exploit was confirmed at runtime", + "title": "Exploit Confirmed", + "type": "boolean" + }, + "payload_sent": { + "description": "The exploit payload or request that was sent", + "title": "Payload Sent", + "type": "string" + }, + "response_summary": { + "description": "Summary of the application response", + "title": "Response Summary", + "type": "string" + }, + "safety_notes": { + "description": "Safety measures taken during verification (sandbox, timeout, etc.)", + "title": "Safety Notes", + "type": "string" + } + }, + "required": [ + "exploit_confirmed", + "payload_sent", + "response_summary", + "safety_notes" + ], + "title": "DastVerificationResult", + "type": "object" + }, + "DataFlowMapRaw": { + "additionalProperties": false, + "description": "Flat harness output for data flow mapper. All list[str], no nesting.", + "properties": { + "flows": { + "description": "One string per data flow. Format: 'source | sink | sanitized(true/false) | file1, file2, ...'. Example: 'request.body | sql.execute | false | src/db.py, src/routes.py'", + "items": { + "type": "string" + }, + "title": "Flows", + "type": "array" + }, + "sanitization_points": { + "description": "One string per sanitization point. Format: 'file_path:line | function_name | type | protects_against'. Example: 'src/utils.py:42 | sanitize_html | html_encoding | CWE-79'", + "items": { + "type": "string" + }, + "title": "Sanitization Points", + "type": "array" + }, + "sinks": { + "description": "One string per security-critical sink. Format: 'sink_type | file_path:line | function_name | notes'. Example: 'sql_execute | src/db.py:55 | run_query | Direct string concatenation'", + "items": { + "type": "string" + }, + "title": "Sinks", + "type": "array" + } + }, + "required": [ + "flows", + "sanitization_points", + "sinks" + ], + "title": "DataFlowMapRaw", + "type": "object" + }, + "DataFlowTrace": { + "additionalProperties": false, + "description": "Flat schema for data flow tracing sub-agent. 4 fields.", + "properties": { + "sink": { + "description": "Security-sensitive operation reached (e.g. 'sql.execute(query)')", + "title": "Sink", + "type": "string" + }, + "sink_reached": { + "description": "Whether tainted data actually reaches the sink", + "title": "Sink Reached", + "type": "boolean" + }, + "source": { + "description": "Where tainted input enters (e.g. 'request.params.id')", + "title": "Source", + "type": "string" + }, + "steps": { + "description": "Ordered list of file:line descriptions showing flow path", + "items": { + "type": "string" + }, + "title": "Steps", + "type": "array" + } + }, + "required": [ + "sink", + "sink_reached", + "source", + "steps" + ], + "title": "DataFlowTrace", + "type": "object" + }, + "DependencyReportRaw": { + "additionalProperties": false, + "description": "Flat harness output for dependency auditor. All list[str], no nesting.", + "properties": { + "known_cves": { + "description": "One string per CVE. Format: 'cve_id | package | installed_version | fixed_version | cvss_score | direct | reachable'. Example: 'CVE-2023-1234 | lodash | 4.17.15 | 4.17.21 | 7.5 | true | unknown'", + "items": { + "type": "string" + }, + "title": "Known Cves", + "type": "array" + }, + "outdated": { + "description": "One string per outdated dep. Format: 'package | current_version | latest_version | direct(true/false)'. Example: 'express | 4.17.0 | 4.18.2 | true'", + "items": { + "type": "string" + }, + "title": "Outdated", + "type": "array" + }, + "sbom": { + "description": "One string per dependency. Format: 'name | version | ecosystem | direct(true/false) | license'. Example: 'express | 4.18.2 | npm | true | MIT'", + "items": { + "type": "string" + }, + "title": "Sbom", + "type": "array" + } + }, + "required": [ + "known_cves", + "outdated", + "sbom" + ], + "title": "DependencyReportRaw", + "type": "object" + }, + "DuplicateCheck": { + "additionalProperties": false, + "description": "DESIGN.md \u00a75.5: quick duplicate check gate for dedup decisions.", + "properties": { + "duplicate_of": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duplicate Of" + }, + "is_duplicate": { + "title": "Is Duplicate", + "type": "boolean" + }, + "reason": { + "title": "Reason", + "type": "string" + } + }, + "required": [ + "duplicate_of", + "is_duplicate", + "reason" + ], + "title": "DuplicateCheck", + "type": "object" + }, + "EnrichedFinding": { + "additionalProperties": false, + "description": "Flat schema for Step 2: finding enrichment. 6 fields.", + "properties": { + "confidence": { + "description": "One of: \"high\", \"medium\", \"low\"", + "title": "Confidence", + "type": "string" + }, + "cwe_id": { + "description": "CWE identifier (e.g. 'CWE-89')", + "title": "Cwe Id", + "type": "string" + }, + "data_flow_summary": { + "description": "Natural language summary of the data flow (string, not nested)", + "title": "Data Flow Summary", + "type": "string" + }, + "description": { + "description": "Detailed description of the vulnerability", + "title": "Description", + "type": "string" + }, + "severity": { + "description": "One of: \"critical\", \"high\", \"medium\", \"low\", \"info\"", + "title": "Severity", + "type": "string" + }, + "title": { + "description": "Human-readable title for the finding", + "title": "Title", + "type": "string" + } + }, + "required": [ + "confidence", + "cwe_id", + "data_flow_summary", + "description", + "severity", + "title" + ], + "title": "EnrichedFinding", + "type": "object" + }, + "ExploitHypothesis": { + "additionalProperties": false, + "description": "Flat schema for exploit construction sub-agent. 3 fields.", + "properties": { + "expected_outcome": { + "description": "What would happen if exploit succeeds", + "title": "Expected Outcome", + "type": "string" + }, + "hypothesis": { + "description": "Natural language description of exploit scenario", + "title": "Hypothesis", + "type": "string" + }, + "payload": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Concrete exploit payload or input", + "title": "Payload" + } + }, + "required": [ + "expected_outcome", + "hypothesis", + "payload" + ], + "title": "ExploitHypothesis", + "type": "object" + }, + "PolicyEvalResult": { + "additionalProperties": false, + "description": "Flat schema for AI policy evaluation. 4 fields.", + "properties": { + "description": { + "description": "How the policy is violated, or 'No violation' if compliant", + "title": "Description", + "type": "string" + }, + "file_path": { + "description": "Primary file where violation occurs, or 'N/A'", + "title": "File Path", + "type": "string" + }, + "severity": { + "description": "Severity: \"high\", \"medium\", or \"low\"", + "title": "Severity", + "type": "string" + }, + "violated": { + "description": "Whether the policy is violated", + "title": "Violated", + "type": "boolean" + } + }, + "required": [ + "description", + "file_path", + "severity", + "violated" + ], + "title": "PolicyEvalResult", + "type": "object" + }, + "ReachabilityGate": { + "additionalProperties": false, + "description": "Reachability assessment for findings without explicit reachability tags.", + "properties": { + "confidence": { + "description": "One of: \"high\", \"medium\", \"low\".", + "title": "Confidence", + "type": "string" + }, + "rationale": { + "title": "Rationale", + "type": "string" + }, + "reachability": { + "description": "One of: \"externally_reachable\", \"requires_auth\", \"internal_only\", \"unreachable\".", + "title": "Reachability", + "type": "string" + } + }, + "required": [ + "confidence", + "rationale", + "reachability" + ], + "title": "ReachabilityGate", + "type": "object" + }, + "ReachabilityProof": { + "additionalProperties": false, + "description": "Flat schema for dependency reachability analysis. 4 fields.", + "properties": { + "call_chain": { + "description": "Import/call chain from app code to vulnerable function", + "items": { + "type": "string" + }, + "title": "Call Chain", + "type": "array" + }, + "direct": { + "description": "Whether the dependency is direct or transitive", + "title": "Direct", + "type": "boolean" + }, + "reachable": { + "description": "Whether the vulnerable function is actually called", + "title": "Reachable", + "type": "boolean" + }, + "vulnerable_function": { + "description": "The vulnerable function/method in the dependency", + "title": "Vulnerable Function", + "type": "string" + } + }, + "required": [ + "call_chain", + "direct", + "reachable", + "vulnerable_function" + ], + "title": "ReachabilityProof", + "type": "object" + }, + "RemediationSuggestion": { + "additionalProperties": false, + "description": "Flat schema for AI-generated remediation suggestion. 3 fields.", + "properties": { + "confidence": { + "description": "Confidence in the fix: \"high\", \"medium\", or \"low\"", + "title": "Confidence", + "type": "string" + }, + "fix_description": { + "description": "Natural language description of the recommended fix", + "title": "Fix Description", + "type": "string" + }, + "patch_diff": { + "description": "Unified diff format patch showing the code changes needed", + "title": "Patch Diff", + "type": "string" + } + }, + "required": [ + "confidence", + "fix_description", + "patch_diff" + ], + "title": "RemediationSuggestion", + "type": "object" + }, + "SanitizationResult": { + "additionalProperties": false, + "description": "Flat schema for sanitization analysis sub-agent. 4 fields.", + "properties": { + "bypass_method": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "How sanitization could be bypassed, if applicable", + "title": "Bypass Method" + }, + "found": { + "description": "Whether any sanitization/validation was found on the path", + "title": "Found", + "type": "boolean" + }, + "sufficient": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Whether sanitization is sufficient to prevent exploit", + "title": "Sufficient" + }, + "type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Type of sanitization (e.g. 'parameterized query', 'html encoding')", + "title": "Type" + } + }, + "required": [ + "bypass_method", + "found", + "sufficient", + "type" + ], + "title": "SanitizationResult", + "type": "object" + }, + "ScanLocationsResult": { + "$defs": { + "VulnLocation": { + "additionalProperties": false, + "description": "Flat schema for Step 1: location scanning. 4 fields.", + "properties": { + "code_snippet": { + "description": "Relevant code snippet around the vulnerability", + "title": "Code Snippet", + "type": "string" + }, + "file_path": { + "description": "Path to the file containing the potential vulnerability", + "title": "File Path", + "type": "string" + }, + "pattern_type": { + "description": "Type of vulnerability pattern detected (e.g. 'sql_injection', 'command_injection')", + "title": "Pattern Type", + "type": "string" + }, + "start_line": { + "description": "Starting line number of the vulnerable code", + "title": "Start Line", + "type": "integer" + } + }, + "required": [ + "code_snippet", + "file_path", + "pattern_type", + "start_line" + ], + "title": "VulnLocation", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Container for Step 1 results.", + "properties": { + "locations": { + "items": { + "$ref": "#/$defs/VulnLocation" + }, + "title": "Locations", + "type": "array" + } + }, + "required": [ + "locations" + ], + "title": "ScanLocationsResult", + "type": "object" + }, + "SecurityContextRaw": { + "additionalProperties": false, + "description": "Flat harness output for security context profiler. All flat, no nesting.", + "properties": { + "auth_details": { + "default": "", + "description": "Brief description of auth implementation details", + "title": "Auth Details", + "type": "string" + }, + "auth_model": { + "description": "Authentication model: jwt, session_cookie, oauth2, api_key, none, or other", + "title": "Auth Model", + "type": "string" + }, + "crypto_usage": { + "description": "One string per crypto usage. Format: 'algorithm | key_size | mode | usage_context | is_weak(true/false)'. Example: 'AES | 256 | GCM | data encryption | false'", + "items": { + "type": "string" + }, + "title": "Crypto Usage", + "type": "array" + }, + "security_signals": { + "description": "Framework security features, security headers, and deployment signals. One signal per entry. Examples: 'CSRF protection enabled', 'HSTS header present', 'Runs in Docker'", + "items": { + "type": "string" + }, + "title": "Security Signals", + "type": "array" + } + }, + "required": [ + "auth_details", + "auth_model", + "crypto_usage", + "security_signals" + ], + "title": "SecurityContextRaw", + "type": "object" + }, + "SeverityClassification": { + "additionalProperties": false, + "description": "DESIGN.md \u00a72.4: quick severity classification gate used in scoring.", + "properties": { + "confidence": { + "title": "Confidence", + "type": "number" + }, + "rationale": { + "title": "Rationale", + "type": "string" + }, + "severity": { + "description": "One of: \"critical\", \"high\", \"medium\", \"low\".", + "title": "Severity", + "type": "string" + } + }, + "required": [ + "confidence", + "rationale", + "severity" + ], + "title": "SeverityClassification", + "type": "object" + }, + "StrategySelection": { + "additionalProperties": false, + "description": "DESIGN.md \u00a75.3: strategy selection gate for HUNT routing.", + "properties": { + "rationale": { + "title": "Rationale", + "type": "string" + }, + "strategies": { + "items": { + "type": "string" + }, + "title": "Strategies", + "type": "array" + } + }, + "required": [ + "rationale", + "strategies" + ], + "title": "StrategySelection", + "type": "object" + }, + "VerdictDecision": { + "additionalProperties": false, + "description": "Flat schema for verdict sub-agent. Uses .ai() not .harness(). 4 fields.", + "properties": { + "confidence": { + "description": "One of: \"high\", \"medium\", \"low\"", + "title": "Confidence", + "type": "string" + }, + "evidence_level": { + "description": "1-6 scale: 1=STATIC_MATCH to 6=FULL_EXPLOIT", + "title": "Evidence Level", + "type": "integer" + }, + "rationale": { + "title": "Rationale", + "type": "string" + }, + "verdict": { + "description": "One of: \"confirmed\", \"likely\", \"inconclusive\", \"not_exploitable\"", + "title": "Verdict", + "type": "string" + } + }, + "required": [ + "confidence", + "evidence_level", + "rationale", + "verdict" + ], + "title": "VerdictDecision", + "type": "object" + } +} diff --git a/internal/secaf/audit/audit_test.go b/internal/secaf/audit/audit_test.go new file mode 100644 index 0000000000..087a0fbc5d --- /dev/null +++ b/internal/secaf/audit/audit_test.go @@ -0,0 +1,1014 @@ +package audit + +// Tests for the audit entry — the body of the sec-af node's `audit` reasoner. +// +// Validation contract (behaviour, derived from src/sec_af/app.py:123): +// +// - the 20 parameters bind with their exact defaults, and the four +// `x or [default]` list fallbacks fire for an EXPLICIT EMPTY LIST as well as +// for an absent key; +// - `enable_dast` never reaches AuditInput (pydantic drops the unknown key), +// so dast_enabled stays False; +// - the pipeline makes exactly four `.call`s, in order, with exactly the +// documented kwargs: recon_phase{repo_path,depth}, +// hunt_phase{repo_path,recon_context,depth}, +// prove_phase{repo_path,hunt_result,depth,max_provers}, +// remediation_phase{repo_path,verified_findings}; +// - hunt_phase receives the RAW recon payload (not the model dump), while +// prove_phase receives hunt.model_dump(); +// - the three checkpoints are written under /.sec-af; +// - agent_invocations = total_selected + len(strategies_run) + 3, and +// findings_not_verified / prove_drop_summary are copied from prove_phase; +// - a non-blank resume_from_checkpoint skips the pipeline entirely; +// - error mapping: every ValueError raised INSIDE audit()'s try -> 400 with +// the raw message and NO note. That is the unknown checkpoint phase AND +// every `model_validate` failure (pydantic's ValidationError subclasses +// ValueError) AND a corrupt checkpoint file (json.JSONDecodeError likewise). +// Any other pipeline failure -> note("Audit pipeline failed: ...", +// ["audit","error"]) then 500 with the "audit execution failed: " prefix +// (Python's stdout `AUDIT ERROR:` print is not reproduced inside codeaf); +// - a bad depth in AuditConfig.from_input (raised OUTSIDE Python's try) +// surfaces as a 500 with no note and no prefix, and a repository that does +// not resolve to a local directory is refused as a 400 with no note. + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/orch" + "github.com/Agent-Field/codeaf/internal/secaf/phases" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// --------------------------------------------------------------------------- +// input binding +// --------------------------------------------------------------------------- + +func TestAuditRequestDefaults(t *testing.T) { + var req AuditRequest + if err := json.Unmarshal([]byte(`{"repo_url":"https://example.test/o/r"}`), &req); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + if req.Depth != "standard" { + t.Errorf("depth = %q, want standard", req.Depth) + } + if req.Branch != "main" { + t.Errorf("branch = %q, want main", req.Branch) + } + if req.SeverityThreshold != "low" { + t.Errorf("severity_threshold = %q, want low", req.SeverityThreshold) + } + if req.IsPr || req.PostPrComments || req.FailOnFindings || req.EnableDast { + t.Error("the four booleans must default to false") + } + if req.CommitSha != nil || req.MaxProvers != nil || req.ResumeFromCheckpoint != nil { + t.Error("the optional parameters must default to nil (Python None)") + } +} + +func TestToAuditInputListFallbacks(t *testing.T) { + t.Run("absent lists take the audit() defaults", func(t *testing.T) { + in := NewAuditRequest().ToAuditInput() + assertStrings(t, "scan_types", in.ScanTypes, []string{"sast", "sca", "secrets", "config"}) + assertStrings(t, "output_formats", in.OutputFormats, []string{"json"}) + assertStrings(t, "compliance_frameworks", in.ComplianceFrameworks, []string{}) + assertStrings(t, "exclude_paths", in.ExcludePaths, []string{"tests/", "vendor/", "node_modules/", ".git/"}) + if in.IncludePaths != nil { + t.Errorf("include_paths = %v, want nil (no fallback in Python)", in.IncludePaths) + } + }) + + t.Run("an EXPLICIT empty list also falls back (Python truthiness)", func(t *testing.T) { + req := NewAuditRequest() + req.ScanTypes = []string{} + req.ExcludePaths = []string{} + in := req.ToAuditInput() + assertStrings(t, "scan_types", in.ScanTypes, []string{"sast", "sca", "secrets", "config"}) + assertStrings(t, "exclude_paths", in.ExcludePaths, []string{"tests/", "vendor/", "node_modules/", ".git/"}) + }) + + t.Run("a non-empty list is passed through", func(t *testing.T) { + req := NewAuditRequest() + req.ScanTypes = []string{"sast"} + req.IncludePaths = []string{"src/"} + in := req.ToAuditInput() + assertStrings(t, "scan_types", in.ScanTypes, []string{"sast"}) + assertStrings(t, "include_paths", in.IncludePaths, []string{"src/"}) + }) +} + +// TestToAuditInputDropsEnableDast pins the Python bug: audit() passes +// enable_dast= to a model whose field is dast_enabled, and pydantic's default +// extra="ignore" throws it away. +func TestToAuditInputDropsEnableDast(t *testing.T) { + req := NewAuditRequest() + req.EnableDast = true + if in := req.ToAuditInput(); in.DastEnabled { + t.Error("dast_enabled = true, want false: audit()'s enable_dast never reaches AuditInput") + } +} + +func TestToAuditInputKeepsUnpassedPydanticDefaults(t *testing.T) { + in := NewAuditRequest().ToAuditInput() + if in.RepoUrls == nil { + t.Error("repo_urls = nil, want [] (default_factory=list)") + } + if in.CustomPolicies == nil { + t.Error("custom_policies = nil, want [] (default_factory=list)") + } + if in.MonitoringMode { + t.Error("monitoring_mode must default to false") + } + if in.BaselinePath != nil { + t.Error("baseline_path must default to nil") + } +} + +// --------------------------------------------------------------------------- +// the pipeline +// --------------------------------------------------------------------------- + +// newAuditNode builds a runner with the audit seams stubbed: a recording +// appx.Fake for the `.call`s and notes, a real orchestrator rooted at repo, and +// a resolveRepo that returns repo. The name is kept from the node's tests. +func newAuditNode(t *testing.T, repo string, fake *appx.Fake) *runner { + t.Helper() + return &runner{ + app: fake, + newOrchestrator: orch.NewAt, + resolveRepo: func(string) (string, error) { + return repo, nil + }, + } +} + +// phasePayloads answers the four phase calls with the minimum schema-valid +// payloads. reconExtra/proveExtra let a test add keys. +func phasePayloads(strategies []string, totalSelected, notVerified int) func(context.Context, string, map[string]any) (map[string]any, error) { + return func(_ context.Context, target string, _ map[string]any) (map[string]any, error) { + switch { + case strings.HasSuffix(target, ".recon_phase"): + return map[string]any{ + "architecture": map[string]any{}, + "data_flows": map[string]any{}, + "dependencies": map[string]any{}, + "config": map[string]any{}, + "security_context": map[string]any{"auth_model": "jwt", "auth_details": "bearer"}, + "languages": []any{"python"}, + "lines_of_code": float64(120), + }, nil + case strings.HasSuffix(target, ".hunt_phase"): + run := make([]any, 0, len(strategies)) + for _, s := range strategies { + run = append(run, s) + } + return map[string]any{"strategies_run": run, "total_raw": float64(0)}, nil + case strings.HasSuffix(target, ".prove_phase"): + return map[string]any{ + "verified": []any{}, + "total_selected": float64(totalSelected), + "total_findings": float64(0), + "not_verified": float64(notVerified), + "drop_summary": map[string]any{ + "demoted_total": float64(2), + "by_reason": map[string]any{"verifier_error": float64(2)}, + "findings": []any{}, + }, + }, nil + case strings.HasSuffix(target, ".remediation_phase"): + return map[string]any{"verified": []any{}}, nil + } + return nil, errors.New("unexpected call target " + target) + } +} + +func TestAuditPipelineCallSequenceAndKwargs(t *testing.T) { + repo := t.TempDir() + fake := &appx.Fake{CallFn: phasePayloads([]string{"injection", "auth"}, 7, 4)} + n := newAuditNode(t, repo, fake) + + got, err := n.handle(context.Background(), map[string]any{ + "repo_url": repo, + "depth": "thorough", + "max_provers": float64(9), + }) + if err != nil { + t.Fatalf("handle: %v", err) + } + + // --- exact targets, in order --- + wantTargets := []string{ + "sec-af.recon_phase", + "sec-af.hunt_phase", + "sec-af.prove_phase", + "sec-af.remediation_phase", + } + if !reflect.DeepEqual(fake.CallTargets(), wantTargets) { + t.Fatalf("call targets = %v, want %v", fake.CallTargets(), wantTargets) + } + + // --- exact kwargs per call --- + assertKeys(t, "recon_phase", fake.Calls[0].Input, "repo_path", "depth") + assertKeys(t, "hunt_phase", fake.Calls[1].Input, "repo_path", "recon_context", "depth") + assertKeys(t, "prove_phase", fake.Calls[2].Input, "repo_path", "hunt_result", "depth", "max_provers") + assertKeys(t, "remediation_phase", fake.Calls[3].Input, "repo_path", "verified_findings") + + for i, call := range fake.Calls { + if call.Input["repo_path"] != repo { + t.Errorf("call %d repo_path = %v, want %q", i, call.Input["repo_path"], repo) + } + } + for _, i := range []int{0, 1, 2} { + if fake.Calls[i].Input["depth"] != "thorough" { + t.Errorf("call %d depth = %v, want thorough", i, fake.Calls[i].Input["depth"]) + } + } + if mp, ok := fake.Calls[2].Input["max_provers"].(*int); !ok || mp == nil || *mp != 9 { + t.Errorf("prove_phase max_provers = %#v, want a pointer to 9", fake.Calls[2].Input["max_provers"]) + } + + // --- hunt_phase gets the RAW recon payload, prove_phase the hunt MODEL --- + reconContext, ok := fake.Calls[1].Input["recon_context"].(map[string]any) + if !ok { + t.Fatalf("recon_context = %#v, want the raw payload map", fake.Calls[1].Input["recon_context"]) + } + if _, present := reconContext["recon_duration_seconds"]; present { + t.Error("recon_context carries recon_duration_seconds: the MODEL dump was forwarded, not the raw payload") + } + huntResult, ok := fake.Calls[2].Input["hunt_result"].(map[string]any) + if !ok { + t.Fatalf("hunt_result = %#v, want hunt.model_dump()", fake.Calls[2].Input["hunt_result"]) + } + for _, key := range []string{"findings", "chains", "total_raw", "strategies_run", "hunt_duration_seconds"} { + if _, present := huntResult[key]; !present { + t.Errorf("hunt_result is missing %q — it is not a full model_dump()", key) + } + } + + // --- checkpoints --- + for _, phase := range []string{"recon", "hunt", "prove"} { + path := filepath.Join(repo, ".sec-af", "checkpoint-"+phase+".json") + if _, statErr := os.Stat(path); statErr != nil { + t.Errorf("checkpoint %s not written: %v", phase, statErr) + } + } + + // --- result bookkeeping --- + result := got + // total_selected(7) + len(strategies_run)(2) + 3 + if result.AgentInvocations != 12 { + t.Errorf("agent_invocations = %d, want 12 (7 + 2 + 3)", result.AgentInvocations) + } + if result.Metadata["findings_not_verified"] != 4 { + t.Errorf("findings_not_verified = %v, want 4", result.Metadata["findings_not_verified"]) + } + summary, ok := result.Metadata["prove_drop_summary"].(map[string]any) + if !ok { + t.Fatalf("prove_drop_summary = %#v, want the payload's own map", result.Metadata["prove_drop_summary"]) + } + // json.Number, not float64: afx.WireNumbers restores the int-vs-float + // split CPython's json.loads makes, so the summary re-serialises into + // metadata as "2" (Python's int) rather than "2.0". + if summary["demoted_total"] != json.Number("2") { + t.Errorf("prove_drop_summary.demoted_total = %#v, want json.Number(\"2\")", summary["demoted_total"]) + } + + // --- the bracketing notes --- + msgs := fake.NoteMessages() + if len(msgs) < 2 || msgs[0] != "Starting SEC-AF audit pipeline" || msgs[len(msgs)-1] != "SEC-AF audit complete" { + t.Errorf("notes = %v, want the start/complete bracket", msgs) + } + if !reflect.DeepEqual(fake.Notes[0].Tags, []string{"audit", "start"}) { + t.Errorf("start note tags = %v, want [audit start]", fake.Notes[0].Tags) + } + last := fake.Notes[len(fake.Notes)-1] + if !reflect.DeepEqual(last.Tags, []string{"audit", "complete"}) { + t.Errorf("complete note tags = %v, want [audit complete]", last.Tags) + } +} + +// TestAuditDefaultsReachThePhases drives the handler with only repo_url and +// checks the defaults the phases observe. +func TestAuditDefaultsReachThePhases(t *testing.T) { + repo := t.TempDir() + fake := &appx.Fake{CallFn: phasePayloads(nil, 0, 0)} + n := newAuditNode(t, repo, fake) + + if _, err := n.handle(context.Background(), map[string]any{"repo_url": repo}); err != nil { + t.Fatalf("handle: %v", err) + } + if fake.Calls[0].Input["depth"] != "standard" { + t.Errorf("recon_phase depth = %v, want the audit() default standard", fake.Calls[0].Input["depth"]) + } + if mp, ok := fake.Calls[2].Input["max_provers"].(*int); !ok || mp != nil { + t.Errorf("prove_phase max_provers = %#v, want a nil *int (Python None)", fake.Calls[2].Input["max_provers"]) + } +} + +// TestAuditResumeSkipsThePipeline pins the resume branch: no `.call`, no +// "Starting SEC-AF audit pipeline" note. +func TestAuditResumeSkipsThePipeline(t *testing.T) { + repo := t.TempDir() + fake := &appx.Fake{CallFn: phasePayloads(nil, 0, 0)} + n := newAuditNode(t, repo, fake) + + _, err := n.handle(context.Background(), map[string]any{ + "repo_url": repo, + "resume_from_checkpoint": "hunt", + }) + // The checkpoint files do not exist, so RunFromCheckpoint fails — which is + // the point: the pipeline was never entered. + if err == nil { + t.Fatal("want a failure reading the missing checkpoint") + } + if len(fake.Calls) != 0 { + t.Errorf("resume made %v calls, want none", fake.CallTargets()) + } + for _, msg := range fake.NoteMessages() { + if msg == "Starting SEC-AF audit pipeline" { + t.Error("resume must not emit the pipeline-start note") + } + } +} + +// TestAuditCheckpointsResumeFromTheirOwnOutput is the round trip the resume +// branch depends on: `_read_checkpoint(phase, schema)` is `schema(**data)`, so +// every checkpoint the pipeline writes must validate when it is read back. A +// finding shape the writer produces but the reader rejects would make +// `audit(resume_from_checkpoint="prove")` a 400 on a healthy run. +func TestAuditCheckpointsResumeFromTheirOwnOutput(t *testing.T) { + repo := t.TempDir() + verified := map[string]any{ + "id": "v1", "fingerprint": "fp1", "title": "SQLi", "description": "d", + "finding_type": "sast", "cwe_id": "CWE-89", "cwe_name": "SQL Injection", + "verdict": "confirmed", "evidence_level": float64(4), "rationale": "r", + "severity": "high", "exploitability_score": 0.9, + "location": map[string]any{"file_path": "a.go", "start_line": float64(1), "end_line": float64(2)}, + "sarif_rule_id": "sast/sql-injection", "sarif_security_severity": 0.9, + "proof": map[string]any{ + "exploit_hypothesis": "h", "verification_method": "static", "evidence_level": float64(4), + }, + "reproduction_steps": []any{map[string]any{"step": float64(1), "description": "d"}}, + "compliance": []any{map[string]any{ + "framework": "OWASP", "control_id": "A03", "control_name": "Injection", + }}, + } + fake := &appx.Fake{CallFn: func(_ context.Context, target string, in map[string]any) (map[string]any, error) { + switch { + case strings.HasSuffix(target, ".prove_phase"): + return map[string]any{ + "verified": []any{verified}, "total_selected": float64(1), + "total_findings": float64(1), "not_verified": float64(0), + }, nil + case strings.HasSuffix(target, ".remediation_phase"): + return map[string]any{"verified": []any{verified}}, nil + } + return phasePayloads([]string{"injection"}, 1, 0)(context.Background(), target, in) + }} + n := newAuditNode(t, repo, fake) + + if _, err := n.handle(context.Background(), map[string]any{"repo_url": repo}); err != nil { + t.Fatalf("handle: %v", err) + } + + for _, phase := range []string{"recon", "hunt", "prove"} { + resumeFake := &appx.Fake{CallFn: phasePayloads([]string{"injection"}, 1, 0)} + resumeNode := newAuditNode(t, repo, resumeFake) + if _, err := resumeNode.handle(context.Background(), map[string]any{ + "repo_url": repo, + "resume_from_checkpoint": phase, + }); err != nil { + t.Errorf("resume from %q failed on checkpoints this pipeline wrote: %v", phase, err) + } + } +} + +// TestAuditBlankResumeRunsThePipeline: `resume_from_checkpoint.strip()` is +// falsy for a whitespace-only value, so the full pipeline runs. +func TestAuditBlankResumeRunsThePipeline(t *testing.T) { + repo := t.TempDir() + fake := &appx.Fake{CallFn: phasePayloads(nil, 0, 0)} + n := newAuditNode(t, repo, fake) + + if _, err := n.handle(context.Background(), map[string]any{ + "repo_url": repo, + "resume_from_checkpoint": " ", + }); err != nil { + t.Fatalf("handle: %v", err) + } + if len(fake.Calls) != 4 { + t.Errorf("calls = %v, want the four phases", fake.CallTargets()) + } +} + +// --------------------------------------------------------------------------- +// error mapping +// --------------------------------------------------------------------------- + +func TestAuditErrorMapping(t *testing.T) { + t.Run("unknown checkpoint phase -> 400, raw message, no note", func(t *testing.T) { + repo := t.TempDir() + fake := &appx.Fake{} + n := newAuditNode(t, repo, fake) + + _, err := n.handle(context.Background(), map[string]any{ + "repo_url": repo, + "resume_from_checkpoint": "nonsense", + }) + exec := asExecuteError(t, err) + if exec.StatusCode != 400 { + t.Errorf("status = %d, want 400", exec.StatusCode) + } + if exec.Message != "Unknown checkpoint phase: nonsense" { + t.Errorf("message = %q, want the raw ValueError text", exec.Message) + } + for _, msg := range fake.NoteMessages() { + if strings.HasPrefix(msg, "Audit pipeline failed") { + t.Error("the 400 branch must not emit the failure note") + } + } + }) + + t.Run("pipeline failure -> note + 500 with the prefix", func(t *testing.T) { + repo := t.TempDir() + fake := &appx.Fake{CallFn: func(context.Context, string, map[string]any) (map[string]any, error) { + return nil, errors.New("recon_phase exploded") + }} + n := newAuditNode(t, repo, fake) + + _, err := n.handle(context.Background(), map[string]any{"repo_url": repo}) + exec := asExecuteError(t, err) + if exec.StatusCode != 500 { + t.Errorf("status = %d, want 500", exec.StatusCode) + } + if !strings.HasPrefix(exec.Message, "audit execution failed: ") { + t.Errorf("message = %q, want the Python prefix", exec.Message) + } + + var failure *appx.NoteCall + for i := range fake.Notes { + if strings.HasPrefix(fake.Notes[i].Message, "Audit pipeline failed: ") { + failure = &fake.Notes[i] + } + } + if failure == nil { + t.Fatalf("no \"Audit pipeline failed\" note; got %v", fake.NoteMessages()) + } + if !reflect.DeepEqual(failure.Tags, []string{"audit", "error"}) { + t.Errorf("failure note tags = %v, want [audit error]", failure.Tags) + } + }) + + // pydantic's ValidationError SUBCLASSES ValueError, so every + // `Model.model_validate(payload)` inside audit()'s try takes the 400 + // branch — raw message, NO note. VERIFIED on the pinned interpreter: + // `ValidationError.__mro__` = (ValidationError, ValueError, Exception, + // BaseException, object). + t.Run("a schema-invalid recon_phase payload -> 400, raw message, no note", func(t *testing.T) { + repo := t.TempDir() + fake := &appx.Fake{CallFn: func(_ context.Context, target string, _ map[string]any) (map[string]any, error) { + if strings.HasSuffix(target, ".recon_phase") { + // app.py:181 `ReconResult.model_validate(recon_dict)` raises. + return map[string]any{"languages": []any{"go"}}, nil + } + return map[string]any{}, nil + }} + n := newAuditNode(t, repo, fake) + + _, err := n.handle(context.Background(), map[string]any{"repo_url": repo}) + exec := asExecuteError(t, err) + if exec.StatusCode != 400 { + t.Errorf("status = %d, want 400", exec.StatusCode) + } + if strings.HasPrefix(exec.Message, "audit execution failed: ") { + t.Errorf("message = %q, want the RAW ValueError text (the 500 prefix is the except-Exception branch)", exec.Message) + } + if !strings.Contains(exec.Message, "ReconResult") { + t.Errorf("message = %q, want the model name", exec.Message) + } + for _, msg := range fake.NoteMessages() { + if strings.HasPrefix(msg, "Audit pipeline failed") { + t.Error("the 400 branch must not emit the failure note") + } + } + }) + + t.Run("a schema-invalid hunt_phase payload -> 400, no note", func(t *testing.T) { + repo := t.TempDir() + fake := &appx.Fake{CallFn: func(_ context.Context, target string, in map[string]any) (map[string]any, error) { + if strings.HasSuffix(target, ".hunt_phase") { + // app.py:191 `HuntResult.model_validate(...)` raises: the + // element is not a RawFinding. + return map[string]any{"findings": []any{map[string]any{"title": "malformed"}}}, nil + } + return phasePayloads(nil, 0, 0)(context.Background(), target, in) + }} + n := newAuditNode(t, repo, fake) + + _, err := n.handle(context.Background(), map[string]any{"repo_url": repo}) + exec := asExecuteError(t, err) + if exec.StatusCode != 400 { + t.Errorf("status = %d, want 400", exec.StatusCode) + } + for _, msg := range fake.NoteMessages() { + if strings.HasPrefix(msg, "Audit pipeline failed") { + t.Error("the 400 branch must not emit the failure note") + } + } + }) + + // orchestrator.py's `_read_checkpoint` does `json.loads(...)`, whose + // JSONDecodeError is also a ValueError -> 400. + t.Run("a corrupt checkpoint file -> 400, no note", func(t *testing.T) { + repo := t.TempDir() + if err := os.MkdirAll(filepath.Join(repo, ".sec-af"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(repo, ".sec-af", "checkpoint-recon.json"), []byte("{not json"), 0o644); err != nil { + t.Fatal(err) + } + fake := &appx.Fake{} + n := newAuditNode(t, repo, fake) + + _, err := n.handle(context.Background(), map[string]any{ + "repo_url": repo, + "resume_from_checkpoint": "recon", + }) + exec := asExecuteError(t, err) + if exec.StatusCode != 400 { + t.Errorf("status = %d, want 400", exec.StatusCode) + } + for _, msg := range fake.NoteMessages() { + if strings.HasPrefix(msg, "Audit pipeline failed") { + t.Error("the 400 branch must not emit the failure note") + } + } + }) + + t.Run("a bad depth fails the constructor -> 500, no note", func(t *testing.T) { + // Python parity: AuditConfig.from_input's ValueError is raised BEFORE + // audit()'s try, so FastAPI answers 500, not 400. + repo := t.TempDir() + fake := &appx.Fake{} + n := newAuditNode(t, repo, fake) + + _, err := n.handle(context.Background(), map[string]any{ + "repo_url": repo, + "depth": "sideways", + }) + exec := asExecuteError(t, err) + if exec.StatusCode != 500 { + t.Errorf("status = %d, want 500", exec.StatusCode) + } + if strings.HasPrefix(exec.Message, "audit execution failed: ") { + t.Errorf("message = %q, want the raw error (this path is not inside the try)", exec.Message) + } + if len(fake.Notes) != 0 { + t.Errorf("notes = %v, want none", fake.NoteMessages()) + } + }) + + t.Run("a repository that does not resolve -> 400, no note", func(t *testing.T) { + fake := &appx.Fake{} + n := newAuditNode(t, t.TempDir(), fake) + n.resolveRepo = ResolveRepo + + _, err := n.handle(context.Background(), map[string]any{"repo_url": "https://example.test/o/r"}) + exec := asExecuteError(t, err) + if exec.StatusCode != 400 { + t.Errorf("status = %d, want 400", exec.StatusCode) + } + if !errors.Is(err, ErrRemoteRepository) { + t.Errorf("err = %v, want it to wrap ErrRemoteRepository", err) + } + if len(fake.Notes) != 0 || len(fake.Calls) != 0 { + t.Errorf("notes = %v, calls = %v, want none", fake.NoteMessages(), fake.CallTargets()) + } + }) +} + +func TestIsBadInput(t *testing.T) { + if !IsBadInput(&orch.UnknownCheckpointPhaseError{Phase: "x"}) { + t.Error("an unknown checkpoint phase must classify as bad input") + } + if !IsBadInput(errors.Join(ErrBadInput, errors.New("wrapped"))) { + t.Error("an error wrapping ErrBadInput must classify as bad input") + } + if IsBadInput(errors.New("boom")) { + t.Error("a plain error must not classify as bad input") + } + + // pydantic's ValidationError subclasses ValueError, so every schema + // failure inside the try is a 400. + if _, err := phases.BindReconResult(map[string]any{}); !IsBadInput(err) { + t.Errorf("a *phases.ValidationError must classify as bad input, got %v", err) + } + if _, err := phases.BindVerifiedFinding(map[string]any{"title": "malformed"}); !IsBadInput(err) { + t.Errorf("a VerifiedFinding validation failure must classify as bad input, got %v", err) + } + if _, err := bindVerifiedList([]any{map[string]any{"title": "malformed"}}); !IsBadInput(err) { + t.Errorf("bindVerifiedList must classify as bad input, got %v", err) + } + // json.JSONDecodeError is a ValueError too. + if err := json.Unmarshal([]byte("{not json"), &map[string]any{}); !IsBadInput(err) { + t.Errorf("a JSON syntax error must classify as bad input, got %v", err) + } + if err := json.Unmarshal([]byte(`{"total_raw":"x"}`), &schemas.HuntResult{}); !IsBadInput(err) { + t.Errorf("a JSON type error must classify as bad input, got %v", err) + } +} + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- + +func asExecuteError(t *testing.T, err error) *Error { + t.Helper() + if err == nil { + t.Fatal("expected an error, got nil") + } + var exec *Error + if !errors.As(err, &exec) { + t.Fatalf("error is not *audit.Error: %T (%v)", err, err) + } + return exec +} + +func assertKeys(t *testing.T, name string, got map[string]any, want ...string) { + t.Helper() + if len(got) != len(want) { + keys := make([]string, 0, len(got)) + for k := range got { + keys = append(keys, k) + } + t.Errorf("%s kwargs = %v, want exactly %v", name, keys, want) + return + } + for _, key := range want { + if _, ok := got[key]; !ok { + t.Errorf("%s kwargs missing %q", name, key) + } + } +} + +func assertStrings(t *testing.T, name string, got, want []string) { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Errorf("%s = %v, want %v", name, got, want) + } +} + +// phasePayloadsWith answers the four phase calls like phasePayloads, then +// applies overrides to the payload of the named phase (an override value of nil +// DELETES the key). +func phasePayloadsWith(phase string, overrides map[string]any) func(context.Context, string, map[string]any) (map[string]any, error) { + base := phasePayloads(nil, 0, 0) + return func(ctx context.Context, target string, input map[string]any) (map[string]any, error) { + out, err := base(ctx, target, input) + if err != nil || !strings.HasSuffix(target, "."+phase) { + return out, err + } + for key, value := range overrides { + if value == nil { + delete(out, key) + continue + } + out[key] = value + } + return out, nil + } +} + +// TestAuditVerifiedKeyErrorsAreStrExc pins the TEXT of the two `KeyError`s +// audit() can raise, which app.py:229-230 interpolates into BOTH the +// "Audit pipeline failed: {exc}" note and the "audit execution failed: {exc}" +// 500 body. +// +// Validation contract (behaviour, from src/sec_af/app.py:201 and :213 — +// `prove_dict["verified"]` and `remediation_dict["verified"]` are bare +// subscripts inside the try): +// +// - a payload with no "verified" key fails the audit; +// - the message is `str(KeyError('verified'))`, which the pinned interpreter +// renders as `'verified'` — the quoted key, with NO `KeyError: ` prefix, +// because app.py interpolates str(exc) and not repr(exc); +// - the failure is an Exception but not a ValueError, so it takes the 500 +// branch (note + prefix), not the 400 one. +func TestAuditVerifiedKeyErrorsAreStrExc(t *testing.T) { + for _, phase := range []string{"prove_phase", "remediation_phase"} { + t.Run(phase, func(t *testing.T) { + repo := t.TempDir() + fake := &appx.Fake{CallFn: phasePayloadsWith(phase, map[string]any{"verified": nil})} + n := newAuditNode(t, repo, fake) + + _, err := n.handle(context.Background(), map[string]any{"repo_url": repo}) + + exec := asExecuteError(t, err) + if exec.StatusCode != 500 { + t.Errorf("status = %d, want 500 (KeyError is not a ValueError)", exec.StatusCode) + } + if want := "audit execution failed: 'verified'"; exec.Message != want { + t.Errorf("500 body = %q, want %q", exec.Message, want) + } + var note string + for _, msg := range fake.NoteMessages() { + if strings.HasPrefix(msg, "Audit pipeline failed: ") { + note = msg + } + } + if want := "Audit pipeline failed: 'verified'"; note != want { + t.Errorf("note = %q, want %q", note, want) + } + }) + } +} + +// TestAuditVerifiedNotIterableIsStrExc pins the other half: `for v in 5` raises +// TypeError, whose str(exc) the pinned interpreter renders as +// `'int' object is not iterable` (and `'NoneType' object is not iterable`, +// `'float' object is not iterable`, `'bool' object is not iterable`) — again +// with no class-name prefix. +// +// Documented residual: Go's encoding/json decodes every JSON number to float64, +// so a numeric payload reports "float" where CPython's json.loads would have +// produced an int and reported "int". That divergence is afx.PyTypeName's, and +// is documented there. +func TestAuditVerifiedNotIterableIsStrExc(t *testing.T) { + for _, tc := range []struct { + name string + payload any + want string + }{ + {"null", nil, "'NoneType' object is not iterable"}, + {"number", float64(5), "'float' object is not iterable"}, + {"bool", true, "'bool' object is not iterable"}, + } { + t.Run(tc.name, func(t *testing.T) { + repo := t.TempDir() + // nil would DELETE the key, which is the KeyError case; wrap it so + // the key stays present with a null value. + answers := phasePayloads(nil, 0, 0) + fake := &appx.Fake{CallFn: func(ctx context.Context, target string, in map[string]any) (map[string]any, error) { + out, err := answers(ctx, target, in) + if err == nil && strings.HasSuffix(target, ".prove_phase") { + out["verified"] = tc.payload + } + return out, err + }} + n := newAuditNode(t, repo, fake) + + _, err := n.handle(context.Background(), map[string]any{"repo_url": repo}) + + exec := asExecuteError(t, err) + if exec.StatusCode != 500 { + t.Errorf("status = %d, want 500 (TypeError is not a ValueError)", exec.StatusCode) + } + if want := "audit execution failed: " + tc.want; exec.Message != want { + t.Errorf("500 body = %q, want %q", exec.Message, want) + } + }) + } +} + +// TestAuditVerifiedElementIsAPydanticValidationError is the third shape: +// the payload ITERATES but an element is not a mapping, so Python reaches +// `VerifiedFinding.model_validate(5)` — a pydantic ValidationError, which +// SUBCLASSES ValueError and therefore takes audit()'s 400 branch (raw message, +// NO note), not the 500 one. VERIFIED on the pinned interpreter: one +// `model_type` error, msg "Input should be a valid dictionary or instance of +// VerifiedFinding", isinstance(exc, ValueError) is True. +// +// A STRING and a DICT are both iterable in Python, so they land here too rather +// than on the TypeError branch — and an EMPTY string or dict iterates zero +// times, yielding an empty verified list and a successful audit. +func TestAuditVerifiedElementIsAPydanticValidationError(t *testing.T) { + for _, tc := range []struct { + name string + payload any + wantErr bool + }{ + {"list of non-dicts", []any{float64(5)}, true}, + {"string iterates characters", "abc", true}, + {"dict iterates keys", map[string]any{"a": float64(1)}, true}, + {"empty string iterates nothing", "", false}, + {"empty dict iterates nothing", map[string]any{}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + repo := t.TempDir() + answers := phasePayloads(nil, 0, 0) + fake := &appx.Fake{CallFn: func(ctx context.Context, target string, in map[string]any) (map[string]any, error) { + out, err := answers(ctx, target, in) + if err == nil && strings.HasSuffix(target, ".prove_phase") { + out["verified"] = tc.payload + } + return out, err + }} + n := newAuditNode(t, repo, fake) + + _, err := n.handle(context.Background(), map[string]any{"repo_url": repo}) + + if !tc.wantErr { + if err != nil { + t.Fatalf("an empty iterable must yield an empty verified list, got %v", err) + } + return + } + exec := asExecuteError(t, err) + if exec.StatusCode != 400 { + t.Errorf("status = %d, want 400 (pydantic's ValidationError subclasses ValueError)", exec.StatusCode) + } + if want := "1 validation error for VerifiedFinding: Input should be a valid dictionary or instance of VerifiedFinding"; exec.Message != want { + t.Errorf("400 body = %q, want %q", exec.Message, want) + } + for _, msg := range fake.NoteMessages() { + if strings.HasPrefix(msg, "Audit pipeline failed") { + t.Error("the 400 branch must not emit the failure note") + } + } + }) + } +} + +// TestAuditDropSummaryIsADotGet pins `prove_dict.get("drop_summary", {...})` +// (app.py:205-208). +// +// Validation contract (behaviour, measured on the pinned interpreter): +// +// - key ABSENT -> the {"demoted_total":0,"by_reason":{},"findings":[]} default; +// - key PRESENT -> the value VERBATIM, whatever it is. `dict.get` never +// inspects the value, so an explicit null stays null, a string stays a +// string, and a list stays a list — all the way into the audit result's +// `metadata.prove_drop_summary`, which is `dict[str, object]` in pydantic +// and accepts any of them. +func TestAuditDropSummaryIsADotGet(t *testing.T) { + defaultSummary := map[string]any{ + "demoted_total": 0, + "by_reason": map[string]int{}, + "findings": []map[string]any{}, + } + + for _, tc := range []struct { + name string + payload any // nil DELETES the key + want any + }{ + {"absent", nil, defaultSummary}, + {"null", (any)(nil), (any)(nil)}, + {"string", "none", "none"}, + // Numbers come back as json.Number: the value is stored UNTYPED and + // re-serialised, so it has to carry the int-vs-float distinction + // CPython's json.loads makes (see afx.WireNumbers). + {"list", []any{float64(1)}, []any{json.Number("1")}}, + {"number", float64(7), json.Number("7")}, + {"object", map[string]any{"demoted_total": float64(3)}, + map[string]any{"demoted_total": json.Number("3")}}, + } { + t.Run(tc.name, func(t *testing.T) { + repo := t.TempDir() + answers := phasePayloads(nil, 0, 0) + fake := &appx.Fake{CallFn: func(ctx context.Context, target string, in map[string]any) (map[string]any, error) { + out, err := answers(ctx, target, in) + if err == nil && strings.HasSuffix(target, ".prove_phase") { + if tc.name == "absent" { + delete(out, "drop_summary") + } else { + out["drop_summary"] = tc.payload + } + } + return out, err + }} + n := newAuditNode(t, repo, fake) + + got, err := n.handle(context.Background(), map[string]any{"repo_url": repo}) + if err != nil { + t.Fatalf("handle: %v", err) + } + result := got + if !reflect.DeepEqual(result.Metadata["prove_drop_summary"], tc.want) { + t.Errorf("metadata.prove_drop_summary = %#v, want %#v", + result.Metadata["prove_drop_summary"], tc.want) + } + }) + } +} + +// TestPhasePayloadErrorTextsCarryNoClassName is the unit-level statement of the +// same rule, including the Source bookkeeping the message deliberately drops: +// str(exc) is the whole message, so nothing may name the exception class and +// nothing may name the phase. +func TestPhasePayloadErrorTextsCarryNoClassName(t *testing.T) { + missing := &missingKeyError{Key: "verified", Source: "prove_phase"} + if got, want := missing.Error(), "'verified'"; got != want { + t.Errorf("missingKeyError = %q, want %q (str(KeyError('verified')))", got, want) + } + if missing.Source != "prove_phase" { + t.Errorf("Source = %q, want it retained for diagnostics", missing.Source) + } + + for _, tc := range []struct{ got, want string }{ + {(¬IterableError{Got: "int"}).Error(), "'int' object is not iterable"}, + {(¬IterableError{Got: "NoneType"}).Error(), "'NoneType' object is not iterable"}, + {(¬IterableError{Got: "float"}).Error(), "'float' object is not iterable"}, + } { + if tc.got != tc.want { + t.Errorf("notIterableError = %q, want %q", tc.got, tc.want) + } + } + + for _, err := range []error{missing, ¬IterableError{Got: "int"}} { + for _, class := range []string{"KeyError", "TypeError", "AttributeError"} { + if strings.Contains(err.Error(), class) { + t.Errorf("%q names an exception class; app.py interpolates str(exc)", err.Error()) + } + } + } +} + +// TestAuditHandlerRunsTheSDKInputValidation pins that the audit entry gets the +// same `_validate_handler_input` layer the 33 registered reasoners do. +// +// Validation contract (behaviour, measured on the pinned interpreter by calling +// the real `app._validate_handler_input(body, input_types["audit"])`): +// +// {"is_pr": "yes"} -> True (bool is a MEMBERSHIP test, not a parse) +// {"is_pr": "no"} -> False +// {"is_pr": 0} -> False +// {"is_pr": null} -> the default False, because is_pr HAS a default +// {"repo_url": 5} -> "5" (str() coercion) +// {"repo_url": null} -> 422 Field 'repo_url' cannot be None +// +// The four coercion cases were json.UnmarshalTypeErrors in an early draft of +// the port, which went straight to afx.Bind. +func TestAuditHandlerRunsTheSDKInputValidation(t *testing.T) { + t.Run("bool coercion", func(t *testing.T) { + for _, tc := range []struct { + value any + want bool + }{ + {"yes", true}, {"true", true}, {"TRUE", true}, {"1", true}, + {"no", false}, {"false", false}, {"", false}, + {float64(0), false}, {float64(2), true}, + {nil, false}, // null on a DEFAULTED parameter is the default + } { + got, err := BindRequest(map[string]any{"repo_url": "u", "is_pr": tc.value}) + if err != nil { + t.Fatalf("is_pr=%#v: %v", tc.value, err) + } + if got.IsPr != tc.want { + t.Errorf("is_pr=%#v bound %v, want %v", tc.value, got.IsPr, tc.want) + } + } + }) + + t.Run("str coercion", func(t *testing.T) { + got, err := BindRequest(map[string]any{"repo_url": float64(5)}) + if err != nil { + t.Fatalf("BindRequest: %v", err) + } + if got.RepoURL != "5" { + t.Errorf("repo_url = %q, want %q", got.RepoURL, "5") + } + // The three non-empty keyword defaults still apply. + if got.Depth != "standard" || got.Branch != "main" || got.SeverityThreshold != "low" { + t.Errorf("defaults lost: %+v", got) + } + }) + + t.Run("null on a defaulted string keeps the default", func(t *testing.T) { + got, err := BindRequest(map[string]any{"repo_url": "u", "branch": nil}) + if err != nil { + t.Fatalf("BindRequest: %v", err) + } + if got.Branch != "main" { + t.Errorf("branch = %q, want main", got.Branch) + } + }) + + t.Run("a null required parameter is a 422", func(t *testing.T) { + repo := t.TempDir() + fake := &appx.Fake{} + n := newAuditNode(t, repo, fake) + + _, err := n.handle(context.Background(), map[string]any{"repo_url": nil}) + exec := asExecuteError(t, err) + if exec.StatusCode != 422 { + t.Errorf("status = %d, want 422 (Python's endpoint rejects the body)", exec.StatusCode) + } + if want := "Field 'repo_url' cannot be None"; exec.Message != want { + t.Errorf("message = %q, want %q", exec.Message, want) + } + for _, msg := range fake.NoteMessages() { + if strings.HasPrefix(msg, "Audit pipeline failed") { + t.Error("input validation runs before the pipeline; no failure note") + } + } + }) +} diff --git a/internal/secaf/audit/local.go b/internal/secaf/audit/local.go new file mode 100644 index 0000000000..830a5ac8d7 --- /dev/null +++ b/internal/secaf/audit/local.go @@ -0,0 +1,179 @@ +package audit + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/Agent-Field/agentfield/sdk/go/ai" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/phases" + "github.com/Agent-Field/codeaf/internal/secaf/reasoners" +) + +// localApp is an appx.App whose Call is answered in process. +// +// The audit pipeline is written as a tree of `.call`s — audit calls the four +// `*_phase` reasoners, and those call the leaf hunters, verifiers and +// remediators — which on the AgentField node each travelled through the control +// plane as a tracked child execution. Inside codeaf there is no control plane, +// so localApp looks each target up in a reasoner registry and runs the handler +// directly. Harness, AI and Note are the wrapped App's own. +type localApp struct { + inner appx.App + reg *reasoners.Registry +} + +var _ appx.App = (*localApp)(nil) + +// WithLocalCalls wraps inner so that every Call resolves to the in-process +// reasoner registry, and returns the wrapper. +// +// The registry is built over the WRAPPER, not over inner: the `*_phase` +// handlers make their nested `.call`s through the App they were registered +// with, and registering them with the wrapper is what keeps those nested calls +// in process too. inner's own Call is never used. +// +// Wrapping an App that is already local returns it unchanged, so a caller that +// wraps defensively does not stack registries. +func WithLocalCalls(inner appx.App) appx.App { + if l, ok := inner.(*localApp); ok { + return l + } + l := &localApp{inner: inner, reg: reasoners.NewRegistry()} + reasoners.RegisterAll(l.reg, l) + return l +} + +// Harness delegates to the wrapped App. +func (l *localApp) Harness(ctx context.Context, prompt string, schema map[string]any, dest any, opts appx.HarnessOptions) (*appx.HarnessResult, error) { + return l.inner.Harness(ctx, prompt, schema, dest, opts) +} + +// AI delegates to the wrapped App. +func (l *localApp) AI(ctx context.Context, prompt string, opts ...ai.Option) (*ai.Response, error) { + return l.inner.AI(ctx, prompt, opts...) +} + +// Note delegates to the wrapped App. +func (l *localApp) Note(ctx context.Context, message string, tags ...string) { + l.inner.Note(ctx, message, tags...) +} + +// Call runs the reasoner target names and returns its result the way the +// control-plane hop returned it. +// +// target is `.` (the phases write `sec-af.recon_phase`), or a +// bare reasoner name, which the SDK qualified with its own node id. A node +// other than phases.NodeID() has nothing behind it here and is refused. +// +// THE JSON HOP IS KEPT ON PURPOSE. Over the wire, the input was marshalled, +// decoded by the node into a fresh map[string]any, and the handler's result was +// marshalled and decoded again into a map[string]any for the caller. Callers +// downstream depend on what that does to values — every number arrives as a +// float64, a typed struct arrives as a plain map, a nil result arrives as a nil +// map (which afx.AsMap reports as NoneType), and a non-object result is an +// error — so both directions round-trip through encoding/json exactly as before +// instead of handing Go values across by reference. +// +// A handler's failure comes back as a *CallError carrying the handler's message +// and NOTHING of its type, which is what the caller saw over the wire: a +// validation failure deep inside prove_phase reached audit as an opaque failed +// execution, not as a bad-input error of audit's own. The one exception is +// cancellation, which unwraps to the context's error so a cancelled audit still +// reads as cancelled. +func (l *localApp) Call(ctx context.Context, target string, input map[string]any) (map[string]any, error) { + name := target + if node, reasoner, qualified := strings.Cut(target, "."); qualified { + if node != phases.NodeID() { + return nil, &CallError{Target: target, Message: fmt.Sprintf("no reasoner %q in this process: node %q is not %q", target, node, phases.NodeID())} + } + name = reasoner + } + handler, ok := l.reg.Lookup(name) + if !ok { + return nil, &CallError{Target: target, Message: fmt.Sprintf("no reasoner %q in this process", target)} + } + + wireInput, err := roundTripInput(input) + if err != nil { + return nil, fmt.Errorf("marshal call payload: %w", err) + } + + result, err := runHandler(ctx, handler, wireInput) + if err != nil { + callErr := &CallError{Target: target, Message: err.Error()} + if ctxErr := ctx.Err(); ctxErr != nil { + callErr.cause = ctxErr + } + return nil, callErr + } + + return roundTripResult(target, result) +} + +// runHandler runs one handler and turns a panic into an error. Over the wire a +// panicking reasoner failed its own execution and the caller saw a failed call; +// in process the same panic would take codeaf down with it. +func runHandler(ctx context.Context, h reasoners.Handler, input map[string]any) (result any, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("reasoner panicked: %v", r) + } + }() + return h(ctx, input) +} + +// roundTripInput is the request half of the hop: marshal, then decode into a +// fresh map[string]any the way the node decoded the request body. +func roundTripInput(input map[string]any) (map[string]any, error) { + b, err := json.Marshal(input) + if err != nil { + return nil, err + } + var out map[string]any + if err := json.Unmarshal(b, &out); err != nil { + return nil, err + } + return out, nil +} + +// roundTripResult is the response half: the handler's value is marshalled and +// decoded into a map[string]any, a JSON null becomes a nil map with no error +// (the SDK's `(nil, nil)` for a null result), and anything that is not an +// object is an error, as the SDK's decode of the execution result was. +func roundTripResult(target string, result any) (map[string]any, error) { + b, err := json.Marshal(result) + if err != nil { + return nil, &CallError{Target: target, Message: fmt.Sprintf("encode result of %s: %v", target, err)} + } + if string(b) == "null" { + return nil, nil + } + var out map[string]any + if err := json.Unmarshal(b, &out); err != nil { + return nil, &CallError{Target: target, Message: fmt.Sprintf("decode result of %s: %v", target, err)} + } + return out, nil +} + +// CallError is a `.call` that did not succeed. Its text is exactly the +// failing reasoner's own message — the error_message the control plane +// reported — because phases fold that text into fallback findings and notes. +type CallError struct { + Target string + Message string + + // cause is set only when the call failed because the context ended, so + // errors.Is(err, context.Canceled) holds without exposing the handler's + // own error types. + cause error +} + +func (e *CallError) Error() string { return e.Message } + +// Unwrap exposes the context's error for a cancelled call and nothing +// otherwise. +func (e *CallError) Unwrap() error { return e.cause } diff --git a/internal/secaf/audit/local_test.go b/internal/secaf/audit/local_test.go new file mode 100644 index 0000000000..25b530092d --- /dev/null +++ b/internal/secaf/audit/local_test.go @@ -0,0 +1,281 @@ +package audit + +// Tests for the in-process Caller. +// +// Validation contract (behaviour, derived from what the control-plane hop did +// to a `.call` — sdk/go/agent's Call and the node's request decoding): +// +// - the wrapper's registry carries exactly the 33 reasoner names, in +// DESIGN.md §3 order, and `audit` is not one of them; +// - `sec-af.` and a bare `` both resolve; any other node, and an +// unknown name, is an error that names the target; +// - input and result each cross a JSON round trip: numbers arrive as float64, +// typed values arrive as plain maps, a null result is a nil map with no +// error, and a non-object result is an error; +// - a handler failure reaches the caller as a *CallError carrying exactly the +// handler's message and none of its type; a cancelled call still unwraps to +// the context's error; a panic is an error, not a crash; +// - Run never uses the wrapped App's own Call: the phase calls and the leaf +// calls under them all resolve in process. + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/phases" + "github.com/Agent-Field/codeaf/internal/secaf/reasoners" +) + +// localWith returns a local App whose registry holds exactly the given handlers, +// for the transport tests that should not run a real reasoner. +func localWith(t *testing.T, handlers map[string]reasoners.Handler) *localApp { + t.Helper() + l := &localApp{inner: &appx.Fake{}, reg: reasoners.NewRegistry()} + for name, h := range handlers { + l.reg.Register(name, h, nil) + } + return l +} + +func TestWithLocalCallsRegistersTheReasonerSurface(t *testing.T) { + app := WithLocalCalls(&appx.Fake{}) + l, ok := app.(*localApp) + if !ok { + t.Fatalf("WithLocalCalls returned %T", app) + } + if got := l.reg.Names(); !reflect.DeepEqual(got, reasoners.Names) { + t.Fatalf("registry = %v, want %v", got, reasoners.Names) + } + if len(l.reg.Names()) != 33 { + t.Errorf("registry holds %d reasoners, want 33", len(l.reg.Names())) + } + if _, ok := l.reg.Lookup(reasoners.NameAudit); ok { + t.Error("audit is the entry point, not a reasoner the pipeline calls") + } + if again := WithLocalCalls(app); again != app { + t.Error("wrapping a local App again must return it unchanged") + } +} + +func TestLocalCallResolvesQualifiedAndBareNames(t *testing.T) { + l := localWith(t, map[string]reasoners.Handler{ + "run_x": func(context.Context, map[string]any) (any, error) { + return map[string]any{"ok": true}, nil + }, + }) + for _, target := range []string{phases.NodeID() + ".run_x", "run_x"} { + got, err := l.Call(context.Background(), target, nil) + if err != nil { + t.Fatalf("Call(%q): %v", target, err) + } + if got["ok"] != true { + t.Errorf("Call(%q) = %v", target, got) + } + } + + for _, target := range []string{"other-node.run_x", "sec-af.run_missing", "run_missing"} { + _, err := l.Call(context.Background(), target, nil) + var callErr *CallError + if !errors.As(err, &callErr) { + t.Fatalf("Call(%q) error = %T %v, want *CallError", target, err, err) + } + if !strings.Contains(callErr.Error(), target) { + t.Errorf("Call(%q) error %q does not name the target", target, callErr.Error()) + } + } +} + +func TestLocalCallRoundTripsThroughJSON(t *testing.T) { + type typed struct { + Count int `json:"count"` + Name string `json:"name"` + } + var seen map[string]any + l := localWith(t, map[string]reasoners.Handler{ + "run_echo": func(_ context.Context, in map[string]any) (any, error) { + seen = in + return struct { + Total int `json:"total"` + Typed typed `json:"typed"` + }{Total: 3, Typed: typed{Count: 2, Name: "n"}}, nil + }, + }) + + n := 9 + got, err := l.Call(context.Background(), "sec-af.run_echo", map[string]any{ + "max_provers": &n, + "none": (*int)(nil), + "item": typed{Count: 1, Name: "a"}, + }) + if err != nil { + t.Fatalf("Call: %v", err) + } + + // The handler sees what the node decoded from the request body. + if seen["max_provers"] != float64(9) { + t.Errorf("max_provers = %#v, want float64(9)", seen["max_provers"]) + } + if v, present := seen["none"]; !present || v != nil { + t.Errorf("none = %#v (present %v), want a present null", v, present) + } + if item, ok := seen["item"].(map[string]any); !ok || item["count"] != float64(1) || item["name"] != "a" { + t.Errorf("item = %#v, want the decoded object", seen["item"]) + } + + // The caller sees what it decoded from the execution result. + if got["total"] != float64(3) { + t.Errorf("total = %#v, want float64(3)", got["total"]) + } + if typedOut, ok := got["typed"].(map[string]any); !ok || typedOut["count"] != float64(2) { + t.Errorf("typed = %#v, want a decoded object", got["typed"]) + } +} + +func TestLocalCallResultShapes(t *testing.T) { + l := localWith(t, map[string]reasoners.Handler{ + "run_null": func(context.Context, map[string]any) (any, error) { return nil, nil }, + "run_list": func(context.Context, map[string]any) (any, error) { return []any{1}, nil }, + }) + + got, err := l.Call(context.Background(), "sec-af.run_null", nil) + if err != nil || got != nil { + t.Errorf("null result = (%#v, %v), want (nil, nil)", got, err) + } + + if _, err := l.Call(context.Background(), "sec-af.run_list", nil); err == nil { + t.Error("a non-object result must be an error, as the SDK's decode was") + } +} + +func TestLocalCallFailuresAreOpaque(t *testing.T) { + l := localWith(t, map[string]reasoners.Handler{ + "run_invalid": func(context.Context, map[string]any) (any, error) { + return nil, &phases.ValidationError{Model: "ReconResult", Errors: []string{"bad"}} + }, + "run_panics": func(context.Context, map[string]any) (any, error) { + panic("boom") + }, + }) + + _, err := l.Call(context.Background(), "sec-af.run_invalid", nil) + var callErr *CallError + if !errors.As(err, &callErr) { + t.Fatalf("error = %T, want *CallError", err) + } + want := (&phases.ValidationError{Model: "ReconResult", Errors: []string{"bad"}}).Error() + if err.Error() != want { + t.Errorf("message = %q, want the handler's own %q", err.Error(), want) + } + var validation *phases.ValidationError + if errors.As(err, &validation) { + t.Error("the handler's error type leaked through the call; the wire hop never carried it") + } + if IsBadInput(err) { + t.Error("a failed child call must not classify as the audit's own bad input") + } + + _, err = l.Call(context.Background(), "sec-af.run_panics", nil) + if err == nil || !strings.Contains(err.Error(), "boom") { + t.Errorf("panic = %v, want an error carrying the panic value", err) + } +} + +func TestLocalCallCancellationUnwraps(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + l := localWith(t, map[string]reasoners.Handler{ + "run_wait": func(ctx context.Context, _ map[string]any) (any, error) { + cancel() + <-ctx.Done() + return nil, ctx.Err() + }, + }) + _, err := l.Call(ctx, "sec-af.run_wait", nil) + if !errors.Is(err, context.Canceled) { + t.Errorf("error = %v, want it to unwrap to context.Canceled", err) + } +} + +func TestLocalCallDelegatesHarnessAINote(t *testing.T) { + fake := &appx.Fake{ + HarnessFn: func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{Result: "r"}, nil + }, + } + app := WithLocalCalls(fake) + if _, err := app.Harness(context.Background(), "p", nil, nil, appx.HarnessOptions{Cwd: "/c"}); err != nil { + t.Fatalf("Harness: %v", err) + } + app.Note(context.Background(), "hello", "t") + _, _ = app.AI(context.Background(), "q") + if len(fake.Harnesses) != 1 || fake.Harnesses[0].Opts.Cwd != "/c" { + t.Errorf("harness calls = %+v", fake.Harnesses) + } + if len(fake.AIs) != 1 || fake.AIs[0].Prompt != "q" { + t.Errorf("ai calls = %+v", fake.AIs) + } + if !reflect.DeepEqual(fake.NoteMessages(), []string{"hello"}) { + t.Errorf("notes = %v", fake.NoteMessages()) + } +} + +// TestRunResolvesEveryCallInProcess drives Run end to end against a repository +// on disk with a harness that fails every run. The audit's outcome does not +// matter here; what does is that the phases and the leaf reasoners under them +// were reached — the harness ran — while the wrapped App's own Call, which +// would have recorded the attempt, was never used. +func TestRunResolvesEveryCallInProcess(t *testing.T) { + repo := t.TempDir() + fake := &appx.Fake{ + HarnessFn: func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{IsError: true, ErrorMessage: "no model in this test"}, nil + }, + } + + _, _ = Run(context.Background(), fake, AuditRequest{ + RepoURL: repo, + Depth: "quick", + Branch: "main", + SeverityThreshold: "low", + }) + + if len(fake.Calls) != 0 { + t.Errorf("the wrapped App's Call was used: %v", fake.CallTargets()) + } + if len(fake.Harnesses) == 0 { + t.Error("no harness ran: the phase calls never reached the leaf reasoners") + } + msgs := fake.NoteMessages() + if len(msgs) == 0 || msgs[0] != "Starting SEC-AF audit pipeline" { + t.Errorf("notes = %v, want the pipeline to have started", msgs) + } +} + +func TestRunRefusesARepositoryThatIsNotLocal(t *testing.T) { + fake := &appx.Fake{} + _, err := Run(context.Background(), fake, AuditRequest{RepoURL: "git@github.com:o/r.git", Depth: "quick"}) + var auditErr *Error + if !errors.As(err, &auditErr) || auditErr.StatusCode != 400 { + t.Fatalf("error = %#v, want a 400 *Error", err) + } + if !errors.Is(err, ErrRemoteRepository) { + t.Errorf("error = %v, want ErrRemoteRepository", err) + } + if len(fake.Notes) != 0 || len(fake.Harnesses) != 0 { + t.Error("a refused repository must not start the pipeline") + } +} + +func TestRunInputValidatesTheBody(t *testing.T) { + _, err := RunInput(context.Background(), &appx.Fake{}, map[string]any{"repo_url": nil}) + var auditErr *Error + if !errors.As(err, &auditErr) || auditErr.StatusCode != 422 { + t.Fatalf("error = %#v, want a 422 *Error", err) + } + if auditErr.Message != "Field 'repo_url' cannot be None" { + t.Errorf("message = %q", auditErr.Message) + } +} diff --git a/internal/secaf/audit/request.go b/internal/secaf/audit/request.go new file mode 100644 index 0000000000..f59500ae36 --- /dev/null +++ b/internal/secaf/audit/request.go @@ -0,0 +1,153 @@ +package audit + +import ( + "encoding/json" + "errors" + "net/http" + + "github.com/Agent-Field/codeaf/internal/secaf/afx" + "github.com/Agent-Field/codeaf/internal/secaf/reasoners" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// AuditRequest transcribes the `audit(...)` signature (app.py:124-144) — all +// twenty parameters, with their exact names, types and defaults. +// +// Nil pointers stand for Python's `None`; nil slices stand for the four +// `list[str] | None = None` parameters, whose `or` fallbacks live in +// ToAuditInput. +// +// RepoURL keeps its wire name, but inside codeaf it must name a local +// directory: ResolveRepo refuses a URL rather than cloning it. +type AuditRequest struct { + RepoURL string `json:"repo_url"` + Depth string `json:"depth"` + Branch string `json:"branch"` + CommitSha *string `json:"commit_sha"` + BaseCommitSha *string `json:"base_commit_sha"` + SeverityThreshold string `json:"severity_threshold"` + ScanTypes []string `json:"scan_types"` + OutputFormats []string `json:"output_formats"` + ComplianceFrameworks []string `json:"compliance_frameworks"` + MaxCostUsd *float64 `json:"max_cost_usd"` + MaxProvers *int `json:"max_provers"` + MaxDurationSeconds *int `json:"max_duration_seconds"` + IncludePaths []string `json:"include_paths"` + ExcludePaths []string `json:"exclude_paths"` + IsPr bool `json:"is_pr"` + PrID *string `json:"pr_id"` + PostPrComments bool `json:"post_pr_comments"` + FailOnFindings bool `json:"fail_on_findings"` + // EnableDast is accepted and then DISCARDED — see ToAuditInput. + EnableDast bool `json:"enable_dast"` + ResumeFromCheckpoint *string `json:"resume_from_checkpoint"` + // CheckpointDir is where the phases' checkpoints are written, in place of + // the `.sec-af` folder inside the repository. It is codeaf's, never on the + // wire: an audit run by codeaf promises the person their folder back as it + // found it, so its checkpoints go to the run's record folder. Empty keeps + // sec-af's own place. + CheckpointDir string `json:"-"` +} + +// NewAuditRequest returns the three non-empty keyword defaults of audit(): +// depth="standard", branch="main", severity_threshold="low". Every other +// parameter's default is None / [] / False, which is the Go zero value. +func NewAuditRequest() AuditRequest { + return AuditRequest{ + Depth: "standard", + Branch: "main", + SeverityThreshold: "low", + } +} + +// UnmarshalJSON seeds audit()'s keyword defaults before decoding. +func (a *AuditRequest) UnmarshalJSON(b []byte) error { + *a = NewAuditRequest() + type alias AuditRequest + return json.Unmarshal(b, (*alias)(a)) +} + +// ToAuditInput ports the `AuditInput(...)` construction at app.py:146. +// +// Python parity, three points: +// +// - the four list fallbacks are `x or [default]`, i.e. PYTHON TRUTHINESS, not +// `is None`. An EXPLICIT EMPTY LIST therefore also falls back to the +// default: `scan_types=[]` yields ["sast","sca","secrets","config"], and +// `exclude_paths=[]` yields the four-entry default. There is no way to ask +// for "no exclusions" through this entry. Reproduced. +// - `include_paths` has NO fallback: it is forwarded as-is, so None stays +// None (scan everything) and [] stays [] (an empty include filter). +// - `enable_dast=enable_dast` is passed to a model that has NO `enable_dast` +// FIELD — AuditInput declares `dast_enabled`. pydantic's default +// `extra="ignore"` DROPS it silently, so `dast_enabled` is False for every +// request no matter what the caller sends. This is a live Python bug — the +// `enable_dast` parameter is inert — and it is reproduced rather than +// fixed: DastEnabled is left at the pydantic default. +// +// The five fields audit() does not pass at all (dast_enabled, repo_urls, +// monitoring_mode, baseline_path, custom_policies) keep their pydantic +// defaults, which is what starting from schemas.NewAuditInput() gives. +func (a AuditRequest) ToAuditInput() schemas.AuditInput { + in := schemas.NewAuditInput() + in.RepoURL = a.RepoURL + in.Depth = a.Depth + in.Branch = a.Branch + in.CommitSha = a.CommitSha + in.BaseCommitSha = a.BaseCommitSha + in.SeverityThreshold = a.SeverityThreshold + in.ScanTypes = orDefault(a.ScanTypes, []string{"sast", "sca", "secrets", "config"}) + in.OutputFormats = orDefault(a.OutputFormats, []string{"json"}) + in.ComplianceFrameworks = orDefault(a.ComplianceFrameworks, []string{}) + in.MaxCostUsd = a.MaxCostUsd + in.MaxProvers = a.MaxProvers + in.MaxDurationSeconds = a.MaxDurationSeconds + in.IncludePaths = a.IncludePaths + in.ExcludePaths = orDefault(a.ExcludePaths, []string{"tests/", "vendor/", "node_modules/", ".git/"}) + in.IsPr = a.IsPr + in.PrID = a.PrID + in.PostPrComments = a.PostPrComments + in.FailOnFindings = a.FailOnFindings + // in.DastEnabled deliberately untouched — see the doc comment. + return in +} + +// BindRequest turns an untyped keyword map into an AuditRequest the way the +// node's `audit` reasoner did, in Python's order: +// +// validated = self._validate_handler_input(body, input_fields) # SDK +// audit_input = AuditInput(**validated) # pydantic +// +// The SDK-level validation is what turns `{"is_pr": "yes"}` into true and +// `{"repo_url": 5}` into "5" instead of a bind error, and what rejects an +// explicit null on a required parameter. Its refusal is an *Error with 422, +// the status Python's endpoint answered with; a failure of the typed bind that +// follows is an *Error with 400. +func BindRequest(input map[string]any) (AuditRequest, error) { + validated, err := reasoners.ValidateHandlerInput(reasoners.NameAudit, input) + if err != nil { + var handlerInput *reasoners.HandlerInputError + if errors.As(err, &handlerInput) { + return AuditRequest{}, &Error{StatusCode: http.StatusUnprocessableEntity, Message: err.Error(), Err: err} + } + return AuditRequest{}, err + } + req, err := afx.Bind[AuditRequest](validated) + if err != nil { + // The pydantic half — `AuditInput(**validated)` at app.py:146 — was + // raised outside audit()'s try and reached FastAPI as a generic 500; + // the Go node answered 400, the bad-input status, because a 500 tells + // the caller nothing about a malformed body. Kept. + return AuditRequest{}, &Error{StatusCode: http.StatusBadRequest, Message: err.Error(), Err: err} + } + return req, nil +} + +// orDefault is Python's `value or default` for a list: an empty (or nil) slice +// is falsy and yields the default. +func orDefault(value, def []string) []string { + if len(value) == 0 { + return def + } + return value +} diff --git a/internal/secaf/audit/resolve.go b/internal/secaf/audit/resolve.go new file mode 100644 index 0000000000..91338e6c7a --- /dev/null +++ b/internal/secaf/audit/resolve.go @@ -0,0 +1,63 @@ +package audit + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +// remotePrefixes are the spellings _resolve_repo treated as something to clone +// (`repo_url.startswith(("https://", "http://", "git@"))`), plus any other +// scheme-qualified address, so a URL is named as a URL in the refusal rather +// than reported as a directory that does not exist. +var remotePrefixes = []string{"https://", "http://", "git@", "ssh://", "git://", "file://"} + +// ErrRemoteRepository is the refusal for a repository given as a URL. Inside +// codeaf the audit reads a checkout that is already on disk; it never clones. +var ErrRemoteRepository = errors.New("the security audit reads a local directory and does not clone repositories") + +// ResolveRepo turns the request's repo_url into the absolute, symlink-free +// directory the audit reads. +// +// It replaces `_resolve_repo(repo_url)` (src/sec_af/app.py:75), and is +// deliberately narrower. The node cloned a URL into a workspaces directory +// (SEC_AF_WORKSPACES_DIR, falling back to ~/.sec-af/workspaces), pulled an +// existing clone, and silently audited SEC_AF_REPO_PATH or its own working +// directory when repo_url was neither a URL nor a directory. Inside codeaf all +// three would be surprises — a network fetch nobody asked for, a directory +// written outside the project, or an audit of the wrong tree — so only an +// existing local directory is accepted, and everything else is refused with a +// sentence that says why. +func ResolveRepo(repoURL string) (string, error) { + trimmed := strings.TrimSpace(repoURL) + if trimmed == "" { + return "", errors.New("no repository to audit: repo_url is empty") + } + for _, prefix := range remotePrefixes { + if strings.HasPrefix(trimmed, prefix) { + return "", fmt.Errorf("%w: %q is a URL; clone it and audit the directory", ErrRemoteRepository, trimmed) + } + } + + abs, err := filepath.Abs(trimmed) + if err != nil { + return "", fmt.Errorf("repository %q: %w", trimmed, err) + } + resolved, err := filepath.EvalSymlinks(abs) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("repository %q does not exist", trimmed) + } + return "", fmt.Errorf("repository %q: %w", trimmed, err) + } + info, err := os.Stat(resolved) + if err != nil { + return "", fmt.Errorf("repository %q: %w", trimmed, err) + } + if !info.IsDir() { + return "", fmt.Errorf("repository %q is not a directory", trimmed) + } + return resolved, nil +} diff --git a/internal/secaf/audit/resolve_test.go b/internal/secaf/audit/resolve_test.go new file mode 100644 index 0000000000..10237d76ce --- /dev/null +++ b/internal/secaf/audit/resolve_test.go @@ -0,0 +1,88 @@ +package audit + +// Tests for repository resolution. +// +// Validation contract: only an existing local directory is audited. It comes +// back absolute and with symlinks resolved; a URL, an empty value, a missing +// path and a regular file are each refused with an error that says which, and +// nothing is cloned, created or fallen back to. + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestResolveRepoReturnsTheResolvedDirectory(t *testing.T) { + dir := t.TempDir() + want, err := filepath.EvalSymlinks(dir) + if err != nil { + t.Fatal(err) + } + + got, err := ResolveRepo(dir) + if err != nil { + t.Fatalf("ResolveRepo: %v", err) + } + if got != want { + t.Errorf("ResolveRepo = %q, want %q", got, want) + } + + link := filepath.Join(t.TempDir(), "link") + if err := os.Symlink(dir, link); err != nil { + t.Skipf("symlink: %v", err) + } + if got, err := ResolveRepo(link); err != nil || got != want { + t.Errorf("ResolveRepo(symlink) = (%q, %v), want %q", got, err, want) + } +} + +func TestResolveRepoMakesARelativePathAbsolute(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + if err := os.Mkdir("sub", 0o755); err != nil { + t.Fatal(err) + } + got, err := ResolveRepo("sub") + if err != nil { + t.Fatalf("ResolveRepo: %v", err) + } + if !filepath.IsAbs(got) || filepath.Base(got) != "sub" { + t.Errorf("ResolveRepo(sub) = %q, want an absolute path ending in sub", got) + } +} + +func TestResolveRepoRefuses(t *testing.T) { + file := filepath.Join(t.TempDir(), "f") + if err := os.WriteFile(file, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + + for _, tc := range []struct { + name, in, want string + remote bool + }{ + {"https", "https://github.com/o/r", "is a URL", true}, + {"http", "http://example.test/r.git", "is a URL", true}, + {"scp-style", "git@github.com:o/r.git", "is a URL", true}, + {"ssh", "ssh://git@example.test/r", "is a URL", true}, + {"empty", " ", "repo_url is empty", false}, + {"missing", filepath.Join(t.TempDir(), "absent"), "does not exist", false}, + {"file", file, "is not a directory", false}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := ResolveRepo(tc.in) + if err == nil { + t.Fatalf("ResolveRepo(%q) = %q, want a refusal", tc.in, got) + } + if !strings.Contains(err.Error(), tc.want) { + t.Errorf("error = %q, want it to say %q", err.Error(), tc.want) + } + if errors.Is(err, ErrRemoteRepository) != tc.remote { + t.Errorf("errors.Is(ErrRemoteRepository) = %v, want %v", !tc.remote, tc.remote) + } + }) + } +} diff --git a/internal/secaf/audit/run.go b/internal/secaf/audit/run.go new file mode 100644 index 0000000000..ce707491ec --- /dev/null +++ b/internal/secaf/audit/run.go @@ -0,0 +1,520 @@ +// Package audit is the security audit's entry point inside codeaf: the request, +// the repository it reads, the four-phase pipeline, and the in-process Caller +// that answers the pipeline's `.call`s. +// +// It replaces the sec-af node's wiring layer (its `audit` reasoner, its +// repository resolution and its control-plane registration). What the node did +// over HTTP — route `sec-af.recon_phase` to a reasoner of the same node, which +// in turn called `sec-af.run_architecture_mapper` and the rest — happens here +// in process: Run wraps the App with WithLocalCalls, whose registry holds the +// same 33 reasoners under the same names. +package audit + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "sort" + "strings" + "time" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/afx" + "github.com/Agent-Field/codeaf/internal/secaf/orch" + "github.com/Agent-Field/codeaf/internal/secaf/phases" + "github.com/Agent-Field/codeaf/internal/secaf/reasoners" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// nowMonotonic is `time.monotonic()`. It is a variable so a test can pin the +// two duration fields the pipeline stamps; production never reassigns it. +// +// Go's time.Now carries a monotonic reading that Sub prefers over the wall +// clock, so a DIFFERENCE has exactly Python's time.monotonic() semantics. +var nowMonotonic = time.Now + +// Error is an audit that produced no result. StatusCode keeps the HTTP status +// the node answered with, because it is how a caller tells the three outcomes +// apart: +// +// 422 the request failed the SDK-level input validation (BindRequest) +// 400 the request was refused: it did not bind, it names no usable +// directory, or the pipeline raised one of Python's ValueError family +// (see IsBadInput) — Message is the raw text +// 500 the audit itself failed — Message carries Python's +// "audit execution failed: " prefix when the failure came from inside +// the pipeline +// +// Err is the underlying cause, for errors.Is and errors.As. +type Error struct { + StatusCode int + Message string + Err error +} + +func (e *Error) Error() string { return e.Message } + +// Unwrap returns the underlying cause. +func (e *Error) Unwrap() error { return e.Err } + +// Run audits one local repository and returns the audit result, running every +// `.call` in the pipeline in process (WithLocalCalls). app supplies Harness, +// AI and Note; its own Call is never used. +// +// A failure is always an *Error. +func Run(ctx context.Context, app appx.App, req AuditRequest) (schemas.SecurityAuditResult, error) { + return newRunner(WithLocalCalls(app)).run(ctx, req) +} + +// RunInput is Run for an untyped keyword map — the body the node's `audit` +// reasoner received — validated and bound by BindRequest first. +func RunInput(ctx context.Context, app appx.App, input map[string]any) (schemas.SecurityAuditResult, error) { + req, err := BindRequest(input) + if err != nil { + return schemas.SecurityAuditResult{}, asError(err, http.StatusBadRequest) + } + return Run(ctx, app, req) +} + +// runner holds the seams the pipeline is tested through: the App whose Call +// the four phase calls go through, the orchestrator constructor, and the +// repository resolver. +type runner struct { + app appx.App + newOrchestrator func(ctx context.Context, app appx.App, in schemas.AuditInput, repoPath string) (*orch.AuditOrchestrator, error) + resolveRepo func(repoURL string) (string, error) +} + +func newRunner(app appx.App) *runner { + return &runner{app: app, newOrchestrator: orch.NewAt, resolveRepo: ResolveRepo} +} + +// asError returns err as an *Error, wrapping it with status when it is not one. +func asError(err error, status int) *Error { + var e *Error + if errors.As(err, &e) { + return e + } + return &Error{StatusCode: status, Message: err.Error(), Err: err} +} + +// run ports the `audit(...)` reasoner (app.py:123-231): +// +// audit_input = AuditInput(...) # bind +// orchestrator = AuditOrchestrator(app, input) # OUTSIDE the try +// repo_path = _resolve_repo(repo_url) # OUTSIDE the try +// orchestrator.repo_path = Path(repo_path) +// orchestrator.checkpoint_dir = repo_path/".sec-af" +// try: +// | +// except ValueError as exc: 400 {"error": str(exc)} +// except Exception as exc: note("Audit pipeline failed: ..."); 500 {"error": "audit execution failed: ..."} +// return result.model_dump() +// +// One deliberate change of order: the repository is resolved BEFORE the +// orchestrator is built, and handed to it (orch.NewAt). Python built the +// orchestrator against SEC_AF_REPO_PATH or the working directory first, so its +// PR-mode diff analysis and AuditConfig.repo_path described whatever directory +// the process started in; inside codeaf that is never the right tree. +func (r *runner) run(ctx context.Context, req AuditRequest) (schemas.SecurityAuditResult, error) { + auditInput := req.ToAuditInput() + + repoPath, err := r.resolveRepo(req.RepoURL) + if err != nil { + // The node cloned a URL and fell back to its working directory, and + // only a failed clone was an error (a 500, raised outside the try). + // Resolution now refuses instead — a URL, a missing path, a file — and + // a refusal is about the request, so it is a 400 with the reason. + return schemas.SecurityAuditResult{}, &Error{StatusCode: http.StatusBadRequest, Message: err.Error(), Err: err} + } + + orchestrator, err := r.newOrchestrator(ctx, r.app, auditInput, repoPath) + if err != nil { + // Python parity: AuditConfig.from_input raises ValueError("'x' is not a + // valid DepthProfile") OUTSIDE the try, so FastAPI answered with a + // generic 500 — not the 400 the ValueError branch would give. Kept as a + // 500, with the message surfaced. See IsBadInput. + return schemas.SecurityAuditResult{}, &Error{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} + } + + if dir := strings.TrimSpace(req.CheckpointDir); dir != "" { + orchestrator.CheckpointDir = dir + } + result, err := r.runPipeline(ctx, orchestrator, req, repoPath) + if err != nil { + if IsBadInput(err) { + // `except ValueError` -> 400 with the RAW message, no note. + return schemas.SecurityAuditResult{}, &Error{StatusCode: http.StatusBadRequest, Message: err.Error(), Err: err} + } + // `except Exception` -> note, then 500 with the prefix (app.py:224-230). + // Python also printed `AUDIT ERROR: ...` to stdout first; inside codeaf + // stdout is the terminal the person is looking at, and the note carries + // the same text, so the print is not reproduced. + r.app.Note(ctx, "Audit pipeline failed: "+err.Error(), "audit", "error") + return schemas.SecurityAuditResult{}, &Error{ + StatusCode: http.StatusInternalServerError, + Message: "audit execution failed: " + err.Error(), + Err: err, + } + } + + // Python returns `result.model_dump()`; SecurityAuditResult marshals to the + // identical snake_case key set. + return result, nil +} + +// handle is the node's `audit` reasoner end to end — bind the keyword map, +// then run — over this runner's seams. It is what the tests drive. +func (r *runner) handle(ctx context.Context, input map[string]any) (schemas.SecurityAuditResult, error) { + req, err := BindRequest(input) + if err != nil { + return schemas.SecurityAuditResult{}, asError(err, http.StatusBadRequest) + } + return r.run(ctx, req) +} + +// runPipeline is the body of audit()'s `try:` block — everything whose failure +// the 400/500 mapping applies to. +func (r *runner) runPipeline( + ctx context.Context, + orchestrator *orch.AuditOrchestrator, + req AuditRequest, + repoPath string, +) (schemas.SecurityAuditResult, error) { + // `if isinstance(resume_from_checkpoint, str) and resume_from_checkpoint.strip():` + // — a whitespace-only value is falsy and takes the full-pipeline branch. + if req.ResumeFromCheckpoint != nil && strings.TrimSpace(*req.ResumeFromCheckpoint) != "" { + return orchestrator.RunFromCheckpoint(ctx, *req.ResumeFromCheckpoint) + } + + app := r.app + nodeID := phases.NodeID() + + app.Note(ctx, "Starting SEC-AF audit pipeline", "audit", "start") + started := nowMonotonic() + + // --- recon_phase -------------------------------------------------------- + reconDict, err := callMap(ctx, app, nodeID, reasoners.NameReconPhase, map[string]any{ + "repo_path": repoPath, + "depth": req.Depth, + }) + if err != nil { + return schemas.SecurityAuditResult{}, err + } + recon, err := phases.BindReconResult(reconDict) + if err != nil { + return schemas.SecurityAuditResult{}, err + } + recon.ReconDurationSeconds = nowMonotonic().Sub(started).Seconds() + if err := orchestrator.WriteCheckpoint(orch.PhaseRecon, recon); err != nil { + return schemas.SecurityAuditResult{}, err + } + + // --- hunt_phase --------------------------------------------------------- + // Python parity: the kwarg is `recon_context=recon_dict` — the RAW payload + // map returned by recon_phase, NOT recon.model_dump(). The two differ: + // recon_duration_seconds was just stamped on the MODEL and is still 0.0 in + // the dict that is passed on. + huntDict, err := callMap(ctx, app, nodeID, reasoners.NameHuntPhase, map[string]any{ + "repo_path": repoPath, + "recon_context": reconDict, + "depth": req.Depth, + }) + if err != nil { + return schemas.SecurityAuditResult{}, err + } + hunt, err := phases.BindHuntResult(huntDict) + if err != nil { + return schemas.SecurityAuditResult{}, err + } + // Python: `time.monotonic() - started - recon.recon_duration_seconds`, i.e. + // the hunt's own share of the elapsed time. + hunt.HuntDurationSeconds = nowMonotonic().Sub(started).Seconds() - recon.ReconDurationSeconds + if err := orchestrator.WriteCheckpoint(orch.PhaseHunt, hunt); err != nil { + return schemas.SecurityAuditResult{}, err + } + + // --- prove_phase -------------------------------------------------------- + // Python parity: here the kwarg IS `hunt.model_dump()` (the model, with the + // duration stamped), unlike recon_context above. + huntDump, err := afx.ToMap(hunt) + if err != nil { + return schemas.SecurityAuditResult{}, err + } + proveDict, err := callMap(ctx, app, nodeID, reasoners.NameProvePhase, map[string]any{ + "repo_path": repoPath, + "hunt_result": huntDump, + "depth": req.Depth, + "max_provers": req.MaxProvers, + }) + if err != nil { + return schemas.SecurityAuditResult{}, err + } + + // Python: `prove_dict["verified"]` — a SUBSCRIPT, so a payload without the + // key raises KeyError (an Exception, not a ValueError => 500). + verifiedRaw, ok := proveDict["verified"] + if !ok { + return schemas.SecurityAuditResult{}, &missingKeyError{Key: "verified", Source: reasoners.NameProvePhase} + } + verified, err := bindVerifiedList(verifiedRaw) + if err != nil { + return schemas.SecurityAuditResult{}, err + } + + orchestrator.FindingsNotVerified = mapInt(proveDict, "not_verified", 0) + // Python: `prove_dict.get("drop_summary", {"demoted_total": 0, "by_reason": {}, "findings": []})` + // (app.py:205-208) — a `.get`, so the default fires ONLY when the key is + // ABSENT. A key that is PRESENT with an odd value (a JSON null, a string, a + // list, a number) is threaded through verbatim and surfaces in the audit + // result's `metadata.prove_drop_summary`, which is `dict[str, object]` and + // accepts anything. + // + // afx.WireNumbers restores the int-vs-float split CPython's json.loads + // makes: the summary is stored UNTYPED and re-serialised into + // `metadata["prove_drop_summary"]`, where a float64 2 would print "2.0" + // against Python's "2". + if summary, present := proveDict["drop_summary"]; present { + orchestrator.ProveDropSummary = afx.WireNumbers(summary) + } else { + orchestrator.ProveDropSummary = orch.NewDropSummary() + } + + if err := orchestrator.WriteCheckpoint(orch.PhaseProve, verified); err != nil { + return schemas.SecurityAuditResult{}, err + } + + // --- remediation_phase -------------------------------------------------- + verifiedDumps := make([]any, 0, len(verified)) + for i := range verified { + dump, dumpErr := afx.ToMap(verified[i]) + if dumpErr != nil { + return schemas.SecurityAuditResult{}, dumpErr + } + verifiedDumps = append(verifiedDumps, dump) + } + remediationDict, err := callMap(ctx, app, nodeID, reasoners.NameRemediationPhase, map[string]any{ + "repo_path": repoPath, + "verified_findings": verifiedDumps, + }) + if err != nil { + return schemas.SecurityAuditResult{}, err + } + remediatedRaw, ok := remediationDict["verified"] + if !ok { + return schemas.SecurityAuditResult{}, &missingKeyError{Key: "verified", Source: reasoners.NameRemediationPhase} + } + verified, err = bindVerifiedList(remediatedRaw) + if err != nil { + return schemas.SecurityAuditResult{}, err + } + + // Python: total_selected + len(hunt.strategies_run) + 3. The "+3" counts the + // three phase reasoners that are not per-finding fan-outs (recon, hunt, + // prove); remediation is not counted, and neither is `audit` itself. + orchestrator.SetAgentInvocations(mapInt(proveDict, "total_selected", 0) + len(hunt.StrategiesRun) + 3) + + result, err := orchestrator.GenerateOutput(ctx, recon, hunt, verified) + if err != nil { + return schemas.SecurityAuditResult{}, err + } + + app.Note(ctx, "SEC-AF audit complete", "audit", "complete") + return result, nil +} + +// ErrBadInput marks a ValueError-class failure raised INSIDE the pipeline, +// i.e. one Python maps to +// +// except ValueError as exc: raise HTTPException(400, detail={"error": str(exc)}) +// +// Wrap an error with %w against this sentinel to have IsBadInput classify it. +var ErrBadInput = errors.New("bad input") + +// IsBadInput reports whether err is one of the ValueError-class failures the +// audit pipeline can raise from inside the mapped region. +// +// THREE families reach it, all of them ValueError subclasses in Python: +// +// 1. `ValueError(f"Unknown checkpoint phase: {phase}")` from +// orchestrator.run_from_checkpoint -> *orch.UnknownCheckpointPhaseError. +// 2. `Model.model_validate(payload)` / `Model(**payload)` anywhere inside the +// try — app.py:181 ReconResult, :191 HuntResult, :203 and :217 +// VerifiedFinding, plus orchestrator.py's `_read_checkpoint`. pydantic's +// ValidationError SUBCLASSES ValueError, so EVERY schema failure inside the +// try takes the 400 branch, not the 500 one -> *phases.ValidationError. +// 3. `json.loads` on a corrupt checkpoint file: `json.JSONDecodeError` is also +// a ValueError -> encoding/json's *SyntaxError / *UnmarshalTypeError, which +// are what orch.ReadCheckpoint / ReadCheckpointList and afx.Bind return for +// the same input. +// +// A failure INSIDE a phase reasoner does not reach this classification with its +// own type: the in-process Call reports it as an opaque *CallError, exactly as +// the control-plane hop reported a failed child execution, so it is a 500. +func IsBadInput(err error) bool { + var unknownPhase *orch.UnknownCheckpointPhaseError + if errors.As(err, &unknownPhase) { + return true + } + var validation *phases.ValidationError + if errors.As(err, &validation) { + return true + } + // json.JSONDecodeError and pydantic's coercion failures are both ValueError + // subclasses; encoding/json reports the same two conditions as these types. + var syntaxErr *json.SyntaxError + if errors.As(err, &syntaxErr) { + return true + } + var typeErr *json.UnmarshalTypeError + if errors.As(err, &typeErr) { + return true + } + return errors.Is(err, ErrBadInput) +} + +// callMap is app.py's +// +// raw = await app.call(f"{NODE_ID}.{name}", **kwargs) +// payload = _as_dict(_unwrap(raw, name), name) +// +// app.py declares its own byte-identical copies of _unwrap/_as_dict alongside +// reasoners/phases.py's; afx owns the single Go implementation. +func callMap(ctx context.Context, app appx.Caller, nodeID, name string, input map[string]any) (map[string]any, error) { + raw, err := app.Call(ctx, nodeID+"."+name, input) + if err != nil { + return nil, err + } + payload, err := afx.Unwrap(raw, name) + if err != nil { + return nil, err + } + return afx.AsMap(payload, name) +} + +// bindVerifiedList is `[VerifiedFinding.model_validate(v) for v in payload]` +// (app.py:201 and :213). +// +// The comprehension has TWO distinct failure modes, and Python maps them to +// DIFFERENT audit responses, so they are kept apart: +// +// - the payload is not ITERABLE -> TypeError, an Exception but not a +// ValueError, so audit()'s `except Exception` answers 500. str(exc) is +// `'int' object is not iterable`, which notIterableError reproduces. +// - the payload iterates but an ELEMENT is not a mapping -> +// `VerifiedFinding.model_validate(5)` raises a pydantic ValidationError, +// which SUBCLASSES ValueError, so audit() answers 400 with +// `Input should be a valid dictionary or instance of VerifiedFinding`. +// A *phases.ValidationError is what IsBadInput routes to that branch. +// +// Iterability follows Python, not Go: a STRING iterates its characters and a +// DICT iterates its keys, so both reach the element branch (and an empty dict +// or empty string yields an empty list, not an error) — only the scalars and +// None are "not iterable". +func bindVerifiedList(payload any) ([]schemas.VerifiedFinding, error) { + items, ok := pyIterate(payload) + if !ok { + return nil, ¬IterableError{Got: afx.PyTypeName(payload)} + } + out := make([]schemas.VerifiedFinding, 0, len(items)) + for _, item := range items { + row, isMap := item.(map[string]any) + if !isMap { + return nil, &phases.ValidationError{ + Model: "VerifiedFinding", + Errors: []string{"Input should be a valid dictionary or instance of VerifiedFinding"}, + } + } + finding, err := phases.BindVerifiedFinding(row) + if err != nil { + return nil, err + } + out = append(out, finding) + } + return out, nil +} + +// pyIterate is `list(x)` for the JSON value kinds a `.call` payload can hold: +// a list yields its elements, a string its CHARACTERS, a dict its KEYS, and +// everything else (numbers, booleans, None) is not iterable. +func pyIterate(payload any) ([]any, bool) { + switch v := payload.(type) { + case []any: + return v, true + case string: + out := make([]any, 0, len(v)) + for _, r := range v { + out = append(out, string(r)) + } + return out, true + case map[string]any: + // Python iterates a dict in INSERTION order, which a decoded Go map + // does not carry; sorting keeps the walk deterministic. The order is + // unobservable here anyway — every key is a string, so whichever comes + // first fails the element branch identically. + keys := make([]string, 0, len(v)) + for key := range v { + keys = append(keys, key) + } + sort.Strings(keys) + out := make([]any, 0, len(keys)) + for _, key := range keys { + out = append(out, key) + } + return out, true + } + return nil, false +} + +// mapInt is `payload.get(key, def)` coerced to an int. +// +// A JSON number arrives as float64 after the JSON hop and as an int when a Go +// caller built the map in process, so both are accepted; anything else +// (including an absent key) yields def. Python would happily store a float in +// findings_not_verified / agent_invocations; the Go fields are typed int, and +// every producer of these keys emits an integer. +func mapInt(payload map[string]any, key string, def int) int { + switch v := payload[key].(type) { + case int: + return v + case int64: + return int(v) + case float64: + return int(v) + case float32: + return int(v) + } + return def +} + +// missingKeyError stands in for Python's KeyError on `prove_dict["verified"]` +// (app.py:201) and `remediation_dict["verified"]` (app.py:213). +// +// The TEXT is `str(exc)`, not `repr(exc)`: app.py:229-230 interpolates the +// exception into `f"Audit pipeline failed: {exc}"` and +// `f"audit execution failed: {exc}"`, and `str(KeyError('verified'))` is +// `'verified'` — the repr of the KEY, with quotes and with no class-name +// prefix. Source is kept as a field because it is useful in tests and logs, but +// it must not appear in Error(). +type missingKeyError struct { + Key string + Source string +} + +func (e *missingKeyError) Error() string { + return "'" + e.Key + "'" +} + +// notIterableError stands in for the TypeError a non-iterable `verified` +// produces. `str(exc)` is `'int' object is not iterable` — again no class-name +// prefix. +// +// Documented residual, inherited from afx.PyTypeName: Go's encoding/json +// decodes every JSON number to float64, so an integral payload says "float" +// where CPython's json.loads would have said "int". +type notIterableError struct{ Got string } + +func (e *notIterableError) Error() string { + return "'" + e.Got + "' object is not iterable" +} diff --git a/internal/secaf/changes.go b/internal/secaf/changes.go new file mode 100644 index 0000000000..fb5e5edcc3 --- /dev/null +++ b/internal/secaf/changes.go @@ -0,0 +1,192 @@ +package secaf + +// What changed, for an audit of the changes. +// +// sec-af had a PR mode that never ran: it read the diff from the process's own +// folder rather than the repository under audit, and on the path every audit +// took it was handed nothing. This is that mode made to work in a person's +// folder: the changes are the branch's commits since it left its base AND the +// edits not yet committed AND the files not yet tracked — the work as it +// stands — and the files near them (sec-af's own "blast radius": the files +// that name a changed file's module) come along, because a change is unsafe +// where its callers meet it. +// +// IT READS GIT AND WRITES NOTHING. Every command here is a read; the audit +// promised the folder back as it found it. + +import ( + "bytes" + "context" + "errors" + "fmt" + "os/exec" + "sort" + "strings" + "time" + + "github.com/Agent-Field/codeaf/internal/secaf/diffanalysis" + "github.com/Agent-Field/codeaf/internal/secaf/focus" +) + +// Changes is the work an audit of the changes looks at. +type Changes struct { + // Base is the commit the changes are measured from, and BaseName how the + // person would say it (`origin/dev`, `main`, the ref they typed). + Base, BaseName string + // Changed is every scannable file that differs from the base; Nearby is + // the files that name one of them; Relevant is both, sorted. + Changed, Nearby, Relevant []string + // Diff is the change itself, cut at [diffBytes]. + Diff string +} + +const ( + // gitWall bounds one git command. + gitWall = 60 * time.Second + // diffBytes is how much of the diff the hunters are shown. The files are + // theirs to read whole; the diff only says where to look. + diffBytes = 48 << 10 + // nearbyFiles bounds the files near the change, so a change to a module + // every file imports does not turn an audit of the changes into an audit + // of everything. + nearbyFiles = 120 +) + +// errNoChanges is an audit of the changes with nothing changed. +var errNoChanges = errors.New("nothing has changed") + +// ReadChanges reads what changed in repo since base, or since the branch's own +// base when base is empty. +func ReadChanges(ctx context.Context, repo, base string) (Changes, error) { + if _, err := git(ctx, repo, "rev-parse", "--verify", "--quiet", "HEAD^{commit}"); err != nil { + return Changes{}, errors.New("an audit of the changes needs a git repository with at least one commit; audit the whole repository instead") + } + changes := Changes{} + if strings.TrimSpace(base) != "" { + commit, err := git(ctx, repo, "rev-parse", "--verify", "--quiet", base+"^{commit}") + if err != nil { + return Changes{}, fmt.Errorf("%s names no commit in this repository", base) + } + changes.Base, changes.BaseName = strings.TrimSpace(commit), base + } else { + changes.Base, changes.BaseName = findBase(ctx, repo) + } + tracked, err := git(ctx, repo, "diff", "--name-only", "--diff-filter=ACMR", changes.Base) + if err != nil { + return Changes{}, fmt.Errorf("read the changes since %s: %w", changes.BaseName, err) + } + untracked, _ := git(ctx, repo, "ls-files", "--others", "--exclude-standard") + seen := map[string]bool{} + for _, file := range append(lines(tracked), lines(untracked)...) { + if file != "" && !seen[file] && diffanalysis.IsScannable(file) { + seen[file] = true + changes.Changed = append(changes.Changed, file) + } + } + sort.Strings(changes.Changed) + if len(changes.Changed) == 0 { + return changes, fmt.Errorf("%w since %s in a file this audit reads (code, not docs or config)", errNoChanges, changes.BaseName) + } + changes.Nearby = nearby(ctx, repo, changes.Changed, seen) + changes.Relevant = append(append([]string(nil), changes.Changed...), changes.Nearby...) + sort.Strings(changes.Relevant) + diff, _ := git(ctx, repo, append([]string{"diff", changes.Base, "--"}, changes.Changed...)...) + if len(diff) > diffBytes { + diff = diff[:diffBytes] + "\n[the diff goes on; read the changed files themselves for the rest]\n" + } + changes.Diff = diff + return changes, nil +} + +// findBase is where this branch left its trunk: the merge base of HEAD with +// the remote's default branch, else with a local trunk by its usual names, and +// HEAD itself when the branch is the trunk — whose changes are then the edits +// not yet committed. +func findBase(ctx context.Context, repo string) (string, string) { + var candidates []string + if ref, err := git(ctx, repo, "symbolic-ref", "--quiet", "--short", "refs/remotes/origin/HEAD"); err == nil { + candidates = append(candidates, strings.TrimSpace(ref)) + } + candidates = append(candidates, "origin/main", "origin/master", "origin/dev", "main", "master", "dev", "trunk") + head, _ := git(ctx, repo, "rev-parse", "HEAD") + head = strings.TrimSpace(head) + for _, candidate := range candidates { + if candidate == "" { + continue + } + if _, err := git(ctx, repo, "rev-parse", "--verify", "--quiet", candidate+"^{commit}"); err != nil { + continue + } + base, err := git(ctx, repo, "merge-base", "HEAD", candidate) + if err != nil || strings.TrimSpace(base) == "" { + continue + } + return strings.TrimSpace(base), candidate + } + return head, "the last commit" +} + +// nearby is the files that name a changed file's module, sec-af's own reading +// of a change's reach (diffanalysis.FileToModule), searched in the working +// tree so files not yet committed count. +func nearby(ctx context.Context, repo string, changed []string, skip map[string]bool) []string { + found := map[string]bool{} + for _, file := range changed { + module := diffanalysis.FileToModule(file) + if module == "" { + continue + } + out, err := git(ctx, repo, "grep", "-l", "--untracked", "--fixed-strings", "-e", module, + "--", "*.py", "*.ts", "*.js", "*.go", "*.java", "*.rb", "*.php", "*.cs", "*.kt", "*.rs") + if err != nil { + continue + } + for _, hit := range lines(out) { + if hit != "" && !skip[hit] && diffanalysis.IsScannable(hit) { + found[hit] = true + } + } + } + out := make([]string, 0, len(found)) + for file := range found { + out = append(out, file) + } + sort.Strings(out) + if len(out) > nearbyFiles { + out = out[:nearbyFiles] + } + return out +} + +// git runs one read-only git command in repo. A command that ran and said no +// (a grep with no match, a ref that is not there) is an error with its exit. +func git(ctx context.Context, repo string, args ...string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, gitWall) + defer cancel() + command := exec.CommandContext(ctx, "git", append([]string{"-C", repo, "--no-optional-locks", "-c", "core.quotePath=false"}, args...)...) + command.Env = append(command.Environ(), "GIT_TERMINAL_PROMPT=0", "GIT_OPTIONAL_LOCKS=0") + var stdout, stderr bytes.Buffer + command.Stdout, command.Stderr = &stdout, &stderr + if err := command.Run(); err != nil { + if message := strings.TrimSpace(stderr.String()); message != "" { + return stdout.String(), fmt.Errorf("%v: %s", err, message) + } + return stdout.String(), err + } + return stdout.String(), nil +} + +func lines(text string) []string { + var out []string + for _, line := range strings.Split(text, "\n") { + if line = strings.TrimSpace(line); line != "" { + out = append(out, line) + } + } + return out +} + +// withFocus is ctx carrying the change to the hunters (internal/secaf/focus). +func withFocus(ctx context.Context, changes Changes) context.Context { + return focus.With(ctx, focus.Focus{Base: changes.BaseName, Changed: changes.Changed, Nearby: changes.Nearby, Diff: changes.Diff}) +} diff --git a/internal/secaf/compliance/mapping.go b/internal/secaf/compliance/mapping.go new file mode 100644 index 0000000000..33289b38ff --- /dev/null +++ b/internal/secaf/compliance/mapping.go @@ -0,0 +1,604 @@ +// Package compliance ports src/sec_af/compliance/mapping.py: the static +// CWE -> framework-control table SEC-AF scores findings against, the lookup and +// normalization helpers around it, the AI fallback for CWEs the table does not +// cover, and the gap aggregation the audit result reports. +// +// The table itself lives in table_gen.go, mechanically derived from the Python +// module by scripts/gen_compliance_table.py — see that file. +package compliance + +import ( + "context" + "fmt" + "reflect" + "sort" + "strings" + "sync" + "unicode" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// severityRank ports the module-level _SEVERITY_RANK table. A severity that is +// not a key ranks 0 (Python's `.get(x, 0)`), which is what makes the +// `str(Severity.HIGH)` quirk in GetComplianceGaps observable — see +// severityKey. +var severityRank = map[string]int{ + "critical": 5, + "high": 4, + "medium": 3, + "low": 2, + "info": 1, +} + +// defaultFrameworks ports _DEFAULT_FRAMEWORKS: the framework list named in the +// AI-fallback prompt when the caller did not restrict the frameworks. +var defaultFrameworks = []string{"OWASP", "PCI-DSS", "SOC2", "HIPAA", "ISO27001"} + +// --------------------------------------------------------------------------- +// AI gate seam +// --------------------------------------------------------------------------- + +// AIGateLike mirrors mapping.py's `_AIGateLike` Protocol, narrowed to the one +// call GetComplianceMappingsHybrid actually makes: +// +// suggestion = await ai_gate.invoke(user=prompt, schema=ComplianceGate) +// +// Python's Protocol is generic over the pydantic model because AIGateWrapper +// serves every gate; a Go method cannot carry a type parameter, so the seam is +// specialised to ComplianceGate here. That also keeps this package free of any +// dependency on internal/gates (which in turn depends on aix, harnessx and the +// SDK) — the production wiring adapts the real gate with AIGateFunc: +// +// compliance.GetComplianceMappingsHybrid(ctx, cwe, frameworks, +// compliance.AIGateFunc(func(ctx context.Context, user string) (schemas.ComplianceGate, error) { +// return gates.Invoke[schemas.ComplianceGate](ctx, gate, user, "") +// })) +// +// A nil AIGateLike is Python's `ai_gate=None`: no fallback, empty result. +type AIGateLike interface { + // InvokeComplianceGate runs the gate with the compliance prompt and + // returns the parsed ComplianceGate. An error is Python's raised + // exception: GetComplianceMappingsHybrid swallows it and returns no + // mappings, exactly like the `except Exception: return []` in mapping.py. + InvokeComplianceGate(ctx context.Context, user string) (schemas.ComplianceGate, error) +} + +// AIGateFunc adapts a plain function to AIGateLike. +type AIGateFunc func(ctx context.Context, user string) (schemas.ComplianceGate, error) + +// InvokeComplianceGate implements AIGateLike. +func (f AIGateFunc) InvokeComplianceGate(ctx context.Context, user string) (schemas.ComplianceGate, error) { + return f(ctx, user) +} + +// --------------------------------------------------------------------------- +// normalization +// --------------------------------------------------------------------------- + +// normalizeCweID ports _normalize_cwe_id: +// +// raw = cwe_id.strip().upper() +// if raw.startswith("CWE-"): return raw +// if raw.startswith("CWE"): return f"CWE-{raw[3:]}" +// return f"CWE-{raw}" +// +// so "89", "cwe89", " CWE-89 " and "CWE89" all normalize to "CWE-89". +// +// Python parity note: `raw[3:]` slices CHARACTERS while Go slices bytes, but +// the slice only runs after an ASCII "CWE" prefix match, where the two agree. +// `str.upper()` is Unicode-aware in Python and can lengthen a string (ß -> SS) +// where strings.ToUpper does not; no CWE identifier contains such a character. +func normalizeCweID(cweID string) string { + raw := strings.ToUpper(pyStrip(cweID)) + if strings.HasPrefix(raw, "CWE-") { + return raw + } + if strings.HasPrefix(raw, "CWE") { + return "CWE-" + raw[3:] + } + return "CWE-" + raw +} + +// normalizeFramework ports _normalize_framework: +// +// framework.strip().lower().replace("_", "-") +// +// It is the comparison key for framework filtering, so "PCI-DSS", "pci-dss" +// and "pci_dss" all select the same controls. +func normalizeFramework(framework string) string { + return strings.ReplaceAll(strings.ToLower(pyStrip(framework)), "_", "-") +} + +// pyStrip reproduces Python's str.strip(): it trims every character for which +// `str.isspace()` is true. That is Go's unicode.IsSpace plus the four +// information separators U+001C..U+001F, which Go does not classify as space. +func pyStrip(s string) string { + return strings.TrimFunc(s, pyIsSpace) +} + +// pyIsSpace reports whether r is whitespace to Python (`str.isspace()`). +func pyIsSpace(r rune) bool { + if r >= 0x1C && r <= 0x1F { + return true + } + return unicode.IsSpace(r) +} + +// --------------------------------------------------------------------------- +// static lookup +// --------------------------------------------------------------------------- + +// GetComplianceMappings ports get_compliance_mappings: the static table lookup +// for one CWE, optionally restricted to a set of frameworks. +// +// Passing a nil or empty frameworks slice is Python's `frameworks=None` / +// `frameworks=[]` — both are falsy, so both mean "no filter". +// +// The returned slice is always freshly allocated and never aliases the table, +// which is what Python's `mapping.model_copy(deep=True)` buys: a caller that +// mutates a returned ComplianceMapping cannot corrupt the table for the next +// caller. ComplianceMapping is a struct of three strings, so a slice copy IS a +// deep copy. +// +// An unmapped CWE yields an empty (non-nil) slice, matching Python's `[]`. +func GetComplianceMappings(cweID string, frameworks []string) []schemas.ComplianceMapping { + normalized := normalizeCweID(cweID) + mappings := ComplianceMap[normalized] + + if len(frameworks) == 0 { + out := make([]schemas.ComplianceMapping, len(mappings)) + copy(out, mappings) + return out + } + + allowed := normalizedFrameworkSet(frameworks) + out := make([]schemas.ComplianceMapping, 0, len(mappings)) + for _, mapping := range mappings { + if _, ok := allowed[normalizeFramework(mapping.Framework)]; ok { + out = append(out, mapping) + } + } + return out +} + +// normalizedFrameworkSet builds Python's `{_normalize_framework(f) for f in frameworks}`. +func normalizedFrameworkSet(frameworks []string) map[string]struct{} { + allowed := make(map[string]struct{}, len(frameworks)) + for _, framework := range frameworks { + allowed[normalizeFramework(framework)] = struct{}{} + } + return allowed +} + +// GetSupportedFrameworks ports get_supported_frameworks: +// +// sorted({m.framework for ms in COMPLIANCE_MAP.values() for m in ms}) +// +// i.e. ["HIPAA", "ISO27001", "OWASP", "PCI-DSS", "SOC2"]. Python sorts strings +// by code point; Go's sort.Strings sorts by UTF-8 byte order, which is the same +// ordering for valid UTF-8. +func GetSupportedFrameworks() []string { + seen := map[string]struct{}{} + for _, mappings := range ComplianceMap { + for _, mapping := range mappings { + seen[mapping.Framework] = struct{}{} + } + } + out := make([]string, 0, len(seen)) + for framework := range seen { + out = append(out, framework) + } + sort.Strings(out) + return out +} + +// --------------------------------------------------------------------------- +// AI fallback + cache +// --------------------------------------------------------------------------- + +// aiCacheKey is the Go form of Python's cache key tuple +// `(normalized_cwe, tuple(sorted({normalized frameworks})) | None)`. +// +// A Go map key must be comparable, so the framework tuple is flattened into a +// single string joined on NUL (which cannot occur in a framework name) and the +// `None` case is carried by a separate bool rather than by the empty string — +// otherwise ("CWE-1", None) and ("CWE-1", ()) would collide, and Python keeps +// them distinct. +type aiCacheKey struct { + cwe string + frameworks string + hasFrameworks bool +} + +// aiComplianceCache ports the module-level _AI_COMPLIANCE_CACHE dict. +// +// Python's dict is safe by accident (the module is only ever driven from a +// single event loop); Go reasoners run genuinely in parallel, so the map is +// guarded by a mutex. The lock is deliberately NOT held across the gate call: +// that keeps a slow LLM round-trip from serialising every other CWE, at the +// cost of reproducing Python's behaviour where two concurrent misses for the +// same key both invoke the gate and the last writer wins. +var ( + aiComplianceCacheMu sync.Mutex + aiComplianceCache = map[aiCacheKey][]schemas.ComplianceMapping{} +) + +// ClearAICache empties the AI fallback cache. +// +// It is the port of the Python tests' `mapping._AI_COMPLIANCE_CACHE.clear()`: +// module-level caches leak across test cases, so every hybrid test starts by +// clearing it. Nothing in the production path calls this. +func ClearAICache() { + aiComplianceCacheMu.Lock() + defer aiComplianceCacheMu.Unlock() + aiComplianceCache = map[aiCacheKey][]schemas.ComplianceMapping{} +} + +// GetComplianceMappingsHybrid ports get_compliance_mappings_hybrid: the static +// table first, and only for a CWE the table does not know does it ask the AI +// gate — caching the answer so the same unknown CWE costs one LLM call per +// process. +// +// Python parity, in order: +// +// 1. the static lookup (already framework-filtered) wins whenever it is +// non-empty — the gate is not even constructed; +// 2. no gate (Python's `ai_gate is None`, Go's nil interface) yields []; +// 3. the cache is keyed by (normalized CWE, sorted normalized framework set or +// None) and stores the ALREADY FILTERED list; +// 4. a gate failure yields [] and is NOT cached, so a transient error does not +// poison the CWE for the rest of the run. Python swallows the exception +// (`except Exception: return []`) and so does this function — which is why +// it returns no error: any diagnostics belong in the AIGateLike +// implementation, where the caller still has the error in hand. +// +// Python parity quirk kept deliberately: the prompt interpolates +// `cwe_description = "Unknown CWE"` — a local constant that is never derived +// from the CWE — so every fallback prompt says "(Unknown CWE)". +func GetComplianceMappingsHybrid( + ctx context.Context, + cweID string, + frameworks []string, + aiGate AIGateLike, +) []schemas.ComplianceMapping { + normalizedCwe := normalizeCweID(cweID) + if cached := GetComplianceMappings(normalizedCwe, frameworks); len(cached) > 0 { + return cached + } + if aiGate == nil { + return []schemas.ComplianceMapping{} + } + + key := aiCacheKey{cwe: normalizedCwe} + frameworkList := append([]string(nil), defaultFrameworks...) + if len(frameworks) > 0 { + key.hasFrameworks = true + key.frameworks = sortedFrameworkTuple(frameworks) + frameworkList = append([]string(nil), frameworks...) + } + + if hit, ok := loadAICache(key); ok { + return hit + } + + prompt := complianceGatePrompt(normalizedCwe, frameworkList) + + suggestion, err := aiGate.InvokeComplianceGate(ctx, prompt) + if err != nil { + // Python parity: `except Exception: return []` — not cached. + return []schemas.ComplianceMapping{} + } + + aiMappings := make([]schemas.ComplianceMapping, 0, len(suggestion.Mappings)) + for _, item := range suggestion.Mappings { + aiMappings = append(aiMappings, schemas.ComplianceMapping{ + Framework: item.Framework, + ControlID: item.ControlID, + ControlName: item.ControlName, + }) + } + if len(frameworks) > 0 { + allowed := normalizedFrameworkSet(frameworks) + filtered := make([]schemas.ComplianceMapping, 0, len(aiMappings)) + for _, mapping := range aiMappings { + if _, ok := allowed[normalizeFramework(mapping.Framework)]; ok { + filtered = append(filtered, mapping) + } + } + aiMappings = filtered + } + + storeAICache(key, aiMappings) + out := make([]schemas.ComplianceMapping, len(aiMappings)) + copy(out, aiMappings) + return out +} + +// complianceGatePrompt builds the AI-fallback prompt byte-for-byte as +// mapping.py does. +// +// Python parity: the `", ".join(framework_list) if framework_list else ...` +// fallback is unreachable — framework_list is either the caller's non-empty +// list or _DEFAULT_FRAMEWORKS — but it is reproduced so the two functions read +// the same. +func complianceGatePrompt(normalizedCwe string, frameworkList []string) string { + const cweDescription = "Unknown CWE" + frameworkPrompt := strings.Join(frameworkList, ", ") + if len(frameworkList) == 0 { + frameworkPrompt = strings.Join(defaultFrameworks, ", ") + } + return fmt.Sprintf( + "Map %s (%s) to compliance framework controls. Frameworks: %s. Return specific control IDs.", + normalizedCwe, cweDescription, frameworkPrompt, + ) +} + +// sortedFrameworkTuple renders `tuple(sorted({_normalize_framework(f) for f in frameworks}))` +// as a NUL-joined string usable as a Go map key. +func sortedFrameworkTuple(frameworks []string) string { + set := normalizedFrameworkSet(frameworks) + names := make([]string, 0, len(set)) + for name := range set { + names = append(names, name) + } + sort.Strings(names) + return strings.Join(names, "\x00") +} + +// loadAICache returns a fresh copy of the cached mappings for key, matching +// Python's `[m.model_copy(deep=True) for m in cache[key]]`. +func loadAICache(key aiCacheKey) ([]schemas.ComplianceMapping, bool) { + aiComplianceCacheMu.Lock() + defer aiComplianceCacheMu.Unlock() + cached, ok := aiComplianceCache[key] + if !ok { + return nil, false + } + out := make([]schemas.ComplianceMapping, len(cached)) + copy(out, cached) + return out, true +} + +// storeAICache stores a private copy of mappings under key. +func storeAICache(key aiCacheKey, mappings []schemas.ComplianceMapping) { + stored := make([]schemas.ComplianceMapping, len(mappings)) + copy(stored, mappings) + aiComplianceCacheMu.Lock() + defer aiComplianceCacheMu.Unlock() + aiComplianceCache[key] = stored +} + +// --------------------------------------------------------------------------- +// gap aggregation +// --------------------------------------------------------------------------- + +// gapKey is Python's aggregation key tuple +// `(mapping.framework, mapping.control_id, mapping.control_name)`. +type gapKey struct { + framework string + controlID string + controlName string +} + +// gapEntry is the mutable accumulator Python keeps in the aggregation dict. +type gapEntry struct { + count int + maxSeverity string + cweIDs []string +} + +// GetComplianceGaps ports get_compliance_gaps: it walks findings, maps each +// one's CWE through the static table, and aggregates per framework control how +// many findings hit it, which CWEs, and the worst severity among them. +// +// The parameter is generic so the two Python call shapes both work unchanged: +// `[]schemas.VerifiedFinding` (what the orchestrator passes) and +// `[]any` / `[]map[string]any` (what the tests pass). Field access is the port +// of `_read_field`, i.e. Python's duck typing — see readField. +// +// Findings without a truthy `cwe_id`, and findings whose CWE the table does not +// know, are skipped (Python `continue`s on both). +// +// The result is sorted by (framework, control_id, control_name), so it is fully +// deterministic even though the aggregation itself runs over a Go map. +func GetComplianceGaps[T any](findings []T) []schemas.ComplianceGap { + aggregated := map[gapKey]*gapEntry{} + order := make([]gapKey, 0, len(findings)) + + for _, finding := range findings { + item := any(finding) + cweID := readField(item, "cwe_id") + severity := severityKey(readField(item, "severity")) + if !pyTruthy(cweID) { + continue + } + + normalizedCwe := normalizeCweID(pyStrOfField(cweID)) + mappings := GetComplianceMappings(normalizedCwe, nil) + if len(mappings) == 0 { + continue + } + + for _, mapping := range mappings { + key := gapKey{mapping.Framework, mapping.ControlID, mapping.ControlName} + entry, ok := aggregated[key] + if !ok { + entry = &gapEntry{maxSeverity: "info"} + aggregated[key] = entry + order = append(order, key) + } + + entry.count++ + if !containsString(entry.cweIDs, normalizedCwe) { + entry.cweIDs = append(entry.cweIDs, normalizedCwe) + } + + // Python parity: an unrecognised severity ranks 0, so it can never + // raise max_severity above the "info" seed — see severityKey for + // why a real VerifiedFinding always takes that branch. + if severityRank[severity] > severityRank[entry.maxSeverity] { + entry.maxSeverity = severity + } + } + } + + gaps := make([]schemas.ComplianceGap, 0, len(order)) + for _, key := range order { + entry := aggregated[key] + cweIDs := append([]string(nil), entry.cweIDs...) + sort.Strings(cweIDs) + gaps = append(gaps, schemas.ComplianceGap{ + Framework: key.framework, + ControlID: key.controlID, + ControlName: key.controlName, + FindingCount: entry.count, + MaxSeverity: entry.maxSeverity, + CweIDs: cweIDs, + }) + } + sort.SliceStable(gaps, func(i, j int) bool { + a, b := gaps[i], gaps[j] + if a.Framework != b.Framework { + return a.Framework < b.Framework + } + if a.ControlID != b.ControlID { + return a.ControlID < b.ControlID + } + return a.ControlName < b.ControlName + }) + return gaps +} + +// readField ports _read_field: +// +// if isinstance(finding, dict): return finding.get(field_name) +// return getattr(finding, field_name, None) +// +// A Go map stands in for the dict branch (any string-keyed map, not just +// map[string]any) and a struct for the attribute branch, where the "attribute +// name" is the json tag — the schemas package tags every field with its exact +// pydantic field name, which is also its Python attribute name. A missing key, +// a missing field, or a nil finding all yield nil, Python's None default. +func readField(finding any, fieldName string) any { + if finding == nil { + return nil + } + rv := reflect.ValueOf(finding) + for rv.Kind() == reflect.Pointer || rv.Kind() == reflect.Interface { + if rv.IsNil() { + return nil + } + rv = rv.Elem() + } + + switch rv.Kind() { + case reflect.Map: + if rv.Type().Key().Kind() != reflect.String { + return nil + } + got := rv.MapIndex(reflect.ValueOf(fieldName).Convert(rv.Type().Key())) + if !got.IsValid() { + return nil + } + return got.Interface() + case reflect.Struct: + rt := rv.Type() + for i := 0; i < rt.NumField(); i++ { + field := rt.Field(i) + if !field.IsExported() { + continue + } + name, _, _ := strings.Cut(field.Tag.Get("json"), ",") + if name == "" { + name = field.Name + } + if name == fieldName { + return rv.Field(i).Interface() + } + } + return nil + default: + return nil + } +} + +// severityKey ports `str(_read_field(finding, "severity") or "low").lower()`. +// +// Python parity — the quirk this function exists to preserve: `Severity` is a +// `class Severity(str, Enum)`, and CPython 3.11 renders such a member with +// Enum.__str__, so `str(Severity.HIGH)` is "Severity.HIGH", not "high". The +// audit orchestrator passes real VerifiedFinding objects, so every production +// call produces "severity.high" — a key that is absent from _SEVERITY_RANK and +// therefore ranks 0, which is why every gap the orchestrator emits carries +// max_severity "info". Only the dict-shaped findings the tests pass (plain +// strings) rank at all. Reproducing the quirk is deliberate; do not "fix" it. +func severityKey(value any) string { + if !pyTruthy(value) { + return "low" + } + return strings.ToLower(pyStrOfField(value)) +} + +// pyStrOfField renders `str(x)` for the field values readField can return. +// +// A plain Go string is itself. A schemas.Severity is a Python `(str, Enum)` +// member and stringifies as "Severity."; every member's name is the +// upper-case of its value, so that is how the name is recovered. Anything else +// falls back to fmt.Sprint, which covers the ints and bools a dict-shaped +// finding might carry. +func pyStrOfField(value any) string { + switch x := value.(type) { + case nil: + return "None" + case string: + return x + case schemas.Severity: + return "Severity." + strings.ToUpper(string(x)) + case bool: + if x { + return "True" + } + return "False" + } + return fmt.Sprint(value) +} + +// pyTruthy reproduces `bool(x)` for the value kinds readField returns: nil, +// False, 0, "" and empty containers are falsy, everything else is truthy. +func pyTruthy(value any) bool { + if value == nil { + return false + } + rv := reflect.ValueOf(value) + switch rv.Kind() { + case reflect.Bool: + return rv.Bool() + case reflect.String: + return rv.Len() != 0 + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return rv.Int() != 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return rv.Uint() != 0 + case reflect.Float32, reflect.Float64: + return rv.Float() != 0 + case reflect.Slice, reflect.Array, reflect.Map: + return rv.Len() != 0 + case reflect.Pointer, reflect.Interface: + if rv.IsNil() { + return false + } + return pyTruthy(rv.Elem().Interface()) + } + return true +} + +// containsString reports whether list already holds value. +func containsString(list []string, value string) bool { + for _, item := range list { + if item == value { + return true + } + } + return false +} diff --git a/internal/secaf/compliance/mapping_test.go b/internal/secaf/compliance/mapping_test.go new file mode 100644 index 0000000000..3509a0da4b --- /dev/null +++ b/internal/secaf/compliance/mapping_test.go @@ -0,0 +1,723 @@ +package compliance + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "sort" + "sync" + "testing" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// requiredCWEs ports tests/test_compliance.py::REQUIRED_CWES. +var requiredCWEs = []string{ + "CWE-78", "CWE-79", "CWE-89", "CWE-90", "CWE-91", "CWE-94", "CWE-917", + "CWE-287", "CWE-306", "CWE-352", "CWE-862", "CWE-863", "CWE-326", + "CWE-327", "CWE-328", "CWE-330", "CWE-916", "CWE-840", "CWE-841", + "CWE-200", "CWE-209", "CWE-312", "CWE-319", "CWE-532", "CWE-829", + "CWE-1104", "CWE-16", "CWE-259", "CWE-321", "CWE-798", "CWE-285", + "CWE-346", "CWE-601", "CWE-918", +} + +// fakeAIGate ports tests/test_compliance.py::_FakeAIGate. +type fakeAIGate struct { + mu sync.Mutex + calls int + prompts []string + response schemas.ComplianceGate + err error +} + +func (f *fakeAIGate) InvokeComplianceGate(_ context.Context, user string) (schemas.ComplianceGate, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls++ + f.prompts = append(f.prompts, user) + if f.err != nil { + return schemas.ComplianceGate{}, f.err + } + return f.response, nil +} + +func (f *fakeAIGate) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.calls +} + +// namespaceFinding stands in for the tests' SimpleNamespace findings: an +// object (not a dict) whose attributes carry the finding fields. +type namespaceFinding struct { + CweID string `json:"cwe_id"` + Severity string `json:"severity"` +} + +func frameworkSet(mappings []schemas.ComplianceMapping) map[string]struct{} { + out := map[string]struct{}{} + for _, mapping := range mappings { + out[mapping.Framework] = struct{}{} + } + return out +} + +func sortedKeys(set map[string]struct{}) []string { + out := make([]string, 0, len(set)) + for key := range set { + out = append(out, key) + } + sort.Strings(out) + return out +} + +// --------------------------------------------------------------------------- +// static table +// --------------------------------------------------------------------------- + +// TestKeyCWEsIncludeRequiredFrameworkMappings ports +// test_key_cwes_include_required_framework_mappings (parametrized). +func TestKeyCWEsIncludeRequiredFrameworkMappings(t *testing.T) { + cases := []struct { + cweID string + owaspControl string + }{ + {"CWE-89", "A03:2021"}, + {"CWE-79", "A03:2021"}, + {"CWE-287", "A07:2021"}, + {"CWE-862", "A01:2021"}, + {"CWE-326", "A02:2021"}, + {"CWE-840", "A04:2021"}, + {"CWE-200", "A01:2021"}, + {"CWE-1104", "A06:2021"}, + {"CWE-16", "A05:2021"}, + {"CWE-918", "A10:2021"}, + } + for _, tc := range cases { + t.Run(tc.cweID, func(t *testing.T) { + mappings := GetComplianceMappings(tc.cweID, nil) + frameworks := frameworkSet(mappings) + for _, want := range []string{"PCI-DSS", "SOC2", "OWASP"} { + if _, ok := frameworks[want]; !ok { + t.Fatalf("%s: framework %q missing from %v", tc.cweID, want, sortedKeys(frameworks)) + } + } + found := false + for _, mapping := range mappings { + if mapping.Framework == "OWASP" && mapping.ControlID == tc.owaspControl { + found = true + } + } + if !found { + t.Fatalf("%s: no OWASP mapping with control id %q", tc.cweID, tc.owaspControl) + } + }) + } +} + +// TestAllRequiredCWEsAreMapped ports test_all_required_cwes_are_mapped. +func TestAllRequiredCWEsAreMapped(t *testing.T) { + got := make([]string, 0, len(ComplianceMap)) + for cwe := range ComplianceMap { + got = append(got, cwe) + } + want := append([]string(nil), requiredCWEs...) + sort.Strings(got) + sort.Strings(want) + if !reflect.DeepEqual(got, want) { + t.Fatalf("COMPLIANCE_MAP keys mismatch\n got: %v\nwant: %v", got, want) + } +} + +// TestGetComplianceMappingsHandlesCweNormalization ports +// test_get_compliance_mappings_handles_cwe_normalization. +func TestGetComplianceMappingsHandlesCweNormalization(t *testing.T) { + normalized := GetComplianceMappings("CWE-89", nil) + shorthand := GetComplianceMappings("89", nil) + mixedCase := GetComplianceMappings("cwe89", nil) + + if len(normalized) == 0 { + t.Fatal("CWE-89 has no mappings") + } + if !reflect.DeepEqual(shorthand, normalized) { + t.Fatalf("\"89\" != \"CWE-89\": %v vs %v", shorthand, normalized) + } + if !reflect.DeepEqual(mixedCase, normalized) { + t.Fatalf("\"cwe89\" != \"CWE-89\": %v vs %v", mixedCase, normalized) + } +} + +// TestNormalizeCweID covers _normalize_cwe_id's three branches directly, +// including the whitespace strip Python's `.strip()` performs. +func TestNormalizeCweID(t *testing.T) { + cases := map[string]string{ + "CWE-89": "CWE-89", + "cwe-89": "CWE-89", + " CWE-89 ": "CWE-89", + "CWE89": "CWE-89", + "cwe89": "CWE-89", + "89": "CWE-89", + "\t89\n": "CWE-89", + "": "CWE-", + "CWE": "CWE-", + "CWE-": "CWE-", + } + for input, want := range cases { + if got := normalizeCweID(input); got != want { + t.Errorf("normalizeCweID(%q) = %q, want %q", input, got, want) + } + } +} + +// TestNormalizeFramework covers _normalize_framework. +func TestNormalizeFramework(t *testing.T) { + cases := map[string]string{ + "PCI-DSS": "pci-dss", + "pci_dss": "pci-dss", + " SOC2 ": "soc2", + "ISO_27001": "iso-27001", + } + for input, want := range cases { + if got := normalizeFramework(input); got != want { + t.Errorf("normalizeFramework(%q) = %q, want %q", input, got, want) + } + } +} + +// TestGetComplianceMappingsCanFilterFrameworks ports +// test_get_compliance_mappings_can_filter_frameworks. +func TestGetComplianceMappingsCanFilterFrameworks(t *testing.T) { + filtered := GetComplianceMappings("CWE-319", []string{"pci-dss", "owasp"}) + got := sortedKeys(frameworkSet(filtered)) + want := []string{"OWASP", "PCI-DSS"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("frameworks = %v, want %v", got, want) + } +} + +// TestGetComplianceMappingsFrameworkFilterWithUnknownFramework ports +// test_get_compliance_mappings_framework_filter_with_unknown_framework. +func TestGetComplianceMappingsFrameworkFilterWithUnknownFramework(t *testing.T) { + filtered := GetComplianceMappings("CWE-89", []string{"owasp", "does-not-exist"}) + got := sortedKeys(frameworkSet(filtered)) + want := []string{"OWASP"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("frameworks = %v, want %v", got, want) + } +} + +// TestGetComplianceMappingsReturnsEmptyForUnmappedCwe ports +// test_get_compliance_mappings_returns_empty_for_unmapped_cwe. The Python +// assertion is `== []`, so the Go result must be empty AND non-nil. +func TestGetComplianceMappingsReturnsEmptyForUnmappedCwe(t *testing.T) { + got := GetComplianceMappings("CWE-9999", nil) + if got == nil { + t.Fatal("want an empty non-nil slice (Python returns []), got nil") + } + if len(got) != 0 { + t.Fatalf("want no mappings, got %v", got) + } +} + +// TestGetComplianceMappingsReturnsDeepCopies ports +// test_get_compliance_mappings_returns_deep_copies: mutating one result must +// not be visible to the next caller, nor corrupt the table. +func TestGetComplianceMappingsReturnsDeepCopies(t *testing.T) { + first := GetComplianceMappings("CWE-89", nil) + second := GetComplianceMappings("CWE-89", nil) + + first[0].ControlName = "mutated" + + if second[0].ControlName == "mutated" { + t.Fatal("second call observed the first call's mutation") + } + if ComplianceMap["CWE-89"][0].ControlName == "mutated" { + t.Fatal("the static table itself was mutated") + } +} + +// TestGetSupportedFrameworksReturnsExpectedSet ports +// test_get_supported_frameworks_returns_expected_set. +func TestGetSupportedFrameworksReturnsExpectedSet(t *testing.T) { + want := []string{"HIPAA", "ISO27001", "OWASP", "PCI-DSS", "SOC2"} + if got := GetSupportedFrameworks(); !reflect.DeepEqual(got, want) { + t.Fatalf("GetSupportedFrameworks() = %v, want %v", got, want) + } +} + +// --------------------------------------------------------------------------- +// gap aggregation +// --------------------------------------------------------------------------- + +// TestGetComplianceGapsAggregatesCountAndMaxSeverity ports +// test_get_compliance_gaps_aggregates_count_and_max_severity. +func TestGetComplianceGapsAggregatesCountAndMaxSeverity(t *testing.T) { + findings := []map[string]any{ + {"cwe_id": "CWE-89", "severity": "high"}, + {"cwe_id": "CWE-79", "severity": "critical"}, + {"cwe_id": "CWE-918", "severity": "medium"}, + } + + gaps := GetComplianceGaps(findings) + + var pciInjection []schemas.ComplianceGap + for _, gap := range gaps { + if gap.Framework == "PCI-DSS" && gap.ControlID == "Req 6.2.4" { + pciInjection = append(pciInjection, gap) + } + } + if len(pciInjection) != 1 { + t.Fatalf("want exactly one PCI-DSS Req 6.2.4 gap, got %d", len(pciInjection)) + } + if pciInjection[0].FindingCount != 2 { + t.Errorf("finding_count = %d, want 2", pciInjection[0].FindingCount) + } + if pciInjection[0].MaxSeverity != "critical" { + t.Errorf("max_severity = %q, want \"critical\"", pciInjection[0].MaxSeverity) + } + if !reflect.DeepEqual(pciInjection[0].CweIDs, []string{"CWE-79", "CWE-89"}) { + t.Errorf("cwe_ids = %v, want [CWE-79 CWE-89]", pciInjection[0].CweIDs) + } +} + +// TestGetComplianceGapsSortsByFrameworkControl asserts the documented ordering +// contract: sorted by (framework, control_id, control_name). +func TestGetComplianceGapsSortsByFrameworkControl(t *testing.T) { + gaps := GetComplianceGaps([]map[string]any{ + {"cwe_id": "CWE-89", "severity": "high"}, + {"cwe_id": "CWE-918", "severity": "low"}, + }) + if len(gaps) == 0 { + t.Fatal("no gaps") + } + for i := 1; i < len(gaps); i++ { + prev, cur := gaps[i-1], gaps[i] + if prev.Framework > cur.Framework || + (prev.Framework == cur.Framework && prev.ControlID > cur.ControlID) || + (prev.Framework == cur.Framework && prev.ControlID == cur.ControlID && prev.ControlName > cur.ControlName) { + t.Fatalf("gaps not sorted at %d: %+v then %+v", i, prev, cur) + } + } +} + +// TestGetComplianceGapsAcceptsObjectFindingsAndNormalizesCwe ports +// test_get_compliance_gaps_accepts_object_findings_and_normalizes_cwe. +func TestGetComplianceGapsAcceptsObjectFindingsAndNormalizesCwe(t *testing.T) { + findings := []namespaceFinding{ + {CweID: "89", Severity: "high"}, + {CweID: "cwe89", Severity: "critical"}, + } + + gaps := GetComplianceGaps(findings) + + var owaspGap *schemas.ComplianceGap + for i := range gaps { + if gaps[i].Framework == "OWASP" && gaps[i].ControlID == "A03:2021" { + owaspGap = &gaps[i] + break + } + } + if owaspGap == nil { + t.Fatal("no OWASP A03:2021 gap") + } + if owaspGap.FindingCount != 2 { + t.Errorf("finding_count = %d, want 2", owaspGap.FindingCount) + } + if owaspGap.MaxSeverity != "critical" { + t.Errorf("max_severity = %q, want \"critical\"", owaspGap.MaxSeverity) + } + if !reflect.DeepEqual(owaspGap.CweIDs, []string{"CWE-89"}) { + t.Errorf("cwe_ids = %v, want [CWE-89]", owaspGap.CweIDs) + } +} + +// TestGetComplianceGapsIgnoresEntriesWithoutValidCwe ports +// test_get_compliance_gaps_ignores_entries_without_valid_cwe. +func TestGetComplianceGapsIgnoresEntriesWithoutValidCwe(t *testing.T) { + findings := []map[string]any{ + {"severity": "critical"}, + {"cwe_id": nil, "severity": "high"}, + {"cwe_id": "CWE-9999", "severity": "critical"}, + } + + gaps := GetComplianceGaps(findings) + if len(gaps) != 0 { + t.Fatalf("want no gaps, got %v", gaps) + } +} + +// TestGetComplianceGapsVerifiedFindingSeverityQuirk pins the CPython behaviour +// documented on severityKey: a real VerifiedFinding carries a +// `class Severity(str, Enum)` member, `str()` of which is "Severity.CRITICAL", +// so the lower-cased key misses _SEVERITY_RANK entirely and max_severity stays +// at its "info" seed. Ground truth captured from +// ~/.agentfield/packages/sec-af/venv/bin/python. +func TestGetComplianceGapsVerifiedFindingSeverityQuirk(t *testing.T) { + finding := schemas.NewVerifiedFinding() + finding.CweID = "CWE-89" + finding.Severity = schemas.SeverityCritical + + gaps := GetComplianceGaps([]schemas.VerifiedFinding{finding}) + + if len(gaps) != 5 { + t.Fatalf("want 5 gaps for CWE-89, got %d", len(gaps)) + } + for _, gap := range gaps { + if gap.MaxSeverity != "info" { + t.Errorf("%s %s: max_severity = %q, want \"info\" (Python str(Severity.CRITICAL) quirk)", + gap.Framework, gap.ControlID, gap.MaxSeverity) + } + if gap.FindingCount != 1 { + t.Errorf("%s %s: finding_count = %d, want 1", gap.Framework, gap.ControlID, gap.FindingCount) + } + } + // Ground truth from Python for the same input, in result order. + wantControls := []string{"§164.312(a)(1)", "A.8.28", "A03:2021", "Req 6.2.4", "CC6"} + wantFrameworks := []string{"HIPAA", "ISO27001", "OWASP", "PCI-DSS", "SOC2"} + for i, gap := range gaps { + if gap.Framework != wantFrameworks[i] || gap.ControlID != wantControls[i] { + t.Fatalf("gap %d = (%s, %s), want (%s, %s)", i, gap.Framework, gap.ControlID, wantFrameworks[i], wantControls[i]) + } + } +} + +// TestGetComplianceGapsPointerFindings proves readField follows pointers, the +// shape a caller holding []*VerifiedFinding would produce. +func TestGetComplianceGapsPointerFindings(t *testing.T) { + gaps := GetComplianceGaps([]*namespaceFinding{{CweID: "CWE-89", Severity: "high"}}) + if len(gaps) != 5 { + t.Fatalf("want 5 gaps, got %d", len(gaps)) + } + if gaps[0].MaxSeverity != "high" { + t.Fatalf("max_severity = %q, want \"high\"", gaps[0].MaxSeverity) + } +} + +// TestReadField covers _read_field's two branches plus the missing-key default. +func TestReadField(t *testing.T) { + if got := readField(map[string]any{"cwe_id": "CWE-1"}, "cwe_id"); got != "CWE-1" { + t.Errorf("dict branch: got %v", got) + } + if got := readField(map[string]any{}, "cwe_id"); got != nil { + t.Errorf("dict missing key: got %v, want nil", got) + } + if got := readField(namespaceFinding{CweID: "CWE-2"}, "cwe_id"); got != "CWE-2" { + t.Errorf("attribute branch: got %v", got) + } + if got := readField(namespaceFinding{}, "not_a_field"); got != nil { + t.Errorf("missing attribute: got %v, want nil", got) + } + if got := readField(nil, "cwe_id"); got != nil { + t.Errorf("nil finding: got %v, want nil", got) + } + if got := readField(map[string]string{"severity": "low"}, "severity"); got != "low" { + t.Errorf("typed map branch: got %v", got) + } +} + +// TestSeverityKey covers `str(x or "low").lower()`. +func TestSeverityKey(t *testing.T) { + cases := []struct { + in any + want string + }{ + {nil, "low"}, + {"", "low"}, + {"HIGH", "high"}, + {"critical", "critical"}, + {schemas.SeverityHigh, "severity.high"}, + {schemas.SeverityInfo, "severity.info"}, + } + for _, tc := range cases { + if got := severityKey(tc.in); got != tc.want { + t.Errorf("severityKey(%#v) = %q, want %q", tc.in, got, tc.want) + } + } +} + +// --------------------------------------------------------------------------- +// hybrid / AI fallback +// --------------------------------------------------------------------------- + +// TestGetComplianceMappingsHybridUsesAIFallbackForUnknownCwe ports +// test_get_compliance_mappings_hybrid_uses_ai_fallback_for_unknown_cwe. +func TestGetComplianceMappingsHybridUsesAIFallbackForUnknownCwe(t *testing.T) { + ClearAICache() + gate := &fakeAIGate{response: schemas.ComplianceGate{ + Mappings: []schemas.ComplianceSuggestion{ + {Framework: "OWASP", ControlID: "A03:2021", ControlName: "Injection"}, + {Framework: "PCI-DSS", ControlID: "Req 6.2.4", ControlName: "Prevent injection attacks"}, + }, + Confidence: "high", + }} + + results := GetComplianceMappingsHybrid(context.Background(), "CWE-9999", nil, gate) + + if gate.callCount() != 1 { + t.Fatalf("gate calls = %d, want 1", gate.callCount()) + } + if len(results) != 2 { + t.Fatalf("results = %v, want 2 mappings", results) + } + got := sortedKeys(frameworkSet(results)) + if !reflect.DeepEqual(got, []string{"OWASP", "PCI-DSS"}) { + t.Fatalf("frameworks = %v, want [OWASP PCI-DSS]", got) + } +} + +// TestGetComplianceMappingsHybridUsesCacheForAIResults ports +// test_get_compliance_mappings_hybrid_uses_cache_for_ai_results. +func TestGetComplianceMappingsHybridUsesCacheForAIResults(t *testing.T) { + ClearAICache() + gate := &fakeAIGate{response: schemas.ComplianceGate{ + Mappings: []schemas.ComplianceSuggestion{ + {Framework: "OWASP", ControlID: "A03:2021", ControlName: "Injection"}, + }, + Confidence: "high", + }} + + first := GetComplianceMappingsHybrid(context.Background(), "CWE-9999", nil, gate) + second := GetComplianceMappingsHybrid(context.Background(), "CWE-9999", nil, gate) + + if gate.callCount() != 1 { + t.Fatalf("gate calls = %d, want 1 (second call must hit the cache)", gate.callCount()) + } + if !reflect.DeepEqual(first, second) { + t.Fatalf("cached result differs: %v vs %v", first, second) + } +} + +// TestGetComplianceMappingsHybridKeepsFastPathForKnownCwe ports +// test_get_compliance_mappings_hybrid_keeps_fast_path_for_known_cwe. +func TestGetComplianceMappingsHybridKeepsFastPathForKnownCwe(t *testing.T) { + ClearAICache() + gate := &fakeAIGate{response: schemas.ComplianceGate{Confidence: "low"}} + + results := GetComplianceMappingsHybrid(context.Background(), "CWE-89", nil, gate) + + if gate.callCount() != 0 { + t.Fatalf("gate calls = %d, want 0 for a table-known CWE", gate.callCount()) + } + if !reflect.DeepEqual(results, GetComplianceMappings("CWE-89", nil)) { + t.Fatalf("hybrid result differs from the static lookup: %v", results) + } +} + +// TestGetComplianceMappingsHybridWithoutGate covers `ai_gate is None`. +func TestGetComplianceMappingsHybridWithoutGate(t *testing.T) { + ClearAICache() + got := GetComplianceMappingsHybrid(context.Background(), "CWE-9999", nil, nil) + if got == nil || len(got) != 0 { + t.Fatalf("want an empty non-nil slice, got %#v", got) + } +} + +// TestGetComplianceMappingsHybridSwallowsGateErrors covers +// `except Exception: return []` — and asserts the failure is NOT cached, so a +// later successful call still reaches the gate. +func TestGetComplianceMappingsHybridSwallowsGateErrors(t *testing.T) { + ClearAICache() + gate := &fakeAIGate{err: errors.New("boom")} + + got := GetComplianceMappingsHybrid(context.Background(), "CWE-9999", nil, gate) + if got == nil || len(got) != 0 { + t.Fatalf("want an empty non-nil slice, got %#v", got) + } + + gate.err = nil + gate.response = schemas.ComplianceGate{ + Mappings: []schemas.ComplianceSuggestion{{Framework: "OWASP", ControlID: "A03:2021", ControlName: "Injection"}}, + Confidence: "high", + } + retry := GetComplianceMappingsHybrid(context.Background(), "CWE-9999", nil, gate) + if len(retry) != 1 { + t.Fatalf("retry after a failure returned %v", retry) + } + if gate.callCount() != 2 { + t.Fatalf("gate calls = %d, want 2 (a failure must not be cached)", gate.callCount()) + } +} + +// TestGetComplianceMappingsHybridFiltersAIMappings asserts the framework filter +// is applied to the gate's answer, not just to the static table. +func TestGetComplianceMappingsHybridFiltersAIMappings(t *testing.T) { + ClearAICache() + gate := &fakeAIGate{response: schemas.ComplianceGate{ + Mappings: []schemas.ComplianceSuggestion{ + {Framework: "OWASP", ControlID: "A03:2021", ControlName: "Injection"}, + {Framework: "PCI-DSS", ControlID: "Req 6.2.4", ControlName: "Prevent injection"}, + }, + Confidence: "high", + }} + + got := GetComplianceMappingsHybrid(context.Background(), "CWE-9999", []string{"owasp"}, gate) + + if len(got) != 1 || got[0].Framework != "OWASP" { + t.Fatalf("want only the OWASP mapping, got %v", got) + } +} + +// TestGetComplianceMappingsHybridCacheKeyDistinguishesFrameworks pins the +// Python cache key: `(cwe, None)` and `(cwe, ("owasp",))` are distinct entries, +// so restricting the frameworks re-queries the gate. +func TestGetComplianceMappingsHybridCacheKeyDistinguishesFrameworks(t *testing.T) { + ClearAICache() + gate := &fakeAIGate{response: schemas.ComplianceGate{ + Mappings: []schemas.ComplianceSuggestion{{Framework: "OWASP", ControlID: "A03:2021", ControlName: "Injection"}}, + Confidence: "high", + }} + + _ = GetComplianceMappingsHybrid(context.Background(), "CWE-9999", nil, gate) + _ = GetComplianceMappingsHybrid(context.Background(), "CWE-9999", []string{"owasp"}, gate) + if gate.callCount() != 2 { + t.Fatalf("gate calls = %d, want 2 (None and ('owasp',) are distinct keys)", gate.callCount()) + } + + // The same framework set spelled differently normalizes to the same key. + _ = GetComplianceMappingsHybrid(context.Background(), "CWE-9999", []string{"OWASP"}, gate) + _ = GetComplianceMappingsHybrid(context.Background(), "CWE-9999", []string{"owasp", "owasp"}, gate) + if gate.callCount() != 2 { + t.Fatalf("gate calls = %d, want 2 (normalized framework sets share a key)", gate.callCount()) + } +} + +// TestGetComplianceMappingsHybridCachedResultsAreCopies asserts a caller cannot +// corrupt the cache by mutating what it was handed (Python model_copy(deep=True)). +func TestGetComplianceMappingsHybridCachedResultsAreCopies(t *testing.T) { + ClearAICache() + gate := &fakeAIGate{response: schemas.ComplianceGate{ + Mappings: []schemas.ComplianceSuggestion{{Framework: "OWASP", ControlID: "A03:2021", ControlName: "Injection"}}, + Confidence: "high", + }} + + first := GetComplianceMappingsHybrid(context.Background(), "CWE-9999", nil, gate) + first[0].ControlName = "mutated" + second := GetComplianceMappingsHybrid(context.Background(), "CWE-9999", nil, gate) + + if second[0].ControlName != "Injection" { + t.Fatalf("cache was corrupted by the caller: %q", second[0].ControlName) + } +} + +// TestComplianceGatePrompt pins the fallback prompt text byte-for-byte, +// including the "(Unknown CWE)" literal. +func TestComplianceGatePrompt(t *testing.T) { + ClearAICache() + gate := &fakeAIGate{response: schemas.ComplianceGate{Confidence: "low"}} + + _ = GetComplianceMappingsHybrid(context.Background(), "cwe9999", nil, gate) + want := "Map CWE-9999 (Unknown CWE) to compliance framework controls. " + + "Frameworks: OWASP, PCI-DSS, SOC2, HIPAA, ISO27001. Return specific control IDs." + if gate.prompts[0] != want { + t.Fatalf("prompt =\n%q\nwant\n%q", gate.prompts[0], want) + } + + ClearAICache() + gate2 := &fakeAIGate{response: schemas.ComplianceGate{Confidence: "low"}} + _ = GetComplianceMappingsHybrid(context.Background(), "CWE-9999", []string{"PCI-DSS", "owasp"}, gate2) + want2 := "Map CWE-9999 (Unknown CWE) to compliance framework controls. " + + "Frameworks: PCI-DSS, owasp. Return specific control IDs." + if gate2.prompts[0] != want2 { + t.Fatalf("prompt =\n%q\nwant\n%q", gate2.prompts[0], want2) + } +} + +// TestGetComplianceMappingsHybridConcurrent exercises the cache under -race: +// Python's dict is single-event-loop safe, the Go port must be mutex safe. +func TestGetComplianceMappingsHybridConcurrent(t *testing.T) { + ClearAICache() + gate := &fakeAIGate{response: schemas.ComplianceGate{ + Mappings: []schemas.ComplianceSuggestion{{Framework: "OWASP", ControlID: "A03:2021", ControlName: "Injection"}}, + Confidence: "high", + }} + + var wg sync.WaitGroup + for i := 0; i < 32; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + cwe := "CWE-90" + string(rune('0'+i%10)) + got := GetComplianceMappingsHybrid(context.Background(), cwe, nil, gate) + if len(got) != 1 { + t.Errorf("%s: got %v", cwe, got) + } + }(i) + } + wg.Wait() +} + +// --------------------------------------------------------------------------- +// generated-table drift guard +// --------------------------------------------------------------------------- + +// TestComplianceMapMatchesPythonSnapshot compares the generated Go table +// against testdata/compliance_map.json, which scripts/gen_compliance_table.py +// writes from the live Python COMPLIANCE_MAP in the same run that writes +// table_gen.go. A hand edit to either file fails here. +func TestComplianceMapMatchesPythonSnapshot(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("testdata", "compliance_map.json")) + if err != nil { + t.Fatalf("read snapshot: %v", err) + } + var snapshot map[string][]schemas.ComplianceMapping + if err := json.Unmarshal(raw, &snapshot); err != nil { + t.Fatalf("decode snapshot: %v", err) + } + if !reflect.DeepEqual(ComplianceMap, snapshot) { + if len(ComplianceMap) != len(snapshot) { + t.Fatalf("table has %d CWEs, snapshot has %d", len(ComplianceMap), len(snapshot)) + } + for cwe, want := range snapshot { + if got := ComplianceMap[cwe]; !reflect.DeepEqual(got, want) { + t.Errorf("%s:\n got: %+v\nwant: %+v", cwe, got, want) + } + } + t.FailNow() + } +} + +// TestGetComplianceGapsFullGroundTruth compares the whole result against the +// output CPython produces for the same three dict findings. Captured with: +// +// PYTHONPATH=src ~/.agentfield/packages/sec-af/venv/bin/python -c " +// from sec_af.compliance.mapping import get_compliance_gaps +// print([(g.framework, g.control_id, g.max_severity, g.finding_count, g.cwe_ids) +// for g in get_compliance_gaps([ +// {'cwe_id':'CWE-89','severity':'high'}, +// {'cwe_id':'CWE-79','severity':'critical'}, +// {'cwe_id':'CWE-918','severity':'medium'}])])" +func TestGetComplianceGapsFullGroundTruth(t *testing.T) { + gaps := GetComplianceGaps([]map[string]any{ + {"cwe_id": "CWE-89", "severity": "high"}, + {"cwe_id": "CWE-79", "severity": "critical"}, + {"cwe_id": "CWE-918", "severity": "medium"}, + }) + + want := []schemas.ComplianceGap{ + {Framework: "HIPAA", ControlID: "\u00a7164.312(a)(1)", ControlName: "Access control", FindingCount: 1, MaxSeverity: "high", CweIDs: []string{"CWE-89"}}, + {Framework: "HIPAA", ControlID: "\u00a7164.312(c)(1)", ControlName: "Integrity", FindingCount: 1, MaxSeverity: "critical", CweIDs: []string{"CWE-79"}}, + {Framework: "HIPAA", ControlID: "\u00a7164.312(e)(1)", ControlName: "Transmission security", FindingCount: 1, MaxSeverity: "medium", CweIDs: []string{"CWE-918"}}, + {Framework: "ISO27001", ControlID: "A.8.20", ControlName: "Network security", FindingCount: 1, MaxSeverity: "medium", CweIDs: []string{"CWE-918"}}, + {Framework: "ISO27001", ControlID: "A.8.28", ControlName: "Secure coding", FindingCount: 2, MaxSeverity: "critical", CweIDs: []string{"CWE-79", "CWE-89"}}, + {Framework: "OWASP", ControlID: "A03:2021", ControlName: "Injection", FindingCount: 2, MaxSeverity: "critical", CweIDs: []string{"CWE-79", "CWE-89"}}, + {Framework: "OWASP", ControlID: "A10:2021", ControlName: "Server-Side Request Forgery", FindingCount: 1, MaxSeverity: "medium", CweIDs: []string{"CWE-918"}}, + {Framework: "PCI-DSS", ControlID: "Req 6", ControlName: "Develop and maintain secure systems and software", FindingCount: 1, MaxSeverity: "medium", CweIDs: []string{"CWE-918"}}, + {Framework: "PCI-DSS", ControlID: "Req 6.2.4", ControlName: "Custom software addresses common coding vulnerabilities", FindingCount: 2, MaxSeverity: "critical", CweIDs: []string{"CWE-79", "CWE-89"}}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls", FindingCount: 1, MaxSeverity: "high", CweIDs: []string{"CWE-89"}}, + {Framework: "SOC2", ControlID: "CC7", ControlName: "System operations", FindingCount: 2, MaxSeverity: "critical", CweIDs: []string{"CWE-79", "CWE-918"}}, + } + if !reflect.DeepEqual(gaps, want) { + if len(gaps) != len(want) { + t.Fatalf("got %d gaps, want %d", len(gaps), len(want)) + } + for i := range want { + if !reflect.DeepEqual(gaps[i], want[i]) { + t.Errorf("gap %d:\n got: %+v\nwant: %+v", i, gaps[i], want[i]) + } + } + t.FailNow() + } +} diff --git a/internal/secaf/compliance/table_gen.go b/internal/secaf/compliance/table_gen.go new file mode 100644 index 0000000000..067b78b89b --- /dev/null +++ b/internal/secaf/compliance/table_gen.go @@ -0,0 +1,258 @@ +// Code generated by scripts/gen_compliance_table.py. DO NOT EDIT. + +package compliance + +import "github.com/Agent-Field/codeaf/internal/secaf/schemas" + +// ComplianceMap ports the module-level COMPLIANCE_MAP table of +// src/sec_af/compliance/mapping.py: the static CWE -> framework-control +// mapping every finding is scored against. +// +// Key order below is the Python dict's literal (insertion) order, and each +// value keeps the Python list order, because GetComplianceMappings returns the +// per-CWE list verbatim and callers compare it positionally. +// +// Regenerate with: +// +// PYTHONPATH=src ~/.agentfield/packages/sec-af/venv/bin/python \ +// go/scripts/gen_compliance_table.py +var ComplianceMap = map[string][]schemas.ComplianceMapping{ + "CWE-78": { + {Framework: "PCI-DSS", ControlID: "Req 6.2.4", ControlName: "Custom software addresses common coding vulnerabilities"}, + {Framework: "SOC2", ControlID: "CC7", ControlName: "System operations"}, + {Framework: "OWASP", ControlID: "A03:2021", ControlName: "Injection"}, + {Framework: "HIPAA", ControlID: "§164.312(c)(1)", ControlName: "Integrity"}, + {Framework: "ISO27001", ControlID: "A.8.28", ControlName: "Secure coding"}, + }, + "CWE-79": { + {Framework: "PCI-DSS", ControlID: "Req 6.2.4", ControlName: "Custom software addresses common coding vulnerabilities"}, + {Framework: "SOC2", ControlID: "CC7", ControlName: "System operations"}, + {Framework: "OWASP", ControlID: "A03:2021", ControlName: "Injection"}, + {Framework: "HIPAA", ControlID: "§164.312(c)(1)", ControlName: "Integrity"}, + {Framework: "ISO27001", ControlID: "A.8.28", ControlName: "Secure coding"}, + }, + "CWE-89": { + {Framework: "PCI-DSS", ControlID: "Req 6.2.4", ControlName: "Custom software addresses common coding vulnerabilities"}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls"}, + {Framework: "OWASP", ControlID: "A03:2021", ControlName: "Injection"}, + {Framework: "HIPAA", ControlID: "§164.312(a)(1)", ControlName: "Access control"}, + {Framework: "ISO27001", ControlID: "A.8.28", ControlName: "Secure coding"}, + }, + "CWE-90": { + {Framework: "PCI-DSS", ControlID: "Req 6.2.4", ControlName: "Custom software addresses common coding vulnerabilities"}, + {Framework: "SOC2", ControlID: "CC7", ControlName: "System operations"}, + {Framework: "OWASP", ControlID: "A03:2021", ControlName: "Injection"}, + {Framework: "HIPAA", ControlID: "§164.312(c)(1)", ControlName: "Integrity"}, + {Framework: "ISO27001", ControlID: "A.8.28", ControlName: "Secure coding"}, + }, + "CWE-91": { + {Framework: "PCI-DSS", ControlID: "Req 6.2.4", ControlName: "Custom software addresses common coding vulnerabilities"}, + {Framework: "SOC2", ControlID: "CC7", ControlName: "System operations"}, + {Framework: "OWASP", ControlID: "A03:2021", ControlName: "Injection"}, + {Framework: "HIPAA", ControlID: "§164.312(c)(1)", ControlName: "Integrity"}, + {Framework: "ISO27001", ControlID: "A.8.28", ControlName: "Secure coding"}, + }, + "CWE-94": { + {Framework: "PCI-DSS", ControlID: "Req 6.2.4", ControlName: "Custom software addresses common coding vulnerabilities"}, + {Framework: "SOC2", ControlID: "CC7", ControlName: "System operations"}, + {Framework: "OWASP", ControlID: "A03:2021", ControlName: "Injection"}, + {Framework: "HIPAA", ControlID: "§164.312(c)(1)", ControlName: "Integrity"}, + {Framework: "ISO27001", ControlID: "A.8.28", ControlName: "Secure coding"}, + }, + "CWE-917": { + {Framework: "PCI-DSS", ControlID: "Req 6.2.4", ControlName: "Custom software addresses common coding vulnerabilities"}, + {Framework: "SOC2", ControlID: "CC7", ControlName: "System operations"}, + {Framework: "OWASP", ControlID: "A03:2021", ControlName: "Injection"}, + {Framework: "HIPAA", ControlID: "§164.312(c)(1)", ControlName: "Integrity"}, + {Framework: "ISO27001", ControlID: "A.8.28", ControlName: "Secure coding"}, + }, + "CWE-287": { + {Framework: "PCI-DSS", ControlID: "Req 8", ControlName: "Identify users and authenticate access to system components"}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls"}, + {Framework: "OWASP", ControlID: "A07:2021", ControlName: "Identification and Authentication Failures"}, + {Framework: "HIPAA", ControlID: "§164.312(d)", ControlName: "Person or entity authentication"}, + {Framework: "ISO27001", ControlID: "A.5.17", ControlName: "Authentication information"}, + }, + "CWE-306": { + {Framework: "PCI-DSS", ControlID: "Req 8", ControlName: "Identify users and authenticate access to system components"}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls"}, + {Framework: "OWASP", ControlID: "A07:2021", ControlName: "Identification and Authentication Failures"}, + {Framework: "HIPAA", ControlID: "§164.312(a)(1)", ControlName: "Access control"}, + {Framework: "ISO27001", ControlID: "A.5.15", ControlName: "Access control"}, + }, + "CWE-352": { + {Framework: "PCI-DSS", ControlID: "Req 6.2.4", ControlName: "Custom software addresses common coding vulnerabilities"}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls"}, + {Framework: "OWASP", ControlID: "A01:2021", ControlName: "Broken Access Control"}, + {Framework: "HIPAA", ControlID: "§164.312(a)(1)", ControlName: "Access control"}, + {Framework: "ISO27001", ControlID: "A.8.5", ControlName: "Secure authentication"}, + }, + "CWE-862": { + {Framework: "PCI-DSS", ControlID: "Req 8", ControlName: "Identify users and authenticate access to system components"}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls"}, + {Framework: "OWASP", ControlID: "A01:2021", ControlName: "Broken Access Control"}, + {Framework: "HIPAA", ControlID: "§164.312(a)(1)", ControlName: "Access control"}, + {Framework: "ISO27001", ControlID: "A.5.15", ControlName: "Access control"}, + }, + "CWE-863": { + {Framework: "PCI-DSS", ControlID: "Req 8", ControlName: "Identify users and authenticate access to system components"}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls"}, + {Framework: "OWASP", ControlID: "A01:2021", ControlName: "Broken Access Control"}, + {Framework: "HIPAA", ControlID: "§164.312(a)(1)", ControlName: "Access control"}, + {Framework: "ISO27001", ControlID: "A.5.18", ControlName: "Access rights"}, + }, + "CWE-326": { + {Framework: "PCI-DSS", ControlID: "Req 3", ControlName: "Protect stored account data"}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls"}, + {Framework: "OWASP", ControlID: "A02:2021", ControlName: "Cryptographic Failures"}, + {Framework: "HIPAA", ControlID: "§164.312(a)(2)(iv)", ControlName: "Encryption and decryption"}, + {Framework: "ISO27001", ControlID: "A.8.24", ControlName: "Use of cryptography"}, + }, + "CWE-327": { + {Framework: "PCI-DSS", ControlID: "Req 3", ControlName: "Protect stored account data"}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls"}, + {Framework: "OWASP", ControlID: "A02:2021", ControlName: "Cryptographic Failures"}, + {Framework: "HIPAA", ControlID: "§164.312(a)(2)(iv)", ControlName: "Encryption and decryption"}, + {Framework: "ISO27001", ControlID: "A.8.24", ControlName: "Use of cryptography"}, + }, + "CWE-328": { + {Framework: "PCI-DSS", ControlID: "Req 3", ControlName: "Protect stored account data"}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls"}, + {Framework: "OWASP", ControlID: "A02:2021", ControlName: "Cryptographic Failures"}, + {Framework: "HIPAA", ControlID: "§164.312(a)(2)(iv)", ControlName: "Encryption and decryption"}, + {Framework: "ISO27001", ControlID: "A.8.24", ControlName: "Use of cryptography"}, + }, + "CWE-330": { + {Framework: "PCI-DSS", ControlID: "Req 3", ControlName: "Protect stored account data"}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls"}, + {Framework: "OWASP", ControlID: "A02:2021", ControlName: "Cryptographic Failures"}, + {Framework: "HIPAA", ControlID: "§164.312(c)(1)", ControlName: "Integrity"}, + {Framework: "ISO27001", ControlID: "A.8.24", ControlName: "Use of cryptography"}, + }, + "CWE-916": { + {Framework: "PCI-DSS", ControlID: "Req 8", ControlName: "Identify users and authenticate access to system components"}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls"}, + {Framework: "OWASP", ControlID: "A07:2021", ControlName: "Identification and Authentication Failures"}, + {Framework: "HIPAA", ControlID: "§164.312(d)", ControlName: "Person or entity authentication"}, + {Framework: "ISO27001", ControlID: "A.8.24", ControlName: "Use of cryptography"}, + }, + "CWE-840": { + {Framework: "PCI-DSS", ControlID: "Req 6", ControlName: "Develop and maintain secure systems and software"}, + {Framework: "SOC2", ControlID: "CC7", ControlName: "System operations"}, + {Framework: "OWASP", ControlID: "A04:2021", ControlName: "Insecure Design"}, + {Framework: "HIPAA", ControlID: "§164.308(a)(1)(ii)(A)", ControlName: "Risk analysis"}, + {Framework: "ISO27001", ControlID: "A.8.25", ControlName: "Secure development lifecycle"}, + }, + "CWE-841": { + {Framework: "PCI-DSS", ControlID: "Req 6", ControlName: "Develop and maintain secure systems and software"}, + {Framework: "SOC2", ControlID: "CC7", ControlName: "System operations"}, + {Framework: "OWASP", ControlID: "A04:2021", ControlName: "Insecure Design"}, + {Framework: "HIPAA", ControlID: "§164.308(a)(1)(ii)(A)", ControlName: "Risk analysis"}, + {Framework: "ISO27001", ControlID: "A.8.25", ControlName: "Secure development lifecycle"}, + }, + "CWE-200": { + {Framework: "PCI-DSS", ControlID: "Req 3", ControlName: "Protect stored account data"}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls"}, + {Framework: "OWASP", ControlID: "A01:2021", ControlName: "Broken Access Control"}, + {Framework: "HIPAA", ControlID: "§164.312(a)(1)", ControlName: "Access control"}, + {Framework: "ISO27001", ControlID: "A.8.12", ControlName: "Data leakage prevention"}, + }, + "CWE-209": { + {Framework: "PCI-DSS", ControlID: "Req 6", ControlName: "Develop and maintain secure systems and software"}, + {Framework: "SOC2", ControlID: "CC7", ControlName: "System operations"}, + {Framework: "OWASP", ControlID: "A09:2021", ControlName: "Security Logging and Monitoring Failures"}, + {Framework: "HIPAA", ControlID: "§164.312(b)", ControlName: "Audit controls"}, + {Framework: "ISO27001", ControlID: "A.8.15", ControlName: "Logging"}, + }, + "CWE-312": { + {Framework: "PCI-DSS", ControlID: "Req 3", ControlName: "Protect stored account data"}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls"}, + {Framework: "OWASP", ControlID: "A02:2021", ControlName: "Cryptographic Failures"}, + {Framework: "HIPAA", ControlID: "§164.312(a)(2)(iv)", ControlName: "Encryption and decryption"}, + {Framework: "ISO27001", ControlID: "A.8.24", ControlName: "Use of cryptography"}, + }, + "CWE-319": { + {Framework: "PCI-DSS", ControlID: "Req 4", ControlName: "Protect cardholder data with strong cryptography during transmission"}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls"}, + {Framework: "OWASP", ControlID: "A02:2021", ControlName: "Cryptographic Failures"}, + {Framework: "HIPAA", ControlID: "§164.312(e)(1)", ControlName: "Transmission security"}, + {Framework: "ISO27001", ControlID: "A.8.20", ControlName: "Network security"}, + }, + "CWE-532": { + {Framework: "PCI-DSS", ControlID: "Req 3", ControlName: "Protect stored account data"}, + {Framework: "SOC2", ControlID: "CC7", ControlName: "System operations"}, + {Framework: "OWASP", ControlID: "A09:2021", ControlName: "Security Logging and Monitoring Failures"}, + {Framework: "HIPAA", ControlID: "§164.312(b)", ControlName: "Audit controls"}, + {Framework: "ISO27001", ControlID: "A.8.15", ControlName: "Logging"}, + }, + "CWE-829": { + {Framework: "PCI-DSS", ControlID: "Req 6", ControlName: "Develop and maintain secure systems and software"}, + {Framework: "SOC2", ControlID: "CC8", ControlName: "Change management"}, + {Framework: "OWASP", ControlID: "A08:2021", ControlName: "Software and Data Integrity Failures"}, + {Framework: "HIPAA", ControlID: "§164.308(a)(1)(ii)(B)", ControlName: "Risk management"}, + {Framework: "ISO27001", ControlID: "A.8.25", ControlName: "Secure development lifecycle"}, + }, + "CWE-1104": { + {Framework: "PCI-DSS", ControlID: "Req 6", ControlName: "Develop and maintain secure systems and software"}, + {Framework: "SOC2", ControlID: "CC8", ControlName: "Change management"}, + {Framework: "OWASP", ControlID: "A06:2021", ControlName: "Vulnerable and Outdated Components"}, + {Framework: "HIPAA", ControlID: "§164.308(a)(1)(ii)(B)", ControlName: "Risk management"}, + {Framework: "ISO27001", ControlID: "A.8.8", ControlName: "Management of technical vulnerabilities"}, + }, + "CWE-16": { + {Framework: "PCI-DSS", ControlID: "Req 2", ControlName: "Apply secure configurations to all system components"}, + {Framework: "SOC2", ControlID: "CC7", ControlName: "System operations"}, + {Framework: "OWASP", ControlID: "A05:2021", ControlName: "Security Misconfiguration"}, + {Framework: "HIPAA", ControlID: "§164.308(a)(1)(ii)(B)", ControlName: "Risk management"}, + {Framework: "ISO27001", ControlID: "A.8.9", ControlName: "Configuration management"}, + }, + "CWE-259": { + {Framework: "PCI-DSS", ControlID: "Req 8", ControlName: "Identify users and authenticate access to system components"}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls"}, + {Framework: "OWASP", ControlID: "A07:2021", ControlName: "Identification and Authentication Failures"}, + {Framework: "HIPAA", ControlID: "§164.312(d)", ControlName: "Person or entity authentication"}, + {Framework: "ISO27001", ControlID: "A.5.17", ControlName: "Authentication information"}, + }, + "CWE-321": { + {Framework: "PCI-DSS", ControlID: "Req 3", ControlName: "Protect stored account data"}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls"}, + {Framework: "OWASP", ControlID: "A02:2021", ControlName: "Cryptographic Failures"}, + {Framework: "HIPAA", ControlID: "§164.312(a)(2)(iv)", ControlName: "Encryption and decryption"}, + {Framework: "ISO27001", ControlID: "A.8.24", ControlName: "Use of cryptography"}, + }, + "CWE-798": { + {Framework: "PCI-DSS", ControlID: "Req 8", ControlName: "Identify users and authenticate access to system components"}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls"}, + {Framework: "OWASP", ControlID: "A07:2021", ControlName: "Identification and Authentication Failures"}, + {Framework: "HIPAA", ControlID: "§164.312(d)", ControlName: "Person or entity authentication"}, + {Framework: "ISO27001", ControlID: "A.5.17", ControlName: "Authentication information"}, + }, + "CWE-285": { + {Framework: "PCI-DSS", ControlID: "Req 8", ControlName: "Identify users and authenticate access to system components"}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls"}, + {Framework: "OWASP", ControlID: "A01:2021", ControlName: "Broken Access Control"}, + {Framework: "HIPAA", ControlID: "§164.312(a)(1)", ControlName: "Access control"}, + {Framework: "ISO27001", ControlID: "A.5.15", ControlName: "Access control"}, + }, + "CWE-346": { + {Framework: "PCI-DSS", ControlID: "Req 8", ControlName: "Identify users and authenticate access to system components"}, + {Framework: "SOC2", ControlID: "CC6", ControlName: "Logical and physical access controls"}, + {Framework: "OWASP", ControlID: "A01:2021", ControlName: "Broken Access Control"}, + {Framework: "HIPAA", ControlID: "§164.312(a)(1)", ControlName: "Access control"}, + {Framework: "ISO27001", ControlID: "A.5.16", ControlName: "Identity management"}, + }, + "CWE-601": { + {Framework: "PCI-DSS", ControlID: "Req 6", ControlName: "Develop and maintain secure systems and software"}, + {Framework: "SOC2", ControlID: "CC7", ControlName: "System operations"}, + {Framework: "OWASP", ControlID: "A01:2021", ControlName: "Broken Access Control"}, + {Framework: "HIPAA", ControlID: "§164.312(c)(1)", ControlName: "Integrity"}, + {Framework: "ISO27001", ControlID: "A.8.28", ControlName: "Secure coding"}, + }, + "CWE-918": { + {Framework: "PCI-DSS", ControlID: "Req 6", ControlName: "Develop and maintain secure systems and software"}, + {Framework: "SOC2", ControlID: "CC7", ControlName: "System operations"}, + {Framework: "OWASP", ControlID: "A10:2021", ControlName: "Server-Side Request Forgery"}, + {Framework: "HIPAA", ControlID: "§164.312(e)(1)", ControlName: "Transmission security"}, + {Framework: "ISO27001", ControlID: "A.8.20", ControlName: "Network security"}, + }, +} diff --git a/internal/secaf/compliance/testdata/compliance_map.json b/internal/secaf/compliance/testdata/compliance_map.json new file mode 100644 index 0000000000..aa2345f88d --- /dev/null +++ b/internal/secaf/compliance/testdata/compliance_map.json @@ -0,0 +1,920 @@ +{ + "CWE-78": [ + { + "framework": "PCI-DSS", + "control_id": "Req 6.2.4", + "control_name": "Custom software addresses common coding vulnerabilities" + }, + { + "framework": "SOC2", + "control_id": "CC7", + "control_name": "System operations" + }, + { + "framework": "OWASP", + "control_id": "A03:2021", + "control_name": "Injection" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(c)(1)", + "control_name": "Integrity" + }, + { + "framework": "ISO27001", + "control_id": "A.8.28", + "control_name": "Secure coding" + } + ], + "CWE-79": [ + { + "framework": "PCI-DSS", + "control_id": "Req 6.2.4", + "control_name": "Custom software addresses common coding vulnerabilities" + }, + { + "framework": "SOC2", + "control_id": "CC7", + "control_name": "System operations" + }, + { + "framework": "OWASP", + "control_id": "A03:2021", + "control_name": "Injection" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(c)(1)", + "control_name": "Integrity" + }, + { + "framework": "ISO27001", + "control_id": "A.8.28", + "control_name": "Secure coding" + } + ], + "CWE-89": [ + { + "framework": "PCI-DSS", + "control_id": "Req 6.2.4", + "control_name": "Custom software addresses common coding vulnerabilities" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A03:2021", + "control_name": "Injection" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(a)(1)", + "control_name": "Access control" + }, + { + "framework": "ISO27001", + "control_id": "A.8.28", + "control_name": "Secure coding" + } + ], + "CWE-90": [ + { + "framework": "PCI-DSS", + "control_id": "Req 6.2.4", + "control_name": "Custom software addresses common coding vulnerabilities" + }, + { + "framework": "SOC2", + "control_id": "CC7", + "control_name": "System operations" + }, + { + "framework": "OWASP", + "control_id": "A03:2021", + "control_name": "Injection" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(c)(1)", + "control_name": "Integrity" + }, + { + "framework": "ISO27001", + "control_id": "A.8.28", + "control_name": "Secure coding" + } + ], + "CWE-91": [ + { + "framework": "PCI-DSS", + "control_id": "Req 6.2.4", + "control_name": "Custom software addresses common coding vulnerabilities" + }, + { + "framework": "SOC2", + "control_id": "CC7", + "control_name": "System operations" + }, + { + "framework": "OWASP", + "control_id": "A03:2021", + "control_name": "Injection" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(c)(1)", + "control_name": "Integrity" + }, + { + "framework": "ISO27001", + "control_id": "A.8.28", + "control_name": "Secure coding" + } + ], + "CWE-94": [ + { + "framework": "PCI-DSS", + "control_id": "Req 6.2.4", + "control_name": "Custom software addresses common coding vulnerabilities" + }, + { + "framework": "SOC2", + "control_id": "CC7", + "control_name": "System operations" + }, + { + "framework": "OWASP", + "control_id": "A03:2021", + "control_name": "Injection" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(c)(1)", + "control_name": "Integrity" + }, + { + "framework": "ISO27001", + "control_id": "A.8.28", + "control_name": "Secure coding" + } + ], + "CWE-917": [ + { + "framework": "PCI-DSS", + "control_id": "Req 6.2.4", + "control_name": "Custom software addresses common coding vulnerabilities" + }, + { + "framework": "SOC2", + "control_id": "CC7", + "control_name": "System operations" + }, + { + "framework": "OWASP", + "control_id": "A03:2021", + "control_name": "Injection" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(c)(1)", + "control_name": "Integrity" + }, + { + "framework": "ISO27001", + "control_id": "A.8.28", + "control_name": "Secure coding" + } + ], + "CWE-287": [ + { + "framework": "PCI-DSS", + "control_id": "Req 8", + "control_name": "Identify users and authenticate access to system components" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A07:2021", + "control_name": "Identification and Authentication Failures" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(d)", + "control_name": "Person or entity authentication" + }, + { + "framework": "ISO27001", + "control_id": "A.5.17", + "control_name": "Authentication information" + } + ], + "CWE-306": [ + { + "framework": "PCI-DSS", + "control_id": "Req 8", + "control_name": "Identify users and authenticate access to system components" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A07:2021", + "control_name": "Identification and Authentication Failures" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(a)(1)", + "control_name": "Access control" + }, + { + "framework": "ISO27001", + "control_id": "A.5.15", + "control_name": "Access control" + } + ], + "CWE-352": [ + { + "framework": "PCI-DSS", + "control_id": "Req 6.2.4", + "control_name": "Custom software addresses common coding vulnerabilities" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A01:2021", + "control_name": "Broken Access Control" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(a)(1)", + "control_name": "Access control" + }, + { + "framework": "ISO27001", + "control_id": "A.8.5", + "control_name": "Secure authentication" + } + ], + "CWE-862": [ + { + "framework": "PCI-DSS", + "control_id": "Req 8", + "control_name": "Identify users and authenticate access to system components" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A01:2021", + "control_name": "Broken Access Control" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(a)(1)", + "control_name": "Access control" + }, + { + "framework": "ISO27001", + "control_id": "A.5.15", + "control_name": "Access control" + } + ], + "CWE-863": [ + { + "framework": "PCI-DSS", + "control_id": "Req 8", + "control_name": "Identify users and authenticate access to system components" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A01:2021", + "control_name": "Broken Access Control" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(a)(1)", + "control_name": "Access control" + }, + { + "framework": "ISO27001", + "control_id": "A.5.18", + "control_name": "Access rights" + } + ], + "CWE-326": [ + { + "framework": "PCI-DSS", + "control_id": "Req 3", + "control_name": "Protect stored account data" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A02:2021", + "control_name": "Cryptographic Failures" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(a)(2)(iv)", + "control_name": "Encryption and decryption" + }, + { + "framework": "ISO27001", + "control_id": "A.8.24", + "control_name": "Use of cryptography" + } + ], + "CWE-327": [ + { + "framework": "PCI-DSS", + "control_id": "Req 3", + "control_name": "Protect stored account data" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A02:2021", + "control_name": "Cryptographic Failures" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(a)(2)(iv)", + "control_name": "Encryption and decryption" + }, + { + "framework": "ISO27001", + "control_id": "A.8.24", + "control_name": "Use of cryptography" + } + ], + "CWE-328": [ + { + "framework": "PCI-DSS", + "control_id": "Req 3", + "control_name": "Protect stored account data" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A02:2021", + "control_name": "Cryptographic Failures" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(a)(2)(iv)", + "control_name": "Encryption and decryption" + }, + { + "framework": "ISO27001", + "control_id": "A.8.24", + "control_name": "Use of cryptography" + } + ], + "CWE-330": [ + { + "framework": "PCI-DSS", + "control_id": "Req 3", + "control_name": "Protect stored account data" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A02:2021", + "control_name": "Cryptographic Failures" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(c)(1)", + "control_name": "Integrity" + }, + { + "framework": "ISO27001", + "control_id": "A.8.24", + "control_name": "Use of cryptography" + } + ], + "CWE-916": [ + { + "framework": "PCI-DSS", + "control_id": "Req 8", + "control_name": "Identify users and authenticate access to system components" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A07:2021", + "control_name": "Identification and Authentication Failures" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(d)", + "control_name": "Person or entity authentication" + }, + { + "framework": "ISO27001", + "control_id": "A.8.24", + "control_name": "Use of cryptography" + } + ], + "CWE-840": [ + { + "framework": "PCI-DSS", + "control_id": "Req 6", + "control_name": "Develop and maintain secure systems and software" + }, + { + "framework": "SOC2", + "control_id": "CC7", + "control_name": "System operations" + }, + { + "framework": "OWASP", + "control_id": "A04:2021", + "control_name": "Insecure Design" + }, + { + "framework": "HIPAA", + "control_id": "§164.308(a)(1)(ii)(A)", + "control_name": "Risk analysis" + }, + { + "framework": "ISO27001", + "control_id": "A.8.25", + "control_name": "Secure development lifecycle" + } + ], + "CWE-841": [ + { + "framework": "PCI-DSS", + "control_id": "Req 6", + "control_name": "Develop and maintain secure systems and software" + }, + { + "framework": "SOC2", + "control_id": "CC7", + "control_name": "System operations" + }, + { + "framework": "OWASP", + "control_id": "A04:2021", + "control_name": "Insecure Design" + }, + { + "framework": "HIPAA", + "control_id": "§164.308(a)(1)(ii)(A)", + "control_name": "Risk analysis" + }, + { + "framework": "ISO27001", + "control_id": "A.8.25", + "control_name": "Secure development lifecycle" + } + ], + "CWE-200": [ + { + "framework": "PCI-DSS", + "control_id": "Req 3", + "control_name": "Protect stored account data" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A01:2021", + "control_name": "Broken Access Control" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(a)(1)", + "control_name": "Access control" + }, + { + "framework": "ISO27001", + "control_id": "A.8.12", + "control_name": "Data leakage prevention" + } + ], + "CWE-209": [ + { + "framework": "PCI-DSS", + "control_id": "Req 6", + "control_name": "Develop and maintain secure systems and software" + }, + { + "framework": "SOC2", + "control_id": "CC7", + "control_name": "System operations" + }, + { + "framework": "OWASP", + "control_id": "A09:2021", + "control_name": "Security Logging and Monitoring Failures" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(b)", + "control_name": "Audit controls" + }, + { + "framework": "ISO27001", + "control_id": "A.8.15", + "control_name": "Logging" + } + ], + "CWE-312": [ + { + "framework": "PCI-DSS", + "control_id": "Req 3", + "control_name": "Protect stored account data" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A02:2021", + "control_name": "Cryptographic Failures" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(a)(2)(iv)", + "control_name": "Encryption and decryption" + }, + { + "framework": "ISO27001", + "control_id": "A.8.24", + "control_name": "Use of cryptography" + } + ], + "CWE-319": [ + { + "framework": "PCI-DSS", + "control_id": "Req 4", + "control_name": "Protect cardholder data with strong cryptography during transmission" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A02:2021", + "control_name": "Cryptographic Failures" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(e)(1)", + "control_name": "Transmission security" + }, + { + "framework": "ISO27001", + "control_id": "A.8.20", + "control_name": "Network security" + } + ], + "CWE-532": [ + { + "framework": "PCI-DSS", + "control_id": "Req 3", + "control_name": "Protect stored account data" + }, + { + "framework": "SOC2", + "control_id": "CC7", + "control_name": "System operations" + }, + { + "framework": "OWASP", + "control_id": "A09:2021", + "control_name": "Security Logging and Monitoring Failures" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(b)", + "control_name": "Audit controls" + }, + { + "framework": "ISO27001", + "control_id": "A.8.15", + "control_name": "Logging" + } + ], + "CWE-829": [ + { + "framework": "PCI-DSS", + "control_id": "Req 6", + "control_name": "Develop and maintain secure systems and software" + }, + { + "framework": "SOC2", + "control_id": "CC8", + "control_name": "Change management" + }, + { + "framework": "OWASP", + "control_id": "A08:2021", + "control_name": "Software and Data Integrity Failures" + }, + { + "framework": "HIPAA", + "control_id": "§164.308(a)(1)(ii)(B)", + "control_name": "Risk management" + }, + { + "framework": "ISO27001", + "control_id": "A.8.25", + "control_name": "Secure development lifecycle" + } + ], + "CWE-1104": [ + { + "framework": "PCI-DSS", + "control_id": "Req 6", + "control_name": "Develop and maintain secure systems and software" + }, + { + "framework": "SOC2", + "control_id": "CC8", + "control_name": "Change management" + }, + { + "framework": "OWASP", + "control_id": "A06:2021", + "control_name": "Vulnerable and Outdated Components" + }, + { + "framework": "HIPAA", + "control_id": "§164.308(a)(1)(ii)(B)", + "control_name": "Risk management" + }, + { + "framework": "ISO27001", + "control_id": "A.8.8", + "control_name": "Management of technical vulnerabilities" + } + ], + "CWE-16": [ + { + "framework": "PCI-DSS", + "control_id": "Req 2", + "control_name": "Apply secure configurations to all system components" + }, + { + "framework": "SOC2", + "control_id": "CC7", + "control_name": "System operations" + }, + { + "framework": "OWASP", + "control_id": "A05:2021", + "control_name": "Security Misconfiguration" + }, + { + "framework": "HIPAA", + "control_id": "§164.308(a)(1)(ii)(B)", + "control_name": "Risk management" + }, + { + "framework": "ISO27001", + "control_id": "A.8.9", + "control_name": "Configuration management" + } + ], + "CWE-259": [ + { + "framework": "PCI-DSS", + "control_id": "Req 8", + "control_name": "Identify users and authenticate access to system components" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A07:2021", + "control_name": "Identification and Authentication Failures" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(d)", + "control_name": "Person or entity authentication" + }, + { + "framework": "ISO27001", + "control_id": "A.5.17", + "control_name": "Authentication information" + } + ], + "CWE-321": [ + { + "framework": "PCI-DSS", + "control_id": "Req 3", + "control_name": "Protect stored account data" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A02:2021", + "control_name": "Cryptographic Failures" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(a)(2)(iv)", + "control_name": "Encryption and decryption" + }, + { + "framework": "ISO27001", + "control_id": "A.8.24", + "control_name": "Use of cryptography" + } + ], + "CWE-798": [ + { + "framework": "PCI-DSS", + "control_id": "Req 8", + "control_name": "Identify users and authenticate access to system components" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A07:2021", + "control_name": "Identification and Authentication Failures" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(d)", + "control_name": "Person or entity authentication" + }, + { + "framework": "ISO27001", + "control_id": "A.5.17", + "control_name": "Authentication information" + } + ], + "CWE-285": [ + { + "framework": "PCI-DSS", + "control_id": "Req 8", + "control_name": "Identify users and authenticate access to system components" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A01:2021", + "control_name": "Broken Access Control" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(a)(1)", + "control_name": "Access control" + }, + { + "framework": "ISO27001", + "control_id": "A.5.15", + "control_name": "Access control" + } + ], + "CWE-346": [ + { + "framework": "PCI-DSS", + "control_id": "Req 8", + "control_name": "Identify users and authenticate access to system components" + }, + { + "framework": "SOC2", + "control_id": "CC6", + "control_name": "Logical and physical access controls" + }, + { + "framework": "OWASP", + "control_id": "A01:2021", + "control_name": "Broken Access Control" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(a)(1)", + "control_name": "Access control" + }, + { + "framework": "ISO27001", + "control_id": "A.5.16", + "control_name": "Identity management" + } + ], + "CWE-601": [ + { + "framework": "PCI-DSS", + "control_id": "Req 6", + "control_name": "Develop and maintain secure systems and software" + }, + { + "framework": "SOC2", + "control_id": "CC7", + "control_name": "System operations" + }, + { + "framework": "OWASP", + "control_id": "A01:2021", + "control_name": "Broken Access Control" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(c)(1)", + "control_name": "Integrity" + }, + { + "framework": "ISO27001", + "control_id": "A.8.28", + "control_name": "Secure coding" + } + ], + "CWE-918": [ + { + "framework": "PCI-DSS", + "control_id": "Req 6", + "control_name": "Develop and maintain secure systems and software" + }, + { + "framework": "SOC2", + "control_id": "CC7", + "control_name": "System operations" + }, + { + "framework": "OWASP", + "control_id": "A10:2021", + "control_name": "Server-Side Request Forgery" + }, + { + "framework": "HIPAA", + "control_id": "§164.312(e)(1)", + "control_name": "Transmission security" + }, + { + "framework": "ISO27001", + "control_id": "A.8.20", + "control_name": "Network security" + } + ] +} diff --git a/internal/secaf/config/ai.go b/internal/secaf/config/ai.go new file mode 100644 index 0000000000..a513163c19 --- /dev/null +++ b/internal/secaf/config/ai.go @@ -0,0 +1,47 @@ +package config + +// AIIntegrationConfig is what remains of the original AIIntegrationConfig once +// the provider machinery is gone: the retry schedule the gates apply around a +// model call, and the model the AI gate asks for. +// +// The sec-af node built this from a dozen environment variables that chose an +// external coding-agent CLI, its binary, its model and the credentials it was +// handed. Inside codeaf none of that exists — the App behind the audit runs +// codeaf's own agent loop and model API, which already know their model, their +// key and their turn budget — so the provider, harness model, turn count, +// binary paths and provider environment were deleted rather than carried as +// settings nothing reads. What is left is plain configuration the codeaf side +// may set; DefaultAIConfig gives the values the node defaulted to. +type AIIntegrationConfig struct { + // AIModel is the model the AI gate names on its `.ai()` calls. Empty means + // "name none", leaving the choice to the App's AI implementation, which is + // what every agent outside the gate already does. + AIModel string `json:"ai_model"` + // MaxRetries is a count of RETRIES after the first attempt, so 0 means one + // attempt and a negative value means none (see gates.runWithRetry). + MaxRetries int `json:"max_retries"` + // InitialBackoffSeconds and MaxBackoffSeconds bound the exponential delay + // between attempts: min(initial * 2**attempt, max). + InitialBackoffSeconds float64 `json:"initial_backoff_seconds"` + MaxBackoffSeconds float64 `json:"max_backoff_seconds"` +} + +// The retry defaults the node read from SEC_AF_AI_MAX_RETRIES, +// SEC_AF_AI_INITIAL_BACKOFF_SECONDS and SEC_AF_AI_MAX_BACKOFF_SECONDS when those +// variables were unset. They are constants now because nothing in codeaf sets +// them from the environment. +const ( + DefaultAIMaxRetries = 3 + DefaultAIInitialBackoffSeconds = 2.0 + DefaultAIMaxBackoffSeconds = 8.0 +) + +// DefaultAIConfig returns the configuration a gate built without one uses: the +// node's retry schedule, and no model of its own. +func DefaultAIConfig() AIIntegrationConfig { + return AIIntegrationConfig{ + MaxRetries: DefaultAIMaxRetries, + InitialBackoffSeconds: DefaultAIInitialBackoffSeconds, + MaxBackoffSeconds: DefaultAIMaxBackoffSeconds, + } +} diff --git a/internal/secaf/config/audit.go b/internal/secaf/config/audit.go new file mode 100644 index 0000000000..801b657bb0 --- /dev/null +++ b/internal/secaf/config/audit.go @@ -0,0 +1,149 @@ +package config + +import ( + "encoding/json" + "fmt" +) + +// AuditConfig ports config.py AuditConfig — the runtime config the orchestrator +// phases consume (DESIGN.md §3 and §9). +// +// IncludePaths is `list[str] | None` in Python and a nil-able slice here: nil +// means "no include filter" and is distinct from an empty list. +type AuditConfig struct { + RepoPath string `json:"repo_path"` + Depth DepthProfile `json:"depth"` + SeverityThreshold string `json:"severity_threshold"` + ScanTypes []string `json:"scan_types"` + OutputFormats []string `json:"output_formats"` + ComplianceFrameworks []string `json:"compliance_frameworks"` + IncludePaths []string `json:"include_paths"` + ExcludePaths []string `json:"exclude_paths"` + Budget BudgetConfig `json:"budget"` +} + +// DefaultAuditConfig builds the pydantic field defaults. repo_path is +// `Field(...)` (required) in Python and has no default; callers supply it. +// +// Fresh slices are returned per call, mirroring `default_factory` — a shared +// package-level slice would let one audit's mutation leak into the next. +func DefaultAuditConfig() AuditConfig { + return AuditConfig{ + Depth: DepthStandard, + SeverityThreshold: "low", + ScanTypes: []string{"sast", "sca", "secrets", "config"}, + OutputFormats: []string{"json"}, + ComplianceFrameworks: []string{}, + IncludePaths: nil, + ExcludePaths: []string{"tests/", "vendor/", "node_modules/", ".git/"}, + Budget: DefaultBudgetConfig(), + } +} + +// UnmarshalJSON seeds the pydantic defaults before decoding. +func (c *AuditConfig) UnmarshalJSON(data []byte) error { + type alias AuditConfig + v := alias(DefaultAuditConfig()) + if err := json.Unmarshal(data, &v); err != nil { + return err + } + *c = AuditConfig(v) + return nil +} + +// AuditInputFields is the projection of schemas.AuditInput that +// AuditConfig.from_input reads. It exists so this package does not have to +// import internal/schemas (which owns the full AuditInput, including the ~12 +// fields from_input ignores): the json tags are the pydantic field names, so +// FromInput can project any AuditInput-shaped value onto it through JSON. +type AuditInputFields struct { + Depth string `json:"depth"` + SeverityThreshold string `json:"severity_threshold"` + ScanTypes []string `json:"scan_types"` + OutputFormats []string `json:"output_formats"` + ComplianceFrameworks []string `json:"compliance_frameworks"` + IncludePaths []string `json:"include_paths"` + ExcludePaths []string `json:"exclude_paths"` + MaxCostUSD *float64 `json:"max_cost_usd"` + MaxProvers *int `json:"max_provers"` + MaxDurationSeconds *int `json:"max_duration_seconds"` +} + +// FromInput ports AuditConfig.from_input (config.py:53, DESIGN.md §8.2): +// +// @classmethod +// def from_input(cls, audit_input: AuditInput, repo_path: str) -> "AuditConfig": +// depth = DepthProfile(audit_input.depth) +// return cls( +// repo_path=repo_path, +// depth=depth, +// severity_threshold=audit_input.severity_threshold, +// scan_types=audit_input.scan_types, +// output_formats=audit_input.output_formats, +// compliance_frameworks=audit_input.compliance_frameworks, +// include_paths=audit_input.include_paths, +// exclude_paths=audit_input.exclude_paths, +// budget=BudgetConfig( +// max_cost_usd=audit_input.max_cost_usd, +// max_provers=audit_input.max_provers, +// max_duration_seconds=audit_input.max_duration_seconds, +// ), +// ) +// +// Two parity points that are easy to get wrong: +// +// - `DepthProfile(audit_input.depth)` is the STRICT enum constructor, not +// _normalize_depth. An unknown depth raises ValueError and the audit fails; +// it does NOT silently become "standard". FromInput returns an error with +// Python's ValueError text. +// - the BudgetConfig is constructed with ONLY the three caps, so the +// percentages and concurrency limits keep their own defaults. +// +// The Python model also carried a `provider` field defaulting to the name of +// the external CLI the node drove. Nothing read it, and inside codeaf there is +// no such CLI, so the field was dropped. +// +// in may be any value that JSON-marshals to the AuditInput shape — in practice +// schemas.AuditInput once that package exists, or AuditInputFields directly. +// Pass the typed struct through FromInputFields when you already have one. +func (AuditConfig) FromInput(in any, repoPath string) (AuditConfig, error) { + fields, ok := in.(AuditInputFields) + if !ok { + b, err := json.Marshal(in) + if err != nil { + return AuditConfig{}, fmt.Errorf("config.FromInput: marshal %T: %w", in, err) + } + if err := json.Unmarshal(b, &fields); err != nil { + return AuditConfig{}, fmt.Errorf("config.FromInput: project %T onto AuditInputFields: %w", in, err) + } + } + return AuditConfig{}.FromInputFields(fields, repoPath) +} + +// FromInputFields is FromInput over the already-projected fields. +func (AuditConfig) FromInputFields(in AuditInputFields, repoPath string) (AuditConfig, error) { + depth := DepthProfile(in.Depth) + if !depth.IsValid() { + // Python: ValueError("'invalid' is not a valid DepthProfile") + return AuditConfig{}, fmt.Errorf("'%s' is not a valid DepthProfile", in.Depth) + } + + c := DefaultAuditConfig() + c.RepoPath = repoPath + c.Depth = depth + c.SeverityThreshold = in.SeverityThreshold + c.ScanTypes = in.ScanTypes + c.OutputFormats = in.OutputFormats + c.ComplianceFrameworks = in.ComplianceFrameworks + c.IncludePaths = in.IncludePaths + c.ExcludePaths = in.ExcludePaths + + // Only the three caps are passed to BudgetConfig(...); everything else keeps + // its pydantic default. + c.Budget = DefaultBudgetConfig() + c.Budget.MaxCostUSD = in.MaxCostUSD + c.Budget.MaxProvers = in.MaxProvers + c.Budget.MaxDurationSeconds = in.MaxDurationSeconds + + return c, nil +} diff --git a/internal/secaf/config/budget.go b/internal/secaf/config/budget.go new file mode 100644 index 0000000000..3a7ea90506 --- /dev/null +++ b/internal/secaf/config/budget.go @@ -0,0 +1,55 @@ +package config + +import "encoding/json" + +// BudgetConfig ports config.py BudgetConfig — the budget-enforcement thresholds +// from DESIGN.md §9.1. +// +// The three caps are `X | None` in Python and pointers here: nil means "no cap", +// which is NOT the same as a zero cap. The percentages and concurrency limits +// have non-zero pydantic defaults, so UnmarshalJSON seeds them (a BudgetConfig +// can arrive over the wire inside a checkpoint or a phase payload, where a Go +// zero value would silently mean "no hunters, no budget"). +type BudgetConfig struct { + MaxCostUSD *float64 `json:"max_cost_usd"` + MaxProvers *int `json:"max_provers"` + MaxDurationSeconds *int `json:"max_duration_seconds"` + + ReconBudgetPct float64 `json:"recon_budget_pct"` + HuntBudgetPct float64 `json:"hunt_budget_pct"` + ProveBudgetPct float64 `json:"prove_budget_pct"` + + MaxConcurrentHunters int `json:"max_concurrent_hunters"` + MaxConcurrentProvers int `json:"max_concurrent_provers"` + + HunterEarlyStopFileThreshold int `json:"hunter_early_stop_file_threshold"` +} + +// DefaultBudgetConfig builds the pydantic defaults verbatim: +// +// recon_budget_pct = 0.10, hunt_budget_pct = 0.45, prove_budget_pct = 0.45 +// max_concurrent_hunters = 4, max_concurrent_provers = 3 +// hunter_early_stop_file_threshold = 30 +// max_cost_usd / max_provers / max_duration_seconds = None +func DefaultBudgetConfig() BudgetConfig { + return BudgetConfig{ + ReconBudgetPct: 0.10, + HuntBudgetPct: 0.45, + ProveBudgetPct: 0.45, + MaxConcurrentHunters: 4, + MaxConcurrentProvers: 3, + HunterEarlyStopFileThreshold: 30, + } +} + +// UnmarshalJSON seeds the pydantic defaults before decoding, so keys absent from +// the payload keep their Python default rather than the Go zero value. +func (b *BudgetConfig) UnmarshalJSON(data []byte) error { + type alias BudgetConfig + v := alias(DefaultBudgetConfig()) + if err := json.Unmarshal(data, &v); err != nil { + return err + } + *b = BudgetConfig(v) + return nil +} diff --git a/internal/secaf/config/config_test.go b/internal/secaf/config/config_test.go new file mode 100644 index 0000000000..8920c74b78 --- /dev/null +++ b/internal/secaf/config/config_test.go @@ -0,0 +1,290 @@ +package config + +import ( + "encoding/json" + "math" + "reflect" + "testing" +) + +// This file ports the config-related tests from tests/test_config.py. Each Go +// test names the Python test it derives from so reviewers can diff coverage. + +func approx(t *testing.T, got, want float64) { + t.Helper() + if math.Abs(got-want) > 1e-9 { + t.Errorf("got %v, want ~%v", got, want) + } +} + +func ptrF(f float64) *float64 { return &f } +func ptrI(i int) *int { return &i } + +// sampleAuditInput mirrors tests/conftest.py::sample_audit_input. +func sampleAuditInput() AuditInputFields { + return AuditInputFields{ + Depth: "standard", + SeverityThreshold: "low", + ScanTypes: []string{"sast", "secrets", "config"}, + OutputFormats: []string{"json", "sarif", "markdown"}, + ComplianceFrameworks: []string{"PCI-DSS", "SOC2", "OWASP"}, + IncludePaths: []string{"src/"}, + ExcludePaths: []string{"tests/", "vendor/", ".git/"}, + MaxCostUSD: ptrF(10.0), + MaxProvers: ptrI(4), + MaxDurationSeconds: ptrI(900), + } +} + +// TestDepthProfileValuesAreStable ports +// test_config.py::test_depth_profile_values_are_stable. +func TestDepthProfileValuesAreStable(t *testing.T) { + if DepthQuick != "quick" { + t.Errorf("DepthQuick = %q", DepthQuick) + } + if DepthStandard != "standard" { + t.Errorf("DepthStandard = %q", DepthStandard) + } + if DepthThorough != "thorough" { + t.Errorf("DepthThorough = %q", DepthThorough) + } +} + +// TestBudgetConfigDefaultsSumTo100Percent ports +// test_config.py::test_budget_config_defaults_sum_to_100_percent. +func TestBudgetConfigDefaultsSumTo100Percent(t *testing.T) { + b := DefaultBudgetConfig() + approx(t, b.ReconBudgetPct+b.HuntBudgetPct+b.ProveBudgetPct, 1.0) + if b.MaxCostUSD != nil || b.MaxProvers != nil || b.MaxDurationSeconds != nil { + t.Errorf("caps default to non-None: %#v", b) + } + // The remaining pydantic defaults, which the Python test does not cover but + // the phases depend on. + if b.MaxConcurrentHunters != 4 || b.MaxConcurrentProvers != 3 || b.HunterEarlyStopFileThreshold != 30 { + t.Errorf("concurrency/threshold defaults = %#v", b) + } +} + +// TestAuditConfigFromInputMapsFieldsAndBudget ports +// test_config.py::test_audit_config_from_input_maps_fields_and_budget. +func TestAuditConfigFromInputMapsFieldsAndBudget(t *testing.T) { + cfg, err := AuditConfig{}.FromInputFields(sampleAuditInput(), "/tmp/sec-af-repo") + if err != nil { + t.Fatalf("FromInputFields: %v", err) + } + if cfg.RepoPath != "/tmp/sec-af-repo" { + t.Errorf("RepoPath = %q", cfg.RepoPath) + } + if cfg.Depth != DepthStandard { + t.Errorf("Depth = %q", cfg.Depth) + } + if !reflect.DeepEqual(cfg.ScanTypes, []string{"sast", "secrets", "config"}) { + t.Errorf("ScanTypes = %#v", cfg.ScanTypes) + } + if !reflect.DeepEqual(cfg.OutputFormats, []string{"json", "sarif", "markdown"}) { + t.Errorf("OutputFormats = %#v", cfg.OutputFormats) + } + if cfg.Budget.MaxCostUSD == nil || *cfg.Budget.MaxCostUSD != 10.0 { + t.Errorf("Budget.MaxCostUSD = %#v", cfg.Budget.MaxCostUSD) + } + if cfg.Budget.MaxProvers == nil || *cfg.Budget.MaxProvers != 4 { + t.Errorf("Budget.MaxProvers = %#v", cfg.Budget.MaxProvers) + } + if cfg.Budget.MaxDurationSeconds == nil || *cfg.Budget.MaxDurationSeconds != 900 { + t.Errorf("Budget.MaxDurationSeconds = %#v", cfg.Budget.MaxDurationSeconds) + } + // BudgetConfig(...) is constructed with only the three caps, so everything + // else keeps its own default. + if cfg.Budget.MaxConcurrentHunters != 4 || cfg.Budget.MaxConcurrentProvers != 3 { + t.Errorf("budget concurrency lost its defaults: %#v", cfg.Budget) + } + approx(t, cfg.Budget.ReconBudgetPct, 0.10) + // Fields from_input does not pass keep their pydantic defaults. + if cfg.SeverityThreshold != "low" { + t.Errorf("SeverityThreshold = %q", cfg.SeverityThreshold) + } + if !reflect.DeepEqual(cfg.IncludePaths, []string{"src/"}) { + t.Errorf("IncludePaths = %#v", cfg.IncludePaths) + } + if !reflect.DeepEqual(cfg.ExcludePaths, []string{"tests/", "vendor/", ".git/"}) { + t.Errorf("ExcludePaths = %#v", cfg.ExcludePaths) + } + if !reflect.DeepEqual(cfg.ComplianceFrameworks, []string{"PCI-DSS", "SOC2", "OWASP"}) { + t.Errorf("ComplianceFrameworks = %#v", cfg.ComplianceFrameworks) + } +} + +// TestAuditConfigRejectsInvalidDepth ports +// test_config.py::test_audit_config_rejects_invalid_depth — from_input uses the +// STRICT enum constructor, so an unknown depth fails rather than falling back to +// STANDARD the way _normalize_depth would. +func TestAuditConfigRejectsInvalidDepth(t *testing.T) { + in := sampleAuditInput() + in.Depth = "invalid" + _, err := AuditConfig{}.FromInputFields(in, "/tmp/sec-af-repo") + if err == nil { + t.Fatal("FromInputFields accepted an invalid depth") + } + if want := "'invalid' is not a valid DepthProfile"; err.Error() != want { + t.Errorf("error = %q, python ValueError = %q", err.Error(), want) + } +} + +// TestFromInputProjectsAnAuditInputShapedValue covers the `in any` projection +// path — the one the node package will use with schemas.AuditInput. +func TestFromInputProjectsAnAuditInputShapedValue(t *testing.T) { + // Deliberately a DIFFERENT struct with extra fields, standing in for + // schemas.AuditInput. + type auditInputLike struct { + RepoURL string `json:"repo_url"` + Branch string `json:"branch"` + Depth string `json:"depth"` + SeverityThreshold string `json:"severity_threshold"` + ScanTypes []string `json:"scan_types"` + OutputFormats []string `json:"output_formats"` + ComplianceFrameworks []string `json:"compliance_frameworks"` + IncludePaths []string `json:"include_paths"` + ExcludePaths []string `json:"exclude_paths"` + MaxCostUSD *float64 `json:"max_cost_usd"` + MaxProvers *int `json:"max_provers"` + MaxDurationSeconds *int `json:"max_duration_seconds"` + IsPR bool `json:"is_pr"` + } + in := auditInputLike{ + RepoURL: "https://github.com/Agent-Field/sec-af", + Branch: "main", + Depth: "thorough", + SeverityThreshold: "high", + ScanTypes: []string{"sast"}, + OutputFormats: []string{"json"}, + MaxCostUSD: ptrF(2.5), + IsPR: true, + } + cfg, err := AuditConfig{}.FromInput(in, "/repo") + if err != nil { + t.Fatalf("FromInput: %v", err) + } + if cfg.Depth != DepthThorough { + t.Errorf("Depth = %q", cfg.Depth) + } + if cfg.SeverityThreshold != "high" { + t.Errorf("SeverityThreshold = %q", cfg.SeverityThreshold) + } + if cfg.Budget.MaxCostUSD == nil || *cfg.Budget.MaxCostUSD != 2.5 { + t.Errorf("Budget.MaxCostUSD = %#v", cfg.Budget.MaxCostUSD) + } + if cfg.RepoPath != "/repo" { + t.Errorf("RepoPath = %q", cfg.RepoPath) + } +} + +// TestNormalizeDepth covers the lenient in-pipeline helper (phases.py:52 and its +// three verbatim copies): case-insensitive, and ANY unknown value becomes +// STANDARD. +func TestNormalizeDepth(t *testing.T) { + cases := []struct { + in string + want DepthProfile + }{ + {"quick", DepthQuick}, + {"standard", DepthStandard}, + {"thorough", DepthThorough}, + {"QUICK", DepthQuick}, + {"Thorough", DepthThorough}, + {"invalid", DepthStandard}, + {"", DepthStandard}, + {"deep", DepthStandard}, + {" quick", DepthStandard}, // python: DepthProfile(" quick") raises -> STANDARD + } + for _, c := range cases { + if got := NormalizeDepth(c.in); got != c.want { + t.Errorf("NormalizeDepth(%q) = %q, want %q", c.in, got, c.want) + } + } + // The business_logic.py `str | DepthProfile` variant: an already-typed + // profile round-trips unchanged. + if got := NormalizeDepth(string(DepthThorough)); got != DepthThorough { + t.Errorf("NormalizeDepth(DepthThorough) = %q", got) + } +} + +// TestDefaultAIConfigKeepsTheNodeDefaults pins the retry schedule the node +// read from SEC_AF_AI_MAX_RETRIES / SEC_AF_AI_INITIAL_BACKOFF_SECONDS / +// SEC_AF_AI_MAX_BACKOFF_SECONDS when they were unset, and that the default +// names no model, leaving the choice to the App. +func TestDefaultAIConfigKeepsTheNodeDefaults(t *testing.T) { + cfg := DefaultAIConfig() + if cfg.MaxRetries != 3 { + t.Errorf("MaxRetries = %d, want 3", cfg.MaxRetries) + } + approx(t, cfg.InitialBackoffSeconds, 2.0) + approx(t, cfg.MaxBackoffSeconds, 8.0) + if cfg.AIModel != "" { + t.Errorf("AIModel = %q, want empty (the App chooses)", cfg.AIModel) + } +} + +// TestDefaultAIConfigIgnoresTheEnvironment: the variables the node read no +// longer steer anything. +func TestDefaultAIConfigIgnoresTheEnvironment(t *testing.T) { + t.Setenv("SEC_AF_AI_MAX_RETRIES", "9") + t.Setenv("SEC_AF_AI_MODEL", "provider/model") + cfg := DefaultAIConfig() + if cfg.MaxRetries != 3 || cfg.AIModel != "" { + t.Errorf("DefaultAIConfig read the environment: %#v", cfg) + } +} + +// TestBudgetConfigUnmarshalSeedsDefaults: a BudgetConfig arriving over the wire +// (checkpoint, phase payload) must not silently become "0 hunters". +func TestBudgetConfigUnmarshalSeedsDefaults(t *testing.T) { + var b BudgetConfig + if err := json.Unmarshal([]byte(`{"max_provers": 7}`), &b); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if b.MaxProvers == nil || *b.MaxProvers != 7 { + t.Errorf("MaxProvers = %#v", b.MaxProvers) + } + if b.MaxConcurrentHunters != 4 || b.MaxConcurrentProvers != 3 || b.HunterEarlyStopFileThreshold != 30 { + t.Errorf("defaults lost: %#v", b) + } + approx(t, b.HuntBudgetPct, 0.45) +} + +// TestAuditConfigUnmarshalSeedsDefaults. +func TestAuditConfigUnmarshalSeedsDefaults(t *testing.T) { + var c AuditConfig + if err := json.Unmarshal([]byte(`{"repo_path": "/r"}`), &c); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if c.RepoPath != "/r" { + t.Errorf("RepoPath = %q", c.RepoPath) + } + if c.Depth != DepthStandard || c.SeverityThreshold != "low" { + t.Errorf("defaults lost: %#v", c) + } + if !reflect.DeepEqual(c.ScanTypes, []string{"sast", "sca", "secrets", "config"}) { + t.Errorf("ScanTypes = %#v", c.ScanTypes) + } + if !reflect.DeepEqual(c.ExcludePaths, []string{"tests/", "vendor/", "node_modules/", ".git/"}) { + t.Errorf("ExcludePaths = %#v", c.ExcludePaths) + } + if c.IncludePaths != nil { + t.Errorf("IncludePaths = %#v, want nil (python None)", c.IncludePaths) + } + if c.Budget.MaxConcurrentHunters != 4 { + t.Errorf("nested budget defaults lost: %#v", c.Budget) + } +} + +// TestDefaultAuditConfigReturnsFreshSlices: pydantic's default_factory hands +// each model its own list; a shared package-level slice would let one audit's +// mutation leak into the next. +func TestDefaultAuditConfigReturnsFreshSlices(t *testing.T) { + a := DefaultAuditConfig() + b := DefaultAuditConfig() + a.ScanTypes[0] = "mutated" + if b.ScanTypes[0] != "sast" { + t.Error("DefaultAuditConfig shares its slices between calls") + } +} diff --git a/internal/secaf/config/depth.go b/internal/secaf/config/depth.go new file mode 100644 index 0000000000..6900acd323 --- /dev/null +++ b/internal/secaf/config/depth.go @@ -0,0 +1,63 @@ +// Package config ports src/sec_af/config.py: the depth profiles, the budget +// thresholds, the runtime AuditConfig the orchestrator phases consume, and the +// retry and model settings the AI gates apply (AIIntegrationConfig). +// +// Nothing here reads the environment. The original read its AI settings from a +// chain of variables when the node booted; inside codeaf the provider half of +// that is gone and the rest are plain values with the node's defaults. +package config + +import "strings" + +// DepthProfile ports config.py DepthProfile — the scan-depth profile +// (DESIGN.md §9). It is a string type so it marshals to and from the same JSON +// values as the Python `class DepthProfile(str, Enum)`. +type DepthProfile string + +// The three profiles, with the exact Python enum values. +const ( + DepthQuick DepthProfile = "quick" + DepthStandard DepthProfile = "standard" + DepthThorough DepthProfile = "thorough" +) + +// IsValid reports whether d is one of the three declared profiles — the +// membership test `DepthProfile(value)` performs in Python. +func (d DepthProfile) IsValid() bool { + switch d { + case DepthQuick, DepthStandard, DepthThorough: + return true + } + return false +} + +// String returns the profile's wire value. +func (d DepthProfile) String() string { return string(d) } + +// NormalizeDepth ports the _normalize_depth helper that SEC-AF repeats verbatim +// in four modules — src/sec_af/reasoners/phases.py:52, +// agents/recon/__init__.py:62, agents/hunt/__init__.py:73 and +// agents/prove/__init__.py:47: +// +// def _normalize_depth(depth: str) -> DepthProfile: +// try: +// return DepthProfile(depth.lower()) +// except ValueError: +// return DepthProfile.STANDARD +// +// Lower-case first, and ANY unrecognised value silently becomes STANDARD. This +// is the LENIENT path used everywhere inside the pipeline; it is deliberately +// NOT what AuditConfig.FromInput does, which uses the strict `DepthProfile(...)` +// constructor and fails on a bad value. +// +// src/sec_af/agents/hunt/business_logic.py:26 declares a fifth copy that accepts +// `str | DepthProfile` and short-circuits when it is already a profile. In Go +// DepthProfile IS a string type, so NormalizeDepth(string(d)) covers both arms +// with the same result. +func NormalizeDepth(depth string) DepthProfile { + p := DepthProfile(strings.ToLower(depth)) + if p.IsValid() { + return p + } + return DepthStandard +} diff --git a/internal/secaf/diffanalysis/diffanalysis.go b/internal/secaf/diffanalysis/diffanalysis.go new file mode 100644 index 0000000000..af7d6e1b24 --- /dev/null +++ b/internal/secaf/diffanalysis/diffanalysis.go @@ -0,0 +1,340 @@ +// Package diffanalysis ports src/sec_af/diff_analysis.py — the diff-aware file +// selection SEC-AF uses in PR mode. +// +// The orchestrator builds one DiffAnalysis per run when the audit input is a PR +// (`is_pr` with a `base_commit_sha`) and feeds AllRelevantFiles to the hunt +// phase as `include_paths`, so the set of files this package computes decides +// what the hunters look at. +// +// self.diff_analysis = analyze_diff( +// str(self.repo_path), input.base_commit_sha, input.commit_sha or "HEAD", +// ) +// +// Everything here shells out to git and swallows failures the way Python does: +// a git that cannot be started, or that takes longer than 30 seconds, yields an +// EMPTY analysis (for the diff) or is skipped (for a blast-radius grep). A git +// that runs and exits non-zero is NOT a failure — Python passes check=False, so +// the (usually empty) stdout is used as-is. +package diffanalysis + +import ( + "bytes" + "context" + "os/exec" + "sort" + "strings" + "time" +) + +// DefaultHeadSHA is the value Python's `head_sha` parameter defaults to: +// +// def analyze_diff(repo_path: str, base_sha: str, head_sha: str = "HEAD") +// +// Go has no default arguments, so callers pass it explicitly. The only +// production caller (orchestrator.py) always supplies `input.commit_sha or +// "HEAD"`, i.e. this constant whenever the input carries no commit. +const DefaultHeadSHA = "HEAD" + +// gitTimeout is the per-invocation `timeout=30` both subprocess.run calls pass. +const gitTimeout = 30 * time.Second + +// DiffAnalysis is the result of analyzing the git diff between base and head. +// +// Ports the diff_analysis.py dataclass of the same name. The three list fields +// have `field(default_factory=list)`, so the zero value of a Go DiffAnalysis +// (nil slices) is NOT the Python default — use NewDiffAnalysis, which every +// constructor path in this package does, so the slices are always non-nil and +// marshal as `[]` rather than `null`. +type DiffAnalysis struct { + ChangedFiles []string `json:"changed_files"` + BlastRadiusFiles []string `json:"blast_radius_files"` + AllRelevantFiles []string `json:"all_relevant_files"` + BaseSHA string `json:"base_sha"` + HeadSHA string `json:"head_sha"` +} + +// NewDiffAnalysis returns the dataclass defaults: three empty lists, +// base_sha "" and head_sha "HEAD". +func NewDiffAnalysis() DiffAnalysis { + return DiffAnalysis{ + ChangedFiles: []string{}, + BlastRadiusFiles: []string{}, + AllRelevantFiles: []string{}, + BaseSHA: "", + HeadSHA: DefaultHeadSHA, + } +} + +// FileCount ports the `file_count` property: the number of files the hunters +// will actually be pointed at. +func (d DiffAnalysis) FileCount() int { return len(d.AllRelevantFiles) } + +// gitRunner runs one git invocation in dir and returns its stdout. +// +// It is a package variable purely so tests can script git, mirroring +// tests/test_diff_analysis.py's +// `monkeypatch.setattr("sec_af.diff_analysis.subprocess.run", _fake_run)`. +// argv includes "git" as its first element so a scripted runner can branch on +// argv[:2] exactly like the Python fake does. +var gitRunner = runGit + +// runGit is the production seam: `subprocess.run(argv, cwd=dir, +// capture_output=True, text=True, timeout=30, check=False)`. +// +// Python parity, in the two ways this differs from a naive exec.Command: +// +// - check=False means a NON-ZERO EXIT IS NOT AN ERROR. `git grep -l` exits 1 +// when nothing matched, and `git diff` exits non-zero for an unknown +// revision; Python reads result.stdout regardless. So an *exec.ExitError is +// reported back as (stdout, nil). +// - timeout=30 raises subprocess.TimeoutExpired, a subprocess.SubprocessError, +// which both call sites catch. A deadline hit here returns an error, which +// routes to the same handling. +// +// A git that cannot be spawned at all (not on PATH, cwd missing) is Python's +// OSError — also an error here, also caught by both call sites. +func runGit(ctx context.Context, dir string, argv []string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, gitTimeout) + defer cancel() + + cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) + cmd.Dir = dir + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + if ctxErr := ctx.Err(); ctxErr != nil { + // The deadline (or the caller's cancellation) killed git: + // subprocess.TimeoutExpired. + return "", ctxErr + } + if err != nil { + if _, isExit := err.(*exec.ExitError); isExit { + // check=False: a non-zero exit is a normal, non-raising outcome. + return stdout.String(), nil + } + return "", err + } + return stdout.String(), nil +} + +// AnalyzeDiff analyzes the git diff to find changed files and their blast +// radius. +// +// Ports src/sec_af/diff_analysis.py analyze_diff. Blast radius = files that +// import from, or are imported by, changed files — approximated by grepping the +// head tree for each changed file's module name. +// +// Never returns an error: every git failure degrades to an empty or partial +// result, exactly as the Python function does. +func AnalyzeDiff(ctx context.Context, repoPath, baseSHA, headSHA string) DiffAnalysis { + empty := NewDiffAnalysis() + empty.BaseSHA = baseSHA + empty.HeadSHA = headSHA + + stdout, err := gitRunner(ctx, repoPath, []string{ + "git", "diff", "--name-only", "--diff-filter=ACMR", baseSHA, headSHA, + }) + if err != nil { + // except (subprocess.SubprocessError, OSError): return DiffAnalysis(...) + return empty + } + + var changed []string + for _, line := range splitLines(stdout) { + if line != "" && IsScannable(line) { + changed = append(changed, line) + } + } + if len(changed) == 0 { + return empty + } + + // `changed` membership is tested per grep hit below; a set keeps that O(1) + // without changing behavior (Python's `file_path not in changed` is a list + // scan over the same elements). + changedSet := make(map[string]struct{}, len(changed)) + for _, f := range changed { + changedSet[f] = struct{}{} + } + + blastRadius := map[string]struct{}{} + for _, changedFile := range changed { + moduleName := FileToModule(changedFile) + if moduleName == "" { + continue + } + + grepOut, grepErr := gitRunner(ctx, repoPath, []string{ + "git", "grep", "-l", moduleName, headSHA, + "--", "*.py", "*.ts", "*.js", "*.go", "*.java", "*.rb", + }) + if grepErr != nil { + // except (subprocess.SubprocessError, OSError): continue + continue + } + for _, line := range splitLines(grepOut) { + if line == "" { + continue + } + // `git grep ` prefixes every hit with ":". + filePath := line + if i := strings.Index(line, ":"); i >= 0 { + filePath = line[i+1:] + } + if _, isChanged := changedSet[filePath]; isChanged { + continue + } + if IsScannable(filePath) { + blastRadius[filePath] = struct{}{} + } + } + } + + out := NewDiffAnalysis() + out.BaseSHA = baseSHA + out.HeadSHA = headSHA + out.ChangedFiles = sortedCopy(changed) + out.BlastRadiusFiles = sortedKeys(blastRadius) + + // all_relevant = sorted(set(changed) | blast_radius) + union := make(map[string]struct{}, len(changedSet)+len(blastRadius)) + for f := range changedSet { + union[f] = struct{}{} + } + for f := range blastRadius { + union[f] = struct{}{} + } + out.AllRelevantFiles = sortedKeys(union) + return out +} + +// skipDirs and skipExtensions are diff_analysis.py's tuples, in order. +var ( + skipDirs = []string{"tests/", "test/", "vendor/", "node_modules/", ".git/", "__pycache__/"} + skipExtensions = []string{".md", ".txt", ".yml", ".yaml", ".json", ".toml", ".cfg", ".ini", ".lock"} +) + +// IsScannable reports whether a file should be included in security scanning. +// +// Ports diff_analysis.py _is_scannable. Exported because the orchestrator's +// PR-mode path and the hunt include_paths plumbing want the same predicate; +// Python keeps it private only because the module is self-contained. +// +// Python parity, two quirks worth stating out loud: +// +// - The directory test is str.startswith, NOT a path-segment containment +// test. "src/tests/x.py" is scannable; only a TOP-LEVEL tests/ dir is +// skipped. ("atests/x.py" is likewise scannable — the prefix is "tests/", +// with the slash.) +// - The extension test is str.endswith and therefore case-sensitive: +// "a.lock" is skipped, "a.LOCK" is not. +func IsScannable(filePath string) bool { + for _, dir := range skipDirs { + if strings.HasPrefix(filePath, dir) { + return false + } + } + for _, ext := range skipExtensions { + if strings.HasSuffix(filePath, ext) { + return false + } + } + return true +} + +// FileToModule converts a file path to the importable module name used for the +// blast-radius grep. +// +// Ports diff_analysis.py _file_to_module. Exported for the same reason as +// IsScannable. +// +// Python parity — the three branches are literal transliterations, quirks +// included (VERIFIED against the venv interpreter): +// +// "src/service/user.py" -> "user" (dots, drop ".py", last segment) +// "src/service/user.test.py" -> "test" (same, so an infix dot wins) +// "foo..py" -> "" (empty last segment; callers skip it) +// "a.js.ts" -> "a" (".ts" stripped, then ".js") +// "a.ts.js" -> "a.ts" (".ts" does not match, ".js" does) +// "a/b.tar.gz" -> "b" (fallback: basename up to first dot) +// "a/.hidden" -> "" (fallback on a dotfile) +func FileToModule(filePath string) string { + if strings.HasSuffix(filePath, ".py") { + dotted := strings.ReplaceAll(filePath, "/", ".") + dotted = strings.TrimSuffix(dotted, ".py") + parts := strings.Split(dotted, ".") + return parts[len(parts)-1] + } + if strings.HasSuffix(filePath, ".ts") || strings.HasSuffix(filePath, ".js") { + base := lastSegment(filePath, "/") + base = strings.TrimSuffix(base, ".ts") + base = strings.TrimSuffix(base, ".js") + return base + } + base := lastSegment(filePath, "/") + return strings.Split(base, ".")[0] +} + +// lastSegment is Python's s.split(sep)[-1]. +func lastSegment(s, sep string) string { + if i := strings.LastIndex(s, sep); i >= 0 { + return s[i+len(sep):] + } + return s +} + +// splitLines reproduces Python's str.splitlines() for the separators git can +// realistically emit. +// +// Python parity: splitlines() splits on "\r\n", "\n" and "\r" (plus \v, \f, +// \x1c-\x1e, \x85, U+2028 and U+2029 — none of which appear in `git diff +// --name-only` or `git grep -l` output, and which are deliberately NOT handled +// here) and produces NO trailing empty element for text ending in a separator. +// strings.Split on "\n" would leave a trailing "" and keep a "\r" on every line +// of CRLF output; both call sites drop empty lines anyway, but the stray "\r" +// would corrupt a path. +func splitLines(s string) []string { + if s == "" { + return nil + } + var out []string + start := 0 + for i := 0; i < len(s); i++ { + switch s[i] { + case '\n': + out = append(out, s[start:i]) + start = i + 1 + case '\r': + out = append(out, s[start:i]) + if i+1 < len(s) && s[i+1] == '\n' { + i++ + } + start = i + 1 + } + } + if start < len(s) { + out = append(out, s[start:]) + } + return out +} + +// sortedCopy is Python's sorted(list) — a new, ascending, non-nil slice. +func sortedCopy(in []string) []string { + out := make([]string, len(in)) + copy(out, in) + sort.Strings(out) + return out +} + +// sortedKeys is Python's sorted(set) — ascending, non-nil even when empty +// (`blast_radius_files` has default_factory=list, never None). +func sortedKeys(set map[string]struct{}) []string { + out := make([]string, 0, len(set)) + for k := range set { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/secaf/diffanalysis/diffanalysis_test.go b/internal/secaf/diffanalysis/diffanalysis_test.go new file mode 100644 index 0000000000..c4a57356bc --- /dev/null +++ b/internal/secaf/diffanalysis/diffanalysis_test.go @@ -0,0 +1,497 @@ +package diffanalysis + +import ( + "context" + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" +) + +// scriptGit swaps the package's git seam for the duration of one test — the Go +// equivalent of `monkeypatch.setattr("sec_af.diff_analysis.subprocess.run", ...)`. +func scriptGit(t *testing.T, fn func(ctx context.Context, dir string, argv []string) (string, error)) { + t.Helper() + prev := gitRunner + gitRunner = fn + t.Cleanup(func() { gitRunner = prev }) +} + +// argvHead is the Go form of the Python fake's `command[:2]` dispatch. +func argvHead(argv []string, n int) string { + if len(argv) < n { + n = len(argv) + } + return strings.Join(argv[:n], " ") +} + +// TestAnalyzeDiffCollectsChangedAndBlastRadius ports +// tests/test_diff_analysis.py::test_analyze_diff_collects_changed_and_blast_radius. +func TestAnalyzeDiffCollectsChangedAndBlastRadius(t *testing.T) { + scriptGit(t, func(_ context.Context, _ string, argv []string) (string, error) { + switch argvHead(argv, 2) { + case "git diff": + return "src/service/user.py\nREADME.md\n", nil + case "git grep": + return "HEAD:src/api/users.py\nHEAD:tests/test_users.py\n", nil + } + t.Fatalf("unexpected command: %v", argv) + return "", nil + }) + + analysis := AnalyzeDiff(context.Background(), "/tmp/repo", "base-sha", "head-sha") + + if analysis.BaseSHA != "base-sha" { + t.Errorf("BaseSHA = %q, want %q", analysis.BaseSHA, "base-sha") + } + if analysis.HeadSHA != "head-sha" { + t.Errorf("HeadSHA = %q, want %q", analysis.HeadSHA, "head-sha") + } + // README.md is filtered by the .md extension rule; tests/test_users.py by + // the tests/ prefix rule. + if want := []string{"src/service/user.py"}; !reflect.DeepEqual(analysis.ChangedFiles, want) { + t.Errorf("ChangedFiles = %v, want %v", analysis.ChangedFiles, want) + } + if want := []string{"src/api/users.py"}; !reflect.DeepEqual(analysis.BlastRadiusFiles, want) { + t.Errorf("BlastRadiusFiles = %v, want %v", analysis.BlastRadiusFiles, want) + } + if want := []string{"src/api/users.py", "src/service/user.py"}; !reflect.DeepEqual(analysis.AllRelevantFiles, want) { + t.Errorf("AllRelevantFiles = %v, want %v", analysis.AllRelevantFiles, want) + } + if analysis.FileCount() != 2 { + t.Errorf("FileCount() = %d, want 2", analysis.FileCount()) + } +} + +// TestAnalyzeDiffReturnsEmptyOnGitFailure ports +// tests/test_diff_analysis.py::test_analyze_diff_returns_empty_on_git_failure — +// Python's `raise OSError("git not available")` from the patched subprocess.run. +func TestAnalyzeDiffReturnsEmptyOnGitFailure(t *testing.T) { + scriptGit(t, func(context.Context, string, []string) (string, error) { + return "", errors.New("git not available") + }) + + analysis := AnalyzeDiff(context.Background(), "/tmp/repo", "base-sha", DefaultHeadSHA) + + if analysis.BaseSHA != "base-sha" { + t.Errorf("BaseSHA = %q, want %q", analysis.BaseSHA, "base-sha") + } + if analysis.HeadSHA != "HEAD" { + t.Errorf("HeadSHA = %q, want %q", analysis.HeadSHA, "HEAD") + } + if len(analysis.ChangedFiles) != 0 || len(analysis.BlastRadiusFiles) != 0 || len(analysis.AllRelevantFiles) != 0 { + t.Fatalf("expected an empty analysis, got %+v", analysis) + } + // The dataclass fields have default_factory=list, so they must marshal as + // [] rather than null even on the failure path. + b, err := json.Marshal(analysis) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if strings.Contains(string(b), "null") { + t.Errorf("empty analysis marshaled with a null list: %s", b) + } +} + +// TestAnalyzeDiffEmptyDiffSkipsGrep proves the `if not changed: return` early +// exit: no blast-radius grep runs at all when every changed path is filtered. +func TestAnalyzeDiffEmptyDiffSkipsGrep(t *testing.T) { + var argvs [][]string + scriptGit(t, func(_ context.Context, _ string, argv []string) (string, error) { + argvs = append(argvs, argv) + if argvHead(argv, 2) == "git diff" { + return "README.md\ndocs/x.md\ntests/a.py\n\n", nil + } + t.Fatalf("grep must not run when nothing changed: %v", argv) + return "", nil + }) + + analysis := AnalyzeDiff(context.Background(), "/tmp/repo", "base", "head") + + if len(argvs) != 1 { + t.Fatalf("expected exactly one git invocation, got %d: %v", len(argvs), argvs) + } + if analysis.FileCount() != 0 { + t.Errorf("FileCount() = %d, want 0", analysis.FileCount()) + } +} + +// TestAnalyzeDiffGitInvocations pins the exact argv of both git calls — the +// port's contract with git, and the thing a "cleanup" refactor would silently +// change. +func TestAnalyzeDiffGitInvocations(t *testing.T) { + var argvs [][]string + var dirs []string + scriptGit(t, func(_ context.Context, dir string, argv []string) (string, error) { + argvs = append(argvs, argv) + dirs = append(dirs, dir) + if argvHead(argv, 2) == "git diff" { + return "src/a.py\n", nil + } + return "", nil + }) + + AnalyzeDiff(context.Background(), "/repo/root", "BASE", "HEADREV") + + wantDiff := []string{"git", "diff", "--name-only", "--diff-filter=ACMR", "BASE", "HEADREV"} + wantGrep := []string{"git", "grep", "-l", "a", "HEADREV", "--", "*.py", "*.ts", "*.js", "*.go", "*.java", "*.rb"} + if len(argvs) != 2 { + t.Fatalf("expected 2 git invocations, got %d: %v", len(argvs), argvs) + } + if !reflect.DeepEqual(argvs[0], wantDiff) { + t.Errorf("diff argv = %v, want %v", argvs[0], wantDiff) + } + if !reflect.DeepEqual(argvs[1], wantGrep) { + t.Errorf("grep argv = %v, want %v", argvs[1], wantGrep) + } + for i, dir := range dirs { + if dir != "/repo/root" { + t.Errorf("invocation %d ran in %q, want %q", i, dir, "/repo/root") + } + } +} + +// TestAnalyzeDiffGrepFailureIsSkipped covers the `except ...: continue` arm: +// one failing grep must not abort the others or the whole analysis. +func TestAnalyzeDiffGrepFailureIsSkipped(t *testing.T) { + scriptGit(t, func(_ context.Context, _ string, argv []string) (string, error) { + if argvHead(argv, 2) == "git diff" { + return "src/a.py\nsrc/b.py\n", nil + } + if argv[3] == "a" { // the module name for src/a.py + return "", errors.New("boom") + } + return "HEAD:src/uses_b.py\n", nil + }) + + analysis := AnalyzeDiff(context.Background(), "/tmp/repo", "base", "HEAD") + + if want := []string{"src/a.py", "src/b.py"}; !reflect.DeepEqual(analysis.ChangedFiles, want) { + t.Errorf("ChangedFiles = %v, want %v", analysis.ChangedFiles, want) + } + if want := []string{"src/uses_b.py"}; !reflect.DeepEqual(analysis.BlastRadiusFiles, want) { + t.Errorf("BlastRadiusFiles = %v, want %v", analysis.BlastRadiusFiles, want) + } +} + +// TestAnalyzeDiffSkipsEmptyModuleNames proves the `if not module_name: continue` +// guard: "foo..py" yields an empty module name, so no grep runs for it. +func TestAnalyzeDiffSkipsEmptyModuleNames(t *testing.T) { + var grepModules []string + scriptGit(t, func(_ context.Context, _ string, argv []string) (string, error) { + if argvHead(argv, 2) == "git diff" { + return "foo..py\nsrc/real.py\n", nil + } + grepModules = append(grepModules, argv[3]) + return "", nil + }) + + AnalyzeDiff(context.Background(), "/tmp/repo", "base", "HEAD") + + if want := []string{"real"}; !reflect.DeepEqual(grepModules, want) { + t.Errorf("grep module names = %v, want %v", grepModules, want) + } +} + +// TestAnalyzeDiffGrepLineWithoutColon covers `line.split(":", 1)[1] if ":" in +// line else line` — `git grep -l` without a tree-ish prints a bare path. +func TestAnalyzeDiffGrepLineWithoutColon(t *testing.T) { + scriptGit(t, func(_ context.Context, _ string, argv []string) (string, error) { + if argvHead(argv, 2) == "git diff" { + return "src/a.py\n", nil + } + return "src/plain.py\nHEAD:src/prefixed.py\n", nil + }) + + analysis := AnalyzeDiff(context.Background(), "/tmp/repo", "base", "HEAD") + + want := []string{"src/plain.py", "src/prefixed.py"} + if !reflect.DeepEqual(analysis.BlastRadiusFiles, want) { + t.Errorf("BlastRadiusFiles = %v, want %v", analysis.BlastRadiusFiles, want) + } +} + +// TestAnalyzeDiffDeduplicatesBlastRadius proves the set semantics of +// `blast_radius: set[str]` — the same hit reported for two changed files +// appears once — and that a changed file is never its own blast radius. +func TestAnalyzeDiffDeduplicatesBlastRadius(t *testing.T) { + scriptGit(t, func(_ context.Context, _ string, argv []string) (string, error) { + if argvHead(argv, 2) == "git diff" { + return "src/a.py\nsrc/b.py\n", nil + } + return "HEAD:src/shared.py\nHEAD:src/a.py\nHEAD:src/b.py\n", nil + }) + + analysis := AnalyzeDiff(context.Background(), "/tmp/repo", "base", "HEAD") + + if want := []string{"src/shared.py"}; !reflect.DeepEqual(analysis.BlastRadiusFiles, want) { + t.Errorf("BlastRadiusFiles = %v, want %v", analysis.BlastRadiusFiles, want) + } + if want := []string{"src/a.py", "src/b.py", "src/shared.py"}; !reflect.DeepEqual(analysis.AllRelevantFiles, want) { + t.Errorf("AllRelevantFiles = %v, want %v", analysis.AllRelevantFiles, want) + } +} + +// TestIsScannable is a table of the ground truth produced by running +// `sec_af.diff_analysis._is_scannable` under +// ~/.agentfield/packages/sec-af/venv/bin/python. +func TestIsScannable(t *testing.T) { + cases := []struct { + path string + want bool + }{ + {"src/service/user.py", true}, + {"README.md", false}, + {"a/b.tar.gz", true}, + {".hidden", true}, + {"a/.hidden", true}, + {"foo..py", true}, + {"main.go", true}, + {"Makefile", true}, + {"x/y", true}, + {"", true}, + {"tests/test_x.py", false}, + {"test/x.py", false}, + {"vendor/x.go", false}, + {"node_modules/x.js", false}, + {".git/config", false}, + {"__pycache__/x.pyc", false}, + // Python parity: startswith, not a path-segment test. + {"atests/x.py", true}, + {"src/tests/x.py", true}, + // Python parity: endswith is case-sensitive. + {"a.lock", false}, + {"a.LOCK", true}, + {"a.Yaml", true}, + {"pkg.toml", false}, + {"conf.cfg", false}, + {"conf.ini", false}, + {"data.json", false}, + {"data.yml", false}, + {"data.yaml", false}, + {"notes.txt", false}, + } + for _, tc := range cases { + if got := IsScannable(tc.path); got != tc.want { + t.Errorf("IsScannable(%q) = %v, want %v", tc.path, got, tc.want) + } + } +} + +// TestFileToModule is a table of the ground truth produced by running +// `sec_af.diff_analysis._file_to_module` under the venv interpreter. +func TestFileToModule(t *testing.T) { + cases := []struct{ path, want string }{ + {"src/service/user.py", "user"}, + {"README.md", "README"}, + {"a/b.tar.gz", "b"}, + {".hidden", ""}, + {"a/.hidden", ""}, + {"foo..py", ""}, + {"a.js.ts", "a"}, + {"a.ts.js", "a.ts"}, + {"lib/x.ts", "x"}, + {"lib/x.js", "x"}, + {"main.go", "main"}, + {"Makefile", "Makefile"}, + {"src/service/user.test.py", "test"}, + {"x/y", "y"}, + {"", ""}, + {"tests/test_x.py", "test_x"}, + {".git/config", "config"}, + } + for _, tc := range cases { + if got := FileToModule(tc.path); got != tc.want { + t.Errorf("FileToModule(%q) = %q, want %q", tc.path, got, tc.want) + } + } +} + +// TestSplitLines pins the Python-splitlines() behavior the two parsers depend +// on: no trailing empty element, and CRLF handled without leaving a stray \r on +// the path. +func TestSplitLines(t *testing.T) { + cases := []struct { + in string + want []string + }{ + {"", nil}, + {"a\n", []string{"a"}}, + {"a\nb\n", []string{"a", "b"}}, + {"a\nb", []string{"a", "b"}}, + {"a\r\nb\r\n", []string{"a", "b"}}, + {"a\rb\r", []string{"a", "b"}}, + {"\n", []string{""}}, + {"a\n\nb\n", []string{"a", "", "b"}}, + } + for _, tc := range cases { + if got := splitLines(tc.in); !reflect.DeepEqual(got, tc.want) { + t.Errorf("splitLines(%q) = %#v, want %#v", tc.in, got, tc.want) + } + } +} + +// TestNewDiffAnalysisDefaults pins the dataclass field defaults. +func TestNewDiffAnalysisDefaults(t *testing.T) { + d := NewDiffAnalysis() + if d.BaseSHA != "" { + t.Errorf("BaseSHA = %q, want empty", d.BaseSHA) + } + if d.HeadSHA != "HEAD" { + t.Errorf("HeadSHA = %q, want %q", d.HeadSHA, "HEAD") + } + if d.ChangedFiles == nil || d.BlastRadiusFiles == nil || d.AllRelevantFiles == nil { + t.Fatalf("default_factory=list fields must be non-nil: %+v", d) + } + if d.FileCount() != 0 { + t.Errorf("FileCount() = %d, want 0", d.FileCount()) + } +} + +// --------------------------------------------------------------------------- +// Real-git end-to-end coverage +// --------------------------------------------------------------------------- + +// gitExec runs a git command in dir with a hermetic environment (no user or +// system config, deterministic identity) and fails the test on error. +func gitExec(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), + "GIT_CONFIG_GLOBAL=/dev/null", + "GIT_CONFIG_SYSTEM=/dev/null", + "GIT_AUTHOR_NAME=sec-af", "GIT_AUTHOR_EMAIL=sec-af@example.com", + "GIT_COMMITTER_NAME=sec-af", "GIT_COMMITTER_EMAIL=sec-af@example.com", + "GIT_TERMINAL_PROMPT=0", + ) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } +} + +func writeFile(t *testing.T, dir, rel, body string) { + t.Helper() + full := filepath.Join(dir, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", full, err) + } + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + t.Fatalf("write %s: %v", full, err) + } +} + +// TestAnalyzeDiffAgainstRealGitRepo drives the UNPATCHED runGit against a +// throwaway repository, so the argv this package builds is validated against +// real git semantics (the `:` grep prefix, the `-- *.py` +// pathspecs matching nested paths, `--diff-filter=ACMR` selecting adds and +// modifications). +func TestAnalyzeDiffAgainstRealGitRepo(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not on PATH") + } + repo := t.TempDir() + gitExec(t, repo, "init", "--quiet", "-b", "main") + + // Base commit: an importer and an unrelated test file already reference the + // module the next commit will touch. + writeFile(t, repo, "src/api/users.py", "from src.service.user import lookup\n") + writeFile(t, repo, "tests/test_users.py", "from src.service.user import lookup\n") + writeFile(t, repo, "docs/user.md", "user docs\n") + gitExec(t, repo, "add", ".") + gitExec(t, repo, "commit", "--quiet", "-m", "base") + + // Head commit: add the module itself plus a filtered-out README change. + writeFile(t, repo, "src/service/user.py", "def lookup():\n return None\n") + writeFile(t, repo, "README.md", "# readme\n") + gitExec(t, repo, "add", ".") + gitExec(t, repo, "commit", "--quiet", "-m", "head") + + analysis := AnalyzeDiff(context.Background(), repo, "HEAD~1", "HEAD") + + if want := []string{"src/service/user.py"}; !reflect.DeepEqual(analysis.ChangedFiles, want) { + t.Errorf("ChangedFiles = %v, want %v", analysis.ChangedFiles, want) + } + // docs/user.md is not a grep pathspec match (*.py/*.ts/*.js/*.go/*.java/*.rb) + // and would be filtered by the .md rule anyway; tests/test_users.py matches + // the grep but is filtered by the tests/ rule. + if want := []string{"src/api/users.py"}; !reflect.DeepEqual(analysis.BlastRadiusFiles, want) { + t.Errorf("BlastRadiusFiles = %v, want %v", analysis.BlastRadiusFiles, want) + } + if want := []string{"src/api/users.py", "src/service/user.py"}; !reflect.DeepEqual(analysis.AllRelevantFiles, want) { + t.Errorf("AllRelevantFiles = %v, want %v", analysis.AllRelevantFiles, want) + } + if analysis.FileCount() != 2 { + t.Errorf("FileCount() = %d, want 2", analysis.FileCount()) + } +} + +// TestAnalyzeDiffRealGitBadRevision proves the check=False contract end to end: +// `git diff` against a revision that does not exist exits non-zero, and Python +// reads its empty stdout rather than raising — so the result is an empty +// analysis, not a crash. +func TestAnalyzeDiffRealGitBadRevision(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not on PATH") + } + repo := t.TempDir() + gitExec(t, repo, "init", "--quiet", "-b", "main") + writeFile(t, repo, "src/a.py", "x = 1\n") + gitExec(t, repo, "add", ".") + gitExec(t, repo, "commit", "--quiet", "-m", "only") + + analysis := AnalyzeDiff(context.Background(), repo, "does-not-exist", "HEAD") + + if analysis.FileCount() != 0 { + t.Fatalf("expected an empty analysis, got %+v", analysis) + } + if analysis.BaseSHA != "does-not-exist" || analysis.HeadSHA != "HEAD" { + t.Errorf("SHAs not carried through: %+v", analysis) + } +} + +// TestRunGitMissingBinaryIsAnError proves the OSError arm of runGit: a binary +// that cannot be spawned is an error (not an empty success), which AnalyzeDiff +// turns into the empty analysis. +func TestRunGitMissingBinaryIsAnError(t *testing.T) { + _, err := runGit(context.Background(), t.TempDir(), []string{"sec-af-no-such-binary-xyz", "diff"}) + if err == nil { + t.Fatal("expected an error for a missing binary") + } +} + +// TestRunGitNonZeroExitIsNotAnError proves the check=False arm of runGit +// directly, without git: a command that prints to stdout and exits 1 yields +// (stdout, nil). +func TestRunGitNonZeroExitIsNotAnError(t *testing.T) { + sh, err := exec.LookPath("sh") + if err != nil { + t.Skip("sh not on PATH") + } + out, err := runGit(context.Background(), t.TempDir(), []string{sh, "-c", "printf 'a\\nb\\n'; exit 1"}) + if err != nil { + t.Fatalf("non-zero exit must not be an error, got %v", err) + } + if out != "a\nb\n" { + t.Errorf("stdout = %q, want %q", out, "a\nb\n") + } +} + +// TestRunGitCancelledContextIsAnError proves the TimeoutExpired arm: a +// cancelled context surfaces as an error, so AnalyzeDiff degrades rather than +// treating a killed git's empty stdout as "nothing changed". +func TestRunGitCancelledContextIsAnError(t *testing.T) { + sh, err := exec.LookPath("sh") + if err != nil { + t.Skip("sh not on PATH") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := runGit(ctx, t.TempDir(), []string{sh, "-c", "echo hi"}); err == nil { + t.Fatal("expected an error for a cancelled context") + } +} diff --git a/internal/secaf/focus/focus.go b/internal/secaf/focus/focus.go new file mode 100644 index 0000000000..799ae5aa35 --- /dev/null +++ b/internal/secaf/focus/focus.go @@ -0,0 +1,72 @@ +// Package focus is what an audit of the changes tells its hunters: which files +// changed, which sit near them, and the change itself. +// +// sec-af's PR mode filtered findings to the changed files only AFTER every +// hunter had scanned the whole repository, so an audit of a ten-line change +// cost a whole audit. Here the scan step is told the change up front and +// reports only what the change touches or makes reachable; the filter after +// the hunt stays, as the backstop it was. +// +// IT RIDES THE CONTEXT, because the hunters are reached through sec-af's own +// tree of calls between reasoners, which carry the context in process and +// nothing else that the algorithm does not already read. An audit of the whole +// repository carries none, and its hunters' prompts are byte for byte the +// prompts sec-af wrote. +package focus + +import ( + "context" + "strings" +) + +// Focus is the change an audit is about. +type Focus struct { + // Base is how the change's starting point is said: `origin/dev`. + Base string + // Changed is the files the change touched; Nearby the files that name + // one of them. + Changed, Nearby []string + // Diff is the change itself, already cut to a size a prompt holds. + Diff string +} + +type key struct{} + +// With is ctx carrying f. +func With(ctx context.Context, f Focus) context.Context { return context.WithValue(ctx, key{}, f) } + +// From is the focus ctx carries, and false for an audit of the whole +// repository. +func From(ctx context.Context) (Focus, bool) { + f, ok := ctx.Value(key{}).(Focus) + return f, ok && len(f.Changed) > 0 +} + +// Section is the words appended to a hunter's scan prompt. +func (f Focus) Section() string { + var b strings.Builder + b.WriteString("\n\nCHANGE UNDER AUDIT:\n") + b.WriteString("This audit covers a change to the repository, not the whole of it. Report only locations in the changed files, ") + b.WriteString("or in the files near them where the change makes the code reachable or exploitable. Code elsewhere is context: ") + b.WriteString("read it to trace where data comes from and goes, but do not report it.\n") + b.WriteString("\nChanged files") + if f.Base != "" { + b.WriteString(" (since " + f.Base + ")") + } + b.WriteString(":\n") + for _, file := range f.Changed { + b.WriteString("- " + file + "\n") + } + if len(f.Nearby) > 0 { + b.WriteString("\nFiles near the change (they name a changed file):\n") + for _, file := range f.Nearby { + b.WriteString("- " + file + "\n") + } + } + if diff := strings.TrimSpace(f.Diff); diff != "" { + b.WriteString("\nThe change (git diff):\n") + b.WriteString(diff) + b.WriteString("\n") + } + return b.String() +} diff --git a/internal/secaf/gates/aigate.go b/internal/secaf/gates/aigate.go new file mode 100644 index 0000000000..f3998e6976 --- /dev/null +++ b/internal/secaf/gates/aigate.go @@ -0,0 +1,265 @@ +package gates + +import ( + "context" + "sync" + + "github.com/Agent-Field/agentfield/sdk/go/ai" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/aix" + "github.com/Agent-Field/codeaf/internal/secaf/config" + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// AIGate ports harness.py AIGateWrapper — the retrying front door to +// `app.ai(system=..., user=..., schema=Model, model=config.ai_model)`. +// +// Python: +// +// class AIGateWrapper(_RetryMixin): +// def __init__(self, app, config=None): +// self.app = app +// self.config = config or AIIntegrationConfig.from_env() +// self._cost_tracker = _CostTracker() +// +// The Go value is safe for concurrent use: the orchestrator's +// _assess_reachability_parallel fans AssessReachability out under a +// semaphore(min(5, n)), so the cost tracker is mutex-guarded and every +// invocation gets its own response capture. +type AIGate struct { + // App is the `.ai(...)` seam. Only AIer is required, which lets a caller + // pass the codeaf App, an appx.Fake, or the orchestrator's budget-checking + // proxy. + App appx.AIer + // Config supplies ai_model plus the retry schedule. + Config config.AIIntegrationConfig + // Sleep is the backoff sleeper; nil means the real clock. Tests set it to + // record the schedule without waiting. + Sleep Sleeper + + mu sync.Mutex + totalCostUSD float64 + invocationCount int +} + +// NewAIGate is `AIGateWrapper(app=app, config=config)`. A nil config takes +// config.DefaultAIConfig(), where Python read `AIIntegrationConfig.from_env()`. +// +// The error return is kept from the days the environment could hold a +// malformed retry count; nothing reads the environment now, so it is always +// nil, and the signature stays so the orchestrator's construction path keeps +// one shape. +func NewAIGate(app appx.AIer, cfg *config.AIIntegrationConfig) (*AIGate, error) { + if cfg != nil { + return &AIGate{App: app, Config: *cfg}, nil + } + return &AIGate{App: app, Config: config.DefaultAIConfig()}, nil +} + +// TotalCostUSD ports the `total_cost_usd` property. +func (g *AIGate) TotalCostUSD() float64 { + g.mu.Lock() + defer g.mu.Unlock() + return g.totalCostUSD +} + +// InvocationCount ports the `invocation_count` property. +func (g *AIGate) InvocationCount() int { + g.mu.Lock() + defer g.mu.Unlock() + return g.invocationCount +} + +// registerInvocation ports _CostTracker.register_invocation. +func (g *AIGate) registerInvocation() { + g.mu.Lock() + g.invocationCount++ + g.mu.Unlock() +} + +// registerCost ports _CostTracker.register_cost: +// +// if cost_usd is None or cost_usd < 0: return +// self.total_cost_usd += cost_usd +func (g *AIGate) registerCost(costUSD *float64) { + if costUSD == nil || *costUSD < 0 { + return + } + g.mu.Lock() + g.totalCostUSD += *costUSD + g.mu.Unlock() +} + +// sleeper resolves the injected Sleeper or the real one. +func (g *AIGate) sleeper() Sleeper { + if g.Sleep != nil { + return g.Sleep + } + return sleepReal +} + +// responseCapture wraps an appx.AIer to keep the last *ai.Response the SDK +// returned, so Invoke can read its cost. One capture is allocated per Invoke +// call, which is what makes concurrent gate use race-free. +type responseCapture struct { + inner appx.AIer + resp *ai.Response +} + +func (c *responseCapture) AI(ctx context.Context, prompt string, opts ...ai.Option) (*ai.Response, error) { + resp, err := c.inner.AI(ctx, prompt, opts...) + if resp != nil { + c.resp = resp + } + return resp, err +} + +// Invoke ports AIGateWrapper.invoke: +// +// self._cost_tracker.register_invocation() +// async def _operation(): +// return await self.app.ai(system=system, user=user, schema=schema, model=self.config.ai_model) +// result = await self._run_with_retry(_operation, self.config) +// self._cost_tracker.register_cost(getattr(result, "cost_usd", None)) +// if isinstance(result, schema): return result +// if isinstance(result, dict): return schema(**result) +// raise AIIntegrationError(f"AI gate returned invalid payload for schema {schema.__name__}") +// +// It is a package-level function rather than a method because Go methods cannot +// carry their own type parameter, and the schema is exactly that. +// +// Python parity notes: +// +// - register_invocation fires ONCE per Invoke, before the first attempt — a +// retried call still counts as one invocation. +// - `system=None` is spelled as the empty string here; aix.Structured omits +// the system message for it, matching the Python SDK, which drops a None +// system. +// - The two isinstance branches collapse into aix.Structured's typed return: +// the Go SDK hands back raw text, aix parses it into T, and a parse failure +// is an error — the same outcome as the trailing `raise`. The dict branch +// exists in Python only because the SDK may hand back an unparsed dict. +// - COST: this is a DELIBERATE, design-doc-mandated divergence. Python reads +// `getattr(result, "cost_usd", None)` off the PARSED PYDANTIC MODEL the SDK +// returns for a `schema=` call (agent_ai.py:898 `return schema(**json_data)`), +// and no gate schema has a cost_usd field — so AIGateWrapper.total_cost_usd +// is permanently 0.0 in the Python node. DESIGN.md §2/§3 specifies the Go +// port take the cost from the SDK response's Usage.Cost instead, which is +// where the real number lives. Everything else about the tracker +// (invocation counting, the negative/None guard, the += accumulation) is +// byte-for-byte Python. +func Invoke[T any](ctx context.Context, g *AIGate, user, system string) (T, error) { + g.registerInvocation() + + capture := &responseCapture{inner: g.App} + // An empty AIModel names no model at all, so the App's own choice stands — + // an `ai.WithModel("")` would be an explicit request for the empty model. + var extra []ai.Option + if g.Config.AIModel != "" { + extra = append(extra, ai.WithModel(g.Config.AIModel)) + } + operation := func() (T, error) { + return aix.StructuredOpts[T](ctx, capture, system, user, extra...) + } + + result, err := runWithRetry(ctx, g.Config, g.sleeper(), operation) + if err != nil { + var zero T + return zero, err + } + if capture.resp != nil && capture.resp.Usage != nil { + g.registerCost(capture.resp.Usage.Cost) + } + return result, nil +} + +// ClassifySeverity ports AIGateWrapper.classify_severity. +// +// The prompt is three adjacent string literals plus the summary after a blank +// line; it is reproduced byte-for-byte, including the trailing space after +// "keep rationale brief." coming from the implicit concatenation. +func (g *AIGate) ClassifySeverity(ctx context.Context, findingSummary string) (schemas.SeverityClassification, error) { + prompt := "Classify severity for this potential security finding. " + + "Use only critical/high/medium/low and keep rationale brief.\n\n" + + findingSummary + return Invoke[schemas.SeverityClassification](ctx, g, prompt, "") +} + +// CheckDuplicate ports AIGateWrapper.check_duplicate: +// +// prompt = ("Decide whether candidate finding is a duplicate of existing finding. " +// "Return duplicate decision only.\n\n" +// f"Candidate: {candidate}\n" +// f"Existing: {existing}") +// +// `candidate` and `existing` are `dict[str, Any]` in Python and land in the +// prompt through an f-string, i.e. `str(dict)` — which is `repr(dict)`, the +// `{'key': 'value'}` spelling with SINGLE quotes and `True`/`False`/`None` +// literals, rendered in the dict's INSERTION order. +// +// They are typed `any` here so a caller can pass a pyfmt.Ordered built in that +// insertion order and get byte-identical output. A plain map[string]any also +// works but renders with SORTED keys (pyfmt.Repr's documented deviation), which +// is only a difference when the Python dict was not already in sorted order. +// +// Python parity: check_duplicate has NO caller in the Python tree — +// agents/dedup.py does its semantic duplicate pass with a direct +// `app.ai(..., schema=DuplicateCheck)` rather than through the gate. It is +// ported for completeness. +func (g *AIGate) CheckDuplicate(ctx context.Context, candidate, existing any) (schemas.DuplicateCheck, error) { + prompt := "Decide whether candidate finding is a duplicate of existing finding. " + + "Return duplicate decision only.\n\n" + + "Candidate: " + pyfmt.Str(candidate) + "\n" + + "Existing: " + pyfmt.Str(existing) + return Invoke[schemas.DuplicateCheck](ctx, g, prompt, "") +} + +// SelectStrategy ports AIGateWrapper.select_strategy: +// +// prompt = ("Select SEC-AF hunt strategies from recon context. Return only selected strategies and rationale.\n" +// f"Depth profile: {depth}\n" +// f"Default candidates: {default_candidates}\n" +// f"Recon summary: {recon_summary}") +// +// Python parity: `default_candidates` is a `list[str]` interpolated by an +// f-string, so it renders as a Python LIST REPR — +// `['injection', 'auth', 'crypto']`, single quotes and ", " separators, `[]` +// when empty. pyfmt.Repr produces exactly that. This is the live call site +// (reasoners/phases.py:262 passes `[s.value for s in default_candidates]`). +func (g *AIGate) SelectStrategy(ctx context.Context, reconSummary, depth string, defaultCandidates []string) (schemas.StrategySelection, error) { + prompt := "Select SEC-AF hunt strategies from recon context. Return only selected strategies and rationale.\n" + + "Depth profile: " + depth + "\n" + + "Default candidates: " + pyfmt.Repr(defaultCandidates) + "\n" + + "Recon summary: " + reconSummary + return Invoke[schemas.StrategySelection](ctx, g, prompt, "") +} + +// AssessReachability ports AIGateWrapper.assess_reachability — the gate the +// orchestrator actually leans on (orchestrator.py:587, inside +// _assess_reachability_parallel). +func (g *AIGate) AssessReachability(ctx context.Context, findingSummary string) (schemas.ReachabilityGate, error) { + prompt := "Assess the reachability of this security finding. " + + "Determine if it is externally_reachable, requires_auth, internal_only, or unreachable. " + + "Consider the attack surface, authentication requirements, and network exposure.\n\n" + + findingSummary + return Invoke[schemas.ReachabilityGate](ctx, g, prompt, "") +} + +// BuildAIIntegration ports harness.py build_ai_integration: +// +// resolved = config or AIIntegrationConfig.from_env() +// return HarnessWrapper(app=app, config=resolved), AIGateWrapper(app=app, config=resolved) +// +// Both wrappers share ONE resolved config instance in Python; they share one +// value here, with config.DefaultAIConfig() standing in for from_env. Nothing +// in the Python tree calls this function — orchestrator.py constructs +// AIGateWrapper directly — so it exists for completeness. +func BuildAIIntegration(app appx.App, cfg *config.AIIntegrationConfig) (*HarnessWrapper, *AIGate, error) { + resolved := config.DefaultAIConfig() + if cfg != nil { + resolved = *cfg + } + return &HarnessWrapper{App: app, Config: resolved}, &AIGate{App: app, Config: resolved}, nil +} diff --git a/internal/secaf/gates/aigate_test.go b/internal/secaf/gates/aigate_test.go new file mode 100644 index 0000000000..3313510448 --- /dev/null +++ b/internal/secaf/gates/aigate_test.go @@ -0,0 +1,346 @@ +package gates + +// Parity tests for harness.py AIGateWrapper. +// +// The four prompt builders are pinned against committed goldens produced by +// go/scripts/gen_golden.py from the same f-strings the Python methods use +// (CLASSIFY_SEVERITY_SUMMARY / CHECK_DUPLICATE_* / SELECT_STRATEGY_* / +// ASSESS_REACHABILITY_SUMMARY there). + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + "time" + + "github.com/Agent-Field/agentfield/sdk/go/ai" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/config" + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" +) + +// The inputs, mirroring gen_golden.py. +const ( + classifySeveritySummary = "SQL injection in app/db/raw.py:42 — request.args['q'] reaches cursor.execute unsanitized." + selectStrategySummary = "General recon summary.\n\nProfile: 3 files, 120 LOC." + assessReachabilitySummary = "Hardcoded AWS key in config/prod.yaml:12, repository is public." +) + +// checkDuplicateCandidate / Existing mirror gen_golden.py's dicts. They are +// pyfmt.Ordered, not maps, because the Python f-string renders the dict in +// INSERTION order and these two are not alphabetically ordered. +var ( + checkDuplicateCandidate = pyfmt.O( + "id", "finding-1", + "file_path", "app/db/raw.py", + "start_line", 42, + "cwe_id", "CWE-89", + "confirmed", true, + "score", 9.5, + "notes", nil, + ) + checkDuplicateExisting = pyfmt.O( + "id", "finding-0", + "file_path", "app/db/raw.py", + "start_line", 41, + "cwe_id", "CWE-89", + "confirmed", false, + "score", 1.0, + "notes", "seen before", + ) +) + +// testConfig is the AIIntegrationConfig every gate test uses: one retry budget +// big enough to observe backoff, and an ai_model the WithModel assertion can +// look for. +func testConfig() config.AIIntegrationConfig { + return config.AIIntegrationConfig{ + AIModel: "minimax/minimax-m2.5", + MaxRetries: 3, + InitialBackoffSeconds: 2.0, + MaxBackoffSeconds: 8.0, + } +} + +// newGate wires an AIGate to a Fake whose AI seam answers with the given JSON, +// and to a sleeper that never waits. +func newGate(t *testing.T, body string) (*AIGate, *appx.Fake) { + t.Helper() + fake := &appx.Fake{ + AIFn: appx.AIJSON(func(string) (json.RawMessage, error) { return json.RawMessage(body), nil }), + } + gate := &AIGate{App: fake, Config: testConfig(), Sleep: func(context.Context, time.Duration) {}} + return gate, fake +} + +// TestAIGatePromptsMatchPython pins the four gate prompts. Each case drives the +// real method through a Fake and compares the prompt the SDK seam received. +func TestAIGatePromptsMatchPython(t *testing.T) { + ctx := context.Background() + + t.Run("classify_severity", func(t *testing.T) { + gate, fake := newGate(t, `{"severity":"high","confidence":0.9,"rationale":"r"}`) + if _, err := gate.ClassifySeverity(ctx, classifySeveritySummary); err != nil { + t.Fatalf("ClassifySeverity: %v", err) + } + assertPrompt(t, fake, golden(t, "ai_gate_classify_severity.txt")) + }) + + t.Run("check_duplicate", func(t *testing.T) { + gate, fake := newGate(t, `{"is_duplicate":false,"duplicate_of":null,"reason":"r"}`) + if _, err := gate.CheckDuplicate(ctx, checkDuplicateCandidate, checkDuplicateExisting); err != nil { + t.Fatalf("CheckDuplicate: %v", err) + } + assertPrompt(t, fake, golden(t, "ai_gate_check_duplicate.txt")) + }) + + t.Run("select_strategy", func(t *testing.T) { + gate, fake := newGate(t, `{"strategies":["injection"],"rationale":"r"}`) + _, err := gate.SelectStrategy(ctx, selectStrategySummary, "standard", []string{"injection", "auth", "crypto"}) + if err != nil { + t.Fatalf("SelectStrategy: %v", err) + } + assertPrompt(t, fake, golden(t, "ai_gate_select_strategy_standard.txt")) + }) + + t.Run("select_strategy with no candidates", func(t *testing.T) { + gate, fake := newGate(t, `{"strategies":[],"rationale":"r"}`) + // Python renders an empty list as "[]", not as "" or "None". + if _, err := gate.SelectStrategy(ctx, selectStrategySummary, "quick", []string{}); err != nil { + t.Fatalf("SelectStrategy: %v", err) + } + assertPrompt(t, fake, golden(t, "ai_gate_select_strategy_empty_candidates.txt")) + }) + + t.Run("assess_reachability", func(t *testing.T) { + gate, fake := newGate(t, `{"reachability":"externally_reachable","rationale":"r","confidence":"high"}`) + if _, err := gate.AssessReachability(ctx, assessReachabilitySummary); err != nil { + t.Fatalf("AssessReachability: %v", err) + } + assertPrompt(t, fake, golden(t, "ai_gate_assess_reachability.txt")) + }) +} + +func assertPrompt(t *testing.T, fake *appx.Fake, want string) { + t.Helper() + if len(fake.AIs) != 1 { + t.Fatalf("expected exactly one .ai() call, got %d", len(fake.AIs)) + } + if got := fake.AIs[0].Prompt; got != want { + t.Errorf("prompt mismatch:\n%s", firstDiff(want, got)) + } +} + +// TestAIGateParsesTheResponse checks the value actually returned, so the +// goldens are not the only thing keeping the methods honest. +func TestAIGateParsesTheResponse(t *testing.T) { + ctx := context.Background() + gate, _ := newGate(t, `{"reachability":"requires_auth","rationale":"behind login","confidence":"medium"}`) + + got, err := gate.AssessReachability(ctx, "summary") + if err != nil { + t.Fatalf("AssessReachability: %v", err) + } + if got.Reachability != "requires_auth" || got.Rationale != "behind login" || got.Confidence != "medium" { + t.Errorf("parsed = %+v", got) + } +} + +// TestAIGatePassesTheConfiguredModel pins `model=self.config.ai_model`, the one +// SDK option AIGateWrapper.invoke sets beyond system/user/schema. +func TestAIGatePassesTheConfiguredModel(t *testing.T) { + ctx := context.Background() + gate, fake := newGate(t, `{"severity":"low","confidence":0.1,"rationale":"r"}`) + + if _, err := gate.ClassifySeverity(ctx, "summary"); err != nil { + t.Fatalf("ClassifySeverity: %v", err) + } + + req := &ai.Request{} + for _, opt := range fake.AIs[0].Opts { + if err := opt(req); err != nil { + t.Fatalf("apply option: %v", err) + } + } + if req.Model != "minimax/minimax-m2.5" { + t.Errorf("Model = %q, want the config's ai_model", req.Model) + } + if req.ResponseFormat == nil || !req.ResponseFormat.JSONSchema.Strict { + t.Error("expected a strict json_schema response format (aix.Structured's WithSchema)") + } + // Python passes system=None for every gate method, so no system message is + // prepended. + for _, message := range req.Messages { + if message.Role == "system" { + t.Error("no gate method passes a system prompt") + } + } +} + +// TestAIGateNamesNoModelByDefault: an empty AIModel adds no model option, so +// the App's own model choice stands. +func TestAIGateNamesNoModelByDefault(t *testing.T) { + fake := &appx.Fake{AIFn: appx.AIJSON(func(string) (json.RawMessage, error) { + return json.RawMessage(`{"severity":"low","confidence":0.1,"rationale":"r"}`), nil + })} + gate, err := NewAIGate(fake, nil) + if err != nil { + t.Fatalf("NewAIGate: %v", err) + } + if _, err := gate.ClassifySeverity(context.Background(), "summary"); err != nil { + t.Fatalf("ClassifySeverity: %v", err) + } + req := &ai.Request{} + for _, opt := range fake.AIs[0].Opts { + if err := opt(req); err != nil { + t.Fatalf("apply option: %v", err) + } + } + if req.Model != "" { + t.Errorf("Model = %q, want none named", req.Model) + } +} + +// TestAIGateCostTracking pins _CostTracker: one invocation per Invoke (not per +// retry attempt), costs summed, and None/negative costs ignored. +// +// Python parity: the SOURCE of the cost differs by design. Python reads +// `getattr(result, "cost_usd", None)` off the parsed pydantic model, which +// never has that attribute, so its total is permanently 0.0; DESIGN.md has the +// Go port read the SDK response's Usage.Cost instead. The accumulation rules +// below are Python's. +func TestAIGateCostTracking(t *testing.T) { + ctx := context.Background() + + costs := []*float64{ptr(0.25), nil, ptr(-1.0), ptr(0.5)} + call := 0 + fake := &appx.Fake{ + AIFn: func(context.Context, string, ...ai.Option) (*ai.Response, error) { + resp := &ai.Response{ + Choices: []ai.Choice{{Message: ai.Message{ + Role: "assistant", + Content: []ai.ContentPart{{Type: "text", Text: `{"severity":"low","confidence":0.1,"rationale":"r"}`}}, + }}}, + Usage: &ai.Usage{Cost: costs[call]}, + } + call++ + return resp, nil + }, + } + gate := &AIGate{App: fake, Config: testConfig()} + + for range costs { + if _, err := gate.ClassifySeverity(ctx, "summary"); err != nil { + t.Fatalf("ClassifySeverity: %v", err) + } + } + + if got := gate.InvocationCount(); got != len(costs) { + t.Errorf("InvocationCount = %d, want %d", got, len(costs)) + } + if got := gate.TotalCostUSD(); got != 0.75 { + t.Errorf("TotalCostUSD = %v, want 0.75 (nil and negative costs are ignored)", got) + } +} + +func ptr[T any](v T) *T { return &v } + +// TestAIGateIsConcurrencySafe exercises the shape the orchestrator uses +// (_assess_reachability_parallel fans out under a semaphore), so the cost +// tracker and the per-call response capture are proven race-free under -race. +func TestAIGateIsConcurrencySafe(t *testing.T) { + ctx := context.Background() + cost := 0.1 + fake := &appx.Fake{ + AIFn: func(context.Context, string, ...ai.Option) (*ai.Response, error) { + return &ai.Response{ + Choices: []ai.Choice{{Message: ai.Message{ + Role: "assistant", + Content: []ai.ContentPart{{Type: "text", Text: `{"reachability":"internal_only","rationale":"r","confidence":"low"}`}}, + }}}, + Usage: &ai.Usage{Cost: &cost}, + }, nil + }, + } + gate := &AIGate{App: fake, Config: testConfig()} + + const n = 16 + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := gate.AssessReachability(ctx, "summary"); err != nil { + t.Errorf("AssessReachability: %v", err) + } + }() + } + wg.Wait() + + if got := gate.InvocationCount(); got != n { + t.Errorf("InvocationCount = %d, want %d", got, n) + } + if got := gate.TotalCostUSD(); got < 1.59 || got > 1.61 { + t.Errorf("TotalCostUSD = %v, want ~1.6", got) + } +} + +// TestNewAIGateUsesSuppliedConfig pins `config or AIIntegrationConfig.from_env()`. +func TestNewAIGateUsesSuppliedConfig(t *testing.T) { + cfg := testConfig() + gate, err := NewAIGate(&appx.Fake{}, &cfg) + if err != nil { + t.Fatalf("NewAIGate: %v", err) + } + if gate.Config.AIModel != cfg.AIModel { + t.Errorf("Config not carried through: %+v", gate.Config) + } + + // With no config it takes the defaults, and never fails. + defaulted, err := NewAIGate(&appx.Fake{}, nil) + if err != nil { + t.Fatalf("NewAIGate(nil config): %v", err) + } + if defaulted.Config != config.DefaultAIConfig() { + t.Errorf("nil config = %+v, want config.DefaultAIConfig()", defaulted.Config) + } +} + +// TestBuildAIIntegrationSharesOneConfig pins build_ai_integration's contract: +// both wrappers get the SAME resolved config value. +func TestBuildAIIntegrationSharesOneConfig(t *testing.T) { + cfg := testConfig() + wrapper, gate, err := BuildAIIntegration(&appx.Fake{}, &cfg) + if err != nil { + t.Fatalf("BuildAIIntegration: %v", err) + } + if wrapper.Config != gate.Config { + t.Error("HarnessWrapper and AIGate must share the resolved config") + } + if wrapper.Config.AIModel != cfg.AIModel { + t.Errorf("config not carried through: %+v", wrapper.Config) + } +} + +// TestAIGateSurfacesNonTransientErrors checks that a hard failure propagates +// rather than being retried or swallowed. +func TestAIGateSurfacesNonTransientErrors(t *testing.T) { + ctx := context.Background() + boom := errors.New("bad request: unsupported model") + fake := &appx.Fake{ + AIFn: func(context.Context, string, ...ai.Option) (*ai.Response, error) { return nil, boom }, + } + gate := &AIGate{App: fake, Config: testConfig(), Sleep: func(context.Context, time.Duration) {}} + + if _, err := gate.ClassifySeverity(ctx, "summary"); err == nil { + t.Fatal("expected an error") + } else if !errors.Is(err, boom) { + t.Errorf("error = %v, want it to wrap %v", err, boom) + } + if len(fake.AIs) != 1 { + t.Errorf("a non-transient error must not be retried; got %d attempts", len(fake.AIs)) + } +} diff --git a/internal/secaf/gates/harnesswrapper.go b/internal/secaf/gates/harnesswrapper.go new file mode 100644 index 0000000000..9512540172 --- /dev/null +++ b/internal/secaf/gates/harnesswrapper.go @@ -0,0 +1,304 @@ +package gates + +import ( + "context" + "reflect" + "strconv" + "sync" + + "golang.org/x/sync/semaphore" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/config" + "github.com/Agent-Field/codeaf/internal/secaf/harnessx" +) + +// maxSchemaRetries ports HarnessWrapper._invoke_with_schema_retry's local +// `max_schema_retries = 3`. +const maxSchemaRetries = 3 + +// HarnessWrapper ports harness.py HarnessWrapper. +// +// NOT ON THE LIVE PATH. Nothing in the Python tree constructs a HarnessWrapper: +// every agent module calls `app.harness(...)` directly and hands the result to +// extract_harness_result (ported as internal/harnessx). The class is ported +// because the port is 1:1 and because its prompt assembly (prompts.go) is +// worth pinning; the invocation machinery below is kept deliberately compact. +type HarnessWrapper struct { + // App is the `.harness(...)` seam. + App appx.Harnesser + // Config supplies the retry schedule. + Config config.AIIntegrationConfig + // Sleep is the backoff sleeper; nil means the real clock. + Sleep Sleeper + + mu sync.Mutex + totalCostUSD float64 + invocationCount int +} + +// NewHarnessWrapper is `HarnessWrapper(app=app, config=config)`; a nil config +// takes config.DefaultAIConfig(), where Python ran AIIntegrationConfig.from_env(). +// The error return is always nil now and is kept for the shape NewAIGate shares. +func NewHarnessWrapper(app appx.Harnesser, cfg *config.AIIntegrationConfig) (*HarnessWrapper, error) { + if cfg != nil { + return &HarnessWrapper{App: app, Config: *cfg}, nil + } + return &HarnessWrapper{App: app, Config: config.DefaultAIConfig()}, nil +} + +// TotalCostUSD ports the `total_cost_usd` property. +func (w *HarnessWrapper) TotalCostUSD() float64 { + w.mu.Lock() + defer w.mu.Unlock() + return w.totalCostUSD +} + +// InvocationCount ports the `invocation_count` property. +func (w *HarnessWrapper) InvocationCount() int { + w.mu.Lock() + defer w.mu.Unlock() + return w.invocationCount +} + +func (w *HarnessWrapper) registerInvocation() { + w.mu.Lock() + w.invocationCount++ + w.mu.Unlock() +} + +// registerCost ports _CostTracker.register_cost. Unlike the AI gate's, this one +// is fed a real number: the Python SDK's HarnessResult carries `cost_usd`, and +// appx.HarnessResult carries CostUSD. +func (w *HarnessWrapper) registerCost(costUSD *float64) { + if costUSD == nil || *costUSD < 0 { + return + } + w.mu.Lock() + w.totalCostUSD += *costUSD + w.mu.Unlock() +} + +func (w *HarnessWrapper) sleeper() Sleeper { + if w.Sleep != nil { + return w.Sleep + } + return sleepReal +} + +// InvokeRequest is the keyword-argument set of HarnessWrapper.invoke: +// +// invoke(*, prompt, schema, cwd, project_dir=None, model=None, +// max_turns=None, max_budget_usd=None, phase=None) +// +// `schema` is the Go type parameter of HarnessInvoke, not a field. The zero +// value of each remaining field is Python's None. +// +// `model`, `max_turns` and `max_budget_usd` are not carried: they were options +// of the external CLI the harness used to drive, and codeaf's harness owns its +// own model and turn budget, so appx.HarnessOptions has nowhere to put them. +type InvokeRequest struct { + Prompt string + Cwd string + ProjectDir string + Phase string +} + +// harnessOptions builds the appx.HarnessOptions for one invocation — the cwd +// and project_dir keyword arguments HarnessWrapper passed to app.harness. The +// rest of what Python passed (provider, model, max_turns, max_budget_usd, the +// provider environment, schema_max_retries=0, an opencode server) configured an +// external CLI that codeaf does not run, and was dropped with it. +func (w *HarnessWrapper) harnessOptions(req InvokeRequest) appx.HarnessOptions { + return appx.HarnessOptions{ + Cwd: req.Cwd, + ProjectDir: req.ProjectDir, + } +} + +// HarnessInvoke ports HarnessWrapper.invoke. +// +// self._cost_tracker.register_invocation() +// enhanced_prompt = f"{_with_phase_guidance(prompt, phase, cwd)}\n\n{_schema_guidance(schema)}" +// result = await self._run_with_retry(_operation, self.config) +// self._cost_tracker.register_cost(result.cost_usd) +// if result.is_error: raise AIIntegrationError(f"Harness failure{f' ({phase})' if phase else ''}: {message}") +// if isinstance(parsed, schema): return parsed +// ... +// if not parsed and not result.is_error: return await self._invoke_with_schema_retry(...) +// raise AIIntegrationError(f"Harness returned invalid payload for schema {schema.__name__}") +// +// Python parity notes: +// +// - The retry wrapper covers the SDK CALL only. A harness that RAN and failed +// comes back with is_error set and a nil Go error, so it never reaches +// _run_with_retry's except clause — it goes straight to the +// AIIntegrationError raise, transient message or not. That is Python's +// behavior too. +// - The phase suffix on the failure message is the empty string when phase is +// falsy, so an empty phase contributes nothing to the text, not " ()". +// - The `isinstance(result, schema)` third branch is Python duck-typing for an +// SDK that returns the model itself; the Go SDK always returns a +// *appx.HarnessResult, so it is unreachable and not ported. +func HarnessInvoke[T any](ctx context.Context, w *HarnessWrapper, req InvokeRequest) (T, error) { + w.registerInvocation() + enhancedPrompt := WithPhaseGuidance(req.Prompt, req.Phase, req.Cwd) + "\n\n" + SchemaGuidance[T]() + + // Python passes prompt=enhanced_prompt down to _invoke_with_schema_retry, + // which never reads it (it rebuilds the prompt from the schema); carried + // through anyway so the two call graphs line up. + retryReq := req + retryReq.Prompt = enhancedPrompt + return harnessAttempt[T](ctx, w, enhancedPrompt, retryReq, 0, false) +} + +// harnessAttempt is the shared body of invoke and _invoke_with_schema_retry. +// schemaRetry == false runs invoke's operation with the caller's prompt; +// schemaRetry == true runs _invoke_with_schema_retry's, which rebuilds the +// prompt from _build_schema_retry_prompt at each level of the recursion. +func harnessAttempt[T any](ctx context.Context, w *HarnessWrapper, prompt string, req InvokeRequest, retryCount int, schemaRetry bool) (T, error) { + var zero T + typeName := reflect.TypeOf((*T)(nil)).Elem().Name() + + if schemaRetry { + // _invoke_with_schema_retry: + // if retry_count >= max_schema_retries: raise AIIntegrationError(...) + // error_detail = f"Retry attempt {retry_count + 1}/{max_schema_retries}" + // retry_task = _build_schema_retry_prompt(schema, error_detail, cwd) + // retry_prompt = f"{_with_phase_guidance(retry_task, phase, cwd)}\n\n{_schema_guidance(schema)}" + if retryCount >= maxSchemaRetries { + return zero, newAIIntegrationError( + "Schema validation failed after " + strconv.Itoa(maxSchemaRetries) + " retries with schema context") + } + errorDetail := "Retry attempt " + strconv.Itoa(retryCount+1) + "/" + strconv.Itoa(maxSchemaRetries) + retryTask := BuildSchemaRetryPrompt[T](errorDetail, req.Cwd) + prompt = WithPhaseGuidance(retryTask, req.Phase, req.Cwd) + "\n\n" + SchemaGuidance[T]() + } + + opts := w.harnessOptions(req) + + type attemptResult struct { + dest *T + res *appx.HarnessResult + } + operation := func() (attemptResult, error) { + dest, res, runErr := harnessx.Run[T](ctx, w.App, prompt, opts) + return attemptResult{dest: dest, res: res}, runErr + } + + out, err := runWithRetry(ctx, w.Config, w.sleeper(), operation) + if err != nil { + return zero, err + } + if out.res != nil { + w.registerCost(out.res.CostUSD) + } + + if out.res != nil && out.res.IsError { + message := out.res.ErrorMessage + if message == "" { + message = "unknown harness error" + } + suffix := "" + if req.Phase != "" { + suffix = " (" + req.Phase + ")" + } + return zero, newAIIntegrationError("Harness failure" + suffix + ": " + message) + } + + if out.res != nil && out.res.Parsed != nil { + if out.dest != nil { + return *out.dest, nil + } + // Python's trailing `raise AIIntegrationError(f"Harness returned + // invalid payload for schema {schema.__name__}")`: reached when parsed + // is truthy but is neither the schema nor a dict. The Go SDK always + // stores the dest pointer it was handed, so this is unreachable here; + // it is kept so the error surface matches. + return zero, newAIIntegrationError("Harness returned invalid payload for schema " + typeName) + } + + // `if not parsed` — the SDK produced no schema-valid object, so recurse with + // the schema spelled out in the prompt. + // + // Python parity: invoke's fall-through calls _invoke_with_schema_retry with + // its DEFAULT retry_count=0, and only the schema-retry path increments. So + // a run that never produces a parsed object makes 1 + max_schema_retries == + // 4 harness calls, and the error details read "Retry attempt 1/3", "2/3", + // "3/3" in that order. + next := 0 + if schemaRetry { + next = retryCount + 1 + } + return harnessAttempt[T](ctx, w, prompt, req, next, true) +} + +// BatchResult is one slot of `asyncio.gather(..., return_exceptions=True)`: +// either the value or the exception, never both. +type BatchResult[T any] struct { + Value T + Err error +} + +// HarnessInvokeBatch ports HarnessWrapper.invoke_batch: +// +// semaphore = asyncio.Semaphore(max_concurrent) if max_concurrent else None +// tasks = [_run_request(r) for r in requests] +// return await asyncio.gather(*tasks, return_exceptions=True) +// +// Python parity notes: +// +// - RESULT ORDER matches REQUEST order (gather preserves it), so results are +// written into a pre-indexed slice rather than collected as they finish. +// - `return_exceptions=True` means one failure does NOT cancel the others; +// every request runs to completion and failures land in their own slot. +// There is no errgroup here for exactly that reason. +// - `max_concurrent=None` (spelled as a non-positive value in Go) means +// UNBOUNDED, which is why the semaphore is conditional. +// - Acquiring the weighted semaphore can fail only if ctx is already done; in +// that case the slot records the ctx error, which is the closest analogue of +// asyncio cancelling a queued task. +func HarnessInvokeBatch[T any](ctx context.Context, w *HarnessWrapper, requests []InvokeRequest, maxConcurrent int) []BatchResult[T] { + results := make([]BatchResult[T], len(requests)) + var sem *semaphore.Weighted + if maxConcurrent > 0 { + sem = semaphore.NewWeighted(int64(maxConcurrent)) + } + + var wg sync.WaitGroup + for i := range requests { + wg.Add(1) + go func(idx int, req InvokeRequest) { + defer wg.Done() + if sem != nil { + if err := sem.Acquire(ctx, 1); err != nil { + results[idx] = BatchResult[T]{Err: err} + return + } + defer sem.Release(1) + } + v, err := HarnessInvoke[T](ctx, w, req) + results[idx] = BatchResult[T]{Value: v, Err: err} + }(i, requests[i]) + } + wg.Wait() + return results +} + +// RunReconAnalysis ports HarnessWrapper.run_recon_analysis — invoke with +// phase="recon". +func RunReconAnalysis[T any](ctx context.Context, w *HarnessWrapper, prompt, cwd, projectDir string) (T, error) { + return HarnessInvoke[T](ctx, w, InvokeRequest{Prompt: prompt, Cwd: cwd, ProjectDir: projectDir, Phase: "recon"}) +} + +// RunHuntAnalysis ports HarnessWrapper.run_hunt_analysis — invoke with +// phase="hunt". +func RunHuntAnalysis[T any](ctx context.Context, w *HarnessWrapper, prompt, cwd, projectDir string) (T, error) { + return HarnessInvoke[T](ctx, w, InvokeRequest{Prompt: prompt, Cwd: cwd, ProjectDir: projectDir, Phase: "hunt"}) +} + +// RunProveAnalysis ports HarnessWrapper.run_prove_analysis — invoke with +// phase="prove". +func RunProveAnalysis[T any](ctx context.Context, w *HarnessWrapper, prompt, cwd, projectDir string) (T, error) { + return HarnessInvoke[T](ctx, w, InvokeRequest{Prompt: prompt, Cwd: cwd, ProjectDir: projectDir, Phase: "prove"}) +} diff --git a/internal/secaf/gates/harnesswrapper_test.go b/internal/secaf/gates/harnesswrapper_test.go new file mode 100644 index 0000000000..4e3825d70a --- /dev/null +++ b/internal/secaf/gates/harnesswrapper_test.go @@ -0,0 +1,367 @@ +package gates + +// Parity tests for harness.py HarnessWrapper. +// +// HarnessWrapper is not on the live path (nothing constructs one in the Python +// tree), so these tests pin the parts that would silently rot: the assembled +// prompt, the appx.HarnessOptions mapping, the cost/invocation bookkeeping, the +// schema-retry recursion depth, and invoke_batch's gather semantics. + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/config" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +func harnessTestConfig() config.AIIntegrationConfig { + return config.AIIntegrationConfig{ + AIModel: "minimax/minimax-m2.5", + MaxRetries: 3, + InitialBackoffSeconds: 2.0, + MaxBackoffSeconds: 8.0, + } +} + +const cweExpansionJSON = `{"additional_cwes":["CWE-918"],"rationale":"because"}` + +// TestHarnessInvokeAssemblesThePrompt pins +// `f"{_with_phase_guidance(prompt, phase, cwd)}\n\n{_schema_guidance(schema)}"`. +func TestHarnessInvokeAssemblesThePrompt(t *testing.T) { + ctx := context.Background() + + fake := &appx.Fake{HarnessFn: appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return json.RawMessage(cweExpansionJSON), nil + })} + wrapper := &HarnessWrapper{App: fake, Config: harnessTestConfig()} + + got, err := HarnessInvoke[schemas.CWEExpansion](ctx, wrapper, InvokeRequest{ + Prompt: "Expand the CWE list.", + Cwd: "/tmp/secaf-hw", + ProjectDir: "/repo", + Phase: "hunt", + }) + if err != nil { + t.Fatalf("HarnessInvoke: %v", err) + } + if len(got.AdditionalCwes) != 1 || got.AdditionalCwes[0] != "CWE-918" { + t.Errorf("parsed = %+v", got) + } + + if len(fake.Harnesses) != 1 { + t.Fatalf("expected one harness call, got %d", len(fake.Harnesses)) + } + want := WithPhaseGuidance("Expand the CWE list.", "hunt", "/tmp/secaf-hw") + + "\n\n" + SchemaGuidance[schemas.CWEExpansion]() + if fake.Harnesses[0].Prompt != want { + t.Errorf("prompt mismatch:\n%s", firstDiff(want, fake.Harnesses[0].Prompt)) + } + // The phase guidance must actually be the HUNT block, not the fallback. + if !strings.Contains(fake.Harnesses[0].Prompt, PhaseGuidance["hunt"]) { + t.Error("prompt does not carry the hunt phase guidance") + } +} + +// TestHarnessInvokeOptions pins the keyword arguments that survive into +// appx.HarnessOptions: cwd and project_dir. Python also passed provider, model, +// max_turns, max_budget_usd and the provider environment, which configured the +// external CLI codeaf does not run. +func TestHarnessInvokeOptions(t *testing.T) { + ctx := context.Background() + fake := &appx.Fake{HarnessFn: appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return json.RawMessage(cweExpansionJSON), nil + })} + wrapper := &HarnessWrapper{App: fake, Config: harnessTestConfig()} + if _, err := HarnessInvoke[schemas.CWEExpansion](ctx, wrapper, InvokeRequest{ + Prompt: "p", Cwd: "/cwd", ProjectDir: "/repo", + }); err != nil { + t.Fatalf("HarnessInvoke: %v", err) + } + if got, want := fake.Harnesses[0].Opts, (appx.HarnessOptions{Cwd: "/cwd", ProjectDir: "/repo"}); got != want { + t.Errorf("opts = %+v, want %+v", got, want) + } +} + +// TestHarnessInvokeErrorResult pins the AIIntegrationError message, including +// the parenthesised phase suffix (absent for an empty phase) and the +// "unknown harness error" default. +func TestHarnessInvokeErrorResult(t *testing.T) { + ctx := context.Background() + + cases := []struct { + name string + phase string + message string + want string + }{ + {"with phase", "prove", "provider exited 1", "Harness failure (prove): provider exited 1"}, + {"without phase", "", "provider exited 1", "Harness failure: provider exited 1"}, + {"missing message", "recon", "", "Harness failure (recon): unknown harness error"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fake := &appx.Fake{HarnessFn: func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{IsError: true, ErrorMessage: tc.message}, nil + }} + wrapper := &HarnessWrapper{App: fake, Config: harnessTestConfig()} + + _, err := HarnessInvoke[schemas.CWEExpansion](ctx, wrapper, InvokeRequest{ + Prompt: "p", Cwd: "/cwd", Phase: tc.phase, + }) + aiErr, ok := AsAIIntegrationError(err) + if !ok { + t.Fatalf("error = %v (%T), want an AIIntegrationError", err, err) + } + if aiErr.Message != tc.want { + t.Errorf("message = %q, want %q", aiErr.Message, tc.want) + } + // A harness that RAN and failed is not retried: is_error comes back + // with a nil transport error, so _run_with_retry never sees an + // exception. Even a "rate limit" message stops here. + if len(fake.Harnesses) != 1 { + t.Errorf("attempts = %d, want 1", len(fake.Harnesses)) + } + }) + } +} + +// TestHarnessInvokeSchemaRetryDepth pins the recursion Python performs when the +// SDK returns no parsed object: one invoke attempt plus max_schema_retries (3) +// schema-context retries, then the "Schema validation failed after 3 retries" +// AIIntegrationError. The retry prompts must carry the schema JSON and the +// 1/3, 2/3, 3/3 error details in order. +func TestHarnessInvokeSchemaRetryDepth(t *testing.T) { + ctx := context.Background() + + fake := &appx.Fake{HarnessFn: func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{}, nil // ran fine, produced nothing parseable + }} + wrapper := &HarnessWrapper{App: fake, Config: harnessTestConfig()} + + _, err := HarnessInvoke[schemas.CWEExpansion](ctx, wrapper, InvokeRequest{ + Prompt: "p", Cwd: "/tmp/secaf-hw", Phase: "hunt", + }) + aiErr, ok := AsAIIntegrationError(err) + if !ok { + t.Fatalf("error = %v (%T), want an AIIntegrationError", err, err) + } + if aiErr.Message != "Schema validation failed after 3 retries with schema context" { + t.Errorf("message = %q", aiErr.Message) + } + if len(fake.Harnesses) != 1+maxSchemaRetries { + t.Fatalf("attempts = %d, want %d", len(fake.Harnesses), 1+maxSchemaRetries) + } + for i, detail := range []string{"Retry attempt 1/3", "Retry attempt 2/3", "Retry attempt 3/3"} { + prompt := fake.Harnesses[i+1].Prompt + if !strings.Contains(prompt, "Error: "+detail) { + t.Errorf("retry %d prompt is missing %q", i+1, detail) + } + if !strings.Contains(prompt, "```json") { + t.Errorf("retry %d prompt is missing the embedded schema", i+1) + } + } +} + +// TestHarnessWrapperCostTracking pins _CostTracker on the harness side, where — +// unlike the AI gate — the SDK really does report a cost. +func TestHarnessWrapperCostTracking(t *testing.T) { + ctx := context.Background() + + costs := []*float64{ptr(0.5), nil, ptr(-2.0), ptr(0.25)} + call := 0 + fake := &appx.Fake{HarnessFn: func(_ context.Context, _ string, _ map[string]any, dest any, _ appx.HarnessOptions) (*appx.HarnessResult, error) { + if err := json.Unmarshal([]byte(cweExpansionJSON), dest); err != nil { + return nil, err + } + res := &appx.HarnessResult{Parsed: dest, CostUSD: costs[call]} + call++ + return res, nil + }} + wrapper := &HarnessWrapper{App: fake, Config: harnessTestConfig()} + + for range costs { + if _, err := HarnessInvoke[schemas.CWEExpansion](ctx, wrapper, InvokeRequest{Prompt: "p", Cwd: "/cwd"}); err != nil { + t.Fatalf("HarnessInvoke: %v", err) + } + } + if got := wrapper.InvocationCount(); got != len(costs) { + t.Errorf("InvocationCount = %d, want %d", got, len(costs)) + } + if got := wrapper.TotalCostUSD(); got != 0.75 { + t.Errorf("TotalCostUSD = %v, want 0.75", got) + } +} + +// TestRunPhaseAnalysisHelpers pins run_recon_analysis / run_hunt_analysis / +// run_prove_analysis — each is invoke with its phase pinned. +func TestRunPhaseAnalysisHelpers(t *testing.T) { + ctx := context.Background() + + cases := []struct { + phase string + run func(*HarnessWrapper) error + }{ + {"recon", func(w *HarnessWrapper) error { + _, err := RunReconAnalysis[schemas.CWEExpansion](ctx, w, "p", "/cwd", "/repo") + return err + }}, + {"hunt", func(w *HarnessWrapper) error { + _, err := RunHuntAnalysis[schemas.CWEExpansion](ctx, w, "p", "/cwd", "/repo") + return err + }}, + {"prove", func(w *HarnessWrapper) error { + _, err := RunProveAnalysis[schemas.CWEExpansion](ctx, w, "p", "/cwd", "/repo") + return err + }}, + } + for _, tc := range cases { + t.Run(tc.phase, func(t *testing.T) { + fake := &appx.Fake{HarnessFn: appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return json.RawMessage(cweExpansionJSON), nil + })} + wrapper := &HarnessWrapper{App: fake, Config: harnessTestConfig()} + if err := tc.run(wrapper); err != nil { + t.Fatalf("run: %v", err) + } + if !strings.Contains(fake.Harnesses[0].Prompt, PhaseGuidance[tc.phase]) { + t.Errorf("prompt does not carry the %s phase guidance", tc.phase) + } + if fake.Harnesses[0].Opts.ProjectDir != "/repo" { + t.Errorf("project_dir = %q", fake.Harnesses[0].Opts.ProjectDir) + } + }) + } +} + +// TestHarnessInvokeBatch pins invoke_batch's gather semantics: results are +// indexed by REQUEST position, one failure does not cancel the rest +// (return_exceptions=True), and max_concurrent bounds the in-flight count. +func TestHarnessInvokeBatch(t *testing.T) { + ctx := context.Background() + + fake := &appx.Fake{HarnessFn: func(_ context.Context, prompt string, _ map[string]any, dest any, _ appx.HarnessOptions) (*appx.HarnessResult, error) { + if strings.Contains(prompt, "Task:\nBOOM") { + return &appx.HarnessResult{IsError: true, ErrorMessage: "provider exited 1"}, nil + } + if err := json.Unmarshal([]byte(cweExpansionJSON), dest); err != nil { + return nil, err + } + return &appx.HarnessResult{Parsed: dest}, nil + }} + wrapper := &HarnessWrapper{App: fake, Config: harnessTestConfig()} + + requests := []InvokeRequest{ + {Prompt: "one", Cwd: "/cwd"}, + {Prompt: "BOOM", Cwd: "/cwd"}, + {Prompt: "three", Cwd: "/cwd"}, + } + results := HarnessInvokeBatch[schemas.CWEExpansion](ctx, wrapper, requests, 2) + + if len(results) != len(requests) { + t.Fatalf("results = %d, want %d", len(results), len(requests)) + } + if results[0].Err != nil || results[2].Err != nil { + t.Errorf("healthy requests failed: %v / %v", results[0].Err, results[2].Err) + } + if results[1].Err == nil { + t.Error("the failing request must record its error in ITS slot") + } + if _, ok := AsAIIntegrationError(results[1].Err); !ok { + t.Errorf("results[1].Err = %v, want an AIIntegrationError", results[1].Err) + } + if len(results[0].Value.AdditionalCwes) != 1 { + t.Errorf("results[0] not parsed: %+v", results[0].Value) + } + if got := fake.MaxConcurrentHarness(); got > 2 { + t.Errorf("max concurrency = %d, want <= 2", got) + } +} + +// TestHarnessInvokeBatchUnbounded pins `max_concurrent=None`: no semaphore, so +// every request may run at once. +func TestHarnessInvokeBatchUnbounded(t *testing.T) { + ctx := context.Background() + + const n = 5 + var ready sync.WaitGroup + ready.Add(n) + release := make(chan struct{}) + + fake := &appx.Fake{HarnessFn: func(_ context.Context, _ string, _ map[string]any, dest any, _ appx.HarnessOptions) (*appx.HarnessResult, error) { + ready.Done() + <-release // hold every call open until all n are in flight + if err := json.Unmarshal([]byte(cweExpansionJSON), dest); err != nil { + return nil, err + } + return &appx.HarnessResult{Parsed: dest}, nil + }} + wrapper := &HarnessWrapper{App: fake, Config: harnessTestConfig()} + + requests := make([]InvokeRequest, n) + for i := range requests { + requests[i] = InvokeRequest{Prompt: "p", Cwd: "/cwd"} + } + + done := make(chan []BatchResult[schemas.CWEExpansion], 1) + go func() { done <- HarnessInvokeBatch[schemas.CWEExpansion](ctx, wrapper, requests, 0) }() + + ready.Wait() // would deadlock if the batch were serialized + close(release) + results := <-done + + for i, r := range results { + if r.Err != nil { + t.Errorf("result[%d]: %v", i, r.Err) + } + } + if got := fake.MaxConcurrentHarness(); got != n { + t.Errorf("max concurrency = %d, want %d (unbounded)", got, n) + } +} + +// TestHarnessInvokeBatchCancelledContext pins the one Go-only branch: a +// semaphore acquire that fails because ctx is already done records the ctx +// error in that slot rather than panicking. +func TestHarnessInvokeBatchCancelledContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + fake := &appx.Fake{HarnessFn: appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return json.RawMessage(cweExpansionJSON), nil + })} + wrapper := &HarnessWrapper{App: fake, Config: harnessTestConfig()} + + results := HarnessInvokeBatch[schemas.CWEExpansion](ctx, wrapper, []InvokeRequest{{Prompt: "p", Cwd: "/cwd"}}, 1) + if len(results) != 1 { + t.Fatalf("results = %d, want 1", len(results)) + } + if !errors.Is(results[0].Err, context.Canceled) { + t.Errorf("err = %v, want context.Canceled", results[0].Err) + } +} + +// TestNewHarnessWrapperUsesSuppliedConfig pins +// `config or AIIntegrationConfig.from_env()`. +func TestNewHarnessWrapperUsesSuppliedConfig(t *testing.T) { + cfg := harnessTestConfig() + wrapper, err := NewHarnessWrapper(&appx.Fake{}, &cfg) + if err != nil { + t.Fatalf("NewHarnessWrapper: %v", err) + } + if wrapper.Config != cfg { + t.Errorf("config not carried through: %+v", wrapper.Config) + } + defaulted, err := NewHarnessWrapper(&appx.Fake{}, nil) + if err != nil { + t.Fatalf("NewHarnessWrapper(nil config): %v", err) + } + if defaulted.Config != config.DefaultAIConfig() { + t.Errorf("nil config = %+v, want config.DefaultAIConfig()", defaulted.Config) + } +} diff --git a/internal/secaf/gates/prompts.go b/internal/secaf/gates/prompts.go new file mode 100644 index 0000000000..e9443687cd --- /dev/null +++ b/internal/secaf/gates/prompts.go @@ -0,0 +1,349 @@ +package gates + +import ( + "reflect" + "sort" + "strings" + "unicode" + + "github.com/Agent-Field/codeaf/internal/secaf/harnessx" + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" +) + +// PhaseGuidance ports harness.py PHASE_GUIDANCE verbatim — the per-phase +// "APPROACH / PROCESS / CONSTRAINTS" block WithPhaseGuidance prepends. +// +// The Python literals are implicit string concatenations with explicit "\n" +// terminators and NO trailing newline on the last line; the Go strings below +// reproduce that byte for byte (verified by the golden test against the Python +// dict). +var PhaseGuidance = map[string]string{ + "recon": "APPROACH: You are performing reconnaissance on a codebase to build an accurate structural map.\n" + + "PROCESS:\n" + + "1. Survey the codebase structure — identify key directories, entry points, and configuration\n" + + "2. Identify the technology stack — languages, frameworks, and external services\n" + + "3. Map security-relevant boundaries — auth layers, data inputs, API surfaces\n" + + "4. Only after surveying, synthesize findings into the required schema\n" + + "CONSTRAINTS:\n" + + "- Report what IS there, not what MIGHT be there\n" + + "- If uncertain about a detail, omit it rather than guess\n" + + "- Prioritize breadth over depth — cover the full surface", + "hunt": "APPROACH: You are hunting for a specific class of security vulnerability with recon context.\n" + + "PROCESS:\n" + + "1. Review the recon context to understand the codebase topology\n" + + "2. Identify files and patterns relevant to your specific vulnerability class\n" + + "3. For each candidate: read the code, trace data flow, assess exploitability\n" + + "4. Only report findings where you have concrete code evidence\n" + + "CONSTRAINTS:\n" + + "- Every finding MUST cite specific file paths and line numbers you have read\n" + + "- Do not report theoretical vulnerabilities without code evidence\n" + + "- False negatives are better than false positives\n" + + "- If a file is sanitized properly, do NOT report it", + "prove": "APPROACH: You are verifying a specific candidate vulnerability for exploitability.\n" + + "PROCESS:\n" + + "1. Read the specific code location cited in the finding\n" + + "2. Trace the data flow from source to sink\n" + + "3. Check for sanitization, validation, or other mitigations on the path\n" + + "4. If exploitable, construct a concrete exploit hypothesis\n" + + "5. Synthesize your verdict with evidence level\n" + + "CONSTRAINTS:\n" + + "- You must READ the actual code — do not rely on the finding description alone\n" + + "- INCONCLUSIVE is a valid verdict — do not force confirmation or denial\n" + + "- Cite specific lines where sanitization exists or is missing\n" + + "- If code has changed since the finding was generated, note the discrepancy", +} + +// defaultPhaseContext is PHASE_GUIDANCE.get(phase, ) — the fallback +// paragraph for any phase name that is not recon/hunt/prove (including the +// empty string, which is what `phase=None` normalizes to). +const defaultPhaseContext = "Build conclusions from repository evidence in iterative passes. " + + "Prefer explicit evidence over speculation, and clearly separate confirmed facts from uncertainty." + +// outputFileName is the basename harness.py appends to cwd when it tells the +// model where to write large output. +const outputFileName = ".agentfield_output.json" + +// WithFileWriteHint ports harness.py _with_file_write_hint: +// +// output_path = Path(cwd) / ".agentfield_output.json" +// return (f"{prompt.rstrip()}\n" +// f"- If output is large or complex, use the file-write pattern and ensure final JSON is written to {output_path}.") +// +// Python parity notes: +// +// - `str.rstrip()` with no argument strips ALL trailing whitespace, not just +// newlines; strings.TrimRight with unicode.IsSpace is the same set, and +// strings.TrimRightFunc(s, unicode.IsSpace) is what TrimSpace uses. Here +// strings.TrimRight over " \t\n\r\v\f" plus TrimRightFunc would be +// equivalent for every real prompt, so TrimRightFunc is used for exactness. +// - `Path(cwd) / name` is NOT filepath.Join. pathlib collapses redundant +// separators and lone "." components but deliberately keeps "..", because +// resolving it without touching the filesystem would be wrong in the +// presence of symlinks; filepath.Join runs Clean, which eats "..". So +// Path("./work/../work")/"f" is "work/../work/f" where Join gives +// "work/f". pyPathJoin below reproduces pathlib. In the live node cwd is +// always an absolute tempfile.mkdtemp() path where the two agree, but the +// divergence is real and the golden test pins it. +func WithFileWriteHint(prompt, cwd string) string { + outputPath := pyPathJoin(cwd, outputFileName) + return pyRstrip(prompt) + "\n" + + "- If output is large or complex, use the file-write pattern and ensure final JSON is written to " + + outputPath + "." +} + +// WithPhaseGuidance ports harness.py _with_phase_guidance: +// +// normalized_phase = (phase or "").strip().lower() +// phase_context = PHASE_GUIDANCE.get(normalized_phase, ) +// constraints = _with_file_write_hint("Constraints:\n- ...\n- ...\n- ...", cwd) +// return (f"Context:\n- {phase_context}\n\n" +// f"{constraints}\n\n" +// f"Task:\n{prompt.rstrip()}\n\n" +// "Output:\n" +// "- Return a single JSON object matching the requested schema.") +// +// `phase` is `str | None` in Python; the Go signature takes a plain string +// because `(phase or "")` maps None and "" to the same normalized value. +func WithPhaseGuidance(prompt, phase, cwd string) string { + normalizedPhase := strings.ToLower(strings.TrimSpace(phase)) + phaseContext, ok := PhaseGuidance[normalizedPhase] + if !ok { + phaseContext = defaultPhaseContext + } + + constraints := WithFileWriteHint( + "Constraints:\n"+ + "- Use evidence-first reasoning; do not speculate beyond available artifacts.\n"+ + "- Keep analysis bounded to the task scope and produce only schema-conformant output.\n"+ + "- Cite concrete repository evidence whenever making security-relevant claims.", + cwd, + ) + + return "Context:\n- " + phaseContext + "\n\n" + + constraints + "\n\n" + + "Task:\n" + pyRstrip(prompt) + "\n\n" + + "Output:\n" + + "- Return a single JSON object matching the requested schema." +} + +// SchemaGuidance ports harness.py _schema_guidance: +// +// field_lines = [] +// for field_name, field in schema.model_fields.items(): +// description = (field.description or "").strip() +// if description: +// field_lines.append(f"- `{field_name}`: {description}") +// if not field_lines: +// return ("Output format:\n" +// "- Return valid JSON only (no markdown fences, no extra text).\n" +// "- Follow the provided Pydantic schema exactly.") +// return ("Output format:\n" +// "- Return valid JSON only (no markdown fences, no extra text).\n" +// "- Follow the provided Pydantic schema exactly.\n" +// "- Field guidance from schema descriptions:\n" +// f"{chr(10).join(field_lines)}") +// +// The two Python inputs — the field ORDER and each field's DESCRIPTION — come +// from different places in Go: +// +// - ORDER is `schema.model_fields.items()`, pydantic declaration order. The +// port's cross-package contract is that a Go struct's fields are declared in +// the same order as its pydantic counterpart's, so reflect.Type field order +// IS model_fields order. (The committed JSON-Schema fixture cannot supply +// it: go/scripts/gen_schemas.py writes with sort_keys=True, so its +// `properties` object is alphabetical.) +// - DESCRIPTION is `FieldInfo.description`, which pydantic copies verbatim +// into `properties[].description` of model_json_schema() — including +// for Optional and nested-model fields, where the description sits beside +// the anyOf/$ref rather than inside it. harnessx.SchemaFor[T] serves that +// document. +// +// A field with no description contributes no line, exactly as in Python; a +// model with no described fields at all takes the short fallback. +func SchemaGuidance[T any]() string { + schema := harnessx.SchemaFor[T]() + properties, _ := schema["properties"].(map[string]any) + + var fieldLines []string + for _, name := range harnessx.JSONFieldNames(reflect.TypeOf((*T)(nil)).Elem()) { + prop, ok := properties[name].(map[string]any) + if !ok { + continue + } + description, _ := prop["description"].(string) + description = strings.TrimSpace(description) + if description == "" { + continue + } + fieldLines = append(fieldLines, "- `"+name+"`: "+description) + } + + const head = "Output format:\n" + + "- Return valid JSON only (no markdown fences, no extra text).\n" + + "- Follow the provided Pydantic schema exactly." + if len(fieldLines) == 0 { + return head + } + return head + "\n" + + "- Field guidance from schema descriptions:\n" + + strings.Join(fieldLines, "\n") +} + +// BuildSchemaRetryPrompt ports harness.py _build_schema_retry_prompt: +// +// output_path = Path(cwd) / ".agentfield_output.json" +// schema_json_str = json.dumps(schema.model_json_schema(), indent=2) +// return (f"The JSON output at {output_path} failed validation.\n" +// f"Error: {error_detail}\n\n" +// f"Your response must conform to this JSON schema:\n" +// f"```json\n{schema_json_str}\n```\n\n" +// f"Rewrite the COMPLETE, corrected JSON to: {output_path}\n" +// f"The file must contain ONLY valid JSON matching the schema above. " +// f"No markdown fences, no extra text, no comments.") +// +// The embedded schema JSON is byte-reproducible from the committed fixture. The +// fixture is written with sort_keys=True, and pydantic's own +// model_json_schema() dict happens to be in sorted key order at EVERY level +// except one: the `properties` object of the root model and of each `$defs` +// entry, which are in field DECLARATION order. declarationOrdered restores +// exactly those, using the Go struct field order reachable from T — verified +// against all 23 committed fixtures. +func BuildSchemaRetryPrompt[T any](errorDetail, cwd string) string { + outputPath := pyPathJoin(cwd, outputFileName) + schemaJSON := pyfmt.Dumps(declarationOrdered[T](harnessx.SchemaFor[T]()), 2) + + return "The JSON output at " + outputPath + " failed validation.\n" + + "Error: " + errorDetail + "\n\n" + + "Your response must conform to this JSON schema:\n" + + "```json\n" + schemaJSON + "\n```\n\n" + + "Rewrite the COMPLETE, corrected JSON to: " + outputPath + "\n" + + "The file must contain ONLY valid JSON matching the schema above. " + + "No markdown fences, no extra text, no comments." +} + +// --------------------------------------------------------------------------- +// reflection helpers +// --------------------------------------------------------------------------- + +// jsonFieldNames and collectFieldOrders live in internal/harnessx (fieldorder.go) +// because internal/aix needs the same declaration order for the `.ai()` request +// schema and cannot import this package (gates imports aix, not the reverse). + +// declarationOrdered rewrites a JSON-Schema document so that the root +// `properties` object, and the `properties` object of every `$defs` entry whose +// name matches a Go struct reachable from T, render in field DECLARATION order +// instead of the sorted order pyfmt.Dumps gives a Go map. Every other object in +// the document keeps sorted rendering, which is what pydantic emits anyway. +// +// Unknown property names (a fixture key with no matching json tag) are appended +// in sorted order rather than dropped, so a drifted fixture degrades to a +// visible ordering difference instead of silently losing schema content. +func declarationOrdered[T any](schema map[string]any) any { + rootType := reflect.TypeOf((*T)(nil)).Elem() + orders := harnessx.FieldOrders(rootType) + + out := make(map[string]any, len(schema)) + for k, v := range schema { + out[k] = v + } + if props, ok := out["properties"].(map[string]any); ok { + out["properties"] = orderProperties(props, orders[rootType.Name()]) + } + if defs, ok := out["$defs"].(map[string]any); ok { + newDefs := make(map[string]any, len(defs)) + for defName, defSchema := range defs { + sub, isMap := defSchema.(map[string]any) + if !isMap { + newDefs[defName] = defSchema + continue + } + copied := make(map[string]any, len(sub)) + for k, v := range sub { + copied[k] = v + } + if props, hasProps := copied["properties"].(map[string]any); hasProps { + copied["properties"] = orderProperties(props, orders[defName]) + } + newDefs[defName] = copied + } + out["$defs"] = newDefs + } + return out +} + +// orderProperties turns a properties map into a pyfmt.Ordered following `order`, +// appending any leftover keys in sorted order. +func orderProperties(props map[string]any, order []string) pyfmt.Ordered { + ordered := make(pyfmt.Ordered, 0, len(props)) + emitted := make(map[string]struct{}, len(props)) + for _, name := range order { + if v, ok := props[name]; ok { + ordered = append(ordered, pyfmt.KV{Key: name, Value: v}) + emitted[name] = struct{}{} + } + } + leftovers := make([]string, 0) + for name := range props { + if _, done := emitted[name]; !done { + leftovers = append(leftovers, name) + } + } + sort.Strings(leftovers) + for _, name := range leftovers { + ordered = append(ordered, pyfmt.KV{Key: name, Value: props[name]}) + } + return ordered +} + +// pyPathJoin reproduces `str(PurePosixPath(dir) / name)`. +// +// PurePath parsing drops empty components (so "a//b" and a trailing "/" both +// normalize) and lone "." components, but PRESERVES "..", which filepath.Clean +// — and therefore filepath.Join — resolves away. Reproducing pathlib rather +// than reusing Join keeps the emitted output path byte-identical for every cwd +// spelling. +// +// The POSIX rule that EXACTLY two leading slashes are significant is honoured: +// pathlib keeps `//` as the root and collapses one or three-or-more to `/` +// (VERIFIED: `"//a//b//"` -> `//a/b/...`, `"///a"` -> `/a/...`, `"//"` -> +// `//...`). One pathlib corner is still deliberately not reproduced, and is +// unreachable from SEC-AF (cwd always comes from os.MkdirTemp): Windows +// drive/UNC handling. +func pyPathJoin(dir, name string) string { + root := "" + rest := dir + if strings.HasPrefix(dir, "/") { + root = "/" + if strings.HasPrefix(dir, "//") && !strings.HasPrefix(dir, "///") { + root = "//" + } + rest = strings.TrimLeft(dir, "/") + } + parts := make([]string, 0, 8) + for _, part := range strings.Split(rest, "/") { + if part == "" || part == "." { + continue + } + parts = append(parts, part) + } + parts = append(parts, name) + return root + strings.Join(parts, "/") +} + +// pyRstrip reproduces Python's str.rstrip() with no argument: strip every +// trailing Unicode whitespace character. +func pyRstrip(s string) string { + return strings.TrimRightFunc(s, isPySpace) +} + +// isPySpace matches the character class str.strip() removes. Python uses +// str.isspace(), which is Unicode's whitespace property plus the ASCII control +// characters \x1c-\x1f; Go's unicode.IsSpace covers the same set except those +// four separators, which are added explicitly. +func isPySpace(r rune) bool { + switch r { + case '\x1c', '\x1d', '\x1e', '\x1f': + return true + } + return unicode.IsSpace(r) +} diff --git a/internal/secaf/gates/prompts_test.go b/internal/secaf/gates/prompts_test.go new file mode 100644 index 0000000000..35869db34c --- /dev/null +++ b/internal/secaf/gates/prompts_test.go @@ -0,0 +1,328 @@ +package gates + +// Parity tests for the pure prompt builders of src/sec_af/harness.py. +// +// Every expectation is a COMMITTED GOLDEN produced by calling the real Python +// function with the SAME literals declared below: +// +// PYTHONPATH=src ~/.agentfield/packages/sec-af/venv/bin/python go/scripts/gen_golden.py +// +// (see gen_golden.py's GATES_CWD / GATES_PROMPT / PHASE_CASES / +// FILE_WRITE_HINT_CASES / GATES_SCHEMAS / SCHEMA_RETRY_ERROR_DETAIL). + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// The inputs, mirroring gen_golden.py. +const ( + goldenCWD = "/tmp/secaf-golden" + // Trailing spaces and blank lines on purpose: _with_phase_guidance rstrips + // the task text, and a Go port that used TrimSpace or TrimSuffix("\n") + // would produce different bytes. + goldenPrompt = "Analyze the repository for SQL injection.\nCite file:line for every claim. \n\n" + schemaRetryErrorDetail = "Retry attempt 1/3" +) + +func golden(t *testing.T, name string) string { + t.Helper() + raw, err := os.ReadFile(filepath.Join("testdata", "golden", name)) + if err != nil { + t.Fatalf("read golden %s: %v (regenerate with go/scripts/gen_golden.py)", name, err) + } + return string(raw) +} + +func firstDiff(want, got string) string { + wantLines, gotLines := splitLines(want), splitLines(got) + n := len(wantLines) + if len(gotLines) < n { + n = len(gotLines) + } + for i := 0; i < n; i++ { + if wantLines[i] != gotLines[i] { + w, _ := json.Marshal(wantLines[i]) + g, _ := json.Marshal(gotLines[i]) + return "first difference at line " + itoa(i+1) + "\n want: " + string(w) + "\n got: " + string(g) + } + } + if len(wantLines) != len(gotLines) { + return "line counts differ: want " + itoa(len(wantLines)) + ", got " + itoa(len(gotLines)) + } + return "(no line differs; check trailing bytes)" +} + +func splitLines(s string) []string { + out := []string{} + start := 0 + for i := 0; i < len(s); i++ { + if s[i] == '\n' { + out = append(out, s[start:i]) + start = i + 1 + } + } + return append(out, s[start:]) +} + +func itoa(n int) string { + if n == 0 { + return "0" + } + var buf [24]byte + i := len(buf) + for n > 0 { + i-- + buf[i] = byte('0' + n%10) + n /= 10 + } + return string(buf[i:]) +} + +// TestPhaseGuidanceMatchesPython pins the three multi-paragraph blocks verbatim +// against the Python dict, so a dropped bullet or a changed em dash is a test +// failure rather than a quietly different instruction to the model. +func TestPhaseGuidanceMatchesPython(t *testing.T) { + var want map[string]string + if err := json.Unmarshal([]byte(golden(t, "phase_guidance.json")), &want); err != nil { + t.Fatalf("parse phase_guidance.json: %v", err) + } + if len(PhaseGuidance) != len(want) { + t.Fatalf("PhaseGuidance has %d entries, want %d", len(PhaseGuidance), len(want)) + } + for phase, wantText := range want { + gotText, ok := PhaseGuidance[phase] + if !ok { + t.Errorf("PhaseGuidance missing %q", phase) + continue + } + if gotText != wantText { + t.Errorf("PhaseGuidance[%q] mismatch:\n%s", phase, firstDiff(wantText, gotText)) + } + } +} + +// TestWithPhaseGuidanceMatchesPython covers every branch of the phase lookup: +// the three known phases, Python's `phase=None` and its `phase=""` twin (which +// normalize to the same string and therefore the same golden), a phase that +// needs strip+lower before it matches, and an unknown phase that takes the +// fallback paragraph. +func TestWithPhaseGuidanceMatchesPython(t *testing.T) { + cases := []struct { + name string + phase string + }{ + {"recon", "recon"}, + {"hunt", "hunt"}, + {"prove", "prove"}, + {"none", ""}, // Python `phase=None` + {"empty", ""}, // Python `phase=""` + {"padded_mixed_case", " Recon "}, + {"unknown", "unknown-phase"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + want := golden(t, "with_phase_guidance_"+tc.name+".txt") + if got := WithPhaseGuidance(goldenPrompt, tc.phase, goldenCWD); got != want { + t.Errorf("mismatch:\n%s", firstDiff(want, got)) + } + }) + } + + // None and "" must be indistinguishable, which is why the Go signature can + // take a plain string. + if WithPhaseGuidance(goldenPrompt, "", goldenCWD) != golden(t, "with_phase_guidance_none.txt") { + t.Error(`phase="" must render the same text Python's phase=None does`) + } +} + +// TestWithFileWriteHintMatchesPython covers the rstrip and the Path join, +// including a cwd with a trailing separator, an empty cwd (Path("") / name == +// name) and a cwd needing normalization. +func TestWithFileWriteHintMatchesPython(t *testing.T) { + cases := []struct { + name string + prompt string + cwd string + }{ + {"basic", "Constraints:\n- first\n- second", goldenCWD}, + {"trailing_whitespace", "keep me\t \n\n ", goldenCWD + "/"}, + {"empty_cwd", "no directory", ""}, + {"relative_cwd", "relative", "./work/../work"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + want := golden(t, "with_file_write_hint_"+tc.name+".txt") + if got := WithFileWriteHint(tc.prompt, tc.cwd); got != want { + t.Errorf("mismatch:\n%s", firstDiff(want, got)) + } + }) + } +} + +// TestPyPathJoinMatchesPathlib transcribes `str(PurePosixPath(cwd) / +// ".agentfield_output.json")` from the pinned interpreter for every cwd +// spelling that exercises a parsing rule. The POSIX "exactly two leading +// slashes" corner is the one this table exists for: pathlib keeps `//` as the +// root and collapses one or three-or-more slashes to `/`. +func TestPyPathJoinMatchesPathlib(t *testing.T) { + const name = ".agentfield_output.json" + for _, tc := range []struct{ cwd, want string }{ + {"", name}, + {"/", "/" + name}, + {"//", "//" + name}, + {"///", "/" + name}, + {"////", "/" + name}, + {"//a", "//a/" + name}, + {"///a", "/a/" + name}, + {"//a//b//", "//a/b/" + name}, + {"//tmp//", "//tmp/" + name}, + {"//.", "//" + name}, + {"//..", "//../" + name}, + {"//a/../b", "//a/../b/" + name}, + {"/a//b", "/a/b/" + name}, + {".", name}, + {"..", "../" + name}, + {"/tmp/..", "/tmp/../" + name}, + {"./work/../work", "work/../work/" + name}, + {"a/./b", "a/b/" + name}, + {"rel/dir", "rel/dir/" + name}, + {"/tmp/x/", "/tmp/x/" + name}, + {"//\u65e5\u672c/x", "//\u65e5\u672c/x/" + name}, + } { + if got := pyPathJoin(tc.cwd, name); got != tc.want { + t.Errorf("pyPathJoin(%q, name) = %q, want %q", tc.cwd, got, tc.want) + } + } +} + +// TestWithFileWriteHintKeepsADoubleSlashRoot is the same rule seen through the +// prompt the model actually receives. VERIFIED: +// `_with_file_write_hint("Constraints:\n- first", "//a//b//")` ends in +// `... written to //a/b/.agentfield_output.json.` +func TestWithFileWriteHintKeepsADoubleSlashRoot(t *testing.T) { + const prompt = "Constraints:\n- first" + for _, tc := range []struct{ cwd, want string }{ + {"//a//b//", prompt + "\n- If output is large or complex, use the file-write pattern and ensure final JSON is written to //a/b/.agentfield_output.json."}, + {"//", prompt + "\n- If output is large or complex, use the file-write pattern and ensure final JSON is written to //.agentfield_output.json."}, + } { + if got := WithFileWriteHint(prompt, tc.cwd); got != tc.want { + t.Errorf("cwd %q:\n got: %q\nwant: %q", tc.cwd, got, tc.want) + } + } +} + +// schemaCase binds a model NAME to the two generic builders instantiated for +// its Go struct. The name is simultaneously the pydantic class name, the +// embedded schema fixture's basename and the golden's suffix — the three-way +// identity the port is built on. +type schemaCase struct { + name string + guidance func() string + retry func(errorDetail, cwd string) string +} + +func newSchemaCase[T any](name string) schemaCase { + return schemaCase{ + name: name, + guidance: SchemaGuidance[T], + retry: BuildSchemaRetryPrompt[T], + } +} + +// schemaCases covers every model go/scripts/gen_schemas.py emits a fixture for. +// Running all 23 through both builders is what makes this a repo-wide check of +// the "Go struct field order == pydantic declaration order" contract: a +// reordered or renamed Go field changes the guidance lines and the schema's +// `properties` order, and both are pinned. +var schemaCases = []schemaCase{ + newSchemaCase[schemas.ArchitectureMapRaw]("ArchitectureMapRaw"), + newSchemaCase[schemas.DependencyReportRaw]("DependencyReportRaw"), + newSchemaCase[schemas.ConfigReportRaw]("ConfigReportRaw"), + newSchemaCase[schemas.DataFlowMapRaw]("DataFlowMapRaw"), + newSchemaCase[schemas.SecurityContextRaw]("SecurityContextRaw"), + newSchemaCase[schemas.ScanLocationsResult]("ScanLocationsResult"), + newSchemaCase[schemas.EnrichedFinding]("EnrichedFinding"), + newSchemaCase[schemas.ChainCorrelationResult]("ChainCorrelationResult"), + newSchemaCase[schemas.DataFlowTrace]("DataFlowTrace"), + newSchemaCase[schemas.SanitizationResult]("SanitizationResult"), + newSchemaCase[schemas.ExploitHypothesis]("ExploitHypothesis"), + newSchemaCase[schemas.ReachabilityProof]("ReachabilityProof"), + newSchemaCase[schemas.DastVerificationResult]("DastVerificationResult"), + newSchemaCase[schemas.CrossServiceFinding]("CrossServiceFinding"), + newSchemaCase[schemas.RemediationSuggestion]("RemediationSuggestion"), + newSchemaCase[schemas.PolicyEvalResult]("PolicyEvalResult"), + newSchemaCase[schemas.VerdictDecision]("VerdictDecision"), + newSchemaCase[schemas.CWEExpansion]("CWEExpansion"), + newSchemaCase[schemas.SeverityClassification]("SeverityClassification"), + newSchemaCase[schemas.DuplicateCheck]("DuplicateCheck"), + newSchemaCase[schemas.StrategySelection]("StrategySelection"), + newSchemaCase[schemas.ReachabilityGate]("ReachabilityGate"), + newSchemaCase[schemas.ComplianceGate]("ComplianceGate"), +} + +// TestSchemaGuidanceMatchesPython pins _schema_guidance for all 23 models. +// +// It is simultaneously the strongest available check that each Go struct +// declares its fields in pydantic order and tags them with the pydantic field +// names: the guidance lines are emitted in Go field order and keyed by json +// tag, while the descriptions come from the pydantic-generated fixture. +// +// The DuplicateCheck / StrategySelection / ComplianceGate goldens are the short +// fallback text — those models describe no field, so `field_lines` is empty. +func TestSchemaGuidanceMatchesPython(t *testing.T) { + for _, tc := range schemaCases { + t.Run(tc.name, func(t *testing.T) { + want := golden(t, "schema_guidance_"+tc.name+".txt") + if got := tc.guidance(); got != want { + t.Errorf("SchemaGuidance[%s] mismatch:\n%s", tc.name, firstDiff(want, got)) + } + }) + } +} + +// TestBuildSchemaRetryPromptMatchesPython pins _build_schema_retry_prompt for +// all 23 models, embedded schema JSON included. +// +// That JSON is `json.dumps(model_json_schema(), indent=2)` in Python — key +// order and all — so a byte match proves three separate things at once: +// pyfmt.Dumps renders a decoded JSON document exactly like CPython's json +// module (including \uXXXX escaping and the ": " separator), the committed +// fixture still equals the live pydantic schema, and declarationOrdered +// correctly restores the one place pydantic's order is not alphabetical (the +// root `properties` object and each `$defs` entry's — ScanLocationsResult and +// ComplianceGate are the two models with `$defs`). +func TestBuildSchemaRetryPromptMatchesPython(t *testing.T) { + for _, tc := range schemaCases { + t.Run(tc.name, func(t *testing.T) { + want := golden(t, "schema_retry_"+tc.name+".txt") + if got := tc.retry(schemaRetryErrorDetail, goldenCWD); got != want { + t.Errorf("BuildSchemaRetryPrompt[%s] mismatch:\n%s", tc.name, firstDiff(want, got)) + } + }) + } +} + +// TestPyRstrip pins the rstrip() character class, which is wider than +// TrimSuffix("\n") and includes the four ASCII separators str.isspace() counts +// but unicode.IsSpace does not. +func TestPyRstrip(t *testing.T) { + cases := map[string]string{ + "a\n": "a", + "a \t\r\n\v\f": "a", + "a\x1c\x1d\x1e\x1f": "a", + "a ": "a", // NBSP is whitespace to both Python and Go + " a ": " a", + "": "", + "\n\n": "", + } + for in, want := range cases { + if got := pyRstrip(in); got != want { + t.Errorf("pyRstrip(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/secaf/gates/retry.go b/internal/secaf/gates/retry.go new file mode 100644 index 0000000000..d7fa05e67e --- /dev/null +++ b/internal/secaf/gates/retry.go @@ -0,0 +1,181 @@ +// Package gates ports src/sec_af/harness.py — the two thin wrappers SEC-AF +// puts between its phases and the AgentField SDK. +// +// - AIGateWrapper (here: AIGate) is LIVE: orchestrator.py builds one in its +// constructor and calls assess_reachability during scoring, phases.py calls +// select_strategy before the hunt fan-out, and compliance/mapping.py calls +// invoke directly for the compliance gate. +// - HarnessWrapper is NOT reachable from any live path — nothing constructs +// one, and build_ai_integration (the only place that would) has no callers. +// It is ported anyway, because the port is meant to be 1:1 and because its +// prompt-assembly helpers (PHASE_GUIDANCE, SchemaGuidance, +// WithPhaseGuidance, WithFileWriteHint, BuildSchemaRetryPrompt) are pure +// string functions worth pinning with golden tests. +// +// The package is called `gates` rather than `harness` so it cannot be confused +// with the SDK's harness package, and because internal/harnessx already owns +// the piece of harness.py-adjacent behavior that IS on the live path +// (extract_harness_result). +package gates + +import ( + "context" + "errors" + "math" + "strings" + "time" + + "github.com/Agent-Field/codeaf/internal/secaf/config" +) + +// transientPatterns ports harness.py _TRANSIENT_PATTERNS, in Python order. +// +// Python parity: these are substring probes against the LOWERCASED error text, +// and three of them ("503", "502", "504", "500") are bare status codes, so any +// message containing those digits anywhere — a port number, a line number, a +// byte count — counts as transient and buys a retry. That is deliberately +// preserved; it is the behavior the live node has. +var transientPatterns = []string{ + "rate limit", + "rate_limit", + "overloaded", + "timeout", + "timed out", + "connection reset", + "connection refused", + "temporarily unavailable", + "service unavailable", + "503", + "502", + "504", + "internal server error", + "500", +} + +// AIIntegrationError ports harness.py's `class AIIntegrationError(RuntimeError)`. +type AIIntegrationError struct { + Message string +} + +func (e *AIIntegrationError) Error() string { return e.Message } + +// newAIIntegrationError is the `raise AIIntegrationError(msg)` shorthand. +func newAIIntegrationError(msg string) error { return &AIIntegrationError{Message: msg} } + +// IsTransientError ports harness.py _is_transient_error: +// +// lowered = error.lower() +// return any(pattern in lowered for pattern in _TRANSIENT_PATTERNS) +func IsTransientError(errText string) bool { + lowered := strings.ToLower(errText) + for _, pattern := range transientPatterns { + if strings.Contains(lowered, pattern) { + return true + } + } + return false +} + +// Sleeper is the injectable stand-in for `await asyncio.sleep(seconds)`. +// +// Tests supply a recorder so the backoff schedule can be asserted without the +// wall-clock wait; production uses sleepReal. +type Sleeper func(ctx context.Context, d time.Duration) + +// sleepReal is the default Sleeper. It honors context cancellation, which +// asyncio.sleep also does (the task is cancelled), so a shutting-down node does +// not sit out an 8-second backoff. +func sleepReal(ctx context.Context, d time.Duration) { + if d <= 0 { + return + } + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + } +} + +// backoffFor computes the delay before the retry that follows attempt n +// (0-based), reproducing +// +// min(config.initial_backoff_seconds * (2 ** attempt), config.max_backoff_seconds) +// +// as a time.Duration. Both operands are seconds-as-float in Python, so the +// multiplication happens in float64 here too and only the final value is +// converted. +func backoffFor(cfg config.AIIntegrationConfig, attempt int) time.Duration { + delay := cfg.InitialBackoffSeconds * math.Pow(2, float64(attempt)) + if delay > cfg.MaxBackoffSeconds { + delay = cfg.MaxBackoffSeconds + } + if delay <= 0 { + return 0 + } + return time.Duration(delay * float64(time.Second)) +} + +// runWithRetry ports _RetryMixin._run_with_retry: +// +// last_error = None +// for attempt in range(config.max_retries + 1): +// try: +// return await operation() +// except Exception as exc: +// last_error = exc +// if attempt >= config.max_retries or not _is_transient_error(str(exc)): +// raise +// await asyncio.sleep(min(initial * 2**attempt, max)) +// if last_error is not None: +// raise last_error +// raise AIIntegrationError("AI operation failed without an error payload") +// +// Python parity notes: +// +// - max_retries is a count of RETRIES, so the loop makes max_retries+1 +// attempts in total; max_retries == 0 means "one attempt, never retry". +// - The transience test runs on `str(exc)`, i.e. the message only. Go's +// err.Error() is the same text. +// - A NEGATIVE max_retries makes `range()` empty, so nothing is ever +// attempted, last_error stays None and the function raises the +// "failed without an error payload" AIIntegrationError. That dead-looking +// branch is reproduced rather than smoothed over, because the config is +// caller-set and nothing stops a caller from setting MaxRetries to -1. +// - There is NO context-cancellation check between attempts beyond the one +// inside the Sleeper: Python's loop has none either, and the operation +// itself carries the ctx. +func runWithRetry[T any](ctx context.Context, cfg config.AIIntegrationConfig, sleep Sleeper, operation func() (T, error)) (T, error) { + var zero T + var lastError error + attempted := false + + for attempt := 0; attempt < cfg.MaxRetries+1; attempt++ { + attempted = true + v, err := operation() + if err == nil { + return v, nil + } + lastError = err + if attempt >= cfg.MaxRetries || !IsTransientError(err.Error()) { + return zero, err + } + sleep(ctx, backoffFor(cfg, attempt)) + } + + if lastError != nil { + return zero, lastError + } + _ = attempted + return zero, newAIIntegrationError("AI operation failed without an error payload") +} + +// AsAIIntegrationError reports whether err is (or wraps) an AIIntegrationError, +// the Go form of `except AIIntegrationError`. +func AsAIIntegrationError(err error) (*AIIntegrationError, bool) { + var target *AIIntegrationError + if errors.As(err, &target) { + return target, true + } + return nil, false +} diff --git a/internal/secaf/gates/retry_test.go b/internal/secaf/gates/retry_test.go new file mode 100644 index 0000000000..5f0f46d459 --- /dev/null +++ b/internal/secaf/gates/retry_test.go @@ -0,0 +1,264 @@ +package gates + +// Parity tests for harness.py _is_transient_error, _RetryMixin._run_with_retry +// and the backoff schedule. + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + "time" + + "github.com/Agent-Field/agentfield/sdk/go/ai" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/config" +) + +// TestIsTransientError pins _TRANSIENT_PATTERNS, including the three bare +// status-code patterns whose over-eagerness is real behavior, not a bug to fix. +func TestIsTransientError(t *testing.T) { + transient := []string{ + "Rate limit exceeded", + "rate_limit_error", + "Model is OVERLOADED", + "request timeout after 60s", + "the call timed out", + "connection reset by peer", + "connection refused", + "Service temporarily unavailable", + "503 Service Unavailable", + "502 Bad Gateway", + "504 Gateway Timeout", + "Internal Server Error", + "HTTP 500", + // Python parity: the bare "500" pattern matches ANY message containing + // those digits. This is over-eager and deliberately preserved. + "failed to parse line 500 of the report", + "listening on port 5027", + } + for _, message := range transient { + if !IsTransientError(message) { + t.Errorf("IsTransientError(%q) = false, want true", message) + } + } + + permanent := []string{ + "", + "bad request: unsupported model", + "invalid api key", + "context length exceeded", + "schema validation failed for field severity", + } + for _, message := range permanent { + if IsTransientError(message) { + t.Errorf("IsTransientError(%q) = true, want false", message) + } + } +} + +// TestBackoffSchedule pins `min(initial * 2**attempt, max)`. +func TestBackoffSchedule(t *testing.T) { + cfg := config.AIIntegrationConfig{InitialBackoffSeconds: 2.0, MaxBackoffSeconds: 8.0} + want := []time.Duration{2 * time.Second, 4 * time.Second, 8 * time.Second, 8 * time.Second} + for attempt, wantDelay := range want { + if got := backoffFor(cfg, attempt); got != wantDelay { + t.Errorf("backoffFor(attempt=%d) = %v, want %v", attempt, got, wantDelay) + } + } + + // Fractional seconds survive the float multiplication. + frac := config.AIIntegrationConfig{InitialBackoffSeconds: 0.5, MaxBackoffSeconds: 10} + if got := backoffFor(frac, 1); got != time.Second { + t.Errorf("backoffFor(0.5s, attempt=1) = %v, want 1s", got) + } + // A zero or negative schedule never sleeps. + if got := backoffFor(config.AIIntegrationConfig{}, 0); got != 0 { + t.Errorf("backoffFor(zero config) = %v, want 0", got) + } +} + +// flakyAI answers with a transient error `failures` times, then succeeds. +func flakyAI(failures int, body string) (func(context.Context, string, ...ai.Option) (*ai.Response, error), *int) { + calls := 0 + fn := func(context.Context, string, ...ai.Option) (*ai.Response, error) { + calls++ + if calls <= failures { + return nil, errors.New("rate limit exceeded, retry later") + } + return &ai.Response{Choices: []ai.Choice{{Message: ai.Message{ + Role: "assistant", + Content: []ai.ContentPart{{Type: "text", Text: body}}, + }}}}, nil + } + return fn, &calls +} + +// recordingSleeper captures the backoff delays instead of waiting them out. +type recordingSleeper struct { + mu sync.Mutex + delays []time.Duration +} + +func (r *recordingSleeper) sleep(_ context.Context, d time.Duration) { + r.mu.Lock() + r.delays = append(r.delays, d) + r.mu.Unlock() +} + +// TestRetryOnTransientError pins the retry loop: max_retries+1 attempts, the +// exact backoff schedule between them, and one invocation counted for the whole +// sequence. +func TestRetryOnTransientError(t *testing.T) { + ctx := context.Background() + fn, calls := flakyAI(2, `{"severity":"high","confidence":0.9,"rationale":"r"}`) + sleeper := &recordingSleeper{} + gate := &AIGate{App: &appx.Fake{AIFn: fn}, Config: testConfig(), Sleep: sleeper.sleep} + + if _, err := gate.ClassifySeverity(ctx, "summary"); err != nil { + t.Fatalf("ClassifySeverity: %v", err) + } + if *calls != 3 { + t.Errorf("attempts = %d, want 3 (two failures then a success)", *calls) + } + want := []time.Duration{2 * time.Second, 4 * time.Second} + if len(sleeper.delays) != len(want) { + t.Fatalf("slept %v, want %v", sleeper.delays, want) + } + for i := range want { + if sleeper.delays[i] != want[i] { + t.Errorf("delay[%d] = %v, want %v", i, sleeper.delays[i], want[i]) + } + } + if got := gate.InvocationCount(); got != 1 { + t.Errorf("InvocationCount = %d, want 1 (retries are not new invocations)", got) + } +} + +// TestRetryGivesUpAfterMaxRetries pins the `attempt >= config.max_retries` +// bound: max_retries+1 attempts total, then the last error propagates. +func TestRetryGivesUpAfterMaxRetries(t *testing.T) { + ctx := context.Background() + fn, calls := flakyAI(99, "") + sleeper := &recordingSleeper{} + gate := &AIGate{App: &appx.Fake{AIFn: fn}, Config: testConfig(), Sleep: sleeper.sleep} + + if _, err := gate.ClassifySeverity(ctx, "summary"); err == nil { + t.Fatal("expected the last error to propagate") + } + if *calls != testConfig().MaxRetries+1 { + t.Errorf("attempts = %d, want max_retries+1 = %d", *calls, testConfig().MaxRetries+1) + } + if len(sleeper.delays) != testConfig().MaxRetries { + t.Errorf("slept %d times, want max_retries = %d", len(sleeper.delays), testConfig().MaxRetries) + } +} + +// TestRetryZeroMaxRetries pins that max_retries=0 means exactly one attempt. +func TestRetryZeroMaxRetries(t *testing.T) { + ctx := context.Background() + fn, calls := flakyAI(99, "") + cfg := testConfig() + cfg.MaxRetries = 0 + sleeper := &recordingSleeper{} + gate := &AIGate{App: &appx.Fake{AIFn: fn}, Config: cfg, Sleep: sleeper.sleep} + + if _, err := gate.ClassifySeverity(ctx, "summary"); err == nil { + t.Fatal("expected an error") + } + if *calls != 1 { + t.Errorf("attempts = %d, want 1", *calls) + } + if len(sleeper.delays) != 0 { + t.Errorf("slept %v, want nothing", sleeper.delays) + } +} + +// TestRetryNegativeMaxRetriesRaisesTheEmptyPayloadError pins the branch Python +// reaches when `range(max_retries + 1)` is empty: nothing is attempted, +// last_error stays None, and AIIntegrationError("AI operation failed without an +// error payload") is raised. SEC_AF_AI_MAX_RETRIES=-1 in the environment is how +// a deployment gets here. +func TestRetryNegativeMaxRetriesRaisesTheEmptyPayloadError(t *testing.T) { + ctx := context.Background() + cfg := testConfig() + cfg.MaxRetries = -1 + fake := &appx.Fake{AIFn: appx.AIJSON(func(string) (json.RawMessage, error) { + return json.RawMessage(`{"severity":"low","confidence":0,"rationale":""}`), nil + })} + gate := &AIGate{App: fake, Config: cfg} + + _, err := gate.ClassifySeverity(ctx, "summary") + if err == nil { + t.Fatal("expected an error") + } + aiErr, ok := AsAIIntegrationError(err) + if !ok { + t.Fatalf("error = %v (%T), want an AIIntegrationError", err, err) + } + if aiErr.Message != "AI operation failed without an error payload" { + t.Errorf("message = %q", aiErr.Message) + } + if len(fake.AIs) != 0 { + t.Errorf("nothing should have been attempted; got %d calls", len(fake.AIs)) + } +} + +// TestRunWithRetryReturnsTheValue is a direct unit test of the generic helper, +// independent of the gate that uses it. +func TestRunWithRetryReturnsTheValue(t *testing.T) { + ctx := context.Background() + cfg := config.AIIntegrationConfig{MaxRetries: 2, InitialBackoffSeconds: 1, MaxBackoffSeconds: 4} + sleeper := &recordingSleeper{} + + attempts := 0 + got, err := runWithRetry(ctx, cfg, sleeper.sleep, func() (string, error) { + attempts++ + if attempts < 2 { + return "", errors.New("connection reset by peer") + } + return "ok", nil + }) + if err != nil { + t.Fatalf("runWithRetry: %v", err) + } + if got != "ok" { + t.Errorf("value = %q, want ok", got) + } + if attempts != 2 { + t.Errorf("attempts = %d, want 2", attempts) + } +} + +// TestAIIntegrationErrorUnwrapping pins that AsAIIntegrationError finds the +// error through a wrapping chain, the Go analogue of `except AIIntegrationError`. +func TestAIIntegrationErrorUnwrapping(t *testing.T) { + base := newAIIntegrationError("boom") + wrapped := errors.Join(errors.New("context"), base) + if got, ok := AsAIIntegrationError(wrapped); !ok || got.Message != "boom" { + t.Errorf("AsAIIntegrationError(wrapped) = %v, %v", got, ok) + } + if _, ok := AsAIIntegrationError(errors.New("plain")); ok { + t.Error("a plain error must not be reported as an AIIntegrationError") + } + if base.Error() != "boom" { + t.Errorf("Error() = %q", base.Error()) + } +} + +// TestSleepRealHonorsContextCancellation guards against a shutting-down node +// sitting out a full 8-second backoff. +func TestSleepRealHonorsContextCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + start := time.Now() + sleepReal(ctx, time.Hour) + if elapsed := time.Since(start); elapsed > time.Second { + t.Errorf("sleepReal ignored cancellation; waited %v", elapsed) + } + // A non-positive delay returns immediately without allocating a timer. + sleepReal(context.Background(), 0) +} diff --git a/internal/secaf/gates/testdata/golden/ai_gate_assess_reachability.txt b/internal/secaf/gates/testdata/golden/ai_gate_assess_reachability.txt new file mode 100644 index 0000000000..1fe73202b8 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/ai_gate_assess_reachability.txt @@ -0,0 +1,3 @@ +Assess the reachability of this security finding. Determine if it is externally_reachable, requires_auth, internal_only, or unreachable. Consider the attack surface, authentication requirements, and network exposure. + +Hardcoded AWS key in config/prod.yaml:12, repository is public. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/ai_gate_check_duplicate.txt b/internal/secaf/gates/testdata/golden/ai_gate_check_duplicate.txt new file mode 100644 index 0000000000..cdbb723368 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/ai_gate_check_duplicate.txt @@ -0,0 +1,4 @@ +Decide whether candidate finding is a duplicate of existing finding. Return duplicate decision only. + +Candidate: {'id': 'finding-1', 'file_path': 'app/db/raw.py', 'start_line': 42, 'cwe_id': 'CWE-89', 'confirmed': True, 'score': 9.5, 'notes': None} +Existing: {'id': 'finding-0', 'file_path': 'app/db/raw.py', 'start_line': 41, 'cwe_id': 'CWE-89', 'confirmed': False, 'score': 1.0, 'notes': 'seen before'} \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/ai_gate_classify_severity.txt b/internal/secaf/gates/testdata/golden/ai_gate_classify_severity.txt new file mode 100644 index 0000000000..c9a6e8ff89 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/ai_gate_classify_severity.txt @@ -0,0 +1,3 @@ +Classify severity for this potential security finding. Use only critical/high/medium/low and keep rationale brief. + +SQL injection in app/db/raw.py:42 — request.args['q'] reaches cursor.execute unsanitized. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/ai_gate_select_strategy_empty_candidates.txt b/internal/secaf/gates/testdata/golden/ai_gate_select_strategy_empty_candidates.txt new file mode 100644 index 0000000000..20ca59a2f9 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/ai_gate_select_strategy_empty_candidates.txt @@ -0,0 +1,6 @@ +Select SEC-AF hunt strategies from recon context. Return only selected strategies and rationale. +Depth profile: quick +Default candidates: [] +Recon summary: General recon summary. + +Profile: 3 files, 120 LOC. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/ai_gate_select_strategy_standard.txt b/internal/secaf/gates/testdata/golden/ai_gate_select_strategy_standard.txt new file mode 100644 index 0000000000..5c98c5c884 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/ai_gate_select_strategy_standard.txt @@ -0,0 +1,6 @@ +Select SEC-AF hunt strategies from recon context. Return only selected strategies and rationale. +Depth profile: standard +Default candidates: ['injection', 'auth', 'crypto'] +Recon summary: General recon summary. + +Profile: 3 files, 120 LOC. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/phase_guidance.json b/internal/secaf/gates/testdata/golden/phase_guidance.json new file mode 100644 index 0000000000..f1b7961246 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/phase_guidance.json @@ -0,0 +1,5 @@ +{ + "hunt": "APPROACH: You are hunting for a specific class of security vulnerability with recon context.\nPROCESS:\n1. Review the recon context to understand the codebase topology\n2. Identify files and patterns relevant to your specific vulnerability class\n3. For each candidate: read the code, trace data flow, assess exploitability\n4. Only report findings where you have concrete code evidence\nCONSTRAINTS:\n- Every finding MUST cite specific file paths and line numbers you have read\n- Do not report theoretical vulnerabilities without code evidence\n- False negatives are better than false positives\n- If a file is sanitized properly, do NOT report it", + "prove": "APPROACH: You are verifying a specific candidate vulnerability for exploitability.\nPROCESS:\n1. Read the specific code location cited in the finding\n2. Trace the data flow from source to sink\n3. Check for sanitization, validation, or other mitigations on the path\n4. If exploitable, construct a concrete exploit hypothesis\n5. Synthesize your verdict with evidence level\nCONSTRAINTS:\n- You must READ the actual code \u2014 do not rely on the finding description alone\n- INCONCLUSIVE is a valid verdict \u2014 do not force confirmation or denial\n- Cite specific lines where sanitization exists or is missing\n- If code has changed since the finding was generated, note the discrepancy", + "recon": "APPROACH: You are performing reconnaissance on a codebase to build an accurate structural map.\nPROCESS:\n1. Survey the codebase structure \u2014 identify key directories, entry points, and configuration\n2. Identify the technology stack \u2014 languages, frameworks, and external services\n3. Map security-relevant boundaries \u2014 auth layers, data inputs, API surfaces\n4. Only after surveying, synthesize findings into the required schema\nCONSTRAINTS:\n- Report what IS there, not what MIGHT be there\n- If uncertain about a detail, omit it rather than guess\n- Prioritize breadth over depth \u2014 cover the full surface" +} diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_ArchitectureMapRaw.txt b/internal/secaf/gates/testdata/golden/schema_guidance_ArchitectureMapRaw.txt new file mode 100644 index 0000000000..71db98723b --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_ArchitectureMapRaw.txt @@ -0,0 +1,10 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. +- Field guidance from schema descriptions: +- `app_type`: Application type: web_api, cli_tool, library, microservice, monolith +- `modules`: One string per module. Format: 'name | path | language | description'. Example: 'auth | src/auth/ | python | Authentication and session management' +- `entry_points`: One string per entry point. Format: 'kind | route_or_id | file_path:line | auth_required'. Example: 'http | POST /api/login | src/routes.py:42 | false' +- `trust_boundaries`: One string per boundary. Format: 'name | source_zone | target_zone | description'. Example: 'API Gateway | external | internal | Rate limiting and auth' +- `services`: One string per external service. Format: 'name | type | endpoint | auth_mechanism'. Example: 'PostgreSQL | database | localhost:5432 | password' +- `api_endpoints`: One string per API endpoint. Format: 'method | path | handler | file_path:line | auth_required | rate_limited'. Example: 'GET | /api/users | get_users | src/api.py:15 | true | false' \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_CWEExpansion.txt b/internal/secaf/gates/testdata/golden/schema_guidance_CWEExpansion.txt new file mode 100644 index 0000000000..aab9b486cb --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_CWEExpansion.txt @@ -0,0 +1,5 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. +- Field guidance from schema descriptions: +- `additional_cwes`: CWE IDs to add beyond baseline, e.g. ['CWE-918', 'CWE-611']. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_ChainCorrelationResult.txt b/internal/secaf/gates/testdata/golden/schema_guidance_ChainCorrelationResult.txt new file mode 100644 index 0000000000..c9a299d759 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_ChainCorrelationResult.txt @@ -0,0 +1,6 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. +- Field guidance from schema descriptions: +- `chains`: Multi-step attack chains found. Format per entry: 'title | finding_id1,finding_id2,... | combined_impact | severity'. Example: 'SSRF to Internal API | f1,f2 | Access internal services | high' +- `duplicate_ids`: Finding IDs that are duplicates missed by programmatic dedup (to drop) \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_ComplianceGate.txt b/internal/secaf/gates/testdata/golden/schema_guidance_ComplianceGate.txt new file mode 100644 index 0000000000..fc721cac70 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_ComplianceGate.txt @@ -0,0 +1,3 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_ConfigReportRaw.txt b/internal/secaf/gates/testdata/golden/schema_guidance_ConfigReportRaw.txt new file mode 100644 index 0000000000..c7937a120d --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_ConfigReportRaw.txt @@ -0,0 +1,6 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. +- Field guidance from schema descriptions: +- `secrets`: One string per secret finding. Format: 'type | file_path:line | match_preview | confidence | is_test(true/false)'. Example: 'aws_access_key | .env:3 | AKIA... | high | false' +- `misconfigs`: One string per misconfiguration. Format: 'category | file_path:line | key | risk | remediation'. Example: 'debug_mode | config.py:15 | DEBUG=True | Exposes stack traces | Set DEBUG=False' \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_CrossServiceFinding.txt b/internal/secaf/gates/testdata/golden/schema_guidance_CrossServiceFinding.txt new file mode 100644 index 0000000000..7d15b4c325 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_CrossServiceFinding.txt @@ -0,0 +1,8 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. +- Field guidance from schema descriptions: +- `chain_description`: Description of the cross-service attack path +- `services_involved`: Service names in the attack chain +- `entry_point`: Public-facing entry point where attack begins +- `impact`: Impact if the cross-service chain is exploited \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_DastVerificationResult.txt b/internal/secaf/gates/testdata/golden/schema_guidance_DastVerificationResult.txt new file mode 100644 index 0000000000..f0c852f3a2 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_DastVerificationResult.txt @@ -0,0 +1,8 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. +- Field guidance from schema descriptions: +- `payload_sent`: The exploit payload or request that was sent +- `response_summary`: Summary of the application response +- `exploit_confirmed`: Whether the exploit was confirmed at runtime +- `safety_notes`: Safety measures taken during verification (sandbox, timeout, etc.) \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_DataFlowMapRaw.txt b/internal/secaf/gates/testdata/golden/schema_guidance_DataFlowMapRaw.txt new file mode 100644 index 0000000000..bedcbcd887 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_DataFlowMapRaw.txt @@ -0,0 +1,7 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. +- Field guidance from schema descriptions: +- `flows`: One string per data flow. Format: 'source | sink | sanitized(true/false) | file1, file2, ...'. Example: 'request.body | sql.execute | false | src/db.py, src/routes.py' +- `sanitization_points`: One string per sanitization point. Format: 'file_path:line | function_name | type | protects_against'. Example: 'src/utils.py:42 | sanitize_html | html_encoding | CWE-79' +- `sinks`: One string per security-critical sink. Format: 'sink_type | file_path:line | function_name | notes'. Example: 'sql_execute | src/db.py:55 | run_query | Direct string concatenation' \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_DataFlowTrace.txt b/internal/secaf/gates/testdata/golden/schema_guidance_DataFlowTrace.txt new file mode 100644 index 0000000000..76546098a5 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_DataFlowTrace.txt @@ -0,0 +1,8 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. +- Field guidance from schema descriptions: +- `source`: Where tainted input enters (e.g. 'request.params.id') +- `sink`: Security-sensitive operation reached (e.g. 'sql.execute(query)') +- `steps`: Ordered list of file:line descriptions showing flow path +- `sink_reached`: Whether tainted data actually reaches the sink \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_DependencyReportRaw.txt b/internal/secaf/gates/testdata/golden/schema_guidance_DependencyReportRaw.txt new file mode 100644 index 0000000000..739755dffb --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_DependencyReportRaw.txt @@ -0,0 +1,7 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. +- Field guidance from schema descriptions: +- `sbom`: One string per dependency. Format: 'name | version | ecosystem | direct(true/false) | license'. Example: 'express | 4.18.2 | npm | true | MIT' +- `known_cves`: One string per CVE. Format: 'cve_id | package | installed_version | fixed_version | cvss_score | direct | reachable'. Example: 'CVE-2023-1234 | lodash | 4.17.15 | 4.17.21 | 7.5 | true | unknown' +- `outdated`: One string per outdated dep. Format: 'package | current_version | latest_version | direct(true/false)'. Example: 'express | 4.17.0 | 4.18.2 | true' \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_DuplicateCheck.txt b/internal/secaf/gates/testdata/golden/schema_guidance_DuplicateCheck.txt new file mode 100644 index 0000000000..fc721cac70 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_DuplicateCheck.txt @@ -0,0 +1,3 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_EnrichedFinding.txt b/internal/secaf/gates/testdata/golden/schema_guidance_EnrichedFinding.txt new file mode 100644 index 0000000000..ec9f96515f --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_EnrichedFinding.txt @@ -0,0 +1,10 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. +- Field guidance from schema descriptions: +- `title`: Human-readable title for the finding +- `description`: Detailed description of the vulnerability +- `cwe_id`: CWE identifier (e.g. 'CWE-89') +- `severity`: One of: "critical", "high", "medium", "low", "info" +- `confidence`: One of: "high", "medium", "low" +- `data_flow_summary`: Natural language summary of the data flow (string, not nested) \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_ExploitHypothesis.txt b/internal/secaf/gates/testdata/golden/schema_guidance_ExploitHypothesis.txt new file mode 100644 index 0000000000..edb2b5d0dc --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_ExploitHypothesis.txt @@ -0,0 +1,7 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. +- Field guidance from schema descriptions: +- `hypothesis`: Natural language description of exploit scenario +- `payload`: Concrete exploit payload or input +- `expected_outcome`: What would happen if exploit succeeds \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_PolicyEvalResult.txt b/internal/secaf/gates/testdata/golden/schema_guidance_PolicyEvalResult.txt new file mode 100644 index 0000000000..a0f6649602 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_PolicyEvalResult.txt @@ -0,0 +1,8 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. +- Field guidance from schema descriptions: +- `violated`: Whether the policy is violated +- `description`: How the policy is violated, or 'No violation' if compliant +- `file_path`: Primary file where violation occurs, or 'N/A' +- `severity`: Severity: "high", "medium", or "low" \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_ReachabilityGate.txt b/internal/secaf/gates/testdata/golden/schema_guidance_ReachabilityGate.txt new file mode 100644 index 0000000000..7adfb60e61 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_ReachabilityGate.txt @@ -0,0 +1,6 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. +- Field guidance from schema descriptions: +- `reachability`: One of: "externally_reachable", "requires_auth", "internal_only", "unreachable". +- `confidence`: One of: "high", "medium", "low". \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_ReachabilityProof.txt b/internal/secaf/gates/testdata/golden/schema_guidance_ReachabilityProof.txt new file mode 100644 index 0000000000..277112b5c4 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_ReachabilityProof.txt @@ -0,0 +1,8 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. +- Field guidance from schema descriptions: +- `vulnerable_function`: The vulnerable function/method in the dependency +- `call_chain`: Import/call chain from app code to vulnerable function +- `reachable`: Whether the vulnerable function is actually called +- `direct`: Whether the dependency is direct or transitive \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_RemediationSuggestion.txt b/internal/secaf/gates/testdata/golden/schema_guidance_RemediationSuggestion.txt new file mode 100644 index 0000000000..2feca45ed9 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_RemediationSuggestion.txt @@ -0,0 +1,7 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. +- Field guidance from schema descriptions: +- `fix_description`: Natural language description of the recommended fix +- `patch_diff`: Unified diff format patch showing the code changes needed +- `confidence`: Confidence in the fix: "high", "medium", or "low" \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_SanitizationResult.txt b/internal/secaf/gates/testdata/golden/schema_guidance_SanitizationResult.txt new file mode 100644 index 0000000000..ddadffa9ab --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_SanitizationResult.txt @@ -0,0 +1,8 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. +- Field guidance from schema descriptions: +- `found`: Whether any sanitization/validation was found on the path +- `type`: Type of sanitization (e.g. 'parameterized query', 'html encoding') +- `sufficient`: Whether sanitization is sufficient to prevent exploit +- `bypass_method`: How sanitization could be bypassed, if applicable \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_ScanLocationsResult.txt b/internal/secaf/gates/testdata/golden/schema_guidance_ScanLocationsResult.txt new file mode 100644 index 0000000000..fc721cac70 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_ScanLocationsResult.txt @@ -0,0 +1,3 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_SecurityContextRaw.txt b/internal/secaf/gates/testdata/golden/schema_guidance_SecurityContextRaw.txt new file mode 100644 index 0000000000..cfbbad79de --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_SecurityContextRaw.txt @@ -0,0 +1,8 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. +- Field guidance from schema descriptions: +- `auth_model`: Authentication model: jwt, session_cookie, oauth2, api_key, none, or other +- `auth_details`: Brief description of auth implementation details +- `crypto_usage`: One string per crypto usage. Format: 'algorithm | key_size | mode | usage_context | is_weak(true/false)'. Example: 'AES | 256 | GCM | data encryption | false' +- `security_signals`: Framework security features, security headers, and deployment signals. One signal per entry. Examples: 'CSRF protection enabled', 'HSTS header present', 'Runs in Docker' \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_SeverityClassification.txt b/internal/secaf/gates/testdata/golden/schema_guidance_SeverityClassification.txt new file mode 100644 index 0000000000..94c3223848 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_SeverityClassification.txt @@ -0,0 +1,5 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. +- Field guidance from schema descriptions: +- `severity`: One of: "critical", "high", "medium", "low". \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_StrategySelection.txt b/internal/secaf/gates/testdata/golden/schema_guidance_StrategySelection.txt new file mode 100644 index 0000000000..fc721cac70 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_StrategySelection.txt @@ -0,0 +1,3 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_guidance_VerdictDecision.txt b/internal/secaf/gates/testdata/golden/schema_guidance_VerdictDecision.txt new file mode 100644 index 0000000000..7980b05dea --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_guidance_VerdictDecision.txt @@ -0,0 +1,7 @@ +Output format: +- Return valid JSON only (no markdown fences, no extra text). +- Follow the provided Pydantic schema exactly. +- Field guidance from schema descriptions: +- `verdict`: One of: "confirmed", "likely", "inconclusive", "not_exploitable" +- `evidence_level`: 1-6 scale: 1=STATIC_MATCH to 6=FULL_EXPLOIT +- `confidence`: One of: "high", "medium", "low" \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_ArchitectureMapRaw.txt b/internal/secaf/gates/testdata/golden/schema_retry_ArchitectureMapRaw.txt new file mode 100644 index 0000000000..0618249008 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_ArchitectureMapRaw.txt @@ -0,0 +1,62 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "Flat harness output for architecture mapper. All list[str], no nesting.", + "properties": { + "app_type": { + "default": "unknown", + "description": "Application type: web_api, cli_tool, library, microservice, monolith", + "title": "App Type", + "type": "string" + }, + "modules": { + "description": "One string per module. Format: 'name | path | language | description'. Example: 'auth | src/auth/ | python | Authentication and session management'", + "items": { + "type": "string" + }, + "title": "Modules", + "type": "array" + }, + "entry_points": { + "description": "One string per entry point. Format: 'kind | route_or_id | file_path:line | auth_required'. Example: 'http | POST /api/login | src/routes.py:42 | false'", + "items": { + "type": "string" + }, + "title": "Entry Points", + "type": "array" + }, + "trust_boundaries": { + "description": "One string per boundary. Format: 'name | source_zone | target_zone | description'. Example: 'API Gateway | external | internal | Rate limiting and auth'", + "items": { + "type": "string" + }, + "title": "Trust Boundaries", + "type": "array" + }, + "services": { + "description": "One string per external service. Format: 'name | type | endpoint | auth_mechanism'. Example: 'PostgreSQL | database | localhost:5432 | password'", + "items": { + "type": "string" + }, + "title": "Services", + "type": "array" + }, + "api_endpoints": { + "description": "One string per API endpoint. Format: 'method | path | handler | file_path:line | auth_required | rate_limited'. Example: 'GET | /api/users | get_users | src/api.py:15 | true | false'", + "items": { + "type": "string" + }, + "title": "Api Endpoints", + "type": "array" + } + }, + "title": "ArchitectureMapRaw", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_CWEExpansion.txt b/internal/secaf/gates/testdata/golden/schema_retry_CWEExpansion.txt new file mode 100644 index 0000000000..e6ae8d7d6a --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_CWEExpansion.txt @@ -0,0 +1,32 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "AI-suggested CWE additions based on recon context.", + "properties": { + "additional_cwes": { + "description": "CWE IDs to add beyond baseline, e.g. ['CWE-918', 'CWE-611'].", + "items": { + "type": "string" + }, + "title": "Additional Cwes", + "type": "array" + }, + "rationale": { + "title": "Rationale", + "type": "string" + } + }, + "required": [ + "additional_cwes", + "rationale" + ], + "title": "CWEExpansion", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_ChainCorrelationResult.txt b/internal/secaf/gates/testdata/golden/schema_retry_ChainCorrelationResult.txt new file mode 100644 index 0000000000..6e770d30a6 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_ChainCorrelationResult.txt @@ -0,0 +1,32 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "Flat harness schema for chain correlation. LLM identifies chains only.", + "properties": { + "chains": { + "description": "Multi-step attack chains found. Format per entry: 'title | finding_id1,finding_id2,... | combined_impact | severity'. Example: 'SSRF to Internal API | f1,f2 | Access internal services | high'", + "items": { + "type": "string" + }, + "title": "Chains", + "type": "array" + }, + "duplicate_ids": { + "description": "Finding IDs that are duplicates missed by programmatic dedup (to drop)", + "items": { + "type": "string" + }, + "title": "Duplicate Ids", + "type": "array" + } + }, + "title": "ChainCorrelationResult", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_ComplianceGate.txt b/internal/secaf/gates/testdata/golden/schema_retry_ComplianceGate.txt new file mode 100644 index 0000000000..c00d06b905 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_ComplianceGate.txt @@ -0,0 +1,55 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "$defs": { + "ComplianceSuggestion": { + "properties": { + "framework": { + "title": "Framework", + "type": "string" + }, + "control_id": { + "title": "Control Id", + "type": "string" + }, + "control_name": { + "title": "Control Name", + "type": "string" + } + }, + "required": [ + "framework", + "control_id", + "control_name" + ], + "title": "ComplianceSuggestion", + "type": "object" + } + }, + "properties": { + "mappings": { + "items": { + "$ref": "#/$defs/ComplianceSuggestion" + }, + "title": "Mappings", + "type": "array" + }, + "confidence": { + "title": "Confidence", + "type": "string" + } + }, + "required": [ + "mappings", + "confidence" + ], + "title": "ComplianceGate", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_ConfigReportRaw.txt b/internal/secaf/gates/testdata/golden/schema_retry_ConfigReportRaw.txt new file mode 100644 index 0000000000..22fc49026d --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_ConfigReportRaw.txt @@ -0,0 +1,32 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "Flat harness output for config scanner. All list[str], no nesting.", + "properties": { + "secrets": { + "description": "One string per secret finding. Format: 'type | file_path:line | match_preview | confidence | is_test(true/false)'. Example: 'aws_access_key | .env:3 | AKIA... | high | false'", + "items": { + "type": "string" + }, + "title": "Secrets", + "type": "array" + }, + "misconfigs": { + "description": "One string per misconfiguration. Format: 'category | file_path:line | key | risk | remediation'. Example: 'debug_mode | config.py:15 | DEBUG=True | Exposes stack traces | Set DEBUG=False'", + "items": { + "type": "string" + }, + "title": "Misconfigs", + "type": "array" + } + }, + "title": "ConfigReportRaw", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_CrossServiceFinding.txt b/internal/secaf/gates/testdata/golden/schema_retry_CrossServiceFinding.txt new file mode 100644 index 0000000000..062fd8300e --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_CrossServiceFinding.txt @@ -0,0 +1,45 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "Flat schema for cross-service attack chain analysis. 4 fields.", + "properties": { + "chain_description": { + "description": "Description of the cross-service attack path", + "title": "Chain Description", + "type": "string" + }, + "services_involved": { + "description": "Service names in the attack chain", + "items": { + "type": "string" + }, + "title": "Services Involved", + "type": "array" + }, + "entry_point": { + "description": "Public-facing entry point where attack begins", + "title": "Entry Point", + "type": "string" + }, + "impact": { + "description": "Impact if the cross-service chain is exploited", + "title": "Impact", + "type": "string" + } + }, + "required": [ + "chain_description", + "services_involved", + "entry_point", + "impact" + ], + "title": "CrossServiceFinding", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_DastVerificationResult.txt b/internal/secaf/gates/testdata/golden/schema_retry_DastVerificationResult.txt new file mode 100644 index 0000000000..1b54f03f33 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_DastVerificationResult.txt @@ -0,0 +1,42 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "Flat schema for DAST-like runtime verification. 4 fields.", + "properties": { + "payload_sent": { + "description": "The exploit payload or request that was sent", + "title": "Payload Sent", + "type": "string" + }, + "response_summary": { + "description": "Summary of the application response", + "title": "Response Summary", + "type": "string" + }, + "exploit_confirmed": { + "description": "Whether the exploit was confirmed at runtime", + "title": "Exploit Confirmed", + "type": "boolean" + }, + "safety_notes": { + "description": "Safety measures taken during verification (sandbox, timeout, etc.)", + "title": "Safety Notes", + "type": "string" + } + }, + "required": [ + "payload_sent", + "response_summary", + "exploit_confirmed", + "safety_notes" + ], + "title": "DastVerificationResult", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_DataFlowMapRaw.txt b/internal/secaf/gates/testdata/golden/schema_retry_DataFlowMapRaw.txt new file mode 100644 index 0000000000..7af7a3dd24 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_DataFlowMapRaw.txt @@ -0,0 +1,40 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "Flat harness output for data flow mapper. All list[str], no nesting.", + "properties": { + "flows": { + "description": "One string per data flow. Format: 'source | sink | sanitized(true/false) | file1, file2, ...'. Example: 'request.body | sql.execute | false | src/db.py, src/routes.py'", + "items": { + "type": "string" + }, + "title": "Flows", + "type": "array" + }, + "sanitization_points": { + "description": "One string per sanitization point. Format: 'file_path:line | function_name | type | protects_against'. Example: 'src/utils.py:42 | sanitize_html | html_encoding | CWE-79'", + "items": { + "type": "string" + }, + "title": "Sanitization Points", + "type": "array" + }, + "sinks": { + "description": "One string per security-critical sink. Format: 'sink_type | file_path:line | function_name | notes'. Example: 'sql_execute | src/db.py:55 | run_query | Direct string concatenation'", + "items": { + "type": "string" + }, + "title": "Sinks", + "type": "array" + } + }, + "title": "DataFlowMapRaw", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_DataFlowTrace.txt b/internal/secaf/gates/testdata/golden/schema_retry_DataFlowTrace.txt new file mode 100644 index 0000000000..d03b8d74a9 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_DataFlowTrace.txt @@ -0,0 +1,45 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "Flat schema for data flow tracing sub-agent. 4 fields.", + "properties": { + "source": { + "description": "Where tainted input enters (e.g. 'request.params.id')", + "title": "Source", + "type": "string" + }, + "sink": { + "description": "Security-sensitive operation reached (e.g. 'sql.execute(query)')", + "title": "Sink", + "type": "string" + }, + "steps": { + "description": "Ordered list of file:line descriptions showing flow path", + "items": { + "type": "string" + }, + "title": "Steps", + "type": "array" + }, + "sink_reached": { + "description": "Whether tainted data actually reaches the sink", + "title": "Sink Reached", + "type": "boolean" + } + }, + "required": [ + "source", + "sink", + "steps", + "sink_reached" + ], + "title": "DataFlowTrace", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_DependencyReportRaw.txt b/internal/secaf/gates/testdata/golden/schema_retry_DependencyReportRaw.txt new file mode 100644 index 0000000000..0f2e499bf6 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_DependencyReportRaw.txt @@ -0,0 +1,40 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "Flat harness output for dependency auditor. All list[str], no nesting.", + "properties": { + "sbom": { + "description": "One string per dependency. Format: 'name | version | ecosystem | direct(true/false) | license'. Example: 'express | 4.18.2 | npm | true | MIT'", + "items": { + "type": "string" + }, + "title": "Sbom", + "type": "array" + }, + "known_cves": { + "description": "One string per CVE. Format: 'cve_id | package | installed_version | fixed_version | cvss_score | direct | reachable'. Example: 'CVE-2023-1234 | lodash | 4.17.15 | 4.17.21 | 7.5 | true | unknown'", + "items": { + "type": "string" + }, + "title": "Known Cves", + "type": "array" + }, + "outdated": { + "description": "One string per outdated dep. Format: 'package | current_version | latest_version | direct(true/false)'. Example: 'express | 4.17.0 | 4.18.2 | true'", + "items": { + "type": "string" + }, + "title": "Outdated", + "type": "array" + } + }, + "title": "DependencyReportRaw", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_DuplicateCheck.txt b/internal/secaf/gates/testdata/golden/schema_retry_DuplicateCheck.txt new file mode 100644 index 0000000000..4c098f42e5 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_DuplicateCheck.txt @@ -0,0 +1,40 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "DESIGN.md \u00a75.5: quick duplicate check gate for dedup decisions.", + "properties": { + "is_duplicate": { + "title": "Is Duplicate", + "type": "boolean" + }, + "duplicate_of": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duplicate Of" + }, + "reason": { + "title": "Reason", + "type": "string" + } + }, + "required": [ + "is_duplicate", + "reason" + ], + "title": "DuplicateCheck", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_EnrichedFinding.txt b/internal/secaf/gates/testdata/golden/schema_retry_EnrichedFinding.txt new file mode 100644 index 0000000000..671e69f99c --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_EnrichedFinding.txt @@ -0,0 +1,54 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "Flat schema for Step 2: finding enrichment. 6 fields.", + "properties": { + "title": { + "description": "Human-readable title for the finding", + "title": "Title", + "type": "string" + }, + "description": { + "description": "Detailed description of the vulnerability", + "title": "Description", + "type": "string" + }, + "cwe_id": { + "description": "CWE identifier (e.g. 'CWE-89')", + "title": "Cwe Id", + "type": "string" + }, + "severity": { + "description": "One of: \"critical\", \"high\", \"medium\", \"low\", \"info\"", + "title": "Severity", + "type": "string" + }, + "confidence": { + "description": "One of: \"high\", \"medium\", \"low\"", + "title": "Confidence", + "type": "string" + }, + "data_flow_summary": { + "description": "Natural language summary of the data flow (string, not nested)", + "title": "Data Flow Summary", + "type": "string" + } + }, + "required": [ + "title", + "description", + "cwe_id", + "severity", + "confidence", + "data_flow_summary" + ], + "title": "EnrichedFinding", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_ExploitHypothesis.txt b/internal/secaf/gates/testdata/golden/schema_retry_ExploitHypothesis.txt new file mode 100644 index 0000000000..7b8d884ef4 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_ExploitHypothesis.txt @@ -0,0 +1,43 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "Flat schema for exploit construction sub-agent. 3 fields.", + "properties": { + "hypothesis": { + "description": "Natural language description of exploit scenario", + "title": "Hypothesis", + "type": "string" + }, + "payload": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Concrete exploit payload or input", + "title": "Payload" + }, + "expected_outcome": { + "description": "What would happen if exploit succeeds", + "title": "Expected Outcome", + "type": "string" + } + }, + "required": [ + "hypothesis", + "expected_outcome" + ], + "title": "ExploitHypothesis", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_PolicyEvalResult.txt b/internal/secaf/gates/testdata/golden/schema_retry_PolicyEvalResult.txt new file mode 100644 index 0000000000..feafd20556 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_PolicyEvalResult.txt @@ -0,0 +1,42 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "Flat schema for AI policy evaluation. 4 fields.", + "properties": { + "violated": { + "description": "Whether the policy is violated", + "title": "Violated", + "type": "boolean" + }, + "description": { + "description": "How the policy is violated, or 'No violation' if compliant", + "title": "Description", + "type": "string" + }, + "file_path": { + "description": "Primary file where violation occurs, or 'N/A'", + "title": "File Path", + "type": "string" + }, + "severity": { + "description": "Severity: \"high\", \"medium\", or \"low\"", + "title": "Severity", + "type": "string" + } + }, + "required": [ + "violated", + "description", + "file_path", + "severity" + ], + "title": "PolicyEvalResult", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_ReachabilityGate.txt b/internal/secaf/gates/testdata/golden/schema_retry_ReachabilityGate.txt new file mode 100644 index 0000000000..9c474097f1 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_ReachabilityGate.txt @@ -0,0 +1,35 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "Reachability assessment for findings without explicit reachability tags.", + "properties": { + "reachability": { + "description": "One of: \"externally_reachable\", \"requires_auth\", \"internal_only\", \"unreachable\".", + "title": "Reachability", + "type": "string" + }, + "rationale": { + "title": "Rationale", + "type": "string" + }, + "confidence": { + "description": "One of: \"high\", \"medium\", \"low\".", + "title": "Confidence", + "type": "string" + } + }, + "required": [ + "reachability", + "rationale", + "confidence" + ], + "title": "ReachabilityGate", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_ReachabilityProof.txt b/internal/secaf/gates/testdata/golden/schema_retry_ReachabilityProof.txt new file mode 100644 index 0000000000..885d73d529 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_ReachabilityProof.txt @@ -0,0 +1,45 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "Flat schema for dependency reachability analysis. 4 fields.", + "properties": { + "vulnerable_function": { + "description": "The vulnerable function/method in the dependency", + "title": "Vulnerable Function", + "type": "string" + }, + "call_chain": { + "description": "Import/call chain from app code to vulnerable function", + "items": { + "type": "string" + }, + "title": "Call Chain", + "type": "array" + }, + "reachable": { + "description": "Whether the vulnerable function is actually called", + "title": "Reachable", + "type": "boolean" + }, + "direct": { + "description": "Whether the dependency is direct or transitive", + "title": "Direct", + "type": "boolean" + } + }, + "required": [ + "vulnerable_function", + "call_chain", + "reachable", + "direct" + ], + "title": "ReachabilityProof", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_RemediationSuggestion.txt b/internal/secaf/gates/testdata/golden/schema_retry_RemediationSuggestion.txt new file mode 100644 index 0000000000..1a51fed5e4 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_RemediationSuggestion.txt @@ -0,0 +1,36 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "Flat schema for AI-generated remediation suggestion. 3 fields.", + "properties": { + "fix_description": { + "description": "Natural language description of the recommended fix", + "title": "Fix Description", + "type": "string" + }, + "patch_diff": { + "description": "Unified diff format patch showing the code changes needed", + "title": "Patch Diff", + "type": "string" + }, + "confidence": { + "description": "Confidence in the fix: \"high\", \"medium\", or \"low\"", + "title": "Confidence", + "type": "string" + } + }, + "required": [ + "fix_description", + "patch_diff", + "confidence" + ], + "title": "RemediationSuggestion", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_SanitizationResult.txt b/internal/secaf/gates/testdata/golden/schema_retry_SanitizationResult.txt new file mode 100644 index 0000000000..5fc1402224 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_SanitizationResult.txt @@ -0,0 +1,63 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "Flat schema for sanitization analysis sub-agent. 4 fields.", + "properties": { + "found": { + "description": "Whether any sanitization/validation was found on the path", + "title": "Found", + "type": "boolean" + }, + "type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Type of sanitization (e.g. 'parameterized query', 'html encoding')", + "title": "Type" + }, + "sufficient": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Whether sanitization is sufficient to prevent exploit", + "title": "Sufficient" + }, + "bypass_method": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "How sanitization could be bypassed, if applicable", + "title": "Bypass Method" + } + }, + "required": [ + "found" + ], + "title": "SanitizationResult", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_ScanLocationsResult.txt b/internal/secaf/gates/testdata/golden/schema_retry_ScanLocationsResult.txt new file mode 100644 index 0000000000..93a7495a22 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_ScanLocationsResult.txt @@ -0,0 +1,58 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "$defs": { + "VulnLocation": { + "description": "Flat schema for Step 1: location scanning. 4 fields.", + "properties": { + "file_path": { + "description": "Path to the file containing the potential vulnerability", + "title": "File Path", + "type": "string" + }, + "start_line": { + "description": "Starting line number of the vulnerable code", + "title": "Start Line", + "type": "integer" + }, + "code_snippet": { + "description": "Relevant code snippet around the vulnerability", + "title": "Code Snippet", + "type": "string" + }, + "pattern_type": { + "description": "Type of vulnerability pattern detected (e.g. 'sql_injection', 'command_injection')", + "title": "Pattern Type", + "type": "string" + } + }, + "required": [ + "file_path", + "start_line", + "code_snippet", + "pattern_type" + ], + "title": "VulnLocation", + "type": "object" + } + }, + "description": "Container for Step 1 results.", + "properties": { + "locations": { + "items": { + "$ref": "#/$defs/VulnLocation" + }, + "title": "Locations", + "type": "array" + } + }, + "title": "ScanLocationsResult", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_SecurityContextRaw.txt b/internal/secaf/gates/testdata/golden/schema_retry_SecurityContextRaw.txt new file mode 100644 index 0000000000..83ca3174d1 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_SecurityContextRaw.txt @@ -0,0 +1,46 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "Flat harness output for security context profiler. All flat, no nesting.", + "properties": { + "auth_model": { + "description": "Authentication model: jwt, session_cookie, oauth2, api_key, none, or other", + "title": "Auth Model", + "type": "string" + }, + "auth_details": { + "default": "", + "description": "Brief description of auth implementation details", + "title": "Auth Details", + "type": "string" + }, + "crypto_usage": { + "description": "One string per crypto usage. Format: 'algorithm | key_size | mode | usage_context | is_weak(true/false)'. Example: 'AES | 256 | GCM | data encryption | false'", + "items": { + "type": "string" + }, + "title": "Crypto Usage", + "type": "array" + }, + "security_signals": { + "description": "Framework security features, security headers, and deployment signals. One signal per entry. Examples: 'CSRF protection enabled', 'HSTS header present', 'Runs in Docker'", + "items": { + "type": "string" + }, + "title": "Security Signals", + "type": "array" + } + }, + "required": [ + "auth_model" + ], + "title": "SecurityContextRaw", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_SeverityClassification.txt b/internal/secaf/gates/testdata/golden/schema_retry_SeverityClassification.txt new file mode 100644 index 0000000000..983147bb72 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_SeverityClassification.txt @@ -0,0 +1,34 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "DESIGN.md \u00a72.4: quick severity classification gate used in scoring.", + "properties": { + "severity": { + "description": "One of: \"critical\", \"high\", \"medium\", \"low\".", + "title": "Severity", + "type": "string" + }, + "confidence": { + "title": "Confidence", + "type": "number" + }, + "rationale": { + "title": "Rationale", + "type": "string" + } + }, + "required": [ + "severity", + "confidence", + "rationale" + ], + "title": "SeverityClassification", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_StrategySelection.txt b/internal/secaf/gates/testdata/golden/schema_retry_StrategySelection.txt new file mode 100644 index 0000000000..8e04a60f14 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_StrategySelection.txt @@ -0,0 +1,31 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "DESIGN.md \u00a75.3: strategy selection gate for HUNT routing.", + "properties": { + "strategies": { + "items": { + "type": "string" + }, + "title": "Strategies", + "type": "array" + }, + "rationale": { + "title": "Rationale", + "type": "string" + } + }, + "required": [ + "strategies", + "rationale" + ], + "title": "StrategySelection", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/schema_retry_VerdictDecision.txt b/internal/secaf/gates/testdata/golden/schema_retry_VerdictDecision.txt new file mode 100644 index 0000000000..d74877d86a --- /dev/null +++ b/internal/secaf/gates/testdata/golden/schema_retry_VerdictDecision.txt @@ -0,0 +1,41 @@ +The JSON output at /tmp/secaf-golden/.agentfield_output.json failed validation. +Error: Retry attempt 1/3 + +Your response must conform to this JSON schema: +```json +{ + "description": "Flat schema for verdict sub-agent. Uses .ai() not .harness(). 4 fields.", + "properties": { + "verdict": { + "description": "One of: \"confirmed\", \"likely\", \"inconclusive\", \"not_exploitable\"", + "title": "Verdict", + "type": "string" + }, + "evidence_level": { + "description": "1-6 scale: 1=STATIC_MATCH to 6=FULL_EXPLOIT", + "title": "Evidence Level", + "type": "integer" + }, + "rationale": { + "title": "Rationale", + "type": "string" + }, + "confidence": { + "description": "One of: \"high\", \"medium\", \"low\"", + "title": "Confidence", + "type": "string" + } + }, + "required": [ + "verdict", + "evidence_level", + "rationale", + "confidence" + ], + "title": "VerdictDecision", + "type": "object" +} +``` + +Rewrite the COMPLETE, corrected JSON to: /tmp/secaf-golden/.agentfield_output.json +The file must contain ONLY valid JSON matching the schema above. No markdown fences, no extra text, no comments. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/with_file_write_hint_basic.txt b/internal/secaf/gates/testdata/golden/with_file_write_hint_basic.txt new file mode 100644 index 0000000000..fd07516f15 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/with_file_write_hint_basic.txt @@ -0,0 +1,4 @@ +Constraints: +- first +- second +- If output is large or complex, use the file-write pattern and ensure final JSON is written to /tmp/secaf-golden/.agentfield_output.json. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/with_file_write_hint_empty_cwd.txt b/internal/secaf/gates/testdata/golden/with_file_write_hint_empty_cwd.txt new file mode 100644 index 0000000000..afd6c08fad --- /dev/null +++ b/internal/secaf/gates/testdata/golden/with_file_write_hint_empty_cwd.txt @@ -0,0 +1,2 @@ +no directory +- If output is large or complex, use the file-write pattern and ensure final JSON is written to .agentfield_output.json. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/with_file_write_hint_relative_cwd.txt b/internal/secaf/gates/testdata/golden/with_file_write_hint_relative_cwd.txt new file mode 100644 index 0000000000..9caa1d76bc --- /dev/null +++ b/internal/secaf/gates/testdata/golden/with_file_write_hint_relative_cwd.txt @@ -0,0 +1,2 @@ +relative +- If output is large or complex, use the file-write pattern and ensure final JSON is written to work/../work/.agentfield_output.json. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/with_file_write_hint_trailing_whitespace.txt b/internal/secaf/gates/testdata/golden/with_file_write_hint_trailing_whitespace.txt new file mode 100644 index 0000000000..d6ea64dffa --- /dev/null +++ b/internal/secaf/gates/testdata/golden/with_file_write_hint_trailing_whitespace.txt @@ -0,0 +1,2 @@ +keep me +- If output is large or complex, use the file-write pattern and ensure final JSON is written to /tmp/secaf-golden/.agentfield_output.json. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/with_phase_guidance_empty.txt b/internal/secaf/gates/testdata/golden/with_phase_guidance_empty.txt new file mode 100644 index 0000000000..9802154de8 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/with_phase_guidance_empty.txt @@ -0,0 +1,15 @@ +Context: +- Build conclusions from repository evidence in iterative passes. Prefer explicit evidence over speculation, and clearly separate confirmed facts from uncertainty. + +Constraints: +- Use evidence-first reasoning; do not speculate beyond available artifacts. +- Keep analysis bounded to the task scope and produce only schema-conformant output. +- Cite concrete repository evidence whenever making security-relevant claims. +- If output is large or complex, use the file-write pattern and ensure final JSON is written to /tmp/secaf-golden/.agentfield_output.json. + +Task: +Analyze the repository for SQL injection. +Cite file:line for every claim. + +Output: +- Return a single JSON object matching the requested schema. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/with_phase_guidance_hunt.txt b/internal/secaf/gates/testdata/golden/with_phase_guidance_hunt.txt new file mode 100644 index 0000000000..a2eb69a0d0 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/with_phase_guidance_hunt.txt @@ -0,0 +1,25 @@ +Context: +- APPROACH: You are hunting for a specific class of security vulnerability with recon context. +PROCESS: +1. Review the recon context to understand the codebase topology +2. Identify files and patterns relevant to your specific vulnerability class +3. For each candidate: read the code, trace data flow, assess exploitability +4. Only report findings where you have concrete code evidence +CONSTRAINTS: +- Every finding MUST cite specific file paths and line numbers you have read +- Do not report theoretical vulnerabilities without code evidence +- False negatives are better than false positives +- If a file is sanitized properly, do NOT report it + +Constraints: +- Use evidence-first reasoning; do not speculate beyond available artifacts. +- Keep analysis bounded to the task scope and produce only schema-conformant output. +- Cite concrete repository evidence whenever making security-relevant claims. +- If output is large or complex, use the file-write pattern and ensure final JSON is written to /tmp/secaf-golden/.agentfield_output.json. + +Task: +Analyze the repository for SQL injection. +Cite file:line for every claim. + +Output: +- Return a single JSON object matching the requested schema. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/with_phase_guidance_none.txt b/internal/secaf/gates/testdata/golden/with_phase_guidance_none.txt new file mode 100644 index 0000000000..9802154de8 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/with_phase_guidance_none.txt @@ -0,0 +1,15 @@ +Context: +- Build conclusions from repository evidence in iterative passes. Prefer explicit evidence over speculation, and clearly separate confirmed facts from uncertainty. + +Constraints: +- Use evidence-first reasoning; do not speculate beyond available artifacts. +- Keep analysis bounded to the task scope and produce only schema-conformant output. +- Cite concrete repository evidence whenever making security-relevant claims. +- If output is large or complex, use the file-write pattern and ensure final JSON is written to /tmp/secaf-golden/.agentfield_output.json. + +Task: +Analyze the repository for SQL injection. +Cite file:line for every claim. + +Output: +- Return a single JSON object matching the requested schema. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/with_phase_guidance_padded_mixed_case.txt b/internal/secaf/gates/testdata/golden/with_phase_guidance_padded_mixed_case.txt new file mode 100644 index 0000000000..4f02ddcddb --- /dev/null +++ b/internal/secaf/gates/testdata/golden/with_phase_guidance_padded_mixed_case.txt @@ -0,0 +1,24 @@ +Context: +- APPROACH: You are performing reconnaissance on a codebase to build an accurate structural map. +PROCESS: +1. Survey the codebase structure — identify key directories, entry points, and configuration +2. Identify the technology stack — languages, frameworks, and external services +3. Map security-relevant boundaries — auth layers, data inputs, API surfaces +4. Only after surveying, synthesize findings into the required schema +CONSTRAINTS: +- Report what IS there, not what MIGHT be there +- If uncertain about a detail, omit it rather than guess +- Prioritize breadth over depth — cover the full surface + +Constraints: +- Use evidence-first reasoning; do not speculate beyond available artifacts. +- Keep analysis bounded to the task scope and produce only schema-conformant output. +- Cite concrete repository evidence whenever making security-relevant claims. +- If output is large or complex, use the file-write pattern and ensure final JSON is written to /tmp/secaf-golden/.agentfield_output.json. + +Task: +Analyze the repository for SQL injection. +Cite file:line for every claim. + +Output: +- Return a single JSON object matching the requested schema. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/with_phase_guidance_prove.txt b/internal/secaf/gates/testdata/golden/with_phase_guidance_prove.txt new file mode 100644 index 0000000000..0978befcce --- /dev/null +++ b/internal/secaf/gates/testdata/golden/with_phase_guidance_prove.txt @@ -0,0 +1,26 @@ +Context: +- APPROACH: You are verifying a specific candidate vulnerability for exploitability. +PROCESS: +1. Read the specific code location cited in the finding +2. Trace the data flow from source to sink +3. Check for sanitization, validation, or other mitigations on the path +4. If exploitable, construct a concrete exploit hypothesis +5. Synthesize your verdict with evidence level +CONSTRAINTS: +- You must READ the actual code — do not rely on the finding description alone +- INCONCLUSIVE is a valid verdict — do not force confirmation or denial +- Cite specific lines where sanitization exists or is missing +- If code has changed since the finding was generated, note the discrepancy + +Constraints: +- Use evidence-first reasoning; do not speculate beyond available artifacts. +- Keep analysis bounded to the task scope and produce only schema-conformant output. +- Cite concrete repository evidence whenever making security-relevant claims. +- If output is large or complex, use the file-write pattern and ensure final JSON is written to /tmp/secaf-golden/.agentfield_output.json. + +Task: +Analyze the repository for SQL injection. +Cite file:line for every claim. + +Output: +- Return a single JSON object matching the requested schema. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/with_phase_guidance_recon.txt b/internal/secaf/gates/testdata/golden/with_phase_guidance_recon.txt new file mode 100644 index 0000000000..4f02ddcddb --- /dev/null +++ b/internal/secaf/gates/testdata/golden/with_phase_guidance_recon.txt @@ -0,0 +1,24 @@ +Context: +- APPROACH: You are performing reconnaissance on a codebase to build an accurate structural map. +PROCESS: +1. Survey the codebase structure — identify key directories, entry points, and configuration +2. Identify the technology stack — languages, frameworks, and external services +3. Map security-relevant boundaries — auth layers, data inputs, API surfaces +4. Only after surveying, synthesize findings into the required schema +CONSTRAINTS: +- Report what IS there, not what MIGHT be there +- If uncertain about a detail, omit it rather than guess +- Prioritize breadth over depth — cover the full surface + +Constraints: +- Use evidence-first reasoning; do not speculate beyond available artifacts. +- Keep analysis bounded to the task scope and produce only schema-conformant output. +- Cite concrete repository evidence whenever making security-relevant claims. +- If output is large or complex, use the file-write pattern and ensure final JSON is written to /tmp/secaf-golden/.agentfield_output.json. + +Task: +Analyze the repository for SQL injection. +Cite file:line for every claim. + +Output: +- Return a single JSON object matching the requested schema. \ No newline at end of file diff --git a/internal/secaf/gates/testdata/golden/with_phase_guidance_unknown.txt b/internal/secaf/gates/testdata/golden/with_phase_guidance_unknown.txt new file mode 100644 index 0000000000..9802154de8 --- /dev/null +++ b/internal/secaf/gates/testdata/golden/with_phase_guidance_unknown.txt @@ -0,0 +1,15 @@ +Context: +- Build conclusions from repository evidence in iterative passes. Prefer explicit evidence over speculation, and clearly separate confirmed facts from uncertainty. + +Constraints: +- Use evidence-first reasoning; do not speculate beyond available artifacts. +- Keep analysis bounded to the task scope and produce only schema-conformant output. +- Cite concrete repository evidence whenever making security-relevant claims. +- If output is large or complex, use the file-write pattern and ensure final JSON is written to /tmp/secaf-golden/.agentfield_output.json. + +Task: +Analyze the repository for SQL injection. +Cite file:line for every claim. + +Output: +- Return a single JSON object matching the requested schema. \ No newline at end of file diff --git a/internal/secaf/harnessx/fieldorder.go b/internal/secaf/harnessx/fieldorder.go new file mode 100644 index 0000000000..bc65417aa6 --- /dev/null +++ b/internal/secaf/harnessx/fieldorder.go @@ -0,0 +1,98 @@ +package harnessx + +import ( + "reflect" + "strings" +) + +// fieldorder.go exposes the DECLARATION order of a Go struct's json fields — +// the Go side of the port's "struct fields are in pydantic declaration order" +// invariant. +// +// It matters because a pydantic `model_json_schema()` is an insertion-ordered +// dict: `properties` renders in field-declaration order, and the Python SDK's +// `_strictify_openai_schema` builds `required` as `list(props.keys())` over +// that dict (agentfield/agent_ai.py:319). A Go `map[string]any` decoded from +// the committed fixture carries no order at all, so anything that has to +// reproduce a pydantic-ordered document — the `.ai()` request schema +// (internal/aix) and the schema block embedded in a retry prompt +// (internal/gates) — recovers the order here, from the Go type. + +// FieldOrders returns, for t and every struct type reachable from it, the +// ordered json field names of that struct keyed by its Go type name. Anonymous +// (embedded) structs are flattened, `json:"-"` fields are dropped, and a field +// with no json tag keeps its Go name — the same rules encoding/json applies. +func FieldOrders(t reflect.Type) map[string][]string { + out := make(map[string][]string) + collectFieldOrders(t, out) + return out +} + +// FieldOrdersFor is FieldOrders for a type parameter. +func FieldOrdersFor[T any]() map[string][]string { + return FieldOrders(reflect.TypeOf((*T)(nil)).Elem()) +} + +func collectFieldOrders(t reflect.Type, out map[string][]string) { + for t.Kind() == reflect.Pointer { + t = t.Elem() + } + switch t.Kind() { + case reflect.Slice, reflect.Array, reflect.Map: + collectFieldOrders(t.Elem(), out) + return + case reflect.Struct: + default: + return + } + name := t.Name() + if name == "" { + return + } + if _, seen := out[name]; seen { + return // also the recursion guard for self-referential models + } + out[name] = JSONFieldNames(t) + for i := 0; i < t.NumField(); i++ { + if !t.Field(i).IsExported() { + continue + } + collectFieldOrders(t.Field(i).Type, out) + } +} + +// JSONFieldNames returns t's json field names in declaration order. +func JSONFieldNames(t reflect.Type) []string { + for t.Kind() == reflect.Pointer { + t = t.Elem() + } + if t.Kind() != reflect.Struct { + return nil + } + names := make([]string, 0, t.NumField()) + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if !f.IsExported() { + continue + } + name, _, _ := strings.Cut(f.Tag.Get("json"), ",") + if name == "-" { + continue + } + if name == "" { + if f.Anonymous { + inner := f.Type + for inner.Kind() == reflect.Pointer { + inner = inner.Elem() + } + if inner.Kind() == reflect.Struct { + names = append(names, JSONFieldNames(inner)...) + continue + } + } + name = f.Name + } + names = append(names, name) + } + return names +} diff --git a/internal/secaf/harnessx/harnessx_test.go b/internal/secaf/harnessx/harnessx_test.go new file mode 100644 index 0000000000..acf71bcad3 --- /dev/null +++ b/internal/secaf/harnessx/harnessx_test.go @@ -0,0 +1,401 @@ +package harnessx + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" +) + +// --------------------------------------------------------------------------- +// fake harness +// --------------------------------------------------------------------------- + +type fakeHarness struct { + gotPrompt string + gotSchema map[string]any + gotDest any + gotOpts appx.HarnessOptions + calls int + + // fill populates dest before the result is returned, simulating the SDK's + // unmarshal into the destination pointer. + fill func(dest any) + res *appx.HarnessResult + err error +} + +func (f *fakeHarness) Harness(ctx context.Context, prompt string, schema map[string]any, dest any, opts appx.HarnessOptions) (*appx.HarnessResult, error) { + f.calls++ + f.gotPrompt = prompt + f.gotSchema = schema + f.gotDest = dest + f.gotOpts = opts + if f.fill != nil { + f.fill(dest) + } + if f.res != nil && f.res.Parsed == nil && !f.res.IsError && f.err == nil { + // The SDK sets Parsed to the dest pointer it was handed on success. + f.res.Parsed = dest + } + return f.res, f.err +} + +// captureStdout runs fn with Diagnostics pointed at a buffer and returns what +// was written. The name is kept from when Extract printed to stdout. +func captureStdout(t *testing.T, fn func()) string { + t.Helper() + orig := Diagnostics + var buf bytes.Buffer + Diagnostics = &buf + defer func() { Diagnostics = orig }() + fn() + return buf.String() +} + +// --------------------------------------------------------------------------- +// schema fixtures +// --------------------------------------------------------------------------- + +// TestEveryFixtureDecodesAsObjectSchema is the committed-fixture health check: +// gen_schemas.py output must be valid JSON, must describe an object, and must +// carry the title pydantic gives it (which is the class name, which is the +// fixture basename — the three-way identity harnessx resolves against). +func TestEveryFixtureDecodesAsObjectSchema(t *testing.T) { + names := FixtureNames() + if len(names) == 0 { + t.Fatal("no schema fixtures embedded — did go:embed lose testdata/schemas?") + } + for _, name := range names { + m, err := LoadFixture(name) + if err != nil { + t.Errorf("%s: %v", name, err) + continue + } + if got := m["type"]; got != "object" { + t.Errorf("%s: type = %v, want \"object\"", name, got) + } + if got, ok := m["title"].(string); !ok || got != name { + t.Errorf("%s: title = %v, want the pydantic class name %q "+ + "(fixture basename must equal the class name and the Go struct name)", name, m["title"], name) + } + if _, ok := m["properties"].(map[string]any); !ok { + t.Errorf("%s: no properties object", name) + } + } +} + +// TestFixtureSetMatchesGeneratorList pins the enumerated model list so a +// fixture added or removed by hand (rather than by gen_schemas.py) is caught. +// Keep this list in sync with go/scripts/gen_schemas.py's MODELS. +func TestFixtureSetMatchesGeneratorList(t *testing.T) { + want := []string{ + "ArchitectureMapRaw", + "CWEExpansion", + "ChainCorrelationResult", + "ComplianceGate", + "ConfigReportRaw", + "CrossServiceFinding", + "DastVerificationResult", + "DataFlowMapRaw", + "DataFlowTrace", + "DependencyReportRaw", + "DuplicateCheck", + "EnrichedFinding", + "ExploitHypothesis", + "PolicyEvalResult", + "ReachabilityGate", + "ReachabilityProof", + "RemediationSuggestion", + "SanitizationResult", + "ScanLocationsResult", + "SecurityContextRaw", + "SeverityClassification", + "StrategySelection", + "VerdictDecision", + } + got := FixtureNames() + if !reflect.DeepEqual(got, want) { + t.Errorf("fixture set =\n %v\nwant\n %v", got, want) + } +} + +// CWEExpansion stands in for the real schemas.CWEExpansion (owned by the +// schemas package) so this package can prove the by-type-name resolution works +// end to end without importing it. +type CWEExpansion struct { + AdditionalCWEs []string `json:"additional_cwes"` + Rationale string `json:"rationale"` +} + +// TestSchemaForResolvesFixtureByGoTypeName: no RegisterSchema call anywhere — +// the Go type's NAME is the lookup key. +func TestSchemaForResolvesFixtureByGoTypeName(t *testing.T) { + got := SchemaFor[CWEExpansion]() + if got["title"] != "CWEExpansion" { + t.Fatalf("SchemaFor[CWEExpansion] title = %v, want the pydantic fixture", got["title"]) + } + // Pydantic marks BOTH fields required and does NOT set additionalProperties; + // an invopop reflection of the Go struct would look different, which is how + // we know the fixture won. + req, _ := got["required"].([]any) + if len(req) != 2 { + t.Errorf("required = %v, want the pydantic 2-entry list", got["required"]) + } + if _, present := got["additionalProperties"]; present { + t.Errorf("fixture unexpectedly carries additionalProperties: %v", got["additionalProperties"]) + } +} + +// unfixturedResult has no committed fixture, so SchemaFor must fall back to +// invopop reflection. +type unfixturedResult struct { + Name string `json:"name"` + Items []string `json:"items"` +} + +func TestSchemaForFallsBackToReflection(t *testing.T) { + got := SchemaFor[unfixturedResult]() + props, ok := got["properties"].(map[string]any) + if !ok { + t.Fatalf("reflected schema has no properties: %#v", got) + } + for _, key := range []string{"name", "items"} { + if _, present := props[key]; !present { + t.Errorf("reflected schema missing property %q: %#v", key, props) + } + } + // ExpandedStruct puts the root properties inline (the SDK's + // DiagnoseOutputFailure reads map["properties"]). + if got["$ref"] != nil { + t.Errorf("reflected schema is a $ref, want ExpandedStruct inline: %#v", got) + } +} + +func TestSchemaForIsCached(t *testing.T) { + a := SchemaFor[CWEExpansion]() + b := SchemaFor[CWEExpansion]() + if reflect.ValueOf(a).Pointer() != reflect.ValueOf(b).Pointer() { + t.Error("SchemaFor re-resolved the schema instead of using the cache") + } +} + +// --------------------------------------------------------------------------- +// Run +// --------------------------------------------------------------------------- + +func TestRunPassesPromptSchemaAndOptions(t *testing.T) { + fh := &fakeHarness{res: &appx.HarnessResult{}} + opts := appx.HarnessOptions{Cwd: "/tmp/secaf-x", ProjectDir: "/repo"} + + dest, res, err := Run[CWEExpansion](context.Background(), fh, "find CWEs", opts) + if err != nil { + t.Fatalf("Run: %v", err) + } + if dest == nil || res == nil { + t.Fatal("Run returned a nil dest/result on success") + } + if fh.gotPrompt != "find CWEs" { + t.Errorf("prompt = %q", fh.gotPrompt) + } + if fh.gotOpts.Cwd != "/tmp/secaf-x" || fh.gotOpts.ProjectDir != "/repo" { + t.Errorf("opts = %#v, want cwd/project_dir passed through unchanged", fh.gotOpts) + } + if fh.gotSchema["title"] != "CWEExpansion" { + t.Errorf("schema = %v, want the committed pydantic fixture", fh.gotSchema["title"]) + } + if _, ok := fh.gotDest.(*CWEExpansion); !ok { + t.Errorf("dest = %T, want *CWEExpansion", fh.gotDest) + } +} + +func TestRunPropagatesTransportError(t *testing.T) { + want := errors.New("provider binary not found") + fh := &fakeHarness{err: want} + _, _, err := Run[CWEExpansion](context.Background(), fh, "p", appx.HarnessOptions{}) + if !errors.Is(err, want) { + t.Errorf("Run error = %v, want the SDK error propagated", err) + } +} + +// TestRunDoesNotErrorOnHarnessFailure: a harness that RAN and failed comes back +// as (result with IsError, nil error) — mirroring the Python SDK, which returns +// a HarnessResult with is_error=True rather than raising. Extract is what turns +// that into an exception. +func TestRunDoesNotErrorOnHarnessFailure(t *testing.T) { + fh := &fakeHarness{res: &appx.HarnessResult{IsError: true, ErrorMessage: "boom"}} + _, res, err := Run[CWEExpansion](context.Background(), fh, "p", appx.HarnessOptions{}) + if err != nil { + t.Fatalf("Run returned an error for a harness-level failure: %v", err) + } + if res == nil || !res.IsError { + t.Fatalf("Run lost the failed result: %#v", res) + } +} + +// --------------------------------------------------------------------------- +// Extract (agents/_utils.py extract_harness_result) +// --------------------------------------------------------------------------- + +func TestExtractReturnsParsedValue(t *testing.T) { + dest := &CWEExpansion{AdditionalCWEs: []string{"CWE-918"}, Rationale: "ssrf surface"} + res := &appx.HarnessResult{Parsed: dest} + + var got CWEExpansion + out := captureStdout(t, func() { + var err error + got, err = Extract(res, dest, "ArchitectureMapper") + if err != nil { + t.Errorf("Extract: %v", err) + } + }) + if out != "" { + t.Errorf("Extract printed diagnostics on the success path: %q", out) + } + if !reflect.DeepEqual(got, *dest) { + t.Errorf("Extract = %#v, want %#v", got, *dest) + } +} + +// TestExtractHarnessErrorRaises pins the RuntimeError text AND the stdout +// diagnostic block, both byte-for-byte against the Python helper: +// +// print(f"[{agent_name}] HARNESS ERROR: {error_message}\n" +// f" turns={num_turns}, duration_ms={duration_ms}\n" +// f" result_text={str(result_text)[:500] if result_text else None}", flush=True) +// raise RuntimeError(f"{agent_name} harness error: {error_message}") +func TestExtractHarnessErrorRaises(t *testing.T) { + res := &appx.HarnessResult{ + IsError: true, + ErrorMessage: "provider exited 1", + Result: "partial text", + NumTurns: 7, + DurationMS: 1234, + } + var err error + out := captureStdout(t, func() { + _, err = Extract(res, &CWEExpansion{}, "DependencyAuditor") + }) + + wantErr := "DependencyAuditor harness error: provider exited 1" + if err == nil || err.Error() != wantErr { + t.Errorf("Extract error = %v, want %q", err, wantErr) + } + wantOut := "[DependencyAuditor] HARNESS ERROR: provider exited 1\n" + + " turns=7, duration_ms=1234\n" + + " result_text=partial text\n" + if out != wantOut { + t.Errorf("diagnostics =\n%q\nwant\n%q", out, wantOut) + } +} + +// TestExtractHarnessErrorEmptyResultTextPrintsNone: Python's +// `str(result_text)[:500] if result_text else None` treats "" as falsy. +func TestExtractHarnessErrorEmptyResultTextPrintsNone(t *testing.T) { + res := &appx.HarnessResult{IsError: true, ErrorMessage: "no output", Result: "", NumTurns: 0, DurationMS: 0} + out := captureStdout(t, func() { _, _ = Extract(res, &CWEExpansion{}, "ConfigScanner") }) + want := "[ConfigScanner] HARNESS ERROR: no output\n turns=0, duration_ms=0\n result_text=None\n" + if out != want { + t.Errorf("diagnostics =\n%q\nwant\n%q", out, want) + } +} + +// TestExtractHarnessErrorTruncatesResultTextByRunes: Python slices str by code +// points, not bytes. +func TestExtractHarnessErrorTruncatesResultTextByRunes(t *testing.T) { + long := strings.Repeat("é", 600) // 600 runes, 1200 bytes + res := &appx.HarnessResult{IsError: true, ErrorMessage: "e", Result: long} + out := captureStdout(t, func() { _, _ = Extract(res, &CWEExpansion{}, "A") }) + + prefix := "[A] HARNESS ERROR: e\n turns=0, duration_ms=0\n result_text=" + body := strings.TrimSuffix(strings.TrimPrefix(out, prefix), "\n") + if n := len([]rune(body)); n != 500 { + t.Errorf("result_text truncated to %d runes, want 500 (python str[:500])", n) + } +} + +// TestExtractParsedNilRaisesTypeError pins the TypeError-equivalent message and +// the debug line. +func TestExtractParsedNilRaisesTypeError(t *testing.T) { + res := &appx.HarnessResult{Parsed: nil} + var err error + out := captureStdout(t, func() { + _, err = Extract(res, &CWEExpansion{}, "SecurityContextProfiler") + }) + wantErr := "SecurityContextProfiler did not return a valid CWEExpansion" + if err == nil || err.Error() != wantErr { + t.Errorf("Extract error = %v, want %q", err, wantErr) + } + wantOut := "[SecurityContextProfiler] harness result type=Result, is_error=False, parsed type=NoneType\n" + if out != wantOut { + t.Errorf("debug line = %q, want %q", out, wantOut) + } +} + +// TestExtractNilResult: Python's getattr(None, ...) defaults land in the same +// TypeError branch. +func TestExtractNilResult(t *testing.T) { + var err error + out := captureStdout(t, func() { + _, err = Extract[CWEExpansion](nil, nil, "Verifier") + }) + if err == nil || err.Error() != "Verifier did not return a valid CWEExpansion" { + t.Errorf("Extract error = %v", err) + } + if want := "[Verifier] harness result type=NoneType, is_error=False, parsed type=NoneType\n"; out != want { + t.Errorf("debug line = %q, want %q", out, want) + } +} + +// --------------------------------------------------------------------------- +// RunExtract +// --------------------------------------------------------------------------- + +func TestRunExtractSuccess(t *testing.T) { + fh := &fakeHarness{ + res: &appx.HarnessResult{}, + fill: func(dest any) { + _ = json.Unmarshal([]byte(`{"additional_cwes":["CWE-611"],"rationale":"xxe"}`), dest) + }, + } + var got CWEExpansion + out := captureStdout(t, func() { + var err error + got, err = RunExtract[CWEExpansion](context.Background(), fh, "p", appx.HarnessOptions{}, "CWEExpander") + if err != nil { + t.Errorf("RunExtract: %v", err) + } + }) + if out != "" { + t.Errorf("RunExtract printed diagnostics on success: %q", out) + } + want := CWEExpansion{AdditionalCWEs: []string{"CWE-611"}, Rationale: "xxe"} + if !reflect.DeepEqual(got, want) { + t.Errorf("RunExtract = %#v, want %#v", got, want) + } +} + +func TestRunExtractPropagatesTransportError(t *testing.T) { + want := errors.New("spawn failed") + fh := &fakeHarness{err: want} + _, err := RunExtract[CWEExpansion](context.Background(), fh, "p", appx.HarnessOptions{}, "A") + if !errors.Is(err, want) { + t.Errorf("RunExtract error = %v, want the transport error", err) + } +} + +func TestRunExtractMapsHarnessFailureToError(t *testing.T) { + fh := &fakeHarness{res: &appx.HarnessResult{IsError: true, ErrorMessage: "schema validation failed"}} + var err error + _ = captureStdout(t, func() { + _, err = RunExtract[CWEExpansion](context.Background(), fh, "p", appx.HarnessOptions{}, "Tracer") + }) + if err == nil || err.Error() != "Tracer harness error: schema validation failed" { + t.Errorf("RunExtract error = %v", err) + } +} diff --git a/internal/secaf/harnessx/run.go b/internal/secaf/harnessx/run.go new file mode 100644 index 0000000000..c6e0eb9cf2 --- /dev/null +++ b/internal/secaf/harnessx/run.go @@ -0,0 +1,160 @@ +package harnessx + +import ( + "context" + "fmt" + "io" + "reflect" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" +) + +// Diagnostics is where Extract writes the diagnostic block Python printed to +// stdout for a failed or unparseable harness run. It is io.Discard by default: +// inside codeaf stdout is the terminal the person is looking at, so the lines +// go nowhere unless the codeaf side points this at a log. Set it before any +// audit runs; it is read concurrently by every agent. +var Diagnostics io.Writer = io.Discard + +// Run is the Go form of +// +// result = await app.harness(prompt=prompt, schema=Model, cwd=..., project_dir=...) +// +// It resolves Model's schema by Go type name (SchemaFor), allocates the +// destination, and calls app.Harness. +// +// The returned error is NON-NIL ONLY when the run could not be made at all. A +// harness that ran and failed (the agent gave up, unparseable output, schema +// validation exhausted) comes back as a nil error with HarnessResult.IsError +// set, exactly as the Python SDK returned a HarnessResult with is_error=True +// rather than raising; appx.HarnessResult documents that contract for the +// codeaf implementation. Turning that into an error is Extract's job, so the +// diagnostic block is written in one place. +// +// *T is always non-nil so callers can hand it straight to Extract; it holds the +// harness-populated value when HarnessResult.Parsed is set and a +// default-seeded value otherwise. +func Run[T any](ctx context.Context, app appx.Harnesser, prompt string, opts appx.HarnessOptions) (*T, *appx.HarnessResult, error) { + schema := SchemaFor[T]() + dest := new(T) + res, err := app.Harness(ctx, prompt, schema, dest, opts) + if err != nil { + return dest, res, err + } + return dest, res, nil +} + +// Extract ports extract_harness_result from src/sec_af/agents/_utils.py exactly: +// +// def extract_harness_result(result, schema, agent_name): +// is_error = bool(getattr(result, "is_error", False)) +// if is_error: +// ... +// print(f"[{agent_name}] HARNESS ERROR: {error_message}\n" +// f" turns={num_turns}, duration_ms={duration_ms}\n" +// f" result_text={str(result_text)[:500] if result_text else None}", +// flush=True) +// raise RuntimeError(f"{agent_name} harness error: {error_message}") +// parsed = getattr(result, "parsed", None) +// if isinstance(parsed, schema): +// return parsed +// debug_message = (...) +// if isinstance(parsed, dict): +// try: return schema.model_validate(parsed) +// except Exception: print(debug_message, flush=True); raise +// print(debug_message, flush=True) +// raise TypeError(f"{agent_name} did not return a valid {schema.__name__}") +// +// dest is the pointer Run handed to app.Harness. An appx.Harnesser sets +// HarnessResult.Parsed to that same pointer once it holds a schema-valid +// object (the SDK runner this seam used to drive did exactly that), so a +// non-nil Parsed means "*dest is populated and schema-valid" — the exact +// condition `isinstance(parsed, schema)` tests in Python. +// +// Python parity notes: +// +// - The diagnostics are reproduced verbatim (written to Diagnostics rather +// than stdout), including the +// `result_text=None` spelling that Python's `... if result_text else None` +// produces for an EMPTY result string (falsy, not just missing), and the +// 500-character (not byte) truncation of Python's str slice. +// - `type(result).__name__` in the debug line is the Python SDK's +// `HarnessResult`; Go's concrete type is `appx.HarnessResult`, so the line +// prints "Result". The line is diagnostic only. +// - The `isinstance(parsed, dict)` branch is Python duck-typing for a SDK that +// may hand back a raw dict. The Go SDK's Parsed is always the `dest` pointer +// it was given, so that branch is unreachable here and is deliberately not +// ported. +func Extract[T any](res *appx.HarnessResult, dest *T, agentName string) (T, error) { + var zero T + typeName := reflect.TypeOf((*T)(nil)).Elem().Name() + + if res == nil { + // Python: getattr(None, "is_error", False) is False and + // getattr(None, "parsed", None) is None, so a missing result falls + // straight to the TypeError branch after the debug line. + fmt.Fprintf(Diagnostics, "[%s] harness result type=%s, is_error=%s, parsed type=%s\n", + agentName, "NoneType", "False", "NoneType") + return zero, fmt.Errorf("%s did not return a valid %s", agentName, typeName) + } + + if res.IsError { + // Python: `str(result_text)[:500] if result_text else None` — an empty + // result string is falsy, so it prints the literal "None". + resultText := "None" + if res.Result != "" { + resultText = runeSlice(res.Result, 500) + } + fmt.Fprintf(Diagnostics, "[%s] HARNESS ERROR: %s\n turns=%d, duration_ms=%d\n result_text=%s\n", + agentName, res.ErrorMessage, res.NumTurns, res.DurationMS, resultText) + return zero, fmt.Errorf("%s harness error: %s", agentName, res.ErrorMessage) + } + + if res.Parsed != nil && dest != nil { + return *dest, nil + } + + parsedType := "NoneType" + if res.Parsed != nil { + parsedType = reflect.TypeOf(res.Parsed).String() + } + fmt.Fprintf(Diagnostics, "[%s] harness result type=%s, is_error=%s, parsed type=%s\n", + agentName, "Result", pyfmt.Str(res.IsError), parsedType) + return zero, fmt.Errorf("%s did not return a valid %s", agentName, typeName) +} + +// RunExtract is the shape every SEC-AF agent module actually uses: +// +// result = await app.harness(prompt=prompt, schema=Model, cwd=harness_cwd, project_dir=repo_path) +// return extract_harness_result(result, Model, "AgentName") +// +// A transport error from app.Harness propagates unchanged (Python would let the +// SDK's exception propagate out of the `await` the same way); everything else +// goes through Extract. +func RunExtract[T any](ctx context.Context, app appx.Harnesser, prompt string, opts appx.HarnessOptions, agentName string) (T, error) { + // THE AGENT'S NAME RIDES THE OPTIONS, so the session it runs is named by + // it on the audit's page; it is the name this agent's errors already say. + if opts.Label == "" { + opts.Label = agentName + } + dest, res, err := Run[T](ctx, app, prompt, opts) + if err != nil { + var zero T + return zero, err + } + return Extract[T](res, dest, agentName) +} + +// runeSlice reproduces Python's s[:n], which counts Unicode code points, not +// bytes. +func runeSlice(s string, n int) string { + if n < 0 { + n = 0 + } + runes := []rune(s) + if len(runes) <= n { + return s + } + return string(runes[:n]) +} diff --git a/internal/secaf/harnessx/schema.go b/internal/secaf/harnessx/schema.go new file mode 100644 index 0000000000..9a7ab5655c --- /dev/null +++ b/internal/secaf/harnessx/schema.go @@ -0,0 +1,148 @@ +// Package harnessx is the single choke point every SEC-AF agent uses to call +// the AgentField harness for structured output. +// +// It replaces two things the Python source does implicitly: +// +// - `app.harness(prompt=..., schema=SomeModel, cwd=..., project_dir=...)` — +// the pydantic model is resolved here into the JSON-schema map the Go SDK +// consumes, by GO TYPE NAME (see SchemaFor); +// - `extract_harness_result(result, SomeModel, "AgentName")` from +// src/sec_af/agents/_utils.py — ported byte-for-byte as Extract, including +// the diagnostic block it prints to stdout. +// +// RunExtract combines both, because `result = await app.harness(...)` followed +// immediately by `extract_harness_result(result, Model, name)` is what every +// single agent module in src/sec_af/agents does. +package harnessx + +import ( + "embed" + "encoding/json" + "io/fs" + "reflect" + "sort" + "strings" + "sync" + + "github.com/invopop/jsonschema" +) + +// embeddedSchemas holds the committed pydantic-generated JSON schemas, one per +// model SEC-AF passes as `schema=`. They are produced by go/scripts/gen_schemas.py +// from the real pydantic classes, so the schema the Go SDK validates against — +// and pretty-prints into the harness prompt's OUTPUT REQUIREMENTS block — is the +// same one the Python node uses: defaulted fields optional, `X | None` nullable, +// extra keys allowed. +// +// Fixture basename == pydantic class name == Go struct name. That three-way +// identity is the cross-package contract the port is built on (see the design +// doc's "Go struct names == pydantic class names exactly" rule): there is NO +// registration call, so a new Run[T] destination type only needs its Go name to +// match the Python class for the right schema to be picked up. +// +//go:embed testdata/schemas/*.json +var embeddedSchemas embed.FS + +// schemaCache memoizes the resolved schema map per concrete type T so the +// embedded-fixture load (or the non-trivial invopop reflection) runs once per +// type. Keyed by reflect.Type; the stored map is treated as immutable by +// callers — the SDK only ever marshals and reads it — so sharing the cached +// value across goroutines is safe. +var schemaCache sync.Map // reflect.Type -> map[string]any + +// SchemaFor resolves the JSON-schema map the Go SDK harness consumes for T. +// +// Resolution order: +// +// 1. testdata/schemas/.json, embedded above. This is +// the pydantic model_json_schema() of the identically named Python class. +// 2. invopop reflection over the Go type, for types with no fixture (ad-hoc +// test structs, and any future destination whose Python counterpart is not +// in gen_schemas.py's MODELS list). +// +// Why the fixture must win: the SDK runs REAL JSON-Schema validation on parsed +// output (harness/runner.go -> runSchemaValidation, santhosh-tekuri/jsonschema/v5) +// and retries the whole harness call when it fails. An invopop reflection marks +// every field required, renders pointer fields non-nullable and sets +// additionalProperties:false, so Python-valid model output would be rejected and +// the node would burn retries and drop findings. +// +// Invopop reflector configuration (fallback path only): +// - ExpandedStruct: inline the root type's own properties at the top level so +// map["properties"] is populated for the SDK's DiagnoseOutputFailure. +// - DoNotReference=false (default): emit a $defs map for nested struct types. +// - Anonymous: suppress the auto-generated $id derived from the package path. +func SchemaFor[T any]() map[string]any { + t := reflect.TypeOf((*T)(nil)).Elem() + if cached, ok := schemaCache.Load(t); ok { + return cached.(map[string]any) + } + + var m map[string]any + if name := t.Name(); name != "" { + // A load failure is the normal "no fixture for this type" case; it also + // covers the impossible-in-practice corrupt-fixture case, which falls + // through to reflection rather than panicking inside an agent. + if loaded, err := LoadFixture(name); err == nil { + m = loaded + } + } + if m == nil { + m = reflectSchema(t) + } + + schemaCache.Store(t, m) + return m +} + +// LoadFixture decodes the committed pydantic schema fixture with the given +// basename (== the pydantic class name). Exported for tests and for tooling +// that needs a schema without a Go type in hand. +func LoadFixture(name string) (map[string]any, error) { + b, err := embeddedSchemas.ReadFile("testdata/schemas/" + name + ".json") + if err != nil { + return nil, err + } + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + return nil, err + } + return m, nil +} + +// FixtureNames lists every committed schema fixture basename, sorted. +func FixtureNames() []string { + entries, err := fs.ReadDir(embeddedSchemas, "testdata/schemas") + if err != nil { + return nil + } + out := make([]string, 0, len(entries)) + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".json") { + continue + } + out = append(out, strings.TrimSuffix(e.Name(), ".json")) + } + sort.Strings(out) + return out +} + +// reflectSchema is the invopop fallback for types with no committed fixture. +func reflectSchema(t reflect.Type) map[string]any { + r := &jsonschema.Reflector{ + ExpandedStruct: true, // root properties inline at top level + DoNotReference: false, // emit $defs for nested types + Anonymous: true, // no auto-generated $id from PkgPath + } + schema := r.ReflectFromType(t) + + b, err := json.Marshal(schema) + if err != nil { + return map[string]any{} + } + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + return map[string]any{} + } + return m +} diff --git a/internal/secaf/harnessx/schema_parity_test.go b/internal/secaf/harnessx/schema_parity_test.go new file mode 100644 index 0000000000..bc374623e7 --- /dev/null +++ b/internal/secaf/harnessx/schema_parity_test.go @@ -0,0 +1,165 @@ +package harnessx + +import ( + "encoding/json" + "reflect" + "sort" + "testing" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// --------------------------------------------------------------------------- +// Fixture <-> Go struct parity +// +// DESIGN.md's cross-agent contract: the pydantic schema fixture for a Go type +// is resolved BY GO TYPE NAME (testdata/schemas/.json), and that +// document is what SchemaFor[T] sends to the harness. Nothing in the compiler +// ties the two together — a renamed json tag, a dropped field or a stale +// regenerated fixture all compile fine and only show up as a model that +// answers in a shape Go cannot unmarshal. +// +// These tests are that tie. `internal/schemas` already gates its structs +// against a pydantic ground-truth dump (model_keys_test.go); this gates the +// OTHER document — the one that actually reaches the LLM — against the same +// structs, and gates the fixture SET against the types harnessx is asked for. +// --------------------------------------------------------------------------- + +// harnessModels is every type harnessx serves a fixture for, keyed by the +// fixture basename. Adding a fixture without adding it here fails +// TestEveryFixtureIsClaimedByAGoType. +func harnessModels() map[string]any { + return map[string]any{ + "ArchitectureMapRaw": schemas.ArchitectureMapRaw{}, + "CWEExpansion": schemas.CWEExpansion{}, + "ChainCorrelationResult": schemas.ChainCorrelationResult{}, + "ComplianceGate": schemas.ComplianceGate{}, + "ConfigReportRaw": schemas.ConfigReportRaw{}, + "CrossServiceFinding": schemas.CrossServiceFinding{}, + "DastVerificationResult": schemas.DastVerificationResult{}, + "DataFlowMapRaw": schemas.DataFlowMapRaw{}, + "DataFlowTrace": schemas.DataFlowTrace{}, + "DependencyReportRaw": schemas.DependencyReportRaw{}, + "DuplicateCheck": schemas.DuplicateCheck{}, + "EnrichedFinding": schemas.EnrichedFinding{}, + "ExploitHypothesis": schemas.ExploitHypothesis{}, + "PolicyEvalResult": schemas.PolicyEvalResult{}, + "ReachabilityGate": schemas.ReachabilityGate{}, + "ReachabilityProof": schemas.ReachabilityProof{}, + "RemediationSuggestion": schemas.RemediationSuggestion{}, + "SanitizationResult": schemas.SanitizationResult{}, + "ScanLocationsResult": schemas.ScanLocationsResult{}, + "SecurityContextRaw": schemas.SecurityContextRaw{}, + "SeverityClassification": schemas.SeverityClassification{}, + "StrategySelection": schemas.StrategySelection{}, + "VerdictDecision": schemas.VerdictDecision{}, + } +} + +// TestEveryFixtureIsClaimedByAGoType keeps the embedded fixture set and the Go +// types in step in both directions: a fixture nothing decodes into is dead +// weight, and a type with no fixture makes SchemaFor panic at runtime. +func TestEveryFixtureIsClaimedByAGoType(t *testing.T) { + models := harnessModels() + fixtures := FixtureNames() + + have := map[string]bool{} + for _, n := range fixtures { + have[n] = true + if _, ok := models[n]; !ok { + t.Errorf("fixture %q has no Go type in harnessModels — add it, or delete the fixture", n) + } + } + for n := range models { + if !have[n] { + t.Errorf("Go type %q has no embedded fixture testdata/schemas/%s.json", n, n) + } + } + if len(fixtures) != len(models) { + t.Errorf("%d fixtures, %d Go types", len(fixtures), len(models)) + } +} + +// TestSchemaFixturesMatchGoStructTags is the field-level gate the package doc +// asked for: the fixture's top-level "properties" key set must equal the Go +// struct's json tag set exactly. A property Go has no field for is silently +// dropped on decode; a field the schema does not declare is one the model is +// never told to produce. +func TestSchemaFixturesMatchGoStructTags(t *testing.T) { + for name, model := range harnessModels() { + name, model := name, model + t.Run(name, func(t *testing.T) { + schema, err := LoadFixture(name) + if err != nil { + t.Fatalf("load fixture: %v", err) + } + props, ok := schema["properties"].(map[string]any) + if !ok { + t.Fatalf("fixture has no top-level properties object") + } + + want := sortedKeys(props) + got := jsonTagsOf(t, model) + if !reflect.DeepEqual(got, want) { + t.Errorf("json tags != schema properties\n Go: %v\n schema: %v", got, want) + } + }) + } +} + +// TestSchemaFixtureRequiredNamesExistInGo checks the other half: every name the +// schema marks required must be a field Go can actually receive. (The reverse +// is NOT asserted — pydantic omits defaulted fields from `required`, which is +// the documented reason these fixtures are used instead of a Go reflection.) +func TestSchemaFixtureRequiredNamesExistInGo(t *testing.T) { + for name, model := range harnessModels() { + name, model := name, model + t.Run(name, func(t *testing.T) { + schema, err := LoadFixture(name) + if err != nil { + t.Fatalf("load fixture: %v", err) + } + tags := map[string]bool{} + for _, tag := range jsonTagsOf(t, model) { + tags[tag] = true + } + req, _ := schema["required"].([]any) + for _, r := range req { + s, ok := r.(string) + if !ok { + t.Fatalf("required entry %v is not a string", r) + } + if !tags[s] { + t.Errorf("required field %q has no Go json tag", s) + } + } + }) + } +} + +// jsonTagsOf returns the sorted json tag names a zero value of model +// marshals to — i.e. exactly the keys the SDK will hand back to a caller. +// Marshaling (rather than reflecting over the struct type) is deliberate: it +// honours embedded structs, `-` tags and any custom MarshalJSON, so it reports +// the wire shape rather than the declaration. +func jsonTagsOf(t *testing.T, model any) []string { + t.Helper() + b, err := json.Marshal(model) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + t.Fatalf("decode marshaled model: %v", err) + } + return sortedKeys(m) +} + +func sortedKeys(m map[string]any) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/secaf/harnessx/testdata/schemas/ArchitectureMapRaw.json b/internal/secaf/harnessx/testdata/schemas/ArchitectureMapRaw.json new file mode 100644 index 0000000000..6e76dc114c --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/ArchitectureMapRaw.json @@ -0,0 +1,53 @@ +{ + "description": "Flat harness output for architecture mapper. All list[str], no nesting.", + "properties": { + "api_endpoints": { + "description": "One string per API endpoint. Format: 'method | path | handler | file_path:line | auth_required | rate_limited'. Example: 'GET | /api/users | get_users | src/api.py:15 | true | false'", + "items": { + "type": "string" + }, + "title": "Api Endpoints", + "type": "array" + }, + "app_type": { + "default": "unknown", + "description": "Application type: web_api, cli_tool, library, microservice, monolith", + "title": "App Type", + "type": "string" + }, + "entry_points": { + "description": "One string per entry point. Format: 'kind | route_or_id | file_path:line | auth_required'. Example: 'http | POST /api/login | src/routes.py:42 | false'", + "items": { + "type": "string" + }, + "title": "Entry Points", + "type": "array" + }, + "modules": { + "description": "One string per module. Format: 'name | path | language | description'. Example: 'auth | src/auth/ | python | Authentication and session management'", + "items": { + "type": "string" + }, + "title": "Modules", + "type": "array" + }, + "services": { + "description": "One string per external service. Format: 'name | type | endpoint | auth_mechanism'. Example: 'PostgreSQL | database | localhost:5432 | password'", + "items": { + "type": "string" + }, + "title": "Services", + "type": "array" + }, + "trust_boundaries": { + "description": "One string per boundary. Format: 'name | source_zone | target_zone | description'. Example: 'API Gateway | external | internal | Rate limiting and auth'", + "items": { + "type": "string" + }, + "title": "Trust Boundaries", + "type": "array" + } + }, + "title": "ArchitectureMapRaw", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/CWEExpansion.json b/internal/secaf/harnessx/testdata/schemas/CWEExpansion.json new file mode 100644 index 0000000000..900051d94f --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/CWEExpansion.json @@ -0,0 +1,23 @@ +{ + "description": "AI-suggested CWE additions based on recon context.", + "properties": { + "additional_cwes": { + "description": "CWE IDs to add beyond baseline, e.g. ['CWE-918', 'CWE-611'].", + "items": { + "type": "string" + }, + "title": "Additional Cwes", + "type": "array" + }, + "rationale": { + "title": "Rationale", + "type": "string" + } + }, + "required": [ + "additional_cwes", + "rationale" + ], + "title": "CWEExpansion", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/ChainCorrelationResult.json b/internal/secaf/harnessx/testdata/schemas/ChainCorrelationResult.json new file mode 100644 index 0000000000..8c2dd5af78 --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/ChainCorrelationResult.json @@ -0,0 +1,23 @@ +{ + "description": "Flat harness schema for chain correlation. LLM identifies chains only.", + "properties": { + "chains": { + "description": "Multi-step attack chains found. Format per entry: 'title | finding_id1,finding_id2,... | combined_impact | severity'. Example: 'SSRF to Internal API | f1,f2 | Access internal services | high'", + "items": { + "type": "string" + }, + "title": "Chains", + "type": "array" + }, + "duplicate_ids": { + "description": "Finding IDs that are duplicates missed by programmatic dedup (to drop)", + "items": { + "type": "string" + }, + "title": "Duplicate Ids", + "type": "array" + } + }, + "title": "ChainCorrelationResult", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/ComplianceGate.json b/internal/secaf/harnessx/testdata/schemas/ComplianceGate.json new file mode 100644 index 0000000000..0bcf2d556f --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/ComplianceGate.json @@ -0,0 +1,46 @@ +{ + "$defs": { + "ComplianceSuggestion": { + "properties": { + "control_id": { + "title": "Control Id", + "type": "string" + }, + "control_name": { + "title": "Control Name", + "type": "string" + }, + "framework": { + "title": "Framework", + "type": "string" + } + }, + "required": [ + "framework", + "control_id", + "control_name" + ], + "title": "ComplianceSuggestion", + "type": "object" + } + }, + "properties": { + "confidence": { + "title": "Confidence", + "type": "string" + }, + "mappings": { + "items": { + "$ref": "#/$defs/ComplianceSuggestion" + }, + "title": "Mappings", + "type": "array" + } + }, + "required": [ + "mappings", + "confidence" + ], + "title": "ComplianceGate", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/ConfigReportRaw.json b/internal/secaf/harnessx/testdata/schemas/ConfigReportRaw.json new file mode 100644 index 0000000000..d650de4460 --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/ConfigReportRaw.json @@ -0,0 +1,23 @@ +{ + "description": "Flat harness output for config scanner. All list[str], no nesting.", + "properties": { + "misconfigs": { + "description": "One string per misconfiguration. Format: 'category | file_path:line | key | risk | remediation'. Example: 'debug_mode | config.py:15 | DEBUG=True | Exposes stack traces | Set DEBUG=False'", + "items": { + "type": "string" + }, + "title": "Misconfigs", + "type": "array" + }, + "secrets": { + "description": "One string per secret finding. Format: 'type | file_path:line | match_preview | confidence | is_test(true/false)'. Example: 'aws_access_key | .env:3 | AKIA... | high | false'", + "items": { + "type": "string" + }, + "title": "Secrets", + "type": "array" + } + }, + "title": "ConfigReportRaw", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/CrossServiceFinding.json b/internal/secaf/harnessx/testdata/schemas/CrossServiceFinding.json new file mode 100644 index 0000000000..cce480fd6c --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/CrossServiceFinding.json @@ -0,0 +1,36 @@ +{ + "description": "Flat schema for cross-service attack chain analysis. 4 fields.", + "properties": { + "chain_description": { + "description": "Description of the cross-service attack path", + "title": "Chain Description", + "type": "string" + }, + "entry_point": { + "description": "Public-facing entry point where attack begins", + "title": "Entry Point", + "type": "string" + }, + "impact": { + "description": "Impact if the cross-service chain is exploited", + "title": "Impact", + "type": "string" + }, + "services_involved": { + "description": "Service names in the attack chain", + "items": { + "type": "string" + }, + "title": "Services Involved", + "type": "array" + } + }, + "required": [ + "chain_description", + "services_involved", + "entry_point", + "impact" + ], + "title": "CrossServiceFinding", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/DastVerificationResult.json b/internal/secaf/harnessx/testdata/schemas/DastVerificationResult.json new file mode 100644 index 0000000000..ad67d81678 --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/DastVerificationResult.json @@ -0,0 +1,33 @@ +{ + "description": "Flat schema for DAST-like runtime verification. 4 fields.", + "properties": { + "exploit_confirmed": { + "description": "Whether the exploit was confirmed at runtime", + "title": "Exploit Confirmed", + "type": "boolean" + }, + "payload_sent": { + "description": "The exploit payload or request that was sent", + "title": "Payload Sent", + "type": "string" + }, + "response_summary": { + "description": "Summary of the application response", + "title": "Response Summary", + "type": "string" + }, + "safety_notes": { + "description": "Safety measures taken during verification (sandbox, timeout, etc.)", + "title": "Safety Notes", + "type": "string" + } + }, + "required": [ + "payload_sent", + "response_summary", + "exploit_confirmed", + "safety_notes" + ], + "title": "DastVerificationResult", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/DataFlowMapRaw.json b/internal/secaf/harnessx/testdata/schemas/DataFlowMapRaw.json new file mode 100644 index 0000000000..8b9f7add0c --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/DataFlowMapRaw.json @@ -0,0 +1,31 @@ +{ + "description": "Flat harness output for data flow mapper. All list[str], no nesting.", + "properties": { + "flows": { + "description": "One string per data flow. Format: 'source | sink | sanitized(true/false) | file1, file2, ...'. Example: 'request.body | sql.execute | false | src/db.py, src/routes.py'", + "items": { + "type": "string" + }, + "title": "Flows", + "type": "array" + }, + "sanitization_points": { + "description": "One string per sanitization point. Format: 'file_path:line | function_name | type | protects_against'. Example: 'src/utils.py:42 | sanitize_html | html_encoding | CWE-79'", + "items": { + "type": "string" + }, + "title": "Sanitization Points", + "type": "array" + }, + "sinks": { + "description": "One string per security-critical sink. Format: 'sink_type | file_path:line | function_name | notes'. Example: 'sql_execute | src/db.py:55 | run_query | Direct string concatenation'", + "items": { + "type": "string" + }, + "title": "Sinks", + "type": "array" + } + }, + "title": "DataFlowMapRaw", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/DataFlowTrace.json b/internal/secaf/harnessx/testdata/schemas/DataFlowTrace.json new file mode 100644 index 0000000000..6f513973d6 --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/DataFlowTrace.json @@ -0,0 +1,36 @@ +{ + "description": "Flat schema for data flow tracing sub-agent. 4 fields.", + "properties": { + "sink": { + "description": "Security-sensitive operation reached (e.g. 'sql.execute(query)')", + "title": "Sink", + "type": "string" + }, + "sink_reached": { + "description": "Whether tainted data actually reaches the sink", + "title": "Sink Reached", + "type": "boolean" + }, + "source": { + "description": "Where tainted input enters (e.g. 'request.params.id')", + "title": "Source", + "type": "string" + }, + "steps": { + "description": "Ordered list of file:line descriptions showing flow path", + "items": { + "type": "string" + }, + "title": "Steps", + "type": "array" + } + }, + "required": [ + "source", + "sink", + "steps", + "sink_reached" + ], + "title": "DataFlowTrace", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/DependencyReportRaw.json b/internal/secaf/harnessx/testdata/schemas/DependencyReportRaw.json new file mode 100644 index 0000000000..05bcb29ec3 --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/DependencyReportRaw.json @@ -0,0 +1,31 @@ +{ + "description": "Flat harness output for dependency auditor. All list[str], no nesting.", + "properties": { + "known_cves": { + "description": "One string per CVE. Format: 'cve_id | package | installed_version | fixed_version | cvss_score | direct | reachable'. Example: 'CVE-2023-1234 | lodash | 4.17.15 | 4.17.21 | 7.5 | true | unknown'", + "items": { + "type": "string" + }, + "title": "Known Cves", + "type": "array" + }, + "outdated": { + "description": "One string per outdated dep. Format: 'package | current_version | latest_version | direct(true/false)'. Example: 'express | 4.17.0 | 4.18.2 | true'", + "items": { + "type": "string" + }, + "title": "Outdated", + "type": "array" + }, + "sbom": { + "description": "One string per dependency. Format: 'name | version | ecosystem | direct(true/false) | license'. Example: 'express | 4.18.2 | npm | true | MIT'", + "items": { + "type": "string" + }, + "title": "Sbom", + "type": "array" + } + }, + "title": "DependencyReportRaw", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/DuplicateCheck.json b/internal/secaf/harnessx/testdata/schemas/DuplicateCheck.json new file mode 100644 index 0000000000..0e11f47091 --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/DuplicateCheck.json @@ -0,0 +1,31 @@ +{ + "description": "DESIGN.md \u00a75.5: quick duplicate check gate for dedup decisions.", + "properties": { + "duplicate_of": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duplicate Of" + }, + "is_duplicate": { + "title": "Is Duplicate", + "type": "boolean" + }, + "reason": { + "title": "Reason", + "type": "string" + } + }, + "required": [ + "is_duplicate", + "reason" + ], + "title": "DuplicateCheck", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/EnrichedFinding.json b/internal/secaf/harnessx/testdata/schemas/EnrichedFinding.json new file mode 100644 index 0000000000..90c73294a0 --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/EnrichedFinding.json @@ -0,0 +1,45 @@ +{ + "description": "Flat schema for Step 2: finding enrichment. 6 fields.", + "properties": { + "confidence": { + "description": "One of: \"high\", \"medium\", \"low\"", + "title": "Confidence", + "type": "string" + }, + "cwe_id": { + "description": "CWE identifier (e.g. 'CWE-89')", + "title": "Cwe Id", + "type": "string" + }, + "data_flow_summary": { + "description": "Natural language summary of the data flow (string, not nested)", + "title": "Data Flow Summary", + "type": "string" + }, + "description": { + "description": "Detailed description of the vulnerability", + "title": "Description", + "type": "string" + }, + "severity": { + "description": "One of: \"critical\", \"high\", \"medium\", \"low\", \"info\"", + "title": "Severity", + "type": "string" + }, + "title": { + "description": "Human-readable title for the finding", + "title": "Title", + "type": "string" + } + }, + "required": [ + "title", + "description", + "cwe_id", + "severity", + "confidence", + "data_flow_summary" + ], + "title": "EnrichedFinding", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/ExploitHypothesis.json b/internal/secaf/harnessx/testdata/schemas/ExploitHypothesis.json new file mode 100644 index 0000000000..a7a7bd2318 --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/ExploitHypothesis.json @@ -0,0 +1,34 @@ +{ + "description": "Flat schema for exploit construction sub-agent. 3 fields.", + "properties": { + "expected_outcome": { + "description": "What would happen if exploit succeeds", + "title": "Expected Outcome", + "type": "string" + }, + "hypothesis": { + "description": "Natural language description of exploit scenario", + "title": "Hypothesis", + "type": "string" + }, + "payload": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Concrete exploit payload or input", + "title": "Payload" + } + }, + "required": [ + "hypothesis", + "expected_outcome" + ], + "title": "ExploitHypothesis", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/PolicyEvalResult.json b/internal/secaf/harnessx/testdata/schemas/PolicyEvalResult.json new file mode 100644 index 0000000000..62e465db99 --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/PolicyEvalResult.json @@ -0,0 +1,33 @@ +{ + "description": "Flat schema for AI policy evaluation. 4 fields.", + "properties": { + "description": { + "description": "How the policy is violated, or 'No violation' if compliant", + "title": "Description", + "type": "string" + }, + "file_path": { + "description": "Primary file where violation occurs, or 'N/A'", + "title": "File Path", + "type": "string" + }, + "severity": { + "description": "Severity: \"high\", \"medium\", or \"low\"", + "title": "Severity", + "type": "string" + }, + "violated": { + "description": "Whether the policy is violated", + "title": "Violated", + "type": "boolean" + } + }, + "required": [ + "violated", + "description", + "file_path", + "severity" + ], + "title": "PolicyEvalResult", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/ReachabilityGate.json b/internal/secaf/harnessx/testdata/schemas/ReachabilityGate.json new file mode 100644 index 0000000000..75b991a947 --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/ReachabilityGate.json @@ -0,0 +1,26 @@ +{ + "description": "Reachability assessment for findings without explicit reachability tags.", + "properties": { + "confidence": { + "description": "One of: \"high\", \"medium\", \"low\".", + "title": "Confidence", + "type": "string" + }, + "rationale": { + "title": "Rationale", + "type": "string" + }, + "reachability": { + "description": "One of: \"externally_reachable\", \"requires_auth\", \"internal_only\", \"unreachable\".", + "title": "Reachability", + "type": "string" + } + }, + "required": [ + "reachability", + "rationale", + "confidence" + ], + "title": "ReachabilityGate", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/ReachabilityProof.json b/internal/secaf/harnessx/testdata/schemas/ReachabilityProof.json new file mode 100644 index 0000000000..bf4f5e9c1c --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/ReachabilityProof.json @@ -0,0 +1,36 @@ +{ + "description": "Flat schema for dependency reachability analysis. 4 fields.", + "properties": { + "call_chain": { + "description": "Import/call chain from app code to vulnerable function", + "items": { + "type": "string" + }, + "title": "Call Chain", + "type": "array" + }, + "direct": { + "description": "Whether the dependency is direct or transitive", + "title": "Direct", + "type": "boolean" + }, + "reachable": { + "description": "Whether the vulnerable function is actually called", + "title": "Reachable", + "type": "boolean" + }, + "vulnerable_function": { + "description": "The vulnerable function/method in the dependency", + "title": "Vulnerable Function", + "type": "string" + } + }, + "required": [ + "vulnerable_function", + "call_chain", + "reachable", + "direct" + ], + "title": "ReachabilityProof", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/RemediationSuggestion.json b/internal/secaf/harnessx/testdata/schemas/RemediationSuggestion.json new file mode 100644 index 0000000000..5722347cc4 --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/RemediationSuggestion.json @@ -0,0 +1,27 @@ +{ + "description": "Flat schema for AI-generated remediation suggestion. 3 fields.", + "properties": { + "confidence": { + "description": "Confidence in the fix: \"high\", \"medium\", or \"low\"", + "title": "Confidence", + "type": "string" + }, + "fix_description": { + "description": "Natural language description of the recommended fix", + "title": "Fix Description", + "type": "string" + }, + "patch_diff": { + "description": "Unified diff format patch showing the code changes needed", + "title": "Patch Diff", + "type": "string" + } + }, + "required": [ + "fix_description", + "patch_diff", + "confidence" + ], + "title": "RemediationSuggestion", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/SanitizationResult.json b/internal/secaf/harnessx/testdata/schemas/SanitizationResult.json new file mode 100644 index 0000000000..07ef4863a3 --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/SanitizationResult.json @@ -0,0 +1,54 @@ +{ + "description": "Flat schema for sanitization analysis sub-agent. 4 fields.", + "properties": { + "bypass_method": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "How sanitization could be bypassed, if applicable", + "title": "Bypass Method" + }, + "found": { + "description": "Whether any sanitization/validation was found on the path", + "title": "Found", + "type": "boolean" + }, + "sufficient": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Whether sanitization is sufficient to prevent exploit", + "title": "Sufficient" + }, + "type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Type of sanitization (e.g. 'parameterized query', 'html encoding')", + "title": "Type" + } + }, + "required": [ + "found" + ], + "title": "SanitizationResult", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/ScanLocationsResult.json b/internal/secaf/harnessx/testdata/schemas/ScanLocationsResult.json new file mode 100644 index 0000000000..77d54fdfe2 --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/ScanLocationsResult.json @@ -0,0 +1,49 @@ +{ + "$defs": { + "VulnLocation": { + "description": "Flat schema for Step 1: location scanning. 4 fields.", + "properties": { + "code_snippet": { + "description": "Relevant code snippet around the vulnerability", + "title": "Code Snippet", + "type": "string" + }, + "file_path": { + "description": "Path to the file containing the potential vulnerability", + "title": "File Path", + "type": "string" + }, + "pattern_type": { + "description": "Type of vulnerability pattern detected (e.g. 'sql_injection', 'command_injection')", + "title": "Pattern Type", + "type": "string" + }, + "start_line": { + "description": "Starting line number of the vulnerable code", + "title": "Start Line", + "type": "integer" + } + }, + "required": [ + "file_path", + "start_line", + "code_snippet", + "pattern_type" + ], + "title": "VulnLocation", + "type": "object" + } + }, + "description": "Container for Step 1 results.", + "properties": { + "locations": { + "items": { + "$ref": "#/$defs/VulnLocation" + }, + "title": "Locations", + "type": "array" + } + }, + "title": "ScanLocationsResult", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/SecurityContextRaw.json b/internal/secaf/harnessx/testdata/schemas/SecurityContextRaw.json new file mode 100644 index 0000000000..43eae9f583 --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/SecurityContextRaw.json @@ -0,0 +1,37 @@ +{ + "description": "Flat harness output for security context profiler. All flat, no nesting.", + "properties": { + "auth_details": { + "default": "", + "description": "Brief description of auth implementation details", + "title": "Auth Details", + "type": "string" + }, + "auth_model": { + "description": "Authentication model: jwt, session_cookie, oauth2, api_key, none, or other", + "title": "Auth Model", + "type": "string" + }, + "crypto_usage": { + "description": "One string per crypto usage. Format: 'algorithm | key_size | mode | usage_context | is_weak(true/false)'. Example: 'AES | 256 | GCM | data encryption | false'", + "items": { + "type": "string" + }, + "title": "Crypto Usage", + "type": "array" + }, + "security_signals": { + "description": "Framework security features, security headers, and deployment signals. One signal per entry. Examples: 'CSRF protection enabled', 'HSTS header present', 'Runs in Docker'", + "items": { + "type": "string" + }, + "title": "Security Signals", + "type": "array" + } + }, + "required": [ + "auth_model" + ], + "title": "SecurityContextRaw", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/SeverityClassification.json b/internal/secaf/harnessx/testdata/schemas/SeverityClassification.json new file mode 100644 index 0000000000..fcc533f1ec --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/SeverityClassification.json @@ -0,0 +1,25 @@ +{ + "description": "DESIGN.md \u00a72.4: quick severity classification gate used in scoring.", + "properties": { + "confidence": { + "title": "Confidence", + "type": "number" + }, + "rationale": { + "title": "Rationale", + "type": "string" + }, + "severity": { + "description": "One of: \"critical\", \"high\", \"medium\", \"low\".", + "title": "Severity", + "type": "string" + } + }, + "required": [ + "severity", + "confidence", + "rationale" + ], + "title": "SeverityClassification", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/StrategySelection.json b/internal/secaf/harnessx/testdata/schemas/StrategySelection.json new file mode 100644 index 0000000000..ab6c4856a8 --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/StrategySelection.json @@ -0,0 +1,22 @@ +{ + "description": "DESIGN.md \u00a75.3: strategy selection gate for HUNT routing.", + "properties": { + "rationale": { + "title": "Rationale", + "type": "string" + }, + "strategies": { + "items": { + "type": "string" + }, + "title": "Strategies", + "type": "array" + } + }, + "required": [ + "strategies", + "rationale" + ], + "title": "StrategySelection", + "type": "object" +} diff --git a/internal/secaf/harnessx/testdata/schemas/VerdictDecision.json b/internal/secaf/harnessx/testdata/schemas/VerdictDecision.json new file mode 100644 index 0000000000..7b93379659 --- /dev/null +++ b/internal/secaf/harnessx/testdata/schemas/VerdictDecision.json @@ -0,0 +1,32 @@ +{ + "description": "Flat schema for verdict sub-agent. Uses .ai() not .harness(). 4 fields.", + "properties": { + "confidence": { + "description": "One of: \"high\", \"medium\", \"low\"", + "title": "Confidence", + "type": "string" + }, + "evidence_level": { + "description": "1-6 scale: 1=STATIC_MATCH to 6=FULL_EXPLOIT", + "title": "Evidence Level", + "type": "integer" + }, + "rationale": { + "title": "Rationale", + "type": "string" + }, + "verdict": { + "description": "One of: \"confirmed\", \"likely\", \"inconclusive\", \"not_exploitable\"", + "title": "Verdict", + "type": "string" + } + }, + "required": [ + "verdict", + "evidence_level", + "rationale", + "confidence" + ], + "title": "VerdictDecision", + "type": "object" +} diff --git a/internal/secaf/limits.go b/internal/secaf/limits.go new file mode 100644 index 0000000000..a09b9ab9b8 --- /dev/null +++ b/internal/secaf/limits.go @@ -0,0 +1,126 @@ +package secaf + +// Each of sec's agents is bounded by what its own work takes. +// +// sec-af gave every agent one cap, fifty turns and thirty minutes +// (SEC_AF_MAX_TURNS, the harness's wait). The owner's whole-repository audit of +// a mid-sized Rust repository on 2026-10-05 measured what each agent actually +// used, and the figures below are that measurement with headroom: about half +// again over the longest session seen, more where a session was cut at the +// old cap. An agent with no measurement keeps the old cap. +// +// agent sessions turns (median, longest) longest time +// hunt scan 11 38, 50 (cut) 10m18s +// hunt enrich 67 7, 30 11m18s +// prove tracer 22 9, 30 4m28s +// prove sanitization 22 7, 27 4m10s +// prove exploit 22 12, 40 10m40s +// recon config scanner 1 50 (cut) 3m43s +// recon architecture 1 32 2m42s +// recon data flow 1 33 3m25s +// recon dependencies 1 12 2m14s +// recon security context 1 11 15s +// dedup (chain) 1 16 10m00s +// +// A person's --max-turns or --session-wall at a shell is one figure for every +// agent, and overrides the table. + +import ( + "path/filepath" + "strings" + "time" + + "github.com/Agent-Field/codeaf/internal/agentsession" + "github.com/Agent-Field/codeaf/internal/agentsession/appx" +) + +// agentLimits is each agent's bounds, by the name its scratch folder starts +// with after `secaf-` (every agent makes one with os.MkdirTemp, and the name is +// the agent's own constant, where its label names the file it is looking at). +// The first prefix that matches wins. +var agentLimits = []struct { + kind string + limits agentsession.Limits +}{ + {"hunt-scan", agentsession.Limits{Turns: 75, Wall: 20 * time.Minute}}, + {"hunt-enrich", agentsession.Limits{Turns: 45, Wall: 20 * time.Minute}}, + {"prove-tracer", agentsession.Limits{Turns: 45, Wall: 10 * time.Minute}}, + {"prove-sanitization", agentsession.Limits{Turns: 40, Wall: 10 * time.Minute}}, + {"prove-exploit", agentsession.Limits{Turns: 60, Wall: 20 * time.Minute}}, + {"recon-config-scanner", agentsession.Limits{Turns: 75, Wall: 15 * time.Minute}}, + {"recon-architecture", agentsession.Limits{Turns: 50, Wall: 10 * time.Minute}}, + {"recon-data-flow", agentsession.Limits{Turns: 50, Wall: 10 * time.Minute}}, + {"recon-dependencies", agentsession.Limits{Turns: 30, Wall: 10 * time.Minute}}, + {"recon-security-context", agentsession.Limits{Turns: 30, Wall: 5 * time.Minute}}, + {"dedup", agentsession.Limits{Turns: 30, Wall: 20 * time.Minute}}, +} + +// limitsFor is the bounds sec gives the agent a session was started for: the +// person's own figure when they set one, else the agent's from the table, and +// zero (the App's default) for an agent the table does not name. +func limitsFor(o options) func(appx.HarnessOptions) agentsession.Limits { + return func(opts appx.HarnessOptions) agentsession.Limits { + own := agentLimitsOf(opts.Cwd) + if o.maxTurns > 0 { + own.Turns = o.maxTurns + } + if o.sessionWall > 0 { + own.Wall = o.sessionWall + } + return own + } +} + +// agentLimitsOf is the table's bounds for the agent whose scratch folder is +// cwd, or zero for one it does not name. +func agentLimitsOf(cwd string) agentsession.Limits { + name, ours := strings.CutPrefix(filepath.Base(strings.TrimSpace(cwd)), "secaf-") + if !ours { + return agentsession.Limits{} + } + for _, row := range agentLimits { + if strings.HasPrefix(name, row.kind+"-") { + return row.limits + } + } + return agentsession.Limits{} +} + +// defaultAgentTurns and defaultAgentWall are sec-af's own one cap, kept for +// every agent the table does not name. +const ( + defaultAgentTurns = 50 + defaultAgentWall = 30 * time.Minute +) + +// sessionsAtOnce and callsAtOnce are how many agent sessions and single model +// calls run at once across the audit: sec-af's own fan-outs multiply — four +// hunters each enriching five locations — and the old harness capped its +// processes at eight for the same reason. --sessions sets the first. +const ( + sessionsAtOnce = 8 + callsAtOnce = 8 +) + +// retries is how many times a model call that failed on the way is sent +// again, past the model API's own ladder of services. +const retries = 2 + +// sessionPolicy is how every one of sec's agent sessions reads and answers. +// These are the figures sec-af's harness ran on, now said here as sec's own +// (agentsession has no defaults; its Policy says why). +var sessionPolicy = agentsession.Policy{ + // The old harness's own count of asks for an answer of the right shape. + FollowUps: 2, + // About a hundred thousand tokens, which every model a person is likely to + // seat holds with room for the answer. + ContextChars: 400_000, + AnswerNow: "Stop reading now and give your answer from what you have found, in the form the system message asks for.", + Tools: agentsession.ToolLimits{ + ReadLines: 400, ReadBytes: 48 << 10, ReadLineRunes: 2000, + ListEntries: 400, GlobMatches: 300, + // A file past two megabytes is a bundle, a dump or a dataset, and its + // lines are not code to trace. + GrepMatches: 200, GrepFileBytes: 2 << 20, GrepLineRunes: 240, + }, +} diff --git a/internal/secaf/markdown.go b/internal/secaf/markdown.go new file mode 100644 index 0000000000..2b7ed8cdf9 --- /dev/null +++ b/internal/secaf/markdown.go @@ -0,0 +1,274 @@ +package secaf + +// sec's readable report, `sec-report.md`, in the words the conversation's +// account uses. +// +// sec-af wrote its own (internal/secaf/output's GenerateReport, kept byte for +// byte with its Python original and its goldens), and it spoke sec-af's +// vocabulary — a finding's "Verdict: inconclusive", "not exploitable", +// "noise reduction" — beside figures that are not true here: its cost was +// always $0.00, its commit `HEAD` and its provider `harness`. A person read +// "unclear" in the chat and "inconclusive" in the file about the same finding. +// So the readable file is written here, from the same result, in the same +// words as the account: confirmed, likely, unclear, ruled out. The JSON and the +// SARIF keep sec-af's field names, because tools read them. +// +// THE EMPTINESS LAW HOLDS IN THE FILE TOO: a figure nobody measured, a field +// the audit left empty, is not written, never written as zero. + +import ( + "fmt" + "sort" + "strings" + "time" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// runFacts is what the run knows about itself that the audit's result does +// not: what it audited, what it spent and how long it took. +type runFacts struct { + scope Scope + changes *Changes + spent float64 + sessions int + calls int + took time.Duration + // cutAt is the phase the run's time ceiling stopped it in, and wall that + // ceiling in words; empty for a run that finished in its time. + cutAt, wall string +} + +// markdownReport is the readable report. +func markdownReport(result schemas.SecurityAuditResult, run runFacts) string { + var b strings.Builder + fmt.Fprintf(&b, "# sec — security audit of %s\n\n", run.scope.Describe()) + var facts []string + if result.Repository != "" { + facts = append(facts, "`"+result.Repository+"`") + } + if result.Branch != nil && strings.TrimSpace(*result.Branch) != "" { + facts = append(facts, "branch `"+*result.Branch+"`") + } + if run.changes != nil { + facts = append(facts, fmt.Sprintf("%d changed file%s since `%s`, %d near them", + len(run.changes.Changed), plural(len(run.changes.Changed)), run.changes.BaseName, len(run.changes.Nearby))) + } + if stamp := strings.TrimSpace(result.Timestamp.String()); stamp != "" { + facts = append(facts, stamp) + } + if len(facts) > 0 { + b.WriteString(strings.Join(facts, " · ") + "\n\n") + } + + if run.cutAt != "" { + fmt.Fprintf(&b, "> **Cut short.** %s\n\n", cutSentence(run.cutAt, run.wall)) + } + b.WriteString("## What it found\n\n") + standing, unclear, ruledOut := sortFindings(result.Findings) + if len(standing) == 0 { + b.WriteString("Nothing it could show exploitable.") + } else { + fmt.Fprintf(&b, "%d problem%s: %d confirmed and %d likely", len(standing), plural(len(standing)), result.Confirmed, result.Likely) + if counts := severityCounts(standing); counts != "" { + b.WriteString(" (" + counts + ")") + } + b.WriteString(".") + } + if len(unclear) > 0 { + fmt.Fprintf(&b, " %d stayed unclear: it could neither show them exploitable nor rule them out.", len(unclear)) + } + if len(ruledOut) > 0 { + fmt.Fprintf(&b, " %d %s ruled out.", len(ruledOut), wasWere(len(ruledOut))) + } + b.WriteString("\n\n") + + if len(standing) > 0 { + b.WriteString("## Confirmed and likely\n\n") + for _, finding := range standing { + writeFinding(&b, finding) + } + } + if len(unclear) > 0 { + b.WriteString("## Unclear\n\nWorth a look by someone who knows the code; the audit could not settle these either way.\n\n") + for _, finding := range unclear { + writeFinding(&b, finding) + } + } + if len(ruledOut) > 0 { + b.WriteString("## Ruled out\n\n") + for _, finding := range ruledOut { + fmt.Fprintf(&b, "- %s", oneLine(finding.Title)) + if where := locationWord(finding.Location); where != "" { + b.WriteString(" · `" + where + "`") + } + if why := oneLine(finding.Rationale); why != "" { + b.WriteString(" — " + why) + } + b.WriteString("\n") + } + b.WriteString("\n") + } + + if len(result.AttackChains) > 0 { + titles := map[string]string{} + for _, finding := range result.Findings { + titles[finding.ID] = oneLine(finding.Title) + } + b.WriteString("## Attack chains\n\nFindings that are worse together than apart.\n\n") + for _, chain := range result.AttackChains { + fmt.Fprintf(&b, "### %s", oneLine(chain.Title)) + if chain.CombinedSeverity != "" { + fmt.Fprintf(&b, " · %s", chain.CombinedSeverity) + } + b.WriteString("\n\n") + if text := oneLine(chain.Description); text != "" { + b.WriteString(text + "\n\n") + } + if impact := oneLine(chain.CombinedImpact); impact != "" { + b.WriteString("- Impact: " + impact + "\n") + } + for _, id := range chain.Findings { + if title := titles[id]; title != "" { + b.WriteString("- " + title + "\n") + } + } + for _, mapping := range chain.MitreAttackMapping { + fmt.Fprintf(&b, "- MITRE ATT&CK %s (%s): %s\n", mapping.TechniqueID, mapping.Tactic, mapping.TechniqueName) + } + b.WriteString("\n") + } + } + + if len(result.ComplianceGaps) > 0 { + b.WriteString("## Compliance gaps\n\n") + for _, gap := range result.ComplianceGaps { + fmt.Fprintf(&b, "- %s %s: %s · %d finding%s, worst %s\n", gap.Framework, gap.ControlID, gap.ControlName, + gap.FindingCount, plural(gap.FindingCount), gap.MaxSeverity) + } + b.WriteString("\n") + } + + var ran []string + if run.sessions > 0 { + ran = append(ran, fmt.Sprintf("%d agent session%s", run.sessions, plural(run.sessions))) + } + if run.calls > 0 { + ran = append(ran, fmt.Sprintf("%d single call%s", run.calls, plural(run.calls))) + } + if run.spent > 0 { + ran = append(ran, fmt.Sprintf("$%.2f", run.spent)) + } + if run.took >= time.Second { + ran = append(ran, run.took.Round(time.Second).String()) + } + if len(ran) > 0 { + b.WriteString("## How it ran\n\n" + strings.Join(ran, " · ") + ", at " + run.scope.Depth + " depth.\n") + } + return strings.TrimRight(b.String(), "\n") + "\n" +} + +// sortFindings splits the findings by how they came out, each most serious +// first. +func sortFindings(findings []schemas.VerifiedFinding) (standing, unclear, ruledOut []schemas.VerifiedFinding) { + for _, finding := range findings { + switch finding.Verdict { + case schemas.VerdictConfirmed, schemas.VerdictLikely: + standing = append(standing, finding) + case schemas.VerdictNotExploitable: + ruledOut = append(ruledOut, finding) + default: + unclear = append(unclear, finding) + } + } + for _, list := range [][]schemas.VerifiedFinding{standing, unclear, ruledOut} { + sort.SliceStable(list, func(i, j int) bool { + if a, b := severityRank(list[i].Severity), severityRank(list[j].Severity); a != b { + return a < b + } + return list[i].Verdict == schemas.VerdictConfirmed && list[j].Verdict != schemas.VerdictConfirmed + }) + } + return standing, unclear, ruledOut +} + +// writeFinding is one finding in full: where, what weakness, why it stands, +// how data reaches it, the attack, and the fix. +func writeFinding(b *strings.Builder, finding schemas.VerifiedFinding) { + fmt.Fprintf(b, "### %s · %s · %s\n\n", finding.Severity, findingWord(finding.Verdict), oneLine(finding.Title)) + if where := locationWord(finding.Location); where != "" { + line := "- Where: `" + where + "`" + if fn := finding.Location.FunctionName; fn != nil && strings.TrimSpace(*fn) != "" { + line += " in `" + strings.TrimSpace(*fn) + "`" + } + b.WriteString(line + "\n") + } + if finding.CweID != "" { + line := "- Weakness: " + finding.CweID + if name := oneLine(finding.CweName); name != "" { + line += " (" + name + ")" + } + if owasp := finding.OwaspCategory; owasp != nil && strings.TrimSpace(*owasp) != "" { + line += " · OWASP " + strings.TrimSpace(*owasp) + } + b.WriteString(line + "\n") + } + if finding.ExploitabilityScore > 0 { + fmt.Fprintf(b, "- Exploitability: %.1f of 10\n", finding.ExploitabilityScore) + } + if why := oneLine(finding.Rationale); why != "" { + b.WriteString("- Why: " + why + "\n") + } + if proof := finding.Proof; proof != nil { + if len(proof.DataFlowTrace) > 0 { + b.WriteString("- How the data gets there:\n") + for _, step := range proof.DataFlowTrace { + where := step.File + if step.Line > 0 { + where = fmt.Sprintf("%s:%d", step.File, step.Line) + } + if strings.TrimSpace(where) != "" { + fmt.Fprintf(b, " - `%s` %s\n", where, oneLine(step.Description)) + } else { + fmt.Fprintf(b, " - %s\n", oneLine(step.Description)) + } + } + } + if attack := oneLine(proof.ExploitHypothesis); attack != "" { + b.WriteString("- The attack: " + attack + "\n") + } + if payload := proof.ExploitPayload; payload != nil && strings.TrimSpace(*payload) != "" { + b.WriteString("- Payload: `" + oneLine(*payload) + "`\n") + } + } + if fix := finding.Remediation; fix != nil { + if text := oneLine(fix.FixDescription); text != "" { + b.WriteString("- Fix: " + text + "\n") + } + if patch := strings.TrimSpace(fix.PatchDiff); patch != "" { + b.WriteString("\n```diff\n" + patch + "\n```\n") + } + } + if len(finding.Compliance) > 0 { + var controls []string + for _, mapping := range finding.Compliance { + controls = append(controls, strings.TrimSpace(mapping.Framework+" "+mapping.ControlID)) + } + b.WriteString("- Compliance: " + strings.Join(controls, ", ") + "\n") + } + b.WriteString("\n") +} + +func wasWere(n int) string { + if n == 1 { + return "was" + } + return "were" +} + +func areIs(n int) string { + if n == 1 { + return "is" + } + return "are" +} diff --git a/internal/secaf/monitoring/monitoring.go b/internal/secaf/monitoring/monitoring.go new file mode 100644 index 0000000000..0987b63293 --- /dev/null +++ b/internal/secaf/monitoring/monitoring.go @@ -0,0 +1,278 @@ +// Package monitoring ports src/sec_af/monitoring.py — continuous monitoring: +// baseline storage and regression detection. +// +// It compares a current SecurityAuditResult against a stored baseline to +// identify new vulnerabilities (regressions) and fixed issues, keyed on the +// finding FINGERPRINT (the stable content hash), never on the finding id. +// +// Python parity — scope: monitoring.py is NOT wired into app.py. AuditInput +// declares `monitoring_mode` and `baseline_path`, but nothing reads them, so no +// reasoner calls save_baseline/compare_with_baseline today. This package is +// ported for 1:1 completeness (and because the baseline file format is a +// user-visible artifact); it must NOT be wired into the Go node either, or the +// two implementations would diverge in behavior rather than only in language. +package monitoring + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// BaselineFinding is one finding as stored in a baseline file. +// +// Ports monitoring.py's BaselineFinding TypedDict. The Go struct's FIELD ORDER +// is the dict-literal order in save_baseline, which is what json.dumps writes +// (Python dicts are insertion-ordered) — see baselineFindingObject. +type BaselineFinding struct { + ID string `json:"id"` + Fingerprint string `json:"fingerprint"` + Title string `json:"title"` + Severity string `json:"severity"` + CweID string `json:"cwe_id"` + Verdict string `json:"verdict"` + FilePath string `json:"file_path"` + StartLine int `json:"start_line"` +} + +// BaselineData is the whole baseline document. +// +// Ports monitoring.py's BaselineData TypedDict. Because Python reads the two +// top-level keys DIFFERENTLY — `baseline.get("commit_sha", "unknown")` tolerates +// an absent key, `baseline["findings"]` raises KeyError on one — a decoded +// BaselineData also remembers which keys the file actually carried. Use +// CommitShaOr and FindingsPresent rather than reading the fields directly when +// the file may not have come from SaveBaseline. +type BaselineData struct { + CommitSha string `json:"commit_sha"` + Timestamp string `json:"timestamp"` + Findings []BaselineFinding `json:"findings"` + + commitShaPresent bool + findingsPresent bool +} + +// UnmarshalJSON decodes the document and records top-level key presence. +// +// Python parity: json.loads on a non-object (a list, a bare number) succeeds +// and the TypedDict cast is a no-op, so the failure surfaces later as a +// TypeError from `baseline["findings"]`. Go cannot decode a non-object into a +// struct at all, so the failure moves up to LoadBaseline — same outcome (an +// error, no result), earlier and with a clearer message. +func (d *BaselineData) UnmarshalJSON(b []byte) error { + var probe map[string]json.RawMessage + if err := json.Unmarshal(b, &probe); err != nil { + return err + } + type alias BaselineData + var a alias + if err := json.Unmarshal(b, &a); err != nil { + return err + } + *d = BaselineData(a) + _, d.commitShaPresent = probe["commit_sha"] + _, d.findingsPresent = probe["findings"] + return nil +} + +// CommitShaOr ports `baseline.get("commit_sha", fallback)`: an ABSENT key +// yields fallback, a present one yields its value even when that value is the +// empty string. +func (d BaselineData) CommitShaOr(fallback string) string { + if !d.commitShaPresent { + return fallback + } + return d.CommitSha +} + +// FindingsPresent reports whether the decoded document carried a "findings" +// key at all — the condition Python's `baseline["findings"]` subscript turns +// into a KeyError. +func (d BaselineData) FindingsPresent() bool { return d.findingsPresent } + +// SaveBaseline saves scan results as a baseline for future comparison. +// +// Ports monitoring.py save_baseline. The bytes are `json.dumps(baseline_data, +// indent=2)` — see pyjson.go for why encoding/json cannot produce them — with +// NO trailing newline, matching `Path(path).write_text(...)`. +// +// Python parity: write_text neither creates parent directories nor tolerates a +// missing one; os.WriteFile behaves the same. The 0o644 mode is Go's explicit +// spelling of what CPython's open() defaults to (0o666 before umask). +func SaveBaseline(result schemas.SecurityAuditResult, path string) error { + doc := BaselineDataFor(result) + if err := os.WriteFile(path, []byte(pyfmt.Dumps(baselineObject(doc), 2)), 0o644); err != nil { + return fmt.Errorf("monitoring: save baseline %q: %w", path, err) + } + return nil +} + +// BaselineDataFor projects a SecurityAuditResult into the baseline document +// save_baseline writes. Exported so callers can inspect (or diff) a baseline +// without touching the filesystem; SaveBaseline is this plus the write. +// +// Python parity: `result.timestamp.isoformat()` — schemas.Timestamp.String() is +// that exact spelling — and `.value` on the two enums, which for a `str, Enum` +// is the plain string. +func BaselineDataFor(result schemas.SecurityAuditResult) BaselineData { + findings := make([]BaselineFinding, 0, len(result.Findings)) + for _, f := range result.Findings { + findings = append(findings, BaselineFinding{ + ID: f.ID, + Fingerprint: f.Fingerprint, + Title: f.Title, + Severity: string(f.Severity), + CweID: f.CweID, + Verdict: string(f.Verdict), + FilePath: f.Location.FilePath, + StartLine: f.Location.StartLine, + }) + } + return BaselineData{ + CommitSha: result.CommitSha, + Timestamp: result.Timestamp.String(), + Findings: findings, + commitShaPresent: true, + findingsPresent: true, + } +} + +// baselineObject renders a BaselineData in the exact key order of +// save_baseline's dict literals. +// +// pyfmt.Ordered, not map[string]any: a Go map has no insertion order and +// pyfmt.Dumps sorts map keys, which would reorder the file. The order of the +// literals in monitoring.py save_baseline is part of the artifact users diff. +func baselineObject(d BaselineData) pyfmt.Ordered { + findings := make([]any, 0, len(d.Findings)) + for _, f := range d.Findings { + findings = append(findings, baselineFindingObject(f)) + } + return pyfmt.O( + "commit_sha", d.CommitSha, + "timestamp", d.Timestamp, + "findings", findings, + ) +} + +// baselineFindingObject renders one finding in save_baseline's dict-literal +// key order. +func baselineFindingObject(f BaselineFinding) pyfmt.Ordered { + return pyfmt.O( + "id", f.ID, + "fingerprint", f.Fingerprint, + "title", f.Title, + "severity", f.Severity, + "cwe_id", f.CweID, + "verdict", f.Verdict, + "file_path", f.FilePath, + "start_line", f.StartLine, + ) +} + +// LoadBaseline loads baseline scan data from a file. +// +// Ports monitoring.py load_baseline: `json.loads(Path(path).read_text( +// encoding="utf-8"))`. A missing file or malformed JSON is an error, exactly as +// Python raises OSError / json.JSONDecodeError. +func LoadBaseline(path string) (BaselineData, error) { + var out BaselineData + b, err := os.ReadFile(path) + if err != nil { + return out, fmt.Errorf("monitoring: load baseline %q: %w", path, err) + } + if err := json.Unmarshal(b, &out); err != nil { + return out, fmt.Errorf("monitoring: parse baseline %q: %w", path, err) + } + return out, nil +} + +// CompareWithBaseline compares current scan results against a stored baseline. +// +// Ports monitoring.py compare_with_baseline. The returned MonitoringResult is +// built from schemas.NewMonitoringResult so the two list fields are `[]` rather +// than null when nothing changed, matching pydantic's default_factory=list. +// +// Python parity, four behaviors that are easy to "clean up" by accident: +// +// - new_findings follows the ORDER of current.findings, and a fingerprint that +// appears twice in the current scan is reported twice (the loop appends per +// finding, it does not deduplicate). +// - fixed_findings follows the FIRST-SEEN order of fingerprints in the +// baseline, but carries the LAST record for a repeated fingerprint — that is +// what the dict comprehension `{f["fingerprint"]: f for f in ...}` produces. +// - unchanged_count counts DISTINCT shared fingerprints (a set intersection), +// so it need not equal len(current.findings) - len(new_findings). +// - regression_detected is `len(new_findings) > 0` only; fixed findings never +// set it. +func CompareWithBaseline(current schemas.SecurityAuditResult, baselinePath string) (schemas.MonitoringResult, error) { + out := schemas.NewMonitoringResult() + + baseline, err := LoadBaseline(baselinePath) + if err != nil { + return out, err + } + if !baseline.FindingsPresent() { + // Python parity: `baseline["findings"]` raises KeyError here. + return out, fmt.Errorf("monitoring: baseline %q has no %q key", baselinePath, "findings") + } + + baselineFingerprints := make(map[string]struct{}, len(baseline.Findings)) + baselineByFP := make(map[string]BaselineFinding, len(baseline.Findings)) + baselineFPOrder := make([]string, 0, len(baseline.Findings)) + for _, f := range baseline.Findings { + if _, seen := baselineByFP[f.Fingerprint]; !seen { + baselineFPOrder = append(baselineFPOrder, f.Fingerprint) + } + baselineFingerprints[f.Fingerprint] = struct{}{} + baselineByFP[f.Fingerprint] = f + } + + currentFingerprints := make(map[string]struct{}, len(current.Findings)) + for _, f := range current.Findings { + currentFingerprints[f.Fingerprint] = struct{}{} + } + + for _, finding := range current.Findings { + if _, inBaseline := baselineFingerprints[finding.Fingerprint]; inBaseline { + continue + } + out.NewFindings = append(out.NewFindings, schemas.RegressionFinding{ + FindingTitle: finding.Title, + FindingID: finding.ID, + Severity: string(finding.Severity), + CweID: finding.CweID, + Status: "new", + }) + } + + for _, fp := range baselineFPOrder { + if _, stillPresent := currentFingerprints[fp]; stillPresent { + continue + } + bf := baselineByFP[fp] + out.FixedFindings = append(out.FixedFindings, schemas.RegressionFinding{ + FindingTitle: bf.Title, + FindingID: bf.ID, + Severity: bf.Severity, + CweID: bf.CweID, + Status: "fixed", + }) + } + + unchanged := 0 + for fp := range baselineFingerprints { + if _, shared := currentFingerprints[fp]; shared { + unchanged++ + } + } + + out.BaselineCommit = baseline.CommitShaOr("unknown") + out.CurrentCommit = current.CommitSha + out.UnchangedCount = unchanged + out.RegressionDetected = len(out.NewFindings) > 0 + return out, nil +} diff --git a/internal/secaf/monitoring/monitoring_test.go b/internal/secaf/monitoring/monitoring_test.go new file mode 100644 index 0000000000..96af20683b --- /dev/null +++ b/internal/secaf/monitoring/monitoring_test.go @@ -0,0 +1,597 @@ +package monitoring + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// makeFinding ports tests/test_monitoring.py::_make_finding. The optional +// arguments Python defaults are spelled out at every call site, since Go has no +// keyword defaults. +func makeFinding(fingerprint, title string, severity schemas.Severity, cweID string) schemas.VerifiedFinding { + f := schemas.NewVerifiedFinding() + f.Fingerprint = fingerprint + f.Title = title + f.Description = "Test" + f.FindingType = schemas.FindingTypeSast + f.CweID = cweID + f.CweName = "SQL Injection" + f.Verdict = schemas.VerdictConfirmed + f.EvidenceLevel = schemas.EvidenceLevelReachabilityConfirmed + f.Rationale = "Test" + f.Severity = severity + f.ExploitabilityScore = 7.5 + f.Location = schemas.Location{FilePath: "app.py", StartLine: 10, EndLine: 15} + f.SarifRuleID = "sec-af/sast/cwe-89" + f.SarifSecuritySeverity = 7.5 + return f +} + +// defaultFinding is _make_finding() with every Python default applied. +func defaultFinding(fingerprint string) schemas.VerifiedFinding { + return makeFinding(fingerprint, "Test Finding", schemas.SeverityHigh, "CWE-89") +} + +// makeResult ports tests/test_monitoring.py::_make_result. +func makeResult(findings []schemas.VerifiedFinding, commit string) schemas.SecurityAuditResult { + r := schemas.NewSecurityAuditResult() + r.Repository = "https://github.com/test/repo" + r.CommitSha = commit + branch := "main" + r.Branch = &branch + r.Timestamp = schemas.NewTimestamp(time.Date(2025, 1, 15, 0, 0, 0, 0, time.UTC)) + r.DepthProfile = "standard" + r.StrategiesUsed = []string{"injection"} + r.Provider = "harness" + r.Findings = findings + r.AttackChains = []schemas.AttackChain{} + r.TotalRawFindings = len(findings) + r.Confirmed = len(findings) + r.NoiseReductionPct = 0.0 + r.BySeverity = map[string]int{} + r.DurationSeconds = 10.0 + r.AgentInvocations = 5 + r.CostUsd = 0.05 + r.CostBreakdown = map[string]float64{} + r.Sarif = "" + return r +} + +func tempBaselinePath(t *testing.T) string { + t.Helper() + return filepath.Join(t.TempDir(), "baseline.json") +} + +// --------------------------------------------------------------------------- +// Ported Python tests +// --------------------------------------------------------------------------- + +// TestSaveAndLoadBaseline ports +// tests/test_monitoring.py::test_save_and_load_baseline. +func TestSaveAndLoadBaseline(t *testing.T) { + result := makeResult([]schemas.VerifiedFinding{defaultFinding("fp-1")}, "abc123") + path := tempBaselinePath(t) + + if err := SaveBaseline(result, path); err != nil { + t.Fatalf("SaveBaseline: %v", err) + } + loaded, err := LoadBaseline(path) + if err != nil { + t.Fatalf("LoadBaseline: %v", err) + } + if loaded.CommitSha != "abc123" { + t.Errorf("CommitSha = %q, want %q", loaded.CommitSha, "abc123") + } + if len(loaded.Findings) != 1 { + t.Fatalf("len(Findings) = %d, want 1", len(loaded.Findings)) + } +} + +// TestCompareDetectsNewFinding ports +// tests/test_monitoring.py::test_compare_detects_new_finding. +func TestCompareDetectsNewFinding(t *testing.T) { + path := tempBaselinePath(t) + baseline := makeResult([]schemas.VerifiedFinding{defaultFinding("fp-1")}, "old") + if err := SaveBaseline(baseline, path); err != nil { + t.Fatalf("SaveBaseline: %v", err) + } + + current := makeResult([]schemas.VerifiedFinding{ + defaultFinding("fp-1"), + makeFinding("fp-2", "New Bug", schemas.SeverityHigh, "CWE-89"), + }, "new") + + result, err := CompareWithBaseline(current, path) + if err != nil { + t.Fatalf("CompareWithBaseline: %v", err) + } + if !result.RegressionDetected { + t.Error("RegressionDetected = false, want true") + } + if len(result.NewFindings) != 1 { + t.Fatalf("len(NewFindings) = %d, want 1", len(result.NewFindings)) + } + if result.NewFindings[0].FindingTitle != "New Bug" { + t.Errorf("NewFindings[0].FindingTitle = %q, want %q", result.NewFindings[0].FindingTitle, "New Bug") + } + if result.UnchangedCount != 1 { + t.Errorf("UnchangedCount = %d, want 1", result.UnchangedCount) + } + // Not asserted by the Python test but implied by the model: the new record + // carries the current finding's identity and status. + if result.NewFindings[0].Status != "new" { + t.Errorf("NewFindings[0].Status = %q, want %q", result.NewFindings[0].Status, "new") + } + if result.NewFindings[0].Severity != "high" || result.NewFindings[0].CweID != "CWE-89" { + t.Errorf("NewFindings[0] severity/cwe = %q/%q, want high/CWE-89", + result.NewFindings[0].Severity, result.NewFindings[0].CweID) + } + if result.BaselineCommit != "old" || result.CurrentCommit != "new" { + t.Errorf("commits = %q/%q, want old/new", result.BaselineCommit, result.CurrentCommit) + } +} + +// TestCompareDetectsFixedFinding ports +// tests/test_monitoring.py::test_compare_detects_fixed_finding. +func TestCompareDetectsFixedFinding(t *testing.T) { + path := tempBaselinePath(t) + baseline := makeResult([]schemas.VerifiedFinding{ + defaultFinding("fp-1"), + defaultFinding("fp-2"), + }, "old") + if err := SaveBaseline(baseline, path); err != nil { + t.Fatalf("SaveBaseline: %v", err) + } + + current := makeResult([]schemas.VerifiedFinding{defaultFinding("fp-1")}, "new") + + result, err := CompareWithBaseline(current, path) + if err != nil { + t.Fatalf("CompareWithBaseline: %v", err) + } + if result.RegressionDetected { + t.Error("RegressionDetected = true, want false — a fixed finding is not a regression") + } + if len(result.FixedFindings) != 1 { + t.Fatalf("len(FixedFindings) = %d, want 1", len(result.FixedFindings)) + } + if result.FixedFindings[0].Status != "fixed" { + t.Errorf("FixedFindings[0].Status = %q, want %q", result.FixedFindings[0].Status, "fixed") + } +} + +// TestCompareNoRegression ports +// tests/test_monitoring.py::test_compare_no_regression. +func TestCompareNoRegression(t *testing.T) { + path := tempBaselinePath(t) + if err := SaveBaseline(makeResult([]schemas.VerifiedFinding{defaultFinding("fp-1")}, "old"), path); err != nil { + t.Fatalf("SaveBaseline: %v", err) + } + + result, err := CompareWithBaseline(makeResult([]schemas.VerifiedFinding{defaultFinding("fp-1")}, "new"), path) + if err != nil { + t.Fatalf("CompareWithBaseline: %v", err) + } + if result.RegressionDetected { + t.Error("RegressionDetected = true, want false") + } + if len(result.NewFindings) != 0 { + t.Errorf("len(NewFindings) = %d, want 0", len(result.NewFindings)) + } + if result.UnchangedCount != 1 { + t.Errorf("UnchangedCount = %d, want 1", result.UnchangedCount) + } +} + +// --------------------------------------------------------------------------- +// Byte-level file-format parity +// --------------------------------------------------------------------------- + +// goldenTitle is gen_golden.py's _BASELINE_TITLE: a quote, a backslash, `<`, +// `>`, `&`, a BMP non-ASCII rune, an em dash, a tab, a newline, DEL, and an +// astral rune. Everything Go's encoder would escape differently from Python's. +const goldenTitle = "SQL injection in \"users\" & café \u2014 \\path\ttab\nnewline\u007f \U0001f600" + +func readGolden(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join("testdata", "golden", name)) + if err != nil { + t.Fatalf("read golden %s: %v", name, err) + } + return string(b) +} + +// goldenResult rebuilds gen_golden.py's `_make_result([_make_finding("fp-1", +// _BASELINE_TITLE, "id-1")])`. +func goldenResult(findings []schemas.VerifiedFinding) schemas.SecurityAuditResult { + return makeResult(findings, "abc123") +} + +// TestSaveBaselineMatchesPythonBytes is the byte-for-byte golden test for +// save_baseline's on-disk format: Python's indent=2 layout AND its +// ensure_ascii=True / no-HTML-escaping string encoding. +// +// Golden produced by go/scripts/gen_golden.py from the real Python function. +// +// NOTE (integration): that script no longer carries a monitoring section — it +// was lost to a concurrent rewrite during the port, so a re-run does NOT +// refresh baseline.json / baseline_empty.json. The committed bytes are the ones +// it produced from the real save_baseline; re-derive by hand if the Python +// changes. See the COVERAGE GAP comment in gen_golden.py. +func TestSaveBaselineMatchesPythonBytes(t *testing.T) { + f := makeFinding("fp-1", goldenTitle, schemas.SeverityHigh, "CWE-89") + f.ID = "id-1" + path := tempBaselinePath(t) + if err := SaveBaseline(goldenResult([]schemas.VerifiedFinding{f}), path); err != nil { + t.Fatalf("SaveBaseline: %v", err) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read back: %v", err) + } + if want := readGolden(t, "baseline.json"); string(got) != want { + t.Errorf("baseline bytes differ from Python\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +// TestSaveBaselineEmptyFindingsMatchesPythonBytes pins the empty-list case, +// which json.dumps collapses to `[]` with no inner newline. +func TestSaveBaselineEmptyFindingsMatchesPythonBytes(t *testing.T) { + path := tempBaselinePath(t) + if err := SaveBaseline(goldenResult(nil), path); err != nil { + t.Fatalf("SaveBaseline: %v", err) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read back: %v", err) + } + if want := readGolden(t, "baseline_empty.json"); string(got) != want { + t.Errorf("baseline bytes differ from Python\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +// TestSaveBaselineHasNoTrailingNewline pins `write_text(json.dumps(...))`: +// json.dumps does not append one and write_text does not either. +func TestSaveBaselineHasNoTrailingNewline(t *testing.T) { + path := tempBaselinePath(t) + if err := SaveBaseline(goldenResult(nil), path); err != nil { + t.Fatalf("SaveBaseline: %v", err) + } + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read back: %v", err) + } + if strings.HasSuffix(string(b), "\n") { + t.Errorf("baseline file ends with a newline: %q", string(b[len(b)-8:])) + } +} + +// TestSaveBaselineRoundTripsThroughLoad proves the written file is still valid +// JSON that decodes to the same values — the escaping is cosmetic, not lossy. +func TestSaveBaselineRoundTripsThroughLoad(t *testing.T) { + f := makeFinding("fp-1", goldenTitle, schemas.SeverityHigh, "CWE-89") + f.ID = "id-1" + path := tempBaselinePath(t) + if err := SaveBaseline(goldenResult([]schemas.VerifiedFinding{f}), path); err != nil { + t.Fatalf("SaveBaseline: %v", err) + } + loaded, err := LoadBaseline(path) + if err != nil { + t.Fatalf("LoadBaseline: %v", err) + } + want := BaselineFinding{ + ID: "id-1", + Fingerprint: "fp-1", + Title: goldenTitle, + Severity: "high", + CweID: "CWE-89", + Verdict: "confirmed", + FilePath: "app.py", + StartLine: 10, + } + if len(loaded.Findings) != 1 || !reflect.DeepEqual(loaded.Findings[0], want) { + t.Errorf("round-tripped finding = %+v, want %+v", loaded.Findings, want) + } + if loaded.Timestamp != "2025-01-15T00:00:00+00:00" { + t.Errorf("Timestamp = %q, want the datetime.isoformat() spelling", loaded.Timestamp) + } +} + +// TestBaselineStringEscaping is a table of `json.dumps(s)` ground truth. Every +// expected value is what the venv interpreter prints for json.dumps of the same +// input. +// +// The encoder under test is pyfmt.Dumps — this package used to carry its own +// copy of CPython's ensure_ascii escaping (pyjson.go) because pyfmt.Dumps did +// not exist yet. The copy is gone; the table stays here, unchanged, because a +// baseline file's titles are LLM-authored prose and this is where a regression +// in that escaping would actually be felt. +func TestBaselineStringEscaping(t *testing.T) { + cases := []struct{ in, want string }{ + {"", `""`}, + {"plain", `"plain"`}, + {`say "hi"`, `"say \"hi\""`}, + {`back\slash`, `"back\\slash"`}, + // NOT escaped by Python (Go's encoder escapes all three). + {" & ", `" & "`}, + {"tab\there", `"tab\there"`}, + {"nl\nhere", `"nl\nhere"`}, + {"cr\rhere", `"cr\rhere"`}, + {"bs\bhere", `"bs\bhere"`}, + {"ff\fhere", `"ff\fhere"`}, + {"\x00\x01\x1f", `"\u0000\u0001\u001f"`}, + {"\x7f", `"\u007f"`}, + {"café", `"caf\u00e9"`}, + {"\u2014", `"\u2014"`}, + {"\u2028\u2029", `"\u2028\u2029"`}, + {"\U0001f600", `"\ud83d\ude00"`}, + } + for _, tc := range cases { + if got := pyfmt.Dumps(tc.in, 2); got != tc.want { + t.Errorf("pyfmt.Dumps(%q) = %s, want %s", tc.in, got, tc.want) + } + } +} + +// TestBaselineIndentLayout pins the container layout rules independently of the +// baseline shape: nesting indentation, the ": " key separator, and empty +// containers collapsing. Same provenance as TestBaselineStringEscaping — the +// expectations came from the venv interpreter and now guard pyfmt.Dumps as this +// package uses it. +func TestBaselineIndentLayout(t *testing.T) { + doc := pyfmt.O( + "a", "x", + "empty_list", []any{}, + "empty_obj", pyfmt.Ordered{}, + "list", []any{pyfmt.O("i", 1, "j", 2), "s"}, + ) + want := `{ + "a": "x", + "empty_list": [], + "empty_obj": {}, + "list": [ + { + "i": 1, + "j": 2 + }, + "s" + ] +}` + if got := pyfmt.Dumps(doc, 2); got != want { + t.Errorf("pyfmt.Dumps layout mismatch\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +// --------------------------------------------------------------------------- +// compare_with_baseline behaviors the Python tests do not cover +// --------------------------------------------------------------------------- + +func writeBaselineJSON(t *testing.T, body string) string { + t.Helper() + path := tempBaselinePath(t) + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatalf("write baseline: %v", err) + } + return path +} + +// TestCompareMissingCommitShaFallsBackToUnknown ports +// `baseline.get("commit_sha", "unknown")`. +func TestCompareMissingCommitShaFallsBackToUnknown(t *testing.T) { + path := writeBaselineJSON(t, `{"timestamp": "2025-01-15T00:00:00+00:00", "findings": []}`) + + result, err := CompareWithBaseline(makeResult(nil, "cur"), path) + if err != nil { + t.Fatalf("CompareWithBaseline: %v", err) + } + if result.BaselineCommit != "unknown" { + t.Errorf("BaselineCommit = %q, want %q", result.BaselineCommit, "unknown") + } +} + +// TestCompareEmptyCommitShaIsNotUnknown proves .get() only falls back for an +// ABSENT key — a present empty string stays empty. +func TestCompareEmptyCommitShaIsNotUnknown(t *testing.T) { + path := writeBaselineJSON(t, `{"commit_sha": "", "timestamp": "t", "findings": []}`) + + result, err := CompareWithBaseline(makeResult(nil, "cur"), path) + if err != nil { + t.Fatalf("CompareWithBaseline: %v", err) + } + if result.BaselineCommit != "" { + t.Errorf("BaselineCommit = %q, want the empty string", result.BaselineCommit) + } +} + +// TestCompareMissingFindingsKeyIsAnError ports the KeyError Python raises for +// `baseline["findings"]` when the key is absent. +func TestCompareMissingFindingsKeyIsAnError(t *testing.T) { + path := writeBaselineJSON(t, `{"commit_sha": "old", "timestamp": "t"}`) + + if _, err := CompareWithBaseline(makeResult(nil, "cur"), path); err == nil { + t.Fatal("expected an error for a baseline with no findings key") + } +} + +// TestCompareMissingBaselineFileIsAnError mirrors the OSError load_baseline +// raises for a path that does not exist. +func TestCompareMissingBaselineFileIsAnError(t *testing.T) { + path := filepath.Join(t.TempDir(), "does-not-exist.json") + if _, err := CompareWithBaseline(makeResult(nil, "cur"), path); err == nil { + t.Fatal("expected an error for a missing baseline file") + } + if _, err := LoadBaseline(path); err == nil { + t.Fatal("expected an error from LoadBaseline for a missing file") + } +} + +// TestLoadBaselineMalformedJSONIsAnError mirrors json.JSONDecodeError. +func TestLoadBaselineMalformedJSONIsAnError(t *testing.T) { + if _, err := LoadBaseline(writeBaselineJSON(t, "{not json")); err == nil { + t.Fatal("expected an error for malformed JSON") + } + // A valid JSON document that is not an object: Python would fail later, at + // the `baseline["findings"]` subscript; Go fails here. Either way, an error. + if _, err := LoadBaseline(writeBaselineJSON(t, "[1, 2, 3]")); err == nil { + t.Fatal("expected an error for a non-object baseline document") + } +} + +// TestCompareNewFindingsFollowCurrentOrderAndKeepDuplicates pins the loop +// semantics: current order is preserved and a repeated new fingerprint is +// reported once per occurrence. +func TestCompareNewFindingsFollowCurrentOrderAndKeepDuplicates(t *testing.T) { + path := tempBaselinePath(t) + if err := SaveBaseline(makeResult(nil, "old"), path); err != nil { + t.Fatalf("SaveBaseline: %v", err) + } + + current := makeResult([]schemas.VerifiedFinding{ + makeFinding("fp-z", "Z", schemas.SeverityLow, "CWE-1"), + makeFinding("fp-a", "A", schemas.SeverityHigh, "CWE-2"), + makeFinding("fp-z", "Z again", schemas.SeverityLow, "CWE-1"), + }, "cur") + + result, err := CompareWithBaseline(current, path) + if err != nil { + t.Fatalf("CompareWithBaseline: %v", err) + } + got := []string{} + for _, f := range result.NewFindings { + got = append(got, f.FindingTitle) + } + if want := []string{"Z", "A", "Z again"}; !reflect.DeepEqual(got, want) { + t.Errorf("NewFindings titles = %v, want %v", got, want) + } + if result.UnchangedCount != 0 { + t.Errorf("UnchangedCount = %d, want 0", result.UnchangedCount) + } +} + +// TestCompareFixedFindingsFollowBaselineFirstSeenOrderWithLastRecord pins the +// `{f["fingerprint"]: f for f in ...}` semantics: FIRST-seen key order, LAST +// value. +func TestCompareFixedFindingsFollowBaselineFirstSeenOrderWithLastRecord(t *testing.T) { + path := writeBaselineJSON(t, `{ + "commit_sha": "old", + "timestamp": "t", + "findings": [ + {"id": "1", "fingerprint": "fp-b", "title": "B first", "severity": "low", "cwe_id": "CWE-1", "verdict": "confirmed", "file_path": "a.py", "start_line": 1}, + {"id": "2", "fingerprint": "fp-a", "title": "A", "severity": "high", "cwe_id": "CWE-2", "verdict": "likely", "file_path": "b.py", "start_line": 2}, + {"id": "3", "fingerprint": "fp-b", "title": "B last", "severity": "medium", "cwe_id": "CWE-3", "verdict": "confirmed", "file_path": "c.py", "start_line": 3} + ] +}`) + + result, err := CompareWithBaseline(makeResult(nil, "cur"), path) + if err != nil { + t.Fatalf("CompareWithBaseline: %v", err) + } + gotTitles := []string{} + gotIDs := []string{} + for _, f := range result.FixedFindings { + gotTitles = append(gotTitles, f.FindingTitle) + gotIDs = append(gotIDs, f.FindingID) + } + if want := []string{"B last", "A"}; !reflect.DeepEqual(gotTitles, want) { + t.Errorf("FixedFindings titles = %v, want %v (first-seen key order, last record)", gotTitles, want) + } + if want := []string{"3", "2"}; !reflect.DeepEqual(gotIDs, want) { + t.Errorf("FixedFindings ids = %v, want %v", gotIDs, want) + } + if result.FixedFindings[0].Severity != "medium" || result.FixedFindings[0].CweID != "CWE-3" { + t.Errorf("fixed record fields came from the wrong duplicate: %+v", result.FixedFindings[0]) + } +} + +// TestCompareUnchangedCountIsDistinctFingerprints proves unchanged_count is a +// SET intersection, so duplicates on either side count once. +func TestCompareUnchangedCountIsDistinctFingerprints(t *testing.T) { + path := tempBaselinePath(t) + baseline := makeResult([]schemas.VerifiedFinding{ + defaultFinding("fp-1"), defaultFinding("fp-1"), defaultFinding("fp-2"), + }, "old") + if err := SaveBaseline(baseline, path); err != nil { + t.Fatalf("SaveBaseline: %v", err) + } + + current := makeResult([]schemas.VerifiedFinding{ + defaultFinding("fp-1"), defaultFinding("fp-1"), defaultFinding("fp-2"), + }, "cur") + + result, err := CompareWithBaseline(current, path) + if err != nil { + t.Fatalf("CompareWithBaseline: %v", err) + } + if result.UnchangedCount != 2 { + t.Errorf("UnchangedCount = %d, want 2 (distinct fingerprints)", result.UnchangedCount) + } + if len(result.NewFindings) != 0 || len(result.FixedFindings) != 0 { + t.Errorf("expected no new/fixed findings, got %+v / %+v", result.NewFindings, result.FixedFindings) + } +} + +// TestCompareResultMarshalsEmptyListsNotNull proves the result uses pydantic's +// default_factory=list shape, so a clean comparison serializes `[]` rather than +// `null`. +func TestCompareResultMarshalsEmptyListsNotNull(t *testing.T) { + path := tempBaselinePath(t) + if err := SaveBaseline(makeResult(nil, "old"), path); err != nil { + t.Fatalf("SaveBaseline: %v", err) + } + result, err := CompareWithBaseline(makeResult(nil, "cur"), path) + if err != nil { + t.Fatalf("CompareWithBaseline: %v", err) + } + b, err := json.Marshal(result) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if strings.Contains(string(b), "null") { + t.Errorf("MonitoringResult marshaled a null list: %s", b) + } + if !strings.Contains(string(b), `"new_findings":[]`) || !strings.Contains(string(b), `"fixed_findings":[]`) { + t.Errorf("expected empty JSON arrays, got %s", b) + } +} + +// TestBaselineDataForProjectsEveryField pins the projection save_baseline +// performs, including the two `.value` enum reads and the nested location. +func TestBaselineDataForProjectsEveryField(t *testing.T) { + f := makeFinding("fp-x", "Title", schemas.SeverityCritical, "CWE-79") + f.ID = "id-x" + f.Verdict = schemas.VerdictLikely + f.Location = schemas.Location{FilePath: "pkg/mod.go", StartLine: 42, EndLine: 44} + + got := BaselineDataFor(makeResult([]schemas.VerifiedFinding{f}, "sha")) + + if got.CommitSha != "sha" { + t.Errorf("CommitSha = %q, want %q", got.CommitSha, "sha") + } + if got.Timestamp != "2025-01-15T00:00:00+00:00" { + t.Errorf("Timestamp = %q", got.Timestamp) + } + want := BaselineFinding{ + ID: "id-x", Fingerprint: "fp-x", Title: "Title", Severity: "critical", + CweID: "CWE-79", Verdict: "likely", FilePath: "pkg/mod.go", StartLine: 42, + } + if len(got.Findings) != 1 || got.Findings[0] != want { + t.Errorf("Findings = %+v, want [%+v]", got.Findings, want) + } +} + +// TestSaveBaselineIntoMissingDirectoryIsAnError mirrors write_text, which does +// not create parent directories. +func TestSaveBaselineIntoMissingDirectoryIsAnError(t *testing.T) { + path := filepath.Join(t.TempDir(), "nope", "baseline.json") + if err := SaveBaseline(makeResult(nil, "x"), path); err == nil { + t.Fatal("expected an error writing into a missing directory") + } +} diff --git a/internal/secaf/monitoring/testdata/golden/baseline.json b/internal/secaf/monitoring/testdata/golden/baseline.json new file mode 100644 index 0000000000..d9645c66a8 --- /dev/null +++ b/internal/secaf/monitoring/testdata/golden/baseline.json @@ -0,0 +1,16 @@ +{ + "commit_sha": "abc123", + "timestamp": "2025-01-15T00:00:00+00:00", + "findings": [ + { + "id": "id-1", + "fingerprint": "fp-1", + "title": "SQL injection in \"users\" & caf\u00e9 \u2014 \\path\ttab\nnewline\u007f \ud83d\ude00", + "severity": "high", + "cwe_id": "CWE-89", + "verdict": "confirmed", + "file_path": "app.py", + "start_line": 10 + } + ] +} \ No newline at end of file diff --git a/internal/secaf/monitoring/testdata/golden/baseline_empty.json b/internal/secaf/monitoring/testdata/golden/baseline_empty.json new file mode 100644 index 0000000000..16268604b1 --- /dev/null +++ b/internal/secaf/monitoring/testdata/golden/baseline_empty.json @@ -0,0 +1,5 @@ +{ + "commit_sha": "abc123", + "timestamp": "2025-01-15T00:00:00+00:00", + "findings": [] +} \ No newline at end of file diff --git a/internal/secaf/orch/budget.go b/internal/secaf/orch/budget.go new file mode 100644 index 0000000000..84c767963f --- /dev/null +++ b/internal/secaf/orch/budget.go @@ -0,0 +1,269 @@ +package orch + +import ( + "context" + "errors" + + "github.com/Agent-Field/agentfield/sdk/go/ai" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + proveagent "github.com/Agent-Field/codeaf/internal/secaf/agents/prove" +) + +// BudgetExhaustedError ports `class BudgetExhausted(RuntimeError)` +// (orchestrator.py:35) — the sentinel the phase proxy raises when a cost or +// duration budget is used up. +type BudgetExhaustedError struct{ Message string } + +func (e *BudgetExhaustedError) Error() string { return e.Message } + +// IsBudgetExhausted reports whether err is (or wraps) a BudgetExhaustedError, +// which is Go's `except BudgetExhausted`. +func IsBudgetExhausted(err error) bool { + var target *BudgetExhaustedError + return errors.As(err, &target) +} + +// checkTimeBudget ports `_check_time_budget`: +// +// if self.max_duration_seconds is None: return +// elapsed = time.monotonic() - self.started_at +// if elapsed > self.max_duration_seconds: +// self.budget_exhausted = True +// raise BudgetExhausted("Duration budget exhausted") +// +// Python parity: the comparison is STRICTLY greater, so an elapsed time exactly +// equal to the limit does not trip it — the opposite of the cost checks, which +// use >=. +func (o *AuditOrchestrator) checkTimeBudget() error { + if o.MaxDurationSeconds == nil { + return nil + } + elapsed := o.elapsedSeconds() + if elapsed > float64(*o.MaxDurationSeconds) { + o.markBudgetExhausted() + return &BudgetExhaustedError{Message: "Duration budget exhausted"} + } + return nil +} + +// phaseBudgetLimit ports `_phase_budget_limit(phase)`: +// +// if self.max_cost_usd is None: return None +// weights = {"recon": recon_budget_pct, "hunt": hunt_budget_pct, "prove": prove_budget_pct} +// return self.max_cost_usd * weights[phase] +// +// The bool result is Python's "not None". Python parity: `weights[phase]` is a +// dict subscript, so an unknown phase name raises KeyError — a programming +// error, not a budget outcome. The Go port reports "no limit" for an unknown +// phase instead of panicking; every call site passes one of the three PhaseOrder +// constants, so the branch is unreachable. +func (o *AuditOrchestrator) phaseBudgetLimit(phase string) (float64, bool) { + if o.MaxCostUSD == nil { + return 0, false + } + var weight float64 + switch phase { + case PhaseRecon: + weight = o.BudgetConfig.ReconBudgetPct + case PhaseHunt: + weight = o.BudgetConfig.HuntBudgetPct + case PhaseProve: + weight = o.BudgetConfig.ProveBudgetPct + default: + return 0, false + } + return *o.MaxCostUSD * weight, true +} + +// checkCostBudget ports `_check_cost_budget(phase)`: +// +// if self.max_cost_usd is not None and self.total_cost_usd >= self.max_cost_usd: +// self.budget_exhausted = True; raise BudgetExhausted("Total budget exhausted") +// phase_limit = self._phase_budget_limit(phase) +// if phase_limit is not None and self.cost_breakdown[phase] >= phase_limit: +// self.budget_exhausted = True; raise BudgetExhausted(f"{phase} budget exhausted") +// +// Both comparisons are >=, so a phase that has spent EXACTLY its allowance is +// already exhausted. +func (o *AuditOrchestrator) checkCostBudget(phase string) error { + o.mu.Lock() + total := o.totalCostUSD + spent := o.costBreakdown[phase] + o.mu.Unlock() + + if o.MaxCostUSD != nil && total >= *o.MaxCostUSD { + o.markBudgetExhausted() + return &BudgetExhaustedError{Message: "Total budget exhausted"} + } + if limit, ok := o.phaseBudgetLimit(phase); ok && spent >= limit { + o.markBudgetExhausted() + return &BudgetExhaustedError{Message: phase + " budget exhausted"} + } + return nil +} + +// budgetOrTimeoutExhausted ports `_budget_or_timeout_exhausted(phase)`: +// +// try: +// self._check_time_budget(); self._check_cost_budget(phase); return False +// except BudgetExhausted: +// return True +// +// Note the side effect the boolean hides: both checks LATCH budget_exhausted +// before raising, so a caller that only reads the bool still sees the flag set. +func (o *AuditOrchestrator) budgetOrTimeoutExhausted(phase string) bool { + if err := o.checkTimeBudget(); err != nil { + return true + } + if err := o.checkCostBudget(phase); err != nil { + return true + } + return false +} + +func (o *AuditOrchestrator) markBudgetExhausted() { + o.mu.Lock() + o.budgetExhausted = true + o.mu.Unlock() +} + +// registerCost ports `_register_cost(phase, cost_usd)`: +// +// if cost_usd is None or cost_usd < 0: return +// self.total_cost_usd += cost_usd +// self.cost_breakdown[phase] += cost_usd +// +// Python parity: a cost of exactly 0.0 IS registered (the guard is `< 0`, not +// `<= 0`), which matters only for the invocation-shaped side effects — there +// are none — but is reproduced. `self.cost_breakdown[phase] += cost` raises +// KeyError for an unknown phase in Python; the Go map simply grows a fourth +// bucket, which GenerateOutput would then report. Unreachable for the same +// reason as phaseBudgetLimit's default branch. +func (o *AuditOrchestrator) registerCost(phase string, costUSD *float64) { + if costUSD == nil || *costUSD < 0 { + return + } + o.mu.Lock() + o.totalCostUSD += *costUSD + o.costBreakdown[phase] += *costUSD + o.mu.Unlock() +} + +// registerInvocation ports `self.agent_invocations += 1`. +func (o *AuditOrchestrator) registerInvocation() { + o.mu.Lock() + o.agentInvocations++ + o.mu.Unlock() +} + +// --------------------------------------------------------------------------- +// _PhaseHarnessProxy +// --------------------------------------------------------------------------- + +// phaseProxy ports `_PhaseHarnessProxy` (orchestrator.py:39) — the App the +// orchestrator hands to every in-process phase so that each harness call is +// budget-checked, counted and costed against that phase's bucket: +// +// class _PhaseHarnessProxy: +// def __init__(self, orchestrator, phase): ... +// async def harness(self, prompt, *, schema=None, cwd=None, **kwargs): +// if self._orchestrator._budget_or_timeout_exhausted(self._phase): +// raise BudgetExhausted(f"{self._phase} budget exhausted") +// result = await self._orchestrator.app.harness(prompt, schema=schema, cwd=cwd, **kwargs) +// self._orchestrator.agent_invocations += 1 +// self._orchestrator._register_cost(self._phase, getattr(result, "cost_usd", None)) +// return result +// +// CAPABILITY SURFACE IS LOAD-BEARING. The Python class defines `harness` and +// NOTHING ELSE — no `ai`, no `note`, no `call`, and no `__getattr__` (VERIFIED +// on the repo interpreter: hasattr(proxy, "ai") is False). Two live behaviours +// depend on that, both reachable from `audit(resume_from_checkpoint=...)` +// (app.py:169 -> run_from_checkpoint -> _run_hunt / _run_prove): +// +// 1. HUNT dedup. agents/dedup.py:150 gates its semantic duplicate pass on +// `has_ai = hasattr(app, "ai") and callable(...)`, which is FALSE for this +// proxy, so the pairwise `.ai(DuplicateCheck)` fan-out never runs and no +// finding is dropped by it. phaseProxy therefore does NOT implement +// appx.AIer, so internal/agents/dedup's `app.(appx.AIer)` probe fails the +// same way. +// 2. PROVE verdict. agents/prove/verdict.py:99 calls `await app.ai(...)` with +// no guard, raising `AttributeError: '_PhaseHarnessProxy' object has no +// attribute 'ai'`; agents/prove/__init__.py's `except BaseException` turns +// that into `verifier_fallback(..., drop_reason="verifier_error")` for +// EVERY finding. Go's run_prove signature needs an AIer, so proveProxy +// (below) supplies one that returns exactly that AttributeError. +// +// Python parity: `note` and `call` are absent for the same reason, and no phase +// the orchestrator drives in process reaches for either (verified by grep over +// src/sec_af/agents/**), so nothing needs a stand-in for them. +type phaseProxy struct { + orch *AuditOrchestrator + phase string +} + +var _ appx.Harnesser = (*phaseProxy)(nil) + +// AttributeError reproduces the CPython AttributeError a phase gets when it +// reaches for an attribute `_PhaseHarnessProxy` does not define. The message is +// byte-identical to CPython's, which matters: agents/prove's demotion classifier +// (`demoteOnError`) inspects `str(exc)` for "unverified"/"verdict"/ +// "validationerror", and this text matches none of them — so a verdict call +// through the proxy demotes with drop_reason "verifier_error", exactly as it +// does in Python. +type AttributeError struct{ Attr string } + +func (e *AttributeError) Error() string { + return "'_PhaseHarnessProxy' object has no attribute '" + e.Attr + "'" +} + +// PhaseProxy exposes the budget-checking wrapper for one phase name +// ("recon", "hunt" or "prove"). It is exported so a caller outside this package +// can drive a phase with the same accounting the orchestrator applies. +// +// The return type is appx.Harnesser, not appx.App: see the phaseProxy doc. +func (o *AuditOrchestrator) PhaseProxy(phase string) appx.Harnesser { + return &phaseProxy{orch: o, phase: phase} +} + +// proveProxy is `_PhaseHarnessProxy(self, "prove")` handed to run_prove, whose +// Go signature (proveagent.HarnessAIer) requires an `AI` method that Python's +// duck-typed call site does not. Every call returns the AttributeError Python +// raises, so the observable outcome — a demoted, INCONCLUSIVE finding with +// drop_reason "verifier_error" — is identical. +type proveProxy struct{ phaseProxy } + +var _ proveagent.HarnessAIer = (*proveProxy)(nil) + +// AI reproduces `_PhaseHarnessProxy.ai` NOT EXISTING. +func (p *proveProxy) AI(context.Context, string, ...ai.Option) (*ai.Response, error) { + return nil, &AttributeError{Attr: "ai"} +} + +// ProvePhaseProxy is PhaseProxy(PhaseProve) with the AI seam described above. +func (o *AuditOrchestrator) ProvePhaseProxy() proveagent.HarnessAIer { + return &proveProxy{phaseProxy{orch: o, phase: PhaseProve}} +} + +// Harness implements appx.Harnesser with the budget guard, the invocation +// counter and the cost registration, in Python's order. +func (p *phaseProxy) Harness(ctx context.Context, prompt string, schema map[string]any, dest any, opts appx.HarnessOptions) (*appx.HarnessResult, error) { + if p.orch.budgetOrTimeoutExhausted(p.phase) { + // Python raises the SAME message whichever check tripped. + return nil, &BudgetExhaustedError{Message: p.phase + " budget exhausted"} + } + res, err := p.orch.App.Harness(ctx, prompt, schema, dest, opts) + if err != nil { + // Python: the `await` raised, so neither the counter nor the cost + // registration below runs. + return res, err + } + p.orch.registerInvocation() + if res != nil { + p.orch.registerCost(p.phase, res.CostUSD) + } else { + // Python: getattr(None, "cost_usd", None) is None -> no-op. + p.orch.registerCost(p.phase, nil) + } + return res, nil +} diff --git a/internal/secaf/orch/checkpoint.go b/internal/secaf/orch/checkpoint.go new file mode 100644 index 0000000000..6182d178f0 --- /dev/null +++ b/internal/secaf/orch/checkpoint.go @@ -0,0 +1,184 @@ +package orch + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/Agent-Field/codeaf/internal/secaf/phases" + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// nowUTC is `datetime.now(UTC)`, as a variable so a test (and +// scripts/gen_golden.py's Python counterpart) can pin the checkpoint's +// created_at. Production never reassigns it. +var nowUTC = func() time.Time { return time.Now().UTC() } + +// checkpointDirPerm / checkpointFilePerm. +// +// Python parity: `Path.mkdir(parents=True, exist_ok=True)` uses mode 0o777 +// masked by the umask, and `Path.write_text` creates with 0o666 masked by the +// umask. Under the usual 022 umask those land on 0755 / 0644, which is what Go +// asks for directly — Go's os package does not apply an implicit mode. +const ( + checkpointDirPerm os.FileMode = 0o755 + checkpointFilePerm os.FileMode = 0o644 +) + +// CheckpointPath ports `_checkpoint_path(phase)`: +// +// return self.checkpoint_dir / f"checkpoint-{phase}.json" +func (o *AuditOrchestrator) CheckpointPath(phase string) string { + return filepath.Join(o.CheckpointDir, "checkpoint-"+phase+".json") +} + +// WriteCheckpoint ports `_write_checkpoint(phase, payload)`: +// +// self.checkpoint_dir.mkdir(parents=True, exist_ok=True) +// path = self._checkpoint_path(phase) +// data = [item.model_dump() for item in payload] if isinstance(payload, list) else payload.model_dump() +// body = {"phase": phase, "created_at": datetime.now(UTC).isoformat(), "data": data} +// path.write_text(json.dumps(body, indent=2), encoding="utf-8") +// +// The FILE BYTES are the contract — a Go node must be able to resume from a +// checkpoint a Python node wrote and vice versa — so the body is rendered with +// pyfmt.Dumps (json.dumps parity: `", "`/`": "` separators become `,`/`": "` +// under indent, floats keep Python's repr, non-ASCII is \uXXXX-escaped and +// `<>&` are NOT escaped) over an INSERTION-ORDERED object, because a Go map +// would sort "created_at" before "data" and "phase" while Python emits them in +// the literal's order. +// +// Python parity: +// +// - `created_at` is a plain STRING produced by `datetime.now(UTC).isoformat()` +// — `2026-01-02T03:04:05.123456+00:00`, with the fraction omitted when the +// microseconds are zero. schemas.Timestamp.String() is that exact spelling. +// - the isinstance(list) branch is only about calling model_dump per element; +// pyfmt.Dumps renders a slice of structs as an array of objects with the +// same field order, so one code path covers both. +// - Python raises on an I/O failure; Go returns the error, and every caller +// propagates it. +// - no trailing newline is written (`json.dumps` produces none, and +// `write_text` adds none). +func (o *AuditOrchestrator) WriteCheckpoint(phase string, payload any) error { + if err := os.MkdirAll(o.CheckpointDir, checkpointDirPerm); err != nil { + return fmt.Errorf("orch: create checkpoint dir: %w", err) + } + body := pyfmt.O( + "phase", phase, + "created_at", schemas.NewTimestamp(nowUTC()).String(), + "data", payload, + ) + path := o.CheckpointPath(phase) + if err := os.WriteFile(path, []byte(pyfmt.Dumps(body, 2)), checkpointFilePerm); err != nil { + return fmt.Errorf("orch: write checkpoint %s: %w", path, err) + } + return nil +} + +// readCheckpointBody decodes the `{"phase", "created_at", "data"}` envelope. +func (o *AuditOrchestrator) readCheckpointBody(phase string) (map[string]json.RawMessage, error) { + raw, err := os.ReadFile(o.CheckpointPath(phase)) + if err != nil { + return nil, err + } + var body map[string]json.RawMessage + if err := json.Unmarshal(raw, &body); err != nil { + return nil, fmt.Errorf("orch: decode checkpoint %s: %w", phase, err) + } + return body, nil +} + +// ReadCheckpoint ports `_read_checkpoint(phase, schema)`: +// +// payload = json.loads(path.read_text(encoding="utf-8")) +// return schema(**payload.get("data", {})) +// +// bind is the `schema(**data)` half: one of internal/phases' checked binders, +// so a hand-edited or stale checkpoint fails HERE — with a ValueError-class +// error the audit handler maps to HTTP 400 — instead of quietly binding to +// default-seeded values. Without it, a checkpoint whose findings carry an +// out-of-vocabulary severity resumed cleanly in Go and raised ValidationError +// in Python. +// +// Python parity: a checkpoint whose "data" key is MISSING falls back to `{}`, +// which for a model with required fields raises and for one with all-defaults +// yields the defaults. Passing `{}` to the binder reproduces both. +func ReadCheckpoint[T any](o *AuditOrchestrator, phase string, bind func(map[string]any) (T, error)) (T, error) { + var out T + body, err := o.readCheckpointBody(phase) + if err != nil { + return out, err + } + data, ok := body["data"] + if !ok || len(data) == 0 { + data = json.RawMessage("{}") + } + row := map[string]any{} + if err := json.Unmarshal(data, &row); err != nil { + return out, fmt.Errorf("orch: decode checkpoint %s data: %w", phase, err) + } + return bind(row) +} + +// ReadCheckpointList ports `_read_checkpoint_list(phase, schema)`: +// +// rows = payload.get("data", []) +// return [schema(**row) for row in rows] +// +// Same story as ReadCheckpoint: `schema(**row)` validates every element, so the +// binder runs per row and the first failure propagates. +func ReadCheckpointList[T any](o *AuditOrchestrator, phase string, bind func(map[string]any) (T, error)) ([]T, error) { + body, err := o.readCheckpointBody(phase) + if err != nil { + return nil, err + } + data, ok := body["data"] + if !ok || len(data) == 0 { + return []T{}, nil + } + var rows []map[string]any + if err := json.Unmarshal(data, &rows); err != nil { + return nil, fmt.Errorf("orch: decode checkpoint %s data: %w", phase, err) + } + out := make([]T, 0, len(rows)) + for _, row := range rows { + item, err := bind(row) + if err != nil { + return nil, err + } + out = append(out, item) + } + return out, nil +} + +// TryLoadCachedRecon ports `_try_load_cached_recon()`: +// +// try: +// return self._read_checkpoint("recon", ReconResult) +// except (FileNotFoundError, Exception): +// return None +// +// Python parity: the except tuple is redundant — `Exception` already covers +// FileNotFoundError — so EVERY failure (missing file, malformed JSON, failed +// validation) yields None. A nil pointer is that None. +func (o *AuditOrchestrator) TryLoadCachedRecon() *schemas.ReconResult { + recon, err := ReadCheckpoint(o, PhaseRecon, phases.BindReconResult) + if err != nil { + return nil + } + return &recon +} + +// mkdirCheckpointDir is `self.checkpoint_dir.mkdir(parents=True, exist_ok=True)`, +// which _generate_output performs before writing the per-framework compliance +// reports. +func mkdirCheckpointDir(dir string) error { + if err := os.MkdirAll(dir, checkpointDirPerm); err != nil { + return fmt.Errorf("orch: create checkpoint dir: %w", err) + } + return nil +} diff --git a/internal/secaf/orch/doc.go b/internal/secaf/orch/doc.go new file mode 100644 index 0000000000..173ddec06f --- /dev/null +++ b/internal/secaf/orch/doc.go @@ -0,0 +1,58 @@ +// Package orch ports src/sec_af/orchestrator.py — the AuditOrchestrator class, +// its budget/cost bookkeeping, its checkpoint format, and the output-generation +// step every entry point funnels through. +// +// # Two paths through this package +// +// The class has TWO ways of driving an audit, and only one of them is reached +// by the live REST API: +// +// - Run (`AuditOrchestrator.run`) is the STREAMING, in-process pipeline: +// fast recon, then deep recon / hunt / prove concurrently, with hunt +// publishing findings to prove through a queue. Every sub-agent is a Go +// function call, so the control plane sees ONE execution. app.py does not +// use it — it issues four `.call`s into internal/phases instead — but +// RunFromCheckpoint shares its `_run_hunt` / `_run_prove` halves, so the +// whole thing is ported. +// - GenerateOutput is the shared tail: severity filtering, the CWE severity +// floor, exploitability scoring, compliance mapping, the counters, the +// SARIF/JSON/Markdown artifacts. Both paths end here, and app.py calls it +// directly after its four `.call`s. +// +// # What the node wiring must know +// +// app.py constructs the orchestrator and then OVERWRITES two fields: +// +// orchestrator = AuditOrchestrator(app=app, input=audit_input) +// repo_path = _resolve_repo(repo_url) +// orchestrator.repo_path = Path(repo_path) +// orchestrator.checkpoint_dir = orchestrator.repo_path / ".sec-af" +// +// so New computes its own repo path from SEC_AF_REPO_PATH/cwd (Python parity — +// including running the PR-mode diff analysis against THAT path, before the +// override) and RepoPath/CheckpointDir are exported for the caller to replace. +// SetRepoPath does both assignments in one call. +// +// codeaf's audit entry (internal/secaf/audit) does not take that road: it +// resolves the repository first and constructs with NewAt, so the diff analysis +// and AuditConfig.repo_path describe the audited repository. New and +// NewWithContext keep the Python order for the in-process Run path. +// +// app.py likewise writes agent_invocations, findings_not_verified and +// prove_drop_summary from the phase results before calling GenerateOutput. +// FindingsNotVerified and ProveDropSummary are exported fields for the same +// reason; the invocation counter is mutex-guarded (see below), so it is written +// through SetAgentInvocations instead. +// +// # Concurrency +// +// Python's orchestrator is single-threaded asyncio, so its counters need no +// locking. The Go port fans phases out across goroutines and the phase proxy is +// shared by all of them, so the cost/invocation bookkeeping is mutex-guarded. +// All four guarded values are unexported and reachable only through +// TotalCostUSD / CostBreakdown / AgentInvocations / SetAgentInvocations / +// BudgetExhausted, so the lock contract cannot be bypassed by a later caller +// that reads a partial result while a phase is still running. +// That is the only structural addition; every value it computes is the value +// Python computes. +package orch diff --git a/internal/secaf/orch/golden_test.go b/internal/secaf/orch/golden_test.go new file mode 100644 index 0000000000..56d7f8080f --- /dev/null +++ b/internal/secaf/orch/golden_test.go @@ -0,0 +1,470 @@ +package orch + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "reflect" + "strconv" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// The goldens in testdata/golden are produced by +// +// PYTHONPATH=src ~/.agentfield/packages/sec-af/venv/bin/python go/scripts/gen_golden_phases.py +// +// by calling the REAL orchestrator.py functions with the fixtures in +// testdata/*.json, with the clock stubbed exactly where this package exposes a +// seam (nowUTC for the checkpoint timestamp, nowMonotonic for the elapsed +// times). Everything the two runtimes must agree on byte-for-byte is compared +// here. + +// --------------------------------------------------------------------------- +// fixtures +// --------------------------------------------------------------------------- + +func readFile(t *testing.T, rel string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join("testdata", filepath.FromSlash(rel))) + if err != nil { + t.Fatalf("read %s: %v", rel, err) + } + return string(b) +} + +func readJSON[T any](t *testing.T, rel string) T { + t.Helper() + var out T + if err := json.Unmarshal([]byte(readFile(t, rel)), &out); err != nil { + t.Fatalf("decode %s: %v", rel, err) + } + return out +} + +func reconFixture(t *testing.T, name string) schemas.ReconResult { + t.Helper() + fixtures := readJSON[map[string]json.RawMessage](t, "recon_fixture.json") + raw, ok := fixtures[name] + if !ok { + t.Fatalf("recon_fixture.json has no %q", name) + } + var recon schemas.ReconResult + if err := json.Unmarshal(raw, &recon); err != nil { + t.Fatalf("decode recon fixture %q: %v", name, err) + } + return recon +} + +func findingsFixture(t *testing.T) []schemas.RawFinding { + t.Helper() + return readJSON[[]schemas.RawFinding](t, "findings_fixture.json") +} + +// newTestOrchestrator builds an orchestrator whose repo/checkpoint paths point +// at a fresh temp dir. +// +// Python parity note reproduced by the caller, not by New: AuditConfig.from_input +// uses the STRICT DepthProfile constructor, so a non-canonical depth cannot +// survive construction. Tests that need one assign Input.Depth afterwards, +// exactly as scripts/gen_golden_phases.py does. +func newTestOrchestrator(t *testing.T, mutate ...func(*schemas.AuditInput)) (*AuditOrchestrator, *appx.Fake) { + t.Helper() + input := schemas.NewAuditInput() + input.RepoURL = "https://example.invalid/repo" + input.Depth = "standard" + for _, fn := range mutate { + fn(&input) + } + fake := &appx.Fake{} + o, err := New(fake, input) + if err != nil { + t.Fatalf("orch.New: %v", err) + } + o.SetRepoPath(t.TempDir()) + return o, fake +} + +func splitKey(t *testing.T, key string) (left, right string) { + t.Helper() + for i := 0; i < len(key); i++ { + if key[i] == '|' { + return key[:i], key[i+1:] + } + } + t.Fatalf("malformed golden key %q", key) + return "", "" +} + +func strategyNames(strategies []schemas.HuntStrategy) []string { + out := make([]string, 0, len(strategies)) + for _, s := range strategies { + out = append(out, string(s)) + } + return out +} + +func normalizeJSON(t *testing.T, v any) any { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var out any + if err := json.Unmarshal(b, &out); err != nil { + t.Fatalf("unmarshal: %v", err) + } + return out +} + +// --------------------------------------------------------------------------- +// pure-function goldens +// --------------------------------------------------------------------------- + +// TestDefaultStrategies_Golden pins the ORCHESTRATOR's variant for every +// (fixture, depth) pair — including the thorough-only python_specific / +// javascript_specific additions the phases variant never makes. +func TestDefaultStrategies_Golden(t *testing.T) { + want := readJSON[map[string][]string](t, "golden/default_strategies.json") + if len(want) == 0 { + t.Fatal("default_strategies.json is empty") + } + for key, expected := range want { + reconName, depth := splitKey(t, key) + o, _ := newTestOrchestrator(t) + o.Input.Depth = depth + got := strategyNames(o.DefaultStrategies(reconFixture(t, reconName))) + if !reflect.DeepEqual(got, expected) { + t.Errorf("DefaultStrategies(%s) at depth %q\n got: %v\nwant: %v", reconName, depth, got, expected) + } + } +} + +// TestDefaultStrategies_DiffersFromPhases guards the two documented +// differences: no XSS ever, and the language-specific pair only at thorough. +func TestDefaultStrategies_DiffersFromPhases(t *testing.T) { + recon := reconFixture(t, "full") // languages: python, typescript + + for _, depth := range []string{"quick", "standard", "thorough"} { + o, _ := newTestOrchestrator(t) + o.Input.Depth = depth + names := strategyNames(o.DefaultStrategies(recon)) + for _, n := range names { + if n == string(schemas.HuntStrategyXSS) { + t.Errorf("depth %q: the orchestrator variant must never add xss (%v)", depth, names) + } + } + hasPython := contains(names, string(schemas.HuntStrategyPythonSpecific)) + hasJS := contains(names, string(schemas.HuntStrategyJavascriptSpecific)) + if depth == "thorough" { + if !hasPython || !hasJS { + t.Errorf("thorough must add both language strategies, got %v", names) + } + } else if hasPython || hasJS { + t.Errorf("depth %q must not add the language strategies, got %v", depth, names) + } + } +} + +func contains(list []string, want string) bool { + for _, v := range list { + if v == want { + return true + } + } + return false +} + +// TestProverCap_Golden pins `_prover_cap()`. +func TestProverCap_Golden(t *testing.T) { + want := readJSON[map[string]int](t, "golden/prover_cap.json") + if len(want) == 0 { + t.Fatal("prover_cap.json is empty") + } + for key, expected := range want { + depth, capSpec := splitKey(t, key) + o, _ := newTestOrchestrator(t, func(in *schemas.AuditInput) { + if capSpec != "null" { + v, err := strconv.Atoi(capSpec) + if err != nil { + t.Fatalf("golden key %q: %v", key, err) + } + in.MaxProvers = &v + } + }) + o.Input.Depth = depth + if got := o.ProverCap(); got != expected { + t.Errorf("ProverCap() at depth %q max_provers %s = %d, want %d", depth, capSpec, got, expected) + } + } +} + +// TestPrioritizeFindings_Golden pins the sort order and the stable tie-break. +func TestPrioritizeFindings_Golden(t *testing.T) { + want := readJSON[[]string](t, "golden/prioritize_findings.json") + o, _ := newTestOrchestrator(t) + findings := findingsFixture(t) + got := make([]string, 0, len(want)) + for _, f := range o.PrioritizeFindings(findings) { + got = append(got, f.ID) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("PrioritizeFindings order\n got: %v\nwant: %v", got, want) + } + if findings[0].ID != "low-high" { + t.Errorf("the input slice was reordered: findings[0] = %q", findings[0].ID) + } +} + +// TestVerifiedFindingFallback_Golden compares the whole model_dump against +// Python's — including the "sec-af//" rule id that +// differs from agents/prove.Fallback's. +func TestVerifiedFindingFallback_Golden(t *testing.T) { + finding := readJSON[schemas.RawFinding](t, "fallback_finding.json") + want := readJSON[any](t, "golden/verified_finding_fallback.json") + got := normalizeJSON(t, VerifiedFindingFallback(finding)) + if !reflect.DeepEqual(got, want) { + gotJSON, _ := json.MarshalIndent(got, "", " ") + wantJSON, _ := json.MarshalIndent(want, "", " ") + t.Errorf("VerifiedFindingFallback\n got: %s\nwant: %s", gotJSON, wantJSON) + } +} + +// TestMergeReconFindingsIntoHunt_Golden ports +// tests/test_recon_findings.py::test_merge_recon_findings_prepends_and_updates_counts +// and compares the whole merged HuntResult with Python's. +func TestMergeReconFindingsIntoHunt_Golden(t *testing.T) { + type goldenShape struct { + Merged json.RawMessage `json:"merged"` + EmptyReconIsIdentity bool `json:"empty_recon_is_identity"` + AlreadyPresentStrategy []string `json:"already_present"` + } + golden := readJSON[goldenShape](t, "golden/merge_recon_findings.json") + + findings := findingsFixture(t) + hunt := schemas.NewHuntResult() + hunt.Findings = []schemas.RawFinding{findings[0]} + hunt.TotalRaw = 1 + hunt.DeduplicatedCount = 1 + hunt.StrategiesRun = []string{"injection"} + reconFindings := []schemas.RawFinding{findings[2]} + + merged := MergeReconFindingsIntoHunt(hunt, reconFindings) + + // The Python test's own four assertions. + if len(merged.Findings) != 2 { + t.Fatalf("merged findings = %d, want 2", len(merged.Findings)) + } + if merged.Findings[0].HunterStrategy != findings[2].HunterStrategy { + t.Errorf("findings[0] must be the recon finding, got %q", merged.Findings[0].HunterStrategy) + } + if merged.TotalRaw != 2 { + t.Errorf("total_raw = %d, want 2", merged.TotalRaw) + } + if merged.DeduplicatedCount != 2 { + t.Errorf("deduplicated_count = %d, want 2", merged.DeduplicatedCount) + } + if merged.StrategiesRun[0] != "recon" { + t.Errorf("strategies_run[0] = %q, want recon", merged.StrategiesRun[0]) + } + + var wantMerged any + if err := json.Unmarshal(golden.Merged, &wantMerged); err != nil { + t.Fatalf("decode merged golden: %v", err) + } + if got := normalizeJSON(t, merged); !reflect.DeepEqual(got, wantMerged) { + gotJSON, _ := json.MarshalIndent(got, "", " ") + wantJSON, _ := json.MarshalIndent(wantMerged, "", " ") + t.Errorf("merged HuntResult\n got: %s\nwant: %s", gotJSON, wantJSON) + } + + // Empty recon findings: the hunt result comes back untouched. + if !golden.EmptyReconIsIdentity { + t.Fatal("golden says the empty-recon case is not the identity; the Python source says it is") + } + identity := MergeReconFindingsIntoHunt(hunt, nil) + if !reflect.DeepEqual(normalizeJSON(t, identity), normalizeJSON(t, hunt)) { + t.Error("MergeReconFindingsIntoHunt(hunt, nil) must return the hunt result unchanged") + } + + // "recon" already present: no second insertion. + pre := schemas.NewHuntResult() + pre.StrategiesRun = []string{"recon", "injection"} + if got := MergeReconFindingsIntoHunt(pre, reconFindings).StrategiesRun; !reflect.DeepEqual(got, golden.AlreadyPresentStrategy) { + t.Errorf("strategies_run = %v, want %v", got, golden.AlreadyPresentStrategy) + } +} + +// --------------------------------------------------------------------------- +// prompts and notes +// --------------------------------------------------------------------------- + +// TestReachabilitySummary_Golden pins the prompt body handed to +// AIGateWrapper.assess_reachability. +func TestReachabilitySummary_Golden(t *testing.T) { + finding := readJSON[schemas.VerifiedFinding](t, "verified_fixture.json") + want := readFile(t, "golden/reachability_summary.txt") + if got := ReachabilitySummary(finding); got != want { + t.Errorf("ReachabilitySummary\n got: %q\nwant: %q", got, want) + } +} + +// TestBuildProgress_And_EmitProgress_Golden pins BOTH the arithmetic +// (progress_fields.json) and the exact note text +// (progress_notes.json — pydantic's model_dump_json, no whitespace, floats with +// a decimal point). +func TestBuildProgress_And_EmitProgress_Golden(t *testing.T) { + type progressCase struct { + phase string + agentsTotal int + agentsCompleted int + findingsSoFar int + } + cases := map[string]progressCase{ + "recon_half": {"recon", 2, 1, 0}, + "recon_full": {"recon", 2, 2, 0}, + "hunt_done": {"hunt", 1, 1, 7}, + "zero_total": {"prove", 0, 0, 0}, + "overshoot": {"prove", 2, 5, 3}, + } + + wantNotes := readJSON[map[string]string](t, "golden/progress_notes.json") + wantFields := readJSON[map[string]map[string]any](t, "golden/progress_fields.json") + if len(wantNotes) != len(cases) { + t.Fatalf("golden has %d cases, the test has %d", len(wantNotes), len(cases)) + } + + // Pin the clock: the generator scripted monotonic() as 100.0 then 102.5, so + // every elapsed read is 2.5s. + base := time.Now() + restore := nowMonotonic + nowMonotonic = func() time.Time { return base.Add(2500 * time.Millisecond) } + defer func() { nowMonotonic = restore }() + + for name, tc := range cases { + name, tc := name, tc + t.Run(name, func(t *testing.T) { + o, fake := newTestOrchestrator(t) + o.StartedAt = base + // The generator set total_cost_usd = 0.123456; the note carries + // round(x, 4) = 0.1235 (banker's rounding). + o.registerCost(PhaseRecon, floatPtr(0.123456)) + + o.EmitProgress(context.Background(), tc.phase, tc.agentsTotal, tc.agentsCompleted, tc.findingsSoFar) + + if len(fake.Notes) != 1 { + t.Fatalf("notes = %d, want 1", len(fake.Notes)) + } + if got := fake.Notes[0].Message; got != wantNotes[name] { + t.Errorf("note message\n got: %s\nwant: %s", got, wantNotes[name]) + } + if want := []string{"audit", "progress", tc.phase}; !reflect.DeepEqual(fake.Notes[0].Tags, want) { + t.Errorf("tags = %v, want %v", fake.Notes[0].Tags, want) + } + + // And the structured fields, so a formatting change cannot hide an + // arithmetic change. + got := normalizeJSON(t, o.BuildProgress(tc.phase, tc.agentsTotal, tc.agentsCompleted, tc.findingsSoFar)) + if !reflect.DeepEqual(got, any(wantFields[name])) { + t.Errorf("progress fields\n got: %#v\nwant: %#v", got, wantFields[name]) + } + }) + } +} + +// TestProgressModelDumpJSON_Golden pins pyfmt.DumpsModelJSON against pydantic's +// serializer independently of the orchestrator's arithmetic. +func TestProgressModelDumpJSON_Golden(t *testing.T) { + want := readJSON[map[string]string](t, "golden/progress_model_dump_json.json") + + cases := map[string]schemas.AuditProgress{ + "unit_progress": { + Phase: "recon", PhaseProgress: 1.0, AgentsTotal: 2, AgentsCompleted: 2, + AgentsRunning: 0, FindingsSoFar: 0, ElapsedSeconds: 2.5, + EstimatedRemainingSeconds: 0.0, CostSoFarUsd: 0.0, + }, + "fractional": { + Phase: "hunt", PhaseProgress: 0.5, AgentsTotal: 4, AgentsCompleted: 2, + AgentsRunning: 2, FindingsSoFar: 13, ElapsedSeconds: 1.25, + EstimatedRemainingSeconds: 1.25, CostSoFarUsd: 0.1235, + }, + } + for name, progress := range cases { + if got := pyfmt.DumpsModelJSON(progress); got != want[name] { + t.Errorf("%s\n got: %s\nwant: %s", name, got, want[name]) + } + } +} + +func floatPtr(f float64) *float64 { return &f } + +// --------------------------------------------------------------------------- +// checkpoints +// --------------------------------------------------------------------------- + +// TestWriteCheckpoint_Golden compares the FILE BYTES with the ones Python +// wrote for the same payload and the same pinned clock. The bytes are the +// cross-runtime contract: a Go node must be able to resume from a Python +// node's checkpoint and vice versa. +func TestWriteCheckpoint_Golden(t *testing.T) { + pinned, err := time.Parse(time.RFC3339Nano, "2026-01-02T03:04:05.123456Z") + if err != nil { + t.Fatalf("parse pinned time: %v", err) + } + restore := nowUTC + nowUTC = func() time.Time { return pinned.UTC() } + defer func() { nowUTC = restore }() + + // The pinned isoformat the generator recorded. + created := readJSON[map[string]string](t, "golden/checkpoint_created_at.json") + if got := schemas.NewTimestamp(nowUTC()).String(); got != created["pinned"] { + t.Fatalf("created_at = %q, want %q", got, created["pinned"]) + } + + o, _ := newTestOrchestrator(t) + + t.Run("model payload", func(t *testing.T) { + if err := o.WriteCheckpoint("recon", reconFixture(t, "minimal")); err != nil { + t.Fatalf("WriteCheckpoint: %v", err) + } + got, err := os.ReadFile(o.CheckpointPath("recon")) + if err != nil { + t.Fatalf("read checkpoint: %v", err) + } + if want := readFile(t, "golden/checkpoint_recon.txt"); string(got) != want { + t.Errorf("checkpoint-recon.json\n got: %s\nwant: %s", got, want) + } + }) + + t.Run("list payload", func(t *testing.T) { + verified := []schemas.VerifiedFinding{readJSON[schemas.VerifiedFinding](t, "verified_fixture.json")} + if err := o.WriteCheckpoint("prove", verified); err != nil { + t.Fatalf("WriteCheckpoint: %v", err) + } + got, err := os.ReadFile(o.CheckpointPath("prove")) + if err != nil { + t.Fatalf("read checkpoint: %v", err) + } + if want := readFile(t, "golden/checkpoint_prove.txt"); string(got) != want { + t.Errorf("checkpoint-prove.json\n got: %s\nwant: %s", got, want) + } + }) + + t.Run("empty list payload", func(t *testing.T) { + if err := o.WriteCheckpoint("prove_empty", []schemas.VerifiedFinding{}); err != nil { + t.Fatalf("WriteCheckpoint: %v", err) + } + got, err := os.ReadFile(o.CheckpointPath("prove_empty")) + if err != nil { + t.Fatalf("read checkpoint: %v", err) + } + if want := readFile(t, "golden/checkpoint_prove_empty.txt"); string(got) != want { + t.Errorf("checkpoint-prove_empty.json\n got: %s\nwant: %s", got, want) + } + }) +} diff --git a/internal/secaf/orch/merge.go b/internal/secaf/orch/merge.go new file mode 100644 index 0000000000..42dc1735cc --- /dev/null +++ b/internal/secaf/orch/merge.go @@ -0,0 +1,152 @@ +package orch + +import ( + "sort" + "strings" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// VerifiedFindingFallback ports the module-level `_verified_finding_fallback` +// (orchestrator.py:672) — a RawFinding promoted to an INCONCLUSIVE +// VerifiedFinding when no proof is available. +// +// VerifiedFinding( +// id=finding.id, fingerprint=finding.fingerprint, title=..., description=..., +// finding_type=..., cwe_id=..., cwe_name=..., owasp_category=..., +// tags=[], verdict=Verdict.INCONCLUSIVE, evidence_level=EvidenceLevel.STATIC_MATCH, +// rationale="Automated proof unavailable; requires manual review.", +// severity=finding.estimated_severity, exploitability_score=0.0, +// location=Location(file_path=..., start_line=..., end_line=..., +// function_name=..., code_snippet=...), +// sarif_rule_id=f"sec-af/{finding.finding_type.value}/{finding.cwe_id.lower()}", +// sarif_security_severity=0.0) +// +// Two things distinguish it from agents/prove.Fallback, which serves the same +// purpose elsewhere: +// +// - the rationale is a FIXED string, not "Verification incomplete: "; +// - the sarif_rule_id is built from the lowercased CWE ID +// ("sec-af/sast/cwe-89"), where prove's is built from the CWE NAME with +// spaces turned into hyphens ("sec-af/sast/sql-injection"). +// +// Python parity: nothing in the Python tree calls this function — every demotion +// path goes through agents/prove.fallback — so it is ported for completeness. +// `related_locations`, `compliance` and `reproduction_steps` keep pydantic's +// empty-list defaults; `drop_reason` stays None, which is what makes a finding +// built here invisible to the orchestrator's drop_reason sweep. +func VerifiedFindingFallback(finding schemas.RawFinding) schemas.VerifiedFinding { + snippet := finding.CodeSnippet + out := schemas.NewVerifiedFinding() + out.ID = finding.ID + out.Fingerprint = finding.Fingerprint + out.Title = finding.Title + out.Description = finding.Description + out.FindingType = finding.FindingType + out.CweID = finding.CweID + out.CweName = finding.CweName + out.OwaspCategory = finding.OwaspCategory + out.Tags = []string{} + out.Verdict = schemas.VerdictInconclusive + out.EvidenceLevel = schemas.EvidenceLevelStaticMatch + out.Rationale = "Automated proof unavailable; requires manual review." + out.Severity = finding.EstimatedSeverity + out.ExploitabilityScore = 0.0 + out.Location = schemas.Location{ + FilePath: finding.FilePath, + StartLine: finding.StartLine, + EndLine: finding.EndLine, + FunctionName: finding.FunctionName, + CodeSnippet: &snippet, + } + out.SarifRuleID = "sec-af/" + string(finding.FindingType) + "/" + strings.ToLower(finding.CweID) + out.SarifSecuritySeverity = 0.0 + return out +} + +// MergeReconFindingsIntoHunt ports the module-level +// `merge_recon_findings_into_hunt` (orchestrator.py:700) — how the findings +// RECON detected on its own (hardcoded secrets, misconfigs, weak TLS) join the +// hunters' output: +// +// if not recon_findings: return hunt +// merged_findings = [*recon_findings, *hunt.findings] +// strategies_run = list(hunt.strategies_run) +// if "recon" not in strategies_run: strategies_run.insert(0, "recon") +// return HuntResult(findings=merged_findings, chains=hunt.chains, +// total_raw=hunt.total_raw + len(recon_findings), +// deduplicated_count=len(merged_findings), +// chain_count=hunt.chain_count, strategies_run=strategies_run, +// hunt_duration_seconds=hunt.hunt_duration_seconds) +// +// Behaviours tests/test_recon_findings.py::test_merge_recon_findings_prepends_and_updates_counts +// pins, all reproduced: +// +// - recon findings are PREPENDED, so findings[0] is a recon finding; +// - total_raw GROWS by the recon count (it is not recomputed); +// - deduplicated_count is REPLACED by the merged length — no dedup pass runs, +// so a recon finding that duplicates a hunter finding survives twice; +// - "recon" is inserted at the FRONT of strategies_run, and only when absent. +// +// Python parity: an empty recon_findings list returns the hunt result +// UNCHANGED — the same value, not a copy — so a caller that mutates the result +// mutates the input. Go returns the same struct value, which copies the header +// but shares the backing arrays: identical aliasing. +func MergeReconFindingsIntoHunt(hunt schemas.HuntResult, reconFindings []schemas.RawFinding) schemas.HuntResult { + if len(reconFindings) == 0 { + return hunt + } + + merged := make([]schemas.RawFinding, 0, len(reconFindings)+len(hunt.Findings)) + merged = append(merged, reconFindings...) + merged = append(merged, hunt.Findings...) + + strategiesRun := make([]string, 0, len(hunt.StrategiesRun)+1) + strategiesRun = append(strategiesRun, hunt.StrategiesRun...) + found := false + for _, s := range strategiesRun { + if s == "recon" { + found = true + break + } + } + if !found { + strategiesRun = append([]string{"recon"}, strategiesRun...) + } + + return schemas.HuntResult{ + Findings: merged, + Chains: hunt.Chains, + TotalRaw: hunt.TotalRaw + len(reconFindings), + DeduplicatedCount: len(merged), + ChainCount: hunt.ChainCount, + StrategiesRun: strategiesRun, + HuntDurationSeconds: hunt.HuntDurationSeconds, + } +} + +// sortedNonEmpty ports `sorted({item for item in values if item})` — the +// framework projection `_merge_recon` applies to +// security_context.framework_security. +// +// Python parity: the filter is TRUTHINESS, so blank entries disappear before +// the set is built; case is preserved, so "Django" and "django" are two +// distinct frameworks. The set-then-sort erases the only nondeterminism. +// reasoners/phases.py's recon_phase carries the same expression, and +// internal/phases has its own copy for the same reason SEC-AF does. +func sortedNonEmpty(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + if value == "" { + continue + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + sort.Strings(out) + return out +} diff --git a/internal/secaf/orch/orch.go b/internal/secaf/orch/orch.go new file mode 100644 index 0000000000..98e521b692 --- /dev/null +++ b/internal/secaf/orch/orch.go @@ -0,0 +1,321 @@ +package orch + +import ( + "context" + "os" + "path/filepath" + "sync" + "time" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/config" + "github.com/Agent-Field/codeaf/internal/secaf/diffanalysis" + "github.com/Agent-Field/codeaf/internal/secaf/gates" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// nowMonotonic is `time.monotonic()` — the clock New stamps StartedAt with and +// elapsedSeconds measures against. It is a variable purely so a test can pin +// the elapsed times that reach the progress notes and duration_seconds; +// production never reassigns it. +// +// Go's time.Now carries a monotonic reading that Sub prefers over the wall +// clock, so the DIFFERENCE has exactly Python's time.monotonic() semantics +// (immune to clock adjustments) even though the absolute value is a wall time. +var nowMonotonic = time.Now + +// PhaseOrder ports `AuditOrchestrator._PHASE_ORDER = ("recon", "hunt", "prove")` +// — the three cost buckets, and the key set of cost_breakdown. +var PhaseOrder = [...]string{"recon", "hunt", "prove"} + +// The three phase names, spelled once so a typo cannot silently create a fourth +// cost bucket. +const ( + PhaseRecon = "recon" + PhaseHunt = "hunt" + PhaseProve = "prove" +) + +// AuditOrchestrator ports the class of the same name (orchestrator.py:53). +// +// Every field Python assigns in `__init__` is here, exported where a caller +// (app.py, and therefore internal/secaf/audit) reads or writes it: +// +// self.app, self.input, self.started_at, self.repo_path, self.checkpoint_dir, +// self.is_pr_mode, self.diff_analysis, self.config, self.budget_config, +// self.max_cost_usd, self.max_duration_seconds, self.total_cost_usd, +// self.cost_breakdown, self.agent_invocations, self.budget_exhausted, +// self.findings_not_verified, self.prove_drop_summary, self.ai_gate +type AuditOrchestrator struct { + // App is the SDK agent. Python stores it as `cast("Any", app)` purely to + // silence the type checker; the capability set it actually uses is + // harness + ai + note (never call — the orchestrator's own pipeline is + // in-process). + App appx.App + // Input is the audit request. + Input schemas.AuditInput + // StartedAt is `time.monotonic()` at construction. Go's time.Now is a wall + // clock, but every read is a DIFFERENCE (`time.monotonic() - self.started_at`) + // and Go's time.Time carries a monotonic reading that Sub prefers, so the + // elapsed values match Python's semantics. + StartedAt time.Time + // RepoPath is `Path(os.getenv("SEC_AF_REPO_PATH", os.getcwd())).resolve()`. + // app.py replaces it right after construction — see SetRepoPath. + RepoPath string + // CheckpointDir is `self.repo_path / ".sec-af"`. + CheckpointDir string + // IsPRMode is `input.is_pr`. + IsPRMode bool + // DiffAnalysis is populated only in PR mode with a base commit; nil is + // Python's None. + DiffAnalysis *diffanalysis.DiffAnalysis + // Config is `AuditConfig.from_input(input, str(repo_path))`. + Config config.AuditConfig + // BudgetConfig is `self.config.budget`. + BudgetConfig config.BudgetConfig + // MaxCostUSD is `input.max_cost_usd` (None => nil => no cost budget). + MaxCostUSD *float64 + // MaxDurationSeconds is `input.max_duration_seconds`. + MaxDurationSeconds *int + // FindingsNotVerified is the count GenerateOutput reports in metadata. + // + // Unlike the two guarded counters it stays an EXPORTED field, because + // every writer runs on the sequential audit path and never inside the + // phase fan-out: run.go's two phases, and app.py's own + // `orchestrator.findings_not_verified = ...` (node/audit.go). + FindingsNotVerified int + // ProveDropSummary is the `{"demoted_total", "by_reason", "findings"}` + // dict, rebuilt at the start of each prove pass and copied verbatim into + // the result metadata. + // + // Typed `any`, not `map[string]any`, because on the app.py path it is + // assigned straight from a `.call` payload with + // `prove_dict.get("drop_summary", {...})` — a `.get`, so the default fires + // ONLY when the key is absent. A key that is present with an odd value (a + // JSON null, a string, a list) is threaded through to + // `metadata["prove_drop_summary"]` verbatim in Python, and a + // `map[string]any` field cannot hold it. See node/audit.go. + ProveDropSummary any + // AIGate is `AIGateWrapper(app=self.app)` — used by + // AssessReachabilityParallel and by GenerateOutput's compliance fallback. + AIGate *gates.AIGate + + // mu guards the cost/invocation counters. Python needs no lock (asyncio is + // single-threaded); the Go phases fan out across goroutines that share one + // proxy, so the bookkeeping must be safe for concurrent use. It guards + // totalCostUSD, costBreakdown, agentInvocations and budgetExhausted. + // + // All four are UNEXPORTED so the contract is enforceable rather than + // merely documented: every access goes through the guarded accessors below + // (TotalCostUSD, CostBreakdown, AgentInvocations, SetAgentInvocations, + // BudgetExhausted) and there is no way for a caller to read one while a + // phase proxy is still writing it. An earlier revision exported the two + // counters, and GenerateOutput read both of them bare — harmless today, + // because Run joins its errgroup before calling it, but a latent race for + // the first caller that reads a partial result (a progress endpoint, an + // early return on budget exhaustion) and one the race detector would only + // catch if a test happened to schedule the overlap. + mu sync.Mutex + totalCostUSD float64 + costBreakdown map[string]float64 + agentInvocations int + budgetExhausted bool +} + +// New ports `AuditOrchestrator.__init__(app, input)`. +// +// self.started_at = time.monotonic() +// self.repo_path = Path(os.getenv("SEC_AF_REPO_PATH", os.getcwd())).resolve() +// self.checkpoint_dir = self.repo_path / ".sec-af" +// self.is_pr_mode = input.is_pr +// if self.is_pr_mode and input.base_commit_sha: +// self.diff_analysis = analyze_diff(str(self.repo_path), input.base_commit_sha, input.commit_sha or "HEAD") +// self.config = AuditConfig.from_input(self.input, str(self.repo_path)) +// ... +// self.ai_gate = AIGateWrapper(app=self.app) +// +// Python parity: +// +// - the diff analysis runs against the CONSTRUCTOR's repo path, before app.py +// substitutes the resolved (possibly cloned) one. Reproduced: callers that +// want the diff against the real checkout must construct after resolving, +// which app.py does not do. +// - `input.base_commit_sha` and `input.commit_sha` are tested with PYTHON +// TRUTHINESS, so an empty string behaves like None — no diff analysis, and +// the head defaults to "HEAD". +// - AIGateWrapper's config falls back to `AIIntegrationConfig.from_env()`; +// here it is config.DefaultAIConfig(), which reads nothing and cannot fail. +// +// Signature parity note: Python's `__init__` is synchronous and carries no +// context, so New takes none. analyze_diff shells out to git, and +// diffanalysis.AnalyzeDiff gives each invocation its own 30s timeout, so +// context.Background() is the honest translation. NewWithContext is for a +// caller that already holds a cancellable context. +func New(app appx.App, input schemas.AuditInput) (*AuditOrchestrator, error) { + return NewWithContext(context.Background(), app, input) +} + +// NewWithContext is New with the context the PR-mode git commands run under. +func NewWithContext(ctx context.Context, app appx.App, input schemas.AuditInput) (*AuditOrchestrator, error) { + return NewAt(ctx, app, input, resolveRepoPath()) +} + +// NewAt is NewWithContext for a repository the caller has already resolved to +// an absolute directory, which is how codeaf's audit entry (internal/secaf/audit) +// constructs it. +// +// It exists because the Python order — construct against SEC_AF_REPO_PATH or the +// process working directory, then overwrite repo_path — runs the PR-mode diff +// analysis and records AuditConfig.repo_path against whatever directory codeaf +// happened to start in. Handing the repository in up front makes all three +// (repo_path, checkpoint_dir, config.repo_path) and the diff analysis agree on +// the repository being audited. +func NewAt(ctx context.Context, app appx.App, input schemas.AuditInput, repoPath string) (*AuditOrchestrator, error) { + o := &AuditOrchestrator{ + App: app, + Input: input, + StartedAt: nowMonotonic(), + RepoPath: repoPath, + CheckpointDir: filepath.Join(repoPath, ".sec-af"), + IsPRMode: input.IsPr, + MaxCostUSD: input.MaxCostUsd, + MaxDurationSeconds: input.MaxDurationSeconds, + FindingsNotVerified: 0, + ProveDropSummary: NewDropSummary(), + costBreakdown: map[string]float64{}, + } + for _, phase := range PhaseOrder { + o.costBreakdown[phase] = 0.0 + } + + if o.IsPRMode && input.BaseCommitSha != nil && *input.BaseCommitSha != "" { + head := "HEAD" + if input.CommitSha != nil && *input.CommitSha != "" { + head = *input.CommitSha + } + analysis := diffanalysis.AnalyzeDiff(ctx, o.RepoPath, *input.BaseCommitSha, head) + o.DiffAnalysis = &analysis + } + + cfg, err := config.AuditConfig{}.FromInput(input, o.RepoPath) + if err != nil { + return nil, err + } + o.Config = cfg + o.BudgetConfig = cfg.Budget + + gate, err := gates.NewAIGate(app, nil) + if err != nil { + return nil, err + } + o.AIGate = gate + + return o, nil +} + +// SetRepoPath performs the two assignments app.py makes right after +// construction: +// +// orchestrator.repo_path = Path(repo_path) +// orchestrator.checkpoint_dir = orchestrator.repo_path / ".sec-af" +// +// Python parity: app.py does NOT re-resolve the path here (it passes the +// already-absolute result of _resolve_repo) and does NOT recompute self.config, +// so AuditConfig.repo_path keeps the constructor's value. Both are reproduced — +// this setter touches exactly the two fields Python touches. +func (o *AuditOrchestrator) SetRepoPath(repoPath string) { + o.RepoPath = repoPath + o.CheckpointDir = filepath.Join(repoPath, ".sec-af") +} + +// resolveRepoPath ports `Path(os.getenv("SEC_AF_REPO_PATH", os.getcwd())).resolve()`. +// +// Python parity: `Path.resolve()` makes the path absolute AND follows symlinks, +// with strict=False so a non-existent path still resolves. Go splits that in +// two: filepath.Abs handles the absolute part and always succeeds for a +// non-empty path; filepath.EvalSymlinks handles the link part but FAILS on a +// path that does not exist, in which case the absolute form is kept — the same +// answer Python gives for a path with no symlinked ancestors, which is every +// path in practice. +func resolveRepoPath() string { + raw := os.Getenv("SEC_AF_REPO_PATH") + if raw == "" { + cwd, err := os.Getwd() + if err != nil { + // os.getcwd() raises in Python too; there is no path that is more + // correct than the relative one. + cwd = "." + } + raw = cwd + } + abs, err := filepath.Abs(raw) + if err != nil { + return raw + } + if resolved, err := filepath.EvalSymlinks(abs); err == nil { + return resolved + } + return abs +} + +// NewDropSummary builds the `{"demoted_total": 0, "by_reason": {}, "findings": []}` +// literal the orchestrator resets prove_drop_summary to. +func NewDropSummary() map[string]any { + return map[string]any{ + "demoted_total": 0, + "by_reason": map[string]int{}, + "findings": []map[string]any{}, + } +} + +// AgentInvocations reports `self.agent_invocations` — the number of harness +// invocations made through the phase proxy. +func (o *AuditOrchestrator) AgentInvocations() int { + o.mu.Lock() + defer o.mu.Unlock() + return o.agentInvocations +} + +// SetAgentInvocations OVERWRITES the counter, which is what app.py:220 does +// (`orchestrator.agent_invocations = total_selected + len(...) + 3`) before +// calling _generate_output. +func (o *AuditOrchestrator) SetAgentInvocations(n int) { + o.mu.Lock() + o.agentInvocations = n + o.mu.Unlock() +} + +// BudgetExhausted reports `self.budget_exhausted`, which latches true the first +// time a budget check trips. +func (o *AuditOrchestrator) BudgetExhausted() bool { + o.mu.Lock() + defer o.mu.Unlock() + return o.budgetExhausted +} + +// TotalCostUSD reports the accumulated harness cost (`self.total_cost_usd`). +func (o *AuditOrchestrator) TotalCostUSD() float64 { + o.mu.Lock() + defer o.mu.Unlock() + return o.totalCostUSD +} + +// CostBreakdown returns a COPY of the per-phase cost map +// (`self.cost_breakdown`), so a caller cannot corrupt the running totals. +func (o *AuditOrchestrator) CostBreakdown() map[string]float64 { + o.mu.Lock() + defer o.mu.Unlock() + out := make(map[string]float64, len(o.costBreakdown)) + for k, v := range o.costBreakdown { + out[k] = v + } + return out +} + +// depthProfile ports `_depth_profile()`: +// +// try: return DepthProfile(self.input.depth.lower()) +// except ValueError: return DepthProfile.STANDARD +func (o *AuditOrchestrator) depthProfile() config.DepthProfile { + return config.NormalizeDepth(o.Input.Depth) +} diff --git a/internal/secaf/orch/orch_test.go b/internal/secaf/orch/orch_test.go new file mode 100644 index 0000000000..b48122d071 --- /dev/null +++ b/internal/secaf/orch/orch_test.go @@ -0,0 +1,1053 @@ +package orch + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/Agent-Field/agentfield/sdk/go/ai" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/compliance" + "github.com/Agent-Field/codeaf/internal/secaf/phases" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// --------------------------------------------------------------------------- +// construction +// --------------------------------------------------------------------------- + +// TestNew_InitializesEveryField mirrors the assignments of Python's __init__. +func TestNew_InitializesEveryField(t *testing.T) { + repo := t.TempDir() + t.Setenv("SEC_AF_REPO_PATH", repo) + + input := schemas.NewAuditInput() + input.RepoURL = "https://example.invalid/repo" + cost := 12.5 + provers := 7 + duration := 900 + input.MaxCostUsd = &cost + input.MaxProvers = &provers + input.MaxDurationSeconds = &duration + + o, err := New(&appx.Fake{}, input) + if err != nil { + t.Fatalf("New: %v", err) + } + + wantRepo := repo + if resolved, err := filepath.EvalSymlinks(repo); err == nil { + wantRepo = resolved + } + if o.RepoPath != wantRepo { + t.Errorf("RepoPath = %q, want %q", o.RepoPath, wantRepo) + } + if want := filepath.Join(wantRepo, ".sec-af"); o.CheckpointDir != want { + t.Errorf("CheckpointDir = %q, want %q", o.CheckpointDir, want) + } + if o.IsPRMode { + t.Error("IsPRMode must default to false") + } + if o.DiffAnalysis != nil { + t.Error("DiffAnalysis must stay nil outside PR mode") + } + if o.Config.RepoPath != wantRepo { + t.Errorf("Config.RepoPath = %q, want %q", o.Config.RepoPath, wantRepo) + } + if o.MaxCostUSD == nil || *o.MaxCostUSD != cost { + t.Errorf("MaxCostUSD = %v, want %v", o.MaxCostUSD, cost) + } + if o.MaxDurationSeconds == nil || *o.MaxDurationSeconds != duration { + t.Errorf("MaxDurationSeconds = %v, want %v", o.MaxDurationSeconds, duration) + } + // budget_config comes from the config, and carries the three input caps. + if o.BudgetConfig.MaxProvers == nil || *o.BudgetConfig.MaxProvers != provers { + t.Errorf("BudgetConfig.MaxProvers = %v, want %v", o.BudgetConfig.MaxProvers, provers) + } + if o.BudgetConfig.MaxConcurrentHunters != 4 || o.BudgetConfig.MaxConcurrentProvers != 3 { + t.Errorf("BudgetConfig concurrency = %d/%d, want 4/3", + o.BudgetConfig.MaxConcurrentHunters, o.BudgetConfig.MaxConcurrentProvers) + } + // cost_breakdown starts as {phase: 0.0} for exactly the three phases. + if got := o.CostBreakdown(); !reflect.DeepEqual(got, map[string]float64{"recon": 0, "hunt": 0, "prove": 0}) { + t.Errorf("CostBreakdown = %v", got) + } + if o.AgentInvocations() != 0 || o.BudgetExhausted() || o.FindingsNotVerified != 0 { + t.Error("counters must start at zero") + } + want := map[string]any{"demoted_total": 0, "by_reason": map[string]int{}, "findings": []map[string]any{}} + if !reflect.DeepEqual(o.ProveDropSummary, want) { + t.Errorf("ProveDropSummary = %#v, want %#v", o.ProveDropSummary, want) + } + if o.AIGate == nil { + t.Error("AIGate must be constructed") + } +} + +// TestNew_RejectsInvalidDepth: AuditConfig.from_input uses the STRICT +// DepthProfile constructor, so an unrecognised depth is a ValueError — which +// app.py maps to HTTP 400. +func TestNew_RejectsInvalidDepth(t *testing.T) { + input := schemas.NewAuditInput() + input.RepoURL = "https://example.invalid/repo" + input.Depth = "QUICK" + if _, err := New(&appx.Fake{}, input); err == nil { + t.Fatal("want an error for depth QUICK") + } +} + +// TestSetRepoPath performs app.py's two post-construction assignments. +func TestSetRepoPath(t *testing.T) { + o, _ := newTestOrchestrator(t) + before := o.Config.RepoPath + o.SetRepoPath("/elsewhere/repo") + if o.RepoPath != "/elsewhere/repo" { + t.Errorf("RepoPath = %q", o.RepoPath) + } + if want := filepath.Join("/elsewhere/repo", ".sec-af"); o.CheckpointDir != want { + t.Errorf("CheckpointDir = %q, want %q", o.CheckpointDir, want) + } + // Python parity: app.py does NOT recompute self.config, so AuditConfig + // keeps the constructor's repo_path. + if o.Config.RepoPath != before { + t.Errorf("Config.RepoPath = %q, want it unchanged (%q)", o.Config.RepoPath, before) + } +} + +// TestNew_PRModeRunsDiffAnalysis: the branch fires only with is_pr AND a +// truthy base_commit_sha. +func TestNew_PRModeRunsDiffAnalysis(t *testing.T) { + cases := []struct { + name string + isPR bool + baseSHA *string + wantDiff bool + }{ + {"not a PR", false, strPtr("abc"), false}, + {"PR without base", true, nil, false}, + {"PR with empty base", true, strPtr(""), false}, + {"PR with base", true, strPtr("abc"), true}, + } + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Setenv("SEC_AF_REPO_PATH", t.TempDir()) + input := schemas.NewAuditInput() + input.RepoURL = "https://example.invalid/repo" + input.IsPr = tc.isPR + input.BaseCommitSha = tc.baseSHA + + o, err := New(&appx.Fake{}, input) + if err != nil { + t.Fatalf("New: %v", err) + } + if got := o.DiffAnalysis != nil; got != tc.wantDiff { + t.Errorf("DiffAnalysis present = %v, want %v", got, tc.wantDiff) + } + if tc.wantDiff && o.DiffAnalysis.HeadSHA != "HEAD" { + t.Errorf("HeadSHA = %q, want HEAD when commit_sha is absent", o.DiffAnalysis.HeadSHA) + } + }) + } +} + +func strPtr(s string) *string { return &s } + +// --------------------------------------------------------------------------- +// budget +// --------------------------------------------------------------------------- + +// TestBudget_CostChecks walks _check_cost_budget's two limits. +func TestBudget_CostChecks(t *testing.T) { + t.Run("no cost budget never trips", func(t *testing.T) { + o, _ := newTestOrchestrator(t) + o.registerCost(PhaseHunt, floatPtr(1000)) + if o.budgetOrTimeoutExhausted(PhaseHunt) { + t.Error("a nil max_cost_usd must never exhaust") + } + }) + + t.Run("total budget trips at >= and wins over the phase limit", func(t *testing.T) { + // The three phase percentages sum to exactly 1.0 (0.10 + 0.45 + 0.45), + // so spending under every phase share also keeps the total under the + // cap — and reaching the cap necessarily saturates a phase too. The + // TOTAL check runs first, so its message is the one Python raises. + o, _ := newTestOrchestrator(t, func(in *schemas.AuditInput) { in.MaxCostUsd = floatPtr(1.0) }) + o.registerCost(PhaseRecon, floatPtr(0.09)) // limit 0.10 + o.registerCost(PhaseHunt, floatPtr(0.44)) // limit 0.45 + o.registerCost(PhaseProve, floatPtr(0.44)) // limit 0.45 + if o.budgetOrTimeoutExhausted(PhaseHunt) { + t.Error("0.97 total with every phase under its share must not exhaust") + } + + o.registerCost(PhaseRecon, floatPtr(0.01)) + o.registerCost(PhaseHunt, floatPtr(0.01)) + o.registerCost(PhaseProve, floatPtr(0.01)) // total 1.0 + if !o.budgetOrTimeoutExhausted(PhaseHunt) { + t.Error("exactly the cap must exhaust (the comparison is >=)") + } + if !o.BudgetExhausted() { + t.Error("the flag must latch") + } + if err := o.checkCostBudget(PhaseHunt); !IsBudgetExhausted(err) || err.Error() != "Total budget exhausted" { + t.Errorf("err = %v, want `Total budget exhausted`", err) + } + }) + + t.Run("per-phase budget uses the configured percentages", func(t *testing.T) { + o, _ := newTestOrchestrator(t, func(in *schemas.AuditInput) { in.MaxCostUsd = floatPtr(10.0) }) + // recon_budget_pct = 0.10 -> a 1.0 allowance. + o.registerCost(PhaseRecon, floatPtr(0.99)) + if o.budgetOrTimeoutExhausted(PhaseRecon) { + t.Error("0.99 < 1.0 must not exhaust the recon phase") + } + o.registerCost(PhaseRecon, floatPtr(0.01)) + err := o.checkCostBudget(PhaseRecon) + if !IsBudgetExhausted(err) || err.Error() != "recon budget exhausted" { + t.Errorf("err = %v, want `recon budget exhausted`", err) + } + // The other phases are untouched. + if o.budgetOrTimeoutExhausted(PhaseHunt) { + t.Error("the hunt phase must still have budget") + } + }) +} + +// TestBudget_TimeCheck: strictly greater, and only when a duration cap is set. +func TestBudget_TimeCheck(t *testing.T) { + base := time.Now() + restore := nowMonotonic + defer func() { nowMonotonic = restore }() + + o, _ := newTestOrchestrator(t, func(in *schemas.AuditInput) { + limit := 10 + in.MaxDurationSeconds = &limit + }) + o.StartedAt = base + + nowMonotonic = func() time.Time { return base.Add(10 * time.Second) } + if err := o.checkTimeBudget(); err != nil { + t.Errorf("exactly the limit must NOT trip (>, not >=): %v", err) + } + nowMonotonic = func() time.Time { return base.Add(10*time.Second + time.Millisecond) } + err := o.checkTimeBudget() + if !IsBudgetExhausted(err) || err.Error() != "Duration budget exhausted" { + t.Errorf("err = %v, want `Duration budget exhausted`", err) + } + + unbounded, _ := newTestOrchestrator(t) + unbounded.StartedAt = base + if err := unbounded.checkTimeBudget(); err != nil { + t.Errorf("a nil max_duration_seconds must never trip: %v", err) + } +} + +// TestRegisterCost reproduces the None/negative guard and the double +// accumulation. +func TestRegisterCost(t *testing.T) { + o, _ := newTestOrchestrator(t) + o.registerCost(PhaseHunt, nil) + o.registerCost(PhaseHunt, floatPtr(-1)) + if o.TotalCostUSD() != 0 { + t.Errorf("total = %v, want 0 after a nil and a negative cost", o.TotalCostUSD()) + } + o.registerCost(PhaseHunt, floatPtr(0)) // exactly zero IS registered + o.registerCost(PhaseHunt, floatPtr(0.25)) + o.registerCost(PhaseProve, floatPtr(0.5)) + if got := o.TotalCostUSD(); got != 0.75 { + t.Errorf("total = %v, want 0.75", got) + } + if got := o.CostBreakdown(); got["hunt"] != 0.25 || got["prove"] != 0.5 || got["recon"] != 0 { + t.Errorf("breakdown = %v", got) + } +} + +// --------------------------------------------------------------------------- +// _PhaseHarnessProxy +// --------------------------------------------------------------------------- + +// TestPhaseProxy_CountsAndCosts is the happy path of the Python class. +func TestPhaseProxy_CountsAndCosts(t *testing.T) { + o, fake := newTestOrchestrator(t) + fake.HarnessFn = func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{CostUSD: floatPtr(0.125)}, nil + } + + proxy := o.PhaseProxy(PhaseHunt) + for i := 0; i < 3; i++ { + if _, err := proxy.Harness(context.Background(), "p", nil, nil, appx.HarnessOptions{}); err != nil { + t.Fatalf("Harness: %v", err) + } + } + if o.AgentInvocations() != 3 { + t.Errorf("AgentInvocations = %d, want 3", o.AgentInvocations()) + } + if got := o.TotalCostUSD(); got != 0.375 { + t.Errorf("total cost = %v, want 0.375", got) + } + if got := o.CostBreakdown()["hunt"]; got != 0.375 { + t.Errorf("hunt bucket = %v, want 0.375", got) + } + if got := o.CostBreakdown()["recon"]; got != 0 { + t.Errorf("recon bucket = %v, want 0", got) + } +} + +// TestPhaseProxy_BudgetExhaustion: the guard fires BEFORE the wrapped harness, +// so a spent phase makes no provider call at all. +func TestPhaseProxy_BudgetExhaustion(t *testing.T) { + o, fake := newTestOrchestrator(t, func(in *schemas.AuditInput) { in.MaxCostUsd = floatPtr(0.2) }) + fake.HarnessFn = func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{CostUSD: floatPtr(0.5)}, nil + } + + proxy := o.PhaseProxy(PhaseProve) + if _, err := proxy.Harness(context.Background(), "first", nil, nil, appx.HarnessOptions{}); err != nil { + t.Fatalf("the first call must succeed: %v", err) + } + _, err := proxy.Harness(context.Background(), "second", nil, nil, appx.HarnessOptions{}) + if !IsBudgetExhausted(err) { + t.Fatalf("err = %v, want a BudgetExhaustedError", err) + } + // Python raises f"{phase} budget exhausted" whichever check tripped. + if err.Error() != "prove budget exhausted" { + t.Errorf("message = %q, want `prove budget exhausted`", err.Error()) + } + if len(fake.Harnesses) != 1 { + t.Errorf("harness calls = %d, want 1 (the second is blocked before the provider)", len(fake.Harnesses)) + } + if o.AgentInvocations() != 1 { + t.Errorf("AgentInvocations = %d, want 1", o.AgentInvocations()) + } + if !o.BudgetExhausted() { + t.Error("the flag must latch") + } +} + +// TestPhaseProxy_TransportErrorSkipsAccounting matches Python, where the raised +// `await` never reaches the counter or the cost registration. +func TestPhaseProxy_TransportErrorSkipsAccounting(t *testing.T) { + o, fake := newTestOrchestrator(t) + fake.HarnessFn = func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return nil, errors.New("provider unreachable") + } + if _, err := o.PhaseProxy(PhaseRecon).Harness(context.Background(), "p", nil, nil, appx.HarnessOptions{}); err == nil { + t.Fatal("want the transport error") + } + if o.AgentInvocations() != 0 || o.TotalCostUSD() != 0 { + t.Errorf("a failed harness must not be counted or costed (%d, %v)", o.AgentInvocations(), o.TotalCostUSD()) + } +} + +// TestPhaseProxy_ExposesOnlyHarness pins the Python class's capability +// surface. `_PhaseHarnessProxy` (orchestrator.py:39) defines `harness` and +// nothing else — VERIFIED on the repo interpreter: +// +// hasattr(proxy, "ai") -> False +// hasattr(proxy, "note") -> False +// hasattr(proxy, "call") -> False +// +// The dedup pass reads exactly that (`hasattr(app, "ai")`, agents/dedup.py:150), +// so a proxy that satisfied appx.AIer would make the Go node issue AI duplicate +// checks Python never issues. +func TestPhaseProxy_ExposesOnlyHarness(t *testing.T) { + o, _ := newTestOrchestrator(t) + + for _, phase := range []string{PhaseRecon, PhaseHunt, PhaseProve} { + proxy := o.PhaseProxy(phase) + if _, ok := proxy.(appx.AIer); ok { + t.Errorf("%s proxy satisfies appx.AIer; Python's hasattr(proxy, \"ai\") is False", phase) + } + if _, ok := proxy.(appx.Noter); ok { + t.Errorf("%s proxy satisfies appx.Noter; Python's hasattr(proxy, \"note\") is False", phase) + } + if _, ok := proxy.(appx.Caller); ok { + t.Errorf("%s proxy satisfies appx.Caller; Python's hasattr(proxy, \"call\") is False", phase) + } + } +} + +// TestProvePhaseProxy_AIRaisesPythonsAttributeError: run_prove's verdict +// sub-agent calls `await app.ai(...)` unguarded (agents/prove/verdict.py:99), +// which against `_PhaseHarnessProxy` raises +// +// AttributeError: '_PhaseHarnessProxy' object has no attribute 'ai' +// +// The Go seam returns that error verbatim instead, and — like an attribute +// lookup that never happened — never reaches the wrapped app and never counts +// or costs anything. +func TestProvePhaseProxy_AIRaisesPythonsAttributeError(t *testing.T) { + o, fake := newTestOrchestrator(t) + fake.AIFn = appx.AIJSON(func(string) (json.RawMessage, error) { return json.RawMessage(`{"ok":true}`), nil }) + + resp, err := o.ProvePhaseProxy().AI(context.Background(), "hello") + if err == nil { + t.Fatal("AI must fail: Python's proxy has no `ai` attribute") + } + if resp != nil { + t.Errorf("resp = %v, want nil", resp) + } + const want = "'_PhaseHarnessProxy' object has no attribute 'ai'" + if err.Error() != want { + t.Errorf("message = %q, want %q", err.Error(), want) + } + var attrErr *AttributeError + if !errors.As(err, &attrErr) { + t.Errorf("err %T is not an *AttributeError", err) + } + if len(fake.AIs) != 0 { + t.Errorf("the wrapped app was reached %d times, want 0", len(fake.AIs)) + } + if o.AgentInvocations() != 0 || o.TotalCostUSD() != 0 { + t.Error("a nonexistent attribute must not be counted or costed") + } +} + +// TestRunProve_DemotesEveryFindingBecauseTheProxyHasNoAI is the observable +// consequence, at the level `audit(resume_from_checkpoint=...)` reaches it. +// +// Python: `_run_prove` hands run_prove the proxy; run_verifier's tracer, +// sanitization and exploit stages succeed through `.harness`, then +// `run_verdict_agent` raises AttributeError, which `_verify`'s +// `except BaseException` (agents/prove/__init__.py:89) turns into +// `verifier_fallback(finding, str(exc), drop_reason="verifier_error")`. So +// EVERY finding comes back INCONCLUSIVE / STATIC_MATCH / score 0.0 with +// tags ["low_confidence"], whatever the harness said. +func TestRunProve_DemotesEveryFindingBecauseTheProxyHasNoAI(t *testing.T) { + o, fake := newTestOrchestrator(t) + fake.HarnessFn = appx.HarnessJSON(func(prompt string, _ appx.HarnessOptions) (json.RawMessage, error) { + switch { + case strings.Contains(prompt, "You are DataFlowTracer"): + return json.Marshal(map[string]any{ + "source": "req.query.id", "sink": "db.query", "steps": []string{"a.go:1"}, "sink_reached": true, + }) + case strings.Contains(prompt, "You are SanitizationAnalyzer"): + return json.Marshal(map[string]any{"found": false}) + case strings.Contains(prompt, "You are ExploitHypothesizer"): + return json.Marshal(map[string]any{ + "hypothesis": "h", "payload": "p", "expected_outcome": "o", + }) + } + return nil, errors.New("unexpected harness prompt") + }) + // A working `.ai()` on the WRAPPED app: the proxy must not reach it. + fake.AIFn = appx.AIJSON(func(string) (json.RawMessage, error) { + return json.Marshal(map[string]any{ + "verdict": "confirmed", "evidence_level": 6, "rationale": "r", "confidence": "high", + }) + }) + + finding := schemas.RawFinding{ + ID: "f1", Fingerprint: "fp1", HunterStrategy: "injection", + Title: "SQLi", Description: "d", FindingType: schemas.FindingTypeSast, + CweID: "CWE-89", CweName: "SQL Injection", + FilePath: "app.go", StartLine: 10, EndLine: 12, CodeSnippet: "q", + EstimatedSeverity: schemas.SeverityHigh, Confidence: schemas.ConfidenceHigh, + } + hunt := schemas.NewHuntResult() + hunt.Findings = []schemas.RawFinding{finding} + + verified, err := o.RunProve(context.Background(), schemas.NewReconResult(), hunt) + if err != nil { + t.Fatalf("RunProve: %v", err) + } + if len(verified) != 1 { + t.Fatalf("verified = %d findings, want 1", len(verified)) + } + got := verified[0] + if got.Verdict != schemas.VerdictInconclusive { + t.Errorf("verdict = %q, want %q", got.Verdict, schemas.VerdictInconclusive) + } + if got.EvidenceLevel != schemas.EvidenceLevelStaticMatch { + t.Errorf("evidence_level = %v, want STATIC_MATCH", got.EvidenceLevel) + } + if got.DropReason == nil || *got.DropReason != "verifier_error" { + t.Errorf("drop_reason = %v, want \"verifier_error\"", got.DropReason) + } + // Python's rationale is `f"Verification incomplete: {str(exc)}"`. Go's + // aix.Structured prefixes every `.ai()` failure with + // "aix.Structured[VerdictDecision]: " (a pre-existing, package-wide + // wrapping unrelated to the proxy), so the CPython text is asserted as a + // substring rather than the whole string. + if !strings.HasPrefix(got.Rationale, "Verification incomplete: ") || + !strings.HasSuffix(got.Rationale, "'_PhaseHarnessProxy' object has no attribute 'ai'") { + t.Errorf("rationale = %q, want `Verification incomplete: ...'_PhaseHarnessProxy' object has no attribute 'ai'`", got.Rationale) + } + if len(got.Tags) == 0 || got.Tags[0] != "low_confidence" { + t.Errorf("tags = %v, want low_confidence first", got.Tags) + } + // _assess_reachability_parallel runs on self.app (not the proxy), so the + // ONE recorded .ai() is the reachability gate — never the verdict agent. + if len(fake.AIs) != 1 { + t.Errorf("recorded .ai() calls = %d, want 1 (the reachability gate only)", len(fake.AIs)) + } +} + +// TestPhaseProxy_ConcurrentAccountingIsRaceFree is the Go-only requirement: +// the phases fan out across goroutines that share one proxy. +func TestPhaseProxy_ConcurrentAccountingIsRaceFree(t *testing.T) { + o, fake := newTestOrchestrator(t) + fake.HarnessFn = func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{CostUSD: floatPtr(0.01)}, nil + } + proxy := o.PhaseProxy(PhaseHunt) + + var wg sync.WaitGroup + for i := 0; i < 50; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = proxy.Harness(context.Background(), "p", nil, nil, appx.HarnessOptions{}) + }() + } + wg.Wait() + + if o.AgentInvocations() != 50 { + t.Errorf("AgentInvocations = %d, want 50", o.AgentInvocations()) + } + if got := o.CostBreakdown()["hunt"]; got < 0.49 || got > 0.51 { + t.Errorf("hunt bucket = %v, want ~0.5", got) + } +} + +// --------------------------------------------------------------------------- +// checkpoints +// --------------------------------------------------------------------------- + +// TestCheckpoints_RoundTrip covers write -> read for both payload shapes plus +// the _try_load_cached_recon fallbacks. +func TestCheckpoints_RoundTrip(t *testing.T) { + o, _ := newTestOrchestrator(t) + + if want := filepath.Join(o.CheckpointDir, "checkpoint-recon.json"); o.CheckpointPath("recon") != want { + t.Errorf("CheckpointPath = %q, want %q", o.CheckpointPath("recon"), want) + } + + recon := reconFixture(t, "full") + if err := o.WriteCheckpoint(PhaseRecon, recon); err != nil { + t.Fatalf("WriteCheckpoint: %v", err) + } + back, err := ReadCheckpoint(o, PhaseRecon, phases.BindReconResult) + if err != nil { + t.Fatalf("ReadCheckpoint: %v", err) + } + if !reflect.DeepEqual(normalizeJSON(t, back), normalizeJSON(t, recon)) { + t.Error("recon checkpoint did not round-trip") + } + + verified := []schemas.VerifiedFinding{readJSON[schemas.VerifiedFinding](t, "verified_fixture.json")} + if err := o.WriteCheckpoint(PhaseProve, verified); err != nil { + t.Fatalf("WriteCheckpoint: %v", err) + } + list, err := ReadCheckpointList(o, PhaseProve, phases.BindVerifiedFinding) + if err != nil { + t.Fatalf("ReadCheckpointList: %v", err) + } + if !reflect.DeepEqual(normalizeJSON(t, list), normalizeJSON(t, verified)) { + t.Error("prove checkpoint did not round-trip") + } + + // The directory is created on demand. + if _, err := os.Stat(o.CheckpointDir); err != nil { + t.Errorf("checkpoint dir was not created: %v", err) + } + + // _try_load_cached_recon: present -> value; absent/corrupt -> nil. + if cached := o.TryLoadCachedRecon(); cached == nil { + t.Error("TryLoadCachedRecon must find the checkpoint just written") + } + other, _ := newTestOrchestrator(t) + if cached := other.TryLoadCachedRecon(); cached != nil { + t.Error("TryLoadCachedRecon must return nil when the file is missing") + } + + // `_read_checkpoint(phase, schema)` is `schema(**data)`: it VALIDATES. + // A hand-edited checkpoint whose finding carries an out-of-vocabulary + // severity is a pydantic ValidationError in Python, and _try_load_cached_recon's + // blanket `except Exception` turns a failed validation into None. + corrupt, _ := newTestOrchestrator(t) + if err := os.MkdirAll(corrupt.CheckpointDir, 0o755); err != nil { + t.Fatal(err) + } + body := `{"phase":"prove","created_at":"2026-01-02T03:04:05+00:00","data":[{"title":"malformed"}]}` + if err := os.WriteFile(corrupt.CheckpointPath(PhaseProve), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + if _, err := ReadCheckpointList(corrupt, PhaseProve, phases.BindVerifiedFinding); err == nil { + t.Error("a checkpoint row that is not a VerifiedFinding must fail, not bind to defaults") + } + reconBody := `{"phase":"recon","created_at":"2026-01-02T03:04:05+00:00","data":{"languages":["go"]}}` + if err := os.WriteFile(corrupt.CheckpointPath(PhaseRecon), []byte(reconBody), 0o644); err != nil { + t.Fatal(err) + } + if _, err := ReadCheckpoint(corrupt, PhaseRecon, phases.BindReconResult); err == nil { + t.Error("a recon checkpoint missing its five required models must fail") + } + if cached := corrupt.TryLoadCachedRecon(); cached != nil { + t.Error("_try_load_cached_recon swallows a failed validation and returns None") + } + if err := os.MkdirAll(other.CheckpointDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(other.CheckpointPath(PhaseRecon), []byte("{not json"), 0o644); err != nil { + t.Fatal(err) + } + if cached := other.TryLoadCachedRecon(); cached != nil { + t.Error("TryLoadCachedRecon must return nil for a corrupt file") + } +} + +// --------------------------------------------------------------------------- +// _assess_reachability_parallel +// --------------------------------------------------------------------------- + +// gateAI is an AIer that answers the reachability gate, records the prompts, and +// stalls so the semaphore bound is observable. +type gateAI struct { + mu sync.Mutex + prompts []string + inflight, peak int + reachability string + fail bool + delay time.Duration + promptsToObserve *[]string +} + +func (g *gateAI) AI(_ context.Context, prompt string, _ ...ai.Option) (*ai.Response, error) { + g.mu.Lock() + g.prompts = append(g.prompts, prompt) + g.inflight++ + if g.inflight > g.peak { + g.peak = g.inflight + } + g.mu.Unlock() + if g.delay > 0 { + time.Sleep(g.delay) + } + defer func() { + g.mu.Lock() + g.inflight-- + g.mu.Unlock() + }() + if g.fail { + return nil, errors.New("gate down") + } + body := `{"reachability":"` + g.reachability + `","rationale":"r","confidence":"high"}` + return &ai.Response{Choices: []ai.Choice{{Message: ai.Message{ + Role: "assistant", Content: []ai.ContentPart{{Type: "text", Text: body}}, + }}}}, nil +} + +func (g *gateAI) peakConcurrency() int { + g.mu.Lock() + defer g.mu.Unlock() + return g.peak +} + +func makeVerified(t *testing.T, n int, tags ...string) []schemas.VerifiedFinding { + t.Helper() + base := readJSON[schemas.VerifiedFinding](t, "verified_fixture.json") + out := make([]schemas.VerifiedFinding, 0, n) + for i := 0; i < n; i++ { + f := base + f.ID = base.ID + "-" + string(rune('a'+i)) + // `[]string{}`, never nil: pydantic's `tags: list[str] = Field( + // default_factory=list)` can never dump `null`, and a nil slice here + // would marshal a checkpoint the Go node's own BindVerifiedFinding + // rejects (as `VerifiedFinding(tags=None)` does in Python). + f.Tags = append([]string{}, tags...) + out = append(out, f) + } + return out +} + +func TestAssessReachabilityParallel(t *testing.T) { + t.Run("nothing to do when every finding is tagged", func(t *testing.T) { + o, _ := newTestOrchestrator(t) + gate := &gateAI{reachability: "externally_reachable"} + o.AIGate.App = gate + verified := makeVerified(t, 3, "requires_auth") + o.AssessReachabilityParallel(context.Background(), verified) + if len(gate.prompts) != 0 { + t.Errorf("gate called %d times, want 0", len(gate.prompts)) + } + for _, f := range verified { + if len(f.Tags) != 1 { + t.Errorf("tags = %v, want the original single tag", f.Tags) + } + } + }) + + t.Run("the gate answer is appended verbatim", func(t *testing.T) { + o, _ := newTestOrchestrator(t) + gate := &gateAI{reachability: "internal_only"} + o.AIGate.App = gate + verified := makeVerified(t, 2) + o.AssessReachabilityParallel(context.Background(), verified) + for _, f := range verified { + if !reflect.DeepEqual(f.Tags, []string{"internal_only"}) { + t.Errorf("tags = %v, want [internal_only]", f.Tags) + } + } + if len(gate.prompts) != 2 { + t.Fatalf("gate called %d times, want 2", len(gate.prompts)) + } + if want := ReachabilitySummary(verified[0]); !strings.Contains(gate.prompts[0], "Finding: ") { + t.Errorf("prompt %q does not embed the summary %q", gate.prompts[0], want) + } + }) + + t.Run("gate failure falls back to requires_auth", func(t *testing.T) { + o, _ := newTestOrchestrator(t) + // Retries would make the test slow; one attempt is enough to observe + // the fallback, so drop the backoff to zero. + o.AIGate.Config.MaxRetries = 1 + o.AIGate.App = &gateAI{fail: true} + verified := makeVerified(t, 2) + o.AssessReachabilityParallel(context.Background(), verified) + for _, f := range verified { + if !reflect.DeepEqual(f.Tags, []string{"requires_auth"}) { + t.Errorf("tags = %v, want [requires_auth]", f.Tags) + } + } + }) + + t.Run("semaphore is min(5, n)", func(t *testing.T) { + o, _ := newTestOrchestrator(t) + gate := &gateAI{reachability: "unreachable", delay: 25 * time.Millisecond} + o.AIGate.App = gate + o.AssessReachabilityParallel(context.Background(), makeVerified(t, 12)) + if got := gate.peakConcurrency(); got > 5 { + t.Errorf("peak concurrency = %d, want <= 5", got) + } + if got := gate.peakConcurrency(); got != 5 { + t.Errorf("peak concurrency = %d, want exactly 5", got) + } + }) + + t.Run("fewer findings than the limit", func(t *testing.T) { + o, _ := newTestOrchestrator(t) + gate := &gateAI{reachability: "unreachable", delay: 20 * time.Millisecond} + o.AIGate.App = gate + o.AssessReachabilityParallel(context.Background(), makeVerified(t, 2)) + if got := gate.peakConcurrency(); got > 2 { + t.Errorf("peak concurrency = %d, want <= 2", got) + } + }) +} + +// --------------------------------------------------------------------------- +// _track_drop / drop summary +// --------------------------------------------------------------------------- + +func TestTrackDrop(t *testing.T) { + o, fake := newTestOrchestrator(t) + o.TrackDrop(context.Background(), "First", nil, "verifier_error") + o.TrackDrop(context.Background(), "Second", strPtr("unverified"), "verdict_unverified") + o.TrackDrop(context.Background(), "Third", strPtr(""), "verifier_error") + + summary := dropSummaryMap(t, o) + if got := summary["demoted_total"]; got != 3 { + t.Errorf("demoted_total = %v, want 3", got) + } + want := map[string]int{"verifier_error": 2, "verdict_unverified": 1} + if got := summary["by_reason"]; !reflect.DeepEqual(got, want) { + t.Errorf("by_reason = %v, want %v", got, want) + } + entries := summary["findings"].([]map[string]any) + if len(entries) != 3 { + t.Fatalf("findings = %d, want 3", len(entries)) + } + if entries[0]["original_verdict"] != nil { + t.Errorf("entries[0].original_verdict = %v, want null", entries[0]["original_verdict"]) + } + if entries[1]["original_verdict"] != "unverified" { + t.Errorf("entries[1].original_verdict = %v", entries[1]["original_verdict"]) + } + + wantNotes := []string{ + "Demoted finding 'First' (verdict=unknown): verifier_error", + "Demoted finding 'Second' (verdict=unverified): verdict_unverified", + // PYTHON TRUTHINESS: an EMPTY verdict also prints "unknown". + "Demoted finding 'Third' (verdict=unknown): verifier_error", + } + if !reflect.DeepEqual(fake.NoteMessages(), wantNotes) { + t.Errorf("notes\n got: %q\nwant: %q", fake.NoteMessages(), wantNotes) + } + // The orchestrator's tags differ from phases' ["prove","drop","demotion"]. + for _, note := range fake.Notes { + if !reflect.DeepEqual(note.Tags, []string{"audit", "prove", "drop"}) { + t.Errorf("tags = %v, want [audit prove drop]", note.Tags) + } + } +} + +// TestRebuildDropSummary is the sweep both run() and _run_prove() perform. +func TestRebuildDropSummary(t *testing.T) { + o, fake := newTestOrchestrator(t) + verified := makeVerified(t, 3) + verified[0].DropReason = strPtr("verifier_error") + verified[1].DropReason = strPtr("") // falsy -> skipped + verified[2].DropReason = nil + + o.rebuildDropSummary(context.Background(), verified) + + if got := dropSummaryMap(t, o)["demoted_total"]; got != 1 { + t.Errorf("demoted_total = %v, want 1 (an empty drop_reason is falsy)", got) + } + if len(fake.Notes) != 1 { + t.Errorf("notes = %d, want 1", len(fake.Notes)) + } +} + +// --------------------------------------------------------------------------- +// _run_dast_verification +// --------------------------------------------------------------------------- + +// TestRunDASTVerification_IsUnreachableInProduction pins the two independent +// reasons DAST never runs, and the behavior of the loop when it is forced. +func TestRunDASTVerification_IsUnreachableInProduction(t *testing.T) { + if enableDast { + t.Fatal("enableDast must be false: AuditInput has no `enable_dast` field, so getattr(...) is always False") + } + + t.Run("no confirmed findings", func(t *testing.T) { + o, fake := newTestOrchestrator(t) + verified := makeVerified(t, 2) + for i := range verified { + verified[i].Verdict = schemas.VerdictInconclusive + } + o.RunDASTVerification(context.Background(), verified) + wantNotes := []string{"No confirmed findings available for DAST step"} + if !reflect.DeepEqual(fake.NoteMessages(), wantNotes) { + t.Errorf("notes = %q, want %q", fake.NoteMessages(), wantNotes) + } + }) + + t.Run("the production seam always raises the arity TypeError", func(t *testing.T) { + o, fake := newTestOrchestrator(t) + verified := makeVerified(t, 2) // the fixture verdict is "confirmed" + o.RunDASTVerification(context.Background(), verified) + for i, f := range verified { + if !contains(f.Tags, "dast_error") { + t.Errorf("verified[%d].tags = %v, want dast_error", i, f.Tags) + } + } + if len(fake.Notes) != 2 { + t.Fatalf("notes = %d, want one per confirmed finding", len(fake.Notes)) + } + for _, note := range fake.Notes { + if !strings.HasPrefix(note.Message, "DAST verifier failed for '") { + t.Errorf("note = %q", note.Message) + } + if !strings.Contains(note.Message, "missing 2 required positional arguments") { + t.Errorf("note %q must carry the arity TypeError", note.Message) + } + if !reflect.DeepEqual(note.Tags, []string{"audit", "prove", "dast", "error"}) { + t.Errorf("tags = %v", note.Tags) + } + } + }) + + t.Run("the ported loop body, with the seam substituted", func(t *testing.T) { + restore := dastVerify + dastVerify = func(context.Context, appx.Harnesser, string, schemas.VerifiedFinding) (dastOutcome, error) { + return dastOutcome{ + ExploitAttempted: true, + ExploitSucceeded: false, + Evidence: "no reflection observed", + Confidence: "medium", + ResponseAnalysis: "the payload was rejected", + }, nil + } + defer func() { dastVerify = restore }() + + o, _ := newTestOrchestrator(t) + verified := makeVerified(t, 1) + proof := schemas.Proof{} + verified[0].Proof = &proof + verified[0].Rationale = "base rationale" + + o.RunDASTVerification(context.Background(), verified) + + if want := []string{"dast_attempted", "dast_not_confirmed"}; !reflect.DeepEqual(verified[0].Tags, want) { + t.Errorf("tags = %v, want %v", verified[0].Tags, want) + } + if want := "base rationale\nDAST: the payload was rejected"; verified[0].Rationale != want { + t.Errorf("rationale = %q, want %q", verified[0].Rationale, want) + } + if verified[0].Proof.PocExecutionOutput == nil { + t.Fatal("poc_execution_output must be filled in") + } + // json.dumps({...}, indent=2) over a dict literal: INSERTION order. + want := "{\n \"exploit_attempted\": true,\n \"exploit_succeeded\": false,\n" + + " \"evidence\": \"no reflection observed\",\n \"confidence\": \"medium\"\n}" + if got := *verified[0].Proof.PocExecutionOutput; got != want { + t.Errorf("poc_execution_output\n got: %q\nwant: %q", got, want) + } + }) +} + +// --------------------------------------------------------------------------- +// run_from_checkpoint +// --------------------------------------------------------------------------- + +// TestRunFromCheckpoint_UnknownPhase pins the ValueError text app.py turns into +// HTTP 400 — including that the message carries the ORIGINAL spelling. +func TestRunFromCheckpoint_UnknownPhase(t *testing.T) { + o, _ := newTestOrchestrator(t) + for _, phase := range []string{"bogus", "Recon!", "", "hunt2"} { + _, err := o.RunFromCheckpoint(context.Background(), phase) + if err == nil { + t.Fatalf("phase %q: want an error", phase) + } + var target *UnknownCheckpointPhaseError + if !errors.As(err, &target) { + t.Fatalf("phase %q: err = %v, want UnknownCheckpointPhaseError", phase, err) + } + if want := "Unknown checkpoint phase: " + phase; err.Error() != want { + t.Errorf("message = %q, want %q", err.Error(), want) + } + } +} + +// TestRunFromCheckpoint_ProveBranch re-runs nothing: it reads all three +// checkpoints and goes straight to GenerateOutput. +func TestRunFromCheckpoint_ProveBranch(t *testing.T) { + compliance.ClearAICache() + o, fake := newTestOrchestrator(t) + fake.AIFn = appx.AIJSON(func(string) (json.RawMessage, error) { + return json.RawMessage(`{"mappings":[],"confidence":"low"}`), nil + }) + + recon := reconFixture(t, "minimal") + hunt := schemas.NewHuntResult() + hunt.TotalRaw = 4 + hunt.StrategiesRun = []string{"injection", "auth"} + verified := makeVerified(t, 2) + + for _, w := range []struct { + phase string + payload any + }{{PhaseRecon, recon}, {PhaseHunt, hunt}, {PhaseProve, verified}} { + if err := o.WriteCheckpoint(w.phase, w.payload); err != nil { + t.Fatalf("WriteCheckpoint(%s): %v", w.phase, err) + } + } + + for _, spelling := range []string{"prove", " PROVE ", "Prove"} { + result, err := o.RunFromCheckpoint(context.Background(), spelling) + if err != nil { + t.Fatalf("RunFromCheckpoint(%q): %v", spelling, err) + } + if result.TotalRawFindings != 4 { + t.Errorf("total_raw_findings = %d, want 4", result.TotalRawFindings) + } + if !reflect.DeepEqual(result.StrategiesUsed, []string{"injection", "auth"}) { + t.Errorf("strategies_used = %v", result.StrategiesUsed) + } + if len(result.Findings) != 2 { + t.Errorf("findings = %d, want 2", len(result.Findings)) + } + } +} + +// TestRunFromCheckpoint_MissingCheckpoint surfaces the file error rather than +// silently producing an empty report. +func TestRunFromCheckpoint_MissingCheckpoint(t *testing.T) { + o, _ := newTestOrchestrator(t) + if _, err := o.RunFromCheckpoint(context.Background(), "prove"); err == nil { + t.Fatal("want an error when checkpoint-recon.json is missing") + } +} + +// dropSummaryMap reads ProveDropSummary as the dict the orchestrator path +// always installs. The field is typed `any` so the app.py path can thread a +// non-dict `.call` value through untouched (see orch.go); every orchestrator +// caller still puts a real dict there. +func dropSummaryMap(t *testing.T, o *AuditOrchestrator) map[string]any { + t.Helper() + summary, ok := o.ProveDropSummary.(map[string]any) + if !ok { + t.Fatalf("ProveDropSummary = %#v, want a map[string]any", o.ProveDropSummary) + } + return summary +} + +// TestGuardedCountersAreSafeToReadWhileAPhaseIsRunning states the lock contract +// as behaviour rather than as a comment. +// +// Validation contract: +// +// - a caller may read agent_invocations / budget_exhausted / total_cost_usd / +// cost_breakdown AT ANY TIME, including while phase goroutines sharing one +// proxy are still incrementing them; +// - the values are the Python ones once the phases have joined. +// +// Python needs no lock (asyncio is single-threaded) and so has no equivalent +// test; this is the obligation the Go fan-out creates. Under -race it fails if +// any of the four accessors, or the writers behind registerInvocation / +// markBudgetExhausted, touches the state unguarded — which is exactly what the +// bare `o.AgentInvocations` and `o.BudgetExhausted` field reads inside +// GenerateOutput used to do. +func TestGuardedCountersAreSafeToReadWhileAPhaseIsRunning(t *testing.T) { + o, fake := newTestOrchestrator(t) + fake.HarnessFn = func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{CostUSD: floatPtr(0.01)}, nil + } + proxy := o.PhaseProxy(PhaseHunt) + + const writers, perWriter = 4, 25 + stop := make(chan struct{}) + + var reader sync.WaitGroup + reader.Add(1) + go func() { + defer reader.Done() + for { + select { + case <-stop: + return + default: + } + // The four reads GenerateOutput performs. + _ = o.AgentInvocations() + _ = o.BudgetExhausted() + _ = o.TotalCostUSD() + _ = o.CostBreakdown() + } + }() + + var phases sync.WaitGroup + phases.Add(writers) + for i := 0; i < writers; i++ { + go func() { + defer phases.Done() + for j := 0; j < perWriter; j++ { + if _, err := proxy.Harness(context.Background(), "p", nil, nil, appx.HarnessOptions{}); err != nil { + t.Error(err) + return + } + } + o.markBudgetExhausted() + }() + } + phases.Wait() + close(stop) + reader.Wait() + + if got := o.AgentInvocations(); got != writers*perWriter { + t.Errorf("AgentInvocations = %d, want %d", got, writers*perWriter) + } + if !o.BudgetExhausted() { + t.Error("BudgetExhausted = false, want true") + } +} diff --git a/internal/secaf/orch/output.go b/internal/secaf/orch/output.go new file mode 100644 index 0000000000..9cfc35365d --- /dev/null +++ b/internal/secaf/orch/output.go @@ -0,0 +1,235 @@ +package orch + +import ( + "context" + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/secaf/compliance" + "github.com/Agent-Field/codeaf/internal/secaf/gates" + "github.com/Agent-Field/codeaf/internal/secaf/output" + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" + "github.com/Agent-Field/codeaf/internal/secaf/scoring" +) + +// severityOrder ports the local `severity_order` map `_generate_output` uses +// for the threshold filter. Note "info" scores 0, which is why a threshold of +// "info" (or an unrecognised threshold) disables filtering entirely. +var severityOrder = map[string]int{ + "critical": 4, + "high": 3, + "medium": 2, + "low": 1, + "info": 0, +} + +// GenerateOutput ports `_generate_output(recon, hunt, verified)` +// (orchestrator.py:365) — the tail every entry point shares. app.py calls it +// directly after its four `.call`s; run() and run_from_checkpoint() call it +// last. +// +// Steps, in Python's order: +// +// 1. THRESHOLD FILTER. `severity_order.get(self.input.severity_threshold.lower(), 0)`; +// when that is > 0, drop every finding scoring below it. A threshold of +// "info" or an unknown string scores 0 and filters NOTHING. +// 2. PER-FINDING ENRICHMENT, in place: the CWE severity floor, the +// exploitability score, sarif_security_severity mirrored from it, and the +// compliance mappings (static table, with the AI gate as fallback). +// 3. COUNTS. verdict counts over the four Verdict members; severity counts +// seeded with every Severity member so a severity with no findings still +// reports 0. +// 4. NOISE REDUCTION. `not_exploitable / hunt.total_raw * 100`, rounded to 2 +// with Python's banker's rounding. Guarded on total_raw > 0. +// 5. ATTACK CHAINS. PotentialChain -> AttackChain, with `description` taking +// the chain's combined_impact (so description and combined_impact are the +// same string) and mitre_attack_mapping left at None. +// 6. COMPLIANCE GAPS, the SecurityAuditResult, then SARIF, then the JSON and +// Markdown reports whose return values are DISCARDED (`_ = generate_json(...)`) +// — they are produced for their side-effect-free validation only. +// 7. The compliance-report loop is NOT carried: see the end of the function. +// +// Python parity details worth stating: +// +// - `verified` is REBOUND by the filter, so the caller's slice is not +// truncated — but the surviving findings are the SAME objects and step 2 +// mutates them. Go reproduces both: a new slice header, sharing elements. +// - `self.input.compliance_frameworks or None` passes None for an EMPTY list, +// which makes get_compliance_mappings_hybrid use its default framework set. +// - `commit_sha=self.input.commit_sha or "HEAD"` is Python truthiness: an +// empty string also becomes "HEAD". +// - `timestamp=datetime.now(UTC)` is read here, not at construction. +// - `duration_seconds` is `time.monotonic() - self.started_at`, i.e. the whole +// orchestrator lifetime, not just this function. +// - the budget note fires only when the flag is latched, and reports +// findings_not_verified — which app.py copies from prove_phase's payload. +func (o *AuditOrchestrator) GenerateOutput( + ctx context.Context, + recon schemas.ReconResult, + hunt schemas.HuntResult, + verified []schemas.VerifiedFinding, +) (schemas.SecurityAuditResult, error) { + _ = recon // Python: `_ = recon` — accepted for symmetry, never read. + + // 1. severity threshold + thresholdValue := severityOrder[strings.ToLower(o.Input.SeverityThreshold)] + if thresholdValue > 0 { + filtered := make([]schemas.VerifiedFinding, 0, len(verified)) + for _, finding := range verified { + if severityOrder[strings.ToLower(string(finding.Severity))] >= thresholdValue { + filtered = append(filtered, finding) + } + } + verified = filtered + } + + // 2. per-finding enrichment + frameworks := o.Input.ComplianceFrameworks + if len(frameworks) == 0 { + // Python: `frameworks=self.input.compliance_frameworks or None` + frameworks = nil + } + gate := o.complianceGate() + for i := range verified { + verified[i].Severity = scoring.ApplyCWESeverityFloor(verified[i].CweID, verified[i].Severity) + verified[i].ExploitabilityScore = scoring.ComputeExploitabilityScore(verified[i]) + verified[i].SarifSecuritySeverity = verified[i].ExploitabilityScore + verified[i].Compliance = compliance.GetComplianceMappingsHybrid(ctx, verified[i].CweID, frameworks, gate) + } + + // 3. counts + verdictCounts := map[schemas.ExploitVerdict]int{ + schemas.VerdictConfirmed: 0, + schemas.VerdictLikely: 0, + schemas.VerdictInconclusive: 0, + schemas.VerdictNotExploitable: 0, + } + severityCounts := map[string]int{} + for _, severity := range schemas.AllSeverities { + severityCounts[string(severity)] = 0 + } + for _, finding := range verified { + // Python: `verdict_counts[finding.verdict] += 1` — a dict subscript, + // which raises KeyError for a verdict outside the four members. The + // Verdict enum has exactly four, so the Go map write is equivalent. + verdictCounts[finding.Verdict]++ + severityCounts[string(finding.Severity)]++ + } + + // 4. noise reduction + totalRaw := hunt.TotalRaw + notExploitable := verdictCounts[schemas.VerdictNotExploitable] + noiseReduction := 0.0 + if totalRaw > 0 { + noiseReduction = float64(notExploitable) / float64(totalRaw) * 100.0 + } + + // 5. attack chains + chains := make([]schemas.AttackChain, 0, len(hunt.Chains)) + for _, chain := range hunt.Chains { + chains = append(chains, schemas.AttackChain{ + ChainID: chain.ChainID, + Title: chain.Title, + Description: chain.CombinedImpact, + Findings: chain.FindingIDs, + CombinedSeverity: chain.EstimatedSeverity, + CombinedImpact: chain.CombinedImpact, + // mitre_attack_mapping is not passed, so it keeps pydantic's None. + MitreAttackMapping: nil, + }) + } + + if o.BudgetExhausted() { + o.App.Note(ctx, + "Budget exhausted; unverified findings: "+strconv.Itoa(o.FindingsNotVerified), + "audit", "budget", "exhausted") + } + + // 6. the result + complianceGaps := compliance.GetComplianceGaps(verified) + + commitSha := "HEAD" + if o.Input.CommitSha != nil && *o.Input.CommitSha != "" { + commitSha = *o.Input.CommitSha + } + branch := o.Input.Branch + + costBreakdown := o.CostBreakdown() + roundedBreakdown := make(map[string]float64, len(costBreakdown)) + for phase, cost := range costBreakdown { + roundedBreakdown[phase] = pyfmt.Round(cost, 4) + } + + result := schemas.NewSecurityAuditResult() + result.Repository = o.Input.RepoURL + result.CommitSha = commitSha + result.Branch = &branch + result.Timestamp = schemas.NewTimestamp(nowUTC()) + result.DepthProfile = o.Input.Depth + result.StrategiesUsed = hunt.StrategiesRun + result.Provider = "harness" + result.Findings = verified + result.AttackChains = chains + result.TotalRawFindings = totalRaw + result.Confirmed = verdictCounts[schemas.VerdictConfirmed] + result.Likely = verdictCounts[schemas.VerdictLikely] + result.Inconclusive = verdictCounts[schemas.VerdictInconclusive] + result.NotExploitable = notExploitable + result.NoiseReductionPct = pyfmt.Round(noiseReduction, 2) + result.BySeverity = severityCounts + result.ComplianceGaps = complianceGaps + result.DurationSeconds = o.elapsedSeconds() + result.AgentInvocations = o.AgentInvocations() + result.CostUsd = pyfmt.Round(o.TotalCostUSD(), 4) + result.CostBreakdown = roundedBreakdown + result.Metadata = map[string]any{ + "findings_not_verified": o.FindingsNotVerified, + "prove_drop_summary": o.ProveDropSummary, + } + result.Sarif = "" + + result.Sarif = output.GenerateSarif(result) + // Python: `_ = generate_json(result, pretty=True)` and `_ = generate_report(result)`. + // Both return values are discarded; the calls stay because they exercise the + // same rendering the caller may ask for later, and because dropping them + // would change nothing except which code runs. + _ = output.GenerateJSON(result, true) + _ = output.GenerateReport(result) + + // 7. THE PER-FRAMEWORK REPORT LOOP IS NOT CARRIED. In sec-af it called + // generate_compliance_report(result, framework) against a function that + // takes one argument, so every audit that named a compliance framework + // failed here, after its whole run, with nothing to show (the Python tree's + // TypeError, which the port reproduced on purpose). The frameworks already + // chose the mappings above; codeaf writes the one compliance report beside + // the audit's other files (internal/secaf's report.go), and nothing is + // written into the repository. + return result, nil +} + +// complianceGate adapts the orchestrator's AIGateWrapper to the narrow seam +// compliance.GetComplianceMappingsHybrid needs: +// +// suggestion = await ai_gate.invoke(user=prompt, schema=ComplianceGate) +// +// A nil AIGate yields a nil seam, which is Python's `ai_gate=None` — the static +// table then has no fallback. +func (o *AuditOrchestrator) complianceGate() compliance.AIGateLike { + if o.AIGate == nil { + return nil + } + return compliance.AIGateFunc(func(ctx context.Context, user string) (schemas.ComplianceGate, error) { + return gates.Invoke[schemas.ComplianceGate](ctx, o.AIGate, user, "") + }) +} + +// elapsedSeconds is `time.monotonic() - self.started_at`. +// +// It reads the clock through nowMonotonic (orch.go) rather than time.Since so a +// test — and scripts/gen_golden_phases.py's Python counterpart, which swaps the +// orchestrator module's `time` name — can pin the elapsed value that lands in +// the progress notes and in duration_seconds. +func (o *AuditOrchestrator) elapsedSeconds() float64 { + return nowMonotonic().Sub(o.StartedAt).Seconds() +} diff --git a/internal/secaf/orch/output_test.go b/internal/secaf/orch/output_test.go new file mode 100644 index 0000000000..2c442c5e51 --- /dev/null +++ b/internal/secaf/orch/output_test.go @@ -0,0 +1,448 @@ +package orch + +import ( + "context" + "encoding/json" + "os" + "reflect" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/compliance" + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" + "github.com/Agent-Field/codeaf/internal/secaf/scoring" +) + +// verifiedWith builds a VerifiedFinding from the fixture with the given +// identity, verdict and severity. +func verifiedWith(t *testing.T, id string, verdict schemas.ExploitVerdict, severity schemas.Severity, cwe string) schemas.VerifiedFinding { + t.Helper() + f := readJSON[schemas.VerifiedFinding](t, "verified_fixture.json") + f.ID = id + f.Fingerprint = "fp-" + id + f.Verdict = verdict + f.Severity = severity + f.CweID = cwe + f.Tags = []string{} + return f +} + +// TestGenerateOutput_SeverityThreshold walks the filter, including the two +// spellings that disable it. +func TestGenerateOutput_SeverityThreshold(t *testing.T) { + compliance.ClearAICache() + + all := []schemas.VerifiedFinding{ + verifiedWith(t, "crit", schemas.VerdictConfirmed, schemas.SeverityCritical, "CWE-79"), + verifiedWith(t, "high", schemas.VerdictLikely, schemas.SeverityHigh, "CWE-79"), + verifiedWith(t, "med", schemas.VerdictInconclusive, schemas.SeverityMedium, "CWE-79"), + verifiedWith(t, "low", schemas.VerdictInconclusive, schemas.SeverityLow, "CWE-79"), + verifiedWith(t, "info", schemas.VerdictNotExploitable, schemas.SeverityInfo, "CWE-79"), + } + + cases := []struct { + threshold string + wantIDs []string + }{ + {"critical", []string{"crit"}}, + {"HIGH", []string{"crit", "high"}}, + {"medium", []string{"crit", "high", "med"}}, + {"low", []string{"crit", "high", "med", "low"}}, + // "info" scores 0, so the filter is skipped entirely. + {"info", []string{"crit", "high", "med", "low", "info"}}, + // An unrecognised threshold also scores 0 — no filtering, no error. + {"bogus", []string{"crit", "high", "med", "low", "info"}}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.threshold, func(t *testing.T) { + o, _ := newTestOrchestrator(t, func(in *schemas.AuditInput) { in.SeverityThreshold = tc.threshold }) + verified := append([]schemas.VerifiedFinding(nil), all...) + + result, err := o.GenerateOutput(context.Background(), schemas.NewReconResult(), schemas.NewHuntResult(), verified) + if err != nil { + t.Fatalf("GenerateOutput: %v", err) + } + got := make([]string, 0, len(result.Findings)) + for _, f := range result.Findings { + got = append(got, f.ID) + } + if !reflect.DeepEqual(got, tc.wantIDs) { + t.Errorf("threshold %q kept %v, want %v", tc.threshold, got, tc.wantIDs) + } + }) + } +} + +// TestGenerateOutput_EnrichesEveryFinding pins the four per-finding mutations. +func TestGenerateOutput_EnrichesEveryFinding(t *testing.T) { + compliance.ClearAICache() + o, _ := newTestOrchestrator(t) + + // CWE-89 carries a severity floor, so a "low" finding must come back raised. + finding := verifiedWith(t, "sqli", schemas.VerdictConfirmed, schemas.SeverityLow, "CWE-89") + before := finding + + result, err := o.GenerateOutput(context.Background(), schemas.NewReconResult(), schemas.NewHuntResult(), + []schemas.VerifiedFinding{finding}) + if err != nil { + t.Fatalf("GenerateOutput: %v", err) + } + got := result.Findings[0] + + wantSeverity := scoring.ApplyCWESeverityFloor(before.CweID, before.Severity) + if got.Severity != wantSeverity { + t.Errorf("severity = %q, want the CWE floor %q", got.Severity, wantSeverity) + } + if got.ExploitabilityScore == 0 { + t.Error("exploitability_score must be recomputed, not left at the input's 0") + } + if got.SarifSecuritySeverity != got.ExploitabilityScore { + t.Errorf("sarif_security_severity = %v, want it mirrored from exploitability_score %v", + got.SarifSecuritySeverity, got.ExploitabilityScore) + } + if got.Compliance == nil { + t.Error("compliance must be replaced by the mapping lookup, never left nil") + } + if len(got.Compliance) == 0 { + t.Error("CWE-89 is in the static compliance table; want at least one mapping") + } +} + +// TestGenerateOutput_CountsAndNoiseReduction pins the counters, the seeded +// by_severity map and the rounded percentage. +func TestGenerateOutput_CountsAndNoiseReduction(t *testing.T) { + compliance.ClearAICache() + // AuditInput's default severity_threshold is "low", which scores 1 and + // would drop the "info" finding before the counters ever see it. "info" + // scores 0 and disables the filter, which is what this test needs. + o, _ := newTestOrchestrator(t, func(in *schemas.AuditInput) { in.SeverityThreshold = "info" }) + + verified := []schemas.VerifiedFinding{ + verifiedWith(t, "a", schemas.VerdictConfirmed, schemas.SeverityHigh, "CWE-79"), + verifiedWith(t, "b", schemas.VerdictConfirmed, schemas.SeverityHigh, "CWE-79"), + verifiedWith(t, "c", schemas.VerdictLikely, schemas.SeverityMedium, "CWE-79"), + verifiedWith(t, "d", schemas.VerdictInconclusive, schemas.SeverityLow, "CWE-79"), + verifiedWith(t, "e", schemas.VerdictNotExploitable, schemas.SeverityInfo, "CWE-79"), + } + hunt := schemas.NewHuntResult() + hunt.TotalRaw = 7 + + result, err := o.GenerateOutput(context.Background(), schemas.NewReconResult(), hunt, verified) + if err != nil { + t.Fatalf("GenerateOutput: %v", err) + } + + if result.Confirmed != 2 || result.Likely != 1 || result.Inconclusive != 1 || result.NotExploitable != 1 { + t.Errorf("verdict counts = %d/%d/%d/%d, want 2/1/1/1", + result.Confirmed, result.Likely, result.Inconclusive, result.NotExploitable) + } + if result.TotalRawFindings != 7 { + t.Errorf("total_raw_findings = %d, want 7 (it comes from hunt.total_raw)", result.TotalRawFindings) + } + + // by_severity is seeded with EVERY Severity member, so a severity with no + // findings still reports 0. The CWE-79 floor may raise some entries, so the + // assertion is on the key set plus the total. + if len(result.BySeverity) != len(schemas.AllSeverities) { + t.Errorf("by_severity keys = %v, want one per Severity member", result.BySeverity) + } + total := 0 + for _, severity := range schemas.AllSeverities { + count, present := result.BySeverity[string(severity)] + if !present { + t.Errorf("by_severity is missing %q", severity) + } + total += count + } + if total != len(verified) { + t.Errorf("by_severity sums to %d, want %d", total, len(verified)) + } + + // 1 not_exploitable of 7 raw -> 14.285714...% -> round(x, 2) = 14.29. + if want := pyfmt.Round(1.0/7.0*100.0, 2); result.NoiseReductionPct != want { + t.Errorf("noise_reduction_pct = %v, want %v", result.NoiseReductionPct, want) + } + if want := 14.29; result.NoiseReductionPct != want { + t.Errorf("noise_reduction_pct = %v, want %v", result.NoiseReductionPct, want) + } +} + +// TestGenerateOutput_NoiseReductionGuard: total_raw of 0 yields 0.0, not a +// division by zero. +func TestGenerateOutput_NoiseReductionGuard(t *testing.T) { + compliance.ClearAICache() + o, _ := newTestOrchestrator(t, func(in *schemas.AuditInput) { in.SeverityThreshold = "info" }) + verified := []schemas.VerifiedFinding{ + verifiedWith(t, "a", schemas.VerdictNotExploitable, schemas.SeverityInfo, "CWE-79"), + } + result, err := o.GenerateOutput(context.Background(), schemas.NewReconResult(), schemas.NewHuntResult(), verified) + if err != nil { + t.Fatalf("GenerateOutput: %v", err) + } + if result.NoiseReductionPct != 0 { + t.Errorf("noise_reduction_pct = %v, want 0", result.NoiseReductionPct) + } +} + +// TestGenerateOutput_AttackChains pins the PotentialChain -> AttackChain +// projection, including `description = combined_impact`. +func TestGenerateOutput_AttackChains(t *testing.T) { + compliance.ClearAICache() + o, _ := newTestOrchestrator(t) + + hunt := schemas.NewHuntResult() + hunt.Chains = []schemas.PotentialChain{{ + ChainID: "chain-1", + Title: "SSRF to RCE", + FindingIDs: []string{"a", "b"}, + CombinedImpact: "internal metadata service reachable, then RCE", + EstimatedSeverity: schemas.SeverityCritical, + }} + + result, err := o.GenerateOutput(context.Background(), schemas.NewReconResult(), hunt, nil) + if err != nil { + t.Fatalf("GenerateOutput: %v", err) + } + if len(result.AttackChains) != 1 { + t.Fatalf("attack_chains = %d, want 1", len(result.AttackChains)) + } + chain := result.AttackChains[0] + if chain.ChainID != "chain-1" || chain.Title != "SSRF to RCE" { + t.Errorf("chain identity = %+v", chain) + } + if chain.Description != chain.CombinedImpact { + t.Errorf("description = %q, want it to equal combined_impact %q", chain.Description, chain.CombinedImpact) + } + if !reflect.DeepEqual(chain.Findings, []string{"a", "b"}) { + t.Errorf("findings = %v", chain.Findings) + } + if chain.CombinedSeverity != schemas.SeverityCritical { + t.Errorf("combined_severity = %q", chain.CombinedSeverity) + } + if chain.MitreAttackMapping != nil { + t.Errorf("mitre_attack_mapping = %v, want nil (it is not passed)", chain.MitreAttackMapping) + } +} + +// TestGenerateOutput_ResultEnvelope pins the scalar fields and the metadata. +func TestGenerateOutput_ResultEnvelope(t *testing.T) { + compliance.ClearAICache() + + base := time.Now() + restoreMono := nowMonotonic + nowMonotonic = func() time.Time { return base.Add(3 * time.Second) } + defer func() { nowMonotonic = restoreMono }() + + pinned, _ := time.Parse(time.RFC3339Nano, "2026-01-02T03:04:05.123456Z") + restoreUTC := nowUTC + nowUTC = func() time.Time { return pinned.UTC() } + defer func() { nowUTC = restoreUTC }() + + sha := "deadbeef" + o, _ := newTestOrchestrator(t, func(in *schemas.AuditInput) { + in.RepoURL = "https://example.invalid/acme/api" + in.Branch = "release/1.2" + in.CommitSha = &sha + in.Depth = "thorough" + }) + o.StartedAt = base + o.SetAgentInvocations(17) + o.FindingsNotVerified = 4 + o.registerCost(PhaseRecon, floatPtr(0.123456)) + o.registerCost(PhaseHunt, floatPtr(1.987654)) + o.TrackDrop(context.Background(), "dropped", nil, "verifier_error") + + hunt := schemas.NewHuntResult() + hunt.StrategiesRun = []string{"injection", "auth"} + + result, err := o.GenerateOutput(context.Background(), schemas.NewReconResult(), hunt, nil) + if err != nil { + t.Fatalf("GenerateOutput: %v", err) + } + + if result.Repository != "https://example.invalid/acme/api" { + t.Errorf("repository = %q", result.Repository) + } + if result.CommitSha != "deadbeef" { + t.Errorf("commit_sha = %q", result.CommitSha) + } + if result.Branch == nil || *result.Branch != "release/1.2" { + t.Errorf("branch = %v", result.Branch) + } + if got := result.Timestamp.String(); got != "2026-01-02T03:04:05.123456+00:00" { + t.Errorf("timestamp = %q", got) + } + if result.DepthProfile != "thorough" { + t.Errorf("depth_profile = %q", result.DepthProfile) + } + if result.Provider != "harness" { + t.Errorf("provider = %q, want harness", result.Provider) + } + if !reflect.DeepEqual(result.StrategiesUsed, []string{"injection", "auth"}) { + t.Errorf("strategies_used = %v", result.StrategiesUsed) + } + if result.AgentInvocations != 17 { + t.Errorf("agent_invocations = %d, want 17", result.AgentInvocations) + } + if result.DurationSeconds != 3 { + t.Errorf("duration_seconds = %v, want 3", result.DurationSeconds) + } + // round(total, 4) and round(each phase, 4) — Python's banker's rounding. + if want := pyfmt.Round(0.123456+1.987654, 4); result.CostUsd != want { + t.Errorf("cost_usd = %v, want %v", result.CostUsd, want) + } + wantBreakdown := map[string]float64{ + "recon": pyfmt.Round(0.123456, 4), + "hunt": pyfmt.Round(1.987654, 4), + "prove": 0, + } + if !reflect.DeepEqual(result.CostBreakdown, wantBreakdown) { + t.Errorf("cost_breakdown = %v, want %v", result.CostBreakdown, wantBreakdown) + } + if got := result.Metadata["findings_not_verified"]; got != 4 { + t.Errorf("metadata.findings_not_verified = %v, want 4", got) + } + if got, ok := result.Metadata["prove_drop_summary"].(map[string]any); !ok || got["demoted_total"] != 1 { + t.Errorf("metadata.prove_drop_summary = %#v", result.Metadata["prove_drop_summary"]) + } + if result.Sarif == "" { + t.Error("sarif must be filled in by generate_sarif") + } + var sarif map[string]any + if err := json.Unmarshal([]byte(result.Sarif), &sarif); err != nil { + t.Errorf("sarif is not valid JSON: %v", err) + } + // policy_violations is never passed, so it keeps the pydantic default. + if result.PolicyViolations == nil || len(result.PolicyViolations) != 0 { + t.Errorf("policy_violations = %v, want []", result.PolicyViolations) + } +} + +// TestGenerateOutput_CommitShaFallback: an absent OR empty commit_sha becomes +// "HEAD" (Python truthiness). +func TestGenerateOutput_CommitShaFallback(t *testing.T) { + compliance.ClearAICache() + for _, sha := range []*string{nil, strPtr("")} { + o, _ := newTestOrchestrator(t, func(in *schemas.AuditInput) { in.CommitSha = sha }) + result, err := o.GenerateOutput(context.Background(), schemas.NewReconResult(), schemas.NewHuntResult(), nil) + if err != nil { + t.Fatalf("GenerateOutput: %v", err) + } + if result.CommitSha != "HEAD" { + t.Errorf("commit_sha = %q, want HEAD", result.CommitSha) + } + } +} + +// TestGenerateOutput_BudgetNote fires only when the flag latched. +func TestGenerateOutput_BudgetNote(t *testing.T) { + compliance.ClearAICache() + + t.Run("silent when the budget held", func(t *testing.T) { + o, fake := newTestOrchestrator(t) + if _, err := o.GenerateOutput(context.Background(), schemas.NewReconResult(), schemas.NewHuntResult(), nil); err != nil { + t.Fatalf("GenerateOutput: %v", err) + } + for _, note := range fake.Notes { + if note.Message[:6] == "Budget" { + t.Errorf("unexpected budget note %q", note.Message) + } + } + }) + + t.Run("reports the unverified count when exhausted", func(t *testing.T) { + o, fake := newTestOrchestrator(t) + o.markBudgetExhausted() + o.FindingsNotVerified = 12 + if _, err := o.GenerateOutput(context.Background(), schemas.NewReconResult(), schemas.NewHuntResult(), nil); err != nil { + t.Fatalf("GenerateOutput: %v", err) + } + want := "Budget exhausted; unverified findings: 12" + found := false + for _, note := range fake.Notes { + if note.Message == want { + found = true + if !reflect.DeepEqual(note.Tags, []string{"audit", "budget", "exhausted"}) { + t.Errorf("tags = %v", note.Tags) + } + } + } + if !found { + t.Errorf("missing %q in %q", want, fake.NoteMessages()) + } + }) +} + +// TestGenerateOutput_ComplianceFrameworksDoNotFailTheAudit: naming +// frameworks used to fail every audit at its very end (sec-af's Python +// TypeError, reproduced by the port). The result is whole, and nothing is +// written into the repository's checkpoint folder. +func TestGenerateOutput_ComplianceFrameworksDoNotFailTheAudit(t *testing.T) { + compliance.ClearAICache() + o, _ := newTestOrchestrator(t, func(in *schemas.AuditInput) { + in.ComplianceFrameworks = []string{"SOC2", "PCI-DSS"} + }) + result, err := o.GenerateOutput(context.Background(), schemas.NewReconResult(), schemas.NewHuntResult(), + []schemas.VerifiedFinding{verifiedWith(t, "a", schemas.VerdictConfirmed, schemas.SeverityHigh, "CWE-89")}) + if err != nil { + t.Fatalf("an audit naming frameworks failed: %v", err) + } + if result.Sarif == "" || len(result.Findings) != 1 { + t.Errorf("the result is not whole: %d findings, sarif %d bytes", len(result.Findings), len(result.Sarif)) + } + if _, statErr := os.Stat(o.CheckpointDir); statErr == nil { + t.Errorf("the checkpoint folder %s was made by the output step", o.CheckpointDir) + } +} + +// TestGenerateOutput_ComplianceFrameworksSelectTheMappings: a non-empty list is +// passed through to get_compliance_mappings_hybrid; an empty one becomes None +// so the default framework set applies. +func TestGenerateOutput_ComplianceFrameworksSelectTheMappings(t *testing.T) { + compliance.ClearAICache() + + // Empty list -> None -> every framework the static table knows for CWE-89. + o, _ := newTestOrchestrator(t) + result, err := o.GenerateOutput(context.Background(), schemas.NewReconResult(), schemas.NewHuntResult(), + []schemas.VerifiedFinding{verifiedWith(t, "a", schemas.VerdictConfirmed, schemas.SeverityHigh, "CWE-89")}) + if err != nil { + t.Fatalf("GenerateOutput: %v", err) + } + unfiltered := result.Findings[0].Compliance + if len(unfiltered) == 0 { + t.Fatal("CWE-89 must map to at least one control") + } + want := compliance.GetComplianceMappings("CWE-89", nil) + if !reflect.DeepEqual(unfiltered, want) { + t.Errorf("compliance = %v, want the unfiltered table %v", unfiltered, want) + } +} + +// TestGenerateOutput_AIGateFallbackForUnknownCWE: a CWE the static table does +// not know reaches the AI gate through the orchestrator's adapter. +func TestGenerateOutput_AIGateFallbackForUnknownCWE(t *testing.T) { + compliance.ClearAICache() + defer compliance.ClearAICache() + + o, fake := newTestOrchestrator(t) + fake.AIFn = appx.AIJSON(func(string) (json.RawMessage, error) { + return json.RawMessage(`{"mappings":[{"framework":"SOC2","control_id":"CC9.9","control_name":"Invented"}],"confidence":"low"}`), nil + }) + + result, err := o.GenerateOutput(context.Background(), schemas.NewReconResult(), schemas.NewHuntResult(), + []schemas.VerifiedFinding{verifiedWith(t, "a", schemas.VerdictConfirmed, schemas.SeverityHigh, "CWE-999999")}) + if err != nil { + t.Fatalf("GenerateOutput: %v", err) + } + got := result.Findings[0].Compliance + if len(got) != 1 || got[0].ControlID != "CC9.9" { + t.Errorf("compliance = %v, want the AI gate's single mapping", got) + } + if len(fake.AIs) == 0 { + t.Error("the AI gate must be consulted for an unknown CWE") + } +} diff --git a/internal/secaf/orch/progress.go b/internal/secaf/orch/progress.go new file mode 100644 index 0000000000..08c0cb79a0 --- /dev/null +++ b/internal/secaf/orch/progress.go @@ -0,0 +1,141 @@ +package orch + +import ( + "context" + + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// EmitProgress ports `_emit_progress` (orchestrator.py:636): +// +// elapsed = time.monotonic() - self.started_at +// safe_total = max(1, agents_total) +// phase_progress = min(1.0, agents_completed / safe_total) +// estimated_total = elapsed / phase_progress if phase_progress > 0 else elapsed +// progress = AuditProgress( +// phase=phase, phase_progress=phase_progress, +// agents_total=agents_total, agents_completed=agents_completed, +// agents_running=max(0, agents_total - agents_completed), +// findings_so_far=findings_so_far, elapsed_seconds=elapsed, +// estimated_remaining_seconds=max(0.0, estimated_total - elapsed), +// cost_so_far_usd=round(self.total_cost_usd, 4)) +// self.app.note(progress.model_dump_json(), tags=["audit", "progress", phase]) +// +// Python parity: +// +// - `agents_completed / safe_total` is TRUE division, so 1/2 is 0.5, not 0; +// - `safe_total = max(1, agents_total)` guards a zero total, and the outer +// min clamps a completed count that overshoots; +// - `estimated_total` divides by phase_progress, so a phase with zero progress +// reports an estimated remaining of 0.0 rather than infinity; +// - `round(self.total_cost_usd, 4)` is Python's BANKER'S rounding +// (pyfmt.Round), not Go's strconv default. +// +// The note MESSAGE is `progress.model_dump_json()` — pydantic's serializer, not +// json.dumps: no space after `:` or `,`, and a float field always carries a +// decimal point (`"phase_progress":1.0`). pyfmt.DumpsModelJSON is that +// spelling; a Go `json.Marshal` would emit `1` and break byte parity with the +// Python node's notes. +func (o *AuditOrchestrator) EmitProgress(ctx context.Context, phase string, agentsTotal, agentsCompleted, findingsSoFar int) { + progress := o.BuildProgress(phase, agentsTotal, agentsCompleted, findingsSoFar) + o.App.Note(ctx, pyfmt.DumpsModelJSON(progress), "audit", "progress", phase) +} + +// BuildProgress is EmitProgress without the note — the AuditProgress value it +// serializes. Split out so a test can assert the arithmetic without scripting a +// fake, and so the note message can be golden-compared. +func (o *AuditOrchestrator) BuildProgress(phase string, agentsTotal, agentsCompleted, findingsSoFar int) schemas.AuditProgress { + elapsed := o.elapsedSeconds() + + safeTotal := agentsTotal + if safeTotal < 1 { + safeTotal = 1 + } + phaseProgress := float64(agentsCompleted) / float64(safeTotal) + if phaseProgress > 1.0 { + phaseProgress = 1.0 + } + + estimatedTotal := elapsed + if phaseProgress > 0 { + estimatedTotal = elapsed / phaseProgress + } + remaining := estimatedTotal - elapsed + if remaining < 0 { + remaining = 0.0 + } + + agentsRunning := agentsTotal - agentsCompleted + if agentsRunning < 0 { + agentsRunning = 0 + } + + return schemas.AuditProgress{ + Phase: phase, + PhaseProgress: phaseProgress, + AgentsTotal: agentsTotal, + AgentsCompleted: agentsCompleted, + AgentsRunning: agentsRunning, + FindingsSoFar: findingsSoFar, + ElapsedSeconds: elapsed, + EstimatedRemainingSeconds: remaining, + CostSoFarUsd: pyfmt.Round(o.TotalCostUSD(), 4), + } +} + +// TrackDrop ports `AuditOrchestrator._track_drop` (orchestrator.py:654) — the +// demotion bookkeeping written into prove_drop_summary, plus its note. +// +// It is the same bookkeeping as reasoners/phases.py's module-level `_track_drop` +// with ONE difference that is easy to miss: the note tags here are +// ["audit", "prove", "drop"], where phases uses ["prove", "drop", "demotion"]. +// +// Python parity: +// +// - `setdefault` recreates a missing "by_reason"/"findings" entry, which the +// Go type assertions reproduce; +// - the recorded `original_verdict` is None (JSON null) when absent; +// - the note's `original_verdict or 'unknown'` is PYTHON TRUTHINESS, so an +// empty string also prints "unknown". +func (o *AuditOrchestrator) TrackDrop(ctx context.Context, findingTitle string, originalVerdict *string, reason string) { + // ProveDropSummary is typed `any` so the app.py path can thread a + // non-dict `.call` value through untouched (see orch.go). Every caller of + // TrackDrop is on the ORCHESTRATOR path, where rebuildDropSummary has just + // installed a real dict, so the assertion always succeeds; the fallback + // mirrors the pre-existing nil guard rather than reproducing the + // AttributeError `None.setdefault(...)` would raise in Python. + summary, isMap := o.ProveDropSummary.(map[string]any) + if !isMap { + summary = NewDropSummary() + o.ProveDropSummary = summary + } + + total, _ := summary["demoted_total"].(int) + summary["demoted_total"] = total + 1 + + byReason, ok := summary["by_reason"].(map[string]int) + if !ok { + byReason = map[string]int{} + summary["by_reason"] = byReason + } + byReason[reason]++ + + findings, ok := summary["findings"].([]map[string]any) + if !ok { + findings = []map[string]any{} + } + entry := map[string]any{"title": findingTitle, "original_verdict": nil, "reason": reason} + if originalVerdict != nil { + entry["original_verdict"] = *originalVerdict + } + summary["findings"] = append(findings, entry) + + verdictLabel := "unknown" + if originalVerdict != nil && *originalVerdict != "" { + verdictLabel = *originalVerdict + } + o.App.Note(ctx, + "Demoted finding '"+findingTitle+"' (verdict="+verdictLabel+"): "+reason, + "audit", "prove", "drop") +} diff --git a/internal/secaf/orch/reachability.go b/internal/secaf/orch/reachability.go new file mode 100644 index 0000000000..0116f84ab9 --- /dev/null +++ b/internal/secaf/orch/reachability.go @@ -0,0 +1,126 @@ +package orch + +import ( + "context" + "strconv" + "sync" + + "golang.org/x/sync/semaphore" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// reachabilityTags ports the `reachability_tags` set: a finding already +// carrying ANY of these needs no assessment. +var reachabilityTags = map[string]struct{}{ + "externally_reachable": {}, + "requires_auth": {}, + "internal_only": {}, + "unreachable": {}, +} + +// ReachabilitySummary builds the exact prompt body `_assess_one` hands to +// AIGateWrapper.assess_reachability (orchestrator.py:580): +// +// summary = (f"Finding: {finding.title}\n" +// f"Description: {finding.description}\n" +// f"CWE: {finding.cwe_id}\n" +// f"File: {finding.location.file_path}:{finding.location.start_line}\n" +// f"Verdict: {finding.verdict.value}") +// +// No trailing newline. `verdict.value` is the enum's string value, which the Go +// enum already is. It is a separate function so the golden test can compare the +// bytes without scripting a gate. +func ReachabilitySummary(finding schemas.VerifiedFinding) string { + return "Finding: " + finding.Title + "\n" + + "Description: " + finding.Description + "\n" + + "CWE: " + finding.CweID + "\n" + + "File: " + finding.Location.FilePath + ":" + strconv.Itoa(finding.Location.StartLine) + "\n" + + "Verdict: " + string(finding.Verdict) +} + +// AssessReachabilityParallel ports `_assess_reachability_parallel(verified)` +// (orchestrator.py:568): +// +// needs_assessment = [f for f in verified if not any(tag in reachability_tags for tag in f.tags)] +// if not needs_assessment: return +// semaphore = asyncio.Semaphore(min(5, len(needs_assessment))) +// async def _assess_one(finding): +// async with semaphore: +// try: +// gate_result = await self.ai_gate.assess_reachability(summary) +// finding.tags.append(gate_result.reachability) +// except Exception: +// finding.tags.append("requires_auth") # safe default +// await asyncio.gather(*[_assess_one(f) for f in needs_assessment]) +// +// Python parity: +// +// - the semaphore is `min(5, len(needs_assessment))`, NOT max(1, ...): the +// early return above guarantees the count is at least 1, so the limit is +// between 1 and 5. +// - EVERY failure — a gate error, a malformed response, anything — appends the +// literal "requires_auth". That is the documented safe default: an +// unassessable finding is treated as authenticated-only rather than +// internet-facing, which lowers its reachability multiplier in scoring. +// - the gate's `reachability` string is appended VERBATIM, with no validation. +// A gate that answers "maybe" puts "maybe" in the tags, and +// scoring.reachabilityMultiplier then falls through to its default. +// - findings are MUTATED IN PLACE. The Go port takes the slice and indexes +// into it, so the caller sees the tags without a return value — the same +// aliasing Python has. +// +// Concurrency: each goroutine appends to a DIFFERENT element's Tags slice, so +// no lock is needed around the append itself; the WaitGroup provides the +// happens-before edge the caller needs to read them. +func (o *AuditOrchestrator) AssessReachabilityParallel(ctx context.Context, verified []schemas.VerifiedFinding) { + needs := make([]int, 0, len(verified)) + for i := range verified { + if !hasReachabilityTag(verified[i].Tags) { + needs = append(needs, i) + } + } + if len(needs) == 0 { + return + } + + limit := 5 + if len(needs) < limit { + limit = len(needs) + } + sem := semaphore.NewWeighted(int64(limit)) + + var wg sync.WaitGroup + for _, idx := range needs { + idx := idx + wg.Add(1) + go func() { + defer wg.Done() + if err := sem.Acquire(ctx, 1); err != nil { + // A cancelled context is one of the exceptions the bare + // `except Exception` swallows. + verified[idx].Tags = append(verified[idx].Tags, "requires_auth") + return + } + defer sem.Release(1) + + gateResult, err := o.AIGate.AssessReachability(ctx, ReachabilitySummary(verified[idx])) + if err != nil { + verified[idx].Tags = append(verified[idx].Tags, "requires_auth") + return + } + verified[idx].Tags = append(verified[idx].Tags, gateResult.Reachability) + }() + } + wg.Wait() +} + +// hasReachabilityTag ports `any(tag in reachability_tags for tag in f.tags)`. +func hasReachabilityTag(tags []string) bool { + for _, tag := range tags { + if _, ok := reachabilityTags[tag]; ok { + return true + } + } + return false +} diff --git a/internal/secaf/orch/run.go b/internal/secaf/orch/run.go new file mode 100644 index 0000000000..82706e5de4 --- /dev/null +++ b/internal/secaf/orch/run.go @@ -0,0 +1,678 @@ +package orch + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + + "golang.org/x/sync/errgroup" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + huntagent "github.com/Agent-Field/codeaf/internal/secaf/agents/hunt" + proveagent "github.com/Agent-Field/codeaf/internal/secaf/agents/prove" + reconagent "github.com/Agent-Field/codeaf/internal/secaf/agents/recon" + "github.com/Agent-Field/codeaf/internal/secaf/config" + "github.com/Agent-Field/codeaf/internal/secaf/phases" + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// enableDast ports `getattr(self.input, "enable_dast", False)` — the guard on +// both DAST branches. +// +// PYTHON QUIRK, REPRODUCED. app.py's `audit` reasoner accepts an `enable_dast` +// parameter and passes it to the AuditInput constructor: +// +// audit_input = AuditInput(..., enable_dast=enable_dast) +// +// but schemas/input.py declares the field as `dast_enabled`, and pydantic's +// default `extra="ignore"` silently DROPS the unknown keyword. VERIFIED: +// `hasattr(AuditInput(repo_url="x", enable_dast=True), "enable_dast")` is False. +// So `getattr(self.input, "enable_dast", False)` is ALWAYS False and neither +// DAST branch has ever executed in production. +// +// It is a package variable rather than a constant so a test can flip it and +// exercise the ported branch; production never assigns it. +var enableDast = false + +// ErrDastVerifierArity is the TypeError orchestrator.py:341 raises. +// +// PYTHON BUG, REPRODUCED. `_run_dast_verification` calls +// +// await run_dast_verifier(_PhaseHarnessProxy(self, "prove"), str(self.repo_path), finding) +// +// with THREE positional arguments, while agents/prove/dast_verifier.py declares +// +// async def run_dast_verifier(app, repo_path, finding, exploit_payload, depth) +// +// — five required parameters. The call raises before the agent runs, the +// surrounding `except Exception` catches it, and the finding is tagged +// "dast_error". (Two further mismatches sit behind it: `finding` is a +// VerifiedFinding where a RawFinding is expected, and the result attributes the +// loop body reads — exploit_attempted, exploit_succeeded, evidence, confidence, +// response_analysis — do not exist on DastVerificationResult, which carries +// payload_sent / response_summary / exploit_confirmed / safety_notes.) +var ErrDastVerifierArity = errors.New( + "run_dast_verifier() missing 2 required positional arguments: 'exploit_payload' and 'depth'") + +// dastOutcome models the attribute set `_run_dast_verification` READS off the +// DAST result. It is deliberately NOT schemas.DastVerificationResult: the +// orchestrator reads five attributes that model does not have (see +// ErrDastVerifierArity). Keeping the shape the Python code expects is what lets +// the ported loop body be written — and tested — as written. +type dastOutcome struct { + ExploitAttempted bool + ExploitSucceeded bool + Evidence string + Confidence string + ResponseAnalysis string +} + +// dastVerify is the `run_dast_verifier(...)` call site, as a seam so a test can +// drive the loop body. The production value always fails, exactly as Python +// does — see ErrDastVerifierArity. +var dastVerify = func(_ context.Context, _ appx.Harnesser, _ string, _ schemas.VerifiedFinding) (dastOutcome, error) { + return dastOutcome{}, ErrDastVerifierArity +} + +// --------------------------------------------------------------------------- +// run() +// --------------------------------------------------------------------------- + +// Run ports `AuditOrchestrator.run()` (orchestrator.py:82) — the STREAMING, +// fully in-process pipeline. +// +// note("Starting SEC-AF streaming orchestrator", tags=["audit","start","streaming"]) +// fast_recon = await self._run_fast_recon(); self._write_checkpoint("recon_fast", fast_recon) +// findings_queue = asyncio.Queue() +// deep_result, hunt, verified = await asyncio.gather( +// self._run_deep_recon_async(fast_recon), +// self._run_hunt_streaming(fast_recon, findings_queue), +// self._run_prove_streaming(findings_queue)) +// self.findings_not_verified = max(0, len(hunt.findings) - len(verified)) +// recon = self._merge_recon(fast_recon, deep_result) +// write "recon", "hunt", "prove" +// await self._assess_reachability_parallel(verified) +// rebuild prove_drop_summary from each finding's drop_reason +// (DAST branch — dead, see enableDast) +// result = await self._generate_output(...) +// note("SEC-AF audit complete", tags=["audit","complete"]) +// +// app.py does NOT use this path: it issues four `.call`s into internal/phases +// and then calls GenerateOutput directly. Run is ported because +// RunFromCheckpoint shares its `_run_hunt`/`_run_prove` halves and because +// DESIGN.md §3 requires the whole class. +// +// Concurrency parity: the three tasks run together, the hunt publishing batches +// to the prove consumer through a queue. Go uses one errgroup (no +// WithContext — `asyncio.gather` does not cancel siblings on failure either), +// so the FIRST error is returned once all three have finished. The Go channel +// is buffered to the strategy count and CLOSED by RunHuntStreaming in place of +// Python's None sentinel. +func (o *AuditOrchestrator) Run(ctx context.Context) (schemas.SecurityAuditResult, error) { + o.App.Note(ctx, "Starting SEC-AF streaming orchestrator", "audit", "start", "streaming") + + fastRecon, err := o.RunFastRecon(ctx) + if err != nil { + return schemas.SecurityAuditResult{}, err + } + if err := o.WriteCheckpoint("recon_fast", fastRecon); err != nil { + return schemas.SecurityAuditResult{}, err + } + + // Buffered to the number of hunters so a producer's send never blocks, + // which is what Python's unbounded asyncio.Queue guarantees. + strategyCount := len(huntagent.SelectStrategies(config.NormalizeDepth(o.Input.Depth))) + findingsQueue := make(chan []schemas.RawFinding, strategyCount) + + var ( + dataFlows schemas.DataFlowMap + securityContext schemas.SecurityContext + hunt schemas.HuntResult + verified []schemas.VerifiedFinding + ) + var g errgroup.Group + g.Go(func() error { + var err error + dataFlows, securityContext, err = o.RunDeepReconAsync(ctx, fastRecon) + return err + }) + g.Go(func() error { + var err error + hunt, err = o.RunHuntStreaming(ctx, fastRecon, findingsQueue) + return err + }) + g.Go(func() error { + verified = o.RunProveStreaming(ctx, findingsQueue) + return nil + }) + if err := g.Wait(); err != nil { + return schemas.SecurityAuditResult{}, err + } + + o.FindingsNotVerified = len(hunt.Findings) - len(verified) + if o.FindingsNotVerified < 0 { + o.FindingsNotVerified = 0 + } + + recon := o.MergeRecon(fastRecon, dataFlows, securityContext) + if err := o.WriteCheckpoint(PhaseRecon, recon); err != nil { + return schemas.SecurityAuditResult{}, err + } + if err := o.WriteCheckpoint(PhaseHunt, hunt); err != nil { + return schemas.SecurityAuditResult{}, err + } + if err := o.WriteCheckpoint(PhaseProve, verified); err != nil { + return schemas.SecurityAuditResult{}, err + } + + o.AssessReachabilityParallel(ctx, verified) + o.rebuildDropSummary(ctx, verified) + + if enableDast { + o.App.Note(ctx, "DAST-like runtime verification enabled", "audit", "prove", "dast") + o.RunDASTVerification(ctx, verified) + } + + result, err := o.GenerateOutput(ctx, recon, hunt, verified) + if err != nil { + return result, err + } + o.App.Note(ctx, "SEC-AF audit complete", "audit", "complete") + return result, nil +} + +// rebuildDropSummary ports the identical three lines run() and _run_prove share: +// +// self.prove_drop_summary = {"demoted_total": 0, "by_reason": {}, "findings": []} +// for finding in verified: +// if finding.drop_reason: +// self._track_drop(finding_title=finding.title, original_verdict=None, reason=finding.drop_reason) +// +// Python parity: `if finding.drop_reason` is TRUTHINESS, so an empty-string +// drop_reason is skipped, and original_verdict is always None here even though +// the demoting code knew it. +func (o *AuditOrchestrator) rebuildDropSummary(ctx context.Context, verified []schemas.VerifiedFinding) { + o.ProveDropSummary = NewDropSummary() + for _, finding := range verified { + if finding.DropReason != nil && *finding.DropReason != "" { + o.TrackDrop(ctx, finding.Title, nil, *finding.DropReason) + } + } +} + +// --------------------------------------------------------------------------- +// run_from_checkpoint() +// --------------------------------------------------------------------------- + +// UnknownCheckpointPhaseError is `ValueError(f"Unknown checkpoint phase: {phase}")`. +// +// app.py maps a ValueError to HTTP 400, so the type matters as much as the +// text: `except ValueError as exc: raise HTTPException(400, {"error": str(exc)})`. +type UnknownCheckpointPhaseError struct{ Phase string } + +func (e *UnknownCheckpointPhaseError) Error() string { + return "Unknown checkpoint phase: " + e.Phase +} + +// RunFromCheckpoint ports `run_from_checkpoint(phase)` (orchestrator.py:121): +// +// normalized_phase = phase.lower().strip() +// if normalized_phase not in {"recon", "hunt", "prove"}: +// raise ValueError(f"Unknown checkpoint phase: {phase}") +// recon: always read from checkpoint-recon.json +// "recon": run hunt, write it, run prove, write it +// "hunt": read hunt, run prove, write it +// "prove": read hunt AND the prove list — nothing is re-run +// return await self._generate_output(recon=recon, hunt=hunt, verified=verified) +// +// Python parity: +// +// - the error message carries the ORIGINAL phase string, not the normalized +// one, so `run_from_checkpoint(" Recon ")` succeeds while +// `run_from_checkpoint("Recon!")` reports "Unknown checkpoint phase: Recon!". +// - normalization is `.lower()` THEN `.strip()`; both orders agree for +// whitespace, and Go's TrimSpace(ToLower(s)) matches for every input that is +// not exotic Unicode whitespace. +// - the "prove" branch reads a LIST of VerifiedFinding, the other two read +// single models — hence the two read helpers. +func (o *AuditOrchestrator) RunFromCheckpoint(ctx context.Context, phase string) (schemas.SecurityAuditResult, error) { + normalized := strings.TrimSpace(strings.ToLower(phase)) + if normalized != PhaseRecon && normalized != PhaseHunt && normalized != PhaseProve { + return schemas.SecurityAuditResult{}, &UnknownCheckpointPhaseError{Phase: phase} + } + + recon, err := ReadCheckpoint(o, PhaseRecon, phases.BindReconResult) + if err != nil { + return schemas.SecurityAuditResult{}, err + } + + var ( + hunt schemas.HuntResult + verified []schemas.VerifiedFinding + ) + switch normalized { + case PhaseRecon: + hunt, err = o.RunHunt(ctx, recon) + if err != nil { + return schemas.SecurityAuditResult{}, err + } + if err = o.WriteCheckpoint(PhaseHunt, hunt); err != nil { + return schemas.SecurityAuditResult{}, err + } + verified, err = o.RunProve(ctx, recon, hunt) + if err != nil { + return schemas.SecurityAuditResult{}, err + } + if err = o.WriteCheckpoint(PhaseProve, verified); err != nil { + return schemas.SecurityAuditResult{}, err + } + case PhaseHunt: + hunt, err = ReadCheckpoint(o, PhaseHunt, phases.BindHuntResult) + if err != nil { + return schemas.SecurityAuditResult{}, err + } + verified, err = o.RunProve(ctx, recon, hunt) + if err != nil { + return schemas.SecurityAuditResult{}, err + } + if err = o.WriteCheckpoint(PhaseProve, verified); err != nil { + return schemas.SecurityAuditResult{}, err + } + default: // PhaseProve + hunt, err = ReadCheckpoint(o, PhaseHunt, phases.BindHuntResult) + if err != nil { + return schemas.SecurityAuditResult{}, err + } + verified, err = ReadCheckpointList(o, PhaseProve, phases.BindVerifiedFinding) + if err != nil { + return schemas.SecurityAuditResult{}, err + } + } + + return o.GenerateOutput(ctx, recon, hunt, verified) +} + +// --------------------------------------------------------------------------- +// recon +// --------------------------------------------------------------------------- + +// RunRecon ports `_run_recon()` (orchestrator.py:149) — the non-streaming RECON +// phase. +// +// Python parity: NOTHING calls it. run() uses _run_fast_recon plus +// _run_deep_recon_async, and run_from_checkpoint always reads recon from disk. +// It is ported for completeness, and it is the only place the PR-mode recon +// cache is consulted with the "Using cached recon for PR-mode scan" wording +// (note the trailing " scan", which _run_fast_recon's otherwise-identical note +// lacks). +func (o *AuditOrchestrator) RunRecon(ctx context.Context) (schemas.ReconResult, error) { + o.App.Note(ctx, "Phase: RECON", "audit", "recon") + if o.IsPRMode { + if cached := o.TryLoadCachedRecon(); cached != nil { + o.App.Note(ctx, "Using cached recon for PR-mode scan", "audit", "recon", "cached") + o.EmitProgress(ctx, PhaseRecon, 1, 1, 0) + return *cached, nil + } + } + recon, err := reconagent.RunRecon(ctx, o.PhaseProxy(PhaseRecon), o.RepoPath, o.Input.Depth) + if err != nil { + return schemas.ReconResult{}, err + } + o.EmitProgress(ctx, PhaseRecon, 1, 1, 0) + return recon, nil +} + +// RunFastRecon ports `_run_fast_recon()` (orchestrator.py:165) — the three +// cheap mappers, plus the PR-mode cache short circuit. +// +// Python parity: the progress event reports agents_total=2 / agents_completed=1 +// on BOTH paths (cached and fresh), because the deep half is still to come. +func (o *AuditOrchestrator) RunFastRecon(ctx context.Context) (schemas.ReconResult, error) { + o.App.Note(ctx, "Phase: FAST RECON", "audit", "recon", "fast") + if o.IsPRMode { + if cached := o.TryLoadCachedRecon(); cached != nil { + o.App.Note(ctx, "Using cached recon for PR-mode", "audit", "recon", "cached") + o.EmitProgress(ctx, PhaseRecon, 2, 1, 0) + return *cached, nil + } + } + fast, err := reconagent.RunFastRecon(ctx, o.PhaseProxy(PhaseRecon), o.RepoPath) + if err != nil { + return schemas.ReconResult{}, err + } + o.EmitProgress(ctx, PhaseRecon, 2, 1, 0) + return fast, nil +} + +// RunDeepReconAsync ports `_run_deep_recon_async(fast_recon)` +// (orchestrator.py:182): +// +// if self.config.depth == DepthProfile.QUICK: +// return fast_recon.data_flows, fast_recon.security_context +// result = await run_deep_recon(app=proxy, repo_path=..., architecture=fast_recon.architecture) +// self._emit_progress(phase="recon", agents_total=2, agents_completed=2, findings_so_far=0) +// return result +// +// Python parity: the QUICK short circuit reads `self.config.depth`, which +// AuditConfig.from_input already normalized — NOT `self._depth_profile()`. The +// two agree, but the config is the one the code consults. It also emits NO +// progress event, so a quick audit reports recon 1/2 and never 2/2. +func (o *AuditOrchestrator) RunDeepReconAsync(ctx context.Context, fastRecon schemas.ReconResult) ( + schemas.DataFlowMap, schemas.SecurityContext, error, +) { + if o.Config.Depth == config.DepthQuick { + return fastRecon.DataFlows, fastRecon.SecurityContext, nil + } + dataFlows, securityContext, err := reconagent.RunDeepRecon( + ctx, o.PhaseProxy(PhaseRecon), o.RepoPath, fastRecon.Architecture) + if err != nil { + return schemas.DataFlowMap{}, schemas.SecurityContext{}, err + } + o.EmitProgress(ctx, PhaseRecon, 2, 2, 0) + return dataFlows, securityContext, nil +} + +// MergeRecon ports `_merge_recon(fast, deep_result)` (orchestrator.py:196): +// +// data_flows, security_context = deep_result +// frameworks = sorted({item for item in security_context.framework_security if item}) +// return ReconResult(architecture=fast.architecture, data_flows=data_flows, +// dependencies=fast.dependencies, config=fast.config, +// security_context=security_context, languages=fast.languages, +// frameworks=frameworks, lines_of_code=fast.lines_of_code, +// file_count=fast.file_count) +// +// Python parity: `recon_duration_seconds` is NOT carried over from `fast` — the +// merged result keeps the pydantic default 0.0 — and the frameworks come from +// the DEEP security context, not the fast one. +func (o *AuditOrchestrator) MergeRecon( + fast schemas.ReconResult, + dataFlows schemas.DataFlowMap, + securityContext schemas.SecurityContext, +) schemas.ReconResult { + return schemas.ReconResult{ + Architecture: fast.Architecture, + DataFlows: dataFlows, + Dependencies: fast.Dependencies, + Config: fast.Config, + SecurityContext: securityContext, + Languages: fast.Languages, + Frameworks: sortedNonEmpty(securityContext.FrameworkSecurity), + LinesOfCode: fast.LinesOfCode, + FileCount: fast.FileCount, + } +} + +// --------------------------------------------------------------------------- +// hunt +// --------------------------------------------------------------------------- + +// huntIncludePaths ports the include-path selection both hunt entry points +// share: +// +// include_paths = self.config.include_paths +// if self.is_pr_mode and self.diff_analysis and self.diff_analysis.changed_files: +// include_paths = self.diff_analysis.all_relevant_files +// self.app.note(f"PR-mode: scanning {file_count} files " +// f"({len(changed_files)} changed + {len(blast_radius_files)} blast radius)", +// tags=["audit", "hunt", "pr-mode"]) +// +// Python parity: the guard needs a NON-EMPTY changed_files list — a diff +// analysis that found nothing leaves the configured include paths in place and +// emits no note. +func (o *AuditOrchestrator) huntIncludePaths(ctx context.Context) []string { + includePaths := o.Config.IncludePaths + if o.IsPRMode && o.DiffAnalysis != nil && len(o.DiffAnalysis.ChangedFiles) > 0 { + includePaths = o.DiffAnalysis.AllRelevantFiles + o.App.Note(ctx, + "PR-mode: scanning "+strconv.Itoa(o.DiffAnalysis.FileCount())+" files ("+ + strconv.Itoa(len(o.DiffAnalysis.ChangedFiles))+" changed + "+ + strconv.Itoa(len(o.DiffAnalysis.BlastRadiusFiles))+" blast radius)", + "audit", "hunt", "pr-mode") + } + return includePaths +} + +// RunHuntStreaming ports `_run_hunt_streaming(recon, findings_queue)` +// (orchestrator.py:211) — run_hunt_streaming plus the recon-finding merge and +// the progress event. +// +// The channel is CLOSED by agents/hunt.RunHuntStreaming in place of Python's +// None sentinel, including when a hunter fails, so the prove consumer always +// terminates. +func (o *AuditOrchestrator) RunHuntStreaming( + ctx context.Context, + recon schemas.ReconResult, + findingsQueue chan<- []schemas.RawFinding, +) (schemas.HuntResult, error) { + o.App.Note(ctx, "Phase: HUNT (streaming)", "audit", "hunt", "streaming") + + includePaths := o.huntIncludePaths(ctx) + + hunt, err := huntagent.RunHuntStreaming( + ctx, + o.PhaseProxy(PhaseHunt), + o.RepoPath, + recon, + findingsQueue, + o.Input.Depth, + o.BudgetConfig.MaxConcurrentHunters, + o.BudgetConfig.HunterEarlyStopFileThreshold, + includePaths, + ) + if err != nil { + return schemas.HuntResult{}, err + } + + hunt = MergeReconFindingsIntoHunt(hunt, reconagent.ExtractReconFindings(recon)) + o.EmitProgress(ctx, PhaseHunt, 1, 1, len(hunt.Findings)) + return hunt, nil +} + +// RunHunt ports `_run_hunt(recon)` (orchestrator.py:263) — the non-streaming +// hunt used by run_from_checkpoint("recon"). +func (o *AuditOrchestrator) RunHunt(ctx context.Context, recon schemas.ReconResult) (schemas.HuntResult, error) { + o.App.Note(ctx, "Phase: HUNT", "audit", "hunt") + + includePaths := o.huntIncludePaths(ctx) + + hunt, err := huntagent.RunHunt( + ctx, + o.PhaseProxy(PhaseHunt), + o.RepoPath, + recon, + o.Input.Depth, + o.BudgetConfig.MaxConcurrentHunters, + o.BudgetConfig.HunterEarlyStopFileThreshold, + includePaths, + ) + if err != nil { + return schemas.HuntResult{}, err + } + + hunt = MergeReconFindingsIntoHunt(hunt, reconagent.ExtractReconFindings(recon)) + o.EmitProgress(ctx, PhaseHunt, 1, 1, len(hunt.Findings)) + return hunt, nil +} + +// --------------------------------------------------------------------------- +// prove +// --------------------------------------------------------------------------- + +// RunProveStreaming ports `_run_prove_streaming(findings_queue)` +// (orchestrator.py:244). +// +// Python parity: it RESETS findings_not_verified to 0 before consuming, so the +// value run() computes afterwards (`len(hunt.findings) - len(verified)`) is the +// one that survives. run_prove_streaming itself cannot fail, so neither can +// this. +func (o *AuditOrchestrator) RunProveStreaming( + ctx context.Context, + findingsQueue <-chan []schemas.RawFinding, +) []schemas.VerifiedFinding { + o.App.Note(ctx, "Phase: PROVE (streaming)", "audit", "prove", "streaming") + + proverCap := o.ProverCap() + o.FindingsNotVerified = 0 + + verified := proveagent.RunProveStreaming( + ctx, + o.ProvePhaseProxy(), + o.RepoPath, + findingsQueue, + o.Input.Depth, + o.BudgetConfig.MaxConcurrentProvers, + proverCap, + ) + o.EmitProgress(ctx, PhaseProve, 1, 1, len(verified)) + return verified +} + +// RunProve ports `_run_prove(recon, hunt)` (orchestrator.py:290) — the +// non-streaming prove used by run_from_checkpoint. +// +// prioritized = self._prioritize_findings(hunt.findings) +// limited_hunt = HuntResult(findings=prioritized[:self._prover_cap()], ) +// self.findings_not_verified = max(0, len(hunt.findings) - len(limited_hunt.findings)) +// verified = await run_prove(app=proxy, ...) +// await self._assess_reachability_parallel(verified) +// rebuild prove_drop_summary +// (DAST branch — dead) +// self._emit_progress(phase="prove", agents_total=1, agents_completed=1, findings_so_far=len(verified)) +// +// Python parity: `recon` is accepted and immediately discarded (`_ = recon`), +// and the limited hunt copies chains/total_raw/deduplicated_count/chain_count/ +// strategies_run/hunt_duration_seconds VERBATIM — only the findings list is +// truncated, so total_raw keeps counting the findings that were dropped. +func (o *AuditOrchestrator) RunProve( + ctx context.Context, + recon schemas.ReconResult, + hunt schemas.HuntResult, +) ([]schemas.VerifiedFinding, error) { + _ = recon // Python: `_ = recon` + + o.App.Note(ctx, "Phase: PROVE", "audit", "prove") + + prioritized := o.PrioritizeFindings(hunt.Findings) + proverCap := o.ProverCap() + if proverCap < len(prioritized) { + prioritized = prioritized[:proverCap] + } + limitedHunt := schemas.HuntResult{ + Findings: prioritized, + Chains: hunt.Chains, + TotalRaw: hunt.TotalRaw, + DeduplicatedCount: hunt.DeduplicatedCount, + ChainCount: hunt.ChainCount, + StrategiesRun: hunt.StrategiesRun, + HuntDurationSeconds: hunt.HuntDurationSeconds, + } + + o.FindingsNotVerified = len(hunt.Findings) - len(limitedHunt.Findings) + if o.FindingsNotVerified < 0 { + o.FindingsNotVerified = 0 + } + + verified, err := proveagent.RunProve( + ctx, + o.ProvePhaseProxy(), + o.RepoPath, + limitedHunt, + o.Input.Depth, + o.BudgetConfig.MaxConcurrentProvers, + ) + if err != nil { + return verified, err + } + + o.AssessReachabilityParallel(ctx, verified) + o.rebuildDropSummary(ctx, verified) + + if enableDast { + o.App.Note(ctx, "DAST-like runtime verification enabled", "audit", "prove", "dast") + o.RunDASTVerification(ctx, verified) + } + + o.EmitProgress(ctx, PhaseProve, 1, 1, len(verified)) + return verified, nil +} + +// RunDASTVerification ports `_run_dast_verification(verified)` +// (orchestrator.py:332). +// +// confirmed = [f for f in verified if f.verdict == Verdict.CONFIRMED] +// if not confirmed: +// note("No confirmed findings available for DAST step", tags=["audit","prove","dast"]); return +// for finding in confirmed: +// try: dast_result = await run_dast_verifier(proxy, str(self.repo_path), finding) +// except Exception as exc: +// finding.tags.append("dast_error") +// note(f"DAST verifier failed for '{finding.title}': {exc}", tags=["audit","prove","dast","error"]) +// continue +// finding.tags.append("dast_attempted" if dast_result.exploit_attempted else "dast_skipped") +// finding.tags.append("dast_confirmed" if dast_result.exploit_succeeded else "dast_not_confirmed") +// finding.rationale = f"{finding.rationale}\nDAST: {dast_result.response_analysis}" +// if finding.proof is not None: +// finding.proof.poc_execution_output = json.dumps({...}, indent=2) +// +// UNREACHABLE IN PRODUCTION on two independent counts — the enableDast guard +// and ErrDastVerifierArity — both documented above. With the production seam +// every confirmed finding takes the except branch: tagged "dast_error", noted, +// and otherwise untouched. +// +// Findings are mutated IN PLACE through the shared backing array, matching +// Python's aliasing. +func (o *AuditOrchestrator) RunDASTVerification(ctx context.Context, verified []schemas.VerifiedFinding) { + confirmed := make([]int, 0, len(verified)) + for i := range verified { + if verified[i].Verdict == schemas.VerdictConfirmed { + confirmed = append(confirmed, i) + } + } + if len(confirmed) == 0 { + o.App.Note(ctx, "No confirmed findings available for DAST step", "audit", "prove", "dast") + return + } + + proxy := o.PhaseProxy(PhaseProve) + for _, idx := range confirmed { + finding := &verified[idx] + + outcome, err := dastVerify(ctx, proxy, o.RepoPath, *finding) + if err != nil { + finding.Tags = append(finding.Tags, "dast_error") + o.App.Note(ctx, + fmt.Sprintf("DAST verifier failed for '%s': %v", finding.Title, err), + "audit", "prove", "dast", "error") + continue + } + + if outcome.ExploitAttempted { + finding.Tags = append(finding.Tags, "dast_attempted") + } else { + finding.Tags = append(finding.Tags, "dast_skipped") + } + if outcome.ExploitSucceeded { + finding.Tags = append(finding.Tags, "dast_confirmed") + } else { + finding.Tags = append(finding.Tags, "dast_not_confirmed") + } + finding.Rationale = finding.Rationale + "\nDAST: " + outcome.ResponseAnalysis + + if finding.Proof != nil { + // Python: json.dumps({...}, indent=2) over a dict literal, so the + // key order is the literal's — pyfmt.O preserves it where a Go map + // would sort. + body := pyfmt.Dumps(pyfmt.O( + "exploit_attempted", outcome.ExploitAttempted, + "exploit_succeeded", outcome.ExploitSucceeded, + "evidence", outcome.Evidence, + "confidence", outcome.Confidence, + ), 2) + finding.Proof.PocExecutionOutput = &body + } + } +} diff --git a/internal/secaf/orch/run_test.go b/internal/secaf/orch/run_test.go new file mode 100644 index 0000000000..b393541231 --- /dev/null +++ b/internal/secaf/orch/run_test.go @@ -0,0 +1,393 @@ +package orch + +import ( + "context" + "encoding/json" + "os" + "reflect" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/compliance" + "github.com/Agent-Field/codeaf/internal/secaf/diffanalysis" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// emptyHarness answers every structured harness call with "{}", which the +// schemas package's default-seeding UnmarshalJSON turns into a fully-defaulted +// model. That is enough to drive the whole in-process pipeline without an LLM: +// no locations found, so no enrichment, no findings, no verification. +func emptyHarness() func(context.Context, string, map[string]any, any, appx.HarnessOptions) (*appx.HarnessResult, error) { + return appx.HarnessJSON(func(string, appx.HarnessOptions) (json.RawMessage, error) { + return json.RawMessage(`{}`), nil + }) +} + +// TestRun_StreamingPipeline drives `run()` end to end with an empty harness and +// asserts the observable contract: the notes, the four checkpoint files, and a +// result that reaches GenerateOutput. +func TestRun_StreamingPipeline(t *testing.T) { + compliance.ClearAICache() + o, fake := newTestOrchestrator(t, func(in *schemas.AuditInput) { in.Depth = "quick" }) + fake.HarnessFn = emptyHarness() + fake.AIFn = appx.AIJSON(func(string) (json.RawMessage, error) { + return json.RawMessage(`{"mappings":[],"confidence":"low"}`), nil + }) + + result, err := o.Run(context.Background()) + if err != nil { + t.Fatalf("Run: %v", err) + } + + msgs := fake.NoteMessages() + if msgs[0] != "Starting SEC-AF streaming orchestrator" { + t.Errorf("first note = %q", msgs[0]) + } + if !reflect.DeepEqual(fake.Notes[0].Tags, []string{"audit", "start", "streaming"}) { + t.Errorf("start tags = %v", fake.Notes[0].Tags) + } + if msgs[len(msgs)-1] != "SEC-AF audit complete" { + t.Errorf("last note = %q", msgs[len(msgs)-1]) + } + for _, want := range []string{"Phase: FAST RECON", "Phase: HUNT (streaming)", "Phase: PROVE (streaming)"} { + if !containsMessage(msgs, want) { + t.Errorf("missing note %q in %q", want, msgs) + } + } + + // run() writes FOUR checkpoints: the fast recon snapshot plus the three + // phase results. + for _, phase := range []string{"recon_fast", "recon", "hunt", "prove"} { + if _, statErr := os.Stat(o.CheckpointPath(phase)); statErr != nil { + t.Errorf("checkpoint-%s.json was not written: %v", phase, statErr) + } + } + + if result.Provider != "harness" { + t.Errorf("provider = %q", result.Provider) + } + if result.DepthProfile != "quick" { + t.Errorf("depth_profile = %q", result.DepthProfile) + } + if o.FindingsNotVerified != 0 { + t.Errorf("findings_not_verified = %d, want 0 for an empty hunt", o.FindingsNotVerified) + } + // Every harness invocation went through a phase proxy, so it was counted. + if o.AgentInvocations() != len(fake.Harnesses) { + t.Errorf("AgentInvocations = %d, want %d (one per harness call)", o.AgentInvocations(), len(fake.Harnesses)) + } +} + +// TestRun_QuickDepthSkipsDeepRecon: _run_deep_recon_async short-circuits on +// self.config.depth, so the merged recon reuses the fast placeholders and no +// second recon progress event is emitted. +func TestRun_QuickDepthSkipsDeepRecon(t *testing.T) { + compliance.ClearAICache() + o, fake := newTestOrchestrator(t, func(in *schemas.AuditInput) { in.Depth = "quick" }) + fake.HarnessFn = emptyHarness() + fake.AIFn = appx.AIJSON(func(string) (json.RawMessage, error) { + return json.RawMessage(`{"mappings":[],"confidence":"low"}`), nil + }) + + fast := schemas.NewReconResult() + fast.SecurityContext.AuthModel = "placeholder" + dataFlows, securityContext, err := o.RunDeepReconAsync(context.Background(), fast) + if err != nil { + t.Fatalf("RunDeepReconAsync: %v", err) + } + if securityContext.AuthModel != "placeholder" { + t.Errorf("quick depth must reuse the fast security context, got %q", securityContext.AuthModel) + } + if !reflect.DeepEqual(dataFlows, fast.DataFlows) { + t.Error("quick depth must reuse the fast data flows") + } + if len(fake.Harnesses) != 0 { + t.Errorf("quick depth must not run the deep mappers (%d harness calls)", len(fake.Harnesses)) + } + if len(fake.Notes) != 0 { + t.Errorf("quick depth emits no progress event, got %q", fake.NoteMessages()) + } +} + +func containsMessage(msgs []string, want string) bool { + for _, m := range msgs { + if m == want { + return true + } + } + return false +} + +// TestMergeRecon pins _merge_recon: the deep half wins for data_flows and +// security_context, the fast half for everything else, frameworks are derived +// from the DEEP context, and recon_duration_seconds is NOT carried over. +func TestMergeRecon(t *testing.T) { + o, _ := newTestOrchestrator(t) + + fast := reconFixture(t, "full") + fast.ReconDurationSeconds = 42.5 + + deepFlows := schemas.NewDataFlowMap() + deepFlows.Sinks = []schemas.Sink{{SinkType: "sql", FilePath: "db.py", Line: 12}} + deepContext := schemas.NewSecurityContext() + deepContext.AuthModel = "oauth2" + deepContext.FrameworkSecurity = []string{"fastapi", "", "fastapi", "Starlette"} + + merged := o.MergeRecon(fast, deepFlows, deepContext) + + if !reflect.DeepEqual(merged.DataFlows, deepFlows) { + t.Error("data_flows must come from the deep half") + } + if merged.SecurityContext.AuthModel != "oauth2" { + t.Errorf("security_context = %+v, want the deep half", merged.SecurityContext) + } + if !reflect.DeepEqual(merged.Architecture, fast.Architecture) { + t.Error("architecture must come from the fast half") + } + if !reflect.DeepEqual(merged.Languages, fast.Languages) { + t.Error("languages must come from the fast half") + } + if merged.LinesOfCode != fast.LinesOfCode || merged.FileCount != fast.FileCount { + t.Error("metrics must come from the fast half") + } + // sorted({item for item in deep.framework_security if item}) — blanks and + // repeats dropped, case preserved. + if want := []string{"Starlette", "fastapi"}; !reflect.DeepEqual(merged.Frameworks, want) { + t.Errorf("frameworks = %v, want %v", merged.Frameworks, want) + } + if merged.ReconDurationSeconds != 0 { + t.Errorf("recon_duration_seconds = %v, want 0 (it is not carried over)", merged.ReconDurationSeconds) + } +} + +// TestRunFastRecon_PRModeCache: in PR mode a previous recon checkpoint short +// circuits the mappers, with the PR-specific note. +func TestRunFastRecon_PRModeCache(t *testing.T) { + t.Run("cache hit", func(t *testing.T) { + o, fake := newTestOrchestrator(t) + o.IsPRMode = true + fake.HarnessFn = emptyHarness() + + cached := reconFixture(t, "full") + if err := o.WriteCheckpoint(PhaseRecon, cached); err != nil { + t.Fatalf("WriteCheckpoint: %v", err) + } + + got, err := o.RunFastRecon(context.Background()) + if err != nil { + t.Fatalf("RunFastRecon: %v", err) + } + if len(fake.Harnesses) != 0 { + t.Errorf("a cache hit must not run any mapper (%d harness calls)", len(fake.Harnesses)) + } + if !reflect.DeepEqual(normalizeJSON(t, got), normalizeJSON(t, cached)) { + t.Error("the cached recon must be returned unchanged") + } + if !containsMessage(fake.NoteMessages(), "Using cached recon for PR-mode") { + t.Errorf("missing the PR-mode cache note in %q", fake.NoteMessages()) + } + }) + + t.Run("cache miss falls through to the mappers", func(t *testing.T) { + o, fake := newTestOrchestrator(t) + o.IsPRMode = true + fake.HarnessFn = emptyHarness() + + if _, err := o.RunFastRecon(context.Background()); err != nil { + t.Fatalf("RunFastRecon: %v", err) + } + if len(fake.Harnesses) != 3 { + t.Errorf("harness calls = %d, want 3 (the three cheap mappers)", len(fake.Harnesses)) + } + }) + + t.Run("the cache is only consulted in PR mode", func(t *testing.T) { + o, fake := newTestOrchestrator(t) + fake.HarnessFn = emptyHarness() + if err := o.WriteCheckpoint(PhaseRecon, reconFixture(t, "full")); err != nil { + t.Fatalf("WriteCheckpoint: %v", err) + } + if _, err := o.RunFastRecon(context.Background()); err != nil { + t.Fatalf("RunFastRecon: %v", err) + } + if len(fake.Harnesses) != 3 { + t.Errorf("harness calls = %d, want 3 — a non-PR run ignores the cache", len(fake.Harnesses)) + } + }) +} + +// TestRunRecon_CacheNoteDiffers guards the one-word difference between +// _run_recon's and _run_fast_recon's cache notes. +func TestRunRecon_CacheNoteDiffers(t *testing.T) { + o, fake := newTestOrchestrator(t) + o.IsPRMode = true + if err := o.WriteCheckpoint(PhaseRecon, reconFixture(t, "full")); err != nil { + t.Fatalf("WriteCheckpoint: %v", err) + } + if _, err := o.RunRecon(context.Background()); err != nil { + t.Fatalf("RunRecon: %v", err) + } + if !containsMessage(fake.NoteMessages(), "Using cached recon for PR-mode scan") { + t.Errorf("_run_recon's note must end in ' scan'; got %q", fake.NoteMessages()) + } +} + +// TestHuntIncludePaths pins the PR-mode include-path substitution and its note. +func TestHuntIncludePaths(t *testing.T) { + t.Run("no PR mode keeps the configured include paths", func(t *testing.T) { + o, fake := newTestOrchestrator(t, func(in *schemas.AuditInput) { + in.IncludePaths = []string{"src/"} + }) + if got := o.huntIncludePaths(context.Background()); !reflect.DeepEqual(got, []string{"src/"}) { + t.Errorf("include paths = %v", got) + } + if len(fake.Notes) != 0 { + t.Errorf("no note expected, got %q", fake.NoteMessages()) + } + }) + + t.Run("PR mode with changed files substitutes the blast radius", func(t *testing.T) { + o, fake := newTestOrchestrator(t, func(in *schemas.AuditInput) { + in.IncludePaths = []string{"src/"} + }) + o.IsPRMode = true + o.DiffAnalysis = &diffanalysis.DiffAnalysis{ + ChangedFiles: []string{"a.py", "b.py"}, + BlastRadiusFiles: []string{"c.py"}, + AllRelevantFiles: []string{"a.py", "b.py", "c.py"}, + } + got := o.huntIncludePaths(context.Background()) + if want := []string{"a.py", "b.py", "c.py"}; !reflect.DeepEqual(got, want) { + t.Errorf("include paths = %v, want %v", got, want) + } + want := "PR-mode: scanning 3 files (2 changed + 1 blast radius)" + if !containsMessage(fake.NoteMessages(), want) { + t.Errorf("missing %q in %q", want, fake.NoteMessages()) + } + for _, note := range fake.Notes { + if note.Message == want && !reflect.DeepEqual(note.Tags, []string{"audit", "hunt", "pr-mode"}) { + t.Errorf("tags = %v", note.Tags) + } + } + }) + + t.Run("PR mode with an EMPTY diff keeps the configured paths", func(t *testing.T) { + o, fake := newTestOrchestrator(t, func(in *schemas.AuditInput) { + in.IncludePaths = []string{"src/"} + }) + o.IsPRMode = true + empty := diffanalysis.NewDiffAnalysis() + o.DiffAnalysis = &empty + if got := o.huntIncludePaths(context.Background()); !reflect.DeepEqual(got, []string{"src/"}) { + t.Errorf("include paths = %v, want the configured ones", got) + } + if len(fake.Notes) != 0 { + t.Errorf("no note expected for an empty diff, got %q", fake.NoteMessages()) + } + }) +} + +// TestRunFromCheckpoint_ReconBranch re-runs hunt and prove and writes both +// checkpoints back. +func TestRunFromCheckpoint_ReconBranch(t *testing.T) { + compliance.ClearAICache() + o, fake := newTestOrchestrator(t, func(in *schemas.AuditInput) { in.Depth = "quick" }) + fake.HarnessFn = emptyHarness() + fake.AIFn = appx.AIJSON(func(string) (json.RawMessage, error) { + return json.RawMessage(`{"mappings":[],"confidence":"low"}`), nil + }) + + if err := o.WriteCheckpoint(PhaseRecon, reconFixture(t, "minimal")); err != nil { + t.Fatalf("WriteCheckpoint: %v", err) + } + + if _, err := o.RunFromCheckpoint(context.Background(), "recon"); err != nil { + t.Fatalf("RunFromCheckpoint: %v", err) + } + for _, phase := range []string{PhaseHunt, PhaseProve} { + if _, statErr := os.Stat(o.CheckpointPath(phase)); statErr != nil { + t.Errorf("checkpoint-%s.json was not written: %v", phase, statErr) + } + } + if !containsMessage(fake.NoteMessages(), "Phase: HUNT") { + t.Errorf("missing the non-streaming HUNT note in %q", fake.NoteMessages()) + } + if !containsMessage(fake.NoteMessages(), "Phase: PROVE") { + t.Errorf("missing the non-streaming PROVE note in %q", fake.NoteMessages()) + } + // The streaming notes must NOT appear on this path. + for _, msg := range fake.NoteMessages() { + if strings.Contains(msg, "(streaming)") { + t.Errorf("unexpected streaming note %q", msg) + } + } +} + +// TestRunFromCheckpoint_HuntBranch reads the hunt checkpoint instead of +// re-running it. +func TestRunFromCheckpoint_HuntBranch(t *testing.T) { + compliance.ClearAICache() + o, fake := newTestOrchestrator(t, func(in *schemas.AuditInput) { in.Depth = "quick" }) + fake.HarnessFn = emptyHarness() + fake.AIFn = appx.AIJSON(func(string) (json.RawMessage, error) { + return json.RawMessage(`{"mappings":[],"confidence":"low"}`), nil + }) + + hunt := schemas.NewHuntResult() + hunt.TotalRaw = 9 + hunt.StrategiesRun = []string{"injection"} + if err := o.WriteCheckpoint(PhaseRecon, reconFixture(t, "minimal")); err != nil { + t.Fatalf("WriteCheckpoint: %v", err) + } + if err := o.WriteCheckpoint(PhaseHunt, hunt); err != nil { + t.Fatalf("WriteCheckpoint: %v", err) + } + + result, err := o.RunFromCheckpoint(context.Background(), "hunt") + if err != nil { + t.Fatalf("RunFromCheckpoint: %v", err) + } + if result.TotalRawFindings != 9 { + t.Errorf("total_raw_findings = %d, want the checkpointed 9", result.TotalRawFindings) + } + if containsMessage(fake.NoteMessages(), "Phase: HUNT") { + t.Errorf("the hunt branch must NOT re-run HUNT; notes = %q", fake.NoteMessages()) + } + if !containsMessage(fake.NoteMessages(), "Phase: PROVE") { + t.Errorf("missing the PROVE note in %q", fake.NoteMessages()) + } +} + +// TestRunProve_LimitsAndCounters pins the limited-hunt projection: only the +// findings list is truncated, and findings_not_verified is the remainder. +func TestRunProve_LimitsAndCounters(t *testing.T) { + o, fake := newTestOrchestrator(t, func(in *schemas.AuditInput) { + cap2 := 2 + in.MaxProvers = &cap2 + }) + fake.HarnessFn = emptyHarness() + fake.AIFn = appx.AIJSON(func(string) (json.RawMessage, error) { + return json.RawMessage(`{"verdict":"inconclusive","evidence_level":1,"rationale":"r","confidence":"low"}`), nil + }) + + hunt := schemas.NewHuntResult() + hunt.Findings = findingsFixture(t) // 5 + hunt.TotalRaw = 11 + hunt.DeduplicatedCount = 5 + hunt.StrategiesRun = []string{"injection"} + + verified, err := o.RunProve(context.Background(), schemas.NewReconResult(), hunt) + if err != nil { + t.Fatalf("RunProve: %v", err) + } + if len(verified) != 2 { + t.Errorf("verified = %d, want 2 (the prover cap)", len(verified)) + } + if o.FindingsNotVerified != 3 { + t.Errorf("findings_not_verified = %d, want 3", o.FindingsNotVerified) + } + if !containsMessage(fake.NoteMessages(), "Phase: PROVE") { + t.Errorf("missing the PROVE note in %q", fake.NoteMessages()) + } +} diff --git a/internal/secaf/orch/strategies.go b/internal/secaf/orch/strategies.go new file mode 100644 index 0000000000..6d34cd2a69 --- /dev/null +++ b/internal/secaf/orch/strategies.go @@ -0,0 +1,167 @@ +package orch + +import ( + "sort" + "strings" + + "github.com/Agent-Field/codeaf/internal/secaf/config" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// DefaultStrategies ports `AuditOrchestrator._default_strategies(recon)` +// (orchestrator.py:507). +// +// This is the ORCHESTRATOR's variant and it is NOT the same list as +// reasoners/phases.py `_default_strategies(recon, depth)`: +// +// base: injection, dos, ssrf, auth, data_exposure, config_secrets +// + crypto when security_context.crypto_usage +// + supply_chain when dependencies.direct_count > 0 +// + api_security when architecture.api_surface +// + business_logic when depth is standard or thorough +// + python_specific when depth is thorough and "python" in the lowered languages +// + javascript_specific when depth is thorough and any language is javascript/typescript +// +// Differences from the phases copy, both deliberate: XSS is NEVER added here +// (phases adds it at standard/thorough), and the two language-specific +// strategies exist only here. +// +// Python parity: +// +// - the depth comes from `self._depth_profile()`, i.e. the orchestrator's own +// input, not a parameter; +// - `"python" in {lang.lower() for lang in recon.languages}` is an EXACT +// match on the lowered name — "python3" does not qualify; +// - the javascript test uses `any(lang.lower() in {"javascript", "typescript"})`, +// so either language alone adds JAVASCRIPT_SPECIFIC, and both together add +// it once; +// - the trailing pass keeps FIRST-seen order and drops repeats. No branch can +// add a duplicate today, but it is reproduced. +func (o *AuditOrchestrator) DefaultStrategies(recon schemas.ReconResult) []schemas.HuntStrategy { + strategies := []schemas.HuntStrategy{ + schemas.HuntStrategyInjection, + schemas.HuntStrategyDos, + schemas.HuntStrategySSRF, + schemas.HuntStrategyAuth, + schemas.HuntStrategyDataExposure, + schemas.HuntStrategyConfigSecrets, + } + if len(recon.SecurityContext.CryptoUsage) > 0 { + strategies = append(strategies, schemas.HuntStrategyCrypto) + } + if recon.Dependencies.DirectCount > 0 { + strategies = append(strategies, schemas.HuntStrategySupplyChain) + } + if len(recon.Architecture.APISurface) > 0 { + strategies = append(strategies, schemas.HuntStrategyAPISecurity) + } + + depth := o.depthProfile() + if depth == config.DepthStandard || depth == config.DepthThorough { + strategies = append(strategies, schemas.HuntStrategyBusinessLogic) + } + if depth == config.DepthThorough && hasLoweredLanguage(recon.Languages, "python") { + strategies = append(strategies, schemas.HuntStrategyPythonSpecific) + } + if depth == config.DepthThorough && hasLoweredLanguage(recon.Languages, "javascript", "typescript") { + strategies = append(strategies, schemas.HuntStrategyJavascriptSpecific) + } + + ordered := make([]schemas.HuntStrategy, 0, len(strategies)) + for _, s := range strategies { + seen := false + for _, o := range ordered { + if o == s { + seen = true + break + } + } + if !seen { + ordered = append(ordered, s) + } + } + return ordered +} + +// hasLoweredLanguage reports whether any entry of languages, lowercased, is one +// of wanted. +func hasLoweredLanguage(languages []string, wanted ...string) bool { + for _, lang := range languages { + lowered := strings.ToLower(lang) + for _, w := range wanted { + if lowered == w { + return true + } + } + } + return false +} + +// PrioritizeFindings ports `AuditOrchestrator._prioritize_findings` +// (orchestrator.py:539) — severity first, confidence second, both descending, +// unknown values scoring 0. +// +// It is byte-identical to reasoners/phases.py's `_prioritize_findings` and to +// agents/prove's `_priority_sort`; SEC-AF carries three copies and each Go +// package ports its own. +// +// Python parity: `sorted(..., reverse=True)` is STABLE (CPython reverses, sorts, +// reverses), so ties keep input order — sort.SliceStable with a strictly-greater +// comparison. The input slice is not mutated. +func (o *AuditOrchestrator) PrioritizeFindings(findings []schemas.RawFinding) []schemas.RawFinding { + severityRank := map[schemas.Severity]int{ + schemas.SeverityCritical: 5, + schemas.SeverityHigh: 4, + schemas.SeverityMedium: 3, + schemas.SeverityLow: 2, + schemas.SeverityInfo: 1, + } + confidenceRank := map[schemas.Confidence]int{ + schemas.ConfidenceHigh: 3, + schemas.ConfidenceMedium: 2, + schemas.ConfidenceLow: 1, + } + + out := make([]schemas.RawFinding, len(findings)) + copy(out, findings) + sort.SliceStable(out, func(i, j int) bool { + si, sj := severityRank[out[i].EstimatedSeverity], severityRank[out[j].EstimatedSeverity] + if si != sj { + return si > sj + } + return confidenceRank[out[i].Confidence] > confidenceRank[out[j].Confidence] + }) + return out +} + +// ProverCap ports `AuditOrchestrator._prover_cap()` (orchestrator.py:557): +// +// defaults = {QUICK: 10, STANDARD: 30, THOROUGH: 10_000} +// default_cap = defaults[self._depth_profile()] +// if self.input.max_provers is None: return default_cap +// return max(0, min(self.input.max_provers, default_cap)) +// +// Same arithmetic as phases' `_prover_cap`, but the depth and max_provers come +// from the orchestrator's own input rather than from arguments. +func (o *AuditOrchestrator) ProverCap() int { + defaultCap := 30 + switch o.depthProfile() { + case config.DepthQuick: + defaultCap = 10 + case config.DepthStandard: + defaultCap = 30 + case config.DepthThorough: + defaultCap = 10_000 + } + if o.Input.MaxProvers == nil { + return defaultCap + } + v := *o.Input.MaxProvers + if v > defaultCap { + v = defaultCap + } + if v < 0 { + v = 0 + } + return v +} diff --git a/internal/secaf/orch/testdata/fallback_finding.json b/internal/secaf/orch/testdata/fallback_finding.json new file mode 100644 index 0000000000..bc337c5d1b --- /dev/null +++ b/internal/secaf/orch/testdata/fallback_finding.json @@ -0,0 +1,20 @@ +{ + "code_snippet": "cursor.execute(f\"SELECT * FROM users WHERE id = {user_id}\")", + "confidence": "high", + "cwe_id": "CWE-89", + "cwe_name": "SQL Injection", + "data_flow": [], + "description": "Potential injection from request parameter", + "end_line": 12, + "estimated_severity": "high", + "file_path": "src/users.py", + "finding_type": "sast", + "fingerprint": "fp-1", + "function_name": "get_user", + "hunter_strategy": "injection", + "id": "raw-1", + "owasp_category": "A03:2021", + "related_files": [], + "start_line": 10, + "title": "Potential SQL injection" +} diff --git a/internal/secaf/orch/testdata/findings_fixture.json b/internal/secaf/orch/testdata/findings_fixture.json new file mode 100644 index 0000000000..e31c0ec2d8 --- /dev/null +++ b/internal/secaf/orch/testdata/findings_fixture.json @@ -0,0 +1,87 @@ +[ + { + "code_snippet": "x", + "confidence": "high", + "cwe_id": "CWE-200", + "cwe_name": "Information Exposure", + "description": "d", + "end_line": 1, + "estimated_severity": "low", + "file_path": "a.py", + "finding_type": "sast", + "fingerprint": "fp-low-high", + "hunter_strategy": "injection", + "id": "low-high", + "related_files": [], + "start_line": 1, + "title": "Low severity, high confidence" + }, + { + "code_snippet": "x", + "confidence": "medium", + "cwe_id": "CWE-287", + "cwe_name": "Improper Authentication", + "description": "d", + "end_line": 2, + "estimated_severity": "medium", + "file_path": "b.py", + "finding_type": "sast", + "fingerprint": "fp-medium-a", + "hunter_strategy": "auth", + "id": "medium-medium-a", + "related_files": [], + "start_line": 2, + "title": "Medium A" + }, + { + "code_snippet": "x", + "confidence": "low", + "cwe_id": "CWE-89", + "cwe_name": "SQL Injection", + "description": "d", + "end_line": 3, + "estimated_severity": "critical", + "file_path": "c.py", + "finding_type": "sast", + "fingerprint": "fp-critical-low", + "hunter_strategy": "injection", + "id": "critical-low", + "related_files": [], + "start_line": 3, + "title": "Critical, low confidence" + }, + { + "code_snippet": "x", + "confidence": "medium", + "cwe_id": "CWE-400", + "cwe_name": "Resource Exhaustion", + "description": "d", + "end_line": 4, + "estimated_severity": "medium", + "file_path": "d.py", + "finding_type": "sast", + "fingerprint": "fp-medium-b", + "hunter_strategy": "dos", + "id": "medium-medium-b", + "related_files": [], + "start_line": 4, + "title": "Medium B" + }, + { + "code_snippet": "x", + "confidence": "low", + "cwe_id": "CWE-79", + "cwe_name": "XSS", + "description": "d", + "end_line": 5, + "estimated_severity": "info", + "file_path": "e.py", + "finding_type": "sast", + "fingerprint": "fp-info", + "hunter_strategy": "xss", + "id": "unknown-unknown", + "related_files": [], + "start_line": 5, + "title": "Unrecognised severity" + } +] diff --git a/internal/secaf/orch/testdata/golden/checkpoint_created_at.json b/internal/secaf/orch/testdata/golden/checkpoint_created_at.json new file mode 100644 index 0000000000..b1cdc37c87 --- /dev/null +++ b/internal/secaf/orch/testdata/golden/checkpoint_created_at.json @@ -0,0 +1,3 @@ +{ + "pinned": "2026-01-02T03:04:05.123456+00:00" +} diff --git a/internal/secaf/orch/testdata/golden/checkpoint_prove.txt b/internal/secaf/orch/testdata/golden/checkpoint_prove.txt new file mode 100644 index 0000000000..858cec25ca --- /dev/null +++ b/internal/secaf/orch/testdata/golden/checkpoint_prove.txt @@ -0,0 +1,44 @@ +{ + "phase": "prove", + "created_at": "2026-01-02T03:04:05.123456+00:00", + "data": [ + { + "id": "vf-1", + "fingerprint": "fp-1", + "title": "SQL injection in user lookup", + "description": "User id flows unsanitised into an f-string query", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "SQL Injection", + "owasp_category": null, + "tags": [], + "verdict": "confirmed", + "evidence_level": 4, + "rationale": "Traced source to sink", + "severity": "high", + "cvss_v4": null, + "epss": null, + "exploitability_score": 0.0, + "proof": null, + "location": { + "file_path": "src/users.py", + "start_line": 10, + "end_line": 12, + "start_column": null, + "end_column": null, + "function_name": null, + "code_snippet": null + }, + "related_locations": [], + "chain_id": null, + "chain_step": null, + "enables": null, + "compliance": [], + "reproduction_steps": [], + "remediation": null, + "sarif_rule_id": "sec-af/sast/sql-injection", + "sarif_security_severity": 0.0, + "drop_reason": null + } + ] +} \ No newline at end of file diff --git a/internal/secaf/orch/testdata/golden/checkpoint_prove_empty.txt b/internal/secaf/orch/testdata/golden/checkpoint_prove_empty.txt new file mode 100644 index 0000000000..77e2cea657 --- /dev/null +++ b/internal/secaf/orch/testdata/golden/checkpoint_prove_empty.txt @@ -0,0 +1,5 @@ +{ + "phase": "prove_empty", + "created_at": "2026-01-02T03:04:05.123456+00:00", + "data": [] +} \ No newline at end of file diff --git a/internal/secaf/orch/testdata/golden/checkpoint_recon.txt b/internal/secaf/orch/testdata/golden/checkpoint_recon.txt new file mode 100644 index 0000000000..21a42248ac --- /dev/null +++ b/internal/secaf/orch/testdata/golden/checkpoint_recon.txt @@ -0,0 +1,43 @@ +{ + "phase": "recon", + "created_at": "2026-01-02T03:04:05.123456+00:00", + "data": { + "architecture": { + "app_type": null, + "modules": [], + "entry_points": [], + "trust_boundaries": [], + "services": [], + "api_surface": [] + }, + "data_flows": { + "flows": [], + "sanitization_points": [], + "sinks": [] + }, + "dependencies": { + "sbom": [], + "known_cves": [], + "outdated": [], + "direct_count": 0, + "transitive_count": 0 + }, + "config": { + "secrets": [], + "misconfigs": [] + }, + "security_context": { + "auth_model": "", + "auth_details": "", + "crypto_usage": [], + "framework_security": [], + "security_headers": [], + "deployment_signals": [] + }, + "languages": [], + "frameworks": [], + "lines_of_code": 0, + "file_count": 0, + "recon_duration_seconds": 0.0 + } +} \ No newline at end of file diff --git a/internal/secaf/orch/testdata/golden/default_strategies.json b/internal/secaf/orch/testdata/golden/default_strategies.json new file mode 100644 index 0000000000..406342f75b --- /dev/null +++ b/internal/secaf/orch/testdata/golden/default_strategies.json @@ -0,0 +1,149 @@ +{ + "full|": [ + "injection", + "dos", + "ssrf", + "auth", + "data_exposure", + "config_secrets", + "crypto", + "supply_chain", + "api_security", + "business_logic" + ], + "full|QUICK": [ + "injection", + "dos", + "ssrf", + "auth", + "data_exposure", + "config_secrets", + "crypto", + "supply_chain", + "api_security" + ], + "full|Thorough": [ + "injection", + "dos", + "ssrf", + "auth", + "data_exposure", + "config_secrets", + "crypto", + "supply_chain", + "api_security", + "business_logic", + "python_specific", + "javascript_specific" + ], + "full|bogus": [ + "injection", + "dos", + "ssrf", + "auth", + "data_exposure", + "config_secrets", + "crypto", + "supply_chain", + "api_security", + "business_logic" + ], + "full|quick": [ + "injection", + "dos", + "ssrf", + "auth", + "data_exposure", + "config_secrets", + "crypto", + "supply_chain", + "api_security" + ], + "full|standard": [ + "injection", + "dos", + "ssrf", + "auth", + "data_exposure", + "config_secrets", + "crypto", + "supply_chain", + "api_security", + "business_logic" + ], + "full|thorough": [ + "injection", + "dos", + "ssrf", + "auth", + "data_exposure", + "config_secrets", + "crypto", + "supply_chain", + "api_security", + "business_logic", + "python_specific", + "javascript_specific" + ], + "minimal|": [ + "injection", + "dos", + "ssrf", + "auth", + "data_exposure", + "config_secrets", + "business_logic" + ], + "minimal|QUICK": [ + "injection", + "dos", + "ssrf", + "auth", + "data_exposure", + "config_secrets" + ], + "minimal|Thorough": [ + "injection", + "dos", + "ssrf", + "auth", + "data_exposure", + "config_secrets", + "business_logic" + ], + "minimal|bogus": [ + "injection", + "dos", + "ssrf", + "auth", + "data_exposure", + "config_secrets", + "business_logic" + ], + "minimal|quick": [ + "injection", + "dos", + "ssrf", + "auth", + "data_exposure", + "config_secrets" + ], + "minimal|standard": [ + "injection", + "dos", + "ssrf", + "auth", + "data_exposure", + "config_secrets", + "business_logic" + ], + "minimal|thorough": [ + "injection", + "dos", + "ssrf", + "auth", + "data_exposure", + "config_secrets", + "business_logic" + ] +} diff --git a/internal/secaf/orch/testdata/golden/merge_recon_findings.json b/internal/secaf/orch/testdata/golden/merge_recon_findings.json new file mode 100644 index 0000000000..7c87730f63 --- /dev/null +++ b/internal/secaf/orch/testdata/golden/merge_recon_findings.json @@ -0,0 +1,60 @@ +{ + "already_present": [ + "recon", + "injection" + ], + "empty_recon_is_identity": true, + "merged": { + "chain_count": 0, + "chains": [], + "deduplicated_count": 2, + "findings": [ + { + "code_snippet": "x", + "confidence": "low", + "cwe_id": "CWE-89", + "cwe_name": "SQL Injection", + "data_flow": null, + "description": "d", + "end_line": 3, + "estimated_severity": "critical", + "file_path": "c.py", + "finding_type": "sast", + "fingerprint": "fp-critical-low", + "function_name": null, + "hunter_strategy": "injection", + "id": "critical-low", + "owasp_category": null, + "related_files": [], + "start_line": 3, + "title": "Critical, low confidence" + }, + { + "code_snippet": "x", + "confidence": "high", + "cwe_id": "CWE-200", + "cwe_name": "Information Exposure", + "data_flow": null, + "description": "d", + "end_line": 1, + "estimated_severity": "low", + "file_path": "a.py", + "finding_type": "sast", + "fingerprint": "fp-low-high", + "function_name": null, + "hunter_strategy": "injection", + "id": "low-high", + "owasp_category": null, + "related_files": [], + "start_line": 1, + "title": "Low severity, high confidence" + } + ], + "hunt_duration_seconds": 0.0, + "strategies_run": [ + "recon", + "injection" + ], + "total_raw": 2 + } +} diff --git a/internal/secaf/orch/testdata/golden/prioritize_findings.json b/internal/secaf/orch/testdata/golden/prioritize_findings.json new file mode 100644 index 0000000000..10314d57f9 --- /dev/null +++ b/internal/secaf/orch/testdata/golden/prioritize_findings.json @@ -0,0 +1,7 @@ +[ + "critical-low", + "medium-medium-a", + "medium-medium-b", + "low-high", + "unknown-unknown" +] diff --git a/internal/secaf/orch/testdata/golden/progress_fields.json b/internal/secaf/orch/testdata/golden/progress_fields.json new file mode 100644 index 0000000000..ba493bc31a --- /dev/null +++ b/internal/secaf/orch/testdata/golden/progress_fields.json @@ -0,0 +1,57 @@ +{ + "hunt_done": { + "agents_completed": 1, + "agents_running": 0, + "agents_total": 1, + "cost_so_far_usd": 0.1235, + "elapsed_seconds": 2.5, + "estimated_remaining_seconds": 0.0, + "findings_so_far": 7, + "phase": "hunt", + "phase_progress": 1.0 + }, + "overshoot": { + "agents_completed": 5, + "agents_running": 0, + "agents_total": 2, + "cost_so_far_usd": 0.1235, + "elapsed_seconds": 2.5, + "estimated_remaining_seconds": 0.0, + "findings_so_far": 3, + "phase": "prove", + "phase_progress": 1.0 + }, + "recon_full": { + "agents_completed": 2, + "agents_running": 0, + "agents_total": 2, + "cost_so_far_usd": 0.1235, + "elapsed_seconds": 2.5, + "estimated_remaining_seconds": 0.0, + "findings_so_far": 0, + "phase": "recon", + "phase_progress": 1.0 + }, + "recon_half": { + "agents_completed": 1, + "agents_running": 1, + "agents_total": 2, + "cost_so_far_usd": 0.1235, + "elapsed_seconds": 2.5, + "estimated_remaining_seconds": 2.5, + "findings_so_far": 0, + "phase": "recon", + "phase_progress": 0.5 + }, + "zero_total": { + "agents_completed": 0, + "agents_running": 0, + "agents_total": 0, + "cost_so_far_usd": 0.1235, + "elapsed_seconds": 2.5, + "estimated_remaining_seconds": 0.0, + "findings_so_far": 0, + "phase": "prove", + "phase_progress": 0.0 + } +} diff --git a/internal/secaf/orch/testdata/golden/progress_model_dump_json.json b/internal/secaf/orch/testdata/golden/progress_model_dump_json.json new file mode 100644 index 0000000000..c75d4ca229 --- /dev/null +++ b/internal/secaf/orch/testdata/golden/progress_model_dump_json.json @@ -0,0 +1,4 @@ +{ + "fractional": "{\"phase\":\"hunt\",\"phase_progress\":0.5,\"agents_total\":4,\"agents_completed\":2,\"agents_running\":2,\"findings_so_far\":13,\"elapsed_seconds\":1.25,\"estimated_remaining_seconds\":1.25,\"cost_so_far_usd\":0.1235}", + "unit_progress": "{\"phase\":\"recon\",\"phase_progress\":1.0,\"agents_total\":2,\"agents_completed\":2,\"agents_running\":0,\"findings_so_far\":0,\"elapsed_seconds\":2.5,\"estimated_remaining_seconds\":0.0,\"cost_so_far_usd\":0.0}" +} diff --git a/internal/secaf/orch/testdata/golden/progress_notes.json b/internal/secaf/orch/testdata/golden/progress_notes.json new file mode 100644 index 0000000000..0314e6a3d8 --- /dev/null +++ b/internal/secaf/orch/testdata/golden/progress_notes.json @@ -0,0 +1,7 @@ +{ + "hunt_done": "{\"phase\":\"hunt\",\"phase_progress\":1.0,\"agents_total\":1,\"agents_completed\":1,\"agents_running\":0,\"findings_so_far\":7,\"elapsed_seconds\":2.5,\"estimated_remaining_seconds\":0.0,\"cost_so_far_usd\":0.1235}", + "overshoot": "{\"phase\":\"prove\",\"phase_progress\":1.0,\"agents_total\":2,\"agents_completed\":5,\"agents_running\":0,\"findings_so_far\":3,\"elapsed_seconds\":2.5,\"estimated_remaining_seconds\":0.0,\"cost_so_far_usd\":0.1235}", + "recon_full": "{\"phase\":\"recon\",\"phase_progress\":1.0,\"agents_total\":2,\"agents_completed\":2,\"agents_running\":0,\"findings_so_far\":0,\"elapsed_seconds\":2.5,\"estimated_remaining_seconds\":0.0,\"cost_so_far_usd\":0.1235}", + "recon_half": "{\"phase\":\"recon\",\"phase_progress\":0.5,\"agents_total\":2,\"agents_completed\":1,\"agents_running\":1,\"findings_so_far\":0,\"elapsed_seconds\":2.5,\"estimated_remaining_seconds\":2.5,\"cost_so_far_usd\":0.1235}", + "zero_total": "{\"phase\":\"prove\",\"phase_progress\":0.0,\"agents_total\":0,\"agents_completed\":0,\"agents_running\":0,\"findings_so_far\":0,\"elapsed_seconds\":2.5,\"estimated_remaining_seconds\":0.0,\"cost_so_far_usd\":0.1235}" +} diff --git a/internal/secaf/orch/testdata/golden/prover_cap.json b/internal/secaf/orch/testdata/golden/prover_cap.json new file mode 100644 index 0000000000..31178745e8 --- /dev/null +++ b/internal/secaf/orch/testdata/golden/prover_cap.json @@ -0,0 +1,11 @@ +{ + "bogus|null": 30, + "quick|3": 3, + "quick|50": 10, + "quick|null": 10, + "standard|-5": 0, + "standard|0": 0, + "standard|null": 30, + "thorough|12345": 10000, + "thorough|null": 10000 +} diff --git a/internal/secaf/orch/testdata/golden/reachability_fallback_tags.json b/internal/secaf/orch/testdata/golden/reachability_fallback_tags.json new file mode 100644 index 0000000000..b563c9a84d --- /dev/null +++ b/internal/secaf/orch/testdata/golden/reachability_fallback_tags.json @@ -0,0 +1,5 @@ +{ + "tags_after_gate_failure": [ + "requires_auth" + ] +} diff --git a/internal/secaf/orch/testdata/golden/reachability_summary.txt b/internal/secaf/orch/testdata/golden/reachability_summary.txt new file mode 100644 index 0000000000..5476e6cc59 --- /dev/null +++ b/internal/secaf/orch/testdata/golden/reachability_summary.txt @@ -0,0 +1,5 @@ +Finding: SQL injection in user lookup +Description: User id flows unsanitised into an f-string query +CWE: CWE-89 +File: src/users.py:10 +Verdict: confirmed \ No newline at end of file diff --git a/internal/secaf/orch/testdata/golden/verified_finding_fallback.json b/internal/secaf/orch/testdata/golden/verified_finding_fallback.json new file mode 100644 index 0000000000..3d36995289 --- /dev/null +++ b/internal/secaf/orch/testdata/golden/verified_finding_fallback.json @@ -0,0 +1,38 @@ +{ + "chain_id": null, + "chain_step": null, + "compliance": [], + "cvss_v4": null, + "cwe_id": "CWE-89", + "cwe_name": "SQL Injection", + "description": "Potential injection from request parameter", + "drop_reason": null, + "enables": null, + "epss": null, + "evidence_level": 1, + "exploitability_score": 0.0, + "finding_type": "sast", + "fingerprint": "fp-1", + "id": "raw-1", + "location": { + "code_snippet": "cursor.execute(f\"SELECT * FROM users WHERE id = {user_id}\")", + "end_column": null, + "end_line": 12, + "file_path": "src/users.py", + "function_name": "get_user", + "start_column": null, + "start_line": 10 + }, + "owasp_category": "A03:2021", + "proof": null, + "rationale": "Automated proof unavailable; requires manual review.", + "related_locations": [], + "remediation": null, + "reproduction_steps": [], + "sarif_rule_id": "sec-af/sast/cwe-89", + "sarif_security_severity": 0.0, + "severity": "high", + "tags": [], + "title": "Potential SQL injection", + "verdict": "inconclusive" +} diff --git a/internal/secaf/orch/testdata/recon_fixture.json b/internal/secaf/orch/testdata/recon_fixture.json new file mode 100644 index 0000000000..0de5f128a7 --- /dev/null +++ b/internal/secaf/orch/testdata/recon_fixture.json @@ -0,0 +1,143 @@ +{ + "full": { + "architecture": { + "api_surface": [ + { + "auth_required": true, + "file_path": "src/api/users.py", + "handler": "list_users", + "line": 12, + "method": "GET", + "path": "/users", + "rate_limited": false + }, + { + "auth_required": false, + "file_path": "src/api/users.py", + "handler": "delete_user", + "line": 40, + "method": "DELETE", + "path": "/users/{id}", + "rate_limited": false + } + ], + "app_type": "web_api", + "entry_points": [], + "modules": [ + { + "dependencies": [], + "language": "Python", + "name": "api", + "path": "src/api" + }, + { + "dependencies": [], + "language": "TypeScript", + "name": "web", + "path": "web" + }, + { + "dependencies": [], + "language": "python", + "name": "legacy", + "path": "legacy" + }, + { + "dependencies": [], + "language": "", + "name": "blank", + "path": "blank" + } + ], + "services": [], + "trust_boundaries": [] + }, + "config": { + "misconfigs": [ + { + "category": "tls", + "file_path": "nginx.conf", + "id": "misconfig-1", + "key": "ssl_protocols", + "line": 3, + "risk": "TLS 1.0 enabled", + "value": "TLSv1" + } + ], + "secrets": [ + { + "confidence": "high", + "file_path": "src/config.py", + "id": "secret-1", + "line": 7, + "match": "API_KEY = \"sk-live-123\"", + "secret_type": "api_key" + } + ] + }, + "data_flows": {}, + "dependencies": { + "direct_count": 5, + "known_cves": [ + { + "cve_id": "CVE-2024-0001", + "cvss_v4_score": 9.1, + "direct": true, + "fixed_version": "2.32.0", + "installed_version": "2.20.0", + "package": "requests" + } + ], + "outdated": [], + "sbom": [], + "transitive_count": 20 + }, + "file_count": 42, + "frameworks": [ + "django" + ], + "languages": [ + "python", + "typescript" + ], + "lines_of_code": 5000, + "security_context": { + "auth_details": "Bearer token", + "auth_model": "jwt", + "crypto_usage": [ + { + "algorithm": "TLSv1.0", + "is_weak": true, + "usage_context": "legacy tls terminator" + }, + { + "algorithm": "AES-256-GCM", + "is_weak": false, + "usage_context": "at-rest encryption" + } + ], + "deployment_signals": [], + "framework_security": [ + "django-rest-framework", + "", + "django-rest-framework", + "Django" + ], + "security_headers": [] + } + }, + "minimal": { + "architecture": {}, + "config": {}, + "data_flows": {}, + "dependencies": {}, + "file_count": 0, + "frameworks": [], + "languages": [], + "lines_of_code": 0, + "security_context": { + "auth_details": "", + "auth_model": "" + } + } +} diff --git a/internal/secaf/orch/testdata/verified_fixture.json b/internal/secaf/orch/testdata/verified_fixture.json new file mode 100644 index 0000000000..21ce32cd4d --- /dev/null +++ b/internal/secaf/orch/testdata/verified_fixture.json @@ -0,0 +1,22 @@ +{ + "cwe_id": "CWE-89", + "cwe_name": "SQL Injection", + "description": "User id flows unsanitised into an f-string query", + "evidence_level": 4, + "exploitability_score": 0.0, + "finding_type": "sast", + "fingerprint": "fp-1", + "id": "vf-1", + "location": { + "end_line": 12, + "file_path": "src/users.py", + "start_line": 10 + }, + "rationale": "Traced source to sink", + "sarif_rule_id": "sec-af/sast/sql-injection", + "sarif_security_severity": 0.0, + "severity": "high", + "tags": [], + "title": "SQL injection in user lookup", + "verdict": "confirmed" +} diff --git a/internal/secaf/output/compliance_report.go b/internal/secaf/output/compliance_report.go new file mode 100644 index 0000000000..fd9d3277c3 --- /dev/null +++ b/internal/secaf/output/compliance_report.go @@ -0,0 +1,329 @@ +package output + +import ( + "fmt" + "sort" + "strings" + "time" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// This file ports src/sec_af/output/compliance_report.py — the +// framework-organised Markdown report meant for PDF conversion. + +// severityIconTable ports the literal dict inside _severity_icon. +var severityIconTable = map[string]string{ + "critical": "CRITICAL", + "high": "HIGH", + "medium": "MEDIUM", + "low": "LOW", +} + +// verdictLabelTable ports the literal dict inside _verdict_label. +var verdictLabelTable = map[string]string{ + "confirmed": "Confirmed", + "likely": "Likely", + "inconclusive": "Inconclusive", + "not_exploitable": "Not Exploitable", +} + +// severityOrder is the fixed iteration order of _render_executive_summary's +// severity table. +var severityOrder = []string{"critical", "high", "medium", "low", "info"} + +// severityIcon ports _severity_icon: note the default is "INFO", so "info" and +// any unknown severity share a label. +func severityIcon(severity string) string { + if icon, ok := severityIconTable[strings.ToLower(severity)]; ok { + return icon + } + return "INFO" +} + +// verdictLabel ports _verdict_label: an unknown verdict is echoed back +// verbatim (`dict.get(v, v)`). +func verdictLabel(verdictValue string) string { + if label, ok := verdictLabelTable[verdictValue]; ok { + return label + } + return verdictValue +} + +// renderComplianceHeader ports _render_header. +// +// `now` is the value Python reads from `datetime.now(UTC)`; it is a parameter +// rather than a call so the golden test can pin it — see +// GenerateComplianceReportAt. +func renderComplianceHeader(result schemas.SecurityAuditResult, now time.Time, heading string) []string { + branch := "N/A" + if result.Branch != nil && *result.Branch != "" { + branch = *result.Branch + } + return []string{ + heading, + "", + fmt.Sprintf("**Generated:** %s", now.UTC().Format("2006-01-02 15:04")+" UTC"), + fmt.Sprintf("**Repository:** %s", result.Repository), + fmt.Sprintf("**Commit:** %s", result.CommitSha), + fmt.Sprintf("**Branch:** %s", branch), + fmt.Sprintf("**Scan Depth:** %s", result.DepthProfile), + fmt.Sprintf("**Total Findings:** %d", len(result.Findings)), + "", + "---", + "", + } +} + +// renderExecutiveSummary ports _render_executive_summary. +func renderExecutiveSummary(result schemas.SecurityAuditResult) []string { + summaryLine := fmt.Sprintf( + "This report covers a security audit of `%s` at commit `%s`. "+ + "The scan identified **%d** findings, of which **%d** are confirmed exploitable.", + result.Repository, headRunes(result.CommitSha, 8), len(result.Findings), result.Confirmed) + + lines := []string{ + "## Executive Summary", + "", + summaryLine, + "", + "### Verdict Distribution", + "", + "| Verdict | Count |", + "|---------|-------|", + fmt.Sprintf("| Confirmed | %d |", result.Confirmed), + fmt.Sprintf("| Likely | %d |", result.Likely), + fmt.Sprintf("| Inconclusive | %d |", result.Inconclusive), + fmt.Sprintf("| Not Exploitable | %d |", result.NotExploitable), + "", + fmt.Sprintf("**Noise Reduction:** %.1f%%", result.NoiseReductionPct), + "", + } + + if len(result.BySeverity) > 0 { + lines = append(lines, + "### Severity Distribution", + "", + "| Severity | Count |", + "|----------|-------|", + ) + for _, severity := range severityOrder { + count := result.BySeverity[severity] + if count > 0 { + lines = append(lines, fmt.Sprintf("| %s | %d |", severityIcon(severity), count)) + } + } + lines = append(lines, "") + } + return lines +} + +// renderComplianceSection ports _render_compliance_section: gaps grouped by +// framework (frameworks sorted, gaps within a framework sorted by control id), +// with the CWE column truncated at five entries. +func renderComplianceSection(result schemas.SecurityAuditResult) []string { + if len(result.ComplianceGaps) == 0 { + return []string{"## Compliance Status", "", "No compliance gaps identified.", ""} + } + + byFramework := map[string][]schemas.ComplianceGap{} + frameworks := make([]string, 0) + for _, gap := range result.ComplianceGaps { + if _, seen := byFramework[gap.Framework]; !seen { + frameworks = append(frameworks, gap.Framework) + } + byFramework[gap.Framework] = append(byFramework[gap.Framework], gap) + } + sort.Strings(frameworks) + + lines := []string{"## Compliance Gap Analysis", ""} + for _, framework := range frameworks { + gaps := append([]schemas.ComplianceGap(nil), byFramework[framework]...) + // Python parity: `sorted(gaps, key=lambda g: g.control_id)` is a STABLE + // sort on the control id alone, so equal control ids keep the order they + // had in result.compliance_gaps. + sort.SliceStable(gaps, func(i, j int) bool { return gaps[i].ControlID < gaps[j].ControlID }) + + lines = append(lines, + fmt.Sprintf("### %s", framework), + "", + "| Control ID | Control Name | Findings | Max Severity | CWEs |", + "|-----------|-------------|----------|-------------|------|", + ) + for _, gap := range gaps { + shown := gap.CweIDs + if len(shown) > 5 { + shown = shown[:5] + } + cweStr := strings.Join(shown, ", ") + if len(gap.CweIDs) > 5 { + cweStr += fmt.Sprintf(" (+%d more)", len(gap.CweIDs)-5) + } + lines = append(lines, fmt.Sprintf("| %s | %s | %d | %s | %s |", + gap.ControlID, gap.ControlName, gap.FindingCount, severityIcon(gap.MaxSeverity), cweStr)) + } + lines = append(lines, "") + } + return lines +} + +// renderFindingsByFramework ports _render_findings_by_framework. +// +// Python parity: a finding is listed once per DISTINCT framework among its +// compliance mappings (the `seen_frameworks` set), and findings with no +// mappings at all land in the trailing "Uncategorized Findings" section. +func renderFindingsByFramework(result schemas.SecurityAuditResult) []string { + if len(result.Findings) == 0 { + return []string{"## Detailed Findings", "", "No findings to report.", ""} + } + + frameworkFindings := map[string][]schemas.VerifiedFinding{} + frameworks := make([]string, 0) + uncategorized := make([]schemas.VerifiedFinding, 0) + + for _, finding := range result.Findings { + if len(finding.Compliance) == 0 { + uncategorized = append(uncategorized, finding) + continue + } + seen := map[string]struct{}{} + for _, mapping := range finding.Compliance { + if _, ok := seen[mapping.Framework]; ok { + continue + } + if _, known := frameworkFindings[mapping.Framework]; !known { + frameworks = append(frameworks, mapping.Framework) + } + frameworkFindings[mapping.Framework] = append(frameworkFindings[mapping.Framework], finding) + seen[mapping.Framework] = struct{}{} + } + } + sort.Strings(frameworks) + + lines := []string{"## Detailed Findings by Framework", ""} + for _, framework := range frameworks { + lines = append(lines, fmt.Sprintf("### %s", framework), "") + for _, finding := range frameworkFindings[framework] { + lines = append(lines, + fmt.Sprintf("#### %s", finding.Title), + "", + fmt.Sprintf("- **Verdict:** %s", verdictLabel(string(finding.Verdict))), + fmt.Sprintf("- **Severity:** %s", severityIcon(string(finding.Severity))), + fmt.Sprintf("- **CWE:** %s (%s)", finding.CweID, finding.CweName), + fmt.Sprintf("- **Location:** `%s:%d`", finding.Location.FilePath, finding.Location.StartLine), + fmt.Sprintf("- **Evidence Level:** %d/6", int(finding.EvidenceLevel)), + fmt.Sprintf("- **Exploitability Score:** %.1f/10", finding.ExploitabilityScore), + "", + ) + if finding.Rationale != "" { + lines = append(lines, fmt.Sprintf("**Rationale:** %s", finding.Rationale), "") + } + complianceLines := make([]string, 0, len(finding.Compliance)) + for _, mapping := range finding.Compliance { + complianceLines = append(complianceLines, + fmt.Sprintf(" - %s %s: %s", mapping.Framework, mapping.ControlID, mapping.ControlName)) + } + if len(complianceLines) > 0 { + lines = append(lines, "**Compliance Mappings:**") + lines = append(lines, complianceLines...) + lines = append(lines, "") + } + } + } + + if len(uncategorized) > 0 { + lines = append(lines, "### Uncategorized Findings", "") + for _, finding := range uncategorized { + lines = append(lines, + fmt.Sprintf("#### %s", finding.Title), + "", + fmt.Sprintf("- **Verdict:** %s", verdictLabel(string(finding.Verdict))), + fmt.Sprintf("- **Severity:** %s", severityIcon(string(finding.Severity))), + fmt.Sprintf("- **CWE:** %s", finding.CweID), + fmt.Sprintf("- **Location:** `%s:%d`", finding.Location.FilePath, finding.Location.StartLine), + "", + ) + } + } + return lines +} + +// renderComplianceFooter ports _render_footer. +func renderComplianceFooter(result schemas.SecurityAuditResult, signature string) []string { + return []string{ + "---", + "", + "## Audit Metadata", + "", + fmt.Sprintf("- **Duration:** %.1fs", result.DurationSeconds), + fmt.Sprintf("- **Agent Invocations:** %d", result.AgentInvocations), + fmt.Sprintf("- **Cost:** $%.2f", result.CostUsd), + fmt.Sprintf("- **Strategies Used:** %s", strings.Join(result.StrategiesUsed, ", ")), + "", + signature, + } +} + +// GenerateComplianceReport ports generate_compliance_report: a +// compliance-focused Markdown report, structured for direct PDF conversion via +// pandoc/weasyprint. +// +// The header stamps the CURRENT time, exactly as the Python function does +// (`datetime.now(UTC)`), so two calls a minute apart differ. Tests and any +// caller that needs a reproducible document use GenerateComplianceReportAt. +func GenerateComplianceReport(result schemas.SecurityAuditResult) string { + return GenerateComplianceReportAt(result, time.Now().UTC()) +} + +// GenerateComplianceReportAt is GenerateComplianceReport with the "Generated:" +// timestamp supplied by the caller. +// +// This seam is Go-only: Python reads the clock inline, so pinning it there +// takes monkeypatching (which is what scripts/gen_golden.py does to produce the +// committed golden). Nothing about the rest of the document depends on it. +func GenerateComplianceReportAt(result schemas.SecurityAuditResult, now time.Time) string { + return GenerateComplianceReportAs(result, now, SecAfCompliance) +} + +// ComplianceBrand is who a compliance report says wrote it: its first line and +// its last. sec-af's is [SecAfCompliance], which the golden holds; codeaf's +// program signs its own (internal/secaf's sec). +type ComplianceBrand struct { + Heading, Signature string +} + +// SecAfCompliance is sec-af's own heading and signature. +var SecAfCompliance = ComplianceBrand{ + Heading: "# SEC-AF Compliance Report", + Signature: "*Report generated by SEC-AF -- Composite Intelligence Security Auditor*", +} + +// GenerateComplianceReportAs is [GenerateComplianceReportAt] signed by brand. +func GenerateComplianceReportAs(result schemas.SecurityAuditResult, now time.Time, brand ComplianceBrand) string { + sections := [][]string{ + renderComplianceHeader(result, now, brand.Heading), + renderExecutiveSummary(result), + renderComplianceSection(result), + renderFindingsByFramework(result), + renderComplianceFooter(result, brand.Signature), + } + lines := make([]string, 0, 64) + for _, section := range sections { + lines = append(lines, section...) + } + return strings.Join(lines, "\n") +} + +// headRunes ports Python's `s[:n]` string slice: it counts CHARACTERS, and a +// string shorter than n is returned whole rather than panicking (which +// s[:n] on a Go string would do). +func headRunes(s string, n int) string { + count := 0 + for i := range s { + if count == n { + return s[:i] + } + count++ + } + return s +} diff --git a/internal/secaf/output/compliance_report_test.go b/internal/secaf/output/compliance_report_test.go new file mode 100644 index 0000000000..f5457247c8 --- /dev/null +++ b/internal/secaf/output/compliance_report_test.go @@ -0,0 +1,204 @@ +package output + +import ( + "strings" + "testing" + "time" +) + +// This file ports tests/test_compliance_report.py. Its `_make_result()` fixture +// is the testdata/audit_result_report.json fixture (gen_golden_output.py +// builds it from the same values, with the finding's uuid4 id pinned). + +// complianceReport renders the ported fixture with the frozen clock. +func complianceReport(t *testing.T) string { + t.Helper() + return GenerateComplianceReportAt(loadFixture(t, "audit_result_report"), goldenComplianceReportAt) +} + +// TestComplianceReportContainsHeader ports test_compliance_report_contains_header. +func TestComplianceReportContainsHeader(t *testing.T) { + report := complianceReport(t) + for _, want := range []string{"# SEC-AF Compliance Report", "https://github.com/test/repo"} { + if !strings.Contains(report, want) { + t.Errorf("report is missing %q", want) + } + } +} + +// TestComplianceReportContainsExecutiveSummary ports +// test_compliance_report_contains_executive_summary. +func TestComplianceReportContainsExecutiveSummary(t *testing.T) { + report := complianceReport(t) + for _, want := range []string{"Executive Summary", "Confirmed"} { + if !strings.Contains(report, want) { + t.Errorf("report is missing %q", want) + } + } +} + +// TestComplianceReportContainsComplianceGaps ports +// test_compliance_report_contains_compliance_gaps. +func TestComplianceReportContainsComplianceGaps(t *testing.T) { + report := complianceReport(t) + for _, want := range []string{"Compliance Gap Analysis", "OWASP", "A03:2021"} { + if !strings.Contains(report, want) { + t.Errorf("report is missing %q", want) + } + } +} + +// TestComplianceReportContainsFindings ports +// test_compliance_report_contains_findings. +func TestComplianceReportContainsFindings(t *testing.T) { + report := complianceReport(t) + for _, want := range []string{"Test Finding", "CWE-89"} { + if !strings.Contains(report, want) { + t.Errorf("report is missing %q", want) + } + } +} + +// TestComplianceReportEmptyFindings ports +// test_compliance_report_empty_findings. +func TestComplianceReportEmptyFindings(t *testing.T) { + report := GenerateComplianceReportAt(loadFixture(t, "audit_result_empty"), goldenComplianceReportAt) + if !strings.Contains(report, "No findings to report") { + t.Error("report is missing the empty-findings notice") + } + // The empty fixture also has no gaps, so the OTHER empty branch fires. + if !strings.Contains(report, "No compliance gaps identified.") { + t.Error("report is missing the empty-gaps notice") + } +} + +// TestComplianceReportContainsMetadata ports +// test_compliance_report_contains_metadata. +func TestComplianceReportContainsMetadata(t *testing.T) { + report := complianceReport(t) + for _, want := range []string{"Audit Metadata", "45.2s"} { + if !strings.Contains(report, want) { + t.Errorf("report is missing %q", want) + } + } +} + +// --------------------------------------------------------------------------- +// behaviours with no Python test +// --------------------------------------------------------------------------- + +// TestGenerateComplianceReportStampsTheClock proves the exported +// GenerateComplianceReport reads the real clock (Python's +// `datetime.now(UTC)`), and that only the header line differs from the pinned +// rendering. +func TestGenerateComplianceReportStampsTheClock(t *testing.T) { + result := loadFixture(t, "audit_result_report") + before := time.Now().UTC() + live := GenerateComplianceReport(result) + after := time.Now().UTC() + + pinned := GenerateComplianceReportAt(result, goldenComplianceReportAt) + liveLines := strings.Split(live, "\n") + pinnedLines := strings.Split(pinned, "\n") + if len(liveLines) != len(pinnedLines) { + t.Fatalf("line counts differ: %d vs %d", len(liveLines), len(pinnedLines)) + } + for i := range liveLines { + if i == 2 { + continue // the "**Generated:**" line + } + if liveLines[i] != pinnedLines[i] { + t.Fatalf("line %d differs outside the header: %q vs %q", i+1, liveLines[i], pinnedLines[i]) + } + } + + // The stamped minute must be one of the minutes the call spanned. + acceptable := map[string]bool{ + "**Generated:** " + before.Format("2006-01-02 15:04") + " UTC": true, + "**Generated:** " + after.Format("2006-01-02 15:04") + " UTC": true, + } + if !acceptable[liveLines[2]] { + t.Errorf("header = %q, want a stamp between %v and %v", liveLines[2], before, after) + } +} + +// TestSeverityIcon covers _severity_icon, whose default is "INFO" — so "info" +// and any unrecognised severity share a label. +func TestSeverityIcon(t *testing.T) { + cases := map[string]string{ + "critical": "CRITICAL", + "CRITICAL": "CRITICAL", + "high": "HIGH", + "medium": "MEDIUM", + "low": "LOW", + "info": "INFO", + "unknown-severity": "INFO", + "": "INFO", + } + for input, want := range cases { + if got := severityIcon(input); got != want { + t.Errorf("severityIcon(%q) = %q, want %q", input, got, want) + } + } +} + +// TestVerdictLabel covers _verdict_label, whose default echoes the input. +func TestVerdictLabel(t *testing.T) { + cases := map[string]string{ + "confirmed": "Confirmed", + "likely": "Likely", + "inconclusive": "Inconclusive", + "not_exploitable": "Not Exploitable", + "weird": "weird", + } + for input, want := range cases { + if got := verdictLabel(input); got != want { + t.Errorf("verdictLabel(%q) = %q, want %q", input, got, want) + } + } +} + +// TestHeadRunes covers Python's `s[:n]` slice, which never panics on a short +// string and counts characters rather than bytes. +func TestHeadRunes(t *testing.T) { + cases := []struct { + in string + n int + want string + }{ + {"abc123def456", 8, "abc123de"}, + {"abc123", 8, "abc123"}, + {"", 8, ""}, + {"naïve-sha", 4, "naïv"}, + {"abc", 0, ""}, + } + for _, tc := range cases { + if got := headRunes(tc.in, tc.n); got != tc.want { + t.Errorf("headRunes(%q, %d) = %q, want %q", tc.in, tc.n, got, tc.want) + } + } +} + +// TestComplianceReportFindingListedOncePerFramework pins the seen_frameworks +// de-duplication: the edge fixture's dup-a carries TWO OWASP mappings and one +// PCI-DSS mapping, so it appears once under each framework heading, not twice +// under OWASP. +func TestComplianceReportFindingListedOncePerFramework(t *testing.T) { + report := GenerateComplianceReportAt(loadFixture(t, "audit_result_edge"), goldenComplianceReportAt) + title := "#### Naïve \"quote\" & handling" + if got := strings.Count(report, title); got != 2 { + t.Errorf("dup-a appears %d times, want 2 (once for OWASP, once for PCI-DSS)", got) + } + if !strings.Contains(report, "### Uncategorized Findings") { + t.Error("findings without compliance mappings must land in Uncategorized Findings") + } +} + +// TestComplianceReportTruncatesCweList pins the "(+N more)" suffix. +func TestComplianceReportTruncatesCweList(t *testing.T) { + report := GenerateComplianceReportAt(loadFixture(t, "audit_result_edge"), goldenComplianceReportAt) + want := "CWE-78, CWE-79, CWE-89, CWE-90, CWE-91 (+2 more)" + if !strings.Contains(report, want) { + t.Errorf("report is missing %q", want) + } +} diff --git a/internal/secaf/output/golden_test.go b/internal/secaf/output/golden_test.go new file mode 100644 index 0000000000..eb0f7cc12e --- /dev/null +++ b/internal/secaf/output/golden_test.go @@ -0,0 +1,181 @@ +package output + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// This file is the byte-for-byte parity gate for internal/output. +// +// scripts/gen_golden_output.py builds four SecurityAuditResult fixtures in +// Python, writes each one to testdata/.json, re-reads it, and writes the +// five artifacts the Python generators produce from it under testdata/golden/. +// The tests below load the SAME fixture files into the Go structs and diff their +// own output against those bytes. +// +// Regenerate after any change to src/sec_af/output/** (the umbrella +// scripts/gen_golden.py calls it too): +// +// PYTHONPATH=src ~/.agentfield/packages/sec-af/venv/bin/python go/scripts/gen_golden_output.py +// +// The fixtures are, in increasing nastiness: +// +// audit_result tests/conftest.py's sample_security_audit_result +// audit_result_empty every "nothing to report" branch at once +// audit_result_edge escaping, float ties, and every truthiness guard +// audit_result_report tests/test_compliance_report.py::_make_result + +// goldenFixtures names the fixtures every generator is checked against. +var goldenFixtures = []string{ + "audit_result", "audit_result_empty", "audit_result_edge", "audit_result_report", + // audit_result_floats is the FLOAT/INT-spelling fixture: it is the only one + // carrying a magnitude outside plain decimal range (1e-7, 8e-05, 1e-5, + // 5e-324, 1e16) and the only one whose `metadata` holds the wire-decoded + // prove_phase drop_summary, whose counts are Python ints. Between them the + // full.json and full_compact.json goldens pin BOTH float rules at once: + // `generate_json(result, pretty=False)` returns model_dump_json() verbatim + // (pydantic-core spells 1e-7 as "1e-7" and 8e-05 as "0.00008") while + // `pretty=True` re-serialises through json.dumps and therefore wants + // repr()'s "1e-07" / "8e-05". + "audit_result_floats", +} + +// goldenComplianceReportAt is the instant scripts/gen_golden.py freezes +// `datetime.now(UTC)` to while rendering the compliance report. +var goldenComplianceReportAt = time.Date(2026, 5, 6, 7, 8, 9, 0, time.UTC) + +// loadFixture reads testdata/.json into the Go model the way CPython +// reads it — which is what the generator does on the Python side +// (`SecurityAuditResult.model_validate(json.loads(payload))`). +// +// The int-vs-float distinction CPython's json.loads makes survives into +// `metadata` (`dict[str, object]`, which keeps whatever the decoder produced), +// so a metadata `2` must not become "2.0" on the way out. +// SecurityAuditResult.UnmarshalJSON decodes with UseNumber for that reason; +// nothing extra is needed here. +func loadFixture(t *testing.T, name string) schemas.SecurityAuditResult { + t.Helper() + raw, err := os.ReadFile(filepath.Join("testdata", name+".json")) + if err != nil { + t.Fatalf("read fixture %s: %v", name, err) + } + var result schemas.SecurityAuditResult + if err := json.Unmarshal(raw, &result); err != nil { + t.Fatalf("decode fixture %s: %v", name, err) + } + return result +} + +// readGolden reads testdata/golden/ verbatim. +func readGolden(t *testing.T, name string) string { + t.Helper() + raw, err := os.ReadFile(filepath.Join("testdata", "golden", name)) + if err != nil { + t.Fatalf("read golden %s: %v", name, err) + } + return string(raw) +} + +// assertGolden diffs got against the golden and, on failure, prints the first +// differing line with its neighbours — a 7KB SARIF document is unreadable as a +// whole-string diff. +func assertGolden(t *testing.T, goldenName, got string) { + t.Helper() + want := readGolden(t, goldenName) + if got == want { + return + } + gotLines := strings.Split(got, "\n") + wantLines := strings.Split(want, "\n") + for i := 0; i < len(gotLines) || i < len(wantLines); i++ { + var gotLine, wantLine string + if i < len(gotLines) { + gotLine = gotLines[i] + } + if i < len(wantLines) { + wantLine = wantLines[i] + } + if gotLine == wantLine { + continue + } + t.Fatalf("%s: first difference at line %d\n go: %q\n python: %q\n(go has %d lines, python %d)", + goldenName, i+1, gotLine, wantLine, len(gotLines), len(wantLines)) + } + t.Fatalf("%s: documents differ only in trailing bytes (go %d bytes, python %d)", + goldenName, len(got), len(want)) +} + +// TestGoldenSarif diffs GenerateSarif against generate_sarif. +func TestGoldenSarif(t *testing.T) { + for _, name := range goldenFixtures { + t.Run(name, func(t *testing.T) { + result := loadFixture(t, name) + assertGolden(t, name+".sarif.json", GenerateSarif(result)) + // render_sarif is generate_sarif under another name. + assertGolden(t, name+".sarif.json", RenderSarif(result)) + }) + } +} + +// TestGoldenGenerateJSON diffs GenerateJSON in both modes against +// generate_json, which is where pydantic's model_dump_json() spelling (and its +// "...Z" datetime) has to be reproduced exactly. +func TestGoldenGenerateJSON(t *testing.T) { + for _, name := range goldenFixtures { + t.Run(name, func(t *testing.T) { + result := loadFixture(t, name) + assertGolden(t, name+".full.json", GenerateJSON(result, true)) + assertGolden(t, name+".full_compact.json", GenerateJSON(result, false)) + }) + } +} + +// TestGoldenSummaryJSON diffs GenerateSummaryJSON against +// generate_summary_json. +func TestGoldenSummaryJSON(t *testing.T) { + for _, name := range goldenFixtures { + t.Run(name, func(t *testing.T) { + assertGolden(t, name+".summary.json", GenerateSummaryJSON(loadFixture(t, name))) + }) + } +} + +// TestGoldenReport diffs GenerateReport against generate_report. +func TestGoldenReport(t *testing.T) { + for _, name := range goldenFixtures { + t.Run(name, func(t *testing.T) { + result := loadFixture(t, name) + assertGolden(t, name+".report.md", GenerateReport(result)) + assertGolden(t, name+".report.md", RenderReport(result)) + }) + } +} + +// TestGoldenComplianceReport diffs GenerateComplianceReportAt against +// generate_compliance_report rendered with the same frozen clock. +func TestGoldenComplianceReport(t *testing.T) { + for _, name := range goldenFixtures { + t.Run(name, func(t *testing.T) { + got := GenerateComplianceReportAt(loadFixture(t, name), goldenComplianceReportAt) + assertGolden(t, name+".compliance_report.md", got) + }) + } +} + +// TestFixtureRoundTrip proves the Go structs lose nothing the fixture carries: +// re-serialising a loaded fixture with pydantic's own spelling reproduces the +// golden full-JSON dump. If a schemas field were missing or mistyped, every +// other golden here would fail with a confusing diff — this one names it. +func TestFixtureRoundTrip(t *testing.T) { + for _, name := range goldenFixtures { + t.Run(name, func(t *testing.T) { + assertGolden(t, name+".full.json", GenerateJSON(loadFixture(t, name), true)) + }) + } +} diff --git a/internal/secaf/output/json_output.go b/internal/secaf/output/json_output.go new file mode 100644 index 0000000000..a1fbcb756c --- /dev/null +++ b/internal/secaf/output/json_output.go @@ -0,0 +1,213 @@ +package output + +import ( + "encoding/json" + "fmt" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// This file ports src/sec_af/output/json_output.py. + +// GenerateJSON ports generate_json: +// +// full_json = result.model_dump_json() +// if not pretty: return full_json +// return json.dumps(json.loads(full_json), indent=2) +// +// so pretty=false yields pydantic's whitespace-free dump and pretty=true the +// two-space-indented re-serialisation of the same document. Both are produced +// here by walking the Go struct directly. +// +// THE TWO BRANCHES DO NOT SPELL FLOATS THE SAME WAY, and that is not a +// formatting detail — it is why each branch configures its own encoder. +// `model_dump_json()` is pydantic-core's Rust serializer; `json.dumps` is +// CPython's, which renders through repr(). They agree on ordinary magnitudes +// and disagree below 1e-4 (VERIFIED on the pinned interpreter): +// +// value pretty=False (pydantic) pretty=True (repr) +// 1e-7 1e-7 1e-07 +// 8e-05 0.00008 8e-05 +// 1.23e-05 0.0000123 1.23e-05 +// +// So Python's `json.loads` round trip is NOT value-preserving in its printed +// form, and a Go port that used one spelling for both branches is wrong for +// one of them. See the pydanticFloat flag in pyjson_local.go, the shared rule +// in pyfmt.PydanticFloat, and the audit_result_floats golden. +// +// See pyjson_local.go for the rest of why encoding/json is not used: the +// ensure_ascii escaping, key order, and the "...Z" datetime form that only +// `model_dump_json()` produces. +func GenerateJSON(result schemas.SecurityAuditResult, pretty bool) string { + if !pretty { + return dumpsPydantic(result) + } + e := &jsonEncoder{indent: 2, keySep: ": ", pydanticTime: true, ensureASCII: true} + e.encode(result, 0) + return e.buf.String() +} + +// RenderJSON ports render_json: the pretty document parsed back into a map. +// +// Python returns `json.loads(generate_json(result, pretty=True))`, a plain +// dict; the Go equivalent is map[string]any with the same JSON-decoded value +// kinds (every number becomes a float64, as it does in Go generally). An +// error is impossible for a document this package just generated, so it is +// reported as a nil map plus the error rather than being swallowed. +func RenderJSON(auditResult schemas.SecurityAuditResult) (map[string]any, error) { + var payload map[string]any + if err := json.Unmarshal([]byte(GenerateJSON(auditResult, true)), &payload); err != nil { + return nil, fmt.Errorf("output.RenderJSON: %w", err) + } + return payload, nil +} + +// buildSummaryStatistics ports _build_summary_statistics. +func buildSummaryStatistics(result schemas.SecurityAuditResult) obj { + return obj{ + {"total_findings", len(result.Findings)}, + {"confirmed", result.Confirmed}, + {"likely", result.Likely}, + {"inconclusive", result.Inconclusive}, + {"not_exploitable", result.NotExploitable}, + {"noise_reduction_pct", result.NoiseReductionPct}, + {"by_severity", result.BySeverity}, + } +} + +// buildSummaryFindings ports _build_summary_findings: the finding view that +// drops proof, tags, compliance and remediation — everything heavy. +func buildSummaryFindings(result schemas.SecurityAuditResult) []any { + out := make([]any, 0, len(result.Findings)) + for _, finding := range result.Findings { + out = append(out, obj{ + {"id", finding.ID}, + {"title", finding.Title}, + {"severity", string(finding.Severity)}, + {"verdict", string(finding.Verdict)}, + {"evidence_level", int(finding.EvidenceLevel)}, + {"exploitability_score", finding.ExploitabilityScore}, + {"cwe_id", finding.CweID}, + {"location", obj{ + {"file", finding.Location.FilePath}, + {"line", finding.Location.StartLine}, + }}, + {"chain_id", finding.ChainID}, + }) + } + return out +} + +// findingsByID ports _findings_by_id. +// +// Python parity: a later finding with a duplicate id overwrites an earlier one +// (dict comprehension semantics), which the map assignment reproduces. +func findingsByID(result schemas.SecurityAuditResult) map[string]schemas.VerifiedFinding { + out := make(map[string]schemas.VerifiedFinding, len(result.Findings)) + for _, finding := range result.Findings { + out[finding.ID] = finding + } + return out +} + +// buildChainSteps ports _build_chain_steps. +// +// Python parity, two quirks preserved: +// - `finding.chain_step or index` is Python truthiness, so a chain_step of 0 +// falls back to the 1-based enumeration index just like a null one does; +// - a chain naming a finding id the result does not contain still produces a +// step, with every finding-derived field null. +func buildChainSteps(chain schemas.AttackChain, findings map[string]schemas.VerifiedFinding) []any { + steps := make([]any, 0, len(chain.Findings)) + for i, findingID := range chain.Findings { + index := i + 1 + finding, ok := findings[findingID] + if !ok { + steps = append(steps, obj{ + {"step", index}, + {"finding_id", findingID}, + {"title", nil}, + {"verdict", nil}, + {"severity", nil}, + {"location", nil}, + }) + continue + } + step := index + if finding.ChainStep != nil && *finding.ChainStep != 0 { + step = *finding.ChainStep + } + steps = append(steps, obj{ + {"step", step}, + {"finding_id", findingID}, + {"title", finding.Title}, + {"verdict", string(finding.Verdict)}, + {"severity", string(finding.Severity)}, + {"location", obj{ + {"file", finding.Location.FilePath}, + {"line", finding.Location.StartLine}, + }}, + }) + } + return steps +} + +// buildAttackChains ports _build_attack_chains. +// +// Python parity: `chain.mitre_attack_mapping or []` turns a null mapping list +// into an empty JSON array rather than null. +func buildAttackChains(result schemas.SecurityAuditResult) []any { + findings := findingsByID(result) + out := make([]any, 0, len(result.AttackChains)) + for _, chain := range result.AttackChains { + mitre := make([]any, 0, len(chain.MitreAttackMapping)) + for _, mapping := range chain.MitreAttackMapping { + mitre = append(mitre, obj{ + {"tactic", mapping.Tactic}, + {"technique_id", mapping.TechniqueID}, + {"technique_name", mapping.TechniqueName}, + }) + } + out = append(out, obj{ + {"chain_id", chain.ChainID}, + {"title", chain.Title}, + {"description", chain.Description}, + {"combined_severity", string(chain.CombinedSeverity)}, + {"combined_impact", chain.CombinedImpact}, + {"findings", chain.Findings}, + {"steps", buildChainSteps(chain, findings)}, + {"mitre_attack_mapping", mitre}, + }) + } + return out +} + +// GenerateSummaryJSON ports generate_summary_json: the compact, dashboard-shaped +// view of a result, serialised with `json.dumps(summary, indent=2)`. +// +// The timestamp here is `result.timestamp.isoformat()` — the "+00:00" spelling, +// NOT the "...Z" one GenerateJSON produces. +func GenerateSummaryJSON(result schemas.SecurityAuditResult) string { + gaps := make([]any, 0, len(result.ComplianceGaps)) + for _, gap := range result.ComplianceGaps { + gaps = append(gaps, gap) + } + + summary := obj{ + {"repository", result.Repository}, + {"commit_sha", result.CommitSha}, + {"timestamp", result.Timestamp.String()}, + {"depth_profile", result.DepthProfile}, + {"summary", buildSummaryStatistics(result)}, + {"findings", buildSummaryFindings(result)}, + {"attack_chains", buildAttackChains(result)}, + {"compliance_gaps", gaps}, + {"performance", obj{ + {"duration_seconds", result.DurationSeconds}, + {"cost_usd", result.CostUsd}, + {"cost_breakdown", result.CostBreakdown}, + {"agent_invocations", result.AgentInvocations}, + }}, + } + return dumpsIndent(summary, 2) +} diff --git a/internal/secaf/output/json_output_test.go b/internal/secaf/output/json_output_test.go new file mode 100644 index 0000000000..b1c057da18 --- /dev/null +++ b/internal/secaf/output/json_output_test.go @@ -0,0 +1,315 @@ +package output + +import ( + "encoding/json" + "math" + "strconv" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// This file ports tests/test_json_output.py — which also carries the one +// generate_report test the Python suite has. Extra report coverage lives in +// report_test.go. + +// jsonDoc parses a generated JSON document into an untyped tree. +func jsonDoc(t *testing.T, raw string) map[string]any { + t.Helper() + var payload map[string]any + if err := json.Unmarshal([]byte(raw), &payload); err != nil { + t.Fatalf("generated JSON is invalid: %v", err) + } + return payload +} + +// TestGenerateJSONPrettyContainsFullFindingPayload ports +// test_generate_json_pretty_contains_full_finding_payload. +func TestGenerateJSONPrettyContainsFullFindingPayload(t *testing.T) { + result := loadFixture(t, "audit_result") + payload := jsonDoc(t, GenerateJSON(result, true)) + findings := sliceAt(t, payload["findings"], "findings") + chains := sliceAt(t, payload["attack_chains"], "attack_chains") + + if len(findings) != 3 { + t.Fatalf("want 3 findings, got %d", len(findings)) + } + if got := mapAt(t, findings[0], "findings[0]")["id"]; got != "finding-confirmed" { + t.Errorf("findings[0].id = %v", got) + } + if got := mapAt(t, findings[1], "findings[1]")["verdict"]; got != "likely" { + t.Errorf("findings[1].verdict = %v", got) + } + if got := mapAt(t, findings[2], "findings[2]")["verdict"]; got != "not_exploitable" { + t.Errorf("findings[2].verdict = %v", got) + } + if got := mapAt(t, chains[0], "attack_chains[0]")["chain_id"]; got != "chain-1" { + t.Errorf("attack_chains[0].chain_id = %v", got) + } + // The "full" payload keeps the heavy sub-objects the summary drops. + if _, ok := mapAt(t, findings[0], "findings[0]")["proof"].(map[string]any); !ok { + t.Error("findings[0].proof is missing from the full payload") + } +} + +// TestGenerateJSONCompactHasNoWhitespaceNewlines ports +// test_generate_json_compact_has_no_whitespace_newlines. +func TestGenerateJSONCompactHasNoWhitespaceNewlines(t *testing.T) { + output := GenerateJSON(loadFixture(t, "audit_result"), false) + + if strings.Contains(output, "\n") { + t.Error("compact output contains a newline") + } + if !strings.HasPrefix(output, "{") { + t.Error("compact output does not start with {") + } + if !strings.HasSuffix(output, "}") { + t.Error("compact output does not end with }") + } + // Python parity: pydantic's compact separators carry no space at all, + // unlike json.dumps' default ", " / ": ". + if strings.Contains(output, `", "`) || strings.Contains(output, `": "`) { + t.Error("compact output uses json.dumps separators, not pydantic's") + } +} + +// TestGenerateSummaryJSONOmitsProofAndContainsStatistics ports +// test_generate_summary_json_omits_proof_and_contains_statistics. +func TestGenerateSummaryJSONOmitsProofAndContainsStatistics(t *testing.T) { + result := loadFixture(t, "audit_result") + payload := jsonDoc(t, GenerateSummaryJSON(result)) + summary := mapAt(t, payload["summary"], "summary") + findings := sliceAt(t, payload["findings"], "findings") + chains := sliceAt(t, payload["attack_chains"], "attack_chains") + performance := mapAt(t, payload["performance"], "performance") + + if summary["total_findings"] != float64(3) { + t.Errorf("summary.total_findings = %v", summary["total_findings"]) + } + if summary["confirmed"] != float64(1) { + t.Errorf("summary.confirmed = %v", summary["confirmed"]) + } + if summary["likely"] != float64(1) { + t.Errorf("summary.likely = %v", summary["likely"]) + } + if summary["not_exploitable"] != float64(1) { + t.Errorf("summary.not_exploitable = %v", summary["not_exploitable"]) + } + if _, present := mapAt(t, findings[0], "findings[0]")["proof"]; present { + t.Error("the summary view leaked the proof object") + } + firstSteps := sliceAt(t, mapAt(t, chains[0], "attack_chains[0]")["steps"], "steps") + if got := mapAt(t, firstSteps[0], "steps[0]")["step"]; got != float64(1) { + t.Errorf("steps[0].step = %v, want 1", got) + } + if performance["cost_usd"] != 3.21 { + t.Errorf("performance.cost_usd = %v, want 3.21", performance["cost_usd"]) + } +} + +// TestRenderJSONReturnsDecodedDictionary ports +// test_render_json_returns_decoded_dictionary. +func TestRenderJSONReturnsDecodedDictionary(t *testing.T) { + payload, err := RenderJSON(loadFixture(t, "audit_result")) + if err != nil { + t.Fatalf("RenderJSON: %v", err) + } + if payload["repository"] != "Agent-Field/sec-af" { + t.Errorf("repository = %v", payload["repository"]) + } + if got := sliceAt(t, payload["findings"], "findings"); len(got) != 3 { + t.Errorf("want 3 findings, got %d", len(got)) + } +} + +// TestGenerateReportIncludesFindingsChainsComplianceAndCost ports +// test_generate_report_includes_findings_chains_compliance_and_cost (which +// lives in tests/test_json_output.py). +func TestGenerateReportIncludesFindingsChainsComplianceAndCost(t *testing.T) { + report := GenerateReport(loadFixture(t, "audit_result")) + + for _, want := range []string{ + "# SEC-AF Security Audit Report", + "## Summary", + "## Findings", + "SQL Injection", + "Missing Authentication", + "## Attack Chains", + "Input to DB read", + "## Compliance Gaps", + "PCI-DSS Req 6.2.4", + "## Performance & Cost", + } { + if !strings.Contains(report, want) { + t.Errorf("report is missing %q", want) + } + } +} + +// --------------------------------------------------------------------------- +// behaviours the Python suite does not cover but the port must not drift on +// --------------------------------------------------------------------------- + +// TestBuildChainStepsFallbacks pins _build_chain_steps' two quirks: a chain +// step of 0 is falsy and falls back to the enumeration index, and a chain +// naming an unknown finding id still yields a step with null detail fields. +func TestBuildChainStepsFallbacks(t *testing.T) { + payload := jsonDoc(t, GenerateSummaryJSON(loadFixture(t, "audit_result_edge"))) + chains := sliceAt(t, payload["attack_chains"], "attack_chains") + steps := sliceAt(t, mapAt(t, chains[0], "attack_chains[0]")["steps"], "steps") + + if len(steps) != 2 { + t.Fatalf("want 2 steps, got %d", len(steps)) + } + // dup-a carries chain_step 0 -> falls back to index 1. + first := mapAt(t, steps[0], "steps[0]") + if first["step"] != float64(1) { + t.Errorf("steps[0].step = %v, want the 1-based index", first["step"]) + } + if first["finding_id"] != "dup-a" { + t.Errorf("steps[0].finding_id = %v", first["finding_id"]) + } + // "not-in-result" is not among the findings -> index and null details. + second := mapAt(t, steps[1], "steps[1]") + if second["step"] != float64(2) { + t.Errorf("steps[1].step = %v, want 2", second["step"]) + } + for _, key := range []string{"title", "verdict", "severity", "location"} { + value, present := second[key] + if !present { + t.Errorf("steps[1].%s is absent; Python emits it as null", key) + } + if value != nil { + t.Errorf("steps[1].%s = %v, want null", key, value) + } + } +} + +// TestBuildAttackChainsNullMitreBecomesEmptyList pins +// `chain.mitre_attack_mapping or []`. +func TestBuildAttackChainsNullMitreBecomesEmptyList(t *testing.T) { + payload := jsonDoc(t, GenerateSummaryJSON(loadFixture(t, "audit_result_edge"))) + chains := sliceAt(t, payload["attack_chains"], "attack_chains") + mitre, present := mapAt(t, chains[0], "attack_chains[0]")["mitre_attack_mapping"] + if !present { + t.Fatal("mitre_attack_mapping is absent") + } + list, ok := mitre.([]any) + if !ok { + t.Fatalf("mitre_attack_mapping = %v (%T), want an empty array", mitre, mitre) + } + if len(list) != 0 { + t.Errorf("mitre_attack_mapping = %v, want []", list) + } +} + +// TestGenerateJSONTimestampSpellings pins the one place the two datetime +// spellings meet: the pydantic dump says "Z", the summary says "+00:00". +func TestGenerateJSONTimestampSpellings(t *testing.T) { + result := loadFixture(t, "audit_result") + + full := jsonDoc(t, GenerateJSON(result, true)) + if full["timestamp"] != "2026-03-04T10:30:00Z" { + t.Errorf("generate_json timestamp = %v, want the pydantic \"Z\" form", full["timestamp"]) + } + summary := jsonDoc(t, GenerateSummaryJSON(result)) + if summary["timestamp"] != "2026-03-04T10:30:00+00:00" { + t.Errorf("generate_summary_json timestamp = %v, want the isoformat form", summary["timestamp"]) + } + + edge := jsonDoc(t, GenerateJSON(loadFixture(t, "audit_result_edge"), true)) + if edge["timestamp"] != "2026-03-04T10:30:00.123456Z" { + t.Errorf("microsecond timestamp = %v", edge["timestamp"]) + } +} + +// TestGenerateJSONFloatSpelling pins the rule the two branches of generate_json +// obey, which is NOT the same rule. +// +// Validation contract (behaviour, measured on the pinned interpreter by running +// the real `generate_json` over a real SecurityAuditResult, one value at a +// time): +// +// - pretty=false is `result.model_dump_json()` VERBATIM — pydantic-core's +// Rust serializer. It writes a DECIMAL form for every magnitude in +// [1e-5, 1e16) (so 8e-05 is "0.00008" and 1e-5 is "0.00001") and an +// exponent form otherwise with UNPADDED exponent digits ("1e-7", not +// "1e-07"); +// - pretty=true is `json.dumps(json.loads(...))`, i.e. CPython repr(), which +// writes "8e-05" and "1e-07" for the same values; +// - the two agree everywhere else, including the 0.30000000000000004 repr +// tie, -0.0, 1e15 (decimal on both), 1e+16, 1e+21 and the 5e-324 denormal. +// +// Before this, both branches used repr() and only the pretty one was right — +// undetected because no committed golden carried a float below 1e-4. +func TestGenerateJSONFloatSpelling(t *testing.T) { + for _, tc := range []struct { + value float64 + compact, pretty string + }{ + {1e-07, "1e-7", "1e-07"}, + {1e-08, "1e-8", "1e-08"}, + {1.5e-07, "1.5e-7", "1.5e-07"}, + {-1e-07, "-1e-7", "-1e-07"}, + {1e-06, "1e-6", "1e-06"}, + {1e-05, "0.00001", "1e-05"}, + {8e-05, "0.00008", "8e-05"}, + {1.23e-05, "0.0000123", "1.23e-05"}, + {9.99e-05, "0.0000999", "9.99e-05"}, + {0.0001, "0.0001", "0.0001"}, + {0.30000000000000004, "0.30000000000000004", "0.30000000000000004"}, + {0.0, "0.0", "0.0"}, + {math.Copysign(0, -1), "-0.0", "-0.0"}, + {10.0, "10.0", "10.0"}, + {1e15, "1000000000000000.0", "1000000000000000.0"}, + {1e16, "1e+16", "1e+16"}, + {1e21, "1e+21", "1e+21"}, + {5e-324, "5e-324", "5e-324"}, + {1234.5, "1234.5", "1234.5"}, + } { + t.Run(strconv.FormatFloat(tc.value, 'g', -1, 64), func(t *testing.T) { + result := schemas.NewSecurityAuditResult() + result.DurationSeconds = tc.value + + if want := `"duration_seconds":` + tc.compact; !strings.Contains(GenerateJSON(result, false), want) { + t.Errorf("compact form does not contain %s\n%s", want, GenerateJSON(result, false)) + } + if want := `"duration_seconds": ` + tc.pretty; !strings.Contains(GenerateJSON(result, true), want) { + t.Errorf("pretty form does not contain %s", want) + } + }) + } +} + +// TestGenerateJSONMetadataKeepsWireIntegers pins the OTHER number rule: the +// untyped `metadata` map (pydantic `dict[str, object]`) keeps whatever the JSON +// decoder produced, and CPython's json.loads produces an `int` for an integer +// literal. Both branches of generate_json therefore print "2", not "2.0". +// +// The live path that puts a value there is app.py:205-208, which copies the +// prove_phase `drop_summary` payload verbatim; on the Go side that payload +// arrives from the SDK's own decoder, where every number is a float64, so the +// int-ness is restored by afx.WireNumbers at the boundary and by +// SecurityAuditResult.UnmarshalJSON's UseNumber when a document is read back. +func TestGenerateJSONMetadataKeepsWireIntegers(t *testing.T) { + var result schemas.SecurityAuditResult + document := `{"metadata":{"findings_not_verified":3,` + + `"prove_drop_summary":{"by_reason":{"verifier_error":2},"demoted_total":2,"findings":[]},` + + `"a_real_float":2.5}}` + if err := json.Unmarshal([]byte(document), &result); err != nil { + t.Fatalf("decode: %v", err) + } + + wantCompact := `"metadata":{"a_real_float":2.5,"findings_not_verified":3,` + + `"prove_drop_summary":{"by_reason":{"verifier_error":2},"demoted_total":2,"findings":[]}}` + if got := GenerateJSON(result, false); !strings.Contains(got, wantCompact) { + t.Errorf("compact metadata\n got: %s\nwant it to contain: %s", got, wantCompact) + } + if got := GenerateJSON(result, true); !strings.Contains(got, `"findings_not_verified": 3,`) { + t.Errorf("pretty metadata spells the wire integer as a float:\n%s", got) + } + if got := GenerateJSON(result, true); !strings.Contains(got, `"a_real_float": 2.5`) { + t.Errorf("pretty metadata lost a genuine float:\n%s", got) + } +} diff --git a/internal/secaf/output/pyjson_equivalence_test.go b/internal/secaf/output/pyjson_equivalence_test.go new file mode 100644 index 0000000000..e73426d7e5 --- /dev/null +++ b/internal/secaf/output/pyjson_equivalence_test.go @@ -0,0 +1,100 @@ +package output + +import ( + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// --------------------------------------------------------------------------- +// dumpsIndent == pyfmt.Dumps (anti-drift gate) +// +// This package keeps its own JSON encoder (pyjson_local.go) because it needs +// the PYDANTIC mode — `model_dump_json()`: no whitespace, raw non-ASCII, a +// "...Z" datetime — which pyfmt.Dumps deliberately does not implement, and +// because the artifact builders spell their documents as unkeyed `obj{{...}}` +// literals that go vet's composites check forbids for pyfmt's KV. +// +// Its OTHER mode, dumpsIndent, is the same function pyfmt.Dumps is: +// `json.dumps(x, indent=n)`. Nothing forces the two to stay equal, so this test +// does — over every value shape the SARIF and summary-JSON documents contain. +// If one encoder is fixed and the other is not, this fails instead of a +// committed artifact silently changing shape. +// --------------------------------------------------------------------------- + +func TestDumpsIndentAgreesWithPyfmtDumps(t *testing.T) { + ts := schemas.Timestamp{Time: time.Date(2025, 1, 15, 0, 0, 0, 0, time.UTC)} + nilStrings := []string(nil) + score := 9.25 + name := "run" + + // Every branch of jsonEncoder.encode/encodeValue that an artifact document + // can reach, in one document. + local := obj{ + {"string", "plain"}, + {"quotes_and_backslash", `say "hi" \ bye`}, + {"html_chars", " & "}, // json.dumps leaves these alone + {"non_ascii", "café — HIPAA §164"}, // ensure_ascii=True escapes these + {"astral", "😀"}, // surrogate pair + {"controls", "\x00\x01\x1f\x7f\n\t"}, // \uXXXX / short escapes + {"empty_string", ""}, + {"true", true}, + {"false", false}, + {"int", 42}, + {"negative_int", -7}, + {"int64", int64(1 << 40)}, + {"float_integral", 10.0}, // "10.0", not "10" + {"float_fraction", 9.25}, + {"float_ptr", &score}, + {"string_ptr", &name}, + {"nil_ptr", (*string)(nil)}, + {"nil_any", nil}, + {"timestamp", ts}, // isoformat "+00:00" spelling in json.dumps mode + {"empty_obj", obj{}}, + {"empty_list", []any{}}, + {"nil_slice", nilStrings}, // null on both sides + {"string_slice", []string{"a", "b"}}, + {"list_of_obj", []any{ + obj{{"i", 1}, {"j", 2}}, + "scalar", + []any{obj{{"deep", true}}}, + }}, + {"nested", obj{ + {"level2", obj{ + {"level3", obj{{"leaf", "x"}}}, + }}, + }}, + } + + for _, indent := range []int{2, 4} { + got := dumpsIndent(local, indent) + want := pyfmt.Dumps(toPyfmtValue(local), indent) + if got != want { + t.Errorf("indent=%d: dumpsIndent and pyfmt.Dumps disagree\n--- dumpsIndent ---\n%s\n--- pyfmt.Dumps ---\n%s", + indent, got, want) + } + } +} + +// toPyfmtValue re-types this package's ordered object as pyfmt's. The two carry +// identical information; only the Go type differs. +func toPyfmtValue(v any) any { + switch x := v.(type) { + case obj: + out := make(pyfmt.Ordered, 0, len(x)) + for _, e := range x { + out = append(out, pyfmt.KV{Key: e.Key, Value: toPyfmtValue(e.Value)}) + } + return out + case []any: + out := make([]any, 0, len(x)) + for _, e := range x { + out = append(out, toPyfmtValue(e)) + } + return out + default: + return v + } +} diff --git a/internal/secaf/output/pyjson_local.go b/internal/secaf/output/pyjson_local.go new file mode 100644 index 0000000000..128e09fd30 --- /dev/null +++ b/internal/secaf/output/pyjson_local.go @@ -0,0 +1,481 @@ +package output + +import ( + "encoding/json" + "reflect" + "sort" + "strconv" + "strings" + "time" + "unicode/utf16" + "unicode/utf8" + + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// This file is the package-local JSON writer the four output generators share. +// It exists because Go's encoding/json and CPython's json module disagree in +// four ways that are all observable in SEC-AF's committed output: +// +// 1. FLOAT SPELLING. Python renders a float with repr(): 10.0 stays "10.0", +// where json.Marshal emits "10". Every exploitability_score, cost and +// duration in the SARIF/JSON artifacts is a float. +// 2. STRING ESCAPING. json.dumps defaults to ensure_ascii=True, escaping every +// non-ASCII rune as \uXXXX (so the HIPAA "§" control ids become §); +// encoding/json emits them raw and instead escapes <, > and & — which +// Python does not touch. +// 3. KEY ORDER. json.dumps walks a dict in insertion order. encoding/json +// sorts map keys, and for a struct follows field order. The SARIF document +// is a hand-built dict literal whose order is part of the artifact. +// 4. DATETIME. `model_dump_json()` renders a UTC datetime as "...Z", while +// `datetime.isoformat()` (used everywhere else in this package) renders +// "...+00:00". schemas.Timestamp's MarshalJSON implements the isoformat +// spelling, so generate_json needs the other one - see pydanticISO. +// 5. PYDANTIC'S OWN FLOAT SPELLING. `model_dump_json()` does not go through +// CPython's repr() at all — it serializes in Rust, which spells 1e-7 as +// "1e-7" (no zero-padded exponent) and 8e-05 as "0.00008". That is a +// DIFFERENT rule from item 1, and it applies only to the compact/pydantic +// branch: `generate_json(result, pretty=True)` re-serialises through +// `json.dumps(json.loads(...))` (json_output.py:12-16) and therefore does +// want repr()'s "1e-07". See pyfmt.PydanticFloat and the pydanticFloat flag. +// 6. WIRE INTEGERS. CPython's json.loads turns an integer literal into an +// `int`, so `str`/`json.dumps` of a decoded payload spells it "2". +// encoding/json decodes EVERY number to float64, which this writer would +// spell "2.0". The compensation is afx.WireNumbers at the boundary where +// such a payload is stored untyped (see internal/secaf/audit/run.go), which +// leaves json.Number values behind; this writer emits their literal +// spelling verbatim. +// +// The json.dumps half of that list is also pyfmt.Dumps' subject (DESIGN.md +// §2b) and the two agree byte for byte — see dumpsIndent. The pydantic half is +// only here: `model_dump_json()` uses different separators, does NOT escape +// non-ASCII, and spells a UTC datetime "...Z" rather than the "+00:00" +// schemas.Timestamp's MarshalJSON (which pyfmt.Dumps honours) produces. + +// --------------------------------------------------------------------------- +// ordered objects +// --------------------------------------------------------------------------- + +// kv is one entry of an ordered JSON object. +type kv struct { + Key string + Value any +} + +// obj is an insertion-ordered JSON object: the Go stand-in for the Python dict +// literals sarif.py and json_output.py build, whose key order is part of the +// artifact (a map[string]any carries none). +// +// It is a package-local type rather than an alias for pyfmt.Ordered so the +// artifact builders can use unkeyed literals — `obj{{"id", x}, {"level", y}}` +// reads like the Python dict it ports, and go vet's composites check forbids +// unkeyed literals of another package's struct. +type obj []kv + +// --------------------------------------------------------------------------- +// entry points +// --------------------------------------------------------------------------- + +// dumpsIndent reproduces `json.dumps(value, indent=n)`: two-space-per-level +// (or n-space) pretty printing, ", " between object keys and their values, no +// trailing newline, and empty containers collapsed to "[]" / "{}". +// +// It shares its semantics with pyfmt.Dumps, the port's canonical json.dumps +// (DESIGN.md §2b), and TestDumpsIndentAgreesWithPyfmtDumps holds the two to +// byte equality over every value shape an artifact document contains — so the +// duplication cannot become a divergence. +// +// The encoder still lives here, rather than delegating, for two reasons that do +// not go away: this package needs the pydantic variant below regardless (there +// is no pyfmt equivalent for model_dump_json), and pyfmt.Dumps takes a +// pyfmt.Ordered whose entries go vet will not let another package build with +// unkeyed literals — which is what makes the artifact builders readable next to +// the Python dicts they port. +func dumpsIndent(value any, indent int) string { + e := &jsonEncoder{indent: indent, keySep: ": ", ensureASCII: true} + e.encode(value, 0) + return e.buf.String() +} + +// dumpsPydantic reproduces pydantic's `BaseModel.model_dump_json()`, which is +// NOT json.dumps and so cannot go through pyfmt.Dumps: +// +// - no whitespace at all, neither after ":" nor after "," (json.dumps' own +// compact form uses ", " and ": "); +// - non-ASCII and DEL emitted raw, where json.dumps escapes them; +// - a UTC datetime spelled "...Z", where schemas.Timestamp's MarshalJSON — +// which pyfmt.Dumps would call — spells it "...+00:00"; +// - floats spelled by pydantic-core's Rust serializer rather than by +// CPython's repr(): 1e-7 is "1e-7" (not "1e-07") and 8e-05 is "0.00008" +// (not "8e-05"). VERIFIED on the pinned interpreter against the real +// SecurityAuditResult. The rule lives in pyfmt.PydanticFloat, which +// pyfmt.DumpsModelJSON already uses, so the port's two model_dump_json +// writers cannot disagree. +func dumpsPydantic(value any) string { + e := &jsonEncoder{indent: 0, keySep: ":", pydanticTime: true, pydanticFloat: true} + e.encode(value, 0) + return e.buf.String() +} + +// jsonEncoder holds one encoding run. +type jsonEncoder struct { + buf strings.Builder + indent int + keySep string + // pydanticTime selects the "...Z" datetime spelling `model_dump_json()` + // produces instead of `datetime.isoformat()`'s "...+00:00". + pydanticTime bool + // pydanticFloat selects pydantic-core's float spelling over CPython's + // repr(). Both branches are reachable and both are correct for their + // caller: `generate_json(result, pretty=False)` returns + // `model_dump_json()` verbatim (pydantic spelling), while + // `generate_json(result, pretty=True)` and every other generator in this + // package round-trip through `json.dumps` (repr spelling). + pydanticFloat bool + // ensureASCII selects json.dumps' default escaping (every rune outside + // printable ASCII becomes \uXXXX). pydantic's serializer does NOT do this: + // it emits non-ASCII and DEL raw, and escapes only C0 control characters. + // VERIFIED against pydantic 2.x + CPython 3.11 in the sec-af venv. + ensureASCII bool +} + +// --------------------------------------------------------------------------- +// encoding +// --------------------------------------------------------------------------- + +func (e *jsonEncoder) encode(value any, depth int) { + switch v := value.(type) { + case nil: + e.buf.WriteString("null") + return + case obj: + e.encodeObject(v, depth) + return + case schemas.Timestamp: + // Handled before the generic struct walk: a Timestamp is a struct, but + // its JSON form is a string. + if e.pydanticTime { + e.writeString(pydanticISO(v)) + } else { + e.writeString(v.String()) + } + return + case string: + e.writeString(v) + return + case bool: + e.writeBool(v) + return + case int: + e.buf.WriteString(strconv.Itoa(v)) + return + case float64: + e.writeFloat(v) + return + case json.Number: + // A number recovered from the wire with its literal spelling intact + // (afx.WireNumbers). It is already valid JSON — and already carries the + // int-vs-float distinction CPython's json.loads makes and Go's + // float64-everything decode destroys — so it is emitted verbatim. + e.buf.WriteString(string(v)) + return + } + + rv := reflect.ValueOf(value) + e.encodeValue(rv, depth) +} + +func (e *jsonEncoder) encodeValue(rv reflect.Value, depth int) { + if !rv.IsValid() { + e.buf.WriteString("null") + return + } + switch rv.Kind() { + case reflect.Pointer, reflect.Interface: + if rv.IsNil() { + e.buf.WriteString("null") + return + } + e.encode(rv.Elem().Interface(), depth) + case reflect.Bool: + e.writeBool(rv.Bool()) + case reflect.String: + e.writeString(rv.String()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + e.buf.WriteString(strconv.FormatInt(rv.Int(), 10)) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + e.buf.WriteString(strconv.FormatUint(rv.Uint(), 10)) + case reflect.Float32, reflect.Float64: + e.writeFloat(rv.Float()) + case reflect.Slice: + if rv.IsNil() { + // Python parity: a pydantic `list | None = None` field dumps as + // null, and that is the only way a nil slice reaches here — the + // schemas package seeds every default_factory list to []. + e.buf.WriteString("null") + return + } + e.encodeArray(rv, depth) + case reflect.Array: + e.encodeArray(rv, depth) + case reflect.Map: + if rv.IsNil() { + e.buf.WriteString("null") + return + } + e.encodeMap(rv, depth) + case reflect.Struct: + e.encodeStruct(rv, depth) + default: + e.buf.WriteString("null") + } +} + +// writeFloat renders a float with the spelling this encoding run's caller +// needs: pydantic-core's for `model_dump_json()`, CPython's repr() for +// `json.dumps`. See the pydanticFloat field. +func (e *jsonEncoder) writeFloat(f float64) { + if e.pydanticFloat { + e.buf.WriteString(pyfmt.PydanticFloat(f)) + return + } + e.buf.WriteString(pyfmt.FormatFloat(f)) +} + +func (e *jsonEncoder) encodeObject(entries obj, depth int) { + if len(entries) == 0 { + e.buf.WriteString("{}") + return + } + e.buf.WriteByte('{') + for i, entry := range entries { + if i > 0 { + e.buf.WriteByte(',') + } + e.newlineIndent(depth + 1) + e.writeString(entry.Key) + e.buf.WriteString(e.keySep) + e.encode(entry.Value, depth+1) + } + e.newlineIndent(depth) + e.buf.WriteByte('}') +} + +func (e *jsonEncoder) encodeArray(rv reflect.Value, depth int) { + n := rv.Len() + if n == 0 { + e.buf.WriteString("[]") + return + } + e.buf.WriteByte('[') + for i := 0; i < n; i++ { + if i > 0 { + e.buf.WriteByte(',') + } + e.newlineIndent(depth + 1) + e.encode(rv.Index(i).Interface(), depth+1) + } + e.newlineIndent(depth) + e.buf.WriteByte(']') +} + +// encodeMap writes a Go map with its keys SORTED. +// +// Python parity divergence (documented in DESIGN.md §2b): CPython walks a dict +// in insertion order, which a Go map does not carry. Sorting is the +// deterministic alternative. It is only observable for the three +// `dict[...]`-typed fields of SecurityAuditResult (by_severity, cost_breakdown, +// metadata); the golden fixtures list those keys in sorted order so the two +// implementations agree byte-for-byte. +func (e *jsonEncoder) encodeMap(rv reflect.Value, depth int) { + if rv.Len() == 0 { + e.buf.WriteString("{}") + return + } + keys := make([]string, 0, rv.Len()) + byKey := make(map[string]reflect.Value, rv.Len()) + for _, key := range rv.MapKeys() { + name := key.String() + keys = append(keys, name) + byKey[name] = rv.MapIndex(key) + } + sort.Strings(keys) + + e.buf.WriteByte('{') + for i, key := range keys { + if i > 0 { + e.buf.WriteByte(',') + } + e.newlineIndent(depth + 1) + e.writeString(key) + e.buf.WriteString(e.keySep) + e.encode(byKey[key].Interface(), depth+1) + } + e.newlineIndent(depth) + e.buf.WriteByte('}') +} + +// encodeStruct writes a struct's exported fields in DECLARATION order, keyed by +// their json tag — which is exactly how pydantic dumps a model, because the Go +// structs in internal/schemas are declared in pydantic field order and tagged +// with the pydantic field names. `json:"-"` fields are skipped; `omitempty` is +// ignored (the schemas package uses none, by design: model_dump emits every +// field). Anonymous exported struct fields without a tag are flattened the way +// encoding/json flattens them. +func (e *jsonEncoder) encodeStruct(rv reflect.Value, depth int) { + entries := make(obj, 0, rv.NumField()) + collectStructFields(&entries, rv) + e.encodeObject(entries, depth) +} + +func collectStructFields(entries *obj, rv reflect.Value) { + rt := rv.Type() + for i := 0; i < rt.NumField(); i++ { + field := rt.Field(i) + if !field.IsExported() { + continue + } + name, _, _ := strings.Cut(field.Tag.Get("json"), ",") + if name == "-" { + continue + } + if name == "" { + if field.Anonymous { + fv := rv.Field(i) + for fv.Kind() == reflect.Pointer && !fv.IsNil() { + fv = fv.Elem() + } + if fv.Kind() == reflect.Struct { + collectStructFields(entries, fv) + continue + } + } + name = field.Name + } + *entries = append(*entries, kv{Key: name, Value: rv.Field(i).Interface()}) + } +} + +// newlineIndent writes the newline + leading spaces json.dumps emits before +// each element when indent is set. With indent == 0 it writes nothing, which +// gives the compact form. +func (e *jsonEncoder) newlineIndent(depth int) { + if e.indent <= 0 { + return + } + e.buf.WriteByte('\n') + e.buf.WriteString(strings.Repeat(" ", e.indent*depth)) +} + +func (e *jsonEncoder) writeBool(b bool) { + if b { + e.buf.WriteString("true") + return + } + e.buf.WriteString("false") +} + +// writeString escapes a JSON string the way whichever Python serializer is +// being reproduced does. +// +// With ensureASCII (json.dumps' default, so every indented artifact) it ports +// CPython's py_encode_basestring_ascii (json/encoder.py): +// +// - `\` and `"` get their short escapes; +// - \b \f \n \r \t get theirs; +// - every other rune outside the printable ASCII range 0x20..0x7E — so +// including DEL — becomes \uXXXX, with a surrogate PAIR above the BMP. +// +// Without it (pydantic's `model_dump_json()`, i.e. GenerateJSON's compact form) +// only `\`, `"` and the C0 control characters are escaped; DEL and every +// non-ASCII rune are emitted raw as UTF-8. VERIFIED: pydantic renders +// "a\x00b\x7fdée😀" as `a\u0000bdée😀`. +// +// Deliberately absent from BOTH modes: Go's default escaping of <, > and & — +// neither Python serializer touches them. +func (e *jsonEncoder) writeString(s string) { + e.buf.WriteByte('"') + for _, r := range s { + switch r { + case '\\': + e.buf.WriteString(`\\`) + case '"': + e.buf.WriteString(`\"`) + case '\b': + e.buf.WriteString(`\b`) + case '\f': + e.buf.WriteString(`\f`) + case '\n': + e.buf.WriteString(`\n`) + case '\r': + e.buf.WriteString(`\r`) + case '\t': + e.buf.WriteString(`\t`) + default: + switch { + case r < 0x20: + e.writeUnicodeEscape(r) + case !e.ensureASCII: + if r == utf8.RuneError { + // An invalid UTF-8 byte; neither Python serializer can + // produce one. Emit the replacement character so the + // document stays valid UTF-8 JSON. + e.buf.WriteRune(utf8.RuneError) + continue + } + e.buf.WriteRune(r) + case r <= 0x7E: + e.buf.WriteByte(byte(r)) + case r == utf8.RuneError: + e.writeUnicodeEscape(0xFFFD) + case r > 0xFFFF: + high, low := utf16.EncodeRune(r) + e.writeUnicodeEscape(high) + e.writeUnicodeEscape(low) + default: + e.writeUnicodeEscape(r) + } + } + } + e.buf.WriteByte('"') +} + +func (e *jsonEncoder) writeUnicodeEscape(r rune) { + const hexDigits = "0123456789abcdef" + e.buf.WriteString(`\u`) + e.buf.WriteByte(hexDigits[(r>>12)&0xF]) + e.buf.WriteByte(hexDigits[(r>>8)&0xF]) + e.buf.WriteByte(hexDigits[(r>>4)&0xF]) + e.buf.WriteByte(hexDigits[r&0xF]) +} + +// pydanticISO renders a datetime the way pydantic-core's JSON serializer does, +// which is NOT `datetime.isoformat()`: +// +// UTC 2026-03-04T10:30:00Z (isoformat: ...+00:00) +// other offset 2026-03-04T10:30:00+05:30 (same as isoformat) +// microseconds 2026-03-04T10:30:00.123456Z (always six digits) +// +// VERIFIED against pydantic 2.x under ~/.agentfield/packages/sec-af/venv. +// Only generate_json needs this spelling, because it is the one function that +// serialises the model with `model_dump_json()` rather than reaching for +// `result.timestamp.isoformat()` itself. +// +// Two Go-specific notes. A Python datetime carries microseconds, so the +// fraction is decided by the microsecond value (a stray sub-microsecond +// nanosecond is truncated away and produces no fraction, as it would in +// Python). And Go has no naive datetime: a zero UTC offset always renders "Z", +// where pydantic would emit no suffix at all for a tz-less datetime — SEC-AF +// only ever builds `datetime.now(UTC)` values, so that case cannot arise. +func pydanticISO(t schemas.Timestamp) string { + micros := t.Nanosecond() / 1000 + rendered := t.Format("2006-01-02T15:04:05") + if micros != 0 { + rendered = t.Truncate(time.Microsecond).Format("2006-01-02T15:04:05.000000") + } + if _, offset := t.Zone(); offset == 0 { + return rendered + "Z" + } + return rendered + t.Format("-07:00") +} diff --git a/internal/secaf/output/pyjson_local_test.go b/internal/secaf/output/pyjson_local_test.go new file mode 100644 index 0000000000..44f55bea5f --- /dev/null +++ b/internal/secaf/output/pyjson_local_test.go @@ -0,0 +1,233 @@ +package output + +import ( + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// This file pins the two JSON writers this package uses against CPython and +// pydantic directly, so a divergence points at the writer rather than surfacing +// as a mystery diff in a 7KB golden. Every `want` below was produced by running +// the quoted expression under ~/.agentfield/packages/sec-af/venv/bin/python. +// +// dumpsIndent shares its contract with pyfmt.Dumps (DESIGN.md §2b); see that +// function's comment for why this package keeps its own encoder. + +// TestDumpsIndentMatchesJSONDumps covers json.dumps(x, indent=2). +func TestDumpsIndentMatchesJSONDumps(t *testing.T) { + cases := []struct { + name string + in any + want string + }{ + { + // json.dumps({"a": 1, "b": [], "c": {}}, indent=2) + name: "empty containers stay on one line", + in: obj{{"a", 1}, {"b", []any{}}, {"c", obj{}}}, + want: "{\n \"a\": 1,\n \"b\": [],\n \"c\": {}\n}", + }, + { + // json.dumps({"f": 1.0, "g": 0.5, "h": -0.0, "i": 1e22, + // "j": 1234567890123456789}, indent=2) + name: "floats use Python repr, ints stay ints", + in: obj{ + {"f", 1.0}, {"g", 0.5}, {"h", negZero()}, {"i", 1e22}, + {"j", int64(1234567890123456789)}, + }, + want: "{\n \"f\": 1.0,\n \"g\": 0.5,\n \"h\": -0.0,\n \"i\": 1e+22,\n \"j\": 1234567890123456789\n}", + }, + { + // json.dumps({"s": 'sec§ naïve — 😀 \x7f \x01 "q" \\ &'}, indent=2) + name: "ensure_ascii escaping, and no escaping of < > &", + in: obj{{"s", "sec§ naïve — 😀 \x7f \x01 \"q\" \\ &"}}, + want: "{\n \"s\": \"sec\\u00a7 na\\u00efve \\u2014 \\ud83d\\ude00 \\u007f \\u0001 \\\"q\\\" \\\\ &\"\n}", + }, + { + // json.dumps([1, [2, [3]]], indent=2) + name: "nested arrays indent per level", + in: []any{1, []any{2, []any{3}}}, + want: "[\n 1,\n [\n 2,\n [\n 3\n ]\n ]\n]", + }, + { + // json.dumps({"n": None, "t": True, "f": False}, indent=2) + name: "null and booleans", + in: obj{{"n", nil}, {"t", true}, {"f", false}}, + want: "{\n \"n\": null,\n \"t\": true,\n \"f\": false\n}", + }, + { + // json.dumps({"x": [{"y": 1}]}, indent=2) + name: "object inside array", + in: obj{{"x", []any{obj{{"y", 1}}}}}, + want: "{\n \"x\": [\n {\n \"y\": 1\n }\n ]\n}", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := dumpsIndent(tc.in, 2); got != tc.want { + t.Errorf("dumpsIndent =\n%q\nwant\n%q", got, tc.want) + } + }) + } +} + +// TestDumpsIndentSortsMapKeys documents the one deliberate deviation from +// CPython: a Go map carries no insertion order, so its keys are emitted sorted. +func TestDumpsIndentSortsMapKeys(t *testing.T) { + got := dumpsIndent(map[string]int{"zeta": 1, "alpha": 2, "mid": 3}, 2) + want := "{\n \"alpha\": 2,\n \"mid\": 3,\n \"zeta\": 1\n}" + if got != want { + t.Errorf("dumpsIndent =\n%q\nwant\n%q", got, want) + } + // Stability is the point: repeat it enough that Go's randomized map + // iteration would show up. + for i := 0; i < 50; i++ { + if dumpsIndent(map[string]int{"zeta": 1, "alpha": 2, "mid": 3}, 2) != want { + t.Fatalf("map key order is not stable (iteration %d)", i) + } + } +} + +// TestDumpsPydanticEscaping pins pydantic's serializer, which — unlike +// json.dumps — leaves non-ASCII and DEL raw and escapes only C0 controls. +// +// VERIFIED: a pydantic model with s='a\x00b\x01c\x7fdée—f😀g<>&' dumps as +// {"s":"a\x00b\x01cdée—f😀g<>&"}. +func TestDumpsPydanticEscaping(t *testing.T) { + got := dumpsPydantic(obj{{"s", "a\x00b\x01c\x7fdée—f😀g<>&"}}) + want := "{\"s\":\"a\\u0000b\\u0001c\x7fdée—f😀g<>&\"}" + if got != want { + t.Errorf("dumpsPydantic =\n%q\nwant\n%q", got, want) + } + + // Short escapes are shared with json.dumps. + // VERIFIED: model_dump_json of 'a\nb\tc\rd\be\ff/g' -> "a\nb\tc\rd\be\ff/g". + if got := dumpsPydantic(obj{{"s", "a\nb\tc\rd\be\ff/g"}}); got != `{"s":"a\nb\tc\rd\be\ff/g"}` { + t.Errorf("dumpsPydantic short escapes = %q", got) + } +} + +// TestDumpsPydanticHasNoWhitespace pins the compact separators. +func TestDumpsPydanticHasNoWhitespace(t *testing.T) { + got := dumpsPydantic(obj{{"a", 1}, {"b", []any{1, 2}}, {"c", obj{{"d", nil}}}}) + want := `{"a":1,"b":[1,2],"c":{"d":null}}` + if got != want { + t.Errorf("dumpsPydantic = %q, want %q", got, want) + } +} + +// TestEncodeStructFollowsDeclarationOrder proves the struct walk keys by json +// tag and preserves pydantic field order — the property GenerateJSON leans on. +func TestEncodeStructFollowsDeclarationOrder(t *testing.T) { + got := dumpsPydantic(schemas.ComplianceGap{ + Framework: "OWASP", + ControlID: "A03:2021", + ControlName: "Injection", + FindingCount: 2, + MaxSeverity: "high", + CweIDs: []string{"CWE-89"}, + }) + want := `{"framework":"OWASP","control_id":"A03:2021","control_name":"Injection",` + + `"finding_count":2,"max_severity":"high","cwe_ids":["CWE-89"]}` + if got != want { + t.Errorf("struct dump =\n%q\nwant\n%q", got, want) + } +} + +// TestEncodeNilSliceAndMap pins the null-vs-empty distinction: a nil slice or +// map is a pydantic `X | None = None` field and dumps as null, while an +// allocated empty one dumps as [] / {}. +func TestEncodeNilSliceAndMap(t *testing.T) { + var nilSlice []string + var nilMap map[string]int + got := dumpsPydantic(obj{ + {"nil_slice", nilSlice}, + {"empty_slice", []string{}}, + {"nil_map", nilMap}, + {"empty_map", map[string]int{}}, + }) + want := `{"nil_slice":null,"empty_slice":[],"nil_map":null,"empty_map":{}}` + if got != want { + t.Errorf("got %q, want %q", got, want) + } +} + +// TestEncodePointers covers the *string / *int fields the schemas use for +// Optional. +func TestEncodePointers(t *testing.T) { + s := "value" + n := 7 + var nilStr *string + got := dumpsPydantic(obj{{"set", &s}, {"num", &n}, {"unset", nilStr}}) + if want := `{"set":"value","num":7,"unset":null}`; got != want { + t.Errorf("got %q, want %q", got, want) + } +} + +// TestPydanticISO pins the "...Z" datetime spelling against the isoformat one +// schemas.Timestamp.String() produces. +// +// VERIFIED against pydantic 2.x: the same instant renders as +// "2026-03-04T10:30:00Z" through model_dump_json and +// "2026-03-04T10:30:00+00:00" through datetime.isoformat. +func TestPydanticISO(t *testing.T) { + cases := []struct { + name string + in time.Time + want string + wantISO string + skipISO bool + }{ + { + name: "utc, no fraction", + in: time.Date(2026, 3, 4, 10, 30, 0, 0, time.UTC), + want: "2026-03-04T10:30:00Z", + wantISO: "2026-03-04T10:30:00+00:00", + }, + { + name: "utc, microseconds", + in: time.Date(2026, 3, 4, 10, 30, 0, 123456000, time.UTC), + want: "2026-03-04T10:30:00.123456Z", + wantISO: "2026-03-04T10:30:00.123456+00:00", + }, + { + name: "utc, trailing zeros kept", + in: time.Date(2026, 3, 4, 10, 30, 0, 120000000, time.UTC), + want: "2026-03-04T10:30:00.120000Z", + wantISO: "2026-03-04T10:30:00.120000+00:00", + }, + { + name: "non-utc offset", + in: time.Date(2026, 3, 4, 10, 30, 0, 0, time.FixedZone("IST", 5*3600+1800)), + want: "2026-03-04T10:30:00+05:30", + wantISO: "2026-03-04T10:30:00+05:30", + }, + { + name: "sub-microsecond nanoseconds are truncated away", + in: time.Date(2026, 3, 4, 10, 30, 0, 500, time.UTC), + want: "2026-03-04T10:30:00Z", + skipISO: true, // schemas.Timestamp.String() keeps its own rounding + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ts := schemas.NewTimestamp(tc.in) + if got := pydanticISO(ts); got != tc.want { + t.Errorf("pydanticISO = %q, want %q", got, tc.want) + } + if !tc.skipISO { + if got := ts.String(); got != tc.wantISO { + t.Errorf("Timestamp.String = %q, want %q", got, tc.wantISO) + } + } + }) + } +} + +// negZero returns -0.0 without tripping the compiler's constant folding, so the +// "-0.0" rendering is genuinely exercised. +func negZero() float64 { + zero := 0.0 + return -zero +} diff --git a/internal/secaf/output/report.go b/internal/secaf/output/report.go new file mode 100644 index 0000000000..11ae0afb55 --- /dev/null +++ b/internal/secaf/output/report.go @@ -0,0 +1,169 @@ +package output + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// This file ports src/sec_af/output/report.py — the human-readable Markdown +// audit report. Every line is byte-identical to the Python one; the golden test +// diffs the whole document. + +// renderSummary ports _render_summary. +// +// Python parity: the branch line is chosen by TRUTHINESS, so an empty branch +// string renders "- Branch: n/a" exactly like a null one. +func renderSummary(result schemas.SecurityAuditResult) []string { + branchLine := "- Branch: n/a" + if result.Branch != nil && *result.Branch != "" { + branchLine = fmt.Sprintf("- Branch: `%s`", *result.Branch) + } + return []string{ + "## Summary", + "", + fmt.Sprintf("- Repository: `%s`", result.Repository), + fmt.Sprintf("- Commit: `%s`", result.CommitSha), + branchLine, + fmt.Sprintf("- Timestamp: `%s`", result.Timestamp.String()), + fmt.Sprintf("- Depth profile: `%s`", result.DepthProfile), + fmt.Sprintf("- Provider: `%s`", result.Provider), + fmt.Sprintf("- Findings: **%d** (confirmed: %d, likely: %d, inconclusive: %d, not exploitable: %d)", + len(result.Findings), result.Confirmed, result.Likely, result.Inconclusive, result.NotExploitable), + fmt.Sprintf("- Noise reduction: **%.1f%%**", result.NoiseReductionPct), + "", + } +} + +// renderFinding ports _render_finding. +func renderFinding(finding schemas.VerifiedFinding) []string { + lines := []string{ + fmt.Sprintf("### %s", finding.Title), + "", + fmt.Sprintf("- ID: `%s`", finding.ID), + fmt.Sprintf("- Verdict: `%s` (evidence level %d)", string(finding.Verdict), int(finding.EvidenceLevel)), + fmt.Sprintf("- Severity: `%s` | Exploitability: **%.1f/10**", string(finding.Severity), finding.ExploitabilityScore), + fmt.Sprintf("- CWE: `%s` (%s)", finding.CweID, finding.CweName), + fmt.Sprintf("- Location: `%s:%d`", finding.Location.FilePath, finding.Location.StartLine), + } + if finding.ChainID != nil && *finding.ChainID != "" { + lines = append(lines, fmt.Sprintf("- Chain: `%s` step %s", *finding.ChainID, chainStepLabel(finding.ChainStep))) + } + if finding.Proof != nil && len(finding.Proof.DataFlowTrace) > 0 { + lines = append(lines, "- Data flow trace:") + for _, step := range finding.Proof.DataFlowTrace { + lines = append(lines, fmt.Sprintf(" - `%s:%d` - %s", step.File, step.Line, step.Description)) + } + } + if finding.Rationale != "" { + lines = append(lines, fmt.Sprintf("- Rationale: %s", finding.Rationale)) + } + return append(lines, "") +} + +// chainStepLabel ports `finding.chain_step or '?'`: Python truthiness, so both +// a null step and a step of 0 render as "?". +func chainStepLabel(step *int) string { + if step == nil || *step == 0 { + return "?" + } + return strconv.Itoa(*step) +} + +// renderAttackChain ports _render_attack_chain. +func renderAttackChain(chain schemas.AttackChain) []string { + quoted := make([]string, 0, len(chain.Findings)) + for _, findingID := range chain.Findings { + quoted = append(quoted, "`"+findingID+"`") + } + lines := []string{ + fmt.Sprintf("### %s", chain.Title), + "", + fmt.Sprintf("- Chain ID: `%s`", chain.ChainID), + fmt.Sprintf("- Combined severity: `%s`", string(chain.CombinedSeverity)), + fmt.Sprintf("- Combined impact: %s", chain.CombinedImpact), + fmt.Sprintf("- Findings: %s", strings.Join(quoted, ", ")), + } + if len(chain.MitreAttackMapping) > 0 { + lines = append(lines, "- MITRE ATT&CK:") + for _, mapping := range chain.MitreAttackMapping { + lines = append(lines, fmt.Sprintf(" - %s (%s): %s", mapping.TechniqueID, mapping.Tactic, mapping.TechniqueName)) + } + } + return append(lines, "") +} + +// GenerateReport ports generate_report: the full Markdown audit report, +// joined with "\n" and carrying NO trailing newline (Python's `"\n".join`). +// +// Python parity divergence: the cost-breakdown section iterates a dict, whose +// order in CPython is insertion order; a Go map carries none, so the phases are +// listed in SORTED key order. Same deviation as the JSON writer's — see +// pyjson_local.go encodeMap. +func GenerateReport(result schemas.SecurityAuditResult) string { + lines := []string{ + "# SEC-AF Security Audit Report", + "", + } + lines = append(lines, renderSummary(result)...) + lines = append(lines, "## Findings", "") + + if len(result.Findings) > 0 { + for _, finding := range result.Findings { + lines = append(lines, renderFinding(finding)...) + } + } else { + lines = append(lines, "No findings.", "") + } + + lines = append(lines, "## Attack Chains", "") + if len(result.AttackChains) > 0 { + for _, chain := range result.AttackChains { + lines = append(lines, renderAttackChain(chain)...) + } + } else { + lines = append(lines, "No attack chains.", "") + } + + lines = append(lines, "## Compliance Gaps", "") + if len(result.ComplianceGaps) > 0 { + for _, gap := range result.ComplianceGaps { + lines = append(lines, fmt.Sprintf("- %s %s: %s (findings: %d, max severity: %s)", + gap.Framework, gap.ControlID, gap.ControlName, gap.FindingCount, gap.MaxSeverity)) + } + lines = append(lines, "") + } else { + lines = append(lines, "No compliance gaps.", "") + } + + lines = append(lines, + "## Performance & Cost", + "", + fmt.Sprintf("- Duration: %.1fs", result.DurationSeconds), + fmt.Sprintf("- Agent invocations: %d", result.AgentInvocations), + fmt.Sprintf("- Cost: $%.2f", result.CostUsd), + "- Cost breakdown:", + ) + if len(result.CostBreakdown) > 0 { + phases := make([]string, 0, len(result.CostBreakdown)) + for phase := range result.CostBreakdown { + phases = append(phases, phase) + } + sort.Strings(phases) + for _, phase := range phases { + lines = append(lines, fmt.Sprintf(" - %s: $%.2f", phase, result.CostBreakdown[phase])) + } + } else { + lines = append(lines, " - n/a") + } + + return strings.Join(lines, "\n") +} + +// RenderReport ports render_report, the alias generate_report is exported under. +func RenderReport(auditResult schemas.SecurityAuditResult) string { + return GenerateReport(auditResult) +} diff --git a/internal/secaf/output/report_test.go b/internal/secaf/output/report_test.go new file mode 100644 index 0000000000..0e198b9cdd --- /dev/null +++ b/internal/secaf/output/report_test.go @@ -0,0 +1,119 @@ +package output + +import ( + "strings" + "testing" +) + +// This file covers output/report.py behaviours the Python suite's single +// substring test (ported in json_output_test.go) does not reach. The +// whole-document parity check lives in golden_test.go. + +// TestReportEmptySections covers every "nothing to report" branch of +// generate_report at once, via the empty fixture. +func TestReportEmptySections(t *testing.T) { + report := GenerateReport(loadFixture(t, "audit_result_empty")) + for _, want := range []string{ + "- Branch: n/a", + "No findings.", + "No attack chains.", + "No compliance gaps.", + " - n/a", + } { + if !strings.Contains(report, want) { + t.Errorf("report is missing %q", want) + } + } +} + +// TestReportBranchTruthiness pins that an EMPTY branch renders "n/a" like a +// null one — Python tests `if result.branch`, not `is not None`. The edge +// fixture carries branch="". +func TestReportBranchTruthiness(t *testing.T) { + report := GenerateReport(loadFixture(t, "audit_result_edge")) + if !strings.Contains(report, "- Branch: n/a") { + t.Error("an empty branch must render \"- Branch: n/a\"") + } + if strings.Contains(report, "- Branch: ``") { + t.Error("an empty branch was rendered as an empty code span") + } +} + +// TestReportChainStepFallback pins `finding.chain_step or '?'`: the edge +// fixture's dup-a has chain_step 0, which is falsy. +func TestReportChainStepFallback(t *testing.T) { + report := GenerateReport(loadFixture(t, "audit_result_edge")) + if !strings.Contains(report, "- Chain: `chain-x` step ?") { + t.Error("a chain_step of 0 must render as \"?\"") + } +} + +// TestChainStepLabel covers the helper directly. +func TestChainStepLabel(t *testing.T) { + zero := 0 + one := 1 + cases := []struct { + in *int + want string + }{ + {nil, "?"}, + {&zero, "?"}, + {&one, "1"}, + } + for _, tc := range cases { + if got := chainStepLabel(tc.in); got != tc.want { + t.Errorf("chainStepLabel(%v) = %q, want %q", tc.in, got, tc.want) + } + } +} + +// TestReportOmitsEmptyRationale pins the `if finding.rationale:` guard: the +// edge fixture's dup-a has an empty rationale and must not get the line. +func TestReportOmitsEmptyRationale(t *testing.T) { + report := GenerateReport(loadFixture(t, "audit_result_edge")) + if strings.Contains(report, "- Rationale: \n") || strings.HasSuffix(report, "- Rationale: ") { + t.Error("an empty rationale produced a bare \"- Rationale:\" line") + } + // The finding that DOES have one still gets it. + if !strings.Contains(report, "- Rationale: Confirmed by trace.") { + t.Error("a non-empty rationale is missing") + } +} + +// TestReportHasNoTrailingNewline pins Python's `"\n".join(lines)`, which ends +// the document on the last content line. +func TestReportHasNoTrailingNewline(t *testing.T) { + for _, name := range goldenFixtures { + report := GenerateReport(loadFixture(t, name)) + if strings.HasSuffix(report, "\n") { + t.Errorf("%s: report ends with a newline", name) + } + } +} + +// TestComplianceReportHasNoTrailingNewline pins the same for the compliance +// report, whose last line is the generator credit. +func TestComplianceReportHasNoTrailingNewline(t *testing.T) { + for _, name := range goldenFixtures { + report := GenerateComplianceReportAt(loadFixture(t, name), goldenComplianceReportAt) + if strings.HasSuffix(report, "\n") { + t.Errorf("%s: compliance report ends with a newline", name) + } + if !strings.HasSuffix(report, "*Report generated by SEC-AF -- Composite Intelligence Security Auditor*") { + t.Errorf("%s: compliance report does not end with the credit line", name) + } + } +} + +// TestReportDataFlowTrace pins the trace block, which only appears when the +// proof carries steps. +func TestReportDataFlowTrace(t *testing.T) { + withTrace := GenerateReport(loadFixture(t, "audit_result")) + if !strings.Contains(withTrace, "- Data flow trace:\n - `src/routes.py:15` - Input source") { + t.Error("the data flow trace block is missing or malformed") + } + withoutTrace := GenerateReport(loadFixture(t, "audit_result_report")) + if strings.Contains(withoutTrace, "- Data flow trace:") { + t.Error("a finding with no proof produced a data flow trace block") + } +} diff --git a/internal/secaf/output/sarif.go b/internal/secaf/output/sarif.go new file mode 100644 index 0000000000..b3175427bd --- /dev/null +++ b/internal/secaf/output/sarif.go @@ -0,0 +1,507 @@ +// Package output ports src/sec_af/output: the four artifact generators the +// orchestrator writes at the end of an audit — SARIF, the full/summary JSON, +// the Markdown report and the compliance report. +// +// Every generator's output is compared byte-for-byte against the Python one by +// golden_test.go, which feeds both implementations the same +// testdata/audit_result.json fixture. That is why this package carries its own +// JSON writer (pyjson_local.go) instead of using encoding/json, and why every +// number and string below is formatted through pyfmt. +package output + +import ( + "fmt" + "sort" + "strings" + "unicode" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// pythonPackageVersion mirrors `src/sec_af/__init__.py::__version__`, which +// sarif.py stamps into the SARIF driver as `semanticVersion`. It is duplicated +// rather than derived because the Go binary has no import of the Python +// package; bump it together with the Python one. +const pythonPackageVersion = "0.1.0" + +// severityToLevel ports _SEVERITY_TO_LEVEL. +var severityToLevel = map[string]string{ + "critical": "error", + "high": "error", + "medium": "warning", + "low": "note", + "info": "note", +} + +// levelRank ports _LEVEL_RANK. +var levelRank = map[string]int{"error": 3, "warning": 2, "note": 1} + +// verdictToPrecision ports _VERDICT_TO_PRECISION. +var verdictToPrecision = map[string]string{ + "confirmed": "very-high", + "likely": "high", + "inconclusive": "medium", + "not_exploitable": "low", +} + +// precisionRank ports _PRECISION_RANK. +var precisionRank = map[string]int{"very-high": 4, "high": 3, "medium": 2, "low": 1} + +// GenerateSarif ports output/sarif.py generate_sarif: the SARIF 2.1.0 document +// for one audit result, serialised with `json.dumps(sarif, indent=2)`. +// +// Findings whose verdict is "not_exploitable" are dropped entirely — they +// appear in neither the results nor the rules — which is what makes the SARIF +// artifact the "signal only" view of an audit. +// +// The key order below is the Python dict literal's order and is part of the +// artifact (see pyjson_local.go). +func GenerateSarif(result schemas.SecurityAuditResult) string { + return GenerateSarifAs(result, SecAfTool) +} + +// GenerateSarifAs is [GenerateSarif] for another tool's identity. +func GenerateSarifAs(result schemas.SecurityAuditResult, tool SarifTool) string { + result.Findings = RuleIDsAs(result.Findings, tool) + included := make([]schemas.VerifiedFinding, 0, len(result.Findings)) + for _, finding := range result.Findings { + if string(finding.Verdict) != "not_exploitable" { + included = append(included, finding) + } + } + + results := make([]any, 0, len(included)) + for _, finding := range included { + results = append(results, buildResult(finding, tool.PropertyPrefix)) + } + + sarif := obj{ + {"$schema", "https://json.schemastore.org/sarif-2.1.0.json"}, + {"version", "2.1.0"}, + {"runs", []any{ + obj{ + {"tool", buildToolSection(included, tool)}, + {"results", results}, + {"automationDetails", obj{ + {"id", fmt.Sprintf("%s/audit/%s/%s", tool.PropertyPrefix, result.Repository, result.Timestamp.String())}, + }}, + }, + }}, + } + return dumpsIndent(sarif, 2) +} + +// SarifTool is who a SARIF log says produced it: the driver's name, its +// version, where to read about it, and the prefix its own properties carry. +// sec-af's is [SecAfTool], which the goldens hold byte for byte; codeaf's +// program names itself (internal/secaf's sec), because a code-scanning tool +// shows the driver's name beside every result, and the program a person ran +// is sec. +type SarifTool struct { + Name string + // SemanticVersion is a semver and Version anything else; an empty one + // is left out of the log. + SemanticVersion, Version string + InformationURI string + // PropertyPrefix leads the driver's own result properties + // (`/verdict`) and its automation id. + PropertyPrefix string +} + +// RuleIDsAs is findings with their SARIF rule ids under tool's prefix in +// place of sec-af's (`sec-af/sast/cwe-78` is `sec/sast/cwe-78`), so a log +// names one tool throughout. sec-af's own identity leaves them as they are; +// the slice is a copy, and the findings handed in are not changed. +func RuleIDsAs(findings []schemas.VerifiedFinding, tool SarifTool) []schemas.VerifiedFinding { + if tool.PropertyPrefix == "" || tool.PropertyPrefix == SecAfTool.PropertyPrefix { + return findings + } + out := append([]schemas.VerifiedFinding(nil), findings...) + for i := range out { + if rest, ok := strings.CutPrefix(out[i].SarifRuleID, SecAfTool.PropertyPrefix+"/"); ok { + out[i].SarifRuleID = tool.PropertyPrefix + "/" + rest + } + } + return out +} + +// SecAfTool is sec-af's own identity, as its Python original wrote it. +var SecAfTool = SarifTool{Name: "SEC-AF", SemanticVersion: pythonPackageVersion, + InformationURI: "https://github.com/Agent-Field/sec-af", PropertyPrefix: "sec-af"} + +// RenderSarif ports render_sarif, the alias generate_sarif is exported under. +func RenderSarif(auditResult schemas.SecurityAuditResult) string { + return GenerateSarif(auditResult) +} + +// buildToolSection ports _build_tool_section: one rule per distinct +// sarif_rule_id, in sorted rule-id order (Python's `sorted(rules_by_id.items())`). +func buildToolSection(findings []schemas.VerifiedFinding, tool SarifTool) obj { + rulesByID := map[string][]schemas.VerifiedFinding{} + for _, finding := range findings { + rulesByID[finding.SarifRuleID] = append(rulesByID[finding.SarifRuleID], finding) + } + ruleIDs := make([]string, 0, len(rulesByID)) + for ruleID := range rulesByID { + ruleIDs = append(ruleIDs, ruleID) + } + sort.Strings(ruleIDs) + + rules := make([]any, 0, len(ruleIDs)) + for _, ruleID := range ruleIDs { + rules = append(rules, buildRule(ruleID, rulesByID[ruleID])) + } + + driver := obj{{"name", tool.Name}} + if tool.SemanticVersion != "" { + driver = append(driver, kv{"semanticVersion", tool.SemanticVersion}) + } + if tool.Version != "" { + driver = append(driver, kv{"version", tool.Version}) + } + if tool.InformationURI != "" { + driver = append(driver, kv{"informationUri", tool.InformationURI}) + } + driver = append(driver, kv{"rules", rules}) + return obj{{"driver", driver}} +} + +// buildRule ports _build_rule. The first finding for the rule id supplies the +// human-readable text; level, security-severity and precision are aggregated +// across every finding that shares the id. +func buildRule(ruleID string, findings []schemas.VerifiedFinding) obj { + representative := findings[0] + maxScore := findings[0].ExploitabilityScore + for _, finding := range findings[1:] { + if finding.ExploitabilityScore > maxScore { + maxScore = finding.ExploitabilityScore + } + } + number := cweNumber(representative.CweID) + return obj{ + {"id", ruleID}, + {"name", ruleName(ruleID)}, + {"shortDescription", obj{{"text", representative.Title + " vulnerability"}}}, + {"fullDescription", obj{{"text", representative.Description}}}, + {"helpUri", "https://cwe.mitre.org/data/definitions/" + number + ".html"}, + {"defaultConfiguration", obj{{"level", maxLevel(findings)}}}, + {"properties", obj{ + {"precision", maxPrecision(findings)}, + {"security-severity", formatSecuritySeverity(maxScore)}, + {"tags", aggregateRuleTags(findings)}, + }}, + } +} + +// buildResult ports _build_result: one SARIF result per included finding. +// +// `relatedLocations` and `codeFlows` are only present when non-empty, matching +// the Python `if related_locations:` / `if code_flows:` guards. +func buildResult(finding schemas.VerifiedFinding, prefix string) obj { + locations := []any{obj{{"physicalLocation", physicalLocation(finding.Location)}}} + + result := obj{ + {"ruleId", finding.SarifRuleID}, + {"level", severityToLevelOf(string(finding.Severity))}, + {"message", obj{{"text", messageText(finding)}}}, + {"locations", locations}, + {"partialFingerprints", obj{{"primaryLocationLineHash", finding.Fingerprint}}}, + {"properties", obj{ + {"security-severity", formatSecuritySeverity(finding.SarifSecuritySeverity)}, + {prefix + "/verdict", string(finding.Verdict)}, + {prefix + "/evidence_level", int(finding.EvidenceLevel)}, + {prefix + "/exploitability_score", finding.ExploitabilityScore}, + {prefix + "/chain_id", finding.ChainID}, + {prefix + "/compliance", complianceList(finding)}, + {"tags", resultTags(finding)}, + }}, + } + + if related := relatedLocations(finding.RelatedLocations); len(related) > 0 { + result = append(result, kv{"relatedLocations", related}) + } + if flows := codeFlows(finding); len(flows) > 0 { + result = append(result, kv{"codeFlows", flows}) + } + return result +} + +// messageText ports _message_text. +func messageText(finding schemas.VerifiedFinding) string { + verdict := strings.ToUpper(string(finding.Verdict)) + return fmt.Sprintf("[%s] %s: %s. Evidence level: %s.", + verdict, finding.Title, finding.Description, finding.EvidenceLevel.Name()) +} + +// physicalLocation ports _physical_location. +// +// Python parity: startColumn/endColumn are emitted on an `is not None` check +// (so a real 0 would be kept) while the snippet is emitted on a TRUTHY check +// (so an empty code_snippet is dropped, exactly like a missing one). +func physicalLocation(location schemas.Location) obj { + region := obj{ + {"startLine", location.StartLine}, + {"endLine", location.EndLine}, + } + if location.StartColumn != nil { + region = append(region, kv{"startColumn", *location.StartColumn}) + } + if location.EndColumn != nil { + region = append(region, kv{"endColumn", *location.EndColumn}) + } + if location.CodeSnippet != nil && *location.CodeSnippet != "" { + region = append(region, kv{"snippet", obj{{"text", *location.CodeSnippet}}}) + } + + return obj{ + {"artifactLocation", obj{ + {"uri", location.FilePath}, + {"uriBaseId", "%SRCROOT%"}, + }}, + {"region", region}, + } +} + +// relatedLocations ports _related_locations: ids are 1-based +// (`enumerate(locations, start=1)`). +func relatedLocations(locations []schemas.Location) []any { + related := make([]any, 0, len(locations)) + for index, location := range locations { + related = append(related, obj{ + {"id", index + 1}, + {"physicalLocation", physicalLocation(location)}, + {"message", obj{{"text", "Related location"}}}, + }) + } + return related +} + +// codeFlows ports _code_flows: a single thread flow built from the proof's +// data-flow trace, or nothing at all when there is no proof or no trace. +func codeFlows(finding schemas.VerifiedFinding) []any { + if finding.Proof == nil || len(finding.Proof.DataFlowTrace) == 0 { + return nil + } + flowLocations := make([]any, 0, len(finding.Proof.DataFlowTrace)) + for _, step := range finding.Proof.DataFlowTrace { + flowLocations = append(flowLocations, obj{ + {"location", obj{ + {"physicalLocation", obj{ + {"artifactLocation", obj{{"uri", step.File}}}, + {"region", obj{{"startLine", step.Line}}}, + }}, + {"message", obj{{"text", step.Description}}}, + }}, + }) + } + return []any{obj{{"threadFlows", []any{obj{{"locations", flowLocations}}}}}} +} + +// severityToLevelOf ports _severity_to_level: an unknown severity maps to +// "warning". +func severityToLevelOf(severity string) string { + if level, ok := severityToLevel[severity]; ok { + return level + } + return "warning" +} + +// maxLevel ports _max_level. Python's max() returns the FIRST element holding +// the maximum key, which the strict `>` comparison reproduces. +func maxLevel(findings []schemas.VerifiedFinding) string { + best := severityToLevelOf(string(findings[0].Severity)) + for _, finding := range findings[1:] { + level := severityToLevelOf(string(finding.Severity)) + if levelRank[level] > levelRank[best] { + best = level + } + } + return best +} + +// maxPrecision ports _max_precision, with the same first-wins tie-break. +func maxPrecision(findings []schemas.VerifiedFinding) string { + precisionOf := func(finding schemas.VerifiedFinding) string { + if precision, ok := verdictToPrecision[string(finding.Verdict)]; ok { + return precision + } + return "medium" + } + best := precisionOf(findings[0]) + for _, finding := range findings[1:] { + precision := precisionOf(finding) + if precisionRank[precision] > precisionRank[best] { + best = precision + } + } + return best +} + +// complianceList ports _compliance_list. +func complianceList(finding schemas.VerifiedFinding) []any { + out := make([]any, 0, len(finding.Compliance)) + for _, mapping := range finding.Compliance { + out = append(out, complianceEntry(mapping.Framework, mapping.ControlID)) + } + return out +} + +// aggregateRuleTags ports _aggregate_rule_tags: the union of every finding's +// base and compliance tags, sorted. +func aggregateRuleTags(findings []schemas.VerifiedFinding) []any { + tags := map[string]struct{}{} + for _, finding := range findings { + for _, tag := range baseTags(finding) { + tags[tag] = struct{}{} + } + for _, tag := range complianceTags(finding) { + tags[tag] = struct{}{} + } + } + return sortedTagList(tags) +} + +// resultTags ports _result_tags. +func resultTags(finding schemas.VerifiedFinding) []any { + tags := map[string]struct{}{} + for _, tag := range baseTags(finding) { + tags[tag] = struct{}{} + } + for _, tag := range complianceTags(finding) { + tags[tag] = struct{}{} + } + return sortedTagList(tags) +} + +// sortedTagList renders a Python `sorted(set_of_tags)`. Python sorts strings by +// code point; Go's byte-order sort agrees for valid UTF-8. +func sortedTagList(tags map[string]struct{}) []any { + names := make([]string, 0, len(tags)) + for tag := range tags { + names = append(names, tag) + } + sort.Strings(names) + out := make([]any, 0, len(names)) + for _, name := range names { + out = append(out, name) + } + return out +} + +// baseTags ports _base_tags: "security", the upper-cased CWE id, the OWASP +// category when set, then the finding's own tags sorted. +func baseTags(finding schemas.VerifiedFinding) []string { + tags := []string{"security", strings.ToUpper(finding.CweID)} + if finding.OwaspCategory != nil && *finding.OwaspCategory != "" { + tags = append(tags, "OWASP-"+*finding.OwaspCategory) + } + own := append([]string(nil), finding.Tags...) + sort.Strings(own) + return append(tags, own...) +} + +// complianceTags ports _compliance_tags. +func complianceTags(finding schemas.VerifiedFinding) []string { + out := make([]string, 0, len(finding.Compliance)) + for _, mapping := range finding.Compliance { + out = append(out, "compliance:"+mapping.Framework+":"+normalizeControlID(mapping.ControlID)) + } + return out +} + +// complianceEntry ports _compliance_entry. +func complianceEntry(framework, controlID string) string { + return framework + ":" + normalizeControlID(controlID) +} + +// normalizeControlID ports _normalize_control_id: +// +// re.sub(r"\s+", "-", control_id.strip()) +// +// so "Req 6.2.4" becomes "Req-6.2.4". Implemented by hand rather than with +// regexp because Python's `\s` for a str pattern is Unicode-aware while Go's +// `\s` is ASCII-only; pyIsSpace below is the Python classification. +func normalizeControlID(controlID string) string { + trimmed := strings.TrimFunc(controlID, pyIsSpace) + var b strings.Builder + b.Grow(len(trimmed)) + inRun := false + for _, r := range trimmed { + if pyIsSpace(r) { + if !inRun { + b.WriteByte('-') + inRun = true + } + continue + } + inRun = false + b.WriteRune(r) + } + return b.String() +} + +// pyIsSpace reports whether r is whitespace to Python (`str.isspace()` and the +// `\s` class of a str regex): Go's unicode.IsSpace plus the four information +// separators U+001C..U+001F. +func pyIsSpace(r rune) bool { + if r >= 0x1C && r <= 0x1F { + return true + } + return unicode.IsSpace(r) +} + +// formatSecuritySeverity ports _format_security_severity: the score clamped to +// [0, 10] and formatted with one decimal. Go's %.1f and Python's `:.1f` both +// round the exact binary value half-to-even, so they agree bit for bit. +func formatSecuritySeverity(score float64) string { + bounded := score + if bounded < 0 { + bounded = 0 + } + if bounded > 10 { + bounded = 10 + } + return fmt.Sprintf("%.1f", bounded) +} + +// ruleName ports _rule_name: the last "/"-separated segment of the rule id, +// split on "-", each non-empty chunk capitalized and concatenated — +// "sec-af/sast/sql-injection" becomes "SqlInjection" — falling back to +// "SecAfRule" when that yields nothing. +func ruleName(ruleID string) string { + segments := strings.Split(ruleID, "/") + rawName := segments[len(segments)-1] + var b strings.Builder + for _, chunk := range strings.Split(rawName, "-") { + if chunk == "" { + continue + } + b.WriteString(pyCapitalize(chunk)) + } + if b.Len() == 0 { + return "SecAfRule" + } + return b.String() +} + +// pyCapitalize ports Python's str.capitalize(): the first character is +// upper-cased and EVERY other character is lower-cased ("sqlINJECTION" -> +// "Sqlinjection"). +func pyCapitalize(s string) string { + if s == "" { + return "" + } + runes := []rune(s) + out := make([]rune, 0, len(runes)) + out = append(out, unicode.ToUpper(runes[0])) + for _, r := range runes[1:] { + out = append(out, unicode.ToLower(r)) + } + return string(out) +} + +// cweNumber ports _cwe_number: the CWE id upper-cased with every "CWE-" +// occurrence removed, so "cwe-89" becomes "89". +func cweNumber(cweID string) string { + return strings.ReplaceAll(strings.ToUpper(cweID), "CWE-", "") +} diff --git a/internal/secaf/output/sarif_test.go b/internal/secaf/output/sarif_test.go new file mode 100644 index 0000000000..ff65ee0563 --- /dev/null +++ b/internal/secaf/output/sarif_test.go @@ -0,0 +1,307 @@ +package output + +import ( + "encoding/json" + "testing" +) + +// This file ports tests/test_sarif.py. The Python tests take the +// `sample_security_audit_result` conftest fixture; the Go tests load the same +// data from testdata/audit_result.json, which scripts/gen_golden.py writes from +// that very fixture. + +// sarifDoc parses a generated SARIF document into an untyped tree. +func sarifDoc(t *testing.T, raw string) map[string]any { + t.Helper() + var payload map[string]any + if err := json.Unmarshal([]byte(raw), &payload); err != nil { + t.Fatalf("generated SARIF is not valid JSON: %v", err) + } + return payload +} + +func mapAt(t *testing.T, value any, what string) map[string]any { + t.Helper() + m, ok := value.(map[string]any) + if !ok { + t.Fatalf("%s: want an object, got %T", what, value) + } + return m +} + +func sliceAt(t *testing.T, value any, what string) []any { + t.Helper() + s, ok := value.([]any) + if !ok { + t.Fatalf("%s: want an array, got %T", what, value) + } + return s +} + +// sarifRun returns runs[0] of a generated document. +func sarifRun(t *testing.T, payload map[string]any) map[string]any { + t.Helper() + runs := sliceAt(t, payload["runs"], "runs") + if len(runs) != 1 { + t.Fatalf("want exactly one run, got %d", len(runs)) + } + return mapAt(t, runs[0], "runs[0]") +} + +func containsAny(list []any, want string) bool { + for _, item := range list { + if s, ok := item.(string); ok && s == want { + return true + } + } + return false +} + +// TestGenerateSarifHasValid210Envelope ports +// test_generate_sarif_has_valid_2_1_0_envelope. +func TestGenerateSarifHasValid210Envelope(t *testing.T) { + result := loadFixture(t, "audit_result") + payload := sarifDoc(t, GenerateSarif(result)) + run := sarifRun(t, payload) + tool := mapAt(t, mapAt(t, run["tool"], "tool")["driver"], "driver") + + if payload["$schema"] != "https://json.schemastore.org/sarif-2.1.0.json" { + t.Errorf("$schema = %v", payload["$schema"]) + } + if payload["version"] != "2.1.0" { + t.Errorf("version = %v", payload["version"]) + } + if tool["name"] != "SEC-AF" { + t.Errorf("driver.name = %v", tool["name"]) + } + if tool["informationUri"] != "https://github.com/Agent-Field/sec-af" { + t.Errorf("driver.informationUri = %v", tool["informationUri"]) + } + wantID := "sec-af/audit/Agent-Field/sec-af/2026-03-04T10:30:00+00:00" + if got := mapAt(t, run["automationDetails"], "automationDetails")["id"]; got != wantID { + t.Errorf("automationDetails.id = %v, want %q", got, wantID) + } +} + +// TestGenerateSarifFiltersNotExploitableAndMapsSeverity ports +// test_generate_sarif_filters_not_exploitable_and_maps_severity. +func TestGenerateSarifFiltersNotExploitableAndMapsSeverity(t *testing.T) { + result := loadFixture(t, "audit_result") + run := sarifRun(t, sarifDoc(t, GenerateSarif(result))) + results := sliceAt(t, run["results"], "results") + + byRule := map[string]map[string]any{} + for _, item := range results { + entry := mapAt(t, item, "result") + byRule[entry["ruleId"].(string)] = entry + } + + if len(results) != 2 { + t.Fatalf("want 2 results, got %d", len(results)) + } + if _, present := byRule["sec-af/sast/xss"]; present { + t.Error("the not_exploitable finding leaked into the SARIF results") + } + if byRule["sec-af/sast/sql-injection"]["level"] != "error" { + t.Errorf("sql-injection level = %v, want \"error\"", byRule["sec-af/sast/sql-injection"]["level"]) + } + if byRule["sec-af/api/missing-authentication"]["level"] != "error" { + t.Errorf("missing-authentication level = %v, want \"error\"", byRule["sec-af/api/missing-authentication"]["level"]) + } +} + +// TestGenerateSarifIncludesComplianceTagsCodeflowAndLocations ports +// test_generate_sarif_includes_compliance_tags_codeflow_and_locations. +func TestGenerateSarifIncludesComplianceTagsCodeflowAndLocations(t *testing.T) { + result := loadFixture(t, "audit_result") + run := sarifRun(t, sarifDoc(t, GenerateSarif(result))) + + var sql map[string]any + for _, item := range sliceAt(t, run["results"], "results") { + entry := mapAt(t, item, "result") + if entry["ruleId"] == "sec-af/sast/sql-injection" { + sql = entry + } + } + if sql == nil { + t.Fatal("no sql-injection result") + } + + properties := mapAt(t, sql["properties"], "properties") + locations := sliceAt(t, sql["locations"], "locations") + physical := mapAt(t, mapAt(t, locations[0], "locations[0]")["physicalLocation"], "physicalLocation") + region := mapAt(t, physical["region"], "region") + related := sliceAt(t, sql["relatedLocations"], "relatedLocations") + codeFlows := sliceAt(t, sql["codeFlows"], "codeFlows") + + if !containsAny(sliceAt(t, properties["sec-af/compliance"], "sec-af/compliance"), "PCI-DSS:Req-6.2.4") { + t.Errorf("sec-af/compliance = %v", properties["sec-af/compliance"]) + } + if !containsAny(sliceAt(t, properties["tags"], "tags"), "compliance:PCI-DSS:Req-6.2.4") { + t.Errorf("tags = %v", properties["tags"]) + } + if region["startLine"] != float64(42) { + t.Errorf("region.startLine = %v, want 42", region["startLine"]) + } + if region["startColumn"] != float64(9) { + t.Errorf("region.startColumn = %v, want 9", region["startColumn"]) + } + firstRelated := mapAt(t, related[0], "relatedLocations[0]") + uri := mapAt(t, mapAt(t, firstRelated["physicalLocation"], "physicalLocation")["artifactLocation"], "artifactLocation")["uri"] + if uri != "src/routes.py" { + t.Errorf("relatedLocations[0] uri = %v", uri) + } + threadFlows := sliceAt(t, mapAt(t, codeFlows[0], "codeFlows[0]")["threadFlows"], "threadFlows") + flowLocations := sliceAt(t, mapAt(t, threadFlows[0], "threadFlows[0]")["locations"], "locations") + if len(flowLocations) != 2 { + t.Errorf("thread flow has %d locations, want 2", len(flowLocations)) + } +} + +// TestGenerateSarifRuleEntriesAggregatePrecisionAndSeverity ports +// test_generate_sarif_rule_entries_aggregate_precision_and_severity. +func TestGenerateSarifRuleEntriesAggregatePrecisionAndSeverity(t *testing.T) { + result := loadFixture(t, "audit_result") + run := sarifRun(t, sarifDoc(t, GenerateSarif(result))) + driver := mapAt(t, mapAt(t, run["tool"], "tool")["driver"], "driver") + + var sqlRule map[string]any + for _, item := range sliceAt(t, driver["rules"], "rules") { + rule := mapAt(t, item, "rule") + if rule["id"] == "sec-af/sast/sql-injection" { + sqlRule = rule + } + } + if sqlRule == nil { + t.Fatal("no sql-injection rule") + } + properties := mapAt(t, sqlRule["properties"], "properties") + + if got := mapAt(t, sqlRule["defaultConfiguration"], "defaultConfiguration")["level"]; got != "error" { + t.Errorf("defaultConfiguration.level = %v, want \"error\"", got) + } + if properties["precision"] != "very-high" { + t.Errorf("precision = %v, want \"very-high\"", properties["precision"]) + } + if properties["security-severity"] != "10.0" { + t.Errorf("security-severity = %v, want \"10.0\"", properties["security-severity"]) + } + if !containsAny(sliceAt(t, properties["tags"], "tags"), "CWE-89") { + t.Errorf("tags = %v", properties["tags"]) + } +} + +// TestGenerateSarifIsStableForSameInput ports +// test_generate_sarif_is_stable_for_same_input. It is the assertion that keeps +// Go map iteration out of the artifact. +func TestGenerateSarifIsStableForSameInput(t *testing.T) { + result := loadFixture(t, "audit_result") + first := GenerateSarif(result) + for i := 0; i < 20; i++ { + if got := GenerateSarif(result); got != first { + t.Fatalf("run %d differs from the first", i) + } + } + if RenderSarif(result) != first { + t.Error("render_sarif differs from generate_sarif") + } +} + +// --------------------------------------------------------------------------- +// helper-level tests (no Python counterpart; they pin the helpers the goldens +// only exercise indirectly) +// --------------------------------------------------------------------------- + +// TestRuleName covers _rule_name, including the "SecAfRule" fallback. +func TestRuleName(t *testing.T) { + cases := map[string]string{ + "sec-af/sast/sql-injection": "SqlInjection", + "sec-af/api/missing-authentication": "MissingAuthentication", + "sec-af/": "SecAfRule", + "": "SecAfRule", + "---": "SecAfRule", + "sec-af/sast/XSS": "Xss", + "sec-af/sast/a--b": "AB", + "noslash": "Noslash", + } + for input, want := range cases { + if got := ruleName(input); got != want { + t.Errorf("ruleName(%q) = %q, want %q", input, got, want) + } + } +} + +// TestCweNumber covers _cwe_number. +func TestCweNumber(t *testing.T) { + cases := map[string]string{ + "CWE-89": "89", + "cwe-89": "89", + "89": "89", + "CWE-CWE-1": "1", + "": "", + } + for input, want := range cases { + if got := cweNumber(input); got != want { + t.Errorf("cweNumber(%q) = %q, want %q", input, got, want) + } + } +} + +// TestNormalizeControlID covers _normalize_control_id: strip, then every run of +// whitespace collapses to a single "-". +func TestNormalizeControlID(t *testing.T) { + cases := map[string]string{ + "Req 6.2.4": "Req-6.2.4", + " Req 6.2.4 ": "Req-6.2.4", + "Req 6.2.4": "Req-6.2.4", + "Req\t6.2.4": "Req-6.2.4", + "A03:2021": "A03:2021", + "": "", + " ": "", + } + for input, want := range cases { + if got := normalizeControlID(input); got != want { + t.Errorf("normalizeControlID(%q) = %q, want %q", input, got, want) + } + } +} + +// TestFormatSecuritySeverity covers _format_security_severity: clamped to +// [0, 10] and rendered with one decimal (half-to-even, like Python's :.1f). +func TestFormatSecuritySeverity(t *testing.T) { + cases := []struct { + in float64 + want string + }{ + {9.9, "9.9"}, + {10.0, "10.0"}, + {11.5, "10.0"}, + {-1.0, "0.0"}, + {0.04, "0.0"}, + {9.25, "9.2"}, + {0.35, "0.3"}, + } + for _, tc := range cases { + if got := formatSecuritySeverity(tc.in); got != tc.want { + t.Errorf("formatSecuritySeverity(%v) = %q, want %q", tc.in, got, tc.want) + } + } +} + +// TestSeverityToLevelOf covers _severity_to_level's default. +func TestSeverityToLevelOf(t *testing.T) { + cases := map[string]string{ + "critical": "error", + "high": "error", + "medium": "warning", + "low": "note", + "info": "note", + "bogus": "warning", + "": "warning", + } + for input, want := range cases { + if got := severityToLevelOf(input); got != want { + t.Errorf("severityToLevelOf(%q) = %q, want %q", input, got, want) + } + } +} diff --git a/internal/secaf/output/testdata/audit_result.json b/internal/secaf/output/testdata/audit_result.json new file mode 100644 index 0000000000..1667eb0d50 --- /dev/null +++ b/internal/secaf/output/testdata/audit_result.json @@ -0,0 +1,234 @@ +{ + "agent_invocations": 24, + "attack_chains": [ + { + "chain_id": "chain-1", + "combined_impact": "Unauthorized DB disclosure", + "combined_severity": "critical", + "description": "Untrusted input reaches SQL sink", + "findings": [ + "finding-confirmed", + "finding-likely" + ], + "mitre_attack_mapping": [ + { + "tactic": "Initial Access", + "technique_id": "T1190", + "technique_name": "Exploit Public-Facing Application" + } + ], + "title": "Input to DB read" + } + ], + "branch": "issue-23-tests", + "by_severity": { + "critical": 1, + "high": 1, + "low": 1 + }, + "commit_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "compliance_gaps": [ + { + "control_id": "Req 6.2.4", + "control_name": "Prevent injection", + "cwe_ids": [ + "CWE-89" + ], + "finding_count": 1, + "framework": "PCI-DSS", + "max_severity": "critical" + } + ], + "confirmed": 1, + "cost_breakdown": { + "hunt": 1.2, + "prove": 1.51, + "recon": 0.5 + }, + "cost_usd": 3.21, + "depth_profile": "standard", + "duration_seconds": 182.4, + "findings": [ + { + "chain_id": "chain-1", + "chain_step": 1, + "compliance": [ + { + "control_id": "Req 6.2.4", + "control_name": "Prevent injection", + "framework": "PCI-DSS" + } + ], + "cvss_v4": null, + "cwe_id": "CWE-89", + "cwe_name": "SQL Injection", + "description": "Unsanitized user input reaches SQL query execution.", + "drop_reason": null, + "enables": [ + "finding-likely" + ], + "epss": null, + "evidence_level": 6, + "exploitability_score": 10.0, + "finding_type": "sast", + "fingerprint": "fp-sql-1", + "id": "finding-confirmed", + "location": { + "code_snippet": "cursor.execute(f\"SELECT * FROM users WHERE id = {user_id}\")", + "end_column": 66, + "end_line": 42, + "file_path": "src/users.py", + "function_name": "lookup_user", + "start_column": 9, + "start_line": 42 + }, + "owasp_category": "A03:2021", + "proof": { + "chain_steps": null, + "data_flow_evidence": null, + "data_flow_trace": [ + { + "description": "Input source", + "file": "src/routes.py", + "line": 15, + "tainted": true + }, + { + "description": "SQL sink", + "file": "src/users.py", + "line": 42, + "tainted": true + } + ], + "evidence_level": 6, + "expected_outcome": "Unauthorized data access", + "exploit_hypothesis": "Inject through id parameter.", + "exploit_payload": "{\"id\": \"1 OR 1=1\"}", + "http_request": null, + "http_response": null, + "poc_code": null, + "poc_execution_output": null, + "reachability": null, + "sanitization_analysis": null, + "verification_method": "manual-review+trace", + "vulnerable_code": "cursor.execute(f\"SELECT * FROM users WHERE id = {user_id}\")" + }, + "rationale": "Source-to-sink path is confirmed and exploitable.", + "related_locations": [ + { + "code_snippet": "user_id = request.json['id']", + "end_column": null, + "end_line": 15, + "file_path": "src/routes.py", + "function_name": null, + "start_column": null, + "start_line": 15 + } + ], + "remediation": null, + "reproduction_steps": [], + "sarif_rule_id": "sec-af/sast/sql-injection", + "sarif_security_severity": 9.9, + "severity": "critical", + "tags": [ + "externally_reachable", + "user-input" + ], + "title": "SQL Injection", + "verdict": "confirmed" + }, + { + "chain_id": null, + "chain_step": null, + "compliance": [], + "cvss_v4": null, + "cwe_id": "CWE-306", + "cwe_name": "Missing Authentication for Critical Function", + "description": "Admin endpoint can be accessed without auth.", + "drop_reason": null, + "enables": null, + "epss": null, + "evidence_level": 2, + "exploitability_score": 4.8, + "finding_type": "api", + "fingerprint": "fp-auth-1", + "id": "finding-likely", + "location": { + "code_snippet": null, + "end_column": null, + "end_line": 11, + "file_path": "src/api/admin.py", + "function_name": null, + "start_column": null, + "start_line": 11 + }, + "owasp_category": "A07:2021", + "proof": null, + "rationale": "Guard checks appear absent on route.", + "related_locations": [], + "remediation": null, + "reproduction_steps": [], + "sarif_rule_id": "sec-af/api/missing-authentication", + "sarif_security_severity": 7.6, + "severity": "high", + "tags": [ + "requires_auth" + ], + "title": "Missing Authentication", + "verdict": "likely" + }, + { + "chain_id": null, + "chain_step": null, + "compliance": [], + "cvss_v4": null, + "cwe_id": "CWE-79", + "cwe_name": "Cross-site Scripting", + "description": "Output is escaped by template engine.", + "drop_reason": null, + "enables": null, + "epss": null, + "evidence_level": 1, + "exploitability_score": 0.6, + "finding_type": "sast", + "fingerprint": "fp-noise-1", + "id": "finding-noise", + "location": { + "code_snippet": null, + "end_column": null, + "end_line": 89, + "file_path": "src/views.py", + "function_name": null, + "start_column": null, + "start_line": 88 + }, + "owasp_category": null, + "proof": null, + "rationale": "Sink auto-escapes output.", + "related_locations": [], + "remediation": null, + "reproduction_steps": [], + "sarif_rule_id": "sec-af/sast/xss", + "sarif_security_severity": 1.9, + "severity": "low", + "tags": [], + "title": "Potential XSS", + "verdict": "not_exploitable" + } + ], + "inconclusive": 0, + "likely": 1, + "metadata": {}, + "noise_reduction_pct": 66.7, + "not_exploitable": 1, + "policy_violations": [], + "provider": "opencode", + "repository": "Agent-Field/sec-af", + "sarif": "{}", + "strategies_used": [ + "injection", + "auth" + ], + "timestamp": "2026-03-04T10:30:00Z", + "total_raw_findings": 6 +} diff --git a/internal/secaf/output/testdata/audit_result_edge.json b/internal/secaf/output/testdata/audit_result_edge.json new file mode 100644 index 0000000000..bd30959fda --- /dev/null +++ b/internal/secaf/output/testdata/audit_result_edge.json @@ -0,0 +1,306 @@ +{ + "agent_invocations": 3, + "attack_chains": [ + { + "chain_id": "chain-x", + "combined_impact": "Impact /", `" /"`}, + // ensure_ascii=True is json.dumps' default: every non-ASCII code point is + // escaped, and an astral one becomes a UTF-16 surrogate pair. + {"latin1", "h\u00e9llo", `"h\u00e9llo"`}, + {"em dash", "a \u2014 b", `"a \u2014 b"`}, + {"cjk", "\u4e16\u754c", `"\u4e16\u754c"`}, + {"astral becomes a surrogate pair", "\U0001F680", `"\ud83d\ude80"`}, + {"NEL", "\u0085", `"\u0085"`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := DumpsCompact(tc.in); got != tc.want { + t.Fatalf("DumpsCompact(%q) = %s, want %s", tc.in, got, tc.want) + } + }) + } +} + +// TestDumpsContainerSeparators pins the two separator regimes: (", ", ": ") with +// no indent, (",", ": ") plus newlines with one. +// +// python -c 'import json; print(json.dumps({"a":1,"b":[1,2]}))' +// python -c 'import json; print(json.dumps({"a":1,"b":[1,2]}, indent=2))' +func TestDumpsContainerSeparators(t *testing.T) { + value := Ordered{{Key: "a", Value: 1}, {Key: "b", Value: []any{1, 2}}} + + if got, want := DumpsCompact(value), `{"a": 1, "b": [1, 2]}`; got != want { + t.Fatalf("compact = %s, want %s", got, want) + } + want := "{\n \"a\": 1,\n \"b\": [\n 1,\n 2\n ]\n}" + if got := Dumps(value, 2); got != want { + t.Fatalf("indent=2 =\n%s\nwant\n%s", got, want) + } + if got := Dumps(value, 0); got != DumpsCompact(value) { + t.Fatalf("Dumps(v, 0) must equal DumpsCompact(v); got %s", got) + } + + // A four-space indent is the same document with a wider gutter. + want4 := "{\n \"a\": 1,\n \"b\": [\n 1,\n 2\n ]\n}" + if got := Dumps(value, 4); got != want4 { + t.Fatalf("indent=4 =\n%s\nwant\n%s", got, want4) + } +} + +// TestDumpsEmptyContainers pins that an empty list/dict stays on one line even +// in indent mode, exactly as Python renders it. +// +// python -c 'import json; print(json.dumps({"a": [], "b": {}}, indent=2))' +func TestDumpsEmptyContainers(t *testing.T) { + value := Ordered{{Key: "a", Value: []any{}}, {Key: "b", Value: Ordered{}}} + want := "{\n \"a\": [],\n \"b\": {}\n}" + if got := Dumps(value, 2); got != want { + t.Fatalf("got\n%s\nwant\n%s", got, want) + } + if got, want := DumpsCompact([]any{}), "[]"; got != want { + t.Fatalf("empty list compact = %s, want %s", got, want) + } + if got, want := DumpsCompact(map[string]any{}), "{}"; got != want { + t.Fatalf("empty map compact = %s, want %s", got, want) + } +} + +// TestDumpsNilContainers documents the one deliberate encoding/json-shaped +// deviation: a NIL Go slice or map is null, not [] / {}. Pydantic never +// produces None for a list field, so this only fires for an unpopulated Go +// value. +func TestDumpsNilContainers(t *testing.T) { + var nilSlice []string + var nilMap map[string]any + var nilPtr *int + var nilIface any + + for name, in := range map[string]any{ + "nil slice": nilSlice, + "nil map": nilMap, + "nil pointer": nilPtr, + "nil interface": nilIface, + } { + if got := DumpsCompact(in); got != "null" { + t.Fatalf("%s = %s, want null", name, got) + } + } +} + +// TestDumpsMapKeysAreSorted pins the documented map-ordering deviation, and +// that an Ordered is the escape hatch that preserves Python's insertion order. +func TestDumpsMapKeysAreSorted(t *testing.T) { + m := map[string]any{"zebra": 1, "Apple": 2, "_under": 3, "apple": 4} + // python -c 'import json; print(json.dumps({...}, sort_keys=True))' + want := `{"Apple": 2, "_under": 3, "apple": 4, "zebra": 1}` + if got := DumpsCompact(m); got != want { + t.Fatalf("map = %s, want %s", got, want) + } + + o := Ordered{{Key: "zebra", Value: 1}, {Key: "Apple", Value: 2}} + if got, want := DumpsCompact(o), `{"zebra": 1, "Apple": 2}`; got != want { + t.Fatalf("Ordered = %s, want %s", got, want) + } +} + +// TestDumpsMapKeyCoercion pins the non-string key spellings json.dumps uses. +// +// python -c 'import json; print(json.dumps({1: "a", True: "b"}))' # keys "1"/"true" +func TestDumpsMapKeyCoercion(t *testing.T) { + if got, want := DumpsCompact(map[int]string{2: "b", 10: "j"}), `{"10": "j", "2": "b"}`; got != want { + t.Fatalf("int keys = %s, want %s", got, want) + } + if got, want := DumpsCompact(map[bool]int{true: 1}), `{"true": 1}`; got != want { + t.Fatalf("bool key = %s, want %s", got, want) + } +} + +// jsonNumberDoc is decoded with UseNumber so integers stay integers, the way +// Python's json.loads distinguishes int from float. +func TestDumpsJSONNumber(t *testing.T) { + dec := json.NewDecoder(strings.NewReader(`{"i": 7, "big": 123456789012345678901234567890, "f": 1.5, "e": 0.00001, "cap": 1E2}`)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + t.Fatalf("decode: %v", err) + } + // python -c 'import json; print(json.dumps(json.loads(s), sort_keys=True))' + want := `{"big": 123456789012345678901234567890, "cap": 100.0, "e": 1e-05, "f": 1.5, "i": 7}` + if got := DumpsCompact(v); got != want { + t.Fatalf("got %s\nwant %s", got, want) + } +} + +// marshalerStamp stands in for schemas.Timestamp: a value type whose +// MarshalJSON produces the exact Python isoformat spelling. +type marshalerStamp struct{ text string } + +func (m marshalerStamp) MarshalJSON() ([]byte, error) { + return json.Marshal(m.text) +} + +// ptrMarshaler exercises the pointer-receiver marshaler path. +type ptrMarshaler struct{ n int } + +func (p *ptrMarshaler) MarshalJSON() ([]byte, error) { return []byte(`{"n": 1}`), nil } + +func TestDumpsHonorsJSONMarshaler(t *testing.T) { + value := Ordered{ + {Key: "ts", Value: marshalerStamp{text: "2026-01-02T03:04:05.123456+00:00"}}, + {Key: "raw", Value: json.RawMessage(`{"b":2,"a":1}`)}, + } + // The marshaler's own bytes are re-rendered through this encoder, so the + // RawMessage picks up Python's ": " separator and sorted keys. + want := `{"ts": "2026-01-02T03:04:05.123456+00:00", "raw": {"a": 1, "b": 2}}` + if got := DumpsCompact(value); got != want { + t.Fatalf("got %s\nwant %s", got, want) + } + + // A marshaler emitting an INTEGER must not be turned into a float by the + // re-render (UseNumber guards that). + if got, want := DumpsCompact(&ptrMarshaler{}), `{"n": 1}`; got != want { + t.Fatalf("pointer marshaler = %s, want %s", got, want) + } +} + +// structFixture pins the struct walk: declaration order, json tag names, +// `json:"-"`, omitempty, embedded flattening and pointer fields. +type Embedded struct { + Inner string `json:"inner"` +} + +type structFixture struct { + Embedded + Zed string `json:"zed"` + Alpha int `json:"alpha"` + Skipped string `json:"-"` + Omitted string `json:"omitted,omitempty"` + Kept string `json:"kept,omitempty"` + Ptr *float64 `json:"ptr"` + NoTag bool +} + +func TestDumpsStructWalk(t *testing.T) { + f := 1.0 + v := structFixture{ + Embedded: Embedded{Inner: "in"}, + Zed: "z", + Alpha: 1, + Skipped: "never", + Kept: "yes", + Ptr: &f, + NoTag: true, + } + // Declaration order, NOT sorted: this is what makes a Go struct stand in + // for a pydantic model_dump()'s insertion order. + want := `{"inner": "in", "zed": "z", "alpha": 1, "kept": "yes", "ptr": 1.0, "NoTag": true}` + if got := DumpsCompact(v); got != want { + t.Fatalf("got %s\nwant %s", got, want) + } + // A pointer to the struct renders identically. + if got := DumpsCompact(&v); got != want { + t.Fatalf("pointer form got %s\nwant %s", got, want) + } +} + +// TestDumpsNestedIndentation pins the indentation of a struct inside a slice +// inside a struct, the shape every model_dump() golden exercises. +func TestDumpsNestedIndentation(t *testing.T) { + type leaf struct { + A int `json:"a"` + } + type root struct { + Leaves []leaf `json:"leaves"` + } + want := "{\n \"leaves\": [\n {\n \"a\": 1\n },\n {\n \"a\": 2\n }\n ]\n}" + if got := Dumps(root{Leaves: []leaf{{A: 1}, {A: 2}}}, 2); got != want { + t.Fatalf("got\n%s\nwant\n%s", got, want) + } +} + +// TestDumpsBytesAreBase64 documents the []byte rendering. No SEC-AF model has a +// bytes field; the branch exists so a stray []byte cannot render as a list of +// integers. +func TestDumpsBytesAreBase64(t *testing.T) { + if got, want := DumpsCompact([]byte("hi")), `"aGk="`; got != want { + t.Fatalf("got %s, want %s", got, want) + } +} diff --git a/internal/secaf/pyfmt/testdata/golden/dumps_ArchitectureMap_compact.txt b/internal/secaf/pyfmt/testdata/golden/dumps_ArchitectureMap_compact.txt new file mode 100644 index 0000000000..44cf8b89b8 --- /dev/null +++ b/internal/secaf/pyfmt/testdata/golden/dumps_ArchitectureMap_compact.txt @@ -0,0 +1 @@ +{"app_type": "web_api", "modules": [{"name": "auth", "path": "app/auth/service.py", "language": "python", "description": "Session issuance \u2014 na\u00efve refresh, \u4e16\u754c emoji \ud83d\ude80", "dependencies": ["jwt", "redis"]}, {"name": "empty_deps", "path": "app/empty.py", "language": "go", "description": null, "dependencies": []}], "entry_points": [{"kind": "http", "identifier": "login", "file_path": "app/api/login.py", "line": 42, "method": "POST", "route": "/v1/login", "auth_required": false}, {"kind": "cli", "identifier": "migrate", "file_path": "tools/migrate.py", "line": 0, "method": null, "route": null, "auth_required": null}], "trust_boundaries": [{"name": "edge", "source_zone": "internet", "target_zone": "app", "description": "TLS terminates at the ingress & waf", "enforcement": []}], "services": [{"name": "postgres", "service_type": "database", "endpoint": "postgres://db:5432", "purpose": null, "auth_mechanism": "password"}], "api_surface": [{"method": "GET", "path": "/v1/users/{id}", "handler": "UserController.show", "file_path": "app/api/users.py", "line": 17, "auth_required": true, "rate_limited": null}]} \ No newline at end of file diff --git a/internal/secaf/pyfmt/testdata/golden/dumps_ArchitectureMap_indent2.txt b/internal/secaf/pyfmt/testdata/golden/dumps_ArchitectureMap_indent2.txt new file mode 100644 index 0000000000..237af425ad --- /dev/null +++ b/internal/secaf/pyfmt/testdata/golden/dumps_ArchitectureMap_indent2.txt @@ -0,0 +1,71 @@ +{ + "app_type": "web_api", + "modules": [ + { + "name": "auth", + "path": "app/auth/service.py", + "language": "python", + "description": "Session issuance \u2014 na\u00efve refresh, \u4e16\u754c emoji \ud83d\ude80", + "dependencies": [ + "jwt", + "redis" + ] + }, + { + "name": "empty_deps", + "path": "app/empty.py", + "language": "go", + "description": null, + "dependencies": [] + } + ], + "entry_points": [ + { + "kind": "http", + "identifier": "login", + "file_path": "app/api/login.py", + "line": 42, + "method": "POST", + "route": "/v1/login", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "migrate", + "file_path": "tools/migrate.py", + "line": 0, + "method": null, + "route": null, + "auth_required": null + } + ], + "trust_boundaries": [ + { + "name": "edge", + "source_zone": "internet", + "target_zone": "app", + "description": "TLS terminates at the ingress & waf", + "enforcement": [] + } + ], + "services": [ + { + "name": "postgres", + "service_type": "database", + "endpoint": "postgres://db:5432", + "purpose": null, + "auth_mechanism": "password" + } + ], + "api_surface": [ + { + "method": "GET", + "path": "/v1/users/{id}", + "handler": "UserController.show", + "file_path": "app/api/users.py", + "line": 17, + "auth_required": true, + "rate_limited": null + } + ] +} \ No newline at end of file diff --git a/internal/secaf/pyfmt/testdata/golden/dumps_ConfigReport_compact.txt b/internal/secaf/pyfmt/testdata/golden/dumps_ConfigReport_compact.txt new file mode 100644 index 0000000000..6fe367162a --- /dev/null +++ b/internal/secaf/pyfmt/testdata/golden/dumps_ConfigReport_compact.txt @@ -0,0 +1 @@ +{"secrets": [{"id": "secret-01", "secret_type": "aws_access_key", "file_path": "config/prod.yaml", "line": 12, "match": "AKIA****0001", "confidence": "high", "is_test_value": false}], "misconfigs": [{"id": "misconfig-01", "category": "tls", "file_path": "config/prod.yaml", "line": null, "key": "SSL_VERIFY", "value": "false", "risk": "critical", "remediation": null}]} \ No newline at end of file diff --git a/internal/secaf/pyfmt/testdata/golden/dumps_ConfigReport_indent2.txt b/internal/secaf/pyfmt/testdata/golden/dumps_ConfigReport_indent2.txt new file mode 100644 index 0000000000..fb114dcd85 --- /dev/null +++ b/internal/secaf/pyfmt/testdata/golden/dumps_ConfigReport_indent2.txt @@ -0,0 +1,25 @@ +{ + "secrets": [ + { + "id": "secret-01", + "secret_type": "aws_access_key", + "file_path": "config/prod.yaml", + "line": 12, + "match": "AKIA****0001", + "confidence": "high", + "is_test_value": false + } + ], + "misconfigs": [ + { + "id": "misconfig-01", + "category": "tls", + "file_path": "config/prod.yaml", + "line": null, + "key": "SSL_VERIFY", + "value": "false", + "risk": "critical", + "remediation": null + } + ] +} \ No newline at end of file diff --git a/internal/secaf/pyfmt/testdata/golden/dumps_DependencyReport_compact.txt b/internal/secaf/pyfmt/testdata/golden/dumps_DependencyReport_compact.txt new file mode 100644 index 0000000000..7ee06475d5 --- /dev/null +++ b/internal/secaf/pyfmt/testdata/golden/dumps_DependencyReport_compact.txt @@ -0,0 +1 @@ +{"sbom": [{"name": "django", "version": "4.2.1", "ecosystem": "pypi", "direct": true, "license": "BSD-3-Clause"}, {"name": "urllib3", "version": "1.26.5", "ecosystem": "pypi", "direct": false, "license": null}], "known_cves": [{"cve_id": "CVE-2023-0001", "package": "django", "installed_version": "4.2.1", "fixed_version": "4.2.5", "cvss_v4_score": 1.0, "epss_score": 0.5, "direct": true, "reachable": true}, {"cve_id": "CVE-2023-0002", "package": "urllib3", "installed_version": "1.26.5", "fixed_version": null, "cvss_v4_score": 1e-05, "epss_score": 1e+16, "direct": false, "reachable": null}, {"cve_id": "CVE-2023-0003", "package": "zlib", "installed_version": "1.2.11", "fixed_version": "1.2.12", "cvss_v4_score": 1000000000000000.0, "epss_score": -0.0, "direct": false, "reachable": false}], "outdated": [], "direct_count": 1, "transitive_count": 0} \ No newline at end of file diff --git a/internal/secaf/pyfmt/testdata/golden/dumps_DependencyReport_indent2.txt b/internal/secaf/pyfmt/testdata/golden/dumps_DependencyReport_indent2.txt new file mode 100644 index 0000000000..dc099c26c6 --- /dev/null +++ b/internal/secaf/pyfmt/testdata/golden/dumps_DependencyReport_indent2.txt @@ -0,0 +1,53 @@ +{ + "sbom": [ + { + "name": "django", + "version": "4.2.1", + "ecosystem": "pypi", + "direct": true, + "license": "BSD-3-Clause" + }, + { + "name": "urllib3", + "version": "1.26.5", + "ecosystem": "pypi", + "direct": false, + "license": null + } + ], + "known_cves": [ + { + "cve_id": "CVE-2023-0001", + "package": "django", + "installed_version": "4.2.1", + "fixed_version": "4.2.5", + "cvss_v4_score": 1.0, + "epss_score": 0.5, + "direct": true, + "reachable": true + }, + { + "cve_id": "CVE-2023-0002", + "package": "urllib3", + "installed_version": "1.26.5", + "fixed_version": null, + "cvss_v4_score": 1e-05, + "epss_score": 1e+16, + "direct": false, + "reachable": null + }, + { + "cve_id": "CVE-2023-0003", + "package": "zlib", + "installed_version": "1.2.11", + "fixed_version": "1.2.12", + "cvss_v4_score": 1000000000000000.0, + "epss_score": -0.0, + "direct": false, + "reachable": false + } + ], + "outdated": [], + "direct_count": 1, + "transitive_count": 0 +} \ No newline at end of file diff --git a/internal/secaf/pyfmt/testdata/golden/dumps_SecurityContext_compact.txt b/internal/secaf/pyfmt/testdata/golden/dumps_SecurityContext_compact.txt new file mode 100644 index 0000000000..1c9e0a7aaf --- /dev/null +++ b/internal/secaf/pyfmt/testdata/golden/dumps_SecurityContext_compact.txt @@ -0,0 +1 @@ +{"auth_model": "jwt", "auth_details": "HS256 \u2014 key from vault\ttab\nnewline \"quoted\" back\\slash", "crypto_usage": [{"algorithm": "AES", "key_size": 256, "mode": "GCM", "usage_context": "at-rest", "is_weak": false}, {"algorithm": "MD5", "key_size": null, "mode": null, "usage_context": null, "is_weak": true}], "framework_security": ["CsrfViewMiddleware", ""], "security_headers": [], "deployment_signals": ["k8s ingress & mTLS", "control chars: \u0000\u001f\u007f\u0085"]} \ No newline at end of file diff --git a/internal/secaf/pyfmt/testdata/golden/dumps_SecurityContext_indent2.txt b/internal/secaf/pyfmt/testdata/golden/dumps_SecurityContext_indent2.txt new file mode 100644 index 0000000000..7a1ebe3312 --- /dev/null +++ b/internal/secaf/pyfmt/testdata/golden/dumps_SecurityContext_indent2.txt @@ -0,0 +1,29 @@ +{ + "auth_model": "jwt", + "auth_details": "HS256 \u2014 key from vault\ttab\nnewline \"quoted\" back\\slash", + "crypto_usage": [ + { + "algorithm": "AES", + "key_size": 256, + "mode": "GCM", + "usage_context": "at-rest", + "is_weak": false + }, + { + "algorithm": "MD5", + "key_size": null, + "mode": null, + "usage_context": null, + "is_weak": true + } + ], + "framework_security": [ + "CsrfViewMiddleware", + "" + ], + "security_headers": [], + "deployment_signals": [ + "k8s ingress & mTLS", + "control chars: \u0000\u001f\u007f\u0085" + ] +} \ No newline at end of file diff --git a/internal/secaf/pyfmt/testdata/golden/dumps_edge_cases_compact.txt b/internal/secaf/pyfmt/testdata/golden/dumps_edge_cases_compact.txt new file mode 100644 index 0000000000..e434a7d08e --- /dev/null +++ b/internal/secaf/pyfmt/testdata/golden/dumps_edge_cases_compact.txt @@ -0,0 +1 @@ +{"bools": [true, false], "empty_dict": {}, "empty_list": [], "escapes": "quote\" backslash\\ tab\t newline\n cr\r ff\f bs\b & / slash", "floats": [1.0, 0.5, 1e-05, -0.0, 1e+16, 1000000000000000.0, 3.141592653589793, 0.1], "ints": [0, -1, 1234567890123456789], "nested": {"a": [{"b": {"c": [1, 2.5, null]}}], "z": 1}, "none": null, "sort_check": {"C": 3, "_": 4, "a": 2, "b": 1, "\u00c9": 5}, "unicode": "h\u00e9llo \u2014 \u4e16\u754c \ud83d\ude80 \u007f"} \ No newline at end of file diff --git a/internal/secaf/pyfmt/testdata/golden/dumps_edge_cases_indent2.txt b/internal/secaf/pyfmt/testdata/golden/dumps_edge_cases_indent2.txt new file mode 100644 index 0000000000..d0f67d61b6 --- /dev/null +++ b/internal/secaf/pyfmt/testdata/golden/dumps_edge_cases_indent2.txt @@ -0,0 +1,47 @@ +{ + "bools": [ + true, + false + ], + "empty_dict": {}, + "empty_list": [], + "escapes": "quote\" backslash\\ tab\t newline\n cr\r ff\f bs\b & / slash", + "floats": [ + 1.0, + 0.5, + 1e-05, + -0.0, + 1e+16, + 1000000000000000.0, + 3.141592653589793, + 0.1 + ], + "ints": [ + 0, + -1, + 1234567890123456789 + ], + "nested": { + "a": [ + { + "b": { + "c": [ + 1, + 2.5, + null + ] + } + } + ], + "z": 1 + }, + "none": null, + "sort_check": { + "C": 3, + "_": 4, + "a": 2, + "b": 1, + "\u00c9": 5 + }, + "unicode": "h\u00e9llo \u2014 \u4e16\u754c \ud83d\ude80 \u007f" +} \ No newline at end of file diff --git a/internal/secaf/pyfmt/testdata/golden/model_dump_json_floats.json b/internal/secaf/pyfmt/testdata/golden/model_dump_json_floats.json new file mode 100644 index 0000000000..4d420613e1 --- /dev/null +++ b/internal/secaf/pyfmt/testdata/golden/model_dump_json_floats.json @@ -0,0 +1,24 @@ +{ + "-0.0": "{\"v\":-0.0}", + "-12.75": "{\"v\":-12.75}", + "0": "{\"v\":0.0}", + "0.0001": "{\"v\":0.0001}", + "0.1": "{\"v\":0.1}", + "0.3333333333333333": "{\"v\":0.3333333333333333}", + "1": "{\"v\":1.0}", + "1.23e-5": "{\"v\":0.0000123}", + "1.23e-6": "{\"v\":1.23e-6}", + "1.5": "{\"v\":1.5}", + "1e-3": "{\"v\":0.001}", + "1e-5": "{\"v\":0.00001}", + "1e-6": "{\"v\":1e-6}", + "1e-7": "{\"v\":1e-7}", + "1e15": "{\"v\":1000000000000000.0}", + "1e16": "{\"v\":1e+16}", + "1e17": "{\"v\":1e+17}", + "1e22": "{\"v\":1e+22}", + "1e300": "{\"v\":1e+300}", + "2.5": "{\"v\":2.5}", + "5e-324": "{\"v\":5e-324}", + "9.9e15": "{\"v\":9900000000000000.0}" +} diff --git a/internal/secaf/pyfmt/testdata/golden/model_dump_json_models.json b/internal/secaf/pyfmt/testdata/golden/model_dump_json_models.json new file mode 100644 index 0000000000..a840f2e674 --- /dev/null +++ b/internal/secaf/pyfmt/testdata/golden/model_dump_json_models.json @@ -0,0 +1,5 @@ +{ + "defaults": "{\"name\":\"\",\"ratio\":-0.0,\"count\":-5,\"flag\":false,\"opt\":null,\"items\":[],\"inner\":null,\"mapping\":{}}", + "rich": "{\"name\":\"h\u00e9llo <&> \\\" \\\\ \\t \ud83d\ude00 \\u0001\",\"ratio\":1.0,\"count\":3,\"flag\":true,\"opt\":null,\"items\":[\"a\",\"b\"],\"inner\":{\"x\":1000000000000000.0,\"y\":0},\"mapping\":{\"a\":2.0,\"b\":0.5}}", + "small_float": "{\"name\":\"x\",\"ratio\":0.00001,\"count\":0,\"flag\":false,\"opt\":null,\"items\":[],\"inner\":null,\"mapping\":{}}" +} diff --git a/internal/secaf/pyfmt/testdata/models_fixture.json b/internal/secaf/pyfmt/testdata/models_fixture.json new file mode 100644 index 0000000000..9170f27969 --- /dev/null +++ b/internal/secaf/pyfmt/testdata/models_fixture.json @@ -0,0 +1,169 @@ +{ + "_readme": [ + "Input fixture for the pyfmt.Dumps parity tests. Each key except this one and", + "'edge_cases' is a pydantic class name; its value is the constructor kwargs.", + "go/scripts/gen_golden.py builds the model, calls model_dump(), and writes", + "json.dumps(dump, indent=2) / json.dumps(dump) into testdata/golden/. The Go", + "test unmarshals the SAME sub-object into the identically named Go struct and", + "compares pyfmt.Dumps(v, 2) / pyfmt.DumpsCompact(v) byte for byte.", + "Every field is supplied explicitly so the comparison exercises Dumps, not the", + "two default-seeding implementations." + ], + "ArchitectureMap": { + "app_type": "web_api", + "modules": [ + { + "name": "auth", + "path": "app/auth/service.py", + "language": "python", + "description": "Session issuance — naïve refresh, 世界 emoji 🚀", + "dependencies": ["jwt", "redis"] + }, + { + "name": "empty_deps", + "path": "app/empty.py", + "language": "go", + "description": null, + "dependencies": [] + } + ], + "entry_points": [ + { + "kind": "http", + "identifier": "login", + "file_path": "app/api/login.py", + "line": 42, + "method": "POST", + "route": "/v1/login", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "migrate", + "file_path": "tools/migrate.py", + "line": 0, + "method": null, + "route": null, + "auth_required": null + } + ], + "trust_boundaries": [ + { + "name": "edge", + "source_zone": "internet", + "target_zone": "app", + "description": "TLS terminates at the ingress & waf", + "enforcement": [] + } + ], + "services": [ + { + "name": "postgres", + "service_type": "database", + "endpoint": "postgres://db:5432", + "purpose": null, + "auth_mechanism": "password" + } + ], + "api_surface": [ + { + "method": "GET", + "path": "/v1/users/{id}", + "handler": "UserController.show", + "file_path": "app/api/users.py", + "line": 17, + "auth_required": true, + "rate_limited": null + } + ] + }, + "DependencyReport": { + "sbom": [ + {"name": "django", "version": "4.2.1", "ecosystem": "pypi", "direct": true, "license": "BSD-3-Clause"}, + {"name": "urllib3", "version": "1.26.5", "ecosystem": "pypi", "direct": false, "license": null} + ], + "known_cves": [ + { + "cve_id": "CVE-2023-0001", + "package": "django", + "installed_version": "4.2.1", + "fixed_version": "4.2.5", + "cvss_v4_score": 1.0, + "epss_score": 0.5, + "direct": true, + "reachable": true + }, + { + "cve_id": "CVE-2023-0002", + "package": "urllib3", + "installed_version": "1.26.5", + "fixed_version": null, + "cvss_v4_score": 1e-05, + "epss_score": 1e16, + "direct": false, + "reachable": null + }, + { + "cve_id": "CVE-2023-0003", + "package": "zlib", + "installed_version": "1.2.11", + "fixed_version": "1.2.12", + "cvss_v4_score": 1000000000000000.0, + "epss_score": -0.0, + "direct": false, + "reachable": false + } + ], + "outdated": [], + "direct_count": 1, + "transitive_count": 0 + }, + "SecurityContext": { + "auth_model": "jwt", + "auth_details": "HS256 — key from vault\ttab\nnewline \"quoted\" back\\slash", + "crypto_usage": [ + {"algorithm": "AES", "key_size": 256, "mode": "GCM", "usage_context": "at-rest", "is_weak": false}, + {"algorithm": "MD5", "key_size": null, "mode": null, "usage_context": null, "is_weak": true} + ], + "framework_security": ["CsrfViewMiddleware", ""], + "security_headers": [], + "deployment_signals": ["k8s ingress & mTLS", "control chars: \u0000\u001f\u007f…"] + }, + "ConfigReport": { + "secrets": [ + { + "id": "secret-01", + "secret_type": "aws_access_key", + "file_path": "config/prod.yaml", + "line": 12, + "match": "AKIA****0001", + "confidence": "high", + "is_test_value": false + } + ], + "misconfigs": [ + { + "id": "misconfig-01", + "category": "tls", + "file_path": "config/prod.yaml", + "line": null, + "key": "SSL_VERIFY", + "value": "false", + "risk": "critical", + "remediation": null + } + ] + }, + "edge_cases": { + "empty_list": [], + "empty_dict": {}, + "none": null, + "bools": [true, false], + "ints": [0, -1, 1234567890123456789], + "floats": [1.0, 0.5, 1e-05, -0.0, 1e16, 1000000000000000.0, 3.141592653589793, 0.1], + "unicode": "héllo — 世界 🚀 \u007f", + "escapes": "quote\" backslash\\ tab\t newline\n cr\r ff\f bs\b & / slash", + "nested": {"a": [{"b": {"c": [1, 2.5, null]}}], "z": 1}, + "sort_check": {"b": 1, "a": 2, "C": 3, "_": 4, "É": 5} + } +} diff --git a/internal/secaf/reasoners/doc.go b/internal/secaf/reasoners/doc.go new file mode 100644 index 0000000000..3321983d96 --- /dev/null +++ b/internal/secaf/reasoners/doc.go @@ -0,0 +1,67 @@ +// Package reasoners ports src/sec_af/reasoners/{__init__,recon,hunt,prove}.py — +// the thin adapter layer between the reasoner surface the pipeline `.call`s and +// the agent functions under internal/agents. +// +// Python builds that surface with a module-level router: +// +// router = AgentRouter(tags=["security", "audit", "red-team"]) # __init__.py +// +// @router.reasoner() +// async def run_architecture_mapper(repo_path: str) -> dict[str, Any]: +// router.note("Architecture mapper starting", tags=["recon", "architecture"]) +// result = await _run_architecture_mapper(router, repo_path) +// return result.model_dump() +// +// Every adapter does the same four things, and this package reproduces them one +// for one: +// +// 1. emit the reasoner's "X starting" note with its exact tags (a few carry +// none — see NameRunLogicBugsHunter and NameRunCWEExpansion); +// 2. materialize the untyped request body into the pydantic models the agent +// function expects (afx.Bind over a typed input struct whose json tags and +// defaults are the Python signature, plus the required-field checks pydantic +// performs — validate.go); +// 3. call the agent function; +// 4. return `result.model_dump()` as a map (afx.ToMap). +// +// # Input validation +// +// Step 2 is not the first thing a request meets. The Python SDK runs +// `Agent._validate_handler_input` over the body BEFORE the decorated function +// is entered, answering 422 for a null on a required parameter and COERCING +// scalars (`"50"` -> 50, `5` -> "5", `"yes"` -> true). handler_input.go ports +// that layer and RegisterAll wraps every handler in it; internal/secaf/audit +// runs the same layer over the audit entry's own request. A refusal is an +// *InputError carrying 422. +// +// # Registration +// +// RegisterAll puts the 33 router reasoners into a Registry in the canonical +// DESIGN.md §3 order and returns its own ordered bookkeeping, which the parity +// test compares with the registry's. The sec-af node mounted the same handlers +// on an SDK router served to the AgentField control plane; inside codeaf the +// registry is read by the in-process Caller (internal/secaf/audit), and the +// router's tags, which only labelled the node for the control plane, are gone. +// +// # Input schemas +// +// Python publishes a JSON Schema per reasoner, derived from the decorated +// function's signature. The exact schemas the Python node registered are +// committed as testdata/python_input_schemas.json and replayed verbatim through +// InputSchema — including Python's own derivation quirks — and the registry +// holds each reasoner's schema beside its handler. input_schemas.go documents +// them and how the capture was made. +// +// A SECOND capture, testdata/python_input_types.json, holds the raw +// `(annotation, default)` pairs the same signatures produce — the input +// `_validate_handler_input` runs on. The two are derived by different SDK code +// paths from the same source, and handler_input_test.go asserts they agree. +// +// # What is NOT here +// +// The `*_phase` bodies (the four reasoners that were the control-plane DAG) live +// in internal/phases; this package only binds their inputs and threads the node +// id. `run_cwe_expansion` likewise delegates to phases.RunCWEExpansion. Keeping +// the split means a phase can be tested without the registration layer and vice +// versa. +package reasoners diff --git a/internal/secaf/reasoners/handler_input.go b/internal/secaf/reasoners/handler_input.go new file mode 100644 index 0000000000..ec8fc218b7 --- /dev/null +++ b/internal/secaf/reasoners/handler_input.go @@ -0,0 +1,495 @@ +package reasoners + +import ( + "bytes" + _ "embed" + "encoding/json" + "fmt" + "math" + "reflect" + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" +) + +// handler_input.go ports the Python SDK's `Agent._validate_handler_input` +// (sdk/python/agentfield/agent.py:1146) — the layer that runs on EVERY reasoner +// request body before the handler function is entered, and whose failures the +// endpoint turns into `JSONResponse(status_code=422, ...)` (agent.py:3122-3134). +// +// # Why the port needs it at all +// +// The Go SDK has no equivalent: `handleExecute` decodes the body into +// map[string]any and hands it straight to the handler, and `InputSchema` is +// only ever published for discovery — nothing validates against it. Without +// this file the port's binding is `afx.Bind`, a bare JSON round trip, which +// diverges from Python in BOTH directions: +// +// {"findings": null, ...} Python 422; Go bound nil and +// on run_deduplicator returned an EMPTY HuntResult with 200 +// {"depth": 5} on recon_phase Python "5"; Go json.UnmarshalTypeError +// {"max_files_without_signal": "50"} Python 50; Go json.UnmarshalTypeError +// {"is_pr": "yes"} on audit Python True; Go json.UnmarshalTypeError +// +// Every producer INSIDE the pipeline emits correctly typed kwargs, so the +// divergence is only visible to a control-plane caller invoking one of the 34 +// registered reasoners directly — which is a first-class caller, not a +// hypothetical (validate.go says the same about hand-built payloads). +// +// This is a DIFFERENT layer from internal/phases' checked binders. Those are +// `Model.model_validate(payload)` for a nested pydantic model reached through a +// `.call`; this is the SDK's own parameter binding, one level above, and it has +// its own rules (scalar coercion, `str()` stringification) that pydantic does +// not share. +// +// # The rules, and the two Python quirks they preserve +// +// For each declared parameter, in signature order (the first offending one +// raises): +// +// absent + has default -> the default (Go: leave the key absent so +// the input struct's +// UnmarshalJSON seeds it) +// absent + required -> "Missing required field: x" [NOT reproduced] +// null + has default -> the default (same treatment as absent) +// null + required -> "Field 'x' cannot be None" +// present -> int(v) / float(v) / str(v) / bool-rules / dict-check / list-check +// / pass-through, per the parameter's annotation +// +// QUIRK 1 — a PEP 604 union is never unwrapped. agent.py:1193-1200 tests +// `expected_type.__origin__ is Union`, and `int | None` is a `types.UnionType` +// with NO `__origin__`, so `max_provers: int | None` reaches the trailing +// pass-through branch: Python leaves `{"max_provers": "5"}` as the STRING "5" +// rather than coercing it to 5. Captured as kind "any" and reproduced. +// +// QUIRK 2 — `bool` is checked AFTER `int`, but with `is`, and `bool is not int`, +// so a bool-annotated field takes the bool branch ("yes"/"1"/"true" -> True, +// anything else stringy -> False) rather than int(). +// +// # The one rule deliberately NOT reproduced +// +// "Missing required field: x". The Go input structs cannot express "this key +// was absent" for a scalar, and the port has always documented the missing +// argument as a divergence (see HunterInput). An absent key keeps the Go zero +// value and the pipeline carries on, where Python answers 422. +// +// # Provenance +// +// testdata/python_input_types.json is generated from the LIVE registry the SDK +// validates against — `app._reasoner_registry[name].input_types` — by +// go/scripts/gen_input_types.py. See that script for the command. A Go-side +// change must never edit the fixture to make a test pass. + +// pythonInputTypesJSON is the committed capture, embedded so the specs ship in +// the binary rather than being read from disk at boot. +// +//go:embed testdata/python_input_types.json +var pythonInputTypesJSON []byte + +// paramKind names which branch of `_validate_handler_input` a parameter takes. +type paramKind string + +const ( + kindStr paramKind = "str" + kindInt paramKind = "int" + kindFloat paramKind = "float" + kindBool paramKind = "bool" + kindDict paramKind = "dict" + kindList paramKind = "list" + // kindAny is the trailing `else: result[name] = value` pass-through. It + // covers every `X | None` parameter in this node (see QUIRK 1) as well as + // `Any | None` (hunt_phase's ai_gate). + kindAny paramKind = "any" +) + +// handlerParam is one row of the capture. +type handlerParam struct { + Name string `json:"name"` + Kind paramKind `json:"kind"` + Annotation string `json:"annotation"` + Required bool `json:"required"` + Default any `json:"default"` +} + +// handlerSpecs maps reasoner id -> its ordered parameter list. Parsed once, at +// package init, so a corrupt or unknown-kind fixture fails the process +// immediately instead of at the first request. +var handlerSpecs = mustParseInputTypes(pythonInputTypesJSON) + +func mustParseInputTypes(raw []byte) map[string][]handlerParam { + var decoded map[string][]handlerParam + if err := json.Unmarshal(raw, &decoded); err != nil { + panic(fmt.Sprintf("reasoners: testdata/python_input_types.json is not a JSON object of parameter lists: %v", err)) + } + if len(decoded) == 0 { + panic("reasoners: testdata/python_input_types.json is empty") + } + for name, params := range decoded { + for _, p := range params { + switch p.Kind { + case kindStr, kindInt, kindFloat, kindBool, kindDict, kindList, kindAny: + default: + panic(fmt.Sprintf("reasoners: %s.%s has unknown kind %q", name, p.Name, p.Kind)) + } + } + } + return decoded +} + +// handlerSpecFor returns the captured parameter list for a reasoner, panicking +// on a name the capture does not know — see ValidateHandlerInput. +func handlerSpecFor(name string) []handlerParam { + params, ok := handlerSpecs[name] + if !ok { + panic(fmt.Sprintf( + "reasoners: no input types for reasoner %q in testdata/python_input_types.json; "+ + "regenerate the capture (see handler_input.go) after changing the Python surface", name)) + } + return params +} + +// HandlerInputError is the Go shape of `_HandlerInputError`, the ValueError +// subclass `_validate_handler_input` raises. Its message is the SDK-constructed +// `safe_message` Python puts in the 422 body verbatim. +type HandlerInputError struct{ Message string } + +func (e *HandlerInputError) Error() string { return e.Message } + +// ValidateHandlerInput is `_validate_handler_input(data, input_types)` for the +// reasoner called name: it returns the validated, coerced keyword map to bind, +// or a *HandlerInputError. +// +// Like Python's, the returned map contains ONLY declared parameters — an +// undeclared key in the body is dropped rather than forwarded. +// +// It PANICS on a name the capture does not know, for the same reason +// InputSchema does: every registration goes through here, so adding a reasoner +// without regenerating the capture fails at boot and in every test instead of +// silently shipping an unvalidated handler. +func ValidateHandlerInput(name string, data map[string]any) (map[string]any, error) { + params := handlerSpecFor(name) + out := make(map[string]any, len(params)) + for _, p := range params { + value, present := data[p.Name] + if !present { + // Python: `result[name] = default`, or + // `raise _HandlerInputError("Missing required field: x")`. The Go + // input structs seed the same defaults from an ABSENT key, and the + // required case is the documented missing-argument divergence. + continue + } + if isPyNone(value) { + if !p.Required { + // Python: `if default is not ...: result[name] = default`. An + // explicit null on a defaulted parameter yields the DEFAULT, + // not None — which is what leaving the key absent produces + // here, since every input struct's UnmarshalJSON seeds exactly + // the Python defaults. + continue + } + return nil, &HandlerInputError{Message: "Field '" + p.Name + "' cannot be None"} + } + coerced, err := coerceParam(p, value) + if err != nil { + return nil, err + } + out[p.Name] = coerced + } + return out, nil +} + +// isPyNone is `value is None` for the shapes a request map can hold. +// +// An untyped nil is what encoding/json leaves for a JSON `null`, which is the +// only case on the wire. A TYPED nil (a nil *int, []string or map) reaches here +// only from an in-process Go caller building the kwargs directly; it marshals +// to `null`, so CPython would see None for it too, and treating it as None +// keeps the two paths from disagreeing. +func isPyNone(value any) bool { + if value == nil { + return true + } + rv := reflect.ValueOf(value) + switch rv.Kind() { + case reflect.Pointer, reflect.Interface, reflect.Map, reflect.Slice: + return rv.IsNil() + } + return false +} + +// coerceParam applies one parameter's branch of agent.py:1202-1236. +func coerceParam(p handlerParam, value any) (any, error) { + // Python wraps the whole coercion in + // `except (ValueError, TypeError): raise _HandlerInputError(f"Invalid value for field '{name}'")`, + // deliberately dropping the inner exception's text. Every failure below is + // one of those two, so they all surface with that message — except the + // dict/list shape checks, which raise _HandlerInputError directly and keep + // their own wording. + invalid := &HandlerInputError{Message: "Invalid value for field '" + p.Name + "'"} + + switch p.Kind { + case kindInt: + n, ok := pyInt(value) + if !ok { + return nil, invalid + } + return n, nil + case kindFloat: + f, ok := pyFloat(value) + if !ok { + return nil, invalid + } + return f, nil + case kindStr: + return pyStr(value), nil + case kindBool: + return pyBool(value), nil + case kindDict: + if reflect.ValueOf(value).Kind() != reflect.Map { + return nil, &HandlerInputError{Message: "Field '" + p.Name + "' must be a dict"} + } + return value, nil + case kindList: + if kind := reflect.ValueOf(value).Kind(); kind != reflect.Slice && kind != reflect.Array { + return nil, &HandlerInputError{Message: "Field '" + p.Name + "' must be a list"} + } + return value, nil + } + // kindAny: `else: result[name] = value`. + return value, nil +} + +// pyInt is `int(value)` for the value kinds a decoded body can hold. +// +// bool -> 1 / 0 +// number -> TRUNCATED toward zero (int(5.7) == 5, int(-5.7) == -5) +// string -> the CPython int() grammar (see parsePyInt) +// anything else (list, dict) -> TypeError +func pyInt(value any) (int, bool) { + switch v := value.(type) { + case bool: + if v { + return 1, true + } + return 0, true + case string: + return parsePyInt(v) + case json.Number: + return parsePyInt(v.String()) + } + rv := reflect.ValueOf(value) + switch rv.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return int(rv.Int()), true + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return int(rv.Uint()), true + case reflect.Float32, reflect.Float64: + f := rv.Float() + if math.IsNaN(f) || math.IsInf(f, 0) { + // Python raises ValueError / OverflowError; JSON cannot express + // either literal, so this is only reachable from a Go caller. + return 0, false + } + return int(f), true + } + return 0, false +} + +// pyFloat is `float(value)`. +// +// Unreachable from the current surface — no reasoner parameter is annotated +// `float` (the only float-ish one, audit's `max_cost_usd: float | None`, is a +// PEP 604 union and therefore kind "any"). Implemented anyway so a future +// signature is handled rather than mis-handled, and pinned by a unit test. +func pyFloat(value any) (float64, bool) { + switch v := value.(type) { + case bool: + if v { + return 1, true + } + return 0, true + case string: + return parsePyFloat(v) + case json.Number: + return parsePyFloat(v.String()) + } + rv := reflect.ValueOf(value) + switch rv.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return float64(rv.Int()), true + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return float64(rv.Uint()), true + case reflect.Float32, reflect.Float64: + return rv.Float(), true + } + return 0, false +} + +// pyStr is `str(value)`. +// +// DOCUMENTED RESIDUAL, the wire-number one afx.PyTypeName carries too: Go's +// encoding/json decodes every JSON number to float64, so an INTEGER literal is +// indistinguishable from a float here. `str()` of an integral value is rendered +// the int way ("5"), which is right for `{"depth": 5}` — the case that +// matters — and wrong for the pathological `{"depth": 5.0}`, where Python says +// "5.0". The same ambiguity applies inside a list/dict rendered through +// pyfmt.Str, which additionally sorts map keys where CPython uses insertion +// order. +func pyStr(value any) string { + switch v := value.(type) { + case string: + return v + case bool: + if v { + return "True" + } + return "False" + case json.Number: + return v.String() + case float32: + return pyStrFloat(float64(v)) + case float64: + return pyStrFloat(v) + } + return pyfmt.Str(wireInts(value)) +} + +// wireInts rewrites every INTEGRAL float64 in a decoded JSON tree as an int, so +// pyfmt.Repr spells it the way CPython would have: `str([1, 2])` is "[1, 2]", +// not "[1.0, 2.0]", because json.loads produced ints. Same heuristic (and same +// residual for an explicit `1.0` literal) as pyStrFloat. +func wireInts(value any) any { + switch v := value.(type) { + case float64: + if !math.IsNaN(v) && !math.IsInf(v, 0) && v == math.Trunc(v) && math.Abs(v) < 1e15 { + return int64(v) + } + case []any: + out := make([]any, len(v)) + for i, item := range v { + out[i] = wireInts(item) + } + return out + case map[string]any: + out := make(map[string]any, len(v)) + for key, item := range v { + out[key] = wireInts(item) + } + return out + } + return value +} + +// pyStrFloat renders a decoded JSON number the way `str()` renders whichever +// Python type json.loads would have produced for it: an integral value is an +// int (no ".0"), anything else is repr(float). +func pyStrFloat(f float64) string { + if !math.IsNaN(f) && !math.IsInf(f, 0) && f == math.Trunc(f) && math.Abs(f) < 1e15 { + return strconv.FormatInt(int64(f), 10) + } + return pyfmt.FormatFloat(f) +} + +// pyBool ports agent.py:1207-1212: +// +// if isinstance(value, bool): value +// elif isinstance(value, str): value.lower() in ("true", "1", "yes") +// else: bool(value) +// +// Note the string rule is a MEMBERSHIP test, not a parse: "no", "0", "false" +// and "" are all False, and so is "TRUE " with a trailing space. +func pyBool(value any) bool { + switch v := value.(type) { + case bool: + return v + case string: + switch strings.ToLower(v) { + case "true", "1", "yes": + return true + } + return false + } + return pyTruthy(value) +} + +// pyTruthy is `bool(x)` for the remaining kinds: a zero number, an empty +// container and None are falsy. +func pyTruthy(value any) bool { + if value == nil { + return false + } + rv := reflect.ValueOf(value) + switch rv.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return rv.Int() != 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return rv.Uint() != 0 + case reflect.Float32, reflect.Float64: + return rv.Float() != 0 + case reflect.Slice, reflect.Array, reflect.Map, reflect.String: + return rv.Len() != 0 + case reflect.Pointer, reflect.Interface: + return !rv.IsNil() + } + return true +} + +// parsePyInt implements CPython's `int(str)` grammar: surrounding whitespace, +// an optional sign, decimal digits, and single underscores BETWEEN digits +// (int("1_0") == 10, while "1__0", "_1" and "1_" all raise). A float spelling +// ("50.5"), a base prefix ("0x10") and the empty string all raise. +// +// Residual: CPython also accepts unicode whitespace and unicode decimal digits; +// only ASCII is handled here. +func parsePyInt(s string) (int, bool) { + digits, ok := stripPyUnderscores(strings.TrimSpace(s)) + if !ok { + return 0, false + } + n, err := strconv.ParseInt(digits, 10, 64) + if err != nil { + return 0, false + } + return int(n), true +} + +// parsePyFloat implements CPython's `float(str)`: whitespace, an optional sign, +// a decimal or exponent form, or the "inf"/"infinity"/"nan" spellings in any +// case; underscores are allowed between digits. A hex float ("0x1p-2") is +// accepted by strconv and REJECTED by Python, so it is rejected here. +func parsePyFloat(s string) (float64, bool) { + text, ok := stripPyUnderscores(strings.TrimSpace(s)) + if !ok { + return 0, false + } + if strings.ContainsAny(text, "xX") { + return 0, false + } + f, err := strconv.ParseFloat(text, 64) + if err != nil { + return 0, false + } + return f, true +} + +// stripPyUnderscores removes PEP 515 digit separators, rejecting the placements +// CPython rejects: an underscore must sit BETWEEN two digits. +func stripPyUnderscores(s string) (string, bool) { + if !strings.Contains(s, "_") { + return s, true + } + var b bytes.Buffer + for i := 0; i < len(s); i++ { + if s[i] != '_' { + b.WriteByte(s[i]) + continue + } + if i == 0 || i+1 >= len(s) || !isASCIIDigit(s[i-1]) || !isASCIIDigit(s[i+1]) { + return "", false + } + } + return b.String(), true +} + +func isASCIIDigit(c byte) bool { return c >= '0' && c <= '9' } diff --git a/internal/secaf/reasoners/handler_input_test.go b/internal/secaf/reasoners/handler_input_test.go new file mode 100644 index 0000000000..98b55ba4aa --- /dev/null +++ b/internal/secaf/reasoners/handler_input_test.go @@ -0,0 +1,550 @@ +package reasoners + +// Tests for handler_input.go — the port of the Python SDK's +// `Agent._validate_handler_input`, which runs on every reasoner request body. +// +// Validation contract (behaviour, MEASURED by calling the real +// `app._validate_handler_input(body, app._reasoner_registry[name].input_types)` +// on the pinned interpreter; every `want` below is that call's output): +// +// - an explicit null on a REQUIRED parameter is rejected with +// `Field 'x' cannot be None`, which the endpoint answers 422 — the case +// that used to bind nil in Go and return an empty result with 200; +// - an explicit null on a DEFAULTED parameter yields the Python DEFAULT +// (depth null -> "standard", max_files_without_signal null -> 30), NOT None; +// - scalars are COERCED, not type-checked: int(v), float(v), str(v) and the +// bool spellings, so `{"depth": 5}` is "5" and +// `{"max_files_without_signal": "50"}` is 50 — both of which afx.Bind alone +// rejected outright; +// - a coercion failure is `Invalid value for field 'x'`, with the inner +// exception's text deliberately dropped; +// - a dict/list-annotated parameter is SHAPE-checked, with its own wording: +// `Field 'x' must be a dict` / `must be a list`; +// - a PEP 604 `X | None` parameter is NEVER unwrapped (types.UnionType has no +// __origin__), so `{"max_provers": "5"}` stays the STRING "5"; +// - an undeclared key is dropped; +// - the one divergence: an ABSENT required key is a 422 in Python +// ("Missing required field: x") and is accepted here, matching the +// missing-argument divergence the port has always documented. + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "reflect" + "sort" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" +) + +// TestValidateHandlerInputMatchesPython replays the measured cases. `want` is +// the SUBSET of Python's validated kwargs that the Go result must carry +// verbatim; keys Python fills from a default and Go leaves absent (so the input +// struct's UnmarshalJSON seeds them) are covered by TestDefaultsAreTheGoSeeds. +func TestValidateHandlerInputMatchesPython(t *testing.T) { + for _, tc := range []struct { + name string + reasoner string + body map[string]any + // want is the SUBSET of Python's validated kwargs the Go result must + // carry verbatim; absent is the set of body keys that must NOT survive. + want map[string]any + absent []string + wantErr string + }{ + // --- null on a required collection: the 200-with-empty-result bug --- + {"deduplicator findings null", NameRunDeduplicator, + map[string]any{"findings": nil, "recon_context": map[string]any{}, "repo_path": "/r"}, + nil, nil, "Field 'findings' cannot be None"}, + {"remediation_phase verified_findings null", NameRemediationPhase, + map[string]any{"repo_path": "/r", "verified_findings": nil}, + nil, nil, "Field 'verified_findings' cannot be None"}, + {"cross_service services null", NameRunCrossServiceAnalyzer, + map[string]any{"repo_path": "/r", "services": nil, "findings_summary": "s", "depth": "standard"}, + nil, nil, "Field 'services' cannot be None"}, + {"recon_phase repo_path null", NameReconPhase, + map[string]any{"repo_path": nil}, nil, nil, "Field 'repo_path' cannot be None"}, + {"audit repo_url null", NameAudit, + map[string]any{"repo_url": nil}, nil, nil, "Field 'repo_url' cannot be None"}, + + // --- an empty list is NOT null and is accepted --- + {"deduplicator findings empty", NameRunDeduplicator, + map[string]any{"findings": []any{}, "recon_context": map[string]any{}, "repo_path": "/r"}, + map[string]any{"findings": []any{}, "recon_context": map[string]any{}, "repo_path": "/r"}, nil, ""}, + + // --- shape checks keep their own wording --- + {"deduplicator findings not a list", NameRunDeduplicator, + map[string]any{"findings": "nope", "recon_context": map[string]any{}, "repo_path": "/r"}, + nil, nil, "Field 'findings' must be a list"}, + {"hunter recon_context not a dict", NameRunInjectionHunter, + map[string]any{"repo_path": "/r", "recon_context": []any{}, "depth": "standard"}, + nil, nil, "Field 'recon_context' must be a dict"}, + + // --- str() coercion --- + {"recon_phase depth from a number", NameReconPhase, + map[string]any{"repo_path": "/r", "depth": float64(5)}, + map[string]any{"repo_path": "/r", "depth": "5"}, nil, ""}, + {"recon_phase depth from a bool", NameReconPhase, + map[string]any{"repo_path": "/r", "depth": true}, + map[string]any{"repo_path": "/r", "depth": "True"}, nil, ""}, + {"recon_phase depth from a list", NameReconPhase, + map[string]any{"repo_path": "/r", "depth": []any{float64(1), float64(2)}}, + map[string]any{"repo_path": "/r", "depth": "[1, 2]"}, nil, ""}, + {"audit repo_url from a number", NameAudit, + map[string]any{"repo_url": float64(5)}, + map[string]any{"repo_url": "5"}, nil, ""}, + + // --- int() coercion --- + {"hunter max_files from a string", NameRunInjectionHunter, + map[string]any{"repo_path": "/r", "recon_context": map[string]any{}, "depth": "standard", + "max_files_without_signal": "50"}, + map[string]any{"max_files_without_signal": 50}, nil, ""}, + {"hunter max_files truncates", NameRunInjectionHunter, + map[string]any{"repo_path": "/r", "recon_context": map[string]any{}, "depth": "standard", + "max_files_without_signal": 30.9}, + map[string]any{"max_files_without_signal": 30}, nil, ""}, + {"hunter max_files from a bool", NameRunInjectionHunter, + map[string]any{"repo_path": "/r", "recon_context": map[string]any{}, "depth": "standard", + "max_files_without_signal": true}, + map[string]any{"max_files_without_signal": 1}, nil, ""}, + {"hunter max_files unparseable", NameRunInjectionHunter, + map[string]any{"repo_path": "/r", "recon_context": map[string]any{}, "depth": "standard", + "max_files_without_signal": "abc"}, + nil, nil, "Invalid value for field 'max_files_without_signal'"}, + {"hunt_phase max_concurrent_hunters from a string", NameHuntPhase, + map[string]any{"repo_path": "/r", "recon_context": map[string]any{}, + "max_concurrent_hunters": "2"}, + map[string]any{"max_concurrent_hunters": 2}, nil, ""}, + + // --- bool() coercion (agent.py's membership test, not a parse) --- + {"audit is_pr yes", NameAudit, map[string]any{"repo_url": "u", "is_pr": "yes"}, + map[string]any{"is_pr": true}, nil, ""}, + {"audit is_pr no", NameAudit, map[string]any{"repo_url": "u", "is_pr": "no"}, + map[string]any{"is_pr": false}, nil, ""}, + {"audit is_pr zero", NameAudit, map[string]any{"repo_url": "u", "is_pr": float64(0)}, + map[string]any{"is_pr": false}, nil, ""}, + + // --- null on a DEFAULTED parameter is the default, not None --- + {"recon_phase depth null", NameReconPhase, + map[string]any{"repo_path": "/r", "depth": nil}, + map[string]any{"repo_path": "/r"}, []string{"depth"}, ""}, + {"hunter max_files null", NameRunInjectionHunter, + map[string]any{"repo_path": "/r", "recon_context": map[string]any{}, "depth": "standard", + "max_files_without_signal": nil}, + map[string]any{"repo_path": "/r"}, []string{"max_files_without_signal"}, ""}, + {"audit is_pr null", NameAudit, map[string]any{"repo_url": "u", "is_pr": nil}, + map[string]any{"repo_url": "u"}, []string{"is_pr"}, ""}, + + // --- QUIRK 1: a PEP 604 union is never unwrapped, so no coercion --- + {"prove_phase max_provers stays a string", NameProvePhase, + map[string]any{"repo_path": "/r", "hunt_result": map[string]any{}, "max_provers": "5"}, + map[string]any{"max_provers": "5"}, nil, ""}, + + // --- an undeclared key is dropped --- + {"recon_phase drops an unknown key", NameReconPhase, + map[string]any{"repo_path": "/r", "unknown_key": float64(1)}, + map[string]any{"repo_path": "/r"}, []string{"unknown_key"}, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := ValidateHandlerInput(tc.reasoner, tc.body) + if tc.wantErr != "" { + if err == nil { + t.Fatalf("accepted %v, want %q", tc.body, tc.wantErr) + } + var hie *HandlerInputError + if !errors.As(err, &hie) { + t.Fatalf("error = %T, want *HandlerInputError", err) + } + if err.Error() != tc.wantErr { + t.Errorf("error = %q, want %q", err.Error(), tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("ValidateHandlerInput: %v", err) + } + for key, want := range tc.want { + if !reflect.DeepEqual(got[key], want) { + t.Errorf("%s = %#v, want %#v", key, got[key], want) + } + } + for _, key := range tc.absent { + if value, present := got[key]; present { + t.Errorf("%s survived as %#v; it must be absent so the input "+ + "struct's seeded default applies", key, value) + } + } + }) + } +} + +// TestValidateHandlerInputDropsNullOnDefaultedParams is the second half of the +// null rule, stated where it is actually observable: a null on a defaulted +// parameter must leave the KEY ABSENT so the input struct's UnmarshalJSON seeds +// the Python default. Keeping the null instead would zero the field. +func TestValidateHandlerInputDropsNullOnDefaultedParams(t *testing.T) { + // `depth` has NO default in the hunter signature, so a null there is an + // error rather than a fallback — only the defaulted parameter is nulled. + got, err := ValidateHandlerInput(NameRunInjectionHunter, map[string]any{ + "repo_path": "/r", "recon_context": map[string]any{}, "depth": "quick", + "max_files_without_signal": nil, + }) + if err != nil { + t.Fatalf("ValidateHandlerInput: %v", err) + } + if _, present := got["max_files_without_signal"]; present { + t.Errorf("max_files_without_signal survived as %#v; it must be absent so "+ + "NewHunterInput's 30 applies", got["max_files_without_signal"]) + } + + raw, err := json.Marshal(got) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var in HunterInput + if err := json.Unmarshal(raw, &in); err != nil { + t.Fatalf("bind: %v", err) + } + if in.MaxFilesWithoutSignal != DefaultMaxFilesWithoutSignal { + t.Errorf("max_files_without_signal = %d, want the Python default %d", + in.MaxFilesWithoutSignal, DefaultMaxFilesWithoutSignal) + } + if in.Depth != "quick" { + t.Errorf("depth = %q, want quick", in.Depth) + } +} + +// TestDefaultsAreTheGoSeeds cross-checks the two captures against the Go input +// structs: every DEFAULT the Python signature declares must be the value the +// corresponding Go constructor seeds, because ValidateHandlerInput reproduces +// "absent or null -> default" by leaving the key out and letting the seed win. +// A drifted Go default would otherwise be invisible. +func TestDefaultsAreTheGoSeeds(t *testing.T) { + for _, tc := range []struct { + reasoner string + seed any + }{ + {NameRunInjectionHunter, NewHunterInput()}, + {NameReconPhase, NewReconPhaseInput()}, + {NameHuntPhase, NewHuntPhaseInput()}, + {NameProvePhase, NewProvePhaseInput()}, + {NameRemediationPhase, NewRemediationPhaseInput()}, + } { + t.Run(tc.reasoner, func(t *testing.T) { + raw, err := json.Marshal(tc.seed) + if err != nil { + t.Fatalf("marshal seed: %v", err) + } + var seeded map[string]any + if err := json.Unmarshal(raw, &seeded); err != nil { + t.Fatalf("decode seed: %v", err) + } + for _, p := range handlerSpecFor(tc.reasoner) { + if p.Required { + continue + } + got, ok := seeded[p.Name] + if !ok { + t.Errorf("%s: the Go input struct has no %q field", tc.reasoner, p.Name) + continue + } + if !sameJSONValue(got, p.Default) { + t.Errorf("%s.%s seed = %#v, Python default = %#v", tc.reasoner, p.Name, got, p.Default) + } + } + }) + } +} + +// sameJSONValue compares a decoded Go seed with a decoded Python default, +// tolerating the int/float64 spread JSON decoding introduces. +func sameJSONValue(got, want any) bool { + gotNum, gotIsNum := got.(float64) + wantNum, wantIsNum := want.(float64) + if gotIsNum && wantIsNum { + return gotNum == wantNum + } + return reflect.DeepEqual(got, want) +} + +// TestInputTypesCoverExactlyTheNodeSurface is the drift guard: the capture +// carries the 33 router reasoners plus `audit`, and nothing else. +func TestInputTypesCoverExactlyTheNodeSurface(t *testing.T) { + want := append([]string{NameAudit}, Names...) + sort.Strings(want) + + got := make([]string, 0, len(handlerSpecs)) + for name := range handlerSpecs { + got = append(got, name) + } + sort.Strings(got) + + if !reflect.DeepEqual(got, want) { + t.Fatalf("captured input-type ids mismatch:\n got = %v\n want = %v", got, want) + } +} + +// TestInputTypesAgreeWithTheInputSchemaCapture cross-checks the two INDEPENDENT +// captures of the same Python signatures: python_input_schemas.json (what the +// node publishes to the control plane, derived by `_types_to_json_schema`) and +// python_input_types.json (what `_validate_handler_input` runs on, the raw +// `(annotation, default)` pairs). They are produced by different SDK code paths +// from the same source, so they must agree on parameter NAMES, on which +// parameters are REQUIRED, and — for the kinds the schema names unambiguously — +// on the kind. +// +// This is what makes the coercion table above non-tautological: a bad +// regeneration of either capture fails here. +func TestInputTypesAgreeWithTheInputSchemaCapture(t *testing.T) { + // _types_to_json_schema's mapping, inverted. {"type":"object"} with no + // additionalProperties is the PEP 604 fall-through, i.e. kind "any". + schemaKind := func(prop map[string]any) paramKind { + switch prop["type"] { + case "string": + return kindStr + case "integer": + return kindInt + case "number": + return kindFloat + case "boolean": + return kindBool + case "array": + return kindList + case "object": + if _, ok := prop["additionalProperties"]; ok { + return kindDict + } + return kindAny + } + return kindAny + } + + for _, name := range InputSchemaNames() { + t.Run(name, func(t *testing.T) { + var schema struct { + Properties map[string]map[string]any `json:"properties"` + Required []string `json:"required"` + } + if err := json.Unmarshal(InputSchema(name), &schema); err != nil { + t.Fatalf("decode schema: %v", err) + } + required := map[string]bool{} + for _, key := range schema.Required { + required[key] = true + } + + params := handlerSpecFor(name) + if len(params) != len(schema.Properties) { + t.Fatalf("%d parameters, schema has %d properties", len(params), len(schema.Properties)) + } + // The schema's `required` list is in PARAMETER order, so it also + // pins the capture's ordering for the required subset. + var requiredInOrder []string + for _, p := range params { + prop, ok := schema.Properties[p.Name] + if !ok { + t.Errorf("%s: not in the published schema", p.Name) + continue + } + if want := schemaKind(prop); p.Kind != want { + t.Errorf("%s: kind %q, schema says %q (%v)", p.Name, p.Kind, want, prop) + } + if p.Required != required[p.Name] { + t.Errorf("%s: required=%v, schema says %v", p.Name, p.Required, required[p.Name]) + } + if p.Required { + requiredInOrder = append(requiredInOrder, p.Name) + } + } + if len(schema.Required) > 0 && !reflect.DeepEqual(requiredInOrder, schema.Required) { + t.Errorf("required order = %v, schema says %v", requiredInOrder, schema.Required) + } + }) + } +} + +// TestEveryRegisteredHandlerIsValidated drives the registered surface the way +// an in-process `.call` arrives, and asserts both ends of the +// divergence: a body Python 422s is refused with the SAME status and message, +// and a body Python accepts is accepted. +func TestEveryRegisteredHandlerIsValidated(t *testing.T) { + handler := nodeHandler(t) + + t.Run("null on a required collection is a 422", func(t *testing.T) { + status, body := postReasoner(t, handler, NameRunDeduplicator, map[string]any{ + "findings": nil, "recon_context": map[string]any{}, "repo_path": "/r", + }) + if status != http.StatusUnprocessableEntity { + t.Fatalf("status = %d, want 422 (Python: Field 'findings' cannot be None); body %v", + status, body) + } + if want := "Field 'findings' cannot be None"; body["error"] != want { + t.Errorf("error = %v, want %q", body["error"], want) + } + }) + + t.Run("null on a required collection reaches remediation_phase too", func(t *testing.T) { + status, body := postReasoner(t, handler, NameRemediationPhase, map[string]any{ + "repo_path": "/r", "verified_findings": nil, + }) + if status != http.StatusUnprocessableEntity { + t.Fatalf("status = %d, want 422; body %v", status, body) + } + if want := "Field 'verified_findings' cannot be None"; body["error"] != want { + t.Errorf("error = %v, want %q", body["error"], want) + } + }) + + t.Run("a string-encoded int is coerced, not rejected", func(t *testing.T) { + // max_files_without_signal="50" is 50 for Python and used to be a + // json.UnmarshalTypeError here. A 200 means the handler ran. + status, body := postReasoner(t, handler, NameRunInjectionHunter, map[string]any{ + "repo_path": t.TempDir(), "recon_context": map[string]any{}, + "depth": "quick", "max_files_without_signal": "50", + }) + // The fake app has no scripted harness, so the reasoner FAILS — but on + // its own work, not on binding: a 422 here would mean the input layer + // rejected a body Python accepts. + if status == http.StatusUnprocessableEntity { + t.Fatalf("max_files_without_signal=\"50\" was rejected; Python coerces it to 50. body %v", body) + } + }) + + t.Run("a number where a string is declared is coerced", func(t *testing.T) { + status, body := postReasoner(t, handler, NameReconPhase, map[string]any{ + "repo_path": t.TempDir(), "depth": float64(5), + }) + // recon_phase fans out to five `.call`s the fake answers with an error, + // so the reasoner FAILS — but with the DAG's error, not a bind error: + // the point is that "5" got through the input layer. + if status == http.StatusUnprocessableEntity { + t.Fatalf("depth=5 was rejected; Python coerces it to \"5\". body %v", body) + } + }) +} + +// nodeHandler builds the registry over an unscripted fake, so a request +// travels the same validation and binding path an in-process `.call` does. +func nodeHandler(t *testing.T) *Registry { + t.Helper() + reg := NewRegistry() + RegisterAll(reg, &appx.Fake{}) + return reg +} + +// postReasoner runs a body through the registered handler for name and returns +// the status the node would have answered with — the *InputError's status for +// a refused body, 500 for any other failure, 200 on success — and an +// `{"error": message}` object for a failure, as the node's HTTP layer wrote it. +func postReasoner(t *testing.T, reg *Registry, name string, body map[string]any) (int, map[string]any) { + t.Helper() + h, ok := reg.Lookup(name) + if !ok { + t.Fatalf("reasoner %q is not registered", name) + } + out, err := h(context.Background(), body) + if err != nil { + status := http.StatusInternalServerError + var inputErr *InputError + if errors.As(err, &inputErr) { + status = inputErr.StatusCode + } + return status, map[string]any{"error": err.Error()} + } + payload, _ := out.(map[string]any) + return http.StatusOK, payload +} + +// TestPyIntGrammar pins CPython's `int(str)`, which `_validate_handler_input` +// calls for every `int`-annotated parameter. VERIFIED on the pinned +// interpreter: int("50")==50, int(" 50 ")==50, int("1_0")==10, int("+7")==7, +// while "0x10", "50.5", "" and "abc" all raise ValueError. +func TestPyIntGrammar(t *testing.T) { + for _, tc := range []struct { + in string + want int + ok bool + }{ + {"50", 50, true}, + {" 50 ", 50, true}, + {"\t50\n", 50, true}, + {"+7", 7, true}, + {"-7", -7, true}, + {"1_0", 10, true}, + {"1_000_000", 1000000, true}, + {"0", 0, true}, + {"0x10", 0, false}, + {"50.5", 0, false}, + {"1e3", 0, false}, + {"", 0, false}, + {"abc", 0, false}, + {"1__0", 0, false}, + {"_1", 0, false}, + {"1_", 0, false}, + } { + got, ok := parsePyInt(tc.in) + if ok != tc.ok || (ok && got != tc.want) { + t.Errorf("int(%q) = (%d, %v), want (%d, %v)", tc.in, got, ok, tc.want, tc.ok) + } + } +} + +// TestPyFloatGrammar pins CPython's `float(str)`. No reasoner parameter is +// annotated `float` today — audit's `max_cost_usd: float | None` is a PEP 604 +// union and therefore kind "any" — so this is the only exercise the branch +// gets. VERIFIED: float("1_0.5")==10.5 and float("inf")==inf, while +// float("0x1p-2") raises where Go's strconv would happily accept it. +func TestPyFloatGrammar(t *testing.T) { + for _, tc := range []struct { + in string + want float64 + ok bool + }{ + {"1.5", 1.5, true}, + {"1e3", 1000, true}, + {" 2 ", 2, true}, + {"1_0.5", 10.5, true}, + {"-0.25", -0.25, true}, + {"0x1p-2", 0, false}, + {"", 0, false}, + {"abc", 0, false}, + } { + got, ok := parsePyFloat(tc.in) + if ok != tc.ok || (ok && got != tc.want) { + t.Errorf("float(%q) = (%v, %v), want (%v, %v)", tc.in, got, ok, tc.want, tc.ok) + } + } + if f, ok := parsePyFloat("inf"); !ok || f <= 0 { + t.Errorf(`float("inf") = (%v, %v), want +Inf`, f, ok) + } +} + +// TestPyStrRendersDecodedJSONTheCPythonWay pins `str(value)` for the value +// kinds a decoded body holds. VERIFIED: str(5)=="5", str(5.5)=="5.5", +// str(True)=="True", str([1, 2])=="[1, 2]", str({'a': 1})=="{'a': 1}". +// +// The wire-number residual is deliberate: Go's decoder cannot tell `5` from +// `5.0`, and the int spelling is the one that matches the realistic body. +func TestPyStrRendersDecodedJSONTheCPythonWay(t *testing.T) { + for _, tc := range []struct { + in any + want string + }{ + {"already", "already"}, + {float64(5), "5"}, + {float64(5.5), "5.5"}, + {float64(-3), "-3"}, + {true, "True"}, + {false, "False"}, + {[]any{float64(1), float64(2)}, "[1, 2]"}, + {[]any{"a", "b"}, "['a', 'b']"}, + {map[string]any{"a": float64(1)}, "{'a': 1}"}, + {[]any{}, "[]"}, + } { + if got := pyStr(tc.in); got != tc.want { + t.Errorf("str(%#v) = %q, want %q", tc.in, got, tc.want) + } + } +} diff --git a/internal/secaf/reasoners/helpers_test.go b/internal/secaf/reasoners/helpers_test.go new file mode 100644 index 0000000000..6e1d2f1057 --- /dev/null +++ b/internal/secaf/reasoners/helpers_test.go @@ -0,0 +1,122 @@ +package reasoners + +import ( + "context" + "encoding/json" + "reflect" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// helpers_test.go holds the fixtures the adapter tests share. + +// newScanFake returns an appx.Fake whose harness answers every call by leaving +// dest untouched and reporting success. For a hunter that means "the location +// scanner found nothing", which is the cheapest way to drive an adapter end to +// end: the hunter returns its empty HuntResult without enriching, and the ONE +// recorded prompt is the scan prompt whose bytes the parity assertions read. +func newScanFake() *appx.Fake { + return &appx.Fake{ + HarnessFn: func(_ context.Context, _ string, _ map[string]any, dest any, _ appx.HarnessOptions) (*appx.HarnessResult, error) { + return &appx.HarnessResult{Parsed: dest}, nil + }, + } +} + +// assertNote asserts that the fake recorded exactly one note, with the given +// message and tags in order. +func assertNote(t *testing.T, fake *appx.Fake, message string, tags ...string) { + t.Helper() + if len(fake.Notes) != 1 { + t.Fatalf("notes = %d (%v), want exactly 1", len(fake.Notes), fake.NoteMessages()) + } + if fake.Notes[0].Message != message { + t.Errorf("note message = %q, want %q", fake.Notes[0].Message, message) + } + if !reflect.DeepEqual(fake.Notes[0].Tags, tags) { + t.Errorf("note tags = %v, want %v", fake.Notes[0].Tags, tags) + } +} + +// assertPromptHasFileBudget asserts the hunter scan prompt names `want` as its +// early-stop file budget. Every hunter phrases the sentence differently, so the +// assertion looks for " files without". +func assertPromptHasFileBudget(t *testing.T, prompt, want string) { + t.Helper() + if !strings.Contains(prompt, want+" files without") { + t.Errorf("prompt does not carry the file budget %q; prompt tail:\n%s", + want, tail(prompt, 400)) + } +} + +func tail(s string, n int) string { + if len(s) <= n { + return s + } + return "..." + s[len(s)-n:] +} + +// fullReconContext is a recon_context that satisfies ReconResult's five +// required nested models without relying on _recon_model's seed — the shape +// hunt_phase forwards to run_deduplicator. +func fullReconContext() map[string]any { + return map[string]any{ + "architecture": map[string]any{}, + "data_flows": map[string]any{}, + "dependencies": map[string]any{}, + "config": map[string]any{}, + "security_context": map[string]any{"auth_model": "jwt", "auth_details": "bearer"}, + } +} + +// rawFindingPayload is a minimal RawFinding dict carrying every required field. +func rawFindingPayload() map[string]any { + return map[string]any{ + "id": "finding-1", + "hunter_strategy": "injection", + "title": "SQL injection", + "description": "unsanitized input reaches a query", + "finding_type": "sast", + "cwe_id": "CWE-89", + "cwe_name": "SQL Injection", + "file_path": "app/db.py", + "start_line": float64(10), + "end_line": float64(12), + "code_snippet": "query(f\"...{x}\")", + "estimated_severity": "high", + "confidence": "high", + "fingerprint": "fp-1", + } +} + +// asMap renders a typed value the way the control plane would (marshal, then +// decode into an untyped map), so a test can compare against Python's +// model_dump() key set. +func asMap(t *testing.T, v any) map[string]any { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var out map[string]any + if err := json.Unmarshal(b, &out); err != nil { + t.Fatalf("unmarshal: %v", err) + } + return out +} + +// verifierProjection is what prove_phase actually sends to run_verifier: +// `finding.for_verifier().model_dump()`. +func verifierProjection(t *testing.T) map[string]any { + t.Helper() + raw, err := bindRawFinding(rawFindingPayload()) + if err != nil { + t.Fatalf("bindRawFinding: %v", err) + } + return asMap(t, raw.ForVerifier()) +} + +var _ = schemas.RawFinding{} diff --git a/internal/secaf/reasoners/hunt.go b/internal/secaf/reasoners/hunt.go new file mode 100644 index 0000000000..e890bfe64f --- /dev/null +++ b/internal/secaf/reasoners/hunt.go @@ -0,0 +1,282 @@ +package reasoners + +import ( + "context" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/afx" + dedupagent "github.com/Agent-Field/codeaf/internal/secaf/agents/dedup" + huntagent "github.com/Agent-Field/codeaf/internal/secaf/agents/hunt" + "github.com/Agent-Field/codeaf/internal/secaf/phases" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// hunt.go ports src/sec_af/reasoners/hunt.py — the twelve hunter adapters, the +// logic-bugs alias and the deduplicator. + +// reconModel ports `_recon_model(recon_context)` (reasoners/hunt.py:26): +// +// normalized = { +// "architecture": {}, "data_flows": {}, "dependencies": {}, "config": {}, +// "security_context": {"auth_model": "unknown", "auth_details": ""}, +// "languages": [], "frameworks": [], "lines_of_code": 0, "file_count": 0, +// } +// normalized.update(recon_context) +// return ReconResult.model_validate(normalized) +// +// The seed exists because ReconResult has five REQUIRED nested models and +// SecurityContext two required scalars: a hunter reasoner invoked with a +// strategy-pruned recon projection (which is what hunt_phase sends) would +// otherwise fail validation. Seeding-then-overlaying is what makes the pruned +// projection bind. +// +// Python parity, and the reason this is a literal dict merge rather than +// "bind with defaults": +// +// - `dict.update` replaces a key WHOLESALE. A recon_context that carries +// `security_context: {"auth_model": "jwt"}` therefore loses the +// auth_details seed and FAILS validation — the seed is not merged field by +// field. Reproduced exactly: the overlay assigns the caller's value for the +// key, it does not deep-merge. +// - the security_context seed is `auth_details: ""`, NOT the "unknown" that +// recon_phase's quick-depth placeholder uses. The two are different values +// in Python and are kept different here. +// - a key present in recon_context with value None overwrites the seed with +// None and then fails validation, same as Python. +func reconModel(reconContext map[string]any) (schemas.ReconResult, error) { + normalized := map[string]any{ + "architecture": map[string]any{}, + "data_flows": map[string]any{}, + "dependencies": map[string]any{}, + "config": map[string]any{}, + "security_context": map[string]any{ + "auth_model": "unknown", + "auth_details": "", + }, + "languages": []any{}, + "frameworks": []any{}, + "lines_of_code": 0, + "file_count": 0, + } + for k, v := range reconContext { + normalized[k] = v + } + return phases.BindReconResult(normalized) +} + +// hunterFunc is one hunter's agent-level entry point, already closed over the +// arguments the adapter binds. +type hunterFunc func(ctx context.Context, app appx.Harnesser, recon schemas.ReconResult) (schemas.HuntResult, error) + +// runHunter ports `_run_hunter(runner, *, repo_path, recon_context, depth, +// max_files_without_signal=30)` (reasoners/hunt.py:41) — minus its TypeError +// cascade, whose OBSERVABLE effect is captured by each caller instead. +// +// The cascade tries four call shapes in order and keeps the first that does not +// raise TypeError: +// +// 1. runner(app=, repo_path=, recon_result=, depth=, max_files_without_signal=) +// 2. runner(app=, repo_path=, recon=, depth=, max_files_without_signal=) +// 3. runner(app=, repo_path=, recon=) +// 4. runner(app, repo_path, recon_model, depth) [positional] +// +// VERIFIED on the repo's own interpreter (by binding each hunter's real +// signature against each shape): the seven hunters that declare `depth` +// — injection, dos, ssrf, auth, xss, business_logic, and logic — take shape 1 +// and receive the caller's max_files_without_signal. The five that do NOT +// declare depth — crypto, data_exposure, supply_chain, config_secrets, +// api_security — fall through to shape 3, which passes NEITHER depth NOR +// max_files_without_signal, so they always run with their own default of 30. +// +// That is a live Python quirk, not an artifact: those five embed the number in +// their prompt ("if you inspect {n} files without credible ..."), so a caller +// that passes max_files_without_signal=50 gets 50 in seven prompts and 30 in +// the other five. It is reproduced (see hunterMaxFilesForNoDepthHunter), and is +// invisible in the live pipeline because hunt_phase always sends 30. +// Shape 4 is unreachable — no hunter's signature rejects shape 3. +func runHunter( + ctx context.Context, + app appx.App, + reconContext map[string]any, + run hunterFunc, +) (map[string]any, error) { + recon, err := reconModel(reconContext) + if err != nil { + return nil, err + } + result, err := run(ctx, app, recon) + if err != nil { + return nil, err + } + return afx.ToMap(result) +} + +// hunterMaxFilesForNoDepthHunter is the value the five depth-less hunters +// actually receive: their OWN default, because _run_hunter's third cascade +// shape omits the keyword entirely. See runHunter. +const hunterMaxFilesForNoDepthHunter = DefaultMaxFilesWithoutSignal + +// RunInjectionHunter ports `run_injection_hunter` (reasoners/hunt.py:72). +func RunInjectionHunter(ctx context.Context, app appx.App, in HunterInput) (map[string]any, error) { + app.Note(ctx, "Injection hunter starting", "hunt", "injection") + return runHunter(ctx, app, in.ReconContext, + func(ctx context.Context, a appx.Harnesser, recon schemas.ReconResult) (schemas.HuntResult, error) { + return huntagent.RunInjectionHunter(ctx, a, in.RepoPath, recon, in.Depth, in.MaxFilesWithoutSignal) + }) +} + +// RunDosHunter ports `run_dos_hunter` (reasoners/hunt.py:89). +func RunDosHunter(ctx context.Context, app appx.App, in HunterInput) (map[string]any, error) { + app.Note(ctx, "DoS hunter starting", "hunt", "dos") + return runHunter(ctx, app, in.ReconContext, + func(ctx context.Context, a appx.Harnesser, recon schemas.ReconResult) (schemas.HuntResult, error) { + return huntagent.RunDosHunter(ctx, a, in.RepoPath, recon, in.Depth, in.MaxFilesWithoutSignal) + }) +} + +// RunSSRFHunter ports `run_ssrf_hunter` (reasoners/hunt.py:106). +func RunSSRFHunter(ctx context.Context, app appx.App, in HunterInput) (map[string]any, error) { + app.Note(ctx, "SSRF hunter starting", "hunt", "ssrf") + return runHunter(ctx, app, in.ReconContext, + func(ctx context.Context, a appx.Harnesser, recon schemas.ReconResult) (schemas.HuntResult, error) { + return huntagent.RunSSRFHunter(ctx, a, in.RepoPath, recon, in.Depth, in.MaxFilesWithoutSignal) + }) +} + +// RunAuthHunter ports `run_auth_hunter` (reasoners/hunt.py:123). +func RunAuthHunter(ctx context.Context, app appx.App, in HunterInput) (map[string]any, error) { + app.Note(ctx, "Auth hunter starting", "hunt", "auth") + return runHunter(ctx, app, in.ReconContext, + func(ctx context.Context, a appx.Harnesser, recon schemas.ReconResult) (schemas.HuntResult, error) { + return huntagent.RunAuthHunter(ctx, a, in.RepoPath, recon, in.Depth, in.MaxFilesWithoutSignal) + }) +} + +// RunXSSHunter ports `run_xss_hunter` (reasoners/hunt.py:140). +func RunXSSHunter(ctx context.Context, app appx.App, in HunterInput) (map[string]any, error) { + app.Note(ctx, "XSS hunter starting", "hunt", "xss") + return runHunter(ctx, app, in.ReconContext, + func(ctx context.Context, a appx.Harnesser, recon schemas.ReconResult) (schemas.HuntResult, error) { + return huntagent.RunXSSHunter(ctx, a, in.RepoPath, recon, in.Depth, in.MaxFilesWithoutSignal) + }) +} + +// RunCryptoHunter ports `run_crypto_hunter` (reasoners/hunt.py:157). +// +// Python parity: run_crypto_hunter's signature has no `depth`, so _run_hunter +// reaches it through the third cascade shape — depth is DROPPED and +// max_files_without_signal falls back to 30. See runHunter. +func RunCryptoHunter(ctx context.Context, app appx.App, in HunterInput) (map[string]any, error) { + app.Note(ctx, "Crypto hunter starting", "hunt", "crypto") + return runHunter(ctx, app, in.ReconContext, + func(ctx context.Context, a appx.Harnesser, recon schemas.ReconResult) (schemas.HuntResult, error) { + return huntagent.RunCryptoHunter(ctx, a, in.RepoPath, recon, hunterMaxFilesForNoDepthHunter) + }) +} + +// RunBusinessLogicHunter ports `run_business_logic_hunter` +// (reasoners/hunt.py:174). +// +// Python parity: the agent function's sixth parameter `depth_prompt: str = ""` +// is never supplied by _run_hunter, so it keeps its default. +func RunBusinessLogicHunter(ctx context.Context, app appx.App, in HunterInput) (map[string]any, error) { + app.Note(ctx, "Business logic hunter starting", "hunt", "business-logic") + return runHunter(ctx, app, in.ReconContext, + func(ctx context.Context, a appx.Harnesser, recon schemas.ReconResult) (schemas.HuntResult, error) { + return huntagent.RunBusinessLogicHunter(ctx, a, in.RepoPath, recon, in.Depth, in.MaxFilesWithoutSignal, "") + }) +} + +// RunLogicBugsHunter ports `run_logic_bugs_hunter` (reasoners/hunt.py:191): +// +// return await run_business_logic_hunter(repo_path=..., recon_context=..., +// depth=..., max_files_without_signal=...) +// +// Python parity, two points that are easy to miss: +// +// - it emits NO note of its own. The note the execution shows is the +// "Business logic hunter starting" one the delegate emits. +// - it delegates to the REASONER, not to agents/hunt/logic.py's +// run_logic_hunter. reasoners/hunt.py imports run_logic_hunter and never +// uses it (a dead import in the Python source), so this reasoner is +// behaviorally indistinguishable from run_business_logic_hunter. +func RunLogicBugsHunter(ctx context.Context, app appx.App, in HunterInput) (map[string]any, error) { + return RunBusinessLogicHunter(ctx, app, in) +} + +// RunDataExposureHunter ports `run_data_exposure_hunter` +// (reasoners/hunt.py:205). Depth-less hunter — see runHunter. +func RunDataExposureHunter(ctx context.Context, app appx.App, in HunterInput) (map[string]any, error) { + app.Note(ctx, "Data exposure hunter starting", "hunt", "data-exposure") + return runHunter(ctx, app, in.ReconContext, + func(ctx context.Context, a appx.Harnesser, recon schemas.ReconResult) (schemas.HuntResult, error) { + return huntagent.RunDataExposureHunter(ctx, a, in.RepoPath, recon, hunterMaxFilesForNoDepthHunter) + }) +} + +// RunSupplyChainHunter ports `run_supply_chain_hunter` +// (reasoners/hunt.py:222). Depth-less hunter — see runHunter. +func RunSupplyChainHunter(ctx context.Context, app appx.App, in HunterInput) (map[string]any, error) { + app.Note(ctx, "Supply chain hunter starting", "hunt", "supply-chain") + return runHunter(ctx, app, in.ReconContext, + func(ctx context.Context, a appx.Harnesser, recon schemas.ReconResult) (schemas.HuntResult, error) { + return huntagent.RunSupplyChainHunter(ctx, a, in.RepoPath, recon, hunterMaxFilesForNoDepthHunter) + }) +} + +// RunConfigSecretsHunter ports `run_config_secrets_hunter` +// (reasoners/hunt.py:239). Depth-less hunter — see runHunter. +func RunConfigSecretsHunter(ctx context.Context, app appx.App, in HunterInput) (map[string]any, error) { + app.Note(ctx, "Config secrets hunter starting", "hunt", "config-secrets") + return runHunter(ctx, app, in.ReconContext, + func(ctx context.Context, a appx.Harnesser, recon schemas.ReconResult) (schemas.HuntResult, error) { + return huntagent.RunConfigSecretsHunter(ctx, a, in.RepoPath, recon, hunterMaxFilesForNoDepthHunter) + }) +} + +// RunAPISecurityHunter ports `run_api_security_hunter` +// (reasoners/hunt.py:256). Depth-less hunter — see runHunter. +func RunAPISecurityHunter(ctx context.Context, app appx.App, in HunterInput) (map[string]any, error) { + app.Note(ctx, "API security hunter starting", "hunt", "api-security") + return runHunter(ctx, app, in.ReconContext, + func(ctx context.Context, a appx.Harnesser, recon schemas.ReconResult) (schemas.HuntResult, error) { + return huntagent.RunAPISecurityHunter(ctx, a, in.RepoPath, recon, hunterMaxFilesForNoDepthHunter) + }) +} + +// RunDeduplicator ports `run_deduplicator(findings, recon_context, repo_path)` +// (reasoners/hunt.py:273): +// +// raw_findings = [RawFinding(**f) for f in findings] +// recon = ReconResult(**recon_context) +// result = await _deduplicate_and_correlate(raw_findings, recon, router, repo_path) +// return result.model_dump() +// +// Python parity: `ReconResult(**recon_context)` here is the RAW constructor — +// it does NOT go through _recon_model, so an incomplete recon_context fails +// validation instead of being seeded. hunt_phase always forwards the caller's +// full recon dump to this reasoner (only the HUNTERS get the pruned +// projection), so the strict bind is the correct one. +func RunDeduplicator(ctx context.Context, app appx.App, in DeduplicatorInput) (map[string]any, error) { + app.Note(ctx, "Deduplicator starting", "hunt", "dedup") + + findings := make([]schemas.RawFinding, 0, len(in.Findings)) + for _, raw := range in.Findings { + finding, err := bindRawFinding(raw) + if err != nil { + return nil, err + } + findings = append(findings, finding) + } + + recon, err := phases.BindReconResult(in.ReconContext) + if err != nil { + return nil, err + } + + result, err := dedupagent.DeduplicateAndCorrelate(ctx, findings, recon, app, in.RepoPath) + if err != nil { + return nil, err + } + return afx.ToMap(result) +} diff --git a/internal/secaf/reasoners/hunt_test.go b/internal/secaf/reasoners/hunt_test.go new file mode 100644 index 0000000000..9a23a0f089 --- /dev/null +++ b/internal/secaf/reasoners/hunt_test.go @@ -0,0 +1,247 @@ +package reasoners + +// Tests for src/sec_af/reasoners/hunt.py. +// +// Validation contract (behaviour, derived from the Python module): +// +// - every hunter reasoner emits its own "X starting" note with its own tags, +// BEFORE the agent function runs; +// - run_logic_bugs_hunter emits NO note of its own and is otherwise +// indistinguishable from run_business_logic_hunter; +// - _recon_model seeds the five required nested models plus +// security_context={"auth_model":"unknown","auth_details":""} and then +// OVERLAYS recon_context wholesale (dict.update semantics — a partial +// security_context replaces the seed and fails validation); +// - the seven hunters whose Python signature declares `depth` receive the +// caller's max_files_without_signal; the five that do not always run with +// 30 (the TypeError cascade drops the keyword); +// - run_deduplicator validates its findings as RawFindings and its +// recon_context as a ReconResult WITHOUT the _recon_model seed; +// - each adapter returns the agent result's model_dump() key set. + +import ( + "context" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" +) + +// hunterAdapter is one hunter reasoner under test. +type hunterAdapter struct { + Name string + Fn func(context.Context, appx.App, HunterInput) (map[string]any, error) + Note string + Tags []string + // TakesDepth reports whether the Python agent signature declares `depth`, + // which is what decides whether max_files_without_signal survives + // _run_hunter's TypeError cascade. + TakesDepth bool + // Gated hunters return early without a harness call when the recon context + // gives them nothing to do, so their prompt cannot be inspected. + Gated bool +} + +func hunterAdapters() []hunterAdapter { + return []hunterAdapter{ + {"run_injection_hunter", RunInjectionHunter, "Injection hunter starting", []string{"hunt", "injection"}, true, false}, + {"run_dos_hunter", RunDosHunter, "DoS hunter starting", []string{"hunt", "dos"}, true, false}, + {"run_ssrf_hunter", RunSSRFHunter, "SSRF hunter starting", []string{"hunt", "ssrf"}, true, false}, + {"run_auth_hunter", RunAuthHunter, "Auth hunter starting", []string{"hunt", "auth"}, true, false}, + {"run_xss_hunter", RunXSSHunter, "XSS hunter starting", []string{"hunt", "xss"}, true, false}, + {"run_crypto_hunter", RunCryptoHunter, "Crypto hunter starting", []string{"hunt", "crypto"}, false, true}, + {"run_business_logic_hunter", RunBusinessLogicHunter, "Business logic hunter starting", []string{"hunt", "business-logic"}, true, false}, + {"run_data_exposure_hunter", RunDataExposureHunter, "Data exposure hunter starting", []string{"hunt", "data-exposure"}, false, false}, + {"run_supply_chain_hunter", RunSupplyChainHunter, "Supply chain hunter starting", []string{"hunt", "supply-chain"}, false, true}, + {"run_config_secrets_hunter", RunConfigSecretsHunter, "Config secrets hunter starting", []string{"hunt", "config-secrets"}, false, false}, + {"run_api_security_hunter", RunAPISecurityHunter, "API security hunter starting", []string{"hunt", "api-security"}, false, true}, + } +} + +func TestHunterAdaptersEmitTheirNote(t *testing.T) { + for _, tc := range hunterAdapters() { + t.Run(tc.Name, func(t *testing.T) { + fake := newScanFake() + if _, err := tc.Fn(context.Background(), fake, HunterInput{ + RepoPath: t.TempDir(), + ReconContext: map[string]any{}, + Depth: "standard", + MaxFilesWithoutSignal: DefaultMaxFilesWithoutSignal, + }); err != nil { + t.Fatalf("%s: %v", tc.Name, err) + } + assertNote(t, fake, tc.Note, tc.Tags...) + }) + } +} + +// TestHunterMaxFilesCascadeParity is the observable half of _run_hunter's +// TypeError cascade: with max_files_without_signal=50, the hunters whose +// signature takes `depth` put 50 in their prompt and the ones that do not put +// 30, because the third cascade shape passes neither keyword. +// +// VERIFIED against the Python source by binding each hunter's real signature +// against each cascade shape on the repo's own interpreter. +func TestHunterMaxFilesCascadeParity(t *testing.T) { + const custom = 50 + + for _, tc := range hunterAdapters() { + if tc.Gated { + continue // no harness call to inspect + } + t.Run(tc.Name, func(t *testing.T) { + fake := newScanFake() + if _, err := tc.Fn(context.Background(), fake, HunterInput{ + RepoPath: t.TempDir(), + ReconContext: map[string]any{}, + Depth: "thorough", + MaxFilesWithoutSignal: custom, + }); err != nil { + t.Fatalf("%s: %v", tc.Name, err) + } + if len(fake.Harnesses) == 0 { + t.Fatalf("%s made no harness call", tc.Name) + } + want := "30" + if tc.TakesDepth { + want = "50" + } + assertPromptHasFileBudget(t, fake.Harnesses[0].Prompt, want) + }) + } +} + +// TestRunLogicBugsHunterDelegates pins the alias: no note of its own, and the +// same prompt run_business_logic_hunter produces for the same input. +func TestRunLogicBugsHunterDelegates(t *testing.T) { + repo := t.TempDir() + in := HunterInput{ + RepoPath: repo, + ReconContext: map[string]any{}, + Depth: "thorough", + MaxFilesWithoutSignal: 42, + } + + aliasFake := newScanFake() + aliasResult, err := RunLogicBugsHunter(context.Background(), aliasFake, in) + if err != nil { + t.Fatalf("RunLogicBugsHunter: %v", err) + } + + directFake := newScanFake() + directResult, err := RunBusinessLogicHunter(context.Background(), directFake, in) + if err != nil { + t.Fatalf("RunBusinessLogicHunter: %v", err) + } + + // The alias emits exactly the delegate's note — one note, not two, and not + // a "logic bugs" one. + assertNote(t, aliasFake, "Business logic hunter starting", "hunt", "business-logic") + + if len(aliasFake.Harnesses) != len(directFake.Harnesses) { + t.Fatalf("harness calls: alias %d, direct %d", len(aliasFake.Harnesses), len(directFake.Harnesses)) + } + for i := range aliasFake.Harnesses { + if aliasFake.Harnesses[i].Prompt != directFake.Harnesses[i].Prompt { + t.Errorf("prompt %d differs between the alias and the delegate", i) + } + } + if len(aliasResult) != len(directResult) { + t.Errorf("result key count: alias %d, direct %d", len(aliasResult), len(directResult)) + } +} + +// TestReconModelSeeds pins _recon_model's normalization. +func TestReconModelSeeds(t *testing.T) { + t.Run("empty context binds with the seeded defaults", func(t *testing.T) { + recon, err := reconModel(map[string]any{}) + if err != nil { + t.Fatalf("reconModel: %v", err) + } + if recon.SecurityContext.AuthModel != "unknown" { + t.Errorf("auth_model = %q, want %q", recon.SecurityContext.AuthModel, "unknown") + } + // Python parity: the hunt.py seed uses "" for auth_details, NOT the + // "unknown" recon_phase's quick-depth placeholder uses. + if recon.SecurityContext.AuthDetails != "" { + t.Errorf("auth_details = %q, want the empty string", recon.SecurityContext.AuthDetails) + } + if recon.LinesOfCode != 0 || recon.FileCount != 0 { + t.Errorf("metrics = (%d, %d), want (0, 0)", recon.LinesOfCode, recon.FileCount) + } + }) + + t.Run("caller values overlay the seed", func(t *testing.T) { + recon, err := reconModel(map[string]any{ + "languages": []any{"python"}, + "lines_of_code": float64(4200), + }) + if err != nil { + t.Fatalf("reconModel: %v", err) + } + if len(recon.Languages) != 1 || recon.Languages[0] != "python" { + t.Errorf("languages = %v, want [python]", recon.Languages) + } + if recon.LinesOfCode != 4200 { + t.Errorf("lines_of_code = %d, want 4200", recon.LinesOfCode) + } + }) + + t.Run("a partial security_context replaces the seed wholesale and fails", func(t *testing.T) { + // Python parity: dict.update replaces the KEY, so the auth_details seed + // is lost and SecurityContext validation fails. + if _, err := reconModel(map[string]any{ + "security_context": map[string]any{"auth_model": "jwt"}, + }); err == nil { + t.Fatal("want a validation error for the partial security_context") + } + }) +} + +// TestRunDeduplicatorValidatesStrictly pins the two binds run_deduplicator does +// — RawFinding per finding, and a RAW ReconResult (no _recon_model seed). +func TestRunDeduplicatorValidatesStrictly(t *testing.T) { + t.Run("an incomplete recon_context fails", func(t *testing.T) { + fake := newScanFake() + _, err := RunDeduplicator(context.Background(), fake, DeduplicatorInput{ + Findings: nil, + ReconContext: map[string]any{}, + RepoPath: t.TempDir(), + }) + if err == nil { + t.Fatal("want a validation error: run_deduplicator does not seed defaults") + } + assertNote(t, fake, "Deduplicator starting", "hunt", "dedup") + }) + + t.Run("a malformed finding fails", func(t *testing.T) { + fake := newScanFake() + _, err := RunDeduplicator(context.Background(), fake, DeduplicatorInput{ + Findings: []map[string]any{{"title": "malformed"}}, + ReconContext: fullReconContext(), + RepoPath: t.TempDir(), + }) + if err == nil { + t.Fatal("want a validation error for a RawFinding missing required fields") + } + }) + + t.Run("a well-formed request returns a HuntResult dump", func(t *testing.T) { + fake := newScanFake() + got, err := RunDeduplicator(context.Background(), fake, DeduplicatorInput{ + Findings: []map[string]any{rawFindingPayload()}, + ReconContext: fullReconContext(), + RepoPath: t.TempDir(), + }) + if err != nil { + t.Fatalf("RunDeduplicator: %v", err) + } + for _, key := range []string{ + "findings", "chains", "total_raw", "deduplicated_count", "chain_count", + "strategies_run", "hunt_duration_seconds", + } { + if _, ok := got[key]; !ok { + t.Errorf("result is missing the %q key (HuntResult.model_dump())", key) + } + } + }) +} diff --git a/internal/secaf/reasoners/input_schemas.go b/internal/secaf/reasoners/input_schemas.go new file mode 100644 index 0000000000..c08a623551 --- /dev/null +++ b/internal/secaf/reasoners/input_schemas.go @@ -0,0 +1,143 @@ +package reasoners + +// input_schemas.go gives every reasoner the SAME input schema the Python node +// publishes to the control plane. +// +// # Why a fixture and not a derivation +// +// The Python SDK does not hand-write these schemas: `@router.reasoner()` stores +// the decorated function's `(annotation, default)` pairs and +// `Agent._types_to_json_schema` (sdk/python/agentfield/agent.py) turns them +// into JSON Schema on demand, once per registration. The Go SDK has no +// equivalent — `RegisterReasoner` stamps every reasoner with the placeholder +// `{"type":"object","additionalProperties":true}` unless the caller passes +// agent.WithInputSchema. Re-deriving the schemas from the Go input structs +// would NOT reproduce Python, because `_type_to_json_schema` has quirks a +// reflection-based Go derivation would "fix" (see below). So the exact bytes +// Python registers are committed as a fixture and replayed verbatim. +// +// # The three quirks the fixture preserves +// +// - `X | None` becomes {"type":"object"}, NOT the base type. The Union branch +// of `_type_to_json_schema` tests `typ.__origin__ is Union`, but a PEP 604 +// union (`str | None`) is a `types.UnionType` with no `__origin__` at all, +// so it falls through to the trailing `return {"type": "object"}`. Every +// optional parameter in this node is spelled the PEP 604 way, so +// `commit_sha: str | None`, `scan_types: list[str] | None`, +// `max_provers: int | None` and `ai_gate: Any | None` are all reported as +// bare objects. +// - `dict[str, Any]` becomes {"type":"object","additionalProperties":true} +// while a plain `dict` would be {"type":"object"} — the two are visibly +// different in the fixture (`recon_context` vs `commit_sha`). +// - a parameter with ANY default is omitted from `required`, and `required` +// itself is omitted when empty. `required` keeps the Python parameter +// ORDER, not alphabetical order — e.g. run_verdict_agent requires +// ["finding","data_flow","sanitization","exploit"]. +// +// None of this is "better" or "worse" than what a Go derivation would produce; +// it is what callers reading `sec-af`'s discovery payload see today, and +// DESIGN.md §0.2 says to reproduce rather than improve. +// +// # Provenance and regeneration +// +// testdata/python_input_schemas.json was captured from a LIVE Python sec-af +// node running agentfield==0.1.131, through the control plane's discovery API +// with input schemas requested: +// +// curl -s "$AGENTFIELD_URL/api/v1/discovery/capabilities?node_id=sec-af&include_input_schema=true" +// +// keyed by reasoner id. It holds all 34 reasoners: the 33 router reasoners plus +// the top-level `audit`. +// +// To regenerate it WITHOUT a running node or control plane — verified to be +// byte-identical to the live capture, because `app.reasoners` is the very +// property the SDK serialises into the registration payload: +// +// PYTHONPATH=/src ~/.agentfield/packages/sec-af/venv/bin/python -c ' +// import json +// from sec_af.app import app +// print(json.dumps({r["id"]: r["input_schema"] for r in app.reasoners}, +// indent=1, sort_keys=True)) +// ' > go/internal/reasoners/testdata/python_input_schemas.json +// +// Regenerate only when the PYTHON signatures change. A Go-side change must +// never edit this file to make a test pass — that would be the Go port quietly +// redefining the contract. + +import ( + "bytes" + _ "embed" + "encoding/json" + "fmt" + "sort" +) + +// pythonInputSchemasJSON is the committed capture, embedded so the schemas ship +// in the binary rather than being read from disk at boot. +// +//go:embed testdata/python_input_schemas.json +var pythonInputSchemasJSON []byte + +// pythonInputSchemas maps reasoner id -> the compacted schema bytes. Parsing +// happens once, at package init, so a corrupt fixture fails the process +// immediately instead of at the first registration. +var pythonInputSchemas = mustParseInputSchemas(pythonInputSchemasJSON) + +// mustParseInputSchemas decodes the fixture and compacts each schema. +// +// Compaction is cosmetic but deliberate: the fixture is stored pretty-printed +// so a human can diff it, while the bytes that go on the wire should look like +// what Python's HTTP client sends (json.dumps with no indent). Compacting also +// normalises the value, so two identical schemas registered from different +// entries are byte-identical. +func mustParseInputSchemas(raw []byte) map[string]json.RawMessage { + var decoded map[string]json.RawMessage + if err := json.Unmarshal(raw, &decoded); err != nil { + panic(fmt.Sprintf("reasoners: testdata/python_input_schemas.json is not a JSON object of schemas: %v", err)) + } + if len(decoded) == 0 { + panic("reasoners: testdata/python_input_schemas.json is empty") + } + + out := make(map[string]json.RawMessage, len(decoded)) + for name, schema := range decoded { + var buf bytes.Buffer + if err := json.Compact(&buf, schema); err != nil { + panic(fmt.Sprintf("reasoners: input schema for %q is not valid JSON: %v", name, err)) + } + out[name] = json.RawMessage(buf.Bytes()) + } + return out +} + +// InputSchema returns the input schema the Python node publishes for the +// reasoner called name, as the registry stores it beside the handler. +// +// It PANICS when the fixture has no entry for name. That is the point: every +// registration goes through here, so adding a reasoner to the Go port without +// regenerating the capture (or renaming one out of sync with Python) crashes at +// registration — loudly, at boot and in every test — instead of silently +// publishing the SDK's `{"type":"object","additionalProperties":true}` +// placeholder and letting the two nodes drift apart in discovery. +// +// The returned slice is a copy, so a caller cannot mutate the shared fixture. +func InputSchema(name string) json.RawMessage { + schema, ok := pythonInputSchemas[name] + if !ok { + panic(fmt.Sprintf( + "reasoners: no input schema for reasoner %q in testdata/python_input_schemas.json; "+ + "regenerate the capture (see input_schemas.go) after changing the Python surface", name)) + } + return append(json.RawMessage(nil), schema...) +} + +// InputSchemaNames returns every reasoner id the fixture carries, sorted. It is +// the "what Python publishes" side of the registration parity assertions. +func InputSchemaNames() []string { + names := make([]string, 0, len(pythonInputSchemas)) + for name := range pythonInputSchemas { + names = append(names, name) + } + sort.Strings(names) + return names +} diff --git a/internal/secaf/reasoners/input_schemas_test.go b/internal/secaf/reasoners/input_schemas_test.go new file mode 100644 index 0000000000..6802121b2b --- /dev/null +++ b/internal/secaf/reasoners/input_schemas_test.go @@ -0,0 +1,256 @@ +package reasoners + +// Input-schema parity for the router surface. +// +// Validation contract (behaviour, derived from what the Python node publishes +// to the control plane — NOT from input_schemas.go): +// +// - every reasoner this package registers carries a schema derived from its +// Python signature; none is left on the SDK's +// `{"type":"object","additionalProperties":true}` placeholder; +// - the schema the registry actually holds for a reasoner (read back through +// Registry.InputSchema) is the schema the Python node published for that +// reasoner id, compared key-order-insensitively; +// - the capture covers exactly the node's surface: the 33 router reasoners +// plus `audit`. A fixture entry with no registration, or a registration +// with no fixture entry, is drift and fails; +// - a registration for a name the capture does not know panics, so drift is +// impossible to ship quietly; +// - the published shapes carry Python's derivation quirks verbatim: a PEP 604 +// `X | None` is {"type":"object"}, a `dict[str, Any]` is +// {"type":"object","additionalProperties":true}, parameters with defaults +// are absent from `required`, and `required` keeps the Python parameter +// order. + +import ( + "encoding/json" + "reflect" + "sort" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" +) + +// sdkPlaceholderSchema is what the SDK stamped on a reasoner registered with +// no schema. A registration carrying it would mean the schema was lost. +const sdkPlaceholderSchema = `{"type":"object","additionalProperties":true}` + +// discoverInputSchemas builds the registry and reads the per-reasoner input +// schemas back out of it, decoded. +// +// Reading back (rather than asserting on InputSchema directly) is what makes +// the test meaningful: it proves the schema survived the registration path and +// inspects what the registry actually holds. +func discoverInputSchemas(t *testing.T) map[string]any { + t.Helper() + + reg := NewRegistry() + RegisterAll(reg, &appx.Fake{}) + + out := map[string]any{} + for _, name := range reg.Names() { + raw, ok := reg.InputSchema(name) + if !ok { + t.Fatalf("registry lists %q but holds no schema for it", name) + } + out[name] = decodeSchema(t, raw) + } + return out +} + +// decodeSchema renders raw JSON as untyped Go values, which makes a comparison +// insensitive to object key order (JSON objects become maps) while staying +// sensitive to array order — `required` is a list whose order is Python's +// parameter order and must be reproduced. +func decodeSchema(t *testing.T, raw []byte) any { + t.Helper() + var v any + if err := json.Unmarshal(raw, &v); err != nil { + t.Fatalf("decode schema %s: %v", raw, err) + } + return v +} + +// TestCaptureCoversExactlyTheNodeSurface pins the two-way containment: the +// capture is neither missing a reasoner this port registers nor carrying one it +// does not. `audit` is in the capture but is the entry internal/secaf/audit +// owns rather than a registered reasoner, so it is added to the expected set +// here. +func TestCaptureCoversExactlyTheNodeSurface(t *testing.T) { + want := append([]string{NameAudit}, Names...) + sort.Strings(want) + + got := InputSchemaNames() + if !reflect.DeepEqual(got, want) { + t.Fatalf("captured schema ids mismatch:\n got = %v\n want = %v", got, want) + } + if len(got) != 34 { + t.Errorf("captured %d schemas, want 34 (audit + 33 router reasoners)", len(got)) + } +} + +// TestEveryRegisteredReasonerPublishesItsPythonSchema is the whole-surface +// assertion: for all 33 router reasoners, what the registry holds equals what +// Python publishes, and nothing was left on the placeholder. +func TestEveryRegisteredReasonerPublishesItsPythonSchema(t *testing.T) { + published := discoverInputSchemas(t) + registered := RegisterAll(NewRegistry(), &appx.Fake{}) + + if len(published) != len(registered) { + t.Fatalf("registry holds %d reasoners, want %d", len(published), len(registered)) + } + + placeholder := decodeSchema(t, []byte(sdkPlaceholderSchema)) + + for _, name := range registered { + got, ok := published[name] + if !ok { + t.Errorf("%s: not present in the registry", name) + continue + } + want := decodeSchema(t, InputSchema(name)) + if !reflect.DeepEqual(got, want) { + t.Errorf("%s: published schema mismatch\n got = %#v\n want = %#v", name, got, want) + } + if reflect.DeepEqual(got, placeholder) { + t.Errorf("%s: published the SDK placeholder schema — WithInputSchema was not applied", name) + } + } +} + +// TestEveryCapturedRouterSchemaIsRegistered walks the other direction: every +// captured id except `audit` must be reachable in the registry. A Python reasoner +// that was never ported would otherwise sit in the capture unnoticed. +func TestEveryCapturedRouterSchemaIsRegistered(t *testing.T) { + published := discoverInputSchemas(t) + + for _, name := range InputSchemaNames() { + if name == NameAudit { + // The audit entry's own spec, used by audit.BindRequest rather + // than registered. + continue + } + if _, ok := published[name]; !ok { + t.Errorf("captured schema %q has no registered reasoner", name) + } + } +} + +// TestRepresentativeSchemasMatchPythonSignatures is the non-tautological half: +// four reasoners whose expected schema is transcribed HERE from the Python +// signature, so a bad regeneration of the capture fails too. Between them they +// cover every mapping the node exercises. +func TestRepresentativeSchemasMatchPythonSignatures(t *testing.T) { + published := discoverInputSchemas(t) + + cases := []struct { + // name is the reasoner id; signature is the Python one it transcribes. + name string + signature string + want string + }{ + { + // The simplest shape: one required `str`. + name: NameRunArchitectureMapper, + signature: "run_architecture_mapper(repo_path: str)", + want: `{"type":"object", + "properties":{"repo_path":{"type":"string"}}, + "required":["repo_path"]}`, + }, + { + // dict[str, Any] -> object + additionalProperties; the `= 30` + // default keeps max_files_without_signal OUT of required, while + // `depth` (no default) stays in — and required is in PARAMETER + // order, not alphabetical. + name: NameRunInjectionHunter, + signature: "run_injection_hunter(repo_path: str, recon_context: dict[str, Any], " + + "depth: str, max_files_without_signal: int = 30)", + want: `{"type":"object", + "properties":{"repo_path":{"type":"string"}, + "recon_context":{"type":"object","additionalProperties":true}, + "depth":{"type":"string"}, + "max_files_without_signal":{"type":"integer"}}, + "required":["repo_path","recon_context","depth"]}`, + }, + { + // `int | None` is a PEP 604 union, which _type_to_json_schema's + // Union branch never sees (no __origin__), so it falls through to + // the {"type":"object"} default instead of {"type":"integer"}. + name: NameProvePhase, + signature: "prove_phase(repo_path: str, hunt_result: dict[str, Any], depth: str = \"standard\", " + + "max_provers: int | None = None, max_concurrent_provers: int = 3)", + want: `{"type":"object", + "properties":{"repo_path":{"type":"string"}, + "hunt_result":{"type":"object","additionalProperties":true}, + "depth":{"type":"string"}, + "max_provers":{"type":"object"}, + "max_concurrent_provers":{"type":"integer"}}, + "required":["repo_path","hunt_result"]}`, + }, + { + // list[dict[str, Any]] -> array whose items carry the dict mapping. + name: NameRemediationPhase, + signature: "remediation_phase(repo_path: str, verified_findings: list[dict[str, Any]], " + + "max_concurrent_remediations: int = 3)", + want: `{"type":"object", + "properties":{"repo_path":{"type":"string"}, + "verified_findings":{"type":"array", + "items":{"type":"object","additionalProperties":true}}, + "max_concurrent_remediations":{"type":"integer"}}, + "required":["repo_path","verified_findings"]}`, + }, + { + // list[str] -> array of string, and a reasoner with NO defaulted + // parameter requires all of them. + name: NameRunCWEExpansion, + signature: "run_cwe_expansion(recon_summary: str, strategies: list[str])", + want: `{"type":"object", + "properties":{"recon_summary":{"type":"string"}, + "strategies":{"type":"array","items":{"type":"string"}}}, + "required":["recon_summary","strategies"]}`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + want := decodeSchema(t, []byte(tc.want)) + + if got := decodeSchema(t, InputSchema(tc.name)); !reflect.DeepEqual(got, want) { + t.Errorf("captured schema for %s does not match the Python signature\n %s\n got = %#v\n want = %#v", + tc.name, tc.signature, got, want) + } + if got := published[tc.name]; !reflect.DeepEqual(got, want) { + t.Errorf("published schema for %s does not match the Python signature\n %s\n got = %#v\n want = %#v", + tc.name, tc.signature, got, want) + } + }) + } +} + +// TestInputSchemaPanicsOnUnknownReasoner pins the loud-drift contract: a +// registration for a name the capture does not carry must crash rather than +// register a reasoner with no schema. +func TestInputSchemaPanicsOnUnknownReasoner(t *testing.T) { + defer func() { + if recover() == nil { + t.Fatal("InputSchema on an unknown reasoner returned instead of panicking") + } + }() + _ = InputSchema("run_not_a_reasoner") +} + +// TestInputSchemaReturnsACopy proves a caller cannot corrupt the shared fixture +// for every later registration — the schemas are handed out as json.RawMessage, +// which is a mutable slice. +func TestInputSchemaReturnsACopy(t *testing.T) { + first := InputSchema(NameRunArchitectureMapper) + original := append(json.RawMessage(nil), first...) + + for i := range first { + first[i] = 'x' + } + + if second := InputSchema(NameRunArchitectureMapper); !reflect.DeepEqual(second, original) { + t.Errorf("mutating a returned schema changed the fixture: %s", second) + } +} diff --git a/internal/secaf/reasoners/inputs.go b/internal/secaf/reasoners/inputs.go new file mode 100644 index 0000000000..6f6c92db51 --- /dev/null +++ b/internal/secaf/reasoners/inputs.go @@ -0,0 +1,343 @@ +package reasoners + +import ( + "encoding/json" + + "github.com/Agent-Field/codeaf/internal/secaf/phases" +) + +// inputs.go declares one struct per reasoner, transcribing the Python +// `async def (...)` signature: json tag == parameter name, field type +// == annotation, and — where the parameter has a non-zero default — a +// `New()` constructor plus an UnmarshalJSON that seeds it before decoding. +// +// The seeding pattern is the schemas package's (`type alias T` strips the +// method set so the inner Unmarshal does not recurse). Its contract matters: +// an ABSENT key keeps the Python default, while a PRESENT key — even one whose +// value is 0 / "" / false / null — overrides it, exactly as Python's +// keyword-argument binding does. +// +// Fields typed `map[string]any` / `[]map[string]any` stand for the Python +// `dict[str, Any]` / `list[dict[str, Any]]` parameters that the adapters hand +// to a pydantic constructor; they are validated in validate.go rather than by +// the bind. +// +// The bind is not the first thing a request meets. handler_input.go runs the +// Python SDK's own `_validate_handler_input` ahead of it — rejecting an +// explicit null on a required parameter, coercing scalars (`"50"` -> 50, +// `5` -> "5", `"yes"` -> true) and shape-checking the dict/list parameters — so +// the map these structs decode has already been through Python's rules. + +// --------------------------------------------------------------------------- +// reasoners/recon.py +// --------------------------------------------------------------------------- + +// RepoPathInput is the signature shared by the three recon reasoners that take +// only a repository path: +// +// run_architecture_mapper(repo_path: str) +// run_dependency_auditor(repo_path: str) +// run_config_scanner(repo_path: str) +type RepoPathInput struct { + RepoPath string `json:"repo_path"` +} + +// ArchitectureInput ports the two deep-recon reasoner signatures: +// +// run_data_flow_mapper(repo_path: str, architecture: dict[str, Any]) +// run_security_context_profiler(repo_path: str, architecture: dict[str, Any]) +// +// `architecture` becomes `ArchitectureMap(**architecture)`. ArchitectureMap has +// no required field of its own, but its five lists hold models that do, so the +// bind goes through phases.BindArchitectureMap rather than a bare afx.Bind. +type ArchitectureInput struct { + RepoPath string `json:"repo_path"` + Architecture map[string]any `json:"architecture"` +} + +// --------------------------------------------------------------------------- +// reasoners/hunt.py +// --------------------------------------------------------------------------- + +// DefaultMaxFilesWithoutSignal ports the `max_files_without_signal: int = 30` +// default every hunter reasoner declares. +const DefaultMaxFilesWithoutSignal = 30 + +// HunterInput is the signature all twelve hunter reasoners share: +// +// run__hunter(repo_path: str, recon_context: dict[str, Any], +// depth: str, max_files_without_signal: int = 30) +// +// `depth` has NO default here (it is positional-or-keyword without one), so a +// request that OMITS it binds the empty string — which every downstream +// _normalize_depth turns into "standard". +// +// This is the ONE rule of handler_input.go's port of the Python SDK's +// `_validate_handler_input` that is deliberately not reproduced: Python answers +// `422 Missing required field: depth` before the handler body runs +// (agent.py:1169-1171), because it can see that the key was absent. The Go +// input structs cannot — an absent scalar is indistinguishable from a zero one +// — so the request proceeds. Everything else that layer does (the +// null-on-required rejection, the int/float/str/bool coercions, the dict/list +// shape checks) IS reproduced; see handler_input.go. +type HunterInput struct { + RepoPath string `json:"repo_path"` + ReconContext map[string]any `json:"recon_context"` + Depth string `json:"depth"` + MaxFilesWithoutSignal int `json:"max_files_without_signal"` +} + +// NewHunterInput returns the Python keyword defaults. +func NewHunterInput() HunterInput { + return HunterInput{MaxFilesWithoutSignal: DefaultMaxFilesWithoutSignal} +} + +// UnmarshalJSON seeds max_files_without_signal=30 before decoding. +func (h *HunterInput) UnmarshalJSON(b []byte) error { + *h = NewHunterInput() + type alias HunterInput + return json.Unmarshal(b, (*alias)(h)) +} + +// DeduplicatorInput ports: +// +// run_deduplicator(findings: list[dict[str, Any]], recon_context: dict[str, Any], repo_path: str) +// +// Field ORDER follows the Python signature (findings first), which is also the +// order the JSON keys are documented in; it has no effect on binding. +type DeduplicatorInput struct { + Findings []map[string]any `json:"findings"` + ReconContext map[string]any `json:"recon_context"` + RepoPath string `json:"repo_path"` +} + +// --------------------------------------------------------------------------- +// reasoners/prove.py +// --------------------------------------------------------------------------- + +// FindingDepthInput ports the three signatures shaped +// `(repo_path, finding, depth)`: +// +// run_dep_reachability(repo_path: str, finding: dict[str, Any], depth: str) +// run_verifier(repo_path: str, finding: dict[str, Any], depth: str) +// run_tracer(repo_path: str, finding: dict[str, Any], depth: str) +type FindingDepthInput struct { + RepoPath string `json:"repo_path"` + Finding map[string]any `json:"finding"` + Depth string `json:"depth"` +} + +// SanitizationInput ports: +// +// run_sanitization_analyzer(repo_path: str, finding: dict[str, Any], +// data_flow: dict[str, Any], depth: str) +type SanitizationInput struct { + RepoPath string `json:"repo_path"` + Finding map[string]any `json:"finding"` + DataFlow map[string]any `json:"data_flow"` + Depth string `json:"depth"` +} + +// ExploitInput ports: +// +// run_exploit_hypothesizer(repo_path: str, finding: dict[str, Any], +// data_flow: dict[str, Any], +// sanitization: dict[str, Any], depth: str) +type ExploitInput struct { + RepoPath string `json:"repo_path"` + Finding map[string]any `json:"finding"` + DataFlow map[string]any `json:"data_flow"` + Sanitization map[string]any `json:"sanitization"` + Depth string `json:"depth"` +} + +// VerdictInput ports: +// +// run_verdict_agent(finding: dict[str, Any], data_flow: dict[str, Any], +// sanitization: dict[str, Any], exploit: dict[str, Any]) +// +// There is no repo_path parameter: the adapter passes the literal "." to +// run_verdict_agent, which never reads it. +type VerdictInput struct { + Finding map[string]any `json:"finding"` + DataFlow map[string]any `json:"data_flow"` + Sanitization map[string]any `json:"sanitization"` + Exploit map[string]any `json:"exploit"` +} + +// RemediationInput ports: +// +// run_remediation(repo_path: str, finding: dict[str, Any]) +// +// `finding` is a VerifiedFinding here (not a RawFinding) — this is the reasoner +// remediation_phase calls. +type RemediationInput struct { + RepoPath string `json:"repo_path"` + Finding map[string]any `json:"finding"` +} + +// RemediationAgentInput ports: +// +// run_remediation_agent(repo_path: str, finding: dict[str, Any], verdict: str, rationale: str) +// +// `finding` is a RawFinding here. Nothing in the pipeline calls this reasoner; +// it is part of the registered surface. +type RemediationAgentInput struct { + RepoPath string `json:"repo_path"` + Finding map[string]any `json:"finding"` + Verdict string `json:"verdict"` + Rationale string `json:"rationale"` +} + +// DastVerifierInput ports: +// +// run_dast_verifier(repo_path: str, finding: dict[str, Any], exploit_payload: str, depth: str) +type DastVerifierInput struct { + RepoPath string `json:"repo_path"` + Finding map[string]any `json:"finding"` + ExploitPayload string `json:"exploit_payload"` + Depth string `json:"depth"` +} + +// CrossServiceInput ports: +// +// run_cross_service_analyzer(repo_path: str, services: list[str], +// findings_summary: str, depth: str) +type CrossServiceInput struct { + RepoPath string `json:"repo_path"` + Services []string `json:"services"` + FindingsSummary string `json:"findings_summary"` + Depth string `json:"depth"` +} + +// --------------------------------------------------------------------------- +// reasoners/phases.py +// --------------------------------------------------------------------------- + +// CWEExpansionInput ports: +// +// run_cwe_expansion(recon_summary: str, strategies: list[str]) +type CWEExpansionInput struct { + ReconSummary string `json:"recon_summary"` + Strategies []string `json:"strategies"` +} + +// ReconPhaseInput ports: +// +// recon_phase(repo_path: str, depth: str = "standard") +type ReconPhaseInput struct { + RepoPath string `json:"repo_path"` + Depth string `json:"depth"` +} + +// NewReconPhaseInput returns the Python keyword defaults. +func NewReconPhaseInput() ReconPhaseInput { + return ReconPhaseInput{Depth: phases.DefaultDepth} +} + +// UnmarshalJSON seeds depth="standard" before decoding. +func (r *ReconPhaseInput) UnmarshalJSON(b []byte) error { + *r = NewReconPhaseInput() + type alias ReconPhaseInput + return json.Unmarshal(b, (*alias)(r)) +} + +// HuntPhaseInput ports: +// +// hunt_phase(repo_path: str, recon_context: dict[str, Any], depth: str = "standard", +// ai_gate: Any | None = None, max_concurrent_hunters: int = 4, +// early_stop_file_threshold: int = 30) +// +// `ai_gate` is kept as a RAW JSON value. It is an object parameter that +// app.py's `.call` into hunt_phase never passes, but hunt_phase is a REGISTERED +// reasoner, so a control-plane caller can send one — and Python binds it (the +// SDK passes an `Any`-hinted parameter through untouched), takes the +// `ai_gate is not None` branch, raises AttributeError on +// `ai_gate.select_strategy(...)` and emits +// +// AI gate failed: 'dict' object has no attribute 'select_strategy', using default strategies +// +// with tags ["hunt","ai_gate","error"] before falling back to the default +// strategies. phases.NewJSONAIGate turns the raw value into exactly that: nil +// (Python's None) for an absent key or `null`, and otherwise a gate that fails +// with the CPython message for that JSON type. Raw bytes rather than `any` +// because encoding/json decodes every number to float64, which would report +// "float" where CPython reports "int". +type HuntPhaseInput struct { + RepoPath string `json:"repo_path"` + ReconContext map[string]any `json:"recon_context"` + Depth string `json:"depth"` + AIGate json.RawMessage `json:"ai_gate"` + MaxConcurrentHunters int `json:"max_concurrent_hunters"` + EarlyStopFileThreshold int `json:"early_stop_file_threshold"` +} + +// NewHuntPhaseInput returns the Python keyword defaults. +func NewHuntPhaseInput() HuntPhaseInput { + return HuntPhaseInput{ + Depth: phases.DefaultDepth, + MaxConcurrentHunters: phases.DefaultMaxConcurrentHunters, + EarlyStopFileThreshold: phases.DefaultEarlyStopFileThreshold, + } +} + +// UnmarshalJSON seeds hunt_phase's three keyword defaults before decoding. +func (h *HuntPhaseInput) UnmarshalJSON(b []byte) error { + *h = NewHuntPhaseInput() + type alias HuntPhaseInput + return json.Unmarshal(b, (*alias)(h)) +} + +// ProvePhaseInput ports: +// +// prove_phase(repo_path: str, hunt_result: dict[str, Any], depth: str = "standard", +// max_provers: int | None = None, max_concurrent_provers: int = 3) +// +// max_provers is a pointer: nil is Python's None (use the depth's cap), and a +// present 0 caps the prover fan-out at zero. +type ProvePhaseInput struct { + RepoPath string `json:"repo_path"` + HuntResult map[string]any `json:"hunt_result"` + Depth string `json:"depth"` + MaxProvers *int `json:"max_provers"` + MaxConcurrentProvers int `json:"max_concurrent_provers"` +} + +// NewProvePhaseInput returns the Python keyword defaults. +func NewProvePhaseInput() ProvePhaseInput { + return ProvePhaseInput{ + Depth: phases.DefaultDepth, + MaxConcurrentProvers: phases.DefaultMaxConcurrentProvers, + } +} + +// UnmarshalJSON seeds prove_phase's keyword defaults before decoding. +func (p *ProvePhaseInput) UnmarshalJSON(b []byte) error { + *p = NewProvePhaseInput() + type alias ProvePhaseInput + return json.Unmarshal(b, (*alias)(p)) +} + +// RemediationPhaseInput ports: +// +// remediation_phase(repo_path: str, verified_findings: list[dict[str, Any]], +// max_concurrent_remediations: int = 3) +type RemediationPhaseInput struct { + RepoPath string `json:"repo_path"` + VerifiedFindings []map[string]any `json:"verified_findings"` + MaxConcurrentRemediations int `json:"max_concurrent_remediations"` +} + +// NewRemediationPhaseInput returns the Python keyword defaults. +func NewRemediationPhaseInput() RemediationPhaseInput { + return RemediationPhaseInput{ + MaxConcurrentRemediations: phases.DefaultMaxConcurrentRemediations, + } +} + +// UnmarshalJSON seeds max_concurrent_remediations=3 before decoding. +func (r *RemediationPhaseInput) UnmarshalJSON(b []byte) error { + *r = NewRemediationPhaseInput() + type alias RemediationPhaseInput + return json.Unmarshal(b, (*alias)(r)) +} diff --git a/internal/secaf/reasoners/inputs_test.go b/internal/secaf/reasoners/inputs_test.go new file mode 100644 index 0000000000..bd49360714 --- /dev/null +++ b/internal/secaf/reasoners/inputs_test.go @@ -0,0 +1,150 @@ +package reasoners + +// Tests for inputs.go — the transcription of each reasoner's Python signature. +// +// Validation contract: +// +// - an ABSENT key yields the Python keyword default; +// - a PRESENT key overrides it, even when its value is falsy (0, "", null), +// which is what Python's keyword binding does; +// - max_provers distinguishes "absent" (None -> the depth's cap) from an +// explicit 0. + +import ( + "context" + "testing" + + "github.com/Agent-Field/codeaf/internal/secaf/afx" + "github.com/Agent-Field/codeaf/internal/secaf/phases" +) + +func TestHunterInputDefaults(t *testing.T) { + got, err := afx.Bind[HunterInput](map[string]any{"repo_path": "/r", "depth": "quick"}) + if err != nil { + t.Fatalf("Bind: %v", err) + } + if got.MaxFilesWithoutSignal != 30 { + t.Errorf("max_files_without_signal = %d, want 30", got.MaxFilesWithoutSignal) + } + + got, err = afx.Bind[HunterInput](map[string]any{"max_files_without_signal": 0}) + if err != nil { + t.Fatalf("Bind: %v", err) + } + if got.MaxFilesWithoutSignal != 0 { + t.Errorf("an explicit 0 must override the default, got %d", got.MaxFilesWithoutSignal) + } +} + +func TestReconPhaseInputDefaults(t *testing.T) { + got, err := afx.Bind[ReconPhaseInput](map[string]any{"repo_path": "/r"}) + if err != nil { + t.Fatalf("Bind: %v", err) + } + if got.Depth != phases.DefaultDepth { + t.Errorf("depth = %q, want %q", got.Depth, phases.DefaultDepth) + } + + got, err = afx.Bind[ReconPhaseInput](map[string]any{"repo_path": "/r", "depth": ""}) + if err != nil { + t.Fatalf("Bind: %v", err) + } + if got.Depth != "" { + t.Errorf("an explicit empty depth must override the default, got %q", got.Depth) + } +} + +func TestHuntPhaseInputDefaults(t *testing.T) { + got, err := afx.Bind[HuntPhaseInput](map[string]any{"repo_path": "/r"}) + if err != nil { + t.Fatalf("Bind: %v", err) + } + if got.Depth != phases.DefaultDepth { + t.Errorf("depth = %q, want %q", got.Depth, phases.DefaultDepth) + } + if got.MaxConcurrentHunters != phases.DefaultMaxConcurrentHunters { + t.Errorf("max_concurrent_hunters = %d, want %d", got.MaxConcurrentHunters, phases.DefaultMaxConcurrentHunters) + } + if got.EarlyStopFileThreshold != phases.DefaultEarlyStopFileThreshold { + t.Errorf("early_stop_file_threshold = %d, want %d", got.EarlyStopFileThreshold, phases.DefaultEarlyStopFileThreshold) + } + // `ai_gate: Any | None = None` — absent binds to Python's None. + if gate := phases.NewJSONAIGate(got.AIGate); gate != nil { + t.Errorf("an absent ai_gate must be None, got %v", gate) + } +} + +// TestHuntPhaseInputBindsAIGate: hunt_phase is a REGISTERED reasoner, so a +// control-plane caller can send `ai_gate`. Python binds the raw JSON (the SDK +// passes an `Any`-hinted parameter straight through) and takes the +// `ai_gate is not None` branch; the Go adapter must reach the same branch and +// report the same CPython type name in the note it produces. +func TestHuntPhaseInputBindsAIGate(t *testing.T) { + cases := []struct { + value any + wantType string + }{ + {map[string]any{"model": "x"}, "dict"}, + {[]any{"a"}, "list"}, + {"gate", "str"}, + {1, "int"}, + {1.5, "float"}, + {true, "bool"}, + } + for _, tc := range cases { + got, err := afx.Bind[HuntPhaseInput](map[string]any{"repo_path": "/r", "ai_gate": tc.value}) + if err != nil { + t.Fatalf("Bind(%v): %v", tc.value, err) + } + gate := phases.NewJSONAIGate(got.AIGate) + if gate == nil { + t.Fatalf("ai_gate=%v must be non-None", tc.value) + } + _, err = gate.SelectStrategy(context.Background(), "", "standard", nil) + want := "'" + tc.wantType + "' object has no attribute 'select_strategy'" + if err == nil || err.Error() != want { + t.Errorf("ai_gate=%v -> err %v, want %q", tc.value, err, want) + } + } + + // An explicit null is Python's None. + got, err := afx.Bind[HuntPhaseInput](map[string]any{"repo_path": "/r", "ai_gate": nil}) + if err != nil { + t.Fatalf("Bind: %v", err) + } + if gate := phases.NewJSONAIGate(got.AIGate); gate != nil { + t.Errorf("ai_gate=null must be None, got %v", gate) + } +} + +func TestProvePhaseInputDefaults(t *testing.T) { + got, err := afx.Bind[ProvePhaseInput](map[string]any{"repo_path": "/r"}) + if err != nil { + t.Fatalf("Bind: %v", err) + } + if got.MaxProvers != nil { + t.Errorf("max_provers = %v, want nil (Python None)", *got.MaxProvers) + } + if got.MaxConcurrentProvers != phases.DefaultMaxConcurrentProvers { + t.Errorf("max_concurrent_provers = %d, want %d", got.MaxConcurrentProvers, phases.DefaultMaxConcurrentProvers) + } + + got, err = afx.Bind[ProvePhaseInput](map[string]any{"repo_path": "/r", "max_provers": 0}) + if err != nil { + t.Fatalf("Bind: %v", err) + } + if got.MaxProvers == nil || *got.MaxProvers != 0 { + t.Errorf("an explicit 0 must bind as a pointer to 0, got %v", got.MaxProvers) + } +} + +func TestRemediationPhaseInputDefaults(t *testing.T) { + got, err := afx.Bind[RemediationPhaseInput](map[string]any{"repo_path": "/r"}) + if err != nil { + t.Fatalf("Bind: %v", err) + } + if got.MaxConcurrentRemediations != phases.DefaultMaxConcurrentRemediations { + t.Errorf("max_concurrent_remediations = %d, want %d", + got.MaxConcurrentRemediations, phases.DefaultMaxConcurrentRemediations) + } +} diff --git a/internal/secaf/reasoners/names.go b/internal/secaf/reasoners/names.go new file mode 100644 index 0000000000..a1a130d437 --- /dev/null +++ b/internal/secaf/reasoners/names.go @@ -0,0 +1,111 @@ +package reasoners + +// names.go declares the SEC-AF reasoner surface: the 33 names the Python +// AgentRouter registers (reasoners/recon.py, hunt.py, prove.py, phases.py) plus +// the one top-level `audit` reasoner app.py registers on the agent itself. +// +// The constants exist so a registration, a `.call` target and a test all spell +// a reasoner the same way; Names fixes the ORDER, which DESIGN.md §3 pins. + +// The 33 router reasoner names, in DESIGN.md §3 order. The order is the Python +// import order of reasoners/__init__.py (recon -> hunt -> prove -> phases) and, +// within each module, the order the `@router.reasoner()` decorators appear. +const ( + // --- reasoners/recon.py --- + + NameRunArchitectureMapper = "run_architecture_mapper" + NameRunDependencyAuditor = "run_dependency_auditor" + NameRunConfigScanner = "run_config_scanner" + NameRunDataFlowMapper = "run_data_flow_mapper" + NameRunSecurityContextProfiler = "run_security_context_profiler" + + // --- reasoners/hunt.py --- + + NameRunInjectionHunter = "run_injection_hunter" + NameRunDosHunter = "run_dos_hunter" + NameRunSSRFHunter = "run_ssrf_hunter" + NameRunAuthHunter = "run_auth_hunter" + NameRunXSSHunter = "run_xss_hunter" + NameRunCryptoHunter = "run_crypto_hunter" + NameRunBusinessLogicHunter = "run_business_logic_hunter" + // NameRunLogicBugsHunter is an ALIAS reasoner: its Python body forwards to + // run_business_logic_hunter (the reasoner, not the agent function), so it + // emits no note of its own and its DAG child is indistinguishable from a + // direct run_business_logic_hunter call. + NameRunLogicBugsHunter = "run_logic_bugs_hunter" + NameRunDataExposureHunter = "run_data_exposure_hunter" + NameRunSupplyChainHunter = "run_supply_chain_hunter" + NameRunConfigSecretsHunter = "run_config_secrets_hunter" + NameRunAPISecurityHunter = "run_api_security_hunter" + NameRunDeduplicator = "run_deduplicator" + + // --- reasoners/prove.py --- + + NameRunDepReachability = "run_dep_reachability" + NameRunVerifier = "run_verifier" + NameRunTracer = "run_tracer" + NameRunSanitizationAnalyzer = "run_sanitization_analyzer" + NameRunExploitHypothesizer = "run_exploit_hypothesizer" + NameRunVerdictAgent = "run_verdict_agent" + NameRunRemediation = "run_remediation" + NameRunRemediationAgent = "run_remediation_agent" + NameRunDastVerifier = "run_dast_verifier" + NameRunCrossServiceAnalyzer = "run_cross_service_analyzer" + + // --- reasoners/phases.py --- + + // NameRunCWEExpansion is REGISTERED but never `.call`ed: hunt_phase invokes + // expand_cwes_for_hunt in process (an `.ai()` call), so this reasoner + // contributes no DAG node. See DESIGN.md §3. + NameRunCWEExpansion = "run_cwe_expansion" + + NameReconPhase = "recon_phase" + NameHuntPhase = "hunt_phase" + NameProvePhase = "prove_phase" + NameRemediationPhase = "remediation_phase" +) + +// NameAudit is the ONE externally driven reasoner. Python registered it with +// `@app.reasoner()` on the Agent, not on the AgentRouter, so it is not part of +// the registry; internal/secaf/audit owns the entry, and the name is still how +// its input specification is looked up (ValidateHandlerInput). +const NameAudit = "audit" + +// Names is the canonical ordered list of the 33 router reasoner names — +// DESIGN.md §3's list verbatim. RegisterAll registers exactly these, in exactly +// this order, and returns the same slice content as its bookkeeping. +var Names = []string{ + NameRunArchitectureMapper, + NameRunDependencyAuditor, + NameRunConfigScanner, + NameRunDataFlowMapper, + NameRunSecurityContextProfiler, + NameRunInjectionHunter, + NameRunDosHunter, + NameRunSSRFHunter, + NameRunAuthHunter, + NameRunXSSHunter, + NameRunCryptoHunter, + NameRunBusinessLogicHunter, + NameRunLogicBugsHunter, + NameRunDataExposureHunter, + NameRunSupplyChainHunter, + NameRunConfigSecretsHunter, + NameRunAPISecurityHunter, + NameRunDeduplicator, + NameRunDepReachability, + NameRunVerifier, + NameRunTracer, + NameRunSanitizationAnalyzer, + NameRunExploitHypothesizer, + NameRunVerdictAgent, + NameRunRemediation, + NameRunRemediationAgent, + NameRunDastVerifier, + NameRunCrossServiceAnalyzer, + NameRunCWEExpansion, + NameReconPhase, + NameHuntPhase, + NameProvePhase, + NameRemediationPhase, +} diff --git a/internal/secaf/reasoners/phases.go b/internal/secaf/reasoners/phases.go new file mode 100644 index 0000000000..f8c4830e79 --- /dev/null +++ b/internal/secaf/reasoners/phases.go @@ -0,0 +1,76 @@ +package reasoners + +import ( + "context" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/phases" +) + +// phases.go ports the five reasoner ENTRY POINTS declared in +// src/sec_af/reasoners/phases.py. The bodies live in internal/phases; these +// adapters only bind the request and thread the node id. +// +// nodeID is a parameter rather than a package variable so a test can pin the +// `.call` targets. RegisterAll reads phases.NodeID() ONCE and closes over the +// result — Python's module-level `NODE_ID = os.getenv("NODE_ID", "sec-af")`, +// which inside codeaf is the constant "sec-af". + +// RunCWEExpansion ports `run_cwe_expansion(recon_summary, strategies)` +// (reasoners/phases.py:139): +// +// additional = await expand_cwes_for_hunt(recon_summary, strategies) +// return {"additional_cwes": additional} +// +// Python parity: no note, and no error path — expand_cwes_for_hunt swallows +// every failure and returns []. It is registered but never `.call`ed +// (hunt_phase runs expand_cwes_for_hunt in process), so it draws no DAG node. +func RunCWEExpansion(ctx context.Context, app appx.App, in CWEExpansionInput) (map[string]any, error) { + return phases.RunCWEExpansion(ctx, app, in.ReconSummary, in.Strategies), nil +} + +// ReconPhase ports the `recon_phase` reasoner (reasoners/phases.py:152). +func ReconPhase(ctx context.Context, app appx.App, nodeID string, in ReconPhaseInput) (map[string]any, error) { + return phases.ReconPhase(ctx, app, nodeID, in.RepoPath, in.Depth) +} + +// HuntPhase ports the `hunt_phase` reasoner (reasoners/phases.py:239). +// +// Python parity: `ai_gate` is always None on the live path (app.py's `.call` +// omits it), so phases.NewJSONAIGate yields nil there. A direct caller that +// supplies one takes the `ai_gate is not None` branch and gets the +// "AI gate failed: ..." note — see HuntPhaseInput. +func HuntPhase(ctx context.Context, app appx.App, nodeID string, in HuntPhaseInput) (map[string]any, error) { + return phases.HuntPhase( + ctx, app, nodeID, + in.RepoPath, + in.ReconContext, + in.Depth, + phases.NewJSONAIGate(in.AIGate), + in.MaxConcurrentHunters, + in.EarlyStopFileThreshold, + ) +} + +// ProvePhase ports the `prove_phase` reasoner (reasoners/phases.py:381). +func ProvePhase(ctx context.Context, app appx.App, nodeID string, in ProvePhaseInput) (map[string]any, error) { + return phases.ProvePhase( + ctx, app, nodeID, + in.RepoPath, + in.HuntResult, + in.Depth, + in.MaxProvers, + in.MaxConcurrentProvers, + ) +} + +// RemediationPhase ports the `remediation_phase` reasoner +// (reasoners/phases.py:487). +func RemediationPhase(ctx context.Context, app appx.App, nodeID string, in RemediationPhaseInput) (map[string]any, error) { + return phases.RemediationPhase( + ctx, app, nodeID, + in.RepoPath, + in.VerifiedFindings, + in.MaxConcurrentRemediations, + ) +} diff --git a/internal/secaf/reasoners/phases_test.go b/internal/secaf/reasoners/phases_test.go new file mode 100644 index 0000000000..a0370db6b4 --- /dev/null +++ b/internal/secaf/reasoners/phases_test.go @@ -0,0 +1,187 @@ +package reasoners + +// Tests for the five reasoner entry points of src/sec_af/reasoners/phases.py +// as seen from the ADAPTER layer: input binding, node-id threading and the +// `.call` targets the phase draws. The phase bodies themselves are covered in +// internal/phases. +// +// Validation contract: +// +// - the four *_phase adapters prefix every `.call` target with the node id, +// the constant "sec-af" — an ambient NODE_ID no longer steers it; +// - hunt_phase is invoked with ai_gate=None, so the AI strategy gate is never +// consulted and the default strategy list is used; +// - run_cwe_expansion emits no note, never fails, and returns +// {"additional_cwes": [...]} — [] when the AI gate is unavailable; +// - the keyword defaults of every phase signature survive a request body that +// omits them. + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" +) + +func TestRunCWEExpansionSwallowsGateFailure(t *testing.T) { + fake := &appx.Fake{} // no AIFn: every .ai() call fails + got, err := RunCWEExpansion(context.Background(), fake, CWEExpansionInput{ + ReconSummary: "python (django), 100 LOC", + Strategies: []string{"injection", "xss"}, + }) + if err != nil { + t.Fatalf("RunCWEExpansion must never fail: %v", err) + } + cwes, ok := got["additional_cwes"].([]string) + if !ok { + t.Fatalf("additional_cwes = %#v, want []string", got["additional_cwes"]) + } + if len(cwes) != 0 { + t.Errorf("additional_cwes = %v, want [] on a failed gate", cwes) + } + if len(fake.Notes) != 0 { + t.Errorf("run_cwe_expansion must emit no note, got %v", fake.NoteMessages()) + } +} + +// TestReconPhaseCallTargetsCarryNodeID pins the DAG edges recon_phase draws and +// the node-id prefix, through the REGISTERED handler (so the nodeID that +// RegisterAll captured is what is exercised). +func TestReconPhaseCallTargetsCarryNodeID(t *testing.T) { + // The node used to read NODE_ID; setting it now must change nothing. + t.Setenv("NODE_ID", "sec-af-go") + + fake := &appx.Fake{ + CallFn: func(_ context.Context, target string, _ map[string]any) (map[string]any, error) { + switch { + case strings.HasSuffix(target, ".run_security_context_profiler"): + return map[string]any{"auth_model": "jwt", "auth_details": "bearer"}, nil + default: + return map[string]any{}, nil + } + }, + } + + out := executeRegistered(t, fake, "recon_phase", map[string]any{ + "repo_path": t.TempDir(), + "depth": "quick", + }) + if out == nil { + t.Fatal("recon_phase returned no payload") + } + + // depth=quick: the three unconditional children only. + want := []string{ + "sec-af.run_architecture_mapper", + "sec-af.run_dependency_auditor", + "sec-af.run_config_scanner", + } + got := fake.CallTargets() + if len(got) != len(want) { + t.Fatalf("call targets = %v, want the %d quick-depth children", got, len(want)) + } + seen := map[string]bool{} + for _, target := range got { + seen[target] = true + } + for _, target := range want { + if !seen[target] { + t.Errorf("missing DAG edge %q (got %v)", target, got) + } + } +} + +// TestPhaseAdaptersBindKeywordDefaults drives each phase through its registered +// handler with a body that omits every optional key, and asserts the Python +// default reached the phase — observed through the kwargs of the `.call` the +// phase makes. +func TestPhaseAdaptersBindKeywordDefaults(t *testing.T) { + t.Run("recon_phase depth defaults to standard", func(t *testing.T) { + fake := &appx.Fake{ + CallFn: func(_ context.Context, target string, _ map[string]any) (map[string]any, error) { + if strings.HasSuffix(target, ".run_security_context_profiler") { + return map[string]any{"auth_model": "jwt", "auth_details": ""}, nil + } + return map[string]any{}, nil + }, + } + executeRegistered(t, fake, "recon_phase", map[string]any{"repo_path": t.TempDir()}) + + // "standard" is not "quick", so the two deep-recon children run. + if len(fake.Calls) != 5 { + t.Fatalf("calls = %v, want 5 (depth defaulted to standard)", fake.CallTargets()) + } + }) + + t.Run("prove_phase max_provers defaults to None", func(t *testing.T) { + fake := &appx.Fake{ + CallFn: func(context.Context, string, map[string]any) (map[string]any, error) { + return map[string]any{}, nil + }, + } + out := executeRegistered(t, fake, "prove_phase", map[string]any{ + "repo_path": t.TempDir(), + "hunt_result": map[string]any{}, + }) + // No findings -> no verifier calls, and the phase reports the empty shape. + if len(fake.Calls) != 0 { + t.Errorf("calls = %v, want none for an empty hunt result", fake.CallTargets()) + } + for _, key := range []string{"verified", "total_selected", "total_findings", "not_verified", "drop_summary"} { + if _, ok := out[key]; !ok { + t.Errorf("prove_phase result is missing %q", key) + } + } + }) + + t.Run("remediation_phase with nothing to remediate", func(t *testing.T) { + fake := &appx.Fake{} + out := executeRegistered(t, fake, "remediation_phase", map[string]any{ + "repo_path": t.TempDir(), + "verified_findings": []any{}, + }) + if _, ok := out["verified"]; !ok { + t.Error("remediation_phase result is missing \"verified\"") + } + if len(fake.Notes) != 2 { + t.Fatalf("notes = %v, want the starting + no-op pair", fake.NoteMessages()) + } + if fake.Notes[1].Message != "No findings need remediation" { + t.Errorf("second note = %q, want the no-op note", fake.Notes[1].Message) + } + }) +} + +// executeRegistered runs one reasoner through the registration path — the +// validation layer, the typed bind and the phase body — as an in-process +// `.call` does. +func executeRegistered(t *testing.T, app appx.App, name string, input map[string]any) map[string]any { + t.Helper() + + reg := NewRegistry() + RegisterAll(reg, app) + h, ok := reg.Lookup(name) + if !ok { + t.Fatalf("reasoner %q is not registered", name) + } + + got, err := h(context.Background(), input) + if err != nil { + var inputErr *InputError + if errors.As(err, &inputErr) { + t.Fatalf("%s: %d %s", name, inputErr.StatusCode, inputErr.Message) + } + t.Fatalf("%s: %v", name, err) + } + + out, ok := got.(map[string]any) + if !ok { + // Every phase returns a map. + b, _ := json.Marshal(got) + t.Fatalf("%s returned %T: %s", name, got, b) + } + return out +} diff --git a/internal/secaf/reasoners/prove.go b/internal/secaf/reasoners/prove.go new file mode 100644 index 0000000000..e8e33dbb0b --- /dev/null +++ b/internal/secaf/reasoners/prove.go @@ -0,0 +1,291 @@ +package reasoners + +import ( + "context" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/afx" + proveagent "github.com/Agent-Field/codeaf/internal/secaf/agents/prove" + remediationagent "github.com/Agent-Field/codeaf/internal/secaf/agents/remediation" + "github.com/Agent-Field/codeaf/internal/secaf/phases" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" + "github.com/Agent-Field/codeaf/internal/secaf/scoring" +) + +// prove.go ports src/sec_af/reasoners/prove.py — the ten PROVE-side adapters +// plus `_coerce_verifier_finding`, the one non-trivial helper in the file. + +// CoerceVerifierFinding ports `_coerce_verifier_finding(finding)` +// (reasoners/prove.py:28): +// +// try: +// return RawFinding.model_validate(finding) +// except Exception: +// view = FindingForVerifier.model_validate(finding) +// return RawFinding( +// id=view.id, +// hunter_strategy="phase_boundary_projection", +// title=view.title, +// description=view.data_flow_summary or view.title, +// finding_type=FindingType.SAST, +// cwe_id=view.cwe_id, +// cwe_name=view.cwe_id, +// file_path=view.file_path, +// start_line=view.start_line, +// end_line=view.end_line, +// function_name=view.function_name, +// code_snippet=view.code_snippet, +// estimated_severity=apply_cwe_severity_floor(view.cwe_id, Severity.MEDIUM), +// confidence=Confidence.MEDIUM, +// fingerprint=view.id, +// ) +// +// This is the reasoner boundary that makes prove_phase work: prove_phase sends +// `finding.for_verifier().model_dump()`, a ten-field FindingForVerifier +// projection, which cannot satisfy RawFinding's twelve required fields — so the +// fallback branch is the LIVE path, not an edge case. See validate.go for why +// the required-field check has to be real. +// +// Python parity in the fallback: +// +// - `cwe_name` is set to the CWE **id**, not a name. Reproduced. +// - `description` uses `view.data_flow_summary or view.title` — Python +// truthiness, so an empty summary falls back to the title. +// - the reconstructed RawFinding's `fingerprint` is the view's id, and its +// `id` is the view's id too, so the two match (a fresh RawFinding would +// have minted two different uuid4s). +// - `related_files` is not passed and keeps `default_factory=list` -> []; +// `data_flow` keeps None. NewRawFinding() seeds exactly that (and then the +// explicit ID/Fingerprint assignments overwrite its minted uuid4s). +// - the `except Exception` is BROAD: any failure to build a RawFinding — not +// just a missing field — takes the fallback. A payload that is neither +// model still fails, on the FindingForVerifier validation. +func CoerceVerifierFinding(finding map[string]any) (schemas.RawFinding, error) { + if raw, err := bindRawFinding(finding); err == nil { + return raw, nil + } + + view, err := bindFindingForVerifier(finding) + if err != nil { + return schemas.RawFinding{}, err + } + + description := view.DataFlowSummary + if description == "" { + description = view.Title + } + + out := schemas.NewRawFinding() + out.ID = view.ID + out.HunterStrategy = "phase_boundary_projection" + out.Title = view.Title + out.Description = description + out.FindingType = schemas.FindingTypeSast + out.CweID = view.CweID + out.CweName = view.CweID + out.FilePath = view.FilePath + out.StartLine = view.StartLine + out.EndLine = view.EndLine + out.FunctionName = view.FunctionName + out.CodeSnippet = view.CodeSnippet + out.EstimatedSeverity = scoring.ApplyCWESeverityFloor(view.CweID, schemas.SeverityMedium) + out.Confidence = schemas.ConfidenceMedium + out.Fingerprint = view.ID + return out, nil +} + +// RunDepReachability ports `run_dep_reachability(repo_path, finding, depth)` +// (reasoners/prove.py:52). +// +// Python parity: `finding` is forwarded to the agent as the RAW DICT — this is +// the one prove reasoner that does not build a model first, because +// agents/prove/dep_reachability.py renders the dict into its prompt directly. +func RunDepReachability(ctx context.Context, app appx.App, in FindingDepthInput) (map[string]any, error) { + app.Note(ctx, "Dependency reachability analyzer starting", "prove", "dep-reachability") + result, err := proveagent.RunDepReachability(ctx, app, in.RepoPath, in.Finding, in.Depth) + if err != nil { + return nil, err + } + return afx.ToMap(result) +} + +// RunVerifier ports `run_verifier(repo_path, finding, depth)` +// (reasoners/prove.py:60) — the reasoner prove_phase fans out over. +func RunVerifier(ctx context.Context, app appx.App, in FindingDepthInput) (map[string]any, error) { + app.Note(ctx, "Verifier starting", "prove", "verifier") + finding, err := CoerceVerifierFinding(in.Finding) + if err != nil { + return nil, err + } + result, err := proveagent.RunVerifier(ctx, app, in.RepoPath, finding, in.Depth) + if err != nil { + return nil, err + } + return afx.ToMap(result) +} + +// RunTracer ports `run_tracer(repo_path, finding, depth)` +// (reasoners/prove.py:69). +// +// Python parity: `RawFinding(**finding)` here is the STRICT constructor, not +// _coerce_verifier_finding — a FindingForVerifier projection sent to this +// reasoner raises rather than being adapted. +func RunTracer(ctx context.Context, app appx.App, in FindingDepthInput) (map[string]any, error) { + app.Note(ctx, "Tracer starting", "prove", "tracer") + finding, err := bindRawFinding(in.Finding) + if err != nil { + return nil, err + } + result, err := proveagent.RunTracer(ctx, app, in.RepoPath, finding, in.Depth) + if err != nil { + return nil, err + } + return afx.ToMap(result) +} + +// RunSanitizationAnalyzer ports +// `run_sanitization_analyzer(repo_path, finding, data_flow, depth)` +// (reasoners/prove.py:77). +func RunSanitizationAnalyzer(ctx context.Context, app appx.App, in SanitizationInput) (map[string]any, error) { + app.Note(ctx, "Sanitization analyzer starting", "prove", "sanitization") + finding, err := bindRawFinding(in.Finding) + if err != nil { + return nil, err + } + flow, err := bindDataFlowTrace(in.DataFlow) + if err != nil { + return nil, err + } + result, err := proveagent.RunSanitizationAnalyzer(ctx, app, in.RepoPath, finding, flow, in.Depth) + if err != nil { + return nil, err + } + return afx.ToMap(result) +} + +// RunExploitHypothesizer ports +// `run_exploit_hypothesizer(repo_path, finding, data_flow, sanitization, depth)` +// (reasoners/prove.py:91). +func RunExploitHypothesizer(ctx context.Context, app appx.App, in ExploitInput) (map[string]any, error) { + app.Note(ctx, "Exploit hypothesizer starting", "prove", "exploit") + finding, err := bindRawFinding(in.Finding) + if err != nil { + return nil, err + } + flow, err := bindDataFlowTrace(in.DataFlow) + if err != nil { + return nil, err + } + sanitization, err := bindSanitizationResult(in.Sanitization) + if err != nil { + return nil, err + } + result, err := proveagent.RunExploitHypothesizer(ctx, app, in.RepoPath, finding, flow, sanitization, in.Depth) + if err != nil { + return nil, err + } + return afx.ToMap(result) +} + +// RunVerdictAgent ports +// `run_verdict_agent(finding, data_flow, sanitization, exploit)` +// (reasoners/prove.py:109). +// +// Python parity: the reasoner has no repo_path parameter and passes the literal +// "." to the agent, which never reads it. +func RunVerdictAgent(ctx context.Context, app appx.App, in VerdictInput) (map[string]any, error) { + app.Note(ctx, "Verdict agent starting", "prove", "verdict") + finding, err := bindRawFinding(in.Finding) + if err != nil { + return nil, err + } + flow, err := bindDataFlowTrace(in.DataFlow) + if err != nil { + return nil, err + } + sanitization, err := bindSanitizationResult(in.Sanitization) + if err != nil { + return nil, err + } + exploit, err := bindExploitHypothesis(in.Exploit) + if err != nil { + return nil, err + } + result, err := proveagent.RunVerdictAgent(ctx, app, ".", finding, flow, sanitization, exploit) + if err != nil { + return nil, err + } + return afx.ToMap(result) +} + +// RunRemediation ports `run_remediation(repo_path, finding)` +// (reasoners/prove.py:131) — the reasoner remediation_phase fans out over. +// +// finding_model = VerifiedFinding(**finding) +// result = await generate_remediation(router, repo_path, finding_model) +// return RemediationSuggestion.model_validate(result).model_dump() +// +// Python parity: the note is "Remediation agent starting" — the same string +// run_remediation_agent uses, tags included. +// +// The trailing `RemediationSuggestion.model_validate(result)` re-validates a +// value that generate_remediation already returned as a RemediationSuggestion. +// It is a no-op round trip; GenerateRemediation returns the typed value here, +// so ToMap is the whole of it. +func RunRemediation(ctx context.Context, app appx.App, in RemediationInput) (map[string]any, error) { + app.Note(ctx, "Remediation agent starting", "prove", "remediation") + finding, err := phases.BindVerifiedFinding(in.Finding) + if err != nil { + return nil, err + } + result, err := remediationagent.GenerateRemediation(ctx, app, in.RepoPath, finding) + if err != nil { + return nil, err + } + return afx.ToMap(result) +} + +// RunRemediationAgent ports +// `run_remediation_agent(repo_path, finding, verdict, rationale)` +// (reasoners/prove.py:145). `finding` is a RawFinding here, unlike +// run_remediation's VerifiedFinding. +func RunRemediationAgent(ctx context.Context, app appx.App, in RemediationAgentInput) (map[string]any, error) { + app.Note(ctx, "Remediation agent starting", "prove", "remediation") + finding, err := bindRawFinding(in.Finding) + if err != nil { + return nil, err + } + result, err := remediationagent.RunRemediation(ctx, app, in.RepoPath, finding, in.Verdict, in.Rationale) + if err != nil { + return nil, err + } + return afx.ToMap(result) +} + +// RunDastVerifier ports +// `run_dast_verifier(repo_path, finding, exploit_payload, depth)` +// (reasoners/prove.py:158). +func RunDastVerifier(ctx context.Context, app appx.App, in DastVerifierInput) (map[string]any, error) { + app.Note(ctx, "DAST verifier starting", "prove", "dast") + finding, err := bindRawFinding(in.Finding) + if err != nil { + return nil, err + } + result, err := proveagent.RunDastVerifier(ctx, app, in.RepoPath, finding, in.ExploitPayload, in.Depth) + if err != nil { + return nil, err + } + return afx.ToMap(result) +} + +// RunCrossServiceAnalyzer ports +// `run_cross_service_analyzer(repo_path, services, findings_summary, depth)` +// (reasoners/prove.py:169). +func RunCrossServiceAnalyzer(ctx context.Context, app appx.App, in CrossServiceInput) (map[string]any, error) { + app.Note(ctx, "Cross-service analyzer starting", "prove", "cross-service") + result, err := proveagent.RunCrossServiceAnalyzer(ctx, app, in.RepoPath, in.Services, in.FindingsSummary, in.Depth) + if err != nil { + return nil, err + } + return afx.ToMap(result) +} diff --git a/internal/secaf/reasoners/prove_test.go b/internal/secaf/reasoners/prove_test.go new file mode 100644 index 0000000000..dc536363ff --- /dev/null +++ b/internal/secaf/reasoners/prove_test.go @@ -0,0 +1,263 @@ +package reasoners + +// Tests for src/sec_af/reasoners/prove.py. +// +// Validation contract: +// +// - each adapter emits its own note (message + tags); run_remediation and +// run_remediation_agent deliberately share the SAME note; +// - _coerce_verifier_finding returns the payload as a RawFinding when it +// validates, and otherwise projects a FindingForVerifier into a RawFinding +// with hunter_strategy="phase_boundary_projection", cwe_name = the CWE ID, +// description falling back to the title, the CWE severity floor applied +// over MEDIUM, confidence MEDIUM and fingerprint = the view's id; +// - the fallback is the LIVE path: prove_phase sends +// finding.for_verifier().model_dump(), which cannot satisfy RawFinding; +// - the strict adapters (tracer, sanitization, exploit, verdict, dast, +// remediation_agent) reject a payload that is not their model. + +import ( + "context" + "testing" + + "github.com/Agent-Field/codeaf/internal/secaf/schemas" + "github.com/Agent-Field/codeaf/internal/secaf/scoring" +) + +func TestCoerceVerifierFindingKeepsAValidRawFinding(t *testing.T) { + got, err := CoerceVerifierFinding(rawFindingPayload()) + if err != nil { + t.Fatalf("CoerceVerifierFinding: %v", err) + } + if got.HunterStrategy != "injection" { + t.Errorf("hunter_strategy = %q, want the payload's own value", got.HunterStrategy) + } + if got.CweName != "SQL Injection" { + t.Errorf("cwe_name = %q, want the payload's own value", got.CweName) + } +} + +func TestCoerceVerifierFindingProjectsTheVerifierView(t *testing.T) { + view := verifierProjection(t) + + got, err := CoerceVerifierFinding(view) + if err != nil { + t.Fatalf("CoerceVerifierFinding: %v", err) + } + + if got.HunterStrategy != "phase_boundary_projection" { + t.Errorf("hunter_strategy = %q, want phase_boundary_projection", got.HunterStrategy) + } + if got.CweName != got.CweID { + t.Errorf("cwe_name = %q, want it to mirror cwe_id %q (Python parity)", got.CweName, got.CweID) + } + if got.FindingType != schemas.FindingTypeSast { + t.Errorf("finding_type = %q, want sast", got.FindingType) + } + if got.Confidence != schemas.ConfidenceMedium { + t.Errorf("confidence = %q, want medium", got.Confidence) + } + wantSeverity := scoring.ApplyCWESeverityFloor(got.CweID, schemas.SeverityMedium) + if got.EstimatedSeverity != wantSeverity { + t.Errorf("estimated_severity = %q, want %q (CWE floor over MEDIUM)", got.EstimatedSeverity, wantSeverity) + } + if got.Fingerprint != view["id"] { + t.Errorf("fingerprint = %q, want the view id %v", got.Fingerprint, view["id"]) + } + if got.ID != view["id"] { + t.Errorf("id = %q, want the view id %v", got.ID, view["id"]) + } + if got.RelatedFiles == nil { + t.Error("related_files = nil, want [] (default_factory=list)") + } +} + +// TestCoerceVerifierFindingDescriptionFallback pins +// `view.data_flow_summary or view.title` — Python truthiness, so an EMPTY +// summary falls back to the title. +func TestCoerceVerifierFindingDescriptionFallback(t *testing.T) { + view := verifierProjection(t) + + view["data_flow_summary"] = "" + got, err := CoerceVerifierFinding(view) + if err != nil { + t.Fatalf("CoerceVerifierFinding: %v", err) + } + if got.Description != got.Title { + t.Errorf("description = %q, want the title %q", got.Description, got.Title) + } + + view["data_flow_summary"] = "src -> sink" + got, err = CoerceVerifierFinding(view) + if err != nil { + t.Fatalf("CoerceVerifierFinding: %v", err) + } + if got.Description != "src -> sink" { + t.Errorf("description = %q, want the data flow summary", got.Description) + } +} + +func TestCoerceVerifierFindingRejectsNeitherModel(t *testing.T) { + if _, err := CoerceVerifierFinding(map[string]any{"title": "malformed"}); err == nil { + t.Fatal("want an error: the payload is neither a RawFinding nor a FindingForVerifier") + } +} + +func TestProveAdapterNotes(t *testing.T) { + repo := t.TempDir() + finding := rawFindingPayload() + + t.Run("run_dep_reachability", func(t *testing.T) { + fake := newScanFake() + if _, err := RunDepReachability(context.Background(), fake, FindingDepthInput{ + RepoPath: repo, Finding: finding, Depth: "standard", + }); err != nil { + t.Fatalf("RunDepReachability: %v", err) + } + assertNote(t, fake, "Dependency reachability analyzer starting", "prove", "dep-reachability") + }) + + t.Run("run_tracer", func(t *testing.T) { + fake := newScanFake() + if _, err := RunTracer(context.Background(), fake, FindingDepthInput{ + RepoPath: repo, Finding: finding, Depth: "standard", + }); err != nil { + t.Fatalf("RunTracer: %v", err) + } + assertNote(t, fake, "Tracer starting", "prove", "tracer") + }) + + t.Run("run_sanitization_analyzer", func(t *testing.T) { + fake := newScanFake() + if _, err := RunSanitizationAnalyzer(context.Background(), fake, SanitizationInput{ + RepoPath: repo, Finding: finding, DataFlow: dataFlowPayload(), Depth: "standard", + }); err != nil { + t.Fatalf("RunSanitizationAnalyzer: %v", err) + } + assertNote(t, fake, "Sanitization analyzer starting", "prove", "sanitization") + }) + + t.Run("run_exploit_hypothesizer", func(t *testing.T) { + fake := newScanFake() + if _, err := RunExploitHypothesizer(context.Background(), fake, ExploitInput{ + RepoPath: repo, Finding: finding, DataFlow: dataFlowPayload(), + Sanitization: sanitizationPayload(), Depth: "standard", + }); err != nil { + t.Fatalf("RunExploitHypothesizer: %v", err) + } + assertNote(t, fake, "Exploit hypothesizer starting", "prove", "exploit") + }) + + t.Run("run_dast_verifier", func(t *testing.T) { + fake := newScanFake() + if _, err := RunDastVerifier(context.Background(), fake, DastVerifierInput{ + RepoPath: repo, Finding: finding, ExploitPayload: "' OR 1=1", Depth: "standard", + }); err != nil { + t.Fatalf("RunDastVerifier: %v", err) + } + assertNote(t, fake, "DAST verifier starting", "prove", "dast") + }) + + t.Run("run_cross_service_analyzer", func(t *testing.T) { + fake := newScanFake() + if _, err := RunCrossServiceAnalyzer(context.Background(), fake, CrossServiceInput{ + RepoPath: repo, Services: []string{"api", "worker"}, FindingsSummary: "s", Depth: "standard", + }); err != nil { + t.Fatalf("RunCrossServiceAnalyzer: %v", err) + } + assertNote(t, fake, "Cross-service analyzer starting", "prove", "cross-service") + }) + + t.Run("run_remediation_agent", func(t *testing.T) { + fake := newScanFake() + if _, err := RunRemediationAgent(context.Background(), fake, RemediationAgentInput{ + RepoPath: repo, Finding: finding, Verdict: "confirmed", Rationale: "reachable", + }); err != nil { + t.Fatalf("RunRemediationAgent: %v", err) + } + // Python parity: the same message and tags run_remediation uses. + assertNote(t, fake, "Remediation agent starting", "prove", "remediation") + }) +} + +// TestRunRemediationNote pins that run_remediation shares run_remediation_agent's +// note verbatim, and that the note fires before the VerifiedFinding bind. +func TestRunRemediationNote(t *testing.T) { + fake := newScanFake() + _, err := RunRemediation(context.Background(), fake, RemediationInput{ + RepoPath: t.TempDir(), + Finding: map[string]any{"title": "malformed"}, + }) + if err == nil { + t.Fatal("want a VerifiedFinding validation error") + } + assertNote(t, fake, "Remediation agent starting", "prove", "remediation") +} + +// TestRunVerdictAgentNote pins the verdict adapter's note. +func TestRunVerdictAgentNote(t *testing.T) { + fake := newScanFake() + _, err := RunVerdictAgent(context.Background(), fake, VerdictInput{ + Finding: map[string]any{"title": "malformed"}, + DataFlow: dataFlowPayload(), + Sanitization: sanitizationPayload(), + Exploit: exploitPayload(), + }) + if err == nil { + t.Fatal("want a RawFinding validation error") + } + assertNote(t, fake, "Verdict agent starting", "prove", "verdict") +} + +// TestRunVerifierNoteFiresBeforeCoercion pins the ordering: the note is emitted +// first, so a finding that coerces to nothing still leaves its trace. +func TestRunVerifierNoteFiresBeforeCoercion(t *testing.T) { + fake := newScanFake() + _, err := RunVerifier(context.Background(), fake, FindingDepthInput{ + RepoPath: t.TempDir(), + Finding: map[string]any{"title": "malformed"}, + Depth: "standard", + }) + if err == nil { + t.Fatal("want an error for a payload that is neither model") + } + assertNote(t, fake, "Verifier starting", "prove", "verifier") +} + +// TestStrictAdaptersRejectMalformedPayloads pins that the adapters using the +// STRICT constructors (not _coerce_verifier_finding) fail on a verifier +// projection, which is what Python's `RawFinding(**finding)` does. +func TestStrictAdaptersRejectMalformedPayloads(t *testing.T) { + view := verifierProjection(t) + repo := t.TempDir() + + if _, err := RunTracer(context.Background(), newScanFake(), FindingDepthInput{ + RepoPath: repo, Finding: view, Depth: "standard", + }); err == nil { + t.Error("run_tracer must reject a FindingForVerifier projection") + } + + if _, err := RunVerdictAgent(context.Background(), newScanFake(), VerdictInput{ + Finding: view, DataFlow: dataFlowPayload(), + Sanitization: sanitizationPayload(), Exploit: exploitPayload(), + }); err == nil { + t.Error("run_verdict_agent must reject a FindingForVerifier projection") + } +} + +func dataFlowPayload() map[string]any { + return map[string]any{ + "source": "request.args", + "sink": "cursor.execute", + "steps": []any{"a", "b"}, + "sink_reached": true, + } +} + +func sanitizationPayload() map[string]any { + return map[string]any{"found": false} +} + +func exploitPayload() map[string]any { + return map[string]any{"hypothesis": "h", "expected_outcome": "o"} +} diff --git a/internal/secaf/reasoners/recon.go b/internal/secaf/reasoners/recon.go new file mode 100644 index 0000000000..1239443901 --- /dev/null +++ b/internal/secaf/reasoners/recon.go @@ -0,0 +1,96 @@ +package reasoners + +import ( + "context" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/afx" + reconagent "github.com/Agent-Field/codeaf/internal/secaf/agents/recon" +) + +// recon.go ports src/sec_af/reasoners/recon.py — the five RECON reasoner +// adapters. Each is three statements in Python: +// +// router.note(" starting", tags=[...]) +// result = await _run_(router, ...) +// return result.model_dump() +// +// The note fires BEFORE the agent function, so a failing mapper still leaves +// its "starting" note in the execution log; that ordering is reproduced. + +// RunArchitectureMapper ports `run_architecture_mapper(repo_path)` +// (reasoners/recon.py:16). +func RunArchitectureMapper(ctx context.Context, app appx.App, in RepoPathInput) (map[string]any, error) { + app.Note(ctx, "Architecture mapper starting", "recon", "architecture") + result, err := reconagent.RunArchitectureMapper(ctx, app, in.RepoPath) + if err != nil { + return nil, err + } + return afx.ToMap(result) +} + +// RunDependencyAuditor ports `run_dependency_auditor(repo_path)` +// (reasoners/recon.py:24). +func RunDependencyAuditor(ctx context.Context, app appx.App, in RepoPathInput) (map[string]any, error) { + app.Note(ctx, "Dependency auditor starting", "recon", "dependencies") + result, err := reconagent.RunDependencyAuditor(ctx, app, in.RepoPath) + if err != nil { + return nil, err + } + return afx.ToMap(result) +} + +// RunConfigScanner ports `run_config_scanner(repo_path)` +// (reasoners/recon.py:32). +func RunConfigScanner(ctx context.Context, app appx.App, in RepoPathInput) (map[string]any, error) { + app.Note(ctx, "Config scanner starting", "recon", "config") + result, err := reconagent.RunConfigScanner(ctx, app, in.RepoPath) + if err != nil { + return nil, err + } + return afx.ToMap(result) +} + +// RunDataFlowMapper ports `run_data_flow_mapper(repo_path, architecture)` +// (reasoners/recon.py:40): +// +// architecture_model = ArchitectureMap(**architecture) +// +// ArchitectureMap has no required field of its OWN, so an empty (or absent) +// `architecture` binds to the pydantic defaults rather than raising — but each +// of its five lists holds a model that DOES have required fields (Module +// name/path/language, EntryPoint kind/identifier/file_path/line, APIEndpoint +// method/path/handler/file_path/line, ...), and pydantic validates those. +// `ArchitectureMap(**{"modules": [{"name": "x"}]})` raises 2 errors on the +// pinned interpreter, so this is bindArchitectureMap, not a bare afx.Bind: the +// reasoner is registered on the router and a control-plane caller can send that +// payload directly. +func RunDataFlowMapper(ctx context.Context, app appx.App, in ArchitectureInput) (map[string]any, error) { + app.Note(ctx, "Data flow mapper starting", "recon", "data-flow") + architecture, err := bindArchitectureMap(in.Architecture) + if err != nil { + return nil, err + } + result, err := reconagent.RunDataFlowMapper(ctx, app, in.RepoPath, architecture) + if err != nil { + return nil, err + } + return afx.ToMap(result) +} + +// RunSecurityContextProfiler ports +// `run_security_context_profiler(repo_path, architecture)` +// (reasoners/recon.py:49). Same `ArchitectureMap(**architecture)` bind, same +// nested validation — see RunDataFlowMapper. +func RunSecurityContextProfiler(ctx context.Context, app appx.App, in ArchitectureInput) (map[string]any, error) { + app.Note(ctx, "Security context profiler starting", "recon", "security-context") + architecture, err := bindArchitectureMap(in.Architecture) + if err != nil { + return nil, err + } + result, err := reconagent.RunSecurityContextProfiler(ctx, app, in.RepoPath, architecture) + if err != nil { + return nil, err + } + return afx.ToMap(result) +} diff --git a/internal/secaf/reasoners/recon_test.go b/internal/secaf/reasoners/recon_test.go new file mode 100644 index 0000000000..be3ef41adf --- /dev/null +++ b/internal/secaf/reasoners/recon_test.go @@ -0,0 +1,100 @@ +package reasoners + +// Tests for src/sec_af/reasoners/recon.py. +// +// Validation contract: +// +// - each of the five adapters emits its own note (message + tags) before the +// agent function runs; +// - the two deep-recon adapters materialize `architecture` with +// ArchitectureMap(**architecture), which has no required field — an empty +// or absent dict binds to the pydantic defaults rather than failing; +// - each adapter returns the agent result's model_dump() key set. + +import ( + "context" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" +) + +func TestReconAdapterNotes(t *testing.T) { + repo := t.TempDir() + + t.Run("run_architecture_mapper", func(t *testing.T) { + fake := newScanFake() + got, err := RunArchitectureMapper(context.Background(), fake, RepoPathInput{RepoPath: repo}) + if err != nil { + t.Fatalf("RunArchitectureMapper: %v", err) + } + assertNote(t, fake, "Architecture mapper starting", "recon", "architecture") + assertHasKeys(t, got, "app_type", "modules", "entry_points", "api_surface", "trust_boundaries") + }) + + t.Run("run_dependency_auditor", func(t *testing.T) { + fake := newScanFake() + got, err := RunDependencyAuditor(context.Background(), fake, RepoPathInput{RepoPath: repo}) + if err != nil { + t.Fatalf("RunDependencyAuditor: %v", err) + } + assertNote(t, fake, "Dependency auditor starting", "recon", "dependencies") + assertHasKeys(t, got, "direct_count", "transitive_count", "known_cves") + }) + + t.Run("run_config_scanner", func(t *testing.T) { + fake := newScanFake() + got, err := RunConfigScanner(context.Background(), fake, RepoPathInput{RepoPath: repo}) + if err != nil { + t.Fatalf("RunConfigScanner: %v", err) + } + assertNote(t, fake, "Config scanner starting", "recon", "config") + assertHasKeys(t, got, "secrets", "misconfigs") + }) + + t.Run("run_data_flow_mapper", func(t *testing.T) { + fake := newScanFake() + got, err := RunDataFlowMapper(context.Background(), fake, ArchitectureInput{ + RepoPath: repo, + Architecture: map[string]any{}, + }) + if err != nil { + t.Fatalf("RunDataFlowMapper: %v", err) + } + assertNote(t, fake, "Data flow mapper starting", "recon", "data-flow") + assertHasKeys(t, got, "flows", "sanitization_points") + }) + + t.Run("run_security_context_profiler", func(t *testing.T) { + fake := newScanFake() + got, err := RunSecurityContextProfiler(context.Background(), fake, ArchitectureInput{ + RepoPath: repo, + Architecture: map[string]any{}, + }) + if err != nil { + t.Fatalf("RunSecurityContextProfiler: %v", err) + } + assertNote(t, fake, "Security context profiler starting", "recon", "security-context") + assertHasKeys(t, got, "auth_model", "auth_details", "crypto_usage", "framework_security") + }) +} + +// TestDeepReconAcceptsAbsentArchitecture pins the "no required field" half of +// the contract: `ArchitectureMap(**{})` succeeds in Python, so a request that +// omits `architecture` entirely must not fail. +func TestDeepReconAcceptsAbsentArchitecture(t *testing.T) { + fake := newScanFake() + if _, err := RunDataFlowMapper(context.Background(), fake, ArchitectureInput{RepoPath: t.TempDir()}); err != nil { + t.Fatalf("RunDataFlowMapper with no architecture: %v", err) + } +} + +func assertHasKeys(t *testing.T, got map[string]any, keys ...string) { + t.Helper() + for _, key := range keys { + if _, ok := got[key]; !ok { + t.Errorf("result is missing the %q key", key) + } + } +} + +var _ appx.App = (*appx.Fake)(nil) diff --git a/internal/secaf/reasoners/register.go b/internal/secaf/reasoners/register.go new file mode 100644 index 0000000000..0999b6d991 --- /dev/null +++ b/internal/secaf/reasoners/register.go @@ -0,0 +1,247 @@ +package reasoners + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/secaf/afx" + "github.com/Agent-Field/codeaf/internal/secaf/phases" +) + +// register.go builds the reasoner surface: the 33 named handlers Python's +// module-level router carried, +// +// router = AgentRouter(tags=["security", "audit", "red-team"]) +// @router.reasoner() +// async def run_x(...): ... +// # app.py: app.include_router(reasoner_router) +// +// held in an in-process Registry instead of mounted on an SDK router. Inside +// codeaf there is no control plane to route a `.call` through: the audit's +// Caller (internal/secaf/audit) looks the target up here and runs the handler +// in the same process, so the names, the order, the input validation and the +// typed binding are exactly what the node registered — only the transport is +// gone. + +// Handler is one reasoner: the untyped keyword map a `.call` carries in, the +// reasoner's result out. It is the shape the SDK's agent.HandlerFunc had, kept +// so every handler below moved across unchanged. +type Handler func(ctx context.Context, input map[string]any) (any, error) + +// Registry is the reasoner table: name -> handler, plus the registration order +// and the input schema each name was registered with. +// +// It is built once by RegisterAll and only read afterwards, so it needs no +// lock; a caller that registers concurrently with lookups is misusing it. +type Registry struct { + names []string + handlers map[string]Handler + schemas map[string]json.RawMessage +} + +// NewRegistry returns an empty registry. +func NewRegistry() *Registry { + return &Registry{ + handlers: map[string]Handler{}, + schemas: map[string]json.RawMessage{}, + } +} + +// Register adds one handler under name. +// +// It PANICS on a name registered twice: the SDK's reasoner table was a map +// where the second registration silently won, and a collision is a wiring bug +// that must fail at construction, not at the first `.call` that reaches the +// wrong handler. +func (r *Registry) Register(name string, h Handler, schema json.RawMessage) { + if _, dup := r.handlers[name]; dup { + panic(fmt.Sprintf("reasoners: %q registered twice", name)) + } + r.names = append(r.names, name) + r.handlers[name] = h + r.schemas[name] = schema +} + +// Lookup returns the handler registered under name. +func (r *Registry) Lookup(name string) (Handler, bool) { + h, ok := r.handlers[name] + return h, ok +} + +// Names returns a copy of the registered names, in registration order. +func (r *Registry) Names() []string { + return append([]string(nil), r.names...) +} + +// InputSchema returns a copy of the input schema name was registered with, and +// false for a name the registry does not carry. +func (r *Registry) InputSchema(name string) (json.RawMessage, bool) { + schema, ok := r.schemas[name] + if !ok { + return nil, false + } + return append(json.RawMessage(nil), schema...), true +} + +// RegisterAll registers every router reasoner on reg, in DESIGN.md §3 order, +// and returns the ordered names it registered. +// +// app is the App every handler closes over — and therefore the App whose Call +// the four `*_phase` handlers make their nested `.call`s through. To keep those +// nested calls in process too, pass the same local-calling App the registry is +// answering for (audit.WithLocalCalls does exactly that). +// +// nodeID is read ONCE, here, from phases.NodeID(): Python's module-level +// `NODE_ID = os.getenv("NODE_ID", "sec-af")`, now a constant. Every `*_phase` +// handler closes over that value, so the `.call` targets are fixed. +func RegisterAll(reg *Registry, app appx.App) []string { + nodeID := phases.NodeID() + r := ®istrar{reg: reg} + + // --- reasoners/recon.py ------------------------------------------------- + register(r, app, NameRunArchitectureMapper, RunArchitectureMapper) + register(r, app, NameRunDependencyAuditor, RunDependencyAuditor) + register(r, app, NameRunConfigScanner, RunConfigScanner) + register(r, app, NameRunDataFlowMapper, RunDataFlowMapper) + register(r, app, NameRunSecurityContextProfiler, RunSecurityContextProfiler) + + // --- reasoners/hunt.py -------------------------------------------------- + register(r, app, NameRunInjectionHunter, RunInjectionHunter) + register(r, app, NameRunDosHunter, RunDosHunter) + register(r, app, NameRunSSRFHunter, RunSSRFHunter) + register(r, app, NameRunAuthHunter, RunAuthHunter) + register(r, app, NameRunXSSHunter, RunXSSHunter) + register(r, app, NameRunCryptoHunter, RunCryptoHunter) + register(r, app, NameRunBusinessLogicHunter, RunBusinessLogicHunter) + register(r, app, NameRunLogicBugsHunter, RunLogicBugsHunter) + register(r, app, NameRunDataExposureHunter, RunDataExposureHunter) + register(r, app, NameRunSupplyChainHunter, RunSupplyChainHunter) + register(r, app, NameRunConfigSecretsHunter, RunConfigSecretsHunter) + register(r, app, NameRunAPISecurityHunter, RunAPISecurityHunter) + register(r, app, NameRunDeduplicator, RunDeduplicator) + + // --- reasoners/prove.py ------------------------------------------------- + register(r, app, NameRunDepReachability, RunDepReachability) + register(r, app, NameRunVerifier, RunVerifier) + register(r, app, NameRunTracer, RunTracer) + register(r, app, NameRunSanitizationAnalyzer, RunSanitizationAnalyzer) + register(r, app, NameRunExploitHypothesizer, RunExploitHypothesizer) + register(r, app, NameRunVerdictAgent, RunVerdictAgent) + register(r, app, NameRunRemediation, RunRemediation) + register(r, app, NameRunRemediationAgent, RunRemediationAgent) + register(r, app, NameRunDastVerifier, RunDastVerifier) + register(r, app, NameRunCrossServiceAnalyzer, RunCrossServiceAnalyzer) + + // --- reasoners/phases.py ------------------------------------------------ + register(r, app, NameRunCWEExpansion, RunCWEExpansion) + registerPhase(r, app, nodeID, NameReconPhase, ReconPhase) + registerPhase(r, app, nodeID, NameHuntPhase, HuntPhase) + registerPhase(r, app, nodeID, NameProvePhase, ProvePhase) + registerPhase(r, app, nodeID, NameRemediationPhase, RemediationPhase) + + return r.names +} + +// registrar is the single registration path: everything that reaches the +// registry is recorded here too, so the returned bookkeeping cannot drift from +// what the registry received. +type registrar struct { + reg *Registry + names []string +} + +// add registers one reasoner, recording it, wrapping it in the input +// validation Python's SDK performed, and attaching the input schema the Python +// node published for that name. +// +// InputSchema panics on a name the capture does not know, so a reasoner added +// here without regenerating testdata/python_input_schemas.json fails at +// registration rather than running unvalidated. See input_schemas.go. +func (r *registrar) add(name string, h Handler) { + r.names = append(r.names, name) + r.reg.Register(name, ValidateHandler(name, h), InputSchema(name)) +} + +// InputError is a request the reasoner refused before running: the input did +// not bind. StatusCode keeps the HTTP status the node answered with — 422 for +// the SDK-level validation Python's endpoint performs, 400 for the audit +// entry's own bad-input family — because that number is how a caller told +// "your input was wrong" from "the audit failed", and the codeaf side may want +// to keep drawing the same line. +type InputError struct { + StatusCode int + Message string +} + +func (e *InputError) Error() string { return e.Message } + +// ValidateHandler wraps h with `_validate_handler_input` for the reasoner +// called name — the check the Python SDK ran on every request body BEFORE the +// handler function was entered (agent.py:3120-3134). +// +// Exported because internal/secaf/audit runs the same layer over the audit +// entry's own request. +// +// Python answered a failure with `JSONResponse(422, {"detail": safe_message})` +// from the endpoint, so the handler never ran. Here the check runs as the first +// thing inside the handler and reports the same status through an +// *InputError. +func ValidateHandler(name string, h Handler) Handler { + // Resolve the spec at REGISTRATION time so an unknown name panics at + // construction, not on the first request — the same loud-drift contract + // InputSchema has. + _ = handlerSpecFor(name) + return func(ctx context.Context, input map[string]any) (any, error) { + validated, err := ValidateHandlerInput(name, input) + if err != nil { + return nil, HandlerInputStatusError(err) + } + return h(ctx, validated) + } +} + +// HandlerInputStatusError maps a *HandlerInputError onto the *InputError that +// carries 422, the status Python's endpoint returned. +func HandlerInputStatusError(err error) error { + return &InputError{StatusCode: http.StatusUnprocessableEntity, Message: err.Error()} +} + +// register adapts a typed reasoner function to a Handler: afx.Bind +// the untyped request map into T (running T's default-seeding UnmarshalJSON, so +// the Python keyword defaults apply to absent keys) and hand it to fn. T is +// inferred from fn. +func register[T any]( + r *registrar, + app appx.App, + name string, + fn func(context.Context, appx.App, T) (map[string]any, error), +) { + r.add(name, func(ctx context.Context, input map[string]any) (any, error) { + in, err := afx.Bind[T](input) + if err != nil { + return nil, err + } + return fn(ctx, app, in) + }) +} + +// registerPhase is register for the four `*_phase` reasoners, which additionally +// need the node id their `.call` targets are prefixed with. +func registerPhase[T any]( + r *registrar, + app appx.App, + nodeID string, + name string, + fn func(context.Context, appx.App, string, T) (map[string]any, error), +) { + r.add(name, func(ctx context.Context, input map[string]any) (any, error) { + in, err := afx.Bind[T](input) + if err != nil { + return nil, err + } + return fn(ctx, app, nodeID, in) + }) +} diff --git a/internal/secaf/reasoners/register_test.go b/internal/secaf/reasoners/register_test.go new file mode 100644 index 0000000000..9e37c82e8f --- /dev/null +++ b/internal/secaf/reasoners/register_test.go @@ -0,0 +1,160 @@ +package reasoners + +// Registration parity for the router surface. +// +// Validation contract (behaviour, derived from src/sec_af/reasoners/*.py and +// DESIGN.md §3 — NOT from register.go): +// +// - the router carries exactly 33 reasoners, with the exact names and in the +// exact order DESIGN.md §3 lists (which is the Python import + decorator +// order); +// - no name is registered twice (the registry panics on a collision rather +// than letting the second registration win); +// - every registered name is actually reachable through Registry.Lookup — +// read back from the registry, not from RegisterAll's own bookkeeping, so +// the two cannot drift together. + +import ( + "context" + "reflect" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" +) + +// pythonSurface is the independent parity checklist: the 33 router reasoner +// names in DESIGN.md §3 order, written out from the Python inventory rather +// than derived from Names, so drift in either direction fails the test. +var pythonSurface = []string{ + // reasoners/recon.py + "run_architecture_mapper", + "run_dependency_auditor", + "run_config_scanner", + "run_data_flow_mapper", + "run_security_context_profiler", + // reasoners/hunt.py + "run_injection_hunter", + "run_dos_hunter", + "run_ssrf_hunter", + "run_auth_hunter", + "run_xss_hunter", + "run_crypto_hunter", + "run_business_logic_hunter", + "run_logic_bugs_hunter", + "run_data_exposure_hunter", + "run_supply_chain_hunter", + "run_config_secrets_hunter", + "run_api_security_hunter", + "run_deduplicator", + // reasoners/prove.py + "run_dep_reachability", + "run_verifier", + "run_tracer", + "run_sanitization_analyzer", + "run_exploit_hypothesizer", + "run_verdict_agent", + "run_remediation", + "run_remediation_agent", + "run_dast_verifier", + "run_cross_service_analyzer", + // reasoners/phases.py + "run_cwe_expansion", + "recon_phase", + "hunt_phase", + "prove_phase", + "remediation_phase", +} + +func TestNamesMatchPythonSurface(t *testing.T) { + if !reflect.DeepEqual(Names, pythonSurface) { + t.Fatalf("Names mismatch:\n got = %v\n want = %v", Names, pythonSurface) + } + if len(Names) != 33 { + t.Fatalf("surface size = %d, want 33", len(Names)) + } +} + +func TestRegisterAllExactOrderedSurface(t *testing.T) { + reg := NewRegistry() + got := RegisterAll(reg, &appx.Fake{}) + + if !reflect.DeepEqual(got, pythonSurface) { + t.Fatalf("registered surface mismatch:\n got = %v\n want = %v", got, pythonSurface) + } + if names := reg.Names(); !reflect.DeepEqual(names, pythonSurface) { + t.Fatalf("registry order mismatch:\n got = %v\n want = %v", names, pythonSurface) + } + + seen := map[string]int{} + for _, name := range got { + seen[name]++ + } + for name, count := range seen { + if count > 1 { + t.Errorf("reasoner %q registered %d times (collision)", name, count) + } + } +} + +// TestRegisterAllReachableInRegistry reads the surface back OUT of the +// registry: every name RegisterAll reports must resolve through Lookup, and the +// registry must hold nothing else. +func TestRegisterAllReachableInRegistry(t *testing.T) { + reg := NewRegistry() + names := RegisterAll(reg, &appx.Fake{}) + + if got := len(reg.Names()); got != len(names) { + t.Errorf("registry holds %d reasoners, want %d", got, len(names)) + } + for _, name := range names { + if h, ok := reg.Lookup(name); !ok || h == nil { + t.Errorf("reasoner %q is not reachable in the registry", name) + } + } + if _, ok := reg.Lookup(NameAudit); ok { + t.Error("audit is the entry point, not a registered reasoner") + } +} + +// TestRegistryRefusesADuplicateName: a second registration under one name is a +// wiring bug, and must fail at construction. +func TestRegistryRefusesADuplicateName(t *testing.T) { + reg := NewRegistry() + h := func(context.Context, map[string]any) (any, error) { return nil, nil } + reg.Register("run_x", h, nil) + defer func() { + if recover() == nil { + t.Fatal("a duplicate registration returned instead of panicking") + } + }() + reg.Register("run_x", h, nil) +} + +// TestRegisterAllHandlersBindDefaults proves the registration path really runs +// afx.Bind into the typed input (and therefore the default-seeding +// UnmarshalJSON): run_config_secrets_hunter is invoked through its REGISTERED +// handler with a body that omits max_files_without_signal, and the prompt the +// harness receives must still carry the Python default of 30. +func TestRegisterAllHandlersBindDefaults(t *testing.T) { + fake := newScanFake() + + reg := NewRegistry() + RegisterAll(reg, fake) + h, ok := reg.Lookup("run_config_secrets_hunter") + if !ok { + t.Fatal("run_config_secrets_hunter is not registered") + } + + if _, err := h(context.Background(), map[string]any{ + "repo_path": t.TempDir(), + "recon_context": map[string]any{}, + "depth": "standard", + }); err != nil { + t.Fatalf("handler: %v", err) + } + + if len(fake.Harnesses) != 1 { + t.Fatalf("harness calls = %d, want 1", len(fake.Harnesses)) + } + assertPromptHasFileBudget(t, fake.Harnesses[0].Prompt, "30") +} diff --git a/internal/secaf/reasoners/testdata/python_input_schemas.json b/internal/secaf/reasoners/testdata/python_input_schemas.json new file mode 100644 index 0000000000..502853a16e --- /dev/null +++ b/internal/secaf/reasoners/testdata/python_input_schemas.json @@ -0,0 +1,774 @@ +{ + "audit": { + "properties": { + "base_commit_sha": { + "type": "object" + }, + "branch": { + "type": "string" + }, + "commit_sha": { + "type": "object" + }, + "compliance_frameworks": { + "type": "object" + }, + "depth": { + "type": "string" + }, + "enable_dast": { + "type": "boolean" + }, + "exclude_paths": { + "type": "object" + }, + "fail_on_findings": { + "type": "boolean" + }, + "include_paths": { + "type": "object" + }, + "is_pr": { + "type": "boolean" + }, + "max_cost_usd": { + "type": "object" + }, + "max_duration_seconds": { + "type": "object" + }, + "max_provers": { + "type": "object" + }, + "output_formats": { + "type": "object" + }, + "post_pr_comments": { + "type": "boolean" + }, + "pr_id": { + "type": "object" + }, + "repo_url": { + "type": "string" + }, + "resume_from_checkpoint": { + "type": "object" + }, + "scan_types": { + "type": "object" + }, + "severity_threshold": { + "type": "string" + } + }, + "required": [ + "repo_url" + ], + "type": "object" + }, + "hunt_phase": { + "properties": { + "ai_gate": { + "type": "object" + }, + "depth": { + "type": "string" + }, + "early_stop_file_threshold": { + "type": "integer" + }, + "max_concurrent_hunters": { + "type": "integer" + }, + "recon_context": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "recon_context" + ], + "type": "object" + }, + "prove_phase": { + "properties": { + "depth": { + "type": "string" + }, + "hunt_result": { + "additionalProperties": true, + "type": "object" + }, + "max_concurrent_provers": { + "type": "integer" + }, + "max_provers": { + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "hunt_result" + ], + "type": "object" + }, + "recon_phase": { + "properties": { + "depth": { + "type": "string" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path" + ], + "type": "object" + }, + "remediation_phase": { + "properties": { + "max_concurrent_remediations": { + "type": "integer" + }, + "repo_path": { + "type": "string" + }, + "verified_findings": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "repo_path", + "verified_findings" + ], + "type": "object" + }, + "run_api_security_hunter": { + "properties": { + "depth": { + "type": "string" + }, + "max_files_without_signal": { + "type": "integer" + }, + "recon_context": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "recon_context", + "depth" + ], + "type": "object" + }, + "run_architecture_mapper": { + "properties": { + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path" + ], + "type": "object" + }, + "run_auth_hunter": { + "properties": { + "depth": { + "type": "string" + }, + "max_files_without_signal": { + "type": "integer" + }, + "recon_context": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "recon_context", + "depth" + ], + "type": "object" + }, + "run_business_logic_hunter": { + "properties": { + "depth": { + "type": "string" + }, + "max_files_without_signal": { + "type": "integer" + }, + "recon_context": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "recon_context", + "depth" + ], + "type": "object" + }, + "run_config_scanner": { + "properties": { + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path" + ], + "type": "object" + }, + "run_config_secrets_hunter": { + "properties": { + "depth": { + "type": "string" + }, + "max_files_without_signal": { + "type": "integer" + }, + "recon_context": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "recon_context", + "depth" + ], + "type": "object" + }, + "run_cross_service_analyzer": { + "properties": { + "depth": { + "type": "string" + }, + "findings_summary": { + "type": "string" + }, + "repo_path": { + "type": "string" + }, + "services": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "repo_path", + "services", + "findings_summary", + "depth" + ], + "type": "object" + }, + "run_crypto_hunter": { + "properties": { + "depth": { + "type": "string" + }, + "max_files_without_signal": { + "type": "integer" + }, + "recon_context": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "recon_context", + "depth" + ], + "type": "object" + }, + "run_cwe_expansion": { + "properties": { + "recon_summary": { + "type": "string" + }, + "strategies": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "recon_summary", + "strategies" + ], + "type": "object" + }, + "run_dast_verifier": { + "properties": { + "depth": { + "type": "string" + }, + "exploit_payload": { + "type": "string" + }, + "finding": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "finding", + "exploit_payload", + "depth" + ], + "type": "object" + }, + "run_data_exposure_hunter": { + "properties": { + "depth": { + "type": "string" + }, + "max_files_without_signal": { + "type": "integer" + }, + "recon_context": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "recon_context", + "depth" + ], + "type": "object" + }, + "run_data_flow_mapper": { + "properties": { + "architecture": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "architecture" + ], + "type": "object" + }, + "run_deduplicator": { + "properties": { + "findings": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + "recon_context": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "findings", + "recon_context", + "repo_path" + ], + "type": "object" + }, + "run_dep_reachability": { + "properties": { + "depth": { + "type": "string" + }, + "finding": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "finding", + "depth" + ], + "type": "object" + }, + "run_dependency_auditor": { + "properties": { + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path" + ], + "type": "object" + }, + "run_dos_hunter": { + "properties": { + "depth": { + "type": "string" + }, + "max_files_without_signal": { + "type": "integer" + }, + "recon_context": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "recon_context", + "depth" + ], + "type": "object" + }, + "run_exploit_hypothesizer": { + "properties": { + "data_flow": { + "additionalProperties": true, + "type": "object" + }, + "depth": { + "type": "string" + }, + "finding": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + }, + "sanitization": { + "additionalProperties": true, + "type": "object" + } + }, + "required": [ + "repo_path", + "finding", + "data_flow", + "sanitization", + "depth" + ], + "type": "object" + }, + "run_injection_hunter": { + "properties": { + "depth": { + "type": "string" + }, + "max_files_without_signal": { + "type": "integer" + }, + "recon_context": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "recon_context", + "depth" + ], + "type": "object" + }, + "run_logic_bugs_hunter": { + "properties": { + "depth": { + "type": "string" + }, + "max_files_without_signal": { + "type": "integer" + }, + "recon_context": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "recon_context", + "depth" + ], + "type": "object" + }, + "run_remediation": { + "properties": { + "finding": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "finding" + ], + "type": "object" + }, + "run_remediation_agent": { + "properties": { + "finding": { + "additionalProperties": true, + "type": "object" + }, + "rationale": { + "type": "string" + }, + "repo_path": { + "type": "string" + }, + "verdict": { + "type": "string" + } + }, + "required": [ + "repo_path", + "finding", + "verdict", + "rationale" + ], + "type": "object" + }, + "run_sanitization_analyzer": { + "properties": { + "data_flow": { + "additionalProperties": true, + "type": "object" + }, + "depth": { + "type": "string" + }, + "finding": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "finding", + "data_flow", + "depth" + ], + "type": "object" + }, + "run_security_context_profiler": { + "properties": { + "architecture": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "architecture" + ], + "type": "object" + }, + "run_ssrf_hunter": { + "properties": { + "depth": { + "type": "string" + }, + "max_files_without_signal": { + "type": "integer" + }, + "recon_context": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "recon_context", + "depth" + ], + "type": "object" + }, + "run_supply_chain_hunter": { + "properties": { + "depth": { + "type": "string" + }, + "max_files_without_signal": { + "type": "integer" + }, + "recon_context": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "recon_context", + "depth" + ], + "type": "object" + }, + "run_tracer": { + "properties": { + "depth": { + "type": "string" + }, + "finding": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "finding", + "depth" + ], + "type": "object" + }, + "run_verdict_agent": { + "properties": { + "data_flow": { + "additionalProperties": true, + "type": "object" + }, + "exploit": { + "additionalProperties": true, + "type": "object" + }, + "finding": { + "additionalProperties": true, + "type": "object" + }, + "sanitization": { + "additionalProperties": true, + "type": "object" + } + }, + "required": [ + "finding", + "data_flow", + "sanitization", + "exploit" + ], + "type": "object" + }, + "run_verifier": { + "properties": { + "depth": { + "type": "string" + }, + "finding": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "finding", + "depth" + ], + "type": "object" + }, + "run_xss_hunter": { + "properties": { + "depth": { + "type": "string" + }, + "max_files_without_signal": { + "type": "integer" + }, + "recon_context": { + "additionalProperties": true, + "type": "object" + }, + "repo_path": { + "type": "string" + } + }, + "required": [ + "repo_path", + "recon_context", + "depth" + ], + "type": "object" + } +} \ No newline at end of file diff --git a/internal/secaf/reasoners/testdata/python_input_types.json b/internal/secaf/reasoners/testdata/python_input_types.json new file mode 100644 index 0000000000..21de1bd2cd --- /dev/null +++ b/internal/secaf/reasoners/testdata/python_input_types.json @@ -0,0 +1,902 @@ +{ + "audit": [ + { + "name": "repo_url", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": false, + "default": "standard" + }, + { + "name": "branch", + "kind": "str", + "annotation": "", + "required": false, + "default": "main" + }, + { + "name": "commit_sha", + "kind": "any", + "annotation": "str | None", + "required": false, + "default": null + }, + { + "name": "base_commit_sha", + "kind": "any", + "annotation": "str | None", + "required": false, + "default": null + }, + { + "name": "severity_threshold", + "kind": "str", + "annotation": "", + "required": false, + "default": "low" + }, + { + "name": "scan_types", + "kind": "any", + "annotation": "list[str] | None", + "required": false, + "default": null + }, + { + "name": "output_formats", + "kind": "any", + "annotation": "list[str] | None", + "required": false, + "default": null + }, + { + "name": "compliance_frameworks", + "kind": "any", + "annotation": "list[str] | None", + "required": false, + "default": null + }, + { + "name": "max_cost_usd", + "kind": "any", + "annotation": "float | None", + "required": false, + "default": null + }, + { + "name": "max_provers", + "kind": "any", + "annotation": "int | None", + "required": false, + "default": null + }, + { + "name": "max_duration_seconds", + "kind": "any", + "annotation": "int | None", + "required": false, + "default": null + }, + { + "name": "include_paths", + "kind": "any", + "annotation": "list[str] | None", + "required": false, + "default": null + }, + { + "name": "exclude_paths", + "kind": "any", + "annotation": "list[str] | None", + "required": false, + "default": null + }, + { + "name": "is_pr", + "kind": "bool", + "annotation": "", + "required": false, + "default": false + }, + { + "name": "pr_id", + "kind": "any", + "annotation": "str | None", + "required": false, + "default": null + }, + { + "name": "post_pr_comments", + "kind": "bool", + "annotation": "", + "required": false, + "default": false + }, + { + "name": "fail_on_findings", + "kind": "bool", + "annotation": "", + "required": false, + "default": false + }, + { + "name": "enable_dast", + "kind": "bool", + "annotation": "", + "required": false, + "default": false + }, + { + "name": "resume_from_checkpoint", + "kind": "any", + "annotation": "str | None", + "required": false, + "default": null + } + ], + "hunt_phase": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "recon_context", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": false, + "default": "standard" + }, + { + "name": "ai_gate", + "kind": "any", + "annotation": "typing.Any | None", + "required": false, + "default": null + }, + { + "name": "max_concurrent_hunters", + "kind": "int", + "annotation": "", + "required": false, + "default": 4 + }, + { + "name": "early_stop_file_threshold", + "kind": "int", + "annotation": "", + "required": false, + "default": 30 + } + ], + "prove_phase": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "hunt_result", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": false, + "default": "standard" + }, + { + "name": "max_provers", + "kind": "any", + "annotation": "int | None", + "required": false, + "default": null + }, + { + "name": "max_concurrent_provers", + "kind": "int", + "annotation": "", + "required": false, + "default": 3 + } + ], + "recon_phase": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": false, + "default": "standard" + } + ], + "remediation_phase": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "verified_findings", + "kind": "list", + "annotation": "list[dict[str, typing.Any]]", + "required": true + }, + { + "name": "max_concurrent_remediations", + "kind": "int", + "annotation": "", + "required": false, + "default": 3 + } + ], + "run_api_security_hunter": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "recon_context", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "max_files_without_signal", + "kind": "int", + "annotation": "", + "required": false, + "default": 30 + } + ], + "run_architecture_mapper": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + } + ], + "run_auth_hunter": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "recon_context", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "max_files_without_signal", + "kind": "int", + "annotation": "", + "required": false, + "default": 30 + } + ], + "run_business_logic_hunter": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "recon_context", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "max_files_without_signal", + "kind": "int", + "annotation": "", + "required": false, + "default": 30 + } + ], + "run_config_scanner": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + } + ], + "run_config_secrets_hunter": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "recon_context", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "max_files_without_signal", + "kind": "int", + "annotation": "", + "required": false, + "default": 30 + } + ], + "run_cross_service_analyzer": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "services", + "kind": "list", + "annotation": "list[str]", + "required": true + }, + { + "name": "findings_summary", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": true + } + ], + "run_crypto_hunter": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "recon_context", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "max_files_without_signal", + "kind": "int", + "annotation": "", + "required": false, + "default": 30 + } + ], + "run_cwe_expansion": [ + { + "name": "recon_summary", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "strategies", + "kind": "list", + "annotation": "list[str]", + "required": true + } + ], + "run_dast_verifier": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "finding", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "exploit_payload", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": true + } + ], + "run_data_exposure_hunter": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "recon_context", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "max_files_without_signal", + "kind": "int", + "annotation": "", + "required": false, + "default": 30 + } + ], + "run_data_flow_mapper": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "architecture", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + } + ], + "run_deduplicator": [ + { + "name": "findings", + "kind": "list", + "annotation": "list[dict[str, typing.Any]]", + "required": true + }, + { + "name": "recon_context", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + } + ], + "run_dep_reachability": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "finding", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": true + } + ], + "run_dependency_auditor": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + } + ], + "run_dos_hunter": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "recon_context", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "max_files_without_signal", + "kind": "int", + "annotation": "", + "required": false, + "default": 30 + } + ], + "run_exploit_hypothesizer": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "finding", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "data_flow", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "sanitization", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": true + } + ], + "run_injection_hunter": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "recon_context", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "max_files_without_signal", + "kind": "int", + "annotation": "", + "required": false, + "default": 30 + } + ], + "run_logic_bugs_hunter": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "recon_context", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "max_files_without_signal", + "kind": "int", + "annotation": "", + "required": false, + "default": 30 + } + ], + "run_remediation": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "finding", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + } + ], + "run_remediation_agent": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "finding", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "verdict", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "rationale", + "kind": "str", + "annotation": "", + "required": true + } + ], + "run_sanitization_analyzer": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "finding", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "data_flow", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": true + } + ], + "run_security_context_profiler": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "architecture", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + } + ], + "run_ssrf_hunter": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "recon_context", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "max_files_without_signal", + "kind": "int", + "annotation": "", + "required": false, + "default": 30 + } + ], + "run_supply_chain_hunter": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "recon_context", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "max_files_without_signal", + "kind": "int", + "annotation": "", + "required": false, + "default": 30 + } + ], + "run_tracer": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "finding", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": true + } + ], + "run_verdict_agent": [ + { + "name": "finding", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "data_flow", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "sanitization", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "exploit", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + } + ], + "run_verifier": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "finding", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": true + } + ], + "run_xss_hunter": [ + { + "name": "repo_path", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "recon_context", + "kind": "dict", + "annotation": "dict[str, typing.Any]", + "required": true + }, + { + "name": "depth", + "kind": "str", + "annotation": "", + "required": true + }, + { + "name": "max_files_without_signal", + "kind": "int", + "annotation": "", + "required": false, + "default": 30 + } + ] +} diff --git a/internal/secaf/reasoners/validate.go b/internal/secaf/reasoners/validate.go new file mode 100644 index 0000000000..5f5aa5bb36 --- /dev/null +++ b/internal/secaf/reasoners/validate.go @@ -0,0 +1,68 @@ +package reasoners + +import ( + "github.com/Agent-Field/codeaf/internal/secaf/phases" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// validate.go supplies what afx.Bind cannot: pydantic's REQUIRED-field, NULL +// and nested-model checks. +// +// Every adapter in this package materializes a request dict with a pydantic +// constructor — `RawFinding(**finding)`, `DataFlowTrace(**data_flow)`, +// `ArchitectureMap(**architecture)`, `ReconResult(**recon_context)` — and those +// RAISE on a payload that omits a required field, hands a non-Optional field an +// explicit null, or carries a nested element missing its OWN required fields. +// json.Unmarshal does none of that: it leaves the Go zero value in place, the +// schemas package's UnmarshalJSON actively seeds pydantic's DEFAULTS (right for +// an optional field, wrong for a required one), and a null even WIPES a seeded +// slice back to nil. +// +// The difference is BEHAVIORAL, not cosmetic, in exactly one place: +// `_coerce_verifier_finding` (prove.py:28) decides between two models by +// whether `RawFinding.model_validate` RAISES. prove_phase always feeds it a +// `FindingForVerifier` projection, which is missing nine of RawFinding's twelve +// required fields — so without a real check every verifier call would take the +// wrong branch and lose the projection's phase_boundary_projection marker, +// its CWE severity floor and its fingerprint. Everywhere else the check simply +// turns a malformed payload into an error, as Python does — and since every +// reasoner here is registered on the router, a control-plane caller can send +// such a payload directly. +// +// internal/phases owns the machinery — the modelSpec table (one entry per +// pydantic class, generated ground truth for the required/non-nullable lists), +// the error type (phases.ValidationError) and the binders. Every wrapper below +// delegates there so that each model's validation surface is transcribed ONCE: +// a second copy could drift and silently change which branch +// `_coerce_verifier_finding` takes, or leave a nested subtree unvalidated. + +// bindRawFinding is `RawFinding(**finding)` / `RawFinding.model_validate(finding)`. +func bindRawFinding(payload map[string]any) (schemas.RawFinding, error) { + return phases.BindRawFinding(payload) +} + +// bindDataFlowTrace is `DataFlowTrace(**data_flow)`. +func bindDataFlowTrace(payload map[string]any) (schemas.DataFlowTrace, error) { + return phases.BindDataFlowTrace(payload) +} + +// bindSanitizationResult is `SanitizationResult(**sanitization)`. +func bindSanitizationResult(payload map[string]any) (schemas.SanitizationResult, error) { + return phases.BindSanitizationResult(payload) +} + +// bindExploitHypothesis is `ExploitHypothesis(**exploit)`. +func bindExploitHypothesis(payload map[string]any) (schemas.ExploitHypothesis, error) { + return phases.BindExploitHypothesis(payload) +} + +// bindFindingForVerifier is `FindingForVerifier.model_validate(finding)` — the +// fallback branch of _coerce_verifier_finding. +func bindFindingForVerifier(payload map[string]any) (schemas.FindingForVerifier, error) { + return phases.BindFindingForVerifier(payload) +} + +// bindArchitectureMap is `ArchitectureMap(**architecture)` (recon.py:43, :52). +func bindArchitectureMap(payload map[string]any) (schemas.ArchitectureMap, error) { + return phases.BindArchitectureMap(payload) +} diff --git a/internal/secaf/reasoners/validate_test.go b/internal/secaf/reasoners/validate_test.go new file mode 100644 index 0000000000..a3d585d757 --- /dev/null +++ b/internal/secaf/reasoners/validate_test.go @@ -0,0 +1,186 @@ +package reasoners + +// Tests for the request binds every adapter performs (validate.go). +// +// Validation contract, taken from pydantic on the pinned interpreter +// (`PYTHONPATH=src ~/.agentfield/packages/sec-af/venv/bin/python`), not from +// the Go code. Each reasoner here is registered on the router (register.go), so +// a control-plane caller can send these payloads DIRECTLY — the DAG's own +// producers are not the only source. +// +// - `ArchitectureMap(**{"modules": [{"name": "x"}]})` raises 2 errors +// (`modules.0.path`, `modules.0.language` missing), so +// run_data_flow_mapper / run_security_context_profiler answer 500 in Python. +// - `ReconResult(**{... "architecture": {"modules": [{"name": "x"}]} ...})` +// raises, so run_deduplicator answers 500 in Python. +// - `DataFlowTrace(source=None, sink="s", steps=None, sink_reached=True)` and +// `ExploitHypothesis(hypothesis=None, expected_outcome=None)` VALIDATE (the +// `mode="before"` validators coerce None to "unknown"/[]), so +// run_sanitization_analyzer / run_exploit_hypothesizer / run_verdict_agent +// answer 200 in Python. + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/agentsession/appx" +) + +// nestedMalformedArchitecture is the payload whose element is missing `path` +// and `language` — valid at the top level, invalid one layer down. +func nestedMalformedArchitecture() map[string]any { + return map[string]any{"modules": []any{map[string]any{"name": "x"}}} +} + +// TestDeepReconValidatesNestedArchitecture covers the two reasoners that bind +// `ArchitectureMap(**architecture)`. ArchitectureMap has no required field of +// its own — TestDeepReconAcceptsAbsentArchitecture pins that an absent one is +// fine — but its list elements do. +func TestDeepReconValidatesNestedArchitecture(t *testing.T) { + repo := t.TempDir() + for _, tc := range []struct { + name string + run func(context.Context, appx.App, ArchitectureInput) (map[string]any, error) + }{ + {"run_data_flow_mapper", RunDataFlowMapper}, + {"run_security_context_profiler", RunSecurityContextProfiler}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := tc.run(context.Background(), newScanFake(), ArchitectureInput{ + RepoPath: repo, Architecture: nestedMalformedArchitecture(), + }) + if err == nil { + t.Fatal("want a validation error: ArchitectureMap(**architecture) raises on this payload") + } + if !strings.Contains(err.Error(), "modules.0.path: field required") { + t.Errorf("error %q does not name the missing nested field", err.Error()) + } + }) + } +} + +// TestRunDeduplicatorValidatesNestedRecon is the same gap on the STRICT +// `ReconResult(**recon_context)` bind run_deduplicator performs (hunt.py:287). +// Its sibling test pins the top-level check; this one pins the four subtrees +// below it. +func TestRunDeduplicatorValidatesNestedRecon(t *testing.T) { + for _, tc := range []struct { + name string + key string + value any + problem string + }{ + {"architecture", "architecture", nestedMalformedArchitecture(), + "architecture.modules.0.path: field required"}, + {"data_flows", "data_flows", + map[string]any{"flows": []any{map[string]any{"source": "a"}}}, + "data_flows.flows.0.sink: field required"}, + {"dependencies", "dependencies", + map[string]any{"sbom": []any{map[string]any{"name": "d"}}}, + "dependencies.sbom.0.version: field required"}, + {"config", "config", + map[string]any{"secrets": []any{map[string]any{"secret_type": "t"}}}, + "config.secrets.0.file_path: field required"}, + } { + t.Run(tc.name, func(t *testing.T) { + recon := fullReconContext() + recon[tc.key] = tc.value + _, err := RunDeduplicator(context.Background(), newScanFake(), DeduplicatorInput{ + Findings: []map[string]any{rawFindingPayload()}, + ReconContext: recon, + RepoPath: t.TempDir(), + }) + if err == nil { + t.Fatal("want a validation error: ReconResult(**recon_context) raises on this payload") + } + if !strings.Contains(err.Error(), tc.problem) { + t.Errorf("error %q does not mention %q", err.Error(), tc.problem) + } + }) + } +} + +// nullDataFlowPayload is `DataFlowTrace(source=None, sink="s", steps=None, +// sink_reached=True)` — VERIFIED to validate as +// `{"source": "unknown", "sink": "s", "steps": [], "sink_reached": true}`. +func nullDataFlowPayload() map[string]any { + return map[string]any{"source": nil, "sink": "s", "steps": nil, "sink_reached": true} +} + +// nullExploitPayload is `ExploitHypothesis(hypothesis=None, +// expected_outcome=None)` — VERIFIED to validate as +// `{"hypothesis": "unknown", "payload": null, "expected_outcome": "unknown"}`. +func nullExploitPayload() map[string]any { + return map[string]any{"hypothesis": nil, "payload": nil, "expected_outcome": nil} +} + +// TestBeforeValidatedNullsAreAcceptedByTheProveAdapters pins that the three +// reasoners taking a DataFlowTrace / ExploitHypothesis kwarg return a RESULT +// for a payload whose before-validated fields are explicitly null, as the +// Python node does — rather than a reasoner error. +func TestBeforeValidatedNullsAreAcceptedByTheProveAdapters(t *testing.T) { + repo := t.TempDir() + finding := rawFindingPayload() + + t.Run("run_sanitization_analyzer", func(t *testing.T) { + got, err := RunSanitizationAnalyzer(context.Background(), newScanFake(), SanitizationInput{ + RepoPath: repo, Finding: finding, DataFlow: nullDataFlowPayload(), Depth: "standard", + }) + if err != nil { + t.Fatalf("RunSanitizationAnalyzer: %v", err) + } + if _, ok := got["found"]; !ok { + t.Errorf("result is missing the %q key: %v", "found", got) + } + }) + + t.Run("run_exploit_hypothesizer", func(t *testing.T) { + got, err := RunExploitHypothesizer(context.Background(), newScanFake(), ExploitInput{ + RepoPath: repo, Finding: finding, DataFlow: nullDataFlowPayload(), + Sanitization: sanitizationPayload(), Depth: "standard", + }) + if err != nil { + t.Fatalf("RunExploitHypothesizer: %v", err) + } + if _, ok := got["hypothesis"]; !ok { + t.Errorf("result is missing the %q key: %v", "hypothesis", got) + } + }) + + t.Run("run_verdict_agent", func(t *testing.T) { + fake := newScanFake() + fake.AIFn = appx.AIJSON(func(string) (json.RawMessage, error) { + return json.RawMessage( + `{"verdict":"likely","evidence_level":2,"rationale":"r","confidence":"medium"}`), nil + }) + got, err := RunVerdictAgent(context.Background(), fake, VerdictInput{ + Finding: finding, DataFlow: nullDataFlowPayload(), + Sanitization: sanitizationPayload(), Exploit: nullExploitPayload(), + }) + if err != nil { + t.Fatalf("RunVerdictAgent: %v", err) + } + if got["verdict"] != "likely" { + t.Errorf("verdict = %v, want likely", got["verdict"]) + } + }) +} + +// TestBeforeValidatedFieldsStillRequireTheirKey is the other half of the same +// rule: a `mode="before"` validator does NOT run for an ABSENT field, so a +// missing `source` is still 1 validation error in Python. +func TestBeforeValidatedFieldsStillRequireTheirKey(t *testing.T) { + flow := nullDataFlowPayload() + delete(flow, "source") + _, err := RunSanitizationAnalyzer(context.Background(), newScanFake(), SanitizationInput{ + RepoPath: t.TempDir(), Finding: rawFindingPayload(), DataFlow: flow, Depth: "standard", + }) + if err == nil { + t.Fatal("want a validation error: a MISSING source is still `missing` in pydantic") + } + if !strings.Contains(err.Error(), "source: field required") { + t.Errorf("error %q does not name the missing field", err.Error()) + } +} diff --git a/internal/secaf/recontext/context.go b/internal/secaf/recontext/context.go new file mode 100644 index 0000000000..db61b2200c --- /dev/null +++ b/internal/secaf/recontext/context.go @@ -0,0 +1,826 @@ +// Package recontext ports src/sec_af/context.py — the strategy-aware RECON +// projections that every HUNT prompt embeds. +// +// The package is named `recontext` rather than `context` for the obvious +// reason: a package called `context` inside this module would shadow the +// standard library's in every file that imports both, and every reasoner in the +// port takes a context.Context. +// +// Two kinds of output live here and they are used very differently: +// +// - prune_recon_for_strategy returns a DICT that travels over the wire as the +// `recon_context` kwarg of a `.call` (reasoners/phases.py:294), so only its +// key SET matters, not its rendering; +// - every recon_context_for_* / *_hints_for_context function returns a STRING +// that is substituted into a prompt template and shipped to the model, so +// its bytes matter exactly. Those are golden-tested against the Python +// functions run over the same fixture (testdata/recon_fixture.json). +package recontext + +import ( + "sort" + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/secaf/afx" + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// Truncation limits, ported from context.py's module constants. +const ( + MaxPrimaryItems = 15 // _MAX_PRIMARY_ITEMS + MaxSecondaryItems = 10 // _MAX_SECONDARY_ITEMS +) + +// BaseReconFields ports _BASE_RECON_FIELDS: the ReconResult fields every +// strategy projection keeps regardless of which sections it asked for. +var BaseReconFields = []string{"languages", "frameworks", "lines_of_code", "file_count"} + +// StrategyContextMap ports STRATEGY_CONTEXT_MAP: strategy VALUE (the +// HuntStrategy enum's string, e.g. "business_logic") -> the ReconResult field +// names its hunter needs. +// +// Python parity: strategies absent from this table — "logic_bugs" is not a +// separate value (it is an alias of "business_logic"), but "python_specific" +// and "javascript_specific" genuinely are missing — fall through to the full +// model_dump() in PruneReconForStrategy. +var StrategyContextMap = map[string][]string{ + "injection": {"architecture", "data_flows", "security_context"}, + "xss": {"architecture", "data_flows", "security_context"}, + "ssrf": {"architecture", "data_flows", "security_context"}, + "auth": {"architecture", "security_context"}, + "crypto": {"security_context"}, + "dos": {"architecture", "data_flows"}, + "data_exposure": {"architecture", "data_flows", "config"}, + "supply_chain": {"dependencies"}, + "config_secrets": {"config", "architecture"}, + "api_security": {"architecture", "security_context", "data_flows"}, + "business_logic": {"architecture", "data_flows", "security_context"}, +} + +// PruneReconForStrategy ports prune_recon_for_strategy: +// +// required_fields = STRATEGY_CONTEXT_MAP.get(strategy) +// if required_fields is None: +// return recon.model_dump() +// include_fields = set(_BASE_RECON_FIELDS) +// include_fields.update(required_fields) +// return recon.model_dump(include=include_fields) +// +// The returned map is the pydantic model_dump() shape: keys are the snake_case +// pydantic field names (== the Go json tags), values are the nested models. +// afx.ToMap keeps the values TYPED rather than round-tripping them through +// JSON, which matters for two reasons: a float field with an integral value +// stays a float (`10.0`, not `10`) when pyfmt.Dumps renders it, and the nested +// structs keep their declaration order for anything that renders them. +// +// Python parity notes: +// +// - KEY ORDER. `model_dump(include={...})` emits the surviving keys in +// ReconResult DECLARATION order (architecture, data_flows, dependencies, +// config, security_context, languages, frameworks, lines_of_code, +// file_count, recon_duration_seconds), not in the include-set's order. A Go +// map has no order at all, which is immaterial here: the only consumer is +// `recon_context=strategy_context` on a `.call`, i.e. a JSON object where +// member order carries no meaning. +// - `recon_duration_seconds` is in NO strategy's projection and is not a base +// field, so it survives only on the unmapped-strategy path that returns the +// full dump. +// - The error return has no Python counterpart; afx.ToMap only fails on a +// non-struct argument, which cannot happen for a ReconResult. +func PruneReconForStrategy(recon schemas.ReconResult, strategy string) (map[string]any, error) { + dump, err := afx.ToMap(recon) + if err != nil { + return nil, err + } + requiredFields, ok := StrategyContextMap[strategy] + if !ok { + return dump, nil + } + + include := make(map[string]struct{}, len(BaseReconFields)+len(requiredFields)) + for _, f := range BaseReconFields { + include[f] = struct{}{} + } + for _, f := range requiredFields { + include[f] = struct{}{} + } + + out := make(map[string]any, len(include)) + for field := range include { + if v, present := dump[field]; present { + out[field] = v + } + } + return out, nil +} + +// --------------------------------------------------------------------------- +// list rendering +// --------------------------------------------------------------------------- + +// limit ports _limit: +// +// rows = [item for item in items if item] +// return rows[:max_items], len(rows) +// +// Python parity: the filter is TRUTHINESS, so empty strings are dropped BEFORE +// the count is taken — a list of ten items with three blanks reports "7 total". +func limit(items []string, maxItems int) ([]string, int) { + rows := make([]string, 0, len(items)) + for _, item := range items { + if item != "" { + rows = append(rows, item) + } + } + if len(rows) > maxItems { + return rows[:maxItems], len(rows) + } + return rows, len(rows) +} + +// renderList ports _render_list: +// +// trimmed, total = _limit(items, max_items) +// if total == 0: return f"{title}: none identified in recon." +// lines = [f"{title}: {total} total, showing top {len(trimmed)}:"] +// lines.extend(f"- {item}" for item in trimmed) +// return "\n".join(lines) +func renderList(title string, items []string, maxItems int) string { + trimmed, total := limit(items, maxItems) + if total == 0 { + return title + ": none identified in recon." + } + var b strings.Builder + b.WriteString(title) + b.WriteString(": ") + b.WriteString(strconv.Itoa(total)) + b.WriteString(" total, showing top ") + b.WriteString(strconv.Itoa(len(trimmed))) + b.WriteString(":") + for _, item := range trimmed { + b.WriteString("\n- ") + b.WriteString(item) + } + return b.String() +} + +// sections joins the top-level blocks the way every builder does: +// "\n\n".join([...]). +func sections(parts ...string) string { return strings.Join(parts, "\n\n") } + +// --------------------------------------------------------------------------- +// small Python-semantics helpers +// --------------------------------------------------------------------------- + +// joinOr reproduces `', '.join(items) or fallback` — an empty join result is +// falsy in Python, so an empty list (and a list of empty strings) yields the +// fallback. +func joinOr(items []string, fallback string) string { + joined := strings.Join(items, ", ") + if joined == "" { + return fallback + } + return joined +} + +// head reproduces Python's `seq[:n]`, which never panics on a short sequence. +func head(items []string, n int) []string { + if len(items) > n { + return items[:n] + } + return items +} + +// str renders an `X | None` scalar the way an f-string does: str(None) is the +// literal "None", str(True) is "True", str(1.0) is "1.0". pyfmt.Str handles the +// pointer deref and the Python spellings for bool/int/float. +func str(v any) string { return pyfmt.Str(v) } + +// strPtr is str() for a `str | None` field. It cannot go through pyfmt.Str, +// which would fall into Repr and QUOTE the string; an f-string interpolating a +// str emits it bare. +func strPtr(p *string) string { + if p == nil { + return "None" + } + return *p +} + +// orStr reproduces `value or fallback` for a `str | None` field: None AND the +// empty string are both falsy and both take the fallback. +func orStr(p *string, fallback string) string { + if p == nil || *p == "" { + return fallback + } + return *p +} + +// orZeroInt reproduces `value or 0` for an `int | None` field. 0 is falsy in +// Python too, so an explicit zero and a missing value render identically. +func orZeroInt(p *int) int { + if p == nil { + return 0 + } + return *p +} + +// truthyBool reproduces Python truthiness for a `bool | None` field: None and +// False are falsy, True is truthy. +func truthyBool(p *bool) bool { return p != nil && *p } + +// isFalse reproduces the `x is False` identity test used by _endpoint_rank_key: +// ONLY an explicit False satisfies it — None does not. +func isFalse(p *bool) bool { return p != nil && !*p } + +// containsAnyLower reports whether any token appears in strings.ToLower(haystack), +// reproducing `any(token in haystack.lower() for token in tokens)`. +// +// Python parity: str.lower() and strings.ToLower are both full Unicode +// lowercasers and agree on everything except a few code points whose lowercase +// form changes length (U+0130). Repository identifiers do not contain those. +func containsAnyLower(haystack string, tokens ...string) bool { + lowered := strings.ToLower(haystack) + for _, token := range tokens { + if strings.Contains(lowered, token) { + return true + } + } + return false +} + +// --------------------------------------------------------------------------- +// ranking helpers +// --------------------------------------------------------------------------- + +// endpointRankKey ports _endpoint_rank_key: +// +// return (0 if endpoint_auth_required is False else 1, +// 0 if endpoint_rate_limited is False else 1) +// +// Python parity: this sorts endpoints that EXPLICITLY declare "no auth" / +// "no rate limit" to the front. An UNKNOWN (None) endpoint ranks with the +// protected ones, not with the unprotected ones — `is False` is an identity +// test, not a truthiness test. +func endpointRankKey(authRequired, rateLimited *bool) (int, int) { + first, second := 1, 1 + if isFalse(authRequired) { + first = 0 + } + if isFalse(rateLimited) { + second = 0 + } + return first, second +} + +// rankedEndpoints reproduces `sorted(api_surface, key=_endpoint_rank_key)`. +// +// Python's sort is STABLE, so endpoints sharing a rank keep their recon order; +// sort.SliceStable is the matching Go primitive. The input slice is copied so +// the caller's ReconResult is not reordered — Python's sorted() also returns a +// new list. +func rankedEndpoints(endpoints []schemas.APIEndpoint) []schemas.APIEndpoint { + out := make([]schemas.APIEndpoint, len(endpoints)) + copy(out, endpoints) + sort.SliceStable(out, func(i, j int) bool { + ai, bi := endpointRankKey(out[i].AuthRequired, out[i].RateLimited) + aj, bj := endpointRankKey(out[j].AuthRequired, out[j].RateLimited) + if ai != aj { + return ai < aj + } + return bi < bj + }) + return out +} + +// cvePriority ports _cve_priority: +// +// reachable_rank = 0 if cve.reachable else 1 +// cvss = cve.cvss_v4_score if cve.cvss_v4_score is not None else -1.0 +// epss = cve.epss_score if cve.epss_score is not None else -1.0 +// direct_rank = 0 if cve.direct else 1 +// return (reachable_rank, -cvss, -epss, direct_rank) +// +// Python parity: reachable uses TRUTHINESS (None ranks with False), while the +// score defaults use an explicit `is not None` check — so a CVE with no CVSS +// sorts BELOW one scored 0.0 (its key component is +1.0 versus -0.0). +func cvePriority(cve schemas.KnownCVE) (int, float64, float64, int) { + reachableRank := 1 + if truthyBool(cve.Reachable) { + reachableRank = 0 + } + cvss := -1.0 + if cve.CvssV4Score != nil { + cvss = *cve.CvssV4Score + } + epss := -1.0 + if cve.EpssScore != nil { + epss = *cve.EpssScore + } + directRank := 1 + if cve.Direct { + directRank = 0 + } + return reachableRank, -cvss, -epss, directRank +} + +// prioritizedCVEs reproduces `sorted(known_cves, key=_cve_priority)` — a stable +// sort on the 4-tuple, compared component by component. +func prioritizedCVEs(cves []schemas.KnownCVE) []schemas.KnownCVE { + out := make([]schemas.KnownCVE, len(cves)) + copy(out, cves) + sort.SliceStable(out, func(i, j int) bool { + ai, bi, ci, di := cvePriority(out[i]) + aj, bj, cj, dj := cvePriority(out[j]) + if ai != aj { + return ai < aj + } + if bi != bj { + return bi < bj + } + if ci != cj { + return ci < cj + } + return di < dj + }) + return out +} + +// weakCryptoFirst reproduces `sorted(crypto_usage, key=lambda u: 0 if u.is_weak else 1)` +// — truthiness again, so an UNKNOWN (None) is_weak sorts with the strong +// algorithms. +func weakCryptoFirst(usages []schemas.CryptoUsage) []schemas.CryptoUsage { + out := make([]schemas.CryptoUsage, len(usages)) + copy(out, usages) + sort.SliceStable(out, func(i, j int) bool { + return truthyBool(out[i].IsWeak) && !truthyBool(out[j].IsWeak) + }) + return out +} + +// --------------------------------------------------------------------------- +// row builders shared by several projections +// --------------------------------------------------------------------------- + +// entryPointRow renders `f"{entry.kind} {entry.route or entry.identifier} ({entry.file_path}:{entry.line})"`. +func entryPointRow(entry schemas.EntryPoint) string { + return entry.Kind + " " + orStr(entry.Route, entry.Identifier) + + " (" + entry.FilePath + ":" + strconv.Itoa(entry.Line) + ")" +} + +// moduleRow renders +// `f"{module.path} ({module.language})" + (f" - {module.description}" if module.description else "")`. +func moduleRow(module schemas.Module) string { + row := module.Path + " (" + module.Language + ")" + if module.Description != nil && *module.Description != "" { + row += " - " + *module.Description + } + return row +} + +// rankedEndpointRow renders the long endpoint line shared by +// recon_context_for_auth and recon_context_for_api_security. +func rankedEndpointRow(endpoint schemas.APIEndpoint) string { + return endpoint.Method + " " + endpoint.Path + " -> " + endpoint.Handler + + " (" + endpoint.FilePath + ":" + strconv.Itoa(endpoint.Line) + + ", auth_required=" + str(endpoint.AuthRequired) + + ", rate_limited=" + str(endpoint.RateLimited) + ")" +} + +// misconfigRow renders +// `f"{m.category} at {m.file_path}:{m.line or 0}; risk={m.risk}; key={m.key or 'n/a'}"`. +func misconfigRow(misconfig schemas.MisconfigFinding) string { + return misconfig.Category + " at " + misconfig.FilePath + ":" + + strconv.Itoa(orZeroInt(misconfig.Line)) + "; risk=" + misconfig.Risk + + "; key=" + orStr(misconfig.Key, "n/a") +} + +// codebaseProfile renders the "N files, N LOC, languages=..., frameworks=..." +// clause that the injection and generic summaries share (with different +// leading words). +func codebaseProfile(recon schemas.ReconResult) string { + return strconv.Itoa(recon.FileCount) + " files, " + strconv.Itoa(recon.LinesOfCode) + + " LOC, languages=" + joinOr(recon.Languages, "unknown") + + ", frameworks=" + joinOr(recon.Frameworks, "unknown") + "." +} + +// --------------------------------------------------------------------------- +// the eight strategy projections + the generic fallback +// --------------------------------------------------------------------------- + +// ReconContextForInjection ports recon_context_for_injection. +// +// Python parity: the flow list prefers UNSANITIZED flows and only falls back to +// every flow when there are none — `unsanitized_flows if unsanitized_flows else +// recon.data_flows.flows`, a truthiness test on the filtered list. +func ReconContextForInjection(recon schemas.ReconResult) string { + flowCandidates := make([]schemas.DataFlow, 0, len(recon.DataFlows.Flows)) + for _, flow := range recon.DataFlows.Flows { + if !flow.Sanitized { + flowCandidates = append(flowCandidates, flow) + } + } + if len(flowCandidates) == 0 { + flowCandidates = recon.DataFlows.Flows + } + + entryPointRows := make([]string, 0, len(recon.Architecture.EntryPoints)) + for _, entry := range recon.Architecture.EntryPoints { + entryPointRows = append(entryPointRows, entryPointRow(entry)) + } + + sinkRows := make([]string, 0, len(recon.DataFlows.Sinks)) + for _, sink := range recon.DataFlows.Sinks { + row := sink.SinkType + " at " + sink.FilePath + ":" + strconv.Itoa(sink.Line) + if sink.FunctionName != nil && *sink.FunctionName != "" { + row += " (" + *sink.FunctionName + ")" + } + sinkRows = append(sinkRows, row) + } + + flowRows := make([]string, 0, len(flowCandidates)) + for _, flow := range flowCandidates { + flowRows = append(flowRows, flow.Source+" -> "+flow.Sink+"; sanitized="+str(flow.Sanitized)+ + "; files="+strings.Join(head(flow.Files, 3), ", ")) + } + + sanitizationRows := make([]string, 0, len(recon.DataFlows.SanitizationPoints)) + for _, point := range recon.DataFlows.SanitizationPoints { + sanitizationRows = append(sanitizationRows, point.FilePath+":"+strconv.Itoa(point.Line)+ + " type="+point.SanitizationType+" protects="+joinOr(point.ProtectsAgainst, "unspecified")) + } + + return sections( + "Injection-focused recon summary.", + "Codebase profile: "+codebaseProfile(recon), + renderList("Entry points likely to receive untrusted input", entryPointRows, MaxPrimaryItems), + renderList("High-value sinks", sinkRows, MaxPrimaryItems), + renderList("Source-to-sink flow candidates (unsanitized first)", flowRows, MaxPrimaryItems), + renderList("Known sanitization points", sanitizationRows, MaxSecondaryItems), + ) +} + +// authModuleTokens is the token tuple recon_context_for_auth scans module +// name/path/description for, in Python order (order is immaterial to `any`, but +// the transcription is kept literal). +var authModuleTokens = []string{"auth", "session", "rbac", "permission", "role", "guard", "middleware", "csrf", "jwt"} + +// authFlowTokens is the token tuple recon_context_for_auth scans data flows for. +var authFlowTokens = []string{"auth", "token", "jwt", "session", "cookie", "csrf", "role", "permission", "scope"} + +// ReconContextForAuth ports recon_context_for_auth. +func ReconContextForAuth(recon schemas.ReconResult) string { + authRelatedModules := make([]string, 0, len(recon.Architecture.Modules)) + for _, module := range recon.Architecture.Modules { + // Python: f"{module.name} {module.path} {(module.description or '')}".lower() + haystack := module.Name + " " + module.Path + " " + orStr(module.Description, "") + if containsAnyLower(haystack, authModuleTokens...) { + authRelatedModules = append(authRelatedModules, moduleRow(module)) + } + } + + endpointRows := make([]string, 0, len(recon.Architecture.APISurface)) + for _, endpoint := range rankedEndpoints(recon.Architecture.APISurface) { + endpointRows = append(endpointRows, rankedEndpointRow(endpoint)) + } + + flowRows := make([]string, 0, len(recon.DataFlows.Flows)) + for _, flow := range recon.DataFlows.Flows { + haystack := flow.Source + " " + flow.Sink + " " + strings.Join(flow.Files, " ") + if !containsAnyLower(haystack, authFlowTokens...) { + continue + } + flowRows = append(flowRows, flow.Source+" -> "+flow.Sink+ + " (files="+strings.Join(head(flow.Files, 3), ", ")+", sanitized="+str(flow.Sanitized)+")") + } + + signals := make([]string, 0, len(recon.SecurityContext.SecurityHeaders)+len(recon.SecurityContext.FrameworkSecurity)) + signals = append(signals, recon.SecurityContext.SecurityHeaders...) + signals = append(signals, recon.SecurityContext.FrameworkSecurity...) + + return sections( + "Authentication/authorization-focused recon summary.", + "Auth model: "+recon.SecurityContext.AuthModel+". Details: "+ + orStr(&recon.SecurityContext.AuthDetails, "none provided")+".", + renderList("Auth/session/RBAC modules and middleware candidates", authRelatedModules, MaxPrimaryItems), + renderList("API endpoints to validate for auth/authz coverage", endpointRows, MaxPrimaryItems), + renderList("Auth/session-relevant data flows", flowRows, MaxSecondaryItems), + renderList("Security headers and framework security signals", signals, MaxSecondaryItems), + ) +} + +// ReconContextForCrypto ports recon_context_for_crypto. +func ReconContextForCrypto(recon schemas.ReconResult) string { + usageRows := make([]string, 0, len(recon.SecurityContext.CryptoUsage)) + for _, usage := range weakCryptoFirst(recon.SecurityContext.CryptoUsage) { + usageRows = append(usageRows, "algorithm="+usage.Algorithm+ + ", key_size="+str(usage.KeySize)+ + ", mode="+strPtr(usage.Mode)+ + ", context="+orStr(usage.UsageContext, "unspecified")+ + ", is_weak="+str(usage.IsWeak)) + } + + secretRows := make([]string, 0, len(recon.Config.Secrets)) + for _, secret := range recon.Config.Secrets { + secretRows = append(secretRows, secret.SecretType+" at "+secret.FilePath+":"+ + strconv.Itoa(secret.Line)+" (confidence="+secret.Confidence+")") + } + + signals := make([]string, 0, len(recon.SecurityContext.DeploymentSignals)+len(recon.SecurityContext.SecurityHeaders)) + signals = append(signals, recon.SecurityContext.DeploymentSignals...) + signals = append(signals, recon.SecurityContext.SecurityHeaders...) + + return sections( + "Cryptography-focused recon summary.", + "Crypto usage entries: "+strconv.Itoa(len(recon.SecurityContext.CryptoUsage))+" total.", + renderList("Algorithms and key handling (weak entries first)", usageRows, MaxPrimaryItems), + renderList("Potential secret/key findings from config scan", secretRows, MaxSecondaryItems), + renderList("Deployment/TLS/security header signals", signals, MaxSecondaryItems), + ) +} + +// dataExposureFlowTokens is the sensitive-domain token tuple +// recon_context_for_data_exposure scans data flows for. +var dataExposureFlowTokens = []string{ + "password", "token", "secret", "credential", "session", "cookie", + "email", "phone", "pii", "ssn", "card", "auth", "user", +} + +// dataExposureMisconfigTokens is the token tuple the same function scans +// misconfigs for. +var dataExposureMisconfigTokens = []string{"log", "debug", "trace", "error", "verbose", "tls", "http", "exposure"} + +// ReconContextForDataExposure ports recon_context_for_data_exposure. +func ReconContextForDataExposure(recon schemas.ReconResult) string { + flowRows := make([]string, 0, len(recon.DataFlows.Flows)) + for _, flow := range recon.DataFlows.Flows { + haystack := flow.Source + " " + flow.Sink + " " + strings.Join(flow.Files, " ") + if !containsAnyLower(haystack, dataExposureFlowTokens...) { + continue + } + flowRows = append(flowRows, flow.Source+" -> "+flow.Sink+"; sanitized="+str(flow.Sanitized)+ + "; files="+strings.Join(head(flow.Files, 3), ", ")) + } + + misconfigRows := make([]string, 0, len(recon.Config.Misconfigs)) + for _, misconfig := range recon.Config.Misconfigs { + // Python: f"{m.category} {m.key or ''} {m.value or ''} {m.risk}".lower() + haystack := misconfig.Category + " " + orStr(misconfig.Key, "") + " " + + orStr(misconfig.Value, "") + " " + misconfig.Risk + if !containsAnyLower(haystack, dataExposureMisconfigTokens...) { + continue + } + misconfigRows = append(misconfigRows, misconfigRow(misconfig)) + } + + endpointRows := make([]string, 0, len(recon.Architecture.APISurface)) + for _, endpoint := range recon.Architecture.APISurface { + endpointRows = append(endpointRows, endpoint.Method+" "+endpoint.Path+ + " ("+endpoint.FilePath+":"+strconv.Itoa(endpoint.Line)+ + ", auth_required="+str(endpoint.AuthRequired)+")") + } + + return sections( + "Data exposure-focused recon summary.", + renderList("Data flows touching likely sensitive domains", flowRows, MaxPrimaryItems), + renderList("Logging/exposure-related misconfig signals", misconfigRows, MaxSecondaryItems), + renderList("Entry points and API surface with exposure risk", endpointRows, MaxSecondaryItems), + ) +} + +// ReconContextForConfigSecrets ports recon_context_for_config_secrets. +func ReconContextForConfigSecrets(recon schemas.ReconResult) string { + secretRows := make([]string, 0, len(recon.Config.Secrets)) + for _, secret := range recon.Config.Secrets { + secretRows = append(secretRows, secret.SecretType+" at "+secret.FilePath+":"+ + strconv.Itoa(secret.Line)+"; confidence="+secret.Confidence+ + "; is_test_value="+str(secret.IsTestValue)) + } + + misconfigRows := make([]string, 0, len(recon.Config.Misconfigs)) + for _, misconfig := range recon.Config.Misconfigs { + misconfigRows = append(misconfigRows, misconfigRow(misconfig)) + } + + signals := make([]string, 0, len(recon.SecurityContext.DeploymentSignals)+len(recon.SecurityContext.FrameworkSecurity)) + signals = append(signals, recon.SecurityContext.DeploymentSignals...) + signals = append(signals, recon.SecurityContext.FrameworkSecurity...) + + return sections( + "Config and secrets-focused recon summary.", + renderList("Detected secret-like findings", secretRows, MaxPrimaryItems), + renderList("Configuration weaknesses from recon", misconfigRows, MaxPrimaryItems), + renderList("Security/deployment context affecting config risk", signals, MaxSecondaryItems), + ) +} + +// ReconContextForSupplyChain ports recon_context_for_supply_chain. +func ReconContextForSupplyChain(recon schemas.ReconResult) string { + cveRows := make([]string, 0, len(recon.Dependencies.KnownCves)) + for _, cve := range prioritizedCVEs(recon.Dependencies.KnownCves) { + cveRows = append(cveRows, cve.CveID+" in "+cve.Package+" "+cve.InstalledVersion+ + " (fixed="+orStr(cve.FixedVersion, "unknown")+ + ", cvss="+str(cve.CvssV4Score)+ + ", epss="+str(cve.EpssScore)+ + ", direct="+str(cve.Direct)+ + ", reachable="+str(cve.Reachable)+")") + } + + outdatedRows := make([]string, 0, len(recon.Dependencies.Outdated)) + for _, dep := range recon.Dependencies.Outdated { + outdatedRows = append(outdatedRows, dep.Package+": "+dep.CurrentVersion+" -> "+ + dep.LatestVersion+" (direct="+str(dep.Direct)+")") + } + + // Python: sorted({f"{dep.ecosystem}: {dep.name}@{dep.version}" for dep in sbom}) + // — a SET comprehension, so duplicate entries collapse, then a lexicographic + // sort. Go's sort.Strings compares bytes, which for UTF-8 is the same order + // as Python's code-point comparison. + seen := make(map[string]struct{}, len(recon.Dependencies.Sbom)) + ecosystems := make([]string, 0, len(recon.Dependencies.Sbom)) + for _, dep := range recon.Dependencies.Sbom { + row := dep.Ecosystem + ": " + dep.Name + "@" + dep.Version + if _, dup := seen[row]; dup { + continue + } + seen[row] = struct{}{} + ecosystems = append(ecosystems, row) + } + sort.Strings(ecosystems) + + return sections( + "Supply-chain-focused recon summary.", + "Dependency inventory: direct="+strconv.Itoa(recon.Dependencies.DirectCount)+ + ", transitive="+strconv.Itoa(recon.Dependencies.TransitiveCount)+ + ", SBOM entries="+strconv.Itoa(len(recon.Dependencies.Sbom))+".", + renderList("Known CVE exposure (reachable/high severity first)", cveRows, MaxPrimaryItems), + renderList("Outdated dependencies", outdatedRows, MaxSecondaryItems), + renderList("Primary dependency ecosystems in this repo", ecosystems, MaxSecondaryItems), + ) +} + +// apiEntryKinds is the entry-point kind allowlist recon_context_for_api_security +// filters on (`entry.kind.lower() in {...}`). +var apiEntryKinds = map[string]struct{}{ + "http": {}, "api": {}, "graphql": {}, "rpc": {}, "route": {}, +} + +// ReconContextForAPISecurity ports recon_context_for_api_security. +func ReconContextForAPISecurity(recon schemas.ReconResult) string { + endpointRows := make([]string, 0, len(recon.Architecture.APISurface)) + for _, endpoint := range rankedEndpoints(recon.Architecture.APISurface) { + endpointRows = append(endpointRows, rankedEndpointRow(endpoint)) + } + + entryRows := make([]string, 0, len(recon.Architecture.EntryPoints)) + for _, entry := range recon.Architecture.EntryPoints { + if _, ok := apiEntryKinds[strings.ToLower(entry.Kind)]; !ok { + continue + } + entryRows = append(entryRows, entry.Kind+" "+orStr(entry.Route, entry.Identifier)+ + " ("+entry.FilePath+":"+strconv.Itoa(entry.Line)+ + ", auth_required="+str(entry.AuthRequired)+")") + } + + boundaryRows := make([]string, 0, len(recon.Architecture.TrustBoundaries)) + for _, boundary := range recon.Architecture.TrustBoundaries { + boundaryRows = append(boundaryRows, boundary.Name+": "+boundary.SourceZone+" -> "+ + boundary.TargetZone+"; enforcement="+joinOr(boundary.Enforcement, "none")) + } + + signals := make([]string, 0, len(recon.SecurityContext.FrameworkSecurity)+len(recon.SecurityContext.DeploymentSignals)) + signals = append(signals, recon.SecurityContext.FrameworkSecurity...) + signals = append(signals, recon.SecurityContext.DeploymentSignals...) + + return sections( + "API security-focused recon summary.", + renderList("API endpoints prioritized by missing auth/rate-limits", endpointRows, MaxPrimaryItems), + renderList("HTTP/API entry points", entryRows, MaxSecondaryItems), + renderList("Trust boundaries relevant to API calls", boundaryRows, MaxSecondaryItems), + renderList("Framework/deployment API security signals", signals, MaxSecondaryItems), + ) +} + +// ReconContextForLogic ports recon_context_for_logic — the projection +// HuntStrategy.BUSINESS_LOGIC (a.k.a. LOGIC_BUGS) uses. +func ReconContextForLogic(recon schemas.ReconResult) string { + moduleRows := make([]string, 0, len(recon.Architecture.Modules)) + for _, module := range recon.Architecture.Modules { + moduleRows = append(moduleRows, moduleRow(module)) + } + + entryRows := make([]string, 0, len(recon.Architecture.EntryPoints)) + for _, entry := range recon.Architecture.EntryPoints { + entryRows = append(entryRows, entryPointRow(entry)) + } + + // Python parity: this one slices five files, not three. + flowRows := make([]string, 0, len(recon.DataFlows.Flows)) + for _, flow := range recon.DataFlows.Flows { + flowRows = append(flowRows, flow.Source+" -> "+flow.Sink+ + "; files="+strings.Join(head(flow.Files, 5), ", ")+ + "; sanitized="+str(flow.Sanitized)) + } + + // Python parity: boundaries and services are unpacked into ONE list, so the + // 10-item cap is shared between them and every boundary row precedes every + // service row. + transitions := make([]string, 0, len(recon.Architecture.TrustBoundaries)+len(recon.Architecture.Services)) + for _, boundary := range recon.Architecture.TrustBoundaries { + transitions = append(transitions, "boundary "+boundary.Name+": "+ + boundary.SourceZone+"->"+boundary.TargetZone) + } + for _, service := range recon.Architecture.Services { + transitions = append(transitions, "service "+service.Name+": type="+service.ServiceType+ + ", endpoint="+orStr(service.Endpoint, "n/a")+ + ", auth="+orStr(service.AuthMechanism, "n/a")) + } + + return sections( + "Business-logic-focused recon summary.", + renderList("Core modules likely to implement workflows and state transitions", moduleRows, MaxPrimaryItems), + renderList("Workflow entry points", entryRows, MaxSecondaryItems), + renderList("Cross-file data/control flow candidates", flowRows, MaxPrimaryItems), + renderList("Trust boundaries and external service transitions", transitions, MaxSecondaryItems), + ) +} + +// ReconContextGeneric ports recon_context_generic — the fallback projection for +// every strategy without a dedicated builder, and the recon summary +// hunt_phase feeds to the CWE-expansion gate (reasoners/phases.py:277). +func ReconContextGeneric(recon schemas.ReconResult) string { + entryRows := make([]string, 0, len(recon.Architecture.EntryPoints)) + for _, entry := range recon.Architecture.EntryPoints { + entryRows = append(entryRows, entryPointRow(entry)) + } + + endpointRows := make([]string, 0, len(recon.Architecture.APISurface)) + for _, endpoint := range recon.Architecture.APISurface { + endpointRows = append(endpointRows, endpoint.Method+" "+endpoint.Path+ + " ("+endpoint.FilePath+":"+strconv.Itoa(endpoint.Line)+")") + } + + flowRows := make([]string, 0, len(recon.DataFlows.Flows)) + for _, flow := range recon.DataFlows.Flows { + flowRows = append(flowRows, flow.Source+" -> "+flow.Sink+"; sanitized="+str(flow.Sanitized)) + } + + return sections( + "General recon summary.", + "Profile: "+codebaseProfile(recon), + renderList("Top entry points", entryRows, MaxSecondaryItems), + renderList("Top API endpoints", endpointRows, MaxSecondaryItems), + renderList("Top data-flow candidates", flowRows, MaxSecondaryItems), + ) +} + +// --------------------------------------------------------------------------- +// hint wrappers and the strategy dispatch +// --------------------------------------------------------------------------- + +// LanguageHintsForContext ports language_hints_for_context — "Build +// language-specific hints from recon-detected languages." +func LanguageHintsForContext(recon schemas.ReconResult) string { + return GetLanguageHints(recon.Languages) +} + +// FrameworkHintsForContext ports framework_hints_for_context — "Build +// framework-specific hints from recon-detected frameworks." +func FrameworkHintsForContext(recon schemas.ReconResult) string { + return GetFrameworkHints(recon.Frameworks) +} + +// strategyBuilders ports the `builders` dict inside get_context_for_strategy. +// +// Python parity: the dict has eight entries but nine strategies reach it, +// because HuntStrategy.LOGIC_BUGS IS HuntStrategy.BUSINESS_LOGIC (an enum value +// alias). Go's schemas.HuntStrategyLogicBugs and schemas.HuntStrategyBusinessLogic +// are likewise the same constant "business_logic", so listing both here would +// be a duplicate-key compile error — the single entry covers both spellings. +var strategyBuilders = map[schemas.HuntStrategy]func(schemas.ReconResult) string{ + schemas.HuntStrategyInjection: ReconContextForInjection, + schemas.HuntStrategyAuth: ReconContextForAuth, + schemas.HuntStrategyCrypto: ReconContextForCrypto, + schemas.HuntStrategyDataExposure: ReconContextForDataExposure, + schemas.HuntStrategyConfigSecrets: ReconContextForConfigSecrets, + schemas.HuntStrategySupplyChain: ReconContextForSupplyChain, + schemas.HuntStrategyAPISecurity: ReconContextForAPISecurity, + schemas.HuntStrategyBusinessLogic: ReconContextForLogic, +} + +// GetContextForStrategy ports get_context_for_strategy: +// +// builder = builders.get(strategy, recon_context_generic) +// return builder(recon) +// +// XSS, SSRF and DOS have entries in STRATEGY_CONTEXT_MAP but NOT here, so their +// prompt context is the generic summary while their pruned dict still drops the +// sections they do not need. (Their hunter modules build their own JSON context +// block instead — agents/hunt/{xss,ssrf,dos}.py.) +func GetContextForStrategy(strategy schemas.HuntStrategy, recon schemas.ReconResult) string { + if builder, ok := strategyBuilders[strategy]; ok { + return builder(recon) + } + return ReconContextGeneric(recon) +} diff --git a/internal/secaf/recontext/context_test.go b/internal/secaf/recontext/context_test.go new file mode 100644 index 0000000000..d3001226d7 --- /dev/null +++ b/internal/secaf/recontext/context_test.go @@ -0,0 +1,542 @@ +package recontext + +// Parity tests for src/sec_af/context.py. +// +// Every expectation is a COMMITTED GOLDEN produced by running the real Python +// function over testdata/recon_fixture.json: +// +// PYTHONPATH=src ~/.agentfield/packages/sec-af/venv/bin/python go/scripts/gen_golden.py +// +// The fixture is deliberately oversized — 19 modules, 17 entry points, 20 API +// endpoints, 20 data flows, 18 sinks, 18 CVEs, 17 secrets, 16 misconfigs, 16 +// crypto entries — so that every _MAX_PRIMARY_ITEMS (15) and +// _MAX_SECONDARY_ITEMS (10) truncation fires, every stable-sort tie is +// exercised, and the "N total, showing top M" counters differ from each other. +// It also carries the awkward cases on purpose: None routes, None +// auth_required/rate_limited/reachable/is_weak, a CVE with no CVSS and no EPSS, +// empty enforcement/protects_against lists, empty-string security signals (which +// _limit's truthiness filter must DROP before counting), a misconfig with +// line=None and one with line=0, duplicate SBOM entries (which the set +// comprehension must collapse), and repeated/aliased/padded language and +// framework names. + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "sort" + "testing" + + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +func loadReconFixture(t *testing.T) schemas.ReconResult { + t.Helper() + raw, err := os.ReadFile(filepath.Join("testdata", "recon_fixture.json")) + if err != nil { + t.Fatalf("read recon_fixture.json: %v", err) + } + var recon schemas.ReconResult + if err := json.Unmarshal(raw, &recon); err != nil { + t.Fatalf("unmarshal ReconResult: %v", err) + } + return recon +} + +func golden(t *testing.T, name string) string { + t.Helper() + raw, err := os.ReadFile(filepath.Join("testdata", "golden", name)) + if err != nil { + t.Fatalf("read golden %s: %v (regenerate with go/scripts/gen_golden.py)", name, err) + } + return string(raw) +} + +func goldenJSON(t *testing.T, name string, dest any) { + t.Helper() + if err := json.Unmarshal([]byte(golden(t, name)), dest); err != nil { + t.Fatalf("parse golden %s: %v", name, err) + } +} + +// firstDiff points at the first differing line of two prompt-sized strings. +func firstDiff(want, got string) string { + wantLines, gotLines := splitLines(want), splitLines(got) + n := len(wantLines) + if len(gotLines) < n { + n = len(gotLines) + } + for i := 0; i < n; i++ { + if wantLines[i] != gotLines[i] { + w, _ := json.Marshal(wantLines[i]) + g, _ := json.Marshal(gotLines[i]) + return "first difference at line " + itoa(i+1) + "\n want: " + string(w) + "\n got: " + string(g) + } + } + if len(wantLines) != len(gotLines) { + return "line counts differ: want " + itoa(len(wantLines)) + ", got " + itoa(len(gotLines)) + } + return "(no line differs; check trailing bytes)" +} + +func splitLines(s string) []string { + out := []string{} + start := 0 + for i := 0; i < len(s); i++ { + if s[i] == '\n' { + out = append(out, s[start:i]) + start = i + 1 + } + } + return append(out, s[start:]) +} + +func itoa(n int) string { + if n == 0 { + return "0" + } + neg := n < 0 + if neg { + n = -n + } + var buf [24]byte + i := len(buf) + for n > 0 { + i-- + buf[i] = byte('0' + n%10) + n /= 10 + } + if neg { + i-- + buf[i] = '-' + } + return string(buf[i:]) +} + +// TestReconContextBuildersMatchPython is the headline parity assertion: all +// eight strategy projections plus the generic fallback, byte for byte. +func TestReconContextBuildersMatchPython(t *testing.T) { + recon := loadReconFixture(t) + + cases := []struct { + golden string + builder func(schemas.ReconResult) string + }{ + {"injection.txt", ReconContextForInjection}, + {"auth.txt", ReconContextForAuth}, + {"crypto.txt", ReconContextForCrypto}, + {"data_exposure.txt", ReconContextForDataExposure}, + {"config_secrets.txt", ReconContextForConfigSecrets}, + {"supply_chain.txt", ReconContextForSupplyChain}, + {"api_security.txt", ReconContextForAPISecurity}, + {"logic.txt", ReconContextForLogic}, + {"generic.txt", ReconContextGeneric}, + } + + for _, tc := range cases { + t.Run(tc.golden, func(t *testing.T) { + want := golden(t, tc.golden) + if got := tc.builder(recon); got != want { + t.Errorf("%s mismatch:\n%s", tc.golden, firstDiff(want, got)) + } + }) + } +} + +// TestGetContextForStrategyDispatch pins which builder every HuntStrategy value +// lands on. The golden stores the SHA-256 of the rendered text rather than the +// text itself (13 strategies x ~5 KB would bloat testdata), which is enough to +// catch a mis-wired dispatch: a strategy routed to the wrong builder produces a +// different digest, and the builders themselves are pinned byte-for-byte above. +// +// Python parity: xss, ssrf and dos have STRATEGY_CONTEXT_MAP entries but no +// builder, so they must hash to the same value as the generic summary; and +// business_logic (a.k.a. LOGIC_BUGS) must hash to recon_context_for_logic's. +func TestGetContextForStrategyDispatch(t *testing.T) { + recon := loadReconFixture(t) + + var want map[string]string + goldenJSON(t, "strategy_dispatch.json", &want) + + if len(want) != len(schemas.AllHuntStrategies) { + t.Fatalf("golden covers %d strategies, schemas.AllHuntStrategies has %d", + len(want), len(schemas.AllHuntStrategies)) + } + + for _, strategy := range schemas.AllHuntStrategies { + value := string(strategy) + sum := sha256.Sum256([]byte(GetContextForStrategy(strategy, recon))) + got := hex.EncodeToString(sum[:]) + if want[value] != got { + t.Errorf("GetContextForStrategy(%q) digest = %s, want %s", value, got, want[value]) + } + } + + // The three mapped-but-builderless strategies fall through to the generic + // summary, and business_logic does not. + genericSum := sha256.Sum256([]byte(ReconContextGeneric(recon))) + generic := hex.EncodeToString(genericSum[:]) + for _, value := range []string{"xss", "ssrf", "dos", "python_specific", "javascript_specific"} { + if want[value] != generic { + t.Errorf("strategy %q should render the generic summary", value) + } + } + logicSum := sha256.Sum256([]byte(ReconContextForLogic(recon))) + if want["business_logic"] != hex.EncodeToString(logicSum[:]) { + t.Error("business_logic should render recon_context_for_logic") + } + // The LOGIC_BUGS alias is the same constant, so it must dispatch identically. + if GetContextForStrategy(schemas.HuntStrategyLogicBugs, recon) != ReconContextForLogic(recon) { + t.Error("HuntStrategy.LOGIC_BUGS must alias BUSINESS_LOGIC") + } +} + +// TestPruneReconForStrategyKeys pins the surviving key set for every strategy +// value plus a miss and the empty string, against +// `sorted(prune_recon_for_strategy(recon, s))` in Python. +func TestPruneReconForStrategyKeys(t *testing.T) { + recon := loadReconFixture(t) + + var want map[string][]string + goldenJSON(t, "prune_keys.json", &want) + + for strategy, wantKeys := range want { + pruned, err := PruneReconForStrategy(recon, strategy) + if err != nil { + t.Fatalf("PruneReconForStrategy(%q): %v", strategy, err) + } + gotKeys := make([]string, 0, len(pruned)) + for key := range pruned { + gotKeys = append(gotKeys, key) + } + sort.Strings(gotKeys) + if len(gotKeys) != len(wantKeys) { + t.Errorf("strategy %q: keys = %v, want %v", strategy, gotKeys, wantKeys) + continue + } + for i := range gotKeys { + if gotKeys[i] != wantKeys[i] { + t.Errorf("strategy %q: keys = %v, want %v", strategy, gotKeys, wantKeys) + break + } + } + } +} + +// TestPruneReconForStrategyRendering compares the full pruned document — every +// nested value, not just the key set — with Python's json.dumps of the same +// dict. +// +// The golden is generated with only the TOP LEVEL sorted, because that is +// exactly the document pyfmt.Dumps produces here: the pruned value is a Go map +// (sorted keys — pyfmt's documented deviation) whose values are typed structs +// (declaration order, which is pydantic's model_dump() order). So a byte match +// proves the nested rendering is identical, including int-vs-float spelling +// (cvss_v4_score 10.0 stays "10.0") and every nullable field. +func TestPruneReconForStrategyRendering(t *testing.T) { + recon := loadReconFixture(t) + + for _, strategy := range []string{"injection", "crypto", "supply_chain", "config_secrets", "unknown_strategy"} { + t.Run(strategy, func(t *testing.T) { + pruned, err := PruneReconForStrategy(recon, strategy) + if err != nil { + t.Fatalf("PruneReconForStrategy: %v", err) + } + want := golden(t, "prune_"+strategy+".json") + if got := pyfmt.Dumps(pruned, 2); got != want { + t.Errorf("prune_%s.json mismatch:\n%s", strategy, firstDiff(want, got)) + } + }) + } +} + +// TestStrategyContextMapMatchesPython pins the table itself, so a typo in a +// field name is caught even for a strategy whose pruned golden is not committed. +func TestStrategyContextMapMatchesPython(t *testing.T) { + want := map[string][]string{ + "injection": {"architecture", "data_flows", "security_context"}, + "xss": {"architecture", "data_flows", "security_context"}, + "ssrf": {"architecture", "data_flows", "security_context"}, + "auth": {"architecture", "security_context"}, + "crypto": {"security_context"}, + "dos": {"architecture", "data_flows"}, + "data_exposure": {"architecture", "data_flows", "config"}, + "supply_chain": {"dependencies"}, + "config_secrets": {"config", "architecture"}, + "api_security": {"architecture", "security_context", "data_flows"}, + "business_logic": {"architecture", "data_flows", "security_context"}, + } + if len(StrategyContextMap) != len(want) { + t.Fatalf("StrategyContextMap has %d entries, want %d", len(StrategyContextMap), len(want)) + } + for key, wantFields := range want { + gotFields, ok := StrategyContextMap[key] + if !ok { + t.Errorf("StrategyContextMap missing %q", key) + continue + } + if len(gotFields) != len(wantFields) { + t.Errorf("%q: %v, want %v", key, gotFields, wantFields) + continue + } + for i := range wantFields { + if gotFields[i] != wantFields[i] { + t.Errorf("%q: %v, want %v", key, gotFields, wantFields) + break + } + } + } + + wantBase := []string{"languages", "frameworks", "lines_of_code", "file_count"} + if len(BaseReconFields) != len(wantBase) { + t.Fatalf("BaseReconFields = %v, want %v", BaseReconFields, wantBase) + } + for i := range wantBase { + if BaseReconFields[i] != wantBase[i] { + t.Fatalf("BaseReconFields = %v, want %v", BaseReconFields, wantBase) + } + } +} + +// TestLimitFiltersFalsyBeforeCounting pins _limit's least obvious behavior: the +// truthiness filter runs BEFORE len() is taken, so blanks never appear in the +// "N total" figure. +func TestLimitFiltersFalsyBeforeCounting(t *testing.T) { + rows, total := limit([]string{"a", "", "b", "", "c"}, 2) + if total != 3 { + t.Errorf("total = %d, want 3 (blanks are dropped before counting)", total) + } + if len(rows) != 2 || rows[0] != "a" || rows[1] != "b" { + t.Errorf("rows = %v, want [a b]", rows) + } + + if got, want := renderList("Title", nil, 5), "Title: none identified in recon."; got != want { + t.Errorf("empty renderList = %q, want %q", got, want) + } + if got, want := renderList("Title", []string{"", ""}, 5), "Title: none identified in recon."; got != want { + t.Errorf("all-blank renderList = %q, want %q", got, want) + } + if got, want := renderList("T", []string{"a", "b"}, 1), "T: 2 total, showing top 1:\n- a"; got != want { + t.Errorf("truncated renderList = %q, want %q", got, want) + } +} + +// TestEndpointRankKeyIsIdentityTest pins the `is False` semantics: an UNKNOWN +// (None) auth_required ranks with the protected endpoints, not the unprotected +// ones. A truthiness test would sort them together and reorder the prompt. +func TestEndpointRankKeyIsIdentityTest(t *testing.T) { + yes, no := true, false + cases := []struct { + auth, rate *bool + wantA int + wantB int + }{ + {&no, &no, 0, 0}, + {&no, &yes, 0, 1}, + {&no, nil, 0, 1}, + {nil, &no, 1, 0}, + {&yes, &yes, 1, 1}, + {nil, nil, 1, 1}, + } + for _, tc := range cases { + a, b := endpointRankKey(tc.auth, tc.rate) + if a != tc.wantA || b != tc.wantB { + t.Errorf("endpointRankKey(%v, %v) = (%d,%d), want (%d,%d)", + tc.auth, tc.rate, a, b, tc.wantA, tc.wantB) + } + } +} + +// TestRankedEndpointsIsStable pins that ties keep recon order — Python's +// sorted() is stable and the prompt's endpoint order is observable output. +func TestRankedEndpointsIsStable(t *testing.T) { + no := false + endpoints := []schemas.APIEndpoint{ + {Path: "/a"}, // rank (1,1) + {Path: "/b", AuthRequired: &no}, // rank (0,1) + {Path: "/c"}, // rank (1,1) + {Path: "/d", AuthRequired: &no, RateLimited: &no}, // rank (0,0) + {Path: "/e", AuthRequired: &no}, // rank (0,1) + } + want := []string{"/d", "/b", "/e", "/a", "/c"} + ranked := rankedEndpoints(endpoints) + for i, endpoint := range ranked { + if endpoint.Path != want[i] { + t.Fatalf("ranked order = %v, want %v", pathsOf(ranked), want) + } + } + // sorted() returns a NEW list; the caller's slice must be untouched. + if endpoints[0].Path != "/a" { + t.Error("rankedEndpoints mutated its input") + } +} + +func pathsOf(endpoints []schemas.APIEndpoint) []string { + out := make([]string, len(endpoints)) + for i, endpoint := range endpoints { + out[i] = endpoint.Path + } + return out +} + +// TestCVEPriorityDefaults pins the asymmetry between the reachable rank +// (truthiness — None ranks with False) and the score defaults (an explicit +// `is not None` check, so a missing CVSS sorts BELOW a 0.0 CVSS). +func TestCVEPriorityDefaults(t *testing.T) { + zero := 0.0 + yes := true + + _, missingCvss, missingEpss, _ := cvePriority(schemas.KnownCVE{}) + if missingCvss != 1.0 || missingEpss != 1.0 { + t.Errorf("missing scores => (%v,%v), want (1,1) — i.e. -(-1.0)", missingCvss, missingEpss) + } + _, zeroCvss, _, _ := cvePriority(schemas.KnownCVE{CvssV4Score: &zero}) + if zeroCvss != 0.0 { + t.Errorf("cvss 0.0 => %v, want 0", zeroCvss) + } + if missingCvss <= zeroCvss { + t.Error("a CVE with no CVSS must sort after one scored 0.0") + } + + rankNone, _, _, directNone := cvePriority(schemas.KnownCVE{}) + if rankNone != 1 || directNone != 1 { + t.Errorf("reachable=None/direct=false => (%d,%d), want (1,1)", rankNone, directNone) + } + rankTrue, _, _, directTrue := cvePriority(schemas.KnownCVE{Reachable: &yes, Direct: true}) + if rankTrue != 0 || directTrue != 0 { + t.Errorf("reachable=True/direct=true => (%d,%d), want (0,0)", rankTrue, directTrue) + } +} + +// TestWeakCryptoFirstIsTruthiness pins that an UNKNOWN is_weak sorts with the +// strong algorithms, and that the sort is stable. +func TestWeakCryptoFirstIsTruthiness(t *testing.T) { + yes, no := true, false + usages := []schemas.CryptoUsage{ + {Algorithm: "AES", IsWeak: &no}, + {Algorithm: "MD5", IsWeak: &yes}, + {Algorithm: "ECDSA", IsWeak: nil}, + {Algorithm: "RC4", IsWeak: &yes}, + } + want := []string{"MD5", "RC4", "AES", "ECDSA"} + for i, usage := range weakCryptoFirst(usages) { + if usage.Algorithm != want[i] { + t.Fatalf("weakCryptoFirst order = %v, want %v", algosOf(weakCryptoFirst(usages)), want) + } + } + if usages[0].Algorithm != "AES" { + t.Error("weakCryptoFirst mutated its input") + } +} + +func algosOf(usages []schemas.CryptoUsage) []string { + out := make([]string, len(usages)) + for i, usage := range usages { + out[i] = usage.Algorithm + } + return out +} + +// TestInjectionPrefersUnsanitizedFlows pins the `unsanitized or all` fallback: +// with at least one unsanitized flow only those are listed; with none, every +// flow is. +func TestInjectionPrefersUnsanitizedFlows(t *testing.T) { + recon := schemas.NewReconResult() + recon.DataFlows.Flows = []schemas.DataFlow{ + {Source: "s1", Sink: "k1", Sanitized: true, Files: []string{}}, + {Source: "s2", Sink: "k2", Sanitized: false, Files: []string{}}, + } + out := ReconContextForInjection(recon) + if !contains(out, "s2 -> k2") || contains(out, "s1 -> k1") { + t.Errorf("with an unsanitized flow present, only it should be listed:\n%s", out) + } + + recon.DataFlows.Flows = []schemas.DataFlow{ + {Source: "s1", Sink: "k1", Sanitized: true, Files: []string{}}, + } + out = ReconContextForInjection(recon) + if !contains(out, "s1 -> k1") { + t.Errorf("with no unsanitized flow, every flow should be listed:\n%s", out) + } +} + +func contains(haystack, needle string) bool { + for i := 0; i+len(needle) <= len(haystack); i++ { + if haystack[i:i+len(needle)] == needle { + return true + } + } + return false +} + +// TestOptionalRenderings pins the f-string spellings of nullable scalars, the +// place a naive port silently swaps "None" for "" or quotes a string. +func TestOptionalRenderings(t *testing.T) { + yes := true + num := 1.0 + n := 7 + s := "text" + + cases := []struct { + name string + got string + want string + }{ + {"nil bool", str((*bool)(nil)), "None"}, + {"true", str(&yes), "True"}, + {"plain bool", str(false), "False"}, + {"nil float", str((*float64)(nil)), "None"}, + {"integral float", str(&num), "1.0"}, + {"nil int", str((*int)(nil)), "None"}, + {"int", str(&n), "7"}, + {"nil str stays bare", strPtr(nil), "None"}, + {"str stays bare (not quoted)", strPtr(&s), "text"}, + } + for _, tc := range cases { + if tc.got != tc.want { + t.Errorf("%s = %q, want %q", tc.name, tc.got, tc.want) + } + } + + empty := "" + if got := orStr(&empty, "n/a"); got != "n/a" { + t.Errorf("orStr(\"\") = %q, want the fallback (empty strings are falsy)", got) + } + zero := 0 + if got := orZeroInt(&zero); got != 0 { + t.Errorf("orZeroInt(0) = %d, want 0", got) + } + if got := orZeroInt(nil); got != 0 { + t.Errorf("orZeroInt(nil) = %d, want 0", got) + } + if got := joinOr(nil, "unknown"); got != "unknown" { + t.Errorf("joinOr(nil) = %q, want unknown", got) + } + if got := joinOr([]string{""}, "unknown"); got != "unknown" { + t.Errorf("joinOr([\"\"]) = %q, want unknown (the join is empty, hence falsy)", got) + } +} + +// TestPruneReconForStrategyDoesNotAliasBaseFields guards the include-set build: +// a mapped strategy must never drop a base field, and must never leak +// recon_duration_seconds. +func TestPruneReconForStrategyDoesNotAliasBaseFields(t *testing.T) { + recon := loadReconFixture(t) + for strategy := range StrategyContextMap { + pruned, err := PruneReconForStrategy(recon, strategy) + if err != nil { + t.Fatalf("PruneReconForStrategy(%q): %v", strategy, err) + } + for _, base := range BaseReconFields { + if _, ok := pruned[base]; !ok { + t.Errorf("strategy %q dropped base field %q", strategy, base) + } + } + if _, ok := pruned["recon_duration_seconds"]; ok { + t.Errorf("strategy %q leaked recon_duration_seconds", strategy) + } + } +} diff --git a/internal/secaf/recontext/hints.go b/internal/secaf/recontext/hints.go new file mode 100644 index 0000000000..e45d567556 --- /dev/null +++ b/internal/secaf/recontext/hints.go @@ -0,0 +1,510 @@ +package recontext + +import "strings" + +// This file ports two Python modules verbatim: +// +// src/sec_af/agents/hunt/_language_hints.py (LANGUAGE_PATTERNS, get_language_hints) +// src/sec_af/agents/hunt/_framework_hints.py (FRAMEWORK_PATTERNS, _FRAMEWORK_ALIASES, +// _normalize_framework, get_framework_hints) +// +// WHY THEY LIVE HERE AND NOT IN internal/agents/hunt +// ------------------------------------------------- +// In Python the dependency runs hunt -> context: `sec_af/context.py` imports +// `get_framework_hints`/`get_language_hints` out of the `sec_af.agents.hunt` +// package, and each hunter module imports `language_hints_for_context` back out +// of `sec_af.context`. Python tolerates that because `agents/hunt/__init__.py` +// is not executed by importing the two leaf hint modules. +// +// Go has no such escape hatch: `internal/agents/hunt` will import +// `internal/recontext` (every hunter needs recon_context_for_*), so if the hint +// tables lived in the hunt package the two packages would import each other and +// the build would fail with an import cycle. The tables are therefore hosted +// here — the package that both sides already depend on — and exported as +// GetLanguageHints / GetFrameworkHints so `internal/agents/hunt` can call them +// directly wherever Python calls them directly. +// +// The table CONTENT is byte-exact with the Python source; every string below is +// a transcription, and the golden tests in hints_test.go compare the rendered +// output against the Python functions run over the same inputs. + +// LanguagePattern is one entry of Python's LANGUAGE_PATTERNS. +// +// Python types it `dict[str, list[str] | str]` and reads it with `.get(key)`, +// so a missing key and an empty list behave identically (both falsy, both skip +// the line). A Go struct with zero values reproduces that exactly. +type LanguagePattern struct { + InjectionSinks []string + SafePatterns []string + DoNotFlag []string + FrameworkSpecifics string +} + +// LanguagePatterns ports _language_hints.py LANGUAGE_PATTERNS. +// +// Keyed by the LOWERCASE language name, because get_language_hints lowercases +// every detected language before the lookup. +var LanguagePatterns = map[string]LanguagePattern{ + "python": { + InjectionSinks: []string{"cursor.execute", "os.system", "subprocess.run", "eval", "exec", "pickle.loads"}, + SafePatterns: []string{ + "parameterized queries with %s placeholders", + "subprocess.run with list args (no shell=True)", + }, + DoNotFlag: []string{ + "Django ORM queries (uses parameterized queries internally)", + "SQLAlchemy text() with bound params", + }, + FrameworkSpecifics: "Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes.", + }, + "javascript": { + InjectionSinks: []string{"eval", "Function()", "innerHTML", "document.write", "child_process.exec", "new Function"}, + SafePatterns: []string{"DOMPurify.sanitize()", "textContent assignment", "parameterized pg queries"}, + DoNotFlag: []string{"React JSX expressions (auto-escaped)", "Angular template bindings (sanitized by default)"}, + FrameworkSpecifics: "Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous.", + }, + "typescript": { + InjectionSinks: []string{"eval", "Function()", "innerHTML", "document.write", "child_process.exec"}, + SafePatterns: []string{"DOMPurify.sanitize()", "textContent assignment", "Prisma parameterized queries"}, + DoNotFlag: []string{"React JSX expressions", "Angular template bindings", "Prisma ORM queries"}, + FrameworkSpecifics: "TypeScript adds type safety but doesn't prevent injection. Check for any type assertions near user input.", + }, + "go": { + InjectionSinks: []string{"fmt.Sprintf into SQL", "exec.Command with user input", "template.HTML()", "os.Exec"}, + SafePatterns: []string{ + "database/sql with ? placeholders", + "html/template (auto-escapes)", + "exec.Command with separate args", + }, + DoNotFlag: []string{ + "GORM parameterized queries", + "html/template default escaping", + "database/sql prepared statements", + }, + FrameworkSpecifics: "Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML.", + }, + "java": { + InjectionSinks: []string{ + "Statement.execute", + "Runtime.exec", + "ProcessBuilder with concatenated strings", + "ScriptEngine.eval", + }, + SafePatterns: []string{"PreparedStatement with ?", "JNDI lookup with allowlist", "OWASP ESAPI encoding"}, + DoNotFlag: []string{"JPA/Hibernate named parameters", "Spring Security CSRF protection", "PreparedStatement usage"}, + FrameworkSpecifics: "Check Spring Boot auto-config. Thymeleaf auto-escapes. JSP needs explicit escaping.", + }, + "ruby": { + InjectionSinks: []string{"eval", "system", "exec", "send", "public_send", "ERB.new with user input"}, + SafePatterns: []string{"ActiveRecord parameterized queries", "Rack::Utils.escape_html", "sanitize helper in Rails"}, + DoNotFlag: []string{ + "ActiveRecord where with hash conditions", + "Rails CSRF protection", + "Rails html_safe on constants", + }, + FrameworkSpecifics: "Rails auto-escapes ERB templates. raw/html_safe bypasses escaping. Check for mass assignment.", + }, + "csharp": { + InjectionSinks: []string{"SqlCommand with concatenation", "Process.Start with user input", "Razor @Html.Raw()"}, + SafePatterns: []string{"SqlParameter", "Entity Framework LINQ", "Razor auto-encoding"}, + DoNotFlag: []string{"Entity Framework LINQ queries", "ASP.NET anti-forgery tokens", "Razor default encoding"}, + FrameworkSpecifics: "ASP.NET Core Razor auto-encodes. @Html.Raw() is dangerous. Check for [ValidateAntiForgeryToken].", + }, +} + +// GetLanguageHints ports _language_hints.py get_language_hints: +// +// detected = [lang.lower() for lang in languages] +// hints = [] +// for lang in detected: +// patterns = LANGUAGE_PATTERNS.get(lang) +// if patterns is None: continue +// section = [f"Language: {lang.upper()}"] +// ... four optional lines ... +// hints.append("\n".join(section)) +// if not hints: return "No language-specific hints available for detected languages." +// return "LANGUAGE-SPECIFIC GUIDANCE:\n" + "\n\n".join(hints) +// +// Python parity notes: +// +// - There is NO deduplication, unlike get_framework_hints. `["Python", +// "python"]` emits the PYTHON section twice, and this reproduces that. +// - The heading uses `lang.upper()` on the ALREADY-LOWERCASED name, so the +// spelling always comes from the table key, never from the caller's casing. +// - Go's strings.ToLower/ToUpper are Unicode-aware like Python's +// str.lower()/str.upper(), with the usual rare divergences on characters +// whose case mapping changes length (Python maps U+0130 to two code points, +// Go to one). No language name in the table is affected. +func GetLanguageHints(languages []string) string { + hints := make([]string, 0, len(languages)) + for _, raw := range languages { + lang := strings.ToLower(raw) + patterns, ok := LanguagePatterns[lang] + if !ok { + continue + } + section := []string{"Language: " + strings.ToUpper(lang)} + if len(patterns.InjectionSinks) > 0 { + section = append(section, " Key sinks: "+strings.Join(patterns.InjectionSinks, ", ")) + } + if len(patterns.SafePatterns) > 0 { + section = append(section, " Safe patterns (skip these): "+strings.Join(patterns.SafePatterns, ", ")) + } + if len(patterns.DoNotFlag) > 0 { + section = append(section, " DO NOT FLAG: "+strings.Join(patterns.DoNotFlag, ", ")) + } + if patterns.FrameworkSpecifics != "" { + section = append(section, " Framework notes: "+patterns.FrameworkSpecifics) + } + hints = append(hints, strings.Join(section, "\n")) + } + if len(hints) == 0 { + return "No language-specific hints available for detected languages." + } + return "LANGUAGE-SPECIFIC GUIDANCE:\n" + strings.Join(hints, "\n\n") +} + +// FrameworkPattern is one entry of Python's FRAMEWORK_PATTERNS +// (`dict[str, dict[str, list[str]]]` — every entry has all four keys, and +// get_framework_hints indexes them directly rather than using .get()). +type FrameworkPattern struct { + SecurityFeatures []string + DoNotFlag []string + WatchFor []string + CommonVulns []string +} + +// FrameworkPatterns ports _framework_hints.py FRAMEWORK_PATTERNS, keyed by the +// NORMALIZED framework name (see normalizeFramework). +var FrameworkPatterns = map[string]FrameworkPattern{ + "django": { + SecurityFeatures: []string{ + "ORM queries are parameterized by default", + "Templates auto-escape variables by default", + "CsrfViewMiddleware enforces CSRF tokens for unsafe methods", + }, + DoNotFlag: []string{ + "Django ORM filter/exclude (parameterized)", + "CSRF with CsrfViewMiddleware active", + "XSS in Django templates (auto-escaped)", + }, + WatchFor: []string{"raw() queries", "mark_safe()", "|safe filter", "CSRF_COOKIE_SECURE=False"}, + CommonVulns: []string{ + "SQL injection via raw SQL and string formatting", + "XSS via unsafe template escape bypasses", + "CSRF weakening via middleware/settings overrides", + }, + }, + "flask": { + SecurityFeatures: []string{ + "Jinja2 auto-escapes in HTML templates", + "Werkzeug handles request parsing safely by default", + "Blueprint/middleware patterns can centralize auth checks", + }, + DoNotFlag: []string{ + "Jinja2 auto-escaped template variables", + "Parameterized SQLAlchemy query usage", + "Server-side session signing with strong SECRET_KEY", + }, + WatchFor: []string{ + "render_template_string() with user input", + "debug=True in production paths", + "string-concatenated SQL in execute()", + "hardcoded SECRET_KEY", + }, + CommonVulns: []string{ + "SSTI through dynamic template rendering", + "XSS when auto-escaping is bypassed", + "session tampering risk with weak secret/config", + }, + }, + "fastapi": { + SecurityFeatures: []string{ + "Pydantic request validation reduces malformed input", + "Dependency injection supports reusable auth guards", + "OpenAPI schema generation improves contract visibility", + }, + DoNotFlag: []string{ + "Pydantic model validation errors", + "Dependency-based auth checks clearly enforced", + "Parameterized SQLAlchemy/async driver usage", + }, + WatchFor: []string{ + "Depends() omitted on privileged routes", + "raw SQL in text()/execute() with interpolation", + "CORS allow_origins=['*'] with credentials", + "unsafe deserialization in background tasks", + }, + CommonVulns: []string{ + "Auth bypass on unprotected routes", + "SQL injection in manually assembled queries", + "CORS misconfiguration exposing credentialed APIs", + }, + }, + "express": { + SecurityFeatures: []string{ + "Router middleware can enforce auth and rate limits", + "helmet can apply secure HTTP headers", + "Validated schema middleware can constrain input", + }, + DoNotFlag: []string{ + "Parameterized ORM/database queries", + "helmet defaults correctly applied", + "router-level auth middleware consistently enforced", + }, + WatchFor: []string{ + "res.send()/res.json() leaking sensitive internals", + "string-built SQL in query()", + "trust proxy misconfiguration", + "open CORS with credentials", + }, + CommonVulns: []string{ + "Authz gaps from missing middleware on routes", + "NoSQL/SQL injection from unsanitized request bodies", + "Open redirect and SSRF through unvalidated URLs", + }, + }, + "nextjs": { + SecurityFeatures: []string{ + "React JSX auto-escapes output by default", + "API routes can share centralized auth middleware", + "Server/client boundaries reduce accidental secret exposure", + }, + DoNotFlag: []string{ + "React JSX escaped rendering", + "getServerSideProps/getServerSession with proper auth checks", + "Typed route handlers with validated schema guards", + }, + WatchFor: []string{ + "dangerouslySetInnerHTML", + "unprotected API routes under /api", + "secret leakage to client bundles", + "rewrites/redirects from untrusted user input", + }, + CommonVulns: []string{ + "XSS through dangerous HTML rendering", + "IDOR/auth bypass in API handlers", + "Sensitive env/config exposure in client-side code", + }, + }, + "spring": { + SecurityFeatures: []string{ + "Spring Security provides auth, CSRF, and filter chain defaults", + "JPA/Hibernate prepared parameter binding by default", + "Bean validation can enforce request constraints", + }, + DoNotFlag: []string{ + "PreparedStatement/JPA named parameter usage", + "Spring Security CSRF/auth filters clearly active", + "Thymeleaf auto-escaped output", + }, + WatchFor: []string{ + "@PreAuthorize missing on sensitive methods", + "JdbcTemplate/raw Statement with concatenation", + "csrf().disable() on browser session flows", + "Actuator endpoints exposed without auth", + }, + CommonVulns: []string{ + "Authz bypass from weak method-level security", + "SQL injection in raw JDBC queries", + "Sensitive management endpoint exposure", + }, + }, + "rails": { + SecurityFeatures: []string{ + "ActiveRecord parameterization for hash/array queries", + "ERB templates auto-escape by default", + "Built-in CSRF protection for non-GET requests", + }, + DoNotFlag: []string{ + "ActiveRecord where with hash conditions", + "Rails protect_from_forgery active", + "ERB escaped output without raw/html_safe", + }, + WatchFor: []string{ + "where/order/find_by_sql with string interpolation", + "raw()/html_safe on untrusted content", + "skip_before_action on auth filters", + "mass assignment via permit!", + }, + CommonVulns: []string{ + "SQL injection in manual query fragments", + "XSS via unsafe output helpers", + "Authz bypass from skipped controller guards", + }, + }, + "aspnet": { + SecurityFeatures: []string{ + "Razor encodes output by default", + "Model binding and data annotations aid validation", + "Anti-forgery tokens support CSRF defense", + }, + DoNotFlag: []string{ + "Entity Framework LINQ parameterized queries", + "Razor default HTML encoding", + "ValidateAntiForgeryToken in state-changing MVC actions", + }, + WatchFor: []string{ + "Html.Raw() on untrusted input", + "FromBody models without validation", + "Authorize missing on privileged endpoints", + "custom SQL built via string interpolation", + }, + CommonVulns: []string{ + "XSS through Html.Raw and unencoded output", + "Auth/authz bypass on unsecured controllers", + "SQL injection in handcrafted query strings", + }, + }, + "react": { + SecurityFeatures: []string{ + "JSX escapes strings before DOM rendering", + "Component model encourages explicit data flow", + "Framework discourages direct DOM mutation", + }, + DoNotFlag: []string{ + "Standard JSX expression rendering", + "textContent assignment for untrusted text", + "sanitized HTML via trusted DOMPurify policy", + }, + WatchFor: []string{ + "dangerouslySetInnerHTML", + "untrusted URL assignment to href/src", + "eval/new Function in client code", + "token/secret exposure in bundles", + }, + CommonVulns: []string{ + "DOM XSS through unsafe HTML injection", + "Open redirect via unvalidated navigation targets", + "Sensitive data exposure in frontend artifacts", + }, + }, + "vue": { + SecurityFeatures: []string{ + "Mustache template interpolation escapes HTML", + "Component props/events provide explicit boundaries", + "Router guards can enforce auth flows", + }, + DoNotFlag: []string{ + "escaped template interpolation {{ value }}", + "validated route guards protecting private routes", + "sanitized content rendered through safe components", + }, + WatchFor: []string{ + "v-html with untrusted data", + "dynamic component/template compilation", + "unsafe URL bindings in href/src", + "client-side auth checks without server enforcement", + }, + CommonVulns: []string{ + "XSS through v-html and unsafe render paths", + "Auth bypass from client-only route protection", + "Open redirect patterns in router navigation", + }, + }, + "angular": { + SecurityFeatures: []string{ + "Template binding sanitization for HTML/URL contexts", + "HttpClient and interceptor patterns support central controls", + "AOT compilation limits runtime template injection vectors", + }, + DoNotFlag: []string{ + "default Angular template binding sanitization", + "HttpClient usage with validated request schemas", + "route guards consistently applied", + }, + WatchFor: []string{ + "bypassSecurityTrustHtml/Url/Script", + "[innerHTML] with unsanitized input", + "direct DOM APIs via ElementRef/nativeElement", + "auth only in client guard without server checks", + }, + CommonVulns: []string{ + "XSS when sanitizer is explicitly bypassed", + "Token leakage in local storage/logging", + "Authorization gaps due to client-only enforcement", + }, + }, +} + +// frameworkAliases ports _framework_hints.py _FRAMEWORK_ALIASES. +var frameworkAliases = map[string]string{ + "next": "nextjs", + "next.js": "nextjs", + "springboot": "spring", + "spring-boot": "spring", + "spring boot": "spring", + "asp.net": "aspnet", + "asp.net core": "aspnet", + "aspnetcore": "aspnet", + "asp net": "aspnet", + "ruby on rails": "rails", +} + +// normalizeFramework ports _normalize_framework: +// +// lowered = value.strip().lower() +// return _FRAMEWORK_ALIASES.get(lowered, lowered) +// +// Python parity: `str.strip()` with no argument strips Unicode whitespace; +// strings.TrimSpace strips the same class (unicode.IsSpace). The two differ +// only on a handful of exotic code points that no framework name contains. +func normalizeFramework(value string) string { + lowered := strings.ToLower(strings.TrimSpace(value)) + if alias, ok := frameworkAliases[lowered]; ok { + return alias + } + return lowered +} + +// GetFrameworkHints ports _framework_hints.py get_framework_hints. +// +// Python parity: the normalized names are deduplicated while PRESERVING FIRST +// APPEARANCE ORDER (Python builds `ordered_unique` with a linear `in` check on +// a list, not a set), so `["Next", "next.js"]` yields one NEXTJS section, and +// unknown frameworks silently drop out of the output but still consume their +// slot in the dedup list. +func GetFrameworkHints(frameworks []string) string { + orderedUnique := make([]string, 0, len(frameworks)) + for _, framework := range frameworks { + normalized := normalizeFramework(framework) + seen := false + for _, existing := range orderedUnique { + if existing == normalized { + seen = true + break + } + } + if seen { + continue + } + orderedUnique = append(orderedUnique, normalized) + } + + sections := make([]string, 0, len(orderedUnique)) + for _, framework := range orderedUnique { + patterns, ok := FrameworkPatterns[framework] + if !ok { + continue + } + section := []string{"Framework: " + strings.ToUpper(framework)} + if len(patterns.SecurityFeatures) > 0 { + section = append(section, " Security features: "+strings.Join(patterns.SecurityFeatures, ", ")) + } + if len(patterns.DoNotFlag) > 0 { + section = append(section, " DO NOT FLAG: "+strings.Join(patterns.DoNotFlag, ", ")) + } + if len(patterns.WatchFor) > 0 { + section = append(section, " Watch for: "+strings.Join(patterns.WatchFor, ", ")) + } + if len(patterns.CommonVulns) > 0 { + section = append(section, " Common vulns: "+strings.Join(patterns.CommonVulns, ", ")) + } + sections = append(sections, strings.Join(section, "\n")) + } + + if len(sections) == 0 { + return "No framework-specific hints available for detected frameworks." + } + return "FRAMEWORK-SPECIFIC GUIDANCE:\n" + strings.Join(sections, "\n\n") +} diff --git a/internal/secaf/recontext/hints_test.go b/internal/secaf/recontext/hints_test.go new file mode 100644 index 0000000000..2af6dec546 --- /dev/null +++ b/internal/secaf/recontext/hints_test.go @@ -0,0 +1,153 @@ +package recontext + +// Parity tests for the two hint tables ported from +// src/sec_af/agents/hunt/_language_hints.py and _framework_hints.py. +// +// The input vectors below are the SAME literals go/scripts/gen_golden.py uses +// (LANGUAGE_HINT_CASES / FRAMEWORK_HINT_CASES); if the two drift apart the +// golden comparison fails, which is the intended alarm. + +import "testing" + +// languageHintCases mirrors gen_golden.py's LANGUAGE_HINT_CASES. +var languageHintCases = []struct { + name string + languages []string +}{ + {"empty", []string{}}, + {"unknown_only", []string{"Rust", "haskell"}}, + {"single", []string{"Python"}}, + {"mixed_case_and_repeat", []string{"Python", "python", "JavaScript", "Rust", "GO"}}, + {"all_known", []string{"python", "javascript", "typescript", "go", "java", "ruby", "csharp"}}, +} + +// frameworkHintCases mirrors gen_golden.py's FRAMEWORK_HINT_CASES. +var frameworkHintCases = []struct { + name string + frameworks []string +}{ + {"empty", []string{}}, + {"unknown_only", []string{"hanami", "phoenix"}}, + {"aliases", []string{"Next", "next.js", "NEXTJS", "Spring Boot", "spring-boot", "ASP.NET Core"}}, + {"padded", []string{" React ", "\tvue\n", "Django"}}, + {"all_known", []string{ + "django", "flask", "fastapi", "express", "nextjs", + "spring", "rails", "aspnet", "react", "vue", "angular", + }}, +} + +// TestGetLanguageHintsMatchesPython pins the whole LANGUAGE_PATTERNS table +// through its rendering — every sink, safe pattern, do-not-flag entry and +// framework note of all seven languages appears in the all_known golden. +// +// The mixed_case_and_repeat case is the interesting one: get_language_hints +// does NOT deduplicate, so "Python" and "python" each emit a PYTHON section, +// and "Rust" (no table entry) contributes nothing. +func TestGetLanguageHintsMatchesPython(t *testing.T) { + for _, tc := range languageHintCases { + t.Run(tc.name, func(t *testing.T) { + want := golden(t, "language_hints_"+tc.name+".txt") + if got := GetLanguageHints(tc.languages); got != want { + t.Errorf("GetLanguageHints(%v) mismatch:\n%s", tc.languages, firstDiff(want, got)) + } + }) + } +} + +// TestGetFrameworkHintsMatchesPython pins FRAMEWORK_PATTERNS and the alias +// table. The aliases case proves Next/next.js/NEXTJS collapse to ONE NEXTJS +// section and Spring Boot/spring-boot to one SPRING; the padded case proves +// `.strip().lower()` runs before the alias lookup. +func TestGetFrameworkHintsMatchesPython(t *testing.T) { + for _, tc := range frameworkHintCases { + t.Run(tc.name, func(t *testing.T) { + want := golden(t, "framework_hints_"+tc.name+".txt") + if got := GetFrameworkHints(tc.frameworks); got != want { + t.Errorf("GetFrameworkHints(%v) mismatch:\n%s", tc.frameworks, firstDiff(want, got)) + } + }) + } +} + +// TestHintsForContextWrappers pins language_hints_for_context / +// framework_hints_for_context, which are the functions every hunter actually +// calls: they feed the ReconResult's own language and framework lists — which +// in the fixture are deliberately messy ("Python" twice, " React " padded, +// "next.js" and "NEXT" aliasing to the same entry, "unknown-fw" unmatched). +func TestHintsForContextWrappers(t *testing.T) { + recon := loadReconFixture(t) + + if got, want := LanguageHintsForContext(recon), golden(t, "language_hints_for_context.txt"); got != want { + t.Errorf("LanguageHintsForContext mismatch:\n%s", firstDiff(want, got)) + } + if got, want := FrameworkHintsForContext(recon), golden(t, "framework_hints_for_context.txt"); got != want { + t.Errorf("FrameworkHintsForContext mismatch:\n%s", firstDiff(want, got)) + } +} + +// TestHintTableKeys guards the table membership itself, so a dropped or +// misspelled key is caught even when no golden covers it. +func TestHintTableKeys(t *testing.T) { + wantLanguages := []string{"python", "javascript", "typescript", "go", "java", "ruby", "csharp"} + if len(LanguagePatterns) != len(wantLanguages) { + t.Errorf("LanguagePatterns has %d entries, want %d", len(LanguagePatterns), len(wantLanguages)) + } + for _, key := range wantLanguages { + if _, ok := LanguagePatterns[key]; !ok { + t.Errorf("LanguagePatterns missing %q", key) + } + } + + wantFrameworks := []string{ + "django", "flask", "fastapi", "express", "nextjs", + "spring", "rails", "aspnet", "react", "vue", "angular", + } + if len(FrameworkPatterns) != len(wantFrameworks) { + t.Errorf("FrameworkPatterns has %d entries, want %d", len(FrameworkPatterns), len(wantFrameworks)) + } + for _, key := range wantFrameworks { + if _, ok := FrameworkPatterns[key]; !ok { + t.Errorf("FrameworkPatterns missing %q", key) + } + } +} + +// TestNormalizeFramework pins _normalize_framework's alias table one entry at a +// time, including the identity case for a name that is not an alias. +func TestNormalizeFramework(t *testing.T) { + cases := map[string]string{ + "next": "nextjs", + "Next": "nextjs", + "next.js": "nextjs", + " NEXT.JS ": "nextjs", + "springboot": "spring", + "spring-boot": "spring", + "spring boot": "spring", + "asp.net": "aspnet", + "asp.net core": "aspnet", + "aspnetcore": "aspnet", + "asp net": "aspnet", + "ruby on rails": "rails", + "django": "django", + "Django": "django", + "unknown-fw": "unknown-fw", + "": "", + } + for in, want := range cases { + if got := normalizeFramework(in); got != want { + t.Errorf("normalizeFramework(%q) = %q, want %q", in, got, want) + } + } +} + +// TestNoHintsFallbacks pins the two "nothing matched" sentences, which are the +// literal strings the prompt templates end up carrying for an unrecognized +// stack. +func TestNoHintsFallbacks(t *testing.T) { + if got, want := GetLanguageHints(nil), "No language-specific hints available for detected languages."; got != want { + t.Errorf("GetLanguageHints(nil) = %q, want %q", got, want) + } + if got, want := GetFrameworkHints(nil), "No framework-specific hints available for detected frameworks."; got != want { + t.Errorf("GetFrameworkHints(nil) = %q, want %q", got, want) + } +} diff --git a/internal/secaf/recontext/testdata/golden/api_security.txt b/internal/secaf/recontext/testdata/golden/api_security.txt new file mode 100644 index 0000000000..1481253500 --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/api_security.txt @@ -0,0 +1,54 @@ +API security-focused recon summary. + +API endpoints prioritized by missing auth/rate-limits: 20 total, showing top 15: +- POST /api/v1/thing/1 -> ThingController.action1 (app/api/thing_1.py:103, auth_required=False, rate_limited=False) +- DELETE /api/v1/thing/3 -> ThingController.action3 (app/api/thing_3.py:109, auth_required=False, rate_limited=False) +- PUT /api/v1/thing/7 -> ThingController.action7 (app/api/thing_7.py:121, auth_required=False, rate_limited=False) +- PATCH /api/v1/thing/9 -> ThingController.action9 (app/api/thing_9.py:127, auth_required=False, rate_limited=False) +- DELETE /api/v1/thing/13 -> ThingController.action13 (app/api/thing_13.py:139, auth_required=False, rate_limited=False) +- GET /api/v1/thing/15 -> ThingController.action15 (app/api/thing_15.py:145, auth_required=False, rate_limited=False) +- PATCH /api/v1/thing/19 -> ThingController.action19 (app/api/thing_19.py:157, auth_required=False, rate_limited=False) +- GET /api/v1/thing/0 -> ThingController.action0 (app/api/thing_0.py:100, auth_required=True, rate_limited=None) +- PUT /api/v1/thing/2 -> ThingController.action2 (app/api/thing_2.py:106, auth_required=None, rate_limited=True) +- PATCH /api/v1/thing/4 -> ThingController.action4 (app/api/thing_4.py:112, auth_required=True, rate_limited=None) +- GET /api/v1/thing/5 -> ThingController.action5 (app/api/thing_5.py:115, auth_required=None, rate_limited=True) +- POST /api/v1/thing/6 -> ThingController.action6 (app/api/thing_6.py:118, auth_required=True, rate_limited=None) +- DELETE /api/v1/thing/8 -> ThingController.action8 (app/api/thing_8.py:124, auth_required=None, rate_limited=True) +- GET /api/v1/thing/10 -> ThingController.action10 (app/api/thing_10.py:130, auth_required=True, rate_limited=None) +- POST /api/v1/thing/11 -> ThingController.action11 (app/api/thing_11.py:133, auth_required=None, rate_limited=True) + +HTTP/API entry points: 11 total, showing top 10: +- http handler_0 (app/entry/e0.py:10, auth_required=True) +- HTTP /v1/resource/1 (app/entry/e1.py:17, auth_required=False) +- api handler_3 (app/entry/e3.py:31, auth_required=True) +- graphql /v1/resource/4 (app/entry/e4.py:38, auth_required=False) +- rpc /v1/resource/5 (app/entry/e5.py:45, auth_required=None) +- route handler_6 (app/entry/e6.py:52, auth_required=True) +- http handler_9 (app/entry/e9.py:73, auth_required=True) +- api /v1/resource/10 (app/entry/e10.py:80, auth_required=False) +- http handler_12 (app/entry/e12.py:94, auth_required=True) +- route /v1/resource/13 (app/entry/e13.py:101, auth_required=False) + +Trust boundaries relevant to API calls: 12 total, showing top 10: +- boundary_0: internet -> app; enforcement=none +- boundary_1: dmz -> db; enforcement=waf_1, mtls_1 +- boundary_2: vpc -> cache; enforcement=waf_2, mtls_2 +- boundary_3: worker -> queue; enforcement=waf_3, mtls_3 +- boundary_4: internet -> app; enforcement=none +- boundary_5: dmz -> db; enforcement=waf_5, mtls_5 +- boundary_6: vpc -> cache; enforcement=waf_6, mtls_6 +- boundary_7: worker -> queue; enforcement=waf_7, mtls_7 +- boundary_8: internet -> app; enforcement=none +- boundary_9: dmz -> db; enforcement=waf_9, mtls_9 + +Framework/deployment API security signals: 13 total, showing top 10: +- django.middleware.csrf.CsrfViewMiddleware +- helmet defaults +- SecurityMiddleware +- rack-protection +- spring-security filter chain +- kubernetes ingress with TLS termination +- docker-compose exposes 5432 +- no WAF in front of /api +- secrets mounted from vault +- readiness probe on /healthz \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/auth.txt b/internal/secaf/recontext/testdata/golden/auth.txt new file mode 100644 index 0000000000..5b5f5864f3 --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/auth.txt @@ -0,0 +1,49 @@ +Authentication/authorization-focused recon summary. + +Auth model: jwt. Details: HS256 JWTs issued by app/auth/service.py, refresh tokens in Redis. + +Auth/session/RBAC modules and middleware candidates: 8 total, showing top 8: +- app/auth/service.py (python) - Session and JWT issuance +- app/auth/rbac.py (python) +- app/web/middleware/csrf.js (javascript) - CSRF middleware +- app/auth/session_store.py (python) - Redis-backed sessions +- app/auth/permissions.py (python) - Role → permission table +- app/common/guard.py (python) - Assorted guard helpers +- db/migrate/2024_add_roles.rb (ruby) - Adds role column +- app/common/jwt_tools.py (python) + +API endpoints to validate for auth/authz coverage: 20 total, showing top 15: +- POST /api/v1/thing/1 -> ThingController.action1 (app/api/thing_1.py:103, auth_required=False, rate_limited=False) +- DELETE /api/v1/thing/3 -> ThingController.action3 (app/api/thing_3.py:109, auth_required=False, rate_limited=False) +- PUT /api/v1/thing/7 -> ThingController.action7 (app/api/thing_7.py:121, auth_required=False, rate_limited=False) +- PATCH /api/v1/thing/9 -> ThingController.action9 (app/api/thing_9.py:127, auth_required=False, rate_limited=False) +- DELETE /api/v1/thing/13 -> ThingController.action13 (app/api/thing_13.py:139, auth_required=False, rate_limited=False) +- GET /api/v1/thing/15 -> ThingController.action15 (app/api/thing_15.py:145, auth_required=False, rate_limited=False) +- PATCH /api/v1/thing/19 -> ThingController.action19 (app/api/thing_19.py:157, auth_required=False, rate_limited=False) +- GET /api/v1/thing/0 -> ThingController.action0 (app/api/thing_0.py:100, auth_required=True, rate_limited=None) +- PUT /api/v1/thing/2 -> ThingController.action2 (app/api/thing_2.py:106, auth_required=None, rate_limited=True) +- PATCH /api/v1/thing/4 -> ThingController.action4 (app/api/thing_4.py:112, auth_required=True, rate_limited=None) +- GET /api/v1/thing/5 -> ThingController.action5 (app/api/thing_5.py:115, auth_required=None, rate_limited=True) +- POST /api/v1/thing/6 -> ThingController.action6 (app/api/thing_6.py:118, auth_required=True, rate_limited=None) +- DELETE /api/v1/thing/8 -> ThingController.action8 (app/api/thing_8.py:124, auth_required=None, rate_limited=True) +- GET /api/v1/thing/10 -> ThingController.action10 (app/api/thing_10.py:130, auth_required=True, rate_limited=None) +- POST /api/v1/thing/11 -> ThingController.action11 (app/api/thing_11.py:133, auth_required=None, rate_limited=True) + +Auth/session-relevant data flows: 5 total, showing top 5: +- request.cookies['session'] -> redis.set (files=app/auth/session_store.py, sanitized=True) +- form['password'] -> logger.info (files=app/auth/service.py, app/obs/telemetry.go, sanitized=False) +- header['Authorization'] -> jwt.decode (files=app/common/jwt_tools.py, sanitized=True) +- session['role'] -> permission_check (files=app/auth/permissions.py, app/auth/rbac.py, sanitized=True) +- token -> cache.set (files=app/common/cache.py, sanitized=True) + +Security headers and framework security signals: 11 total, showing top 10: +- Content-Security-Policy: default-src 'self' +- X-Content-Type-Options: nosniff +- Strict-Transport-Security: max-age=31536000 +- X-Frame-Options: DENY +- Referrer-Policy: no-referrer +- Permissions-Policy: geolocation=() +- django.middleware.csrf.CsrfViewMiddleware +- helmet defaults +- SecurityMiddleware +- rack-protection \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/config_secrets.txt b/internal/secaf/recontext/testdata/golden/config_secrets.txt new file mode 100644 index 0000000000..08d9de2f59 --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/config_secrets.txt @@ -0,0 +1,47 @@ +Config and secrets-focused recon summary. + +Detected secret-like findings: 17 total, showing top 15: +- aws_access_key at config/env_0.yaml:3; confidence=high; is_test_value=False +- github_token at config/env_1.yaml:4; confidence=medium; is_test_value=True +- private_key at config/env_2.yaml:5; confidence=low; is_test_value=None +- slack_webhook at config/env_3.yaml:6; confidence=high; is_test_value=False +- generic_api_key at config/env_4.yaml:7; confidence=medium; is_test_value=True +- aws_access_key at config/env_5.yaml:8; confidence=low; is_test_value=None +- github_token at config/env_6.yaml:9; confidence=high; is_test_value=False +- private_key at config/env_7.yaml:10; confidence=medium; is_test_value=True +- slack_webhook at config/env_8.yaml:11; confidence=low; is_test_value=None +- generic_api_key at config/env_9.yaml:12; confidence=high; is_test_value=False +- aws_access_key at config/env_10.yaml:13; confidence=medium; is_test_value=True +- github_token at config/env_11.yaml:14; confidence=low; is_test_value=None +- private_key at config/env_12.yaml:15; confidence=high; is_test_value=False +- slack_webhook at config/env_13.yaml:16; confidence=medium; is_test_value=True +- generic_api_key at config/env_14.yaml:17; confidence=low; is_test_value=None + +Configuration weaknesses from recon: 16 total, showing top 15: +- logging at config/app_0.ini:0; risk=high; key=LOG_LEVEL +- tls at config/app_1.ini:13; risk=critical; key=SSL_VERIFY +- cors at config/app_2.ini:14; risk=high; key=ALLOW_ORIGIN +- headers at config/app_3.ini:0; risk=medium; key=X_FRAME_OPTIONS +- debug at config/app_4.ini:16; risk=critical; key=DJANGO_DEBUG +- storage at config/app_5.ini:0; risk=critical; key=BUCKET_ACL +- auth at config/app_6.ini:18; risk=medium; key=SESSION_TIMEOUT +- http at config/app_7.ini:19; risk=high; key=REDIRECT_HTTPS +- secrets at config/app_8.ini:20; risk=medium; key=n/a +- trace at config/app_9.ini:21; risk=low; key=OTEL_TRACE_ALL +- network at config/app_10.ini:0; risk=medium; key=BIND_ADDR +- errors at config/app_11.ini:23; risk=high; key=SHOW_STACKTRACE +- cache at config/app_12.ini:24; risk=low; key=CACHE_TTL +- db at config/app_13.ini:25; risk=critical; key=SSLMODE +- queue at config/app_14.ini:26; risk=low; key=PREFETCH + +Security/deployment context affecting config risk: 13 total, showing top 10: +- kubernetes ingress with TLS termination +- docker-compose exposes 5432 +- no WAF in front of /api +- secrets mounted from vault +- readiness probe on /healthz +- single replica for the worker +- TLS 1.2 minimum +- internal service mesh mTLS +- django.middleware.csrf.CsrfViewMiddleware +- helmet defaults \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/crypto.txt b/internal/secaf/recontext/testdata/golden/crypto.txt new file mode 100644 index 0000000000..ca7dae7d36 --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/crypto.txt @@ -0,0 +1,44 @@ +Cryptography-focused recon summary. + +Crypto usage entries: 16 total. + +Algorithms and key handling (weak entries first): 16 total, showing top 15: +- algorithm=MD5, key_size=None, mode=None, context=password hashing, is_weak=True +- algorithm=RSA, key_size=1024, mode=None, context=token signing, is_weak=True +- algorithm=DES, key_size=56, mode=CBC, context=legacy export, is_weak=True +- algorithm=HMAC-SHA1, key_size=160, mode=None, context=webhook signature, is_weak=True +- algorithm=RC4, key_size=128, mode=None, context=unspecified, is_weak=True +- algorithm=SHA-1, key_size=None, mode=None, context=checksum, is_weak=True +- algorithm=AES, key_size=128, mode=ECB, context=legacy blob, is_weak=True +- algorithm=3DES, key_size=168, mode=CBC, context=legacy tape, is_weak=True +- algorithm=AES, key_size=256, mode=GCM, context=at-rest encryption, is_weak=False +- algorithm=SHA-256, key_size=None, mode=None, context=unspecified, is_weak=False +- algorithm=ChaCha20, key_size=256, mode=Poly1305, context=transport, is_weak=False +- algorithm=bcrypt, key_size=None, mode=None, context=password hashing, is_weak=False +- algorithm=ECDSA, key_size=256, mode=None, context=JWT signing, is_weak=None +- algorithm=PBKDF2, key_size=None, mode=None, context=key derivation, is_weak=False +- algorithm=Ed25519, key_size=256, mode=None, context=package signing, is_weak=False + +Potential secret/key findings from config scan: 17 total, showing top 10: +- aws_access_key at config/env_0.yaml:3 (confidence=high) +- github_token at config/env_1.yaml:4 (confidence=medium) +- private_key at config/env_2.yaml:5 (confidence=low) +- slack_webhook at config/env_3.yaml:6 (confidence=high) +- generic_api_key at config/env_4.yaml:7 (confidence=medium) +- aws_access_key at config/env_5.yaml:8 (confidence=low) +- github_token at config/env_6.yaml:9 (confidence=high) +- private_key at config/env_7.yaml:10 (confidence=medium) +- slack_webhook at config/env_8.yaml:11 (confidence=low) +- generic_api_key at config/env_9.yaml:12 (confidence=high) + +Deployment/TLS/security header signals: 14 total, showing top 10: +- kubernetes ingress with TLS termination +- docker-compose exposes 5432 +- no WAF in front of /api +- secrets mounted from vault +- readiness probe on /healthz +- single replica for the worker +- TLS 1.2 minimum +- internal service mesh mTLS +- Content-Security-Policy: default-src 'self' +- X-Content-Type-Options: nosniff \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/data_exposure.txt b/internal/secaf/recontext/testdata/golden/data_exposure.txt new file mode 100644 index 0000000000..d614c850e7 --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/data_exposure.txt @@ -0,0 +1,32 @@ +Data exposure-focused recon summary. + +Data flows touching likely sensitive domains: 8 total, showing top 8: +- request.cookies['session'] -> redis.set; sanitized=True; files=app/auth/session_store.py +- form['password'] -> logger.info; sanitized=False; files=app/auth/service.py, app/obs/telemetry.go +- header['Authorization'] -> jwt.decode; sanitized=True; files=app/common/jwt_tools.py +- query['email'] -> smtp.send; sanitized=False; files=app/notify/mailer.rb +- body['card_number'] -> stripe.Charge.create; sanitized=True; files=app/billing/payments.go, app/billing/core.py +- session['role'] -> permission_check; sanitized=True; files=app/auth/permissions.py, app/auth/rbac.py +- token -> cache.set; sanitized=True; files=app/common/cache.py +- request.headers['X-User-Phone'] -> audit_log; sanitized=False; files=app/obs/audit.py + +Logging/exposure-related misconfig signals: 7 total, showing top 7: +- logging at config/app_0.ini:0; risk=high; key=LOG_LEVEL +- tls at config/app_1.ini:13; risk=critical; key=SSL_VERIFY +- debug at config/app_4.ini:16; risk=critical; key=DJANGO_DEBUG +- http at config/app_7.ini:19; risk=high; key=REDIRECT_HTTPS +- trace at config/app_9.ini:21; risk=low; key=OTEL_TRACE_ALL +- errors at config/app_11.ini:23; risk=high; key=SHOW_STACKTRACE +- exposure at config/app_15.ini:0; risk=medium; key=ADMIN_PATH + +Entry points and API surface with exposure risk: 20 total, showing top 10: +- GET /api/v1/thing/0 (app/api/thing_0.py:100, auth_required=True) +- POST /api/v1/thing/1 (app/api/thing_1.py:103, auth_required=False) +- PUT /api/v1/thing/2 (app/api/thing_2.py:106, auth_required=None) +- DELETE /api/v1/thing/3 (app/api/thing_3.py:109, auth_required=False) +- PATCH /api/v1/thing/4 (app/api/thing_4.py:112, auth_required=True) +- GET /api/v1/thing/5 (app/api/thing_5.py:115, auth_required=None) +- POST /api/v1/thing/6 (app/api/thing_6.py:118, auth_required=True) +- PUT /api/v1/thing/7 (app/api/thing_7.py:121, auth_required=False) +- DELETE /api/v1/thing/8 (app/api/thing_8.py:124, auth_required=None) +- PATCH /api/v1/thing/9 (app/api/thing_9.py:127, auth_required=False) \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/framework_hints_aliases.txt b/internal/secaf/recontext/testdata/golden/framework_hints_aliases.txt new file mode 100644 index 0000000000..a2be8c7e99 --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/framework_hints_aliases.txt @@ -0,0 +1,18 @@ +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: NEXTJS + Security features: React JSX auto-escapes output by default, API routes can share centralized auth middleware, Server/client boundaries reduce accidental secret exposure + DO NOT FLAG: React JSX escaped rendering, getServerSideProps/getServerSession with proper auth checks, Typed route handlers with validated schema guards + Watch for: dangerouslySetInnerHTML, unprotected API routes under /api, secret leakage to client bundles, rewrites/redirects from untrusted user input + Common vulns: XSS through dangerous HTML rendering, IDOR/auth bypass in API handlers, Sensitive env/config exposure in client-side code + +Framework: SPRING + Security features: Spring Security provides auth, CSRF, and filter chain defaults, JPA/Hibernate prepared parameter binding by default, Bean validation can enforce request constraints + DO NOT FLAG: PreparedStatement/JPA named parameter usage, Spring Security CSRF/auth filters clearly active, Thymeleaf auto-escaped output + Watch for: @PreAuthorize missing on sensitive methods, JdbcTemplate/raw Statement with concatenation, csrf().disable() on browser session flows, Actuator endpoints exposed without auth + Common vulns: Authz bypass from weak method-level security, SQL injection in raw JDBC queries, Sensitive management endpoint exposure + +Framework: ASPNET + Security features: Razor encodes output by default, Model binding and data annotations aid validation, Anti-forgery tokens support CSRF defense + DO NOT FLAG: Entity Framework LINQ parameterized queries, Razor default HTML encoding, ValidateAntiForgeryToken in state-changing MVC actions + Watch for: Html.Raw() on untrusted input, FromBody models without validation, Authorize missing on privileged endpoints, custom SQL built via string interpolation + Common vulns: XSS through Html.Raw and unencoded output, Auth/authz bypass on unsecured controllers, SQL injection in handcrafted query strings \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/framework_hints_all_known.txt b/internal/secaf/recontext/testdata/golden/framework_hints_all_known.txt new file mode 100644 index 0000000000..f87fa9acec --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/framework_hints_all_known.txt @@ -0,0 +1,66 @@ +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +Framework: FLASK + Security features: Jinja2 auto-escapes in HTML templates, Werkzeug handles request parsing safely by default, Blueprint/middleware patterns can centralize auth checks + DO NOT FLAG: Jinja2 auto-escaped template variables, Parameterized SQLAlchemy query usage, Server-side session signing with strong SECRET_KEY + Watch for: render_template_string() with user input, debug=True in production paths, string-concatenated SQL in execute(), hardcoded SECRET_KEY + Common vulns: SSTI through dynamic template rendering, XSS when auto-escaping is bypassed, session tampering risk with weak secret/config + +Framework: FASTAPI + Security features: Pydantic request validation reduces malformed input, Dependency injection supports reusable auth guards, OpenAPI schema generation improves contract visibility + DO NOT FLAG: Pydantic model validation errors, Dependency-based auth checks clearly enforced, Parameterized SQLAlchemy/async driver usage + Watch for: Depends() omitted on privileged routes, raw SQL in text()/execute() with interpolation, CORS allow_origins=['*'] with credentials, unsafe deserialization in background tasks + Common vulns: Auth bypass on unprotected routes, SQL injection in manually assembled queries, CORS misconfiguration exposing credentialed APIs + +Framework: EXPRESS + Security features: Router middleware can enforce auth and rate limits, helmet can apply secure HTTP headers, Validated schema middleware can constrain input + DO NOT FLAG: Parameterized ORM/database queries, helmet defaults correctly applied, router-level auth middleware consistently enforced + Watch for: res.send()/res.json() leaking sensitive internals, string-built SQL in query(), trust proxy misconfiguration, open CORS with credentials + Common vulns: Authz gaps from missing middleware on routes, NoSQL/SQL injection from unsanitized request bodies, Open redirect and SSRF through unvalidated URLs + +Framework: NEXTJS + Security features: React JSX auto-escapes output by default, API routes can share centralized auth middleware, Server/client boundaries reduce accidental secret exposure + DO NOT FLAG: React JSX escaped rendering, getServerSideProps/getServerSession with proper auth checks, Typed route handlers with validated schema guards + Watch for: dangerouslySetInnerHTML, unprotected API routes under /api, secret leakage to client bundles, rewrites/redirects from untrusted user input + Common vulns: XSS through dangerous HTML rendering, IDOR/auth bypass in API handlers, Sensitive env/config exposure in client-side code + +Framework: SPRING + Security features: Spring Security provides auth, CSRF, and filter chain defaults, JPA/Hibernate prepared parameter binding by default, Bean validation can enforce request constraints + DO NOT FLAG: PreparedStatement/JPA named parameter usage, Spring Security CSRF/auth filters clearly active, Thymeleaf auto-escaped output + Watch for: @PreAuthorize missing on sensitive methods, JdbcTemplate/raw Statement with concatenation, csrf().disable() on browser session flows, Actuator endpoints exposed without auth + Common vulns: Authz bypass from weak method-level security, SQL injection in raw JDBC queries, Sensitive management endpoint exposure + +Framework: RAILS + Security features: ActiveRecord parameterization for hash/array queries, ERB templates auto-escape by default, Built-in CSRF protection for non-GET requests + DO NOT FLAG: ActiveRecord where with hash conditions, Rails protect_from_forgery active, ERB escaped output without raw/html_safe + Watch for: where/order/find_by_sql with string interpolation, raw()/html_safe on untrusted content, skip_before_action on auth filters, mass assignment via permit! + Common vulns: SQL injection in manual query fragments, XSS via unsafe output helpers, Authz bypass from skipped controller guards + +Framework: ASPNET + Security features: Razor encodes output by default, Model binding and data annotations aid validation, Anti-forgery tokens support CSRF defense + DO NOT FLAG: Entity Framework LINQ parameterized queries, Razor default HTML encoding, ValidateAntiForgeryToken in state-changing MVC actions + Watch for: Html.Raw() on untrusted input, FromBody models without validation, Authorize missing on privileged endpoints, custom SQL built via string interpolation + Common vulns: XSS through Html.Raw and unencoded output, Auth/authz bypass on unsecured controllers, SQL injection in handcrafted query strings + +Framework: REACT + Security features: JSX escapes strings before DOM rendering, Component model encourages explicit data flow, Framework discourages direct DOM mutation + DO NOT FLAG: Standard JSX expression rendering, textContent assignment for untrusted text, sanitized HTML via trusted DOMPurify policy + Watch for: dangerouslySetInnerHTML, untrusted URL assignment to href/src, eval/new Function in client code, token/secret exposure in bundles + Common vulns: DOM XSS through unsafe HTML injection, Open redirect via unvalidated navigation targets, Sensitive data exposure in frontend artifacts + +Framework: VUE + Security features: Mustache template interpolation escapes HTML, Component props/events provide explicit boundaries, Router guards can enforce auth flows + DO NOT FLAG: escaped template interpolation {{ value }}, validated route guards protecting private routes, sanitized content rendered through safe components + Watch for: v-html with untrusted data, dynamic component/template compilation, unsafe URL bindings in href/src, client-side auth checks without server enforcement + Common vulns: XSS through v-html and unsafe render paths, Auth bypass from client-only route protection, Open redirect patterns in router navigation + +Framework: ANGULAR + Security features: Template binding sanitization for HTML/URL contexts, HttpClient and interceptor patterns support central controls, AOT compilation limits runtime template injection vectors + DO NOT FLAG: default Angular template binding sanitization, HttpClient usage with validated request schemas, route guards consistently applied + Watch for: bypassSecurityTrustHtml/Url/Script, [innerHTML] with unsanitized input, direct DOM APIs via ElementRef/nativeElement, auth only in client guard without server checks + Common vulns: XSS when sanitizer is explicitly bypassed, Token leakage in local storage/logging, Authorization gaps due to client-only enforcement \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/framework_hints_empty.txt b/internal/secaf/recontext/testdata/golden/framework_hints_empty.txt new file mode 100644 index 0000000000..17572567c2 --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/framework_hints_empty.txt @@ -0,0 +1 @@ +No framework-specific hints available for detected frameworks. \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/framework_hints_for_context.txt b/internal/secaf/recontext/testdata/golden/framework_hints_for_context.txt new file mode 100644 index 0000000000..ce24e9f9fb --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/framework_hints_for_context.txt @@ -0,0 +1,30 @@ +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides + +Framework: NEXTJS + Security features: React JSX auto-escapes output by default, API routes can share centralized auth middleware, Server/client boundaries reduce accidental secret exposure + DO NOT FLAG: React JSX escaped rendering, getServerSideProps/getServerSession with proper auth checks, Typed route handlers with validated schema guards + Watch for: dangerouslySetInnerHTML, unprotected API routes under /api, secret leakage to client bundles, rewrites/redirects from untrusted user input + Common vulns: XSS through dangerous HTML rendering, IDOR/auth bypass in API handlers, Sensitive env/config exposure in client-side code + +Framework: SPRING + Security features: Spring Security provides auth, CSRF, and filter chain defaults, JPA/Hibernate prepared parameter binding by default, Bean validation can enforce request constraints + DO NOT FLAG: PreparedStatement/JPA named parameter usage, Spring Security CSRF/auth filters clearly active, Thymeleaf auto-escaped output + Watch for: @PreAuthorize missing on sensitive methods, JdbcTemplate/raw Statement with concatenation, csrf().disable() on browser session flows, Actuator endpoints exposed without auth + Common vulns: Authz bypass from weak method-level security, SQL injection in raw JDBC queries, Sensitive management endpoint exposure + +Framework: REACT + Security features: JSX escapes strings before DOM rendering, Component model encourages explicit data flow, Framework discourages direct DOM mutation + DO NOT FLAG: Standard JSX expression rendering, textContent assignment for untrusted text, sanitized HTML via trusted DOMPurify policy + Watch for: dangerouslySetInnerHTML, untrusted URL assignment to href/src, eval/new Function in client code, token/secret exposure in bundles + Common vulns: DOM XSS through unsafe HTML injection, Open redirect via unvalidated navigation targets, Sensitive data exposure in frontend artifacts + +Framework: VUE + Security features: Mustache template interpolation escapes HTML, Component props/events provide explicit boundaries, Router guards can enforce auth flows + DO NOT FLAG: escaped template interpolation {{ value }}, validated route guards protecting private routes, sanitized content rendered through safe components + Watch for: v-html with untrusted data, dynamic component/template compilation, unsafe URL bindings in href/src, client-side auth checks without server enforcement + Common vulns: XSS through v-html and unsafe render paths, Auth bypass from client-only route protection, Open redirect patterns in router navigation \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/framework_hints_padded.txt b/internal/secaf/recontext/testdata/golden/framework_hints_padded.txt new file mode 100644 index 0000000000..1b99225aec --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/framework_hints_padded.txt @@ -0,0 +1,18 @@ +FRAMEWORK-SPECIFIC GUIDANCE: +Framework: REACT + Security features: JSX escapes strings before DOM rendering, Component model encourages explicit data flow, Framework discourages direct DOM mutation + DO NOT FLAG: Standard JSX expression rendering, textContent assignment for untrusted text, sanitized HTML via trusted DOMPurify policy + Watch for: dangerouslySetInnerHTML, untrusted URL assignment to href/src, eval/new Function in client code, token/secret exposure in bundles + Common vulns: DOM XSS through unsafe HTML injection, Open redirect via unvalidated navigation targets, Sensitive data exposure in frontend artifacts + +Framework: VUE + Security features: Mustache template interpolation escapes HTML, Component props/events provide explicit boundaries, Router guards can enforce auth flows + DO NOT FLAG: escaped template interpolation {{ value }}, validated route guards protecting private routes, sanitized content rendered through safe components + Watch for: v-html with untrusted data, dynamic component/template compilation, unsafe URL bindings in href/src, client-side auth checks without server enforcement + Common vulns: XSS through v-html and unsafe render paths, Auth bypass from client-only route protection, Open redirect patterns in router navigation + +Framework: DJANGO + Security features: ORM queries are parameterized by default, Templates auto-escape variables by default, CsrfViewMiddleware enforces CSRF tokens for unsafe methods + DO NOT FLAG: Django ORM filter/exclude (parameterized), CSRF with CsrfViewMiddleware active, XSS in Django templates (auto-escaped) + Watch for: raw() queries, mark_safe(), |safe filter, CSRF_COOKIE_SECURE=False + Common vulns: SQL injection via raw SQL and string formatting, XSS via unsafe template escape bypasses, CSRF weakening via middleware/settings overrides \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/framework_hints_unknown_only.txt b/internal/secaf/recontext/testdata/golden/framework_hints_unknown_only.txt new file mode 100644 index 0000000000..17572567c2 --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/framework_hints_unknown_only.txt @@ -0,0 +1 @@ +No framework-specific hints available for detected frameworks. \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/generic.txt b/internal/secaf/recontext/testdata/golden/generic.txt new file mode 100644 index 0000000000..5a8d2cf90a --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/generic.txt @@ -0,0 +1,39 @@ +General recon summary. + +Profile: 2417 files, 184213 LOC, languages=Python, JavaScript, Go, Ruby, Rust, python, frameworks=Django, next.js, Spring Boot, React , unknown-fw, NEXT, Vue. + +Top entry points: 17 total, showing top 10: +- http handler_0 (app/entry/e0.py:10) +- HTTP /v1/resource/1 (app/entry/e1.py:17) +- cli /v1/resource/2 (app/entry/e2.py:24) +- api handler_3 (app/entry/e3.py:31) +- graphql /v1/resource/4 (app/entry/e4.py:38) +- rpc /v1/resource/5 (app/entry/e5.py:45) +- route handler_6 (app/entry/e6.py:52) +- cron /v1/resource/7 (app/entry/e7.py:59) +- queue /v1/resource/8 (app/entry/e8.py:66) +- http handler_9 (app/entry/e9.py:73) + +Top API endpoints: 20 total, showing top 10: +- GET /api/v1/thing/0 (app/api/thing_0.py:100) +- POST /api/v1/thing/1 (app/api/thing_1.py:103) +- PUT /api/v1/thing/2 (app/api/thing_2.py:106) +- DELETE /api/v1/thing/3 (app/api/thing_3.py:109) +- PATCH /api/v1/thing/4 (app/api/thing_4.py:112) +- GET /api/v1/thing/5 (app/api/thing_5.py:115) +- POST /api/v1/thing/6 (app/api/thing_6.py:118) +- PUT /api/v1/thing/7 (app/api/thing_7.py:121) +- DELETE /api/v1/thing/8 (app/api/thing_8.py:124) +- PATCH /api/v1/thing/9 (app/api/thing_9.py:127) + +Top data-flow candidates: 20 total, showing top 10: +- request.args['q'] -> cursor.execute; sanitized=False +- request.cookies['session'] -> redis.set; sanitized=True +- request.json['url'] -> requests.get; sanitized=False +- form['password'] -> logger.info; sanitized=False +- header['Authorization'] -> jwt.decode; sanitized=True +- query['email'] -> smtp.send; sanitized=False +- body['card_number'] -> stripe.Charge.create; sanitized=True +- path_param['id'] -> orm.filter; sanitized=True +- upload.filename -> open; sanitized=False +- env['DEBUG'] -> template.render; sanitized=False \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/injection.txt b/internal/secaf/recontext/testdata/golden/injection.txt new file mode 100644 index 0000000000..90eacedd64 --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/injection.txt @@ -0,0 +1,64 @@ +Injection-focused recon summary. + +Codebase profile: 2417 files, 184213 LOC, languages=Python, JavaScript, Go, Ruby, Rust, python, frameworks=Django, next.js, Spring Boot, React , unknown-fw, NEXT, Vue. + +Entry points likely to receive untrusted input: 17 total, showing top 15: +- http handler_0 (app/entry/e0.py:10) +- HTTP /v1/resource/1 (app/entry/e1.py:17) +- cli /v1/resource/2 (app/entry/e2.py:24) +- api handler_3 (app/entry/e3.py:31) +- graphql /v1/resource/4 (app/entry/e4.py:38) +- rpc /v1/resource/5 (app/entry/e5.py:45) +- route handler_6 (app/entry/e6.py:52) +- cron /v1/resource/7 (app/entry/e7.py:59) +- queue /v1/resource/8 (app/entry/e8.py:66) +- http handler_9 (app/entry/e9.py:73) +- api /v1/resource/10 (app/entry/e10.py:80) +- grpc /v1/resource/11 (app/entry/e11.py:87) +- http handler_12 (app/entry/e12.py:94) +- route /v1/resource/13 (app/entry/e13.py:101) +- cli /v1/resource/14 (app/entry/e14.py:108) + +High-value sinks: 18 total, showing top 15: +- sql at app/sink/k0.py:200 +- command at app/sink/k1.py:202 (sink_fn_1) +- template at app/sink/k2.py:204 (sink_fn_2) +- file at app/sink/k3.py:206 (sink_fn_3) +- network at app/sink/k4.py:208 +- deserialization at app/sink/k5.py:210 (sink_fn_5) +- sql at app/sink/k6.py:212 (sink_fn_6) +- command at app/sink/k7.py:214 (sink_fn_7) +- template at app/sink/k8.py:216 +- file at app/sink/k9.py:218 (sink_fn_9) +- network at app/sink/k10.py:220 (sink_fn_10) +- deserialization at app/sink/k11.py:222 (sink_fn_11) +- sql at app/sink/k12.py:224 +- command at app/sink/k13.py:226 (sink_fn_13) +- template at app/sink/k14.py:228 (sink_fn_14) + +Source-to-sink flow candidates (unsanitized first): 13 total, showing top 13: +- request.args['q'] -> cursor.execute; sanitized=False; files=app/search/index.ts, app/db/raw.py, app/util/a.py +- request.json['url'] -> requests.get; sanitized=False; files=app/integrations/webhooks.go, app/net/client.go +- form['password'] -> logger.info; sanitized=False; files=app/auth/service.py, app/obs/telemetry.go +- query['email'] -> smtp.send; sanitized=False; files=app/notify/mailer.rb +- upload.filename -> open; sanitized=False; files=app/media/uploads.py, app/common/cache.py, app/common/guard.py +- env['DEBUG'] -> template.render; sanitized=False; files=app/web/render.py +- request.body -> eval; sanitized=False; files=app/admin/views.py +- websocket.msg -> subprocess.run; sanitized=False; files=app/ws/handler.py, app/ws/exec.py +- cli.argv -> os.system; sanitized=False; files=tools/run.py +- queue.payload -> pickle.loads; sanitized=False; files=app/worker/task.py +- request.args['redirect'] -> HttpResponseRedirect; sanitized=False; files=app/web/redirect.py +- request.headers['X-User-Phone'] -> audit_log; sanitized=False; files=app/obs/audit.py +- graphql.variables -> db.query; sanitized=False; files=app/api/graphql.ts + +Known sanitization points: 12 total, showing top 10: +- app/sanitize/s0.py:5 type=escape protects=unspecified +- app/sanitize/s1.py:9 type=parameterize protects=CWE-80, CWE-90 +- app/sanitize/s2.py:13 type=allowlist protects=CWE-81, CWE-91 +- app/sanitize/s3.py:17 type=encode protects=unspecified +- app/sanitize/s4.py:21 type=escape protects=CWE-83, CWE-93 +- app/sanitize/s5.py:25 type=parameterize protects=CWE-84, CWE-94 +- app/sanitize/s6.py:29 type=allowlist protects=unspecified +- app/sanitize/s7.py:33 type=encode protects=CWE-86, CWE-96 +- app/sanitize/s8.py:37 type=escape protects=CWE-87, CWE-97 +- app/sanitize/s9.py:41 type=parameterize protects=unspecified \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/language_hints_all_known.txt b/internal/secaf/recontext/testdata/golden/language_hints_all_known.txt new file mode 100644 index 0000000000..8d45c2f5c2 --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/language_hints_all_known.txt @@ -0,0 +1,42 @@ +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: JAVASCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec, new Function + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, parameterized pg queries + DO NOT FLAG: React JSX expressions (auto-escaped), Angular template bindings (sanitized by default) + Framework notes: Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous. + +Language: TYPESCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, Prisma parameterized queries + DO NOT FLAG: React JSX expressions, Angular template bindings, Prisma ORM queries + Framework notes: TypeScript adds type safety but doesn't prevent injection. Check for any type assertions near user input. + +Language: GO + Key sinks: fmt.Sprintf into SQL, exec.Command with user input, template.HTML(), os.Exec + Safe patterns (skip these): database/sql with ? placeholders, html/template (auto-escapes), exec.Command with separate args + DO NOT FLAG: GORM parameterized queries, html/template default escaping, database/sql prepared statements + Framework notes: Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML. + +Language: JAVA + Key sinks: Statement.execute, Runtime.exec, ProcessBuilder with concatenated strings, ScriptEngine.eval + Safe patterns (skip these): PreparedStatement with ?, JNDI lookup with allowlist, OWASP ESAPI encoding + DO NOT FLAG: JPA/Hibernate named parameters, Spring Security CSRF protection, PreparedStatement usage + Framework notes: Check Spring Boot auto-config. Thymeleaf auto-escapes. JSP needs explicit escaping. + +Language: RUBY + Key sinks: eval, system, exec, send, public_send, ERB.new with user input + Safe patterns (skip these): ActiveRecord parameterized queries, Rack::Utils.escape_html, sanitize helper in Rails + DO NOT FLAG: ActiveRecord where with hash conditions, Rails CSRF protection, Rails html_safe on constants + Framework notes: Rails auto-escapes ERB templates. raw/html_safe bypasses escaping. Check for mass assignment. + +Language: CSHARP + Key sinks: SqlCommand with concatenation, Process.Start with user input, Razor @Html.Raw() + Safe patterns (skip these): SqlParameter, Entity Framework LINQ, Razor auto-encoding + DO NOT FLAG: Entity Framework LINQ queries, ASP.NET anti-forgery tokens, Razor default encoding + Framework notes: ASP.NET Core Razor auto-encodes. @Html.Raw() is dangerous. Check for [ValidateAntiForgeryToken]. \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/language_hints_empty.txt b/internal/secaf/recontext/testdata/golden/language_hints_empty.txt new file mode 100644 index 0000000000..3268b9620a --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/language_hints_empty.txt @@ -0,0 +1 @@ +No language-specific hints available for detected languages. \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/language_hints_for_context.txt b/internal/secaf/recontext/testdata/golden/language_hints_for_context.txt new file mode 100644 index 0000000000..75940849da --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/language_hints_for_context.txt @@ -0,0 +1,30 @@ +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: JAVASCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec, new Function + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, parameterized pg queries + DO NOT FLAG: React JSX expressions (auto-escaped), Angular template bindings (sanitized by default) + Framework notes: Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous. + +Language: GO + Key sinks: fmt.Sprintf into SQL, exec.Command with user input, template.HTML(), os.Exec + Safe patterns (skip these): database/sql with ? placeholders, html/template (auto-escapes), exec.Command with separate args + DO NOT FLAG: GORM parameterized queries, html/template default escaping, database/sql prepared statements + Framework notes: Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML. + +Language: RUBY + Key sinks: eval, system, exec, send, public_send, ERB.new with user input + Safe patterns (skip these): ActiveRecord parameterized queries, Rack::Utils.escape_html, sanitize helper in Rails + DO NOT FLAG: ActiveRecord where with hash conditions, Rails CSRF protection, Rails html_safe on constants + Framework notes: Rails auto-escapes ERB templates. raw/html_safe bypasses escaping. Check for mass assignment. + +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/language_hints_mixed_case_and_repeat.txt b/internal/secaf/recontext/testdata/golden/language_hints_mixed_case_and_repeat.txt new file mode 100644 index 0000000000..cbead73cf1 --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/language_hints_mixed_case_and_repeat.txt @@ -0,0 +1,24 @@ +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. + +Language: JAVASCRIPT + Key sinks: eval, Function(), innerHTML, document.write, child_process.exec, new Function + Safe patterns (skip these): DOMPurify.sanitize(), textContent assignment, parameterized pg queries + DO NOT FLAG: React JSX expressions (auto-escaped), Angular template bindings (sanitized by default) + Framework notes: Check for React/Vue/Angular-specific XSS patterns. React auto-escapes JSX. Vue v-html is dangerous. + +Language: GO + Key sinks: fmt.Sprintf into SQL, exec.Command with user input, template.HTML(), os.Exec + Safe patterns (skip these): database/sql with ? placeholders, html/template (auto-escapes), exec.Command with separate args + DO NOT FLAG: GORM parameterized queries, html/template default escaping, database/sql prepared statements + Framework notes: Go's html/template auto-escapes. text/template does NOT. Check for text/template serving HTML. \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/language_hints_single.txt b/internal/secaf/recontext/testdata/golden/language_hints_single.txt new file mode 100644 index 0000000000..935dd64ca6 --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/language_hints_single.txt @@ -0,0 +1,6 @@ +LANGUAGE-SPECIFIC GUIDANCE: +Language: PYTHON + Key sinks: cursor.execute, os.system, subprocess.run, eval, exec, pickle.loads + Safe patterns (skip these): parameterized queries with %s placeholders, subprocess.run with list args (no shell=True) + DO NOT FLAG: Django ORM queries (uses parameterized queries internally), SQLAlchemy text() with bound params + Framework notes: Check for Django/Flask/FastAPI-specific patterns. Django auto-escapes templates. Flask Jinja2 auto-escapes. \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/language_hints_unknown_only.txt b/internal/secaf/recontext/testdata/golden/language_hints_unknown_only.txt new file mode 100644 index 0000000000..3268b9620a --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/language_hints_unknown_only.txt @@ -0,0 +1 @@ +No language-specific hints available for detected languages. \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/logic.txt b/internal/secaf/recontext/testdata/golden/logic.txt new file mode 100644 index 0000000000..bb89d686ee --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/logic.txt @@ -0,0 +1,59 @@ +Business-logic-focused recon summary. + +Core modules likely to implement workflows and state transitions: 19 total, showing top 15: +- app/auth/service.py (python) - Session and JWT issuance +- app/auth/rbac.py (python) +- app/billing/core.py (python) - Invoice state machine — handles refunds +- app/web/middleware/csrf.js (javascript) - CSRF middleware +- app/billing/payments.go (go) - Charge orchestration +- app/reports/render.py (python) - PDF and CSV export +- app/search/index.ts (typescript) - Elasticsearch query builder +- app/auth/session_store.py (python) - Redis-backed sessions +- app/notify/mailer.rb (ruby) +- app/auth/permissions.py (python) - Role → permission table +- app/media/uploads.py (python) - S3 multipart upload helper +- app/integrations/webhooks.go (go) - Outbound webhook dispatcher +- app/api/graphql.ts (typescript) - GraphQL resolvers — naïve depth limit +- app/admin/views.py (python) - Django admin overrides +- app/common/guard.py (python) - Assorted guard helpers + +Workflow entry points: 17 total, showing top 10: +- http handler_0 (app/entry/e0.py:10) +- HTTP /v1/resource/1 (app/entry/e1.py:17) +- cli /v1/resource/2 (app/entry/e2.py:24) +- api handler_3 (app/entry/e3.py:31) +- graphql /v1/resource/4 (app/entry/e4.py:38) +- rpc /v1/resource/5 (app/entry/e5.py:45) +- route handler_6 (app/entry/e6.py:52) +- cron /v1/resource/7 (app/entry/e7.py:59) +- queue /v1/resource/8 (app/entry/e8.py:66) +- http handler_9 (app/entry/e9.py:73) + +Cross-file data/control flow candidates: 20 total, showing top 15: +- request.args['q'] -> cursor.execute; files=app/search/index.ts, app/db/raw.py, app/util/a.py, app/util/b.py; sanitized=False +- request.cookies['session'] -> redis.set; files=app/auth/session_store.py; sanitized=True +- request.json['url'] -> requests.get; files=app/integrations/webhooks.go, app/net/client.go; sanitized=False +- form['password'] -> logger.info; files=app/auth/service.py, app/obs/telemetry.go; sanitized=False +- header['Authorization'] -> jwt.decode; files=app/common/jwt_tools.py; sanitized=True +- query['email'] -> smtp.send; files=app/notify/mailer.rb; sanitized=False +- body['card_number'] -> stripe.Charge.create; files=app/billing/payments.go, app/billing/core.py; sanitized=True +- path_param['id'] -> orm.filter; files=app/api/thing_1.py; sanitized=True +- upload.filename -> open; files=app/media/uploads.py, app/common/cache.py, app/common/guard.py, app/x/y.py, app/x/z.py; sanitized=False +- env['DEBUG'] -> template.render; files=app/web/render.py; sanitized=False +- request.body -> eval; files=app/admin/views.py; sanitized=False +- websocket.msg -> subprocess.run; files=app/ws/handler.py, app/ws/exec.py; sanitized=False +- cli.argv -> os.system; files=tools/run.py; sanitized=False +- queue.payload -> pickle.loads; files=app/worker/task.py; sanitized=False +- request.args['redirect'] -> HttpResponseRedirect; files=app/web/redirect.py; sanitized=False + +Trust boundaries and external service transitions: 17 total, showing top 10: +- boundary boundary_0: internet->app +- boundary boundary_1: dmz->db +- boundary boundary_2: vpc->cache +- boundary boundary_3: worker->queue +- boundary boundary_4: internet->app +- boundary boundary_5: dmz->db +- boundary boundary_6: vpc->cache +- boundary boundary_7: worker->queue +- boundary boundary_8: internet->app +- boundary boundary_9: dmz->db \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/prune_config_secrets.json b/internal/secaf/recontext/testdata/golden/prune_config_secrets.json new file mode 100644 index 0000000000..16fbd90dbd --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/prune_config_secrets.json @@ -0,0 +1,988 @@ +{ + "architecture": { + "app_type": "web_api", + "modules": [ + { + "name": "auth_service", + "path": "app/auth/service.py", + "language": "python", + "description": "Session and JWT issuance", + "dependencies": [ + "jwt", + "redis" + ] + }, + { + "name": "rbac", + "path": "app/auth/rbac.py", + "language": "python", + "description": null, + "dependencies": [] + }, + { + "name": "billing", + "path": "app/billing/core.py", + "language": "python", + "description": "Invoice state machine \u2014 handles refunds", + "dependencies": [ + "stripe" + ] + }, + { + "name": "csrf_guard", + "path": "app/web/middleware/csrf.js", + "language": "javascript", + "description": "CSRF middleware", + "dependencies": [] + }, + { + "name": "payments", + "path": "app/billing/payments.go", + "language": "go", + "description": "Charge orchestration", + "dependencies": [ + "stripe-go" + ] + }, + { + "name": "reporting", + "path": "app/reports/render.py", + "language": "python", + "description": "PDF and CSV export", + "dependencies": [ + "weasyprint" + ] + }, + { + "name": "search", + "path": "app/search/index.ts", + "language": "typescript", + "description": "Elasticsearch query builder", + "dependencies": [ + "@elastic/elasticsearch" + ] + }, + { + "name": "session_store", + "path": "app/auth/session_store.py", + "language": "python", + "description": "Redis-backed sessions", + "dependencies": [ + "redis" + ] + }, + { + "name": "notifications", + "path": "app/notify/mailer.rb", + "language": "ruby", + "description": null, + "dependencies": [ + "mail" + ] + }, + { + "name": "permission_matrix", + "path": "app/auth/permissions.py", + "language": "python", + "description": "Role \u2192 permission table", + "dependencies": [] + }, + { + "name": "uploads", + "path": "app/media/uploads.py", + "language": "python", + "description": "S3 multipart upload helper", + "dependencies": [ + "boto3" + ] + }, + { + "name": "webhooks", + "path": "app/integrations/webhooks.go", + "language": "go", + "description": "Outbound webhook dispatcher", + "dependencies": [] + }, + { + "name": "graph_api", + "path": "app/api/graphql.ts", + "language": "typescript", + "description": "GraphQL resolvers \u2014 na\u00efve depth limit", + "dependencies": [ + "graphql" + ] + }, + { + "name": "admin_panel", + "path": "app/admin/views.py", + "language": "python", + "description": "Django admin overrides", + "dependencies": [ + "django" + ] + }, + { + "name": "guard_utils", + "path": "app/common/guard.py", + "language": "python", + "description": "Assorted guard helpers", + "dependencies": [] + }, + { + "name": "cache", + "path": "app/common/cache.py", + "language": "python", + "description": "Memoization wrappers", + "dependencies": [] + }, + { + "name": "migrations", + "path": "db/migrate/2024_add_roles.rb", + "language": "ruby", + "description": "Adds role column", + "dependencies": [] + }, + { + "name": "jwt_tools", + "path": "app/common/jwt_tools.py", + "language": "python", + "description": null, + "dependencies": [ + "pyjwt" + ] + }, + { + "name": "telemetry", + "path": "app/obs/telemetry.go", + "language": "go", + "description": "OTel exporter", + "dependencies": [] + } + ], + "entry_points": [ + { + "kind": "http", + "identifier": "handler_0", + "file_path": "app/entry/e0.py", + "line": 10, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "HTTP", + "identifier": "handler_1", + "file_path": "app/entry/e1.py", + "line": 17, + "method": "POST", + "route": "/v1/resource/1", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_2", + "file_path": "app/entry/e2.py", + "line": 24, + "method": null, + "route": "/v1/resource/2", + "auth_required": null + }, + { + "kind": "api", + "identifier": "handler_3", + "file_path": "app/entry/e3.py", + "line": 31, + "method": "PUT", + "route": null, + "auth_required": true + }, + { + "kind": "graphql", + "identifier": "handler_4", + "file_path": "app/entry/e4.py", + "line": 38, + "method": "GET", + "route": "/v1/resource/4", + "auth_required": false + }, + { + "kind": "rpc", + "identifier": "handler_5", + "file_path": "app/entry/e5.py", + "line": 45, + "method": "POST", + "route": "/v1/resource/5", + "auth_required": null + }, + { + "kind": "route", + "identifier": "handler_6", + "file_path": "app/entry/e6.py", + "line": 52, + "method": null, + "route": null, + "auth_required": true + }, + { + "kind": "cron", + "identifier": "handler_7", + "file_path": "app/entry/e7.py", + "line": 59, + "method": "PUT", + "route": "/v1/resource/7", + "auth_required": false + }, + { + "kind": "queue", + "identifier": "handler_8", + "file_path": "app/entry/e8.py", + "line": 66, + "method": "GET", + "route": "/v1/resource/8", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_9", + "file_path": "app/entry/e9.py", + "line": 73, + "method": "POST", + "route": null, + "auth_required": true + }, + { + "kind": "api", + "identifier": "handler_10", + "file_path": "app/entry/e10.py", + "line": 80, + "method": null, + "route": "/v1/resource/10", + "auth_required": false + }, + { + "kind": "grpc", + "identifier": "handler_11", + "file_path": "app/entry/e11.py", + "line": 87, + "method": "PUT", + "route": "/v1/resource/11", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_12", + "file_path": "app/entry/e12.py", + "line": 94, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "route", + "identifier": "handler_13", + "file_path": "app/entry/e13.py", + "line": 101, + "method": "POST", + "route": "/v1/resource/13", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_14", + "file_path": "app/entry/e14.py", + "line": 108, + "method": null, + "route": "/v1/resource/14", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_15", + "file_path": "app/entry/e15.py", + "line": 115, + "method": "PUT", + "route": null, + "auth_required": true + }, + { + "kind": "websocket", + "identifier": "handler_16", + "file_path": "app/entry/e16.py", + "line": 122, + "method": "GET", + "route": "/v1/resource/16", + "auth_required": false + } + ], + "trust_boundaries": [ + { + "name": "boundary_0", + "source_zone": "internet", + "target_zone": "app", + "description": "Transition 0", + "enforcement": [] + }, + { + "name": "boundary_1", + "source_zone": "dmz", + "target_zone": "db", + "description": "Transition 1", + "enforcement": [ + "waf_1", + "mtls_1" + ] + }, + { + "name": "boundary_2", + "source_zone": "vpc", + "target_zone": "cache", + "description": "Transition 2", + "enforcement": [ + "waf_2", + "mtls_2" + ] + }, + { + "name": "boundary_3", + "source_zone": "worker", + "target_zone": "queue", + "description": "Transition 3", + "enforcement": [ + "waf_3", + "mtls_3" + ] + }, + { + "name": "boundary_4", + "source_zone": "internet", + "target_zone": "app", + "description": "Transition 4", + "enforcement": [] + }, + { + "name": "boundary_5", + "source_zone": "dmz", + "target_zone": "db", + "description": "Transition 5", + "enforcement": [ + "waf_5", + "mtls_5" + ] + }, + { + "name": "boundary_6", + "source_zone": "vpc", + "target_zone": "cache", + "description": "Transition 6", + "enforcement": [ + "waf_6", + "mtls_6" + ] + }, + { + "name": "boundary_7", + "source_zone": "worker", + "target_zone": "queue", + "description": "Transition 7", + "enforcement": [ + "waf_7", + "mtls_7" + ] + }, + { + "name": "boundary_8", + "source_zone": "internet", + "target_zone": "app", + "description": "Transition 8", + "enforcement": [] + }, + { + "name": "boundary_9", + "source_zone": "dmz", + "target_zone": "db", + "description": "Transition 9", + "enforcement": [ + "waf_9", + "mtls_9" + ] + }, + { + "name": "boundary_10", + "source_zone": "vpc", + "target_zone": "cache", + "description": "Transition 10", + "enforcement": [ + "waf_10", + "mtls_10" + ] + }, + { + "name": "boundary_11", + "source_zone": "worker", + "target_zone": "queue", + "description": "Transition 11", + "enforcement": [ + "waf_11", + "mtls_11" + ] + } + ], + "services": [ + { + "name": "postgres", + "service_type": "database", + "endpoint": "postgres://db:5432", + "purpose": "primary store", + "auth_mechanism": "password" + }, + { + "name": "redis", + "service_type": "cache", + "endpoint": null, + "purpose": "sessions", + "auth_mechanism": null + }, + { + "name": "stripe", + "service_type": "payments", + "endpoint": "https://api.stripe.com", + "purpose": null, + "auth_mechanism": "api_key" + }, + { + "name": "s3", + "service_type": "object_store", + "endpoint": "https://s3.amazonaws.com", + "purpose": "uploads", + "auth_mechanism": "iam" + }, + { + "name": "smtp", + "service_type": "mail", + "endpoint": null, + "purpose": "transactional email", + "auth_mechanism": null + } + ], + "api_surface": [ + { + "method": "GET", + "path": "/api/v1/thing/0", + "handler": "ThingController.action0", + "file_path": "app/api/thing_0.py", + "line": 100, + "auth_required": true, + "rate_limited": null + }, + { + "method": "POST", + "path": "/api/v1/thing/1", + "handler": "ThingController.action1", + "file_path": "app/api/thing_1.py", + "line": 103, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PUT", + "path": "/api/v1/thing/2", + "handler": "ThingController.action2", + "file_path": "app/api/thing_2.py", + "line": 106, + "auth_required": null, + "rate_limited": true + }, + { + "method": "DELETE", + "path": "/api/v1/thing/3", + "handler": "ThingController.action3", + "file_path": "app/api/thing_3.py", + "line": 109, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PATCH", + "path": "/api/v1/thing/4", + "handler": "ThingController.action4", + "file_path": "app/api/thing_4.py", + "line": 112, + "auth_required": true, + "rate_limited": null + }, + { + "method": "GET", + "path": "/api/v1/thing/5", + "handler": "ThingController.action5", + "file_path": "app/api/thing_5.py", + "line": 115, + "auth_required": null, + "rate_limited": true + }, + { + "method": "POST", + "path": "/api/v1/thing/6", + "handler": "ThingController.action6", + "file_path": "app/api/thing_6.py", + "line": 118, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PUT", + "path": "/api/v1/thing/7", + "handler": "ThingController.action7", + "file_path": "app/api/thing_7.py", + "line": 121, + "auth_required": false, + "rate_limited": false + }, + { + "method": "DELETE", + "path": "/api/v1/thing/8", + "handler": "ThingController.action8", + "file_path": "app/api/thing_8.py", + "line": 124, + "auth_required": null, + "rate_limited": true + }, + { + "method": "PATCH", + "path": "/api/v1/thing/9", + "handler": "ThingController.action9", + "file_path": "app/api/thing_9.py", + "line": 127, + "auth_required": false, + "rate_limited": false + }, + { + "method": "GET", + "path": "/api/v1/thing/10", + "handler": "ThingController.action10", + "file_path": "app/api/thing_10.py", + "line": 130, + "auth_required": true, + "rate_limited": null + }, + { + "method": "POST", + "path": "/api/v1/thing/11", + "handler": "ThingController.action11", + "file_path": "app/api/thing_11.py", + "line": 133, + "auth_required": null, + "rate_limited": true + }, + { + "method": "PUT", + "path": "/api/v1/thing/12", + "handler": "ThingController.action12", + "file_path": "app/api/thing_12.py", + "line": 136, + "auth_required": true, + "rate_limited": null + }, + { + "method": "DELETE", + "path": "/api/v1/thing/13", + "handler": "ThingController.action13", + "file_path": "app/api/thing_13.py", + "line": 139, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PATCH", + "path": "/api/v1/thing/14", + "handler": "ThingController.action14", + "file_path": "app/api/thing_14.py", + "line": 142, + "auth_required": null, + "rate_limited": true + }, + { + "method": "GET", + "path": "/api/v1/thing/15", + "handler": "ThingController.action15", + "file_path": "app/api/thing_15.py", + "line": 145, + "auth_required": false, + "rate_limited": false + }, + { + "method": "POST", + "path": "/api/v1/thing/16", + "handler": "ThingController.action16", + "file_path": "app/api/thing_16.py", + "line": 148, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PUT", + "path": "/api/v1/thing/17", + "handler": "ThingController.action17", + "file_path": "app/api/thing_17.py", + "line": 151, + "auth_required": null, + "rate_limited": true + }, + { + "method": "DELETE", + "path": "/api/v1/thing/18", + "handler": "ThingController.action18", + "file_path": "app/api/thing_18.py", + "line": 154, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PATCH", + "path": "/api/v1/thing/19", + "handler": "ThingController.action19", + "file_path": "app/api/thing_19.py", + "line": 157, + "auth_required": false, + "rate_limited": false + } + ] + }, + "config": { + "secrets": [ + { + "id": "secret-00", + "secret_type": "aws_access_key", + "file_path": "config/env_0.yaml", + "line": 3, + "match": "AKIA****0000", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-01", + "secret_type": "github_token", + "file_path": "config/env_1.yaml", + "line": 4, + "match": "AKIA****0001", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-02", + "secret_type": "private_key", + "file_path": "config/env_2.yaml", + "line": 5, + "match": "AKIA****0002", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-03", + "secret_type": "slack_webhook", + "file_path": "config/env_3.yaml", + "line": 6, + "match": "AKIA****0003", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-04", + "secret_type": "generic_api_key", + "file_path": "config/env_4.yaml", + "line": 7, + "match": "AKIA****0004", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-05", + "secret_type": "aws_access_key", + "file_path": "config/env_5.yaml", + "line": 8, + "match": "AKIA****0005", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-06", + "secret_type": "github_token", + "file_path": "config/env_6.yaml", + "line": 9, + "match": "AKIA****0006", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-07", + "secret_type": "private_key", + "file_path": "config/env_7.yaml", + "line": 10, + "match": "AKIA****0007", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-08", + "secret_type": "slack_webhook", + "file_path": "config/env_8.yaml", + "line": 11, + "match": "AKIA****0008", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-09", + "secret_type": "generic_api_key", + "file_path": "config/env_9.yaml", + "line": 12, + "match": "AKIA****0009", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-10", + "secret_type": "aws_access_key", + "file_path": "config/env_10.yaml", + "line": 13, + "match": "AKIA****0010", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-11", + "secret_type": "github_token", + "file_path": "config/env_11.yaml", + "line": 14, + "match": "AKIA****0011", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-12", + "secret_type": "private_key", + "file_path": "config/env_12.yaml", + "line": 15, + "match": "AKIA****0012", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-13", + "secret_type": "slack_webhook", + "file_path": "config/env_13.yaml", + "line": 16, + "match": "AKIA****0013", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-14", + "secret_type": "generic_api_key", + "file_path": "config/env_14.yaml", + "line": 17, + "match": "AKIA****0014", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-15", + "secret_type": "aws_access_key", + "file_path": "config/env_15.yaml", + "line": 18, + "match": "AKIA****0015", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-16", + "secret_type": "github_token", + "file_path": "config/env_16.yaml", + "line": 19, + "match": "AKIA****0016", + "confidence": "medium", + "is_test_value": true + } + ], + "misconfigs": [ + { + "id": "misconfig-00", + "category": "logging", + "file_path": "config/app_0.ini", + "line": null, + "key": "LOG_LEVEL", + "value": "DEBUG", + "risk": "high", + "remediation": "Debug logging in production" + }, + { + "id": "misconfig-01", + "category": "tls", + "file_path": "config/app_1.ini", + "line": 13, + "key": "SSL_VERIFY", + "value": "false", + "risk": "critical", + "remediation": "TLS verification disabled" + }, + { + "id": "misconfig-02", + "category": "cors", + "file_path": "config/app_2.ini", + "line": 14, + "key": "ALLOW_ORIGIN", + "value": "*", + "risk": "high", + "remediation": null + }, + { + "id": "misconfig-03", + "category": "headers", + "file_path": "config/app_3.ini", + "line": 0, + "key": "X_FRAME_OPTIONS", + "value": null, + "risk": "medium", + "remediation": "Missing clickjacking header" + }, + { + "id": "misconfig-04", + "category": "debug", + "file_path": "config/app_4.ini", + "line": 16, + "key": "DJANGO_DEBUG", + "value": "True", + "risk": "critical", + "remediation": "Debug mode enabled" + }, + { + "id": "misconfig-05", + "category": "storage", + "file_path": "config/app_5.ini", + "line": null, + "key": "BUCKET_ACL", + "value": "public-read", + "risk": "critical", + "remediation": "Public bucket" + }, + { + "id": "misconfig-06", + "category": "auth", + "file_path": "config/app_6.ini", + "line": 18, + "key": "SESSION_TIMEOUT", + "value": "999999", + "risk": "medium", + "remediation": null + }, + { + "id": "misconfig-07", + "category": "http", + "file_path": "config/app_7.ini", + "line": 19, + "key": "REDIRECT_HTTPS", + "value": "false", + "risk": "high", + "remediation": "Plain HTTP allowed" + }, + { + "id": "misconfig-08", + "category": "secrets", + "file_path": "config/app_8.ini", + "line": 20, + "key": null, + "value": "inline", + "risk": "medium", + "remediation": "Inline secret" + }, + { + "id": "misconfig-09", + "category": "trace", + "file_path": "config/app_9.ini", + "line": 21, + "key": "OTEL_TRACE_ALL", + "value": "true", + "risk": "low", + "remediation": "Verbose tracing" + }, + { + "id": "misconfig-10", + "category": "network", + "file_path": "config/app_10.ini", + "line": null, + "key": "BIND_ADDR", + "value": "0.0.0.0", + "risk": "medium", + "remediation": null + }, + { + "id": "misconfig-11", + "category": "errors", + "file_path": "config/app_11.ini", + "line": 23, + "key": "SHOW_STACKTRACE", + "value": "true", + "risk": "high", + "remediation": "Error stacktraces exposed" + }, + { + "id": "misconfig-12", + "category": "cache", + "file_path": "config/app_12.ini", + "line": 24, + "key": "CACHE_TTL", + "value": "0", + "risk": "low", + "remediation": null + }, + { + "id": "misconfig-13", + "category": "db", + "file_path": "config/app_13.ini", + "line": 25, + "key": "SSLMODE", + "value": "disable", + "risk": "critical", + "remediation": "Database TLS off" + }, + { + "id": "misconfig-14", + "category": "queue", + "file_path": "config/app_14.ini", + "line": 26, + "key": "PREFETCH", + "value": "1000", + "risk": "low", + "remediation": null + }, + { + "id": "misconfig-15", + "category": "exposure", + "file_path": "config/app_15.ini", + "line": null, + "key": "ADMIN_PATH", + "value": "/admin", + "risk": "medium", + "remediation": "Admin surface exposed" + } + ] + }, + "file_count": 2417, + "frameworks": [ + "Django", + "next.js", + "Spring Boot", + " React ", + "unknown-fw", + "NEXT", + "Vue" + ], + "languages": [ + "Python", + "JavaScript", + "Go", + "Ruby", + "Rust", + "python" + ], + "lines_of_code": 184213 +} \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/prune_crypto.json b/internal/secaf/recontext/testdata/golden/prune_crypto.json new file mode 100644 index 0000000000..672f300221 --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/prune_crypto.json @@ -0,0 +1,166 @@ +{ + "file_count": 2417, + "frameworks": [ + "Django", + "next.js", + "Spring Boot", + " React ", + "unknown-fw", + "NEXT", + "Vue" + ], + "languages": [ + "Python", + "JavaScript", + "Go", + "Ruby", + "Rust", + "python" + ], + "lines_of_code": 184213, + "security_context": { + "auth_model": "jwt", + "auth_details": "HS256 JWTs issued by app/auth/service.py, refresh tokens in Redis", + "crypto_usage": [ + { + "algorithm": "AES", + "key_size": 256, + "mode": "GCM", + "usage_context": "at-rest encryption", + "is_weak": false + }, + { + "algorithm": "MD5", + "key_size": null, + "mode": null, + "usage_context": "password hashing", + "is_weak": true + }, + { + "algorithm": "RSA", + "key_size": 1024, + "mode": null, + "usage_context": "token signing", + "is_weak": true + }, + { + "algorithm": "SHA-256", + "key_size": null, + "mode": null, + "usage_context": null, + "is_weak": false + }, + { + "algorithm": "DES", + "key_size": 56, + "mode": "CBC", + "usage_context": "legacy export", + "is_weak": true + }, + { + "algorithm": "HMAC-SHA1", + "key_size": 160, + "mode": null, + "usage_context": "webhook signature", + "is_weak": true + }, + { + "algorithm": "ChaCha20", + "key_size": 256, + "mode": "Poly1305", + "usage_context": "transport", + "is_weak": false + }, + { + "algorithm": "bcrypt", + "key_size": null, + "mode": null, + "usage_context": "password hashing", + "is_weak": false + }, + { + "algorithm": "RC4", + "key_size": 128, + "mode": null, + "usage_context": null, + "is_weak": true + }, + { + "algorithm": "ECDSA", + "key_size": 256, + "mode": null, + "usage_context": "JWT signing", + "is_weak": null + }, + { + "algorithm": "PBKDF2", + "key_size": null, + "mode": null, + "usage_context": "key derivation", + "is_weak": false + }, + { + "algorithm": "SHA-1", + "key_size": null, + "mode": null, + "usage_context": "checksum", + "is_weak": true + }, + { + "algorithm": "AES", + "key_size": 128, + "mode": "ECB", + "usage_context": "legacy blob", + "is_weak": true + }, + { + "algorithm": "Ed25519", + "key_size": 256, + "mode": null, + "usage_context": "package signing", + "is_weak": false + }, + { + "algorithm": "3DES", + "key_size": 168, + "mode": "CBC", + "usage_context": "legacy tape", + "is_weak": true + }, + { + "algorithm": "Argon2id", + "key_size": null, + "mode": null, + "usage_context": "password hashing", + "is_weak": null + } + ], + "framework_security": [ + "django.middleware.csrf.CsrfViewMiddleware", + "", + "helmet defaults", + "SecurityMiddleware", + "rack-protection", + "spring-security filter chain" + ], + "security_headers": [ + "Content-Security-Policy: default-src 'self'", + "X-Content-Type-Options: nosniff", + "Strict-Transport-Security: max-age=31536000", + "X-Frame-Options: DENY", + "Referrer-Policy: no-referrer", + "", + "Permissions-Policy: geolocation=()" + ], + "deployment_signals": [ + "kubernetes ingress with TLS termination", + "docker-compose exposes 5432", + "no WAF in front of /api", + "secrets mounted from vault", + "readiness probe on /healthz", + "single replica for the worker", + "TLS 1.2 minimum", + "internal service mesh mTLS" + ] + } +} \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/prune_injection.json b/internal/secaf/recontext/testdata/golden/prune_injection.json new file mode 100644 index 0000000000..b6b32a5c5e --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/prune_injection.json @@ -0,0 +1,1509 @@ +{ + "architecture": { + "app_type": "web_api", + "modules": [ + { + "name": "auth_service", + "path": "app/auth/service.py", + "language": "python", + "description": "Session and JWT issuance", + "dependencies": [ + "jwt", + "redis" + ] + }, + { + "name": "rbac", + "path": "app/auth/rbac.py", + "language": "python", + "description": null, + "dependencies": [] + }, + { + "name": "billing", + "path": "app/billing/core.py", + "language": "python", + "description": "Invoice state machine \u2014 handles refunds", + "dependencies": [ + "stripe" + ] + }, + { + "name": "csrf_guard", + "path": "app/web/middleware/csrf.js", + "language": "javascript", + "description": "CSRF middleware", + "dependencies": [] + }, + { + "name": "payments", + "path": "app/billing/payments.go", + "language": "go", + "description": "Charge orchestration", + "dependencies": [ + "stripe-go" + ] + }, + { + "name": "reporting", + "path": "app/reports/render.py", + "language": "python", + "description": "PDF and CSV export", + "dependencies": [ + "weasyprint" + ] + }, + { + "name": "search", + "path": "app/search/index.ts", + "language": "typescript", + "description": "Elasticsearch query builder", + "dependencies": [ + "@elastic/elasticsearch" + ] + }, + { + "name": "session_store", + "path": "app/auth/session_store.py", + "language": "python", + "description": "Redis-backed sessions", + "dependencies": [ + "redis" + ] + }, + { + "name": "notifications", + "path": "app/notify/mailer.rb", + "language": "ruby", + "description": null, + "dependencies": [ + "mail" + ] + }, + { + "name": "permission_matrix", + "path": "app/auth/permissions.py", + "language": "python", + "description": "Role \u2192 permission table", + "dependencies": [] + }, + { + "name": "uploads", + "path": "app/media/uploads.py", + "language": "python", + "description": "S3 multipart upload helper", + "dependencies": [ + "boto3" + ] + }, + { + "name": "webhooks", + "path": "app/integrations/webhooks.go", + "language": "go", + "description": "Outbound webhook dispatcher", + "dependencies": [] + }, + { + "name": "graph_api", + "path": "app/api/graphql.ts", + "language": "typescript", + "description": "GraphQL resolvers \u2014 na\u00efve depth limit", + "dependencies": [ + "graphql" + ] + }, + { + "name": "admin_panel", + "path": "app/admin/views.py", + "language": "python", + "description": "Django admin overrides", + "dependencies": [ + "django" + ] + }, + { + "name": "guard_utils", + "path": "app/common/guard.py", + "language": "python", + "description": "Assorted guard helpers", + "dependencies": [] + }, + { + "name": "cache", + "path": "app/common/cache.py", + "language": "python", + "description": "Memoization wrappers", + "dependencies": [] + }, + { + "name": "migrations", + "path": "db/migrate/2024_add_roles.rb", + "language": "ruby", + "description": "Adds role column", + "dependencies": [] + }, + { + "name": "jwt_tools", + "path": "app/common/jwt_tools.py", + "language": "python", + "description": null, + "dependencies": [ + "pyjwt" + ] + }, + { + "name": "telemetry", + "path": "app/obs/telemetry.go", + "language": "go", + "description": "OTel exporter", + "dependencies": [] + } + ], + "entry_points": [ + { + "kind": "http", + "identifier": "handler_0", + "file_path": "app/entry/e0.py", + "line": 10, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "HTTP", + "identifier": "handler_1", + "file_path": "app/entry/e1.py", + "line": 17, + "method": "POST", + "route": "/v1/resource/1", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_2", + "file_path": "app/entry/e2.py", + "line": 24, + "method": null, + "route": "/v1/resource/2", + "auth_required": null + }, + { + "kind": "api", + "identifier": "handler_3", + "file_path": "app/entry/e3.py", + "line": 31, + "method": "PUT", + "route": null, + "auth_required": true + }, + { + "kind": "graphql", + "identifier": "handler_4", + "file_path": "app/entry/e4.py", + "line": 38, + "method": "GET", + "route": "/v1/resource/4", + "auth_required": false + }, + { + "kind": "rpc", + "identifier": "handler_5", + "file_path": "app/entry/e5.py", + "line": 45, + "method": "POST", + "route": "/v1/resource/5", + "auth_required": null + }, + { + "kind": "route", + "identifier": "handler_6", + "file_path": "app/entry/e6.py", + "line": 52, + "method": null, + "route": null, + "auth_required": true + }, + { + "kind": "cron", + "identifier": "handler_7", + "file_path": "app/entry/e7.py", + "line": 59, + "method": "PUT", + "route": "/v1/resource/7", + "auth_required": false + }, + { + "kind": "queue", + "identifier": "handler_8", + "file_path": "app/entry/e8.py", + "line": 66, + "method": "GET", + "route": "/v1/resource/8", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_9", + "file_path": "app/entry/e9.py", + "line": 73, + "method": "POST", + "route": null, + "auth_required": true + }, + { + "kind": "api", + "identifier": "handler_10", + "file_path": "app/entry/e10.py", + "line": 80, + "method": null, + "route": "/v1/resource/10", + "auth_required": false + }, + { + "kind": "grpc", + "identifier": "handler_11", + "file_path": "app/entry/e11.py", + "line": 87, + "method": "PUT", + "route": "/v1/resource/11", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_12", + "file_path": "app/entry/e12.py", + "line": 94, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "route", + "identifier": "handler_13", + "file_path": "app/entry/e13.py", + "line": 101, + "method": "POST", + "route": "/v1/resource/13", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_14", + "file_path": "app/entry/e14.py", + "line": 108, + "method": null, + "route": "/v1/resource/14", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_15", + "file_path": "app/entry/e15.py", + "line": 115, + "method": "PUT", + "route": null, + "auth_required": true + }, + { + "kind": "websocket", + "identifier": "handler_16", + "file_path": "app/entry/e16.py", + "line": 122, + "method": "GET", + "route": "/v1/resource/16", + "auth_required": false + } + ], + "trust_boundaries": [ + { + "name": "boundary_0", + "source_zone": "internet", + "target_zone": "app", + "description": "Transition 0", + "enforcement": [] + }, + { + "name": "boundary_1", + "source_zone": "dmz", + "target_zone": "db", + "description": "Transition 1", + "enforcement": [ + "waf_1", + "mtls_1" + ] + }, + { + "name": "boundary_2", + "source_zone": "vpc", + "target_zone": "cache", + "description": "Transition 2", + "enforcement": [ + "waf_2", + "mtls_2" + ] + }, + { + "name": "boundary_3", + "source_zone": "worker", + "target_zone": "queue", + "description": "Transition 3", + "enforcement": [ + "waf_3", + "mtls_3" + ] + }, + { + "name": "boundary_4", + "source_zone": "internet", + "target_zone": "app", + "description": "Transition 4", + "enforcement": [] + }, + { + "name": "boundary_5", + "source_zone": "dmz", + "target_zone": "db", + "description": "Transition 5", + "enforcement": [ + "waf_5", + "mtls_5" + ] + }, + { + "name": "boundary_6", + "source_zone": "vpc", + "target_zone": "cache", + "description": "Transition 6", + "enforcement": [ + "waf_6", + "mtls_6" + ] + }, + { + "name": "boundary_7", + "source_zone": "worker", + "target_zone": "queue", + "description": "Transition 7", + "enforcement": [ + "waf_7", + "mtls_7" + ] + }, + { + "name": "boundary_8", + "source_zone": "internet", + "target_zone": "app", + "description": "Transition 8", + "enforcement": [] + }, + { + "name": "boundary_9", + "source_zone": "dmz", + "target_zone": "db", + "description": "Transition 9", + "enforcement": [ + "waf_9", + "mtls_9" + ] + }, + { + "name": "boundary_10", + "source_zone": "vpc", + "target_zone": "cache", + "description": "Transition 10", + "enforcement": [ + "waf_10", + "mtls_10" + ] + }, + { + "name": "boundary_11", + "source_zone": "worker", + "target_zone": "queue", + "description": "Transition 11", + "enforcement": [ + "waf_11", + "mtls_11" + ] + } + ], + "services": [ + { + "name": "postgres", + "service_type": "database", + "endpoint": "postgres://db:5432", + "purpose": "primary store", + "auth_mechanism": "password" + }, + { + "name": "redis", + "service_type": "cache", + "endpoint": null, + "purpose": "sessions", + "auth_mechanism": null + }, + { + "name": "stripe", + "service_type": "payments", + "endpoint": "https://api.stripe.com", + "purpose": null, + "auth_mechanism": "api_key" + }, + { + "name": "s3", + "service_type": "object_store", + "endpoint": "https://s3.amazonaws.com", + "purpose": "uploads", + "auth_mechanism": "iam" + }, + { + "name": "smtp", + "service_type": "mail", + "endpoint": null, + "purpose": "transactional email", + "auth_mechanism": null + } + ], + "api_surface": [ + { + "method": "GET", + "path": "/api/v1/thing/0", + "handler": "ThingController.action0", + "file_path": "app/api/thing_0.py", + "line": 100, + "auth_required": true, + "rate_limited": null + }, + { + "method": "POST", + "path": "/api/v1/thing/1", + "handler": "ThingController.action1", + "file_path": "app/api/thing_1.py", + "line": 103, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PUT", + "path": "/api/v1/thing/2", + "handler": "ThingController.action2", + "file_path": "app/api/thing_2.py", + "line": 106, + "auth_required": null, + "rate_limited": true + }, + { + "method": "DELETE", + "path": "/api/v1/thing/3", + "handler": "ThingController.action3", + "file_path": "app/api/thing_3.py", + "line": 109, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PATCH", + "path": "/api/v1/thing/4", + "handler": "ThingController.action4", + "file_path": "app/api/thing_4.py", + "line": 112, + "auth_required": true, + "rate_limited": null + }, + { + "method": "GET", + "path": "/api/v1/thing/5", + "handler": "ThingController.action5", + "file_path": "app/api/thing_5.py", + "line": 115, + "auth_required": null, + "rate_limited": true + }, + { + "method": "POST", + "path": "/api/v1/thing/6", + "handler": "ThingController.action6", + "file_path": "app/api/thing_6.py", + "line": 118, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PUT", + "path": "/api/v1/thing/7", + "handler": "ThingController.action7", + "file_path": "app/api/thing_7.py", + "line": 121, + "auth_required": false, + "rate_limited": false + }, + { + "method": "DELETE", + "path": "/api/v1/thing/8", + "handler": "ThingController.action8", + "file_path": "app/api/thing_8.py", + "line": 124, + "auth_required": null, + "rate_limited": true + }, + { + "method": "PATCH", + "path": "/api/v1/thing/9", + "handler": "ThingController.action9", + "file_path": "app/api/thing_9.py", + "line": 127, + "auth_required": false, + "rate_limited": false + }, + { + "method": "GET", + "path": "/api/v1/thing/10", + "handler": "ThingController.action10", + "file_path": "app/api/thing_10.py", + "line": 130, + "auth_required": true, + "rate_limited": null + }, + { + "method": "POST", + "path": "/api/v1/thing/11", + "handler": "ThingController.action11", + "file_path": "app/api/thing_11.py", + "line": 133, + "auth_required": null, + "rate_limited": true + }, + { + "method": "PUT", + "path": "/api/v1/thing/12", + "handler": "ThingController.action12", + "file_path": "app/api/thing_12.py", + "line": 136, + "auth_required": true, + "rate_limited": null + }, + { + "method": "DELETE", + "path": "/api/v1/thing/13", + "handler": "ThingController.action13", + "file_path": "app/api/thing_13.py", + "line": 139, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PATCH", + "path": "/api/v1/thing/14", + "handler": "ThingController.action14", + "file_path": "app/api/thing_14.py", + "line": 142, + "auth_required": null, + "rate_limited": true + }, + { + "method": "GET", + "path": "/api/v1/thing/15", + "handler": "ThingController.action15", + "file_path": "app/api/thing_15.py", + "line": 145, + "auth_required": false, + "rate_limited": false + }, + { + "method": "POST", + "path": "/api/v1/thing/16", + "handler": "ThingController.action16", + "file_path": "app/api/thing_16.py", + "line": 148, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PUT", + "path": "/api/v1/thing/17", + "handler": "ThingController.action17", + "file_path": "app/api/thing_17.py", + "line": 151, + "auth_required": null, + "rate_limited": true + }, + { + "method": "DELETE", + "path": "/api/v1/thing/18", + "handler": "ThingController.action18", + "file_path": "app/api/thing_18.py", + "line": 154, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PATCH", + "path": "/api/v1/thing/19", + "handler": "ThingController.action19", + "file_path": "app/api/thing_19.py", + "line": 157, + "auth_required": false, + "rate_limited": false + } + ] + }, + "data_flows": { + "flows": [ + { + "source": "request.args['q']", + "path": [ + { + "file_path": "app/search/index.ts", + "line": 20, + "component": "step_0_a", + "operation": "read" + }, + { + "file_path": "app/util/b.py", + "line": 40, + "component": "step_0_b", + "operation": "write" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/search/index.ts", + "app/db/raw.py", + "app/util/a.py", + "app/util/b.py" + ] + }, + { + "source": "request.cookies['session']", + "path": [ + { + "file_path": "app/auth/session_store.py", + "line": 21, + "component": "step_1_a", + "operation": "read" + }, + { + "file_path": "app/auth/session_store.py", + "line": 41, + "component": "step_1_b", + "operation": "write" + } + ], + "sink": "redis.set", + "sanitized": true, + "files": [ + "app/auth/session_store.py" + ] + }, + { + "source": "request.json['url']", + "path": [ + { + "file_path": "app/integrations/webhooks.go", + "line": 22, + "component": "step_2_a", + "operation": "read" + }, + { + "file_path": "app/net/client.go", + "line": 42, + "component": "step_2_b", + "operation": "write" + } + ], + "sink": "requests.get", + "sanitized": false, + "files": [ + "app/integrations/webhooks.go", + "app/net/client.go" + ] + }, + { + "source": "form['password']", + "path": [ + { + "file_path": "app/auth/service.py", + "line": 23, + "component": "step_3_a", + "operation": "read" + }, + { + "file_path": "app/obs/telemetry.go", + "line": 43, + "component": "step_3_b", + "operation": "write" + } + ], + "sink": "logger.info", + "sanitized": false, + "files": [ + "app/auth/service.py", + "app/obs/telemetry.go" + ] + }, + { + "source": "header['Authorization']", + "path": [ + { + "file_path": "app/common/jwt_tools.py", + "line": 24, + "component": "step_4_a", + "operation": "read" + }, + { + "file_path": "app/common/jwt_tools.py", + "line": 44, + "component": "step_4_b", + "operation": "write" + } + ], + "sink": "jwt.decode", + "sanitized": true, + "files": [ + "app/common/jwt_tools.py" + ] + }, + { + "source": "query['email']", + "path": [ + { + "file_path": "app/notify/mailer.rb", + "line": 25, + "component": "step_5_a", + "operation": "read" + }, + { + "file_path": "app/notify/mailer.rb", + "line": 45, + "component": "step_5_b", + "operation": "write" + } + ], + "sink": "smtp.send", + "sanitized": false, + "files": [ + "app/notify/mailer.rb" + ] + }, + { + "source": "body['card_number']", + "path": [ + { + "file_path": "app/billing/payments.go", + "line": 26, + "component": "step_6_a", + "operation": "read" + }, + { + "file_path": "app/billing/core.py", + "line": 46, + "component": "step_6_b", + "operation": "write" + } + ], + "sink": "stripe.Charge.create", + "sanitized": true, + "files": [ + "app/billing/payments.go", + "app/billing/core.py" + ] + }, + { + "source": "path_param['id']", + "path": [ + { + "file_path": "app/api/thing_1.py", + "line": 27, + "component": "step_7_a", + "operation": "read" + }, + { + "file_path": "app/api/thing_1.py", + "line": 47, + "component": "step_7_b", + "operation": "write" + } + ], + "sink": "orm.filter", + "sanitized": true, + "files": [ + "app/api/thing_1.py" + ] + }, + { + "source": "upload.filename", + "path": [ + { + "file_path": "app/media/uploads.py", + "line": 28, + "component": "step_8_a", + "operation": "read" + }, + { + "file_path": "app/x/w.py", + "line": 48, + "component": "step_8_b", + "operation": "write" + } + ], + "sink": "open", + "sanitized": false, + "files": [ + "app/media/uploads.py", + "app/common/cache.py", + "app/common/guard.py", + "app/x/y.py", + "app/x/z.py", + "app/x/w.py" + ] + }, + { + "source": "env['DEBUG']", + "path": [ + { + "file_path": "app/web/render.py", + "line": 29, + "component": "step_9_a", + "operation": "read" + }, + { + "file_path": "app/web/render.py", + "line": 49, + "component": "step_9_b", + "operation": "write" + } + ], + "sink": "template.render", + "sanitized": false, + "files": [ + "app/web/render.py" + ] + }, + { + "source": "request.body", + "path": [ + { + "file_path": "app/admin/views.py", + "line": 30, + "component": "step_10_a", + "operation": "read" + }, + { + "file_path": "app/admin/views.py", + "line": 50, + "component": "step_10_b", + "operation": "write" + } + ], + "sink": "eval", + "sanitized": false, + "files": [ + "app/admin/views.py" + ] + }, + { + "source": "websocket.msg", + "path": [ + { + "file_path": "app/ws/handler.py", + "line": 31, + "component": "step_11_a", + "operation": "read" + }, + { + "file_path": "app/ws/exec.py", + "line": 51, + "component": "step_11_b", + "operation": "write" + } + ], + "sink": "subprocess.run", + "sanitized": false, + "files": [ + "app/ws/handler.py", + "app/ws/exec.py" + ] + }, + { + "source": "cli.argv", + "path": [ + { + "file_path": "tools/run.py", + "line": 32, + "component": "step_12_a", + "operation": "read" + }, + { + "file_path": "tools/run.py", + "line": 52, + "component": "step_12_b", + "operation": "write" + } + ], + "sink": "os.system", + "sanitized": false, + "files": [ + "tools/run.py" + ] + }, + { + "source": "queue.payload", + "path": [ + { + "file_path": "app/worker/task.py", + "line": 33, + "component": "step_13_a", + "operation": "read" + }, + { + "file_path": "app/worker/task.py", + "line": 53, + "component": "step_13_b", + "operation": "write" + } + ], + "sink": "pickle.loads", + "sanitized": false, + "files": [ + "app/worker/task.py" + ] + }, + { + "source": "request.args['redirect']", + "path": [ + { + "file_path": "app/web/redirect.py", + "line": 34, + "component": "step_14_a", + "operation": "read" + }, + { + "file_path": "app/web/redirect.py", + "line": 54, + "component": "step_14_b", + "operation": "write" + } + ], + "sink": "HttpResponseRedirect", + "sanitized": false, + "files": [ + "app/web/redirect.py" + ] + }, + { + "source": "session['role']", + "path": [ + { + "file_path": "app/auth/permissions.py", + "line": 35, + "component": "step_15_a", + "operation": "read" + }, + { + "file_path": "app/auth/rbac.py", + "line": 55, + "component": "step_15_b", + "operation": "write" + } + ], + "sink": "permission_check", + "sanitized": true, + "files": [ + "app/auth/permissions.py", + "app/auth/rbac.py" + ] + }, + { + "source": "request.files['avatar']", + "path": [ + { + "file_path": "app/media/uploads.py", + "line": 36, + "component": "step_16_a", + "operation": "read" + }, + { + "file_path": "app/media/uploads.py", + "line": 56, + "component": "step_16_b", + "operation": "write" + } + ], + "sink": "s3.put_object", + "sanitized": true, + "files": [ + "app/media/uploads.py" + ] + }, + { + "source": "token", + "path": [ + { + "file_path": "app/common/cache.py", + "line": 37, + "component": "step_17_a", + "operation": "read" + }, + { + "file_path": "app/common/cache.py", + "line": 57, + "component": "step_17_b", + "operation": "write" + } + ], + "sink": "cache.set", + "sanitized": true, + "files": [ + "app/common/cache.py" + ] + }, + { + "source": "request.headers['X-User-Phone']", + "path": [ + { + "file_path": "app/obs/audit.py", + "line": 38, + "component": "step_18_a", + "operation": "read" + }, + { + "file_path": "app/obs/audit.py", + "line": 58, + "component": "step_18_b", + "operation": "write" + } + ], + "sink": "audit_log", + "sanitized": false, + "files": [ + "app/obs/audit.py" + ] + }, + { + "source": "graphql.variables", + "path": [ + { + "file_path": "app/api/graphql.ts", + "line": 39, + "component": "step_19_a", + "operation": "read" + }, + { + "file_path": "app/api/graphql.ts", + "line": 59, + "component": "step_19_b", + "operation": "write" + } + ], + "sink": "db.query", + "sanitized": false, + "files": [ + "app/api/graphql.ts" + ] + } + ], + "sanitization_points": [ + { + "file_path": "app/sanitize/s0.py", + "line": 5, + "function_name": null, + "sanitization_type": "escape", + "protects_against": [] + }, + { + "file_path": "app/sanitize/s1.py", + "line": 9, + "function_name": "clean_1", + "sanitization_type": "parameterize", + "protects_against": [ + "CWE-80", + "CWE-90" + ] + }, + { + "file_path": "app/sanitize/s2.py", + "line": 13, + "function_name": "clean_2", + "sanitization_type": "allowlist", + "protects_against": [ + "CWE-81", + "CWE-91" + ] + }, + { + "file_path": "app/sanitize/s3.py", + "line": 17, + "function_name": "clean_3", + "sanitization_type": "encode", + "protects_against": [] + }, + { + "file_path": "app/sanitize/s4.py", + "line": 21, + "function_name": "clean_4", + "sanitization_type": "escape", + "protects_against": [ + "CWE-83", + "CWE-93" + ] + }, + { + "file_path": "app/sanitize/s5.py", + "line": 25, + "function_name": null, + "sanitization_type": "parameterize", + "protects_against": [ + "CWE-84", + "CWE-94" + ] + }, + { + "file_path": "app/sanitize/s6.py", + "line": 29, + "function_name": "clean_6", + "sanitization_type": "allowlist", + "protects_against": [] + }, + { + "file_path": "app/sanitize/s7.py", + "line": 33, + "function_name": "clean_7", + "sanitization_type": "encode", + "protects_against": [ + "CWE-86", + "CWE-96" + ] + }, + { + "file_path": "app/sanitize/s8.py", + "line": 37, + "function_name": "clean_8", + "sanitization_type": "escape", + "protects_against": [ + "CWE-87", + "CWE-97" + ] + }, + { + "file_path": "app/sanitize/s9.py", + "line": 41, + "function_name": "clean_9", + "sanitization_type": "parameterize", + "protects_against": [] + }, + { + "file_path": "app/sanitize/s10.py", + "line": 45, + "function_name": null, + "sanitization_type": "allowlist", + "protects_against": [ + "CWE-89", + "CWE-99" + ] + }, + { + "file_path": "app/sanitize/s11.py", + "line": 49, + "function_name": "clean_11", + "sanitization_type": "encode", + "protects_against": [ + "CWE-90", + "CWE-100" + ] + } + ], + "sinks": [ + { + "sink_type": "sql", + "file_path": "app/sink/k0.py", + "line": 200, + "function_name": null, + "exploitability_notes": null + }, + { + "sink_type": "command", + "file_path": "app/sink/k1.py", + "line": 202, + "function_name": "sink_fn_1", + "exploitability_notes": "note 1" + }, + { + "sink_type": "template", + "file_path": "app/sink/k2.py", + "line": 204, + "function_name": "sink_fn_2", + "exploitability_notes": "note 2" + }, + { + "sink_type": "file", + "file_path": "app/sink/k3.py", + "line": 206, + "function_name": "sink_fn_3", + "exploitability_notes": null + }, + { + "sink_type": "network", + "file_path": "app/sink/k4.py", + "line": 208, + "function_name": null, + "exploitability_notes": "note 4" + }, + { + "sink_type": "deserialization", + "file_path": "app/sink/k5.py", + "line": 210, + "function_name": "sink_fn_5", + "exploitability_notes": "note 5" + }, + { + "sink_type": "sql", + "file_path": "app/sink/k6.py", + "line": 212, + "function_name": "sink_fn_6", + "exploitability_notes": null + }, + { + "sink_type": "command", + "file_path": "app/sink/k7.py", + "line": 214, + "function_name": "sink_fn_7", + "exploitability_notes": "note 7" + }, + { + "sink_type": "template", + "file_path": "app/sink/k8.py", + "line": 216, + "function_name": null, + "exploitability_notes": "note 8" + }, + { + "sink_type": "file", + "file_path": "app/sink/k9.py", + "line": 218, + "function_name": "sink_fn_9", + "exploitability_notes": null + }, + { + "sink_type": "network", + "file_path": "app/sink/k10.py", + "line": 220, + "function_name": "sink_fn_10", + "exploitability_notes": "note 10" + }, + { + "sink_type": "deserialization", + "file_path": "app/sink/k11.py", + "line": 222, + "function_name": "sink_fn_11", + "exploitability_notes": "note 11" + }, + { + "sink_type": "sql", + "file_path": "app/sink/k12.py", + "line": 224, + "function_name": null, + "exploitability_notes": null + }, + { + "sink_type": "command", + "file_path": "app/sink/k13.py", + "line": 226, + "function_name": "sink_fn_13", + "exploitability_notes": "note 13" + }, + { + "sink_type": "template", + "file_path": "app/sink/k14.py", + "line": 228, + "function_name": "sink_fn_14", + "exploitability_notes": "note 14" + }, + { + "sink_type": "file", + "file_path": "app/sink/k15.py", + "line": 230, + "function_name": "sink_fn_15", + "exploitability_notes": null + }, + { + "sink_type": "network", + "file_path": "app/sink/k16.py", + "line": 232, + "function_name": null, + "exploitability_notes": "note 16" + }, + { + "sink_type": "deserialization", + "file_path": "app/sink/k17.py", + "line": 234, + "function_name": "sink_fn_17", + "exploitability_notes": "note 17" + } + ] + }, + "file_count": 2417, + "frameworks": [ + "Django", + "next.js", + "Spring Boot", + " React ", + "unknown-fw", + "NEXT", + "Vue" + ], + "languages": [ + "Python", + "JavaScript", + "Go", + "Ruby", + "Rust", + "python" + ], + "lines_of_code": 184213, + "security_context": { + "auth_model": "jwt", + "auth_details": "HS256 JWTs issued by app/auth/service.py, refresh tokens in Redis", + "crypto_usage": [ + { + "algorithm": "AES", + "key_size": 256, + "mode": "GCM", + "usage_context": "at-rest encryption", + "is_weak": false + }, + { + "algorithm": "MD5", + "key_size": null, + "mode": null, + "usage_context": "password hashing", + "is_weak": true + }, + { + "algorithm": "RSA", + "key_size": 1024, + "mode": null, + "usage_context": "token signing", + "is_weak": true + }, + { + "algorithm": "SHA-256", + "key_size": null, + "mode": null, + "usage_context": null, + "is_weak": false + }, + { + "algorithm": "DES", + "key_size": 56, + "mode": "CBC", + "usage_context": "legacy export", + "is_weak": true + }, + { + "algorithm": "HMAC-SHA1", + "key_size": 160, + "mode": null, + "usage_context": "webhook signature", + "is_weak": true + }, + { + "algorithm": "ChaCha20", + "key_size": 256, + "mode": "Poly1305", + "usage_context": "transport", + "is_weak": false + }, + { + "algorithm": "bcrypt", + "key_size": null, + "mode": null, + "usage_context": "password hashing", + "is_weak": false + }, + { + "algorithm": "RC4", + "key_size": 128, + "mode": null, + "usage_context": null, + "is_weak": true + }, + { + "algorithm": "ECDSA", + "key_size": 256, + "mode": null, + "usage_context": "JWT signing", + "is_weak": null + }, + { + "algorithm": "PBKDF2", + "key_size": null, + "mode": null, + "usage_context": "key derivation", + "is_weak": false + }, + { + "algorithm": "SHA-1", + "key_size": null, + "mode": null, + "usage_context": "checksum", + "is_weak": true + }, + { + "algorithm": "AES", + "key_size": 128, + "mode": "ECB", + "usage_context": "legacy blob", + "is_weak": true + }, + { + "algorithm": "Ed25519", + "key_size": 256, + "mode": null, + "usage_context": "package signing", + "is_weak": false + }, + { + "algorithm": "3DES", + "key_size": 168, + "mode": "CBC", + "usage_context": "legacy tape", + "is_weak": true + }, + { + "algorithm": "Argon2id", + "key_size": null, + "mode": null, + "usage_context": "password hashing", + "is_weak": null + } + ], + "framework_security": [ + "django.middleware.csrf.CsrfViewMiddleware", + "", + "helmet defaults", + "SecurityMiddleware", + "rack-protection", + "spring-security filter chain" + ], + "security_headers": [ + "Content-Security-Policy: default-src 'self'", + "X-Content-Type-Options: nosniff", + "Strict-Transport-Security: max-age=31536000", + "X-Frame-Options: DENY", + "Referrer-Policy: no-referrer", + "", + "Permissions-Policy: geolocation=()" + ], + "deployment_signals": [ + "kubernetes ingress with TLS termination", + "docker-compose exposes 5432", + "no WAF in front of /api", + "secrets mounted from vault", + "readiness probe on /healthz", + "single replica for the worker", + "TLS 1.2 minimum", + "internal service mesh mTLS" + ] + } +} \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/prune_keys.json b/internal/secaf/recontext/testdata/golden/prune_keys.json new file mode 100644 index 0000000000..bd33641b68 --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/prune_keys.json @@ -0,0 +1,142 @@ +{ + "": [ + "architecture", + "config", + "data_flows", + "dependencies", + "file_count", + "frameworks", + "languages", + "lines_of_code", + "recon_duration_seconds", + "security_context" + ], + "api_security": [ + "architecture", + "data_flows", + "file_count", + "frameworks", + "languages", + "lines_of_code", + "security_context" + ], + "auth": [ + "architecture", + "file_count", + "frameworks", + "languages", + "lines_of_code", + "security_context" + ], + "business_logic": [ + "architecture", + "data_flows", + "file_count", + "frameworks", + "languages", + "lines_of_code", + "security_context" + ], + "config_secrets": [ + "architecture", + "config", + "file_count", + "frameworks", + "languages", + "lines_of_code" + ], + "crypto": [ + "file_count", + "frameworks", + "languages", + "lines_of_code", + "security_context" + ], + "data_exposure": [ + "architecture", + "config", + "data_flows", + "file_count", + "frameworks", + "languages", + "lines_of_code" + ], + "dos": [ + "architecture", + "data_flows", + "file_count", + "frameworks", + "languages", + "lines_of_code" + ], + "injection": [ + "architecture", + "data_flows", + "file_count", + "frameworks", + "languages", + "lines_of_code", + "security_context" + ], + "javascript_specific": [ + "architecture", + "config", + "data_flows", + "dependencies", + "file_count", + "frameworks", + "languages", + "lines_of_code", + "recon_duration_seconds", + "security_context" + ], + "python_specific": [ + "architecture", + "config", + "data_flows", + "dependencies", + "file_count", + "frameworks", + "languages", + "lines_of_code", + "recon_duration_seconds", + "security_context" + ], + "ssrf": [ + "architecture", + "data_flows", + "file_count", + "frameworks", + "languages", + "lines_of_code", + "security_context" + ], + "supply_chain": [ + "dependencies", + "file_count", + "frameworks", + "languages", + "lines_of_code" + ], + "unknown_strategy": [ + "architecture", + "config", + "data_flows", + "dependencies", + "file_count", + "frameworks", + "languages", + "lines_of_code", + "recon_duration_seconds", + "security_context" + ], + "xss": [ + "architecture", + "data_flows", + "file_count", + "frameworks", + "languages", + "lines_of_code", + "security_context" + ] +} diff --git a/internal/secaf/recontext/testdata/golden/prune_supply_chain.json b/internal/secaf/recontext/testdata/golden/prune_supply_chain.json new file mode 100644 index 0000000000..6f56f2662e --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/prune_supply_chain.json @@ -0,0 +1,381 @@ +{ + "dependencies": { + "sbom": [ + { + "name": "django", + "version": "4.2.1", + "ecosystem": "pypi", + "direct": true, + "license": "BSD-3-Clause" + }, + { + "name": "requests", + "version": "2.31.0", + "ecosystem": "pypi", + "direct": true, + "license": "Apache-2.0" + }, + { + "name": "urllib3", + "version": "1.26.5", + "ecosystem": "pypi", + "direct": false, + "license": "MIT" + }, + { + "name": "jinja2", + "version": "3.1.2", + "ecosystem": "pypi", + "direct": false, + "license": null + }, + { + "name": "lodash", + "version": "4.17.19", + "ecosystem": "npm", + "direct": true, + "license": "MIT" + }, + { + "name": "express", + "version": "4.17.1", + "ecosystem": "npm", + "direct": true, + "license": "MIT" + }, + { + "name": "minimist", + "version": "1.2.5", + "ecosystem": "npm", + "direct": false, + "license": "MIT" + }, + { + "name": "axios", + "version": "0.21.1", + "ecosystem": "npm", + "direct": false, + "license": "MIT" + }, + { + "name": "github.com/gin-gonic/gin", + "version": "1.9.0", + "ecosystem": "go", + "direct": true, + "license": "MIT" + }, + { + "name": "golang.org/x/net", + "version": "0.7.0", + "ecosystem": "go", + "direct": false, + "license": "BSD-3-Clause" + }, + { + "name": "rails", + "version": "7.0.4", + "ecosystem": "rubygems", + "direct": true, + "license": "MIT" + }, + { + "name": "nokogiri", + "version": "1.13.6", + "ecosystem": "rubygems", + "direct": false, + "license": "MIT" + }, + { + "name": "django", + "version": "4.2.1", + "ecosystem": "pypi", + "direct": true, + "license": "BSD-3-Clause" + }, + { + "name": "requests", + "version": "2.31.0", + "ecosystem": "pypi", + "direct": true, + "license": "Apache-2.0" + } + ], + "known_cves": [ + { + "cve_id": "CVE-2023-0001", + "package": "django", + "installed_version": "4.2.1", + "fixed_version": "4.2.5", + "cvss_v4_score": 9.8, + "epss_score": 0.5, + "direct": true, + "reachable": true + }, + { + "cve_id": "CVE-2023-0002", + "package": "urllib3", + "installed_version": "1.26.5", + "fixed_version": null, + "cvss_v4_score": 7.5, + "epss_score": 0.1, + "direct": false, + "reachable": false + }, + { + "cve_id": "CVE-2023-0003", + "package": "lodash", + "installed_version": "4.17.19", + "fixed_version": "4.17.21", + "cvss_v4_score": 9.8, + "epss_score": 0.9, + "direct": true, + "reachable": true + }, + { + "cve_id": "CVE-2023-0004", + "package": "minimist", + "installed_version": "1.2.5", + "fixed_version": "1.2.6", + "cvss_v4_score": null, + "epss_score": null, + "direct": false, + "reachable": null + }, + { + "cve_id": "CVE-2023-0005", + "package": "axios", + "installed_version": "0.21.1", + "fixed_version": "0.21.2", + "cvss_v4_score": 5.3, + "epss_score": 0.02, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0006", + "package": "jinja2", + "installed_version": "3.1.2", + "fixed_version": null, + "cvss_v4_score": 6.1, + "epss_score": null, + "direct": false, + "reachable": false + }, + { + "cve_id": "CVE-2023-0007", + "package": "express", + "installed_version": "4.17.1", + "fixed_version": "4.18.0", + "cvss_v4_score": 4.3, + "epss_score": 0.005, + "direct": true, + "reachable": null + }, + { + "cve_id": "CVE-2023-0008", + "package": "nokogiri", + "installed_version": "1.13.6", + "fixed_version": "1.13.10", + "cvss_v4_score": 8.8, + "epss_score": 0.3, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0009", + "package": "golang.org/x/net", + "installed_version": "0.7.0", + "fixed_version": "0.17.0", + "cvss_v4_score": 7.5, + "epss_score": 0.44, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0010", + "package": "rails", + "installed_version": "7.0.4", + "fixed_version": "7.0.7", + "cvss_v4_score": 9.1, + "epss_score": 0.6, + "direct": true, + "reachable": false + }, + { + "cve_id": "CVE-2023-0011", + "package": "gin", + "installed_version": "1.9.0", + "fixed_version": null, + "cvss_v4_score": 3.7, + "epss_score": 0.001, + "direct": true, + "reachable": null + }, + { + "cve_id": "CVE-2023-0012", + "package": "requests", + "installed_version": "2.31.0", + "fixed_version": "2.32.0", + "cvss_v4_score": 6.5, + "epss_score": 0.07, + "direct": true, + "reachable": true + }, + { + "cve_id": "CVE-2023-0013", + "package": "pyyaml", + "installed_version": "5.3", + "fixed_version": "5.4", + "cvss_v4_score": 9.8, + "epss_score": 0.5, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0014", + "package": "log4j", + "installed_version": "2.14.0", + "fixed_version": "2.17.1", + "cvss_v4_score": 10.0, + "epss_score": 0.97, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0015", + "package": "openssl", + "installed_version": "1.1.1k", + "fixed_version": "1.1.1t", + "cvss_v4_score": 7.4, + "epss_score": 0.12, + "direct": false, + "reachable": null + }, + { + "cve_id": "CVE-2023-0016", + "package": "zlib", + "installed_version": "1.2.11", + "fixed_version": "1.2.12", + "cvss_v4_score": 8.2, + "epss_score": 0.04, + "direct": false, + "reachable": false + }, + { + "cve_id": "CVE-2023-0017", + "package": "libxml2", + "installed_version": "2.9.10", + "fixed_version": null, + "cvss_v4_score": null, + "epss_score": 0.25, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0018", + "package": "curl", + "installed_version": "7.68.0", + "fixed_version": "7.88.0", + "cvss_v4_score": 8.1, + "epss_score": null, + "direct": true, + "reachable": true + } + ], + "outdated": [ + { + "package": "pkg-0", + "current_version": "1.0.0", + "latest_version": "2.0.0", + "direct": true + }, + { + "package": "pkg-1", + "current_version": "1.1.0", + "latest_version": "2.1.0", + "direct": false + }, + { + "package": "pkg-2", + "current_version": "1.2.0", + "latest_version": "2.2.0", + "direct": true + }, + { + "package": "pkg-3", + "current_version": "1.3.0", + "latest_version": "2.3.0", + "direct": false + }, + { + "package": "pkg-4", + "current_version": "1.4.0", + "latest_version": "2.4.0", + "direct": true + }, + { + "package": "pkg-5", + "current_version": "1.5.0", + "latest_version": "2.5.0", + "direct": false + }, + { + "package": "pkg-6", + "current_version": "1.6.0", + "latest_version": "2.6.0", + "direct": true + }, + { + "package": "pkg-7", + "current_version": "1.7.0", + "latest_version": "2.7.0", + "direct": false + }, + { + "package": "pkg-8", + "current_version": "1.8.0", + "latest_version": "2.8.0", + "direct": true + }, + { + "package": "pkg-9", + "current_version": "1.9.0", + "latest_version": "2.9.0", + "direct": false + }, + { + "package": "pkg-10", + "current_version": "1.10.0", + "latest_version": "2.10.0", + "direct": true + }, + { + "package": "pkg-11", + "current_version": "1.11.0", + "latest_version": "2.11.0", + "direct": false + } + ], + "direct_count": 7, + "transitive_count": 143 + }, + "file_count": 2417, + "frameworks": [ + "Django", + "next.js", + "Spring Boot", + " React ", + "unknown-fw", + "NEXT", + "Vue" + ], + "languages": [ + "Python", + "JavaScript", + "Go", + "Ruby", + "Rust", + "python" + ], + "lines_of_code": 184213 +} \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/prune_unknown_strategy.json b/internal/secaf/recontext/testdata/golden/prune_unknown_strategy.json new file mode 100644 index 0000000000..576776d002 --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/prune_unknown_strategy.json @@ -0,0 +1,2189 @@ +{ + "architecture": { + "app_type": "web_api", + "modules": [ + { + "name": "auth_service", + "path": "app/auth/service.py", + "language": "python", + "description": "Session and JWT issuance", + "dependencies": [ + "jwt", + "redis" + ] + }, + { + "name": "rbac", + "path": "app/auth/rbac.py", + "language": "python", + "description": null, + "dependencies": [] + }, + { + "name": "billing", + "path": "app/billing/core.py", + "language": "python", + "description": "Invoice state machine \u2014 handles refunds", + "dependencies": [ + "stripe" + ] + }, + { + "name": "csrf_guard", + "path": "app/web/middleware/csrf.js", + "language": "javascript", + "description": "CSRF middleware", + "dependencies": [] + }, + { + "name": "payments", + "path": "app/billing/payments.go", + "language": "go", + "description": "Charge orchestration", + "dependencies": [ + "stripe-go" + ] + }, + { + "name": "reporting", + "path": "app/reports/render.py", + "language": "python", + "description": "PDF and CSV export", + "dependencies": [ + "weasyprint" + ] + }, + { + "name": "search", + "path": "app/search/index.ts", + "language": "typescript", + "description": "Elasticsearch query builder", + "dependencies": [ + "@elastic/elasticsearch" + ] + }, + { + "name": "session_store", + "path": "app/auth/session_store.py", + "language": "python", + "description": "Redis-backed sessions", + "dependencies": [ + "redis" + ] + }, + { + "name": "notifications", + "path": "app/notify/mailer.rb", + "language": "ruby", + "description": null, + "dependencies": [ + "mail" + ] + }, + { + "name": "permission_matrix", + "path": "app/auth/permissions.py", + "language": "python", + "description": "Role \u2192 permission table", + "dependencies": [] + }, + { + "name": "uploads", + "path": "app/media/uploads.py", + "language": "python", + "description": "S3 multipart upload helper", + "dependencies": [ + "boto3" + ] + }, + { + "name": "webhooks", + "path": "app/integrations/webhooks.go", + "language": "go", + "description": "Outbound webhook dispatcher", + "dependencies": [] + }, + { + "name": "graph_api", + "path": "app/api/graphql.ts", + "language": "typescript", + "description": "GraphQL resolvers \u2014 na\u00efve depth limit", + "dependencies": [ + "graphql" + ] + }, + { + "name": "admin_panel", + "path": "app/admin/views.py", + "language": "python", + "description": "Django admin overrides", + "dependencies": [ + "django" + ] + }, + { + "name": "guard_utils", + "path": "app/common/guard.py", + "language": "python", + "description": "Assorted guard helpers", + "dependencies": [] + }, + { + "name": "cache", + "path": "app/common/cache.py", + "language": "python", + "description": "Memoization wrappers", + "dependencies": [] + }, + { + "name": "migrations", + "path": "db/migrate/2024_add_roles.rb", + "language": "ruby", + "description": "Adds role column", + "dependencies": [] + }, + { + "name": "jwt_tools", + "path": "app/common/jwt_tools.py", + "language": "python", + "description": null, + "dependencies": [ + "pyjwt" + ] + }, + { + "name": "telemetry", + "path": "app/obs/telemetry.go", + "language": "go", + "description": "OTel exporter", + "dependencies": [] + } + ], + "entry_points": [ + { + "kind": "http", + "identifier": "handler_0", + "file_path": "app/entry/e0.py", + "line": 10, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "HTTP", + "identifier": "handler_1", + "file_path": "app/entry/e1.py", + "line": 17, + "method": "POST", + "route": "/v1/resource/1", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_2", + "file_path": "app/entry/e2.py", + "line": 24, + "method": null, + "route": "/v1/resource/2", + "auth_required": null + }, + { + "kind": "api", + "identifier": "handler_3", + "file_path": "app/entry/e3.py", + "line": 31, + "method": "PUT", + "route": null, + "auth_required": true + }, + { + "kind": "graphql", + "identifier": "handler_4", + "file_path": "app/entry/e4.py", + "line": 38, + "method": "GET", + "route": "/v1/resource/4", + "auth_required": false + }, + { + "kind": "rpc", + "identifier": "handler_5", + "file_path": "app/entry/e5.py", + "line": 45, + "method": "POST", + "route": "/v1/resource/5", + "auth_required": null + }, + { + "kind": "route", + "identifier": "handler_6", + "file_path": "app/entry/e6.py", + "line": 52, + "method": null, + "route": null, + "auth_required": true + }, + { + "kind": "cron", + "identifier": "handler_7", + "file_path": "app/entry/e7.py", + "line": 59, + "method": "PUT", + "route": "/v1/resource/7", + "auth_required": false + }, + { + "kind": "queue", + "identifier": "handler_8", + "file_path": "app/entry/e8.py", + "line": 66, + "method": "GET", + "route": "/v1/resource/8", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_9", + "file_path": "app/entry/e9.py", + "line": 73, + "method": "POST", + "route": null, + "auth_required": true + }, + { + "kind": "api", + "identifier": "handler_10", + "file_path": "app/entry/e10.py", + "line": 80, + "method": null, + "route": "/v1/resource/10", + "auth_required": false + }, + { + "kind": "grpc", + "identifier": "handler_11", + "file_path": "app/entry/e11.py", + "line": 87, + "method": "PUT", + "route": "/v1/resource/11", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_12", + "file_path": "app/entry/e12.py", + "line": 94, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "route", + "identifier": "handler_13", + "file_path": "app/entry/e13.py", + "line": 101, + "method": "POST", + "route": "/v1/resource/13", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_14", + "file_path": "app/entry/e14.py", + "line": 108, + "method": null, + "route": "/v1/resource/14", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_15", + "file_path": "app/entry/e15.py", + "line": 115, + "method": "PUT", + "route": null, + "auth_required": true + }, + { + "kind": "websocket", + "identifier": "handler_16", + "file_path": "app/entry/e16.py", + "line": 122, + "method": "GET", + "route": "/v1/resource/16", + "auth_required": false + } + ], + "trust_boundaries": [ + { + "name": "boundary_0", + "source_zone": "internet", + "target_zone": "app", + "description": "Transition 0", + "enforcement": [] + }, + { + "name": "boundary_1", + "source_zone": "dmz", + "target_zone": "db", + "description": "Transition 1", + "enforcement": [ + "waf_1", + "mtls_1" + ] + }, + { + "name": "boundary_2", + "source_zone": "vpc", + "target_zone": "cache", + "description": "Transition 2", + "enforcement": [ + "waf_2", + "mtls_2" + ] + }, + { + "name": "boundary_3", + "source_zone": "worker", + "target_zone": "queue", + "description": "Transition 3", + "enforcement": [ + "waf_3", + "mtls_3" + ] + }, + { + "name": "boundary_4", + "source_zone": "internet", + "target_zone": "app", + "description": "Transition 4", + "enforcement": [] + }, + { + "name": "boundary_5", + "source_zone": "dmz", + "target_zone": "db", + "description": "Transition 5", + "enforcement": [ + "waf_5", + "mtls_5" + ] + }, + { + "name": "boundary_6", + "source_zone": "vpc", + "target_zone": "cache", + "description": "Transition 6", + "enforcement": [ + "waf_6", + "mtls_6" + ] + }, + { + "name": "boundary_7", + "source_zone": "worker", + "target_zone": "queue", + "description": "Transition 7", + "enforcement": [ + "waf_7", + "mtls_7" + ] + }, + { + "name": "boundary_8", + "source_zone": "internet", + "target_zone": "app", + "description": "Transition 8", + "enforcement": [] + }, + { + "name": "boundary_9", + "source_zone": "dmz", + "target_zone": "db", + "description": "Transition 9", + "enforcement": [ + "waf_9", + "mtls_9" + ] + }, + { + "name": "boundary_10", + "source_zone": "vpc", + "target_zone": "cache", + "description": "Transition 10", + "enforcement": [ + "waf_10", + "mtls_10" + ] + }, + { + "name": "boundary_11", + "source_zone": "worker", + "target_zone": "queue", + "description": "Transition 11", + "enforcement": [ + "waf_11", + "mtls_11" + ] + } + ], + "services": [ + { + "name": "postgres", + "service_type": "database", + "endpoint": "postgres://db:5432", + "purpose": "primary store", + "auth_mechanism": "password" + }, + { + "name": "redis", + "service_type": "cache", + "endpoint": null, + "purpose": "sessions", + "auth_mechanism": null + }, + { + "name": "stripe", + "service_type": "payments", + "endpoint": "https://api.stripe.com", + "purpose": null, + "auth_mechanism": "api_key" + }, + { + "name": "s3", + "service_type": "object_store", + "endpoint": "https://s3.amazonaws.com", + "purpose": "uploads", + "auth_mechanism": "iam" + }, + { + "name": "smtp", + "service_type": "mail", + "endpoint": null, + "purpose": "transactional email", + "auth_mechanism": null + } + ], + "api_surface": [ + { + "method": "GET", + "path": "/api/v1/thing/0", + "handler": "ThingController.action0", + "file_path": "app/api/thing_0.py", + "line": 100, + "auth_required": true, + "rate_limited": null + }, + { + "method": "POST", + "path": "/api/v1/thing/1", + "handler": "ThingController.action1", + "file_path": "app/api/thing_1.py", + "line": 103, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PUT", + "path": "/api/v1/thing/2", + "handler": "ThingController.action2", + "file_path": "app/api/thing_2.py", + "line": 106, + "auth_required": null, + "rate_limited": true + }, + { + "method": "DELETE", + "path": "/api/v1/thing/3", + "handler": "ThingController.action3", + "file_path": "app/api/thing_3.py", + "line": 109, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PATCH", + "path": "/api/v1/thing/4", + "handler": "ThingController.action4", + "file_path": "app/api/thing_4.py", + "line": 112, + "auth_required": true, + "rate_limited": null + }, + { + "method": "GET", + "path": "/api/v1/thing/5", + "handler": "ThingController.action5", + "file_path": "app/api/thing_5.py", + "line": 115, + "auth_required": null, + "rate_limited": true + }, + { + "method": "POST", + "path": "/api/v1/thing/6", + "handler": "ThingController.action6", + "file_path": "app/api/thing_6.py", + "line": 118, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PUT", + "path": "/api/v1/thing/7", + "handler": "ThingController.action7", + "file_path": "app/api/thing_7.py", + "line": 121, + "auth_required": false, + "rate_limited": false + }, + { + "method": "DELETE", + "path": "/api/v1/thing/8", + "handler": "ThingController.action8", + "file_path": "app/api/thing_8.py", + "line": 124, + "auth_required": null, + "rate_limited": true + }, + { + "method": "PATCH", + "path": "/api/v1/thing/9", + "handler": "ThingController.action9", + "file_path": "app/api/thing_9.py", + "line": 127, + "auth_required": false, + "rate_limited": false + }, + { + "method": "GET", + "path": "/api/v1/thing/10", + "handler": "ThingController.action10", + "file_path": "app/api/thing_10.py", + "line": 130, + "auth_required": true, + "rate_limited": null + }, + { + "method": "POST", + "path": "/api/v1/thing/11", + "handler": "ThingController.action11", + "file_path": "app/api/thing_11.py", + "line": 133, + "auth_required": null, + "rate_limited": true + }, + { + "method": "PUT", + "path": "/api/v1/thing/12", + "handler": "ThingController.action12", + "file_path": "app/api/thing_12.py", + "line": 136, + "auth_required": true, + "rate_limited": null + }, + { + "method": "DELETE", + "path": "/api/v1/thing/13", + "handler": "ThingController.action13", + "file_path": "app/api/thing_13.py", + "line": 139, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PATCH", + "path": "/api/v1/thing/14", + "handler": "ThingController.action14", + "file_path": "app/api/thing_14.py", + "line": 142, + "auth_required": null, + "rate_limited": true + }, + { + "method": "GET", + "path": "/api/v1/thing/15", + "handler": "ThingController.action15", + "file_path": "app/api/thing_15.py", + "line": 145, + "auth_required": false, + "rate_limited": false + }, + { + "method": "POST", + "path": "/api/v1/thing/16", + "handler": "ThingController.action16", + "file_path": "app/api/thing_16.py", + "line": 148, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PUT", + "path": "/api/v1/thing/17", + "handler": "ThingController.action17", + "file_path": "app/api/thing_17.py", + "line": 151, + "auth_required": null, + "rate_limited": true + }, + { + "method": "DELETE", + "path": "/api/v1/thing/18", + "handler": "ThingController.action18", + "file_path": "app/api/thing_18.py", + "line": 154, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PATCH", + "path": "/api/v1/thing/19", + "handler": "ThingController.action19", + "file_path": "app/api/thing_19.py", + "line": 157, + "auth_required": false, + "rate_limited": false + } + ] + }, + "config": { + "secrets": [ + { + "id": "secret-00", + "secret_type": "aws_access_key", + "file_path": "config/env_0.yaml", + "line": 3, + "match": "AKIA****0000", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-01", + "secret_type": "github_token", + "file_path": "config/env_1.yaml", + "line": 4, + "match": "AKIA****0001", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-02", + "secret_type": "private_key", + "file_path": "config/env_2.yaml", + "line": 5, + "match": "AKIA****0002", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-03", + "secret_type": "slack_webhook", + "file_path": "config/env_3.yaml", + "line": 6, + "match": "AKIA****0003", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-04", + "secret_type": "generic_api_key", + "file_path": "config/env_4.yaml", + "line": 7, + "match": "AKIA****0004", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-05", + "secret_type": "aws_access_key", + "file_path": "config/env_5.yaml", + "line": 8, + "match": "AKIA****0005", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-06", + "secret_type": "github_token", + "file_path": "config/env_6.yaml", + "line": 9, + "match": "AKIA****0006", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-07", + "secret_type": "private_key", + "file_path": "config/env_7.yaml", + "line": 10, + "match": "AKIA****0007", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-08", + "secret_type": "slack_webhook", + "file_path": "config/env_8.yaml", + "line": 11, + "match": "AKIA****0008", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-09", + "secret_type": "generic_api_key", + "file_path": "config/env_9.yaml", + "line": 12, + "match": "AKIA****0009", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-10", + "secret_type": "aws_access_key", + "file_path": "config/env_10.yaml", + "line": 13, + "match": "AKIA****0010", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-11", + "secret_type": "github_token", + "file_path": "config/env_11.yaml", + "line": 14, + "match": "AKIA****0011", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-12", + "secret_type": "private_key", + "file_path": "config/env_12.yaml", + "line": 15, + "match": "AKIA****0012", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-13", + "secret_type": "slack_webhook", + "file_path": "config/env_13.yaml", + "line": 16, + "match": "AKIA****0013", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-14", + "secret_type": "generic_api_key", + "file_path": "config/env_14.yaml", + "line": 17, + "match": "AKIA****0014", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-15", + "secret_type": "aws_access_key", + "file_path": "config/env_15.yaml", + "line": 18, + "match": "AKIA****0015", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-16", + "secret_type": "github_token", + "file_path": "config/env_16.yaml", + "line": 19, + "match": "AKIA****0016", + "confidence": "medium", + "is_test_value": true + } + ], + "misconfigs": [ + { + "id": "misconfig-00", + "category": "logging", + "file_path": "config/app_0.ini", + "line": null, + "key": "LOG_LEVEL", + "value": "DEBUG", + "risk": "high", + "remediation": "Debug logging in production" + }, + { + "id": "misconfig-01", + "category": "tls", + "file_path": "config/app_1.ini", + "line": 13, + "key": "SSL_VERIFY", + "value": "false", + "risk": "critical", + "remediation": "TLS verification disabled" + }, + { + "id": "misconfig-02", + "category": "cors", + "file_path": "config/app_2.ini", + "line": 14, + "key": "ALLOW_ORIGIN", + "value": "*", + "risk": "high", + "remediation": null + }, + { + "id": "misconfig-03", + "category": "headers", + "file_path": "config/app_3.ini", + "line": 0, + "key": "X_FRAME_OPTIONS", + "value": null, + "risk": "medium", + "remediation": "Missing clickjacking header" + }, + { + "id": "misconfig-04", + "category": "debug", + "file_path": "config/app_4.ini", + "line": 16, + "key": "DJANGO_DEBUG", + "value": "True", + "risk": "critical", + "remediation": "Debug mode enabled" + }, + { + "id": "misconfig-05", + "category": "storage", + "file_path": "config/app_5.ini", + "line": null, + "key": "BUCKET_ACL", + "value": "public-read", + "risk": "critical", + "remediation": "Public bucket" + }, + { + "id": "misconfig-06", + "category": "auth", + "file_path": "config/app_6.ini", + "line": 18, + "key": "SESSION_TIMEOUT", + "value": "999999", + "risk": "medium", + "remediation": null + }, + { + "id": "misconfig-07", + "category": "http", + "file_path": "config/app_7.ini", + "line": 19, + "key": "REDIRECT_HTTPS", + "value": "false", + "risk": "high", + "remediation": "Plain HTTP allowed" + }, + { + "id": "misconfig-08", + "category": "secrets", + "file_path": "config/app_8.ini", + "line": 20, + "key": null, + "value": "inline", + "risk": "medium", + "remediation": "Inline secret" + }, + { + "id": "misconfig-09", + "category": "trace", + "file_path": "config/app_9.ini", + "line": 21, + "key": "OTEL_TRACE_ALL", + "value": "true", + "risk": "low", + "remediation": "Verbose tracing" + }, + { + "id": "misconfig-10", + "category": "network", + "file_path": "config/app_10.ini", + "line": null, + "key": "BIND_ADDR", + "value": "0.0.0.0", + "risk": "medium", + "remediation": null + }, + { + "id": "misconfig-11", + "category": "errors", + "file_path": "config/app_11.ini", + "line": 23, + "key": "SHOW_STACKTRACE", + "value": "true", + "risk": "high", + "remediation": "Error stacktraces exposed" + }, + { + "id": "misconfig-12", + "category": "cache", + "file_path": "config/app_12.ini", + "line": 24, + "key": "CACHE_TTL", + "value": "0", + "risk": "low", + "remediation": null + }, + { + "id": "misconfig-13", + "category": "db", + "file_path": "config/app_13.ini", + "line": 25, + "key": "SSLMODE", + "value": "disable", + "risk": "critical", + "remediation": "Database TLS off" + }, + { + "id": "misconfig-14", + "category": "queue", + "file_path": "config/app_14.ini", + "line": 26, + "key": "PREFETCH", + "value": "1000", + "risk": "low", + "remediation": null + }, + { + "id": "misconfig-15", + "category": "exposure", + "file_path": "config/app_15.ini", + "line": null, + "key": "ADMIN_PATH", + "value": "/admin", + "risk": "medium", + "remediation": "Admin surface exposed" + } + ] + }, + "data_flows": { + "flows": [ + { + "source": "request.args['q']", + "path": [ + { + "file_path": "app/search/index.ts", + "line": 20, + "component": "step_0_a", + "operation": "read" + }, + { + "file_path": "app/util/b.py", + "line": 40, + "component": "step_0_b", + "operation": "write" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/search/index.ts", + "app/db/raw.py", + "app/util/a.py", + "app/util/b.py" + ] + }, + { + "source": "request.cookies['session']", + "path": [ + { + "file_path": "app/auth/session_store.py", + "line": 21, + "component": "step_1_a", + "operation": "read" + }, + { + "file_path": "app/auth/session_store.py", + "line": 41, + "component": "step_1_b", + "operation": "write" + } + ], + "sink": "redis.set", + "sanitized": true, + "files": [ + "app/auth/session_store.py" + ] + }, + { + "source": "request.json['url']", + "path": [ + { + "file_path": "app/integrations/webhooks.go", + "line": 22, + "component": "step_2_a", + "operation": "read" + }, + { + "file_path": "app/net/client.go", + "line": 42, + "component": "step_2_b", + "operation": "write" + } + ], + "sink": "requests.get", + "sanitized": false, + "files": [ + "app/integrations/webhooks.go", + "app/net/client.go" + ] + }, + { + "source": "form['password']", + "path": [ + { + "file_path": "app/auth/service.py", + "line": 23, + "component": "step_3_a", + "operation": "read" + }, + { + "file_path": "app/obs/telemetry.go", + "line": 43, + "component": "step_3_b", + "operation": "write" + } + ], + "sink": "logger.info", + "sanitized": false, + "files": [ + "app/auth/service.py", + "app/obs/telemetry.go" + ] + }, + { + "source": "header['Authorization']", + "path": [ + { + "file_path": "app/common/jwt_tools.py", + "line": 24, + "component": "step_4_a", + "operation": "read" + }, + { + "file_path": "app/common/jwt_tools.py", + "line": 44, + "component": "step_4_b", + "operation": "write" + } + ], + "sink": "jwt.decode", + "sanitized": true, + "files": [ + "app/common/jwt_tools.py" + ] + }, + { + "source": "query['email']", + "path": [ + { + "file_path": "app/notify/mailer.rb", + "line": 25, + "component": "step_5_a", + "operation": "read" + }, + { + "file_path": "app/notify/mailer.rb", + "line": 45, + "component": "step_5_b", + "operation": "write" + } + ], + "sink": "smtp.send", + "sanitized": false, + "files": [ + "app/notify/mailer.rb" + ] + }, + { + "source": "body['card_number']", + "path": [ + { + "file_path": "app/billing/payments.go", + "line": 26, + "component": "step_6_a", + "operation": "read" + }, + { + "file_path": "app/billing/core.py", + "line": 46, + "component": "step_6_b", + "operation": "write" + } + ], + "sink": "stripe.Charge.create", + "sanitized": true, + "files": [ + "app/billing/payments.go", + "app/billing/core.py" + ] + }, + { + "source": "path_param['id']", + "path": [ + { + "file_path": "app/api/thing_1.py", + "line": 27, + "component": "step_7_a", + "operation": "read" + }, + { + "file_path": "app/api/thing_1.py", + "line": 47, + "component": "step_7_b", + "operation": "write" + } + ], + "sink": "orm.filter", + "sanitized": true, + "files": [ + "app/api/thing_1.py" + ] + }, + { + "source": "upload.filename", + "path": [ + { + "file_path": "app/media/uploads.py", + "line": 28, + "component": "step_8_a", + "operation": "read" + }, + { + "file_path": "app/x/w.py", + "line": 48, + "component": "step_8_b", + "operation": "write" + } + ], + "sink": "open", + "sanitized": false, + "files": [ + "app/media/uploads.py", + "app/common/cache.py", + "app/common/guard.py", + "app/x/y.py", + "app/x/z.py", + "app/x/w.py" + ] + }, + { + "source": "env['DEBUG']", + "path": [ + { + "file_path": "app/web/render.py", + "line": 29, + "component": "step_9_a", + "operation": "read" + }, + { + "file_path": "app/web/render.py", + "line": 49, + "component": "step_9_b", + "operation": "write" + } + ], + "sink": "template.render", + "sanitized": false, + "files": [ + "app/web/render.py" + ] + }, + { + "source": "request.body", + "path": [ + { + "file_path": "app/admin/views.py", + "line": 30, + "component": "step_10_a", + "operation": "read" + }, + { + "file_path": "app/admin/views.py", + "line": 50, + "component": "step_10_b", + "operation": "write" + } + ], + "sink": "eval", + "sanitized": false, + "files": [ + "app/admin/views.py" + ] + }, + { + "source": "websocket.msg", + "path": [ + { + "file_path": "app/ws/handler.py", + "line": 31, + "component": "step_11_a", + "operation": "read" + }, + { + "file_path": "app/ws/exec.py", + "line": 51, + "component": "step_11_b", + "operation": "write" + } + ], + "sink": "subprocess.run", + "sanitized": false, + "files": [ + "app/ws/handler.py", + "app/ws/exec.py" + ] + }, + { + "source": "cli.argv", + "path": [ + { + "file_path": "tools/run.py", + "line": 32, + "component": "step_12_a", + "operation": "read" + }, + { + "file_path": "tools/run.py", + "line": 52, + "component": "step_12_b", + "operation": "write" + } + ], + "sink": "os.system", + "sanitized": false, + "files": [ + "tools/run.py" + ] + }, + { + "source": "queue.payload", + "path": [ + { + "file_path": "app/worker/task.py", + "line": 33, + "component": "step_13_a", + "operation": "read" + }, + { + "file_path": "app/worker/task.py", + "line": 53, + "component": "step_13_b", + "operation": "write" + } + ], + "sink": "pickle.loads", + "sanitized": false, + "files": [ + "app/worker/task.py" + ] + }, + { + "source": "request.args['redirect']", + "path": [ + { + "file_path": "app/web/redirect.py", + "line": 34, + "component": "step_14_a", + "operation": "read" + }, + { + "file_path": "app/web/redirect.py", + "line": 54, + "component": "step_14_b", + "operation": "write" + } + ], + "sink": "HttpResponseRedirect", + "sanitized": false, + "files": [ + "app/web/redirect.py" + ] + }, + { + "source": "session['role']", + "path": [ + { + "file_path": "app/auth/permissions.py", + "line": 35, + "component": "step_15_a", + "operation": "read" + }, + { + "file_path": "app/auth/rbac.py", + "line": 55, + "component": "step_15_b", + "operation": "write" + } + ], + "sink": "permission_check", + "sanitized": true, + "files": [ + "app/auth/permissions.py", + "app/auth/rbac.py" + ] + }, + { + "source": "request.files['avatar']", + "path": [ + { + "file_path": "app/media/uploads.py", + "line": 36, + "component": "step_16_a", + "operation": "read" + }, + { + "file_path": "app/media/uploads.py", + "line": 56, + "component": "step_16_b", + "operation": "write" + } + ], + "sink": "s3.put_object", + "sanitized": true, + "files": [ + "app/media/uploads.py" + ] + }, + { + "source": "token", + "path": [ + { + "file_path": "app/common/cache.py", + "line": 37, + "component": "step_17_a", + "operation": "read" + }, + { + "file_path": "app/common/cache.py", + "line": 57, + "component": "step_17_b", + "operation": "write" + } + ], + "sink": "cache.set", + "sanitized": true, + "files": [ + "app/common/cache.py" + ] + }, + { + "source": "request.headers['X-User-Phone']", + "path": [ + { + "file_path": "app/obs/audit.py", + "line": 38, + "component": "step_18_a", + "operation": "read" + }, + { + "file_path": "app/obs/audit.py", + "line": 58, + "component": "step_18_b", + "operation": "write" + } + ], + "sink": "audit_log", + "sanitized": false, + "files": [ + "app/obs/audit.py" + ] + }, + { + "source": "graphql.variables", + "path": [ + { + "file_path": "app/api/graphql.ts", + "line": 39, + "component": "step_19_a", + "operation": "read" + }, + { + "file_path": "app/api/graphql.ts", + "line": 59, + "component": "step_19_b", + "operation": "write" + } + ], + "sink": "db.query", + "sanitized": false, + "files": [ + "app/api/graphql.ts" + ] + } + ], + "sanitization_points": [ + { + "file_path": "app/sanitize/s0.py", + "line": 5, + "function_name": null, + "sanitization_type": "escape", + "protects_against": [] + }, + { + "file_path": "app/sanitize/s1.py", + "line": 9, + "function_name": "clean_1", + "sanitization_type": "parameterize", + "protects_against": [ + "CWE-80", + "CWE-90" + ] + }, + { + "file_path": "app/sanitize/s2.py", + "line": 13, + "function_name": "clean_2", + "sanitization_type": "allowlist", + "protects_against": [ + "CWE-81", + "CWE-91" + ] + }, + { + "file_path": "app/sanitize/s3.py", + "line": 17, + "function_name": "clean_3", + "sanitization_type": "encode", + "protects_against": [] + }, + { + "file_path": "app/sanitize/s4.py", + "line": 21, + "function_name": "clean_4", + "sanitization_type": "escape", + "protects_against": [ + "CWE-83", + "CWE-93" + ] + }, + { + "file_path": "app/sanitize/s5.py", + "line": 25, + "function_name": null, + "sanitization_type": "parameterize", + "protects_against": [ + "CWE-84", + "CWE-94" + ] + }, + { + "file_path": "app/sanitize/s6.py", + "line": 29, + "function_name": "clean_6", + "sanitization_type": "allowlist", + "protects_against": [] + }, + { + "file_path": "app/sanitize/s7.py", + "line": 33, + "function_name": "clean_7", + "sanitization_type": "encode", + "protects_against": [ + "CWE-86", + "CWE-96" + ] + }, + { + "file_path": "app/sanitize/s8.py", + "line": 37, + "function_name": "clean_8", + "sanitization_type": "escape", + "protects_against": [ + "CWE-87", + "CWE-97" + ] + }, + { + "file_path": "app/sanitize/s9.py", + "line": 41, + "function_name": "clean_9", + "sanitization_type": "parameterize", + "protects_against": [] + }, + { + "file_path": "app/sanitize/s10.py", + "line": 45, + "function_name": null, + "sanitization_type": "allowlist", + "protects_against": [ + "CWE-89", + "CWE-99" + ] + }, + { + "file_path": "app/sanitize/s11.py", + "line": 49, + "function_name": "clean_11", + "sanitization_type": "encode", + "protects_against": [ + "CWE-90", + "CWE-100" + ] + } + ], + "sinks": [ + { + "sink_type": "sql", + "file_path": "app/sink/k0.py", + "line": 200, + "function_name": null, + "exploitability_notes": null + }, + { + "sink_type": "command", + "file_path": "app/sink/k1.py", + "line": 202, + "function_name": "sink_fn_1", + "exploitability_notes": "note 1" + }, + { + "sink_type": "template", + "file_path": "app/sink/k2.py", + "line": 204, + "function_name": "sink_fn_2", + "exploitability_notes": "note 2" + }, + { + "sink_type": "file", + "file_path": "app/sink/k3.py", + "line": 206, + "function_name": "sink_fn_3", + "exploitability_notes": null + }, + { + "sink_type": "network", + "file_path": "app/sink/k4.py", + "line": 208, + "function_name": null, + "exploitability_notes": "note 4" + }, + { + "sink_type": "deserialization", + "file_path": "app/sink/k5.py", + "line": 210, + "function_name": "sink_fn_5", + "exploitability_notes": "note 5" + }, + { + "sink_type": "sql", + "file_path": "app/sink/k6.py", + "line": 212, + "function_name": "sink_fn_6", + "exploitability_notes": null + }, + { + "sink_type": "command", + "file_path": "app/sink/k7.py", + "line": 214, + "function_name": "sink_fn_7", + "exploitability_notes": "note 7" + }, + { + "sink_type": "template", + "file_path": "app/sink/k8.py", + "line": 216, + "function_name": null, + "exploitability_notes": "note 8" + }, + { + "sink_type": "file", + "file_path": "app/sink/k9.py", + "line": 218, + "function_name": "sink_fn_9", + "exploitability_notes": null + }, + { + "sink_type": "network", + "file_path": "app/sink/k10.py", + "line": 220, + "function_name": "sink_fn_10", + "exploitability_notes": "note 10" + }, + { + "sink_type": "deserialization", + "file_path": "app/sink/k11.py", + "line": 222, + "function_name": "sink_fn_11", + "exploitability_notes": "note 11" + }, + { + "sink_type": "sql", + "file_path": "app/sink/k12.py", + "line": 224, + "function_name": null, + "exploitability_notes": null + }, + { + "sink_type": "command", + "file_path": "app/sink/k13.py", + "line": 226, + "function_name": "sink_fn_13", + "exploitability_notes": "note 13" + }, + { + "sink_type": "template", + "file_path": "app/sink/k14.py", + "line": 228, + "function_name": "sink_fn_14", + "exploitability_notes": "note 14" + }, + { + "sink_type": "file", + "file_path": "app/sink/k15.py", + "line": 230, + "function_name": "sink_fn_15", + "exploitability_notes": null + }, + { + "sink_type": "network", + "file_path": "app/sink/k16.py", + "line": 232, + "function_name": null, + "exploitability_notes": "note 16" + }, + { + "sink_type": "deserialization", + "file_path": "app/sink/k17.py", + "line": 234, + "function_name": "sink_fn_17", + "exploitability_notes": "note 17" + } + ] + }, + "dependencies": { + "sbom": [ + { + "name": "django", + "version": "4.2.1", + "ecosystem": "pypi", + "direct": true, + "license": "BSD-3-Clause" + }, + { + "name": "requests", + "version": "2.31.0", + "ecosystem": "pypi", + "direct": true, + "license": "Apache-2.0" + }, + { + "name": "urllib3", + "version": "1.26.5", + "ecosystem": "pypi", + "direct": false, + "license": "MIT" + }, + { + "name": "jinja2", + "version": "3.1.2", + "ecosystem": "pypi", + "direct": false, + "license": null + }, + { + "name": "lodash", + "version": "4.17.19", + "ecosystem": "npm", + "direct": true, + "license": "MIT" + }, + { + "name": "express", + "version": "4.17.1", + "ecosystem": "npm", + "direct": true, + "license": "MIT" + }, + { + "name": "minimist", + "version": "1.2.5", + "ecosystem": "npm", + "direct": false, + "license": "MIT" + }, + { + "name": "axios", + "version": "0.21.1", + "ecosystem": "npm", + "direct": false, + "license": "MIT" + }, + { + "name": "github.com/gin-gonic/gin", + "version": "1.9.0", + "ecosystem": "go", + "direct": true, + "license": "MIT" + }, + { + "name": "golang.org/x/net", + "version": "0.7.0", + "ecosystem": "go", + "direct": false, + "license": "BSD-3-Clause" + }, + { + "name": "rails", + "version": "7.0.4", + "ecosystem": "rubygems", + "direct": true, + "license": "MIT" + }, + { + "name": "nokogiri", + "version": "1.13.6", + "ecosystem": "rubygems", + "direct": false, + "license": "MIT" + }, + { + "name": "django", + "version": "4.2.1", + "ecosystem": "pypi", + "direct": true, + "license": "BSD-3-Clause" + }, + { + "name": "requests", + "version": "2.31.0", + "ecosystem": "pypi", + "direct": true, + "license": "Apache-2.0" + } + ], + "known_cves": [ + { + "cve_id": "CVE-2023-0001", + "package": "django", + "installed_version": "4.2.1", + "fixed_version": "4.2.5", + "cvss_v4_score": 9.8, + "epss_score": 0.5, + "direct": true, + "reachable": true + }, + { + "cve_id": "CVE-2023-0002", + "package": "urllib3", + "installed_version": "1.26.5", + "fixed_version": null, + "cvss_v4_score": 7.5, + "epss_score": 0.1, + "direct": false, + "reachable": false + }, + { + "cve_id": "CVE-2023-0003", + "package": "lodash", + "installed_version": "4.17.19", + "fixed_version": "4.17.21", + "cvss_v4_score": 9.8, + "epss_score": 0.9, + "direct": true, + "reachable": true + }, + { + "cve_id": "CVE-2023-0004", + "package": "minimist", + "installed_version": "1.2.5", + "fixed_version": "1.2.6", + "cvss_v4_score": null, + "epss_score": null, + "direct": false, + "reachable": null + }, + { + "cve_id": "CVE-2023-0005", + "package": "axios", + "installed_version": "0.21.1", + "fixed_version": "0.21.2", + "cvss_v4_score": 5.3, + "epss_score": 0.02, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0006", + "package": "jinja2", + "installed_version": "3.1.2", + "fixed_version": null, + "cvss_v4_score": 6.1, + "epss_score": null, + "direct": false, + "reachable": false + }, + { + "cve_id": "CVE-2023-0007", + "package": "express", + "installed_version": "4.17.1", + "fixed_version": "4.18.0", + "cvss_v4_score": 4.3, + "epss_score": 0.005, + "direct": true, + "reachable": null + }, + { + "cve_id": "CVE-2023-0008", + "package": "nokogiri", + "installed_version": "1.13.6", + "fixed_version": "1.13.10", + "cvss_v4_score": 8.8, + "epss_score": 0.3, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0009", + "package": "golang.org/x/net", + "installed_version": "0.7.0", + "fixed_version": "0.17.0", + "cvss_v4_score": 7.5, + "epss_score": 0.44, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0010", + "package": "rails", + "installed_version": "7.0.4", + "fixed_version": "7.0.7", + "cvss_v4_score": 9.1, + "epss_score": 0.6, + "direct": true, + "reachable": false + }, + { + "cve_id": "CVE-2023-0011", + "package": "gin", + "installed_version": "1.9.0", + "fixed_version": null, + "cvss_v4_score": 3.7, + "epss_score": 0.001, + "direct": true, + "reachable": null + }, + { + "cve_id": "CVE-2023-0012", + "package": "requests", + "installed_version": "2.31.0", + "fixed_version": "2.32.0", + "cvss_v4_score": 6.5, + "epss_score": 0.07, + "direct": true, + "reachable": true + }, + { + "cve_id": "CVE-2023-0013", + "package": "pyyaml", + "installed_version": "5.3", + "fixed_version": "5.4", + "cvss_v4_score": 9.8, + "epss_score": 0.5, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0014", + "package": "log4j", + "installed_version": "2.14.0", + "fixed_version": "2.17.1", + "cvss_v4_score": 10.0, + "epss_score": 0.97, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0015", + "package": "openssl", + "installed_version": "1.1.1k", + "fixed_version": "1.1.1t", + "cvss_v4_score": 7.4, + "epss_score": 0.12, + "direct": false, + "reachable": null + }, + { + "cve_id": "CVE-2023-0016", + "package": "zlib", + "installed_version": "1.2.11", + "fixed_version": "1.2.12", + "cvss_v4_score": 8.2, + "epss_score": 0.04, + "direct": false, + "reachable": false + }, + { + "cve_id": "CVE-2023-0017", + "package": "libxml2", + "installed_version": "2.9.10", + "fixed_version": null, + "cvss_v4_score": null, + "epss_score": 0.25, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0018", + "package": "curl", + "installed_version": "7.68.0", + "fixed_version": "7.88.0", + "cvss_v4_score": 8.1, + "epss_score": null, + "direct": true, + "reachable": true + } + ], + "outdated": [ + { + "package": "pkg-0", + "current_version": "1.0.0", + "latest_version": "2.0.0", + "direct": true + }, + { + "package": "pkg-1", + "current_version": "1.1.0", + "latest_version": "2.1.0", + "direct": false + }, + { + "package": "pkg-2", + "current_version": "1.2.0", + "latest_version": "2.2.0", + "direct": true + }, + { + "package": "pkg-3", + "current_version": "1.3.0", + "latest_version": "2.3.0", + "direct": false + }, + { + "package": "pkg-4", + "current_version": "1.4.0", + "latest_version": "2.4.0", + "direct": true + }, + { + "package": "pkg-5", + "current_version": "1.5.0", + "latest_version": "2.5.0", + "direct": false + }, + { + "package": "pkg-6", + "current_version": "1.6.0", + "latest_version": "2.6.0", + "direct": true + }, + { + "package": "pkg-7", + "current_version": "1.7.0", + "latest_version": "2.7.0", + "direct": false + }, + { + "package": "pkg-8", + "current_version": "1.8.0", + "latest_version": "2.8.0", + "direct": true + }, + { + "package": "pkg-9", + "current_version": "1.9.0", + "latest_version": "2.9.0", + "direct": false + }, + { + "package": "pkg-10", + "current_version": "1.10.0", + "latest_version": "2.10.0", + "direct": true + }, + { + "package": "pkg-11", + "current_version": "1.11.0", + "latest_version": "2.11.0", + "direct": false + } + ], + "direct_count": 7, + "transitive_count": 143 + }, + "file_count": 2417, + "frameworks": [ + "Django", + "next.js", + "Spring Boot", + " React ", + "unknown-fw", + "NEXT", + "Vue" + ], + "languages": [ + "Python", + "JavaScript", + "Go", + "Ruby", + "Rust", + "python" + ], + "lines_of_code": 184213, + "recon_duration_seconds": 42.5, + "security_context": { + "auth_model": "jwt", + "auth_details": "HS256 JWTs issued by app/auth/service.py, refresh tokens in Redis", + "crypto_usage": [ + { + "algorithm": "AES", + "key_size": 256, + "mode": "GCM", + "usage_context": "at-rest encryption", + "is_weak": false + }, + { + "algorithm": "MD5", + "key_size": null, + "mode": null, + "usage_context": "password hashing", + "is_weak": true + }, + { + "algorithm": "RSA", + "key_size": 1024, + "mode": null, + "usage_context": "token signing", + "is_weak": true + }, + { + "algorithm": "SHA-256", + "key_size": null, + "mode": null, + "usage_context": null, + "is_weak": false + }, + { + "algorithm": "DES", + "key_size": 56, + "mode": "CBC", + "usage_context": "legacy export", + "is_weak": true + }, + { + "algorithm": "HMAC-SHA1", + "key_size": 160, + "mode": null, + "usage_context": "webhook signature", + "is_weak": true + }, + { + "algorithm": "ChaCha20", + "key_size": 256, + "mode": "Poly1305", + "usage_context": "transport", + "is_weak": false + }, + { + "algorithm": "bcrypt", + "key_size": null, + "mode": null, + "usage_context": "password hashing", + "is_weak": false + }, + { + "algorithm": "RC4", + "key_size": 128, + "mode": null, + "usage_context": null, + "is_weak": true + }, + { + "algorithm": "ECDSA", + "key_size": 256, + "mode": null, + "usage_context": "JWT signing", + "is_weak": null + }, + { + "algorithm": "PBKDF2", + "key_size": null, + "mode": null, + "usage_context": "key derivation", + "is_weak": false + }, + { + "algorithm": "SHA-1", + "key_size": null, + "mode": null, + "usage_context": "checksum", + "is_weak": true + }, + { + "algorithm": "AES", + "key_size": 128, + "mode": "ECB", + "usage_context": "legacy blob", + "is_weak": true + }, + { + "algorithm": "Ed25519", + "key_size": 256, + "mode": null, + "usage_context": "package signing", + "is_weak": false + }, + { + "algorithm": "3DES", + "key_size": 168, + "mode": "CBC", + "usage_context": "legacy tape", + "is_weak": true + }, + { + "algorithm": "Argon2id", + "key_size": null, + "mode": null, + "usage_context": "password hashing", + "is_weak": null + } + ], + "framework_security": [ + "django.middleware.csrf.CsrfViewMiddleware", + "", + "helmet defaults", + "SecurityMiddleware", + "rack-protection", + "spring-security filter chain" + ], + "security_headers": [ + "Content-Security-Policy: default-src 'self'", + "X-Content-Type-Options: nosniff", + "Strict-Transport-Security: max-age=31536000", + "X-Frame-Options: DENY", + "Referrer-Policy: no-referrer", + "", + "Permissions-Policy: geolocation=()" + ], + "deployment_signals": [ + "kubernetes ingress with TLS termination", + "docker-compose exposes 5432", + "no WAF in front of /api", + "secrets mounted from vault", + "readiness probe on /healthz", + "single replica for the worker", + "TLS 1.2 minimum", + "internal service mesh mTLS" + ] + } +} \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/golden/strategy_dispatch.json b/internal/secaf/recontext/testdata/golden/strategy_dispatch.json new file mode 100644 index 0000000000..14b329d45a --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/strategy_dispatch.json @@ -0,0 +1,15 @@ +{ + "api_security": "82771f025cbc1e0b1641ddf58f8dfef0ed953bb003adc8f374feb89b39d487a3", + "auth": "b60513ba49bcbe930024c756c324c146c4db1c8286610906a84d94a37549fc53", + "business_logic": "5233f410243248527401314d2421f5ea1c882243342797767d9fe66656d058bf", + "config_secrets": "c8f201b7dad4c7f3f5b5839ed74ea7346dd21a36e49f9b03d8a1c40a37892670", + "crypto": "c7f937f4cebe00d387c8d603052795e13c79b0a5e7d2724ba7c3bb135b1e72d9", + "data_exposure": "6401321c6a9b97bcb9d7039f448944a4888d4ed2e255cb674fb093d2327bd7b3", + "dos": "aab1ed8b0c727bc8cbafdd01a4356359893fa0ec252a33f8e3a6f82aa99f1593", + "injection": "dbc7d5b08698797bdaa3c460840b389ef5e91d6e937ce466396e22de25d894d8", + "javascript_specific": "aab1ed8b0c727bc8cbafdd01a4356359893fa0ec252a33f8e3a6f82aa99f1593", + "python_specific": "aab1ed8b0c727bc8cbafdd01a4356359893fa0ec252a33f8e3a6f82aa99f1593", + "ssrf": "aab1ed8b0c727bc8cbafdd01a4356359893fa0ec252a33f8e3a6f82aa99f1593", + "supply_chain": "13dde4da65c9c8ef0a41a25f7d8e3d2ce70f7a0ee207392304f6bdf8cbde0011", + "xss": "aab1ed8b0c727bc8cbafdd01a4356359893fa0ec252a33f8e3a6f82aa99f1593" +} diff --git a/internal/secaf/recontext/testdata/golden/supply_chain.txt b/internal/secaf/recontext/testdata/golden/supply_chain.txt new file mode 100644 index 0000000000..f297901d2a --- /dev/null +++ b/internal/secaf/recontext/testdata/golden/supply_chain.txt @@ -0,0 +1,44 @@ +Supply-chain-focused recon summary. + +Dependency inventory: direct=7, transitive=143, SBOM entries=14. + +Known CVE exposure (reachable/high severity first): 18 total, showing top 15: +- CVE-2023-0014 in log4j 2.14.0 (fixed=2.17.1, cvss=10.0, epss=0.97, direct=False, reachable=True) +- CVE-2023-0003 in lodash 4.17.19 (fixed=4.17.21, cvss=9.8, epss=0.9, direct=True, reachable=True) +- CVE-2023-0001 in django 4.2.1 (fixed=4.2.5, cvss=9.8, epss=0.5, direct=True, reachable=True) +- CVE-2023-0013 in pyyaml 5.3 (fixed=5.4, cvss=9.8, epss=0.5, direct=False, reachable=True) +- CVE-2023-0008 in nokogiri 1.13.6 (fixed=1.13.10, cvss=8.8, epss=0.3, direct=False, reachable=True) +- CVE-2023-0018 in curl 7.68.0 (fixed=7.88.0, cvss=8.1, epss=None, direct=True, reachable=True) +- CVE-2023-0009 in golang.org/x/net 0.7.0 (fixed=0.17.0, cvss=7.5, epss=0.44, direct=False, reachable=True) +- CVE-2023-0012 in requests 2.31.0 (fixed=2.32.0, cvss=6.5, epss=0.07, direct=True, reachable=True) +- CVE-2023-0005 in axios 0.21.1 (fixed=0.21.2, cvss=5.3, epss=0.02, direct=False, reachable=True) +- CVE-2023-0017 in libxml2 2.9.10 (fixed=unknown, cvss=None, epss=0.25, direct=False, reachable=True) +- CVE-2023-0010 in rails 7.0.4 (fixed=7.0.7, cvss=9.1, epss=0.6, direct=True, reachable=False) +- CVE-2023-0016 in zlib 1.2.11 (fixed=1.2.12, cvss=8.2, epss=0.04, direct=False, reachable=False) +- CVE-2023-0002 in urllib3 1.26.5 (fixed=unknown, cvss=7.5, epss=0.1, direct=False, reachable=False) +- CVE-2023-0015 in openssl 1.1.1k (fixed=1.1.1t, cvss=7.4, epss=0.12, direct=False, reachable=None) +- CVE-2023-0006 in jinja2 3.1.2 (fixed=unknown, cvss=6.1, epss=None, direct=False, reachable=False) + +Outdated dependencies: 12 total, showing top 10: +- pkg-0: 1.0.0 -> 2.0.0 (direct=True) +- pkg-1: 1.1.0 -> 2.1.0 (direct=False) +- pkg-2: 1.2.0 -> 2.2.0 (direct=True) +- pkg-3: 1.3.0 -> 2.3.0 (direct=False) +- pkg-4: 1.4.0 -> 2.4.0 (direct=True) +- pkg-5: 1.5.0 -> 2.5.0 (direct=False) +- pkg-6: 1.6.0 -> 2.6.0 (direct=True) +- pkg-7: 1.7.0 -> 2.7.0 (direct=False) +- pkg-8: 1.8.0 -> 2.8.0 (direct=True) +- pkg-9: 1.9.0 -> 2.9.0 (direct=False) + +Primary dependency ecosystems in this repo: 12 total, showing top 10: +- go: github.com/gin-gonic/gin@1.9.0 +- go: golang.org/x/net@0.7.0 +- npm: axios@0.21.1 +- npm: express@4.17.1 +- npm: lodash@4.17.19 +- npm: minimist@1.2.5 +- pypi: django@4.2.1 +- pypi: jinja2@3.1.2 +- pypi: requests@2.31.0 +- pypi: urllib3@1.26.5 \ No newline at end of file diff --git a/internal/secaf/recontext/testdata/recon_fixture.json b/internal/secaf/recontext/testdata/recon_fixture.json new file mode 100644 index 0000000000..bf916bb218 --- /dev/null +++ b/internal/secaf/recontext/testdata/recon_fixture.json @@ -0,0 +1,2189 @@ +{ + "architecture": { + "app_type": "web_api", + "modules": [ + { + "name": "auth_service", + "path": "app/auth/service.py", + "language": "python", + "description": "Session and JWT issuance", + "dependencies": [ + "jwt", + "redis" + ] + }, + { + "name": "rbac", + "path": "app/auth/rbac.py", + "language": "python", + "description": null, + "dependencies": [] + }, + { + "name": "billing", + "path": "app/billing/core.py", + "language": "python", + "description": "Invoice state machine — handles refunds", + "dependencies": [ + "stripe" + ] + }, + { + "name": "csrf_guard", + "path": "app/web/middleware/csrf.js", + "language": "javascript", + "description": "CSRF middleware", + "dependencies": [] + }, + { + "name": "payments", + "path": "app/billing/payments.go", + "language": "go", + "description": "Charge orchestration", + "dependencies": [ + "stripe-go" + ] + }, + { + "name": "reporting", + "path": "app/reports/render.py", + "language": "python", + "description": "PDF and CSV export", + "dependencies": [ + "weasyprint" + ] + }, + { + "name": "search", + "path": "app/search/index.ts", + "language": "typescript", + "description": "Elasticsearch query builder", + "dependencies": [ + "@elastic/elasticsearch" + ] + }, + { + "name": "session_store", + "path": "app/auth/session_store.py", + "language": "python", + "description": "Redis-backed sessions", + "dependencies": [ + "redis" + ] + }, + { + "name": "notifications", + "path": "app/notify/mailer.rb", + "language": "ruby", + "description": null, + "dependencies": [ + "mail" + ] + }, + { + "name": "permission_matrix", + "path": "app/auth/permissions.py", + "language": "python", + "description": "Role → permission table", + "dependencies": [] + }, + { + "name": "uploads", + "path": "app/media/uploads.py", + "language": "python", + "description": "S3 multipart upload helper", + "dependencies": [ + "boto3" + ] + }, + { + "name": "webhooks", + "path": "app/integrations/webhooks.go", + "language": "go", + "description": "Outbound webhook dispatcher", + "dependencies": [] + }, + { + "name": "graph_api", + "path": "app/api/graphql.ts", + "language": "typescript", + "description": "GraphQL resolvers — naïve depth limit", + "dependencies": [ + "graphql" + ] + }, + { + "name": "admin_panel", + "path": "app/admin/views.py", + "language": "python", + "description": "Django admin overrides", + "dependencies": [ + "django" + ] + }, + { + "name": "guard_utils", + "path": "app/common/guard.py", + "language": "python", + "description": "Assorted guard helpers", + "dependencies": [] + }, + { + "name": "cache", + "path": "app/common/cache.py", + "language": "python", + "description": "Memoization wrappers", + "dependencies": [] + }, + { + "name": "migrations", + "path": "db/migrate/2024_add_roles.rb", + "language": "ruby", + "description": "Adds role column", + "dependencies": [] + }, + { + "name": "jwt_tools", + "path": "app/common/jwt_tools.py", + "language": "python", + "description": null, + "dependencies": [ + "pyjwt" + ] + }, + { + "name": "telemetry", + "path": "app/obs/telemetry.go", + "language": "go", + "description": "OTel exporter", + "dependencies": [] + } + ], + "entry_points": [ + { + "kind": "http", + "identifier": "handler_0", + "file_path": "app/entry/e0.py", + "line": 10, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "HTTP", + "identifier": "handler_1", + "file_path": "app/entry/e1.py", + "line": 17, + "method": "POST", + "route": "/v1/resource/1", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_2", + "file_path": "app/entry/e2.py", + "line": 24, + "method": null, + "route": "/v1/resource/2", + "auth_required": null + }, + { + "kind": "api", + "identifier": "handler_3", + "file_path": "app/entry/e3.py", + "line": 31, + "method": "PUT", + "route": null, + "auth_required": true + }, + { + "kind": "graphql", + "identifier": "handler_4", + "file_path": "app/entry/e4.py", + "line": 38, + "method": "GET", + "route": "/v1/resource/4", + "auth_required": false + }, + { + "kind": "rpc", + "identifier": "handler_5", + "file_path": "app/entry/e5.py", + "line": 45, + "method": "POST", + "route": "/v1/resource/5", + "auth_required": null + }, + { + "kind": "route", + "identifier": "handler_6", + "file_path": "app/entry/e6.py", + "line": 52, + "method": null, + "route": null, + "auth_required": true + }, + { + "kind": "cron", + "identifier": "handler_7", + "file_path": "app/entry/e7.py", + "line": 59, + "method": "PUT", + "route": "/v1/resource/7", + "auth_required": false + }, + { + "kind": "queue", + "identifier": "handler_8", + "file_path": "app/entry/e8.py", + "line": 66, + "method": "GET", + "route": "/v1/resource/8", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_9", + "file_path": "app/entry/e9.py", + "line": 73, + "method": "POST", + "route": null, + "auth_required": true + }, + { + "kind": "api", + "identifier": "handler_10", + "file_path": "app/entry/e10.py", + "line": 80, + "method": null, + "route": "/v1/resource/10", + "auth_required": false + }, + { + "kind": "grpc", + "identifier": "handler_11", + "file_path": "app/entry/e11.py", + "line": 87, + "method": "PUT", + "route": "/v1/resource/11", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_12", + "file_path": "app/entry/e12.py", + "line": 94, + "method": "GET", + "route": null, + "auth_required": true + }, + { + "kind": "route", + "identifier": "handler_13", + "file_path": "app/entry/e13.py", + "line": 101, + "method": "POST", + "route": "/v1/resource/13", + "auth_required": false + }, + { + "kind": "cli", + "identifier": "handler_14", + "file_path": "app/entry/e14.py", + "line": 108, + "method": null, + "route": "/v1/resource/14", + "auth_required": null + }, + { + "kind": "http", + "identifier": "handler_15", + "file_path": "app/entry/e15.py", + "line": 115, + "method": "PUT", + "route": null, + "auth_required": true + }, + { + "kind": "websocket", + "identifier": "handler_16", + "file_path": "app/entry/e16.py", + "line": 122, + "method": "GET", + "route": "/v1/resource/16", + "auth_required": false + } + ], + "trust_boundaries": [ + { + "name": "boundary_0", + "source_zone": "internet", + "target_zone": "app", + "description": "Transition 0", + "enforcement": [] + }, + { + "name": "boundary_1", + "source_zone": "dmz", + "target_zone": "db", + "description": "Transition 1", + "enforcement": [ + "waf_1", + "mtls_1" + ] + }, + { + "name": "boundary_2", + "source_zone": "vpc", + "target_zone": "cache", + "description": "Transition 2", + "enforcement": [ + "waf_2", + "mtls_2" + ] + }, + { + "name": "boundary_3", + "source_zone": "worker", + "target_zone": "queue", + "description": "Transition 3", + "enforcement": [ + "waf_3", + "mtls_3" + ] + }, + { + "name": "boundary_4", + "source_zone": "internet", + "target_zone": "app", + "description": "Transition 4", + "enforcement": [] + }, + { + "name": "boundary_5", + "source_zone": "dmz", + "target_zone": "db", + "description": "Transition 5", + "enforcement": [ + "waf_5", + "mtls_5" + ] + }, + { + "name": "boundary_6", + "source_zone": "vpc", + "target_zone": "cache", + "description": "Transition 6", + "enforcement": [ + "waf_6", + "mtls_6" + ] + }, + { + "name": "boundary_7", + "source_zone": "worker", + "target_zone": "queue", + "description": "Transition 7", + "enforcement": [ + "waf_7", + "mtls_7" + ] + }, + { + "name": "boundary_8", + "source_zone": "internet", + "target_zone": "app", + "description": "Transition 8", + "enforcement": [] + }, + { + "name": "boundary_9", + "source_zone": "dmz", + "target_zone": "db", + "description": "Transition 9", + "enforcement": [ + "waf_9", + "mtls_9" + ] + }, + { + "name": "boundary_10", + "source_zone": "vpc", + "target_zone": "cache", + "description": "Transition 10", + "enforcement": [ + "waf_10", + "mtls_10" + ] + }, + { + "name": "boundary_11", + "source_zone": "worker", + "target_zone": "queue", + "description": "Transition 11", + "enforcement": [ + "waf_11", + "mtls_11" + ] + } + ], + "services": [ + { + "name": "postgres", + "service_type": "database", + "endpoint": "postgres://db:5432", + "purpose": "primary store", + "auth_mechanism": "password" + }, + { + "name": "redis", + "service_type": "cache", + "endpoint": null, + "purpose": "sessions", + "auth_mechanism": null + }, + { + "name": "stripe", + "service_type": "payments", + "endpoint": "https://api.stripe.com", + "purpose": null, + "auth_mechanism": "api_key" + }, + { + "name": "s3", + "service_type": "object_store", + "endpoint": "https://s3.amazonaws.com", + "purpose": "uploads", + "auth_mechanism": "iam" + }, + { + "name": "smtp", + "service_type": "mail", + "endpoint": null, + "purpose": "transactional email", + "auth_mechanism": null + } + ], + "api_surface": [ + { + "method": "GET", + "path": "/api/v1/thing/0", + "handler": "ThingController.action0", + "file_path": "app/api/thing_0.py", + "line": 100, + "auth_required": true, + "rate_limited": null + }, + { + "method": "POST", + "path": "/api/v1/thing/1", + "handler": "ThingController.action1", + "file_path": "app/api/thing_1.py", + "line": 103, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PUT", + "path": "/api/v1/thing/2", + "handler": "ThingController.action2", + "file_path": "app/api/thing_2.py", + "line": 106, + "auth_required": null, + "rate_limited": true + }, + { + "method": "DELETE", + "path": "/api/v1/thing/3", + "handler": "ThingController.action3", + "file_path": "app/api/thing_3.py", + "line": 109, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PATCH", + "path": "/api/v1/thing/4", + "handler": "ThingController.action4", + "file_path": "app/api/thing_4.py", + "line": 112, + "auth_required": true, + "rate_limited": null + }, + { + "method": "GET", + "path": "/api/v1/thing/5", + "handler": "ThingController.action5", + "file_path": "app/api/thing_5.py", + "line": 115, + "auth_required": null, + "rate_limited": true + }, + { + "method": "POST", + "path": "/api/v1/thing/6", + "handler": "ThingController.action6", + "file_path": "app/api/thing_6.py", + "line": 118, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PUT", + "path": "/api/v1/thing/7", + "handler": "ThingController.action7", + "file_path": "app/api/thing_7.py", + "line": 121, + "auth_required": false, + "rate_limited": false + }, + { + "method": "DELETE", + "path": "/api/v1/thing/8", + "handler": "ThingController.action8", + "file_path": "app/api/thing_8.py", + "line": 124, + "auth_required": null, + "rate_limited": true + }, + { + "method": "PATCH", + "path": "/api/v1/thing/9", + "handler": "ThingController.action9", + "file_path": "app/api/thing_9.py", + "line": 127, + "auth_required": false, + "rate_limited": false + }, + { + "method": "GET", + "path": "/api/v1/thing/10", + "handler": "ThingController.action10", + "file_path": "app/api/thing_10.py", + "line": 130, + "auth_required": true, + "rate_limited": null + }, + { + "method": "POST", + "path": "/api/v1/thing/11", + "handler": "ThingController.action11", + "file_path": "app/api/thing_11.py", + "line": 133, + "auth_required": null, + "rate_limited": true + }, + { + "method": "PUT", + "path": "/api/v1/thing/12", + "handler": "ThingController.action12", + "file_path": "app/api/thing_12.py", + "line": 136, + "auth_required": true, + "rate_limited": null + }, + { + "method": "DELETE", + "path": "/api/v1/thing/13", + "handler": "ThingController.action13", + "file_path": "app/api/thing_13.py", + "line": 139, + "auth_required": false, + "rate_limited": false + }, + { + "method": "PATCH", + "path": "/api/v1/thing/14", + "handler": "ThingController.action14", + "file_path": "app/api/thing_14.py", + "line": 142, + "auth_required": null, + "rate_limited": true + }, + { + "method": "GET", + "path": "/api/v1/thing/15", + "handler": "ThingController.action15", + "file_path": "app/api/thing_15.py", + "line": 145, + "auth_required": false, + "rate_limited": false + }, + { + "method": "POST", + "path": "/api/v1/thing/16", + "handler": "ThingController.action16", + "file_path": "app/api/thing_16.py", + "line": 148, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PUT", + "path": "/api/v1/thing/17", + "handler": "ThingController.action17", + "file_path": "app/api/thing_17.py", + "line": 151, + "auth_required": null, + "rate_limited": true + }, + { + "method": "DELETE", + "path": "/api/v1/thing/18", + "handler": "ThingController.action18", + "file_path": "app/api/thing_18.py", + "line": 154, + "auth_required": true, + "rate_limited": null + }, + { + "method": "PATCH", + "path": "/api/v1/thing/19", + "handler": "ThingController.action19", + "file_path": "app/api/thing_19.py", + "line": 157, + "auth_required": false, + "rate_limited": false + } + ] + }, + "data_flows": { + "flows": [ + { + "source": "request.args['q']", + "path": [ + { + "file_path": "app/search/index.ts", + "line": 20, + "component": "step_0_a", + "operation": "read" + }, + { + "file_path": "app/util/b.py", + "line": 40, + "component": "step_0_b", + "operation": "write" + } + ], + "sink": "cursor.execute", + "sanitized": false, + "files": [ + "app/search/index.ts", + "app/db/raw.py", + "app/util/a.py", + "app/util/b.py" + ] + }, + { + "source": "request.cookies['session']", + "path": [ + { + "file_path": "app/auth/session_store.py", + "line": 21, + "component": "step_1_a", + "operation": "read" + }, + { + "file_path": "app/auth/session_store.py", + "line": 41, + "component": "step_1_b", + "operation": "write" + } + ], + "sink": "redis.set", + "sanitized": true, + "files": [ + "app/auth/session_store.py" + ] + }, + { + "source": "request.json['url']", + "path": [ + { + "file_path": "app/integrations/webhooks.go", + "line": 22, + "component": "step_2_a", + "operation": "read" + }, + { + "file_path": "app/net/client.go", + "line": 42, + "component": "step_2_b", + "operation": "write" + } + ], + "sink": "requests.get", + "sanitized": false, + "files": [ + "app/integrations/webhooks.go", + "app/net/client.go" + ] + }, + { + "source": "form['password']", + "path": [ + { + "file_path": "app/auth/service.py", + "line": 23, + "component": "step_3_a", + "operation": "read" + }, + { + "file_path": "app/obs/telemetry.go", + "line": 43, + "component": "step_3_b", + "operation": "write" + } + ], + "sink": "logger.info", + "sanitized": false, + "files": [ + "app/auth/service.py", + "app/obs/telemetry.go" + ] + }, + { + "source": "header['Authorization']", + "path": [ + { + "file_path": "app/common/jwt_tools.py", + "line": 24, + "component": "step_4_a", + "operation": "read" + }, + { + "file_path": "app/common/jwt_tools.py", + "line": 44, + "component": "step_4_b", + "operation": "write" + } + ], + "sink": "jwt.decode", + "sanitized": true, + "files": [ + "app/common/jwt_tools.py" + ] + }, + { + "source": "query['email']", + "path": [ + { + "file_path": "app/notify/mailer.rb", + "line": 25, + "component": "step_5_a", + "operation": "read" + }, + { + "file_path": "app/notify/mailer.rb", + "line": 45, + "component": "step_5_b", + "operation": "write" + } + ], + "sink": "smtp.send", + "sanitized": false, + "files": [ + "app/notify/mailer.rb" + ] + }, + { + "source": "body['card_number']", + "path": [ + { + "file_path": "app/billing/payments.go", + "line": 26, + "component": "step_6_a", + "operation": "read" + }, + { + "file_path": "app/billing/core.py", + "line": 46, + "component": "step_6_b", + "operation": "write" + } + ], + "sink": "stripe.Charge.create", + "sanitized": true, + "files": [ + "app/billing/payments.go", + "app/billing/core.py" + ] + }, + { + "source": "path_param['id']", + "path": [ + { + "file_path": "app/api/thing_1.py", + "line": 27, + "component": "step_7_a", + "operation": "read" + }, + { + "file_path": "app/api/thing_1.py", + "line": 47, + "component": "step_7_b", + "operation": "write" + } + ], + "sink": "orm.filter", + "sanitized": true, + "files": [ + "app/api/thing_1.py" + ] + }, + { + "source": "upload.filename", + "path": [ + { + "file_path": "app/media/uploads.py", + "line": 28, + "component": "step_8_a", + "operation": "read" + }, + { + "file_path": "app/x/w.py", + "line": 48, + "component": "step_8_b", + "operation": "write" + } + ], + "sink": "open", + "sanitized": false, + "files": [ + "app/media/uploads.py", + "app/common/cache.py", + "app/common/guard.py", + "app/x/y.py", + "app/x/z.py", + "app/x/w.py" + ] + }, + { + "source": "env['DEBUG']", + "path": [ + { + "file_path": "app/web/render.py", + "line": 29, + "component": "step_9_a", + "operation": "read" + }, + { + "file_path": "app/web/render.py", + "line": 49, + "component": "step_9_b", + "operation": "write" + } + ], + "sink": "template.render", + "sanitized": false, + "files": [ + "app/web/render.py" + ] + }, + { + "source": "request.body", + "path": [ + { + "file_path": "app/admin/views.py", + "line": 30, + "component": "step_10_a", + "operation": "read" + }, + { + "file_path": "app/admin/views.py", + "line": 50, + "component": "step_10_b", + "operation": "write" + } + ], + "sink": "eval", + "sanitized": false, + "files": [ + "app/admin/views.py" + ] + }, + { + "source": "websocket.msg", + "path": [ + { + "file_path": "app/ws/handler.py", + "line": 31, + "component": "step_11_a", + "operation": "read" + }, + { + "file_path": "app/ws/exec.py", + "line": 51, + "component": "step_11_b", + "operation": "write" + } + ], + "sink": "subprocess.run", + "sanitized": false, + "files": [ + "app/ws/handler.py", + "app/ws/exec.py" + ] + }, + { + "source": "cli.argv", + "path": [ + { + "file_path": "tools/run.py", + "line": 32, + "component": "step_12_a", + "operation": "read" + }, + { + "file_path": "tools/run.py", + "line": 52, + "component": "step_12_b", + "operation": "write" + } + ], + "sink": "os.system", + "sanitized": false, + "files": [ + "tools/run.py" + ] + }, + { + "source": "queue.payload", + "path": [ + { + "file_path": "app/worker/task.py", + "line": 33, + "component": "step_13_a", + "operation": "read" + }, + { + "file_path": "app/worker/task.py", + "line": 53, + "component": "step_13_b", + "operation": "write" + } + ], + "sink": "pickle.loads", + "sanitized": false, + "files": [ + "app/worker/task.py" + ] + }, + { + "source": "request.args['redirect']", + "path": [ + { + "file_path": "app/web/redirect.py", + "line": 34, + "component": "step_14_a", + "operation": "read" + }, + { + "file_path": "app/web/redirect.py", + "line": 54, + "component": "step_14_b", + "operation": "write" + } + ], + "sink": "HttpResponseRedirect", + "sanitized": false, + "files": [ + "app/web/redirect.py" + ] + }, + { + "source": "session['role']", + "path": [ + { + "file_path": "app/auth/permissions.py", + "line": 35, + "component": "step_15_a", + "operation": "read" + }, + { + "file_path": "app/auth/rbac.py", + "line": 55, + "component": "step_15_b", + "operation": "write" + } + ], + "sink": "permission_check", + "sanitized": true, + "files": [ + "app/auth/permissions.py", + "app/auth/rbac.py" + ] + }, + { + "source": "request.files['avatar']", + "path": [ + { + "file_path": "app/media/uploads.py", + "line": 36, + "component": "step_16_a", + "operation": "read" + }, + { + "file_path": "app/media/uploads.py", + "line": 56, + "component": "step_16_b", + "operation": "write" + } + ], + "sink": "s3.put_object", + "sanitized": true, + "files": [ + "app/media/uploads.py" + ] + }, + { + "source": "token", + "path": [ + { + "file_path": "app/common/cache.py", + "line": 37, + "component": "step_17_a", + "operation": "read" + }, + { + "file_path": "app/common/cache.py", + "line": 57, + "component": "step_17_b", + "operation": "write" + } + ], + "sink": "cache.set", + "sanitized": true, + "files": [ + "app/common/cache.py" + ] + }, + { + "source": "request.headers['X-User-Phone']", + "path": [ + { + "file_path": "app/obs/audit.py", + "line": 38, + "component": "step_18_a", + "operation": "read" + }, + { + "file_path": "app/obs/audit.py", + "line": 58, + "component": "step_18_b", + "operation": "write" + } + ], + "sink": "audit_log", + "sanitized": false, + "files": [ + "app/obs/audit.py" + ] + }, + { + "source": "graphql.variables", + "path": [ + { + "file_path": "app/api/graphql.ts", + "line": 39, + "component": "step_19_a", + "operation": "read" + }, + { + "file_path": "app/api/graphql.ts", + "line": 59, + "component": "step_19_b", + "operation": "write" + } + ], + "sink": "db.query", + "sanitized": false, + "files": [ + "app/api/graphql.ts" + ] + } + ], + "sanitization_points": [ + { + "file_path": "app/sanitize/s0.py", + "line": 5, + "function_name": null, + "sanitization_type": "escape", + "protects_against": [] + }, + { + "file_path": "app/sanitize/s1.py", + "line": 9, + "function_name": "clean_1", + "sanitization_type": "parameterize", + "protects_against": [ + "CWE-80", + "CWE-90" + ] + }, + { + "file_path": "app/sanitize/s2.py", + "line": 13, + "function_name": "clean_2", + "sanitization_type": "allowlist", + "protects_against": [ + "CWE-81", + "CWE-91" + ] + }, + { + "file_path": "app/sanitize/s3.py", + "line": 17, + "function_name": "clean_3", + "sanitization_type": "encode", + "protects_against": [] + }, + { + "file_path": "app/sanitize/s4.py", + "line": 21, + "function_name": "clean_4", + "sanitization_type": "escape", + "protects_against": [ + "CWE-83", + "CWE-93" + ] + }, + { + "file_path": "app/sanitize/s5.py", + "line": 25, + "function_name": null, + "sanitization_type": "parameterize", + "protects_against": [ + "CWE-84", + "CWE-94" + ] + }, + { + "file_path": "app/sanitize/s6.py", + "line": 29, + "function_name": "clean_6", + "sanitization_type": "allowlist", + "protects_against": [] + }, + { + "file_path": "app/sanitize/s7.py", + "line": 33, + "function_name": "clean_7", + "sanitization_type": "encode", + "protects_against": [ + "CWE-86", + "CWE-96" + ] + }, + { + "file_path": "app/sanitize/s8.py", + "line": 37, + "function_name": "clean_8", + "sanitization_type": "escape", + "protects_against": [ + "CWE-87", + "CWE-97" + ] + }, + { + "file_path": "app/sanitize/s9.py", + "line": 41, + "function_name": "clean_9", + "sanitization_type": "parameterize", + "protects_against": [] + }, + { + "file_path": "app/sanitize/s10.py", + "line": 45, + "function_name": null, + "sanitization_type": "allowlist", + "protects_against": [ + "CWE-89", + "CWE-99" + ] + }, + { + "file_path": "app/sanitize/s11.py", + "line": 49, + "function_name": "clean_11", + "sanitization_type": "encode", + "protects_against": [ + "CWE-90", + "CWE-100" + ] + } + ], + "sinks": [ + { + "sink_type": "sql", + "file_path": "app/sink/k0.py", + "line": 200, + "function_name": null, + "exploitability_notes": null + }, + { + "sink_type": "command", + "file_path": "app/sink/k1.py", + "line": 202, + "function_name": "sink_fn_1", + "exploitability_notes": "note 1" + }, + { + "sink_type": "template", + "file_path": "app/sink/k2.py", + "line": 204, + "function_name": "sink_fn_2", + "exploitability_notes": "note 2" + }, + { + "sink_type": "file", + "file_path": "app/sink/k3.py", + "line": 206, + "function_name": "sink_fn_3", + "exploitability_notes": null + }, + { + "sink_type": "network", + "file_path": "app/sink/k4.py", + "line": 208, + "function_name": null, + "exploitability_notes": "note 4" + }, + { + "sink_type": "deserialization", + "file_path": "app/sink/k5.py", + "line": 210, + "function_name": "sink_fn_5", + "exploitability_notes": "note 5" + }, + { + "sink_type": "sql", + "file_path": "app/sink/k6.py", + "line": 212, + "function_name": "sink_fn_6", + "exploitability_notes": null + }, + { + "sink_type": "command", + "file_path": "app/sink/k7.py", + "line": 214, + "function_name": "sink_fn_7", + "exploitability_notes": "note 7" + }, + { + "sink_type": "template", + "file_path": "app/sink/k8.py", + "line": 216, + "function_name": null, + "exploitability_notes": "note 8" + }, + { + "sink_type": "file", + "file_path": "app/sink/k9.py", + "line": 218, + "function_name": "sink_fn_9", + "exploitability_notes": null + }, + { + "sink_type": "network", + "file_path": "app/sink/k10.py", + "line": 220, + "function_name": "sink_fn_10", + "exploitability_notes": "note 10" + }, + { + "sink_type": "deserialization", + "file_path": "app/sink/k11.py", + "line": 222, + "function_name": "sink_fn_11", + "exploitability_notes": "note 11" + }, + { + "sink_type": "sql", + "file_path": "app/sink/k12.py", + "line": 224, + "function_name": null, + "exploitability_notes": null + }, + { + "sink_type": "command", + "file_path": "app/sink/k13.py", + "line": 226, + "function_name": "sink_fn_13", + "exploitability_notes": "note 13" + }, + { + "sink_type": "template", + "file_path": "app/sink/k14.py", + "line": 228, + "function_name": "sink_fn_14", + "exploitability_notes": "note 14" + }, + { + "sink_type": "file", + "file_path": "app/sink/k15.py", + "line": 230, + "function_name": "sink_fn_15", + "exploitability_notes": null + }, + { + "sink_type": "network", + "file_path": "app/sink/k16.py", + "line": 232, + "function_name": null, + "exploitability_notes": "note 16" + }, + { + "sink_type": "deserialization", + "file_path": "app/sink/k17.py", + "line": 234, + "function_name": "sink_fn_17", + "exploitability_notes": "note 17" + } + ] + }, + "dependencies": { + "sbom": [ + { + "name": "django", + "version": "4.2.1", + "ecosystem": "pypi", + "direct": true, + "license": "BSD-3-Clause" + }, + { + "name": "requests", + "version": "2.31.0", + "ecosystem": "pypi", + "direct": true, + "license": "Apache-2.0" + }, + { + "name": "urllib3", + "version": "1.26.5", + "ecosystem": "pypi", + "direct": false, + "license": "MIT" + }, + { + "name": "jinja2", + "version": "3.1.2", + "ecosystem": "pypi", + "direct": false, + "license": null + }, + { + "name": "lodash", + "version": "4.17.19", + "ecosystem": "npm", + "direct": true, + "license": "MIT" + }, + { + "name": "express", + "version": "4.17.1", + "ecosystem": "npm", + "direct": true, + "license": "MIT" + }, + { + "name": "minimist", + "version": "1.2.5", + "ecosystem": "npm", + "direct": false, + "license": "MIT" + }, + { + "name": "axios", + "version": "0.21.1", + "ecosystem": "npm", + "direct": false, + "license": "MIT" + }, + { + "name": "github.com/gin-gonic/gin", + "version": "1.9.0", + "ecosystem": "go", + "direct": true, + "license": "MIT" + }, + { + "name": "golang.org/x/net", + "version": "0.7.0", + "ecosystem": "go", + "direct": false, + "license": "BSD-3-Clause" + }, + { + "name": "rails", + "version": "7.0.4", + "ecosystem": "rubygems", + "direct": true, + "license": "MIT" + }, + { + "name": "nokogiri", + "version": "1.13.6", + "ecosystem": "rubygems", + "direct": false, + "license": "MIT" + }, + { + "name": "django", + "version": "4.2.1", + "ecosystem": "pypi", + "direct": true, + "license": "BSD-3-Clause" + }, + { + "name": "requests", + "version": "2.31.0", + "ecosystem": "pypi", + "direct": true, + "license": "Apache-2.0" + } + ], + "known_cves": [ + { + "cve_id": "CVE-2023-0001", + "package": "django", + "installed_version": "4.2.1", + "fixed_version": "4.2.5", + "cvss_v4_score": 9.8, + "epss_score": 0.5, + "direct": true, + "reachable": true + }, + { + "cve_id": "CVE-2023-0002", + "package": "urllib3", + "installed_version": "1.26.5", + "fixed_version": null, + "cvss_v4_score": 7.5, + "epss_score": 0.1, + "direct": false, + "reachable": false + }, + { + "cve_id": "CVE-2023-0003", + "package": "lodash", + "installed_version": "4.17.19", + "fixed_version": "4.17.21", + "cvss_v4_score": 9.8, + "epss_score": 0.9, + "direct": true, + "reachable": true + }, + { + "cve_id": "CVE-2023-0004", + "package": "minimist", + "installed_version": "1.2.5", + "fixed_version": "1.2.6", + "cvss_v4_score": null, + "epss_score": null, + "direct": false, + "reachable": null + }, + { + "cve_id": "CVE-2023-0005", + "package": "axios", + "installed_version": "0.21.1", + "fixed_version": "0.21.2", + "cvss_v4_score": 5.3, + "epss_score": 0.02, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0006", + "package": "jinja2", + "installed_version": "3.1.2", + "fixed_version": null, + "cvss_v4_score": 6.1, + "epss_score": null, + "direct": false, + "reachable": false + }, + { + "cve_id": "CVE-2023-0007", + "package": "express", + "installed_version": "4.17.1", + "fixed_version": "4.18.0", + "cvss_v4_score": 4.3, + "epss_score": 0.005, + "direct": true, + "reachable": null + }, + { + "cve_id": "CVE-2023-0008", + "package": "nokogiri", + "installed_version": "1.13.6", + "fixed_version": "1.13.10", + "cvss_v4_score": 8.8, + "epss_score": 0.3, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0009", + "package": "golang.org/x/net", + "installed_version": "0.7.0", + "fixed_version": "0.17.0", + "cvss_v4_score": 7.5, + "epss_score": 0.44, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0010", + "package": "rails", + "installed_version": "7.0.4", + "fixed_version": "7.0.7", + "cvss_v4_score": 9.1, + "epss_score": 0.6, + "direct": true, + "reachable": false + }, + { + "cve_id": "CVE-2023-0011", + "package": "gin", + "installed_version": "1.9.0", + "fixed_version": null, + "cvss_v4_score": 3.7, + "epss_score": 0.001, + "direct": true, + "reachable": null + }, + { + "cve_id": "CVE-2023-0012", + "package": "requests", + "installed_version": "2.31.0", + "fixed_version": "2.32.0", + "cvss_v4_score": 6.5, + "epss_score": 0.07, + "direct": true, + "reachable": true + }, + { + "cve_id": "CVE-2023-0013", + "package": "pyyaml", + "installed_version": "5.3", + "fixed_version": "5.4", + "cvss_v4_score": 9.8, + "epss_score": 0.5, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0014", + "package": "log4j", + "installed_version": "2.14.0", + "fixed_version": "2.17.1", + "cvss_v4_score": 10.0, + "epss_score": 0.97, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0015", + "package": "openssl", + "installed_version": "1.1.1k", + "fixed_version": "1.1.1t", + "cvss_v4_score": 7.4, + "epss_score": 0.12, + "direct": false, + "reachable": null + }, + { + "cve_id": "CVE-2023-0016", + "package": "zlib", + "installed_version": "1.2.11", + "fixed_version": "1.2.12", + "cvss_v4_score": 8.2, + "epss_score": 0.04, + "direct": false, + "reachable": false + }, + { + "cve_id": "CVE-2023-0017", + "package": "libxml2", + "installed_version": "2.9.10", + "fixed_version": null, + "cvss_v4_score": null, + "epss_score": 0.25, + "direct": false, + "reachable": true + }, + { + "cve_id": "CVE-2023-0018", + "package": "curl", + "installed_version": "7.68.0", + "fixed_version": "7.88.0", + "cvss_v4_score": 8.1, + "epss_score": null, + "direct": true, + "reachable": true + } + ], + "outdated": [ + { + "package": "pkg-0", + "current_version": "1.0.0", + "latest_version": "2.0.0", + "direct": true + }, + { + "package": "pkg-1", + "current_version": "1.1.0", + "latest_version": "2.1.0", + "direct": false + }, + { + "package": "pkg-2", + "current_version": "1.2.0", + "latest_version": "2.2.0", + "direct": true + }, + { + "package": "pkg-3", + "current_version": "1.3.0", + "latest_version": "2.3.0", + "direct": false + }, + { + "package": "pkg-4", + "current_version": "1.4.0", + "latest_version": "2.4.0", + "direct": true + }, + { + "package": "pkg-5", + "current_version": "1.5.0", + "latest_version": "2.5.0", + "direct": false + }, + { + "package": "pkg-6", + "current_version": "1.6.0", + "latest_version": "2.6.0", + "direct": true + }, + { + "package": "pkg-7", + "current_version": "1.7.0", + "latest_version": "2.7.0", + "direct": false + }, + { + "package": "pkg-8", + "current_version": "1.8.0", + "latest_version": "2.8.0", + "direct": true + }, + { + "package": "pkg-9", + "current_version": "1.9.0", + "latest_version": "2.9.0", + "direct": false + }, + { + "package": "pkg-10", + "current_version": "1.10.0", + "latest_version": "2.10.0", + "direct": true + }, + { + "package": "pkg-11", + "current_version": "1.11.0", + "latest_version": "2.11.0", + "direct": false + } + ], + "direct_count": 7, + "transitive_count": 143 + }, + "config": { + "secrets": [ + { + "id": "secret-00", + "secret_type": "aws_access_key", + "file_path": "config/env_0.yaml", + "line": 3, + "match": "AKIA****0000", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-01", + "secret_type": "github_token", + "file_path": "config/env_1.yaml", + "line": 4, + "match": "AKIA****0001", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-02", + "secret_type": "private_key", + "file_path": "config/env_2.yaml", + "line": 5, + "match": "AKIA****0002", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-03", + "secret_type": "slack_webhook", + "file_path": "config/env_3.yaml", + "line": 6, + "match": "AKIA****0003", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-04", + "secret_type": "generic_api_key", + "file_path": "config/env_4.yaml", + "line": 7, + "match": "AKIA****0004", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-05", + "secret_type": "aws_access_key", + "file_path": "config/env_5.yaml", + "line": 8, + "match": "AKIA****0005", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-06", + "secret_type": "github_token", + "file_path": "config/env_6.yaml", + "line": 9, + "match": "AKIA****0006", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-07", + "secret_type": "private_key", + "file_path": "config/env_7.yaml", + "line": 10, + "match": "AKIA****0007", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-08", + "secret_type": "slack_webhook", + "file_path": "config/env_8.yaml", + "line": 11, + "match": "AKIA****0008", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-09", + "secret_type": "generic_api_key", + "file_path": "config/env_9.yaml", + "line": 12, + "match": "AKIA****0009", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-10", + "secret_type": "aws_access_key", + "file_path": "config/env_10.yaml", + "line": 13, + "match": "AKIA****0010", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-11", + "secret_type": "github_token", + "file_path": "config/env_11.yaml", + "line": 14, + "match": "AKIA****0011", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-12", + "secret_type": "private_key", + "file_path": "config/env_12.yaml", + "line": 15, + "match": "AKIA****0012", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-13", + "secret_type": "slack_webhook", + "file_path": "config/env_13.yaml", + "line": 16, + "match": "AKIA****0013", + "confidence": "medium", + "is_test_value": true + }, + { + "id": "secret-14", + "secret_type": "generic_api_key", + "file_path": "config/env_14.yaml", + "line": 17, + "match": "AKIA****0014", + "confidence": "low", + "is_test_value": null + }, + { + "id": "secret-15", + "secret_type": "aws_access_key", + "file_path": "config/env_15.yaml", + "line": 18, + "match": "AKIA****0015", + "confidence": "high", + "is_test_value": false + }, + { + "id": "secret-16", + "secret_type": "github_token", + "file_path": "config/env_16.yaml", + "line": 19, + "match": "AKIA****0016", + "confidence": "medium", + "is_test_value": true + } + ], + "misconfigs": [ + { + "id": "misconfig-00", + "category": "logging", + "file_path": "config/app_0.ini", + "line": null, + "key": "LOG_LEVEL", + "value": "DEBUG", + "risk": "high", + "remediation": "Debug logging in production" + }, + { + "id": "misconfig-01", + "category": "tls", + "file_path": "config/app_1.ini", + "line": 13, + "key": "SSL_VERIFY", + "value": "false", + "risk": "critical", + "remediation": "TLS verification disabled" + }, + { + "id": "misconfig-02", + "category": "cors", + "file_path": "config/app_2.ini", + "line": 14, + "key": "ALLOW_ORIGIN", + "value": "*", + "risk": "high", + "remediation": null + }, + { + "id": "misconfig-03", + "category": "headers", + "file_path": "config/app_3.ini", + "line": 0, + "key": "X_FRAME_OPTIONS", + "value": null, + "risk": "medium", + "remediation": "Missing clickjacking header" + }, + { + "id": "misconfig-04", + "category": "debug", + "file_path": "config/app_4.ini", + "line": 16, + "key": "DJANGO_DEBUG", + "value": "True", + "risk": "critical", + "remediation": "Debug mode enabled" + }, + { + "id": "misconfig-05", + "category": "storage", + "file_path": "config/app_5.ini", + "line": null, + "key": "BUCKET_ACL", + "value": "public-read", + "risk": "critical", + "remediation": "Public bucket" + }, + { + "id": "misconfig-06", + "category": "auth", + "file_path": "config/app_6.ini", + "line": 18, + "key": "SESSION_TIMEOUT", + "value": "999999", + "risk": "medium", + "remediation": null + }, + { + "id": "misconfig-07", + "category": "http", + "file_path": "config/app_7.ini", + "line": 19, + "key": "REDIRECT_HTTPS", + "value": "false", + "risk": "high", + "remediation": "Plain HTTP allowed" + }, + { + "id": "misconfig-08", + "category": "secrets", + "file_path": "config/app_8.ini", + "line": 20, + "key": null, + "value": "inline", + "risk": "medium", + "remediation": "Inline secret" + }, + { + "id": "misconfig-09", + "category": "trace", + "file_path": "config/app_9.ini", + "line": 21, + "key": "OTEL_TRACE_ALL", + "value": "true", + "risk": "low", + "remediation": "Verbose tracing" + }, + { + "id": "misconfig-10", + "category": "network", + "file_path": "config/app_10.ini", + "line": null, + "key": "BIND_ADDR", + "value": "0.0.0.0", + "risk": "medium", + "remediation": null + }, + { + "id": "misconfig-11", + "category": "errors", + "file_path": "config/app_11.ini", + "line": 23, + "key": "SHOW_STACKTRACE", + "value": "true", + "risk": "high", + "remediation": "Error stacktraces exposed" + }, + { + "id": "misconfig-12", + "category": "cache", + "file_path": "config/app_12.ini", + "line": 24, + "key": "CACHE_TTL", + "value": "0", + "risk": "low", + "remediation": null + }, + { + "id": "misconfig-13", + "category": "db", + "file_path": "config/app_13.ini", + "line": 25, + "key": "SSLMODE", + "value": "disable", + "risk": "critical", + "remediation": "Database TLS off" + }, + { + "id": "misconfig-14", + "category": "queue", + "file_path": "config/app_14.ini", + "line": 26, + "key": "PREFETCH", + "value": "1000", + "risk": "low", + "remediation": null + }, + { + "id": "misconfig-15", + "category": "exposure", + "file_path": "config/app_15.ini", + "line": null, + "key": "ADMIN_PATH", + "value": "/admin", + "risk": "medium", + "remediation": "Admin surface exposed" + } + ] + }, + "security_context": { + "auth_model": "jwt", + "auth_details": "HS256 JWTs issued by app/auth/service.py, refresh tokens in Redis", + "crypto_usage": [ + { + "algorithm": "AES", + "key_size": 256, + "mode": "GCM", + "usage_context": "at-rest encryption", + "is_weak": false + }, + { + "algorithm": "MD5", + "key_size": null, + "mode": null, + "usage_context": "password hashing", + "is_weak": true + }, + { + "algorithm": "RSA", + "key_size": 1024, + "mode": null, + "usage_context": "token signing", + "is_weak": true + }, + { + "algorithm": "SHA-256", + "key_size": null, + "mode": null, + "usage_context": null, + "is_weak": false + }, + { + "algorithm": "DES", + "key_size": 56, + "mode": "CBC", + "usage_context": "legacy export", + "is_weak": true + }, + { + "algorithm": "HMAC-SHA1", + "key_size": 160, + "mode": null, + "usage_context": "webhook signature", + "is_weak": true + }, + { + "algorithm": "ChaCha20", + "key_size": 256, + "mode": "Poly1305", + "usage_context": "transport", + "is_weak": false + }, + { + "algorithm": "bcrypt", + "key_size": null, + "mode": null, + "usage_context": "password hashing", + "is_weak": false + }, + { + "algorithm": "RC4", + "key_size": 128, + "mode": null, + "usage_context": null, + "is_weak": true + }, + { + "algorithm": "ECDSA", + "key_size": 256, + "mode": null, + "usage_context": "JWT signing", + "is_weak": null + }, + { + "algorithm": "PBKDF2", + "key_size": null, + "mode": null, + "usage_context": "key derivation", + "is_weak": false + }, + { + "algorithm": "SHA-1", + "key_size": null, + "mode": null, + "usage_context": "checksum", + "is_weak": true + }, + { + "algorithm": "AES", + "key_size": 128, + "mode": "ECB", + "usage_context": "legacy blob", + "is_weak": true + }, + { + "algorithm": "Ed25519", + "key_size": 256, + "mode": null, + "usage_context": "package signing", + "is_weak": false + }, + { + "algorithm": "3DES", + "key_size": 168, + "mode": "CBC", + "usage_context": "legacy tape", + "is_weak": true + }, + { + "algorithm": "Argon2id", + "key_size": null, + "mode": null, + "usage_context": "password hashing", + "is_weak": null + } + ], + "framework_security": [ + "django.middleware.csrf.CsrfViewMiddleware", + "", + "helmet defaults", + "SecurityMiddleware", + "rack-protection", + "spring-security filter chain" + ], + "security_headers": [ + "Content-Security-Policy: default-src 'self'", + "X-Content-Type-Options: nosniff", + "Strict-Transport-Security: max-age=31536000", + "X-Frame-Options: DENY", + "Referrer-Policy: no-referrer", + "", + "Permissions-Policy: geolocation=()" + ], + "deployment_signals": [ + "kubernetes ingress with TLS termination", + "docker-compose exposes 5432", + "no WAF in front of /api", + "secrets mounted from vault", + "readiness probe on /healthz", + "single replica for the worker", + "TLS 1.2 minimum", + "internal service mesh mTLS" + ] + }, + "languages": [ + "Python", + "JavaScript", + "Go", + "Ruby", + "Rust", + "python" + ], + "frameworks": [ + "Django", + "next.js", + "Spring Boot", + " React ", + "unknown-fw", + "NEXT", + "Vue" + ], + "lines_of_code": 184213, + "file_count": 2417, + "recon_duration_seconds": 42.5 +} diff --git a/internal/secaf/report.go b/internal/secaf/report.go new file mode 100644 index 0000000000..a0989c981e --- /dev/null +++ b/internal/secaf/report.go @@ -0,0 +1,280 @@ +package secaf + +// What the audit hands back: the files of its full report, written in the +// run's record folder, and the short account of them the conversation reads. +// +// THE ACCOUNT IS IN A PERSON'S WORDS. sec-af's own vocabulary — a finding's +// "verdict", "verified", "not_exploitable" — is its machinery, and this house +// draws none of it: a finding here is confirmed, likely, unclear or ruled out. +// The JSON report keeps sec-af's field names, because it is data a tool reads. + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/Agent-Field/codeaf/internal/buildinfo" + "github.com/Agent-Field/codeaf/internal/secaf/output" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// Report file names in the record folder: the program's own name on each, as +// it is on every surface. +const ( + reportJSON = "sec-report.json" + reportMarkdown = "sec-report.md" + reportSARIF = "sec-report.sarif" + // reportCompliance is written only when the audit was asked to map its + // findings to compliance frameworks. + reportCompliance = "sec-compliance.md" +) + +// sarifTool is who sec's SARIF says produced it. A code-scanning tool shows +// the driver's name beside every result, and the program a person ran is sec, +// so the log names it, its build and its home rather than sec-af's. +func sarifTool() output.SarifTool { + return output.SarifTool{Name: Name, Version: buildinfo.Revision(), + InformationURI: "https://github.com/Agent-Field/CodeAF", PropertyPrefix: Name} +} + +// complianceBrand is how sec signs its compliance report. +var complianceBrand = output.ComplianceBrand{ + Heading: "# sec — compliance report", + Signature: "*Written by sec, the security audit codeaf carries.*", +} + +// reportFiles writes the full report into folder and answers the paths it +// wrote, in a fixed order. A folder of "" writes nothing. +func reportFiles(folder string, result schemas.SecurityAuditResult, compliance bool, run runFacts) ([]string, error) { + if strings.TrimSpace(folder) == "" { + return nil, nil + } + if err := os.MkdirAll(folder, 0o700); err != nil { + return nil, err + } + // THE FILES SAY WHAT THIS RUN MEASURED. sec-af's own result carries a + // cost and an agent count it never filled — $0.00 and the calls between + // its own parts — and a provider word ("harness") that is not true here. + // The run's own figures go in their place, and the SARIF is written again + // under sec's name. + result.CostUsd = run.spent + result.AgentInvocations = run.sessions + run.calls + result.Provider = "codeaf" + result.Findings = output.RuleIDsAs(result.Findings, sarifTool()) + result.Sarif = output.GenerateSarifAs(result, sarifTool()) + files := []struct { + name string + body []byte + }{ + {reportMarkdown, []byte(markdownReport(result, run))}, + {reportJSON, []byte(output.GenerateJSON(result, true) + "\n")}, + {reportSARIF, []byte(result.Sarif)}, + } + if compliance { + files = append(files, struct { + name string + body []byte + }{reportCompliance, []byte(output.GenerateComplianceReportAs(result, time.Now().UTC(), complianceBrand))}) + } + var written []string + for _, file := range files { + if len(strings.TrimSpace(string(file.body))) == 0 { + continue + } + path := filepath.Join(folder, file.name) + if err := os.WriteFile(path, file.body, 0o600); err != nil { + return written, err + } + written = append(written, path) + } + return written, nil +} + +// findingWord is how a finding came out, in a person's words. +func findingWord(verdict schemas.ExploitVerdict) string { + switch verdict { + case schemas.VerdictConfirmed: + return "confirmed" + case schemas.VerdictLikely: + return "likely" + case schemas.VerdictNotExploitable: + return "ruled out" + } + return "unclear" +} + +// severityRank orders severities most serious first. +func severityRank(severity schemas.Severity) int { + switch severity { + case schemas.SeverityCritical: + return 0 + case schemas.SeverityHigh: + return 1 + case schemas.SeverityMedium: + return 2 + case schemas.SeverityLow: + return 3 + } + return 4 +} + +// summaryFindings is how many findings the account lists one by one; the rest +// are counted, and the report holds them all. +const summaryFindings = 12 + +// summary is the account of a finished audit the conversation reads: what was +// audited, what it found in counts, the findings that stand one per line with +// where and how to fix, and where the full report is. +func summary(scope Scope, changes *Changes, result schemas.SecurityAuditResult, files []string, cutAt, wall string) string { + standing := make([]schemas.VerifiedFinding, 0, len(result.Findings)) + for _, finding := range result.Findings { + if finding.Verdict == schemas.VerdictConfirmed || finding.Verdict == schemas.VerdictLikely { + standing = append(standing, finding) + } + } + // THE FIRST LINE SAYS WHAT IT FOUND, because the first line is what the + // project's index keeps of a run (session's taskOutcome) and all another + // conversation reads of it there. It used to say only what was audited, + // so a window beside the one that ran it read "Security audit of the + // whole repository." and went looking through the disk for the rest. + var b strings.Builder + fmt.Fprintf(&b, "Security audit of %s", scope.Describe()) + if changes != nil { + fmt.Fprintf(&b, " (%d changed files since %s, %d files near them)", len(changes.Changed), changes.BaseName, len(changes.Nearby)) + } + if len(standing) == 0 { + b.WriteString(": nothing it could show exploitable") + if result.Inconclusive > 0 { + fmt.Fprintf(&b, "; %d finding%s stayed unclear and %s in the report", result.Inconclusive, plural(result.Inconclusive), areIs(result.Inconclusive)) + } + b.WriteString(".\n") + } else { + fmt.Fprintf(&b, ": %d problem%s, %d confirmed and %d likely", len(standing), plural(len(standing)), result.Confirmed, result.Likely) + if counts := severityCounts(standing); counts != "" { + b.WriteString(" (" + counts + ")") + } + b.WriteString(".\n") + } + if cutAt != "" { + b.WriteString(cutSentence(cutAt, wall) + "\n") + } + if len(standing) > 0 { + if result.Inconclusive > 0 || result.NotExploitable > 0 { + fmt.Fprintf(&b, "%d more stayed unclear and %d %s ruled out.\n", result.Inconclusive, result.NotExploitable, wasWere(result.NotExploitable)) + } + b.WriteString("\n") + sort.SliceStable(standing, func(i, j int) bool { + if a, b := severityRank(standing[i].Severity), severityRank(standing[j].Severity); a != b { + return a < b + } + return standing[i].Verdict == schemas.VerdictConfirmed && standing[j].Verdict != schemas.VerdictConfirmed + }) + for index, finding := range standing { + if index == summaryFindings { + fmt.Fprintf(&b, "- and %d more in the report\n", len(standing)-index) + break + } + fmt.Fprintf(&b, "- %s · %s · %s", finding.Severity, findingWord(finding.Verdict), oneLine(finding.Title)) + if where := locationWord(finding.Location); where != "" { + b.WriteString(" · " + where) + } + if finding.CweID != "" { + b.WriteString(" · " + finding.CweID) + } + if fix := fixWord(finding.Remediation); fix != "" { + b.WriteString("\n fix: " + fix) + } + b.WriteString("\n") + } + } + if len(result.AttackChains) > 0 { + fmt.Fprintf(&b, "\n%d attack chain%s link findings together; the report draws them.\n", len(result.AttackChains), plural(len(result.AttackChains))) + } + if len(files) > 0 { + b.WriteString("\nFull report: " + strings.Join(files, ", ") + "\n") + } + return strings.TrimSpace(b.String()) +} + +func severityCounts(findings []schemas.VerifiedFinding) string { + counts := map[schemas.Severity]int{} + for _, finding := range findings { + counts[finding.Severity]++ + } + var parts []string + for _, severity := range []schemas.Severity{schemas.SeverityCritical, schemas.SeverityHigh, schemas.SeverityMedium, schemas.SeverityLow, schemas.SeverityInfo} { + if counts[severity] > 0 { + parts = append(parts, fmt.Sprintf("%d %s", counts[severity], severity)) + } + } + return strings.Join(parts, ", ") +} + +func locationWord(location schemas.Location) string { + if strings.TrimSpace(location.FilePath) == "" { + return "" + } + if location.StartLine > 0 { + return fmt.Sprintf("%s:%d", location.FilePath, location.StartLine) + } + return location.FilePath +} + +// fixWord is a finding's fix in one line, cut where it runs long. +func fixWord(remediation *schemas.RemediationSuggestion) string { + if remediation == nil { + return "" + } + fix := oneLine(remediation.FixDescription) + if runes := []rune(fix); len(runes) > 220 { + fix = string(runes[:220]) + "…" + } + return fix +} + +func oneLine(text string) string { + return strings.Join(strings.Fields(text), " ") +} + +func plural(n int) string { + if n == 1 { + return "" + } + return "s" +} + +// outcomeLine is a finished audit in one line, for its ending: how many +// problems stand, or that none does. +func outcomeLine(result schemas.SecurityAuditResult, cutAt string) string { + standing := result.Confirmed + result.Likely + line := "it found nothing it could show exploitable" + if standing > 0 { + line = fmt.Sprintf("it found %d problem%s: %d confirmed, %d likely", standing, plural(standing), result.Confirmed, result.Likely) + } + if cutAt != "" { + line += ", before its time ceiling cut it short during " + cutAt + } + return line +} + +// cutSentence says that the run's time ceiling stopped it, in which phase, and +// what that left undone. +func cutSentence(phase, wall string) string { + ceiling := "its time ceiling" + if wall != "" { + ceiling += " of " + wall + } + undone := "the work after it did not run" + switch phase { + case stageWords[stageProve]: + undone = "findings whose tests had not finished are marked unclear, and no fixes were written" + case stageWords[stageRemediation]: + undone = "the fixes it had not finished are missing" + case stageWords[stageHunt]: + undone = "the hunters still running were stopped, and their findings were not tested" + } + return fmt.Sprintf("It reached %s during %s, so %s.", ceiling, phase, undone) +} diff --git a/internal/secaf/schemas/compliance.go b/internal/secaf/schemas/compliance.go new file mode 100644 index 0000000000..99c87dc44f --- /dev/null +++ b/internal/secaf/schemas/compliance.go @@ -0,0 +1,31 @@ +package schemas + +// This file ports src/sec_af/schemas/compliance.py — the compliance framework +// data models (DESIGN.md section 10). Both models are fully required; neither +// needs default seeding. + +// ComplianceMapping maps a finding onto one control of one framework. +// +// Ports schemas/compliance.py ComplianceMapping. +type ComplianceMapping struct { + Framework string `json:"framework"` + ControlID string `json:"control_id"` + ControlName string `json:"control_name"` +} + +// ComplianceGap is an aggregate: one framework control with unresolved +// findings against it. +// +// Ports schemas/compliance.py ComplianceGap. Python parity: `cwe_ids` is a +// REQUIRED `list[str]` (no default_factory), so a zero-value Go struct +// marshals it as null — which is what Python does too when the caller passes +// nothing, because pydantic refuses to construct the model at all. Callers +// always supply it. +type ComplianceGap struct { + Framework string `json:"framework"` + ControlID string `json:"control_id"` + ControlName string `json:"control_name"` + FindingCount int `json:"finding_count"` + MaxSeverity string `json:"max_severity"` + CweIDs []string `json:"cwe_ids"` +} diff --git a/internal/secaf/schemas/defaults.go b/internal/secaf/schemas/defaults.go new file mode 100644 index 0000000000..ca1060da4b --- /dev/null +++ b/internal/secaf/schemas/defaults.go @@ -0,0 +1,552 @@ +package schemas + +import ( + "bytes" + "encoding/json" +) + +// This file implements non-zero-default seeding for every struct that has at +// least one field whose pydantic default is not the Go zero value — see the +// package doc for the contract. Each such struct gets: +// +// - `NewX() X`, the pydantic-default value. Go code that BUILDS an X must use +// it, otherwise `default_factory=list` fields marshal as null instead of [] +// and the non-zero scalar defaults are missing. +// - `UnmarshalJSON`, which seeds NewX() before decoding, so an absent key +// keeps the default while a present key (even false/0/""/null) overrides it +// — matching pydantic's model_validate. +// +// The `type alias X` trick strips X's methods so the inner json.Unmarshal does +// not recurse; nested field types keep their own UnmarshalJSON and seeding. +// +// Python parity: the uuid4 default_factory fields (RawFinding.ID/Fingerprint, +// PotentialChain.ChainID, SecretFinding.ID, MisconfigFinding.ID, +// VerifiedFinding.ID) are minted by the CONSTRUCTORS only. UnmarshalJSON +// deliberately leaves them at "" when the key is absent so that decoding a +// payload never mints a fresh identity mid-pipeline; the live path always +// carries the field. + +// --- hunt.go --- + +// NewScanLocationsResult returns ScanLocationsResult's pydantic defaults +// (locations=[]). +func NewScanLocationsResult() ScanLocationsResult { + return ScanLocationsResult{Locations: []VulnLocation{}} +} + +// UnmarshalJSON seeds ScanLocationsResult's list default. +func (s *ScanLocationsResult) UnmarshalJSON(b []byte) error { + *s = NewScanLocationsResult() + type alias ScanLocationsResult + return json.Unmarshal(b, (*alias)(s)) +} + +// NewRawFinding returns RawFinding's pydantic defaults: fresh uuid4 ID and +// Fingerprint, related_files=[]. data_flow stays nil (Python default None). +func NewRawFinding() RawFinding { + return RawFinding{ + ID: NewUUID4(), + RelatedFiles: []string{}, + Fingerprint: NewUUID4(), + } +} + +// UnmarshalJSON seeds RawFinding.RelatedFiles=[]. ID and Fingerprint are NOT +// seeded (see the file header). +func (r *RawFinding) UnmarshalJSON(b []byte) error { + *r = RawFinding{RelatedFiles: []string{}} + type alias RawFinding + return json.Unmarshal(b, (*alias)(r)) +} + +// NewPotentialChain returns PotentialChain's pydantic defaults: fresh uuid4 +// ChainID, finding_ids=[]. +func NewPotentialChain() PotentialChain { + return PotentialChain{ChainID: NewUUID4(), FindingIDs: []string{}} +} + +// UnmarshalJSON seeds PotentialChain.FindingIDs=[]. ChainID is NOT seeded. +func (p *PotentialChain) UnmarshalJSON(b []byte) error { + *p = PotentialChain{FindingIDs: []string{}} + type alias PotentialChain + return json.Unmarshal(b, (*alias)(p)) +} + +// NewHuntResult returns HuntResult's pydantic defaults (three empty lists, +// zeroed counters). +func NewHuntResult() HuntResult { + return HuntResult{ + Findings: []RawFinding{}, + Chains: []PotentialChain{}, + StrategiesRun: []string{}, + } +} + +// UnmarshalJSON seeds HuntResult's three list defaults. +func (h *HuntResult) UnmarshalJSON(b []byte) error { + *h = NewHuntResult() + type alias HuntResult + return json.Unmarshal(b, (*alias)(h)) +} + +// NewDeduplicatedResult returns DeduplicatedResult's pydantic defaults. +func NewDeduplicatedResult() DeduplicatedResult { + return DeduplicatedResult{Findings: []RawFinding{}, Chains: []PotentialChain{}} +} + +// UnmarshalJSON seeds DeduplicatedResult's two list defaults. +func (d *DeduplicatedResult) UnmarshalJSON(b []byte) error { + *d = NewDeduplicatedResult() + type alias DeduplicatedResult + return json.Unmarshal(b, (*alias)(d)) +} + +// NewChainCorrelationResult returns ChainCorrelationResult's pydantic defaults. +func NewChainCorrelationResult() ChainCorrelationResult { + return ChainCorrelationResult{Chains: []string{}, DuplicateIDs: []string{}} +} + +// UnmarshalJSON seeds ChainCorrelationResult's two list defaults. +func (c *ChainCorrelationResult) UnmarshalJSON(b []byte) error { + *c = NewChainCorrelationResult() + type alias ChainCorrelationResult + return json.Unmarshal(b, (*alias)(c)) +} + +// --- input.go --- + +// NewAuditInput returns AuditInput's pydantic defaults: branch="main", +// depth="standard", severity_threshold="low", +// scan_types=["sast","sca","secrets","config"], output_formats=["json"], +// exclude_paths=["tests/","vendor/","node_modules/",".git/"], and empty +// compliance_frameworks / repo_urls / custom_policies. +func NewAuditInput() AuditInput { + return AuditInput{ + Branch: "main", + Depth: "standard", + SeverityThreshold: "low", + ScanTypes: []string{"sast", "sca", "secrets", "config"}, + OutputFormats: []string{"json"}, + ComplianceFrameworks: []string{}, + ExcludePaths: []string{"tests/", "vendor/", "node_modules/", ".git/"}, + RepoUrls: []string{}, + CustomPolicies: []string{}, + } +} + +// UnmarshalJSON seeds AuditInput's pydantic defaults before decoding, so +// afx.Bind[AuditInput](payload) behaves like AuditInput.model_validate(payload). +func (a *AuditInput) UnmarshalJSON(b []byte) error { + *a = NewAuditInput() + type alias AuditInput + return json.Unmarshal(b, (*alias)(a)) +} + +// --- output.go --- + +// NewAttackChain returns AttackChain's pydantic defaults (findings=[]). +func NewAttackChain() AttackChain { + return AttackChain{Findings: []string{}} +} + +// UnmarshalJSON seeds AttackChain.Findings=[]. +func (a *AttackChain) UnmarshalJSON(b []byte) error { + *a = NewAttackChain() + type alias AttackChain + return json.Unmarshal(b, (*alias)(a)) +} + +// NewServiceDefinition returns ServiceDefinition's pydantic defaults. +func NewServiceDefinition() ServiceDefinition { + return ServiceDefinition{APIEndpoints: []string{}, Dependencies: []string{}} +} + +// UnmarshalJSON seeds ServiceDefinition's two list defaults. +func (s *ServiceDefinition) UnmarshalJSON(b []byte) error { + *s = NewServiceDefinition() + type alias ServiceDefinition + return json.Unmarshal(b, (*alias)(s)) +} + +// NewMonitoringResult returns MonitoringResult's pydantic defaults. +func NewMonitoringResult() MonitoringResult { + return MonitoringResult{ + NewFindings: []RegressionFinding{}, + FixedFindings: []RegressionFinding{}, + } +} + +// UnmarshalJSON seeds MonitoringResult's two list defaults. +func (m *MonitoringResult) UnmarshalJSON(b []byte) error { + *m = NewMonitoringResult() + type alias MonitoringResult + return json.Unmarshal(b, (*alias)(m)) +} + +// NewPolicyViolation returns PolicyViolation's pydantic defaults +// (severity="medium"). +func NewPolicyViolation() PolicyViolation { + return PolicyViolation{Severity: "medium"} +} + +// UnmarshalJSON seeds PolicyViolation.Severity="medium". +func (p *PolicyViolation) UnmarshalJSON(b []byte) error { + *p = NewPolicyViolation() + type alias PolicyViolation + return json.Unmarshal(b, (*alias)(p)) +} + +// NewSecurityAuditResult returns SecurityAuditResult's pydantic defaults: the +// five list fields empty, the three dict fields empty. +func NewSecurityAuditResult() SecurityAuditResult { + return SecurityAuditResult{ + StrategiesUsed: []string{}, + Findings: []VerifiedFinding{}, + AttackChains: []AttackChain{}, + BySeverity: map[string]int{}, + ComplianceGaps: []ComplianceGap{}, + PolicyViolations: []PolicyViolation{}, + CostBreakdown: map[string]float64{}, + Metadata: map[string]any{}, + } +} + +// UnmarshalJSON seeds SecurityAuditResult's container defaults and decodes with +// UseNumber. +// +// UseNumber is here for exactly one field: `Metadata map[string]any`, the port +// of pydantic's `metadata: dict[str, object]`. An `object`-typed value keeps +// whatever the DECODER produced, and CPython's json.loads produces an `int` for +// an integer literal and a `float` otherwise — so `{"demoted_total": 2}` +// re-serialises as "2" in Python. Go's default decode turns every number into +// float64 and would spell it "2.0". UseNumber affects ONLY values decoded into +// `interface{}`; the model's typed int/float fields are untouched, exactly as +// pydantic's are. +// +// The same compensation is applied on the live path by afx.WireNumbers, where +// the drop summary arrives already decoded from the SDK's own reader (see +// internal/secaf/audit/run.go). +func (s *SecurityAuditResult) UnmarshalJSON(b []byte) error { + *s = NewSecurityAuditResult() + type alias SecurityAuditResult + dec := json.NewDecoder(bytes.NewReader(b)) + dec.UseNumber() + return dec.Decode((*alias)(s)) +} + +// NewAuditMetrics returns AuditMetrics's pydantic defaults +// (cost_breakdown={}). +func NewAuditMetrics() AuditMetrics { + return AuditMetrics{CostBreakdown: map[string]float64{}} +} + +// UnmarshalJSON seeds AuditMetrics.CostBreakdown={}. +func (a *AuditMetrics) UnmarshalJSON(b []byte) error { + *a = NewAuditMetrics() + type alias AuditMetrics + return json.Unmarshal(b, (*alias)(a)) +} + +// --- prove.go --- + +// NewDataFlowEvidence returns DataFlowEvidence's pydantic defaults (steps=[]). +func NewDataFlowEvidence() DataFlowEvidence { + return DataFlowEvidence{Steps: []DataFlowStep{}} +} + +// UnmarshalJSON seeds DataFlowEvidence.Steps=[]. +func (d *DataFlowEvidence) UnmarshalJSON(b []byte) error { + *d = NewDataFlowEvidence() + type alias DataFlowEvidence + return json.Unmarshal(b, (*alias)(d)) +} + +// NewReachabilityEvidence returns ReachabilityEvidence's pydantic defaults +// (call_chain=[]). +func NewReachabilityEvidence() ReachabilityEvidence { + return ReachabilityEvidence{CallChain: []string{}} +} + +// UnmarshalJSON seeds ReachabilityEvidence.CallChain=[]. +func (r *ReachabilityEvidence) UnmarshalJSON(b []byte) error { + *r = NewReachabilityEvidence() + type alias ReachabilityEvidence + return json.Unmarshal(b, (*alias)(r)) +} + +// NewVerifiedFinding returns VerifiedFinding's pydantic defaults: fresh uuid4 +// ID, and tags / related_locations / compliance / reproduction_steps empty. +func NewVerifiedFinding() VerifiedFinding { + v := verifiedFindingListDefaults() + v.ID = NewUUID4() + return v +} + +// verifiedFindingListDefaults is NewVerifiedFinding without the uuid4 ID — +// what UnmarshalJSON seeds. +func verifiedFindingListDefaults() VerifiedFinding { + return VerifiedFinding{ + Tags: []string{}, + RelatedLocations: []Location{}, + Compliance: []ComplianceMapping{}, + ReproductionSteps: []ReproductionStep{}, + } +} + +// UnmarshalJSON seeds VerifiedFinding's four list defaults. ID is NOT seeded. +func (v *VerifiedFinding) UnmarshalJSON(b []byte) error { + *v = verifiedFindingListDefaults() + type alias VerifiedFinding + return json.Unmarshal(b, (*alias)(v)) +} + +// --- recon.go --- + +// NewArchitectureMap returns ArchitectureMap's pydantic defaults (five empty +// lists; app_type stays nil). +func NewArchitectureMap() ArchitectureMap { + return ArchitectureMap{ + Modules: []Module{}, + EntryPoints: []EntryPoint{}, + TrustBoundaries: []TrustBoundary{}, + Services: []Service{}, + APISurface: []APIEndpoint{}, + } +} + +// UnmarshalJSON seeds ArchitectureMap's five list defaults. +func (a *ArchitectureMap) UnmarshalJSON(b []byte) error { + *a = NewArchitectureMap() + type alias ArchitectureMap + return json.Unmarshal(b, (*alias)(a)) +} + +// NewModule returns Module's pydantic defaults (dependencies=[]). +func NewModule() Module { return Module{Dependencies: []string{}} } + +// UnmarshalJSON seeds Module.Dependencies=[]. +func (m *Module) UnmarshalJSON(b []byte) error { + *m = NewModule() + type alias Module + return json.Unmarshal(b, (*alias)(m)) +} + +// NewTrustBoundary returns TrustBoundary's pydantic defaults (enforcement=[]). +func NewTrustBoundary() TrustBoundary { return TrustBoundary{Enforcement: []string{}} } + +// UnmarshalJSON seeds TrustBoundary.Enforcement=[]. +func (t *TrustBoundary) UnmarshalJSON(b []byte) error { + *t = NewTrustBoundary() + type alias TrustBoundary + return json.Unmarshal(b, (*alias)(t)) +} + +// NewSanitizationPoint returns SanitizationPoint's pydantic defaults +// (protects_against=[]). +func NewSanitizationPoint() SanitizationPoint { + return SanitizationPoint{ProtectsAgainst: []string{}} +} + +// UnmarshalJSON seeds SanitizationPoint.ProtectsAgainst=[]. +func (s *SanitizationPoint) UnmarshalJSON(b []byte) error { + *s = NewSanitizationPoint() + type alias SanitizationPoint + return json.Unmarshal(b, (*alias)(s)) +} + +// NewDataFlow returns DataFlow's pydantic defaults (path=[], files=[]). +func NewDataFlow() DataFlow { + return DataFlow{Path: []ReconDataFlowStep{}, Files: []string{}} +} + +// UnmarshalJSON seeds DataFlow's two list defaults. +func (d *DataFlow) UnmarshalJSON(b []byte) error { + *d = NewDataFlow() + type alias DataFlow + return json.Unmarshal(b, (*alias)(d)) +} + +// NewDataFlowMap returns DataFlowMap's pydantic defaults (three empty lists). +func NewDataFlowMap() DataFlowMap { + return DataFlowMap{ + Flows: []DataFlow{}, + SanitizationPoints: []SanitizationPoint{}, + Sinks: []Sink{}, + } +} + +// UnmarshalJSON seeds DataFlowMap's three list defaults. +func (d *DataFlowMap) UnmarshalJSON(b []byte) error { + *d = NewDataFlowMap() + type alias DataFlowMap + return json.Unmarshal(b, (*alias)(d)) +} + +// NewDependencyReport returns DependencyReport's pydantic defaults (three +// empty lists, zeroed counters). +func NewDependencyReport() DependencyReport { + return DependencyReport{ + Sbom: []Dependency{}, + KnownCves: []KnownCVE{}, + Outdated: []OutdatedDep{}, + } +} + +// UnmarshalJSON seeds DependencyReport's three list defaults. +func (d *DependencyReport) UnmarshalJSON(b []byte) error { + *d = NewDependencyReport() + type alias DependencyReport + return json.Unmarshal(b, (*alias)(d)) +} + +// NewSecretFinding returns SecretFinding's pydantic defaults (fresh uuid4 ID). +func NewSecretFinding() SecretFinding { return SecretFinding{ID: NewUUID4()} } + +// NewMisconfigFinding returns MisconfigFinding's pydantic defaults (fresh +// uuid4 ID). +func NewMisconfigFinding() MisconfigFinding { return MisconfigFinding{ID: NewUUID4()} } + +// NewConfigReport returns ConfigReport's pydantic defaults (secrets=[], +// misconfigs=[]). +func NewConfigReport() ConfigReport { + return ConfigReport{Secrets: []SecretFinding{}, Misconfigs: []MisconfigFinding{}} +} + +// UnmarshalJSON seeds ConfigReport's two list defaults. +func (c *ConfigReport) UnmarshalJSON(b []byte) error { + *c = NewConfigReport() + type alias ConfigReport + return json.Unmarshal(b, (*alias)(c)) +} + +// NewSecurityContext returns SecurityContext's pydantic defaults (four empty +// lists). +func NewSecurityContext() SecurityContext { + return SecurityContext{ + CryptoUsage: []CryptoUsage{}, + FrameworkSecurity: []string{}, + SecurityHeaders: []string{}, + DeploymentSignals: []string{}, + } +} + +// UnmarshalJSON seeds SecurityContext's four list defaults. +func (s *SecurityContext) UnmarshalJSON(b []byte) error { + *s = NewSecurityContext() + type alias SecurityContext + return json.Unmarshal(b, (*alias)(s)) +} + +// NewReconResult returns ReconResult's pydantic defaults: languages=[], +// frameworks=[], and — because the five nested models are REQUIRED in Python +// and therefore always constructed by the caller — each nested model at ITS +// own defaults rather than a half-built zero value. +func NewReconResult() ReconResult { + return ReconResult{ + Architecture: NewArchitectureMap(), + DataFlows: NewDataFlowMap(), + Dependencies: NewDependencyReport(), + Config: NewConfigReport(), + SecurityContext: NewSecurityContext(), + Languages: []string{}, + Frameworks: []string{}, + } +} + +// UnmarshalJSON seeds ReconResult's defaults. The nested models re-seed +// themselves through their own UnmarshalJSON when their key is present; when a +// key is ABSENT the seeded default survives, which is what Python's +// `_recon_model` normalization (reasoners/phases.py) relies on. +func (r *ReconResult) UnmarshalJSON(b []byte) error { + *r = NewReconResult() + type alias ReconResult + return json.Unmarshal(b, (*alias)(r)) +} + +// NewArchitectureMapRaw returns ArchitectureMapRaw's pydantic defaults +// (app_type="unknown", five empty lists). +func NewArchitectureMapRaw() ArchitectureMapRaw { + return ArchitectureMapRaw{ + AppType: "unknown", + Modules: []string{}, + EntryPoints: []string{}, + TrustBoundaries: []string{}, + Services: []string{}, + APIEndpoints: []string{}, + } +} + +// UnmarshalJSON seeds ArchitectureMapRaw's defaults. +func (a *ArchitectureMapRaw) UnmarshalJSON(b []byte) error { + *a = NewArchitectureMapRaw() + type alias ArchitectureMapRaw + return json.Unmarshal(b, (*alias)(a)) +} + +// NewDataFlowMapRaw returns DataFlowMapRaw's pydantic defaults (three empty +// lists). +func NewDataFlowMapRaw() DataFlowMapRaw { + return DataFlowMapRaw{ + Flows: []string{}, + SanitizationPoints: []string{}, + Sinks: []string{}, + } +} + +// UnmarshalJSON seeds DataFlowMapRaw's three list defaults. +func (d *DataFlowMapRaw) UnmarshalJSON(b []byte) error { + *d = NewDataFlowMapRaw() + type alias DataFlowMapRaw + return json.Unmarshal(b, (*alias)(d)) +} + +// NewDependencyReportRaw returns DependencyReportRaw's pydantic defaults. +func NewDependencyReportRaw() DependencyReportRaw { + return DependencyReportRaw{Sbom: []string{}, KnownCves: []string{}, Outdated: []string{}} +} + +// UnmarshalJSON seeds DependencyReportRaw's three list defaults. +func (d *DependencyReportRaw) UnmarshalJSON(b []byte) error { + *d = NewDependencyReportRaw() + type alias DependencyReportRaw + return json.Unmarshal(b, (*alias)(d)) +} + +// NewConfigReportRaw returns ConfigReportRaw's pydantic defaults. +func NewConfigReportRaw() ConfigReportRaw { + return ConfigReportRaw{Secrets: []string{}, Misconfigs: []string{}} +} + +// UnmarshalJSON seeds ConfigReportRaw's two list defaults. +func (c *ConfigReportRaw) UnmarshalJSON(b []byte) error { + *c = NewConfigReportRaw() + type alias ConfigReportRaw + return json.Unmarshal(b, (*alias)(c)) +} + +// NewSecurityContextRaw returns SecurityContextRaw's pydantic defaults +// (auth_details="" — the Go zero value — plus two empty lists). +func NewSecurityContextRaw() SecurityContextRaw { + return SecurityContextRaw{CryptoUsage: []string{}, SecuritySignals: []string{}} +} + +// UnmarshalJSON seeds SecurityContextRaw's two list defaults. +func (s *SecurityContextRaw) UnmarshalJSON(b []byte) error { + *s = NewSecurityContextRaw() + type alias SecurityContextRaw + return json.Unmarshal(b, (*alias)(s)) +} + +// --- prove.go (before-validator models that also need list seeding) --- + +// NewDataFlowTrace returns DataFlowTrace with a non-nil Steps slice. Python +// declares `steps` as REQUIRED (no default_factory), so this is not a pydantic +// default — it exists so Go code that builds a trace does not emit null where +// the model always carries a list. +func NewDataFlowTrace() DataFlowTrace { return DataFlowTrace{Steps: []string{}} } + +// NewReachabilityProof returns ReachabilityProof with a non-nil CallChain. +// As with DataFlowTrace, `call_chain` is REQUIRED in Python. +func NewReachabilityProof() ReachabilityProof { + return ReachabilityProof{CallChain: []string{}} +} diff --git a/internal/secaf/schemas/doc.go b/internal/secaf/schemas/doc.go new file mode 100644 index 0000000000..9ed1673fb6 --- /dev/null +++ b/internal/secaf/schemas/doc.go @@ -0,0 +1,99 @@ +// Package schemas ports every pydantic model and enum SEC-AF exchanges over a +// JSON boundary: `src/sec_af/schemas/*.py` (compliance, gates, hunt, input, +// output, prove, recon, views) plus `PolicyEvalResult` from +// `src/sec_af/policies.py`, which `evaluate_policy` hands to +// `app.harness(schema=...)`. +// +// # Parity rules (docs/DESIGN.md §0.2, §2, §3) +// +// Struct name == pydantic class name, json tag == pydantic field name, and +// Go field declaration order == Python field declaration order (encoding/json +// emits struct fields in declaration order, so a marshaled Go value reproduces +// `model_dump()`'s key order exactly — the parity test in +// model_keys_test.go asserts the ordered key list, not just the set). +// +// NO `omitempty` anywhere. Python reasoners return `model_dump()`, which emits +// every field including the ones that are None. Where Python uses +// `model_dump(exclude_none=True)` the call site drops nulls through +// afx.DropNulls; it is never a property of the struct. +// +// `Optional[X]` / `X | None` maps to a Go pointer (`*string`, `*int`, +// `*float64`, `*bool`, `*SomeModel`) so an unset value marshals to JSON null +// exactly as pydantic does. The two exceptions are `list[X] | None` and +// `dict[K, V] | None`: a nil Go slice/map already marshals to null and a +// non-nil empty one to `[]` / `{}`, which is precisely the pydantic behavior, +// so those stay plain slices/maps (a pointer would only add a second nil to +// reason about). +// +// # Default seeding (the pr-af `schemas/defaults.go` pattern) +// +// Go's json.Unmarshal leaves an absent key at the Go zero value; pydantic +// fills the declared default. Every struct with at least one non-zero pydantic +// default therefore gets, in defaults.go: +// +// - an exported constructor `NewX() X` returning the pydantic-default value. +// Use it whenever Go code BUILDS an X — a zero-value `X{}` marshals +// `default_factory=list` fields as `null` where pydantic emits `[]`, and +// misses the non-zero scalar defaults (`AuditInput.Depth == "standard"`, +// `ArchitectureMapRaw.AppType == "unknown"`, …). +// - an `UnmarshalJSON` that seeds `NewX()` before decoding, so an absent key +// keeps the default while a present key — `false`, `0`, `""` — overrides +// it. +// +// An explicit `null` is the ONE case seeding cannot get right on its own, and +// it goes the other way: encoding/json treats a null as a no-op for a scalar +// and as ZERO-THE-VALUE for a slice/map/pointer, so `{"findings": null}` binds +// cleanly AND wipes the seeded `[]`, emitting `"findings": null` — a shape +// pydantic can never produce for a non-Optional field (`HuntResult(findings= +// None)` raises). Only `X | None` fields accept a null, plus the handful of +// required fields whose `mode="before"` validator maps None onto a value +// (`DataFlowTrace.source` -> "unknown"). That distinction is generated ground +// truth (`accepts_null` in testdata/model_keys.json) and is enforced one layer +// up, by internal/phases' checked binders — so `afx.Bind[X](map)` alone is +// NOT `X.model_validate(dict)`; `phases.BindX` is. +// +// The `type alias X` trick inside each UnmarshalJSON strips X's methods so the +// inner json.Unmarshal does not recurse; nested field types keep their own +// UnmarshalJSON and therefore their own seeding. +// +// Models whose every field is required or zero-defaulted have no constructor +// and no UnmarshalJSON — their zero value already matches pydantic. The parity +// test enumerates all 80 distinct models and fails if one that needs seeding +// lacks it. +// +// `default_factory=lambda: str(uuid4())` fields (`RawFinding.ID`, +// `RawFinding.Fingerprint`, `PotentialChain.ChainID`, `SecretFinding.ID`, +// `MisconfigFinding.ID`, `VerifiedFinding.ID`) are seeded by the constructors +// with a fresh RFC 4122 v4 string from NewUUID4 (uuid.go, crypto/rand — the +// port takes no new third-party dependency). They are deliberately NOT seeded +// in UnmarshalJSON: decoding is how a value crosses a reasoner boundary, and a +// payload that omits `id` should not silently mint a NEW identity on every +// hop. Python has the same hazard; the live path always carries the field. +// +// # Enums +// +// Python's `class X(str, Enum)` maps to `type X string` with one constant per +// member; `class EvidenceLevel(IntEnum)` maps to `type EvidenceLevel int`. +// Each has `Valid()` and a `ParseX` helper. Note that `HuntStrategy` has a +// value alias — Python's `LOGIC_BUGS = "business_logic"` is the SAME member as +// `BUSINESS_LOGIC`, not a distinct one (`HuntStrategy.LOGIC_BUGS is +// HuntStrategy.BUSINESS_LOGIC` is True) — so Go declares both constants with +// the same value and `AllHuntStrategies` lists the value once. +// +// Python parity: `str(Severity.HIGH)` is "Severity.HIGH", not "high" (3.11 +// Enum.__str__ on a str mixin), while `str(EvidenceLevel.FULL_EXPLOIT)` is "6" +// (3.11 IntEnum.__str__ == int.__str__). Go's string/int enums print their +// value, so any prompt builder that interpolates an enum must use `.value` / +// `int(...)` on the Python side to match — none of the ported prompt builders +// interpolate a bare enum member. +// +// # Name collisions +// +// `prove.py` re-declares `Location`, `CvssV4Score`, `EpssScore` and +// `ReproductionStep` byte-identically to `output.py`; the Go package keeps one +// struct each (declared in output.go, which is what `schemas/__init__.py` +// re-exports) and the parity test asserts both Python declarations really do +// have the same shape. `recon.DataFlowStep` and `prove.DataFlowStep` are +// DIFFERENT models sharing a name; Go follows `schemas/__init__.py`, which +// re-exports prove's as `DataFlowStep` and recon's as `ReconDataFlowStep`. +package schemas diff --git a/internal/secaf/schemas/gates.go b/internal/secaf/schemas/gates.go new file mode 100644 index 0000000000..3c6b7d5564 --- /dev/null +++ b/internal/secaf/schemas/gates.go @@ -0,0 +1,98 @@ +package schemas + +// This file ports src/sec_af/schemas/gates.py — the FLAT schemas handed to +// `.ai()` gate calls (DESIGN.md §2.4/§2.5: .ai() schemas must stay flat). +// Every field is required in Python except DuplicateCheck.duplicate_of, so +// only that one model has a nullable field and none need default seeding. + +// SeverityClassification is the quick severity classification gate used in +// scoring (DESIGN.md §2.4). +// +// Ports schemas/gates.py SeverityClassification. +type SeverityClassification struct { + // Severity is one of: "critical", "high", "medium", "low". + Severity string `json:"severity"` + Confidence float64 `json:"confidence"` + Rationale string `json:"rationale"` +} + +// DuplicateCheck is the quick duplicate-check gate for dedup decisions +// (DESIGN.md §5.5). +// +// Ports schemas/gates.py DuplicateCheck. +type DuplicateCheck struct { + IsDuplicate bool `json:"is_duplicate"` + DuplicateOf *string `json:"duplicate_of"` + Reason string `json:"reason"` +} + +// StrategySelection is the strategy-selection gate for HUNT routing +// (DESIGN.md §5.3). +// +// Ports schemas/gates.py StrategySelection. +type StrategySelection struct { + Strategies []string `json:"strategies"` + Rationale string `json:"rationale"` +} + +// CWEExpansion carries AI-suggested CWE additions derived from recon context. +// +// Ports schemas/gates.py CWEExpansion. +type CWEExpansion struct { + // AdditionalCwes are CWE IDs to add beyond the hunter's baseline, e.g. + // ['CWE-918', 'CWE-611']. + AdditionalCwes []string `json:"additional_cwes"` + Rationale string `json:"rationale"` +} + +// RelevanceGate is the relevance/noise filter gate for candidate findings +// (DESIGN.md §2.4). +// +// Ports schemas/gates.py RelevanceGate. +type RelevanceGate struct { + IsRelevant bool `json:"is_relevant"` + Confidence float64 `json:"confidence"` + Reason string `json:"reason"` +} + +// VerdictGate is the binary verdict gate for simple cases (DESIGN.md §2.4, +// §6.3). +// +// Ports schemas/gates.py VerdictGate. +type VerdictGate struct { + Confirmed bool `json:"confirmed"` + Confidence float64 `json:"confidence"` + Reason string `json:"reason"` +} + +// ComplianceSuggestion is one AI-suggested framework control mapping. +// +// Ports schemas/gates.py ComplianceSuggestion. It has the same shape as +// ComplianceMapping but is a distinct pydantic class (and therefore a distinct +// harness/ai schema fixture), so Go keeps it distinct too. +type ComplianceSuggestion struct { + Framework string `json:"framework"` + ControlID string `json:"control_id"` + ControlName string `json:"control_name"` +} + +// ComplianceGate is the AI half of get_compliance_mappings_hybrid. +// +// Ports schemas/gates.py ComplianceGate. +type ComplianceGate struct { + Mappings []ComplianceSuggestion `json:"mappings"` + Confidence string `json:"confidence"` +} + +// ReachabilityGate is the reachability assessment for findings that carry no +// explicit reachability tag. +// +// Ports schemas/gates.py ReachabilityGate. +type ReachabilityGate struct { + // Reachability is one of: "externally_reachable", "requires_auth", + // "internal_only", "unreachable". + Reachability string `json:"reachability"` + Rationale string `json:"rationale"` + // Confidence is one of: "high", "medium", "low". + Confidence string `json:"confidence"` +} diff --git a/internal/secaf/schemas/hunt.go b/internal/secaf/schemas/hunt.go new file mode 100644 index 0000000000..fb763cb921 --- /dev/null +++ b/internal/secaf/schemas/hunt.go @@ -0,0 +1,358 @@ +package schemas + +import ( + "strconv" + "strings" +) + +// This file ports src/sec_af/schemas/hunt.py — the HUNT phase enums, the flat +// two-step scan/enrich harness schemas, and the RawFinding / chain / result +// models (DESIGN.md §5.2-§5.5). + +// FindingType is the finding taxonomy (DESIGN.md §5.4). +// +// Ports schemas/hunt.py FindingType (`class FindingType(str, Enum)`). +type FindingType string + +// FindingType members, in Python declaration order. +const ( + FindingTypeSast FindingType = "sast" + FindingTypeSca FindingType = "sca" + FindingTypeSecrets FindingType = "secrets" + FindingTypeConfig FindingType = "config" + FindingTypeLogic FindingType = "logic" + FindingTypeAPI FindingType = "api" +) + +// AllFindingTypes lists every FindingType in Python declaration order. +var AllFindingTypes = []FindingType{ + FindingTypeSast, FindingTypeSca, FindingTypeSecrets, + FindingTypeConfig, FindingTypeLogic, FindingTypeAPI, +} + +// Valid reports whether f is one of the declared members. +func (f FindingType) Valid() bool { + for _, v := range AllFindingTypes { + if f == v { + return true + } + } + return false +} + +// ParseFindingType is the Go equivalent of `FindingType(s)`: it returns the +// member with value s, or an error (pydantic raises ValidationError). +func ParseFindingType(s string) (FindingType, error) { + v := FindingType(s) + if !v.Valid() { + return "", &EnumValueError{Enum: "FindingType", Value: s} + } + return v, nil +} + +// Severity is the severity scale (DESIGN.md §5.4, §7). +// +// Ports schemas/hunt.py Severity (`class Severity(str, Enum)`). +type Severity string + +// Severity members, in Python declaration order (decreasing urgency). +const ( + SeverityCritical Severity = "critical" + SeverityHigh Severity = "high" + SeverityMedium Severity = "medium" + SeverityLow Severity = "low" + SeverityInfo Severity = "info" +) + +// AllSeverities lists every Severity in Python declaration order. +var AllSeverities = []Severity{ + SeverityCritical, SeverityHigh, SeverityMedium, SeverityLow, SeverityInfo, +} + +// Valid reports whether s is one of the declared members. +func (s Severity) Valid() bool { + for _, v := range AllSeverities { + if s == v { + return true + } + } + return false +} + +// ParseSeverity is the Go equivalent of `Severity(s)`. +func ParseSeverity(s string) (Severity, error) { + v := Severity(s) + if !v.Valid() { + return "", &EnumValueError{Enum: "Severity", Value: s} + } + return v, nil +} + +// Confidence is the confidence scale for provisional findings +// (DESIGN.md §5.4). +// +// Ports schemas/hunt.py Confidence (`class Confidence(str, Enum)`). +type Confidence string + +// Confidence members, in Python declaration order. +const ( + ConfidenceHigh Confidence = "high" + ConfidenceMedium Confidence = "medium" + ConfidenceLow Confidence = "low" +) + +// AllConfidences lists every Confidence in Python declaration order. +var AllConfidences = []Confidence{ConfidenceHigh, ConfidenceMedium, ConfidenceLow} + +// Valid reports whether c is one of the declared members. +func (c Confidence) Valid() bool { + for _, v := range AllConfidences { + if c == v { + return true + } + } + return false +} + +// ParseConfidence is the Go equivalent of `Confidence(s)`. +func ParseConfidence(s string) (Confidence, error) { + v := Confidence(s) + if !v.Valid() { + return "", &EnumValueError{Enum: "Confidence", Value: s} + } + return v, nil +} + +// HuntStrategy is the strategy catalog for hunters (DESIGN.md §5.2, §5.3). +// +// Ports schemas/hunt.py HuntStrategy (`class HuntStrategy(str, Enum)`). +// +// Python parity: `LOGIC_BUGS = "business_logic"` repeats BUSINESS_LOGIC's +// value, so Python's Enum machinery makes LOGIC_BUGS an ALIAS for the same +// member — `HuntStrategy.LOGIC_BUGS is HuntStrategy.BUSINESS_LOGIC` is True and +// `list(HuntStrategy)` yields the value once. Go declares both constants (so +// call sites can use either spelling) but AllHuntStrategies lists +// "business_logic" once, matching `list(HuntStrategy)`. +type HuntStrategy string + +// HuntStrategy members, in Python declaration order. +const ( + HuntStrategyInjection HuntStrategy = "injection" + HuntStrategyXSS HuntStrategy = "xss" + HuntStrategyDos HuntStrategy = "dos" + HuntStrategySSRF HuntStrategy = "ssrf" + HuntStrategyAuth HuntStrategy = "auth" + HuntStrategyCrypto HuntStrategy = "crypto" + HuntStrategyBusinessLogic HuntStrategy = "business_logic" + HuntStrategyLogicBugs HuntStrategy = "business_logic" // Python alias of BUSINESS_LOGIC + HuntStrategyDataExposure HuntStrategy = "data_exposure" + HuntStrategySupplyChain HuntStrategy = "supply_chain" + HuntStrategyConfigSecrets HuntStrategy = "config_secrets" + HuntStrategyAPISecurity HuntStrategy = "api_security" + HuntStrategyPythonSpecific HuntStrategy = "python_specific" + HuntStrategyJavascriptSpecific HuntStrategy = "javascript_specific" +) + +// AllHuntStrategies lists every distinct HuntStrategy value in Python +// declaration order — i.e. `[s.value for s in HuntStrategy]`, which excludes +// the LOGIC_BUGS alias. +var AllHuntStrategies = []HuntStrategy{ + HuntStrategyInjection, HuntStrategyXSS, HuntStrategyDos, HuntStrategySSRF, + HuntStrategyAuth, HuntStrategyCrypto, HuntStrategyBusinessLogic, + HuntStrategyDataExposure, HuntStrategySupplyChain, HuntStrategyConfigSecrets, + HuntStrategyAPISecurity, HuntStrategyPythonSpecific, HuntStrategyJavascriptSpecific, +} + +// Valid reports whether h is one of the declared members. +func (h HuntStrategy) Valid() bool { + for _, v := range AllHuntStrategies { + if h == v { + return true + } + } + return false +} + +// ParseHuntStrategy is the Go equivalent of `HuntStrategy(s)`. +func ParseHuntStrategy(s string) (HuntStrategy, error) { + v := HuntStrategy(s) + if !v.Valid() { + return "", &EnumValueError{Enum: "HuntStrategy", Value: s} + } + return v, nil +} + +// EnumValueError is what the ParseX helpers return for an unknown value. It +// stands in for the `ValueError: 'x' is not a valid Severity` pydantic/Enum +// raises. +type EnumValueError struct { + Enum string + Value string +} + +func (e *EnumValueError) Error() string { + return "'" + e.Value + "' is not a valid " + e.Enum +} + +// VulnLocation is the flat schema for hunt Step 1 (location scanning). +// +// Ports schemas/hunt.py VulnLocation. +type VulnLocation struct { + FilePath string `json:"file_path"` + StartLine int `json:"start_line"` + CodeSnippet string `json:"code_snippet"` + // PatternType e.g. 'sql_injection', 'command_injection'. + PatternType string `json:"pattern_type"` +} + +// EnrichedFinding is the flat schema for hunt Step 2 (finding enrichment). +// +// Ports schemas/hunt.py EnrichedFinding. Severity/Confidence are plain strings +// here (not the enums) because this is a raw harness output schema. +type EnrichedFinding struct { + Title string `json:"title"` + Description string `json:"description"` + CweID string `json:"cwe_id"` + // Severity is one of: "critical", "high", "medium", "low", "info". + Severity string `json:"severity"` + // Confidence is one of: "high", "medium", "low". + Confidence string `json:"confidence"` + // DataFlowSummary is a natural-language summary (string, not nested). + DataFlowSummary string `json:"data_flow_summary"` +} + +// ScanLocationsResult is the container for hunt Step 1 results. +// +// Ports schemas/hunt.py ScanLocationsResult. Seeded (defaults.go): +// locations `[]`. +type ScanLocationsResult struct { + Locations []VulnLocation `json:"locations"` +} + +// RawFinding is a potential vulnerability produced by a hunter +// (DESIGN.md §5.4). +// +// Ports schemas/hunt.py RawFinding. Seeded (defaults.go): related_files `[]`, +// and NewRawFinding mints ID and Fingerprint as fresh uuid4 strings (Python's +// `default_factory=lambda: str(uuid4())` on BOTH fields — the fingerprint is a +// random uuid by default, NOT a content hash; agents/hunt overwrites it with a +// real fingerprint before dedup). +type RawFinding struct { + ID string `json:"id"` + HunterStrategy string `json:"hunter_strategy"` + Title string `json:"title"` + Description string `json:"description"` + FindingType FindingType `json:"finding_type"` + CweID string `json:"cwe_id"` + CweName string `json:"cwe_name"` + OwaspCategory *string `json:"owasp_category"` + FilePath string `json:"file_path"` + StartLine int `json:"start_line"` + EndLine int `json:"end_line"` + FunctionName *string `json:"function_name"` + CodeSnippet string `json:"code_snippet"` + EstimatedSeverity Severity `json:"estimated_severity"` + Confidence Confidence `json:"confidence"` + DataFlow []ReconDataFlowStep `json:"data_flow"` + RelatedFiles []string `json:"related_files"` + Fingerprint string `json:"fingerprint"` +} + +// ForVerifier projects the finding onto what the verifier pipeline needs. +// +// Ports schemas/hunt.py RawFinding.for_verifier(). Python parity: the +// data_flow summary joins `f"{step.file_path}:{step.line} {step.operation}"` +// with "\n", and is "" when data_flow is empty or None — note Python tests +// `if self.data_flow:`, so an EMPTY list yields "" just like None does. +func (r RawFinding) ForVerifier() FindingForVerifier { + summary := "" + if len(r.DataFlow) > 0 { + parts := make([]string, 0, len(r.DataFlow)) + for _, step := range r.DataFlow { + parts = append(parts, step.FilePath+":"+strconv.Itoa(step.Line)+" "+step.Operation) + } + summary = strings.Join(parts, "\n") + } + return FindingForVerifier{ + ID: r.ID, + Title: r.Title, + Description: r.Description, + FilePath: r.FilePath, + StartLine: r.StartLine, + EndLine: r.EndLine, + CodeSnippet: r.CodeSnippet, + CweID: r.CweID, + FunctionName: r.FunctionName, + DataFlowSummary: summary, + } +} + +// ForDedup projects the finding onto what the deduplicator needs. +// +// Ports schemas/hunt.py RawFinding.for_dedup(). Python passes +// `self.finding_type.value` / `self.estimated_severity.value`; the Go enums +// already ARE their values. +func (r RawFinding) ForDedup() FindingForDedup { + return FindingForDedup{ + ID: r.ID, + Fingerprint: r.Fingerprint, + Title: r.Title, + FilePath: r.FilePath, + StartLine: r.StartLine, + CweID: r.CweID, + FindingType: string(r.FindingType), + EstimatedSeverity: string(r.EstimatedSeverity), + } +} + +// PotentialChain is a potential multi-step attack chain before proof +// (DESIGN.md §5.5). +// +// Ports schemas/hunt.py PotentialChain. Seeded (defaults.go): finding_ids `[]` +// and NewPotentialChain mints ChainID as a fresh uuid4. +type PotentialChain struct { + ChainID string `json:"chain_id"` + Title string `json:"title"` + FindingIDs []string `json:"finding_ids"` + CombinedImpact string `json:"combined_impact"` + EstimatedSeverity Severity `json:"estimated_severity"` +} + +// HuntResult is the deduplicated and correlated hunt output (DESIGN.md §5.5). +// +// Ports schemas/hunt.py HuntResult. Seeded (defaults.go): findings, chains and +// strategies_run `[]`; the counters and duration default to the Go zero value. +type HuntResult struct { + Findings []RawFinding `json:"findings"` + Chains []PotentialChain `json:"chains"` + TotalRaw int `json:"total_raw"` + DeduplicatedCount int `json:"deduplicated_count"` + ChainCount int `json:"chain_count"` + StrategiesRun []string `json:"strategies_run"` + HuntDurationSeconds float64 `json:"hunt_duration_seconds"` +} + +// DeduplicatedResult is the dedup lane output before PROVE prioritization +// (DESIGN.md §5.5). +// +// Ports schemas/hunt.py DeduplicatedResult. Seeded (defaults.go): findings and +// chains `[]`. +type DeduplicatedResult struct { + Findings []RawFinding `json:"findings"` + Chains []PotentialChain `json:"chains"` + DroppedDuplicates int `json:"dropped_duplicates"` + KeptFindings int `json:"kept_findings"` +} + +// ChainCorrelationResult is the flat harness schema for chain correlation: +// the LLM identifies chains only. +// +// Ports schemas/hunt.py ChainCorrelationResult. Seeded (defaults.go): both +// list fields `[]`. +type ChainCorrelationResult struct { + // Chains: one entry per multi-step attack chain, formatted + // "title | finding_id1,finding_id2,... | combined_impact | severity". + Chains []string `json:"chains"` + // DuplicateIDs are finding IDs the programmatic dedup missed (to drop). + DuplicateIDs []string `json:"duplicate_ids"` +} diff --git a/internal/secaf/schemas/input.go b/internal/secaf/schemas/input.go new file mode 100644 index 0000000000..b5f8ab2893 --- /dev/null +++ b/internal/secaf/schemas/input.go @@ -0,0 +1,89 @@ +package schemas + +// This file ports src/sec_af/schemas/input.py — the REST API input contract +// for `sec-af.audit` (DESIGN.md §8.2). + +// AuditInput is the input for a `sec-af.audit` execution (DESIGN.md §8.2). +// +// Ports schemas/input.py AuditInput. Seeded (defaults.go): branch="main", +// depth="standard", severity_threshold="low", +// scan_types=["sast","sca","secrets","config"], output_formats=["json"], +// exclude_paths=["tests/","vendor/","node_modules/",".git/"], and +// compliance_frameworks / repo_urls / custom_policies `[]`. +// +// `repo_url` is the only REQUIRED field — binding a payload without it must +// fail (ported from tests/test_schemas.py::test_schema_validation_and_required_fields); +// see Validate. +type AuditInput struct { + // RepoURL is the git repository URL to audit. REQUIRED. + RepoURL string `json:"repo_url"` + // Branch to audit. + Branch string `json:"branch"` + // CommitSha to audit. + CommitSha *string `json:"commit_sha"` + // BaseCommitSha is the base commit for diff-aware PR scanning. + BaseCommitSha *string `json:"base_commit_sha"` + // Depth is the scan depth profile: quick|standard|thorough. + Depth string `json:"depth"` + // SeverityThreshold is the minimum severity to report: + // critical|high|medium|low|info. + SeverityThreshold string `json:"severity_threshold"` + ScanTypes []string `json:"scan_types"` + OutputFormats []string `json:"output_formats"` + ComplianceFrameworks []string `json:"compliance_frameworks"` + // MaxCostUsd is the budget cap in USD. + MaxCostUsd *float64 `json:"max_cost_usd"` + // MaxProvers caps parallel provers. + MaxProvers *int `json:"max_provers"` + // MaxDurationSeconds caps execution time. + MaxDurationSeconds *int `json:"max_duration_seconds"` + // IncludePaths restricts the scan to these repository paths. + IncludePaths []string `json:"include_paths"` + ExcludePaths []string `json:"exclude_paths"` + // IsPr reports whether the scan is for a pull request. + IsPr bool `json:"is_pr"` + // PrID is the pull request identifier. + PrID *string `json:"pr_id"` + // PostPrComments posts findings as PR comments. + PostPrComments bool `json:"post_pr_comments"` + // FailOnFindings returns a non-zero status for CI gating. + FailOnFindings bool `json:"fail_on_findings"` + // DastEnabled turns on DAST-like runtime exploit verification (sandbox). + DastEnabled bool `json:"dast_enabled"` + // RepoUrls are additional repository URLs for cross-service analysis. + RepoUrls []string `json:"repo_urls"` + // MonitoringMode enables continuous monitoring (compare against baseline). + MonitoringMode bool `json:"monitoring_mode"` + // BaselinePath points at baseline scan results for regression detection. + BaselinePath *string `json:"baseline_path"` + // CustomPolicies are org-specific security policy rules to evaluate, + // e.g. 'All endpoints must require authentication'. + CustomPolicies []string `json:"custom_policies"` +} + +// Validate reproduces the one required-field constraint pydantic enforces on +// AuditInput: `repo_url` has no default, so `AuditInput(branch="main")` raises +// ValidationError. Go's json.Unmarshal cannot express "required", so the +// binding call site calls Validate. +// +// Python parity: this is the ONLY constraint on AuditInput — the schema +// declares no ge/le bounds and no Literal, so depth / severity_threshold / +// scan_types are free-form strings validated (or not) downstream, exactly as +// in Python. +func (a AuditInput) Validate() error { + if a.RepoURL == "" { + return &MissingFieldError{Model: "AuditInput", Field: "repo_url"} + } + return nil +} + +// MissingFieldError stands in for pydantic's ValidationError on a missing +// required field. +type MissingFieldError struct { + Model string + Field string +} + +func (e *MissingFieldError) Error() string { + return e.Model + ": field required: " + e.Field +} diff --git a/internal/secaf/schemas/model_keys_test.go b/internal/secaf/schemas/model_keys_test.go new file mode 100644 index 0000000000..b2a0fb3c77 --- /dev/null +++ b/internal/secaf/schemas/model_keys_test.go @@ -0,0 +1,502 @@ +package schemas + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "reflect" + "regexp" + "sort" + "testing" +) + +// This file is the exhaustive schema parity gate. testdata/model_keys.json is +// generated from the LIVE pydantic models by go/scripts/gen_model_keys.py +// (regenerate with: +// +// PYTHONPATH=$PWD/src ~/.agentfield/packages/sec-af/venv/bin/python \ +// go/scripts/gen_model_keys.py +// +// from the repo root). For every model it records model_dump()'s ordered key +// list, which fields are required, and what a minimally-constructed instance +// serialises each non-required field to. The tests below assert the Go structs +// reproduce all of it, so a pydantic field that is added, renamed, reordered or +// re-defaulted in Python fails the Go build's test step instead of silently +// diverging on the wire. + +// --------------------------------------------------------------------------- +// The generated ground truth +// --------------------------------------------------------------------------- + +type modelSpec struct { + PythonModule string `json:"python_module"` + PythonClass string `json:"python_class"` + GoName string `json:"go_name"` + DuplicateOf *string `json:"duplicate_of"` + Keys []string `json:"keys"` + Required []string `json:"required"` + NullFields []string `json:"null_fields"` + EmptyListFields []string `json:"empty_list_fields"` + EmptyDictFields []string `json:"empty_dict_fields"` + ScalarDefaults map[string]any `json:"scalar_defaults"` + UUIDDefaults []string `json:"uuid_defaults"` +} + +type enumSpec struct { + PythonModule string `json:"python_module"` + Members map[string]any `json:"members"` + Kind string `json:"kind"` +} + +type groundTruth struct { + Models []modelSpec `json:"models"` + Enums map[string]enumSpec `json:"enums"` + SchemasAll []string `json:"schemas_all"` +} + +func loadGroundTruth(t *testing.T) groundTruth { + t.Helper() + raw, err := os.ReadFile("testdata/model_keys.json") + if err != nil { + t.Fatalf("read testdata/model_keys.json: %v", err) + } + var gt groundTruth + if err := json.Unmarshal(raw, >); err != nil { + t.Fatalf("decode testdata/model_keys.json: %v", err) + } + if len(gt.Models) == 0 { + t.Fatal("testdata/model_keys.json has no models") + } + return gt +} + +// --------------------------------------------------------------------------- +// The Go side: one default-constructed value per ported model. +// +// The value is NewX() where the model has pydantic defaults (defaults.go) and +// the zero value otherwise — i.e. exactly what Go code gets when it builds the +// struct the intended way. That is what the parity assertions run against, so +// a missing constructor shows up as a `null` where Python emits `[]`. +// --------------------------------------------------------------------------- + +func goModelRegistry() map[string]any { + return map[string]any{ + // --- compliance.go --- + "ComplianceMapping": ComplianceMapping{}, + "ComplianceGap": ComplianceGap{}, + // --- gates.go --- + "SeverityClassification": SeverityClassification{}, + "DuplicateCheck": DuplicateCheck{}, + "StrategySelection": StrategySelection{}, + "CWEExpansion": CWEExpansion{}, + "RelevanceGate": RelevanceGate{}, + "VerdictGate": VerdictGate{}, + "ComplianceSuggestion": ComplianceSuggestion{}, + "ComplianceGate": ComplianceGate{}, + "ReachabilityGate": ReachabilityGate{}, + // --- hunt.go --- + "VulnLocation": VulnLocation{}, + "EnrichedFinding": EnrichedFinding{}, + "ScanLocationsResult": NewScanLocationsResult(), + "RawFinding": NewRawFinding(), + "PotentialChain": NewPotentialChain(), + "HuntResult": NewHuntResult(), + "DeduplicatedResult": NewDeduplicatedResult(), + "ChainCorrelationResult": NewChainCorrelationResult(), + // --- input.go --- + "AuditInput": NewAuditInput(), + // --- output.go --- + "Location": Location{}, + "CvssV4Score": CvssV4Score{}, + "EpssScore": EpssScore{}, + "MitreMapping": MitreMapping{}, + "AttackChain": NewAttackChain(), + "ReproductionStep": ReproductionStep{}, + "ServiceDefinition": NewServiceDefinition(), + "CrossServiceFinding": CrossServiceFinding{}, + "RegressionFinding": RegressionFinding{}, + "MonitoringResult": NewMonitoringResult(), + "PolicyViolation": NewPolicyViolation(), + "SecurityAuditResult": NewSecurityAuditResult(), + "AuditProgress": AuditProgress{}, + "AuditMetrics": NewAuditMetrics(), + // --- prove.go --- + "DataFlowTrace": NewDataFlowTrace(), + "ReachabilityProof": NewReachabilityProof(), + "SanitizationResult": SanitizationResult{}, + "ExploitHypothesis": ExploitHypothesis{}, + "DastVerificationResult": DastVerificationResult{}, + "VerdictDecision": VerdictDecision{}, + "RemediationSuggestion": RemediationSuggestion{}, + "DataFlowStep": DataFlowStep{}, + "DataFlowEvidence": NewDataFlowEvidence(), + "SanitizationAnalysis": SanitizationAnalysis{}, + "HttpEvidence": HttpEvidence{}, + "ReachabilityEvidence": NewReachabilityEvidence(), + "ChainStep": ChainStep{}, + "Proof": Proof{}, + "ProverSignal": ProverSignal{}, + "VerifiedFinding": NewVerifiedFinding(), + // --- recon.go --- + "Module": NewModule(), + "EntryPoint": EntryPoint{}, + "TrustBoundary": NewTrustBoundary(), + "Service": Service{}, + "APIEndpoint": APIEndpoint{}, + "ArchitectureMap": NewArchitectureMap(), + "ReconDataFlowStep": ReconDataFlowStep{}, + "SanitizationPoint": NewSanitizationPoint(), + "Sink": Sink{}, + "DataFlow": NewDataFlow(), + "DataFlowMap": NewDataFlowMap(), + "Dependency": Dependency{}, + "KnownCVE": KnownCVE{}, + "OutdatedDep": OutdatedDep{}, + "DependencyReport": NewDependencyReport(), + "SecretFinding": NewSecretFinding(), + "MisconfigFinding": NewMisconfigFinding(), + "ConfigReport": NewConfigReport(), + "CryptoUsage": CryptoUsage{}, + "SecurityContext": NewSecurityContext(), + "ReconResult": NewReconResult(), + "ArchitectureMapRaw": NewArchitectureMapRaw(), + "DataFlowMapRaw": NewDataFlowMapRaw(), + "DependencyReportRaw": NewDependencyReportRaw(), + "ConfigReportRaw": NewConfigReportRaw(), + "SecurityContextRaw": NewSecurityContextRaw(), + // --- views.go --- + "FindingForVerifier": FindingForVerifier{}, + "FindingForDedup": FindingForDedup{}, + "FindingForReachability": FindingForReachability{}, + // --- policies.go --- + "PolicyEvalResult": PolicyEvalResult{}, + } +} + +// orderedKeys returns the top-level object keys of b in the order they appear. +// encoding/json emits struct fields in declaration order, so this exposes the +// Go field order for comparison with pydantic's model_dump() key order. +func orderedKeys(t *testing.T, b []byte) []string { + t.Helper() + dec := json.NewDecoder(bytes.NewReader(b)) + tok, err := dec.Token() + if err != nil { + t.Fatalf("read opening token: %v", err) + } + if d, ok := tok.(json.Delim); !ok || d != '{' { + t.Fatalf("expected a JSON object, got %v", tok) + } + var keys []string + depth := 0 + for dec.More() || depth > 0 { + tok, err := dec.Token() + if err != nil { + t.Fatalf("read token: %v", err) + } + if d, ok := tok.(json.Delim); ok { + switch d { + case '{', '[': + depth++ + case '}', ']': + depth-- + } + continue + } + if depth == 0 { + key, ok := tok.(string) + if !ok { + t.Fatalf("expected an object key, got %T", tok) + } + keys = append(keys, key) + // Skip this key's value wholesale. + var v json.RawMessage + if err := dec.Decode(&v); err != nil { + t.Fatalf("skip value of %q: %v", key, err) + } + } + } + return keys +} + +// asAny re-decodes b into map[string]any so values can be compared with the +// generated JSON on equal footing (both sides get float64 for every number). +func asAny(t *testing.T, b []byte) map[string]any { + t.Helper() + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + t.Fatalf("decode: %v", err) + } + return m +} + +// --------------------------------------------------------------------------- +// V1 — every Python model has a Go struct, and vice versa. +// --------------------------------------------------------------------------- + +func TestModelRegistryCoversEveryPydanticModel(t *testing.T) { + gt := loadGroundTruth(t) + registry := goModelRegistry() + + seen := map[string]bool{} + for _, spec := range gt.Models { + seen[spec.GoName] = true + if _, ok := registry[spec.GoName]; !ok { + t.Errorf("pydantic model %s.%s has no Go struct %q in the registry", + spec.PythonModule, spec.PythonClass, spec.GoName) + } + } + for name := range registry { + if !seen[name] { + t.Errorf("Go struct %q is in the registry but no pydantic model maps to it", name) + } + } + if len(registry) != 80 { + t.Errorf("registry has %d models, want 80 (update this count deliberately)", len(registry)) + } +} + +// --------------------------------------------------------------------------- +// V2 — marshaling a Go default value reproduces model_dump()'s ORDERED key +// list. This is the field-name, field-count and field-order gate in one. +// --------------------------------------------------------------------------- + +func TestModelKeysMatchPydanticDump(t *testing.T) { + gt := loadGroundTruth(t) + registry := goModelRegistry() + + for _, spec := range gt.Models { + spec := spec + t.Run(spec.PythonModule+"."+spec.PythonClass, func(t *testing.T) { + value, ok := registry[spec.GoName] + if !ok { + t.Skipf("no Go struct for %s (reported by TestModelRegistryCoversEveryPydanticModel)", spec.GoName) + } + b, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal %s: %v", spec.GoName, err) + } + got := orderedKeys(t, b) + if !reflect.DeepEqual(got, spec.Keys) { + t.Errorf("%s json keys\n got: %v\nwant: %v", spec.GoName, got, spec.Keys) + } + }) + } +} + +// --------------------------------------------------------------------------- +// V3 — every non-required field serialises to what pydantic's default +// serialises to: null for Optional, [] for default_factory=list, {} for +// default_factory=dict, and the literal value for a non-zero scalar default. +// This is what proves "NO omitempty", "Optional -> pointer" and the +// defaults.go seeding are all correct at once. +// --------------------------------------------------------------------------- + +func TestModelDefaultsMatchPydanticDefaults(t *testing.T) { + gt := loadGroundTruth(t) + registry := goModelRegistry() + uuidRe := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`) + + for _, spec := range gt.Models { + spec := spec + t.Run(spec.PythonModule+"."+spec.PythonClass, func(t *testing.T) { + value, ok := registry[spec.GoName] + if !ok { + t.Skipf("no Go struct for %s", spec.GoName) + } + b, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal %s: %v", spec.GoName, err) + } + got := asAny(t, b) + + for _, field := range spec.NullFields { + if v, present := got[field]; !present || v != nil { + t.Errorf("%s.%s = %#v, want null (pydantic Optional default None)", spec.GoName, field, v) + } + } + for _, field := range spec.EmptyListFields { + v, present := got[field] + if !present { + t.Errorf("%s.%s missing", spec.GoName, field) + continue + } + list, isList := v.([]any) + if !isList || len(list) != 0 { + t.Errorf("%s.%s = %#v, want [] (pydantic default_factory=list; a nil Go slice marshals to null)", + spec.GoName, field, v) + } + } + for _, field := range spec.EmptyDictFields { + v, present := got[field] + if !present { + t.Errorf("%s.%s missing", spec.GoName, field) + continue + } + obj, isObj := v.(map[string]any) + if !isObj || len(obj) != 0 { + t.Errorf("%s.%s = %#v, want {} (pydantic default_factory=dict)", spec.GoName, field, v) + } + } + for field, want := range spec.ScalarDefaults { + if !reflect.DeepEqual(got[field], want) { + t.Errorf("%s.%s = %#v, want %#v (pydantic default)", spec.GoName, field, got[field], want) + } + } + for _, field := range spec.UUIDDefaults { + s, isStr := got[field].(string) + if !isStr || !uuidRe.MatchString(s) { + t.Errorf("%s.%s = %#v, want an RFC 4122 v4 uuid string (pydantic default_factory=lambda: str(uuid4()))", + spec.GoName, field, got[field]) + } + } + }) + } +} + +// --------------------------------------------------------------------------- +// V4 — the four models prove.py re-declares byte-identically to output.py +// really are identical, which is what licenses the Go package to keep one +// struct per name. +// --------------------------------------------------------------------------- + +func TestDuplicateDeclarationsAreIdentical(t *testing.T) { + gt := loadGroundTruth(t) + byName := map[string]modelSpec{} + for _, spec := range gt.Models { + if spec.DuplicateOf == nil { + byName[spec.GoName] = spec + } + } + + dupes := 0 + for _, spec := range gt.Models { + if spec.DuplicateOf == nil { + continue + } + dupes++ + canonical, ok := byName[*spec.DuplicateOf] + if !ok { + t.Fatalf("%s.%s says it duplicates %q, which is not declared anywhere", + spec.PythonModule, spec.PythonClass, *spec.DuplicateOf) + } + if !reflect.DeepEqual(spec.Keys, canonical.Keys) { + t.Errorf("%s.%s keys %v != %s.%s keys %v — the Go package cannot keep one struct for both", + spec.PythonModule, spec.PythonClass, spec.Keys, + canonical.PythonModule, canonical.PythonClass, canonical.Keys) + } + if !reflect.DeepEqual(spec.NullFields, canonical.NullFields) { + t.Errorf("%s.%s null fields %v != %s null fields %v", + spec.PythonModule, spec.PythonClass, spec.NullFields, canonical.GoName, canonical.NullFields) + } + if !reflect.DeepEqual(spec.Required, canonical.Required) { + t.Errorf("%s.%s required %v != %s required %v", + spec.PythonModule, spec.PythonClass, spec.Required, canonical.GoName, canonical.Required) + } + } + if dupes != 4 { + t.Errorf("found %d duplicate declarations, want 4 (Location, CvssV4Score, EpssScore, ReproductionStep)", dupes) + } +} + +// --------------------------------------------------------------------------- +// V5 — enum members and values match Python exactly, aliases included. +// --------------------------------------------------------------------------- + +func TestEnumMembersMatchPython(t *testing.T) { + gt := loadGroundTruth(t) + + goEnums := map[string]map[string]any{ + "FindingType": { + "SAST": string(FindingTypeSast), "SCA": string(FindingTypeSca), + "SECRETS": string(FindingTypeSecrets), "CONFIG": string(FindingTypeConfig), + "LOGIC": string(FindingTypeLogic), "API": string(FindingTypeAPI), + }, + "Severity": { + "CRITICAL": string(SeverityCritical), "HIGH": string(SeverityHigh), + "MEDIUM": string(SeverityMedium), "LOW": string(SeverityLow), + "INFO": string(SeverityInfo), + }, + "Confidence": { + "HIGH": string(ConfidenceHigh), "MEDIUM": string(ConfidenceMedium), + "LOW": string(ConfidenceLow), + }, + "HuntStrategy": { + "INJECTION": string(HuntStrategyInjection), "XSS": string(HuntStrategyXSS), + "DOS": string(HuntStrategyDos), "SSRF": string(HuntStrategySSRF), + "AUTH": string(HuntStrategyAuth), "CRYPTO": string(HuntStrategyCrypto), + "BUSINESS_LOGIC": string(HuntStrategyBusinessLogic), + "LOGIC_BUGS": string(HuntStrategyLogicBugs), + "DATA_EXPOSURE": string(HuntStrategyDataExposure), + "SUPPLY_CHAIN": string(HuntStrategySupplyChain), + "CONFIG_SECRETS": string(HuntStrategyConfigSecrets), + "API_SECURITY": string(HuntStrategyAPISecurity), + "PYTHON_SPECIFIC": string(HuntStrategyPythonSpecific), + "JAVASCRIPT_SPECIFIC": string(HuntStrategyJavascriptSpecific), + }, + "Verdict": { + "CONFIRMED": string(VerdictConfirmed), "LIKELY": string(VerdictLikely), + "INCONCLUSIVE": string(VerdictInconclusive), "NOT_EXPLOITABLE": string(VerdictNotExploitable), + }, + "EvidenceLevel": { + "STATIC_MATCH": float64(EvidenceLevelStaticMatch), + "FLOW_IDENTIFIED": float64(EvidenceLevelFlowIdentified), + "REACHABILITY_CONFIRMED": float64(EvidenceLevelReachabilityConfirmed), + "SANITIZATION_BYPASSABLE": float64(EvidenceLevelSanitizationBypassable), + "EXPLOIT_SCENARIO_VALIDATED": float64(EvidenceLevelExploitScenarioValidated), + "FULL_EXPLOIT": float64(EvidenceLevelFullExploit), + }, + } + + if len(goEnums) != len(gt.Enums) { + t.Errorf("Go declares %d enums, Python has %d", len(goEnums), len(gt.Enums)) + } + for name, spec := range gt.Enums { + got, ok := goEnums[name] + if !ok { + t.Errorf("python enum %s has no Go counterpart", name) + continue + } + if !reflect.DeepEqual(got, spec.Members) { + t.Errorf("enum %s members\n got: %v\nwant: %v", name, sortedPairs(got), sortedPairs(spec.Members)) + } + } +} + +func sortedPairs(m map[string]any) []string { + out := make([]string, 0, len(m)) + for k, v := range m { + out = append(out, fmt.Sprintf("%s=%v", k, v)) + } + sort.Strings(out) + return out +} + +// --------------------------------------------------------------------------- +// V6 — every name sec_af.schemas.__init__.__all__ re-exports exists in the Go +// package under the SAME name. This is the cross-agent contract harnessx +// relies on to resolve testdata/schemas/.json. +// --------------------------------------------------------------------------- + +func TestSchemasInitExportsHaveGoTypes(t *testing.T) { + gt := loadGroundTruth(t) + if len(gt.SchemasAll) != 62 { + t.Errorf("sec_af.schemas.__all__ has %d names, want 62 (update deliberately)", len(gt.SchemasAll)) + } + + known := map[string]bool{} + for name := range goModelRegistry() { + known[name] = true + } + // The enums are types too, and __all__ re-exports them. + for _, name := range []string{"FindingType", "Severity", "Confidence", "HuntStrategy", "Verdict", "EvidenceLevel"} { + known[name] = true + } + // Python parity: __init__.py aliases recon.DataFlowStep as + // ReconDataFlowStep, which the registry already carries under that name. + for _, name := range gt.SchemasAll { + if !known[name] { + t.Errorf("sec_af.schemas.__all__ exports %q with no Go type of that name", name) + } + } +} diff --git a/internal/secaf/schemas/output.go b/internal/secaf/schemas/output.go new file mode 100644 index 0000000000..ce0d94ce96 --- /dev/null +++ b/internal/secaf/schemas/output.go @@ -0,0 +1,206 @@ +package schemas + +// This file ports src/sec_af/schemas/output.py — the output and orchestration +// payloads (DESIGN.md §7, §12.3). +// +// Location, CvssV4Score, EpssScore and ReproductionStep are declared here even +// though prove.py declares byte-identical copies: schemas/__init__.py +// re-exports output.py's, and the Go package needs exactly one struct per name +// (harnessx resolves the embedded pydantic schema fixture by Go type name). + +// Location is source-location metadata for a finding reference +// (DESIGN.md §7.1). +// +// Ports schemas/output.py Location (== schemas/prove.py Location). +type Location struct { + FilePath string `json:"file_path"` + StartLine int `json:"start_line"` + EndLine int `json:"end_line"` + StartColumn *int `json:"start_column"` + EndColumn *int `json:"end_column"` + FunctionName *string `json:"function_name"` + CodeSnippet *string `json:"code_snippet"` +} + +// CvssV4Score holds CVSS v4 scoring details (DESIGN.md §7.1). +// +// Ports schemas/output.py CvssV4Score (== schemas/prove.py CvssV4Score). +type CvssV4Score struct { + Vector string `json:"vector"` + BaseScore float64 `json:"base_score"` + Severity string `json:"severity"` + Automatable bool `json:"automatable"` + SubsequentImpact bool `json:"subsequent_impact"` +} + +// EpssScore holds EPSS probability details (DESIGN.md §7.1). +// +// Ports schemas/output.py EpssScore (== schemas/prove.py EpssScore). +type EpssScore struct { + Score float64 `json:"score"` + Percentile float64 `json:"percentile"` + Date string `json:"date"` +} + +// MitreMapping is a MITRE ATT&CK mapping for an attack chain +// (DESIGN.md §7.2). +// +// Ports schemas/output.py MitreMapping. +type MitreMapping struct { + Tactic string `json:"tactic"` + TechniqueID string `json:"technique_id"` + TechniqueName string `json:"technique_name"` +} + +// AttackChain is a verified multi-step exploit chain (DESIGN.md §7.2). +// +// Ports schemas/output.py AttackChain. Seeded (defaults.go): findings `[]`. +// `mitre_attack_mapping` is `list[MitreMapping] | None` with NO +// default_factory, so a nil Go slice marshaling to null is exactly right. +type AttackChain struct { + ChainID string `json:"chain_id"` + Title string `json:"title"` + Description string `json:"description"` + Findings []string `json:"findings"` + CombinedSeverity Severity `json:"combined_severity"` + CombinedImpact string `json:"combined_impact"` + MitreAttackMapping []MitreMapping `json:"mitre_attack_mapping"` +} + +// ReproductionStep is one reproduction instruction for analysts +// (DESIGN.md §7.1). +// +// Ports schemas/output.py ReproductionStep (== schemas/prove.py +// ReproductionStep). +type ReproductionStep struct { + Step int `json:"step"` + Description string `json:"description"` + Command *string `json:"command"` + ExpectedOutput *string `json:"expected_output"` +} + +// ServiceDefinition is a service node in a multi-repo architecture. +// +// Ports schemas/output.py ServiceDefinition. Seeded (defaults.go): +// api_endpoints and dependencies `[]`. +type ServiceDefinition struct { + Name string `json:"name"` + RepoURL string `json:"repo_url"` + APIEndpoints []string `json:"api_endpoints"` + // Dependencies are the names of services this one depends on. + Dependencies []string `json:"dependencies"` +} + +// CrossServiceFinding is the flat schema for cross-service attack chain +// analysis. +// +// Ports schemas/output.py CrossServiceFinding. +type CrossServiceFinding struct { + ChainDescription string `json:"chain_description"` + ServicesInvolved []string `json:"services_involved"` + EntryPoint string `json:"entry_point"` + Impact string `json:"impact"` +} + +// RegressionFinding is a finding that appeared (or disappeared) since the +// baseline scan. +// +// Ports schemas/output.py RegressionFinding. +type RegressionFinding struct { + FindingTitle string `json:"finding_title"` + FindingID string `json:"finding_id"` + Severity string `json:"severity"` + CweID string `json:"cwe_id"` + // Status is one of: "new", "fixed", "unchanged". + Status string `json:"status"` +} + +// MonitoringResult is the result of comparing the current scan against a +// baseline. +// +// Ports schemas/output.py MonitoringResult. Seeded (defaults.go): +// new_findings and fixed_findings `[]`. +type MonitoringResult struct { + BaselineCommit string `json:"baseline_commit"` + CurrentCommit string `json:"current_commit"` + NewFindings []RegressionFinding `json:"new_findings"` + FixedFindings []RegressionFinding `json:"fixed_findings"` + UnchangedCount int `json:"unchanged_count"` + RegressionDetected bool `json:"regression_detected"` +} + +// PolicyViolation is a violation of an org-specific security policy. +// +// Ports schemas/output.py PolicyViolation. Seeded (defaults.go): +// severity="medium". +type PolicyViolation struct { + // Policy is the policy rule that was violated. + Policy string `json:"policy"` + // ViolationDescription explains how the code violates the policy. + ViolationDescription string `json:"violation_description"` + FilePath string `json:"file_path"` + Severity string `json:"severity"` +} + +// SecurityAuditResult is the top-level SEC-AF audit output (DESIGN.md §7.3). +// +// Ports schemas/output.py SecurityAuditResult. Seeded (defaults.go): the five +// list fields `[]`, the three dict fields `{}`. +// +// Timestamp is a pydantic `datetime`; see timestamp.go for the exact wire +// format (`datetime.isoformat()` via FastAPI's jsonable_encoder). +type SecurityAuditResult struct { + Repository string `json:"repository"` + CommitSha string `json:"commit_sha"` + Branch *string `json:"branch"` + Timestamp Timestamp `json:"timestamp"` + DepthProfile string `json:"depth_profile"` + StrategiesUsed []string `json:"strategies_used"` + Provider string `json:"provider"` + Findings []VerifiedFinding `json:"findings"` + AttackChains []AttackChain `json:"attack_chains"` + TotalRawFindings int `json:"total_raw_findings"` + Confirmed int `json:"confirmed"` + Likely int `json:"likely"` + Inconclusive int `json:"inconclusive"` + NotExploitable int `json:"not_exploitable"` + NoiseReductionPct float64 `json:"noise_reduction_pct"` + BySeverity map[string]int `json:"by_severity"` + ComplianceGaps []ComplianceGap `json:"compliance_gaps"` + PolicyViolations []PolicyViolation `json:"policy_violations"` + DurationSeconds float64 `json:"duration_seconds"` + AgentInvocations int `json:"agent_invocations"` + CostUsd float64 `json:"cost_usd"` + CostBreakdown map[string]float64 `json:"cost_breakdown"` + Metadata map[string]any `json:"metadata"` + Sarif string `json:"sarif"` +} + +// AuditProgress is an orchestrator phase progress event (DESIGN.md §12.3). +// +// Ports schemas/output.py AuditProgress. Every field is required. +type AuditProgress struct { + Phase string `json:"phase"` + PhaseProgress float64 `json:"phase_progress"` + AgentsTotal int `json:"agents_total"` + AgentsCompleted int `json:"agents_completed"` + AgentsRunning int `json:"agents_running"` + FindingsSoFar int `json:"findings_so_far"` + ElapsedSeconds float64 `json:"elapsed_seconds"` + EstimatedRemainingSeconds float64 `json:"estimated_remaining_seconds"` + CostSoFarUsd float64 `json:"cost_so_far_usd"` +} + +// AuditMetrics holds run-level performance and budget metrics +// (DESIGN.md §7.3, §9.1). +// +// Ports schemas/output.py AuditMetrics. Seeded (defaults.go): +// cost_breakdown `{}`. +type AuditMetrics struct { + DurationSeconds float64 `json:"duration_seconds"` + AgentInvocations int `json:"agent_invocations"` + CostUsd float64 `json:"cost_usd"` + CostBreakdown map[string]float64 `json:"cost_breakdown"` + BudgetExhausted bool `json:"budget_exhausted"` + FindingsNotVerified int `json:"findings_not_verified"` +} diff --git a/internal/secaf/schemas/policies.go b/internal/secaf/schemas/policies.go new file mode 100644 index 0000000000..5afa4f38ee --- /dev/null +++ b/internal/secaf/schemas/policies.go @@ -0,0 +1,26 @@ +package schemas + +// This file ports the one pydantic model declared OUTSIDE src/sec_af/schemas/ +// that crosses a JSON boundary: PolicyEvalResult from src/sec_af/policies.py, +// which `evaluate_policy` hands to `app.harness(schema=PolicyEvalResult)`. +// It lives in this package so harnessx can resolve its embedded pydantic +// schema fixture by Go type name like every other harness schema; the rest of +// policies.py (prompt building, evaluate_policy) belongs to internal/policies. +// +// config.py's BaseModels (BudgetConfig, AuditConfig, AIIntegrationConfig) are +// deliberately NOT here — they are process configuration, never serialized +// across a reasoner boundary, and live in internal/config. + +// PolicyEvalResult is the flat schema for AI policy evaluation. 4 fields. +// +// Ports policies.py PolicyEvalResult. Every field is required. +type PolicyEvalResult struct { + // Violated reports whether the policy is violated. + Violated bool `json:"violated"` + // Description explains how the policy is violated, or is 'No violation'. + Description string `json:"description"` + // FilePath is the primary file where the violation occurs, or 'N/A'. + FilePath string `json:"file_path"` + // Severity is "high", "medium" or "low". + Severity string `json:"severity"` +} diff --git a/internal/secaf/schemas/prove.go b/internal/secaf/schemas/prove.go new file mode 100644 index 0000000000..589e72bffb --- /dev/null +++ b/internal/secaf/schemas/prove.go @@ -0,0 +1,640 @@ +package schemas + +import ( + "bytes" + "encoding/json" + "fmt" + "strconv" +) + +// This file ports src/sec_af/schemas/prove.py — the PROVE phase enums, the +// flat sub-agent harness/ai schemas, the evidence artifacts and VerifiedFinding +// (DESIGN.md §6.3-§6.4, §7.1). +// +// prove.py also re-declares Location, CvssV4Score, EpssScore and +// ReproductionStep byte-identically to output.py. Go keeps ONE struct each, +// declared in output.go (which is what schemas/__init__.py re-exports); the +// parity test proves the two Python declarations really do match. + +// ExploitVerdict is the exploitability verdict vocabulary (DESIGN.md §6.3). +// +// Ports schemas/prove.py Verdict (`class Verdict(str, Enum)`). The Go name +// differs from the pydantic one because codeaf keeps exactly one type called +// Verdict, in internal/taxonomy (a law in that package enforces it); this one +// is never a harness destination, so no schema fixture is keyed by its name. +type ExploitVerdict string + +// Verdict members, in Python declaration order. +const ( + VerdictConfirmed ExploitVerdict = "confirmed" + VerdictLikely ExploitVerdict = "likely" + VerdictInconclusive ExploitVerdict = "inconclusive" + VerdictNotExploitable ExploitVerdict = "not_exploitable" +) + +// AllVerdicts lists every Verdict in Python declaration order. +var AllVerdicts = []ExploitVerdict{ + VerdictConfirmed, VerdictLikely, VerdictInconclusive, VerdictNotExploitable, +} + +// Valid reports whether v is one of the declared members. +func (v ExploitVerdict) Valid() bool { + for _, m := range AllVerdicts { + if v == m { + return true + } + } + return false +} + +// ParseVerdict is the Go equivalent of `Verdict(s)`. +func ParseVerdict(s string) (ExploitVerdict, error) { + v := ExploitVerdict(s) + if !v.Valid() { + return "", &EnumValueError{Enum: "Verdict", Value: s} + } + return v, nil +} + +// EvidenceLevel is the six-level evidence strength hierarchy (DESIGN.md §6.3). +// +// Ports schemas/prove.py EvidenceLevel (`class EvidenceLevel(IntEnum)`), so it +// is an INT on the wire — `model_dump(mode="json")` emits 1..6, and +// `EvidenceLevel.FULL_EXPLOIT == 6` is True in Python. +type EvidenceLevel int + +// EvidenceLevel members, in Python declaration order (increasing strength). +const ( + EvidenceLevelStaticMatch EvidenceLevel = 1 + EvidenceLevelFlowIdentified EvidenceLevel = 2 + EvidenceLevelReachabilityConfirmed EvidenceLevel = 3 + EvidenceLevelSanitizationBypassable EvidenceLevel = 4 + EvidenceLevelExploitScenarioValidated EvidenceLevel = 5 + EvidenceLevelFullExploit EvidenceLevel = 6 +) + +// AllEvidenceLevels lists every EvidenceLevel in Python declaration order. +var AllEvidenceLevels = []EvidenceLevel{ + EvidenceLevelStaticMatch, EvidenceLevelFlowIdentified, + EvidenceLevelReachabilityConfirmed, EvidenceLevelSanitizationBypassable, + EvidenceLevelExploitScenarioValidated, EvidenceLevelFullExploit, +} + +// evidenceLevelNames maps each level to its Python member name. +var evidenceLevelNames = map[EvidenceLevel]string{ + EvidenceLevelStaticMatch: "STATIC_MATCH", + EvidenceLevelFlowIdentified: "FLOW_IDENTIFIED", + EvidenceLevelReachabilityConfirmed: "REACHABILITY_CONFIRMED", + EvidenceLevelSanitizationBypassable: "SANITIZATION_BYPASSABLE", + EvidenceLevelExploitScenarioValidated: "EXPLOIT_SCENARIO_VALIDATED", + EvidenceLevelFullExploit: "FULL_EXPLOIT", +} + +// Valid reports whether e is one of the six declared levels. +func (e EvidenceLevel) Valid() bool { _, ok := evidenceLevelNames[e]; return ok } + +// Name returns the Python member name ("FULL_EXPLOIT"), or "" for an +// undeclared value. +func (e EvidenceLevel) Name() string { return evidenceLevelNames[e] } + +// String renders the level the way Python 3.11 renders an IntEnum: as the +// decimal number, NOT as "EvidenceLevel.FULL_EXPLOIT" (3.11 gave IntEnum +// int.__str__). Use Name() for the symbolic form. +func (e EvidenceLevel) String() string { return strconv.Itoa(int(e)) } + +// ParseEvidenceLevel is the Go equivalent of `EvidenceLevel(i)`. +func ParseEvidenceLevel(i int) (EvidenceLevel, error) { + v := EvidenceLevel(i) + if !v.Valid() { + return 0, &EnumValueError{Enum: "EvidenceLevel", Value: strconv.Itoa(i)} + } + return v, nil +} + +// --------------------------------------------------------------------------- +// Flat sub-agent schemas +// --------------------------------------------------------------------------- + +// DataFlowTrace is the flat schema for the data-flow tracing sub-agent. +// +// Ports schemas/prove.py DataFlowTrace, including its three `mode="before"` +// field_validators (see UnmarshalJSON). +type DataFlowTrace struct { + // Source is where tainted input enters, e.g. "request.params.id". + Source string `json:"source"` + // Sink is the security-sensitive operation reached, e.g. "sql.execute(query)". + Sink string `json:"sink"` + // Steps is an ordered list of file:line descriptions showing the flow path. + Steps []string `json:"steps"` + // SinkReached reports whether tainted data actually reaches the sink. + SinkReached bool `json:"sink_reached"` +} + +// UnmarshalJSON ports DataFlowTrace's `_coerce_to_str` / `_coerce_steps` +// before-validators: LLMs sometimes return dicts or lists where a flat string +// (or list of strings) is declared, and Python coerces rather than failing. +func (d *DataFlowTrace) UnmarshalJSON(b []byte) error { + var raw struct { + Source json.RawMessage `json:"source"` + Sink json.RawMessage `json:"sink"` + Steps json.RawMessage `json:"steps"` + SinkReached bool `json:"sink_reached"` + } + if err := json.Unmarshal(b, &raw); err != nil { + return err + } + d.Source = coerceToStr(raw.Source) + d.Sink = coerceToStr(raw.Sink) + d.Steps = coerceToStrList(raw.Steps) + d.SinkReached = raw.SinkReached + return nil +} + +// ReachabilityProof is the flat schema for dependency reachability analysis. +// +// Ports schemas/prove.py ReachabilityProof, including its two before-validators. +type ReachabilityProof struct { + // VulnerableFunction is the vulnerable function/method in the dependency. + VulnerableFunction string `json:"vulnerable_function"` + // CallChain is the import/call chain from app code to that function. + CallChain []string `json:"call_chain"` + // Reachable reports whether the vulnerable function is actually called. + Reachable bool `json:"reachable"` + // Direct reports whether the dependency is direct or transitive. + Direct bool `json:"direct"` +} + +// UnmarshalJSON ports ReachabilityProof's `_coerce_to_str` / +// `_coerce_call_chain` before-validators. +// +// Python parity: `_coerce_to_str` here does NOT have the dict-key probing that +// DataFlowTrace's version has — a dict lands on `str(v)`. coerceToStr handles +// both by taking a `probeDictKeys` flag; ReachabilityProof passes false. +func (r *ReachabilityProof) UnmarshalJSON(b []byte) error { + var raw struct { + VulnerableFunction json.RawMessage `json:"vulnerable_function"` + CallChain json.RawMessage `json:"call_chain"` + Reachable bool `json:"reachable"` + Direct bool `json:"direct"` + } + if err := json.Unmarshal(b, &raw); err != nil { + return err + } + r.VulnerableFunction = coerceToStrNoProbe(raw.VulnerableFunction) + r.CallChain = coerceToStrList(raw.CallChain) + r.Reachable = raw.Reachable + r.Direct = raw.Direct + return nil +} + +// SanitizationResult is the flat schema for the sanitization analysis +// sub-agent. +// +// Ports schemas/prove.py SanitizationResult. +type SanitizationResult struct { + // Found reports whether any sanitization/validation was found on the path. + Found bool `json:"found"` + // Type is e.g. "parameterized query", "html encoding". + Type *string `json:"type"` + // Sufficient reports whether the sanitization prevents the exploit. + Sufficient *bool `json:"sufficient"` + // BypassMethod describes how the sanitization could be bypassed. + BypassMethod *string `json:"bypass_method"` +} + +// ExploitHypothesis is the flat schema for the exploit construction sub-agent. +// +// Ports schemas/prove.py ExploitHypothesis, including its before-validators. +type ExploitHypothesis struct { + // Hypothesis is a natural-language description of the exploit scenario. + Hypothesis string `json:"hypothesis"` + // Payload is a concrete exploit payload or input. + Payload *string `json:"payload"` + // ExpectedOutcome is what would happen if the exploit succeeds. + ExpectedOutcome string `json:"expected_outcome"` +} + +// UnmarshalJSON ports ExploitHypothesis's `_coerce_to_str` / `_coerce_payload` +// before-validators. Python parity: `_coerce_payload` maps None to None but +// stringifies anything else (including a falsy 0 or "" — unlike +// `_coerce_to_str`, which maps falsy values to "unknown"). +func (e *ExploitHypothesis) UnmarshalJSON(b []byte) error { + var raw struct { + Hypothesis json.RawMessage `json:"hypothesis"` + Payload json.RawMessage `json:"payload"` + ExpectedOutcome json.RawMessage `json:"expected_outcome"` + } + if err := json.Unmarshal(b, &raw); err != nil { + return err + } + e.Hypothesis = coerceToStrNoProbe(raw.Hypothesis) + e.ExpectedOutcome = coerceToStrNoProbe(raw.ExpectedOutcome) + e.Payload = coercePayload(raw.Payload) + return nil +} + +// DastVerificationResult is the flat schema for DAST-like runtime verification. +// +// Ports schemas/prove.py DastVerificationResult. +type DastVerificationResult struct { + PayloadSent string `json:"payload_sent"` + ResponseSummary string `json:"response_summary"` + ExploitConfirmed bool `json:"exploit_confirmed"` + SafetyNotes string `json:"safety_notes"` +} + +// VerdictDecision is the flat schema for the verdict sub-agent. It goes +// through `.ai()`, not `.harness()`. +// +// Ports schemas/prove.py VerdictDecision. Verdict/EvidenceLevel are a plain +// string/int here because this is the raw model output, validated downstream. +type VerdictDecision struct { + // Verdict is one of: "confirmed", "likely", "inconclusive", "not_exploitable". + Verdict string `json:"verdict"` + // EvidenceLevel is 1..6: 1=STATIC_MATCH to 6=FULL_EXPLOIT. + EvidenceLevel int `json:"evidence_level"` + Rationale string `json:"rationale"` + // Confidence is one of: "high", "medium", "low". + Confidence string `json:"confidence"` +} + +// RemediationSuggestion is the flat schema for an AI-generated remediation. +// +// Ports schemas/prove.py RemediationSuggestion. +type RemediationSuggestion struct { + FixDescription string `json:"fix_description"` + // PatchDiff is a unified-diff patch showing the code changes needed. + PatchDiff string `json:"patch_diff"` + // Confidence is "high", "medium" or "low". + Confidence string `json:"confidence"` +} + +// --------------------------------------------------------------------------- +// Evidence artifacts +// --------------------------------------------------------------------------- + +// DataFlowStep is one step in a source-to-sink proof trace (DESIGN.md §6.4). +// +// Ports schemas/prove.py DataFlowStep — the model schemas/__init__.py +// re-exports under the bare name. recon.py's same-named model is +// ReconDataFlowStep. +type DataFlowStep struct { + File string `json:"file"` + Line int `json:"line"` + Description string `json:"description"` + Tainted bool `json:"tainted"` +} + +// DataFlowEvidence is the grouped data-flow evidence artifact +// (DESIGN.md §6.4). +// +// Ports schemas/prove.py DataFlowEvidence. Seeded (defaults.go): steps `[]`. +type DataFlowEvidence struct { + Steps []DataFlowStep `json:"steps"` + Source *string `json:"source"` + Sink *string `json:"sink"` + SinkReached bool `json:"sink_reached"` +} + +// SanitizationAnalysis is the sanitization effectiveness analysis +// (DESIGN.md §6.4). +// +// Ports schemas/prove.py SanitizationAnalysis. +type SanitizationAnalysis struct { + SanitizationFound bool `json:"sanitization_found"` + SanitizationType *string `json:"sanitization_type"` + SanitizationSufficient *bool `json:"sanitization_sufficient"` + BypassPossible *bool `json:"bypass_possible"` + BypassMethod *string `json:"bypass_method"` +} + +// HttpEvidence is the HTTP request/response evidence artifact +// (DESIGN.md §6.4). +// +// Ports schemas/prove.py HttpEvidence. `headers` is `dict[str, str] | None`; +// a nil Go map already marshals to null, so no pointer is needed. +type HttpEvidence struct { + Method *string `json:"method"` + URL *string `json:"url"` + Headers map[string]string `json:"headers"` + Body *string `json:"body"` + HighlightedSegment *string `json:"highlighted_segment"` +} + +// ReachabilityEvidence is the reachability evidence for dependency findings +// (DESIGN.md §6.4). +// +// Ports schemas/prove.py ReachabilityEvidence. Seeded (defaults.go): +// call_chain `[]`. +type ReachabilityEvidence struct { + VulnerableFunction string `json:"vulnerable_function"` + CallChain []string `json:"call_chain"` + Reachable bool `json:"reachable"` + DirectDependency bool `json:"direct_dependency"` +} + +// ChainStep is a chain evidence link across findings (DESIGN.md §6.4). +// +// Ports schemas/prove.py ChainStep. +type ChainStep struct { + StepNumber int `json:"step_number"` + FindingID string `json:"finding_id"` + Description string `json:"description"` + Enables string `json:"enables"` +} + +// Proof is the evidence artifact supporting a final verdict (DESIGN.md §6.4). +// +// Ports schemas/prove.py Proof. Every optional field is `X | None` with NO +// default_factory, so a nil slice marshaling to null is exactly right. +type Proof struct { + ExploitHypothesis string `json:"exploit_hypothesis"` + VerificationMethod string `json:"verification_method"` + EvidenceLevel EvidenceLevel `json:"evidence_level"` + DataFlowTrace []DataFlowStep `json:"data_flow_trace"` + DataFlowEvidence *DataFlowEvidence `json:"data_flow_evidence"` + SanitizationAnalysis *SanitizationAnalysis `json:"sanitization_analysis"` + VulnerableCode *string `json:"vulnerable_code"` + ExploitPayload *string `json:"exploit_payload"` + ExpectedOutcome *string `json:"expected_outcome"` + PocCode *string `json:"poc_code"` + PocExecutionOutput *string `json:"poc_execution_output"` + HTTPRequest *HttpEvidence `json:"http_request"` + HTTPResponse *HttpEvidence `json:"http_response"` + Reachability *ReachabilityEvidence `json:"reachability"` + ChainSteps []ChainStep `json:"chain_steps"` +} + +// ProverSignal is the depth-first expansion signal from a prover +// (DESIGN.md §6.6). +// +// Ports schemas/prove.py ProverSignal. +type ProverSignal struct { + Expand bool `json:"expand"` + ExpansionReason *string `json:"expansion_reason"` + ExpansionStrategy *string `json:"expansion_strategy"` + ExpansionTarget *string `json:"expansion_target"` +} + +// VerifiedFinding is a finding fully assessed by the PROVE phase +// (DESIGN.md §7.1). +// +// Ports schemas/prove.py VerifiedFinding. Seeded (defaults.go): tags, +// related_locations, compliance and reproduction_steps `[]`, and +// NewVerifiedFinding mints ID as a fresh uuid4. +type VerifiedFinding struct { + ID string `json:"id"` + Fingerprint string `json:"fingerprint"` + Title string `json:"title"` + Description string `json:"description"` + FindingType FindingType `json:"finding_type"` + CweID string `json:"cwe_id"` + CweName string `json:"cwe_name"` + OwaspCategory *string `json:"owasp_category"` + Tags []string `json:"tags"` + Verdict ExploitVerdict `json:"verdict"` + EvidenceLevel EvidenceLevel `json:"evidence_level"` + Rationale string `json:"rationale"` + Severity Severity `json:"severity"` + CvssV4 *CvssV4Score `json:"cvss_v4"` + Epss *EpssScore `json:"epss"` + ExploitabilityScore float64 `json:"exploitability_score"` + Proof *Proof `json:"proof"` + Location Location `json:"location"` + RelatedLocations []Location `json:"related_locations"` + ChainID *string `json:"chain_id"` + ChainStep *int `json:"chain_step"` + Enables []string `json:"enables"` + Compliance []ComplianceMapping `json:"compliance"` + ReproductionSteps []ReproductionStep `json:"reproduction_steps"` + Remediation *RemediationSuggestion `json:"remediation"` + SarifRuleID string `json:"sarif_rule_id"` + SarifSecuritySeverity float64 `json:"sarif_security_severity"` + DropReason *string `json:"drop_reason"` +} + +// --------------------------------------------------------------------------- +// before-validator helpers (schemas/prove.py @field_validator(mode="before")) +// --------------------------------------------------------------------------- + +// coerceToStr ports DataFlowTrace._coerce_to_str: a JSON string passes through +// UNCHANGED (Python's `if isinstance(v, str): return v` runs before the falsy +// check, so "" stays ""); a JSON object is probed for the first of +// value/name/description/path/text whose value is a string; anything else +// (including a probed-but-unmatched object) falls back to Python's +// `str(v) if v else "unknown"`. +func coerceToStr(raw json.RawMessage) string { + if len(raw) == 0 { + // Absent key. Python raises ValidationError on a missing required + // field; the Go port keeps the zero value and lets the caller decide. + return "" + } + if s, ok := jsonString(raw); ok { + return s + } + var obj map[string]json.RawMessage + if err := json.Unmarshal(raw, &obj); err == nil { + // Python parity: the probe requires the value to BE a string; a + // non-string under "name" falls through to the next candidate key. + for _, key := range []string{"value", "name", "description", "path", "text"} { + if v, ok := obj[key]; ok { + if sv, isStr := jsonString(v); isStr { + return sv + } + } + } + } + return coerceToStrNoProbe(raw) +} + +// coerceToStrNoProbe ports the plainer `_coerce_to_str` on ReachabilityProof +// and ExploitHypothesis: a string passes through UNCHANGED (including ""), +// everything else becomes `str(v) if v else "unknown"`. +func coerceToStrNoProbe(raw json.RawMessage) string { + if len(raw) == 0 { + return "" + } + if s, ok := jsonString(raw); ok { + return s + } + if isFalsyJSON(raw) { + return "unknown" + } + return pyStr(raw) +} + +// coercePayload ports ExploitHypothesis._coerce_payload: None stays None, a +// string passes through, anything else is `str(v)` — with NO falsy special +// case, so a JSON 0 becomes "0" rather than "unknown". +func coercePayload(raw json.RawMessage) *string { + if len(raw) == 0 || isJSONNull(raw) { + return nil + } + if s, ok := jsonString(raw); ok { + return &s + } + out := pyStr(raw) + return &out +} + +// coerceToStrList ports `_coerce_steps` / `_coerce_call_chain`: a JSON array +// becomes a list with every non-string element stringified via `str()`; +// anything else becomes `[str(v)] if v else []`. +func coerceToStrList(raw json.RawMessage) []string { + if len(raw) == 0 { + return []string{} + } + var items []json.RawMessage + if err := json.Unmarshal(raw, &items); err == nil { + out := make([]string, 0, len(items)) + for _, item := range items { + if s, ok := jsonString(item); ok { + out = append(out, s) // Python parity: str items are kept as-is + continue + } + out = append(out, pyStr(item)) + } + return out + } + if isFalsyJSON(raw) { + return []string{} + } + if s, ok := jsonString(raw); ok { + return []string{s} + } + return []string{pyStr(raw)} +} + +// jsonString reports whether raw is a JSON STRING and returns its value. It +// exists because json.Unmarshal happily decodes a JSON null into a Go string +// (leaving it ""), which would make a null look like an empty string — and the +// two take different branches in every one of Python's before-validators. +func jsonString(raw json.RawMessage) (string, bool) { + var s *string + if err := json.Unmarshal(raw, &s); err != nil || s == nil { + return "", false + } + return *s, true +} + +// isJSONNull reports whether raw decodes to JSON null. +func isJSONNull(raw json.RawMessage) bool { + var v any + return json.Unmarshal(raw, &v) == nil && v == nil +} + +// isFalsyJSON reports whether raw is one of the JSON values Python considers +// falsy: null, false, 0, 0.0, "", [] and {}. +func isFalsyJSON(raw json.RawMessage) bool { + var v any + if err := json.Unmarshal(raw, &v); err != nil { + return false + } + switch t := v.(type) { + case nil: + return true + case bool: + return !t + case float64: + return t == 0 + case string: + return t == "" + case []any: + return len(t) == 0 + case map[string]any: + return len(t) == 0 + } + return false +} + +// pyStr renders a decoded JSON value the way Python's `str()` renders the +// object pydantic received: True/False/None, single-quoted strings, +// {'k': v} dicts in INSERTION order, [a, b] lists. +// +// It walks the raw JSON with a token decoder rather than decoding into +// map[string]any precisely so dict key order survives — Python's dict repr +// follows insertion order and a Go map would scramble it. +// +// These branches only fire on malformed model output, and the resulting string +// is prose fed back to an LLM, never parsed. Known divergence: a number is +// emitted as its JSON literal, so an exponent-form literal like `1e3` renders +// as "1e3" where Python's `str(json.loads("1e3"))` gives "1000.0". +func pyStr(raw json.RawMessage) string { + var buf bytes.Buffer + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if err := writePyRepr(&buf, dec); err != nil { + return string(raw) + } + return buf.String() +} + +// writePyRepr consumes exactly one JSON value from dec and writes its Python +// repr to buf. +func writePyRepr(buf *bytes.Buffer, dec *json.Decoder) error { + tok, err := dec.Token() + if err != nil { + return err + } + switch t := tok.(type) { + case json.Delim: + switch t { + case '{': + buf.WriteByte('{') + first := true + for dec.More() { + keyTok, err := dec.Token() + if err != nil { + return err + } + if !first { + buf.WriteString(", ") + } + first = false + key, _ := keyTok.(string) + buf.WriteString("'" + key + "': ") + if err := writePyRepr(buf, dec); err != nil { + return err + } + } + if _, err := dec.Token(); err != nil { // closing '}' + return err + } + buf.WriteByte('}') + return nil + case '[': + buf.WriteByte('[') + first := true + for dec.More() { + if !first { + buf.WriteString(", ") + } + first = false + if err := writePyRepr(buf, dec); err != nil { + return err + } + } + if _, err := dec.Token(); err != nil { // closing ']' + return err + } + buf.WriteByte(']') + return nil + } + return fmt.Errorf("unexpected delimiter %v", t) + case nil: + buf.WriteString("None") + case bool: + if t { + buf.WriteString("True") + } else { + buf.WriteString("False") + } + case json.Number: + buf.WriteString(t.String()) + case string: + buf.WriteString("'" + t + "'") + default: + buf.WriteString(fmt.Sprintf("%v", t)) + } + return nil +} diff --git a/internal/secaf/schemas/recon.go b/internal/secaf/schemas/recon.go new file mode 100644 index 0000000000..f9e1375d9f --- /dev/null +++ b/internal/secaf/schemas/recon.go @@ -0,0 +1,353 @@ +package schemas + +// This file ports src/sec_af/schemas/recon.py — the RECON phase schemas +// (DESIGN.md §4.3), both the structured models that flow downstream and the +// FLAT `*Raw` models the harness actually produces. +// +// Python parity: recon.py declares a `DataFlowStep` that is a DIFFERENT model +// from prove.py's `DataFlowStep`. `schemas/__init__.py` re-exports prove's +// under the bare name and recon's as `ReconDataFlowStep`, so Go does the same. + +// Module is a module-level architecture element (DESIGN.md §4.3). +// +// Ports schemas/recon.py Module. +type Module struct { + Name string `json:"name"` + Path string `json:"path"` + Language string `json:"language"` + Description *string `json:"description"` + Dependencies []string `json:"dependencies"` +} + +// EntryPoint is an executable entry point (HTTP, CLI, event) (DESIGN.md §4.3). +// +// Ports schemas/recon.py EntryPoint. +type EntryPoint struct { + Kind string `json:"kind"` + Identifier string `json:"identifier"` + FilePath string `json:"file_path"` + Line int `json:"line"` + Method *string `json:"method"` + Route *string `json:"route"` + AuthRequired *bool `json:"auth_required"` +} + +// TrustBoundary is a trust transition location (DESIGN.md §4.3). +// +// Ports schemas/recon.py TrustBoundary. +type TrustBoundary struct { + Name string `json:"name"` + SourceZone string `json:"source_zone"` + TargetZone string `json:"target_zone"` + Description string `json:"description"` + Enforcement []string `json:"enforcement"` +} + +// Service is a service dependency or external integration (DESIGN.md §4.3). +// +// Ports schemas/recon.py Service. +type Service struct { + Name string `json:"name"` + ServiceType string `json:"service_type"` + Endpoint *string `json:"endpoint"` + Purpose *string `json:"purpose"` + AuthMechanism *string `json:"auth_mechanism"` +} + +// APIEndpoint is one exposed API surface entry (DESIGN.md §4.3). +// +// Ports schemas/recon.py APIEndpoint. +type APIEndpoint struct { + Method string `json:"method"` + Path string `json:"path"` + Handler string `json:"handler"` + FilePath string `json:"file_path"` + Line int `json:"line"` + AuthRequired *bool `json:"auth_required"` + RateLimited *bool `json:"rate_limited"` +} + +// ArchitectureMap is the architecture mapper output (DESIGN.md §4.3). +// +// Ports schemas/recon.py ArchitectureMap. Seeded (defaults.go): all five list +// fields default to `[]`. +type ArchitectureMap struct { + AppType *string `json:"app_type"` + Modules []Module `json:"modules"` + EntryPoints []EntryPoint `json:"entry_points"` + TrustBoundaries []TrustBoundary `json:"trust_boundaries"` + Services []Service `json:"services"` + APISurface []APIEndpoint `json:"api_surface"` +} + +// ReconDataFlowStep is an intermediate transformation step in a data flow +// (DESIGN.md §4.3). +// +// Ports schemas/recon.py DataFlowStep, which `schemas/__init__.py` re-exports +// as `ReconDataFlowStep` to leave the bare name to prove.py's different model. +// This is the one used by DataFlow.path and RawFinding.data_flow. +type ReconDataFlowStep struct { + FilePath string `json:"file_path"` + Line int `json:"line"` + Component string `json:"component"` + Operation string `json:"operation"` +} + +// SanitizationPoint is a location where tainted data is sanitized +// (DESIGN.md §4.3). +// +// Ports schemas/recon.py SanitizationPoint. Seeded (defaults.go): +// protects_against defaults to `[]`. +type SanitizationPoint struct { + FilePath string `json:"file_path"` + Line int `json:"line"` + FunctionName *string `json:"function_name"` + SanitizationType string `json:"sanitization_type"` + ProtectsAgainst []string `json:"protects_against"` +} + +// Sink is a security-critical sink reached by application data +// (DESIGN.md §4.3). +// +// Ports schemas/recon.py Sink. +type Sink struct { + SinkType string `json:"sink_type"` + FilePath string `json:"file_path"` + Line int `json:"line"` + FunctionName *string `json:"function_name"` + ExploitabilityNotes *string `json:"exploitability_notes"` +} + +// DataFlow is one input-to-sink path (DESIGN.md §4.3). +// +// Ports schemas/recon.py DataFlow. Seeded (defaults.go): path and files +// default to `[]`. +type DataFlow struct { + Source string `json:"source"` + Path []ReconDataFlowStep `json:"path"` + Sink string `json:"sink"` + Sanitized bool `json:"sanitized"` + Files []string `json:"files"` +} + +// DataFlowMap is the aggregated data-flow analysis output (DESIGN.md §4.3). +// +// Ports schemas/recon.py DataFlowMap. Seeded (defaults.go): all three list +// fields default to `[]`. +type DataFlowMap struct { + Flows []DataFlow `json:"flows"` + SanitizationPoints []SanitizationPoint `json:"sanitization_points"` + Sinks []Sink `json:"sinks"` +} + +// Dependency is one software bill-of-materials entry (DESIGN.md §4.3). +// +// Ports schemas/recon.py Dependency. +type Dependency struct { + Name string `json:"name"` + Version string `json:"version"` + Ecosystem string `json:"ecosystem"` + Direct bool `json:"direct"` + License *string `json:"license"` +} + +// KnownCVE is a known CVE affecting a dependency (DESIGN.md §4.3). +// +// Ports schemas/recon.py KnownCVE. +type KnownCVE struct { + CveID string `json:"cve_id"` + Package string `json:"package"` + InstalledVersion string `json:"installed_version"` + FixedVersion *string `json:"fixed_version"` + CvssV4Score *float64 `json:"cvss_v4_score"` + EpssScore *float64 `json:"epss_score"` + Direct bool `json:"direct"` + Reachable *bool `json:"reachable"` +} + +// OutdatedDep is a dependency that lags behind the latest available version +// (DESIGN.md §4.3). +// +// Ports schemas/recon.py OutdatedDep. +type OutdatedDep struct { + Package string `json:"package"` + CurrentVersion string `json:"current_version"` + LatestVersion string `json:"latest_version"` + Direct bool `json:"direct"` +} + +// DependencyReport is the dependency auditor output (DESIGN.md §4.3). +// +// Ports schemas/recon.py DependencyReport. Seeded (defaults.go): sbom, +// known_cves and outdated default to `[]`. +type DependencyReport struct { + Sbom []Dependency `json:"sbom"` + KnownCves []KnownCVE `json:"known_cves"` + Outdated []OutdatedDep `json:"outdated"` + DirectCount int `json:"direct_count"` + TransitiveCount int `json:"transitive_count"` +} + +// SecretFinding is a discovered hardcoded secret or credential +// (DESIGN.md §4.3). +// +// Ports schemas/recon.py SecretFinding. Seeded (defaults.go): ID gets a fresh +// uuid4 from NewSecretFinding. +type SecretFinding struct { + ID string `json:"id"` + SecretType string `json:"secret_type"` + FilePath string `json:"file_path"` + Line int `json:"line"` + Match string `json:"match"` + Confidence string `json:"confidence"` + IsTestValue *bool `json:"is_test_value"` +} + +// MisconfigFinding is an insecure configuration finding (DESIGN.md §4.3). +// +// Ports schemas/recon.py MisconfigFinding. Seeded (defaults.go): ID gets a +// fresh uuid4 from NewMisconfigFinding. +type MisconfigFinding struct { + ID string `json:"id"` + Category string `json:"category"` + FilePath string `json:"file_path"` + Line *int `json:"line"` + Key *string `json:"key"` + Value *string `json:"value"` + Risk string `json:"risk"` + Remediation *string `json:"remediation"` +} + +// ConfigReport is the config scanner output (DESIGN.md §4.3). +// +// Ports schemas/recon.py ConfigReport. Seeded (defaults.go): secrets and +// misconfigs default to `[]`. +type ConfigReport struct { + Secrets []SecretFinding `json:"secrets"` + Misconfigs []MisconfigFinding `json:"misconfigs"` +} + +// CryptoUsage is one cryptography usage profile entry (DESIGN.md §4.3). +// +// Ports schemas/recon.py CryptoUsage. +type CryptoUsage struct { + Algorithm string `json:"algorithm"` + KeySize *int `json:"key_size"` + Mode *string `json:"mode"` + UsageContext *string `json:"usage_context"` + IsWeak *bool `json:"is_weak"` +} + +// SecurityContext is the security context profiler output (DESIGN.md §4.3). +// +// Ports schemas/recon.py SecurityContext. Seeded (defaults.go): the four list +// fields default to `[]`. +type SecurityContext struct { + AuthModel string `json:"auth_model"` + AuthDetails string `json:"auth_details"` + CryptoUsage []CryptoUsage `json:"crypto_usage"` + FrameworkSecurity []string `json:"framework_security"` + SecurityHeaders []string `json:"security_headers"` + DeploymentSignals []string `json:"deployment_signals"` +} + +// ReconResult is the comprehensive RECON context (DESIGN.md §4.3). +// +// Ports schemas/recon.py ReconResult. Seeded (defaults.go): languages and +// frameworks default to `[]`; the five nested models are REQUIRED in Python, +// so NewReconResult seeds them with their own defaults rather than leaving +// half-built zero values. +type ReconResult struct { + Architecture ArchitectureMap `json:"architecture"` + DataFlows DataFlowMap `json:"data_flows"` + Dependencies DependencyReport `json:"dependencies"` + Config ConfigReport `json:"config"` + SecurityContext SecurityContext `json:"security_context"` + Languages []string `json:"languages"` + Frameworks []string `json:"frameworks"` + LinesOfCode int `json:"lines_of_code"` + FileCount int `json:"file_count"` + ReconDurationSeconds float64 `json:"recon_duration_seconds"` +} + +// --------------------------------------------------------------------------- +// Flat harness schemas for RECON agents +// --------------------------------------------------------------------------- +// These are what the LLM actually produces via .harness() calls: only flat +// fields (string, []string), no nesting. agents/recon/_parsers.py converts +// them into the structured schemas above. +// --------------------------------------------------------------------------- + +// ArchitectureMapRaw is the flat harness output for the architecture mapper. +// +// Ports schemas/recon.py ArchitectureMapRaw. Seeded (defaults.go): +// app_type="unknown", all five list fields `[]`. +type ArchitectureMapRaw struct { + // AppType is one of: web_api, cli_tool, library, microservice, monolith. + AppType string `json:"app_type"` + // Modules: one string per module, "name | path | language | description". + Modules []string `json:"modules"` + // EntryPoints: "kind | route_or_id | file_path:line | auth_required". + EntryPoints []string `json:"entry_points"` + // TrustBoundaries: "name | source_zone | target_zone | description". + TrustBoundaries []string `json:"trust_boundaries"` + // Services: "name | type | endpoint | auth_mechanism". + Services []string `json:"services"` + // APIEndpoints: "method | path | handler | file_path:line | auth_required | rate_limited". + APIEndpoints []string `json:"api_endpoints"` +} + +// DataFlowMapRaw is the flat harness output for the data flow mapper. +// +// Ports schemas/recon.py DataFlowMapRaw. Seeded (defaults.go): all three list +// fields `[]`. +type DataFlowMapRaw struct { + // Flows: "source | sink | sanitized(true/false) | file1, file2, ...". + Flows []string `json:"flows"` + // SanitizationPoints: "file_path:line | function_name | type | protects_against". + SanitizationPoints []string `json:"sanitization_points"` + // Sinks: "sink_type | file_path:line | function_name | notes". + Sinks []string `json:"sinks"` +} + +// DependencyReportRaw is the flat harness output for the dependency auditor. +// +// Ports schemas/recon.py DependencyReportRaw. Seeded (defaults.go): all three +// list fields `[]`. +type DependencyReportRaw struct { + // Sbom: "name | version | ecosystem | direct(true/false) | license". + Sbom []string `json:"sbom"` + // KnownCves: "cve_id | package | installed_version | fixed_version | cvss_score | direct | reachable". + KnownCves []string `json:"known_cves"` + // Outdated: "package | current_version | latest_version | direct(true/false)". + Outdated []string `json:"outdated"` +} + +// ConfigReportRaw is the flat harness output for the config scanner. +// +// Ports schemas/recon.py ConfigReportRaw. Seeded (defaults.go): both list +// fields `[]`. +type ConfigReportRaw struct { + // Secrets: "type | file_path:line | match_preview | confidence | is_test(true/false)". + Secrets []string `json:"secrets"` + // Misconfigs: "category | file_path:line | key | risk | remediation". + Misconfigs []string `json:"misconfigs"` +} + +// SecurityContextRaw is the flat harness output for the security context +// profiler. +// +// Ports schemas/recon.py SecurityContextRaw. Seeded (defaults.go): +// auth_details="" (the Go zero value, so only the list fields actually need +// seeding), crypto_usage and security_signals `[]`. +type SecurityContextRaw struct { + // AuthModel is one of: jwt, session_cookie, oauth2, api_key, none, other. + AuthModel string `json:"auth_model"` + // AuthDetails is a brief description of the auth implementation. + AuthDetails string `json:"auth_details"` + // CryptoUsage: "algorithm | key_size | mode | usage_context | is_weak(true/false)". + CryptoUsage []string `json:"crypto_usage"` + // SecuritySignals: framework security features, security headers and + // deployment signals, one per entry. + SecuritySignals []string `json:"security_signals"` +} diff --git a/internal/secaf/schemas/schemas_test.go b/internal/secaf/schemas/schemas_test.go new file mode 100644 index 0000000000..8727c5d132 --- /dev/null +++ b/internal/secaf/schemas/schemas_test.go @@ -0,0 +1,895 @@ +package schemas + +import ( + "encoding/json" + "reflect" + "testing" +) + +// This file ports tests/test_schemas.py plus the conftest.py fixtures it +// consumes, and adds the default-seeding / projection / coercion tests the +// Python suite gets for free from pydantic. +// +// Every expected value here was verified against the live models with +// `PYTHONPATH=src ~/.agentfield/packages/sec-af/venv/bin/python`. + +func strp(s string) *string { return &s } +func intp(i int) *int { return &i } + +func mustMarshal(t *testing.T, v any) []byte { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatalf("marshal %T: %v", v, err) + } + return b +} + +func mustUnmarshal[T any](t *testing.T, data string) T { + t.Helper() + var v T + if err := json.Unmarshal([]byte(data), &v); err != nil { + t.Fatalf("unmarshal %q into %T: %v", data, v, err) + } + return v +} + +// --------------------------------------------------------------------------- +// conftest.py fixtures +// --------------------------------------------------------------------------- + +// sampleVerifiedFindings ports conftest.py::sample_verified_findings. +// +// Python parity: the fixture passes `tags={"a","b"}` — a SET — into a +// `list[str]` field, which pydantic coerces in lax mode. Set iteration order is +// undefined in Python, so the Go port fixes a deterministic order +// (docs/DESIGN.md §0.2: make non-determinism explicit). No assertion depends +// on the order. +func sampleVerifiedFindings() []VerifiedFinding { + sql := NewVerifiedFinding() + sql.ID = "finding-confirmed" + sql.Fingerprint = "fp-sql-1" + sql.Title = "SQL Injection" + sql.Description = "Unsanitized user input reaches SQL query execution." + sql.FindingType = FindingTypeSast + sql.CweID = "CWE-89" + sql.CweName = "SQL Injection" + sql.OwaspCategory = strp("A03:2021") + sql.Tags = []string{"externally_reachable", "user-input"} + sql.Verdict = VerdictConfirmed + sql.EvidenceLevel = EvidenceLevelFullExploit + sql.Rationale = "Source-to-sink path is confirmed and exploitable." + sql.Severity = SeverityCritical + sql.ExploitabilityScore = 10.0 + sql.Proof = &Proof{ + ExploitHypothesis: "Inject through id parameter.", + VerificationMethod: "manual-review+trace", + EvidenceLevel: EvidenceLevelFullExploit, + DataFlowTrace: []DataFlowStep{ + {File: "src/routes.py", Line: 15, Description: "Input source", Tainted: true}, + {File: "src/users.py", Line: 42, Description: "SQL sink", Tainted: true}, + }, + VulnerableCode: strp(`cursor.execute(f"SELECT * FROM users WHERE id = {user_id}")`), + ExploitPayload: strp(`{"id": "1 OR 1=1"}`), + ExpectedOutcome: strp("Unauthorized data access"), + } + sql.Location = Location{ + FilePath: "src/users.py", + StartLine: 42, + EndLine: 42, + StartColumn: intp(9), + EndColumn: intp(66), + FunctionName: strp("lookup_user"), + CodeSnippet: strp(`cursor.execute(f"SELECT * FROM users WHERE id = {user_id}")`), + } + sql.RelatedLocations = []Location{{ + FilePath: "src/routes.py", + StartLine: 15, + EndLine: 15, + CodeSnippet: strp("user_id = request.json['id']"), + }} + sql.ChainID = strp("chain-1") + sql.ChainStep = intp(1) + sql.Enables = []string{"finding-likely"} + sql.Compliance = []ComplianceMapping{{ + Framework: "PCI-DSS", ControlID: "Req 6.2.4", ControlName: "Prevent injection", + }} + sql.SarifRuleID = "sec-af/sast/sql-injection" + sql.SarifSecuritySeverity = 9.9 + + likely := NewVerifiedFinding() + likely.ID = "finding-likely" + likely.Fingerprint = "fp-auth-1" + likely.Title = "Missing Authentication" + likely.Description = "Admin endpoint can be accessed without auth." + likely.FindingType = FindingTypeAPI + likely.CweID = "CWE-306" + likely.CweName = "Missing Authentication for Critical Function" + likely.OwaspCategory = strp("A07:2021") + likely.Tags = []string{"requires_auth"} + likely.Verdict = VerdictLikely + likely.EvidenceLevel = EvidenceLevelFlowIdentified + likely.Rationale = "Guard checks appear absent on route." + likely.Severity = SeverityHigh + likely.ExploitabilityScore = 4.8 + likely.Location = Location{FilePath: "src/api/admin.py", StartLine: 11, EndLine: 11} + likely.SarifRuleID = "sec-af/api/missing-authentication" + likely.SarifSecuritySeverity = 7.6 + + notExploitable := NewVerifiedFinding() + notExploitable.ID = "finding-noise" + notExploitable.Fingerprint = "fp-noise-1" + notExploitable.Title = "Potential XSS" + notExploitable.Description = "Output is escaped by template engine." + notExploitable.FindingType = FindingTypeSast + notExploitable.CweID = "CWE-79" + notExploitable.CweName = "Cross-site Scripting" + notExploitable.Verdict = VerdictNotExploitable + notExploitable.EvidenceLevel = EvidenceLevelStaticMatch + notExploitable.Rationale = "Sink auto-escapes output." + notExploitable.Severity = SeverityLow + notExploitable.ExploitabilityScore = 0.6 + notExploitable.Location = Location{FilePath: "src/views.py", StartLine: 88, EndLine: 89} + notExploitable.SarifRuleID = "sec-af/sast/xss" + notExploitable.SarifSecuritySeverity = 1.9 + + return []VerifiedFinding{sql, likely, notExploitable} +} + +// sampleSecurityAuditResult ports conftest.py::sample_security_audit_result. +func sampleSecurityAuditResult() SecurityAuditResult { + r := NewSecurityAuditResult() + r.Repository = "Agent-Field/sec-af" + r.CommitSha = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + r.Branch = strp("issue-23-tests") + r.Timestamp = mustUnmarshalNoT[Timestamp](t0JSON) + r.DepthProfile = "standard" + r.StrategiesUsed = []string{"injection", "auth"} + r.Provider = "opencode" + r.Findings = sampleVerifiedFindings() + + chain := NewAttackChain() + chain.ChainID = "chain-1" + chain.Title = "Input to DB read" + chain.Description = "Untrusted input reaches SQL sink" + chain.Findings = []string{"finding-confirmed", "finding-likely"} + chain.CombinedSeverity = SeverityCritical + chain.CombinedImpact = "Unauthorized DB disclosure" + chain.MitreAttackMapping = []MitreMapping{{ + Tactic: "Initial Access", TechniqueID: "T1190", + TechniqueName: "Exploit Public-Facing Application", + }} + r.AttackChains = []AttackChain{chain} + + r.TotalRawFindings = 6 + r.Confirmed = 1 + r.Likely = 1 + r.Inconclusive = 0 + r.NotExploitable = 1 + r.NoiseReductionPct = 66.7 + r.BySeverity = map[string]int{"critical": 1, "high": 1, "low": 1} + r.ComplianceGaps = []ComplianceGap{{ + Framework: "PCI-DSS", ControlID: "Req 6.2.4", ControlName: "Prevent injection", + FindingCount: 1, MaxSeverity: "critical", CweIDs: []string{"CWE-89"}, + }} + r.DurationSeconds = 182.4 + r.AgentInvocations = 24 + r.CostUsd = 3.21 + r.CostBreakdown = map[string]float64{"recon": 0.5, "hunt": 1.2, "prove": 1.51} + r.Sarif = "{}" + return r +} + +// t0JSON is conftest.py's datetime(2026, 3, 4, 10, 30, 0, tzinfo=UTC), in the +// exact wire form Python emits (see timestamp.go). +const t0JSON = `"2026-03-04T10:30:00+00:00"` + +// mustUnmarshalNoT is mustUnmarshal without a *testing.T, for fixture builders. +func mustUnmarshalNoT[T any](data string) T { + var v T + if err := json.Unmarshal([]byte(data), &v); err != nil { + panic(err) + } + return v +} + +// --------------------------------------------------------------------------- +// test_schemas.py::test_schema_validation_and_required_fields +// --------------------------------------------------------------------------- + +// Python asserts pydantic raises ValidationError for three payloads that omit +// required fields. Go's json.Unmarshal has no notion of "required", so the +// port splits the assertion: +// +// - AuditInput, the ONE model the port validates at runtime (it is the +// reasoner's public input contract), gets a real Validate() check. +// - For RawFinding and VerifiedFinding the contract is asserted against the +// generated pydantic ground truth: the fields Python's test omits really +// are required, so any Go code building one must set them. +func TestSchemaValidationAndRequiredFields(t *testing.T) { + // _validate(AuditInput, {"branch": "main"}) raises: repo_url is required. + in := mustUnmarshal[AuditInput](t, `{"branch":"main"}`) + if err := in.Validate(); err == nil { + t.Error("AuditInput{branch: main}.Validate() = nil, want a missing-repo_url error") + } + // ...and the defaults still seeded around the missing field. + if in.Branch != "main" || in.Depth != "standard" { + t.Errorf("AuditInput = %+v, want branch=main depth=standard", in) + } + // A payload WITH repo_url validates. + ok := mustUnmarshal[AuditInput](t, `{"repo_url":"https://github.com/Agent-Field/sec-af"}`) + if err := ok.Validate(); err != nil { + t.Errorf("AuditInput{repo_url: ...}.Validate() = %v, want nil", err) + } + + gt := loadGroundTruth(t) + requiredOf := func(goName string) map[string]bool { + for _, spec := range gt.Models { + if spec.GoName == goName && spec.DuplicateOf == nil { + set := map[string]bool{} + for _, f := range spec.Required { + set[f] = true + } + return set + } + } + t.Fatalf("no ground truth for %s", goName) + return nil + } + // RawFinding payload in the Python test omits everything but + // hunter_strategy/title/description — those omissions must be required. + rawRequired := requiredOf("RawFinding") + for _, field := range []string{"finding_type", "cwe_id", "cwe_name", "file_path", + "start_line", "end_line", "code_snippet", "estimated_severity", "confidence"} { + if !rawRequired[field] { + t.Errorf("RawFinding.%s should be required in pydantic", field) + } + } + // VerifiedFinding payload in the Python test omits fingerprint and location. + verifiedRequired := requiredOf("VerifiedFinding") + for _, field := range []string{"fingerprint", "location", "sarif_rule_id", "sarif_security_severity"} { + if !verifiedRequired[field] { + t.Errorf("VerifiedFinding.%s should be required in pydantic", field) + } + } + + // assert sample_verified_findings[0].location.file_path == "src/users.py" + if got := sampleVerifiedFindings()[0].Location.FilePath; got != "src/users.py" { + t.Errorf("sample_verified_findings[0].location.file_path = %q, want src/users.py", got) + } +} + +// --------------------------------------------------------------------------- +// test_schemas.py::test_schema_roundtrip_serialization +// --------------------------------------------------------------------------- + +func TestSchemaRoundtripSerialization(t *testing.T) { + sample := sampleSecurityAuditResult() + payload := mustMarshal(t, sample) + restored := mustUnmarshal[SecurityAuditResult](t, string(payload)) + + if restored.Repository != sample.Repository { + t.Errorf("repository = %q, want %q", restored.Repository, sample.Repository) + } + if restored.Findings[0].Fingerprint != "fp-sql-1" { + t.Errorf("findings[0].fingerprint = %q, want fp-sql-1", restored.Findings[0].Fingerprint) + } + if restored.Findings[1].Verdict != VerdictLikely { + t.Errorf("findings[1].verdict = %q, want likely", restored.Findings[1].Verdict) + } + if restored.Findings[2].Verdict != VerdictNotExploitable { + t.Errorf("findings[2].verdict = %q, want not_exploitable", restored.Findings[2].Verdict) + } + // The round trip must be byte-stable: model_dump -> model_validate -> + // model_dump is the identity in Python, and must be here too. + if again := mustMarshal(t, restored); string(again) != string(payload) { + t.Errorf("round trip changed the JSON\n got: %s\nwant: %s", again, payload) + } +} + +// --------------------------------------------------------------------------- +// test_schemas.py::test_enum_values_are_stable +// --------------------------------------------------------------------------- + +func TestEnumValuesAreStable(t *testing.T) { + if FindingTypeSast != "sast" { + t.Errorf("FindingType.SAST = %q, want sast", FindingTypeSast) + } + if FindingTypeAPI != "api" { + t.Errorf("FindingType.API = %q, want api", FindingTypeAPI) + } + if SeverityCritical != "critical" { + t.Errorf("Severity.CRITICAL = %q, want critical", SeverityCritical) + } + if ConfidenceHigh != "high" { + t.Errorf("Confidence.HIGH = %q, want high", ConfidenceHigh) + } + if VerdictConfirmed != "confirmed" { + t.Errorf("Verdict.CONFIRMED = %q, want confirmed", VerdictConfirmed) + } + if EvidenceLevelFullExploit != 6 { + t.Errorf("EvidenceLevel.FULL_EXPLOIT = %d, want 6", EvidenceLevelFullExploit) + } + if HuntStrategyDos != "dos" { + t.Errorf("HuntStrategy.DOS = %q, want dos", HuntStrategyDos) + } + if HuntStrategyConfigSecrets != "config_secrets" { + t.Errorf("HuntStrategy.CONFIG_SECRETS = %q, want config_secrets", HuntStrategyConfigSecrets) + } + // Python parity: LOGIC_BUGS is an ALIAS of BUSINESS_LOGIC, not a member. + if HuntStrategyLogicBugs != HuntStrategyBusinessLogic { + t.Errorf("HuntStrategy.LOGIC_BUGS = %q, want it to alias BUSINESS_LOGIC (%q)", + HuntStrategyLogicBugs, HuntStrategyBusinessLogic) + } + if len(AllHuntStrategies) != 13 { + t.Errorf("len(list(HuntStrategy)) = %d, want 13 (14 constants, one alias)", len(AllHuntStrategies)) + } +} + +// TestEnumValidAndParse covers the Valid()/ParseX helpers, which stand in for +// Python's `Severity("high")` constructor and its ValueError. +func TestEnumValidAndParse(t *testing.T) { + if !SeverityHigh.Valid() || Severity("nope").Valid() { + t.Error("Severity.Valid() is wrong") + } + if v, err := ParseSeverity("critical"); err != nil || v != SeverityCritical { + t.Errorf("ParseSeverity(critical) = %q, %v", v, err) + } + if _, err := ParseSeverity("blocker"); err == nil { + t.Error("ParseSeverity(blocker) = nil error, want ValueError equivalent") + } else if err.Error() != "'blocker' is not a valid Severity" { + t.Errorf("ParseSeverity(blocker) error = %q", err.Error()) + } + if _, err := ParseFindingType("nope"); err == nil { + t.Error("ParseFindingType(nope) should fail") + } + if _, err := ParseConfidence("nope"); err == nil { + t.Error("ParseConfidence(nope) should fail") + } + if _, err := ParseVerdict("nope"); err == nil { + t.Error("ParseVerdict(nope) should fail") + } + // The alias parses, because it is the same VALUE. + if v, err := ParseHuntStrategy("business_logic"); err != nil || v != HuntStrategyLogicBugs { + t.Errorf("ParseHuntStrategy(business_logic) = %q, %v", v, err) + } + if _, err := ParseHuntStrategy("logic_bugs"); err == nil { + t.Error("ParseHuntStrategy(logic_bugs) should fail — it is a member NAME, not a value") + } + for _, level := range AllEvidenceLevels { + if v, err := ParseEvidenceLevel(int(level)); err != nil || v != level { + t.Errorf("ParseEvidenceLevel(%d) = %d, %v", level, v, err) + } + } + if _, err := ParseEvidenceLevel(7); err == nil { + t.Error("ParseEvidenceLevel(7) should fail") + } + // Python parity: str(IntEnum) is the number in 3.11; Name() gives the + // symbolic form. + if got := EvidenceLevelFullExploit.String(); got != "6" { + t.Errorf("EvidenceLevel.String() = %q, want 6", got) + } + if got := EvidenceLevelFullExploit.Name(); got != "FULL_EXPLOIT" { + t.Errorf("EvidenceLevel.Name() = %q, want FULL_EXPLOIT", got) + } + if got := EvidenceLevel(9).Name(); got != "" { + t.Errorf("EvidenceLevel(9).Name() = %q, want empty", got) + } +} + +// --------------------------------------------------------------------------- +// test_schemas.py::test_json_schema_generation_contains_expected_fields +// +// Python inspects model_json_schema(); the Go port has no jsonschema +// generator in this package (harnessx embeds the pydantic-generated fixtures), +// so the same four assertions run against the generated ground truth, which is +// the same source: the live pydantic models. +// --------------------------------------------------------------------------- + +func TestJSONSchemaGenerationContainsExpectedFields(t *testing.T) { + gt := loadGroundTruth(t) + find := func(goName string) modelSpec { + for _, spec := range gt.Models { + if spec.GoName == goName && spec.DuplicateOf == nil { + return spec + } + } + t.Fatalf("no ground truth for %s", goName) + return modelSpec{} + } + has := func(list []string, want string) bool { + for _, v := range list { + if v == want { + return true + } + } + return false + } + + auditInput := find("AuditInput") + finding := find("VerifiedFinding") + result := find("SecurityAuditResult") + + if !has(auditInput.Keys, "repo_url") { + t.Error("repo_url missing from AuditInput properties") + } + if !has(auditInput.Required, "repo_url") { + t.Error("repo_url missing from AuditInput required") + } + if !has(finding.Keys, "fingerprint") { + t.Error("fingerprint missing from VerifiedFinding properties") + } + if !has(finding.Keys, "location") { + t.Error("location missing from VerifiedFinding properties") + } + if !has(result.Keys, "findings") { + t.Error("findings missing from SecurityAuditResult properties") + } + if !has(result.Required, "repository") { + t.Error("repository missing from SecurityAuditResult required") + } +} + +// --------------------------------------------------------------------------- +// test_schemas.py::test_recon_hunt_output_and_gate_models_instantiate +// --------------------------------------------------------------------------- + +func TestReconHuntOutputAndGateModelsInstantiate(t *testing.T) { + recon := NewReconResult() + recon.Architecture = NewArchitectureMap() + recon.Architecture.AppType = strp("web") + recon.DataFlows = NewDataFlowMap() + recon.Dependencies = NewDependencyReport() + recon.Dependencies.DirectCount = 1 + recon.Dependencies.TransitiveCount = 2 + recon.Config = NewConfigReport() + recon.SecurityContext = NewSecurityContext() + recon.SecurityContext.AuthModel = "jwt" + recon.SecurityContext.AuthDetails = "bearer token" + recon.Languages = []string{"python"} + recon.Frameworks = []string{"fastapi"} + recon.LinesOfCode = 1200 + recon.FileCount = 34 + recon.ReconDurationSeconds = 12.5 + + hunt := NewHuntResult() + hunt.TotalRaw = 2 + hunt.DeduplicatedCount = 2 + hunt.ChainCount = 1 + hunt.StrategiesRun = []string{"injection"} + + attackChain := NewAttackChain() + attackChain.ChainID = "chain-123" + attackChain.Title = "Privilege escalation path" + attackChain.Description = "Two-step attack" + attackChain.Findings = []string{"f1", "f2"} + attackChain.CombinedSeverity = SeverityHigh + attackChain.CombinedImpact = "Privilege escalation" + attackChain.MitreAttackMapping = []MitreMapping{{ + Tactic: "Privilege Escalation", + TechniqueID: "T1068", + TechniqueName: "Exploitation for Privilege Escalation", + }} + + progress := AuditProgress{ + Phase: "prove", PhaseProgress: 0.75, AgentsTotal: 6, AgentsCompleted: 4, + AgentsRunning: 2, FindingsSoFar: 3, ElapsedSeconds: 45.0, + EstimatedRemainingSeconds: 15.0, CostSoFarUsd: 1.23, + } + metrics := NewAuditMetrics() + metrics.DurationSeconds = 180.0 + metrics.AgentInvocations = 22 + metrics.CostUsd = 2.9 + + compliance := ComplianceGap{ + Framework: "PCI-DSS", ControlID: "Req 6.2.4", ControlName: "Prevent injection", + FindingCount: 2, MaxSeverity: "critical", CweIDs: []string{"CWE-79", "CWE-89"}, + } + gate := SeverityClassification{Severity: "high", Confidence: 0.9, Rationale: "validated"} + complianceGate := ComplianceGate{ + Mappings: []ComplianceSuggestion{{ + Framework: "OWASP", ControlID: "A03:2021", ControlName: "Injection", + }}, + Confidence: "high", + } + + if recon.LinesOfCode != 1200 { + t.Errorf("recon.lines_of_code = %d, want 1200", recon.LinesOfCode) + } + if hunt.TotalRaw != 2 { + t.Errorf("hunt.total_raw = %d, want 2", hunt.TotalRaw) + } + if attackChain.MitreAttackMapping == nil { + t.Error("attack_chain.mitre_attack_mapping is nil") + } + if progress.Phase != "prove" { + t.Errorf("progress.phase = %q, want prove", progress.Phase) + } + if metrics.BudgetExhausted { + t.Error("metrics.budget_exhausted = true, want false") + } + if compliance.Framework != "PCI-DSS" { + t.Errorf("compliance.framework = %q, want PCI-DSS", compliance.Framework) + } + if gate.Severity != "high" { + t.Errorf("gate.severity = %q, want high", gate.Severity) + } + if complianceGate.Mappings[0].Framework != "OWASP" { + t.Errorf("compliance_gate.mappings[0].framework = %q, want OWASP", complianceGate.Mappings[0].Framework) + } +} + +// --------------------------------------------------------------------------- +// test_schemas.py::test_model_validate_accepts_nested_dictionaries +// --------------------------------------------------------------------------- + +func TestModelValidateAcceptsNestedDictionaries(t *testing.T) { + payload := `{ + "repository": "Agent-Field/sec-af", + "commit_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "branch": "main", + "timestamp": "2026-03-04T12:00:00+00:00", + "depth_profile": "quick", + "provider": "opencode", + "sarif": "{}", + "findings": [ + { + "fingerprint": "fp-1", + "title": "Weak hash", + "description": "Uses md5", + "finding_type": "sast", + "cwe_id": "CWE-327", + "cwe_name": "Broken crypto", + "verdict": "likely", + "evidence_level": 2, + "rationale": "hash algorithm is weak", + "severity": "medium", + "exploitability_score": 1.5, + "location": {"file_path": "src/auth.py", "start_line": 7, "end_line": 7}, + "sarif_rule_id": "sec-af/sast/weak-hash", + "sarif_security_severity": 4.1, + "compliance": [ + {"framework": "OWASP", "control_id": "A02:2021", "control_name": "Cryptographic Failures"} + ] + } + ] + }` + + model := mustUnmarshal[SecurityAuditResult](t, payload) + if model.Findings[0].Location.FilePath != "src/auth.py" { + t.Errorf("findings[0].location.file_path = %q, want src/auth.py", model.Findings[0].Location.FilePath) + } + if model.Findings[0].Location.StartLine != 7 || model.Findings[0].Location.EndLine != 7 { + t.Errorf("findings[0].location lines = %d..%d, want 7..7", + model.Findings[0].Location.StartLine, model.Findings[0].Location.EndLine) + } + if len(model.Findings[0].Compliance) != 1 || model.Findings[0].Compliance[0].Framework != "OWASP" { + t.Errorf("findings[0].compliance = %+v, want one OWASP mapping", model.Findings[0].Compliance) + } + if model.Findings[0].EvidenceLevel != EvidenceLevelFlowIdentified { + t.Errorf("findings[0].evidence_level = %d, want 2", model.Findings[0].EvidenceLevel) + } + // The unspecified nested defaults survive the decode. + if model.Findings[0].Tags == nil || len(model.Findings[0].Tags) != 0 { + t.Errorf("findings[0].tags = %#v, want []", model.Findings[0].Tags) + } + if model.AttackChains == nil || len(model.AttackChains) != 0 { + t.Errorf("attack_chains = %#v, want []", model.AttackChains) + } + if model.Metadata == nil || len(model.Metadata) != 0 { + t.Errorf("metadata = %#v, want {}", model.Metadata) + } +} + +// --------------------------------------------------------------------------- +// Default seeding: "{}" seeds pydantic's defaults, a present key overrides — +// including false / 0 / "" / null. Derived from the package contract. +// --------------------------------------------------------------------------- + +func TestAuditInputDefaultSeeding(t *testing.T) { + in := mustUnmarshal[AuditInput](t, "{}") + if in.Branch != "main" { + t.Errorf("Branch = %q, want main", in.Branch) + } + if in.Depth != "standard" { + t.Errorf("Depth = %q, want standard", in.Depth) + } + if in.SeverityThreshold != "low" { + t.Errorf("SeverityThreshold = %q, want low", in.SeverityThreshold) + } + if !reflect.DeepEqual(in.ScanTypes, []string{"sast", "sca", "secrets", "config"}) { + t.Errorf("ScanTypes = %v", in.ScanTypes) + } + if !reflect.DeepEqual(in.OutputFormats, []string{"json"}) { + t.Errorf("OutputFormats = %v", in.OutputFormats) + } + if !reflect.DeepEqual(in.ExcludePaths, []string{"tests/", "vendor/", "node_modules/", ".git/"}) { + t.Errorf("ExcludePaths = %v", in.ExcludePaths) + } + if in.IncludePaths != nil { + t.Errorf("IncludePaths = %v, want nil (Optional default None)", in.IncludePaths) + } + if in.MaxCostUsd != nil || in.MaxProvers != nil || in.MaxDurationSeconds != nil { + t.Error("budget caps should default to nil") + } + + // Present values — including explicitly empty ones — override. + override := mustUnmarshal[AuditInput](t, + `{"repo_url":"r","branch":"","depth":"quick","scan_types":[],"is_pr":true,"exclude_paths":null}`) + if override.Branch != "" { + t.Errorf("explicit empty branch = %q, want \"\"", override.Branch) + } + if override.Depth != "quick" { + t.Errorf("Depth = %q, want quick", override.Depth) + } + if override.ScanTypes == nil || len(override.ScanTypes) != 0 { + t.Errorf("explicit [] scan_types = %#v, want empty non-nil", override.ScanTypes) + } + if !override.IsPr { + t.Error("is_pr = false, want true") + } + if override.ExcludePaths != nil { + t.Errorf("explicit null exclude_paths = %#v, want nil", override.ExcludePaths) + } +} + +func TestNestedDefaultSeeding(t *testing.T) { + // A ReconResult decoded from "{}" has every nested model at ITS defaults, + // which is what reasoners/phases.py::_recon_model normalization relies on. + recon := mustUnmarshal[ReconResult](t, "{}") + b := mustMarshal(t, recon) + var got map[string]any + if err := json.Unmarshal(b, &got); err != nil { + t.Fatal(err) + } + arch, _ := got["architecture"].(map[string]any) + if arch == nil { + t.Fatalf("architecture = %#v", got["architecture"]) + } + for _, key := range []string{"modules", "entry_points", "trust_boundaries", "services", "api_surface"} { + list, ok := arch[key].([]any) + if !ok || len(list) != 0 { + t.Errorf("architecture.%s = %#v, want []", key, arch[key]) + } + } + // A PRESENT nested key re-seeds through the nested UnmarshalJSON. + recon2 := mustUnmarshal[ReconResult](t, `{"architecture":{"app_type":"web_api"}}`) + if recon2.Architecture.AppType == nil || *recon2.Architecture.AppType != "web_api" { + t.Errorf("architecture.app_type = %v", recon2.Architecture.AppType) + } + if recon2.Architecture.Modules == nil || len(recon2.Architecture.Modules) != 0 { + t.Errorf("architecture.modules = %#v, want []", recon2.Architecture.Modules) + } +} + +func TestArchitectureMapRawDefaultSeeding(t *testing.T) { + raw := mustUnmarshal[ArchitectureMapRaw](t, "{}") + if raw.AppType != "unknown" { + t.Errorf("app_type = %q, want unknown", raw.AppType) + } + // An explicit empty string overrides the default (pydantic parity). + raw2 := mustUnmarshal[ArchitectureMapRaw](t, `{"app_type":""}`) + if raw2.AppType != "" { + t.Errorf("explicit empty app_type = %q, want \"\"", raw2.AppType) + } +} + +func TestUUIDDefaultsAreNotSeededOnDecode(t *testing.T) { + // Python parity note (doc.go): a decode must NOT mint a new identity. + rf := mustUnmarshal[RawFinding](t, "{}") + if rf.ID != "" || rf.Fingerprint != "" { + t.Errorf("decoded RawFinding minted id=%q fingerprint=%q, want empty", rf.ID, rf.Fingerprint) + } + if rf.RelatedFiles == nil || len(rf.RelatedFiles) != 0 { + t.Errorf("related_files = %#v, want []", rf.RelatedFiles) + } + // The constructor DOES mint them. + built := NewRawFinding() + if built.ID == "" || built.Fingerprint == "" || built.ID == built.Fingerprint { + t.Errorf("NewRawFinding gave id=%q fingerprint=%q", built.ID, built.Fingerprint) + } + if vf := mustUnmarshal[VerifiedFinding](t, "{}"); vf.ID != "" { + t.Errorf("decoded VerifiedFinding minted id=%q", vf.ID) + } + if pc := mustUnmarshal[PotentialChain](t, "{}"); pc.ChainID != "" { + t.Errorf("decoded PotentialChain minted chain_id=%q", pc.ChainID) + } +} + +// --------------------------------------------------------------------------- +// views.py projections (RawFinding.for_verifier / for_dedup) +// --------------------------------------------------------------------------- + +func TestRawFindingForVerifier(t *testing.T) { + base := NewRawFinding() + base.ID = "raw-1" + base.HunterStrategy = "injection" + base.Title = "SQLi" + base.Description = "desc" + base.FindingType = FindingTypeSast + base.CweID = "CWE-89" + base.CweName = "SQL Injection" + base.OwaspCategory = strp("A03:2021") + base.FilePath = "src/users.py" + base.StartLine = 42 + base.EndLine = 44 + base.FunctionName = strp("lookup_user") + base.CodeSnippet = "cursor.execute(q)" + base.EstimatedSeverity = SeverityCritical + base.Confidence = ConfidenceHigh + base.RelatedFiles = []string{"src/routes.py"} + base.Fingerprint = "fp-1" + + withFlow := base + withFlow.DataFlow = []ReconDataFlowStep{ + {FilePath: "src/routes.py", Line: 15, Component: "route", Operation: "read body"}, + {FilePath: "src/users.py", Line: 42, Component: "db", Operation: "execute"}, + } + + got := mustMarshal(t, withFlow.ForVerifier()) + want := `{"id":"raw-1","title":"SQLi","description":"desc","file_path":"src/users.py",` + + `"start_line":42,"end_line":44,"code_snippet":"cursor.execute(q)","cwe_id":"CWE-89",` + + `"function_name":"lookup_user","data_flow_summary":"src/routes.py:15 read body\nsrc/users.py:42 execute"}` + if string(got) != want { + t.Errorf("for_verifier()\n got: %s\nwant: %s", got, want) + } + + // Python parity: `if self.data_flow:` — an EMPTY list yields "" exactly + // like None does. + empty := base + empty.DataFlow = []ReconDataFlowStep{} + if s := empty.ForVerifier().DataFlowSummary; s != "" { + t.Errorf("empty data_flow summary = %q, want \"\"", s) + } + if s := base.ForVerifier().DataFlowSummary; s != "" { + t.Errorf("nil data_flow summary = %q, want \"\"", s) + } + + gotDedup := mustMarshal(t, withFlow.ForDedup()) + wantDedup := `{"id":"raw-1","fingerprint":"fp-1","title":"SQLi","file_path":"src/users.py",` + + `"start_line":42,"cwe_id":"CWE-89","finding_type":"sast","estimated_severity":"critical"}` + if string(gotDedup) != wantDedup { + t.Errorf("for_dedup()\n got: %s\nwant: %s", gotDedup, wantDedup) + } +} + +// --------------------------------------------------------------------------- +// prove.py @field_validator(mode="before") coercions. +// Every expectation below was produced by the Python models directly. +// --------------------------------------------------------------------------- + +func TestDataFlowTraceCoercion(t *testing.T) { + cases := []struct{ in, want string }{ + {`{"source":{"value":"req.id"},"sink":"sql","steps":["a","b"],"sink_reached":true}`, + `{"source":"req.id","sink":"sql","steps":["a","b"],"sink_reached":true}`}, + {`{"source":{"zzz":1},"sink":["x",2],"steps":"oneval","sink_reached":false}`, + `{"source":"{'zzz': 1}","sink":"['x', 2]","steps":["oneval"],"sink_reached":false}`}, + {`{"source":{},"sink":0,"steps":null,"sink_reached":false}`, + `{"source":"unknown","sink":"unknown","steps":[],"sink_reached":false}`}, + {`{"source":42,"sink":"","steps":[1,true,null],"sink_reached":true}`, + `{"source":"42","sink":"","steps":["1","True","None"],"sink_reached":true}`}, + {`{"source":{"name":5,"description":"d"},"sink":"s","steps":[],"sink_reached":false}`, + `{"source":"d","sink":"s","steps":[],"sink_reached":false}`}, + // null / false are falsy non-strings -> "unknown"; nested containers + // stringify with Python's repr. + {`{"source":null,"sink":false,"steps":[[1,"a"],{"k":2}],"sink_reached":false}`, + `{"source":"unknown","sink":"unknown","steps":["[1, 'a']","{'k': 2}"],"sink_reached":false}`}, + // The dict probe skips a null "value" and lands on "path". + {`{"source":{"path":"p","value":null},"sink":1.5,"steps":{},"sink_reached":false}`, + `{"source":"p","sink":"1.5","steps":[],"sink_reached":false}`}, + } + for _, tc := range cases { + got := mustMarshal(t, mustUnmarshal[DataFlowTrace](t, tc.in)) + if string(got) != tc.want { + t.Errorf("DataFlowTrace(%s)\n got: %s\nwant: %s", tc.in, got, tc.want) + } + } +} + +func TestReachabilityProofCoercion(t *testing.T) { + cases := []struct{ in, want string }{ + {`{"vulnerable_function":{"a":1},"call_chain":"x","reachable":true,"direct":false}`, + `{"vulnerable_function":"{'a': 1}","call_chain":["x"],"reachable":true,"direct":false}`}, + {`{"vulnerable_function":"","call_chain":null,"reachable":false,"direct":true}`, + `{"vulnerable_function":"","call_chain":[],"reachable":false,"direct":true}`}, + {`{"vulnerable_function":7,"call_chain":[1,"b"],"reachable":false,"direct":true}`, + `{"vulnerable_function":"7","call_chain":["1","b"],"reachable":false,"direct":true}`}, + // Python parity: ReachabilityProof's _coerce_to_str has NO dict probe, + // and null is falsy -> "unknown". + {`{"vulnerable_function":null,"call_chain":{},"reachable":false,"direct":false}`, + `{"vulnerable_function":"unknown","call_chain":[],"reachable":false,"direct":false}`}, + } + for _, tc := range cases { + got := mustMarshal(t, mustUnmarshal[ReachabilityProof](t, tc.in)) + if string(got) != tc.want { + t.Errorf("ReachabilityProof(%s)\n got: %s\nwant: %s", tc.in, got, tc.want) + } + } +} + +func TestExploitHypothesisCoercion(t *testing.T) { + cases := []struct{ in, want string }{ + {`{"hypothesis":{"a":1},"payload":null,"expected_outcome":""}`, + `{"hypothesis":"{'a': 1}","payload":null,"expected_outcome":""}`}, + {`{"hypothesis":"h","payload":0,"expected_outcome":"o"}`, + `{"hypothesis":"h","payload":"0","expected_outcome":"o"}`}, + {`{"hypothesis":"h","payload":{"k":"v"},"expected_outcome":"o"}`, + `{"hypothesis":"h","payload":"{'k': 'v'}","expected_outcome":"o"}`}, + {`{"hypothesis":"h","payload":"","expected_outcome":"o"}`, + `{"hypothesis":"h","payload":"","expected_outcome":"o"}`}, + {`{"hypothesis":null,"payload":[1,2],"expected_outcome":false}`, + `{"hypothesis":"unknown","payload":"[1, 2]","expected_outcome":"unknown"}`}, + } + for _, tc := range cases { + got := mustMarshal(t, mustUnmarshal[ExploitHypothesis](t, tc.in)) + if string(got) != tc.want { + t.Errorf("ExploitHypothesis(%s)\n got: %s\nwant: %s", tc.in, got, tc.want) + } + } +} + +// TestPyStrPreservesDictOrder pins the ordered dict repr — a Go map would +// scramble it, which is why pyStr walks the raw JSON with a token decoder. +func TestPyStrPreservesDictOrder(t *testing.T) { + in := `{"vulnerable_function":{"zeta":1,"alpha":2,"mid":{"b":true,"a":null}},"call_chain":[],"reachable":false,"direct":false}` + got := mustUnmarshal[ReachabilityProof](t, in).VulnerableFunction + want := "{'zeta': 1, 'alpha': 2, 'mid': {'b': True, 'a': None}}" + if got != want { + t.Errorf("pyStr\n got: %s\nwant: %s", got, want) + } +} + +// --------------------------------------------------------------------------- +// Optional / null parity spot-checks that the generated gate cannot express: +// a POINTER field set to a value round-trips, and dict/list-typed Optionals +// distinguish null from empty. +// --------------------------------------------------------------------------- + +func TestOptionalPointerRoundTrip(t *testing.T) { + ev := HttpEvidence{ + Method: strp("POST"), + URL: strp("https://example.test/login"), + Headers: map[string]string{"content-type": "application/json"}, + } + b := mustMarshal(t, ev) + want := `{"method":"POST","url":"https://example.test/login",` + + `"headers":{"content-type":"application/json"},"body":null,"highlighted_segment":null}` + if string(b) != want { + t.Errorf("HttpEvidence\n got: %s\nwant: %s", b, want) + } + back := mustUnmarshal[HttpEvidence](t, string(b)) + if !reflect.DeepEqual(back, ev) { + t.Errorf("round trip = %+v, want %+v", back, ev) + } + + // An empty (but non-nil) headers map marshals to {} — distinct from null. + empty := HttpEvidence{Headers: map[string]string{}} + if got := string(mustMarshal(t, empty)); got != + `{"method":null,"url":null,"headers":{},"body":null,"highlighted_segment":null}` { + t.Errorf("empty headers = %s", got) + } +} + +func TestProofOptionalListsMarshalNull(t *testing.T) { + // Python parity: Proof's list fields are `list[X] | None` with NO + // default_factory, so an unset one is null, not []. + b := mustMarshal(t, Proof{}) + var got map[string]any + if err := json.Unmarshal(b, &got); err != nil { + t.Fatal(err) + } + for _, key := range []string{"data_flow_trace", "chain_steps"} { + if got[key] != nil { + t.Errorf("Proof.%s = %#v, want null", key, got[key]) + } + } + // ...while an explicitly empty list stays []. + b2 := mustMarshal(t, Proof{DataFlowTrace: []DataFlowStep{}}) + var got2 map[string]any + if err := json.Unmarshal(b2, &got2); err != nil { + t.Fatal(err) + } + if list, ok := got2["data_flow_trace"].([]any); !ok || len(list) != 0 { + t.Errorf("Proof.data_flow_trace = %#v, want []", got2["data_flow_trace"]) + } +} diff --git a/internal/secaf/schemas/testdata/model_keys.json b/internal/secaf/schemas/testdata/model_keys.json new file mode 100644 index 0000000000..10982939ab --- /dev/null +++ b/internal/secaf/schemas/testdata/model_keys.json @@ -0,0 +1,2944 @@ +{ + "_generated_by": "go/scripts/gen_model_keys.py", + "models": [ + { + "python_module": "sec_af.schemas.compliance", + "python_class": "ComplianceMapping", + "go_name": "ComplianceMapping", + "duplicate_of": null, + "keys": [ + "framework", + "control_id", + "control_name" + ], + "required": [ + "framework", + "control_id", + "control_name" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.compliance", + "python_class": "ComplianceGap", + "go_name": "ComplianceGap", + "duplicate_of": null, + "keys": [ + "framework", + "control_id", + "control_name", + "finding_count", + "max_severity", + "cwe_ids" + ], + "required": [ + "framework", + "control_id", + "control_name", + "finding_count", + "max_severity", + "cwe_ids" + ], + "accepts_null": [], + "int_fields": [ + "finding_count" + ], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.gates", + "python_class": "SeverityClassification", + "go_name": "SeverityClassification", + "duplicate_of": null, + "keys": [ + "severity", + "confidence", + "rationale" + ], + "required": [ + "severity", + "confidence", + "rationale" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [ + "confidence" + ], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.gates", + "python_class": "DuplicateCheck", + "go_name": "DuplicateCheck", + "duplicate_of": null, + "keys": [ + "is_duplicate", + "duplicate_of", + "reason" + ], + "required": [ + "is_duplicate", + "reason" + ], + "accepts_null": [ + "duplicate_of" + ], + "int_fields": [], + "float_fields": [], + "null_fields": [ + "duplicate_of" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.gates", + "python_class": "StrategySelection", + "go_name": "StrategySelection", + "duplicate_of": null, + "keys": [ + "strategies", + "rationale" + ], + "required": [ + "strategies", + "rationale" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.gates", + "python_class": "CWEExpansion", + "go_name": "CWEExpansion", + "duplicate_of": null, + "keys": [ + "additional_cwes", + "rationale" + ], + "required": [ + "additional_cwes", + "rationale" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.gates", + "python_class": "RelevanceGate", + "go_name": "RelevanceGate", + "duplicate_of": null, + "keys": [ + "is_relevant", + "confidence", + "reason" + ], + "required": [ + "is_relevant", + "confidence", + "reason" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [ + "confidence" + ], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.gates", + "python_class": "VerdictGate", + "go_name": "VerdictGate", + "duplicate_of": null, + "keys": [ + "confirmed", + "confidence", + "reason" + ], + "required": [ + "confirmed", + "confidence", + "reason" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [ + "confidence" + ], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.gates", + "python_class": "ComplianceSuggestion", + "go_name": "ComplianceSuggestion", + "duplicate_of": null, + "keys": [ + "framework", + "control_id", + "control_name" + ], + "required": [ + "framework", + "control_id", + "control_name" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.gates", + "python_class": "ComplianceGate", + "go_name": "ComplianceGate", + "duplicate_of": null, + "keys": [ + "mappings", + "confidence" + ], + "required": [ + "mappings", + "confidence" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.gates", + "python_class": "ReachabilityGate", + "go_name": "ReachabilityGate", + "duplicate_of": null, + "keys": [ + "reachability", + "rationale", + "confidence" + ], + "required": [ + "reachability", + "rationale", + "confidence" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.hunt", + "python_class": "VulnLocation", + "go_name": "VulnLocation", + "duplicate_of": null, + "keys": [ + "file_path", + "start_line", + "code_snippet", + "pattern_type" + ], + "required": [ + "file_path", + "start_line", + "code_snippet", + "pattern_type" + ], + "accepts_null": [], + "int_fields": [ + "start_line" + ], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.hunt", + "python_class": "EnrichedFinding", + "go_name": "EnrichedFinding", + "duplicate_of": null, + "keys": [ + "title", + "description", + "cwe_id", + "severity", + "confidence", + "data_flow_summary" + ], + "required": [ + "title", + "description", + "cwe_id", + "severity", + "confidence", + "data_flow_summary" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.hunt", + "python_class": "ScanLocationsResult", + "go_name": "ScanLocationsResult", + "duplicate_of": null, + "keys": [ + "locations" + ], + "required": [], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [ + "locations" + ], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.hunt", + "python_class": "RawFinding", + "go_name": "RawFinding", + "duplicate_of": null, + "keys": [ + "id", + "hunter_strategy", + "title", + "description", + "finding_type", + "cwe_id", + "cwe_name", + "owasp_category", + "file_path", + "start_line", + "end_line", + "function_name", + "code_snippet", + "estimated_severity", + "confidence", + "data_flow", + "related_files", + "fingerprint" + ], + "required": [ + "hunter_strategy", + "title", + "description", + "finding_type", + "cwe_id", + "cwe_name", + "file_path", + "start_line", + "end_line", + "code_snippet", + "estimated_severity", + "confidence" + ], + "accepts_null": [ + "owasp_category", + "function_name", + "data_flow" + ], + "int_fields": [ + "start_line", + "end_line" + ], + "float_fields": [], + "null_fields": [ + "owasp_category", + "function_name", + "data_flow" + ], + "empty_list_fields": [ + "related_files" + ], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [ + "id", + "fingerprint" + ] + }, + { + "python_module": "sec_af.schemas.hunt", + "python_class": "PotentialChain", + "go_name": "PotentialChain", + "duplicate_of": null, + "keys": [ + "chain_id", + "title", + "finding_ids", + "combined_impact", + "estimated_severity" + ], + "required": [ + "title", + "combined_impact", + "estimated_severity" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [ + "finding_ids" + ], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [ + "chain_id" + ] + }, + { + "python_module": "sec_af.schemas.hunt", + "python_class": "HuntResult", + "go_name": "HuntResult", + "duplicate_of": null, + "keys": [ + "findings", + "chains", + "total_raw", + "deduplicated_count", + "chain_count", + "strategies_run", + "hunt_duration_seconds" + ], + "required": [], + "accepts_null": [], + "int_fields": [ + "total_raw", + "deduplicated_count", + "chain_count" + ], + "float_fields": [ + "hunt_duration_seconds" + ], + "null_fields": [], + "empty_list_fields": [ + "findings", + "chains", + "strategies_run" + ], + "empty_dict_fields": [], + "scalar_defaults": { + "total_raw": 0, + "deduplicated_count": 0, + "chain_count": 0, + "hunt_duration_seconds": 0.0 + }, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.hunt", + "python_class": "DeduplicatedResult", + "go_name": "DeduplicatedResult", + "duplicate_of": null, + "keys": [ + "findings", + "chains", + "dropped_duplicates", + "kept_findings" + ], + "required": [], + "accepts_null": [], + "int_fields": [ + "dropped_duplicates", + "kept_findings" + ], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [ + "findings", + "chains" + ], + "empty_dict_fields": [], + "scalar_defaults": { + "dropped_duplicates": 0, + "kept_findings": 0 + }, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.hunt", + "python_class": "ChainCorrelationResult", + "go_name": "ChainCorrelationResult", + "duplicate_of": null, + "keys": [ + "chains", + "duplicate_ids" + ], + "required": [], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [ + "chains", + "duplicate_ids" + ], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.input", + "python_class": "AuditInput", + "go_name": "AuditInput", + "duplicate_of": null, + "keys": [ + "repo_url", + "branch", + "commit_sha", + "base_commit_sha", + "depth", + "severity_threshold", + "scan_types", + "output_formats", + "compliance_frameworks", + "max_cost_usd", + "max_provers", + "max_duration_seconds", + "include_paths", + "exclude_paths", + "is_pr", + "pr_id", + "post_pr_comments", + "fail_on_findings", + "dast_enabled", + "repo_urls", + "monitoring_mode", + "baseline_path", + "custom_policies" + ], + "required": [ + "repo_url" + ], + "accepts_null": [ + "commit_sha", + "base_commit_sha", + "max_cost_usd", + "max_provers", + "max_duration_seconds", + "include_paths", + "pr_id", + "baseline_path" + ], + "int_fields": [ + "max_provers", + "max_duration_seconds" + ], + "float_fields": [ + "max_cost_usd" + ], + "null_fields": [ + "commit_sha", + "base_commit_sha", + "max_cost_usd", + "max_provers", + "max_duration_seconds", + "include_paths", + "pr_id", + "baseline_path" + ], + "empty_list_fields": [ + "compliance_frameworks", + "repo_urls", + "custom_policies" + ], + "empty_dict_fields": [], + "scalar_defaults": { + "branch": "main", + "depth": "standard", + "severity_threshold": "low", + "scan_types": [ + "sast", + "sca", + "secrets", + "config" + ], + "output_formats": [ + "json" + ], + "exclude_paths": [ + "tests/", + "vendor/", + "node_modules/", + ".git/" + ], + "is_pr": false, + "post_pr_comments": false, + "fail_on_findings": false, + "dast_enabled": false, + "monitoring_mode": false + }, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.output", + "python_class": "Location", + "go_name": "Location", + "duplicate_of": null, + "keys": [ + "file_path", + "start_line", + "end_line", + "start_column", + "end_column", + "function_name", + "code_snippet" + ], + "required": [ + "file_path", + "start_line", + "end_line" + ], + "accepts_null": [ + "start_column", + "end_column", + "function_name", + "code_snippet" + ], + "int_fields": [ + "start_line", + "end_line", + "start_column", + "end_column" + ], + "float_fields": [], + "null_fields": [ + "start_column", + "end_column", + "function_name", + "code_snippet" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.output", + "python_class": "CvssV4Score", + "go_name": "CvssV4Score", + "duplicate_of": null, + "keys": [ + "vector", + "base_score", + "severity", + "automatable", + "subsequent_impact" + ], + "required": [ + "vector", + "base_score", + "severity", + "automatable", + "subsequent_impact" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [ + "base_score" + ], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.output", + "python_class": "EpssScore", + "go_name": "EpssScore", + "duplicate_of": null, + "keys": [ + "score", + "percentile", + "date" + ], + "required": [ + "score", + "percentile", + "date" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [ + "score", + "percentile" + ], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.output", + "python_class": "MitreMapping", + "go_name": "MitreMapping", + "duplicate_of": null, + "keys": [ + "tactic", + "technique_id", + "technique_name" + ], + "required": [ + "tactic", + "technique_id", + "technique_name" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.output", + "python_class": "AttackChain", + "go_name": "AttackChain", + "duplicate_of": null, + "keys": [ + "chain_id", + "title", + "description", + "findings", + "combined_severity", + "combined_impact", + "mitre_attack_mapping" + ], + "required": [ + "chain_id", + "title", + "description", + "combined_severity", + "combined_impact" + ], + "accepts_null": [ + "mitre_attack_mapping" + ], + "int_fields": [], + "float_fields": [], + "null_fields": [ + "mitre_attack_mapping" + ], + "empty_list_fields": [ + "findings" + ], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.output", + "python_class": "ReproductionStep", + "go_name": "ReproductionStep", + "duplicate_of": null, + "keys": [ + "step", + "description", + "command", + "expected_output" + ], + "required": [ + "step", + "description" + ], + "accepts_null": [ + "command", + "expected_output" + ], + "int_fields": [ + "step" + ], + "float_fields": [], + "null_fields": [ + "command", + "expected_output" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.output", + "python_class": "ServiceDefinition", + "go_name": "ServiceDefinition", + "duplicate_of": null, + "keys": [ + "name", + "repo_url", + "api_endpoints", + "dependencies" + ], + "required": [ + "name", + "repo_url" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [ + "api_endpoints", + "dependencies" + ], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.output", + "python_class": "CrossServiceFinding", + "go_name": "CrossServiceFinding", + "duplicate_of": null, + "keys": [ + "chain_description", + "services_involved", + "entry_point", + "impact" + ], + "required": [ + "chain_description", + "services_involved", + "entry_point", + "impact" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.output", + "python_class": "RegressionFinding", + "go_name": "RegressionFinding", + "duplicate_of": null, + "keys": [ + "finding_title", + "finding_id", + "severity", + "cwe_id", + "status" + ], + "required": [ + "finding_title", + "finding_id", + "severity", + "cwe_id", + "status" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.output", + "python_class": "MonitoringResult", + "go_name": "MonitoringResult", + "duplicate_of": null, + "keys": [ + "baseline_commit", + "current_commit", + "new_findings", + "fixed_findings", + "unchanged_count", + "regression_detected" + ], + "required": [ + "baseline_commit", + "current_commit" + ], + "accepts_null": [], + "int_fields": [ + "unchanged_count" + ], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [ + "new_findings", + "fixed_findings" + ], + "empty_dict_fields": [], + "scalar_defaults": { + "unchanged_count": 0, + "regression_detected": false + }, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.output", + "python_class": "PolicyViolation", + "go_name": "PolicyViolation", + "duplicate_of": null, + "keys": [ + "policy", + "violation_description", + "file_path", + "severity" + ], + "required": [ + "policy", + "violation_description", + "file_path" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": { + "severity": "medium" + }, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.output", + "python_class": "SecurityAuditResult", + "go_name": "SecurityAuditResult", + "duplicate_of": null, + "keys": [ + "repository", + "commit_sha", + "branch", + "timestamp", + "depth_profile", + "strategies_used", + "provider", + "findings", + "attack_chains", + "total_raw_findings", + "confirmed", + "likely", + "inconclusive", + "not_exploitable", + "noise_reduction_pct", + "by_severity", + "compliance_gaps", + "policy_violations", + "duration_seconds", + "agent_invocations", + "cost_usd", + "cost_breakdown", + "metadata", + "sarif" + ], + "required": [ + "repository", + "commit_sha", + "timestamp", + "depth_profile", + "provider", + "sarif" + ], + "accepts_null": [ + "branch" + ], + "int_fields": [ + "total_raw_findings", + "confirmed", + "likely", + "inconclusive", + "not_exploitable", + "agent_invocations" + ], + "float_fields": [ + "noise_reduction_pct", + "duration_seconds", + "cost_usd" + ], + "null_fields": [ + "branch" + ], + "empty_list_fields": [ + "strategies_used", + "findings", + "attack_chains", + "compliance_gaps", + "policy_violations" + ], + "empty_dict_fields": [ + "by_severity", + "cost_breakdown", + "metadata" + ], + "scalar_defaults": { + "total_raw_findings": 0, + "confirmed": 0, + "likely": 0, + "inconclusive": 0, + "not_exploitable": 0, + "noise_reduction_pct": 0.0, + "duration_seconds": 0.0, + "agent_invocations": 0, + "cost_usd": 0.0 + }, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.output", + "python_class": "AuditProgress", + "go_name": "AuditProgress", + "duplicate_of": null, + "keys": [ + "phase", + "phase_progress", + "agents_total", + "agents_completed", + "agents_running", + "findings_so_far", + "elapsed_seconds", + "estimated_remaining_seconds", + "cost_so_far_usd" + ], + "required": [ + "phase", + "phase_progress", + "agents_total", + "agents_completed", + "agents_running", + "findings_so_far", + "elapsed_seconds", + "estimated_remaining_seconds", + "cost_so_far_usd" + ], + "accepts_null": [], + "int_fields": [ + "agents_total", + "agents_completed", + "agents_running", + "findings_so_far" + ], + "float_fields": [ + "phase_progress", + "elapsed_seconds", + "estimated_remaining_seconds", + "cost_so_far_usd" + ], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.output", + "python_class": "AuditMetrics", + "go_name": "AuditMetrics", + "duplicate_of": null, + "keys": [ + "duration_seconds", + "agent_invocations", + "cost_usd", + "cost_breakdown", + "budget_exhausted", + "findings_not_verified" + ], + "required": [ + "duration_seconds", + "agent_invocations", + "cost_usd" + ], + "accepts_null": [], + "int_fields": [ + "agent_invocations", + "findings_not_verified" + ], + "float_fields": [ + "duration_seconds", + "cost_usd" + ], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [ + "cost_breakdown" + ], + "scalar_defaults": { + "budget_exhausted": false, + "findings_not_verified": 0 + }, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "DataFlowTrace", + "go_name": "DataFlowTrace", + "duplicate_of": null, + "keys": [ + "source", + "sink", + "steps", + "sink_reached" + ], + "required": [ + "source", + "sink", + "steps", + "sink_reached" + ], + "accepts_null": [ + "source", + "sink", + "steps" + ], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "ReachabilityProof", + "go_name": "ReachabilityProof", + "duplicate_of": null, + "keys": [ + "vulnerable_function", + "call_chain", + "reachable", + "direct" + ], + "required": [ + "vulnerable_function", + "call_chain", + "reachable", + "direct" + ], + "accepts_null": [ + "vulnerable_function", + "call_chain" + ], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "SanitizationResult", + "go_name": "SanitizationResult", + "duplicate_of": null, + "keys": [ + "found", + "type", + "sufficient", + "bypass_method" + ], + "required": [ + "found" + ], + "accepts_null": [ + "type", + "sufficient", + "bypass_method" + ], + "int_fields": [], + "float_fields": [], + "null_fields": [ + "type", + "sufficient", + "bypass_method" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "ExploitHypothesis", + "go_name": "ExploitHypothesis", + "duplicate_of": null, + "keys": [ + "hypothesis", + "payload", + "expected_outcome" + ], + "required": [ + "hypothesis", + "expected_outcome" + ], + "accepts_null": [ + "hypothesis", + "payload", + "expected_outcome" + ], + "int_fields": [], + "float_fields": [], + "null_fields": [ + "payload" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "DastVerificationResult", + "go_name": "DastVerificationResult", + "duplicate_of": null, + "keys": [ + "payload_sent", + "response_summary", + "exploit_confirmed", + "safety_notes" + ], + "required": [ + "payload_sent", + "response_summary", + "exploit_confirmed", + "safety_notes" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "VerdictDecision", + "go_name": "VerdictDecision", + "duplicate_of": null, + "keys": [ + "verdict", + "evidence_level", + "rationale", + "confidence" + ], + "required": [ + "verdict", + "evidence_level", + "rationale", + "confidence" + ], + "accepts_null": [], + "int_fields": [ + "evidence_level" + ], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "RemediationSuggestion", + "go_name": "RemediationSuggestion", + "duplicate_of": null, + "keys": [ + "fix_description", + "patch_diff", + "confidence" + ], + "required": [ + "fix_description", + "patch_diff", + "confidence" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "DataFlowStep", + "go_name": "DataFlowStep", + "duplicate_of": null, + "keys": [ + "file", + "line", + "description", + "tainted" + ], + "required": [ + "file", + "line", + "description", + "tainted" + ], + "accepts_null": [], + "int_fields": [ + "line" + ], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "DataFlowEvidence", + "go_name": "DataFlowEvidence", + "duplicate_of": null, + "keys": [ + "steps", + "source", + "sink", + "sink_reached" + ], + "required": [], + "accepts_null": [ + "source", + "sink" + ], + "int_fields": [], + "float_fields": [], + "null_fields": [ + "source", + "sink" + ], + "empty_list_fields": [ + "steps" + ], + "empty_dict_fields": [], + "scalar_defaults": { + "sink_reached": false + }, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "SanitizationAnalysis", + "go_name": "SanitizationAnalysis", + "duplicate_of": null, + "keys": [ + "sanitization_found", + "sanitization_type", + "sanitization_sufficient", + "bypass_possible", + "bypass_method" + ], + "required": [ + "sanitization_found" + ], + "accepts_null": [ + "sanitization_type", + "sanitization_sufficient", + "bypass_possible", + "bypass_method" + ], + "int_fields": [], + "float_fields": [], + "null_fields": [ + "sanitization_type", + "sanitization_sufficient", + "bypass_possible", + "bypass_method" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "HttpEvidence", + "go_name": "HttpEvidence", + "duplicate_of": null, + "keys": [ + "method", + "url", + "headers", + "body", + "highlighted_segment" + ], + "required": [], + "accepts_null": [ + "method", + "url", + "headers", + "body", + "highlighted_segment" + ], + "int_fields": [], + "float_fields": [], + "null_fields": [ + "method", + "url", + "headers", + "body", + "highlighted_segment" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "ReachabilityEvidence", + "go_name": "ReachabilityEvidence", + "duplicate_of": null, + "keys": [ + "vulnerable_function", + "call_chain", + "reachable", + "direct_dependency" + ], + "required": [ + "vulnerable_function", + "reachable", + "direct_dependency" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [ + "call_chain" + ], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "ChainStep", + "go_name": "ChainStep", + "duplicate_of": null, + "keys": [ + "step_number", + "finding_id", + "description", + "enables" + ], + "required": [ + "step_number", + "finding_id", + "description", + "enables" + ], + "accepts_null": [], + "int_fields": [ + "step_number" + ], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "Proof", + "go_name": "Proof", + "duplicate_of": null, + "keys": [ + "exploit_hypothesis", + "verification_method", + "evidence_level", + "data_flow_trace", + "data_flow_evidence", + "sanitization_analysis", + "vulnerable_code", + "exploit_payload", + "expected_outcome", + "poc_code", + "poc_execution_output", + "http_request", + "http_response", + "reachability", + "chain_steps" + ], + "required": [ + "exploit_hypothesis", + "verification_method", + "evidence_level" + ], + "accepts_null": [ + "data_flow_trace", + "data_flow_evidence", + "sanitization_analysis", + "vulnerable_code", + "exploit_payload", + "expected_outcome", + "poc_code", + "poc_execution_output", + "http_request", + "http_response", + "reachability", + "chain_steps" + ], + "int_fields": [ + "evidence_level" + ], + "float_fields": [], + "null_fields": [ + "data_flow_trace", + "data_flow_evidence", + "sanitization_analysis", + "vulnerable_code", + "exploit_payload", + "expected_outcome", + "poc_code", + "poc_execution_output", + "http_request", + "http_response", + "reachability", + "chain_steps" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "ProverSignal", + "go_name": "ProverSignal", + "duplicate_of": null, + "keys": [ + "expand", + "expansion_reason", + "expansion_strategy", + "expansion_target" + ], + "required": [], + "accepts_null": [ + "expansion_reason", + "expansion_strategy", + "expansion_target" + ], + "int_fields": [], + "float_fields": [], + "null_fields": [ + "expansion_reason", + "expansion_strategy", + "expansion_target" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": { + "expand": false + }, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "VerifiedFinding", + "go_name": "VerifiedFinding", + "duplicate_of": null, + "keys": [ + "id", + "fingerprint", + "title", + "description", + "finding_type", + "cwe_id", + "cwe_name", + "owasp_category", + "tags", + "verdict", + "evidence_level", + "rationale", + "severity", + "cvss_v4", + "epss", + "exploitability_score", + "proof", + "location", + "related_locations", + "chain_id", + "chain_step", + "enables", + "compliance", + "reproduction_steps", + "remediation", + "sarif_rule_id", + "sarif_security_severity", + "drop_reason" + ], + "required": [ + "fingerprint", + "title", + "description", + "finding_type", + "cwe_id", + "cwe_name", + "verdict", + "evidence_level", + "rationale", + "severity", + "exploitability_score", + "location", + "sarif_rule_id", + "sarif_security_severity" + ], + "accepts_null": [ + "owasp_category", + "cvss_v4", + "epss", + "proof", + "chain_id", + "chain_step", + "enables", + "remediation", + "drop_reason" + ], + "int_fields": [ + "evidence_level", + "chain_step" + ], + "float_fields": [ + "exploitability_score", + "sarif_security_severity" + ], + "null_fields": [ + "owasp_category", + "cvss_v4", + "epss", + "proof", + "chain_id", + "chain_step", + "enables", + "remediation", + "drop_reason" + ], + "empty_list_fields": [ + "tags", + "related_locations", + "compliance", + "reproduction_steps" + ], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [ + "id" + ] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "Location", + "go_name": "Location", + "duplicate_of": "Location", + "keys": [ + "file_path", + "start_line", + "end_line", + "start_column", + "end_column", + "function_name", + "code_snippet" + ], + "required": [ + "file_path", + "start_line", + "end_line" + ], + "accepts_null": [ + "start_column", + "end_column", + "function_name", + "code_snippet" + ], + "int_fields": [ + "start_line", + "end_line", + "start_column", + "end_column" + ], + "float_fields": [], + "null_fields": [ + "start_column", + "end_column", + "function_name", + "code_snippet" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "CvssV4Score", + "go_name": "CvssV4Score", + "duplicate_of": "CvssV4Score", + "keys": [ + "vector", + "base_score", + "severity", + "automatable", + "subsequent_impact" + ], + "required": [ + "vector", + "base_score", + "severity", + "automatable", + "subsequent_impact" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [ + "base_score" + ], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "EpssScore", + "go_name": "EpssScore", + "duplicate_of": "EpssScore", + "keys": [ + "score", + "percentile", + "date" + ], + "required": [ + "score", + "percentile", + "date" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [ + "score", + "percentile" + ], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.prove", + "python_class": "ReproductionStep", + "go_name": "ReproductionStep", + "duplicate_of": "ReproductionStep", + "keys": [ + "step", + "description", + "command", + "expected_output" + ], + "required": [ + "step", + "description" + ], + "accepts_null": [ + "command", + "expected_output" + ], + "int_fields": [ + "step" + ], + "float_fields": [], + "null_fields": [ + "command", + "expected_output" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "Module", + "go_name": "Module", + "duplicate_of": null, + "keys": [ + "name", + "path", + "language", + "description", + "dependencies" + ], + "required": [ + "name", + "path", + "language" + ], + "accepts_null": [ + "description" + ], + "int_fields": [], + "float_fields": [], + "null_fields": [ + "description" + ], + "empty_list_fields": [ + "dependencies" + ], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "EntryPoint", + "go_name": "EntryPoint", + "duplicate_of": null, + "keys": [ + "kind", + "identifier", + "file_path", + "line", + "method", + "route", + "auth_required" + ], + "required": [ + "kind", + "identifier", + "file_path", + "line" + ], + "accepts_null": [ + "method", + "route", + "auth_required" + ], + "int_fields": [ + "line" + ], + "float_fields": [], + "null_fields": [ + "method", + "route", + "auth_required" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "TrustBoundary", + "go_name": "TrustBoundary", + "duplicate_of": null, + "keys": [ + "name", + "source_zone", + "target_zone", + "description", + "enforcement" + ], + "required": [ + "name", + "source_zone", + "target_zone", + "description" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [ + "enforcement" + ], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "Service", + "go_name": "Service", + "duplicate_of": null, + "keys": [ + "name", + "service_type", + "endpoint", + "purpose", + "auth_mechanism" + ], + "required": [ + "name", + "service_type" + ], + "accepts_null": [ + "endpoint", + "purpose", + "auth_mechanism" + ], + "int_fields": [], + "float_fields": [], + "null_fields": [ + "endpoint", + "purpose", + "auth_mechanism" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "APIEndpoint", + "go_name": "APIEndpoint", + "duplicate_of": null, + "keys": [ + "method", + "path", + "handler", + "file_path", + "line", + "auth_required", + "rate_limited" + ], + "required": [ + "method", + "path", + "handler", + "file_path", + "line" + ], + "accepts_null": [ + "auth_required", + "rate_limited" + ], + "int_fields": [ + "line" + ], + "float_fields": [], + "null_fields": [ + "auth_required", + "rate_limited" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "ArchitectureMap", + "go_name": "ArchitectureMap", + "duplicate_of": null, + "keys": [ + "app_type", + "modules", + "entry_points", + "trust_boundaries", + "services", + "api_surface" + ], + "required": [], + "accepts_null": [ + "app_type" + ], + "int_fields": [], + "float_fields": [], + "null_fields": [ + "app_type" + ], + "empty_list_fields": [ + "modules", + "entry_points", + "trust_boundaries", + "services", + "api_surface" + ], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "DataFlowStep", + "go_name": "ReconDataFlowStep", + "duplicate_of": null, + "keys": [ + "file_path", + "line", + "component", + "operation" + ], + "required": [ + "file_path", + "line", + "component", + "operation" + ], + "accepts_null": [], + "int_fields": [ + "line" + ], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "SanitizationPoint", + "go_name": "SanitizationPoint", + "duplicate_of": null, + "keys": [ + "file_path", + "line", + "function_name", + "sanitization_type", + "protects_against" + ], + "required": [ + "file_path", + "line", + "sanitization_type" + ], + "accepts_null": [ + "function_name" + ], + "int_fields": [ + "line" + ], + "float_fields": [], + "null_fields": [ + "function_name" + ], + "empty_list_fields": [ + "protects_against" + ], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "Sink", + "go_name": "Sink", + "duplicate_of": null, + "keys": [ + "sink_type", + "file_path", + "line", + "function_name", + "exploitability_notes" + ], + "required": [ + "sink_type", + "file_path", + "line" + ], + "accepts_null": [ + "function_name", + "exploitability_notes" + ], + "int_fields": [ + "line" + ], + "float_fields": [], + "null_fields": [ + "function_name", + "exploitability_notes" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "DataFlow", + "go_name": "DataFlow", + "duplicate_of": null, + "keys": [ + "source", + "path", + "sink", + "sanitized", + "files" + ], + "required": [ + "source", + "sink", + "sanitized" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [ + "path", + "files" + ], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "DataFlowMap", + "go_name": "DataFlowMap", + "duplicate_of": null, + "keys": [ + "flows", + "sanitization_points", + "sinks" + ], + "required": [], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [ + "flows", + "sanitization_points", + "sinks" + ], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "Dependency", + "go_name": "Dependency", + "duplicate_of": null, + "keys": [ + "name", + "version", + "ecosystem", + "direct", + "license" + ], + "required": [ + "name", + "version", + "ecosystem", + "direct" + ], + "accepts_null": [ + "license" + ], + "int_fields": [], + "float_fields": [], + "null_fields": [ + "license" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "KnownCVE", + "go_name": "KnownCVE", + "duplicate_of": null, + "keys": [ + "cve_id", + "package", + "installed_version", + "fixed_version", + "cvss_v4_score", + "epss_score", + "direct", + "reachable" + ], + "required": [ + "cve_id", + "package", + "installed_version", + "direct" + ], + "accepts_null": [ + "fixed_version", + "cvss_v4_score", + "epss_score", + "reachable" + ], + "int_fields": [], + "float_fields": [ + "cvss_v4_score", + "epss_score" + ], + "null_fields": [ + "fixed_version", + "cvss_v4_score", + "epss_score", + "reachable" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "OutdatedDep", + "go_name": "OutdatedDep", + "duplicate_of": null, + "keys": [ + "package", + "current_version", + "latest_version", + "direct" + ], + "required": [ + "package", + "current_version", + "latest_version", + "direct" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "DependencyReport", + "go_name": "DependencyReport", + "duplicate_of": null, + "keys": [ + "sbom", + "known_cves", + "outdated", + "direct_count", + "transitive_count" + ], + "required": [], + "accepts_null": [], + "int_fields": [ + "direct_count", + "transitive_count" + ], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [ + "sbom", + "known_cves", + "outdated" + ], + "empty_dict_fields": [], + "scalar_defaults": { + "direct_count": 0, + "transitive_count": 0 + }, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "SecretFinding", + "go_name": "SecretFinding", + "duplicate_of": null, + "keys": [ + "id", + "secret_type", + "file_path", + "line", + "match", + "confidence", + "is_test_value" + ], + "required": [ + "secret_type", + "file_path", + "line", + "match", + "confidence" + ], + "accepts_null": [ + "is_test_value" + ], + "int_fields": [ + "line" + ], + "float_fields": [], + "null_fields": [ + "is_test_value" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [ + "id" + ] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "MisconfigFinding", + "go_name": "MisconfigFinding", + "duplicate_of": null, + "keys": [ + "id", + "category", + "file_path", + "line", + "key", + "value", + "risk", + "remediation" + ], + "required": [ + "category", + "file_path", + "risk" + ], + "accepts_null": [ + "line", + "key", + "value", + "remediation" + ], + "int_fields": [ + "line" + ], + "float_fields": [], + "null_fields": [ + "line", + "key", + "value", + "remediation" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [ + "id" + ] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "ConfigReport", + "go_name": "ConfigReport", + "duplicate_of": null, + "keys": [ + "secrets", + "misconfigs" + ], + "required": [], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [ + "secrets", + "misconfigs" + ], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "CryptoUsage", + "go_name": "CryptoUsage", + "duplicate_of": null, + "keys": [ + "algorithm", + "key_size", + "mode", + "usage_context", + "is_weak" + ], + "required": [ + "algorithm" + ], + "accepts_null": [ + "key_size", + "mode", + "usage_context", + "is_weak" + ], + "int_fields": [ + "key_size" + ], + "float_fields": [], + "null_fields": [ + "key_size", + "mode", + "usage_context", + "is_weak" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "SecurityContext", + "go_name": "SecurityContext", + "duplicate_of": null, + "keys": [ + "auth_model", + "auth_details", + "crypto_usage", + "framework_security", + "security_headers", + "deployment_signals" + ], + "required": [ + "auth_model", + "auth_details" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [ + "crypto_usage", + "framework_security", + "security_headers", + "deployment_signals" + ], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "ReconResult", + "go_name": "ReconResult", + "duplicate_of": null, + "keys": [ + "architecture", + "data_flows", + "dependencies", + "config", + "security_context", + "languages", + "frameworks", + "lines_of_code", + "file_count", + "recon_duration_seconds" + ], + "required": [ + "architecture", + "data_flows", + "dependencies", + "config", + "security_context" + ], + "accepts_null": [], + "int_fields": [ + "lines_of_code", + "file_count" + ], + "float_fields": [ + "recon_duration_seconds" + ], + "null_fields": [], + "empty_list_fields": [ + "languages", + "frameworks" + ], + "empty_dict_fields": [], + "scalar_defaults": { + "lines_of_code": 0, + "file_count": 0, + "recon_duration_seconds": 0.0 + }, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "ArchitectureMapRaw", + "go_name": "ArchitectureMapRaw", + "duplicate_of": null, + "keys": [ + "app_type", + "modules", + "entry_points", + "trust_boundaries", + "services", + "api_endpoints" + ], + "required": [], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [ + "modules", + "entry_points", + "trust_boundaries", + "services", + "api_endpoints" + ], + "empty_dict_fields": [], + "scalar_defaults": { + "app_type": "unknown" + }, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "DataFlowMapRaw", + "go_name": "DataFlowMapRaw", + "duplicate_of": null, + "keys": [ + "flows", + "sanitization_points", + "sinks" + ], + "required": [], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [ + "flows", + "sanitization_points", + "sinks" + ], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "DependencyReportRaw", + "go_name": "DependencyReportRaw", + "duplicate_of": null, + "keys": [ + "sbom", + "known_cves", + "outdated" + ], + "required": [], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [ + "sbom", + "known_cves", + "outdated" + ], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "ConfigReportRaw", + "go_name": "ConfigReportRaw", + "duplicate_of": null, + "keys": [ + "secrets", + "misconfigs" + ], + "required": [], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [ + "secrets", + "misconfigs" + ], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.recon", + "python_class": "SecurityContextRaw", + "go_name": "SecurityContextRaw", + "duplicate_of": null, + "keys": [ + "auth_model", + "auth_details", + "crypto_usage", + "security_signals" + ], + "required": [ + "auth_model" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [ + "crypto_usage", + "security_signals" + ], + "empty_dict_fields": [], + "scalar_defaults": { + "auth_details": "" + }, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.views", + "python_class": "FindingForVerifier", + "go_name": "FindingForVerifier", + "duplicate_of": null, + "keys": [ + "id", + "title", + "description", + "file_path", + "start_line", + "end_line", + "code_snippet", + "cwe_id", + "function_name", + "data_flow_summary" + ], + "required": [ + "id", + "title", + "description", + "file_path", + "start_line", + "end_line", + "code_snippet", + "cwe_id" + ], + "accepts_null": [ + "function_name" + ], + "int_fields": [ + "start_line", + "end_line" + ], + "float_fields": [], + "null_fields": [ + "function_name" + ], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": { + "data_flow_summary": "" + }, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.views", + "python_class": "FindingForDedup", + "go_name": "FindingForDedup", + "duplicate_of": null, + "keys": [ + "id", + "fingerprint", + "title", + "file_path", + "start_line", + "cwe_id", + "finding_type", + "estimated_severity" + ], + "required": [ + "id", + "fingerprint", + "title", + "file_path", + "start_line", + "cwe_id", + "finding_type", + "estimated_severity" + ], + "accepts_null": [], + "int_fields": [ + "start_line" + ], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.schemas.views", + "python_class": "FindingForReachability", + "go_name": "FindingForReachability", + "duplicate_of": null, + "keys": [ + "title", + "description", + "cwe_id", + "file_path", + "start_line", + "verdict" + ], + "required": [ + "title", + "description", + "cwe_id", + "file_path", + "start_line", + "verdict" + ], + "accepts_null": [], + "int_fields": [ + "start_line" + ], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + }, + { + "python_module": "sec_af.policies", + "python_class": "PolicyEvalResult", + "go_name": "PolicyEvalResult", + "duplicate_of": null, + "keys": [ + "violated", + "description", + "file_path", + "severity" + ], + "required": [ + "violated", + "description", + "file_path", + "severity" + ], + "accepts_null": [], + "int_fields": [], + "float_fields": [], + "null_fields": [], + "empty_list_fields": [], + "empty_dict_fields": [], + "scalar_defaults": {}, + "uuid_defaults": [] + } + ], + "enums": { + "FindingType": { + "python_module": "sec_af.schemas.hunt", + "members": { + "SAST": "sast", + "SCA": "sca", + "SECRETS": "secrets", + "CONFIG": "config", + "LOGIC": "logic", + "API": "api" + }, + "kind": "str" + }, + "Severity": { + "python_module": "sec_af.schemas.hunt", + "members": { + "CRITICAL": "critical", + "HIGH": "high", + "MEDIUM": "medium", + "LOW": "low", + "INFO": "info" + }, + "kind": "str" + }, + "Confidence": { + "python_module": "sec_af.schemas.hunt", + "members": { + "HIGH": "high", + "MEDIUM": "medium", + "LOW": "low" + }, + "kind": "str" + }, + "HuntStrategy": { + "python_module": "sec_af.schemas.hunt", + "members": { + "INJECTION": "injection", + "XSS": "xss", + "DOS": "dos", + "SSRF": "ssrf", + "AUTH": "auth", + "CRYPTO": "crypto", + "BUSINESS_LOGIC": "business_logic", + "LOGIC_BUGS": "business_logic", + "DATA_EXPOSURE": "data_exposure", + "SUPPLY_CHAIN": "supply_chain", + "CONFIG_SECRETS": "config_secrets", + "API_SECURITY": "api_security", + "PYTHON_SPECIFIC": "python_specific", + "JAVASCRIPT_SPECIFIC": "javascript_specific" + }, + "kind": "str" + }, + "Verdict": { + "python_module": "sec_af.schemas.prove", + "members": { + "CONFIRMED": "confirmed", + "LIKELY": "likely", + "INCONCLUSIVE": "inconclusive", + "NOT_EXPLOITABLE": "not_exploitable" + }, + "kind": "str" + }, + "EvidenceLevel": { + "python_module": "sec_af.schemas.prove", + "members": { + "STATIC_MATCH": 1, + "FLOW_IDENTIFIED": 2, + "REACHABILITY_CONFIRMED": 3, + "SANITIZATION_BYPASSABLE": 4, + "EXPLOIT_SCENARIO_VALIDATED": 5, + "FULL_EXPLOIT": 6 + }, + "kind": "int" + } + }, + "schemas_all": [ + "APIEndpoint", + "ArchitectureMap", + "AttackChain", + "AuditInput", + "AuditMetrics", + "AuditProgress", + "ChainStep", + "ComplianceGate", + "ComplianceGap", + "ComplianceMapping", + "ComplianceSuggestion", + "Confidence", + "ConfigReport", + "CryptoUsage", + "CvssV4Score", + "DataFlow", + "DataFlowEvidence", + "DataFlowMap", + "DataFlowStep", + "DeduplicatedResult", + "Dependency", + "DependencyReport", + "DuplicateCheck", + "EntryPoint", + "EpssScore", + "EvidenceLevel", + "FindingForDedup", + "FindingForReachability", + "FindingForVerifier", + "FindingType", + "HttpEvidence", + "HuntResult", + "HuntStrategy", + "KnownCVE", + "Location", + "MisconfigFinding", + "MitreMapping", + "Module", + "OutdatedDep", + "PotentialChain", + "Proof", + "ProverSignal", + "RawFinding", + "ReachabilityEvidence", + "ReconDataFlowStep", + "ReconResult", + "RelevanceGate", + "ReproductionStep", + "SanitizationAnalysis", + "SanitizationPoint", + "SecretFinding", + "SecurityAuditResult", + "SecurityContext", + "Service", + "Severity", + "SeverityClassification", + "Sink", + "StrategySelection", + "TrustBoundary", + "Verdict", + "VerdictGate", + "VerifiedFinding" + ] +} diff --git a/internal/secaf/schemas/timestamp.go b/internal/secaf/schemas/timestamp.go new file mode 100644 index 0000000000..d4c9876d95 --- /dev/null +++ b/internal/secaf/schemas/timestamp.go @@ -0,0 +1,129 @@ +package schemas + +import ( + "encoding/json" + "fmt" + "math" + "strings" + "time" +) + +// Timestamp is the JSON representation of a pydantic `datetime` field +// (`SecurityAuditResult.timestamp`). +// +// Python parity (docs/DESIGN.md §2, VERIFIED): a reasoner returns +// `model_dump()`, which leaves the field as a real datetime object; the +// AgentField Python SDK then hands the result to FastAPI, whose +// `jsonable_encoder` serialises a datetime with `datetime.isoformat()`. That +// produces +// +// 2026-01-02T03:04:05.123456+00:00 (microseconds present) +// 2026-01-02T03:04:05+00:00 (microseconds exactly zero -> omitted) +// +// i.e. always a numeric UTC offset (never the "Z" shorthand), and a fractional +// part that is either absent or exactly six digits — `isoformat()` never emits +// 1, 3 or 9 digits, and never a `.000000` fraction (a zero microsecond +// component means no fraction at all). +// +// UnmarshalJSON is deliberately more permissive than MarshalJSON: it accepts +// RFC 3339 with or without a fractional part, with "Z" or a numeric offset, and +// a naive (offset-less) timestamp — pydantic accepts all of those, and +// `model_dump(mode="json")` (used by the schema-fixture generator) emits the +// "Z" form. +type Timestamp struct { + time.Time +} + +// NewTimestamp wraps t. +func NewTimestamp(t time.Time) Timestamp { return Timestamp{Time: t} } + +// pythonISOLayoutMicros is `datetime.isoformat()` with microseconds. +const pythonISOLayoutMicros = "2006-01-02T15:04:05.000000-07:00" + +// pythonISOLayoutSeconds is `datetime.isoformat()` when microseconds are zero. +const pythonISOLayoutSeconds = "2006-01-02T15:04:05-07:00" + +// String renders the timestamp exactly as Python's `datetime.isoformat()`. +// +// The guard tests the MICROSECOND component, not the nanosecond one. Go's clock +// has nanosecond resolution while `datetime` has only microsecond resolution +// (`datetime.resolution == 1µs`), so a reading whose sub-second part is 1-999ns +// has a microsecond component of zero: Python prints no fraction, and a +// `t.Nanosecond() == 0` guard would print `.000000`. The window is the first +// microsecond of every second, and the producer is an untruncated +// `time.Now().UTC()` (orch.nowUTC), so it reaches the audit result timestamp, +// the checkpoint `created_at`, the SARIF automationDetails.id and the +// monitoring baseline. +func (t Timestamp) String() string { + truncated := t.Truncate(time.Microsecond) + if truncated.Nanosecond() == 0 { + return truncated.Format(pythonISOLayoutSeconds) + } + // Python parity: isoformat() emits microseconds, so truncate the extra + // nanosecond digits Go carries rather than rounding them. + return truncated.Format(pythonISOLayoutMicros) +} + +// MarshalJSON emits the `datetime.isoformat()` string as a JSON string. +func (t Timestamp) MarshalJSON() ([]byte, error) { + return json.Marshal(t.String()) +} + +// timestampLayouts are tried in order by UnmarshalJSON. time.RFC3339Nano +// handles both "Z" and numeric offsets with or without a fraction; the rest +// cover the naive (offset-less) and space-separated forms pydantic also +// accepts (`datetime(...)` from "2026-01-02 03:04:05+00:00" parses fine). +var timestampLayouts = []string{ + time.RFC3339Nano, + "2006-01-02T15:04:05.999999999", + "2006-01-02T15:04:05", + "2006-01-02 15:04:05.999999999Z07:00", + "2006-01-02 15:04:05Z07:00", + "2006-01-02 15:04:05.999999999", + "2006-01-02 15:04:05", + "2006-01-02", +} + +// UnmarshalJSON accepts everything pydantic accepts for a datetime field: +// +// - a JSON string in any of timestampLayouts (the wire form is always the +// first one), +// - a JSON number, read as a Unix epoch in seconds with a fractional part +// carrying sub-second precision — VERIFIED: pydantic turns 1767322445 into +// 2026-01-02T02:54:05+00:00 and 1767322445.5 into ...:05.500000+00:00, +// - JSON null, which leaves the zero Timestamp (Go-only: the field is +// required in pydantic, but a null must not fail the whole decode). +func (t *Timestamp) UnmarshalJSON(b []byte) error { + trimmed := strings.TrimSpace(string(b)) + if trimmed == "null" { + *t = Timestamp{} + return nil + } + if len(trimmed) > 0 && trimmed[0] != '"' { + var epoch float64 + if err := json.Unmarshal(b, &epoch); err != nil { + return fmt.Errorf("schemas.Timestamp: expected a JSON string or epoch number: %w", err) + } + sec, frac := math.Modf(epoch) + // Python parity: pydantic keeps microsecond precision on a float epoch. + nsec := math.Round(frac*1e6) * 1e3 + *t = Timestamp{Time: time.Unix(int64(sec), int64(nsec)).UTC()} + return nil + } + var s string + if err := json.Unmarshal(b, &s); err != nil { + return fmt.Errorf("schemas.Timestamp: expected a JSON string: %w", err) + } + s = strings.TrimSpace(s) + if s == "" { + *t = Timestamp{} + return nil + } + for _, layout := range timestampLayouts { + if parsed, err := time.Parse(layout, s); err == nil { + *t = Timestamp{Time: parsed} + return nil + } + } + return fmt.Errorf("schemas.Timestamp: cannot parse %q as an ISO-8601 datetime", s) +} diff --git a/internal/secaf/schemas/timestamp_test.go b/internal/secaf/schemas/timestamp_test.go new file mode 100644 index 0000000000..d132b99c9d --- /dev/null +++ b/internal/secaf/schemas/timestamp_test.go @@ -0,0 +1,241 @@ +package schemas + +import ( + "encoding/json" + "testing" + "time" +) + +// Every expectation here was produced by the CPython 3.11 datetime the port +// mirrors, e.g. +// +// datetime(2026,1,2,3,4,5,123456,tzinfo=UTC).isoformat() +// -> '2026-01-02T03:04:05.123456+00:00' +// +// See docs/DESIGN.md §2 (VERIFIED) and timestamp.go. + +func TestTimestampMarshalMatchesPythonIsoformat(t *testing.T) { + cases := []struct { + name string + in time.Time + want string + }{ + { + "microseconds present", + time.Date(2026, 1, 2, 3, 4, 5, 123456000, time.UTC), + `"2026-01-02T03:04:05.123456+00:00"`, + }, + { + // Python parity: isoformat() omits the fraction entirely when + // microseconds are exactly 0 — it never writes ".000000". + "microseconds zero", + time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), + `"2026-01-02T03:04:05+00:00"`, + }, + { + // ...and never trims to fewer than 6 digits when non-zero. + "trailing-zero microseconds keep all six digits", + time.Date(2026, 1, 2, 3, 4, 5, 100000000, time.UTC), + `"2026-01-02T03:04:05.100000+00:00"`, + }, + { + "one microsecond", + time.Date(2026, 1, 2, 3, 4, 5, 1000, time.UTC), + `"2026-01-02T03:04:05.000001+00:00"`, + }, + { + // Python parity: a numeric offset, never "Z". + "non-UTC offset", + time.Date(2026, 1, 2, 3, 4, 5, 123456000, time.FixedZone("", -5*3600)), + `"2026-01-02T03:04:05.123456-05:00"`, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := json.Marshal(NewTimestamp(tc.in)) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if string(got) != tc.want { + t.Errorf("got %s, want %s", got, tc.want) + } + }) + } +} + +func TestTimestampMarshalTruncatesSubMicrosecond(t *testing.T) { + // Go carries nanoseconds; Python's isoformat() only has microseconds, and + // truncates rather than rounds. + ts := NewTimestamp(time.Date(2026, 1, 2, 3, 4, 5, 123456999, time.UTC)) + got, err := json.Marshal(ts) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if string(got) != `"2026-01-02T03:04:05.123456+00:00"` { + t.Errorf("got %s, want the microsecond-truncated form", got) + } +} + +// TestTimestampSubMicrosecondNanosOmitTheFraction is the 1-999ns window. +// +// Go's time.Now has nanosecond resolution, Python's datetime only microsecond +// resolution (`datetime.resolution == 1µs`), so a reading with 1-999ns has a +// ZERO microsecond component. VERIFIED on the pinned interpreter: +// +// datetime(2026,1,2,3,4,5,0,tzinfo=UTC).isoformat() -> "2026-01-02T03:04:05+00:00" +// +// — no fraction at all. `isoformat()` cannot emit ".000000". +func TestTimestampSubMicrosecondNanosOmitTheFraction(t *testing.T) { + cases := []struct { + nanos int + want string + }{ + {0, "2026-01-02T03:04:05+00:00"}, + {1, "2026-01-02T03:04:05+00:00"}, + {500, "2026-01-02T03:04:05+00:00"}, + {999, "2026-01-02T03:04:05+00:00"}, + {1000, "2026-01-02T03:04:05.000001+00:00"}, + {123456000, "2026-01-02T03:04:05.123456+00:00"}, + } + for _, tc := range cases { + ts := NewTimestamp(time.Date(2026, 1, 2, 3, 4, 5, tc.nanos, time.UTC)) + if got := ts.String(); got != tc.want { + t.Errorf("%dns -> %q, want %q", tc.nanos, got, tc.want) + } + } +} + +func TestTimestampUnmarshalAcceptsBothRepresentations(t *testing.T) { + want := time.Date(2026, 1, 2, 3, 4, 5, 123456000, time.UTC) + // The two representations the port must tolerate: FastAPI's + // jsonable_encoder form (numeric offset) and pydantic's + // model_dump(mode="json") form ("Z"), plus the fraction-free variants. + for _, in := range []string{ + `"2026-01-02T03:04:05.123456+00:00"`, + `"2026-01-02T03:04:05.123456Z"`, + `"2026-01-02T03:04:05.123456"`, + } { + var ts Timestamp + if err := json.Unmarshal([]byte(in), &ts); err != nil { + t.Fatalf("unmarshal %s: %v", in, err) + } + if !ts.UTC().Equal(want) { + t.Errorf("unmarshal %s = %v, want %v", in, ts.Time, want) + } + } + + secondsOnly := time.Date(2026, 3, 4, 10, 30, 0, 0, time.UTC) + for _, in := range []string{ + `"2026-03-04T10:30:00+00:00"`, + `"2026-03-04T10:30:00Z"`, + `"2026-03-04T10:30:00"`, + } { + var ts Timestamp + if err := json.Unmarshal([]byte(in), &ts); err != nil { + t.Fatalf("unmarshal %s: %v", in, err) + } + if !ts.UTC().Equal(secondsOnly) { + t.Errorf("unmarshal %s = %v, want %v", in, ts.Time, secondsOnly) + } + } +} + +func TestTimestampRoundTrip(t *testing.T) { + for _, in := range []string{ + `"2026-01-02T03:04:05.123456+00:00"`, + `"2026-03-04T10:30:00+00:00"`, + `"2026-01-02T03:04:05.123456-05:00"`, + } { + var ts Timestamp + if err := json.Unmarshal([]byte(in), &ts); err != nil { + t.Fatalf("unmarshal %s: %v", in, err) + } + out, err := json.Marshal(ts) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if string(out) != in { + t.Errorf("round trip %s -> %s", in, out) + } + } + // The "Z" form normalises to the isoformat form on the way out — the port + // always EMITS what Python emits. + var ts Timestamp + if err := json.Unmarshal([]byte(`"2026-01-02T03:04:05.123456Z"`), &ts); err != nil { + t.Fatal(err) + } + if out, _ := json.Marshal(ts); string(out) != `"2026-01-02T03:04:05.123456+00:00"` { + t.Errorf("Z form re-emitted as %s", out) + } +} + +func TestTimestampUnmarshalNullAndErrors(t *testing.T) { + var ts Timestamp + if err := json.Unmarshal([]byte(`null`), &ts); err != nil { + t.Errorf("null: %v", err) + } + if !ts.IsZero() { + t.Errorf("null gave %v, want the zero Timestamp", ts.Time) + } + if err := json.Unmarshal([]byte(`"not a date"`), &ts); err == nil { + t.Error("a garbage string should fail to parse") + } + if err := json.Unmarshal([]byte(`{}`), &ts); err == nil { + t.Error("a JSON object should fail") + } +} + +// TestTimestampUnmarshalEpochNumber matches pydantic, which reads a bare number +// as a Unix epoch. VERIFIED against the live model: +// +// SecurityAuditResult(timestamp=1767322445).timestamp.isoformat() +// -> '2026-01-02T02:54:05+00:00' +// SecurityAuditResult(timestamp=1767322445.5).timestamp.isoformat() +// -> '2026-01-02T02:54:05.500000+00:00' +func TestTimestampUnmarshalEpochNumber(t *testing.T) { + cases := []struct{ in, want string }{ + {`1767322445`, `"2026-01-02T02:54:05+00:00"`}, + {`1767322445.5`, `"2026-01-02T02:54:05.500000+00:00"`}, + } + for _, tc := range cases { + var ts Timestamp + if err := json.Unmarshal([]byte(tc.in), &ts); err != nil { + t.Fatalf("unmarshal %s: %v", tc.in, err) + } + got, err := json.Marshal(ts) + if err != nil { + t.Fatal(err) + } + if string(got) != tc.want { + t.Errorf("epoch %s -> %s, want %s", tc.in, got, tc.want) + } + } +} + +// TestTimestampUnmarshalSpaceSeparated matches pydantic, which accepts +// "2026-01-02 03:04:05+00:00" as well as the "T"-separated form. +func TestTimestampUnmarshalSpaceSeparated(t *testing.T) { + var ts Timestamp + if err := json.Unmarshal([]byte(`"2026-01-02 03:04:05+00:00"`), &ts); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if got, _ := json.Marshal(ts); string(got) != `"2026-01-02T03:04:05+00:00"` { + t.Errorf("space-separated re-emitted as %s", got) + } +} + +func TestTimestampInsideSecurityAuditResult(t *testing.T) { + r := NewSecurityAuditResult() + r.Timestamp = NewTimestamp(time.Date(2026, 3, 4, 10, 30, 0, 0, time.UTC)) + b, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + var got map[string]any + if err := json.Unmarshal(b, &got); err != nil { + t.Fatal(err) + } + if got["timestamp"] != "2026-03-04T10:30:00+00:00" { + t.Errorf("timestamp = %#v", got["timestamp"]) + } +} diff --git a/internal/secaf/schemas/uuid.go b/internal/secaf/schemas/uuid.go new file mode 100644 index 0000000000..2058ec0a78 --- /dev/null +++ b/internal/secaf/schemas/uuid.go @@ -0,0 +1,40 @@ +package schemas + +import ( + "crypto/rand" + "encoding/hex" + "fmt" +) + +// NewUUID4 returns a random RFC 4122 version-4 UUID in the canonical +// 8-4-4-4-12 lowercase hex form, e.g. "1b4e28ba-2fa1-4d1d-883f-9e0d5a1e0f2b". +// +// Ports Python's `default_factory=lambda: str(uuid4())` (schemas/hunt.py, +// schemas/recon.py, schemas/prove.py). The port takes no new third-party +// dependency (docs/DESIGN.md §0.6), so this builds the value from crypto/rand +// directly: 16 random bytes with the version nibble forced to 4 and the variant +// bits forced to 10xx, exactly as CPython's `uuid.uuid4()` does. +// +// It panics only if the system CSPRNG fails, which crypto/rand documents as +// unrecoverable — the same posture as `uuid.uuid4()`, which has no failure mode +// a caller could act on. +func NewUUID4() string { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + panic(fmt.Sprintf("schemas.NewUUID4: crypto/rand failed: %v", err)) + } + b[6] = (b[6] & 0x0f) | 0x40 // version 4 + b[8] = (b[8] & 0x3f) | 0x80 // variant 10xx (RFC 4122) + + var out [36]byte + hex.Encode(out[0:8], b[0:4]) + out[8] = '-' + hex.Encode(out[9:13], b[4:6]) + out[13] = '-' + hex.Encode(out[14:18], b[6:8]) + out[18] = '-' + hex.Encode(out[19:23], b[8:10]) + out[23] = '-' + hex.Encode(out[24:36], b[10:16]) + return string(out[:]) +} diff --git a/internal/secaf/schemas/uuid_test.go b/internal/secaf/schemas/uuid_test.go new file mode 100644 index 0000000000..a1dbeb748d --- /dev/null +++ b/internal/secaf/schemas/uuid_test.go @@ -0,0 +1,55 @@ +package schemas + +import ( + "regexp" + "testing" +) + +// uuid4Re matches the canonical RFC 4122 v4 form CPython's `str(uuid4())` +// produces: 32 lowercase hex digits in 8-4-4-4-12 groups, version nibble 4, +// variant nibble 8|9|a|b. +var uuid4Re = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`) + +func TestNewUUID4Format(t *testing.T) { + for i := 0; i < 200; i++ { + got := NewUUID4() + if len(got) != 36 { + t.Fatalf("NewUUID4() = %q, want 36 chars", got) + } + if !uuid4Re.MatchString(got) { + t.Fatalf("NewUUID4() = %q, not an RFC 4122 v4 uuid", got) + } + } +} + +func TestNewUUID4IsUnique(t *testing.T) { + seen := make(map[string]struct{}, 1000) + for i := 0; i < 1000; i++ { + v := NewUUID4() + if _, dup := seen[v]; dup { + t.Fatalf("NewUUID4() repeated %q after %d draws", v, i) + } + seen[v] = struct{}{} + } +} + +func TestConstructorsMintDistinctUUIDs(t *testing.T) { + // Python parity: each default_factory=lambda: str(uuid4()) call is + // independent, so RawFinding gets two DIFFERENT uuids. + rf := NewRawFinding() + if rf.ID == rf.Fingerprint { + t.Errorf("NewRawFinding minted the same uuid twice: %q", rf.ID) + } + for name, got := range map[string]string{ + "RawFinding.ID": rf.ID, + "RawFinding.Fingerprint": rf.Fingerprint, + "PotentialChain.ChainID": NewPotentialChain().ChainID, + "SecretFinding.ID": NewSecretFinding().ID, + "MisconfigFinding.ID": NewMisconfigFinding().ID, + "VerifiedFinding.ID": NewVerifiedFinding().ID, + } { + if !uuid4Re.MatchString(got) { + t.Errorf("%s = %q, want a uuid4", name, got) + } + } +} diff --git a/internal/secaf/schemas/views.go b/internal/secaf/schemas/views.go new file mode 100644 index 0000000000..82a50b3e87 --- /dev/null +++ b/internal/secaf/schemas/views.go @@ -0,0 +1,52 @@ +package schemas + +// This file ports src/sec_af/schemas/views.py — the phase-boundary projections +// that give each consumer only the fields it needs (contextual fidelity). +// The builders live on RawFinding (hunt.go: ForVerifier / ForDedup). + +// FindingForVerifier is what the verifier pipeline needs from a RawFinding. +// +// Ports schemas/views.py FindingForVerifier. Seeded (defaults.go): +// data_flow_summary="" — which is the Go zero value, so the struct needs no +// constructor; it is listed here for completeness. +type FindingForVerifier struct { + ID string `json:"id"` + Title string `json:"title"` + Description string `json:"description"` + FilePath string `json:"file_path"` + StartLine int `json:"start_line"` + EndLine int `json:"end_line"` + CodeSnippet string `json:"code_snippet"` + CweID string `json:"cwe_id"` + FunctionName *string `json:"function_name"` + // DataFlowSummary is the flattened data flow path. + DataFlowSummary string `json:"data_flow_summary"` +} + +// FindingForDedup is what the deduplicator needs. 8 fields. +// +// Ports schemas/views.py FindingForDedup. Every field is required. +type FindingForDedup struct { + ID string `json:"id"` + Fingerprint string `json:"fingerprint"` + Title string `json:"title"` + FilePath string `json:"file_path"` + StartLine int `json:"start_line"` + CweID string `json:"cwe_id"` + FindingType string `json:"finding_type"` + EstimatedSeverity string `json:"estimated_severity"` +} + +// FindingForReachability is what the reachability gate needs. 6 fields. +// +// Ports schemas/views.py FindingForReachability. (Its docstring says "5 +// fields"; the class declares 6 — Python parity: the docstring is stale, the +// field list is authoritative.) Every field is required. +type FindingForReachability struct { + Title string `json:"title"` + Description string `json:"description"` + CweID string `json:"cwe_id"` + FilePath string `json:"file_path"` + StartLine int `json:"start_line"` + Verdict string `json:"verdict"` +} diff --git a/internal/secaf/scope.go b/internal/secaf/scope.go new file mode 100644 index 0000000000..5dea1fe84b --- /dev/null +++ b/internal/secaf/scope.go @@ -0,0 +1,92 @@ +package secaf + +// What a brief asks the audit to look at. +// +// A run started from the chat is handed nothing but its brief — the flags of a +// shell line never reach it — so the brief carries the scope in a few plain +// words, the ones the guide teaches the model (internal/programguide): +// `whole repository`, or `changes` with an optional `since `, and +// optionally `quick` or `thorough`. Anything else in the brief is the person's +// own words about the audit, kept on the report. +// +// A BRIEF THAT SAYS NOTHING ABOUT SCOPE IS THE WHOLE REPOSITORY. That is what +// sec-af always audited, and an audit that guessed at a narrower scope would +// report less than was asked without saying so. + +import ( + "regexp" + "strings" +) + +// Scope is the audit a brief asks for. +type Scope struct { + // Changes is an audit of what changed since Base rather than of the whole + // repository. + Changes bool + // Base is the revision the changes are measured from; empty is the + // branch's own base, found when the audit starts ([findBase]). + Base string + // Depth is `quick`, `standard` or `thorough`. + Depth string + // Words is the brief as the person or the conversation wrote it. + Words string +} + +var ( + // changesWord opens a brief that asks for the changes: `changes`, `the + // changes`, `diff`, `changes on this branch`, `changes since main`. + changesWord = regexp.MustCompile(`(?i)^\s*(?:only\s+)?(?:the\s+)?(?:changes|changed\s+files|diff)\b`) + // sinceRef is the base a brief names: `since `, `against `, + // `from `, `vs `. + sinceRef = regexp.MustCompile(`(?i)\b(?:since|against|from|vs\.?|versus|relative\s+to)\s+([^\s,;]+)`) + // depthWord is a depth the brief names anywhere. + depthWord = regexp.MustCompile(`(?i)\b(quick|thorough)\b`) +) + +// ReadScope reads a brief, with the shell's own flags over it: a flag the +// person typed wins over a word in the brief. +func ReadScope(brief string, changesFlag bool, baseFlag, depthFlag string) Scope { + scope := Scope{Depth: "standard", Words: strings.TrimSpace(brief)} + first := firstLine(scope.Words) + if changesWord.MatchString(first) { + scope.Changes = true + if match := sinceRef.FindStringSubmatch(first); match != nil { + scope.Base = strings.Trim(match[1], "`'\".") + } + } + if match := depthWord.FindStringSubmatch(first); match != nil { + scope.Depth = strings.ToLower(match[1]) + } + if changesFlag || strings.TrimSpace(baseFlag) != "" { + scope.Changes = true + } + if base := strings.TrimSpace(baseFlag); base != "" { + scope.Base = base + } + if depth := strings.ToLower(strings.TrimSpace(depthFlag)); depth != "" { + scope.Depth = depth + } + return scope +} + +// Describe is the scope in a few words, for the first line of the report. +func (s Scope) Describe() string { + what := "the whole repository" + if s.Changes { + what = "the changes on this branch" + if s.Base != "" { + what = "the changes since " + s.Base + } + } + if s.Depth != "" && s.Depth != "standard" { + what += ", " + s.Depth + } + return what +} + +func firstLine(text string) string { + if at := strings.IndexByte(text, '\n'); at >= 0 { + return text[:at] + } + return text +} diff --git a/internal/secaf/scoring/scoring.go b/internal/secaf/scoring/scoring.go new file mode 100644 index 0000000000..4d8bec8d40 --- /dev/null +++ b/internal/secaf/scoring/scoring.go @@ -0,0 +1,238 @@ +// Package scoring ports src/sec_af/scoring.py — the deterministic +// exploitability scoring engine: the severity/evidence/reachability weight +// tables, the CWE severity floors, and the score → label mapping. +// +// Everything here is pure: no LLM, no I/O. The orchestrator calls +// ComputeExploitabilityScore for every VerifiedFinding in _generate_output and +// ApplyCWESeverityFloor before that. +package scoring + +import ( + "sort" + "strings" + + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// SeverityWeights is the base weight per severity label. +// +// Ports scoring.py SEVERITY_WEIGHTS. Keyed by the ENUM VALUE (Python indexes +// it with `finding.severity.value`), which is what schemas.Severity already is. +var SeverityWeights = map[schemas.Severity]float64{ + schemas.SeverityCritical: 10.0, + schemas.SeverityHigh: 8.0, + schemas.SeverityMedium: 5.0, + schemas.SeverityLow: 3.0, + schemas.SeverityInfo: 1.0, +} + +// EvidenceMultipliers scales the base weight by how strong the proof is. +// +// Ports scoring.py EVIDENCE_MULTIPLIERS (declared there in decreasing order; +// the map has no order, the values are what matter). +var EvidenceMultipliers = map[schemas.EvidenceLevel]float64{ + schemas.EvidenceLevelFullExploit: 1.0, + schemas.EvidenceLevelExploitScenarioValidated: 0.9, + schemas.EvidenceLevelSanitizationBypassable: 0.7, + schemas.EvidenceLevelReachabilityConfirmed: 0.5, + schemas.EvidenceLevelFlowIdentified: 0.3, + schemas.EvidenceLevelStaticMatch: 0.1, +} + +// ReachabilityMultipliers scales the score by how exposed the code path is. +// +// Ports scoring.py REACHABILITY_MULTIPLIERS. +var ReachabilityMultipliers = map[string]float64{ + "externally_reachable": 1.0, + "internally_reachable": 0.7, + "requires_auth": 0.5, + "requires_admin": 0.3, +} + +// reachabilityTagOrder is the order _reachability_multiplier probes the tag +// set. Order matters: a finding tagged both "externally_reachable" and +// "requires_auth" scores as externally reachable. +// +// Ports the tuple literal in scoring.py _reachability_multiplier. +var reachabilityTagOrder = []string{ + "externally_reachable", + "internally_reachable", + "requires_auth", + "requires_admin", +} + +// severityOrder ranks the severity labels so a floor can be compared against a +// current severity. +// +// Ports scoring.py _SEVERITY_ORDER. +var severityOrder = map[string]int{ + "critical": 4, + "high": 3, + "medium": 2, + "low": 1, + "info": 0, +} + +// CWESeverityFloor is the minimum severity for well-known vulnerability +// classes. LLMs consistently underrate injection and RCE, so this is a hard +// floor: CWE-78 can never be reported as "medium". +// +// Ports scoring.py CWE_SEVERITY_FLOOR — byte-for-byte the same 18 entries. +var CWESeverityFloor = map[string]schemas.Severity{ + // Remote Code Execution / Command Injection — always critical + "CWE-78": schemas.SeverityCritical, + "CWE-77": schemas.SeverityCritical, + "CWE-94": schemas.SeverityCritical, + "CWE-95": schemas.SeverityCritical, + "CWE-96": schemas.SeverityCritical, + // SQL Injection — always critical + "CWE-89": schemas.SeverityCritical, + // Deserialization — always critical + "CWE-502": schemas.SeverityCritical, + // SSRF — at least high + "CWE-918": schemas.SeverityHigh, + // Authentication Bypass — at least high + "CWE-287": schemas.SeverityHigh, + "CWE-290": schemas.SeverityHigh, + "CWE-306": schemas.SeverityHigh, + // Hardcoded Credentials — at least high + "CWE-798": schemas.SeverityHigh, + // Path Traversal — at least high + "CWE-22": schemas.SeverityHigh, + // XXE — at least high + "CWE-611": schemas.SeverityHigh, + // XSS — at least medium (already is, but explicit) + "CWE-79": schemas.SeverityMedium, + // Broken Access Control — at least high + "CWE-840": schemas.SeverityHigh, + "CWE-862": schemas.SeverityHigh, + "CWE-863": schemas.SeverityHigh, +} + +// ApplyCWESeverityFloor upgrades a severity when the CWE has a known minimum +// floor, and otherwise returns it unchanged. +// +// Ports scoring.py apply_cwe_severity_floor. Python parity: the comparison is +// `_SEVERITY_ORDER.get(label, 0) > _SEVERITY_ORDER.get(current.value, 0)`, so +// an UNKNOWN current severity ranks as 0 (info) and any floor beats it. +func ApplyCWESeverityFloor(cweID string, current schemas.Severity) schemas.Severity { + floor, ok := CWESeverityFloor[cweID] + if !ok { + return current + } + if severityOrder[string(floor)] > severityOrder[string(current)] { + return floor + } + return current +} + +// reachabilityMultiplier picks the multiplier from the finding's tags. +// +// Ports scoring.py _reachability_multiplier, including its two fallbacks: +// NO tags at all means "assume externally reachable" (1.0), while tags that +// exist but say nothing about reachability mean "requires_auth" (0.5). The +// comment in scoring.py records why: defaulting everything to 0.5 scored +// critical CWEs at 2.5/10 whenever reachability assessment was not wired into +// the DAG path. +// +// Tags are lower-cased before matching (Python builds a set comprehension of +// `tag.lower()`), so a finding tagged "EXTERNALLY_REACHABLE" matches. +func reachabilityMultiplier(finding schemas.VerifiedFinding) float64 { + normalized := make(map[string]struct{}, len(finding.Tags)) + for _, tag := range finding.Tags { + normalized[strings.ToLower(tag)] = struct{}{} + } + for _, key := range reachabilityTagOrder { + if _, ok := normalized[key]; ok { + return ReachabilityMultipliers[key] + } + } + if len(normalized) == 0 { + return ReachabilityMultipliers["externally_reachable"] + } + return ReachabilityMultipliers["requires_auth"] +} + +// ComputeExploitabilityScore is the 0-10 exploitability score for a finding: +// severity weight × evidence multiplier × reachability multiplier × chain +// bonus, clamped to [0, 10] and rounded to 2 decimals. +// +// Ports scoring.py compute_exploitability_score. The chain bonus is 2.0 when +// the finding belongs to an attack chain, else 1.0. +// +// Python parity divergence (deliberate, documented): Python indexes +// SEVERITY_WEIGHTS / EVIDENCE_MULTIPLIERS directly, so an out-of-vocabulary +// severity or evidence level raises KeyError. Pydantic makes that unreachable +// in Python (both fields are enums), but Go's schemas.Severity is a string +// type that a hand-built struct could set to anything. Go treats an unknown +// key as weight 0.0 — the score collapses to 0.0 rather than panicking mid +// audit. +// +// The final `round(..., 2)` goes through pyfmt.Round, which reproduces CPython's +// round-half-to-EVEN on the exact binary value — NOT math.Round(x*100)/100, +// which rounds half away from zero and accumulates the scaling error. +func ComputeExploitabilityScore(finding schemas.VerifiedFinding) float64 { + severityWeight := SeverityWeights[finding.Severity] + evidenceMultiplier := EvidenceMultipliers[finding.EvidenceLevel] + reachability := reachabilityMultiplier(finding) + chainBonus := 1.0 + if finding.ChainID != nil && *finding.ChainID != "" { + chainBonus = 2.0 + } + + score := severityWeight * evidenceMultiplier * reachability * chainBonus + if score < 0.0 { + score = 0.0 + } + if score > 10.0 { + score = 10.0 + } + return pyfmt.Round(score, 2) +} + +// ComputePriorityRank returns the findings sorted by exploitability score, +// highest first, as a NEW slice (Python's `sorted()` copies; the input order is +// untouched). +// +// Ports scoring.py compute_priority_rank. Python's sorted() is stable and +// `reverse=True` preserves the original order among equal keys, so Go uses +// sort.SliceStable — sort.Slice would be free to reorder ties and break +// tests/test_scoring.py::test_compute_priority_rank_is_stable_for_equal_scores. +// The score is computed ONCE per finding (Python's `key=` does the same), +// which also keeps the comparator cheap. +func ComputePriorityRank(findings []schemas.VerifiedFinding) []schemas.VerifiedFinding { + out := make([]schemas.VerifiedFinding, len(findings)) + copy(out, findings) + scores := make([]float64, len(out)) + idx := make([]int, len(out)) + for i := range out { + scores[i] = ComputeExploitabilityScore(out[i]) + idx[i] = i + } + sort.SliceStable(idx, func(a, b int) bool { return scores[idx[a]] > scores[idx[b]] }) + ranked := make([]schemas.VerifiedFinding, len(out)) + for i, j := range idx { + ranked[i] = out[j] + } + return ranked +} + +// AssignSeverityLabel maps an exploitability score onto a severity label. +// +// Ports scoring.py assign_severity_label. Boundaries are inclusive on the +// lower end: >=9 critical, >=7 high, >=4 medium, >=1 low, else info. +func AssignSeverityLabel(score float64) string { + switch { + case score >= 9.0: + return "critical" + case score >= 7.0: + return "high" + case score >= 4.0: + return "medium" + case score >= 1.0: + return "low" + default: + return "info" + } +} diff --git a/internal/secaf/scoring/scoring_test.go b/internal/secaf/scoring/scoring_test.go new file mode 100644 index 0000000000..a0e7b78703 --- /dev/null +++ b/internal/secaf/scoring/scoring_test.go @@ -0,0 +1,466 @@ +package scoring + +import ( + "reflect" + "testing" + + "github.com/Agent-Field/codeaf/internal/secaf/pyfmt" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// This file ports tests/test_scoring.py. Every expected number was produced by +// running the Python functions in the sec-af venv against the same inputs +// (`PYTHONPATH=src ~/.agentfield/packages/sec-af/venv/bin/python`). +// +// Two assertions in tests/test_scoring.py are STALE — they encode the OLD +// "missing reachability tags default to requires_auth (0.5)" behavior that +// scoring.py's own comment says was deliberately changed to +// externally_reachable (1.0): +// +// - test_compute_exploitability_score_defaults_reachability_when_missing +// expects 1.05; the live code returns 2.1. +// - test_reachability_multipliers_and_default_behavior[set()] expects 2.5; +// the live code returns 5.0. +// +// The port follows the CODE (verified against the interpreter), which is what +// the audit pipeline actually computes, and the two tests below carry a note +// where they diverge from the Python file. See the PR's parity notes. + +// makeFinding ports tests/test_scoring.py::make_finding. +// +// Python parity: the fixture passes `tags` as a SET, which pydantic coerces to +// a list in an undefined order. Go takes an ordered slice; the scoring code +// lower-cases into a set, so order never matters. +func makeFinding(severity schemas.Severity, evidenceLevel schemas.EvidenceLevel, tags []string, chainID *string) schemas.VerifiedFinding { + f := schemas.NewVerifiedFinding() + f.Fingerprint = "abc123" + f.Title = "Sample finding" + f.Description = "Sample description" + f.FindingType = schemas.FindingTypeSast + f.CweID = "CWE-89" + f.CweName = "SQL Injection" + if tags != nil { + f.Tags = tags + } + f.Verdict = schemas.VerdictConfirmed + f.EvidenceLevel = evidenceLevel + f.Rationale = "Reasonable rationale" + f.Severity = severity + f.ExploitabilityScore = 0.0 + f.Location = schemas.Location{FilePath: "app.py", StartLine: 10, EndLine: 10} + f.ChainID = chainID + f.SarifRuleID = "sec-af/sast/sql-injection" + f.SarifSecuritySeverity = 0.0 + return f +} + +func strptr(s string) *string { return &s } + +// --------------------------------------------------------------------------- +// test_compute_exploitability_score_respects_severity_weights +// --------------------------------------------------------------------------- + +func TestComputeExploitabilityScoreRespectsSeverityWeights(t *testing.T) { + cases := []struct { + severity schemas.Severity + want float64 + }{ + {schemas.SeverityCritical, 10.0}, + {schemas.SeverityHigh, 8.0}, + {schemas.SeverityMedium, 5.0}, + {schemas.SeverityLow, 3.0}, + {schemas.SeverityInfo, 1.0}, + } + for _, tc := range cases { + t.Run(string(tc.severity), func(t *testing.T) { + f := makeFinding(tc.severity, schemas.EvidenceLevelFullExploit, []string{"externally_reachable"}, nil) + if got := ComputeExploitabilityScore(f); got != tc.want { + t.Errorf("= %v, want %v", got, tc.want) + } + }) + } +} + +// --------------------------------------------------------------------------- +// test_compute_exploitability_score_applies_chain_bonus_and_clamps_to_ten +// --------------------------------------------------------------------------- + +func TestComputeExploitabilityScoreAppliesChainBonusAndClampsToTen(t *testing.T) { + f := makeFinding(schemas.SeverityCritical, schemas.EvidenceLevelFullExploit, + []string{"externally_reachable"}, strptr("chain-1")) + // 10 * 1.0 * 1.0 * 2.0 = 20 -> clamped to 10. + if got := ComputeExploitabilityScore(f); got != 10.0 { + t.Errorf("= %v, want 10.0", got) + } + // A chain bonus that does NOT hit the clamp is still visible. + mid := makeFinding(schemas.SeverityMedium, schemas.EvidenceLevelFlowIdentified, + []string{"requires_auth"}, strptr("c1")) + if got := ComputeExploitabilityScore(mid); got != 1.5 { + t.Errorf("chained medium = %v, want 1.5", got) + } + // Python parity: `if finding.chain_id:` — an EMPTY chain_id is falsy, so + // it earns no bonus. + empty := makeFinding(schemas.SeverityMedium, schemas.EvidenceLevelFullExploit, + []string{"externally_reachable"}, strptr("")) + if got := ComputeExploitabilityScore(empty); got != 5.0 { + t.Errorf("empty chain_id = %v, want 5.0 (no bonus)", got) + } +} + +// --------------------------------------------------------------------------- +// test_compute_exploitability_score_uses_partial_flow_and_internal_reachability +// --------------------------------------------------------------------------- + +func TestComputeExploitabilityScoreUsesPartialFlowAndInternalReachability(t *testing.T) { + f := makeFinding(schemas.SeverityMedium, schemas.EvidenceLevelReachabilityConfirmed, + []string{"internally_reachable"}, nil) + if got := ComputeExploitabilityScore(f); got != 1.75 { + t.Errorf("= %v, want 1.75", got) + } +} + +// --------------------------------------------------------------------------- +// test_compute_exploitability_score_uses_requires_admin_and_unverified +// --------------------------------------------------------------------------- + +func TestComputeExploitabilityScoreUsesRequiresAdminAndUnverified(t *testing.T) { + f := makeFinding(schemas.SeverityHigh, schemas.EvidenceLevelStaticMatch, + []string{"requires_admin"}, nil) + // 8.0 * 0.1 * 0.3 = 0.24000000000000002 -> round(,2) -> 0.24 + if got := ComputeExploitabilityScore(f); got != 0.24 { + t.Errorf("= %v, want 0.24", got) + } +} + +// --------------------------------------------------------------------------- +// test_compute_exploitability_score_defaults_reachability_when_missing +// +// STALE IN PYTHON: tests/test_scoring.py asserts 1.05 (the old requires_auth +// default). scoring.py now defaults an EMPTY tag set to externally_reachable +// (1.0), so the value is 3.0 * 0.7 * 1.0 = 2.1. Verified against the +// interpreter. +// --------------------------------------------------------------------------- + +func TestComputeExploitabilityScoreDefaultsReachabilityWhenMissing(t *testing.T) { + f := makeFinding(schemas.SeverityLow, schemas.EvidenceLevelSanitizationBypassable, nil, nil) + if got := ComputeExploitabilityScore(f); got != 2.1 { + t.Errorf("= %v, want 2.1 (empty tags -> externally_reachable)", got) + } +} + +// --------------------------------------------------------------------------- +// test_compute_exploitability_score_is_deterministic +// --------------------------------------------------------------------------- + +func TestComputeExploitabilityScoreIsDeterministic(t *testing.T) { + f := makeFinding(schemas.SeverityHigh, schemas.EvidenceLevelExploitScenarioValidated, + []string{"requires_auth"}, nil) + if ComputeExploitabilityScore(f) != ComputeExploitabilityScore(f) { + t.Error("two calls disagreed") + } +} + +// --------------------------------------------------------------------------- +// test_reachability_multipliers_and_default_behavior +// +// STALE IN PYTHON for the empty-set row: the file asserts 2.5, the live code +// returns 5.0 (see the file header). +// --------------------------------------------------------------------------- + +func TestReachabilityMultipliersAndDefaultBehavior(t *testing.T) { + cases := []struct { + name string + tags []string + want float64 + }{ + // Tags are lower-cased before matching. + {"EXTERNALLY_REACHABLE", []string{"EXTERNALLY_REACHABLE"}, 5.0}, + {"internally_reachable", []string{"internally_reachable"}, 3.5}, + {"requires_auth", []string{"requires_auth"}, 2.5}, + {"requires_admin", []string{"requires_admin"}, 1.5}, + // A tag that says nothing about reachability -> requires_auth (0.5). + {"custom_tag", []string{"custom_tag"}, 2.5}, + // NO tags at all -> externally_reachable (1.0). Python file says 2.5. + {"empty", []string{}, 5.0}, + {"nil", nil, 5.0}, + // Probe order: externally_reachable wins over requires_admin. + {"probe order", []string{"requires_admin", "externally_reachable"}, 5.0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := makeFinding(schemas.SeverityMedium, schemas.EvidenceLevelFullExploit, tc.tags, nil) + if got := ComputeExploitabilityScore(f); got != tc.want { + t.Errorf("= %v, want %v", got, tc.want) + } + }) + } +} + +// --------------------------------------------------------------------------- +// test_compute_exploitability_score_lower_bound_is_zeroish_for_low_signal +// --------------------------------------------------------------------------- + +func TestComputeExploitabilityScoreLowerBoundIsZeroishForLowSignal(t *testing.T) { + f := makeFinding(schemas.SeverityInfo, schemas.EvidenceLevelStaticMatch, + []string{"requires_admin"}, nil) + // 1.0 * 0.1 * 0.3 = 0.030000000000000002 -> 0.03 + if got := ComputeExploitabilityScore(f); got != 0.03 { + t.Errorf("= %v, want 0.03", got) + } +} + +// --------------------------------------------------------------------------- +// test_compute_exploitability_score_does_not_depend_on_optional_cvss_or_epss_fields +// --------------------------------------------------------------------------- + +func TestComputeExploitabilityScoreIgnoresOptionalCvssAndEpss(t *testing.T) { + plain := makeFinding(schemas.SeverityHigh, schemas.EvidenceLevelExploitScenarioValidated, + []string{"requires_auth"}, nil) + withOptional := plain + withOptional.CvssV4 = &schemas.CvssV4Score{ + Vector: "CVSS:4.0/AV:N/AC:L/PR:N/UI:N/VC:H/VI:H/VA:H", + BaseScore: 9.3, + Severity: "critical", + Automatable: true, + SubsequentImpact: true, + } + withOptional.Epss = &schemas.EpssScore{Score: 0.81, Percentile: 0.95, Date: "2026-03-04"} + + // 8.0 * 0.9 * 0.5 = 3.6000000000000005 -> 3.6 + if got := ComputeExploitabilityScore(plain); got != 3.6 { + t.Errorf("plain = %v, want 3.6", got) + } + if got := ComputeExploitabilityScore(withOptional); got != 3.6 { + t.Errorf("with cvss/epss = %v, want 3.6", got) + } +} + +// --------------------------------------------------------------------------- +// test_compute_priority_rank_sorts_descending +// --------------------------------------------------------------------------- + +func TestComputePriorityRankSortsDescending(t *testing.T) { + low := makeFinding(schemas.SeverityInfo, schemas.EvidenceLevelStaticMatch, []string{"requires_admin"}, nil) + low.ID = "low" + medium := makeFinding(schemas.SeverityMedium, schemas.EvidenceLevelFlowIdentified, []string{"requires_auth"}, nil) + medium.ID = "medium" + high := makeFinding(schemas.SeverityCritical, schemas.EvidenceLevelFullExploit, []string{"externally_reachable"}, nil) + high.ID = "high" + + input := []schemas.VerifiedFinding{medium, high, low} + ranked := ComputePriorityRank(input) + + gotIDs := []string{ranked[0].ID, ranked[1].ID, ranked[2].ID} + if !reflect.DeepEqual(gotIDs, []string{"high", "medium", "low"}) { + t.Errorf("ranked = %v, want [high medium low]", gotIDs) + } + // Python's sorted() returns a NEW list; the input must be untouched. + inputIDs := []string{input[0].ID, input[1].ID, input[2].ID} + if !reflect.DeepEqual(inputIDs, []string{"medium", "high", "low"}) { + t.Errorf("input was mutated: %v", inputIDs) + } +} + +// --------------------------------------------------------------------------- +// test_compute_priority_rank_is_stable_for_equal_scores +// --------------------------------------------------------------------------- + +func TestComputePriorityRankIsStableForEqualScores(t *testing.T) { + first := makeFinding(schemas.SeverityLow, schemas.EvidenceLevelFlowIdentified, []string{"requires_auth"}, nil) + first.ID = "first" + second := makeFinding(schemas.SeverityLow, schemas.EvidenceLevelFlowIdentified, []string{"requires_auth"}, nil) + second.ID = "second" + + ranked := ComputePriorityRank([]schemas.VerifiedFinding{first, second}) + if ranked[0].ID != "first" || ranked[1].ID != "second" { + t.Errorf("ranked = [%s %s], want [first second] (Python sorted() is stable)", ranked[0].ID, ranked[1].ID) + } + + // A longer run of ties, to catch a non-stable sort that only shows up + // above the insertion-sort cutoff. + var many []schemas.VerifiedFinding + for i := 0; i < 40; i++ { + f := makeFinding(schemas.SeverityLow, schemas.EvidenceLevelFlowIdentified, []string{"requires_auth"}, nil) + f.ID = string(rune('a' + i%26)) + f.Fingerprint = f.ID + "-" + string(rune('0'+i/26)) + many = append(many, f) + } + rankedMany := ComputePriorityRank(many) + for i := range many { + if rankedMany[i].Fingerprint != many[i].Fingerprint { + t.Fatalf("tie at %d reordered: %q != %q", i, rankedMany[i].Fingerprint, many[i].Fingerprint) + } + } + + // An empty input returns an empty (non-nil) slice, not nil. + if got := ComputePriorityRank(nil); got == nil || len(got) != 0 { + t.Errorf("ComputePriorityRank(nil) = %#v, want an empty slice", got) + } +} + +// --------------------------------------------------------------------------- +// test_assign_severity_label +// --------------------------------------------------------------------------- + +func TestAssignSeverityLabel(t *testing.T) { + cases := []struct { + score float64 + label string + }{ + {10.0, "critical"}, + {9.0, "critical"}, + {8.9, "high"}, + {7.0, "high"}, + {6.9, "medium"}, + {4.0, "medium"}, + {3.9, "low"}, + {1.0, "low"}, + {0.9, "info"}, + {0.0, "info"}, + {-1.0, "info"}, + } + for _, tc := range cases { + if got := AssignSeverityLabel(tc.score); got != tc.label { + t.Errorf("AssignSeverityLabel(%v) = %q, want %q", tc.score, got, tc.label) + } + } +} + +// --------------------------------------------------------------------------- +// apply_cwe_severity_floor — not covered by tests/test_scoring.py, so these +// are derived from the function's contract and verified against the +// interpreter. +// --------------------------------------------------------------------------- + +func TestApplyCWESeverityFloor(t *testing.T) { + cases := []struct { + cwe string + current schemas.Severity + want schemas.Severity + }{ + // A floor above the current severity upgrades it. + {"CWE-78", schemas.SeverityMedium, schemas.SeverityCritical}, + {"CWE-918", schemas.SeverityInfo, schemas.SeverityHigh}, + {"CWE-79", schemas.SeverityLow, schemas.SeverityMedium}, + // A floor at or below the current severity leaves it alone. + {"CWE-78", schemas.SeverityCritical, schemas.SeverityCritical}, + {"CWE-79", schemas.SeverityHigh, schemas.SeverityHigh}, + {"CWE-22", schemas.SeverityCritical, schemas.SeverityCritical}, + // A CWE with no floor is untouched. + {"CWE-9999", schemas.SeverityLow, schemas.SeverityLow}, + {"", schemas.SeverityInfo, schemas.SeverityInfo}, + } + for _, tc := range cases { + if got := ApplyCWESeverityFloor(tc.cwe, tc.current); got != tc.want { + t.Errorf("ApplyCWESeverityFloor(%q, %q) = %q, want %q", tc.cwe, tc.current, got, tc.want) + } + } +} + +func TestScoringTablesMatchPython(t *testing.T) { + if len(CWESeverityFloor) != 18 { + t.Errorf("CWE_SEVERITY_FLOOR has %d entries, want 18", len(CWESeverityFloor)) + } + wantCritical := []string{"CWE-78", "CWE-77", "CWE-94", "CWE-95", "CWE-96", "CWE-89", "CWE-502"} + for _, cwe := range wantCritical { + if CWESeverityFloor[cwe] != schemas.SeverityCritical { + t.Errorf("CWE_SEVERITY_FLOOR[%s] = %q, want critical", cwe, CWESeverityFloor[cwe]) + } + } + wantHigh := []string{"CWE-918", "CWE-287", "CWE-290", "CWE-306", "CWE-798", "CWE-22", "CWE-611", "CWE-840", "CWE-862", "CWE-863"} + for _, cwe := range wantHigh { + if CWESeverityFloor[cwe] != schemas.SeverityHigh { + t.Errorf("CWE_SEVERITY_FLOOR[%s] = %q, want high", cwe, CWESeverityFloor[cwe]) + } + } + if CWESeverityFloor["CWE-79"] != schemas.SeverityMedium { + t.Errorf("CWE_SEVERITY_FLOOR[CWE-79] = %q, want medium", CWESeverityFloor["CWE-79"]) + } + + wantWeights := map[schemas.Severity]float64{ + schemas.SeverityCritical: 10.0, schemas.SeverityHigh: 8.0, schemas.SeverityMedium: 5.0, + schemas.SeverityLow: 3.0, schemas.SeverityInfo: 1.0, + } + if !reflect.DeepEqual(SeverityWeights, wantWeights) { + t.Errorf("SEVERITY_WEIGHTS = %v, want %v", SeverityWeights, wantWeights) + } + wantEvidence := map[schemas.EvidenceLevel]float64{ + schemas.EvidenceLevelFullExploit: 1.0, schemas.EvidenceLevelExploitScenarioValidated: 0.9, + schemas.EvidenceLevelSanitizationBypassable: 0.7, schemas.EvidenceLevelReachabilityConfirmed: 0.5, + schemas.EvidenceLevelFlowIdentified: 0.3, schemas.EvidenceLevelStaticMatch: 0.1, + } + if !reflect.DeepEqual(EvidenceMultipliers, wantEvidence) { + t.Errorf("EVIDENCE_MULTIPLIERS = %v, want %v", EvidenceMultipliers, wantEvidence) + } + wantReach := map[string]float64{ + "externally_reachable": 1.0, "internally_reachable": 0.7, + "requires_auth": 0.5, "requires_admin": 0.3, + } + if !reflect.DeepEqual(ReachabilityMultipliers, wantReach) { + t.Errorf("REACHABILITY_MULTIPLIERS = %v, want %v", ReachabilityMultipliers, wantReach) + } +} + +// TestEvidenceMultiplierSweep pins every (severity, evidence) pair against the +// values the Python function returns for an externally-reachable finding. +func TestEvidenceMultiplierSweep(t *testing.T) { + want := map[schemas.EvidenceLevel]float64{ + schemas.EvidenceLevelStaticMatch: 1.0, + schemas.EvidenceLevelFlowIdentified: 3.0, + schemas.EvidenceLevelReachabilityConfirmed: 5.0, + schemas.EvidenceLevelSanitizationBypassable: 7.0, + schemas.EvidenceLevelExploitScenarioValidated: 9.0, + schemas.EvidenceLevelFullExploit: 10.0, + } + for level, expected := range want { + f := makeFinding(schemas.SeverityCritical, level, []string{"externally_reachable"}, nil) + if got := ComputeExploitabilityScore(f); got != expected { + t.Errorf("critical @ %s = %v, want %v", level.Name(), got, expected) + } + } +} + +// TestRoundingMatchesPythonRound pins the rounding scoring relies on +// (pyfmt.Round) against the values Python's round(x, 2) produces, including the +// cases where naive math.Round(x*100)/100 diverges. The scoring package used to +// carry a private copy of this helper; it now delegates to internal/pyfmt, and +// this test guards the values scoring actually depends on. +func TestRoundingMatchesPythonRound(t *testing.T) { + cases := []struct { + in float64 + want float64 + }{ + {0.24000000000000002, 0.24}, + {0.030000000000000002, 0.03}, + {3.6000000000000005, 3.6}, + {2.0999999999999996, 2.1}, + {1.75, 1.75}, + // Exact ties round to even, not away from zero: Python round(0.125, 2) + // is 0.12 and round(0.135, 2) is 0.14 (0.135 is really 0.13500...0028). + {0.125, 0.12}, + {0.135, 0.14}, + {10.0, 10.0}, + {0.0, 0.0}, + } + for _, tc := range cases { + if got := pyfmt.Round(tc.in, 2); got != tc.want { + t.Errorf("pyfmt.Round(%v, 2) = %v, want %v", tc.in, got, tc.want) + } + } +} + +// TestUnknownSeverityOrEvidenceCollapsesToZero documents the ONE deliberate +// divergence from Python: Python raises KeyError (pydantic makes it +// unreachable), Go treats the missing weight as 0. +func TestUnknownSeverityOrEvidenceCollapsesToZero(t *testing.T) { + bad := makeFinding(schemas.Severity("blocker"), schemas.EvidenceLevelFullExploit, + []string{"externally_reachable"}, nil) + if got := ComputeExploitabilityScore(bad); got != 0.0 { + t.Errorf("unknown severity = %v, want 0", got) + } + badLevel := makeFinding(schemas.SeverityCritical, schemas.EvidenceLevel(9), + []string{"externally_reachable"}, nil) + if got := ComputeExploitabilityScore(badLevel); got != 0.0 { + t.Errorf("unknown evidence level = %v, want 0", got) + } +} diff --git a/internal/secaf/secaf.go b/internal/secaf/secaf.go new file mode 100644 index 0000000000..5f66f4654c --- /dev/null +++ b/internal/secaf/secaf.go @@ -0,0 +1,373 @@ +// Package secaf is sec: sec-af, the security auditor, as a program +// codeaf carries and runs, and nothing else can. It takes one brief — the +// whole repository, or the changes on this branch — reads the folder it is +// handed without changing it, hunts for vulnerabilities, sets each against the +// code to show it exploitable or rule it out, suggests fixes for the ones that +// stand, and ends with one report. +// +// IT WAS A SEPARATE PROGRAM. sec-af (github.com/Agent-Field/sec-af) was an +// AgentField node: a control plane carried its calls, a router key it held +// paid for its models, and a coding-agent binary ran each of its agent +// sessions. It was copied into codeaf once, at the tag codeaf-absorb (47d57d7), +// and lives on only here (docs/design/security-audit/ABSORB.md). Its algorithm +// — the phases, the twelve hunters, the four-agent proof chain, the prompts — +// is in the packages below this one and is its own; what it runs on is +// codeaf's (internal/agentsession). +// +// IT HAS NO ENTRY POINT OF ITS OWN. What codeaf needs of it is a +// delegate.Delegate value, and its command's body takes a delegate.Host, which +// only codeaf makes: `/sec` in the chat, `codeaf sec` at +// a shell, and `propose_task` with `via: "sec"`. +package secaf + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "runtime/debug" + "strings" + "time" + + "github.com/Agent-Field/codeaf/internal/agentsession" + "github.com/Agent-Field/codeaf/internal/delegate" + "github.com/Agent-Field/codeaf/internal/programguide" + "github.com/Agent-Field/codeaf/internal/secaf/audit" +) + +// Name is the program's name: its chat command and its shell verb. +const Name = "sec" + +// The stages the audit reports, in order. They are sec-af's own phases, and +// `starting` and `report` around them. +const ( + stageStarting = "starting" + stageRecon = "recon" + stageHunt = "hunt" + stageProve = "prove" + stageRemediation = "remediation" + stageReport = "report" +) + +// Stages is every stage the audit reports, in the order it reaches them. +var Stages = []string{stageStarting, stageRecon, stageHunt, stageProve, stageRemediation, stageReport} + +// stageWords is the stages in a person's words, for the task's row. +// +// THEY ARE sec-af's OWN PHASE NAMES, the words its notes on the page already +// use (`RECON phase complete`), so the row, the page's step headings and the +// notes under them say one thing (the owner's call, 2026-10-05). +var stageWords = map[string]string{ + stageStarting: "starting", + stageRecon: "recon", + stageHunt: "hunt", + stageProve: "prove", + stageRemediation: "remediate", + stageReport: "report", +} + +// Unattended is the audit's own ceilings: $5 and four hours, which stop a run +// nobody is watching. TIME IS THE CEILING A STANDARD AUDIT MEETS FIRST: one of +// a mid-sized Rust repository on 2026-10-05 made 1,952 calls, spent $2.40 of +// its $5 and was cut by two hours in prove with no fixes written, so the hours +// were doubled and the dollars left where they were. +var Unattended = delegate.Ceilings{CostUSD: 5, Hours: 4} + +// Program is sec as codeaf carries it. +var Program = delegate.Delegate{ + Name: Name, + Summary: "a security audit of the repository or of its changes", + Guide: programguide.Sec, + Lands: delegate.LandsText, + // A BARE `/sec` AUDITS THE REPOSITORY IT IS STARTED IN, which + // is what the name already says. + DefaultBrief: "whole repository", + // ITS BRIEF IS ITS SCOPE WORDS AND NOTHING ELSE (ReadScope), however the + // run was asked for. + Words: true, + Args: "[changes [since ]] [quick | thorough]", + // A TYPED RUN IS TITLED BY WHAT IT AUDITS, because its brief is a few + // scope words that say nothing as a title (`quick`, `changes`). + Title: func(brief string) string { return "Security audit of " + ReadScope(brief, false, "", "").Describe() }, + Unattended: Unattended, + // THE OFFER IS A FIX, AND THE PERSON SAYS WHETHER. A report that found + // something wants mending; mending is another program's work and more of + // the person's money, so the conversation offers it and waits. + FollowUp: "If it found confirmed or likely problems, offer to hand them to senior-dev as one task that fixes them, " + + "naming which; propose that task only after the person says yes.", + ModelFlag: "model", + CrewFlags: crewFlags, + StageWords: stageWords, + Present: presentActions, + Default: "run", + Page: Name, + Commands: []delegate.Command{runCommand}, +} + +// crewFlags is the conversation's crew as the audit's own flags: the working +// seat (or the models the person asked for) answers the agent sessions that +// read the code, and the light seat the single structured calls. +func crewFlags(crew delegate.Crew) []string { + var flags []string + model := crew.Hands + if len(crew.Asked) > 0 { + model = crew.Asked[0] + } + // THE IDS GO ON AS THE CREW SPELLS THEM, which is how the run's model API + // reads them: a first segment naming a connected service is that service, + // and anything else is the default service's (modelsource.Split). + if model = strings.TrimSpace(model); model != "" { + flags = append(flags, "--model", model) + } + if light := strings.TrimSpace(crew.Light); light != "" { + flags = append(flags, "--light", light) + } + return flags +} + +// options is one run's flags and brief. +type options struct { + brief string + changes bool + base string + depth string + model string + light string + severity string + maxProvers int + compliance string + sessions int + maxTurns int + sessionWall time.Duration +} + +var runCommand = delegate.Command{ + Name: "run", + Usage: "[flags] [-- ]", + Summary: "audits the repository or its changes and reports what stands", + Bind: bindRun, +} + +func bindRun(fs *flag.FlagSet) delegate.Body { + var o options + fs.BoolVar(&o.changes, "changes", false, "audit only the changes since the branch's base") + fs.StringVar(&o.base, "base", "", "the commit or branch the changes are measured from") + fs.StringVar(&o.depth, "depth", "", "quick, standard or thorough (default: standard)") + fs.StringVar(&o.model, "model", "", "the model that reads the code (default: the run's work seat)") + fs.StringVar(&o.light, "light", "", "the model for single structured calls (default: --model)") + fs.StringVar(&o.severity, "severity", "low", "the least severe finding to report: info, low, medium, high") + fs.IntVar(&o.maxProvers, "max-provers", 0, "the most findings to test (default: the depth's)") + fs.StringVar(&o.compliance, "compliance", "", "frameworks to map findings to, such as owasp,pci-dss") + fs.IntVar(&o.sessions, "sessions", sessionsAtOnce, "how many agent sessions run at once") + fs.IntVar(&o.maxTurns, "max-turns", 0, "most turns of any agent session (default: the agent's own)") + fs.DurationVar(&o.sessionWall, "session-wall", 0, "longest any agent session runs (default: the agent's own)") + return func(ctx context.Context, host delegate.Host, args []string) error { + o.brief = strings.Join(args, " ") + run(ctx, host, o, os.Stderr) + return nil + } +} + +// run is the body: hello first, the audit, and exactly one terminal. +// +// AN ENDING'S MESSAGE IS THE REASON ALONE, never opening on sec's name. codeaf +// puts the program's name and which ending it was in front of it (`sec did not +// finish: …`, `sec stopped on its own ceiling: …`, `sec finished: …`), and a +// message that said `sec did not finish: …` itself was read out twice. +func run(ctx context.Context, host delegate.Host, o options, notes io.Writer) { + host.Hello(Stages) + defer func() { + if recovered := recover(); recovered != nil { + _, _ = fmt.Fprintf(notes, "[sec] panic: %v\n%s", recovered, debug.Stack()) + host.Terminal(delegate.Ending{Status: delegate.StatusCrashed, + Message: fmt.Sprintf("it broke: %v", recovered)}) + } + }() + host.Terminal(runAudit(ctx, host, o, notes)) +} + +// reportReserve is the time kept back from the run's wall for writing the +// report, so an audit cut by its hours still says what it found. +const reportReserve = 3 * time.Minute + +// runAudit is the audit from the brief to its ending. +func runAudit(ctx context.Context, host delegate.Host, o options, notes io.Writer) delegate.Ending { + started := time.Now() + scope := ReadScope(o.brief, o.changes, o.base, o.depth) + switch scope.Depth { + case "quick", "standard", "thorough": + default: + return delegate.Ending{Status: delegate.StatusFail, + Message: fmt.Sprintf("it could not start: the depth %q is not quick, standard or thorough", scope.Depth)} + } + if o.sessions < 1 { + return delegate.Ending{Status: delegate.StatusFail, Message: fmt.Sprintf("it could not start: --sessions is %d, and at least one agent session has to run", o.sessions)} + } + root, err := filepath.EvalSymlinks(host.Workspace()) + if err != nil { + return delegate.Ending{Status: delegate.StatusFail, Message: "it could not start: " + err.Error()} + } + host.Stage(delegate.StageRecord{Stage: stageStarting, Status: "running", Data: stageData(map[string]any{ + "scope": scope.Describe(), "depth": scope.Depth})}) + var changes *Changes + if scope.Changes { + read, err := ReadChanges(ctx, root, scope.Base) + if err != nil { + if errors.Is(err, errNoChanges) { + return delegate.Ending{Status: delegate.StatusPass, Message: "it had nothing to audit: " + err.Error(), + Deliverable: "Security audit of " + scope.Describe() + ": " + err.Error() + ", so there was nothing to audit.", + Extra: map[string]any{"status": "pass"}} + } + return delegate.Ending{Status: delegate.StatusFail, Message: "it could not start: " + err.Error()} + } + changes = &read + host.Stage(delegate.StageRecord{Stage: stageStarting, Status: "changes", Data: stageData(map[string]any{ + "changed": len(read.Changed), "nearby": len(read.Nearby), "base": read.BaseName})}) + } + client, err := agentsession.NewClient(host.Models(), retries) + if err != nil { + return delegate.Ending{Status: delegate.StatusFail, Message: "it could not start: " + err.Error()} + } + records := "" + if recorder, ok := host.(delegate.Recorder); ok { + records = recorder.Records() + } + light := o.light + if light == "" { + light = o.model + } + watching, phases := newWatch(host) + app, err := agentsession.New(client, agentsession.Config{ + Root: root, Work: "a security audit", SessionModel: o.model, AIModel: light, + Sessions: o.sessions, Calls: callsAtOnce, MaxTurns: defaultAgentTurns, SessionWall: defaultAgentWall, + Policy: sessionPolicy, Limits: limitsFor(o), Watch: watching, + }) + if err != nil { + return delegate.Ending{Status: delegate.StatusCrashed, Message: "it could not start: " + err.Error()} + } + request := auditRequest(root, scope, changes, o, records) + // THE CHECKPOINTS GO WITH THE RECORDS, never into the person's folder. A + // run with no record folder keeps them in a scratch folder of its own and + // removes it when it ends. + if records != "" { + request.CheckpointDir = filepath.Join(records, "checkpoints") + } else if scratch, err := os.MkdirTemp("", "sec-"); err == nil { + defer os.RemoveAll(scratch) + request.CheckpointDir = scratch + } + // THE RUN'S HOURS ARE THE AUDIT'S, LESS THE TIME TO WRITE IT DOWN. codeaf + // stops a program at its wall whatever it is doing; an audit that stops + // itself a little before has the minutes to say what it found. + auditCtx := ctx + if wall := host.Ceilings().Elapsed(); wall > 2*reportReserve { + var cancel context.CancelFunc + auditCtx, cancel = context.WithTimeout(ctx, wall-reportReserve-time.Since(started)) + defer cancel() + } + if changes != nil { + auditCtx = withFocus(auditCtx, *changes) + } + result, err := audit.Run(auditCtx, app, request) + spent, sessions, calls := app.Spent() + if err != nil { + _, _ = fmt.Fprintf(notes, "[sec] the audit ended without a result: %v\n", err) + ending := delegate.Ending{CostUSD: spent, Reason: err.Error(), + Extra: map[string]any{"sessions": sessions, "calls": calls}} + switch { + case errors.Is(err, agentsession.ErrCeiling): + ending.Status = delegate.StatusBudget + ending.Message = "it reached the run's dollar ceiling before it finished, so it has no report" + case errors.Is(auditCtx.Err(), context.DeadlineExceeded) && ctx.Err() == nil: + ending.Status = delegate.StatusBudget + ending.Message = "it reached the run's time ceiling before it finished, so it has no report" + case ctx.Err() != nil: + ending.Status = delegate.StatusFail + ending.Message = "it was stopped before it finished" + default: + ending.Status = delegate.StatusFail + ending.Message = firstSentence(err.Error()) + } + return ending + } + // A RESULT THE WALL CUT SAYS SO. sec-af finishes a phase whose agents were + // stopped — a test that did not finish leaves its finding unclear, a fix + // that was not written is missing — and hands back a whole-looking result. + // The run stopped itself at its time ceiling, so the account and the report + // say where, rather than reading as an audit that ran to its end. + cut := "" + if errors.Is(auditCtx.Err(), context.DeadlineExceeded) && ctx.Err() == nil { + cut = stageWords[phases.current()] + } + host.Stage(delegate.StageRecord{Stage: stageReport, Status: "running"}) + files, err := reportFiles(records, result, len(request.ComplianceFrameworks) > 0, runFacts{ + scope: scope, changes: changes, spent: spent, sessions: sessions, calls: calls, took: time.Since(started), + cutAt: cut, wall: host.Ceilings().TimeWord()}) + if err != nil { + _, _ = fmt.Fprintf(notes, "[sec] could not write the full report: %v\n", err) + } + text := summary(scope, changes, result, files, cut, host.Ceilings().TimeWord()) + return delegate.Ending{ + Status: delegate.StatusPass, + Message: outcomeLine(result, cut), + CostUSD: spent, + Deliverable: text, + Extra: map[string]any{ + "status": "pass", "reports": files, "sessions": sessions, "calls": calls, "cut_at": cut, + "confirmed": result.Confirmed, "likely": result.Likely, + "inconclusive": result.Inconclusive, "ruled_out": result.NotExploitable, + }, + } +} + +// auditRequest is sec-af's own request for this run. +func auditRequest(root string, scope Scope, changes *Changes, o options, records string) audit.AuditRequest { + request := audit.NewAuditRequest() + request.RepoURL = root + request.Depth = scope.Depth + if severity := strings.TrimSpace(o.severity); severity != "" { + request.SeverityThreshold = severity + } + if o.maxProvers > 0 { + provers := o.maxProvers + request.MaxProvers = &provers + } + if frameworks := splitList(o.compliance); len(frameworks) > 0 { + request.ComplianceFrameworks = frameworks + } + if changes != nil { + // AN AUDIT OF THE CHANGES KEEPS WHAT IT FOUND IN THEM AND NEAR THEM + // (sec-af's own include filter, applied after the hunt), and tests + // fewer findings than a whole repository's, unless told otherwise. + request.IncludePaths = append([]string(nil), changes.Relevant...) + if request.MaxProvers == nil { + provers := changesProvers + request.MaxProvers = &provers + } + } + return request +} + +// changesProvers is how many findings an audit of the changes tests by +// default: half a standard audit's thirty. +const changesProvers = 15 + +func splitList(text string) []string { + var out []string + for _, part := range strings.Split(text, ",") { + if part = strings.TrimSpace(part); part != "" { + out = append(out, part) + } + } + return out +} + +func firstSentence(text string) string { + text = firstLine(strings.TrimSpace(text)) + if runes := []rune(text); len(runes) > 300 { + text = string(runes[:300]) + "…" + } + return text +} diff --git a/internal/secaf/secaf_test.go b/internal/secaf/secaf_test.go new file mode 100644 index 0000000000..05c81e5b6e --- /dev/null +++ b/internal/secaf/secaf_test.go @@ -0,0 +1,559 @@ +package secaf + +import ( + "context" + "encoding/json" + "io" + "io/fs" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/Agent-Field/agentfield/sdk/go/ai" + "github.com/Agent-Field/codeaf/internal/agentsession/appx" + "github.com/Agent-Field/codeaf/internal/delegate" + "github.com/Agent-Field/codeaf/internal/secaf/schemas" +) + +// stubHost is the host a run of the program is handed: what it says is kept, +// and its models are the stub API's. +type stubHost struct { + mu sync.Mutex + workspace string + records string + api delegate.ModelAPI + ceilings delegate.Ceilings + stages []delegate.StageRecord + steps []delegate.StepRecord + ending *delegate.Ending +} + +func (h *stubHost) Workspace() string { return h.workspace } +func (h *stubHost) Ceilings() delegate.Ceilings { return h.ceilings } +func (h *stubHost) Models() delegate.ModelAPI { return h.api } +func (h *stubHost) Records() string { return h.records } +func (h *stubHost) Hello([]string) {} +func (h *stubHost) Stage(s delegate.StageRecord) { + h.mu.Lock() + h.stages = append(h.stages, s) + h.mu.Unlock() +} +func (h *stubHost) Step(s delegate.StepRecord) { + h.mu.Lock() + h.steps = append(h.steps, s) + h.mu.Unlock() +} +func (h *stubHost) Terminal(e delegate.Ending) { h.mu.Lock(); h.ending = &e; h.mu.Unlock() } + +// stubModels is a model API that answers every call with the smallest value +// its schema allows — and, for the location scanner, one real location — so +// the whole of sec-af's pipeline runs with no model behind it. +type stubModels struct { + mu sync.Mutex + calls int + threads map[string]bool + scanned []string +} + +func (m *stubModels) serve(t *testing.T) delegate.ModelAPI { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + var body struct { + Messages []ai.Message `json:"messages"` + ResponseFormat *ai.ResponseFormat `json:"response_format"` + Thread string `json:"prompt_cache_key"` + } + if err := json.Unmarshal(raw, &body); err != nil { + t.Errorf("a call does not parse: %v", err) + } + m.mu.Lock() + m.calls++ + if m.threads == nil { + m.threads = map[string]bool{} + } + m.threads[body.Thread] = true + m.mu.Unlock() + text := "no chains" + if schema := schemaOf(body.ResponseFormat, body.Messages); schema != nil { + value := minimal(schema, schema) + if title, _ := schema["title"].(string); title == "ScanLocationsResult" { + user := body.Messages[len(body.Messages)-1].Content[0].Text + m.mu.Lock() + m.scanned = append(m.scanned, user) + m.mu.Unlock() + value = map[string]any{"locations": []any{map[string]any{ + "file_path": "app/views.py", "start_line": 4, "code_snippet": "os.popen(cmd)", "pattern_type": "command_injection"}}} + } + encoded, _ := json.Marshal(value) + text = string(encoded) + } + cost := 0.001 + _ = json.NewEncoder(w).Encode(ai.Response{Choices: []ai.Choice{{Message: textMessage("assistant", text)}}, Usage: &ai.Usage{Cost: &cost}}) + })) + t.Cleanup(server.Close) + return delegate.ModelAPI{BaseURL: server.URL + "/v1", Token: "stub"} +} + +func textMessage(role, text string) ai.Message { + return ai.Message{Role: role, Content: []ai.ContentPart{{Type: "text", Text: text}}} +} + +// schemaOf is the schema a call asks its answer to meet: its response format, +// or the one an agent session's system message carries. +func schemaOf(format *ai.ResponseFormat, messages []ai.Message) map[string]any { + var raw []byte + if format != nil && format.JSONSchema != nil { + raw = format.JSONSchema.Schema + } else if len(messages) > 0 && messages[0].Role == "system" { + system := messages[0].Content[0].Text + if at := strings.Index(system, "code fence:\n"); at >= 0 { + raw = []byte(system[at+len("code fence:\n"):]) + } + } + var schema map[string]any + if json.Unmarshal(raw, &schema) != nil { + return nil + } + return schema +} + +// minimal is the smallest value a schema allows: every required field, the +// first of every enum, an empty list. +func minimal(schema, root map[string]any) any { + if ref, ok := schema["$ref"].(string); ok { + defs, _ := root["$defs"].(map[string]any) + target, _ := defs[strings.TrimPrefix(ref, "#/$defs/")].(map[string]any) + return minimal(target, root) + } + if values, ok := schema["enum"].([]any); ok && len(values) > 0 { + return values[0] + } + if options, ok := schema["anyOf"].([]any); ok && len(options) > 0 { + first, _ := options[0].(map[string]any) + return minimal(first, root) + } + kind := schema["type"] + if list, ok := kind.([]any); ok && len(list) > 0 { + kind = list[0] + } + switch kind { + case "object": + out := map[string]any{} + properties, _ := schema["properties"].(map[string]any) + required, _ := schema["required"].([]any) + for _, name := range required { + if property, ok := properties[name.(string)].(map[string]any); ok { + out[name.(string)] = minimal(property, root) + } + } + return out + case "array": + return []any{} + case "string": + return "x" + case "integer": + return 1 + case "number": + return 0.5 + case "boolean": + return false + case "null": + return nil + } + return "x" +} + +func vulnerableRepo(t *testing.T) string { + t.Helper() + root := t.TempDir() + for rel, text := range map[string]string{ + "app/views.py": "import os\n\ndef run(request):\n return os.popen(request.args['cmd']).read()\n", + "app/models.py": "class User:\n password = 'hunter2'\n", + "requirements.txt": "flask==0.12\n", + } { + path := filepath.Join(root, rel) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + real, _ := filepath.EvalSymlinks(root) + return real +} + +// snapshot is every file under root and its bytes. +func snapshot(t *testing.T, root string) map[string]string { + t.Helper() + files := map[string]string{} + _ = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err == nil && !info.IsDir() { + data, _ := os.ReadFile(path) + files[path] = string(data) + } + return nil + }) + return files +} + +// THE WHOLE AUDIT RUNS ON CODEAF'S ROAD AND LEAVES THE FOLDER AS IT WAS. sec-af's +// pipeline, from recon to the report, runs over the stub model API: every +// call reaches it as its own thread, the program reports its phases and its +// sessions, the full report lands in the record folder, the conversation's +// account names it, and not one byte of the repository changed. +func TestAnAuditRunsTheWholePipelineAndChangesNothing(t *testing.T) { + repo := vulnerableRepo(t) + before := snapshot(t, repo) + models := &stubModels{} + host := &stubHost{workspace: repo, records: t.TempDir(), api: models.serve(t)} + run(context.Background(), host, options{brief: "whole repository quick", severity: "low", sessions: 4, maxTurns: 5}, io.Discard) + + if host.ending == nil { + t.Fatal("the run wrote no ending") + } + ending := *host.ending + if ending.Status != delegate.StatusPass { + t.Fatalf("the audit ended %s: %s (%s)", ending.Status, ending.Message, ending.Reason) + } + if ending.Extra["status"] != "pass" { + t.Fatalf("the ending's data says %v", ending.Extra["status"]) + } + // THE FIRST LINE SAYS WHAT IT FOUND, because it is all the project's index + // keeps of the run for another conversation to read. + if first := firstLine(ending.Deliverable); !strings.HasPrefix(first, "Security audit of the whole repository, quick: ") || + !strings.Contains(first, "findings stayed unclear and are in the report") { + t.Fatalf("the account begins %q", first) + } + readable, _ := os.ReadFile(filepath.Join(host.records, reportMarkdown)) + for _, want := range []string{"# sec — security audit of the whole repository, quick", "## What it found", "## How it ran"} { + if !strings.Contains(string(readable), want) { + t.Errorf("the readable report lacks %q:\n%s", want, readable) + } + } + for _, unwanted := range []string{"Verdict", "inconclusive", "not exploitable", "$0.00", "Provider"} { + if strings.Contains(string(readable), unwanted) { + t.Errorf("the readable report says %q:\n%s", unwanted, readable) + } + } + for _, name := range []string{reportMarkdown, reportJSON, reportSARIF} { + path := filepath.Join(host.records, name) + if _, err := os.Stat(path); err != nil { + t.Errorf("the report %s was not written: %v", name, err) + } + if !strings.Contains(ending.Deliverable, path) { + t.Errorf("the account does not name %s:\n%s", path, ending.Deliverable) + } + } + if after := snapshot(t, repo); len(after) != len(before) { + t.Fatalf("the repository changed: %d files before, %d after", len(before), len(after)) + } else { + for path, text := range before { + if after[path] != text { + t.Fatalf("%s changed", path) + } + } + } + var stages []string + for _, stage := range host.stages { + stages = append(stages, stage.Stage+":"+stage.Status) + } + for _, want := range []string{"recon:running", "hunt:running", "prove:running", "report:running"} { + if !strings.Contains(strings.Join(stages, " "), want) { + t.Errorf("the stages %v lack %s", stages, want) + } + } + sessions := map[string]bool{} + observed := map[string]string{} + for _, step := range host.steps { + if step.Tool == toolSession { + sessions[step.Command] = true + observed[step.Command] = step.Observation + } + } + // EACH HUNTER'S SESSIONS SAY WHICH HUNTER AND WHAT IT FOUND: the scan how + // many places it found, the enrichment the finding it made of one. + if got := observed["injection hunter · scan"]; !strings.HasPrefix(got, "1 location · ") { + t.Errorf("the injection hunter's scan says %q", got) + } + if got := observed["injection hunter · app/views.py:4 · x"]; !strings.HasPrefix(got, "x · ") { + t.Errorf("the injection hunter's enrichment says %q", got) + } + sarif, _ := os.ReadFile(filepath.Join(host.records, reportSARIF)) + if !strings.Contains(string(sarif), `"name": "sec"`) || strings.Contains(string(sarif), "SEC-AF") || strings.Contains(string(sarif), "sec-af/") { + t.Errorf("the SARIF does not name sec as its tool:\n%.600s", sarif) + } + for _, want := range []string{"architecture mapper", "injection hunter · scan", "deciding agent", "dependency checker"} { + if !sessions[want] { + t.Errorf("no %s session was recorded; sessions were %v", want, keys(sessions)) + } + } + if len(models.threads) < 10 || models.calls < 10 { + t.Fatalf("the pipeline made %d calls on %d threads", models.calls, len(models.threads)) + } + for _, prompt := range models.scanned { + if strings.Contains(prompt, "CHANGE UNDER AUDIT") { + t.Fatal("an audit of the whole repository told its hunters about a change") + } + } +} + +// AN AUDIT OF THE CHANGES TELLS ITS HUNTERS WHAT CHANGED: the changed file, +// uncommitted, and the change itself; and with nothing changed it ends at once, +// having spent nothing. +func TestAnAuditOfTheChangesFocusesItsHunters(t *testing.T) { + repo := vulnerableRepo(t) + gitIn(t, repo, "init", "-q", "-b", "main") + gitIn(t, repo, "add", ".") + gitIn(t, repo, "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "base") + + models := &stubModels{} + host := &stubHost{workspace: repo, records: t.TempDir(), api: models.serve(t)} + run(context.Background(), host, options{brief: "changes", severity: "low", sessions: 4, maxTurns: 5}, io.Discard) + if host.ending == nil || host.ending.Status != delegate.StatusPass || !strings.Contains(host.ending.Message, "nothing to audit") { + t.Fatalf("an audit with nothing changed ended %+v", host.ending) + } + if models.calls != 0 { + t.Fatalf("an audit with nothing changed made %d model calls", models.calls) + } + + if err := os.WriteFile(filepath.Join(repo, "app/views.py"), []byte("import os\n\ndef run(request):\n return os.system(request.args['cmd'])\n"), 0o644); err != nil { + t.Fatal(err) + } + host = &stubHost{workspace: repo, records: t.TempDir(), api: models.serve(t)} + run(context.Background(), host, options{brief: "changes quick", severity: "low", sessions: 4, maxTurns: 5}, io.Discard) + if host.ending == nil || host.ending.Status != delegate.StatusPass { + t.Fatalf("an audit of the changes ended %+v", host.ending) + } + if !strings.Contains(host.ending.Deliverable, "(1 changed files since the last commit") && !strings.Contains(host.ending.Deliverable, "1 changed files since") { + t.Fatalf("the account does not say what changed:\n%s", host.ending.Deliverable) + } + if len(models.scanned) == 0 { + t.Fatal("no hunter scanned") + } + for _, prompt := range models.scanned { + if !strings.Contains(prompt, "CHANGE UNDER AUDIT") || !strings.Contains(prompt, "- app/views.py") || !strings.Contains(prompt, "os.system") { + t.Fatalf("a hunter was not told the change:\n%s", prompt) + } + } +} + +func gitIn(t *testing.T, dir string, args ...string) { + t.Helper() + command := exec.Command("git", append([]string{"-C", dir}, args...)...) + if out, err := command.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } +} + +func keys(set map[string]bool) []string { + out := make([]string, 0, len(set)) + for key := range set { + out = append(out, key) + } + sort.Strings(out) + return out +} + +// THE BRIEF'S FEW WORDS ARE READ AS THE GUIDE TEACHES THEM, and a shell's +// flags win over them. +func TestTheBriefSaysTheScope(t *testing.T) { + for _, tc := range []struct { + brief string + changes bool + base, depth string + }{ + {"whole repository", false, "", "standard"}, + {"", false, "", "standard"}, + {"changes", true, "", "standard"}, + {"the changes since origin/main, quick", true, "origin/main", "quick"}, + {"Changes against v1.2.0 thorough", true, "v1.2.0", "thorough"}, + {"diff vs main", true, "main", "standard"}, + {"look for injection in the login changes", false, "", "standard"}, + } { + got := ReadScope(tc.brief, false, "", "") + if got.Changes != tc.changes || got.Base != tc.base || got.Depth != tc.depth { + t.Errorf("%q read as %+v", tc.brief, got) + } + } + if got := ReadScope("whole repository quick", true, "dev", "thorough"); !got.Changes || got.Base != "dev" || got.Depth != "thorough" { + t.Errorf("the flags did not win: %+v", got) + } +} + +// The crew's working seat reads the code and its light seat makes the single +// calls; a model the person asked for reads the code in its place. +func TestTheCrewSeatsTheAudit(t *testing.T) { + if got := strings.Join(crewFlags(delegate.Crew{Hands: "v/hands", Light: "v/light"}), " "); got != "--model v/hands --light v/light" { + t.Fatalf("crew flags %q", got) + } + if got := strings.Join(crewFlags(delegate.Crew{Hands: "v/hands", Asked: []string{"v/asked", "v/other"}}), " "); got != "--model v/asked" { + t.Fatalf("asked flags %q", got) + } + if err := Program.Validate(); err != nil { + t.Fatal(err) + } +} + +// NO MACHINERY WORDS REACH THE PAGE: sec-af's notes say what its provers +// decide in its own words, and the page says them in a person's. +func TestThePageSpeaksAPersonsWords(t *testing.T) { + read := presentActions() + for _, action := range []delegate.Action{ + {Kind: delegate.ActionStep, Tool: toolNote, Step: stageProve, Command: "PROVE phase complete: 3 verified"}, + {Kind: delegate.ActionStep, Tool: toolNote, Step: stageProve, Command: "Verdict agent starting"}, + {Kind: delegate.ActionStep, Tool: toolSession, Step: stageProve, Command: "verdict agent", Observation: "answered · 2 turns · 0 reads"}, + {Kind: delegate.ActionStage, Stage: stageProve, Status: "done", Data: stageData(map[string]any{"note": plainWords("PROVE phase complete: 2 verified")})}, + } { + shown, ok := read(action) + if !ok { + t.Fatalf("%+v was left off the page", action) + } + if lower := strings.ToLower(shown.Text + " " + shown.Outcome); strings.Contains(lower, "verdict") || strings.Contains(lower, "verified") { + t.Errorf("the page says %q", shown.Text) + } + if shown.Step != "prove" { + t.Errorf("%q is under %q", shown.Text, shown.Step) + } + } + for stage := range stageWords { + if stepWords[stage] == "" { + t.Errorf("the stage %s has no step word", stage) + } + } +} + +// THE PAGE SPEAKS sec-af's PHASE NAMES, from the row to the notes: its step +// headings are the phases' own, its running and closing lines are sec-af's +// notes, and the notes that expose plumbing are reworded or left off. +func TestThePageUsesSecAfsPhaseNames(t *testing.T) { + read := presentActions() + for _, tc := range []struct { + action delegate.Action + step string + text string + }{ + {delegate.Action{Kind: delegate.ActionStage, Stage: stageRecon, Status: "running", Data: stageData(map[string]any{"note": "RECON phase starting"})}, "recon", "RECON phase starting"}, + {delegate.Action{Kind: delegate.ActionStage, Stage: stageRecon, Status: "done", Data: stageData(map[string]any{"note": "RECON phase complete"})}, "recon", "RECON phase complete"}, + {delegate.Action{Kind: delegate.ActionStage, Stage: stageRemediation, Status: "running"}, "remediate", "remediate"}, + } { + shown, ok := read(tc.action) + if !ok || shown.Step != tc.step || shown.Text != tc.text { + t.Errorf("%s %s read as %+v (%v), want %q under %q", tc.action.Stage, tc.action.Status, shown, ok, tc.text, tc.step) + } + } + for stage, word := range stageWords { + if stage != stageStarting && stepWords[stage] != word { + t.Errorf("the row says %q for %s and the page %q", word, stage, stepWords[stage]) + } + } + if got := rewriteNote("CWE expansion suggested 8 additional CWEs"); got != "" { + t.Errorf("the CWE expansion note was kept: %q", got) + } + if got := rewriteNote("HUNT found 6 fingerprint-unique findings, running semantic dedup"); got != "HUNT found 6 distinct findings, merging duplicates" { + t.Errorf("the dedup note reads %q", got) + } + if got := plainWords("Verifier starting"); got != "Verifier starting" { + t.Errorf("sec-af's Verifier became %q", got) + } +} + +// THE ACCOUNT'S FIRST LINE SAYS WHAT STANDS, since the project's index keeps +// only that line of it, and sec's brief is handed to it as its words alone +// (delegate.Delegate.Words), however the run was asked for. +func TestTheAccountsFirstLineSaysWhatItFound(t *testing.T) { + result := schemas.SecurityAuditResult{Confirmed: 1, Likely: 1, Inconclusive: 3, NotExploitable: 1, Findings: []schemas.VerifiedFinding{ + {Title: "Command injection", Severity: "critical", Verdict: schemas.VerdictConfirmed}, + {Title: "Open redirect", Severity: "medium", Verdict: schemas.VerdictLikely}, + }} + account := summary(Scope{Depth: "standard"}, nil, result, []string{"/records/sec-report.md"}, "", "4h") + if first := firstLine(account); first != "Security audit of the whole repository: 2 problems, 1 confirmed and 1 likely (1 critical, 1 medium)." { + t.Fatalf("the account begins %q", first) + } + if !strings.Contains(account, "\n3 more stayed unclear and 1 was ruled out.\n") { + t.Fatalf("the account does not count the rest:\n%s", account) + } + if !Program.Words { + t.Fatal("sec is handed a composed brief, and reads its headings as its scope") + } +} + +// A RUN ITS TIME CEILING CUT SAYS SO, in its account, its one-line ending and +// its report, and says what the cut left undone; one that finished says none +// of it. +func TestARunCutByItsTimeCeilingSaysWhere(t *testing.T) { + result := schemas.SecurityAuditResult{Confirmed: 9, Likely: 5} + scope := Scope{Depth: "standard"} + account := summary(scope, nil, result, nil, "prove", "2h") + if !strings.Contains(account, "It reached its time ceiling of 2h during prove, so findings whose tests had not finished are marked unclear, and no fixes were written.") { + t.Fatalf("the account does not say it was cut:\n%s", account) + } + if got := outcomeLine(result, "prove"); got != "it found 14 problems: 9 confirmed, 5 likely, before its time ceiling cut it short during prove" { + t.Fatalf("the ending says %q", got) + } + if report := markdownReport(result, runFacts{scope: scope, cutAt: "remediate", wall: "2h"}); !strings.Contains(report, "> **Cut short.** It reached its time ceiling of 2h during remediate, so the fixes it had not finished are missing.") { + t.Fatalf("the report does not say it was cut:\n%s", report) + } + if whole := summary(scope, nil, result, nil, "", "2h"); strings.Contains(whole, "time ceiling") { + t.Fatalf("a finished run says it was cut:\n%s", whole) + } + if got := rewriteNote("Demoted finding 'SSRF via unvalidated S3 endpoint' (decision=unknown): verifier_error"); got != "'SSRF via unvalidated S3 endpoint' stays unclear: its test did not finish" { + t.Fatalf("the demotion note reads %q", got) + } +} + +// AN ENDING'S MESSAGE IS THE REASON ALONE. codeaf puts `sec did not finish:` +// (or which ending it was) in front of it, and a message that opened on sec's +// own name was read out twice: `sec did not finish: sec did not start: …`. +func TestAnEndingsMessageDoesNotSayTheProgramsName(t *testing.T) { + host := &stubHost{workspace: t.TempDir(), records: t.TempDir()} + run(context.Background(), host, options{brief: "whole repository", depth: "deep", sessions: 1, maxTurns: 1}, io.Discard) + if host.ending == nil || host.ending.Status != delegate.StatusFail || + host.ending.Message != `it could not start: the depth "deep" is not quick, standard or thorough` { + t.Fatalf("a refused depth ended %+v", host.ending) + } + if strings.HasPrefix(host.ending.Message, "sec ") { + t.Fatalf("the ending opens on sec's name: %q", host.ending.Message) + } +} + +// EACH OF SEC'S AGENTS RUNS UNDER ITS OWN FIGURES, found by its scratch +// folder's name; a person's flag is one figure for all of them; and every row +// of the table names an agent this audit has, so a renamed agent cannot fall +// silently back to the default. +func TestEachAgentHasItsOwnLimitsAndAFlagOverridesThem(t *testing.T) { + limits := limitsFor(options{}) + if got := limits(appx.HarnessOptions{Cwd: "/tmp/secaf-hunt-scan-123456"}); got.Turns != 75 || got.Wall != 20*time.Minute { + t.Fatalf("the location scanner runs under %+v", got) + } + if got := limits(appx.HarnessOptions{Cwd: "/tmp/secaf-hunt-enrich-api_security-42"}); got.Turns != 45 { + t.Fatalf("an enricher runs under %+v", got) + } + if got := limits(appx.HarnessOptions{Cwd: "/tmp/secaf-remediation-7"}); got.Turns != 0 || got.Wall != 0 { + t.Fatalf("an agent the table does not name runs under %+v, want the default", got) + } + if got := limitsFor(options{maxTurns: 12, sessionWall: time.Minute})(appx.HarnessOptions{Cwd: "/tmp/secaf-hunt-scan-1"}); got.Turns != 12 || got.Wall != time.Minute { + t.Fatalf("a person's flags gave %+v", got) + } + var sources strings.Builder + _ = filepath.WalkDir("agents", func(path string, entry fs.DirEntry, err error) error { + if err == nil && !entry.IsDir() && strings.HasSuffix(path, ".go") && !strings.HasSuffix(path, "_test.go") { + raw, _ := os.ReadFile(path) + sources.Write(raw) + } + return nil + }) + for _, row := range agentLimits { + if !strings.Contains(sources.String(), `"`+row.kind+`"`) && !strings.Contains(sources.String(), `"secaf-`+row.kind+`-`) { + t.Errorf("the table bounds %q, which no agent's scratch folder is named for", row.kind) + } + } +} diff --git a/internal/seniordev/seniordev.go b/internal/seniordev/seniordev.go index 909d18e3a0..b8ebf81055 100644 --- a/internal/seniordev/seniordev.go +++ b/internal/seniordev/seniordev.go @@ -98,7 +98,12 @@ var Program = delegate.Delegate{ Notes: ".senior-dev", // IT LISTENS: the person's words from its page and the conversation's // `say` reach its model between steps until it hands in (app/steering.go). - Listens: true, + Listens: true, + // An hour-long change nobody may be watching stops on its own at $10 and + // three hours, or sooner where the conversation has less left. + Unattended: delegate.SeniorDevCeilings, + // --high is the pool its coder routes on: the model a shell run names. + ModelFlag: "high", CrewFlags: crewFlags, // What a person reads for its stages, one plain word per phase: getting // ready, doing the work (every inner stage of a model turn included), diff --git a/internal/session/agent.go b/internal/session/agent.go index 913c4a7c1d..5d5d62fcdd 100644 --- a/internal/session/agent.go +++ b/internal/session/agent.go @@ -1948,6 +1948,12 @@ func (a *Agent) startTurnLocked(ctx context.Context, user userMessage, watcher * // terms. Everything else on the queue drains exactly as it always has. a.liftSteersLocked(hub) _, unanswered := a.drainSteeringLocked(hub) + // A PROGRAM'S ENDING THIS TURN COULD NOT ANSWER IS SAID ANYWAY, in + // the program's own words (program_outcome.go), and a person's stop + // is told as theirs. The cause is still the stop's: the turn's own + // cancel(nil) is deferred first, so it runs after this. + door, stopped := stopCause(turnCtx) + a.programOutcomeUnansweredLocked(hub, completed, stopped && door == StopByPerson) // AND THE SECOND LOOK AT A YOUNG COMMAND IS LET GO OF WITH THE TURN // IT WAS ARMED IN. It re-checks this turn's number before it touches // anything, so a leftover is inert either way; stopping it here is diff --git a/internal/session/delegate_door.go b/internal/session/delegate_door.go index 5d06d17933..14295e1b4a 100644 --- a/internal/session/delegate_door.go +++ b/internal/session/delegate_door.go @@ -40,6 +40,11 @@ type DelegateRow struct { Description string // Lands is delegate.LandsTree or delegate.LandsText. Lands string + // Args is what the row shows after the name; empty is ``. + Args string `json:",omitempty"` + // BriefOptional says a bare `/` runs the program's default brief + // ([delegate.Delegate.DefaultBrief]) rather than asking for one. + BriefOptional bool `json:",omitempty"` } // DelegateReport is the programs this conversation can hand work to, as the @@ -59,7 +64,8 @@ func (c Config) delegateReport() DelegateReport { if lands == "" { lands = delegate.LandsTree } - report.Rows = append(report.Rows, DelegateRow{Name: program.Name, Description: program.Summary, Lands: lands}) + report.Rows = append(report.Rows, DelegateRow{Name: program.Name, Description: program.Summary, Lands: lands, + Args: program.Args, BriefOptional: strings.TrimSpace(program.DefaultBrief) != ""}) } return report } @@ -74,6 +80,18 @@ func (c Config) delegateNames() []string { return names } +// delegateNamed finds a program this launch carries by its name, without the +// refusal [Agent.delegateFor] words for one it does not. +func (c Config) delegateNamed(name string) (delegate.Delegate, bool) { + name = strings.TrimSpace(name) + for _, program := range c.Delegates { + if name != "" && program.Name == name { + return program, true + } + } + return delegate.Delegate{}, false +} + // mayDelegate says whether this belt may hand work to a program: it is the // conversation's own hand-off predicate with one more condition, that this // build carries at least one. A task node never delegates, for the reason it @@ -392,6 +410,16 @@ func (a *Agent) delegateFor(name string) (delegate.Delegate, error) { return delegate.Delegate{}, DelegateUnknownError{Named: name, Have: a.config.delegateNames()} } +// programWords is the brief a program that reads its brief as words is +// handed from a proposal: the words the model wrote, or the program's default +// brief when it wrote none ([delegate.Delegate.Words]). +func programWords(brief string, program delegate.Delegate) string { + if brief = strings.TrimSpace(brief); brief != "" { + return brief + } + return strings.TrimSpace(program.DefaultBrief) +} + // StartDelegate hands one person-authored brief to the named program. It is // `/ `'s door and it answers what StartTask answers: the id the // row wears, the title, a note about where the work stands (always empty here) @@ -402,10 +430,13 @@ func (a *Agent) delegateFor(name string) (delegate.Delegate, error) { // linked. func (a *Agent) StartDelegate(ctx context.Context, name, brief string) (uint64, string, string, error) { brief = strings.TrimSpace(brief) + program, err := a.delegateFor(name) + if brief == "" && err == nil { + brief = strings.TrimSpace(program.DefaultBrief) + } if brief == "" { return 0, "", "", errors.New("/" + strings.TrimSpace(name) + " needs a brief: the whole task, in words") } - program, err := a.delegateFor(name) if err != nil { return 0, "", "", err } @@ -422,9 +453,14 @@ func (a *Agent) StartDelegate(ctx context.Context, name, brief string) (uint64, } id := g.reserve() title := taskPersonTitle(brief) + if program.Title != nil { + if own := strings.TrimSpace(program.Title(brief)); own != "" { + title = own + } + } note := "" - if program.Name == "senior-dev" { - note = a.seniorDevCeilings(a.Usage().CostUSD).Summary() + if !program.Unattended.IsZero() { + note = a.programCeilings(&program, a.Usage().CostUSD).Summary() } if err := a.startKnownTaskRunVia(ctx, id, title, brief, nil, delegateStand(folder), "", &program); err != nil { return 0, "", "", err diff --git a/internal/session/delegate_door_test.go b/internal/session/delegate_door_test.go index 5d36139455..e429a63136 100644 --- a/internal/session/delegate_door_test.go +++ b/internal/session/delegate_door_test.go @@ -2,6 +2,7 @@ package session import ( "context" + "encoding/json" "flag" "os" "path/filepath" @@ -10,6 +11,7 @@ import ( "strings" "testing" + "github.com/Agent-Field/agentfield/sdk/go/ai" "github.com/Agent-Field/codeaf/internal/config" "github.com/Agent-Field/codeaf/internal/crewroute" "github.com/Agent-Field/codeaf/internal/delegate" @@ -21,13 +23,20 @@ import ( // never starts its process — the run engine is a double here — so its command // is a body that is never called. func testPrograms(name string) []delegate.Delegate { - return []delegate.Delegate{{ + program := delegate.Delegate{ Name: name, Summary: "a fake program", Default: "run", Page: name, Guide: "For work a fake does, with a brief that names the fake's files.", Commands: []delegate.Command{{Name: "run", Bind: func(*flag.FlagSet) delegate.Body { return func(context.Context, delegate.Host, []string) error { return nil } }}}, - }} + } + // A fake called senior-dev carries senior-dev's own unattended ceilings, + // because what the tests that name it hold is how a run of a program with + // ceilings of its own is bounded. + if name == "senior-dev" { + program.Unattended = delegate.SeniorDevCeilings + } + return []delegate.Delegate{program} } // The whole road from the door to the branch: `/fake ` starts a run @@ -341,6 +350,89 @@ func TestAProgramsRunThatDidNotFinishIsEndedInItsStore(t *testing.T) { } } +// A BARE START OF A PROGRAM WHOSE NAME SAYS THE WORK runs its default brief, +// and starts under its own ceilings, which the start note names; one with no +// default brief is still asked for one. +func TestABareStartRunsTheProgramsDefaultBrief(t *testing.T) { + double := newBeltRunDouble("done") + registerBeltRunEngine(t, double) + registry := testPrograms("audit") + registry[0].Lands, registry[0].DefaultBrief = delegate.LandsText, "whole repository" + registry[0].Unattended = delegate.Ceilings{CostUSD: 5, Hours: 2} + registry[0].Title = func(brief string) string { return "Audit of the " + brief } + registry = append(registry, testPrograms("fake")...) + agent, _ := newTestAgent(t, beltRunCompleter{text: "unused"}, func(config *Config) { + config.Workspace = newTestRepo(t) + config.Place = Place{Dir: t.TempDir()} + config.Delegates = registry + }) + _, title, note, err := agent.StartDelegate(context.Background(), "audit", " ") + if err != nil { + t.Fatal(err) + } + if title != "Audit of the whole repository" || note != "up to $5.00 and 2h" { + t.Fatalf("title %q, note %q", title, note) + } + <-double.entered + double.mu.Lock() + spec := double.spec + double.mu.Unlock() + if spec.Brief != "whole repository" || spec.Delegate == nil || spec.Delegate.Name != "audit" { + t.Fatalf("the run was handed %q for %v", spec.Brief, spec.Delegate) + } + if _, _, _, err := agent.StartDelegate(context.Background(), "fake", ""); err == nil || !strings.Contains(err.Error(), "/fake needs a brief") { + t.Fatalf("a program with no default brief started bare: %v", err) + } +} + +// A PROPOSED RUN OF A PROGRAM WHOSE BRIEF IS WORDS IS HANDED THE WORDS. The +// chat's proposal composes a brief under its own headings, and sec read its +// scope off that brief's first line: a proposed `whole repository thorough` +// ran at standard depth, because the first line was `WHAT THE PERSON ASKED +// FOR`. And the row it leaves in the project's index points at its record +// folder, where its report is, and not at the repository it changed nothing in. +func TestAProposedProgramWhoseBriefIsWordsIsHandedOnlyItsWords(t *testing.T) { + double := newBeltRunDouble("done") + registerBeltRunEngine(t, double) + registry := testPrograms("audit") + registry[0].Lands, registry[0].DefaultBrief, registry[0].Words = delegate.LandsText, "whole repository", true + arguments, _ := json.Marshal(taskArguments{ + Title: "Second audit of the repository", Summary: "a thorough pass to confirm the first", + Brief: "whole repository thorough", Deliverable: "a report with every finding", Acceptance: "every phase finishes", + Via: "audit", + }) + completer := &routedCompleter{parent: []step{ + func(context.Context, []ai.Message) (*ai.Response, error) { + return toolResponse("call-task", "propose_task", string(arguments)), nil + }, + finalText("started"), + }} + var index string + agent, _ := newTestAgent(t, completer, func(config *Config) { + config.Workspace = newTestRepo(t) + config.Place = Place{Dir: filepath.Join(t.TempDir(), "conversation")} + config.AskConsent = false + config.Delegates = registry + index = config.taskIndexFile() + }) + collect(t, mustSubmit(t, agent, "can we run a second, thorough audit to confirm these?")) + <-double.entered + double.mu.Lock() + spec := double.spec + double.mu.Unlock() + if spec.Brief != "whole repository thorough" { + t.Fatalf("the program was handed %q, want only its words", spec.Brief) + } + agent.beltMu.Lock() + records := plandb.TaskDir(filepath.Dir(agent.beltRun.store.Path()), agent.beltRun.root) + agent.beltMu.Unlock() + endBeltRun(t, agent, double) + rows := ReadTaskIndex(index) + if len(rows) != 1 || rows[0].ArtifactURI != "file://"+records { + t.Fatalf("the index rows = %+v, want one pointing at the record folder %s", rows, records) + } +} + func TestStartDelegateRefusesANameThisMachineDoesNotHave(t *testing.T) { double := newBeltRunDouble("done") registerBeltRunEngine(t, double) diff --git a/internal/session/prefixbudget_test.go b/internal/session/prefixbudget_test.go index 2d60236ff9..2529b73780 100644 --- a/internal/session/prefixbudget_test.go +++ b/internal/session/prefixbudget_test.go @@ -553,9 +553,18 @@ const fixedPrefixTarget = 48_000 // block to 34,927 over 24 tools. The measured fixed prefix is 57,218 bytes: // 94 above its preceding cap. The fixed waiver rises by exactly 94 for this // combined belt; the lean arm remains within its 49,590-byte cap. +// +// 2026-10-05, sec (sec-af's security audit), the second program codeaf +// carries. Its guide (programguide.Sec, 386 bytes) is one more item of the +// hand-off paragraph's list, which is all the prefix pays for a program: its +// manual page and its report-turn wording ride no request. Measured against dev +// 7e3033564 it cost the fixed prefix 366 bytes and the lean 229; 2026-10-07, +// merged with dev baa2d7f0f, whose page carries a program's guide on the lean +// arm too, it costs each 396 (dev measures 57,150 and 49,572; with sec, +// 57,546 and 49,968). Both waivers are dev's plus exactly that 396. const ( - fixedPrefixWaiver = 9_218 - leanPrefixWaiver = 18_090 + fixedPrefixWaiver = 9_614 + leanPrefixWaiver = 18_486 ) // THE LEAN PROFILE GETS A BUDGET OF ITS OWN (2026-09-10, the prompt diet's lane @@ -680,7 +689,10 @@ const leanWindow = 16_000 // own guide but does not read the platform's carried registry: Windows carries // none, while this budget must still weigh the Unix shipping maximum. func prefixPrograms() []delegate.Delegate { - return []delegate.Delegate{{Name: "senior-dev", Guide: programguide.SeniorDev, Lands: delegate.LandsTree}} + return []delegate.Delegate{ + {Name: "sec", Guide: programguide.Sec, Lands: delegate.LandsText}, + {Name: "senior-dev", Guide: programguide.SeniorDev, Lands: delegate.LandsTree}, + } } // widestPage is the page at its heaviest: prompts/system.md with every one of diff --git a/internal/session/program_outcome.go b/internal/session/program_outcome.go index 33c47e2db4..ab347534a2 100644 --- a/internal/session/program_outcome.go +++ b/internal/session/program_outcome.go @@ -113,6 +113,18 @@ type programOutcome struct { row uint64 program string verdict programVerdict + // report is a program that lands text ([delegate.LandsText]): its ending + // is an answer that changed nothing in the folder, so there is no branch + // to read, check or offer, and the turn is told so. + report bool + // followUp is the program's own sentence for what to offer once it has + // finished ([delegate.Delegate.FollowUp]); empty is none. + followUp string + // account is the run's ending as the note handed it to the turn — the + // landing line, and a report program's whole answer — kept so a turn that + // could not finish its own answer still leaves it with the person + // ([Agent.programOutcomeUnansweredLocked]). It is not saved with the hold. + account string programAttempt } @@ -129,6 +141,12 @@ func programOutcomeNote(outcome programOutcome, line string, costUSD float64) st } b.WriteString("\n") b.WriteString(programNextStep(outcome)) + if outcome.report { + // A REPORT HAS NO CODE TO LOCATE. The paragraph below is about the + // branch a program that edits files leaves, and handed to the turn after + // an answer it sent the model looking for a branch that was never cut. + return b.String() + } b.WriteString("\n\nBefore any checks or edits, locate the run's code. If the ending names a branch whose copy was removed, run each check in a temporary worktree that one command adds and always removes: `tmp=$(mktemp -d) && git -C worktree add -q --detach \"$tmp\" && (cd \"$tmp\" && ); status=$?; git -C worktree remove --force \"$tmp\"; exit $status`. A worktree left behind stays registered in the person's repository. Do not run checks in the person's checkout: it still holds their code, not the run's result, and tests can leave files there. If the ending says its copy was kept, or the run worked in a folder with no git history, use the folder it names instead.") return b.String() } @@ -136,6 +154,9 @@ func programOutcomeNote(outcome programOutcome, line string, costUSD float64) st // programNextStep is what to do about one ending, in one sentence the model // reads with the playbook it expands. func programNextStep(o programOutcome) string { + if o.report { + return programReportStep(o) + } left := programAutoRetries - o.auto switch o.verdict { case programPassed: @@ -154,6 +175,27 @@ func programNextStep(o programOutcome) string { return fmt.Sprintf("It handed in no finished change. Read what it got done on its branch, then hand the work back to %s with a brief sharpened by what is missing, which counts as fixing it yourself; finish only a trivial gap in your own worktree. You may send it back %d more time%s on your own.", o.program, left, plural(left)) } +// programReportStep is what to do about the ending of a program that lands a +// report rather than a change. Its answer is above, in the note; nothing in the +// folder moved, so there is nothing to check on a branch and nothing to merge. +func programReportStep(o programOutcome) string { + left := programAutoRetries - o.auto + switch o.verdict { + case programPassed, programUnverified: + step := "It finished, and its report is above; it changed nothing in the folder. Tell the person in a short plain summary what it found, most serious first, and where its full report is." + if follow := strings.TrimSpace(o.followUp); follow != "" { + step += " " + follow + } + return step + " Start nothing on this turn." + case programLimit: + return "It stopped on a limit, so another run spends more of the person's money: do not hand it back. Tell the person briefly what it found before it stopped, and ask whether to spend more." + } + if left <= 0 || o.verdict != programCrashed { + return "It did not finish its report. Tell the person plainly what it got through and why it stopped, and let them decide whether to run it again." + } + return fmt.Sprintf("It broke rather than finished. If the cause looks passing (a network or provider failure), hand the same work to %s again; otherwise tell the person plainly. You may send it back %d more time%s on your own.", o.program, left, plural(left)) +} + // rememberProgramOutcomeLocked keeps a program's ending until the person next // speaks, so later automatic turns still know a hand-off is a re-attempt. // The caller holds a.mu. @@ -272,7 +314,15 @@ func (a *Agent) programAttemptFor(row uint64) programAttempt { func (a *Agent) programLandingNote(run *beltRun, summary RunSummary, line string) userMessage { outcome := programOutcome{ row: run.row, program: programName(run.delegate), verdict: programVerdictOf(summary), - programAttempt: a.programAttemptFor(run.row), + programAttempt: a.programAttemptFor(run.row), account: strings.TrimSpace(line), + } + if run.delegate != nil && !run.delegate.LandsTree() { + outcome.report, outcome.followUp = true, run.delegate.FollowUp + // A REPORT THAT ARRIVED IS A FINISHED ANSWER. `unverified` is about a + // change nobody's checks passed, and a report has no checks to run. + if outcome.verdict == programUnverified { + outcome.verdict = programPassed + } } a.mu.Lock() a.programHold = &outcome @@ -413,9 +463,53 @@ func programLimitLine(run *beltRun, summary RunSummary, landing RunLanding) stri if spent > 0 { spentWord = fmt.Sprintf("spent $%.2f", spent) } + if run.delegate != nil && !run.delegate.LandsTree() { + // A REPORT HAS NO WORK IN THE FOLDER to point at: it changed nothing + // there, and what it found before it stopped is in its ending. + return fmt.Sprintf("%s stopped at %s %s limit · %s · it changed nothing in %s", programName(run.delegate), scope, limit, spentWord, run.ground) + } where := "in the folder " + run.ground if landing.Branch != "" { where = "on branch " + landing.Branch + " in " + run.ground } return fmt.Sprintf("%s stopped at %s %s limit · %s · its work is %s", programName(run.delegate), scope, limit, spentWord, where) } + +// programOutcomeUnansweredLocked writes a program's own account of its ending +// into the conversation when the turn its ending woke did not finish an answer. +// The caller holds a.mu, at the end of that turn. +// +// A PROGRAM'S ENDING IS NEVER LOST TO A FAILED REPLY. On 2026-10-05 a two-hour +// sec run ended, the turn it woke read its report and began a good summary, +// and every model it was offered was cut mid-reply; the partial summary was +// kept as an interrupted message the chat does not draw, and the person saw +// a done card and nothing else. The program's account is what the turn was +// woken to relay, so when the turn could not, the conversation says that it +// could not and gives the account itself — once per run, as an authored line, +// the way a run's limit line is written ([Agent.recordProgramLimit]). +// +// AND IT SAYS WHY THE ANSWER IS MISSING. A turn the person stopped did not +// complete either, and telling them the chat "could not" finish would be the +// record misdescribing their own key: their stop keeps the account and names +// itself instead. +func (a *Agent) programOutcomeUnansweredLocked(hub *eventHub, completed, personStopped bool) { + outcome := a.programOutcomeNow + if completed || outcome == nil || outcome.account == "" || a.programAnsweredFor == outcome.row { + return + } + a.programAnsweredFor = outcome.row + why := "but the chat could not finish its answer to it" + if personStopped { + why = "and you stopped the chat's answer to it" + } + text := fmt.Sprintf("%s ended, %s, so here is what %s said:\n\n%s", + outcome.program, why, outcome.program, outcome.account) + note := userText(text) + note.authored = true + a.recordUserLocked(note) + event := Event{Kind: EventNotice, Text: text} + hub.send(event) + for _, watcher := range a.taskWatchers { + watcher.send(event) + } +} diff --git a/internal/session/program_outcome_test.go b/internal/session/program_outcome_test.go index 0896e5509b..44e5e41db0 100644 --- a/internal/session/program_outcome_test.go +++ b/internal/session/program_outcome_test.go @@ -506,3 +506,141 @@ func TestAProgramOutcomeCountsAHandBackAsFixingItYourself(t *testing.T) { } } } + +// A PROGRAM THAT ANSWERS WITH A REPORT IS NEVER SENT LOOKING FOR A BRANCH. Its +// ending changed nothing in the folder, so its note carries none of the +// worktree, check or merge advice a change gets: it says what the report is +// for, the program's own offer, and that nothing starts on this turn. +func TestAReportProgramsEndingAsksForASummaryAndStartsNothing(t *testing.T) { + const offer = "If it found problems, offer to hand them to senior-dev to fix." + note := func(verdict programVerdict, auto int) string { + return programOutcomeNote(programOutcome{row: 9, program: "sec", verdict: verdict, + report: true, followUp: offer, programAttempt: programAttempt{attempt: auto + 1, auto: auto}}, "done · ran 14m", 0.8) + } + passed := note(programPassed, 0) + for _, want := range []string{"[sec ended — for you to act on] task 9 · passed · run 1 · $0.80", + "it changed nothing in the folder", offer, "Start nothing on this turn."} { + if !strings.Contains(passed, want) { + t.Fatalf("a finished report's note lacks %q:\n%s", want, passed) + } + } + for _, verdict := range []programVerdict{programPassed, programFailed, programLimit, programCrashed} { + got := note(verdict, 0) + for _, unwanted := range []string{"worktree", "merge", "branch"} { + if strings.Contains(got, unwanted) { + t.Errorf("a %s report's note speaks of a %s:\n%s", verdict, unwanted, got) + } + } + } + if limit := note(programLimit, 0); !strings.Contains(limit, "ask whether to spend more") || strings.Contains(limit, offer) { + t.Fatalf("a report that stopped on a limit is not told to ask first:\n%s", limit) + } + if failed := note(programFailed, 0); strings.Contains(failed, "hand the same work") { + t.Fatalf("a report that did not finish is sent back on codeaf's own:\n%s", failed) + } + if crashed := note(programCrashed, programAutoRetries); strings.Contains(crashed, "hand the same work") { + t.Fatalf("a crash after the last retry is still sent back:\n%s", crashed) + } +} + +// A report program's ending that named no `pass` is still a finished answer, +// and its limit line points at nothing in the folder, because it changed +// nothing there. +func TestAReportProgramsRunReadsAsFinishedAndItsLimitLineNamesNoWork(t *testing.T) { + a := programConversation(t, nil) + program := testPrograms("sec")[0] + program.Lands = delegate.LandsText + store, err := plandb.Open(filepath.Join(t.TempDir(), planStoreFilename), "the audit", planRootID, "The audit", "") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + run := &beltRun{row: 2, root: planRootID, store: store, delegate: &program, ground: "/repo", costCeiling: 2} + note := a.programLandingNote(run, RunSummary{Outcome: beltRunOutcomeDone}, "done") + if note.programOutcome == nil || note.programOutcome.verdict != programPassed || !note.programOutcome.report { + t.Fatalf("a finished report reads as %+v, want a passed report", note.programOutcome) + } + line := programLimitLine(run, RunSummary{Outcome: "incomplete", Limit: RunLimitCost, USD: 2}, RunLanding{}) + if !strings.Contains(line, "it changed nothing in /repo") || strings.Contains(line, "its work is") { + t.Fatalf("a report's limit line = %q", line) + } +} + +// failingCompleter is a model that cannot answer at all: every call is +// refused for want of a key, which ends a turn at once. +type failingCompleter struct{ calls atomic.Int32 } + +func (f *failingCompleter) CompleteWithMessages(context.Context, []ai.Message, ...ai.Option) (*ai.Response, error) { + f.calls.Add(1) + return nil, provider.ErrNoAPIKey +} + +// A PROGRAM'S ENDING IS NEVER LOST TO A FAILED REPLY. The turn its landing +// wakes is the one that tells the person what the program found; when that turn +// cannot finish an answer, the program's own account is written into the +// conversation as the session's line — once, however many turns fail after it. +func TestAProgramsEndingIsSaidWhenTheTurnItWokeCannotAnswer(t *testing.T) { + completer := &failingCompleter{} + agent, _ := newTestAgent(t, completer, func(config *Config) { + config.AskConsent = false + // THE JOURNAL IS WHAT MARKS A LINE AS THE SESSION'S, so the conversation + // has one, as every conversation a person opens does. + config.SessionFile = filepath.Join(t.TempDir(), "conversation.jsonl") + }) + store, err := plandb.Open(filepath.Join(t.TempDir(), planStoreFilename), "the run", "1", "Audit", "whole repository") + if err != nil { + t.Fatal(err) + } + defer store.Close() + program := testPrograms("sec")[0] + program.Lands = delegate.LandsText + run := &beltRun{store: store, root: store.RootID(), row: 1, delegate: &program, ground: "/project"} + const account = "Found 2 problems: 1 confirmed and 1 likely.\n- critical · confirmed · Command injection · app/views.py:4" + agent.deliverBeltRunLanding(run, RunSummary{Outcome: beltRunOutcomeDone, Result: account}, RunLanding{}) + beltRunWaitFor(t, "the failed outcome turn to end", func() bool { + agent.mu.Lock() + defer agent.mu.Unlock() + return completer.calls.Load() > 0 && !agent.running + }) + said := 0 + for _, entry := range agent.Transcript() { + if strings.Contains(entry.Text, "sec ended, but the chat could not finish its answer to it") { + said++ + if entry.Role != "aside" || !strings.Contains(entry.Text, "app/views.py:4") { + t.Fatalf("the ending was said as %s: %q", entry.Role, entry.Text) + } + } + } + if said != 1 { + t.Fatalf("the program's ending was said %d times after the turn failed", said) + } + // A later failed turn does not say it again. + agent.mu.Lock() + agent.programOutcomeUnansweredLocked(nil, false, false) + agent.mu.Unlock() + again := 0 + for _, entry := range agent.Transcript() { + if strings.Contains(entry.Text, "could not finish its answer") { + again++ + } + } + if again != 1 { + t.Fatalf("the ending was said %d times", again) + } + // A TURN THE PERSON STOPPED IS TOLD AS THEIRS, and still keeps the account: + // their key did not make the chat fail. + agent.mu.Lock() + agent.programOutcomeNow = &programOutcome{row: 2, program: "sec", account: account} + agent.programOutcomeUnansweredLocked(nil, false, true) + agent.mu.Unlock() + stopped := 0 + for _, entry := range agent.Transcript() { + if strings.Contains(entry.Text, "sec ended, and you stopped the chat's answer to it, so here is what sec said:") && + strings.Contains(entry.Text, "app/views.py:4") { + stopped++ + } + } + if stopped != 1 { + t.Fatalf("a stopped answer was said %d times as the person's stop", stopped) + } +} diff --git a/internal/session/prompts/program-outcome.md b/internal/session/prompts/program-outcome.md index 173682072d..63aa3bb22c 100644 --- a/internal/session/prompts/program-outcome.md +++ b/internal/session/prompts/program-outcome.md @@ -2,6 +2,8 @@ A program you handed work to has just ended. Its ending is for you to act on, no The note ends with a line in brackets: the task, how it came out (passed, unverified, failed, limit, crashed), and which run on this work it was. The line under it says what to do now. Follow it. +Some programs answer with a report and change nothing in the folder. Their note says so in its last line, and for them everything below about branches, worktrees, checks and merging does not apply: there is no branch. Read the report in the note, tell the person in a short plain summary what it found (most serious first) and where the full report is, offer the next step the note names, and start nothing on this turn. A fix the person then asks for is a new piece of work, proposed in a later turn like any other. + For every outcome, bringing the program's branch into the person's branch or checkout, or pushing it, is the person's call. Never merge, rebase, cherry-pick or otherwise bring it over on this wake turn. You may offer it and stop; do it only when the person asks in a later message. Never push the program's branch (or anything) on this wake turn. When the ending says a push sends the work to its remote branch, offer that push and stop, and push only when the person asks in a later message. Checking and fixing work on the program's branch in a temporary worktree does not bring it into theirs. Wherever this page says to fix or finish something yourself, handing it back to the same program with propose_task and via counts as doing it yourself, and is the better road for anything beyond a line or two: on this turn the program carries on from its own branch, with what it learned on the run. Edit its branch in your own worktree only for a trivial gap. Once it can be sent back no more, tell the person what still does not work, where the work is, and what you would try next. diff --git a/internal/session/session.go b/internal/session/session.go index 9d6a424a90..a68999f59c 100644 --- a/internal/session/session.go +++ b/internal/session/session.go @@ -2785,6 +2785,11 @@ type Agent struct { // woken with, nil for every other turn: a hand-off it makes is a re-attempt // of that run ([Agent.programRetryRefusal]). Cleared with owedAsks. programOutcomeNow *programOutcome + // programAnsweredFor is the row whose program ending was last written + // into the conversation because the turn it woke did not finish its + // answer ([Agent.programOutcomeUnansweredLocked]), so a run's account is + // written once however many turns then fail. + programAnsweredFor uint64 // programHold is the last program ending since the person's own words. It // survives wake turns and reloads from the conversation's sidecar record. programHold *programOutcome diff --git a/internal/session/task.go b/internal/session/task.go index 2d119f259c..df8a6ec57a 100644 --- a/internal/session/task.go +++ b/internal/session/task.go @@ -922,6 +922,17 @@ func (p *stagedProposal) Commit(ctx context.Context) (string, bool, error) { return taskReceipt(p.id, spec, state, p.stand, elsewhere), false, nil } +// proposalBrief is what a proposal committed to a run hands its work: the +// composed brief every task gets, or, for a program that reads its brief as +// words, the words alone, the way a typed `/` hands them +// ([delegate.Delegate.Words]). +func proposalBrief(spec taskSpec, via *delegate.Delegate) string { + if via != nil && via.Words { + return programWords(spec.brief, *via) + } + return composeBrief(briefWhole, spec.request, spec.brief, spec.deliverable, spec.acceptance, "", spec.admission, spec.origin, taskCopy{}) +} + // commitProposalToRun is the run road of an approved proposal: an approved // hand-off under the bash belt, and every hand-off that names a delegate, is a // RUN and never a session-tree node. It keeps the id the card showed, carries @@ -953,7 +964,7 @@ func (a *Agent) commitProposalToRun(ctx context.Context, p *stagedProposal, spec } via = &m } - description := composeBrief(briefWhole, spec.request, spec.brief, spec.deliverable, spec.acceptance, "", spec.admission, spec.origin, taskCopy{}) + description := proposalBrief(spec, via) // THE RUN OUTLIVES THE TURN THAT LAUNCHED IT, AND NOT THE CONVERSATION. // This context is the turn's, and the turn cancels it on its way out // (agent.go, `defer cancel(nil)`); a run driven under it would be stopped @@ -1315,8 +1326,8 @@ func (a *Agent) openTask(ctx context.Context, id uint64, spec taskSpec, elsewher deadline = a.taskClockNow().Add(countdown) } question := newTaskQuestion(id, spec, elsewhere, deadline, a.config) - if spec.via == "senior-dev" { - question.notice.Ceiling = a.seniorDevCeilings(a.usage.CostUSD).Summary() + if program, known := a.config.delegateNamed(spec.via); known && !program.Unattended.IsZero() { + question.notice.Ceiling = a.programCeilings(&program, a.usage.CostUSD).Summary() } if a.taskAnswers == nil { a.taskAnswers = make(map[uint64]*taskQuestion, 1) diff --git a/internal/session/task_run_belt.go b/internal/session/task_run_belt.go index e9ad450ba5..0aa29718a4 100644 --- a/internal/session/task_run_belt.go +++ b/internal/session/task_run_belt.go @@ -71,9 +71,10 @@ func runCostLeft(limit, spent float64) float64 { return left } -// seniorDevCeilings caps the conversation's remaining allowance at the -// unattended run defaults, so an unset conversation limit is still finite. -func (a *Agent) seniorDevCeilings(spent float64) delegate.Ceilings { +// programCeilings caps the conversation's remaining allowance at the +// program's unattended ceilings ([delegate.Delegate.Unattended]), so an unset +// conversation limit is still finite for a program that names its own. +func (a *Agent) programCeilings(program *delegate.Delegate, spent float64) delegate.Ceilings { wallLeft, _ := a.config.Budget.Left() if a.config.Budget.Wall > 0 && !a.startedAt.IsZero() { wallLeft = a.config.Budget.Wall - time.Since(a.startedAt) @@ -84,7 +85,33 @@ func (a *Agent) seniorDevCeilings(spent float64) delegate.Ceilings { return (delegate.Ceilings{ CostUSD: runCostLeft(a.railCap(0), spent), Hours: wallLeft.Hours(), - }).SeniorDev() + }).CappedBy(programUnattended(program)) +} + +// programUnattended is a program's own ceilings, none for no program. +func programUnattended(program *delegate.Delegate) delegate.Ceilings { + if program == nil { + return delegate.Ceilings{} + } + return program.Unattended +} + +// holdProgramCeilings writes onto the run which ceilings it starts under and +// whether the conversation's own limit, rather than the program's unattended +// one, is the one that will stop it — the words its ending is told in. A run +// of no program keeps none. +// +// A PROGRAM WITH NO CEILINGS OF ITS OWN IS STOPPED BY THE CONVERSATION'S, and +// its ending says so. Skipping it left the run holding no ceiling and no +// word for whose it was, and its ending named "the run's $0.00 limit". +func (a *Agent) holdProgramCeilings(run *beltRun, spec RunSpec) { + if run.delegate == nil { + return + } + own := programUnattended(run.delegate) + run.costCeiling, run.timeCeiling = spec.CostUSD, spec.Elapsed.Hours() + run.conversationCostLimit = a.railCap(0) > 0 && (own.CostUSD <= 0 || runCostLeft(a.railCap(0), a.Usage().CostUSD) <= own.CostUSD) + run.conversationTimeLimit = a.programConversationTimeLimit(own) } // RunSpec is one run as the door hands it to the engine: the store to drive, @@ -677,11 +704,7 @@ func (a *Agent) startAdmittedBeltRun(ctx context.Context, engine RunEngine, g *T Model: run.crewWorker(), }) spec := a.beltRunSpec(run, brief) - if programName(via) == "senior-dev" { - run.costCeiling, run.timeCeiling = spec.CostUSD, spec.Elapsed.Hours() - run.conversationCostLimit = a.railCap(0) > 0 && runCostLeft(a.railCap(0), a.Usage().CostUSD) <= delegate.DefaultSeniorDevCostUSD - run.conversationTimeLimit = a.seniorDevConversationTimeLimit() - } + a.holdProgramCeilings(run, spec) go a.driveBeltRun(runCtx, engine, run, spec) return false, nil } @@ -767,17 +790,21 @@ func (run *beltRun) wishedFrom(ctx context.Context) { run.thinking, run.carry, run.crewEffort = wish.thinking, wish.carry, crewWishOf(ctx).effort } -// seniorDevConversationTimeLimit reports whether the person's remaining wall -// limit, rather than the unattended default, is the one that will stop the run. -func (a *Agent) seniorDevConversationTimeLimit() bool { +// programConversationTimeLimit reports whether the person's remaining wall +// limit, rather than the program's unattended one, is the one that will stop +// the run. +func (a *Agent) programConversationTimeLimit(own delegate.Ceilings) bool { if a.config.Budget.Wall <= 0 { return false } + if own.Hours <= 0 { + return true + } remaining := a.config.Budget.Wall if !a.startedAt.IsZero() { remaining -= time.Since(a.startedAt) } - return remaining <= time.Duration(delegate.DefaultSeniorDevHours*float64(time.Hour)) + return remaining <= time.Duration(own.Hours*float64(time.Hour)) } // lockBeltStart takes the conversation's start lock, the one door every road @@ -1023,8 +1050,8 @@ func (a *Agent) beltRunSpec(run *beltRun, brief string) RunSpec { } } cost := runCostLeft(a.railCap(0), a.Usage().CostUSD) - if programName(run.delegate) == "senior-dev" { - ceilings := a.seniorDevCeilings(a.Usage().CostUSD) + if !programUnattended(run.delegate).IsZero() { + ceilings := a.programCeilings(run.delegate, a.Usage().CostUSD) cost, wallLeft = ceilings.CostUSD, ceilings.Elapsed() } admission := run.admission @@ -1835,11 +1862,7 @@ func (a *Agent) startPendingBeltRun(ctx context.Context, run *beltRun) (RunSpec, return RunSpec{}, false } spec := a.beltRunSpec(run, run.brief) - if programName(run.delegate) == "senior-dev" { - run.costCeiling, run.timeCeiling = spec.CostUSD, spec.Elapsed.Hours() - run.conversationCostLimit = a.railCap(0) > 0 && runCostLeft(a.railCap(0), a.Usage().CostUSD) <= delegate.DefaultSeniorDevCostUSD - run.conversationTimeLimit = a.seniorDevConversationTimeLimit() - } + a.holdProgramCeilings(run, spec) return spec, true } diff --git a/internal/session/task_run_index.go b/internal/session/task_run_index.go index f4bd2a7378..e8d3cee92a 100644 --- a/internal/session/task_run_index.go +++ b/internal/session/task_run_index.go @@ -25,8 +25,11 @@ package session // index would be the same work named twice in one answer. import ( + "path/filepath" "strconv" "strings" + + "github.com/Agent-Field/codeaf/internal/plandb" ) // indexRunRow appends one published run row to the project's index. A job's @@ -71,6 +74,9 @@ func (a *Agent) indexRunRow(notice TaskNotice) { entry.Where, entry.Ground, entry.Mode, entry.Rung = where.Dir, where.Ground, where.Mode, where.Rung } entry.ArtifactURI = taskArtifactURI(worktree, notice.Branch, notice.Merge) + if records := a.beltRunReportFolder(notice.ID); records != "" { + entry.ArtifactURI = taskURI(records) + } if notice.State.settled() { entry.Cost = notice.CostUSD if entry.Cost == 0 { @@ -95,6 +101,25 @@ func (a *Agent) beltRunSpent(id uint64) float64 { return 0 } +// beltRunReportFolder is the record folder of the live run with this id when +// a program that answers with a report has it, and "" for any other run. +// +// A REPORT'S ROW POINTS AT THE REPORT. A program that lands text works on the +// person's folder in place and changes nothing there, so the artifact the row +// would otherwise name is the repository itself, which says nothing about where +// the report went: another conversation reading this row searched the disk for +// it, and opened a different run's report first. The report is written in the +// task's record folder (delegate.EnvRecords), so that folder is the artifact. +func (a *Agent) beltRunReportFolder(id uint64) string { + a.beltMu.Lock() + defer a.beltMu.Unlock() + run := a.beltRun + if run == nil || run.row != id || run.delegate == nil || run.delegate.LandsTree() || run.store == nil { + return "" + } + return plandb.TaskDir(filepath.Dir(run.store.Path()), run.root) +} + // presenceBeltRuns is the live run this conversation has out, and each hand-off // that joined it and has not settled, one presence row each under the id its // index row carries — the join another window makes to know the running row in diff --git a/internal/tui3/delegate.go b/internal/tui3/delegate.go index 582aaf9404..3d81e3528c 100644 --- a/internal/tui3/delegate.go +++ b/internal/tui3/delegate.go @@ -47,6 +47,8 @@ var ( baseCommands = append([]command(nil), commands...) delegateRowsMu sync.Mutex delegateRows map[string]bool + // delegateBareRuns is the installed rows a bare `/` runs. + delegateBareRuns map[string]bool ) // installDelegateCommands rebuilds the live command table as the literal plus @@ -59,23 +61,30 @@ func installDelegateCommands(rows []session.DelegateRow) []string { delegateRowsMu.Lock() defer delegateRowsMu.Unlock() table := append([]command(nil), baseCommands...) - installed := map[string]bool{} + installed, optional := map[string]bool{}, map[string]bool{} var refused []string for _, row := range rows { name := strings.TrimSpace(row.Name) if name == "" { continue } - candidate := command{name: name, args: "", desc: row.Description} + args := strings.TrimSpace(row.Args) + if args == "" { + args = "" + } + candidate := command{name: name, args: args, desc: row.Description} if err := checkCommands(append(append([]command(nil), table...), candidate)); err != nil || baseNames()[name] { refused = append(refused, name+": its name is already a command here — not added") continue } table = append(table, candidate) installed[name] = true + if row.BriefOptional { + optional[name] = true + } } commands = table - delegateRows = installed + delegateRows, delegateBareRuns = installed, optional return refused } @@ -91,6 +100,14 @@ func baseNames() map[string]bool { return names } +// delegateRunsBare says a bare `/` runs the program's own default brief +// rather than asking for one (session.DelegateRow.BriefOptional). +func delegateRunsBare(name string) bool { + delegateRowsMu.Lock() + defer delegateRowsMu.Unlock() + return delegateBareRuns[name] +} + // isDelegateCommand says whether a typed word is one of the installed rows. func isDelegateCommand(name string) bool { delegateRowsMu.Lock() @@ -128,7 +145,7 @@ func (a *app) installDelegates() tea.Cmd { // answer lands as a task start, on the message `/task` lands on. func (a *app) runDelegateCommand(name, brief string) tea.Cmd { brief = strings.TrimSpace(brief) - if brief == "" { + if brief == "" && !delegateRunsBare(name) { a.note("usage: /" + name + delegateUsageWordTail) return nil } diff --git a/internal/tui3/delegate_test.go b/internal/tui3/delegate_test.go index 739a01c0ea..6d2605ae87 100644 --- a/internal/tui3/delegate_test.go +++ b/internal/tui3/delegate_test.go @@ -118,6 +118,31 @@ func TestADelegateRowWithNoBriefSaysItsUsage(t *testing.T) { } } +// A PROGRAM WHOSE NAME SAYS THE WORK RUNS BARE. Its row shows its own +// argument shape, and `/` alone goes to its door with no brief, where +// the program's default brief is put in (session.Agent.StartDelegate). +func TestABareRunProgramsRowShowsItsArgsAndStartsWithNoBrief(t *testing.T) { + a, fake := newDelegateApp(t, session.DelegateRow{Name: "audit", Description: "an audit", + Args: "[changes] [quick | thorough]", BriefOptional: true}) + shown := false + for _, c := range commands { + if c.name == "audit" && c.args == "[changes] [quick | thorough]" { + shown = true + } + } + if !shown { + t.Fatal("the row does not show the program's own arguments") + } + cmd := a.slash("/audit") + if cmd == nil { + t.Fatal("a bare /audit opened no door") + } + settleDoor(t, a, cmd) + if len(fake.started) != 1 || fake.started[0] != "audit: " { + t.Fatalf("StartDelegate was asked %v", fake.started) + } +} + func TestAProgramNamedLikeABuiltInCommandIsNotInstalled(t *testing.T) { newDelegateApp(t, session.DelegateRow{Name: "task", Description: "an impostor"}) if isDelegateCommand("task") {