Add support for YARA data collection

vulnerabilities/improvers/__init__.py

@@ -32,6 +32,7 @@
 )
 from vulnerabilities.pipelines.v2_improvers import flag_ghost_packages as flag_ghost_packages_v2
 from vulnerabilities.pipelines.v2_improvers import unfurl_version_range as unfurl_version_range_v2
+from vulnerabilities.pipelines.v2_improvers import yara_rules
 from vulnerabilities.utils import create_registry
 
 IMPROVERS_REGISTRY = create_registry(
@@ -72,5 +73,6 @@
         unfurl_version_range_v2.UnfurlVersionRangePipeline,
         compute_advisory_todo.ComputeToDo,
         collect_ssvc_trees.CollectSSVCPipeline,
+        yara_rules.YaraRulesImproverPipeline,
     ]
 )

vulnerabilities/migrations/0106_detectionrule.py

@@ -0,0 +1,68 @@
+# Generated by Django 4.2.25 on 2025-12-16 12:27
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0105_packagecommitpatch_patch_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="DetectionRule",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                (
+                    "rule_type",
+                    models.CharField(
+                        choices=[
+                            ("yara", "Yara"),
+                            ("yara-x", "Yara-X"),
+                            ("sigma", "Sigma"),
+                            ("clamav", "ClamAV"),
+                            ("suricata", "Suricata"),
+                        ],
+                        help_text="The type of the detection rule content (e.g., YARA, Sigma).",
+                        max_length=50,
+                    ),
+                ),
+                (
+                    "source_url",
+                    models.URLField(
+                        help_text="URL to the original source or reference for this rule.",
+                        max_length=1024,
+                    ),
+                ),
+                (
+                    "rule_metadata",
+                    models.JSONField(
+                        blank=True,
+                        help_text="Additional structured data such as tags, or author information.",
+                        null=True,
+                    ),
+                ),
+                (
+                    "rule_text",
+                    models.TextField(help_text="The content of the detection signature."),
+                ),
+                (
+                    "advisory",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="detection_rules",
+                        to="vulnerabilities.advisoryv2",
+                    ),
+                ),
+            ],
+        ),
+    ]

vulnerabilities/models.py

@@ -3489,3 +3489,45 @@ def __str__(self):
 
     class Meta:
         unique_together = ("vector", "source_advisory")
+
+
+class DetectionRuleTypes(models.TextChoices):
+    """Defines the supported formats for security detection rules."""
+
+    YARA = "yara", "Yara"
+    YARA_X = "yara-x", "Yara-X"
+    SIGMA = "sigma", "Sigma"
+    CLAMAV = "clamav", "ClamAV"
+    SURICATA = "suricata", "Suricata"
+
+
+class DetectionRule(models.Model):
+    """
+    A Detection Rule is code used to identify malicious activity or security threats.
+    """
+
+    rule_type = models.CharField(
+        max_length=50,
+        choices=DetectionRuleTypes.choices,
+        help_text="The type of the detection rule content (e.g., YARA, Sigma).",
+    )
+
+    source_url = models.URLField(
+        max_length=1024, help_text="URL to the original source or reference for this rule."
+    )
+
+    rule_metadata = models.JSONField(
+        null=True,
+        blank=True,
+        help_text="Additional structured data such as tags, or author information.",
+    )
+
+    rule_text = models.TextField(help_text="The content of the detection signature.")
+
+    advisory = models.ForeignKey(
+        AdvisoryV2,
+        related_name="detection_rules",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+    )

vulnerabilities/pipelines/v2_improvers/yara_rules.py

@@ -0,0 +1,119 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+from pathlib import Path
+
+from aboutcode.pipeline import LoopProgress
+from fetchcode.vcs import fetch_via_vcs
+
+from vulnerabilities.models import DetectionRule
+from vulnerabilities.models import DetectionRuleTypes
+from vulnerabilities.pipelines import VulnerableCodePipeline
+
+
+class YaraRulesImproverPipeline(VulnerableCodePipeline):
+    pipeline_id = "yara_rules"
+
+    repo_urls = [
+        "git+https://github.com/elastic/protections-artifacts",
+        "git+https://github.com/Yara-Rules/rules",
+        "git+https://github.com/Xumeiquer/yara-forensics",
+        "git+https://github.com/reversinglabs/reversinglabs-yara-rules",
+        "git+https://github.com/advanced-threat-research/Yara-Rules",
+        "git+https://github.com/bartblaze/Yara-rules",
+        "git+https://github.com/godaddy/yara-rules",  # archived
+        "git+https://github.com/SupportIntelligence/Icewater",
+        "git+https://github.com/jeFF0Falltrades/YARA-Signatures",
+        "git+https://github.com/tjnel/yara_repo",
+        "git+https://github.com/JPCERTCC/jpcert-yara",
+        "git+https://github.com/mikesxrs/Open-Source-YARA-rules",
+        "git+https://github.com/fboldewin/YARA-rules",
+        "git+https://github.com/h3x2b/yara-rules",
+    ]
+
+    license_urls = """
+    https://github.com/elastic/protections-artifacts/blob/main/LICENSE.txt
+    https://github.com/Yara-Rules/rules/blob/master/LICENSE
+    https://github.com/Xumeiquer/yara-forensics/blob/master/LICENSE
+    https://github.com/reversinglabs/reversinglabs-yara-rules/blob/develop/LICENSE
+    https://github.com/advanced-threat-research/Yara-Rules/blob/master/LICENSE
+    https://github.com/bartblaze/Yara-rules/blob/master/LICENSE
+    https://github.com/godaddy/yara-rules/blob/master/LICENSE.md
+    https://github.com/SupportIntelligence/Icewater/blob/master/LICENSE
+    https://github.com/jeFF0Falltrades/YARA-Signatures/blob/master/LICENSE.md
+    https://github.com/tjnel/yara_repo/blob/master/LICENSE
+    https://github.com/JPCERTCC/jpcert-yara/blob/main/LICENSE
+
+    NO-LICENSE: https://github.com/mikesxrs/Open-Source-YARA-rules/
+    NO-LICENSE: https://github.com/fboldewin/YARA-rules
+    NO-LICENSE: https://github.com/h3x2b/yara-rules
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.vcs_responses = []
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.clone_repos,
+            cls.collect_and_store_rules,
+            cls.clean_downloads,
+        )
+
+    def clone_repos(self):
+        for url in self.repo_urls:
+            self.log(f"Cloning `{url}`")
+            try:
+                response = fetch_via_vcs(url)
+                if response:
+                    self.vcs_responses.append((response, url))
+            except Exception as e:
+                self.log(f"Failed to clone {url}: {e}")
+
+    def collect_and_store_rules(self):
+        for vcs_response, repo_url in self.vcs_responses:
+            base_directory = Path(vcs_response.dest_dir)
+            yara_files = [
+                p
+                for p in base_directory.rglob("*")
+                if p.suffix in (".yar", ".yara") and p.is_file()
+            ]
+
+            rules_count = len(yara_files)
+            self.log(f"Processing {rules_count:,d} rules from {repo_url}")
+
+            progress = LoopProgress(total_iterations=rules_count, logger=self.log)
+            for file_path in progress.iter(yara_files):
+                if not file_path.exists() or not file_path.is_file():
+                    self.log(
+                        f"Skipping file as it no longer exists or is not a file: {file_path}",
+                        level="warning",
+                    )
+                    continue
+
+                raw_text = file_path.read_text(encoding="utf-8", errors="ignore")
+                if not raw_text:
+                    continue
+
+                DetectionRule.objects.update_or_create(
+                    rule_text=raw_text,
+                    rule_type=DetectionRuleTypes.YARA,
+                    advisory=None,
+                )
+
+    def clean_downloads(self):
+        for vcs_response, _ in self.vcs_responses:
+            if vcs_response:
+                self.log(f"Removing cloned repository: {vcs_response.dest_dir}")
+                vcs_response.delete()
+
+        self.vcs_responses = []
+
+    def on_failure(self):
+        self.clean_downloads()

vulnerabilities/tests/pipelines/v2_improvers/test_yara.py

@@ -0,0 +1,32 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+
+from vulnerabilities.models import DetectionRule
+from vulnerabilities.pipelines.v2_improvers.yara_rules import YaraRulesImproverPipeline
+
+BASE_DIR = Path(__file__).resolve().parent
+TEST_REPO_DIR = (BASE_DIR / "../../test_data/yara").resolve()
+
+
+@pytest.mark.django_db
+def test_collect_and_store_rules_from_test_repo_dir():
+    mock_vcs_response = MagicMock()
+    mock_vcs_response.dest_dir = str(TEST_REPO_DIR)
+
+    improver = YaraRulesImproverPipeline()
+    improver.vcs_responses = [(mock_vcs_response, "https://github.com/mock/repo")]
+    improver.collect_and_store_rules()
+
+    assert DetectionRule.objects.exists()
+    assert DetectionRule.objects.count() == 4

vulnerabilities/tests/test_data/yara/Linux_Backdoor_Bash.yar

@@ -0,0 +1,45 @@
+rule Windows_Trojan_Xeno_f92ffb82 {
+    meta:
+        author = "Elastic Security"
+        id = "f92ffb82-b743-4df1-9d6b-2afa3b7bb61f"
+        fingerprint = "2ae1aebd652afb7da5799f46883205b1f3a5c5b01e975b526640407d9bd0d22c"
+        creation_date = "2024-10-10"
+        last_modified = "2024-10-24"
+        threat_name = "Windows.Trojan.Xeno"
+        reference_sample = "22dbdbcdd4c8b6899006f9f07e87c19b6a2947eeff8cc89c653309379b388cf4"
+        severity = 50
+        arch_context = "x86"
+        scan_context = "file, memory"
+        license = "Elastic License v2"
+        os = "windows"
+    strings:
+        $a1 = { 28 00 00 0A 7D 0E 00 00 04 02 7B 0E 00 00 04 28 29 00 00 0A 07 7B 03 00 00 04 02 7B 0E 00 00 04 6F 2A 00 00 0A 3A F2 00 00 00 02 7B 07 00 00 04 02 7B 09 00 00 04 6F 32 00 00 06 6F 2B 00 00 0A }
+    condition:
+        all of them
+}
+
+rule Windows_Trojan_Xeno_89f9f060 {
+    meta:
+        author = "Elastic Security"
+        id = "89f9f060-afc8-427d-ad36-3672016efdf6"
+        fingerprint = "ddc5bf8c6d5140cb9ea2fbd9b6f1aaab60f506dcd6161a26961958efa4aa42e1"
+        creation_date = "2024-10-25"
+        last_modified = "2024-11-26"
+        threat_name = "Windows.Trojan.Xeno"
+        reference_sample = "b74733d68e95220ab0630a68ddf973b0c959fd421628e639c1b91e465ba9299b"
+        severity = 100
+        arch_context = "x86"
+        scan_context = "file, memory"
+        license = "Elastic License v2"
+        os = "windows"
+    strings:
+        $sc_1 = { 8B 44 24 04 89 C6 FF 56 08 68 00 04 00 00 6A 08 50 FF 16 89 C3 8B 06 89 83 2C 01 00 00 8B 46 04 89 83 30 01 00 00 8B 46 08 89 83 34 01 00 00 8B 46 0C 89 83 38 }
+        $sc_2 = { 55 48 89 E5 48 83 EC 40 49 89 CC 41 FF 54 24 10 48 89 C1 BA 08 00 00 00 41 B8 00 04 00 00 41 FF 14 24 48 89 C3 49 8B 04 24 48 89 83 90 01 00 00 49 8B 44 24 08 }
+        $str_1 = "SharpInjector" ascii fullword
+        $str_2 = "HEAVENSGATE_NON_OPERATIONAL" ascii fullword
+        $str_3 = "ChromeDecryptor" ascii fullword
+        $str_4 = "DataExtractionStructs" ascii fullword
+        $str_5 = "XenoStealer" ascii fullword
+    condition:
+        (($sc_1 or $sc_2) and ($str_1 or $str_2)) and (1 of ($str_3, $str_4, $str_5))
+}

vulnerabilities/tests/test_data/yara/Linux_Backdoor_Python.yar

@@ -0,0 +1,19 @@
+rule Linux_Backdoor_Python_00606bac {
+    meta:
+        author = "Elastic Security"
+        id = "00606bac-83eb-4a58-82d2-e4fd16d30846"
+        fingerprint = "cce1d0e7395a74c04f15ff95f6de7fd7d5f46ede83322b832df74133912c0b17"
+        creation_date = "2021-01-12"
+        last_modified = "2021-09-16"
+        threat_name = "Linux.Backdoor.Python"
+        reference_sample = "b3e3728d43535f47a1c15b915c2d29835d9769a9dc69eb1b16e40d5ba1b98460"
+        severity = 100
+        arch_context = "x86"
+        scan_context = "file, memory"
+        license = "Elastic License v2"
+        os = "linux"
+    strings:
+        $a = { F4 01 83 45 F8 01 8B 45 F8 0F B6 00 84 C0 75 F2 83 45 F8 01 8B }
+    condition:
+        all of them
+}

vulnerabilities/tests/test_data/yara/Linux_Exploit_CVE_2022_0847.yar

@@ -0,0 +1,27 @@
+rule Linux_Exploit_CVE_2022_0847_e831c285 {
+    meta:
+        author = "Elastic Security"
+        id = "e831c285-b2b9-49f3-a87c-3deb806e31e4"
+        fingerprint = "376b791f9bb5f48d0f41ead4e48b5bcc74cb68002bb7c170760428ace169457e"
+        creation_date = "2022-03-10"
+        last_modified = "2022-03-14"
+        threat_name = "Linux.Exploit.CVE-2022-0847"
+        reference_sample = "c6b2cef2f2bc04e3ae33e0d368eb39eb5ea38d1bca390df47f7096117c1aecca"
+        severity = 100
+        arch_context = "x86"
+        scan_context = "file, memory"
+        license = "Elastic License v2"
+        os = "linux"
+    strings:
+        $pp = "prepare_pipe"
+        $s1 = "splice failed"
+        $s2 = "short splice"
+        $s3 = "short write"
+        $s4 = "hijacking suid binary"
+        $s5 = "Usage: %s TARGETFILE OFFSET DATA"
+        $s6 = "Usage: %s SUID"
+        $bs1 = { B8 00 10 00 00 81 7D EC 00 10 00 00 0F 46 45 EC 89 45 FC 8B 55 FC 48 8B 45 D8 48 83 C0 04 8B 00 48 8D 35 }
+        $bs2 = { B8 00 10 00 00 81 7D F0 00 10 00 00 0F 46 45 F0 89 45 F8 8B 55 F8 48 8B 45 D8 8B 00 48 }
+    condition:
+        ($pp and 2 of ($s*)) or (all of ($bs*))
+}