merging llm-redactor to main

.gitignore

@@ -163,4 +163,6 @@ pyrightconfig.json
 
 # direnv files, used by load python venv
 .direnv/
-.envrc
+.envrc
+.local.env
+.envrc

README.md

@@ -40,3 +40,7 @@ A `Makefile` is provided with the following targets:
 - `htmlcov`  : run the unit tests and generate a full report in htmlcov/
 
 Testing and coverage requires standing up a local testbed. For details, see [Design](docs/design.md).
+
+
+## Adding `llm-redactor` branch
+For the redactor feature.

data/custom_fields.json

@@ -1 +1,207 @@
-{"custom_fields":[{"id":2,"name":"Discord ID","description":"ID used to link user to their discord account, to enable integration","customized_type":"user","field_format":"string","regexp":"","min_length":null,"max_length":null,"is_required":false,"is_filter":false,"searchable":false,"multiple":false,"default_value":"","visible":true},{"id":4,"name":"syncdata","description":"Metadata used to sync the ticket with external sources.\r\n\r\nCurrent format is thread-id|zulu-timestamp.\r\n\r\nNot recommend to edit.","customized_type":"issue","field_format":"string","regexp":"","min_length":null,"max_length":null,"is_required":false,"is_filter":true,"searchable":false,"multiple":false,"default_value":"","visible":false,"trackers":[{"id":2,"name":"Infra-Field"},{"id":4,"name":"Software-Dev"},{"id":6,"name":"Infra-Config"},{"id":8,"name":"External-Comms-Intake"},{"id":9,"name":"Outreach-Partnerships"},{"id":10,"name":"Admin"},{"id":17,"name":"Research"},{"id":18,"name":"Mutual-Aid-Action"},{"id":19,"name":"SCN-Space"}],"roles":[{"id":3,"name":"Administrator"}]},{"id":5,"name":"To/CC","description":"Contains the To and Cc headers from the email that created the ticket.","customized_type":"issue","field_format":"string","regexp":"","min_length":null,"max_length":null,"is_required":false,"is_filter":false,"searchable":true,"multiple":false,"default_value":"","visible":true,"trackers":[{"id":2,"name":"Infra-Field"},{"id":4,"name":"Software-Dev"},{"id":6,"name":"Infra-Config"},{"id":8,"name":"External-Comms-Intake"},{"id":9,"name":"Outreach-Partnerships"},{"id":10,"name":"Admin"},{"id":13,"name":"Test-Reject"},{"id":17,"name":"Research"},{"id":18,"name":"Mutual-Aid-Action"},{"id":19,"name":"SCN-Space"}],"roles":[]}]}
+{
+  "custom_fields": [
+    {
+      "id": 2,
+      "name": "Discord ID",
+      "description": "ID used to link user to their discord account, to enable integration",
+      "customized_type": "user",
+      "field_format": "string",
+      "regexp": "",
+      "min_length": null,
+      "max_length": null,
+      "is_required": false,
+      "is_filter": false,
+      "searchable": false,
+      "multiple": false,
+      "default_value": "",
+      "visible": true,
+      "editable": true
+    },
+    {
+      "id": 4,
+      "name": "syncdata",
+      "description": "Metadata used to sync the ticket with external sources.\r\n\r\nCurrent format is thread-id|zulu-timestamp.\r\n\r\nNot recommend to edit.",
+      "customized_type": "issue",
+      "field_format": "string",
+      "regexp": "",
+      "min_length": null,
+      "max_length": null,
+      "is_required": false,
+      "is_filter": true,
+      "searchable": false,
+      "multiple": false,
+      "default_value": "",
+      "visible": false,
+      "editable": true,
+      "trackers": [
+        {
+          "id": 2,
+          "name": "Infra-Field"
+        },
+        {
+          "id": 4,
+          "name": "Software-Dev"
+        },
+        {
+          "id": 6,
+          "name": "Infra-Config"
+        },
+        {
+          "id": 8,
+          "name": "External-Comms-Intake"
+        },
+        {
+          "id": 9,
+          "name": "Outreach-Partnerships"
+        },
+        {
+          "id": 10,
+          "name": "Admin"
+        },
+        {
+          "id": 17,
+          "name": "Research"
+        },
+        {
+          "id": 18,
+          "name": "Mutual-Aid-Action"
+        },
+        {
+          "id": 19,
+          "name": "SCN-Space"
+        }
+      ],
+      "roles": [
+        {
+          "id": 3,
+          "name": "Administrator"
+        }
+      ]
+    },
+    {
+      "id": 5,
+      "name": "To/CC",
+      "description": "Contains the To and Cc headers from the email that created the ticket.",
+      "customized_type": "issue",
+      "field_format": "string",
+      "regexp": "",
+      "min_length": null,
+      "max_length": null,
+      "is_required": false,
+      "is_filter": false,
+      "searchable": true,
+      "multiple": false,
+      "default_value": "",
+      "visible": true,
+      "editable": true,
+      "trackers": [
+        {
+          "id": 2,
+          "name": "Infra-Field"
+        },
+        {
+          "id": 4,
+          "name": "Software-Dev"
+        },
+        {
+          "id": 6,
+          "name": "Infra-Config"
+        },
+        {
+          "id": 8,
+          "name": "External-Comms-Intake"
+        },
+        {
+          "id": 9,
+          "name": "Outreach-Partnerships"
+        },
+        {
+          "id": 10,
+          "name": "Admin"
+        },
+        {
+          "id": 13,
+          "name": "Test-Reject"
+        },
+        {
+          "id": 17,
+          "name": "Research"
+        },
+        {
+          "id": 18,
+          "name": "Mutual-Aid-Action"
+        },
+        {
+          "id": 19,
+          "name": "SCN-Space"
+        }
+      ],
+      "roles": []
+    },
+    {
+      "id": 6,
+      "name": "redacted",
+      "description": "Keys and values for the redacted fields in the ticket.",
+      "customized_type": "issue",
+      "field_format": "string",
+      "regexp": "",
+      "min_length": null,
+      "max_length": null,
+      "is_required": false,
+      "is_filter": false,
+      "searchable": false,
+      "multiple": false,
+      "default_value": "",
+      "visible": false,
+      "editable": true,
+      "trackers": [
+        {
+          "id": 2,
+          "name": "Infra-Field"
+        },
+        {
+          "id": 4,
+          "name": "Software-Dev"
+        },
+        {
+          "id": 6,
+          "name": "Infra-Config"
+        },
+        {
+          "id": 8,
+          "name": "External-Comms-Intake"
+        },
+        {
+          "id": 9,
+          "name": "Outreach-Partnerships"
+        },
+        {
+          "id": 10,
+          "name": "Admin"
+        },
+        {
+          "id": 13,
+          "name": "Test-Reject"
+        },
+        {
+          "id": 17,
+          "name": "Research"
+        },
+        {
+          "id": 18,
+          "name": "Mutual-Aid-Action"
+        },
+        {
+          "id": 19,
+          "name": "SCN-Space"
+        }
+      ],
+      "roles": [
+        {
+          "id": 3,
+          "name": "Administrator"
+        }
+      ]
+    }
+  ]
+}

finetuning/.gitignore

@@ -0,0 +1,15 @@
+venv/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.Python
+*.so
+*.egg
+*.egg-info/
+dist/
+build/
+pii-redactor-fused/
+pii-redactor-finetune.tar
+data/
+.DS_Store