diff --git a/lambdas/ack_backend/src/ack_processor.py b/lambdas/ack_backend/src/ack_processor.py index 6fdaae20b..b1a517dbc 100644 --- a/lambdas/ack_backend/src/ack_processor.py +++ b/lambdas/ack_backend/src/ack_processor.py @@ -6,7 +6,11 @@ from common.batch.eof_utils import is_eof_message from convert_message_to_ack_row import convert_message_to_ack_row from logging_decorators import ack_lambda_handler_logging_decorator -from update_ack_file import complete_batch_file_process, update_ack_file +from update_ack_file import ( + complete_batch_file_process, + update_csv_ack_file, + update_json_ack_file, +) @ack_lambda_handler_logging_decorator @@ -52,7 +56,9 @@ def lambda_handler(event, _): ack_data_rows.append(convert_message_to_ack_row(message, created_at_formatted_string)) increment_records_failed_count(message_id) - update_ack_file(file_key, created_at_formatted_string, ack_data_rows) + update_csv_ack_file(file_key, created_at_formatted_string, ack_data_rows) + + update_json_ack_file(message_id, supplier, file_key, created_at_formatted_string, ack_data_rows) if file_processing_complete: complete_batch_file_process(message_id, supplier, vaccine_type, created_at_formatted_string, file_key) diff --git a/lambdas/ack_backend/src/constants.py b/lambdas/ack_backend/src/constants.py index d157d02ba..5cea694ab 100644 --- a/lambdas/ack_backend/src/constants.py +++ b/lambdas/ack_backend/src/constants.py @@ -4,6 +4,8 @@ TEMP_ACK_DIR = "TempAck" BATCH_FILE_PROCESSING_DIR = "processing" BATCH_FILE_ARCHIVE_DIR = "archive" +BATCH_REPORT_TITLE = "Immunisation FHIR API Batch Report" +BATCH_REPORT_VERSION = 1 LAMBDA_FUNCTION_NAME_PREFIX = "ack_processor" DEFAULT_STREAM_NAME = "immunisation-fhir-api-internal-dev-splunk-firehose" diff --git a/lambdas/ack_backend/src/update_ack_file.py b/lambdas/ack_backend/src/update_ack_file.py index a4969c765..4fd7e0c0c 100644 --- a/lambdas/ack_backend/src/update_ack_file.py +++ b/lambdas/ack_backend/src/update_ack_file.py @@ -1,21 +1,34 @@ """Functions for uploading the data to the ack file""" +import json import os import time -from datetime import datetime +from copy import deepcopy +from datetime import datetime, timezone from io import BytesIO, StringIO from botocore.exceptions import ClientError from common.aws_s3_utils import move_file -from common.batch.audit_table import get_record_count_and_failures_by_message_id, update_audit_table_item +from common.batch.audit_table import ( + get_ingestion_start_time_by_message_id, + get_record_count_and_failures_by_message_id, + update_audit_table_item, +) from common.clients import get_s3_client, logger from common.log_decorator import generate_and_send_logs -from common.models.batch_constants import ACK_BUCKET_NAME, SOURCE_BUCKET_NAME, AuditTableKeys, FileStatus +from common.models.batch_constants import ( + ACK_BUCKET_NAME, + SOURCE_BUCKET_NAME, + AuditTableKeys, + FileStatus, +) from constants import ( ACK_HEADERS, BATCH_FILE_ARCHIVE_DIR, BATCH_FILE_PROCESSING_DIR, + BATCH_REPORT_TITLE, + BATCH_REPORT_VERSION, COMPLETED_ACK_DIR, DEFAULT_STREAM_NAME, LAMBDA_FUNCTION_NAME_PREFIX, @@ -25,6 +38,63 @@ STREAM_NAME = os.getenv("SPLUNK_FIREHOSE_NAME", DEFAULT_STREAM_NAME) +def _generated_date() -> str: + return datetime.now(timezone.utc).isoformat()[:-13] + ".000Z" + + +def _make_ack_data_dict_identifier_information( + supplier: str, raw_ack_filename: str, message_id: str, ingestion_start_time: int +) -> dict: + return { + "system": BATCH_REPORT_TITLE, + "version": BATCH_REPORT_VERSION, + "generatedDate": "", # will be filled on completion + "provider": supplier, + "filename": raw_ack_filename, + "messageHeaderId": message_id, + "summary": { + "ingestionTime": { + "start": ingestion_start_time, + } + }, + "failures": [], + } + + +def _add_ack_data_dict_summary( + existing_ack_data_dict: dict, + total_ack_rows_processed: int, + successful_record_count: int, + total_failures: int, + ingestion_end_time_seconds: int, +) -> dict: + ack_data_dict = deepcopy(existing_ack_data_dict) + ack_data_dict["generatedDate"] = _generated_date() + ack_data_dict_summary_ingestion_time = { + "start": ack_data_dict["summary"]["ingestionTime"]["start"], + "end": ingestion_end_time_seconds, + } + ack_data_dict_summary = { + "totalRecords": total_ack_rows_processed, + "succeeded": successful_record_count, + "failed": total_failures, + "ingestionTime": ack_data_dict_summary_ingestion_time, + } + ack_data_dict["summary"] = ack_data_dict_summary + return ack_data_dict + + +def _make_json_ack_data_row(ack_data_row: dict) -> dict: + return { + "rowId": int(ack_data_row["MESSAGE_HEADER_ID"].split("^")[-1]), + "responseCode": ack_data_row["RESPONSE_CODE"], + "responseDisplay": ack_data_row["RESPONSE_DISPLAY"], + "severity": ack_data_row["ISSUE_SEVERITY"], + "localId": ack_data_row["LOCAL_ID"], + "operationOutcome": ack_data_row["OPERATION_OUTCOME"], + } + + def create_ack_data( created_at_formatted_string: str, local_id: str, @@ -71,28 +141,46 @@ def complete_batch_file_process( the audit table status""" start_time = time.time() + # finish CSV file ack_filename = f"{file_key.replace('.csv', f'_BusAck_{created_at_formatted_string}.csv')}" move_file(ACK_BUCKET_NAME, f"{TEMP_ACK_DIR}/{ack_filename}", f"{COMPLETED_ACK_DIR}/{ack_filename}") move_file(SOURCE_BUCKET_NAME, f"{BATCH_FILE_PROCESSING_DIR}/{file_key}", f"{BATCH_FILE_ARCHIVE_DIR}/{file_key}") total_ack_rows_processed, total_failures = get_record_count_and_failures_by_message_id(message_id) - update_audit_table_item( - file_key=file_key, message_id=message_id, attrs_to_update={AuditTableKeys.STATUS: FileStatus.PROCESSED} - ) + successful_record_count = total_ack_rows_processed - total_failures # Consider creating time utils and using datetime instead of time - ingestion_end_time = time.strftime("%Y%m%dT%H%M%S00", time.gmtime()) - successful_record_count = total_ack_rows_processed - total_failures + time_now = time.gmtime(time.time()) + ingestion_end_time = time.strftime("%Y%m%dT%H%M%S00", time_now) update_audit_table_item( file_key=file_key, message_id=message_id, attrs_to_update={ AuditTableKeys.RECORDS_SUCCEEDED: successful_record_count, AuditTableKeys.INGESTION_END_TIME: ingestion_end_time, + AuditTableKeys.STATUS: FileStatus.PROCESSED, }, ) + # finish JSON file + json_ack_filename = f"{file_key.replace('.csv', f'_BusAck_{created_at_formatted_string}.json')}" + temp_ack_file_key = f"{TEMP_ACK_DIR}/{json_ack_filename}" + ack_data_dict = obtain_current_json_ack_content(message_id, supplier, file_key, temp_ack_file_key) + + ack_data_dict = _add_ack_data_dict_summary( + ack_data_dict, + total_ack_rows_processed, + successful_record_count, + total_failures, + int(time.strftime("%s", time_now)), + ) + + # Upload ack_data_dict to S3 + json_bytes = BytesIO(json.dumps(ack_data_dict, indent=2).encode("utf-8")) + get_s3_client().upload_fileobj(json_bytes, ACK_BUCKET_NAME, temp_ack_file_key) + move_file(ACK_BUCKET_NAME, f"{TEMP_ACK_DIR}/{json_ack_filename}", f"{COMPLETED_ACK_DIR}/{json_ack_filename}") + result = { "message_id": message_id, "file_key": file_key, @@ -127,14 +215,14 @@ def log_batch_file_process(start_time: float, result: dict, function_name: str) generate_and_send_logs(STREAM_NAME, start_time, base_log_data, additional_log_data) -def obtain_current_ack_content(temp_ack_file_key: str) -> StringIO: +def obtain_current_csv_ack_content(temp_ack_file_key: str) -> StringIO: """Returns the current ack file content if the file exists, or else initialises the content with the ack headers.""" try: # If ack file exists in S3 download the contents existing_ack_file = get_s3_client().get_object(Bucket=ACK_BUCKET_NAME, Key=temp_ack_file_key) existing_content = existing_ack_file["Body"].read().decode("utf-8") except ClientError as error: - # If ack file does not exist in S3 create a new file containing the headers only + # If ack file does not exist in S3 create a new file containing the identifier information if error.response["Error"]["Code"] in ("404", "NoSuchKey"): logger.info("No existing ack file found in S3 - creating new file") existing_content = "|".join(ACK_HEADERS) + "\n" @@ -147,7 +235,34 @@ def obtain_current_ack_content(temp_ack_file_key: str) -> StringIO: return accumulated_csv_content -def update_ack_file( +def obtain_current_json_ack_content(message_id: str, supplier: str, file_key: str, temp_ack_file_key: str) -> dict: + """Returns the current ack file content if the file exists, or else initialises the content with the ack headers.""" + try: + # If ack file exists in S3 download the contents + existing_ack_file = get_s3_client().get_object(Bucket=ACK_BUCKET_NAME, Key=temp_ack_file_key) + except ClientError as error: + # If ack file does not exist in S3 create a new file containing the headers only + if error.response["Error"]["Code"] in ("404", "NoSuchKey"): + logger.info("No existing JSON ack file found in S3 - creating new file") + + ingestion_start_time = get_ingestion_start_time_by_message_id(message_id) + raw_ack_filename = file_key.split(".")[0] + + # Generate the initial fields + return _make_ack_data_dict_identifier_information( + supplier, + raw_ack_filename, + message_id, + ingestion_start_time, + ) + else: + logger.error("error whilst obtaining current JSON ack content: %s", error) + raise + + return json.loads(existing_ack_file["Body"].read().decode("utf-8")) + + +def update_csv_ack_file( file_key: str, created_at_formatted_string: str, ack_data_rows: list, @@ -155,8 +270,7 @@ def update_ack_file( """Updates the ack file with the new data row based on the given arguments""" ack_filename = f"{file_key.replace('.csv', f'_BusAck_{created_at_formatted_string}.csv')}" temp_ack_file_key = f"{TEMP_ACK_DIR}/{ack_filename}" - completed_ack_file_key = f"{COMPLETED_ACK_DIR}/{ack_filename}" - accumulated_csv_content = obtain_current_ack_content(temp_ack_file_key) + accumulated_csv_content = obtain_current_csv_ack_content(temp_ack_file_key) for row in ack_data_rows: data_row_str = [str(item) for item in row.values()] @@ -166,4 +280,25 @@ def update_ack_file( csv_file_like_object = BytesIO(accumulated_csv_content.getvalue().encode("utf-8")) get_s3_client().upload_fileobj(csv_file_like_object, ACK_BUCKET_NAME, temp_ack_file_key) - logger.info("Ack file updated to %s: %s", ACK_BUCKET_NAME, completed_ack_file_key) + logger.info("Ack file updated to %s: %s", ACK_BUCKET_NAME, temp_ack_file_key) + + +def update_json_ack_file( + message_id: str, + supplier: str, + file_key: str, + created_at_formatted_string: str, + ack_data_rows: list, +) -> None: + """Updates the ack file with the new data row based on the given arguments""" + ack_filename = f"{file_key.replace('.csv', f'_BusAck_{created_at_formatted_string}.json')}" + temp_ack_file_key = f"{TEMP_ACK_DIR}/{ack_filename}" + ack_data_dict = obtain_current_json_ack_content(message_id, supplier, file_key, temp_ack_file_key) + + for row in ack_data_rows: + ack_data_dict["failures"].append(_make_json_ack_data_row(row)) + + # Upload ack_data_dict to S3 + json_bytes = BytesIO(json.dumps(ack_data_dict, indent=2).encode("utf-8")) + get_s3_client().upload_fileobj(json_bytes, ACK_BUCKET_NAME, temp_ack_file_key) + logger.info("JSON ack file updated to %s: %s", ACK_BUCKET_NAME, temp_ack_file_key) diff --git a/lambdas/ack_backend/tests/test_ack_processor.py b/lambdas/ack_backend/tests/test_ack_processor.py index cc3ca7a0d..d474bc8de 100644 --- a/lambdas/ack_backend/tests/test_ack_processor.py +++ b/lambdas/ack_backend/tests/test_ack_processor.py @@ -23,7 +23,9 @@ from utils.utils_for_ack_backend_tests import ( add_audit_entry_to_table, generate_sample_existing_ack_content, + generate_sample_existing_json_ack_content, validate_ack_file_content, + validate_json_ack_file_content, ) from utils.values_for_ack_backend_tests import ( EXPECTED_ACK_LAMBDA_RESPONSE_FOR_SUCCESS, @@ -94,15 +96,20 @@ def assert_ack_and_source_file_locations_correct( source_file_key: str, tmp_ack_file_key: str, complete_ack_file_key: str, + tmp_json_ack_file_key: str, + complete_json_ack_file_key: str, is_complete: bool, ) -> None: """Helper function to check the ack and source files have not been moved as the processing is not yet complete""" if is_complete: ack_file = self.s3_client.get_object(Bucket=BucketNames.DESTINATION, Key=complete_ack_file_key) + json_ack_file = self.s3_client.get_object(Bucket=BucketNames.DESTINATION, Key=complete_json_ack_file_key) else: ack_file = self.s3_client.get_object(Bucket=BucketNames.DESTINATION, Key=tmp_ack_file_key) + json_ack_file = self.s3_client.get_object(Bucket=BucketNames.DESTINATION, Key=tmp_json_ack_file_key) self.assertIsNotNone(ack_file["Body"].read()) + self.assertIsNotNone(json_ack_file["Body"].read()) full_src_file_key = f"archive/{source_file_key}" if is_complete else f"processing/{source_file_key}" src_file = self.s3_client.get_object(Bucket=BucketNames.SOURCE, Key=full_src_file_key) @@ -135,6 +142,8 @@ def test_lambda_handler_main_multiple_records(self): """Test lambda handler with multiple records.""" # Set up an audit entry which does not yet have record_count recorded add_audit_entry_to_table(self.dynamodb_client, "row") + existing_json_file_content = deepcopy(ValidValues.json_ack_initial_content) + existing_json_file_content["messageHeaderId"] = "row" # First array of messages. Rows 1 to 3 array_of_messages_one = [ { @@ -199,12 +208,23 @@ def test_lambda_handler_main_multiple_records(self): ], existing_file_content=ValidValues.ack_headers, ) + validate_json_ack_file_content( + self.s3_client, + [ + *array_of_messages_one, + *array_of_messages_two, + *array_of_messages_three, + ], + existing_file_content=existing_json_file_content, + ) self.assert_audit_entry_counts_equal("row", expected_entry_counts) def test_lambda_handler_main(self): """Test lambda handler with consistent ack_file_name and message_template.""" # Set up an audit entry which does not yet have record_count recorded add_audit_entry_to_table(self.dynamodb_client, "row") + existing_json_file_content = deepcopy(ValidValues.json_ack_initial_content) + existing_json_file_content["messageHeaderId"] = "row" test_cases = [ { "description": "Multiple messages: all with diagnostics (failure messages)", @@ -246,11 +266,16 @@ def test_lambda_handler_main(self): }, ) validate_ack_file_content(self.s3_client, test_case["messages"]) + validate_json_ack_file_content(self.s3_client, test_case["messages"], existing_json_file_content) self.s3_client.delete_object( Bucket=BucketNames.DESTINATION, Key=MOCK_MESSAGE_DETAILS.temp_ack_file_key, ) + self.s3_client.delete_object( + Bucket=BucketNames.DESTINATION, + Key=MOCK_MESSAGE_DETAILS.temp_json_ack_file_key, + ) def test_lambda_handler_updates_ack_file_but_does_not_mark_complete_when_records_still_remaining(self): """ @@ -264,6 +289,9 @@ def test_lambda_handler_updates_ack_file_but_does_not_mark_complete_when_records # Original source file had 100 records add_audit_entry_to_table(self.dynamodb_client, mock_batch_message_id, record_count=100) + existing_json_file_content = deepcopy(ValidValues.json_ack_initial_content) + existing_json_file_content["messageHeaderId"] = mock_batch_message_id + array_of_failure_messages = [ { **BASE_FAILURE_MESSAGE, @@ -287,10 +315,17 @@ def test_lambda_handler_updates_ack_file_but_does_not_mark_complete_when_records [*array_of_failure_messages], existing_file_content=ValidValues.ack_headers, ) + validate_json_ack_file_content( + self.s3_client, + [*array_of_failure_messages], + existing_file_content=existing_json_file_content, + ) self.assert_ack_and_source_file_locations_correct( MOCK_MESSAGE_DETAILS.file_key, MOCK_MESSAGE_DETAILS.temp_ack_file_key, MOCK_MESSAGE_DETAILS.archive_ack_file_key, + MOCK_MESSAGE_DETAILS.temp_json_ack_file_key, + MOCK_MESSAGE_DETAILS.archive_json_ack_file_key, is_complete=False, ) self.assert_audit_entry_status_equals(mock_batch_message_id, "Preprocessed") @@ -317,6 +352,12 @@ def test_lambda_handler_updates_ack_file_and_marks_complete_when_all_records_pro Key=MOCK_MESSAGE_DETAILS.temp_ack_file_key, Body=StringIO(existing_ack_content).getvalue(), ) + existing_json_ack_content = generate_sample_existing_json_ack_content(mock_batch_message_id) + self.s3_client.put_object( + Bucket=BucketNames.DESTINATION, + Key=MOCK_MESSAGE_DETAILS.temp_json_ack_file_key, + Body=json.dumps(existing_json_ack_content), + ) array_of_failure_messages = [ { @@ -331,11 +372,16 @@ def test_lambda_handler_updates_ack_file_and_marks_complete_when_all_records_pro all_messages_plus_eof = deepcopy(array_of_failure_messages) all_messages_plus_eof.append(MOCK_MESSAGE_DETAILS.eof_message) test_event = {"Records": [{"body": json.dumps(all_messages_plus_eof)}]} + expected_entry_counts = { "record_count": "100", "records_succeeded": "49", "records_failed": "51", } + # Include summary counts in expected JSON content + existing_json_ack_content["summary"]["totalRecords"] = int(expected_entry_counts["record_count"]) + existing_json_ack_content["summary"]["succeeded"] = int(expected_entry_counts["records_succeeded"]) + existing_json_ack_content["summary"]["failed"] = int(expected_entry_counts["records_failed"]) response = lambda_handler(event=test_event, context={}) @@ -343,10 +389,18 @@ def test_lambda_handler_updates_ack_file_and_marks_complete_when_all_records_pro validate_ack_file_content( self.s3_client, [*array_of_failure_messages], existing_file_content=existing_ack_content, is_complete=True ) + validate_json_ack_file_content( + self.s3_client, + [*array_of_failure_messages], + existing_file_content=existing_json_ack_content, + is_complete=True, + ) self.assert_ack_and_source_file_locations_correct( MOCK_MESSAGE_DETAILS.file_key, MOCK_MESSAGE_DETAILS.temp_ack_file_key, MOCK_MESSAGE_DETAILS.archive_ack_file_key, + MOCK_MESSAGE_DETAILS.temp_json_ack_file_key, + MOCK_MESSAGE_DETAILS.archive_json_ack_file_key, is_complete=True, ) self.assert_audit_entry_status_equals(mock_batch_message_id, "Processed") diff --git a/lambdas/ack_backend/tests/test_splunk_logging.py b/lambdas/ack_backend/tests/test_splunk_logging.py index a8e0e69c0..8fe1f4d44 100644 --- a/lambdas/ack_backend/tests/test_splunk_logging.py +++ b/lambdas/ack_backend/tests/test_splunk_logging.py @@ -48,6 +48,12 @@ def setUp(self): self.ack_bucket_patcher = patch("update_ack_file.ACK_BUCKET_NAME", BucketNames.DESTINATION) self.ack_bucket_patcher.start() + self.get_ingestion_start_time_by_message_id_patcher = patch( + "update_ack_file.get_ingestion_start_time_by_message_id" + ) + self.mock_get_ingestion_start_time_by_message_id = self.get_ingestion_start_time_by_message_id_patcher.start() + self.mock_get_ingestion_start_time_by_message_id.return_value = 3456 + def tearDown(self): GenericTearDown(self.s3_client) @@ -99,10 +105,10 @@ def extract_all_call_args_for_logger_error(self, mock_logger): def expected_lambda_handler_logs(self, success: bool, number_of_rows, ingestion_complete=False, diagnostics=None): """Returns the expected logs for the lambda handler function.""" # Mocking of timings is such that the time taken is 2 seconds for each row, - # plus 2 seconds for the handler if it succeeds (i.e. it calls update_ack_file) or 1 second if it doesn't; - # plus an extra second if ingestion is complete + # plus 2 seconds for the handler if it succeeds (i.e. it calls update_*_ack_file) or 1 second if it doesn't; + # plus an extra 2 seconds if ingestion is complete if success: - time_taken = f"{number_of_rows * 2 + 3}.0s" if ingestion_complete else f"{number_of_rows * 2 + 1}.0s" + time_taken = f"{number_of_rows * 2 + 4}.0s" if ingestion_complete else f"{number_of_rows * 2 + 1}.0s" else: time_taken = f"{number_of_rows * 2 + 1}.0s" @@ -429,6 +435,7 @@ def test_splunk_update_ack_file_logged(self): patch("common.log_decorator.logger") as mock_logger, # noqa: E999 patch("update_ack_file.get_record_count_and_failures_by_message_id", return_value=(99, 2)), patch("update_ack_file.update_audit_table_item") as mock_update_audit_table_item, # noqa: E999 + patch("update_ack_file.move_file"), # noqa: E999 patch("ack_processor.increment_records_failed_count"), # noqa: E999 ): # noqa: E999 mock_datetime.now.return_value = ValidValues.fixed_datetime @@ -443,7 +450,7 @@ def test_splunk_update_ack_file_logged(self): expected_secondlast_logger_info_data = { **ValidValues.upload_ack_file_expected_log, "message_id": "test", - "time_taken": "1.0s", + "time_taken": "2.0s", } expected_last_logger_info_data = self.expected_lambda_handler_logs( success=True, number_of_rows=99, ingestion_complete=True @@ -465,7 +472,7 @@ def test_splunk_update_ack_file_logged(self): ] ) - self.assertEqual(mock_update_audit_table_item.call_count, 2) + self.assertEqual(mock_update_audit_table_item.call_count, 1) if __name__ == "__main__": diff --git a/lambdas/ack_backend/tests/test_update_ack_file.py b/lambdas/ack_backend/tests/test_update_ack_file.py index 6b9cd163f..285f2c03d 100644 --- a/lambdas/ack_backend/tests/test_update_ack_file.py +++ b/lambdas/ack_backend/tests/test_update_ack_file.py @@ -1,5 +1,7 @@ """Tests for the functions in the update_ack_file module.""" +import copy +import json import os import unittest from io import StringIO @@ -19,19 +21,24 @@ ) from utils.utils_for_ack_backend_tests import ( MOCK_MESSAGE_DETAILS, - generate_expected_ack_content, generate_expected_ack_file_row, + generate_expected_json_ack_file_element, generate_sample_existing_ack_content, + generate_sample_existing_json_ack_content, obtain_current_ack_file_content, + obtain_current_json_ack_file_content, setup_existing_ack_file, ) from utils.values_for_ack_backend_tests import DefaultValues, ValidValues with patch.dict("os.environ", MOCK_ENVIRONMENT_DICT): from update_ack_file import ( + complete_batch_file_process, create_ack_data, - obtain_current_ack_content, - update_ack_file, + obtain_current_csv_ack_content, + obtain_current_json_ack_content, + update_csv_ack_file, + update_json_ack_file, ) firehose_client = boto3_client("firehose", region_name=REGION_NAME) @@ -44,7 +51,7 @@ class TestUpdateAckFile(unittest.TestCase): def setUp(self) -> None: self.s3_client = boto3_client("s3", region_name=REGION_NAME) - GenericSetUp(self.s3_client) + GenericSetUp(s3_client=self.s3_client) # MOCK SOURCE FILE WITH 100 ROWS TO SIMULATE THE SCENARIO WHERE THE ACK FILE IS NOT FULL. # TODO: Test all other scenarios. @@ -59,25 +66,37 @@ def setUp(self) -> None: self.ack_bucket_patcher = patch("update_ack_file.ACK_BUCKET_NAME", BucketNames.DESTINATION) self.ack_bucket_patcher.start() + self.source_bucket_patcher = patch("update_ack_file.SOURCE_BUCKET_NAME", BucketNames.SOURCE) + self.source_bucket_patcher.start() + + self.get_ingestion_start_time_by_message_id_patcher = patch( + "update_ack_file.get_ingestion_start_time_by_message_id" + ) + self.mock_get_ingestion_start_time_by_message_id = self.get_ingestion_start_time_by_message_id_patcher.start() + self.mock_get_ingestion_start_time_by_message_id.return_value = 3456 + + self.get_record_and_failure_count_patcher = patch("update_ack_file.get_record_count_and_failures_by_message_id") + self.mock_get_record_and_failure_count = self.get_record_and_failure_count_patcher.start() + + self.update_audit_table_item_patcher = patch("update_ack_file.update_audit_table_item") + self.mock_update_audit_table_item = self.update_audit_table_item_patcher.start() + + self.datetime_patcher = patch("update_ack_file.time") + self.mock_datetime = self.datetime_patcher.start() + self.mock_datetime.strftime.return_value = "7890" + + self.generated_date_patcher = patch("update_ack_file._generated_date") + self.mock_generated_date = self.generated_date_patcher.start() + self.mock_generated_date.return_value = "2026-02-09T17:26:00.000Z" + + self.generate_send_patcher = patch("update_ack_file.generate_and_send_logs") + self.mock_generate_send = self.generate_send_patcher.start() def tearDown(self) -> None: - GenericTearDown(self.s3_client) - - def validate_ack_file_content( - self, - incoming_messages: list[dict], - existing_file_content: str = ValidValues.ack_headers, - ) -> None: - """ - Obtains the ack file content and ensures that it matches the expected content (expected content is based - on the incoming messages). - """ - actual_ack_file_content = obtain_current_ack_file_content(self.s3_client) - expected_ack_file_content = generate_expected_ack_content(incoming_messages, existing_file_content) - self.assertEqual(expected_ack_file_content, actual_ack_file_content) + GenericTearDown(s3_client=self.s3_client) - def test_update_ack_file(self): - """Test that update_ack_file correctly creates the ack file when there was no existing ack file""" + def test_update_csv_ack_file(self): + """Test that update_csv_ack_file correctly creates the ack file when there was no existing ack file""" test_cases = [ { @@ -132,7 +151,7 @@ def test_update_ack_file(self): for test_case in test_cases: with self.subTest(test_case["description"]): - update_ack_file( + update_csv_ack_file( file_key=MOCK_MESSAGE_DETAILS.file_key, created_at_formatted_string=MOCK_MESSAGE_DETAILS.created_at_formatted_string, ack_data_rows=test_case["input_rows"], @@ -157,7 +176,7 @@ def test_update_ack_file_existing(self): ValidValues.ack_data_success_dict, ValidValues.ack_data_failure_dict, ] - update_ack_file( + update_csv_ack_file( file_key=MOCK_MESSAGE_DETAILS.file_key, created_at_formatted_string=MOCK_MESSAGE_DETAILS.created_at_formatted_string, ack_data_rows=ack_data_rows, @@ -233,18 +252,204 @@ def test_create_ack_data(self): ) self.assertEqual(result, test_case["expected_result"]) - def test_obtain_current_ack_content_file_no_existing(self): - """Test that when the ack file does not yet exist, obtain_current_ack_content returns the ack headers only.""" - result = obtain_current_ack_content(MOCK_MESSAGE_DETAILS.temp_ack_file_key) + def test_obtain_current_csv_ack_content_file_no_existing(self): + """Test that when the ack file does not yet exist, obtain_current_csv_ack_content returns the ack headers only.""" + result = obtain_current_csv_ack_content(MOCK_MESSAGE_DETAILS.temp_ack_file_key) self.assertEqual(result.getvalue(), ValidValues.ack_headers) - def test_obtain_current_ack_content_file_exists(self): + def test_obtain_current_csv_ack_content_file_exists(self): """Test that the existing ack file content is retrieved and new rows are added.""" existing_content = generate_sample_existing_ack_content() setup_existing_ack_file(MOCK_MESSAGE_DETAILS.temp_ack_file_key, existing_content, self.s3_client) - result = obtain_current_ack_content(MOCK_MESSAGE_DETAILS.temp_ack_file_key) + result = obtain_current_csv_ack_content(MOCK_MESSAGE_DETAILS.temp_ack_file_key) self.assertEqual(result.getvalue(), existing_content) + def test_update_json_ack_file(self): + """Test that update_json_ack_file correctly creates the ack file when there was no existing ack file""" + + test_cases = [ + { + "description": "Single failure row", + "input_rows": [ValidValues.ack_data_failure_dict], + "expected_elements": [ + generate_expected_json_ack_file_element( + success=False, imms_id=DefaultValues.imms_id, diagnostics="DIAGNOSTICS" + ) + ], + }, + { + "description": "With multiple rows", + "input_rows": [ + {**ValidValues.ack_data_failure_dict, "IMMS_ID": "TEST_IMMS_ID_1"}, + ValidValues.ack_data_failure_dict, + ValidValues.ack_data_failure_dict, + ], + "expected_elements": [ + generate_expected_json_ack_file_element( + success=False, + imms_id="TEST_IMMS_ID_1", + diagnostics="DIAGNOSTICS", + ), + generate_expected_json_ack_file_element(success=False, imms_id="", diagnostics="DIAGNOSTICS"), + generate_expected_json_ack_file_element(success=False, imms_id="", diagnostics="DIAGNOSTICS"), + ], + }, + { + "description": "Multiple rows With different diagnostics", + "input_rows": [ + { + **ValidValues.ack_data_failure_dict, + "OPERATION_OUTCOME": "Error 1", + }, + { + **ValidValues.ack_data_failure_dict, + "OPERATION_OUTCOME": "Error 2", + }, + { + **ValidValues.ack_data_failure_dict, + "OPERATION_OUTCOME": "Error 3", + }, + ], + "expected_elements": [ + generate_expected_json_ack_file_element(success=False, imms_id="", diagnostics="Error 1"), + generate_expected_json_ack_file_element(success=False, imms_id="", diagnostics="Error 2"), + generate_expected_json_ack_file_element(success=False, imms_id="", diagnostics="Error 3"), + ], + }, + ] + + for test_case in test_cases: + with self.subTest(test_case["description"]): + update_json_ack_file( + message_id=MOCK_MESSAGE_DETAILS.message_id, + supplier=MOCK_MESSAGE_DETAILS.supplier, + file_key=MOCK_MESSAGE_DETAILS.file_key, + created_at_formatted_string=MOCK_MESSAGE_DETAILS.created_at_formatted_string, + ack_data_rows=test_case["input_rows"], + ) + + actual_ack_file_content = obtain_current_json_ack_file_content( + self.s3_client, MOCK_MESSAGE_DETAILS.temp_json_ack_file_key + ) + expected_ack_file_content = copy.deepcopy(ValidValues.json_ack_initial_content) + for element in test_case["expected_elements"]: + expected_ack_file_content["failures"].append(element) + self.assertEqual(expected_ack_file_content, actual_ack_file_content) + self.s3_client.delete_object( + Bucket=BucketNames.DESTINATION, + Key=MOCK_MESSAGE_DETAILS.temp_json_ack_file_key, + ) + + def test_update_json_ack_file_existing(self): + """Test that update_json_ack_file correctly updates the ack file when there was an existing ack file""" + # Mock existing content in the ack file + existing_content = generate_sample_existing_json_ack_content() + setup_existing_ack_file( + MOCK_MESSAGE_DETAILS.temp_json_ack_file_key, json.dumps(existing_content), self.s3_client + ) + + ack_data_rows = [ + ValidValues.ack_data_failure_dict, + ] + update_json_ack_file( + message_id=MOCK_MESSAGE_DETAILS.message_id, + supplier=MOCK_MESSAGE_DETAILS.supplier, + file_key=MOCK_MESSAGE_DETAILS.file_key, + created_at_formatted_string=MOCK_MESSAGE_DETAILS.created_at_formatted_string, + ack_data_rows=ack_data_rows, + ) + + actual_ack_file_content = obtain_current_json_ack_file_content( + self.s3_client, MOCK_MESSAGE_DETAILS.temp_json_ack_file_key + ) + + expected_rows = [ + generate_expected_json_ack_file_element(success=False, imms_id="", diagnostics="DIAGNOSTICS"), + ] + expected_ack_file_content = existing_content + expected_ack_file_content["failures"].append(expected_rows[0]) + + self.assertEqual(expected_ack_file_content, actual_ack_file_content) + + def test_obtain_current_json_ack_content_file_no_existing(self): + """Test that when the json ack file does not yet exist, obtain_current_json_ack_content returns the ack headers only.""" + result = obtain_current_json_ack_content( + MOCK_MESSAGE_DETAILS.message_id, + MOCK_MESSAGE_DETAILS.supplier, + MOCK_MESSAGE_DETAILS.file_key, + MOCK_MESSAGE_DETAILS.temp_json_ack_file_key, + ) + self.assertEqual(result, ValidValues.json_ack_initial_content) + + def test_obtain_current_json_ack_content_file_exists(self): + """Test that the existing json ack file content is retrieved and new elements are added.""" + existing_content = generate_sample_existing_json_ack_content() + setup_existing_ack_file( + MOCK_MESSAGE_DETAILS.temp_json_ack_file_key, json.dumps(existing_content), self.s3_client + ) + result = obtain_current_json_ack_content( + MOCK_MESSAGE_DETAILS.message_id, + MOCK_MESSAGE_DETAILS.supplier, + MOCK_MESSAGE_DETAILS.file_key, + MOCK_MESSAGE_DETAILS.temp_json_ack_file_key, + ) + self.assertEqual(result, existing_content) + + def test_complete_batch_file_process_json_ack_file(self): + """Test that complete_batch_file_process completes and moves the JSON ack file.""" + generate_sample_existing_json_ack_content() + self.s3_client.put_object( + Bucket=BucketNames.SOURCE, + Key=f"processing/{MOCK_MESSAGE_DETAILS.file_key}", + Body="dummy content", + ) + update_csv_ack_file( + file_key=MOCK_MESSAGE_DETAILS.file_key, + created_at_formatted_string=MOCK_MESSAGE_DETAILS.created_at_formatted_string, + ack_data_rows=[ValidValues.ack_data_failure_dict], + ) + update_json_ack_file( + message_id=MOCK_MESSAGE_DETAILS.message_id, + supplier=MOCK_MESSAGE_DETAILS.supplier, + file_key=MOCK_MESSAGE_DETAILS.file_key, + created_at_formatted_string=MOCK_MESSAGE_DETAILS.created_at_formatted_string, + ack_data_rows=[ValidValues.ack_data_failure_dict], + ) + + self.mock_get_record_and_failure_count.return_value = 10, 1 + + complete_batch_file_process( + message_id=MOCK_MESSAGE_DETAILS.message_id, + supplier=MOCK_MESSAGE_DETAILS.supplier, + vaccine_type=MOCK_MESSAGE_DETAILS.vaccine_type, + created_at_formatted_string="20211120T12000000", + file_key=MOCK_MESSAGE_DETAILS.file_key, + ) + result = obtain_current_json_ack_content( + MOCK_MESSAGE_DETAILS.message_id, + MOCK_MESSAGE_DETAILS.supplier, + MOCK_MESSAGE_DETAILS.file_key, + MOCK_MESSAGE_DETAILS.archive_json_ack_file_key, + ) + self.assertEqual(result, ValidValues.json_ack_complete_content) + + def test_update_json_ack_file_with_empty_ack_data_rows(self): + """Test that update_json_ack_file correctly updates the ack file when given an empty list""" + # Mock existing content in the ack file + existing_content = generate_sample_existing_json_ack_content() + setup_existing_ack_file( + MOCK_MESSAGE_DETAILS.temp_json_ack_file_key, json.dumps(existing_content), self.s3_client + ) + + # Should not raise an exception + update_json_ack_file( + message_id=MOCK_MESSAGE_DETAILS.message_id, + supplier=MOCK_MESSAGE_DETAILS.supplier, + file_key=MOCK_MESSAGE_DETAILS.file_key, + created_at_formatted_string=MOCK_MESSAGE_DETAILS.created_at_formatted_string, + ack_data_rows=[], + ) + if __name__ == "__main__": unittest.main() diff --git a/lambdas/ack_backend/tests/test_update_ack_file_flow.py b/lambdas/ack_backend/tests/test_update_ack_file_flow.py index a286cb950..d26320fc7 100644 --- a/lambdas/ack_backend/tests/test_update_ack_file_flow.py +++ b/lambdas/ack_backend/tests/test_update_ack_file_flow.py @@ -33,15 +33,22 @@ def setUp(self): self.logger_patcher = patch("update_ack_file.logger") self.mock_logger = self.logger_patcher.start() + self.firehose_patcher = patch("common.clients.global_firehose_client") + self.mock_firehose = self.firehose_patcher.start() + self.update_audit_table_item_patcher = patch("update_ack_file.update_audit_table_item") self.mock_update_audit_table_item = self.update_audit_table_item_patcher.start() self.get_record_and_failure_count_patcher = patch("update_ack_file.get_record_count_and_failures_by_message_id") self.mock_get_record_and_failure_count = self.get_record_and_failure_count_patcher.start() + self.get_ingestion_start_time_patcher = patch("update_ack_file.get_ingestion_start_time_by_message_id") + self.mock_get_ingestion_start_time = self.get_ingestion_start_time_patcher.start() def tearDown(self): self.logger_patcher.stop() + self.firehose_patcher.stop() self.update_audit_table_item_patcher.stop() self.get_record_and_failure_count_patcher.stop() + self.get_ingestion_start_time_patcher.stop() def test_audit_table_updated_correctly_when_ack_process_complete(self): """VED-167 - Test that the audit table has been updated correctly""" @@ -58,6 +65,7 @@ def test_audit_table_updated_correctly_when_ack_process_complete(self): Bucket=self.ack_bucket_name, Key=f"TempAck/audit_table_test_BusAck_{mock_created_at_string}.csv" ) self.mock_get_record_and_failure_count.return_value = 10, 2 + self.mock_get_ingestion_start_time.return_value = 1769781283 # Act update_ack_file.complete_batch_file_process( @@ -70,4 +78,44 @@ def test_audit_table_updated_correctly_when_ack_process_complete(self): # Assert: Only check audit table interactions self.mock_get_record_and_failure_count.assert_called_once_with(message_id) - self.assertEqual(self.mock_update_audit_table_item.call_count, 2) + self.assertEqual(self.mock_update_audit_table_item.call_count, 1) + + def test_source_file_moved_when_ack_process_complete(self): + """VED-167 - Test that the source file has been moved correctly""" + # Setup + message_id = "msg-audit-table" + mock_created_at_string = "created_at_formatted_string" + file_key = "audit_table_test.csv" + self.s3_client.put_object( + Bucket=self.source_bucket_name, + Key=f"processing/{file_key}", + Body="dummy content", + ) + self.s3_client.put_object( + Bucket=self.ack_bucket_name, Key=f"TempAck/audit_table_test_BusAck_{mock_created_at_string}.csv" + ) + self.mock_get_record_and_failure_count.return_value = 10, 2 + self.mock_get_ingestion_start_time.return_value = 1769781283 + + # Assert that the source file is not yet in the archive folder + with self.assertRaises(self.s3_client.exceptions.NoSuchKey): + archived_obj = self.s3_client.get_object( + Bucket=self.source_bucket_name, + Key=f"archive/{file_key}", + ) + + # Act + update_ack_file.complete_batch_file_process( + message_id=message_id, + supplier="queue-audit-table-supplier", + vaccine_type="vaccine-type", + created_at_formatted_string=mock_created_at_string, + file_key=file_key, + ) + + # Assert that the source file has been moved into the archive folder + archived_obj = self.s3_client.get_object( + Bucket=self.source_bucket_name, + Key=f"archive/{file_key}", + ) + self.assertIsNotNone(archived_obj) diff --git a/lambdas/ack_backend/tests/utils/utils_for_ack_backend_tests.py b/lambdas/ack_backend/tests/utils/utils_for_ack_backend_tests.py index 9781b051a..9a129670d 100644 --- a/lambdas/ack_backend/tests/utils/utils_for_ack_backend_tests.py +++ b/lambdas/ack_backend/tests/utils/utils_for_ack_backend_tests.py @@ -1,6 +1,7 @@ """Utils functions for the ack backend tests""" import json +from copy import deepcopy from typing import Optional from boto3 import client as boto3_client @@ -130,3 +131,105 @@ def validate_ack_file_content( ) expected_ack_file_content = generate_expected_ack_content(incoming_messages, existing_file_content) assert expected_ack_file_content == actual_ack_file_content + + +def obtain_current_json_ack_file_content( + s3_client, temp_ack_file_key: str = MOCK_MESSAGE_DETAILS.temp_ack_file_key +) -> dict: + """Obtains the ack file content from the destination bucket.""" + retrieved_object = s3_client.get_object(Bucket=BucketNames.DESTINATION, Key=temp_ack_file_key) + return json.loads(retrieved_object["Body"].read().decode("utf-8")) + + +def obtain_completed_json_ack_file_content( + s3_client, complete_ack_file_key: str = MOCK_MESSAGE_DETAILS.archive_ack_file_key +) -> dict: + """Obtains the ack file content from the forwardedFile directory""" + retrieved_object = s3_client.get_object(Bucket=BucketNames.DESTINATION, Key=complete_ack_file_key) + return json.loads(retrieved_object["Body"].read().decode("utf-8")) + + +def generate_expected_json_ack_file_element( + success: bool, + imms_id: str = MOCK_MESSAGE_DETAILS.imms_id, + diagnostics: str = None, + row_id: str = MOCK_MESSAGE_DETAILS.row_id, + local_id: str = MOCK_MESSAGE_DETAILS.local_id, + created_at_formatted_string: str = MOCK_MESSAGE_DETAILS.created_at_formatted_string, +) -> dict: + """Create an ack element, containing the given message details.""" + if success: + return None # we no longer process success elements + else: + return { + "rowId": int(row_id.split("^")[-1]), + "responseCode": "30002", + "responseDisplay": "Business Level Response Value - Processing Error", + "severity": "Fatal", + "localId": local_id, + "operationOutcome": "" if not diagnostics else diagnostics, + } + + +def generate_sample_existing_json_ack_content(message_id: str = "test_file_id") -> dict: + """Returns sample ack file content with a single failure row.""" + sample_content = deepcopy(ValidValues.json_ack_initial_content) + sample_content["messageHeaderId"] = message_id + sample_content["failures"].append(generate_expected_json_ack_file_element(success=False)) + return sample_content + + +def generate_expected_json_ack_content( + incoming_messages: list[dict], existing_content: str = ValidValues.json_ack_initial_content +) -> dict: + """Returns the expected_json_ack_file_content based on the incoming messages""" + for message in incoming_messages: + # Determine diagnostics based on the diagnostics value in the incoming message + diagnostics_dictionary = message.get("diagnostics", {}) + diagnostics = ( + diagnostics_dictionary.get("error_message", "") + if isinstance(diagnostics_dictionary, dict) + else "Unable to determine diagnostics issue" + ) + + # Create the ack row based on the incoming message details + ack_element = generate_expected_json_ack_file_element( + success=diagnostics == "", + row_id=message.get("row_id", MOCK_MESSAGE_DETAILS.row_id), + created_at_formatted_string=message.get( + "created_at_formatted_string", + MOCK_MESSAGE_DETAILS.created_at_formatted_string, + ), + local_id=message.get("local_id", MOCK_MESSAGE_DETAILS.local_id), + imms_id=("" if diagnostics else message.get("imms_id", MOCK_MESSAGE_DETAILS.imms_id)), + diagnostics=diagnostics, + ) + + existing_content["failures"].append(ack_element) + + return existing_content + + +def validate_json_ack_file_content( + s3_client, + incoming_messages: list[dict], + existing_file_content: str = ValidValues.json_ack_initial_content, + is_complete: bool = False, +) -> None: + """ + Obtains the ack file content and ensures that it matches the expected content (expected content is based + on the incoming messages). + """ + actual_ack_file_content = ( + obtain_current_json_ack_file_content(s3_client, MOCK_MESSAGE_DETAILS.temp_json_ack_file_key) + if not is_complete + else obtain_completed_json_ack_file_content(s3_client, MOCK_MESSAGE_DETAILS.archive_json_ack_file_key) + ) + existing_file_content_copy = deepcopy(existing_file_content) + expected_ack_file_content = generate_expected_json_ack_content(incoming_messages, existing_file_content_copy) + + # NB: disregard real-time generated fields + actual_ack_file_content["generatedDate"] = expected_ack_file_content["generatedDate"] + actual_ack_file_content["summary"]["ingestionTime"] = expected_ack_file_content["summary"]["ingestionTime"] + + assert expected_ack_file_content == actual_ack_file_content diff --git a/lambdas/ack_backend/tests/utils/values_for_ack_backend_tests.py b/lambdas/ack_backend/tests/utils/values_for_ack_backend_tests.py index 706debe26..b429b1273 100644 --- a/lambdas/ack_backend/tests/utils/values_for_ack_backend_tests.py +++ b/lambdas/ack_backend/tests/utils/values_for_ack_backend_tests.py @@ -106,6 +106,12 @@ def __init__( self.archive_ack_file_key = ( f"forwardedFile/{vaccine_type}_Vaccinations_v5_{ods_code}_20210730T12000000_BusAck_20211120T12000000.csv" ) + self.temp_json_ack_file_key = ( + f"TempAck/{vaccine_type}_Vaccinations_v5_{ods_code}_20210730T12000000_BusAck_20211120T12000000.json" + ) + self.archive_json_ack_file_key = ( + f"forwardedFile/{vaccine_type}_Vaccinations_v5_{ods_code}_20210730T12000000_BusAck_20211120T12000000.json" + ) self.vaccine_type = vaccine_type self.ods_code = ods_code self.supplier = supplier @@ -259,6 +265,48 @@ class ValidValues: "message": "Record processing complete", } + json_ack_initial_content = { + "system": "Immunisation FHIR API Batch Report", + "version": 1, + "generatedDate": "", + "filename": "RSV_Vaccinations_v5_X26_20210730T12000000", + "provider": "RAVS", + "messageHeaderId": "test_file_id", + "summary": {"ingestionTime": {"start": 3456}}, + "failures": [], + } + + json_ack_complete_content = { + "system": "Immunisation FHIR API Batch Report", + "version": 1, + "generatedDate": "2026-02-09T17:26:00.000Z", + "filename": "RSV_Vaccinations_v5_X26_20210730T12000000", + "provider": "RAVS", + "messageHeaderId": "test_file_id", + "summary": {"totalRecords": 10, "succeeded": 9, "failed": 1, "ingestionTime": {"start": 3456, "end": 7890}}, + "failures": [ + { + "rowId": 1, + "responseCode": "30002", + "responseDisplay": "Business Level Response Value - Processing Error", + "severity": "Fatal", + "localId": "test_system_uri^testabc", + "operationOutcome": "DIAGNOSTICS", + } + ], + } + + json_ack_data_failure_dict = ( + { + "rowId": DefaultValues.row_id, + "responseCode": "30002", + "responseDisplay": "Business Level Response Value - Processing Error", + "severity": "Fatal", + "localId": DefaultValues.local_id, + "operationOutcome": "DIAGNOSTICS", + }, + ) + class InvalidValues: """Invalid values for use in tests""" diff --git a/lambdas/shared/src/common/batch/audit_table.py b/lambdas/shared/src/common/batch/audit_table.py index 58598757c..bb82e8146 100644 --- a/lambdas/shared/src/common/batch/audit_table.py +++ b/lambdas/shared/src/common/batch/audit_table.py @@ -1,3 +1,4 @@ +from datetime import datetime from typing import Optional, Tuple from common.clients import get_dynamodb_client, logger @@ -111,6 +112,25 @@ def _build_audit_table_update_log_message(file_key: str, message_id: str, attrs_ ) +def get_ingestion_start_time_by_message_id(event_message_id: str) -> int: + """Retrieves ingestion start time by unique event message ID""" + # Required by JSON ack file + audit_record = dynamodb_client.get_item( + TableName=AUDIT_TABLE_NAME, Key={AuditTableKeys.MESSAGE_ID: {"S": event_message_id}} + ) + + ingestion_start_time_str = audit_record.get("Item", {}).get(AuditTableKeys.INGESTION_START_TIME, {}).get("S") + if not ingestion_start_time_str: + return 0 + try: + ingestion_start_time = int( + (datetime.strptime(ingestion_start_time_str, "%Y%m%dT%H%M%S00") - datetime(1970, 1, 1)).total_seconds() + ) + except ValueError: + return 0 + return ingestion_start_time + + def get_record_count_and_failures_by_message_id(event_message_id: str) -> tuple[int, int]: """Retrieves total record count and total failures by unique event message ID""" audit_record = dynamodb_client.get_item( diff --git a/lambdas/shared/tests/test_common/batch/test_audit_table.py b/lambdas/shared/tests/test_common/batch/test_audit_table.py index 75b9763ca..b23fbea09 100644 --- a/lambdas/shared/tests/test_common/batch/test_audit_table.py +++ b/lambdas/shared/tests/test_common/batch/test_audit_table.py @@ -27,6 +27,7 @@ from common.batch.audit_table import ( NOTHING_TO_UPDATE_ERROR_MESSAGE, create_audit_table_item, + get_ingestion_start_time_by_message_id, get_record_count_and_failures_by_message_id, increment_records_failed_count, update_audit_table_item, @@ -301,6 +302,50 @@ def test_get_record_count_and_failures_by_message_id_returns_zero_if_values_not_ self.assertEqual(record_count, 0) self.assertEqual(failed_count, 0) + def test_get_ingestion_start_time_by_message_id_returns_the_ingestion_start_time(self): + """Test that get_ingestion_start_time_by_message_id retrieves the integer value of the ingestion start time""" + ravs_rsv_test_file = FileDetails("RSV", "RAVS", "X26") + expected_table_entry = { + **MockFileDetails.rsv_ravs.audit_table_entry, + "status": {"S": FileStatus.PREPROCESSED}, + "ingestion_start_time": {"S": "20260130T16093500"}, + } + + dynamodb_client.put_item(TableName=AUDIT_TABLE_NAME, Item=expected_table_entry) + + ingestion_start_time = get_ingestion_start_time_by_message_id(ravs_rsv_test_file.message_id) + + self.assertEqual(ingestion_start_time, 1769789375) + + def test_get_ingestion_start_time_by_message_id_returns_zero_if_values_not_set(self): + """Test that if the ingestion start time has not yet been set on the audit item then zero is returned""" + ravs_rsv_test_file = FileDetails("RSV", "RAVS", "X26") + expected_table_entry = { + **MockFileDetails.rsv_ravs.audit_table_entry, + "status": {"S": FileStatus.PREPROCESSED}, + } + + dynamodb_client.put_item(TableName=AUDIT_TABLE_NAME, Item=expected_table_entry) + + ingestion_start_time = get_ingestion_start_time_by_message_id(ravs_rsv_test_file.message_id) + + self.assertEqual(ingestion_start_time, 0) + + def test_get_ingestion_start_time_by_message_id_returns_zero_if_format_invalid(self): + """Test that if the ingestion start time has been set with an invalid value then zero is returned""" + ravs_rsv_test_file = FileDetails("RSV", "RAVS", "X26") + expected_table_entry = { + **MockFileDetails.rsv_ravs.audit_table_entry, + "status": {"S": FileStatus.PREPROCESSED}, + "ingestion_start_time": {"S": "1769789375"}, + } + + dynamodb_client.put_item(TableName=AUDIT_TABLE_NAME, Item=expected_table_entry) + + ingestion_start_time = get_ingestion_start_time_by_message_id(ravs_rsv_test_file.message_id) + + self.assertEqual(ingestion_start_time, 0) + def test_increment_records_failed_count(self): """Checks audit table correctly increments the records_failed count""" ravs_rsv_test_file = FileDetails("RSV", "RAVS", "X26") diff --git a/tests/e2e_automation/features/APITests/create.feature b/tests/e2e_automation/features/APITests/create.feature index 4c96444f0..4430cd4c5 100644 --- a/tests/e2e_automation/features/APITests/create.feature +++ b/tests/e2e_automation/features/APITests/create.feature @@ -1,232 +1,252 @@ @Create_Feature @functional Feature: Create the immunization event for a patient -@Delete_cleanUp @smoke -Scenario Outline: Verify that the POST Create API for different vaccine types - Given Valid token is generated for the '' - And Valid json payload is created with Patient '' and vaccine_type '' - When Trigger the post create request - Then The request will be successful with the status code '201' - And The location key and Etag in header will contain the Immunization Id and version - And The X-Request-ID and X-Correlation-ID keys in header will populate correctly - And The imms event table will be populated with the correct data for 'created' event - And The delta table will be populated with the correct data for created event - - Examples: - | Patient | vaccine_type| Supplier | - |Random | COVID | Postman_Auth | - |Random | RSV | RAVS | - |Random | FLU | MAVIS | - |Random | MMR | Postman_Auth | - |Random | MENACWY | TPP | - |Random | 3IN1 | TPP | - |Random | MMRV | EMIS | - |Random | PERTUSSIS | EMIS | - |Random | SHINGLES | EMIS | - |Random | PNEUMOCOCCAL| EMIS | - |Random | 4IN1 | TPP | - |Random | 6IN1 | EMIS | - |Random | HIB | TPP | - |Random | MENB | TPP | - |Random | ROTAVIRUS | MEDICUS | - |Random | HEPB | EMIS | - |Random | BCG | MEDICUS | - -@Delete_cleanUp @vaccine_type_6IN1 @patient_id_Random @supplier_name_EMIS -Scenario: Verify that VACCINATION_PROCEDURE_TERM, VACCINE_PRODUCT_TERM, SITE_OF_VACCINATION_TERM, ROUTE_OF_VACCINATION_TERM fields are mapped to respective text fields in imms delta table - Given Valid json payload is created where vaccination terms has text field populated - When Trigger the post create request - Then The request will be successful with the status code '201' - And The location key and Etag in header will contain the Immunization Id and version - And The terms are mapped to the respective text fields in imms delta table - -@Delete_cleanUp @vaccine_type_BCG @patient_id_Random @supplier_name_EMIS -Scenario: Verify that VACCINATION_PROCEDURE_TERM, VACCINE_PRODUCT_TERM fields are mapped to first instance of coding.display fields in imms delta table - Given Valid json payload is created where vaccination terms has multiple instances of coding - When Trigger the post create request - Then The request will be successful with the status code '201' - And The location key and Etag in header will contain the Immunization Id and version - And The terms are mapped to first instance of coding.display fields in imms delta table - -@Delete_cleanUp @vaccine_type_HEPB @patient_id_Random @supplier_name_MEDICUS -Scenario: Verify that VACCINATION_PROCEDURE_TERM, VACCINE_PRODUCT_TERM, SITE_OF_VACCINATION_TERM, ROUTE_OF_VACCINATION_TERM fields are mapped to correct instance of coding.display fields in imms delta table - Given Valid json payload is created where vaccination terms has multiple instance of coding with different coding system - When Trigger the post create request - Then The request will be successful with the status code '201' - And The location key and Etag in header will contain the Immunization Id and version - And The terms are mapped to correct instance of coding.display fields in imms delta table - -@Delete_cleanUp @vaccine_type_PERTUSSIS @patient_id_Random @supplier_name_EMIS -Scenario: Verify that VACCINATION_PROCEDURE_TERM, VACCINE_PRODUCT_TERM, SITE_OF_VACCINATION_TERM, ROUTE_OF_VACCINATION_TERM fields are mapped to coding.display in imms delta table in case of only one instance of coding - Given Valid json payload is created where vaccination terms has one instance of coding with no text or value string field - When Trigger the post create request - Then The request will be successful with the status code '201' - And The location key and Etag in header will contain the Immunization Id and version - And The terms are mapped to correct coding.display fields in imms delta table - -@Delete_cleanUp @vaccine_type_HIB @patient_id_Random @supplier_name_TPP -Scenario: Verify that VACCINATION_PROCEDURE_TERM, VACCINE_PRODUCT_TERM, SITE_OF_VACCINATION_TERM, ROUTE_OF_VACCINATION_TERM fields are blank in imms delta table if no text or value string or display field is present - Given Valid json payload is created where vaccination terms has no text or value string or display field - When Trigger the post create request - Then The request will be successful with the status code '201' - And The location key and Etag in header will contain the Immunization Id and version - And The terms are blank in imms delta table - -Scenario Outline: Verify that the POST Create API for different supplier fails on access denied - Given Valid token is generated for the '' - And Valid json payload is created with Patient '' and vaccine_type '' - When Trigger the post create request - Then The request will be unsuccessful with the status code '403' - And The Response JSONs should contain correct error message for 'unauthorized_access' access - Examples: - | Patient | vaccine_type| Supplier | - |Random | COVID | MAVIS | - |Random | RSV | MAVIS | - |Random | RSV | SONAR | - -@Delete_cleanUp @supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Mod11_NHS -Scenario: Verify that the POST Create API for invalid but Mod11 compliant NHS Number - Given Valid json payload is created - When Trigger the post create request - Then The request will be successful with the status code '201' - And The location key and Etag in header will contain the Immunization Id and version - And The X-Request-ID and X-Correlation-ID keys in header will populate correctly - And The imms event table will be populated with the correct data for 'created' event - And The delta table will be populated with the correct data for created event - -@supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random -Scenario Outline: Verify that the POST Create API will fail if doseNumberPositiveInt is not valid - Given Valid json payload is created where doseNumberPositiveInt is '' - When Trigger the post create request - Then The request will be unsuccessful with the status code '400' - And The Response JSONs should contain correct error message for '' - Examples: - | doseNumberPositiveInt | error_type | - | -1 | doseNumberPositiveInt_PositiveInteger | - | 0 | doseNumberPositiveInt_PositiveInteger | - | 10 | doseNumberPositiveInt_ValidRange | - - -@Delete_cleanUp @supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random -Scenario: Verify that the POST Create API will be successful if all date field has valid past date - Given Valid json payload is created where date fields has past date - When Trigger the post create request - Then The request will be successful with the status code '201' - And The location key and Etag in header will contain the Immunization Id and version - - -@supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random -Scenario Outline: Verify that the POST Create API will fail if occurrenceDateTime has future or invalid formatted date - Given Valid json payload is created where occurrenceDateTime has invalid '' date - When Trigger the post create request - Then The request will be unsuccessful with the status code '400' - And The Response JSONs should contain correct error message for '' - Examples: - | Date | error_type | - | future_occurrence | invalid_OccurrenceDateTime | - | invalid_format | invalid_OccurrenceDateTime | - | nonexistent | invalid_OccurrenceDateTime | - | empty | invalid_OccurrenceDateTime | - | none | empty_OccurrenceDateTime | - -@supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random -Scenario Outline: Verify that the POST Create API will fail if recorded has future or invalid formatted date - Given Valid json payload is created where recorded has invalid '' date - When Trigger the post create request - Then The request will be unsuccessful with the status code '400' - And The Response JSONs should contain correct error message for '' - Examples: - | Date | error_type | - | future_date | invalid_recorded | - | invalid_format | invalid_recorded | - | nonexistent | invalid_recorded | - | empty | invalid_recorded | - | none | empty_recorded | - -@supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random -Scenario Outline: Verify that the POST Create API will fail if patient's data of birth has future or invalid formatted date - Given Valid json payload is created where date of birth has invalid '' date - When Trigger the post create request - Then The request will be unsuccessful with the status code '400' - And The Response JSONs should contain correct error message for '' - Examples: - | Date | error_type | - | future_date | future_DateOfBirth | - | invalid_format | invalid_DateOfBirth | - | nonexistent | invalid_DateOfBirth | - | empty | invalid_DateOfBirth | - | none | missing_DateOfBirth | - -@supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random -Scenario Outline: Verify that the POST Create API will fail if expiration date has invalid formatted date - Given Valid json payload is created where expiration date has invalid '' date - When Trigger the post create request - Then The request will be unsuccessful with the status code '400' - And The Response JSONs should contain correct error message for 'invalid_expirationDate' - Examples: - | Date | - | invalid_format | - | nonexistent | - | empty | - -@supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random -Scenario Outline: Verify that the POST Create API will fail if nhs number is invalid - Given Valid json payload is created where Nhs number is invalid '' - When Trigger the post create request - Then The request will be unsuccessful with the status code '400' - And The Response JSONs should contain correct error message for '' - Examples: - |invalid_NhsNumber |error_type | - |1234567890 |invalid_mod11_nhsnumber | - |12345678 |invalid_nhsnumber_length | - -@supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random -Scenario Outline: Verify that the POST Create API will fail if patient forename is invalid - Given Valid json payload is created where patient forename is '' - When Trigger the post create request - Then The request will be unsuccessful with the status code '400' - And The Response JSONs should contain correct error message for '' - Examples: - | forename | error_type | - | empty | empty_forename | - | missing | no_forename | - | white_space_array | empty_forename | - | single_value_max_len | max_len_forename | - | max_len_array | max_item_forename | - -@supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random -Scenario Outline: Verify that the POST Create API will fail if patient surname is invalid - Given Valid json payload is created where patient surname is '' - When Trigger the post create request - Then The request will be unsuccessful with the status code '400' - And The Response JSONs should contain correct error message for '' - Examples: - | surname | error_type | - | empty | empty_surname | - | missing | no_surname | - | white_space | empty_surname | - | name_length_36 | max_len_surname | - -@supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random -Scenario: Verify that the POST Create API will fail if patient name is empty - Given Valid json payload is created where patient name is empty - When Trigger the post create request - Then The request will be unsuccessful with the status code '400' - And The Response JSONs should contain correct error message for 'empty_forename_surname' - -@supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random -Scenario Outline: Verify that the POST Create API will fail if patient gender is invalid - Given Valid json payload is created where patient gender is '' - When Trigger the post create request - Then The request will be unsuccessful with the status code '400' - And The Response JSONs should contain correct error message for '' - Examples: - | gender | error_type | - | random_text | invalid_gender | - | empty | empty_gender | - | number | should_be_string | - | gender_code | invalid_gender | - | missing | missing_gender | - - - + @Delete_cleanUp + Scenario Outline: Verify that the POST Create API for different vaccine types + Given Valid token is generated for the '' + And Valid json payload is created with Patient '' and vaccine_type '' + When Trigger the post create request + Then The request will be successful with the status code '201' + And The location key and Etag in header will contain the Immunization Id and version + And The X-Request-ID and X-Correlation-ID keys in header will populate correctly + And The imms event table will be populated with the correct data for 'created' event + And The delta table will be populated with the correct data for created event + + Examples: + | Patient | vaccine_type | Supplier | + | Random | COVID | Postman_Auth | + | Random | RSV | RAVS | + | Random | FLU | MAVIS | + | Random | MMR | Postman_Auth | + | Random | MENACWY | TPP | + | Random | 3IN1 | TPP | + | Random | MMRV | EMIS | + | Random | PERTUSSIS | EMIS | + | Random | SHINGLES | EMIS | + | Random | PNEUMOCOCCAL | EMIS | + | Random | 4IN1 | TPP | + | Random | 6IN1 | EMIS | + | Random | HIB | TPP | + | Random | MENB | TPP | + | Random | ROTAVIRUS | MEDICUS | + | Random | HEPB | EMIS | + | Random | BCG | MEDICUS | + + @Delete_cleanUp @vaccine_type_6IN1 @patient_id_Random @supplier_name_EMIS + Scenario: Verify that VACCINATION_PROCEDURE_TERM, VACCINE_PRODUCT_TERM, SITE_OF_VACCINATION_TERM, ROUTE_OF_VACCINATION_TERM fields are mapped to respective text fields in imms delta table + Given Valid json payload is created where vaccination terms has text field populated + When Trigger the post create request + Then The request will be successful with the status code '201' + And The location key and Etag in header will contain the Immunization Id and version + And The terms are mapped to the respective text fields in imms delta table + + @Delete_cleanUp @vaccine_type_BCG @patient_id_Random @supplier_name_EMIS + Scenario: Verify that VACCINATION_PROCEDURE_TERM, VACCINE_PRODUCT_TERM fields are mapped to first instance of coding.display fields in imms delta table + Given Valid json payload is created where vaccination terms has multiple instances of coding + When Trigger the post create request + Then The request will be successful with the status code '201' + And The location key and Etag in header will contain the Immunization Id and version + And The terms are mapped to first instance of coding.display fields in imms delta table + + @Delete_cleanUp @vaccine_type_HEPB @patient_id_Random @supplier_name_MEDICUS + Scenario: Verify that VACCINATION_PROCEDURE_TERM, VACCINE_PRODUCT_TERM, SITE_OF_VACCINATION_TERM, ROUTE_OF_VACCINATION_TERM fields are mapped to correct instance of coding.display fields in imms delta table + Given Valid json payload is created where vaccination terms has multiple instance of coding with different coding system + When Trigger the post create request + Then The request will be successful with the status code '201' + And The location key and Etag in header will contain the Immunization Id and version + And The terms are mapped to correct instance of coding.display fields in imms delta table + + @smoke + @Delete_cleanUp @vaccine_type_PERTUSSIS @patient_id_Random @supplier_name_EMIS + Scenario: Verify that VACCINATION_PROCEDURE_TERM, VACCINE_PRODUCT_TERM, SITE_OF_VACCINATION_TERM, ROUTE_OF_VACCINATION_TERM fields are mapped to coding.display in imms delta table in case of only one instance of coding + Given Valid json payload is created where vaccination terms has one instance of coding with no text or value string field + When Trigger the post create request + Then The request will be successful with the status code '201' + And The location key and Etag in header will contain the Immunization Id and version + And The terms are mapped to correct coding.display fields in imms delta table + + @smoke + @Delete_cleanUp @vaccine_type_HIB @patient_id_Random @supplier_name_TPP + Scenario: Verify that VACCINATION_PROCEDURE_TERM, VACCINE_PRODUCT_TERM, SITE_OF_VACCINATION_TERM, ROUTE_OF_VACCINATION_TERM fields are blank in imms delta table if no text or value string or display field is present + Given Valid json payload is created where vaccination terms has no text or value string or display field + When Trigger the post create request + Then The request will be successful with the status code '201' + And The location key and Etag in header will contain the Immunization Id and version + And The terms are blank in imms delta table + + @smoke + Scenario Outline: Verify that the POST Create API for different supplier fails on access denied + Given Valid token is generated for the '' + And Valid json payload is created with Patient '' and vaccine_type '' + When Trigger the post create request + Then The request will be unsuccessful with the status code '403' + And The Response JSONs should contain correct error message for 'unauthorized_access' access + Examples: + | Patient | vaccine_type | Supplier | + | Random | COVID | MAVIS | + | Random | RSV | MAVIS | + | Random | RSV | SONAR | + + @Delete_cleanUp @supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Mod11_NHS + Scenario: Verify that the POST Create API for invalid but Mod11 compliant NHS Number + Given Valid json payload is created + When Trigger the post create request + Then The request will be successful with the status code '201' + And The location key and Etag in header will contain the Immunization Id and version + And The X-Request-ID and X-Correlation-ID keys in header will populate correctly + And The imms event table will be populated with the correct data for 'created' event + And The delta table will be populated with the correct data for created event + + @smoke + @supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random + Scenario Outline: Verify that the POST Create API will fail if doseNumberPositiveInt is not valid + Given Valid json payload is created where doseNumberPositiveInt is '' + When Trigger the post create request + Then The request will be unsuccessful with the status code '400' + And The Response JSONs should contain correct error message for '' + Examples: + | doseNumberPositiveInt | error_type | + | -1 | doseNumberPositiveInt_PositiveInteger | + | 0 | doseNumberPositiveInt_PositiveInteger | + | 10 | doseNumberPositiveInt_ValidRange | + + @smoke + @Delete_cleanUp @supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random + Scenario: Verify that the POST Create API will be successful if all date field has valid past date + Given Valid json payload is created where date fields has past date + When Trigger the post create request + Then The request will be successful with the status code '201' + And The location key and Etag in header will contain the Immunization Id and version + + + @supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random + Scenario Outline: Verify that the POST Create API will fail if occurrenceDateTime has future or invalid formatted date + Given Valid json payload is created where occurrenceDateTime has invalid '' date + When Trigger the post create request + Then The request will be unsuccessful with the status code '400' + And The Response JSONs should contain correct error message for '' + Examples: + | Date | error_type | + | future_occurrence | invalid_OccurrenceDateTime | + | invalid_format | invalid_OccurrenceDateTime | + | nonexistent | invalid_OccurrenceDateTime | + | empty | invalid_OccurrenceDateTime | + | none | empty_OccurrenceDateTime | + + @supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random + Scenario Outline: Verify that the POST Create API will fail if recorded has future or invalid formatted date + Given Valid json payload is created where recorded has invalid '' date + When Trigger the post create request + Then The request will be unsuccessful with the status code '400' + And The Response JSONs should contain correct error message for '' + Examples: + | Date | error_type | + | future_date | invalid_recorded | + | invalid_format | invalid_recorded | + | nonexistent | invalid_recorded | + | empty | invalid_recorded | + | none | empty_recorded | + + @supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random + Scenario Outline: Verify that the POST Create API will fail if patient's data of birth has future or invalid formatted date + Given Valid json payload is created where date of birth has invalid '' date + When Trigger the post create request + Then The request will be unsuccessful with the status code '400' + And The Response JSONs should contain correct error message for '' + Examples: + | Date | error_type | + | future_date | future_DateOfBirth | + | invalid_format | invalid_DateOfBirth | + | nonexistent | invalid_DateOfBirth | + | empty | invalid_DateOfBirth | + | none | missing_DateOfBirth | + + @smoke + @supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random + Scenario Outline: Verify that the POST Create API will fail if expiration date has invalid formatted date + Given Valid json payload is created where expiration date has invalid '' date + When Trigger the post create request + Then The request will be unsuccessful with the status code '400' + And The Response JSONs should contain correct error message for 'invalid_expirationDate' + Examples: + | Date | + | invalid_format | + | nonexistent | + | empty | + + @smoke + @supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random + Scenario Outline: Verify that the POST Create API will fail if nhs number is invalid + Given Valid json payload is created where Nhs number is invalid '' + When Trigger the post create request + Then The request will be unsuccessful with the status code '400' + And The Response JSONs should contain correct error message for '' + Examples: + | invalid_NhsNumber | error_type | + | 1234567890 | invalid_mod11_nhsnumber | + | 12345678 | invalid_nhsnumber_length | + + @smoke + @supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random + Scenario Outline: Verify that the POST Create API will fail if patient forename is invalid + Given Valid json payload is created where patient forename is '' + When Trigger the post create request + Then The request will be unsuccessful with the status code '400' + And The Response JSONs should contain correct error message for '' + Examples: + | forename | error_type | + | empty | empty_forename | + | missing | no_forename | + | white_space_array | empty_forename | + | single_value_max_len | max_len_forename | + | max_len_array | max_item_forename | + + @smoke + @supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random + Scenario Outline: Verify that the POST Create API will fail if patient surname is invalid + Given Valid json payload is created where patient surname is '' + When Trigger the post create request + Then The request will be unsuccessful with the status code '400' + And The Response JSONs should contain correct error message for '' + Examples: + | surname | error_type | + | empty | empty_surname | + | missing | no_surname | + | white_space | empty_surname | + | name_length_36 | max_len_surname | + + @smoke + @supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random + Scenario: Verify that the POST Create API will fail if patient name is empty + Given Valid json payload is created where patient name is empty + When Trigger the post create request + Then The request will be unsuccessful with the status code '400' + And The Response JSONs should contain correct error message for 'empty_forename_surname' + + @supplier_name_Postman_Auth @vaccine_type_RSV @patient_id_Random + Scenario Outline: Verify that the POST Create API will fail if patient gender is invalid + Given Valid json payload is created where patient gender is '' + When Trigger the post create request + Then The request will be unsuccessful with the status code '400' + And The Response JSONs should contain correct error message for '' + Examples: + | gender | error_type | + | random_text | invalid_gender | + | empty | empty_gender | + | number | should_be_string | + | gender_code | invalid_gender | + | missing | missing_gender | + + @smoke + @Delete_cleanUp @supplier_name_TPP @vaccine_type_BCG @patient_id_Random + Scenario: Verify that the POST Create API will fail when exiting Unique Id and no_unique_id_uri is used in the request + Given Valid json payload is created + When Trigger the post create request + Then The request will be successful with the status code '201' + And The location key and Etag in header will contain the Immunization Id and version + And The X-Request-ID and X-Correlation-ID keys in header will populate correctly + And The imms event table will be populated with the correct data for 'created' event + And The delta table will be populated with the correct data for created event + When Trigger another post create request with same unique_id and unique_id_uri + Then The request will be unsuccessful with the status code '422' + And The Response JSONs should contain correct error message for 'duplicate' diff --git a/tests/e2e_automation/features/APITests/steps/common_steps.py b/tests/e2e_automation/features/APITests/steps/common_steps.py index 191282b4b..5a118aea9 100644 --- a/tests/e2e_automation/features/APITests/steps/common_steps.py +++ b/tests/e2e_automation/features/APITests/steps/common_steps.py @@ -213,7 +213,13 @@ def validate_imms_event_table_by_operation(context, operation: Operation): @then(parsers.parse("The Response JSONs should contain correct error message for Imms_id '{errorName}'")) def validateForbiddenAccess(context, errorName): error_response = parse_error_response(context.response.json()) - validate_error_response(error_response, errorName, imms_id=context.ImmsID) + if errorName == "duplicate": + identifier = ( + f"{context.immunization_object.identifier[0].system}#{context.immunization_object.identifier[0].value}" + ) + validate_error_response(error_response, errorName, identifier=identifier) + else: + validate_error_response(error_response, errorName, imms_id=context.ImmsID) print(f"\n Error Response - \n {error_response}") diff --git a/tests/e2e_automation/features/APITests/steps/test_create_steps.py b/tests/e2e_automation/features/APITests/steps/test_create_steps.py index 2e6b4dfb1..12225f447 100644 --- a/tests/e2e_automation/features/APITests/steps/test_create_steps.py +++ b/tests/e2e_automation/features/APITests/steps/test_create_steps.py @@ -4,7 +4,7 @@ from venv import logger import pytest_check as check -from pytest_bdd import given, parsers, scenarios, then +from pytest_bdd import given, parsers, scenarios, then, when from src.dynamoDB.dynamo_db_helper import ( fetch_immunization_events_detail, fetch_immunization_int_delta_detail_by_immsID, @@ -25,7 +25,7 @@ from utilities.text_helper import get_text from utilities.vaccination_constants import ROUTE_MAP, SITE_MAP, VACCINATION_PROCEDURE_MAP, VACCINE_CODE_MAP -from .common_steps import valid_json_payload_is_created +from .common_steps import Trigger_the_post_create_request, valid_json_payload_is_created scenarios("APITests/create.feature") @@ -315,3 +315,11 @@ def create_request_with_invalid_gender(context, gender): def create_request_with_empty_nam(context): valid_json_payload_is_created(context) context.immunization_object.contained[1].name = None + + +@when("Trigger another post create request with same unique_id and unique_id_uri") +def trigger_post_create_with_same_unique_id(context): + valid_json_payload_is_created(context) + context.immunization_object.identifier[0].value = context.create_object.identifier[0].value + context.immunization_object.identifier[0].system = context.create_object.identifier[0].system + Trigger_the_post_create_request(context) diff --git a/tests/e2e_automation/features/APITests/steps/test_search_steps.py b/tests/e2e_automation/features/APITests/steps/test_search_steps.py index 6dd56e140..a78a73c6b 100644 --- a/tests/e2e_automation/features/APITests/steps/test_search_steps.py +++ b/tests/e2e_automation/features/APITests/steps/test_search_steps.py @@ -59,7 +59,6 @@ def TriggerSearchGetRequest(context): ) print(f"\n Search Get Parameters - \n {context.params}") context.response = http_requests_session.get(context.url, params=context.params, headers=context.headers) - print(f"\n Search Get Response - \n {context.response.json()}") @@ -73,7 +72,6 @@ def TriggerSearchPostRequest(context): ) print(f"\n Search Post Request - \n {context.request}") context.response = http_requests_session.post(context.url, headers=context.headers, data=context.request) - print(f"\n Search Post Response - \n {context.response.json()}") @@ -94,10 +92,8 @@ def TriggerSearchPostRequest(context): ) def send_invalid_param_get_request(context, NHSNumber, DiseaseType): get_search_get_url_header(context) - NHSNumber = normalize_param(NHSNumber) DiseaseType = normalize_param(DiseaseType) - context.params = convert_to_form_data( set_request_data(NHSNumber, DiseaseType, datetime.today().strftime("%Y-%m-%d")) ) @@ -150,10 +146,6 @@ def send_invalid_param_post_request(context, NHSNumber, DiseaseType): ) def send_invalid_date_get_request(context, DateFrom, DateTo): get_search_get_url_header(context) - - # DateFrom = normalize_param(DateFrom.lower()) - # DateTo = normalize_param(DateTo.lower()) - context.params = convert_to_form_data(set_request_data(9001066569, context.vaccine_type, DateFrom, DateTo)) print(f"\n Search Get parameters - \n {context.params}") context.response = http_requests_session.get(context.url, params=context.params, headers=context.headers) @@ -176,10 +168,6 @@ def send_invalid_date_get_request(context, DateFrom, DateTo): ) def send_invalid_param_post_request_with_dates(context, DateFrom, DateTo): get_search_post_url_header(context) - - # DateFrom = normalize_param(DateFrom.lower()) - # DateTo = normalize_param(DateTo) - context.request = convert_to_form_data(set_request_data(9001066569, context.vaccine_type, DateFrom, DateTo)) print(f"\n Search Post request - \n {context.request}") context.response = http_requests_session.post(context.url, headers=context.headers, data=context.request) @@ -192,7 +180,6 @@ def send_invalid_param_post_request_with_dates(context, DateFrom, DateTo): ) def send_valid_param_get_request(context, NHSNumber, vaccine_type, DateFrom, DateTo): get_search_get_url_header(context) - context.params = convert_to_form_data(set_request_data(NHSNumber, vaccine_type, DateFrom, DateTo)) print(f"\n Search Get parameters - \n {context.params}") context.response = http_requests_session.get(context.url, params=context.params, headers=context.headers) @@ -205,7 +192,6 @@ def send_valid_param_get_request(context, NHSNumber, vaccine_type, DateFrom, Dat ) def send_valid_param_post_request(context, NHSNumber, vaccine_type, DateFrom, DateTo): get_search_post_url_header(context) - context.request = convert_to_form_data(set_request_data(NHSNumber, vaccine_type, DateFrom, DateTo)) print(f"\n Search Get parameters - \n {context.request}") context.response = http_requests_session.post(context.url, headers=context.headers, data=context.request) @@ -263,18 +249,13 @@ def send_valid_param_get_request_with_include_and_dates(context, NHSNumber, vacc def validateDateRange(context): data = context.response.json() context.parsed_search_object = parse_FHIR_immunization_response(data) - params = getattr(context, "params", getattr(context, "request", {})) - if isinstance(params, str): parsed = parse_qs(params) params = {k: v[0] for k, v in parsed.items()} if parsed else {} - dateFrom = params.get("-date.from") dateTo = params.get("-date.to") - assert context.parsed_search_object.entry, "No entries found in the search response." - for entry in context.parsed_search_object.entry: if entry.resource.resourceType == "Immunization": occurrence_date = entry.resource.occurrenceDateTime @@ -294,7 +275,6 @@ def validateDateRange(context): def validateImmsID(context): data = context.response.json() context.parsed_search_object = parse_FHIR_immunization_response(data) - assert context.parsed_search_object.resourceType == "Bundle", ( f"expected resourceType to be 'Bundle' but got {context.parsed_search_object.resourceType}" ) @@ -310,18 +290,12 @@ def validateImmsID(context): assert context.parsed_search_object.total >= 1, ( f"expected total to be greater than or equal to 1 but got {context.parsed_search_object.total}" ) - context.created_event = find_entry_by_Imms_id(context.parsed_search_object, context.ImmsID) - if context.created_event is None: raise AssertionError(f"No object found with Immunisation ID {context.ImmsID} in the search response.") - patient_reference = getattr(context.created_event.resource.patient, "reference", None) - if not patient_reference: raise ValueError("Patient reference is missing in the found event.") - - # Assign to context for further usage context.Patient_fullUrl = patient_reference @@ -338,13 +312,11 @@ def validateJsonImms(context): def validateJsonPat(context): response_patient_entry = find_patient_by_fullurl(context.parsed_search_object) assert response_patient_entry is not None, f"No Patient found with fullUrl {context.Patient_fullUrl}" - response_patient = response_patient_entry.resource expected_nhs_number = context.create_object.contained[1].identifier[0].value actual_nhs_number = response_patient.identifier[0].value expected_system = context.create_object.contained[1].identifier[0].system actual_system = response_patient.identifier[0].system - fields_to_compare = [ ("fullUrl", context.Patient_fullUrl, response_patient_entry.fullUrl), ("resourceType", "Patient", response_patient.resourceType), @@ -352,7 +324,6 @@ def validateJsonPat(context): ("identifier.system", expected_system, actual_system), ("identifier.value", expected_nhs_number, actual_nhs_number), ] - for name, expected, actual in fields_to_compare: check.is_true(expected == actual, f"Expected {name}: {expected}, Actual {actual}") @@ -361,14 +332,10 @@ def validateJsonPat(context): def validate_correct_immunization_event(context): data = context.response.json() context.parsed_search_object = parse_FHIR_immunization_response(data) - context.created_event = context.parsed_search_object.entry[0] if context.parsed_search_object.entry else None - if context.created_event is None: raise AssertionError(f"No object found with Immunisation ID {context.ImmsID} in the search response.") - validateJsonImms(context) - assert context.parsed_search_object.resourceType == "Bundle", ( f"expected resourceType to be 'Bundle' but got {context.parsed_search_object.resourceType}" ) @@ -392,24 +359,18 @@ def validate_correct_immunization_event_with_elements(context): assert response.get("resourceType") == "Bundle", "resourceType should be 'Bundle'" assert response.get("type") == "searchset", "type should be 'searchset'" assert isinstance(response.get("entry"), list) and len(response["entry"]) > 0, " entry list is missing or empty" - - # Link validation link = response.get("link", [{}])[0] link_url = link.get("url") assert link_url is not None, " link[0].url is missing" assert link_url.startswith(context.baseUrl), f"link[0].url should start with '{context.baseUrl}', got '{link_url}'" - - # Entry resource validation resource = response["entry"][0].get("resource", {}) assert resource.get("resourceType") == "Immunization", "resourceType should be 'Immunization'" assert "id" in resource, "resource.id is missing" assert "meta" in resource and "versionId" in resource["meta"], " meta.versionId is missing" - assert resource["id"] == context.ImmsID, f"resource.id mismatch: expected '{context.ImmsID}', got '{resource['id']}'" assert str(resource["meta"]["versionId"]) == str(context.expected_version), ( f"meta.versionId mismatch: expected '{context.expected_version}', got '{resource['meta']['versionId']}'" ) - assert response.get("total") == 1, "total should be 1" @@ -419,13 +380,10 @@ def validate_empty_immunization_event(context): assert response.get("resourceType") == "Bundle", "resourceType should be 'Bundle'" assert response.get("type") == "searchset", "type should be 'searchset'" assert isinstance(response.get("entry"), list) and len(response["entry"]) == 0, " entry list should be empty" - - # Link validation link = response.get("link", [{}])[0] link_url = link.get("url") assert link_url is not None, " link[0].url is missing" assert link_url == f"{context.baseUrl}/Immunization?identifier=None", ( f"link[0].url should be '{context.baseUrl}/Immunization?identifier=None', got '{link_url}'" ) - assert response.get("total") == 0, "total should be 0" diff --git a/tests/e2e_automation/features/APITests/steps/test_status_steps.py b/tests/e2e_automation/features/APITests/steps/test_status_steps.py index b6236e2a5..759a0ca44 100644 --- a/tests/e2e_automation/features/APITests/steps/test_status_steps.py +++ b/tests/e2e_automation/features/APITests/steps/test_status_steps.py @@ -36,10 +36,8 @@ def send_request_to_aws_backend(context: ScenarioContext) -> None: # Let exception be raised if expected env var is not present aws_domain_name: str = os.environ["AWS_DOMAIN_NAME"] backend_status_url = "https://" + aws_domain_name + "/status" - with pytest.raises(requests.exceptions.ConnectionError) as e: requests.get(backend_status_url) - context.response = str(e) @@ -52,7 +50,6 @@ def send_unauthenticated_request_to_api(context: ScenarioContext) -> None: def check_status_response_healthy(context: ScenarioContext) -> None: status_response = context.response.json() assertion_failure_msg = f"Status response assertions failed. Res: {status_response}" - assert status_response.get("status") == "pass", assertion_failure_msg assert status_response.get("checks", {}).get("healthcheck", {}).get("status") == "pass", assertion_failure_msg diff --git a/tests/e2e_automation/features/APITests/steps/test_update_steps.py b/tests/e2e_automation/features/APITests/steps/test_update_steps.py index 45c95214b..dd60b56e0 100644 --- a/tests/e2e_automation/features/APITests/steps/test_update_steps.py +++ b/tests/e2e_automation/features/APITests/steps/test_update_steps.py @@ -34,10 +34,8 @@ def validate_delta_table_for_updated_event(context): context.aws_profile_name, context.ImmsID, context.S3_env, context.expected_version ) assert items, f"Items not found in response for ImmsID: {context.ImmsID}" - delta_items = [i for i in items if i.get("Operation") == Operation.updated.value] assert delta_items, f"No item found for ImmsID: {context.ImmsID}" - latest_delta_record = max(delta_items, key=lambda x: x.get("DateTimeStamp", 0)) validate_imms_delta_record_with_created_event( context, create_obj, latest_delta_record, Operation.updated.value, ActionFlag.updated.value diff --git a/tests/e2e_automation/features/batchTests/Steps/batch_common_steps.py b/tests/e2e_automation/features/batchTests/Steps/batch_common_steps.py index 95f06f65e..679f71751 100644 --- a/tests/e2e_automation/features/batchTests/Steps/batch_common_steps.py +++ b/tests/e2e_automation/features/batchTests/Steps/batch_common_steps.py @@ -22,10 +22,12 @@ save_record_to_batch_files_directory, ) from utilities.batch_file_helper import ( - read_and_validate_bus_ack_file_content, + read_and_validate_csv_bus_ack_file_content, validate_bus_ack_file_for_error, validate_bus_ack_file_for_successful_records, validate_inf_ack_file, + validate_json_bus_ack_file_failure_records, + validate_json_bus_ack_file_structure_and_metadata, ) from utilities.batch_S3_buckets import upload_file_to_S3, wait_and_read_ack_file, wait_for_file_to_move_archive from utilities.enums import ActionFlag, ActionMap, Operation @@ -72,7 +74,6 @@ def wrapper(*args, **kwargs): context.vaccine_df = pd.DataFrame() # fallback to empty return None - return func(*args, **kwargs) return wrapper @@ -97,7 +98,8 @@ def batch_file_upload_in_s3_bucket(context): @then("file will be moved to destination bucket and inf ack file will be created") def ack_file_will_be_moved_to_destination_bucket(context): - context.fileContent = wait_and_read_ack_file(context, "ack") + result = wait_and_read_ack_file(context, "ack") + context.fileContent = result["csv"] assert context.fileContent, f"File not found in destination bucket after timeout: {context.forwarded_prefix}" @@ -107,23 +109,48 @@ def all_records_are_processed_successfully_in_the_inf_ack_file(context): assert all_valid, "One or more records failed validation checks" -@then("bus ack file will be created") +@then("bus ack files will be created") def file_will_be_moved_to_destination_bucket(context): - context.fileContent = wait_and_read_ack_file(context, "forwardedFile") - assert context.fileContent, f"File not found in destination bucket after timeout: {context.forwarded_prefix}" + result = wait_and_read_ack_file(context, "forwardedFile") + assert isinstance(result, dict), f"Expected both CSV and JSON ACK files but got: {type(result)}" + context.fileContent = result.get("csv") + context.fileContentJson = result.get("json") + assert context.fileContent, ( + f"BUS Ack csv File not found in destination bucket after timeout: {context.forwarded_prefix}" + ) + assert context.fileContentJson, ( + f"BUS Ack JSON file not found in destination bucket after timeout: {context.forwarded_prefix}" + ) -@then("bus ack will not have any entry of successfully processed records") +@then("CSV bus ack will not have any entry of successfully processed records") def all_records_are_processed_successfully_in_the_batch_file(context): - file_rows = read_and_validate_bus_ack_file_content(context) + file_rows = read_and_validate_csv_bus_ack_file_content(context) all_valid = validate_bus_ack_file_for_successful_records(context, file_rows) assert all_valid, "One or more records failed validation checks" +@then("Json bus ack will only contain file metadata and no failure record entry") +def json_bus_ack_will_only_contain_file_metadata_and_no_record_entries(context): + json_content = context.fileContentJson + assert json_content is not None, "BUS Ack JSON content is None" + validate_json_bus_ack_file_structure_and_metadata(context) + success = validate_json_bus_ack_file_failure_records(context, expected_failure=False) + assert success, "Failed to validate JSON bus ack file failure records" + + +@then("Json bus ack will only contain file metadata and correct failure record entries") +def json_bus_ack_will_only_contain_file_metadata_and_correct_failure_record_entries(context): + json_content = context.fileContentJson + assert json_content is not None, "BUS Ack JSON content is None" + validate_json_bus_ack_file_structure_and_metadata(context) + success = validate_json_bus_ack_file_failure_records(context, expected_failure=True) + assert success, "Failed to validate JSON bus ack file failure records" + + @then("Audit table will have correct status, queue name and record count for the processed batch file") def validate_imms_audit_table(context): table_query_response = fetch_batch_audit_table_detail(context.aws_profile_name, context.filename, context.S3_env) - assert isinstance(table_query_response, list) and table_query_response, ( f"Item not found in response for filename: {context.filename}" ) @@ -208,9 +235,9 @@ def validate_imms_event_table_for_all_records_in_batch_file(context, operation: validate_to_compare_batch_record_with_event_table_record(context, batch_record, created_event) -@then("all records are rejected in the bus ack file and no imms id is generated") +@then("all rejected records are listed in the csv bus ack file and no imms id is generated") def all_record_are_rejected_for_given_field_name(context): - file_rows = read_and_validate_bus_ack_file_content(context) + file_rows = read_and_validate_csv_bus_ack_file_content(context) all_valid = validate_bus_ack_file_for_error(context, file_rows) assert all_valid, "One or more records failed validation checks" diff --git a/tests/e2e_automation/features/batchTests/Steps/test_batch_file_validation_steps.py b/tests/e2e_automation/features/batchTests/Steps/test_batch_file_validation_steps.py index 5ad954795..357997c0e 100644 --- a/tests/e2e_automation/features/batchTests/Steps/test_batch_file_validation_steps.py +++ b/tests/e2e_automation/features/batchTests/Steps/test_batch_file_validation_steps.py @@ -75,7 +75,9 @@ def batch_file_with_additional_column_is_created(datatable, context): @then("file will be moved to destination bucket and inf ack file will be created for duplicate batch file upload") def file_will_be_moved_to_destination_bucket(context): - context.fileContent = wait_and_read_ack_file(context, "ack", duplicate_inf_files=True) + result = wait_and_read_ack_file(context, "ack", duplicate_inf_files=True) + assert result is not None, f"File not found in destination bucket after timeout: {context.forwarded_prefix}" + context.fileContent = result["csv"] assert context.fileContent, f"File not found in destination bucket after timeout: {context.forwarded_prefix}" @@ -87,8 +89,8 @@ def failed_inf_ack_file(context): @then("bus ack file will not be created") def file_will_not_be_moved_to_destination_bucket(context): - context.fileContent = wait_and_read_ack_file(context, "forwardedFile", timeout=10, duplicate_bus_files=True) - assert context.fileContent is None, f"File found in destination bucket: {context.forwarded_prefix}" + result = wait_and_read_ack_file(context, "forwardedFile", timeout=10, duplicate_bus_files=True) + assert result is None, f"Unexpected BUS ACK file(s) found in destination bucket: {context.forwarded_prefix}" @then( @@ -96,7 +98,6 @@ def file_will_not_be_moved_to_destination_bucket(context): ) def validate_imms_audit_table(context, status, queue_name, error_details): table_query_response = fetch_batch_audit_table_detail(context.aws_profile_name, context.filename, context.S3_env) - assert isinstance(table_query_response, list) and table_query_response, ( f"Item not found in response for filename: {context.filename}" ) diff --git a/tests/e2e_automation/features/batchTests/Steps/test_delete_batch_steps.py b/tests/e2e_automation/features/batchTests/Steps/test_delete_batch_steps.py index 67d2e178d..6d54c3d20 100644 --- a/tests/e2e_automation/features/batchTests/Steps/test_delete_batch_steps.py +++ b/tests/e2e_automation/features/batchTests/Steps/test_delete_batch_steps.py @@ -72,7 +72,6 @@ def upload_batch_file_to_s3_for_update_with_mandatory_field_missing(context): # Build base record record = build_batch_file(context) context.vaccine_df = pd.DataFrame([record.dict()]) - base_fields = { "NHS_NUMBER": context.create_object.contained[1].identifier[0].value, "PERSON_FORENAME": context.create_object.contained[1].name[0].given[0], @@ -85,7 +84,6 @@ def upload_batch_file_to_s3_for_update_with_mandatory_field_missing(context): "UNIQUE_ID_URI": context.create_object.identifier[0].system, } context.vaccine_df.loc[0, list(base_fields.keys())] = list(base_fields.values()) - create_batch_file(context) context.vaccine_df.loc[0, "IMMS_ID"] = context.ImmsID diff --git a/tests/e2e_automation/features/batchTests/Steps/test_update_batch_steps.py b/tests/e2e_automation/features/batchTests/Steps/test_update_batch_steps.py index 166884355..76a430700 100644 --- a/tests/e2e_automation/features/batchTests/Steps/test_update_batch_steps.py +++ b/tests/e2e_automation/features/batchTests/Steps/test_update_batch_steps.py @@ -1,22 +1,37 @@ +import uuid + import pandas as pd from pytest_bdd import given, scenarios, then, when from src.objectModels.batch.batch_file_builder import build_batch_file -from utilities.batch_file_helper import read_and_validate_bus_ack_file_content +from utilities.batch_file_helper import ( + read_and_validate_csv_bus_ack_file_content, + validate_json_bus_ack_file_failure_records, + validate_json_bus_ack_file_structure_and_metadata, +) from utilities.enums import GenderCode from utilities.error_constants import ERROR_MAP from features.APITests.steps.common_steps import ( The_request_will_have_status_code, + Trigger_the_post_create_request, send_update_for_immunization_event, valid_json_payload_is_created, validate_etag_in_header, validate_imms_event_table_by_operation, + validateCreateLocation, validVaccinationRecordIsCreated, ) -from features.APITests.steps.test_create_steps import validate_imms_delta_table_by_ImmsID -from features.APITests.steps.test_update_steps import validate_delta_table_for_updated_event +from features.APITests.steps.test_create_steps import ( + validate_imms_delta_table_by_ImmsID, +) +from features.APITests.steps.test_update_steps import ( + validate_delta_table_for_updated_event, +) -from .batch_common_steps import build_dataFrame_using_datatable, create_batch_file +from .batch_common_steps import ( + build_dataFrame_using_datatable, + create_batch_file, +) scenarios("batchTests/update_batch.feature") @@ -38,6 +53,17 @@ def create_valid_vaccination_record_through_api(context): print(f"Created Immunization record with ImmsID: {context.ImmsID}") +@given( + "vaccination record exists in the API where batch file includes update records for missing mandatory fields and a duplicate entry" +) +def create_valid_vaccination_record_with_missing_mandatory_fields(context): + valid_json_payload_is_created(context) + context.immunization_object.identifier[0].value = f"Fail-missing-mandatory-fields-{str(uuid.uuid4())}-duplicate" + Trigger_the_post_create_request(context) + The_request_will_have_status_code(context, 201) + validateCreateLocation(context) + + @when("An update to above vaccination record is made through batch file upload") def upload_batch_file_to_s3_for_update(context): record = build_batch_file(context) @@ -104,12 +130,11 @@ def api_request_will_be_successful_and_tables_will_be_updated_correctly(context) validate_delta_table_for_updated_event(context) -@when("Update to above vaccination record is made through batch file upload with mandatory field missing") +@when("records for same event are uploaded via batch file with missing mandatory fields and duplicated record") def upload_batch_file_to_s3_for_update_with_mandatory_field_missing(context): # Build base record record = build_batch_file(context) context.vaccine_df = pd.DataFrame([record.dict()]) - base_fields = { "NHS_NUMBER": context.create_object.contained[1].identifier[0].value, "PERSON_FORENAME": context.create_object.contained[1].name[0].given[0], @@ -121,11 +146,8 @@ def upload_batch_file_to_s3_for_update_with_mandatory_field_missing(context): "UNIQUE_ID": context.create_object.identifier[0].value, "UNIQUE_ID_URI": context.create_object.identifier[0].system, } - context.vaccine_df.loc[0, list(base_fields.keys())] = list(base_fields.values()) - - context.vaccine_df = pd.concat([context.vaccine_df.loc[[0]]] * 19, ignore_index=True) - + context.vaccine_df = pd.concat([context.vaccine_df.loc[[0]]] * 20, ignore_index=True) missing_cases = { 0: {"SITE_CODE": "", "PERSON_SURNAME": "empty_site_code"}, 1: {"SITE_CODE_TYPE_URI": "", "PERSON_SURNAME": "empty_site_code_uri"}, @@ -142,48 +164,60 @@ def upload_batch_file_to_s3_for_update_with_mandatory_field_missing(context): 12: {"UNIQUE_ID": " ", "PERSON_SURNAME": "no_unique_id"}, 13: {"UNIQUE_ID_URI": " ", "PERSON_SURNAME": "no_unique_id_uri"}, 14: {"PRIMARY_SOURCE": " ", "PERSON_SURNAME": "no_primary_source"}, - 15: {"VACCINATION_PROCEDURE_CODE": " ", "PERSON_SURNAME": "empty_procedure_code"}, + 15: { + "VACCINATION_PROCEDURE_CODE": " ", + "PERSON_SURNAME": "empty_procedure_code", + }, 16: {"PRIMARY_SOURCE": "test", "PERSON_SURNAME": "no_primary_source"}, 17: {"ACTION_FLAG": "", "PERSON_SURNAME": "invalid_action_flag"}, 18: {"ACTION_FLAG": " ", "PERSON_SURNAME": "invalid_action_flag"}, + 19: {"ACTION_FLAG": "New", "PERSON_SURNAME": "duplicate"}, } - # Apply all missing-field modifications for row_idx, updates in missing_cases.items(): for col, value in updates.items(): context.vaccine_df.loc[row_idx, col] = value - create_batch_file(context) -@then("bus ack will have error records for all the updated records in the batch file") +@then("csv bus ack will have error records for all the updated records in the batch file") def all_records_are_processed_successfully_in_the_batch_file(context): - file_rows = read_and_validate_bus_ack_file_content(context, False, True) + file_rows = read_and_validate_csv_bus_ack_file_content(context, False, True) all_valid = validate_bus_ack_file_for_error_by_surname(context, file_rows) assert all_valid, "One or more records failed validation checks" +@then("json bus ack will have error records for all the updated records in the batch file") +def json_bus_ack_will_have_error_records_for_all_updated_records_in_batch_file(context): + json_content = context.fileContentJson + assert json_content is not None, "BUS Ack JSON content is None" + validate_json_bus_ack_file_structure_and_metadata(context) + success = validate_json_bus_ack_file_failure_records( + context, expected_failure=True, use_username_for_error_lookup=True + ) + assert success, "Failed to validate JSON bus ack file failure records" + + def validate_bus_ack_file_for_error_by_surname(context, file_rows) -> bool: if not file_rows: print("No rows found in BUS ACK file for failed records") return False - overall_valid = True - for batch_idx, row in context.vaccine_df.iterrows(): bus_ack_row_number = batch_idx + 2 - row_data_list = file_rows.get(bus_ack_row_number) - if not row_data_list: print(f"Batch row {batch_idx}: No BUS ACK entry found for row number {bus_ack_row_number}") overall_valid = False continue - surname = str(row.get("PERSON_SURNAME", "")).strip() expected_error = surname expected_diagnostic = ERROR_MAP.get(expected_error, {}).get("diagnostics") - + if expected_error == "duplicate" and expected_diagnostic: + expected_diagnostic = expected_diagnostic.replace( + "", + f"{context.immunization_object.identifier[0].system}#{context.immunization_object.identifier[0].value}", + ) for row_data in row_data_list: i = row_data["row"] fields = row_data["fields"] @@ -197,7 +231,6 @@ def validate_bus_ack_file_for_error_by_surname(context, file_rows) -> bool: imms_id = fields[11] operation_outcome = fields[12] message_delivery = fields[13] - if header_response_code != "Fatal Error": print(f"Row {i}: HEADER_RESPONSE_CODE is not 'Fatal Error'") row_valid = False @@ -219,14 +252,11 @@ def validate_bus_ack_file_for_error_by_surname(context, file_rows) -> bool: if message_delivery != "False": print(f"Row {i}: MESSAGE_DELIVERY is not 'False'") row_valid = False - if operation_outcome != expected_diagnostic: print( f"Row {i}: operation_outcome '{operation_outcome}' does not match " f"expected diagnostics '{expected_diagnostic}' for surname '{expected_error}'" ) row_valid = False - overall_valid = overall_valid and row_valid - return overall_valid diff --git a/tests/e2e_automation/features/batchTests/batch_file_validation.feature b/tests/e2e_automation/features/batchTests/batch_file_validation.feature index ccd80ecf8..4ede0e083 100644 --- a/tests/e2e_automation/features/batchTests/batch_file_validation.feature +++ b/tests/e2e_automation/features/batchTests/batch_file_validation.feature @@ -1,102 +1,105 @@ -@Batch_File_Validation_Feature +@Batch_File_Validation_Feature Feature: Validate the file level and columns validations for vaccination batch file -@vaccine_type_COVID @supplier_name_MAVIS -Scenario Outline: verify that vaccination file will be rejected if file name format is invalid - Given batch file is created for below data with filename and extension - | patient_id | unique_id | - | Random | Valid_NhsNumber | - When batch file is uploaded in s3 bucket - Then file will be moved to destination bucket and inf ack file will be created - And inf ack file has failure status for processed batch file - And bus ack file will not be created - And Audit table will have '', '' and '' for the processed batch file + @vaccine_type_COVID @supplier_name_MAVIS + Scenario Outline: verify that vaccination file will be rejected if file name format is invalid + Given batch file is created for below data with filename and extension + | patient_id | unique_id | + | Random | Valid_NhsNumber | + When batch file is uploaded in s3 bucket + Then file will be moved to destination bucket and inf ack file will be created + And inf ack file has failure status for processed batch file + And bus ack file will not be created + And Audit table will have '', '' and '' for the processed batch file - Examples: - | invalidFilename | file_extension | status | queue_name | error_details | - | HP_Vaccinations_v5_YGM41 | csv | Failed | unknown_unknown | Initial file validation failed: invalid file key | - | HPV_Vaccinations_v5_YGM41 | pdf | Failed | unknown_unknown | Initial file validation failed: invalid file key | - | HPV_Vaccination_v5_YGM41 | csv | Failed | unknown_unknown | Initial file validation failed: invalid file key | - | HPV_Vaccinations_v0_YGM41 | csv | Failed | unknown_unknown | Initial file validation failed: invalid file key | - | HPV_Vaccinations_v0_ABC12 | csv | Failed | unknown_unknown | Initial file validation failed: invalid file key | - -@vaccine_type_HPV @supplier_name_MAVIS -Scenario: verify that vaccination file will be rejected if the processed file is duplicate - Given batch file is created for below data as full dataset - | patient_id | unique_id | - | Random | Valid_NhsNumber | - When batch file is uploaded in s3 bucket - Then file will be moved to destination bucket and inf ack file will be created - And inf ack file has success status for processed batch file - And bus ack file will be created - And bus ack will not have any entry of successfully processed records - And Audit table will have correct status, queue name and record count for the processed batch file - When same batch file is uploaded again in s3 bucket - Then file will be moved to destination bucket and inf ack file will be created for duplicate batch file upload - And inf ack file has failure status for processed batch file - And bus ack file will not be created - And Audit table will have 'Not processed - Duplicate', 'MAVIS_HPV' and 'None' for the processed batch file + Examples: + | invalidFilename | file_extension | status | queue_name | error_details | + | HP_Vaccinations_v5_YGM41 | csv | Failed | unknown_unknown | Initial file validation failed: invalid file key | + | HPV_Vaccinations_v5_YGM41 | pdf | Failed | unknown_unknown | Initial file validation failed: invalid file key | + | HPV_Vaccination_v5_YGM41 | csv | Failed | unknown_unknown | Initial file validation failed: invalid file key | + | HPV_Vaccinations_v0_YGM41 | csv | Failed | unknown_unknown | Initial file validation failed: invalid file key | + | HPV_Vaccinations_v0_ABC12 | csv | Failed | unknown_unknown | Initial file validation failed: invalid file key | -@vaccine_type_FLU @supplier_name_MAVIS -Scenario: verify that vaccination file will be rejected if file is empty - Given Empty batch file is created - When batch file is uploaded in s3 bucket - Then file will be moved to destination bucket and inf ack file will be created - And inf ack file has success status for processed batch file - And bus ack file will not be created - And Audit table will have 'Not processed - Empty file', 'MAVIS_FLU' and 'None' for the processed batch file + @vaccine_type_HPV @supplier_name_MAVIS + Scenario: verify that vaccination file will be rejected if the processed file is duplicate + Given batch file is created for below data as full dataset + | patient_id | unique_id | + | Random | Valid_NhsNumber | + When batch file is uploaded in s3 bucket + Then file will be moved to destination bucket and inf ack file will be created + And inf ack file has success status for processed batch file + And bus ack files will be created + And CSV bus ack will not have any entry of successfully processed records + And Json bus ack will only contain file metadata and no failure record entry + And Audit table will have correct status, queue name and record count for the processed batch file + When same batch file is uploaded again in s3 bucket + Then file will be moved to destination bucket and inf ack file will be created for duplicate batch file upload + And inf ack file has failure status for processed batch file + And bus ack file will not be created + And Audit table will have 'Not processed - Duplicate', 'MAVIS_HPV' and 'None' for the processed batch file -@vaccine_type_MENACWY @supplier_name_TPP -Scenario: verify that vaccination file will be rejected if columns are missing - Given batch file is created with missing columns for below data - | patient_id | unique_id | - | Random | Valid_NhsNumber | - When batch file is uploaded in s3 bucket - Then file will be moved to destination bucket and inf ack file will be created - And inf ack file has failure status for processed batch file - And bus ack file will not be created - And Audit table will have 'Failed', 'TPP_MENACWY' and 'File headers are invalid.' for the processed batch file + @vaccine_type_FLU @supplier_name_MAVIS + Scenario: verify that vaccination file will be rejected if file is empty + Given Empty batch file is created + When batch file is uploaded in s3 bucket + Then file will be moved to destination bucket and inf ack file will be created + And inf ack file has success status for processed batch file + And bus ack file will not be created + And Audit table will have 'Not processed - Empty file', 'MAVIS_FLU' and 'None' for the processed batch file -@vaccine_type_COVID @supplier_name_EMIS -Scenario: verify that vaccination file will be rejected if column order is invalid - Given batch file is created with invalid column order for below data - | patient_id | unique_id | - | Random | Valid_NhsNumber | - When batch file is uploaded in s3 bucket - Then file will be moved to destination bucket and inf ack file will be created - And inf ack file has failure status for processed batch file - And bus ack file will not be created - And Audit table will have 'Failed', 'EMIS_COVID' and 'File headers are invalid.' for the processed batch file + @vaccine_type_MENACWY @supplier_name_TPP + Scenario: verify that vaccination file will be rejected if columns are missing + Given batch file is created with missing columns for below data + | patient_id | unique_id | + | Random | Valid_NhsNumber | + When batch file is uploaded in s3 bucket + Then file will be moved to destination bucket and inf ack file will be created + And inf ack file has failure status for processed batch file + And bus ack file will not be created + And Audit table will have 'Failed', 'TPP_MENACWY' and 'File headers are invalid.' for the processed batch file -@vaccine_type_FLU @supplier_name_SONAR -Scenario: verify that vaccination file will be rejected if file delimiter is invalid - Given batch file is created with invalid delimiter for below data - | patient_id | unique_id | - | Random | Valid_NhsNumber | - When batch file is uploaded in s3 bucket - Then file will be moved to destination bucket and inf ack file will be created - And inf ack file has failure status for processed batch file - And bus ack file will not be created - And Audit table will have 'Failed', 'SONAR_FLU' and 'File headers are invalid.' for the processed batch file + @smoke + @vaccine_type_COVID @supplier_name_EMIS + Scenario: verify that vaccination file will be rejected if column order is invalid + Given batch file is created with invalid column order for below data + | patient_id | unique_id | + | Random | Valid_NhsNumber | + When batch file is uploaded in s3 bucket + Then file will be moved to destination bucket and inf ack file will be created + And inf ack file has failure status for processed batch file + And bus ack file will not be created + And Audit table will have 'Failed', 'EMIS_COVID' and 'File headers are invalid.' for the processed batch file -@vaccine_type_3IN1 @supplier_name_TPP -Scenario: verify that vaccination file will be rejected if one of the column name is invalid - Given batch file is created with invalid column name for patient surname for below data - | patient_id | unique_id | - | Random | Valid_NhsNumber | - When batch file is uploaded in s3 bucket - Then file will be moved to destination bucket and inf ack file will be created - And inf ack file has failure status for processed batch file - And bus ack file will not be created - And Audit table will have 'Failed', 'TPP_3IN1' and 'File headers are invalid.' for the processed batch file + @smoke + @vaccine_type_FLU @supplier_name_SONAR + Scenario: verify that vaccination file will be rejected if file delimiter is invalid + Given batch file is created with invalid delimiter for below data + | patient_id | unique_id | + | Random | Valid_NhsNumber | + When batch file is uploaded in s3 bucket + Then file will be moved to destination bucket and inf ack file will be created + And inf ack file has failure status for processed batch file + And bus ack file will not be created + And Audit table will have 'Failed', 'SONAR_FLU' and 'File headers are invalid.' for the processed batch file -@vaccine_type_3IN1 @supplier_name_EMIS -Scenario: verify that vaccination file will be rejected if additional column is present - Given batch file is created with additional column person age for below data - | patient_id | unique_id | - | Random | Valid_NhsNumber | - When batch file is uploaded in s3 bucket - Then file will be moved to destination bucket and inf ack file will be created - And inf ack file has failure status for processed batch file - And bus ack file will not be created - And Audit table will have 'Failed', 'EMIS_3IN1' and 'File headers are invalid.' for the processed batch file \ No newline at end of file + @vaccine_type_3IN1 @supplier_name_TPP + Scenario: verify that vaccination file will be rejected if one of the column name is invalid + Given batch file is created with invalid column name for patient surname for below data + | patient_id | unique_id | + | Random | Valid_NhsNumber | + When batch file is uploaded in s3 bucket + Then file will be moved to destination bucket and inf ack file will be created + And inf ack file has failure status for processed batch file + And bus ack file will not be created + And Audit table will have 'Failed', 'TPP_3IN1' and 'File headers are invalid.' for the processed batch file + + @vaccine_type_3IN1 @supplier_name_EMIS + Scenario: verify that vaccination file will be rejected if additional column is present + Given batch file is created with additional column person age for below data + | patient_id | unique_id | + | Random | Valid_NhsNumber | + When batch file is uploaded in s3 bucket + Then file will be moved to destination bucket and inf ack file will be created + And inf ack file has failure status for processed batch file + And bus ack file will not be created + And Audit table will have 'Failed', 'EMIS_3IN1' and 'File headers are invalid.' for the processed batch file \ No newline at end of file diff --git a/tests/e2e_automation/features/batchTests/create_batch.feature b/tests/e2e_automation/features/batchTests/create_batch.feature index e3c731379..494f5df19 100644 --- a/tests/e2e_automation/features/batchTests/create_batch.feature +++ b/tests/e2e_automation/features/batchTests/create_batch.feature @@ -14,8 +14,9 @@ Scenario: Verify that full dataset vaccination record will be created through ba When batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file - And bus ack file will be created - And bus ack will not have any entry of successfully processed records + And bus ack files will be created + And CSV bus ack will not have any entry of successfully processed records + And Json bus ack will only contain file metadata and no failure record entry And Audit table will have correct status, queue name and record count for the processed batch file And The imms event table will be populated with the correct data for 'created' event for records in batch file And The delta table will be populated with the correct data for all created records in batch file @@ -33,8 +34,9 @@ Scenario: Verify that minimum dataset vaccination record will be created through When batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file - And bus ack file will be created - And bus ack will not have any entry of successfully processed records + And bus ack files will be created + And CSV bus ack will not have any entry of successfully processed records + And Json bus ack will only contain file metadata and no failure record entry And Audit table will have correct status, queue name and record count for the processed batch file And The imms event table will be populated with the correct data for 'created' event for records in batch file And The delta table will be populated with the correct data for all created records in batch file @@ -50,8 +52,9 @@ Scenario: Verify that vaccination record will be get rejected if date_and_time i When batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file - And bus ack file will be created - And all records are rejected in the bus ack file and no imms id is generated + And bus ack files will be created + And all rejected records are listed in the csv bus ack file and no imms id is generated + And Json bus ack will only contain file metadata and correct failure record entries And Audit table will have correct status, queue name and record count for the processed batch file @vaccine_type_6IN1 @supplier_name_EMIS @@ -65,8 +68,9 @@ Scenario: verify that vaccination record will be get rejected if recorded_date i When batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file - And bus ack file will be created - And all records are rejected in the bus ack file and no imms id is generated + And bus ack files will be created + And all rejected records are listed in the csv bus ack file and no imms id is generated + And Json bus ack will only contain file metadata and correct failure record entries And Audit table will have correct status, queue name and record count for the processed batch file @vaccine_type_4IN1 @supplier_name_TPP @@ -78,8 +82,9 @@ Scenario: verify that vaccination record will be get rejected if expiry_date is When batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file - And bus ack file will be created - And all records are rejected in the bus ack file and no imms id is generated + And bus ack files will be created + And all rejected records are listed in the csv bus ack file and no imms id is generated + And Json bus ack will only contain file metadata and correct failure record entries And Audit table will have correct status, queue name and record count for the processed batch file @vaccine_type_FLU @supplier_name_MAVIS @@ -93,8 +98,9 @@ Scenario: verify that vaccination record will be get rejected if Person date of When batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file - And bus ack file will be created - And all records are rejected in the bus ack file and no imms id is generated + And bus ack files will be created + And all rejected records are listed in the csv bus ack file and no imms id is generated + And Json bus ack will only contain file metadata and correct failure record entries And Audit table will have correct status, queue name and record count for the processed batch file @vaccine_type_FLU @supplier_name_MAVIS @@ -116,8 +122,9 @@ Scenario: verify that vaccination record will be get rejected if Person nhs numb When batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file - And bus ack file will be created - And all records are rejected in the bus ack file and no imms id is generated + And bus ack files will be created + And all rejected records are listed in the csv bus ack file and no imms id is generated + And Json bus ack will only contain file metadata and correct failure record entries And Audit table will have correct status, queue name and record count for the processed batch file @vaccine_type_BCG @supplier_name_TPP @@ -129,8 +136,9 @@ Scenario: verify that vaccination record will be get successful if performer is When batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file - And bus ack file will be created - And bus ack will not have any entry of successfully processed records + And bus ack files will be created + And CSV bus ack will not have any entry of successfully processed records + And Json bus ack will only contain file metadata and no failure record entry And Audit table will have correct status, queue name and record count for the processed batch file And The imms event table will be populated with the correct data for 'created' event for records in batch file And The delta table will be populated with the correct data for all created records in batch file @@ -153,8 +161,9 @@ Scenario: verify that vaccination record will be get successful with different v When batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file - And bus ack file will be created - And bus ack will not have any entry of successfully processed records + And bus ack files will be created + And CSV bus ack will not have any entry of successfully processed records + And Json bus ack will only contain file metadata and no failure record entry And Audit table will have correct status, queue name and record count for the processed batch file And The imms event table will be populated with the correct data for 'created' event for records in batch file And The delta table will be populated with the correct data for all created records in batch file @@ -185,8 +194,9 @@ Scenario: verify that vaccination record will be get rejected if mandatory field When batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file - And bus ack file will be created - And all records are rejected in the bus ack file and no imms id is generated + And bus ack files will be created + And all rejected records are listed in the csv bus ack file and no imms id is generated + And Json bus ack will only contain file metadata and correct failure record entries And Audit table will have correct status, queue name and record count for the processed batch file @delete_cleanup_batch @vaccine_type_HIB @supplier_name_EMIS @@ -199,8 +209,9 @@ Scenario: verify that vaccination record will be successful if mandatory field f When batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file - And bus ack file will be created - And bus ack will not have any entry of successfully processed records + And bus ack files will be created + And CSV bus ack will not have any entry of successfully processed records + And Json bus ack will only contain file metadata and no failure record entry And Audit table will have correct status, queue name and record count for the processed batch file And The imms event table will be populated with the correct data for 'created' event for records in batch file And The delta table will be populated with the correct data for all created records in batch file @@ -217,8 +228,9 @@ Scenario: verify that vaccination record will be get successful if action flag h When batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file - And bus ack file will be created - And bus ack will not have any entry of successfully processed records + And bus ack files will be created + And CSV bus ack will not have any entry of successfully processed records + And Json bus ack will only contain file metadata and no failure record entry And Audit table will have correct status, queue name and record count for the processed batch file And The imms event table will be populated with the correct data for 'created' event for records in batch file And The delta table will be populated with the correct data for all created records in batch file @@ -245,8 +257,9 @@ Scenario: verify that vaccination record will be get rejected if non mandatory f When batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file - And bus ack file will be created - And all records are rejected in the bus ack file and no imms id is generated + And bus ack files will be created + And all rejected records are listed in the csv bus ack file and no imms id is generated + And Json bus ack will only contain file metadata and correct failure record entries And Audit table will have correct status, queue name and record count for the processed batch file @delete_cleanup_batch @vaccine_type_3IN1 @supplier_name_TPP @@ -270,8 +283,9 @@ Scenario: verify that vaccination record will be get successful if non mandatory When batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file - And bus ack file will be created - And bus ack will not have any entry of successfully processed records + And bus ack files will be created + And CSV bus ack will not have any entry of successfully processed records + And Json bus ack will only contain file metadata and no failure record entry And Audit table will have correct status, queue name and record count for the processed batch file And The imms event table will be populated with the correct data for 'created' event for records in batch file And The delta table will be populated with the correct data for all created records in batch file diff --git a/tests/e2e_automation/features/batchTests/delete_batch.feature b/tests/e2e_automation/features/batchTests/delete_batch.feature index 53b19a8b4..ecc48ca20 100644 --- a/tests/e2e_automation/features/batchTests/delete_batch.feature +++ b/tests/e2e_automation/features/batchTests/delete_batch.feature @@ -14,8 +14,9 @@ Scenario: Delete immunization event for a patient through batch file When batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file - And bus ack file will be created - And bus ack will not have any entry of successfully processed records + And bus ack files will be created + And CSV bus ack will not have any entry of successfully processed records + And Json bus ack will only contain file metadata and no failure record entry And Audit table will have correct status, queue name and record count for the processed batch file And The imms event table will be populated with the correct data for 'deleted' event for records in batch file And The delta table will be populated with the correct data for all created records in batch file @@ -29,8 +30,9 @@ Scenario: Verify that the API vaccination record will be successful deleted by b And batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file - And bus ack file will be created - And bus ack will not have any entry of successfully processed records + And bus ack files will be created + And CSV bus ack will not have any entry of successfully processed records + And Json bus ack will only contain file metadata and no failure record entry And Audit table will have correct status, queue name and record count for the processed batch file And The imms event table status will be updated to delete and no change to record detail And The delta table will have delete entry with no change to record detail @@ -42,7 +44,8 @@ Scenario: Verify that the API vaccination record will be successful deleted and And batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file - And bus ack file will be created - And bus ack will not have any entry of successfully processed records + And bus ack files will be created + And CSV bus ack will not have any entry of successfully processed records + And Json bus ack will only contain file metadata and no failure record entry And The imms event table status will be updated to delete and no change to record detail And The delta table will have delete entry with no change to record detail diff --git a/tests/e2e_automation/features/batchTests/update_batch.feature b/tests/e2e_automation/features/batchTests/update_batch.feature index 4b819ae3b..86e9fce40 100644 --- a/tests/e2e_automation/features/batchTests/update_batch.feature +++ b/tests/e2e_automation/features/batchTests/update_batch.feature @@ -1,63 +1,68 @@ @Update_Batch_Feature @functional Feature: Create the immunization event for a patient through batch file and update the record from batch or Api calls -@smoke -@delete_cleanup_batch @vaccine_type_MMR @supplier_name_TPP -Scenario: Update immunization event for a patient through batch file - Given batch file is created for below data as full dataset and each record has a valid update record in the same file - | patient_id | unique_id | - | Random | Valid_NhsNumber | - | InvalidInPDS | InvalidInPDS_NhsNumber| - | SFlag | SFlag_NhsNumber | - | Mod11_NHS | Mod11_NhSNumber | - | OldNHSNo | OldNHSNo | - When batch file is uploaded in s3 bucket - Then file will be moved to destination bucket and inf ack file will be created - And inf ack file has success status for processed batch file - And bus ack file will be created - And bus ack will not have any entry of successfully processed records - And Audit table will have correct status, queue name and record count for the processed batch file - And The imms event table will be populated with the correct data for 'updated' event for records in batch file - And The delta table will be populated with the correct data for all created records in batch file - And The delta table will be populated with the correct data for all updated records in batch file + @smoke + @delete_cleanup_batch @vaccine_type_MMR @supplier_name_TPP + Scenario: Update immunization event for a patient through batch file + Given batch file is created for below data as full dataset and each record has a valid update record in the same file + | patient_id | unique_id | + | Random | Valid_NhsNumber | + | InvalidInPDS | InvalidInPDS_NhsNumber | + | SFlag | SFlag_NhsNumber | + | Mod11_NHS | Mod11_NhSNumber | + | OldNHSNo | OldNHSNo | + When batch file is uploaded in s3 bucket + Then file will be moved to destination bucket and inf ack file will be created + And inf ack file has success status for processed batch file + And bus ack files will be created + And CSV bus ack will not have any entry of successfully processed records + And Json bus ack will only contain file metadata and no failure record entry + And Audit table will have correct status, queue name and record count for the processed batch file + And The imms event table will be populated with the correct data for 'updated' event for records in batch file + And The delta table will be populated with the correct data for all created records in batch file + And The delta table will be populated with the correct data for all updated records in batch file -@Delete_cleanUp @vaccine_type_ROTAVIRUS @patient_id_Random @supplier_name_EMIS -Scenario: Verify that the API vaccination record will be successful updated by batch file upload - Given I have created a valid vaccination record through API - And The delta and imms event table will be populated with the correct data for api created event - When An update to above vaccination record is made through batch file upload - And batch file is uploaded in s3 bucket - Then file will be moved to destination bucket and inf ack file will be created - And inf ack file has success status for processed batch file - And bus ack file will be created - And bus ack will not have any entry of successfully processed records - And Audit table will have correct status, queue name and record count for the processed batch file - And The imms event table will be populated with the correct data for 'updated' event for records in batch file - And The delta table will be populated with the correct data for all updated records in batch file + @Delete_cleanUp @vaccine_type_ROTAVIRUS @patient_id_Random @supplier_name_EMIS + Scenario: Verify that the API vaccination record will be successful updated by batch file upload + Given I have created a valid vaccination record through API + And The delta and imms event table will be populated with the correct data for api created event + When An update to above vaccination record is made through batch file upload + And batch file is uploaded in s3 bucket + Then file will be moved to destination bucket and inf ack file will be created + And inf ack file has success status for processed batch file + And bus ack files will be created + And CSV bus ack will not have any entry of successfully processed records + And Json bus ack will only contain file metadata and no failure record entry + And Audit table will have correct status, queue name and record count for the processed batch file + And The imms event table will be populated with the correct data for 'updated' event for records in batch file + And The delta table will be populated with the correct data for all updated records in batch file -@Delete_cleanUp @vaccine_type_6IN1 @patient_id_Random @supplier_name_TPP -Scenario: Verify that the batch vaccination record will be successful updated by API request - Given batch file is created for below data as full dataset - | patient_id | unique_id | - | Random | Valid_NhsNumber | - When batch file is uploaded in s3 bucket - Then file will be moved to destination bucket and inf ack file will be created - And inf ack file has success status for processed batch file - And bus ack file will be created - And bus ack will not have any entry of successfully processed records - And Audit table will have correct status, queue name and record count for the processed batch file - And The imms event table will be populated with the correct data for 'created' event for records in batch file - And The delta table will be populated with the correct data for all created records in batch file - When Send a update for Immunization event created with vaccination detail being updated through API request - Then Api request will be successful and tables will be updated correctly + @Delete_cleanUp @vaccine_type_6IN1 @patient_id_Random @supplier_name_TPP + Scenario: Verify that the batch vaccination record will be successful updated by API request + Given batch file is created for below data as full dataset + | patient_id | unique_id | + | Random | Valid_NhsNumber | + When batch file is uploaded in s3 bucket + Then file will be moved to destination bucket and inf ack file will be created + And inf ack file has success status for processed batch file + And bus ack files will be created + And CSV bus ack will not have any entry of successfully processed records + And Json bus ack will only contain file metadata and no failure record entry + And Audit table will have correct status, queue name and record count for the processed batch file + And The imms event table will be populated with the correct data for 'created' event for records in batch file + And The delta table will be populated with the correct data for all created records in batch file + When Send a update for Immunization event created with vaccination detail being updated through API request + Then Api request will be successful and tables will be updated correctly -@Delete_cleanUp @vaccine_type_RSV @patient_id_Random @supplier_name_RAVS -Scenario: Verify that the API vaccination record will be successful updated and batch file will fail upload due to mandatory field missing - Given I have created a valid vaccination record through API - When Update to above vaccination record is made through batch file upload with mandatory field missing - And batch file is uploaded in s3 bucket - Then file will be moved to destination bucket and inf ack file will be created - And inf ack file has success status for processed batch file - And bus ack file will be created - And bus ack will have error records for all the updated records in the batch file - And The delta and imms event table will be populated with the correct data for api created event + @smoke + @Delete_cleanUp @vaccine_type_RSV @patient_id_Random @supplier_name_RAVS + Scenario: Verify API succeeds while batch upload fails due to missing mandatory fields and duplicate record + Given vaccination record exists in the API where batch file includes update records for missing mandatory fields and a duplicate entry + When records for same event are uploaded via batch file with missing mandatory fields and duplicated record + And batch file is uploaded in s3 bucket + Then file will be moved to destination bucket and inf ack file will be created + And inf ack file has success status for processed batch file + And bus ack files will be created + And csv bus ack will have error records for all the updated records in the batch file + And json bus ack will have error records for all the updated records in the batch file + And The delta and imms event table will be populated with the correct data for api created event \ No newline at end of file diff --git a/tests/e2e_automation/features/conftest.py b/tests/e2e_automation/features/conftest.py index b0d5f018c..530e631df 100644 --- a/tests/e2e_automation/features/conftest.py +++ b/tests/e2e_automation/features/conftest.py @@ -132,18 +132,24 @@ def pytest_bdd_after_scenario(request, feature, scenario): print("Skipping delete: ImmsID is None") if "delete_cleanup_batch" in tags: - get_tokens(context, context.supplier_name) - context.vaccine_df["IMMS_ID_CLEAN"] = ( - context.vaccine_df["IMMS_ID"].astype(str).str.replace("Immunization#", "", regex=False) - ) - - for imms_id in context.vaccine_df["IMMS_ID_CLEAN"].dropna().unique(): - delete_url = f"{context.url}/{imms_id}" - print(f"Sending DELETE request to: {delete_url}") - response = http_requests_session.delete(delete_url, headers=context.headers) - - assert response.status_code == 204, ( - f" Failed to delete {imms_id}: expected 204, got {response.status_code}. Response: {response.text}" + if "IMMS_ID" in context.vaccine_df.columns and context.vaccine_df["IMMS_ID"].notna().any(): + get_tokens(context, context.supplier_name) + + context.vaccine_df["IMMS_ID_CLEAN"] = ( + context.vaccine_df["IMMS_ID"].astype(str).str.replace("Immunization#", "", regex=False) ) - print("✅ All IMMS_IDs deleted successfully.") + for imms_id in context.vaccine_df["IMMS_ID_CLEAN"].dropna().unique(): + delete_url = f"{context.url}/{imms_id}" + print(f"Sending DELETE request to: {delete_url}") + + response = http_requests_session.delete(delete_url, headers=context.headers) + + assert response.status_code == 204, ( + f"Failed to delete {imms_id}: expected 204, got {response.status_code}. Response: {response.text}" + ) + + print("All IMMS_IDs deleted successfully.") + + else: + print(" No IMMS_ID column or no values present as test failed due to as exception — skipping delete cleanup.") diff --git a/tests/e2e_automation/src/objectModels/batch/batch_report_object.py b/tests/e2e_automation/src/objectModels/batch/batch_report_object.py new file mode 100644 index 000000000..e783fc28a --- /dev/null +++ b/tests/e2e_automation/src/objectModels/batch/batch_report_object.py @@ -0,0 +1,51 @@ +from datetime import datetime +from typing import List, Optional + +from pydantic import BaseModel, validator + + +class IngestionTime(BaseModel): + start: int + end: int + + @validator("end") + def validate_order(cls, v, values): + start = values.get("start") + if start is not None and v <= start: + raise ValueError("ingestionTime.end must be greater than start") + return v + + +class Summary(BaseModel): + totalRecords: int + succeeded: int + failed: int + ingestionTime: IngestionTime + + +class FailureDetail(BaseModel): + rowId: int + responseCode: str + responseDisplay: str + severity: str + localId: str + operationOutcome: str + + +class BatchReport(BaseModel): + system: str + version: int + generatedDate: str + filename: str + provider: str + messageHeaderId: str + summary: Summary + failures: Optional[List[FailureDetail]] + + @validator("generatedDate") + def validate_generated_date(cls, v): + try: + datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%fZ") + except ValueError: + raise ValueError("generatedDate must be ISO‑8601 format ending with 'Z'") + return v diff --git a/tests/e2e_automation/utilities/api_fhir_immunization_helper.py b/tests/e2e_automation/utilities/api_fhir_immunization_helper.py index 69c5f182d..c9569d842 100644 --- a/tests/e2e_automation/utilities/api_fhir_immunization_helper.py +++ b/tests/e2e_automation/utilities/api_fhir_immunization_helper.py @@ -97,7 +97,7 @@ def is_valid_nhs_number(nhs_number: str) -> bool: return check_digit == digits[9] -def validate_error_response(error_response, errorName: str, imms_id: str = "", version: str = ""): +def validate_error_response(error_response, errorName: str, imms_id: str = "", version: str = "", identifier: str = ""): uuid_obj = uuid.UUID(error_response.id, version=4) check.is_true(isinstance(uuid_obj, uuid.UUID), f"Id is not UUID {error_response.id}") @@ -111,6 +111,11 @@ def validate_error_response(error_response, errorName: str, imms_id: str = "", v case "invalid_etag": expected_diagnostics = ERROR_MAP.get("invalid_etag", {}).get("diagnostics", "").replace("", version) fields_to_compare.append(("Diagnostics", expected_diagnostics, error_response.issue[0].diagnostics)) + case "duplicate": + expected_diagnostics = ( + ERROR_MAP.get("duplicate", {}).get("diagnostics", "").replace("", identifier) + ) + fields_to_compare.append(("Diagnostics", expected_diagnostics, error_response.issue[0].diagnostics)) case _: actual_diagnostics = ( error_response.issue[0] diff --git a/tests/e2e_automation/utilities/batch_S3_buckets.py b/tests/e2e_automation/utilities/batch_S3_buckets.py index f468569d9..abe44d4ab 100644 --- a/tests/e2e_automation/utilities/batch_S3_buckets.py +++ b/tests/e2e_automation/utilities/batch_S3_buckets.py @@ -1,3 +1,4 @@ +import os import time import boto3 @@ -50,15 +51,25 @@ def wait_and_read_ack_file( ): s3 = boto3.client("s3") destination_bucket = f"immunisation-batch-{context.S3_env}-data-destinations" + source_filename = context.filename base_name = source_filename.replace(f".{context.file_extension}", "") forwarded_prefix = f"{folderName}/{base_name}" context.forwarded_prefix = forwarded_prefix - print(f"Waiting for file starting with '{forwarded_prefix}' in bucket: {destination_bucket}") + print(f"Waiting for ACK files starting with '{forwarded_prefix}' in bucket: {destination_bucket}") elapsed = 0 + if folderName == "forwardedFile": + expected_extensions = {".csv", ".json"} + print("[MODE] Expecting BOTH CSV and JSON ACK files") + else: + expected_extensions = {".csv"} + print("[MODE] Expecting ONLY CSV ACK file") + + found_files = {} + while elapsed < timeout: try: response = s3.list_objects_v2(Bucket=destination_bucket, Prefix=forwarded_prefix) @@ -66,23 +77,35 @@ def wait_and_read_ack_file( if not contents: print(f"[WAIT] No files found yet... ({elapsed}s)") - elif duplicate_inf_files and len(contents) == 1: - print(f"[WAIT] Waiting for more INF files... ({elapsed}s)") - elif duplicate_bus_files: - if len(contents) > 1: - print(f"[ERROR] Unexpected second BUS file detected: {contents}") - return "Unexpected duplicate BUS file found" - elif len(contents) == 1: - print(f"[WAIT] Only one BUS file seen so far... ({elapsed}s)") else: - sorted_objects = sorted(contents, key=lambda obj: obj["LastModified"], reverse=True) - key = sorted_objects[0]["Key"] - print(f"[FOUND] File located: {key}") + if duplicate_inf_files and len(contents) == 1: + print(f"[WAIT] Waiting for more INF files... ({elapsed}s)") + + elif duplicate_bus_files: + if len(contents) > len(expected_extensions): + print(f"[ERROR] Unexpected extra BUS files detected: {contents}") + return "Unexpected duplicate BUS file found" + elif len(contents) < len(expected_extensions): + print(f"[WAIT] Not all BUS ACK files arrived yet... ({elapsed}s)") + + for obj in contents: + key = obj["Key"] + ext = os.path.splitext(key)[1].lower() + + if ext in expected_extensions and ext not in found_files: + print(f"[FOUND] {ext} file located: {key}") + s3_obj = s3.get_object(Bucket=destination_bucket, Key=key) + file_data = s3_obj["Body"].read().decode("utf-8") + found_files[ext] = file_data + print(f"[SUCCESS] Loaded {ext} file ({len(file_data)} bytes)") + + if expected_extensions.issubset(found_files.keys()): + print("[COMPLETE] All expected ACK files received") + + if expected_extensions == {".csv"}: + return {"csv": found_files[".csv"]} - obj = s3.get_object(Bucket=destination_bucket, Key=key) - file_data = obj["Body"].read().decode("utf-8") - print(f"[SUCCESS] File contents loaded ({len(file_data)} bytes)") - return file_data + return {"csv": found_files[".csv"], "json": found_files[".json"]} time.sleep(interval) elapsed += interval @@ -91,5 +114,5 @@ def wait_and_read_ack_file( print(f"[ERROR] S3 access failed: {e}") return None - print(f"[TIMEOUT] No file found with prefix '{forwarded_prefix}' after {timeout} seconds") + print(f"[TIMEOUT] ACK files not found with prefix '{forwarded_prefix}' after {timeout} seconds") return None diff --git a/tests/e2e_automation/utilities/batch_file_helper.py b/tests/e2e_automation/utilities/batch_file_helper.py index 3cbc66e62..f4e9f8f15 100644 --- a/tests/e2e_automation/utilities/batch_file_helper.py +++ b/tests/e2e_automation/utilities/batch_file_helper.py @@ -1,3 +1,7 @@ +import json + +from src.objectModels.batch.batch_report_object import BatchReport + from utilities.error_constants import ERROR_MAP @@ -184,8 +188,7 @@ def validate_bus_ack_file_for_error(context, file_rows) -> bool: return overall_valid -def read_and_validate_bus_ack_file_content(context, by_local_id: bool = True, by_row_number: bool = False) -> dict: - # Prevent invalid combinations +def read_and_validate_csv_bus_ack_file_content(context, by_local_id: bool = True, by_row_number: bool = False) -> dict: if by_local_id and by_row_number: raise ValueError("Choose only one mode: by_local_id OR by_row_number") @@ -248,3 +251,128 @@ def read_and_validate_bus_ack_file_content(context, by_local_id: bool = True, by return file_rows raise ValueError("You must select either by_local_id=True or by_row_number=True") + + +def validate_json_bus_ack_file_structure_and_metadata(context): + data = json.loads(context.fileContentJson) + report = BatchReport(**data) + assert report.system == "Immunisation FHIR API Batch Report", ( + f"Expected system 'Immunisation FHIR API Batch Report', got '{report.system}'" + ) + assert report.version == 1, f"Expected version 1, got {report.version}" + assert report.filename == context.filename.replace(f".{context.file_extension}", ""), ( + f"Expected filename '{context.filename}' without extension, got '{report.filename}'" + ) + assert report.provider == context.supplier_name, ( + f"Expected provider '{context.supplier_name}', got '{report.provider}'" + ) + + expected_row_count = len(context.vaccine_df) + + expected_success_count = context.vaccine_df[ + (~context.vaccine_df["UNIQUE_ID"].str.startswith("Fail-", na=False)) + & (context.vaccine_df["UNIQUE_ID"].str.strip() != "") + ].shape[0] + + expected_failure_count = context.vaccine_df[ + (context.vaccine_df["UNIQUE_ID"].str.startswith("Fail-", na=False)) + | (context.vaccine_df["UNIQUE_ID"].str.strip() == "") + ].shape[0] + + assert report.summary.totalRecords == expected_row_count, ( + f"Expected totalRecords {expected_row_count}, got {report.summary.totalRecords}" + ) + assert report.summary.succeeded == expected_success_count, ( + f"Expected success count {expected_success_count}, got {report.summary.succeeded}" + ) + assert report.summary.failed == expected_failure_count, ( + f"Expected failure count {expected_failure_count}, got {report.summary.failed}" + ) + + +def validate_json_bus_ack_file_failure_records( + context, expected_failure: bool = True, use_username_for_error_lookup: bool = False +): + data = json.loads(context.fileContentJson) + report = BatchReport(**data) + failures = report.failures or [] + + if not expected_failure: + if not failures: + return True + print(f"Found {len(failures)} failure records in BUS ACK file as not expected") + return False + + fail_mask = context.vaccine_df["UNIQUE_ID"].str.startswith("Fail-", na=False) | ( + context.vaccine_df["UNIQUE_ID"].str.strip() == "" + ) + fail_df = context.vaccine_df[fail_mask] + + expected_local_ids = set(fail_df["UNIQUE_ID"].astype(str) + "^" + fail_df["UNIQUE_ID_URI"].astype(str)) + + overall_valid = True + + for failure in failures: + row_valid = True + + row_id = failure.rowId + local_id = failure.localId + operation_outcome = failure.operationOutcome + + if local_id not in expected_local_ids: + print(f"Failure rowId {row_id}: localId '{local_id}' not expected") + row_valid = False + + if failure.responseCode != "30002": + print(f"Failure rowId {row_id}: responseCode != '30002'") + row_valid = False + + if failure.responseDisplay != "Business Level Response Value - Processing Error": + print(f"Failure rowId {row_id}: responseDisplay incorrect") + row_valid = False + + if failure.severity != "Fatal": + print(f"Failure rowId {row_id}: severity != 'Fatal'") + row_valid = False + + try: + df_row = context.vaccine_df.loc[row_id - 1] + expected_error = get_expected_error(df_row, use_username_for_error_lookup) + + expected_diagnostic = ERROR_MAP.get(expected_error, {}).get("diagnostics") + + # Duplicate case + if expected_error == "duplicate" and expected_diagnostic: + expected_diagnostic = expected_diagnostic.replace( + "", + f"{context.immunization_object.identifier[0].system}#" + f"{context.immunization_object.identifier[0].value}", + ) + + if operation_outcome != expected_diagnostic: + print( + f"Failure rowId {row_id}: operationOutcome mismatch. " + f"Expected '{expected_diagnostic}', got '{operation_outcome}'" + ) + row_valid = False + + except Exception as e: + print(f"Failure rowId {row_id}: error resolving expected diagnostics: {e}") + row_valid = False + + overall_valid = overall_valid and row_valid + + return overall_valid + + +def get_expected_error(df_row, use_surname: bool): + prefix = str(df_row["UNIQUE_ID"]).strip() + + if prefix in ["", " ", "nan"]: + return df_row.get("PERSON_SURNAME", "").strip() + + if use_surname: + return str(df_row.get("PERSON_SURNAME", "")).strip() + + parts = prefix.split("-") + return parts[2] if len(parts) > 2 else "invalid_prefix_format" diff --git a/tests/e2e_automation/utilities/context.py b/tests/e2e_automation/utilities/context.py index 1d576fdb2..5cfd8b5fc 100644 --- a/tests/e2e_automation/utilities/context.py +++ b/tests/e2e_automation/utilities/context.py @@ -44,4 +44,6 @@ def __init__(self): self.supplier_ods_code = None self.working_directory = None self.fileContent = None + self.fileContentJson = None + self.forwarded_prefix = None self.delta_cache = None diff --git a/tests/e2e_automation/utilities/error_constants.py b/tests/e2e_automation/utilities/error_constants.py index fb6dfc627..3c80de88e 100644 --- a/tests/e2e_automation/utilities/error_constants.py +++ b/tests/e2e_automation/utilities/error_constants.py @@ -242,4 +242,8 @@ "code": "INVARIANT", "diagnostics": "Validation errors: Immunization resource version: in the request headers is invalid.", }, + "duplicate": { + "code": "DUPLICATE", + "diagnostics": "The provided identifier: is duplicated", + }, }