From c2141d1f1e15882222160b6484291d66131bd0b6 Mon Sep 17 00:00:00 2001 From: lxcxjxhx Date: Sun, 8 Nov 2026 02:18:01 +0800 Subject: [PATCH 1/2] Python: write and read chat history files as UTF-8 --- .../ai/onnx/services/onnx_gen_ai_completion_base.py | 2 +- python/semantic_kernel/contents/chat_history.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/semantic_kernel/connectors/ai/onnx/services/onnx_gen_ai_completion_base.py b/python/semantic_kernel/connectors/ai/onnx/services/onnx_gen_ai_completion_base.py index 9bafaa38b31f..15ba394165b9 100644 --- a/python/semantic_kernel/connectors/ai/onnx/services/onnx_gen_ai_completion_base.py +++ b/python/semantic_kernel/connectors/ai/onnx/services/onnx_gen_ai_completion_base.py @@ -40,7 +40,7 @@ def __init__(self, ai_model_path: str, **kwargs) -> None: raise ImportError("onnxruntime-genai is not installed.") try: json_gen_ai_config = os.path.join(ai_model_path + "/genai_config.json") - with open(json_gen_ai_config) as file: + with open(json_gen_ai_config, encoding="utf-8") as file: config: dict = json.load(file) enable_multi_modality = "vision" in config.get("model", {}) model = OnnxRuntimeGenAi.Model(ai_model_path) diff --git a/python/semantic_kernel/contents/chat_history.py b/python/semantic_kernel/contents/chat_history.py index 3a0a66fa1e2b..c0ba7cb9f752 100644 --- a/python/semantic_kernel/contents/chat_history.py +++ b/python/semantic_kernel/contents/chat_history.py @@ -427,7 +427,7 @@ def store_chat_history_to_file(self, file_path: str) -> None: file_path: The path to the file where the serialized data will be stored. """ json_str = self.serialize() - with open(file_path, "w") as local_file: + with open(file_path, "w", encoding="utf-8") as local_file: local_file.write(json_str) @classmethod @@ -442,6 +442,6 @@ def load_chat_history_from_file(cls: type[_T], file_path: str) -> _T: Returns: ChatHistory: The deserialized ChatHistory instance. """ - with open(file_path) as file: + with open(file_path, encoding="utf-8") as file: json_str = file.read() return cls.restore_chat_history(json_str) From 3fb8d302669c9bd5534d778462bf9c30fc5558b5 Mon Sep 17 00:00:00 2001 From: lxcxjxhx Date: Mon, 9 Nov 2026 16:31:14 +0800 Subject: [PATCH 2/2] Python: pin the UTF-8 file encodings in tests Both the chat history file round trip and the ONNX `genai_config.json` read relied on the host's default encoding in their tests, so removing the explicit `encoding="utf-8"` left them green. Assert the encoding each open requests, and add a non-ASCII round trip for the chat history. Verified by reverting the source encodings: the two new assertions fail, and both suites pass with them (`113 passed`). --- .../services/test_onnx_chat_completion.py | 14 ++++++++ .../tests/unit/contents/test_chat_history.py | 36 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/python/tests/unit/connectors/ai/onnx/services/test_onnx_chat_completion.py b/python/tests/unit/connectors/ai/onnx/services/test_onnx_chat_completion.py index b7f064926a89..ad84771f64f5 100644 --- a/python/tests/unit/connectors/ai/onnx/services/test_onnx_chat_completion.py +++ b/python/tests/unit/connectors/ai/onnx/services/test_onnx_chat_completion.py @@ -20,6 +20,20 @@ def test_onnx_chat_completion_with_valid_env_variable(gen_ai_config, model, toke assert not service.enable_multi_modality +@patch("builtins.open", new_callable=mock_open, read_data=json.dumps(gen_ai_config)) +@patch("onnxruntime_genai.Model") +@patch("onnxruntime_genai.Tokenizer") +def test_onnx_config_is_read_as_utf8(gen_ai_config, model, tokenizer, onnx_unit_test_env): + """The genai_config read must name UTF-8 rather than inherit the host default.""" + OnnxGenAIChatCompletion(template=ONNXTemplate.PHI3, env_file_path="test.env") + + config_calls = [ + call for call in open.call_args_list if call.args and str(call.args[0]).endswith("genai_config.json") + ] + assert config_calls, [call.args for call in open.call_args_list] + assert all(call.kwargs.get("encoding") == "utf-8" for call in config_calls), config_calls + + @patch("builtins.open", new_callable=mock_open, read_data=json.dumps(gen_ai_config_vision)) @patch("onnxruntime_genai.Model") @patch("onnxruntime_genai.Tokenizer") diff --git a/python/tests/unit/contents/test_chat_history.py b/python/tests/unit/contents/test_chat_history.py index 45d5d0f23706..66b44393abfe 100644 --- a/python/tests/unit/contents/test_chat_history.py +++ b/python/tests/unit/contents/test_chat_history.py @@ -747,3 +747,39 @@ def test_chat_history_prompt_round_trip_keeps_entities(text: str): restored = ChatHistory.from_rendered_prompt(chat_history.to_prompt()) assert [message.content for message in restored.messages] == [text, text] + + +def test_chat_history_file_round_trip_keeps_non_ascii(tmp_path): + """A history with accented and CJK text has to survive the file round trip.""" + history = ChatHistory() + history.add_user_message("héllo — 世界") + + file_path = tmp_path / "chat_history_utf8.json" + history.store_chat_history_to_file(file_path) + + # The stored bytes are UTF-8 no matter what the host's default encoding is. + assert "héllo — 世界".encode() in file_path.read_bytes() + + restored = ChatHistory.load_chat_history_from_file(file_path) + assert restored.messages[0].content == "héllo — 世界" + + +def test_chat_history_file_io_requests_utf8_explicitly(tmp_path, monkeypatch): + """Both file opens have to name the encoding instead of relying on the default.""" + recorded = [] + real_open = open + + def spy_open(*args, **kwargs): + recorded.append(kwargs.get("encoding")) + return real_open(*args, **kwargs) + + monkeypatch.setattr("builtins.open", spy_open) + + history = ChatHistory() + history.add_user_message("héllo — 世界") + file_path = tmp_path / "chat_history_utf8.json" + history.store_chat_history_to_file(file_path) + ChatHistory.load_chat_history_from_file(file_path) + + assert recorded, "the history never touched the filesystem" + assert all(encoding == "utf-8" for encoding in recorded), recorded