Skip to content

Commit 2301eb4

Browse files
gh-85943: Fix BytesWarning in the struct format cache under -bb (GH-153627)
Normalize bytes format strings to str before using them as the cache key, so that equal str and bytes formats no longer collide and get compared. Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> (cherry picked from commit 190d2ff)
1 parent 05f0b61 commit 2301eb4

3 files changed

Lines changed: 36 additions & 4 deletions

File tree

Lib/test/test_struct.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,16 @@ def test_calcsize(self):
176176
self.assertGreaterEqual(struct.calcsize('n'), struct.calcsize('i'))
177177
self.assertGreaterEqual(struct.calcsize('n'), struct.calcsize('P'))
178178

179+
def test_cache_bytes_vs_str_bb(self):
180+
# Mixing str and bytes formats must not raise BytesWarning under -bb.
181+
code = (
182+
'import struct\n'
183+
'struct.calcsize(b"!d"); struct.calcsize("!d")\n'
184+
'struct.calcsize(">d"); struct.calcsize(b">d")\n'
185+
'struct.Struct(b"i"); struct.Struct("i")\n'
186+
)
187+
assert_python_ok('-bb', '-c', code)
188+
179189
def test_integers(self):
180190
# Integer tests (bBhHiIlLqQnN).
181191
import binascii
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Fix :mod:`struct` functions raising :exc:`BytesWarning` under the ``-bb``
2+
command line option when a :class:`str` format is used after an equal
3+
:class:`bytes` format (or vice versa). The internal format cache no longer
4+
mixes :class:`str` and :class:`bytes` keys.

Modules/_struct.c

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2289,32 +2289,50 @@ static PyType_Spec PyStructType_spec = {
22892289
static int
22902290
cache_struct_converter(PyObject *module, PyObject *fmt, PyStructObject **ptr)
22912291
{
2292-
PyObject * s_object;
2292+
PyObject *s_object;
2293+
PyObject *key;
22932294
_structmodulestate *state = get_struct_state(module);
22942295

22952296
if (fmt == NULL) {
22962297
Py_SETREF(*ptr, NULL);
22972298
return 1;
22982299
}
22992300

2300-
if (PyDict_GetItemRef(state->cache, fmt, &s_object) < 0) {
2301+
/* Use a str cache key: an equal str and bytes would collide and be
2302+
compared, raising BytesWarning under -bb. */
2303+
if (PyBytes_Check(fmt)) {
2304+
key = PyUnicode_DecodeASCII(PyBytes_AS_STRING(fmt),
2305+
PyBytes_GET_SIZE(fmt), "surrogateescape");
2306+
if (key == NULL) {
2307+
return 0;
2308+
}
2309+
}
2310+
else {
2311+
key = Py_NewRef(fmt);
2312+
}
2313+
2314+
if (PyDict_GetItemRef(state->cache, key, &s_object) < 0) {
2315+
Py_DECREF(key);
23012316
return 0;
23022317
}
23032318
if (s_object != NULL) {
2319+
Py_DECREF(key);
23042320
*ptr = (PyStructObject *)s_object;
23052321
return Py_CLEANUP_SUPPORTED;
23062322
}
23072323

2308-
s_object = PyObject_CallOneArg(state->PyStructType, fmt);
2324+
s_object = PyObject_CallOneArg(state->PyStructType, key);
23092325
if (s_object != NULL) {
23102326
if (PyDict_GET_SIZE(state->cache) >= MAXCACHE)
23112327
PyDict_Clear(state->cache);
23122328
/* Attempt to cache the result */
2313-
if (PyDict_SetItem(state->cache, fmt, s_object) == -1)
2329+
if (PyDict_SetItem(state->cache, key, s_object) == -1)
23142330
PyErr_Clear();
2331+
Py_DECREF(key);
23152332
*ptr = (PyStructObject *)s_object;
23162333
return Py_CLEANUP_SUPPORTED;
23172334
}
2335+
Py_DECREF(key);
23182336
return 0;
23192337
}
23202338

0 commit comments

Comments
 (0)