fixup! gh-153569: Add tokenizer source and cursor primitives

Author: pablogsal

Modules/_testinternalcapi/tokenizer.c

@@ -103,12 +103,16 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
             check(line.start == 6 && line.end == 9 &&
                       line.implicit_newline && !line.contains_nul,
                   "wrong second source line") < 0 ||
+            check(!_PyTok_SourceLineIsImplicit(&source, 1) &&
+                      _PyTok_SourceLineIsImplicit(&source, 2),
+                  "wrong early implicit newline flags") < 0 ||
             check(_PyTok_SourceLine(&source, 3, &line) == 0,
                   "cannot find third source line") < 0 ||
             check(line.contains_nul, "missing null byte flag") < 0 ||
             check(_PyTok_SourceLine(&source, final_line, &line) == 0,
                   "cannot find final source line") < 0 ||
-            check(line.implicit_newline,
+            check(line.implicit_newline &&
+                      _PyTok_SourceLineIsImplicit(&source, final_line),
                   "missing late implicit newline flag") < 0) {
         goto error;
     }
@@ -154,7 +158,11 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
             check(_PyTok_SourceLine(&source, final_line + 1, &line) == 0,
                   "cannot find virtual EOF line") < 0 ||
             check(line.start == source.len && line.end == source.len,
-                  "wrong virtual EOF line") < 0) {
+                  "wrong virtual EOF line") < 0 ||
+            check(!_PyTok_SourceLineIsImplicit(&source, 0) &&
+                      !_PyTok_SourceLineIsImplicit(
+                          &source, final_line + 1),
+                  "virtual or invalid line is implicit") < 0) {
         goto error;
     }
 

Parser/tokenizer/cursor.h

@@ -12,8 +12,10 @@ typedef struct {
     int lineno;
 } _PyTok_Cursor;
 
+/* Move to the start of a 1-based line. Both setters preserve the cursor on
+   error. */
 PyAPI_FUNC(int) _PyTok_CursorSetLine(_PyTok_Cursor *, int);
-/* A line-boundary offset selects the following line. */
+/* Move to an offset. A line boundary selects the following line. */
 PyAPI_FUNC(int) _PyTok_CursorSetOffset(_PyTok_Cursor *, _PyTok_Off);
 
 static inline void
@@ -24,7 +26,9 @@ _PyTok_CursorInit(_PyTok_Cursor *cursor, const _PyTok_SourceText *source)
     };
 }
 
-/* EOF with pos before line_end reports byte-column overflow. */
+/* Read one byte from the current line, including its terminating newline.
+   EOF marks the line boundary, not necessarily the end of the source. It is
+   also returned if advancing would make the byte column unrepresentable. */
 static inline int
 _PyTok_CursorAdvance(_PyTok_Cursor *cursor)
 {
@@ -41,15 +45,18 @@ _PyTok_CursorAdvance(_PyTok_Cursor *cursor)
     return Py_CHARMASK(cursor->source->bytes[cursor->pos++]);
 }
 
+/* Return the byte at a nonnegative distance within the current line, or EOF
+   if the distance reaches or crosses the line boundary. */
 static inline int
 _PyTok_CursorPeek(const _PyTok_Cursor *cursor, int distance)
 {
     assert(cursor->source != NULL);
     assert(cursor->pos >= cursor->line_start);
     assert(cursor->pos <= cursor->line_end);
     assert(cursor->line_end <= cursor->source->len);
-    if ((size_t)distance >=
-            (size_t)(cursor->line_end - cursor->pos)) {
+    assert(distance >= 0);
+    if (distance < 0 ||
+            distance >= cursor->line_end - cursor->pos) {
         return EOF;
     }
     return Py_CHARMASK(cursor->source->bytes[cursor->pos + distance]);

Parser/tokenizer/source.c

@@ -75,14 +75,8 @@ reserve_checkpoints(_PyTok_SourceText *source, int needed)
         PyErr_NoMemory();
         return -1;
     }
-    if ((size_t)cap > (size_t)PY_SSIZE_T_MAX /
-            sizeof(*source->line_checkpoints)) {
-        PyErr_NoMemory();
-        return -1;
-    }
-    _PyTok_Off *checkpoints = PyMem_Realloc(
-        source->line_checkpoints,
-        (size_t)cap * sizeof(*source->line_checkpoints));
+    _PyTok_Off *checkpoints = source->line_checkpoints;
+    PyMem_Resize(checkpoints, _PyTok_Off, cap);
     if (checkpoints == NULL) {
         PyErr_NoMemory();
         return -1;
@@ -190,10 +184,11 @@ _PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span,
     return source->bytes == NULL ? "" : source->bytes + span.start;
 }
 
-static int
-line_is_implicit(const _PyTok_SourceText *source, int lineno)
+int
+_PyTok_SourceLineIsImplicit(const _PyTok_SourceText *source, int lineno)
 {
-    if ((lineno - 1) / 8 >= source->implicit_cap) {
+    if (lineno < 1 || lineno > source->nlines ||
+            (lineno - 1) / 8 >= source->implicit_cap) {
         return 0;
     }
     return (source->implicit_lines[(lineno - 1) / 8] >>
@@ -251,7 +246,7 @@ _PyTok_SourceLine(const _PyTok_SourceText *source, int lineno,
     *line = (_PyTok_Line){
         .start = start,
         .end = end,
-        .implicit_newline = line_is_implicit(source, lineno),
+        .implicit_newline = _PyTok_SourceLineIsImplicit(source, lineno),
         .contains_nul = memchr(
             source->bytes + start, 0, end - start) != NULL,
     };

Parser/tokenizer/source.h

@@ -5,11 +5,13 @@
 
 typedef Py_ssize_t _PyTok_Off;
 
+/* Half-open byte offsets into a _PyTok_SourceText. */
 typedef struct {
     _PyTok_Off start;
     _PyTok_Off end;
 } _PyTok_Span;
 
+/* Lines are 1-based and byte columns are 0-based. */
 typedef struct {
     int lineno;
     int byte_col;
@@ -20,6 +22,7 @@ typedef enum {
     _PYTOK_AFFINITY_RIGHT,
 } _PyTok_Affinity;
 
+/* The half-open range includes the terminating newline when present. */
 typedef struct {
     _PyTok_Off start;
     _PyTok_Off end;
@@ -39,19 +42,27 @@ typedef struct {
 } _PyTok_SourceText;
 
 PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *);
-/* Clear invalidates all cursors and span views for the source. */
+/* Clear invalidates all cursors, spans, and views for the source. */
 PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *);
-/* Append one logical line. The implicit_newline flag means that its newline
-   terminator was synthesized. The input must not point into source storage. */
+/* Append one nonempty logical line and return its start offset. The input may
+   contain one newline, as its final byte. An unterminated line must be the
+   final line. implicit_newline means that the final newline was synthesized.
+   The input must not point into source storage. */
 PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine(
     _PyTok_SourceText *source, const char *bytes, Py_ssize_t len,
     int implicit_newline);
 /* The returned view is invalidated by SourceAppendLine and SourceClear. */
 PyAPI_FUNC(const char *) _PyTok_SourceSpanView(
     const _PyTok_SourceText *, _PyTok_Span, Py_ssize_t *);
+/* Look up a 1-based line. Empty and newline-terminated sources have an empty
+   virtual line at EOF. */
 PyAPI_FUNC(int) _PyTok_SourceLine(
     const _PyTok_SourceText *, int, _PyTok_Line *);
-/* At a line boundary, affinity selects the preceding or following line. */
+/* Return false for invalid line numbers and the virtual EOF line. */
+PyAPI_FUNC(int) _PyTok_SourceLineIsImplicit(
+    const _PyTok_SourceText *, int);
+/* At a line boundary, left affinity selects the preceding line at its end;
+   right affinity selects the following line at byte column zero. */
 PyAPI_FUNC(int) _PyTok_SourceLocation(
     const _PyTok_SourceText *, _PyTok_Off, _PyTok_Affinity, _PyTok_Loc *);