Skip to content

Commit 6282ef6

Browse files
authored
[3.11] gh-99891: Fix infinite recursion in the tokenizer when showing warnings (GH-99893) (GH-99896)
Automerge-Triggered-By: GH:pablogsal. (cherry picked from commit 417206a) Co-authored-by: Pablo Galindo Salgado <Pablogsal@gmail.com>
1 parent 0076ca4 commit 6282ef6

File tree

4 files changed

+24
-0
lines changed

4 files changed

+24
-0
lines changed

Lib/test/test_source_encoding.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,18 @@ def test_file_parse_error_multiline(self):
161161
finally:
162162
os.unlink(TESTFN)
163163

164+
def test_tokenizer_fstring_warning_in_first_line(self):
165+
source = "0b1and 2"
166+
with open(TESTFN, "w") as fd:
167+
fd.write("{}".format(source))
168+
try:
169+
retcode, stdout, stderr = script_helper.assert_python_ok(TESTFN)
170+
self.assertIn(b"SyntaxWarning: invalid binary litera", stderr)
171+
self.assertEqual(stderr.count(source.encode()), 1)
172+
finally:
173+
os.unlink(TESTFN)
174+
175+
164176
class AbstractSourceEncodingTest:
165177

166178
def test_default_coding(self):
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Fix a bug in the tokenizer that could cause infinite recursion when showing
2+
syntax warnings that happen in the first line of the source. Patch by Pablo
3+
Galindo

Parser/tokenizer.c

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ tok_new(void)
8888
tok->async_def_nl = 0;
8989
tok->interactive_underflow = IUNDERFLOW_NORMAL;
9090
tok->str = NULL;
91+
tok->report_warnings = 1;
9192
return tok;
9293
}
9394

@@ -1186,6 +1187,10 @@ indenterror(struct tok_state *tok)
11861187
static int
11871188
parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...)
11881189
{
1190+
if (!tok->report_warnings) {
1191+
return 0;
1192+
}
1193+
11891194
PyObject *errmsg;
11901195
va_list vargs;
11911196
#ifdef HAVE_STDARG_PROTOTYPES
@@ -2194,6 +2199,9 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
21942199
return encoding;
21952200
}
21962201
}
2202+
// We don't want to report warnings here because it could cause infinite recursion
2203+
// if fetching the encoding shows a warning.
2204+
tok->report_warnings = 0;
21972205
while (tok->lineno < 2 && tok->done == E_OK) {
21982206
_PyTokenizer_Get(tok, &p_start, &p_end);
21992207
}

Parser/tokenizer.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ struct tok_state {
8484
NEWLINE token after it. */
8585
/* How to proceed when asked for a new token in interactive mode */
8686
enum interactive_underflow_t interactive_underflow;
87+
int report_warnings;
8788
};
8889

8990
extern struct tok_state *_PyTokenizer_FromString(const char *, int);

0 commit comments

Comments
 (0)