-
-
Notifications
You must be signed in to change notification settings - Fork 33.1k
gh-140149: use PyBytesWriter
in action_helpers.c
's _build_concatenated_bytes
; 3x faster bytes
concat in the parser
#140150
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from 5 commits
692a4f0
904fe91
bde6eaf
e5b07d6
9955759
3d375ce
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
Speed up parsing bytes literals concatenation by using PyBytesWriter API and | ||
a single memory allocation (about 3x faster) | ||
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -1612,19 +1612,46 @@ _build_concatenated_bytes(Parser *p, asdl_expr_seq *strings, int lineno, | |
Py_ssize_t len = asdl_seq_LEN(strings); | ||
assert(len > 0); | ||
|
||
PyObject* res = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); | ||
|
||
/* Bytes literals never get a kind, but just for consistency | ||
since they are represented as Constant nodes, we'll mirror | ||
the same behavior as unicode strings for determining the | ||
kind. */ | ||
PyObject* kind = asdl_seq_GET(strings, 0)->v.Constant.kind; | ||
PyObject *kind = asdl_seq_GET(strings, 0)->v.Constant.kind; | ||
|
||
Py_ssize_t total = 0; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. bit of a meta question: How much performance change is it to not pre-calculate the length? The precalculation + |
||
for (Py_ssize_t i = 0; i < len; i++) { | ||
expr_ty elem = asdl_seq_GET(strings, i); | ||
PyObject *bytes = elem->v.Constant.value; | ||
Py_ssize_t part = PyBytes_GET_SIZE(bytes); | ||
if (part > PY_SSIZE_T_MAX - total) { | ||
PyErr_NoMemory(); | ||
return NULL; | ||
} | ||
total += part; | ||
} | ||
|
||
PyBytesWriter *writer = PyBytesWriter_Create(total); | ||
if (writer == NULL) { | ||
return NULL; | ||
} | ||
char *out = PyBytesWriter_GetData(writer); | ||
|
||
for (Py_ssize_t i = 0; i < len; i++) { | ||
expr_ty elem = asdl_seq_GET(strings, i); | ||
vstinner marked this conversation as resolved.
Show resolved
Hide resolved
|
||
PyBytes_Concat(&res, elem->v.Constant.value); | ||
PyObject *bytes = elem->v.Constant.value; | ||
Py_ssize_t part = PyBytes_GET_SIZE(bytes); | ||
if (part > 0) { | ||
memcpy(out, PyBytes_AS_STRING(bytes), part); | ||
out += part; | ||
} | ||
} | ||
if (!res || _PyArena_AddPyObject(arena, res) < 0) { | ||
Py_XDECREF(res); | ||
|
||
PyObject *res = PyBytesWriter_Finish(writer); | ||
if (res == NULL) { | ||
return NULL; | ||
} | ||
if (_PyArena_AddPyObject(arena, res) < 0) { | ||
Py_DECREF(res); | ||
return NULL; | ||
} | ||
return _PyAST_Constant(res, kind, lineno, col_offset, end_lineno, end_col_offset, p->arena); | ||
|
Uh oh!
There was an error while loading. Please reload this page.