Release Python GIL when possible

[grantjenks]

I'm looking to search several million source files using the tree-sitter query interface. I'd love to use a thread pool in Python for this. Would it be possible to make a thread-safe Query.captures method that released the GIL?

I tried doing it in https://github.com/grantjenks/py-tree-sitter-languages but couldn't figure it out this evening (not sure it's possible with the current interface).

[ObserverOfTime]

PR welcome.

This might be slightly easier to implement with recent changes to GIL in 3.12:
https://docs.python.org/3.12/whatsnew/3.12.html#pep-684-a-per-interpreter-gil

[ObserverOfTime]

Another thing to consider is the free-threaded build in Python 3.13.

[ObserverOfTime]

I think we can't actually do this since the order of the results matters, right? @amaanq

[amaanq]

if we allocate a separate cursor per call to matches or captures, it'd be doable

[ObserverOfTime]

Nevermind, I misunderstood the issue.

[m4ul3r]

Probably too hacky of a solution for this, but I used this patch to get tree-sitter working with python3.14t after rebuilding tree-sitter wheel.

--- tree_sitter/binding/module.c	2026-01-11 00:39:49.190491421 -0500
+++ tree_sitter/binding/module.c	2026-01-11 00:19:10.842396744 -0500
@@ -48,20 +48,7 @@
     Py_XDECREF(state->re_compile);
 }
 
-static struct PyModuleDef module_definition = {
-    .m_base = PyModuleDef_HEAD_INIT,
-    .m_name = "_binding",
-    .m_doc = NULL,
-    .m_size = sizeof(ModuleState),
-    .m_free = module_free,
-};
-
-PyMODINIT_FUNC PyInit__binding(void) {
-    PyObject *module = PyModule_Create(&module_definition);
-    if (module == NULL) {
-        return NULL;
-    }
-
+static int module_exec(PyObject *module) {
     ModuleState *state = PyModule_GetState(module);
 
     ts_set_allocator(PyMem_Malloc, PyMem_Calloc, PyMem_Realloc, PyMem_Free);
@@ -113,7 +100,7 @@
         (PyModule_AddObjectRef(module, "Range", (PyObject *)state->range_type) < 0) ||
         (PyModule_AddObjectRef(module, "Tree", (PyObject *)state->tree_type) < 0) ||
         (PyModule_AddObjectRef(module, "TreeCursor", (PyObject *)state->tree_cursor_type) < 0)) {
-        goto cleanup;
+        return -1;
     }
 
     state->query_error = PyErr_NewExceptionWithDoc(
@@ -122,23 +109,24 @@
         PyExc_ValueError, NULL);
     if (state->query_error == NULL ||
         PyModule_AddObjectRef(module, "QueryError", state->query_error) < 0) {
-        goto cleanup;
+        return -1;
     }
 
     state->re_compile = import_attribute("re", "compile");
     if (state->re_compile == NULL) {
-        goto cleanup;
+        return -1;
     }
 
     PyObject *int_enum = import_attribute("enum", "IntEnum");
     if (int_enum == NULL) {
-        goto cleanup;
+        return -1;
     }
     state->log_type_type = (PyTypeObject *)PyObject_CallFunction(
         int_enum, "s{sisi}", "LogType", "PARSE", TSLogTypeParse, "LEX", TSLogTypeLex);
     if (state->log_type_type == NULL ||
         PyModule_AddObjectRef(module, "LogType", (PyObject *)state->log_type_type) < 0) {
-        goto cleanup;
+        Py_DECREF(int_enum);
+        return -1;
     };
     Py_DECREF(int_enum);
 
@@ -147,12 +135,26 @@
                             TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION);
     PyModule_AddStringConstant(module, "__version__", PY_TS_VERSION);
 
+    return 0;
+}
+
+static PyModuleDef_Slot module_slots[] = {
 #ifdef Py_GIL_DISABLED
-    PyUnstable_Module_SetGIL(module, Py_MOD_GIL_USED);
+    {Py_mod_gil, Py_MOD_GIL_USED},
 #endif
-    return module;
+    {Py_mod_exec, module_exec},
+    {0, NULL}
+};
 
-cleanup:
-    Py_XDECREF(module);
-    return NULL;
+static struct PyModuleDef module_definition = {
+    .m_base = PyModuleDef_HEAD_INIT,
+    .m_name = "_binding",
+    .m_doc = NULL,
+    .m_size = sizeof(ModuleState),
+    .m_slots = module_slots,
+    .m_free = module_free,
+};
+
+PyMODINIT_FUNC PyInit__binding(void) {
+    return PyModuleDef_Init(&module_definition);
 }

[reneleonhardt]

Thank you so much for your motivation to fix this 4 years old issue @m4ul3r! ❤️

Could you open a pull request and add python 3.13t and 3.14t to the test matrix to see if free-threaded and normal versions function as expected?