From bbbdbc387ee1701beb064f8618b5a78f3a01dc9d Mon Sep 17 00:00:00 2001 From: Megh Parikh Date: Sun, 30 Mar 2025 17:07:12 -0400 Subject: [PATCH 1/5] gh-131918: Add _ThreadLocalSqliteConnection in dbm.sqlite --- Lib/dbm/sqlite3.py | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 7e0ae2a29e3a64..76ab6d7336544b 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -3,6 +3,8 @@ from pathlib import Path from contextlib import suppress, closing from collections.abc import MutableMapping +from threading import current_thread +from weakref import ref BUILD_TABLE = """ CREATE TABLE IF NOT EXISTS Dict ( @@ -32,6 +34,34 @@ def _normalize_uri(path): uri = uri.replace("//", "/") return uri +class _ThreadLocalSqliteConnection: + def __init__(self, uri: str): + self._uri = uri + self._conn = {} + + def conn(self): + thread = current_thread() + idt = id(thread) + def thread_deleted(_, idt=idt): + # When the thread is deleted, remove the local dict. + # Note that this is suboptimal if the thread object gets + # caught in a reference loop. We would like to be called + # as soon as the OS-level thread ends instead. + if self._conn is not None: + dct = self._conn.dicts.pop(idt) + wrthread = ref(thread, thread_deleted) + try: + conn = sqlite3.connect(self._uri, autocommit=True, uri=True) + self._conn[id(thread)] = conn + return conn + except sqlite3.Error as exc: + raise error(str(exc)) + + def close(self): + for t, conn in self._conn.items(): + conn.close() + del self._conn[t] + class _Database(MutableMapping): @@ -59,15 +89,11 @@ def __init__(self, path, /, *, flag, mode): # We use the URI format when opening the database. uri = _normalize_uri(path) uri = f"{uri}?mode={flag}" - - try: - self._cx = sqlite3.connect(uri, autocommit=True, uri=True) - except sqlite3.Error as exc: - raise error(str(exc)) + self._cx = _ThreadLocalSqliteConnection(uri) # This is an optimization only; it's ok if it fails. with suppress(sqlite3.OperationalError): - self._cx.execute("PRAGMA journal_mode = wal") + self._cx.conn().execute("PRAGMA journal_mode = wal") if flag == "rwc": self._execute(BUILD_TABLE) @@ -76,7 +102,7 @@ def _execute(self, *args, **kwargs): if not self._cx: raise error(_ERR_CLOSED) try: - return closing(self._cx.execute(*args, **kwargs)) + return closing(self._cx.conn().execute(*args, **kwargs)) except sqlite3.Error as exc: raise error(str(exc)) From 3fd9f934fd09c238389ad912c2c9fdfc1396d409 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sun, 30 Mar 2025 21:14:12 +0000 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2025-03-30-21-14-11.gh-issue-131918.X5ibxj.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2025-03-30-21-14-11.gh-issue-131918.X5ibxj.rst diff --git a/Misc/NEWS.d/next/Library/2025-03-30-21-14-11.gh-issue-131918.X5ibxj.rst b/Misc/NEWS.d/next/Library/2025-03-30-21-14-11.gh-issue-131918.X5ibxj.rst new file mode 100644 index 00000000000000..eb06ed03807f1e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-30-21-14-11.gh-issue-131918.X5ibxj.rst @@ -0,0 +1 @@ +Fixes dbm.sqlite3 for multi-threaded use-cases by using thread-local connections. From 9a6b1c0b2eea55ed4052d1c226b7cb465f202a24 Mon Sep 17 00:00:00 2001 From: Megh Parikh Date: Sun, 30 Mar 2025 17:23:54 -0400 Subject: [PATCH 3/5] Typos --- Lib/dbm/sqlite3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 76ab6d7336544b..3b0e9eb9fda707 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -48,7 +48,7 @@ def thread_deleted(_, idt=idt): # caught in a reference loop. We would like to be called # as soon as the OS-level thread ends instead. if self._conn is not None: - dct = self._conn.dicts.pop(idt) + self._conn.pop(idt) wrthread = ref(thread, thread_deleted) try: conn = sqlite3.connect(self._uri, autocommit=True, uri=True) @@ -60,7 +60,7 @@ def thread_deleted(_, idt=idt): def close(self): for t, conn in self._conn.items(): conn.close() - del self._conn[t] + self._conn = {} class _Database(MutableMapping): From d0eec4628b29002228d48a6bd9a77950583f4565 Mon Sep 17 00:00:00 2001 From: Megh Parikh Date: Sun, 30 Mar 2025 17:25:39 -0400 Subject: [PATCH 4/5] More typo bugs --- Lib/dbm/sqlite3.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 3b0e9eb9fda707..f170523d6bb600 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -42,6 +42,8 @@ def __init__(self, uri: str): def conn(self): thread = current_thread() idt = id(thread) + if idt in self._conn: + return self._conn[idt] def thread_deleted(_, idt=idt): # When the thread is deleted, remove the local dict. # Note that this is suboptimal if the thread object gets From 5ebb7742c7cb1cbbc327337d04f6224db0a298b9 Mon Sep 17 00:00:00 2001 From: Megh Parikh Date: Sun, 30 Mar 2025 17:51:39 -0400 Subject: [PATCH 5/5] Actually close the connection --- Lib/dbm/sqlite3.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index f170523d6bb600..a6955e641dd767 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -50,7 +50,8 @@ def thread_deleted(_, idt=idt): # caught in a reference loop. We would like to be called # as soon as the OS-level thread ends instead. if self._conn is not None: - self._conn.pop(idt) + conn = self._conn.pop(idt) + conn.close() wrthread = ref(thread, thread_deleted) try: conn = sqlite3.connect(self._uri, autocommit=True, uri=True)