gh-100414: Add SQLite backend to dbm (#114481)

erlend-aasland · rhettinger · serhiy-storchaka · web-flow · commit dd5e4d90789b · 2024-02-14T11:14:56.000Z
Co-authored-by: Raymond Hettinger &lt;rhettinger@users.noreply.github.com&gt;
Co-authored-by: Serhiy Storchaka &lt;storchaka@gmail.com&gt;
Co-authored-by: Mariusz Felisiak &lt;felisiak.mariusz@gmail.com&gt;
diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst
@@ -8,8 +8,13 @@
 
 --------------
 
-:mod:`dbm` is a generic interface to variants of the DBM database ---
-:mod:`dbm.gnu` or :mod:`dbm.ndbm`.  If none of these modules is installed, the
+:mod:`dbm` is a generic interface to variants of the DBM database:
+
+* :mod:`dbm.sqlite3`
+* :mod:`dbm.gnu`
+* :mod:`dbm.ndbm`
+
+If none of these modules are installed, the
 slow-but-simple implementation in module :mod:`dbm.dumb` will be used.  There
 is a `third party interface <https://www.jcea.es/programacion/pybsddb.htm>`_ to
 the Oracle Berkeley DB.
@@ -25,8 +30,8 @@ the Oracle Berkeley DB.
 .. function:: whichdb(filename)
 
    This function attempts to guess which of the several simple database modules
-   available --- :mod:`dbm.gnu`, :mod:`dbm.ndbm` or :mod:`dbm.dumb` --- should
-   be used to open a given file.
+   available --- :mod:`dbm.sqlite3`, :mod:`dbm.gnu`, :mod:`dbm.ndbm`,
+   or :mod:`dbm.dumb` --- should be used to open a given file.
 
    Return one of the following values:
 
@@ -144,6 +149,46 @@ then prints out the contents of the database::
 
 The individual submodules are described in the following sections.
 
+:mod:`dbm.sqlite3` --- SQLite backend for dbm
+---------------------------------------------
+
+.. module:: dbm.sqlite3
+   :platform: All
+   :synopsis: SQLite backend for dbm
+
+.. versionadded:: 3.13
+
+**Source code:** :source:`Lib/dbm/sqlite3.py`
+
+--------------
+
+This module uses the standard library :mod:`sqlite3` module to provide an
+SQLite backend for the :mod:`dbm` module.
+The files created by :mod:`dbm.sqlite3` can thus be opened by :mod:`sqlite3`,
+or any other SQLite browser, including the SQLite CLI.
+
+.. function:: open(filename, /, flag="r", mode=0o666)
+
+   Open an SQLite database.
+   The returned object behaves like a :term:`mapping`,
+   implements a :meth:`!close` method,
+   and supports a "closing" context manager via the :keyword:`with` keyword.
+
+   :param filename:
+      The path to the database to be opened.
+   :type filename: :term:`path-like object`
+
+   :param str flag:
+
+      * ``'r'`` (default): |flag_r|
+      * ``'w'``: |flag_w|
+      * ``'c'``: |flag_c|
+      * ``'n'``: |flag_n|
+
+   :param mode:
+      The Unix file access mode of the file (default: octal ``0o666``),
+      used only when the database has to be created.
+
 
 :mod:`dbm.gnu` --- GNU database manager
 ---------------------------------------
diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst
@@ -231,6 +231,16 @@ dis
   the ``show_offsets`` parameter.
   (Contributed by Irit Katriel in :gh:`112137`.)
 
+dbm
+---
+
+* Add :meth:`dbm.gnu.gdbm.clear` and :meth:`dbm.ndbm.ndbm.clear`  methods that remove all items
+  from the database.
+  (Contributed by Donghee Na in :gh:`107122`.)
+
+* Add new :mod:`dbm.sqlite3` backend.
+  (Contributed by Raymond Hettinger and Erlend E. Aasland in :gh:`100414`.)
+
 doctest
 -------
 
diff --git a/Lib/dbm/__init__.py b/Lib/dbm/__init__.py
@@ -5,7 +5,7 @@
         import dbm
         d = dbm.open(file, 'w', 0o666)
 
-The returned object is a dbm.gnu, dbm.ndbm or dbm.dumb object, dependent on the
+The returned object is a dbm.sqlite3, dbm.gnu, dbm.ndbm or dbm.dumb database object, dependent on the
 type of database being opened (determined by the whichdb function) in the case
 of an existing dbm. If the dbm does not exist and the create or new flag ('c'
 or 'n') was specified, the dbm type will be determined by the availability of
@@ -38,7 +38,7 @@
 class error(Exception):
     pass
 
-_names = ['dbm.gnu', 'dbm.ndbm', 'dbm.dumb']
+_names = ['dbm.gnu', 'dbm.ndbm', 'dbm.sqlite3', 'dbm.dumb']
 _defaultmod = None
 _modules = {}
 
@@ -164,6 +164,10 @@ def whichdb(filename):
     if len(s) != 4:
         return ""
 
+    # Check for SQLite3 header string.
+    if s16 == b"SQLite format 3\0":
+        return "dbm.sqlite3"
+
     # Convert to 4-byte int in native byte order -- return "" if impossible
     try:
         (magic,) = struct.unpack("=l", s)
diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py
@@ -0,0 +1,141 @@
+import os
+import sqlite3
+import sys
+from pathlib import Path
+from contextlib import suppress, closing
+from collections.abc import MutableMapping
+
+BUILD_TABLE = """
+  CREATE TABLE IF NOT EXISTS Dict (
+    key BLOB UNIQUE NOT NULL,
+    value BLOB NOT NULL
+  )
+"""
+GET_SIZE = "SELECT COUNT (key) FROM Dict"
+LOOKUP_KEY = "SELECT value FROM Dict WHERE key = CAST(? AS BLOB)"
+STORE_KV = "REPLACE INTO Dict (key, value) VALUES (CAST(? AS BLOB), CAST(? AS BLOB))"
+DELETE_KEY = "DELETE FROM Dict WHERE key = CAST(? AS BLOB)"
+ITER_KEYS = "SELECT key FROM Dict"
+
+
+class error(OSError):
+    pass
+
+
+_ERR_CLOSED = "DBM object has already been closed"
+_ERR_REINIT = "DBM object does not support reinitialization"
+
+
+def _normalize_uri(path):
+    path = Path(path)
+    uri = path.absolute().as_uri()
+    while "//" in uri:
+        uri = uri.replace("//", "/")
+    return uri
+
+
+class _Database(MutableMapping):
+
+    def __init__(self, path, /, *, flag, mode):
+        if hasattr(self, "_cx"):
+            raise error(_ERR_REINIT)
+
+        path = os.fsdecode(path)
+        match flag:
+            case "r":
+                flag = "ro"
+            case "w":
+                flag = "rw"
+            case "c":
+                flag = "rwc"
+                Path(path).touch(mode=mode, exist_ok=True)
+            case "n":
+                flag = "rwc"
+                Path(path).unlink(missing_ok=True)
+                Path(path).touch(mode=mode)
+            case _:
+                raise ValueError("Flag must be one of 'r', 'w', 'c', or 'n', "
+                                 f"not {flag!r}")
+
+        # We use the URI format when opening the database.
+        uri = _normalize_uri(path)
+        uri = f"{uri}?mode={flag}"
+
+        try:
+            self._cx = sqlite3.connect(uri, autocommit=True, uri=True)
+        except sqlite3.Error as exc:
+            raise error(str(exc))
+
+        # This is an optimization only; it's ok if it fails.
+        with suppress(sqlite3.OperationalError):
+            self._cx.execute("PRAGMA journal_mode = wal")
+
+        if flag == "rwc":
+            self._execute(BUILD_TABLE)
+
+    def _execute(self, *args, **kwargs):
+        if not self._cx:
+            raise error(_ERR_CLOSED)
+        try:
+            return closing(self._cx.execute(*args, **kwargs))
+        except sqlite3.Error as exc:
+            raise error(str(exc))
+
+    def __len__(self):
+        with self._execute(GET_SIZE) as cu:
+            row = cu.fetchone()
+        return row[0]
+
+    def __getitem__(self, key):
+        with self._execute(LOOKUP_KEY, (key,)) as cu:
+            row = cu.fetchone()
+        if not row:
+            raise KeyError(key)
+        return row[0]
+
+    def __setitem__(self, key, value):
+        self._execute(STORE_KV, (key, value))
+
+    def __delitem__(self, key):
+        with self._execute(DELETE_KEY, (key,)) as cu:
+            if not cu.rowcount:
+                raise KeyError(key)
+
+    def __iter__(self):
+        try:
+            with self._execute(ITER_KEYS) as cu:
+                for row in cu:
+                    yield row[0]
+        except sqlite3.Error as exc:
+            raise error(str(exc))
+
+    def close(self):
+        if self._cx:
+            self._cx.close()
+            self._cx = None
+
+    def keys(self):
+        return list(super().keys())
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+
+def open(filename, /, flag="r", mode=0o666):
+    """Open a dbm.sqlite3 database and return the dbm object.
+
+    The 'filename' parameter is the name of the database file.
+
+    The optional 'flag' parameter can be one of ...:
+        'r' (default): open an existing database for read only access
+        'w': open an existing database for read/write access
+        'c': create a database if it does not exist; open for read/write access
+        'n': always create a new, empty database; open for read/write access
+
+    The optional 'mode' parameter is the Unix file access mode of the database;
+    only used when creating a new database. Default: 0o666.
+    """
+    return _Database(filename, flag=flag, mode=mode)
diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py
@@ -6,6 +6,13 @@
 from test.support import import_helper
 from test.support import os_helper
 
+
+try:
+    from dbm import sqlite3 as dbm_sqlite3
+except ImportError:
+    dbm_sqlite3 = None
+
+
 try:
     from dbm import ndbm
 except ImportError:
@@ -213,6 +220,27 @@ def test_whichdb_ndbm(self):
         for path in fnames:
             self.assertIsNone(self.dbm.whichdb(path))
 
+    @unittest.skipUnless(dbm_sqlite3, reason='Test requires dbm.sqlite3')
+    def test_whichdb_sqlite3(self):
+        # Databases created by dbm.sqlite3 are detected correctly.
+        with dbm_sqlite3.open(_fname, "c") as db:
+            db["key"] = "value"
+        self.assertEqual(self.dbm.whichdb(_fname), "dbm.sqlite3")
+
+    @unittest.skipUnless(dbm_sqlite3, reason='Test requires dbm.sqlite3')
+    def test_whichdb_sqlite3_existing_db(self):
+        # Existing sqlite3 databases are detected correctly.
+        sqlite3 = import_helper.import_module("sqlite3")
+        try:
+            # Create an empty database.
+            with sqlite3.connect(_fname) as cx:
+                cx.execute("CREATE TABLE dummy(database)")
+                cx.commit()
+        finally:
+            cx.close()
+        self.assertEqual(self.dbm.whichdb(_fname), "dbm.sqlite3")
+
+
     def setUp(self):
         self.addCleanup(cleaunup_test_dir)
         setup_test_dir()
diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py
diff --git a/Misc/NEWS.d/next/Library/2024-01-23-13-03-22.gh-issue-100414.5kTdU5.rst b/Misc/NEWS.d/next/Library/2024-01-23-13-03-22.gh-issue-100414.5kTdU5.rst
