From 8cdb980c2e002aa98870f51f9f368917381019ab Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 2 Nov 2021 01:04:46 +0100 Subject: [PATCH 01/12] bpo-45243: Use sqlite3 connection limit context manager in test suite --- Lib/test/test_sqlite3/test_dbapi.py | 35 +++++++++++++++------ Lib/test/test_sqlite3/test_regression.py | 24 +++++++------- Lib/test/test_sqlite3/test_userfunctions.py | 9 ++++++ 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 34895be018078f..1737855ae16d5d 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -29,7 +29,6 @@ from test.support import ( SHORT_TIMEOUT, - bigmemtest, check_disallow_instantiation, threading_helper, ) @@ -48,6 +47,23 @@ def managed_connect(*args, in_mem=False, **kwargs): unlink(TESTFN) +# Helper for temporary memory databases +def memory_database(): + cx = sqlite.connect(":memory:") + return contextlib.closing(cx) + + +# Temporarily limits the maximum size of any string or BLOB or table row, in +# bytes. +@contextlib.contextmanager +def cx_limit(cx, category=sqlite.SQLITE_LIMIT_LENGTH, limit=128): + try: + _prev = cx.setlimit(category, limit) + yield limit + finally: + cx.setlimit(category, _prev) + + class ModuleTests(unittest.TestCase): def test_api_level(self): self.assertEqual(sqlite.apilevel, "2.0", @@ -910,14 +926,15 @@ def test_cursor_executescript_with_surrogates(self): insert into a(s) values ('\ud8ff'); """) - @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') - @bigmemtest(size=2**31, memuse=3, dry_run=False) - def test_cursor_executescript_too_large_script(self, maxsize): - con = sqlite.connect(":memory:") - cur = con.cursor() - for size in 2**31-1, 2**31: - with self.assertRaises(sqlite.DataError): - cur.executescript("create table a(s);".ljust(size)) + def test_cursor_executescript_too_large_script(self): + msg = "query string is too large" + with memory_database() as cx, cx_limit(cx) as lim: + for sz in lim, lim+1: + with self.subTest(sz=sz): + self.assertRaisesRegex( + sqlite.DataError, msg, cx.executescript, + "create table a(s);".ljust(sz) + ) def test_cursor_executescript_tx_control(self): con = sqlite.connect(":memory:") diff --git a/Lib/test/test_sqlite3/test_regression.py b/Lib/test/test_sqlite3/test_regression.py index 3d71809d9c11cf..4ca18ef9571d5d 100644 --- a/Lib/test/test_sqlite3/test_regression.py +++ b/Lib/test/test_sqlite3/test_regression.py @@ -28,7 +28,8 @@ import functools from test import support -from .test_dbapi import managed_connect +from .test_dbapi import memory_database, managed_connect, cx_limit + class RegressionTests(unittest.TestCase): def setUp(self): @@ -356,17 +357,16 @@ def test_surrogates(self): self.assertRaises(UnicodeEncodeError, cur.execute, "select '\ud8ff'") self.assertRaises(UnicodeEncodeError, cur.execute, "select '\udcff'") - @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') - @support.bigmemtest(size=2**31, memuse=4, dry_run=False) - def test_large_sql(self, maxsize): - # Test two cases: size+1 > INT_MAX and size+1 <= INT_MAX. - for size in (2**31, 2**31-2): - con = sqlite.connect(":memory:") - sql = "select 1".ljust(size) - self.assertRaises(sqlite.DataError, con, sql) - cur = con.cursor() - self.assertRaises(sqlite.DataError, cur.execute, sql) - del sql + def test_large_sql(self): + msg = "query string is too large" + with memory_database() as cx, cx_limit(cx) as lim: + for sz in lim, lim+1: + with self.subTest(sz=sz): + sql = "select 1".ljust(sz) + self.assertRaisesRegex(sqlite.DataError, msg, cx, sql) + cu = cx.cursor() + self.assertRaisesRegex(sqlite.DataError, msg, + cu.execute, sql) def test_commit_cursor_reset(self): """ diff --git a/Lib/test/test_sqlite3/test_userfunctions.py b/Lib/test/test_sqlite3/test_userfunctions.py index ad408475b73af7..8acc723418d728 100644 --- a/Lib/test/test_sqlite3/test_userfunctions.py +++ b/Lib/test/test_sqlite3/test_userfunctions.py @@ -31,6 +31,7 @@ import sqlite3 as sqlite from test.support import bigmemtest +from .test_dbapi import cx_limit def with_tracebacks(strings, traceback=True): @@ -223,6 +224,14 @@ def test_func_error_on_create(self): with self.assertRaises(sqlite.OperationalError): self.con.create_function("bla", -100, lambda x: 2*x) + def test_func_too_many_args(self): + category = sqlite.SQLITE_LIMIT_FUNCTION_ARG + msg = "too many arguments on function" + with cx_limit(self.con, category=category, limit=1): + with self.assertRaisesRegex(sqlite.OperationalError, msg): + self.con.create_function("toomany", 2, lambda: "") + self.con.execute("select toomany(1, 2)"); + def test_func_ref_count(self): def getfunc(): def f(): From d68d6d844ba3be161a548c9574841d746481ab68 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 2 Nov 2021 10:40:25 +0100 Subject: [PATCH 02/12] Add test for too many SQL variables --- Lib/test/test_sqlite3/test_dbapi.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 1737855ae16d5d..d228051b093d14 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -540,6 +540,13 @@ def __getitem__(slf, x): with self.assertRaises(ZeroDivisionError): self.cu.execute("select name from test where name=?", L()) + def test_execute_too_many_params(self): + category = sqlite.SQLITE_LIMIT_VARIABLE_NUMBER + msg = "too many SQL variables" + with cx_limit(self.cx, category=category, limit=1): + with self.assertRaisesRegex(sqlite.OperationalError, msg): + self.cu.execute("insert into test values(?, ?)", (1, 2)) + def test_execute_dict_mapping(self): self.cu.execute("insert into test(name) values ('foo')") self.cu.execute("select name from test where name=:name", {"name": "foo"}) From 22adb063987f7ec9412f7d0d3e7efe873cf63345 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 2 Nov 2021 10:54:11 +0100 Subject: [PATCH 03/12] Adjust comment --- Lib/test/test_sqlite3/test_dbapi.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index d228051b093d14..23974da868c8dd 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -53,8 +53,7 @@ def memory_database(): return contextlib.closing(cx) -# Temporarily limits the maximum size of any string or BLOB or table row, in -# bytes. +# Temporarily limit a database connection parameter @contextlib.contextmanager def cx_limit(cx, category=sqlite.SQLITE_LIMIT_LENGTH, limit=128): try: From cb5624974e5567900eb2d6fa20d66e53ac733d21 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 2 Nov 2021 13:00:01 +0100 Subject: [PATCH 04/12] Address review --- Lib/test/test_sqlite3/test_dbapi.py | 7 +++++-- Lib/test/test_sqlite3/test_userfunctions.py | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 23974da868c8dd..2ffe2296875b98 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -542,9 +542,12 @@ def __getitem__(slf, x): def test_execute_too_many_params(self): category = sqlite.SQLITE_LIMIT_VARIABLE_NUMBER msg = "too many SQL variables" - with cx_limit(self.cx, category=category, limit=1): + with cx_limit(self.cx, category=category, limit=2): + self.cu.execute("select * from test where name=? and income=?", + ("a", 1)) with self.assertRaisesRegex(sqlite.OperationalError, msg): - self.cu.execute("insert into test values(?, ?)", (1, 2)) + self.cu.execute("insert into test values(?, ?, ?)", + ("a", 2, "b")) def test_execute_dict_mapping(self): self.cu.execute("insert into test(name) values ('foo')") diff --git a/Lib/test/test_sqlite3/test_userfunctions.py b/Lib/test/test_sqlite3/test_userfunctions.py index 8acc723418d728..560711e2843b87 100644 --- a/Lib/test/test_sqlite3/test_userfunctions.py +++ b/Lib/test/test_sqlite3/test_userfunctions.py @@ -229,8 +229,8 @@ def test_func_too_many_args(self): msg = "too many arguments on function" with cx_limit(self.con, category=category, limit=1): with self.assertRaisesRegex(sqlite.OperationalError, msg): - self.con.create_function("toomany", 2, lambda: "") - self.con.execute("select toomany(1, 2)"); + self.con.create_function("addint", 2, lambda x, y: x+y) + self.con.execute("select addint(1, 2)"); def test_func_ref_count(self): def getfunc(): From 3a75c2db2ef04f64dce62483e0a9e8d6d27018d2 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 4 Nov 2021 13:59:24 +0100 Subject: [PATCH 05/12] Address review: test lim-1 and lim --- Lib/test/test_sqlite3/test_dbapi.py | 9 +++------ Lib/test/test_sqlite3/test_regression.py | 16 +++++++++------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 2ffe2296875b98..c36f23f66de430 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -938,12 +938,9 @@ def test_cursor_executescript_with_surrogates(self): def test_cursor_executescript_too_large_script(self): msg = "query string is too large" with memory_database() as cx, cx_limit(cx) as lim: - for sz in lim, lim+1: - with self.subTest(sz=sz): - self.assertRaisesRegex( - sqlite.DataError, msg, cx.executescript, - "create table a(s);".ljust(sz) - ) + cx.executescript("select 'almost too large'".ljust(lim-1)) + with self.assertRaisesRegex(sqlite.DataError, msg): + cx.executescript("select 'too large'".ljust(lim)) def test_cursor_executescript_tx_control(self): con = sqlite.connect(":memory:") diff --git a/Lib/test/test_sqlite3/test_regression.py b/Lib/test/test_sqlite3/test_regression.py index 4ca18ef9571d5d..158f4cf86f55ca 100644 --- a/Lib/test/test_sqlite3/test_regression.py +++ b/Lib/test/test_sqlite3/test_regression.py @@ -360,13 +360,15 @@ def test_surrogates(self): def test_large_sql(self): msg = "query string is too large" with memory_database() as cx, cx_limit(cx) as lim: - for sz in lim, lim+1: - with self.subTest(sz=sz): - sql = "select 1".ljust(sz) - self.assertRaisesRegex(sqlite.DataError, msg, cx, sql) - cu = cx.cursor() - self.assertRaisesRegex(sqlite.DataError, msg, - cu.execute, sql) + cu = cx.cursor() + + cx("select 1".ljust(lim-1)) + # use a different SQL statement; don't reuse from the LRU cache + cu.execute("select 2".ljust(lim-1)) + + sql = "select 3".ljust(lim) + self.assertRaisesRegex(sqlite.DataError, msg, cx, sql) + self.assertRaisesRegex(sqlite.DataError, msg, cu.execute, sql) def test_commit_cursor_reset(self): """ From a6da63452050efb7cc279636cdc260889b764f64 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 4 Nov 2021 15:00:23 +0100 Subject: [PATCH 06/12] Address review: make it clear which function is failing --- Lib/test/test_sqlite3/test_userfunctions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_sqlite3/test_userfunctions.py b/Lib/test/test_sqlite3/test_userfunctions.py index 560711e2843b87..14d5011b0f2061 100644 --- a/Lib/test/test_sqlite3/test_userfunctions.py +++ b/Lib/test/test_sqlite3/test_userfunctions.py @@ -228,8 +228,8 @@ def test_func_too_many_args(self): category = sqlite.SQLITE_LIMIT_FUNCTION_ARG msg = "too many arguments on function" with cx_limit(self.con, category=category, limit=1): + self.con.create_function("addint", 2, lambda x, y: x+y) with self.assertRaisesRegex(sqlite.OperationalError, msg): - self.con.create_function("addint", 2, lambda x, y: x+y) self.con.execute("select addint(1, 2)"); def test_func_ref_count(self): From b98d6232714d6f22acc5ed250549afaedfee4f4a Mon Sep 17 00:00:00 2001 From: Erlend Egeberg Aasland Date: Fri, 5 Nov 2021 13:55:05 +0100 Subject: [PATCH 07/12] Update Lib/test/test_sqlite3/test_dbapi.py Co-authored-by: Serhiy Storchaka --- Lib/test/test_sqlite3/test_dbapi.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index f10772111adda2..7831ad50d9907a 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -668,12 +668,11 @@ def __getitem__(slf, x): def test_execute_too_many_params(self): category = sqlite.SQLITE_LIMIT_VARIABLE_NUMBER msg = "too many SQL variables" - with cx_limit(self.cx, category=category, limit=2): - self.cu.execute("select * from test where name=? and income=?", - ("a", 1)) + with cx_limit(self.cx, category=category, limit=1): + self.cu.execute("select * from test where name=?", ("a",)) with self.assertRaisesRegex(sqlite.OperationalError, msg): - self.cu.execute("insert into test values(?, ?, ?)", - ("a", 2, "b")) + self.cu.execute("select * from test where name=? and income=?", + ("a", 1)) def test_execute_dict_mapping(self): self.cu.execute("insert into test(name) values ('foo')") From 9521fc1754946fe6d9c2d5d190c63ff42d74eced Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 5 Nov 2021 13:59:44 +0100 Subject: [PATCH 08/12] Simplify test_func_too_many_args further --- Lib/test/test_sqlite3/test_userfunctions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_sqlite3/test_userfunctions.py b/Lib/test/test_sqlite3/test_userfunctions.py index 14d5011b0f2061..b4144bb68a0283 100644 --- a/Lib/test/test_sqlite3/test_userfunctions.py +++ b/Lib/test/test_sqlite3/test_userfunctions.py @@ -228,9 +228,8 @@ def test_func_too_many_args(self): category = sqlite.SQLITE_LIMIT_FUNCTION_ARG msg = "too many arguments on function" with cx_limit(self.con, category=category, limit=1): - self.con.create_function("addint", 2, lambda x, y: x+y) with self.assertRaisesRegex(sqlite.OperationalError, msg): - self.con.execute("select addint(1, 2)"); + self.con.execute("select max(1, 2)"); def test_func_ref_count(self): def getfunc(): From c53ffd7841a0dd63c6e09244131be0aaeab6e028 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 5 Nov 2021 14:02:45 +0100 Subject: [PATCH 09/12] Add single-argument test --- Lib/test/test_sqlite3/test_userfunctions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_sqlite3/test_userfunctions.py b/Lib/test/test_sqlite3/test_userfunctions.py index b4144bb68a0283..d906f2b8ed1c7a 100644 --- a/Lib/test/test_sqlite3/test_userfunctions.py +++ b/Lib/test/test_sqlite3/test_userfunctions.py @@ -228,6 +228,7 @@ def test_func_too_many_args(self): category = sqlite.SQLITE_LIMIT_FUNCTION_ARG msg = "too many arguments on function" with cx_limit(self.con, category=category, limit=1): + self.con.execute("select round(1.1)"); with self.assertRaisesRegex(sqlite.OperationalError, msg): self.con.execute("select max(1, 2)"); From 6197b3a8ab5e9834ca0efb86d0b1beddc4d1ab7f Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 5 Nov 2021 14:07:07 +0100 Subject: [PATCH 10/12] Simplify sql statements in test_execute_too_many_params --- Lib/test/test_sqlite3/test_dbapi.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 7831ad50d9907a..d6f9dd1b0cf718 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -669,10 +669,9 @@ def test_execute_too_many_params(self): category = sqlite.SQLITE_LIMIT_VARIABLE_NUMBER msg = "too many SQL variables" with cx_limit(self.cx, category=category, limit=1): - self.cu.execute("select * from test where name=?", ("a",)) + self.cu.execute("select round(?)", (1.1,)) with self.assertRaisesRegex(sqlite.OperationalError, msg): - self.cu.execute("select * from test where name=? and income=?", - ("a", 1)) + self.cu.execute("select max(?, ?)", (1, 2)) def test_execute_dict_mapping(self): self.cu.execute("insert into test(name) values ('foo')") From 032bfd5a9941d1672f830d5c9580bda226ab0895 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 5 Nov 2021 14:18:05 +0100 Subject: [PATCH 11/12] Use abs iso. round build-in round is missing if SQLITE_OMIT_FLOATING_POINT is defined --- Lib/test/test_sqlite3/test_dbapi.py | 2 +- Lib/test/test_sqlite3/test_userfunctions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index d6f9dd1b0cf718..935d6c383bc33d 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -669,7 +669,7 @@ def test_execute_too_many_params(self): category = sqlite.SQLITE_LIMIT_VARIABLE_NUMBER msg = "too many SQL variables" with cx_limit(self.cx, category=category, limit=1): - self.cu.execute("select round(?)", (1.1,)) + self.cu.execute("select abs(?)", (-1,)) with self.assertRaisesRegex(sqlite.OperationalError, msg): self.cu.execute("select max(?, ?)", (1, 2)) diff --git a/Lib/test/test_sqlite3/test_userfunctions.py b/Lib/test/test_sqlite3/test_userfunctions.py index d906f2b8ed1c7a..62a11a5431b7b8 100644 --- a/Lib/test/test_sqlite3/test_userfunctions.py +++ b/Lib/test/test_sqlite3/test_userfunctions.py @@ -228,7 +228,7 @@ def test_func_too_many_args(self): category = sqlite.SQLITE_LIMIT_FUNCTION_ARG msg = "too many arguments on function" with cx_limit(self.con, category=category, limit=1): - self.con.execute("select round(1.1)"); + self.con.execute("select abs(-1)"); with self.assertRaisesRegex(sqlite.OperationalError, msg): self.con.execute("select max(1, 2)"); From c257c65bccba60ca98f3c43f0940760567529463 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 5 Nov 2021 14:36:13 +0100 Subject: [PATCH 12/12] Make sure we test against correct limit in test_execute_too_many_params --- Lib/test/test_sqlite3/test_dbapi.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 935d6c383bc33d..6628eee975d355 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -669,9 +669,10 @@ def test_execute_too_many_params(self): category = sqlite.SQLITE_LIMIT_VARIABLE_NUMBER msg = "too many SQL variables" with cx_limit(self.cx, category=category, limit=1): - self.cu.execute("select abs(?)", (-1,)) + self.cu.execute("select * from test where id=?", (1,)) with self.assertRaisesRegex(sqlite.OperationalError, msg): - self.cu.execute("select max(?, ?)", (1, 2)) + self.cu.execute("select * from test where id!=? and id!=?", + (1, 2)) def test_execute_dict_mapping(self): self.cu.execute("insert into test(name) values ('foo')")