From 3a2b2aabb85fff3f143398431ec7658a3db8186f Mon Sep 17 00:00:00 2001 From: Matthias Fetzer Date: Mon, 15 May 2023 20:15:39 +0200 Subject: [PATCH 1/5] Allow to turn off column store --- src/crate/client/sqlalchemy/compiler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/crate/client/sqlalchemy/compiler.py b/src/crate/client/sqlalchemy/compiler.py index 7e6dad7d..69493abc 100644 --- a/src/crate/client/sqlalchemy/compiler.py +++ b/src/crate/client/sqlalchemy/compiler.py @@ -128,6 +128,9 @@ def get_column_specification(self, column, **kwargs): colspec += " INDEX OFF" + if column.dialect_options['crate'].get('columnstore') is False: + colspec += " STORAGE WITH (columnstore = false)" + return colspec def visit_computed_column(self, generated): From b70f90c2dbed25db67f7114ad4c6619e651edcfb Mon Sep 17 00:00:00 2001 From: Matthias Fetzer Date: Mon, 29 May 2023 11:08:26 +0200 Subject: [PATCH 2/5] Columnstore: Adding tests and error handling for non TEXT columns --- src/crate/client/sqlalchemy/compiler.py | 6 +++ .../sqlalchemy/tests/create_table_test.py | 37 ++++++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/crate/client/sqlalchemy/compiler.py b/src/crate/client/sqlalchemy/compiler.py index 69493abc..efa88d17 100644 --- a/src/crate/client/sqlalchemy/compiler.py +++ b/src/crate/client/sqlalchemy/compiler.py @@ -25,6 +25,7 @@ import sqlalchemy as sa from sqlalchemy.dialects.postgresql.base import PGCompiler from sqlalchemy.sql import compiler +from sqlalchemy.types import String from .types import MutableDict, _Craty, Geopoint, Geoshape from .sa_version import SA_VERSION, SA_1_4 @@ -129,6 +130,11 @@ def get_column_specification(self, column, **kwargs): colspec += " INDEX OFF" if column.dialect_options['crate'].get('columnstore') is False: + if not isinstance(column.type, (String, )): + raise sa.exc.CompileError( + "Controlling the columnstore is only allowed for STRING columns" + ) + colspec += " STORAGE WITH (columnstore = false)" return colspec diff --git a/src/crate/client/sqlalchemy/tests/create_table_test.py b/src/crate/client/sqlalchemy/tests/create_table_test.py index 7eca2628..91f54bca 100644 --- a/src/crate/client/sqlalchemy/tests/create_table_test.py +++ b/src/crate/client/sqlalchemy/tests/create_table_test.py @@ -20,6 +20,7 @@ # software solely pursuant to the terms of the relevant commercial agreement. import sqlalchemy as sa + try: from sqlalchemy.orm import declarative_base except ImportError: @@ -31,7 +32,6 @@ from unittest import TestCase from unittest.mock import patch, MagicMock - fake_cursor = MagicMock(name='fake_cursor') FakeCursor = MagicMock(name='FakeCursor', spec=Cursor) FakeCursor.return_value = fake_cursor @@ -77,6 +77,7 @@ class DummyTable(self.Base): __tablename__ = 'dummy' pk = sa.Column(sa.String, primary_key=True) obj_col = sa.Column(Object) + self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE dummy (\n\tpk STRING NOT NULL, \n\tobj_col OBJECT, ' @@ -91,6 +92,7 @@ class DummyTable(self.Base): } pk = sa.Column(sa.String, primary_key=True) p = sa.Column(sa.String) + self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE t (\n\t' @@ -105,6 +107,7 @@ class DummyTable(self.Base): __tablename__ = 't' ts = sa.Column(sa.BigInteger, primary_key=True) p = sa.Column(sa.BigInteger, sa.Computed("date_trunc('day', ts)")) + self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE t (\n\t' @@ -119,6 +122,7 @@ class DummyTable(self.Base): __tablename__ = 't' ts = sa.Column(sa.BigInteger, primary_key=True) p = sa.Column(sa.BigInteger, sa.Computed("date_trunc('day', ts)", persisted=False)) + with self.assertRaises(sa.exc.CompileError): self.Base.metadata.create_all(bind=self.engine) @@ -131,6 +135,7 @@ class DummyTable(self.Base): } pk = sa.Column(sa.String, primary_key=True) p = sa.Column(sa.String) + self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE t (\n\t' @@ -166,6 +171,7 @@ class DummyTable(self.Base): } pk = sa.Column(sa.String, primary_key=True) p = sa.Column(sa.String, primary_key=True) + self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE t (\n\t' @@ -207,6 +213,7 @@ def test_column_pk_nullable(self): class DummyTable(self.Base): __tablename__ = 't' pk = sa.Column(sa.String, primary_key=True, nullable=True) + with self.assertRaises(sa.exc.CompileError): self.Base.metadata.create_all(bind=self.engine) @@ -230,5 +237,33 @@ class DummyTable(self.Base): __tablename__ = 't' pk = sa.Column(sa.String, primary_key=True) a = sa.Column(Geopoint, crate_index=False) + + with self.assertRaises(sa.exc.CompileError): + self.Base.metadata.create_all(bind=self.engine) + + def test_text_column_without_columnstore(self): + class DummyTable(self.Base): + __tablename__ = 't' + pk = sa.Column(sa.String, primary_key=True) + a = sa.Column(sa.String, crate_columnstore=False) + b = sa.Column(sa.String, crate_columnstore=True) + c = sa.Column(sa.String) + + self.Base.metadata.create_all(bind=self.engine) + + fake_cursor.execute.assert_called_with( + ('\nCREATE TABLE t (\n\t' + 'pk STRING NOT NULL, \n\t' + 'a STRING STORAGE WITH (columnstore = false), \n\t' + 'b STRING, \n\t' + 'c STRING, \n\t' + 'PRIMARY KEY (pk)\n)\n\n'), ()) + + def test_non_text_column_without_columnstore(self): + class DummyTable(self.Base): + __tablename__ = 't' + pk = sa.Column(sa.String, primary_key=True) + a = sa.Column(sa.Integer, crate_columnstore=False) + with self.assertRaises(sa.exc.CompileError): self.Base.metadata.create_all(bind=self.engine) From da8cae3e2ff236ab98f655b49d4f07d9381504c1 Mon Sep 17 00:00:00 2001 From: Matthias Fetzer Date: Mon, 29 May 2023 11:12:45 +0200 Subject: [PATCH 3/5] Columnstore: Adding docs for controlling the columnstore --- docs/sqlalchemy.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/sqlalchemy.rst b/docs/sqlalchemy.rst index 00bf5d00..b16f2abf 100644 --- a/docs/sqlalchemy.rst +++ b/docs/sqlalchemy.rst @@ -205,6 +205,7 @@ system `: ... more_details = sa.Column(types.ObjectArray) ... name_ft = sa.Column(sa.String) ... quote_ft = sa.Column(sa.String) + ... even_more_details = sa.Column(sa.String, crate_columnstore=False) ... ... __mapper_args__ = { ... 'exclude_properties': ['name_ft', 'quote_ft'] @@ -220,6 +221,7 @@ In this example, we: - Use standard SQLAlchemy types for the ``id``, ``name``, and ``quote`` columns - Use ``nullable=False`` to define a ``NOT NULL`` constraint - Disable indexing of the ``name`` column using ``crate_index=False`` +- Disable the columnstore of the ``even_more_details`` column using ``crate_columnstore=False`` - Define a computed column ``name_normalized`` (based on ``name``) that translates into a generated column - Use the `Object`_ extension type for the ``details`` column From b839e114420bdbd424564030bd133df2f1952672 Mon Sep 17 00:00:00 2001 From: Matthias Fetzer Date: Mon, 29 May 2023 11:38:21 +0200 Subject: [PATCH 4/5] Columnstore: Revert (auto) reformatting --- src/crate/client/sqlalchemy/tests/create_table_test.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/crate/client/sqlalchemy/tests/create_table_test.py b/src/crate/client/sqlalchemy/tests/create_table_test.py index 91f54bca..f876655d 100644 --- a/src/crate/client/sqlalchemy/tests/create_table_test.py +++ b/src/crate/client/sqlalchemy/tests/create_table_test.py @@ -20,7 +20,6 @@ # software solely pursuant to the terms of the relevant commercial agreement. import sqlalchemy as sa - try: from sqlalchemy.orm import declarative_base except ImportError: @@ -32,6 +31,7 @@ from unittest import TestCase from unittest.mock import patch, MagicMock + fake_cursor = MagicMock(name='fake_cursor') FakeCursor = MagicMock(name='FakeCursor', spec=Cursor) FakeCursor.return_value = fake_cursor @@ -77,7 +77,6 @@ class DummyTable(self.Base): __tablename__ = 'dummy' pk = sa.Column(sa.String, primary_key=True) obj_col = sa.Column(Object) - self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE dummy (\n\tpk STRING NOT NULL, \n\tobj_col OBJECT, ' @@ -92,7 +91,6 @@ class DummyTable(self.Base): } pk = sa.Column(sa.String, primary_key=True) p = sa.Column(sa.String) - self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE t (\n\t' @@ -107,7 +105,6 @@ class DummyTable(self.Base): __tablename__ = 't' ts = sa.Column(sa.BigInteger, primary_key=True) p = sa.Column(sa.BigInteger, sa.Computed("date_trunc('day', ts)")) - self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE t (\n\t' @@ -122,7 +119,6 @@ class DummyTable(self.Base): __tablename__ = 't' ts = sa.Column(sa.BigInteger, primary_key=True) p = sa.Column(sa.BigInteger, sa.Computed("date_trunc('day', ts)", persisted=False)) - with self.assertRaises(sa.exc.CompileError): self.Base.metadata.create_all(bind=self.engine) @@ -135,7 +131,6 @@ class DummyTable(self.Base): } pk = sa.Column(sa.String, primary_key=True) p = sa.Column(sa.String) - self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE t (\n\t' @@ -171,7 +166,6 @@ class DummyTable(self.Base): } pk = sa.Column(sa.String, primary_key=True) p = sa.Column(sa.String, primary_key=True) - self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE t (\n\t' @@ -213,7 +207,6 @@ def test_column_pk_nullable(self): class DummyTable(self.Base): __tablename__ = 't' pk = sa.Column(sa.String, primary_key=True, nullable=True) - with self.assertRaises(sa.exc.CompileError): self.Base.metadata.create_all(bind=self.engine) @@ -237,7 +230,6 @@ class DummyTable(self.Base): __tablename__ = 't' pk = sa.Column(sa.String, primary_key=True) a = sa.Column(Geopoint, crate_index=False) - with self.assertRaises(sa.exc.CompileError): self.Base.metadata.create_all(bind=self.engine) From 29804b96dd0b47ae1bcc66a4c10f66d7d2872421 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 8 Jun 2023 20:10:22 +0200 Subject: [PATCH 5/5] Columnstore: Add changelog item --- CHANGES.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 141b0667..0a7196f9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -5,6 +5,9 @@ Changes for crate Unreleased ========== +- SQLAlchemy DDL: Allow turning off column store using ``crate_columnstore=False``. + Thanks, @fetzerms. + 2023/04/18 0.31.1 =================