Skip to content

Commit a5e749c

Browse files
committed
Fixed SQL rendering of special DDL table options in CrateDDLCompiler.
Configuring special table options like `crate_"translog.durability"` was not possible, because they got rendered into SQL DDL in uppercase letters. That fails like: SQLParseException[Invalid property "TRANSLOG.DURABILITY" passed to [ALTER | CREATE] TABLE statement]
1 parent 7461258 commit a5e749c

File tree

4 files changed

+56
-11
lines changed

4 files changed

+56
-11
lines changed

CHANGES.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
`check_uniqueness_factory`
1212
- Added `table_kwargs` context manager to enable pandas/Dask to support
1313
CrateDB dialect table options.
14+
- Fixed SQL rendering of special DDL table options in `CrateDDLCompiler`.
15+
Before, configuring `crate_"translog.durability"` was not possible.
1416

1517
## 2024/06/13 0.37.0
1618
- Added support for CrateDB's [FLOAT_VECTOR] data type and its accompanying

src/sqlalchemy_cratedb/compiler.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -100,13 +100,13 @@ def crate_before_execute(conn, clauseelement, multiparams, params, *args, **kwar
100100
class CrateDDLCompiler(compiler.DDLCompiler):
101101

102102
__special_opts_tmpl = {
103-
'PARTITIONED_BY': ' PARTITIONED BY ({0})'
103+
'partitioned_by': ' PARTITIONED BY ({0})'
104104
}
105105
__clustered_opts_tmpl = {
106-
'NUMBER_OF_SHARDS': ' INTO {0} SHARDS',
107-
'CLUSTERED_BY': ' BY ({0})',
106+
'number_of_shards': ' INTO {0} SHARDS',
107+
'clustered_by': ' BY ({0})',
108108
}
109-
__clustered_opt_tmpl = ' CLUSTERED{CLUSTERED_BY}{NUMBER_OF_SHARDS}'
109+
__clustered_opt_tmpl = ' CLUSTERED{clustered_by}{number_of_shards}'
110110

111111
def get_column_specification(self, column, **kwargs):
112112
colspec = self.preparer.format_column(column) + " " + \
@@ -162,7 +162,7 @@ def post_create_table(self, table):
162162
table_opts = []
163163

164164
opts = dict(
165-
(k[len(self.dialect.name) + 1:].upper(), v)
165+
(k[len(self.dialect.name) + 1:], v)
166166
for k, v, in table.kwargs.items()
167167
if k.startswith('%s_' % self.dialect.name)
168168
)

tests/create_table_test.py

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ class DummyTable(self.Base):
154154
('\nCREATE TABLE t (\n\t'
155155
'pk STRING NOT NULL, \n\t'
156156
'PRIMARY KEY (pk)\n'
157-
') CLUSTERED INTO 3 SHARDS WITH (NUMBER_OF_REPLICAS = 2)\n\n'),
157+
') CLUSTERED INTO 3 SHARDS WITH (number_of_replicas = 2)\n\n'),
158158
())
159159

160160
def test_table_clustered_by_and_number_of_shards(self):
@@ -175,6 +175,21 @@ class DummyTable(self.Base):
175175
') CLUSTERED BY (p) INTO 3 SHARDS\n\n'),
176176
())
177177

178+
def test_table_translog_durability(self):
179+
class DummyTable(self.Base):
180+
__tablename__ = 't'
181+
__table_args__ = {
182+
'crate_"translog.durability"': "'async'",
183+
}
184+
pk = sa.Column(sa.String, primary_key=True)
185+
self.Base.metadata.create_all(bind=self.engine)
186+
fake_cursor.execute.assert_called_with(
187+
('\nCREATE TABLE t (\n\t'
188+
'pk STRING NOT NULL, \n\t'
189+
'PRIMARY KEY (pk)\n'
190+
""") WITH ("translog.durability" = 'async')\n\n"""),
191+
())
192+
178193
def test_column_object_array(self):
179194
class DummyTable(self.Base):
180195
__tablename__ = 't'

tests/test_support_pandas.py

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
import sys
33

44
import pytest
5-
import sqlalchemy as sa
65
from sqlalchemy.exc import ProgrammingError
76
from sqlalchemy.orm import sessionmaker
87

@@ -42,10 +41,7 @@ def test_table_kwargs_partitioned_by(cratedb_service):
4241
cratedb_service.database.refresh_table(TABLE_NAME)
4342

4443
# Inquire table cardinality.
45-
metadata = sa.MetaData()
46-
query = sa.select(sa.func.count()).select_from(sa.Table(TABLE_NAME, metadata))
47-
results = session.execute(query)
48-
count = results.scalar()
44+
count = cratedb_service.database.count_records(TABLE_NAME)
4945

5046
# Compare outcome.
5147
assert count == INSERT_RECORDS
@@ -55,6 +51,38 @@ def test_table_kwargs_partitioned_by(cratedb_service):
5551
assert 'PARTITIONED BY ("time")' in ddl[0][0]
5652

5753

54+
@pytest.mark.skipif(sys.version_info < (3, 8), reason="Feature not supported on Python 3.7 and earlier")
55+
@pytest.mark.skipif(SA_VERSION < SA_2_0, reason="Feature not supported on SQLAlchemy 1.4 and earlier")
56+
def test_table_kwargs_translog_durability(cratedb_service):
57+
"""
58+
Validate adding CrateDB dialect table option `translog.durability` at runtime.
59+
"""
60+
61+
engine = cratedb_service.database.engine
62+
63+
# Insert records from pandas dataframe.
64+
with table_kwargs(**{'crate_"translog.durability"': "'async'"}):
65+
df.to_sql(
66+
TABLE_NAME,
67+
engine,
68+
if_exists="replace",
69+
index=False,
70+
)
71+
72+
# Synchronize writes.
73+
cratedb_service.database.refresh_table(TABLE_NAME)
74+
75+
# Inquire table cardinality.
76+
count = cratedb_service.database.count_records(TABLE_NAME)
77+
78+
# Compare outcome.
79+
assert count == INSERT_RECORDS
80+
81+
# Validate SQL DDL.
82+
ddl = cratedb_service.database.run_sql(f"SHOW CREATE TABLE {TABLE_NAME}")
83+
assert """"translog.durability" = 'ASYNC'""" in ddl[0][0]
84+
85+
5886
@pytest.mark.skipif(sys.version_info < (3, 8), reason="Feature not supported on Python 3.7 and earlier")
5987
@pytest.mark.skipif(SA_VERSION < SA_2_0, reason="Feature not supported on SQLAlchemy 1.4 and earlier")
6088
def test_table_kwargs_unknown(cratedb_service):

0 commit comments

Comments
 (0)