From fac7355f69198a9263ba9f09de3b4fda2a2ed68e Mon Sep 17 00:00:00 2001 From: Romanas Sonkinas Date: Thu, 27 Jul 2023 15:28:04 +0100 Subject: [PATCH 01/19] Added automatic conversion of UUIDs to strings. --- .gitignore | 1 + CHANGES.txt | 1 + src/crate/client/http.py | 4 +++- src/crate/client/test_http.py | 15 +++++++++++++++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3b32ddeb..be2a312f 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ htmlcov/ out/ parts/ tmp/ +env/ diff --git a/CHANGES.txt b/CHANGES.txt index d04a31a2..19b3db02 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -5,6 +5,7 @@ Changes for crate Unreleased ========== +- Properly handle Python-native UUID types in SQL parameters 2023/07/17 0.33.0 ================= diff --git a/src/crate/client/http.py b/src/crate/client/http.py index d4522612..76df121c 100644 --- a/src/crate/client/http.py +++ b/src/crate/client/http.py @@ -35,6 +35,8 @@ from time import time from datetime import datetime, date, timezone from decimal import Decimal +from uuid import UUID + from urllib3 import connection_from_url from urllib3.connection import HTTPConnection from urllib3.exceptions import ( @@ -86,7 +88,7 @@ class CrateJsonEncoder(json.JSONEncoder): epoch_naive = datetime(1970, 1, 1) def default(self, o): - if isinstance(o, Decimal): + if isinstance(o, (Decimal, UUID)): return str(o) if isinstance(o, datetime): if o.tzinfo is not None: diff --git a/src/crate/client/test_http.py b/src/crate/client/test_http.py index 0f7afa35..de287834 100644 --- a/src/crate/client/test_http.py +++ b/src/crate/client/test_http.py @@ -35,9 +35,12 @@ from threading import Thread, Event from decimal import Decimal import datetime as dt + import urllib3.exceptions from base64 import b64decode from urllib.parse import urlparse, parse_qs + +import uuid from setuptools.ssl_support import find_ca_bundle from .http import Client, CrateJsonEncoder, _get_socket_opts, _remove_certs_for_non_https @@ -287,6 +290,18 @@ def test_socket_options_contain_keepalive(self): ) client.close() + @patch(REQUEST, autospec=True) + def test_uuid_serialization(self, request): + client = Client(servers="localhost:4200") + request.return_value = fake_response(200) + + uid = uuid.uuid4() + client.sql('insert into my_table (str_col) values (?)', (uid,)) + + data = json.loads(request.call_args[1]['data']) + self.assertEqual(data['args'], [str(uid)]) + client.close() + @patch(REQUEST, fail_sometimes) class ThreadSafeHttpClientTest(TestCase): From 9919250caa470dfdafb4afd914435824c3fca77c Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sun, 30 Jul 2023 16:47:10 +0200 Subject: [PATCH 02/19] Chore: Update documentation content - Add missing credits to change log - Update to crate-docs 2.1.1 - Improve wording - Update intersphinx reference to SQLAlchemy 2.x --- CHANGES.txt | 2 +- docs/build.json | 2 +- docs/by-example/sqlalchemy/advanced-querying.rst | 5 +++-- docs/conf.py | 2 +- docs/sqlalchemy.rst | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 19b3db02..429e9c4f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -28,7 +28,7 @@ Unreleased - Allow handling datetime values tagged with time zone info when inserting or updating. -- SQLAlchemy: Fix SQL statement caching for CrateDB's ``OBJECT`` type. +- SQLAlchemy: Fix SQL statement caching for CrateDB's ``OBJECT`` type. Thanks, @faymarie. - SQLAlchemy: Refactor ``OBJECT`` type to use SQLAlchemy's JSON type infrastructure. diff --git a/docs/build.json b/docs/build.json index 49cbd2be..5647caf4 100644 --- a/docs/build.json +++ b/docs/build.json @@ -1,5 +1,5 @@ { "schemaVersion": 1, "label": "docs build", - "message": "2.1.0" + "message": "2.1.1" } diff --git a/docs/by-example/sqlalchemy/advanced-querying.rst b/docs/by-example/sqlalchemy/advanced-querying.rst index 9108bb49..7c4d6781 100644 --- a/docs/by-example/sqlalchemy/advanced-querying.rst +++ b/docs/by-example/sqlalchemy/advanced-querying.rst @@ -5,8 +5,9 @@ SQLAlchemy: Advanced querying ============================= This section of the documentation demonstrates running queries using a fulltext -index with analyzer, queries using counting and aggregations, and support for -the ``INSERT...FROM SELECT`` construct, all using the CrateDB SQLAlchemy dialect. +index with an analyzer, queries using counting and aggregations, and support for +the ``INSERT...FROM SELECT`` and ``INSERT...RETURNING`` constructs, all using the +CrateDB SQLAlchemy dialect. .. rubric:: Table of Contents diff --git a/docs/conf.py b/docs/conf.py index 59cc622f..3804b4b6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,7 +11,7 @@ intersphinx_mapping.update({ 'py': ('https://docs.python.org/3/', None), - 'sa': ('https://docs.sqlalchemy.org/en/14/', None), + 'sa': ('https://docs.sqlalchemy.org/en/20/', None), 'urllib3': ('https://urllib3.readthedocs.io/en/1.26.13/', None), 'dask': ('https://docs.dask.org/en/stable/', None), 'pandas': ('https://pandas.pydata.org/docs/', None), diff --git a/docs/sqlalchemy.rst b/docs/sqlalchemy.rst index c3d0c7af..c31396ab 100644 --- a/docs/sqlalchemy.rst +++ b/docs/sqlalchemy.rst @@ -28,7 +28,7 @@ The CrateDB SQLAlchemy dialect is validated to work with SQLAlchemy versions .. SEEALSO:: For general help using SQLAlchemy, consult the :ref:`SQLAlchemy tutorial - ` or the `SQLAlchemy library`_. + ` or the `SQLAlchemy library`_. Supplementary information about the CrateDB SQLAlchemy dialect can be found in the :ref:`data types appendix `. From d2d44a5c6f9f882402d6cb63d4e5c4895cf06050 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sun, 30 Jul 2023 16:50:08 +0200 Subject: [PATCH 03/19] Chore: Satisfy linter admonition flake8 E721 The admonition was: E721 do not compare types, for exact checks use `is` / `is not`, for instance checks use `isinstance()`. Apparently, it has been added to flake8 6.1.0, released on July 29, 2023 --- src/crate/testing/layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crate/testing/layer.py b/src/crate/testing/layer.py index 5fd6d8fd..ef8bfe2b 100644 --- a/src/crate/testing/layer.py +++ b/src/crate/testing/layer.py @@ -248,7 +248,7 @@ def __init__(self, transport_port or '4300-4399', settings) # ES 5 cannot parse 'True'/'False' as booleans so convert to lowercase - start_cmd = (crate_exec, ) + tuple(["-C%s=%s" % ((key, str(value).lower()) if type(value) == bool else (key, value)) + start_cmd = (crate_exec, ) + tuple(["-C%s=%s" % ((key, str(value).lower()) if isinstance(value, bool) else (key, value)) for key, value in settings.items()]) self._wd = wd = os.path.join(CrateLayer.tmpdir, 'crate_layer', name) From d01167393390864b978ee0e66b076720317de2ec Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Tue, 5 Sep 2023 01:31:56 +0200 Subject: [PATCH 04/19] SQLAlchemy: Fix handling URL parameters `timeout` and `pool_size` --- CHANGES.txt | 2 + .../by-example/sqlalchemy/getting-started.rst | 41 +++++++++++++++++-- src/crate/client/http.py | 5 ++- .../sqlalchemy/tests/connection_test.py | 16 ++++++++ 4 files changed, 59 insertions(+), 5 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 429e9c4f..f4ebd26f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,6 +6,8 @@ Unreleased ========== - Properly handle Python-native UUID types in SQL parameters +- SQLAlchemy: Fix handling URL parameters ``timeout`` and ``pool_size`` + 2023/07/17 0.33.0 ================= diff --git a/docs/by-example/sqlalchemy/getting-started.rst b/docs/by-example/sqlalchemy/getting-started.rst index c64964dc..33e8f75d 100644 --- a/docs/by-example/sqlalchemy/getting-started.rst +++ b/docs/by-example/sqlalchemy/getting-started.rst @@ -46,8 +46,8 @@ Create an SQLAlchemy :doc:`Session `: >>> Base = declarative_base() -Connection string -================= +Connect +======= In SQLAlchemy, a connection is established using the ``create_engine`` function. This function takes a connection string, actually an `URL`_, that varies from @@ -65,7 +65,9 @@ to a different server the following syntax can be used: >>> sa.create_engine('crate://otherserver:4200') Engine(crate://otherserver:4200) -Since CrateDB is a clustered database running on multiple servers, it is +Multiple Hosts +-------------- +Because CrateDB is a clustered database running on multiple servers, it is recommended to connect to all of them. This enables the DB-API layer to use round-robin to distribute the load and skip a server if it becomes unavailable. In order to make the driver aware of multiple servers, use @@ -76,6 +78,8 @@ the ``connect_args`` parameter like so: ... }) Engine(crate://) +TLS Options +----------- As defined in :ref:`https_connection`, the client validates SSL server certificates by default. To configure this further, use e.g. the ``ca_cert`` attribute within the ``connect_args``, like: @@ -96,6 +100,37 @@ In order to disable SSL verification, use ``verify_ssl_cert = False``, like: ... 'verify_ssl_cert': False, ... }) +Timeout Options +--------------- +In order to configure TCP timeout options, use the ``timeout`` parameter within +``connect_args``, + + >>> timeout_engine = sa.create_engine('crate://localhost/', connect_args={'timeout': 42.42}) + >>> timeout_engine.raw_connection().driver_connection.client._pool_kw["timeout"] + 42.42 + +or use the ``timeout`` URL parameter within the database connection URL. + + >>> timeout_engine = sa.create_engine('crate://localhost/?timeout=42.42') + >>> timeout_engine.raw_connection().driver_connection.client._pool_kw["timeout"] + 42.42 + +Pool Size +--------- + +In order to configure the database connection pool size, use the ``pool_size`` +parameter within ``connect_args``, + + >>> timeout_engine = sa.create_engine('crate://localhost/', connect_args={'pool_size': 20}) + >>> timeout_engine.raw_connection().driver_connection.client._pool_kw["maxsize"] + 20 + +or use the ``pool_size`` URL parameter within the database connection URL. + + >>> timeout_engine = sa.create_engine('crate://localhost/?pool_size=20') + >>> timeout_engine.raw_connection().driver_connection.client._pool_kw["maxsize"] + 20 + Basic DDL operations ==================== diff --git a/src/crate/client/http.py b/src/crate/client/http.py index 76df121c..ca45bae8 100644 --- a/src/crate/client/http.py +++ b/src/crate/client/http.py @@ -255,10 +255,11 @@ def _pool_kw_args(verify_ssl_cert, ca_cert, client_cert, client_key, 'cert_reqs': ssl.CERT_REQUIRED if verify_ssl_cert else ssl.CERT_NONE, 'cert_file': client_cert, 'key_file': client_key, - 'timeout': timeout, } + if timeout is not None: + kw['timeout'] = float(timeout) if pool_size is not None: - kw['maxsize'] = pool_size + kw['maxsize'] = int(pool_size) return kw diff --git a/src/crate/client/sqlalchemy/tests/connection_test.py b/src/crate/client/sqlalchemy/tests/connection_test.py index 4e22489b..f1a560e9 100644 --- a/src/crate/client/sqlalchemy/tests/connection_test.py +++ b/src/crate/client/sqlalchemy/tests/connection_test.py @@ -83,6 +83,22 @@ def test_connection_server_uri_https_with_credentials(self): conn.close() engine.dispose() + def test_connection_server_uri_parameter_timeout(self): + engine = sa.create_engine( + "crate://otherhost:19201/?timeout=42.42") + conn = engine.raw_connection() + self.assertEqual(conn.driver_connection.client._pool_kw["timeout"], 42.42) + conn.close() + engine.dispose() + + def test_connection_server_uri_parameter_pool_size(self): + engine = sa.create_engine( + "crate://otherhost:19201/?pool_size=20") + conn = engine.raw_connection() + self.assertEqual(conn.driver_connection.client._pool_kw["maxsize"], 20) + conn.close() + engine.dispose() + def test_connection_multiple_server_http(self): engine = sa.create_engine( "crate://", connect_args={ From 2e9ccfd9c32514c4624c9072aad7ecd55ff653eb Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Tue, 5 Sep 2023 16:54:21 +0200 Subject: [PATCH 05/19] Chore: Mitigate deprecation warnings about `unittest.makeSuite` --- src/crate/client/sqlalchemy/tests/__init__.py | 5 +++- src/crate/client/tests.py | 24 ++++++++++--------- src/crate/testing/tests.py | 7 ++++-- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/crate/client/sqlalchemy/tests/__init__.py b/src/crate/client/sqlalchemy/tests/__init__.py index 6102cb5a..606c545c 100644 --- a/src/crate/client/sqlalchemy/tests/__init__.py +++ b/src/crate/client/sqlalchemy/tests/__init__.py @@ -10,7 +10,7 @@ monkeypatch_amend_select_sa14() monkeypatch_add_connectionfairy_driver_connection() -from unittest import TestSuite, makeSuite +from unittest import TestLoader, TestSuite from .connection_test import SqlAlchemyConnectionTest from .dict_test import SqlAlchemyDictTypeTest from .datetime_test import SqlAlchemyDateAndDateTimeTest @@ -27,6 +27,9 @@ from .query_caching import SqlAlchemyQueryCompilationCaching +makeSuite = TestLoader().loadTestsFromTestCase + + def test_suite_unit(): tests = TestSuite() tests.addTest(makeSuite(SqlAlchemyConnectionTest)) diff --git a/src/crate/client/tests.py b/src/crate/client/tests.py index 4ce9c950..026fb56f 100644 --- a/src/crate/client/tests.py +++ b/src/crate/client/tests.py @@ -59,6 +59,8 @@ from .sqlalchemy.tests import test_suite_unit as sqlalchemy_test_suite_unit from .sqlalchemy.tests import test_suite_integration as sqlalchemy_test_suite_integration +makeSuite = unittest.TestLoader().loadTestsFromTestCase + log = logging.getLogger('crate.testing.layer') ch = logging.StreamHandler() ch.setLevel(logging.ERROR) @@ -336,17 +338,17 @@ def test_suite(): flags = (doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS) # Unit tests. - suite.addTest(unittest.makeSuite(CursorTest)) - suite.addTest(unittest.makeSuite(HttpClientTest)) - suite.addTest(unittest.makeSuite(KeepAliveClientTest)) - suite.addTest(unittest.makeSuite(ThreadSafeHttpClientTest)) - suite.addTest(unittest.makeSuite(ParamsTest)) - suite.addTest(unittest.makeSuite(ConnectionTest)) - suite.addTest(unittest.makeSuite(RetryOnTimeoutServerTest)) - suite.addTest(unittest.makeSuite(RequestsCaBundleTest)) - suite.addTest(unittest.makeSuite(TestUsernameSentAsHeader)) - suite.addTest(unittest.makeSuite(TestCrateJsonEncoder)) - suite.addTest(unittest.makeSuite(TestDefaultSchemaHeader)) + suite.addTest(makeSuite(CursorTest)) + suite.addTest(makeSuite(HttpClientTest)) + suite.addTest(makeSuite(KeepAliveClientTest)) + suite.addTest(makeSuite(ThreadSafeHttpClientTest)) + suite.addTest(makeSuite(ParamsTest)) + suite.addTest(makeSuite(ConnectionTest)) + suite.addTest(makeSuite(RetryOnTimeoutServerTest)) + suite.addTest(makeSuite(RequestsCaBundleTest)) + suite.addTest(makeSuite(TestUsernameSentAsHeader)) + suite.addTest(makeSuite(TestCrateJsonEncoder)) + suite.addTest(makeSuite(TestDefaultSchemaHeader)) suite.addTest(sqlalchemy_test_suite_unit()) suite.addTest(doctest.DocTestSuite('crate.client.connection')) suite.addTest(doctest.DocTestSuite('crate.client.http')) diff --git a/src/crate/testing/tests.py b/src/crate/testing/tests.py index fb08f7ab..2a6e06d0 100644 --- a/src/crate/testing/tests.py +++ b/src/crate/testing/tests.py @@ -24,8 +24,11 @@ from .test_layer import LayerUtilsTest, LayerTest +makeSuite = unittest.TestLoader().loadTestsFromTestCase + + def test_suite(): suite = unittest.TestSuite() - suite.addTest(unittest.makeSuite(LayerUtilsTest)) - suite.addTest(unittest.makeSuite(LayerTest)) + suite.addTest(makeSuite(LayerUtilsTest)) + suite.addTest(makeSuite(LayerTest)) return suite From c574896d667f1194cec67d46854caf59e03bb0bf Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Wed, 6 Sep 2023 08:27:34 +0200 Subject: [PATCH 06/19] Documentation: Add link to "by-example" section to primary navigation --- docs/index.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 15b43df5..fea76cad 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -160,6 +160,11 @@ The DB API driver and the SQLAlchemy dialect support :ref:`CrateDB's data types please consult the :ref:`data-types` and :ref:`SQLAlchemy extension types ` documentation pages. +.. toctree:: + :maxdepth: 2 + + data-types + Examples ======== @@ -173,6 +178,11 @@ Examples connect to CrateDB using `pandas`_, and how to load and export data. - The `Apache Superset`_ and `FIWARE QuantumLeap data historian`_ projects. +.. toctree:: + :maxdepth: 2 + + by-example/index + ******************* Project information From 524936e5b9f963e3375d388c3542a1531d6189f3 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Wed, 6 Sep 2023 08:56:21 +0200 Subject: [PATCH 07/19] Documentation: Improve text layout/flow --- docs/by-example/index.rst | 9 ++------- docs/index.rst | 4 +--- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/docs/by-example/index.rst b/docs/by-example/index.rst index 589beb99..39c503e4 100644 --- a/docs/by-example/index.rst +++ b/docs/by-example/index.rst @@ -4,13 +4,8 @@ By example ########## - -***** -About -***** - -This part of the documentation contains examples how to use the CrateDB Python -client. +This part of the documentation enumerates different kinds of examples how to +use the CrateDB Python client. DB API, HTTP, and BLOB interfaces diff --git a/docs/index.rst b/docs/index.rst index fea76cad..c166b513 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -36,11 +36,9 @@ Documentation For general help about the Python Database API, or SQLAlchemy, please consult `PEP 249`_, the `SQLAlchemy tutorial`_, and the general `SQLAlchemy documentation`_. - For more detailed information about how to install the client driver, how to connect to a CrateDB cluster, and how to run queries, consult the resources -referenced below. A dedicated section demonstrates how to use the :ref:`blob -storage capabilities ` of CrateDB. +referenced below. .. toctree:: :titlesonly: From c276159bb1d6dddbcf9d36f310aa9b8512fd3ec7 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 16 Feb 2023 15:21:23 +0100 Subject: [PATCH 08/19] Dependencies: Permit installation with urllib3 2.0 --- CHANGES.txt | 9 +++++++++ docs/by-example/https.rst | 15 +++++++++++++++ docs/connect.rst | 11 +++++++++++ setup.py | 2 +- src/crate/client/connection.py | 2 ++ src/crate/client/http.py | 24 +++++++++++++++++++++++- 6 files changed, 61 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index f4ebd26f..2ffe8765 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,15 @@ Unreleased - Properly handle Python-native UUID types in SQL parameters - SQLAlchemy: Fix handling URL parameters ``timeout`` and ``pool_size`` +- Permit installation with urllib3 v2, see also `urllib3 v2.0 roadmap`_ + and `urllib3 v2.0 migration guide`_. You can optionally retain support + for TLS 1.0 and TLS 1.1, but a few other outdated use-cases of X.509 + certificate details are immanent, like no longer accepting the long + deprecated ``commonName`` attribute. Instead, going forward, only the + ``subjectAltName`` attribute will be used. + +.. _urllib3 v2.0 migration guide: https://urllib3.readthedocs.io/en/latest/v2-migration-guide.html +.. _urllib3 v2.0 roadmap: https://urllib3.readthedocs.io/en/stable/v2-roadmap.html 2023/07/17 0.33.0 diff --git a/docs/by-example/https.rst b/docs/by-example/https.rst index cc6da50b..4bbd408e 100644 --- a/docs/by-example/https.rst +++ b/docs/by-example/https.rst @@ -110,3 +110,18 @@ The connection will also fail when providing an invalid CA certificate: Traceback (most recent call last): ... crate.client.exceptions.ConnectionError: Server not available, exception: HTTPSConnectionPool... + + +Relaxing minimum SSL version +============================ + +urrlib3 v2 dropped support for TLS 1.0 and TLS 1.1 by default, see `Modern security by default - +HTTPS requires TLS 1.2+`_. If you need to re-enable it, use the ``ssl_relax_minimum_version`` flag, +which will configure ``kwargs["ssl_minimum_version"] = ssl.TLSVersion.MINIMUM_SUPPORTED``. + + >>> client = HttpClient([crate_host], ssl_relax_minimum_version=True, verify_ssl_cert=False) + >>> client.server_infos(crate_host) + ('https://localhost:65534', 'test', '0.0.0') + + +.. _Modern security by default - HTTPS requires TLS 1.2+: https://urllib3.readthedocs.io/en/latest/v2-migration-guide.html#https-requires-tls-1-2 diff --git a/docs/connect.rst b/docs/connect.rst index 44c25b04..944fe263 100644 --- a/docs/connect.rst +++ b/docs/connect.rst @@ -139,6 +139,16 @@ Here, replace ```` with the path to the client certificate file, and verification. In such circumstances, you can combine the two methods above to do both at once. +Relaxing minimum SSL version +............................ + +urrlib3 v2 dropped support for TLS 1.0 and TLS 1.1 by default, see `Modern security by default - +HTTPS requires TLS 1.2+`_. If you need to re-enable it, use the ``ssl_relax_minimum_version`` flag, +which will configure ``kwargs["ssl_minimum_version"] = ssl.TLSVersion.MINIMUM_SUPPORTED``. + + >>> connection = client.connect(..., ssl_relax_minimum_version=True) + + Timeout ------- @@ -268,6 +278,7 @@ Once you're connected, you can :ref:`query CrateDB `. .. _client-side random load balancing: https://en.wikipedia.org/wiki/Load_balancing_(computing)#Client-side_random_load_balancing +.. _Modern security by default - HTTPS requires TLS 1.2+: https://urllib3.readthedocs.io/en/latest/v2-migration-guide.html#https-requires-tls-1-2 .. _Python Database API Specification v2.0: https://www.python.org/dev/peps/pep-0249/ .. _round-robin DNS: https://en.wikipedia.org/wiki/Round-robin_DNS .. _sample application: https://github.com/crate/crate-sample-apps/tree/main/python-flask diff --git a/setup.py b/setup.py index 5d4ee00b..fa28639d 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ def read(path): 'crate = crate.client.sqlalchemy:CrateDialect' ] }, - install_requires=['urllib3>=1.9,<2'], + install_requires=['urllib3<2.1'], extras_require=dict( sqlalchemy=['sqlalchemy>=1.0,<2.1', 'geojson>=2.5.0,<4', diff --git a/src/crate/client/connection.py b/src/crate/client/connection.py index db4ce473..03b5b444 100644 --- a/src/crate/client/connection.py +++ b/src/crate/client/connection.py @@ -38,6 +38,7 @@ def __init__(self, error_trace=False, cert_file=None, key_file=None, + ssl_relax_minimum_version=False, username=None, password=None, schema=None, @@ -138,6 +139,7 @@ def __init__(self, error_trace=error_trace, cert_file=cert_file, key_file=key_file, + ssl_relax_minimum_version=ssl_relax_minimum_version, username=username, password=password, schema=schema, diff --git a/src/crate/client/http.py b/src/crate/client/http.py index ca45bae8..0cce7bda 100644 --- a/src/crate/client/http.py +++ b/src/crate/client/http.py @@ -37,6 +37,7 @@ from decimal import Decimal from uuid import UUID +import urllib3 from urllib3 import connection_from_url from urllib3.connection import HTTPConnection from urllib3.exceptions import ( @@ -48,6 +49,8 @@ SSLError, ) from urllib3.util.retry import Retry + +from crate.client._pep440 import Version from crate.client.exceptions import ( ConnectionError, BlobLocationNotFoundException, @@ -274,6 +277,19 @@ def _remove_certs_for_non_https(server, kwargs): return kwargs +def _update_pool_kwargs_for_ssl_minimum_version(server, kwargs): + """ + On urllib3 v2, re-add support for TLS 1.0 and TLS 1.1. + + https://urllib3.readthedocs.io/en/latest/v2-migration-guide.html#https-requires-tls-1-2 + """ + if Version(urllib3.__version__) >= Version("2"): + from urllib3.util import parse_url + scheme, _, host, port, *_ = parse_url(server) + if scheme == "https": + kwargs["ssl_minimum_version"] = ssl.TLSVersion.MINIMUM_SUPPORTED + + def _create_sql_payload(stmt, args, bulk_args): if not isinstance(stmt, str): raise ValueError('stmt is not a string') @@ -304,7 +320,7 @@ def _get_socket_opts(keepalive=True, # always use TCP keepalive opts = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)] - # hasattr check because some of the options depend on system capabilities + # hasattr check because some options depend on system capabilities # see https://docs.python.org/3/library/socket.html#socket.SOMAXCONN if hasattr(socket, 'TCP_KEEPIDLE') and tcp_keepidle is not None: opts.append((socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, tcp_keepidle)) @@ -340,6 +356,7 @@ def __init__(self, error_trace=False, cert_file=None, key_file=None, + ssl_relax_minimum_version=False, username=None, password=None, schema=None, @@ -380,6 +397,7 @@ def __init__(self, 'socket_tcp_keepintvl': socket_tcp_keepintvl, 'socket_tcp_keepcnt': socket_tcp_keepcnt, }) + self.ssl_relax_minimum_version = ssl_relax_minimum_version self.backoff_factor = backoff_factor self.server_pool = {} self._update_server_pool(servers, **pool_kw) @@ -400,6 +418,10 @@ def close(self): def _create_server(self, server, **pool_kw): kwargs = _remove_certs_for_non_https(server, pool_kw) + # After updating to urllib3 v2, optionally retain support for TLS 1.0 and TLS 1.1, + # in order to support connectivity to older versions of CrateDB. + if self.ssl_relax_minimum_version: + _update_pool_kwargs_for_ssl_minimum_version(server, kwargs) self.server_pool[server] = Server(server, **kwargs) def _update_server_pool(self, servers, **pool_kw): From 279434c93f9eadd98093621dbf7eac375aaf3c68 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 28 Sep 2023 14:46:37 +0200 Subject: [PATCH 09/19] SQLAlchemy: Improve DDL compiler to ignore foreign key constraints --- CHANGES.txt | 1 + src/crate/client/sqlalchemy/compiler.py | 5 + src/crate/client/sqlalchemy/tests/__init__.py | 3 +- .../client/sqlalchemy/tests/compiler_test.py | 104 +++++++++++++++++- 4 files changed, 111 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 2ffe8765..d09a90ee 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -13,6 +13,7 @@ Unreleased certificate details are immanent, like no longer accepting the long deprecated ``commonName`` attribute. Instead, going forward, only the ``subjectAltName`` attribute will be used. +- SQLAlchemy: Improve DDL compiler to ignore foreign key constraints .. _urllib3 v2.0 migration guide: https://urllib3.readthedocs.io/en/latest/v2-migration-guide.html .. _urllib3 v2.0 roadmap: https://urllib3.readthedocs.io/en/stable/v2-roadmap.html diff --git a/src/crate/client/sqlalchemy/compiler.py b/src/crate/client/sqlalchemy/compiler.py index 3ae7a7cb..1995e050 100644 --- a/src/crate/client/sqlalchemy/compiler.py +++ b/src/crate/client/sqlalchemy/compiler.py @@ -178,6 +178,11 @@ def post_create_table(self, table): ', '.join(sorted(table_opts))) return special_options + def visit_foreign_key_constraint(self, constraint, **kw): + """ + CrateDB does not support foreign key constraints. + """ + return None class CrateTypeCompiler(compiler.GenericTypeCompiler): diff --git a/src/crate/client/sqlalchemy/tests/__init__.py b/src/crate/client/sqlalchemy/tests/__init__.py index 606c545c..d6d37493 100644 --- a/src/crate/client/sqlalchemy/tests/__init__.py +++ b/src/crate/client/sqlalchemy/tests/__init__.py @@ -14,7 +14,7 @@ from .connection_test import SqlAlchemyConnectionTest from .dict_test import SqlAlchemyDictTypeTest from .datetime_test import SqlAlchemyDateAndDateTimeTest -from .compiler_test import SqlAlchemyCompilerTest +from .compiler_test import SqlAlchemyCompilerTest, SqlAlchemyDDLCompilerTest from .update_test import SqlAlchemyUpdateTest from .match_test import SqlAlchemyMatchTest from .bulk_test import SqlAlchemyBulkTest @@ -36,6 +36,7 @@ def test_suite_unit(): tests.addTest(makeSuite(SqlAlchemyDictTypeTest)) tests.addTest(makeSuite(SqlAlchemyDateAndDateTimeTest)) tests.addTest(makeSuite(SqlAlchemyCompilerTest)) + tests.addTest(makeSuite(SqlAlchemyDDLCompilerTest)) tests.addTest(ParametrizedTestCase.parametrize(SqlAlchemyCompilerTest, param={"server_version_info": None})) tests.addTest(ParametrizedTestCase.parametrize(SqlAlchemyCompilerTest, param={"server_version_info": (4, 0, 12)})) tests.addTest(ParametrizedTestCase.parametrize(SqlAlchemyCompilerTest, param={"server_version_info": (4, 1, 10)})) diff --git a/src/crate/client/sqlalchemy/tests/compiler_test.py b/src/crate/client/sqlalchemy/tests/compiler_test.py index 5d5cc89e..fa24e1a6 100644 --- a/src/crate/client/sqlalchemy/tests/compiler_test.py +++ b/src/crate/client/sqlalchemy/tests/compiler_test.py @@ -19,8 +19,10 @@ # with Crate these terms will supersede the license and you may use the # software solely pursuant to the terms of the relevant commercial agreement. from textwrap import dedent -from unittest import mock, skipIf +from unittest import mock, skipIf, TestCase +from unittest.mock import MagicMock, patch +from crate.client.cursor import Cursor from crate.client.sqlalchemy.compiler import crate_before_execute import sqlalchemy as sa @@ -30,6 +32,8 @@ from crate.client.sqlalchemy.types import ObjectType from crate.client.test_util import ParametrizedTestCase +from crate.testing.settings import crate_host + class SqlAlchemyCompilerTest(ParametrizedTestCase): @@ -244,3 +248,101 @@ def test_insert_manyvalues(self): mock.call(mock.ANY, 'INSERT INTO mytable (name) VALUES (?), (?)', ('foo_2', 'foo_3'), None), mock.call(mock.ANY, 'INSERT INTO mytable (name) VALUES (?)', ('foo_4', ), None), ]) + + +FakeCursor = MagicMock(name='FakeCursor', spec=Cursor) + + +class CompilerTestCase(TestCase): + """ + A base class for providing mocking infrastructure to validate the DDL compiler. + """ + + def setUp(self): + self.engine = sa.create_engine(f"crate://{crate_host}") + self.metadata = sa.MetaData(schema="testdrive") + self.session = sa.orm.Session(bind=self.engine) + self.setup_mock() + + def setup_mock(self): + """ + Set up a fake cursor, in order to intercept query execution. + """ + + self.fake_cursor = MagicMock(name="fake_cursor") + FakeCursor.return_value = self.fake_cursor + + self.executed_statement = None + self.fake_cursor.execute = self.execute_wrapper + + def execute_wrapper(self, query, *args, **kwargs): + """ + Receive the SQL query expression, and store it. + """ + self.executed_statement = query + return self.fake_cursor + + +@patch('crate.client.connection.Cursor', FakeCursor) +class SqlAlchemyDDLCompilerTest(CompilerTestCase): + """ + Verify a few scenarios regarding the DDL compiler. + """ + + def test_ddl_with_foreign_keys(self): + """ + Verify the CrateDB dialect properly ignores foreign key constraints. + """ + + Base = sa.orm.declarative_base(metadata=self.metadata) + + class RootStore(Base): + """The main store.""" + + __tablename__ = "root" + + id = sa.Column(sa.Integer, primary_key=True) + name = sa.Column(sa.String) + + items = sa.orm.relationship( + "ItemStore", + back_populates="root", + passive_deletes=True, + ) + + class ItemStore(Base): + """The auxiliary store.""" + + __tablename__ = "item" + + id = sa.Column(sa.Integer, primary_key=True) + name = sa.Column(sa.String) + root_id = sa.Column( + sa.Integer, + sa.ForeignKey( + f"{RootStore.__tablename__}.id", + ondelete="CASCADE", + ), + ) + root = sa.orm.relationship(RootStore, back_populates="items") + + self.metadata.create_all(self.engine, tables=[RootStore.__table__], checkfirst=False) + self.assertEqual(self.executed_statement, dedent(""" + CREATE TABLE testdrive.root ( + \tid INT NOT NULL, + \tname STRING, + \tPRIMARY KEY (id) + ) + + """)) # noqa: W291 + + self.metadata.create_all(self.engine, tables=[ItemStore.__table__], checkfirst=False) + self.assertEqual(self.executed_statement, dedent(""" + CREATE TABLE testdrive.item ( + \tid INT NOT NULL, + \tname STRING, + \troot_id INT, + \tPRIMARY KEY (id) + ) + + """)) # noqa: W291, W293 From 9eaf38d60d607ecee8a52867a2d17df245bc0522 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 28 Sep 2023 15:26:58 +0200 Subject: [PATCH 10/19] SQLAlchemy: Improve DDL compiler to ignore unique key constraints --- CHANGES.txt | 3 +- src/crate/client/sqlalchemy/compiler.py | 7 +++++ .../client/sqlalchemy/tests/compiler_test.py | 31 ++++++++++++++++++- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index d09a90ee..4fbb10b9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -13,7 +13,8 @@ Unreleased certificate details are immanent, like no longer accepting the long deprecated ``commonName`` attribute. Instead, going forward, only the ``subjectAltName`` attribute will be used. -- SQLAlchemy: Improve DDL compiler to ignore foreign key constraints +- SQLAlchemy: Improve DDL compiler to ignore foreign key and uniqueness + constraints .. _urllib3 v2.0 migration guide: https://urllib3.readthedocs.io/en/latest/v2-migration-guide.html .. _urllib3 v2.0 roadmap: https://urllib3.readthedocs.io/en/stable/v2-roadmap.html diff --git a/src/crate/client/sqlalchemy/compiler.py b/src/crate/client/sqlalchemy/compiler.py index 1995e050..4906dcba 100644 --- a/src/crate/client/sqlalchemy/compiler.py +++ b/src/crate/client/sqlalchemy/compiler.py @@ -184,6 +184,13 @@ def visit_foreign_key_constraint(self, constraint, **kw): """ return None + def visit_unique_constraint(self, constraint, **kw): + """ + CrateDB does not support unique key constraints. + """ + return None + + class CrateTypeCompiler(compiler.GenericTypeCompiler): def visit_string(self, type_, **kw): diff --git a/src/crate/client/sqlalchemy/tests/compiler_test.py b/src/crate/client/sqlalchemy/tests/compiler_test.py index fa24e1a6..f9b4eef8 100644 --- a/src/crate/client/sqlalchemy/tests/compiler_test.py +++ b/src/crate/client/sqlalchemy/tests/compiler_test.py @@ -27,6 +27,10 @@ import sqlalchemy as sa from sqlalchemy.sql import text, Update +try: + from sqlalchemy.orm import declarative_base +except ImportError: + from sqlalchemy.ext.declarative import declarative_base from crate.client.sqlalchemy.sa_version import SA_VERSION, SA_1_4, SA_2_0 from crate.client.sqlalchemy.types import ObjectType @@ -294,7 +298,7 @@ def test_ddl_with_foreign_keys(self): Verify the CrateDB dialect properly ignores foreign key constraints. """ - Base = sa.orm.declarative_base(metadata=self.metadata) + Base = declarative_base(metadata=self.metadata) class RootStore(Base): """The main store.""" @@ -346,3 +350,28 @@ class ItemStore(Base): ) """)) # noqa: W291, W293 + + def test_ddl_with_unique_key(self): + """ + Verify the CrateDB dialect properly ignores unique key constraints. + """ + + Base = declarative_base(metadata=self.metadata) + + class FooBar(Base): + """The entity.""" + + __tablename__ = "foobar" + + id = sa.Column(sa.Integer, primary_key=True) + name = sa.Column(sa.String, unique=True) + + self.metadata.create_all(self.engine, tables=[FooBar.__table__], checkfirst=False) + self.assertEqual(self.executed_statement, dedent(""" + CREATE TABLE testdrive.foobar ( + \tid INT NOT NULL, + \tname STRING, + \tPRIMARY KEY (id) + ) + + """)) # noqa: W291 From 47ad0223f9a376b93f449d2f868855b86aa029ac Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 28 Sep 2023 16:34:15 +0200 Subject: [PATCH 11/19] SQLAlchemy: Adjust DDL compiler improvements to emit warnings --- src/crate/client/sqlalchemy/compiler.py | 5 ++ .../client/sqlalchemy/tests/compiler_test.py | 89 ++++++++++++------- 2 files changed, 64 insertions(+), 30 deletions(-) diff --git a/src/crate/client/sqlalchemy/compiler.py b/src/crate/client/sqlalchemy/compiler.py index 4906dcba..cdc07728 100644 --- a/src/crate/client/sqlalchemy/compiler.py +++ b/src/crate/client/sqlalchemy/compiler.py @@ -20,6 +20,7 @@ # software solely pursuant to the terms of the relevant commercial agreement. import string +import warnings from collections import defaultdict import sqlalchemy as sa @@ -182,12 +183,16 @@ def visit_foreign_key_constraint(self, constraint, **kw): """ CrateDB does not support foreign key constraints. """ + warnings.warn("CrateDB does not support foreign key constraints, " + "they will be omitted when generating DDL statements.") return None def visit_unique_constraint(self, constraint, **kw): """ CrateDB does not support unique key constraints. """ + warnings.warn("CrateDB does not support unique constraints, " + "they will be omitted when generating DDL statements.") return None diff --git a/src/crate/client/sqlalchemy/tests/compiler_test.py b/src/crate/client/sqlalchemy/tests/compiler_test.py index f9b4eef8..44cb16ce 100644 --- a/src/crate/client/sqlalchemy/tests/compiler_test.py +++ b/src/crate/client/sqlalchemy/tests/compiler_test.py @@ -18,6 +18,7 @@ # However, if you have executed another commercial license agreement # with Crate these terms will supersede the license and you may use the # software solely pursuant to the terms of the relevant commercial agreement. +import warnings from textwrap import dedent from unittest import mock, skipIf, TestCase from unittest.mock import MagicMock, patch @@ -27,6 +28,9 @@ import sqlalchemy as sa from sqlalchemy.sql import text, Update + +from crate.testing.util import ExtraAssertions + try: from sqlalchemy.orm import declarative_base except ImportError: @@ -288,7 +292,7 @@ def execute_wrapper(self, query, *args, **kwargs): @patch('crate.client.connection.Cursor', FakeCursor) -class SqlAlchemyDDLCompilerTest(CompilerTestCase): +class SqlAlchemyDDLCompilerTest(CompilerTestCase, ExtraAssertions): """ Verify a few scenarios regarding the DDL compiler. """ @@ -330,26 +334,39 @@ class ItemStore(Base): ) root = sa.orm.relationship(RootStore, back_populates="items") - self.metadata.create_all(self.engine, tables=[RootStore.__table__], checkfirst=False) - self.assertEqual(self.executed_statement, dedent(""" - CREATE TABLE testdrive.root ( - \tid INT NOT NULL, - \tname STRING, - \tPRIMARY KEY (id) - ) - - """)) # noqa: W291 - - self.metadata.create_all(self.engine, tables=[ItemStore.__table__], checkfirst=False) - self.assertEqual(self.executed_statement, dedent(""" - CREATE TABLE testdrive.item ( - \tid INT NOT NULL, - \tname STRING, - \troot_id INT, - \tPRIMARY KEY (id) - ) - - """)) # noqa: W291, W293 + with warnings.catch_warnings(record=True) as w: + + # Cause all warnings to always be triggered. + warnings.simplefilter("always") + + # Verify SQL DDL statement. + self.metadata.create_all(self.engine, tables=[RootStore.__table__], checkfirst=False) + self.assertEqual(self.executed_statement, dedent(""" + CREATE TABLE testdrive.root ( + \tid INT NOT NULL, + \tname STRING, + \tPRIMARY KEY (id) + ) + + """)) # noqa: W291, W293 + + # Verify SQL DDL statement. + self.metadata.create_all(self.engine, tables=[ItemStore.__table__], checkfirst=False) + self.assertEqual(self.executed_statement, dedent(""" + CREATE TABLE testdrive.item ( + \tid INT NOT NULL, + \tname STRING, + \troot_id INT, + \tPRIMARY KEY (id) + ) + + """)) # noqa: W291, W293 + + # Verify if corresponding warning is emitted. + self.assertEqual(len(w), 1) + self.assertIsSubclass(w[-1].category, UserWarning) + self.assertIn("CrateDB does not support foreign key constraints, " + "they will be omitted when generating DDL statements.", str(w[-1].message)) def test_ddl_with_unique_key(self): """ @@ -366,12 +383,24 @@ class FooBar(Base): id = sa.Column(sa.Integer, primary_key=True) name = sa.Column(sa.String, unique=True) - self.metadata.create_all(self.engine, tables=[FooBar.__table__], checkfirst=False) - self.assertEqual(self.executed_statement, dedent(""" - CREATE TABLE testdrive.foobar ( - \tid INT NOT NULL, - \tname STRING, - \tPRIMARY KEY (id) - ) - - """)) # noqa: W291 + with warnings.catch_warnings(record=True) as w: + + # Cause all warnings to always be triggered. + warnings.simplefilter("always") + + # Verify SQL DDL statement. + self.metadata.create_all(self.engine, tables=[FooBar.__table__], checkfirst=False) + self.assertEqual(self.executed_statement, dedent(""" + CREATE TABLE testdrive.foobar ( + \tid INT NOT NULL, + \tname STRING, + \tPRIMARY KEY (id) + ) + + """)) # noqa: W291, W293 + + # Verify if corresponding warning is emitted. + self.assertEqual(len(w), 1) + self.assertIsSubclass(w[-1].category, UserWarning) + self.assertIn("CrateDB does not support unique constraints, " + "they will be omitted when generating DDL statements.", str(w[-1].message)) From ac8b2f4d8a9006288d3e78e495e34603aa619120 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 28 Sep 2023 15:39:29 +0200 Subject: [PATCH 12/19] Improve exception handling: Properly raise `IntegrityError` exceptions ... when receiving `DuplicateKeyException` errors from CrateDB, instead of the more general `ProgrammingError`. --- CHANGES.txt | 2 ++ src/crate/client/http.py | 13 +++++++++++++ src/crate/client/test_http.py | 26 ++++++++++++++++++++++++-- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 4fbb10b9..808430c7 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -15,6 +15,8 @@ Unreleased ``subjectAltName`` attribute will be used. - SQLAlchemy: Improve DDL compiler to ignore foreign key and uniqueness constraints +- DBAPI: Properly raise ``IntegrityError`` exceptions instead of + ``ProgrammingError``, when CrateDB raises a ``DuplicateKeyException``. .. _urllib3 v2.0 migration guide: https://urllib3.readthedocs.io/en/latest/v2-migration-guide.html .. _urllib3 v2.0 roadmap: https://urllib3.readthedocs.io/en/stable/v2-roadmap.html diff --git a/src/crate/client/http.py b/src/crate/client/http.py index 0cce7bda..1318cca2 100644 --- a/src/crate/client/http.py +++ b/src/crate/client/http.py @@ -56,6 +56,7 @@ BlobLocationNotFoundException, DigestNotFoundException, ProgrammingError, + IntegrityError, ) @@ -191,6 +192,18 @@ def _ex_to_message(ex): def _raise_for_status(response): + """ + Properly raise `IntegrityError` exceptions for CrateDB's `DuplicateKeyException` errors. + """ + try: + return _raise_for_status_real(response) + except ProgrammingError as ex: + if "DuplicateKeyException" in ex.message: + raise IntegrityError(ex.message, error_trace=ex.error_trace) from ex + raise + + +def _raise_for_status_real(response): """ make sure that only crate.exceptions are raised that are defined in the DB-API specification """ message = '' diff --git a/src/crate/client/test_http.py b/src/crate/client/test_http.py index de287834..fe20d386 100644 --- a/src/crate/client/test_http.py +++ b/src/crate/client/test_http.py @@ -44,8 +44,7 @@ from setuptools.ssl_support import find_ca_bundle from .http import Client, CrateJsonEncoder, _get_socket_opts, _remove_certs_for_non_https -from .exceptions import ConnectionError, ProgrammingError - +from .exceptions import ConnectionError, ProgrammingError, IntegrityError REQUEST = 'crate.client.http.Server.request' CA_CERT_PATH = find_ca_bundle() @@ -91,6 +90,17 @@ def bad_bulk_response(): return r +def duplicate_key_exception(): + r = fake_response(409, 'Conflict') + r.data = json.dumps({ + "error": { + "code": 4091, + "message": "DuplicateKeyException[A document with the same primary key exists already]" + } + }).encode() + return r + + def fail_sometimes(*args, **kwargs): if random.randint(1, 100) % 10 == 0: raise urllib3.exceptions.MaxRetryError(None, '/_sql', '') @@ -302,6 +312,18 @@ def test_uuid_serialization(self, request): self.assertEqual(data['args'], [str(uid)]) client.close() + @patch(REQUEST, fake_request(duplicate_key_exception())) + def test_duplicate_key_error(self): + """ + Verify that an `IntegrityError` is raised on duplicate key errors, + instead of the more general `ProgrammingError`. + """ + client = Client(servers="localhost:4200") + with self.assertRaises(IntegrityError) as cm: + client.sql('INSERT INTO testdrive (foo) VALUES (42)') + self.assertEqual(cm.exception.message, + "DuplicateKeyException[A document with the same primary key exists already]") + @patch(REQUEST, fail_sometimes) class ThreadSafeHttpClientTest(TestCase): From 40cf2e9873a8b5da09ba59ca9a317f4da1fe4276 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 May 2023 15:05:45 +0000 Subject: [PATCH 13/19] Update sphinx requirement from <7,>=3.5 to >=3.5,<8 Updates the requirements on [sphinx](https://github.com/sphinx-doc/sphinx) to permit the latest version. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v3.5.0...v7.0.0) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fa28639d..8fadedd7 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ def read(path): # `test_http.py` needs `setuptools.ssl_support` 'setuptools<57', ], - doc=['sphinx>=3.5,<7', + doc=['sphinx>=3.5,<8', 'crate-docs-theme>=0.26.5'], ), python_requires='>=3.6', From 0a482acbef2734e8bd30ce60a1e29bf306f9379f Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 28 Sep 2023 21:40:22 +0200 Subject: [PATCH 14/19] Maintenance: Remove use of deprecated `setuptools.ssl_support` As a replacement for `find_ca_bundle`, use `certifi.where()`. --- setup.py | 3 +-- src/crate/client/test_http.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 8fadedd7..ca00d565 100644 --- a/setup.py +++ b/setup.py @@ -67,14 +67,13 @@ def read(path): 'zope.testing>=4,<6', 'zope.testrunner>=5,<7', 'zc.customdoctests>=1.0.1,<2', + 'certifi', 'createcoverage>=1,<2', 'dask', 'stopit>=1.1.2,<2', 'flake8>=4,<7', 'pandas', 'pytz', - # `test_http.py` needs `setuptools.ssl_support` - 'setuptools<57', ], doc=['sphinx>=3.5,<8', 'crate-docs-theme>=0.26.5'], diff --git a/src/crate/client/test_http.py b/src/crate/client/test_http.py index fe20d386..8e547963 100644 --- a/src/crate/client/test_http.py +++ b/src/crate/client/test_http.py @@ -41,13 +41,13 @@ from urllib.parse import urlparse, parse_qs import uuid -from setuptools.ssl_support import find_ca_bundle +import certifi from .http import Client, CrateJsonEncoder, _get_socket_opts, _remove_certs_for_non_https from .exceptions import ConnectionError, ProgrammingError, IntegrityError REQUEST = 'crate.client.http.Server.request' -CA_CERT_PATH = find_ca_bundle() +CA_CERT_PATH = certifi.where() def fake_request(response=None): From c9257178ab6473fab082ff613ca5b03e691c665d Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 28 Sep 2023 21:42:24 +0200 Subject: [PATCH 15/19] CI: Configure Dependabot to check for versions of GitHub actions --- .github/dependabot.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 91abb11f..8efce62c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,3 +9,8 @@ updates: directory: "/" # Location of package manifests schedule: interval: "weekly" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" From 7c4a4c9ab45315339ebb6bf96304b935ca53a333 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 07:07:14 +0000 Subject: [PATCH 16/19] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/nightly.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/tests.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 621d914c..d0f88fff 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -34,7 +34,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Initialize CodeQL uses: github/codeql-action/init@v2 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0ac96596..b9e89cf8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Acquire sources - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 4f8424b5..9941d180 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -37,7 +37,7 @@ jobs: PIP_ALLOW_PRERELEASE: ${{ matrix.pip-allow-prerelease }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4dad813f..4d35c3c2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest if: startsWith(github.event.ref, 'refs/tags') steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fd4fa6e2..58f086d1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -52,7 +52,7 @@ jobs: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: From 471f026fa5e764ac80c1a47c366600e3d5ca0ca0 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 28 Sep 2023 20:52:46 +0200 Subject: [PATCH 17/19] Documentation: Add section about using `gen_random_text_uuid` as auto-PK --- docs/sqlalchemy.rst | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/sqlalchemy.rst b/docs/sqlalchemy.rst index c31396ab..8c399a5c 100644 --- a/docs/sqlalchemy.rst +++ b/docs/sqlalchemy.rst @@ -300,6 +300,41 @@ would translate into the following declarative model: >>> log.id ... + +Auto-generated primary key +.......................... + +CrateDB 4.5.0 added the :ref:`gen_random_text_uuid() ` +scalar function, which can also be used within an SQL DDL statement, in order to automatically +assign random identifiers to newly inserted records on the server side. + +In this spirit, it is suitable to be used as a ``PRIMARY KEY`` constraint for SQLAlchemy. + +A table schema like this + +.. code-block:: sql + + CREATE TABLE "doc"."items" ( + "id" STRING DEFAULT gen_random_text_uuid() NOT NULL PRIMARY KEY, + "name" STRING + ) + +would translate into the following declarative model: + + >>> class Item(Base): + ... + ... __tablename__ = 'items' + ... + ... id = sa.Column("id", sa.String, server_default=func.gen_random_text_uuid(), primary_key=True) + ... name = sa.Column("name", sa.String) + + >>> item = Item(name="Foobar") + >>> session.add(item) + >>> session.commit() + >>> item.id + ... + + .. _using-extension-types: Extension types From c1ac6790753c1947eba3ec6e8901392e551bcc1c Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 16 Feb 2023 15:21:23 +0100 Subject: [PATCH 18/19] SQLAlchemy: Ignore SQL "FOR UPDATE" clause, CrateDB does not support it --- CHANGES.txt | 1 + src/crate/client/sqlalchemy/compiler.py | 7 +++++ .../client/sqlalchemy/tests/compiler_test.py | 30 ++++++++++++++++++- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 808430c7..59471850 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -17,6 +17,7 @@ Unreleased constraints - DBAPI: Properly raise ``IntegrityError`` exceptions instead of ``ProgrammingError``, when CrateDB raises a ``DuplicateKeyException``. +- SQLAlchemy: Ignore SQL's ``FOR UPDATE`` clause. Thanks, @surister. .. _urllib3 v2.0 migration guide: https://urllib3.readthedocs.io/en/latest/v2-migration-guide.html .. _urllib3 v2.0 roadmap: https://urllib3.readthedocs.io/en/stable/v2-roadmap.html diff --git a/src/crate/client/sqlalchemy/compiler.py b/src/crate/client/sqlalchemy/compiler.py index cdc07728..767ad638 100644 --- a/src/crate/client/sqlalchemy/compiler.py +++ b/src/crate/client/sqlalchemy/compiler.py @@ -309,3 +309,10 @@ def limit_clause(self, select, **kw): Generate OFFSET / LIMIT clause, PostgreSQL-compatible. """ return PGCompiler.limit_clause(self, select, **kw) + + def for_update_clause(self, select, **kw): + # CrateDB does not support the `INSERT ... FOR UPDATE` clause. + # See https://github.com/crate/crate-python/issues/577. + warnings.warn("CrateDB does not support the 'INSERT ... FOR UPDATE' clause, " + "it will be omitted when generating SQL statements.") + return '' diff --git a/src/crate/client/sqlalchemy/tests/compiler_test.py b/src/crate/client/sqlalchemy/tests/compiler_test.py index 44cb16ce..9c08154b 100644 --- a/src/crate/client/sqlalchemy/tests/compiler_test.py +++ b/src/crate/client/sqlalchemy/tests/compiler_test.py @@ -43,7 +43,7 @@ from crate.testing.settings import crate_host -class SqlAlchemyCompilerTest(ParametrizedTestCase): +class SqlAlchemyCompilerTest(ParametrizedTestCase, ExtraAssertions): def setUp(self): self.crate_engine = sa.create_engine('crate://') @@ -257,6 +257,34 @@ def test_insert_manyvalues(self): mock.call(mock.ANY, 'INSERT INTO mytable (name) VALUES (?)', ('foo_4', ), None), ]) + def test_for_update(self): + """ + Verify the `CrateCompiler.for_update_clause` method to + omit the clause, since CrateDB does not support it. + """ + + with warnings.catch_warnings(record=True) as w: + + # By default, warnings from a loop will only be emitted once. + # This scenario tests exactly this behaviour, to verify logs + # don't get flooded. + warnings.simplefilter("once") + + selectable = self.mytable.select().with_for_update() + _ = str(selectable.compile(bind=self.crate_engine)) + + selectable = self.mytable.select().with_for_update() + statement = str(selectable.compile(bind=self.crate_engine)) + + # Verify SQL statement. + self.assertEqual(statement, "SELECT mytable.name, mytable.data \nFROM mytable") + + # Verify if corresponding warning is emitted, once. + self.assertEqual(len(w), 1) + self.assertIsSubclass(w[-1].category, UserWarning) + self.assertIn("CrateDB does not support the 'INSERT ... FOR UPDATE' clause, " + "it will be omitted when generating SQL statements.", str(w[-1].message)) + FakeCursor = MagicMock(name='FakeCursor', spec=Cursor) From 45a1d7f7abc8056ec7b26a54cfedea9b4a3abb1b Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Fri, 29 Sep 2023 16:12:07 +0200 Subject: [PATCH 19/19] Release 0.34.0 --- CHANGES.txt | 9 +++++++-- src/crate/client/__init__.py | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 59471850..3ccfd634 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -5,7 +5,12 @@ Changes for crate Unreleased ========== -- Properly handle Python-native UUID types in SQL parameters + +2023/09/29 0.34.0 +================= + +- Properly handle Python-native UUID types in SQL parameters. Thanks, + @SStorm. - SQLAlchemy: Fix handling URL parameters ``timeout`` and ``pool_size`` - Permit installation with urllib3 v2, see also `urllib3 v2.0 roadmap`_ and `urllib3 v2.0 migration guide`_. You can optionally retain support @@ -14,7 +19,7 @@ Unreleased deprecated ``commonName`` attribute. Instead, going forward, only the ``subjectAltName`` attribute will be used. - SQLAlchemy: Improve DDL compiler to ignore foreign key and uniqueness - constraints + constraints. - DBAPI: Properly raise ``IntegrityError`` exceptions instead of ``ProgrammingError``, when CrateDB raises a ``DuplicateKeyException``. - SQLAlchemy: Ignore SQL's ``FOR UPDATE`` clause. Thanks, @surister. diff --git a/src/crate/client/__init__.py b/src/crate/client/__init__.py index bf1c1648..3d67a541 100644 --- a/src/crate/client/__init__.py +++ b/src/crate/client/__init__.py @@ -29,7 +29,7 @@ # version string read from setup.py using a regex. Take care not to break the # regex! -__version__ = "0.33.0" +__version__ = "0.34.0" apilevel = "2.0" threadsafety = 2