From ec3e78213714d358924cadf506faf2d9f90ef7ac Mon Sep 17 00:00:00 2001 From: Andrew Chang-DeWitt Date: Fri, 29 Oct 2021 17:06:00 -0400 Subject: [PATCH 1/6] Bump versions in example requirements. --- example/requirements/prod.txt | 2 +- example_sync/requirements/prod.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/example/requirements/prod.txt b/example/requirements/prod.txt index d875b7a..e60595e 100644 --- a/example/requirements/prod.txt +++ b/example/requirements/prod.txt @@ -1 +1 @@ -https://github.com/cheese-drawer/lib-python-db-wrapper/releases/download/2.1.0/db_wrapper-2.1.0-py3-none-any.whl +https://github.com/cheese-drawer/lib-python-db-wrapper/releases/download/2.3.0/db_wrapper-2.3.0-py3-none-any.whl diff --git a/example_sync/requirements/prod.txt b/example_sync/requirements/prod.txt index d875b7a..e60595e 100644 --- a/example_sync/requirements/prod.txt +++ b/example_sync/requirements/prod.txt @@ -1 +1 @@ -https://github.com/cheese-drawer/lib-python-db-wrapper/releases/download/2.1.0/db_wrapper-2.1.0-py3-none-any.whl +https://github.com/cheese-drawer/lib-python-db-wrapper/releases/download/2.3.0/db_wrapper-2.3.0-py3-none-any.whl From 781b688a70514640e304385448432d5ca32ccf18 Mon Sep 17 00:00:00 2001 From: Andrew Chang-DeWitt Date: Fri, 29 Oct 2021 17:35:26 -0400 Subject: [PATCH 2/6] Update query to use UUID for id. --- db_wrapper/model/async_model.py | 2 +- db_wrapper/model/base.py | 4 ++-- db_wrapper/model/sync_model.py | 2 +- test/test_model.py | 33 ++++++++++++++++----------------- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/db_wrapper/model/async_model.py b/db_wrapper/model/async_model.py index 07a5bdf..73cb23c 100644 --- a/db_wrapper/model/async_model.py +++ b/db_wrapper/model/async_model.py @@ -89,7 +89,7 @@ def __init__( super().__init__(table, return_constructor) self._client = client - async def one_by_id(self, id_value: str, changes: Dict[str, Any]) -> T: + async def one_by_id(self, id_value: UUID, changes: Dict[str, Any]) -> T: """Apply changes to row with given id. Arguments: diff --git a/db_wrapper/model/base.py b/db_wrapper/model/base.py index 7d0a8b0..fb24805 100644 --- a/db_wrapper/model/base.py +++ b/db_wrapper/model/base.py @@ -134,7 +134,7 @@ class UpdateABC(CRUDABC[T]): def _query_one_by_id( self, - id_value: str, + id_value: UUID, changes: Dict[str, Any] ) -> sql.Composed: """Build Query to apply changes to row with given id.""" @@ -157,7 +157,7 @@ def compose_changes(changes: Dict[str, Any]) -> sql.Composed: ).format( table=self._table, changes=compose_changes(changes), - id_value=sql.Literal(id_value), + id_value=sql.Literal(str(id_value)), ) return query diff --git a/db_wrapper/model/sync_model.py b/db_wrapper/model/sync_model.py index 3676a9a..7d65602 100644 --- a/db_wrapper/model/sync_model.py +++ b/db_wrapper/model/sync_model.py @@ -88,7 +88,7 @@ def __init__( super().__init__(table, return_constructor) self._client = client - def one_by_id(self, id_value: str, changes: Dict[str, Any]) -> T: + def one_by_id(self, id_value: UUID, changes: Dict[str, Any]) -> T: """Apply changes to row with given id. Arguments: diff --git a/test/test_model.py b/test/test_model.py index 0f9e686..85f21b9 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -34,12 +34,9 @@ ) -# Generic doesn't need a more descriptive name -# pylint: disable=invalid-name -T = TypeVar('T', bound=ModelData) - - -def setupAsync(query_result: List[T]) -> Tuple[AsyncModel[T], AsyncClient]: +def setupAsync( + query_result: List[ModelData] +) -> Tuple[AsyncModel[ModelData], AsyncClient]: """Setup helper that returns instances of both a Model & a Client. Mocks the execute_and_return method on the Client instance to skip @@ -60,12 +57,14 @@ def setupAsync(query_result: List[T]) -> Tuple[AsyncModel[T], AsyncClient]: return_value=query_result) # init a real model with mocked client - model = AsyncModel[Any](client, 'test') + model = AsyncModel[Any](client, 'test', ModelData) return model, client -def setupSync(query_result: List[T]) -> Tuple[SyncModel[T], SyncClient]: +def setupSync( + query_result: List[ModelData] +) -> Tuple[SyncModel[ModelData], SyncClient]: """Setup helper that returns instances of both a Model & a Client. Mocks the execute_and_return method on the Client instance to skip @@ -86,7 +85,7 @@ def setupSync(query_result: List[T]) -> Tuple[SyncModel[T], SyncClient]: return_value=query_result) # init a real model with mocked client - model = SyncModel[Any](client, 'test') + model = SyncModel[ModelData](client, 'test', ModelData) return model, client @@ -100,8 +99,8 @@ async def test_it_correctly_builds_query_with_given_id(self) -> None: async_model, async_client = setupAsync([item]) sync_model, sync_client = setupSync([item]) - await async_model.read.one_by_id(str(item.id)) - sync_model.read.one_by_id(str(item.id)) + await async_model.read.one_by_id(item.id) + sync_model.read.one_by_id(item.id) async_query_composed = cast( helpers.AsyncMock, async_client.execute_and_return).call_args[0][0] @@ -124,8 +123,8 @@ async def test_it_returns_a_single_result(self) -> None: item = ModelData(id=uuid4()) async_model, _ = setupAsync([item]) sync_model, _ = setupSync([item]) - results = [await async_model.read.one_by_id(str(item.id)), - sync_model.read.one_by_id(str(item.id))] + results = [await async_model.read.one_by_id(item.id), + sync_model.read.one_by_id(item.id)] for result in results: with self.subTest(): @@ -139,11 +138,11 @@ async def test_it_raises_exception_if_more_than_one_result(self) -> None: with self.subTest(): with self.assertRaises(UnexpectedMultipleResults): - await async_model.read.one_by_id(str(item.id)) + await async_model.read.one_by_id(item.id) with self.subTest(): with self.assertRaises(UnexpectedMultipleResults): - sync_model.read.one_by_id(str(item.id)) + sync_model.read.one_by_id(item.id) @ helpers.async_test async def test_it_raises_exception_if_no_result_to_return(self) -> None: @@ -265,8 +264,8 @@ async def test_it_returns_the_new_record(self) -> None: sync_model, _ = setupSync([updated]) results = [ - await async_model.update.one_by_id(str(item.id), {'b': 'c'}), - sync_model.update.one_by_id(str(item.id), {'b': 'c'}) + await async_model.update.one_by_id(item.id, {'b': 'c'}), + sync_model.update.one_by_id(item.id, {'b': 'c'}) ] for result in results: From 15af9cad26a9ced3c00189e2f4f0c7157407511d Mon Sep 17 00:00:00 2001 From: Andrew Chang-DeWitt Date: Fri, 29 Oct 2021 17:52:38 -0400 Subject: [PATCH 3/6] Update tests. --- test/test_model.py | 84 +++++++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 38 deletions(-) diff --git a/test/test_model.py b/test/test_model.py index 85f21b9..d9de694 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -12,6 +12,7 @@ Any, List, Tuple, + Type, TypeVar, ) from uuid import uuid4 @@ -34,9 +35,13 @@ ) +T = TypeVar('T', bound=ModelData) + + def setupAsync( - query_result: List[ModelData] -) -> Tuple[AsyncModel[ModelData], AsyncClient]: + query_result: List[T], + model_data: Type[T] +) -> Tuple[AsyncModel[T], AsyncClient]: """Setup helper that returns instances of both a Model & a Client. Mocks the execute_and_return method on the Client instance to skip @@ -54,17 +59,18 @@ def setupAsync( # mock client's sql execution method client.execute_and_return = helpers.AsyncMock( # type:ignore - return_value=query_result) + return_value=[i.dict() for i in query_result]) # init a real model with mocked client - model = AsyncModel[Any](client, 'test', ModelData) + model = AsyncModel[Any](client, 'test', model_data) return model, client def setupSync( - query_result: List[ModelData] -) -> Tuple[SyncModel[ModelData], SyncClient]: + query_result: List[T], + model_data: Type[T] +) -> Tuple[SyncModel[T], SyncClient]: """Setup helper that returns instances of both a Model & a Client. Mocks the execute_and_return method on the Client instance to skip @@ -82,10 +88,10 @@ def setupSync( # mock client's sql execution method client.execute_and_return = helpers.MagicMock( # type:ignore - return_value=query_result) + return_value=[i.dict() for i in query_result]) # init a real model with mocked client - model = SyncModel[ModelData](client, 'test', ModelData) + model = SyncModel[T](client, 'test', model_data) return model, client @@ -96,8 +102,8 @@ class TestReadOneById(TestCase): @helpers.async_test async def test_it_correctly_builds_query_with_given_id(self) -> None: item = ModelData(id=uuid4()) - async_model, async_client = setupAsync([item]) - sync_model, sync_client = setupSync([item]) + async_model, async_client = setupAsync([item], ModelData) + sync_model, sync_client = setupSync([item], ModelData) await async_model.read.one_by_id(item.id) sync_model.read.one_by_id(item.id) @@ -121,8 +127,8 @@ async def test_it_correctly_builds_query_with_given_id(self) -> None: @helpers.async_test async def test_it_returns_a_single_result(self) -> None: item = ModelData(id=uuid4()) - async_model, _ = setupAsync([item]) - sync_model, _ = setupSync([item]) + async_model, _ = setupAsync([item], ModelData) + sync_model, _ = setupSync([item], ModelData) results = [await async_model.read.one_by_id(item.id), sync_model.read.one_by_id(item.id)] @@ -133,8 +139,8 @@ async def test_it_returns_a_single_result(self) -> None: @helpers.async_test async def test_it_raises_exception_if_more_than_one_result(self) -> None: item = ModelData(id=uuid4()) - async_model, _ = setupAsync([item, item]) - sync_model, _ = setupSync([item, item]) + async_model, _ = setupAsync([item, item], ModelData) + sync_model, _ = setupSync([item, item], ModelData) with self.subTest(): with self.assertRaises(UnexpectedMultipleResults): @@ -146,18 +152,20 @@ async def test_it_raises_exception_if_more_than_one_result(self) -> None: @ helpers.async_test async def test_it_raises_exception_if_no_result_to_return(self) -> None: + empty_async: List[ModelData] = [] + empty_sync: List[ModelData] = [] async_model: AsyncModel[ModelData] sync_model: SyncModel[ModelData] - async_model, _ = setupAsync([]) - sync_model, _ = setupSync([]) + async_model, _ = setupAsync(empty_async, ModelData) + sync_model, _ = setupSync(empty_sync, ModelData) with self.subTest(): with self.assertRaises(NoResultFound): - await async_model.read.one_by_id('id') + await async_model.read.one_by_id(uuid4()) with self.subTest(): with self.assertRaises(NoResultFound): - sync_model.read.one_by_id('id') + sync_model.read.one_by_id(uuid4()) class TestCreateOne(TestCase): @@ -175,8 +183,8 @@ async def test_it_correctly_builds_query_with_given_data(self) -> None: 'a': 'a', 'b': 'b', }) - async_model, async_client = setupAsync([item]) - sync_model, sync_client = setupSync([item]) + async_model, async_client = setupAsync([item], TestCreateOne.Item) + sync_model, sync_client = setupSync([item], TestCreateOne.Item) await async_model.create.one(item) sync_model.create.one(item) @@ -203,8 +211,8 @@ async def test_it_returns_the_new_record(self) -> None: 'a': 'a', 'b': 'b', }) - async_model, _ = setupAsync([item]) - sync_model, _ = setupSync([item]) + async_model, _ = setupAsync([item], TestCreateOne.Item) + sync_model, _ = setupSync([item], TestCreateOne.Item) results = [await async_model.create.one(item), sync_model.create.one(item)] @@ -229,11 +237,11 @@ async def test_it_correctly_builds_query_with_given_data(self) -> None: 'a': 'a', 'b': 'b', }) - async_model, async_client = setupAsync([item]) - sync_model, sync_client = setupSync([item]) + async_model, async_client = setupAsync([item], TestUpdateOne.Item) + sync_model, sync_client = setupSync([item], TestUpdateOne.Item) - await async_model.update.one_by_id(str(item.id), {'b': 'c'}) - sync_model.update.one_by_id(str(item.id), {'b': 'c'}) + await async_model.update.one_by_id(item.id, {'b': 'c'}) + sync_model.update.one_by_id(item.id, {'b': 'c'}) async_query_composed = cast( helpers.AsyncMock, async_client.execute_and_return).call_args[0][0] @@ -260,8 +268,8 @@ async def test_it_returns_the_new_record(self) -> None: }) # mock result updated = TestUpdateOne.Item(**{**item.dict(), 'b': 'c'}) - async_model, _ = setupAsync([updated]) - sync_model, _ = setupSync([updated]) + async_model, _ = setupAsync([updated], TestUpdateOne.Item) + sync_model, _ = setupSync([updated], TestUpdateOne.Item) results = [ await async_model.update.one_by_id(item.id, {'b': 'c'}), @@ -288,8 +296,8 @@ async def test_it_correctly_builds_query_with_given_data(self) -> None: 'a': 'a', 'b': 'b', }) - async_model, async_client = setupAsync([item]) - sync_model, sync_client = setupSync([item]) + async_model, async_client = setupAsync([item], TestDeleteOneById.Item) + sync_model, sync_client = setupSync([item], TestDeleteOneById.Item) await async_model.delete.one_by_id(str(item.id)) sync_model.delete.one_by_id(str(item.id)) @@ -316,8 +324,8 @@ async def test_it_returns_the_deleted_record(self) -> None: 'a': 'a', 'b': 'b', }) - async_model, _ = setupAsync([item]) - sync_model, _ = setupSync([item]) + async_model, _ = setupAsync([item], TestDeleteOneById.Item) + sync_model, _ = setupSync([item], TestDeleteOneById.Item) results = [await async_model.delete.one_by_id(str(item.id)), sync_model.delete.one_by_id(str(item.id))] @@ -347,8 +355,8 @@ class AsyncExtendedModel(AsyncModel[Item]): read: AsyncReadExtended def __init__(self, client: AsyncClient) -> None: - super().__init__(client, 'extended_model') - self.read = AsyncReadExtended(self.client, self.table) + super().__init__(client, 'extended_model', Item) + self.read = AsyncReadExtended(self.client, self.table, Item) class SyncReadExtended(SyncRead[Item]): """Extending Read with additional query.""" @@ -361,11 +369,11 @@ class SyncExtendedModel(SyncModel[Item]): read: SyncReadExtended def __init__(self, client: SyncClient) -> None: - super().__init__(client, 'extended_model') - self.read = SyncReadExtended(self.client, self.table) + super().__init__(client, 'extended_model', Item) + self.read = SyncReadExtended(self.client, self.table, Item) - _, async_client = setupAsync([Item(**{"id": uuid4()})]) - _, sync_client = setupSync([Item(**{"id": uuid4()})]) + _, async_client = setupAsync([Item(**{"id": uuid4()})], Item) + _, sync_client = setupSync([Item(**{"id": uuid4()})], Item) self.models = [AsyncExtendedModel(async_client), SyncExtendedModel(sync_client)] From bd9e5b32653d89698dcdf22b3c776b523253abac Mon Sep 17 00:00:00 2001 From: Andrew Chang-DeWitt Date: Wed, 5 Apr 2023 15:18:56 -0500 Subject: [PATCH 4/6] Switch from single connection to pool. --- db_wrapper/client/async_client.py | 19 ++++++++++--------- db_wrapper/connection.py | 18 +++++++++--------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/db_wrapper/client/async_client.py b/db_wrapper/client/async_client.py index dc9ac12..afed995 100644 --- a/db_wrapper/client/async_client.py +++ b/db_wrapper/client/async_client.py @@ -12,10 +12,10 @@ Dict) import aiopg -from psycopg2.extras import register_uuid, RealDictRow +from psycopg2.extras import register_uuid, RealDictCursor, RealDictRow # type: ignore from psycopg2 import sql -from db_wrapper.connection import ConnectionParameters, connect +from db_wrapper.connection import ConnectionParameters, get_pool # add uuid support to psycopg2 & Postgres register_uuid() @@ -33,18 +33,19 @@ class AsyncClient: """ _connection_params: ConnectionParameters - _connection: aiopg.Connection + _pool: aiopg.Pool def __init__(self, connection_params: ConnectionParameters) -> None: self._connection_params = connection_params async def connect(self) -> None: - """Connect to the database.""" - self._connection = await connect(self._connection_params) + """Create a database connection pool.""" + self._pool = await get_pool(self._connection_params) async def disconnect(self) -> None: - """Disconnect from the database.""" - await self._connection.close() + """Close database connection pool.""" + self._pool.close() + await self._pool.wait_closed() # PENDS python 3.9 support in pylint # pylint: disable=unsubscriptable-object @@ -81,7 +82,7 @@ async def execute( Returns: None """ - async with self._connection.cursor() as cursor: + with (await self._pool.cursor(cursor_factory=RealDictCursor) ) as cursor: await self._execute_query(cursor, query, params) # PENDS python 3.9 support in pylint @@ -101,7 +102,7 @@ async def execute_and_return( Returns: List containing all the rows that matched the query. """ - async with self._connection.cursor() as cursor: + with (await self._pool.cursor(cursor_factory=RealDictCursor) ) as cursor: await self._execute_query(cursor, query, params) result: List[RealDictRow] = await cursor.fetchall() diff --git a/db_wrapper/connection.py b/db_wrapper/connection.py index 18777e0..0574d55 100644 --- a/db_wrapper/connection.py +++ b/db_wrapper/connection.py @@ -41,18 +41,18 @@ async def _try_connect( dsn = f"dbname={database} user={user} password={password} " \ f"host={host} port={port}" + # return await aiopg.create_pool(dsn) + # PENDS python 3.9 support in pylint # pylint: disable=unsubscriptable-object - connection: Optional[aiopg.Connection] = None + pool: Optional[aiopg.Connection] = None LOGGER.info(f"Attempting to connect to database {database} as " f"{user}@{host}:{port}...") - while connection is None: + while pool is None: try: - connection = await aiopg.connect( - dsn, - cursor_factory=RealDictCursor) + pool = await aiopg.create_pool(dsn) except psycopg2OpError as err: print(type(err)) if retries > 12: @@ -67,7 +67,7 @@ async def _try_connect( await asyncio.sleep(5) return await _try_connect(connection_params, retries + 1) - return connection + return pool def _sync_try_connect( @@ -112,10 +112,10 @@ def _sync_try_connect( # PENDS python 3.9 support in pylint # pylint: disable=unsubscriptable-object -async def connect( +async def get_pool( connection_params: ConnectionParameters -) -> aiopg.Connection: - """Establish database connection.""" +) -> aiopg.Pool: + """Establish database connection pool.""" return await _try_connect(connection_params) From fca95d5d8c28b1501a51294cf6129b4941b08f5b Mon Sep 17 00:00:00 2001 From: Andrew Chang-DeWitt Date: Wed, 5 Apr 2023 15:20:09 -0500 Subject: [PATCH 5/6] Bump version. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2b4a682..ead47ee 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="db_wrapper", - version="2.3.0", + version="2.4.0", author="Andrew Chang-DeWitt", author_email="andrew@andrew-chang-dewitt.dev", description=short_description, From 2e3bbf44262499c4a84bef9a6795f92d87bbaa7a Mon Sep 17 00:00:00 2001 From: Andrew Chang-DeWitt Date: Wed, 5 Apr 2023 15:22:56 -0500 Subject: [PATCH 6/6] Update scripts & examples. --- example/requirements/prod.txt | 2 +- scripts/install | 66 +++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100755 scripts/install diff --git a/example/requirements/prod.txt b/example/requirements/prod.txt index e60595e..c642fd0 100644 --- a/example/requirements/prod.txt +++ b/example/requirements/prod.txt @@ -1 +1 @@ -https://github.com/cheese-drawer/lib-python-db-wrapper/releases/download/2.3.0/db_wrapper-2.3.0-py3-none-any.whl +-e ../ diff --git a/scripts/install b/scripts/install new file mode 100755 index 0000000..b5d3924 --- /dev/null +++ b/scripts/install @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +# +# NAVIGATE TO CORRECT DIRECTORY +# + +# start by going to script dir so all movements +# from here are relative +SCRIPT_DIR=`dirname $(realpath "$0")` +cd $SCRIPT_DIR +# go up to root +cd .. + + +# +# INSTALL FROM LISTS +# + +function dev { + echo "" + echo "Installing dev requirements..." + echo "" + dev_result=0 + + # install from dev list + pip install -r requirements/dev.txt +} + +function prod { + echo "" + echo "Installing prod requirements..." + echo "" + prod_result=0 + + # install from dev list + pip install -r requirements/prod.txt +} + +# Install dev, prod, or all requirements depending on argument given +if [ $# -eq 0 ]; then + dev + prod + + if [[ $dev_result != 0 && $prod_result != 0 ]]; then + echo "Errors found in both dev & prod installation. See output above." + exit $dev_result + elif [[ $dev_result != 0 && $prod_result == 0 ]]; then + echo "Errors found in dev installation. See output above." + exit $dev_result + elif [[ $dev_result == 0 && $prod_result != 0 ]]; then + echo "Errors found in prod installation. See output above." + exit $prod_result + else + exit 0 + fi + +elif [[ $1 == 'dev' || $1 == 'development' ]]; then + dev ${@:2} + exit $dev_result +elif [[ $1 == 'prod' || $1 == 'production' ]]; then + prod ${@:2} + exit $prod_result +else + echo "Bad argument given, either specify \`dev\` or \`prod\` requirements by giving either word as your first argument to this script, or run both by giving no arguments." + exit 1 +fi