8000 Merge pull request #37 from hartym/master · python-bonobo/bonobo-sqlalchemy@cee36b0 · GitHub 8000
[go: up one dir, main page]
Skip to content

Commit cee36b0

Browse files
hartymhartym
authored
Merge pull request #37 from hartym/master
[misc] Update requirements, fix options in Select, switch from edgy.project to medikit.
2 parents 05479c6 + 10b1d35 commit cee36b0

File tree

8 files changed

+94
-46
lines changed

8 files changed

+94
-46
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
/.coverage
77
/.idea
88
/.python*-*
9+
/.release
910
/build
1011
/dist
1112
/htmlcov

‎Makefile

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
1-
# This file has been auto-generated.
2-
# All changes will be lost, see Projectfile.
3-
#
4-
# Updated at 2017-07-16 10:29:30.567692
1+
# This file has been auto-generated by Medikit. All changes will be lost.
2+
# Updated on 2017-10-21.
53

64
PACKAGE ?= bonobo_sqlalchemy
75
PYTHON ?= $(shell which python)
@@ -18,24 +16,34 @@ YAPF ?= $(PYTHON) -m yapf
1816
YAPF_OPTIONS ?= -rip
1917
VERSION ?= $(shell git describe 2>/dev/null || echo dev)
2018

21-
.PHONY: clean format install install-dev test
19+
.PHONY: clean format install install-dev test update update-requirements
2220

2321
# Installs the local project dependencies.
2422
install:
2523
if [ -z "$(QUICK)" ]; then \
26-
$(PIP) install -U pip wheel $(PYTHON_PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_FILE) ; \
24+
$(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_FILE) ; \
2725
fi
2826

2927
# Installs the local project dependencies, including development-only libraries.
3028
install-dev:
3129
if [ -z "$(QUICK)" ]; then \
32-
$(PIP) install -U pip wheel $(PYTHON_PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \
30+
$(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \
3331
fi
3432

3533
# Cleans up the local mess.
3634
clean:
3735
rm -rf build dist *.egg-info
3836

37+
# Update project artifacts using medikit, after installing it eventually.
38+
update:
39+
python -c 'import medikit; print(medikit.__version__)' || pip install medikit;
40+
$(PYTHON) -m medikit update
41+
42+
# Remove requirements files and update project artifacts using medikit, after installing it eventually.
43+
update-requirements:
44+
rm -rf requirements*.txt
45+
$(MAKE) update
46+
3947
test: install-dev
4048
$(PYTEST) $(PYTEST_OPTIONS) tests
4149

‎Projectfile

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# bonobo-sqlalchemy (see github.com/python-bonobo/bonobo-sqlalchemy)
22

3-
from edgy.project import require
3+
from medikit import require
44

55
pytest = require('pytest')
66
python = require('python')
@@ -16,13 +16,12 @@ python.setup(
1616
author_email='romain@dorgueil.net',
1717
)
1818

19-
bonobo_version = '>=0.4,<0.5'
19+
bonobo_version = '>= 0.5.1'
20+
2021
python.add_requirements(
2122
'bonobo ' + bonobo_version,
2223
'SQLAlchemy >=1.1,<1.2',
23-
dev=[
24-
'bonobo[dev] ' + bonobo_version,
25-
],
24+
dev=['bonobo[dev] ' + bonobo_version],
2625
)
2726

2827
# vim: ft=python:

‎bonobo_sqlalchemy/readers.py

Lines changed: 43 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,51 @@
44

55

66
class Select(Configurable):
7-
query = Option(str, positional=True, required=True, default='SELECT 1') # type: str
8-
pack_size = Option(int, default=1000) # type: int
9-
limit = Option(int) # type: int
7+
"""
8+
Reads data from a database using a SQL query and a limit-offset based pagination.
109
11-
engine = Service('sqlalchemy.engine') # type: str
10+
Example:
1211
13-
def __call__(self, engine):
14-
query = self.query.strip()
15-
if query[-1] == ';':
16-
query = query[0:-1]
12+
.. code-block:: python
13+
14+
Select('SELECT * from foo;')
15+
16+
Caveats:
17+
18+
We're using "limit-offset" pagination, but limit-offset pagination can be inconsistent.
19+
20+
Suppose a user moves from page n to n+1 while simultaneously a new element is inserted into page n. This will cause
21+
both a duplication (the previously-final element of page n is pushed into page n+1) and an omission (the new
22+
element). Alternatively consider an element removed from page n just as the user moves to page n+1. The previously
23+
initial element of page n+1 will be shifted to page n and be omitted.
24+
25+
A better implementation could be to use database-side cursors, to have the external system mark the last row
26+
extracted and "stabilize" pagination. Here is an example of how this can be done (although it's not implemented in
27+
bonobo-sqlalchemy, for now).
28+
29+
.. code-block:: sql
30+
31+
-- We must be in a transaction
32+
BEGIN;
33+
-- Open a cursor for a query
34+
DECLARE select_cursor CURSOR FOR SELECT * FROM foo;
35+
-- Retrieve ten rows
36+
FETCH 10 FROM select_cursor;
37+
-- ...
38+
-- Retrieve ten more from where we left off
39+
FETCH 10 FROM select_cursor;
40+
-- All done
41+
COMMIT;
42+
43+
"""
44+
query = Option(str, positional=True, default='SELECT 1', __doc__='The actual SQL query to run.') # type: str
45+
pack_size = Option(int, required=False, default=1000, __doc__='How many rows to retrieve at once.') # type: int
46+
limit = Option(int, required=False, __doc__='Maximum rows to retrieve, in total.') # type: int
47+
48+
engine = Service('sqlalchemy.engine', __doc__='Database connection (an sqlalchemy.engine).') # type: str
49+
50+
def call(self, engine):
51+
query = self.query.strip(' \n;')
1752

1853
offset = 0
1954
while not self.limit or offset * self.pack_size < self.limit:

‎bonobo_sqlalchemy/writers.py

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,12 @@ class InsertOrUpdate(Configurable):
2323
discriminant = Option(tuple, default=('id', )) # type: tuple
2424
created_at_field = Option(str, default='created_at') # type: str
2525
updated_at_field = Option(str, default='updated_at') # type: str
26-
allowed_operations = Option(tuple, default=(INSERT, UPDATE, )) # type: tuple
26+
allowed_operations = Option(
27+
tuple, default=(
28+
INSERT,
29+
UPDATE,
30+
)
31+
) # type: tuple
2732
buffer_size = Option(int, default=1000) # type: int
2833

2934
engine = Service('sqlalchemy.engine') # type: str
@@ -156,8 +161,8 @@ def insert_or_update(self, table, connection, row):
156161
return row
157162

158163
def find(self, connection, table, row):
159-
sql = select([table]
160-
).where(and_(*(getattr(table.c, col) == row.get(col) for col in self.discriminant))).limit(1)
164+
sql = select([table]).where(and_(*(getattr(table.c, col) == row.get(col) for col in self.discriminant))
165+
).limit(1)
161166
row = connection.execute(sql).fetchone()
162167
return dict(row) if row else None
163168

‎requirements-dev.txt

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,21 @@
11
-e .[dev]
22
appdirs==1.4.3
3-
bonobo==0.4.3
4-
certifi==2017.4.17
3+
bonobo==0.5.1
4+
certifi==2017.7.27.1
55
chardet==3.0.4
66
colorama==0.3.9
77
coverage==4.4.1
8-
fs==2.0.4
9-
idna==2.5
8+
fs==2.0.12
9+
idna==2.6
1010
packaging==16.8
1111
pbr==3.1.1
12-
psutil==5.2.2
12+
psutil==5.4.0
1313
py==1.4.34
1414
pyparsing==2.2.0
1515
pytest-cov==2.5.1
16-
pytest==3.1.3
16+
pytest==3.2.3
1717
pytz==2017.2
18-
requests==2.18.1
19-
six==1.10.0
20-
stevedore==1.24.0
21-
urllib3==1.21.1
18+
requests==2.18.4
19+
six==1.11.0
20+
stevedore==1.27.1
21+
urllib3==1.22

‎requirements.txt

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,18 @@
11
-e .
22
appdirs==1.4.3
3-
bonobo==0.4.3
4-
certifi==2017.4.17
3+
bonobo==0.5.1
4+
certifi==2017.7.27.1
55
chardet==3.0.4
66
colorama==0.3.9
7-
fs==2.0.4
8-
idna==2.5
7+
fs==2.0.12
8+
idna==2.6
99
packaging==16.8
1010
pbr==3.1.1
11-
psutil==5.2.2
11+
psutil==5.4.0
1212
pyparsing==2.2.0
1313
pytz==2017.2
14-
requests==2.18.1
15-
six==1.10.0
16-
sqlalchemy==1.1.11
17-
stevedore==1.24.0
18-
urllib3==1.21.1
14+
requests==2.18.4
15+
six==1.11.0
16+
sqlalchemy==1.1.14
17+
stevedore==1.27.1
18+
urllib3==1.22

‎setup.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# This file is autogenerated by edgy.project code generator.
1+
# This file is autogenerated by medikit code generator.
22
# All changes will be overwritten.
33

44
from setuptools import setup, find_packages
@@ -51,10 +51,10 @@ def execfile(fname, globs, locs=None):
5151
classifiers=classifiers,
5252
packages=find_packages(exclude=['ez_setup', 'example', 'test']),
5353
include_package_data=True,
54-
install_requires=['SQLAlchemy (>= 1.1, < 1.2)', 'bonobo (>= 0.4, < 0.5)'],
54+
install_requires=['SQLAlchemy (>= 1.1, < 1.2)', 'bonobo (>= 0.5.1)'],
5555
extras_require={
5656
'dev':
57-
['bonobo (>= 0.4, < 0.5)', 'coverage (>= 4.4, < 5.0)', 'pytest (>= 3.1, < 4.0)', 'pytest-cov (>= 2.5, < 3.0)']
57+
['bonobo (>= 0.5.1)', 'coverage (>= 4.4, < 5.0)', 'pytest (>= 3.1, < 4.0)', 'pytest-cov (>= 2.5, < 3.0)']
5858
},
5959
url='https://www.bonobo-project.org/with/sqlalchemy',
6060
download_url='https://github.com/python-bonobo/bonobo-sqlalchemy/tarball/{version}'.format(version=version),

0 commit comments

Comments
 (0)
0