8000 Merge pull request #39 from hartym/develop · python-bonobo/bonobo-sqlalchemy@e517fc8 · GitHub
[go: up one dir, main page]
Skip to content

Commit e517fc8

Browse files
hartymhartym
authored
Merge pull request #39 from hartym/develop
Develop
2 parents e39b498 + 240dfaa commit e517fc8

File tree

12 files changed

+429
-77
lines changed

12 file changed

+429
-77
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
/.cache
66
/.coverage
77
/.idea
8+
/.medikit
89
/.medikit-pipeline
910
/.python*-*
1011
/.release

‎.travis.yml

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
language: python
2+
sudo: required
3+
services:
4+
- docker
25
python:
3-
- 3.5
4-
- 3.5-dev
5-
- 3.6
6-
- 3.6-dev
7-
- nightly
6+
- 3.5
7+
- 3.5-dev
8+
- 3.6
9+
- 3.6-dev
10+
- nightly
811
install:
9-
- make install-dev
10-
- pip install coveralls
12+
- make install-dev
13+
- pip install coveralls docker psycopg2-binary
1114
script:
12-
- make clean test
15+
- make clean test
1316
after_success:
14-
- coveralls
17+
- coveralls

‎Makefile

Lines changed: 58 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,52 +1,82 @@
1-
# Generated by Medikit 0.4.5 on 2018-01-01.
1+
# Generated by Medikit 0.6.3 on 2018-06-11.
22
# All changes will be overriden.
3+
# Edit Projectfile and run “make update” (or “medikit update”) to regenerate.
34

45
PACKAGE ?= bonobo_sqlalchemy
5-
PYTHON ?= $(shell which python)
6+
PYTHON ?= $(shell which python || echo python)
67
PYTHON_BASENAME ?= $(shell basename $(PYTHON))
78
PYTHON_DIRNAME ?= $(shell dirname $(PYTHON))
89
PYTHON_REQUIREMENTS_FILE ?= requirements.txt
10+
PYTHON_REQUIREMENTS_INLINE ?=
911
PYTHON_REQUIREMENTS_DEV_FILE ?= requirements-dev.txt
12+
PYTHON_REQUIREMENTS_DEV_INLINE ?=
1013
QUICK ?=
11-
PIP ?= $(PYTHON_DIRNAME)/pip
14+
PIP ?= $(PYTHON) -m pip
1215
PIP_INSTALL_OPTIONS ?=
1316
VERSION ?= $(shell git describe 2>/dev/null || git rev-parse --short HEAD)
1417
PYTEST ?= $(PYTHON_DIRNAME)/pytest
1518
PYTEST_OPTIONS ?= --capture=no --cov=$(PACKAGE) --cov-report html
1619
YAPF ?= $(PYTHON) -m yapf
1720
YAPF_OPTIONS ?= -rip
21+
MEDIKIT ?= $(PYTHON) -m medikit
22+
MEDIKIT_UPDATE_OPTIONS ?=
23+
MEDIKIT_VERSION ?= 0.6.3
1824

19-
.PHONY: clean format install install-dev test update update-requirements
25+
.PHONY: clean format help install install-dev medikit quick test update update-requirements
2026

21-
# Installs the local project dependencies.
22-
install:
23-
if [ -z "$(QUICK)" ]; then \
24-
$(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_FILE) ; \
25-
fi
27+
install: .medikit/install ## Installs the project.
28+
.medikit/install: $(PYTHON_REQUIREMENTS_FILE) setup.py
29+
$(eval target := $(shell echo $@ | rev | cut -d/ -f1 | rev))
30+
ifeq ($(filter quick,$(MAKECMDGOALS)),quick)
31+
@printf "Skipping \033[36m%s\033[0m because of \033[36mquick\033[0m target.\n" $(target)
32+
else ifneq ($(QUICK),)
33+
@printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target)
34+
else
35+
@printf "Applying \033[36m%s\033[0m target...\n" $(target)
36+
$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel
37+
$(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_INLINE) -r $(PYTHON_REQUIREMENTS_FILE)
38+
@mkdir -p .medikit; touch $@
39+
endif
2640

27-
# Installs the local project dependencies, including development-only libraries.
28-
install-dev:
29-
if [ -z "$(QUICK)" ]; then \
30-
$(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \
31-
fi
41+
clean: ## Cleans up the working copy.
42+
rm -rf build dist *.egg-info .medikit/install .medikit/install-dev
43+
find . -name __pycache__ -type d | xargs rm -rf
3244

33-
# Cleans up the local mess.
34-
clean:
35-
rm -rf build dist *.egg-info
45+
install-dev: .medikit/install-dev ## Installs the project (with dev dependencies).
46+
.medikit/install-dev: $(PYTHON_REQUIREMENTS_DEV_FILE) setup.py
47+
$(eval target := $(shell echo $@ | rev | cut -d/ -f1 | rev))
48+
ifeq ($(filter quick,$(MAKECMDGOALS)),quick)
49+
@printf "Skipping \033[36m%s\033[0m because of \033[36mquick\033[0m target.\n" $(target)
50+
else ifneq ($(QUICK),)
51+
@printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target)
52+
else
53+
@printf "Applying \033[36m%s\033[0m target...\n" $(target)
54+
$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel
55+
$(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_DEV_INLINE) -r $(PYTHON_REQUIREMENTS_DEV_FILE)
56+
@mkdir -p .medikit; touch $@
57+
endif
3658

37-
# Update project artifacts using medikit, after installing it eventually.
38-
update:
39-
python -c 'import medikit; print(medikit.__version__)' || pip install medikit;
40-
$(PYTHON) -m medikit update
59+
quick: #
60+
@printf ""
4161

42-
# Remove requirements files and update project artifacts using medikit, after installing it eventually.
43-
update-requirements:
44-
rm -rf requirements*.txt
45-
$(MAKE) update
46-
47-
test: install-dev
62+
test: install-dev ## Runs the test suite.
4863
$(PYTEST) $(PYTEST_OPTIONS) tests
4964

50-
format: install-dev
65+
format: install-dev ## Reformats the whole python codebase using yapf.
5166
$(YAPF) $(YAPF_OPTIONS) .
5267
$(YAPF) $(YAPF_OPTIONS) Projectfile
68+
69+
medikit: # Checks installed medikit version and updates it if it is outdated.
70+
@$(PYTHON) -c 'import medikit, pip, sys; from packaging.version import Version; sys.exit(0 if (Version(medikit.__version__) >= Version("$(MEDIKIT_VERSION)")) and (Version(pip.__version__) < Version("10")) else 1)' || $(PYTHON) -m pip install -U "pip ~=10.0" "medikit>=$(MEDIKIT_VERSION)"
71+
72+
update: medikit ## Update project artifacts using medikit.
73+
$(MEDIKIT) update $(MEDIKIT_UPDATE_OPTIONS)
74+
75+
update-requirements: ## Update project artifacts using medikit, including requirements files.
76+
MEDIKIT_UPDATE_OPTIONS="--override-requirements" $(MAKE) update
77+
78+
help: ## Shows available commands.
79+
@echo "Available commands:"
80+
@echo
81+
@grep -E '^[a-zA-Z_-]+:.*?##[\s]?.*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?##"}; {printf " make \033[36m%-30s\033[0m %s\n", $$1, $$2}'
82+
@echo

‎Projectfile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ yapf = require('yapf')
88

99
python.setup(
1010
name='bonobo_sqlalchemy',
11+
python_requires='>=3.5',
1112
description='Bonobo SQLAlchemy Extension',
1213
license='Apache License, Version 2.0',
1314
url='https://www.bonobo-project.org/with/sqlalchemy',

‎bonobo_sqlalchemy/readers.py

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,11 +52,15 @@ def __call__(self, context, *, engine):
5252
query = self.query.strip(' \n;')
5353

5454
offset = 0
55+
max_limit = self.limit or self.pack_size
56+
57+
assert self.pack_size > 0, 'Pack size must be > 0 for now.'
58+
5559
while not self.limit or offset * self.pack_size < self.limit:
5660
results = engine.execute(
5761
'{query} LIMIT {limit}{offset}'.format(
5862
query=query,
59-
limit=self.pack_size,
63+
limit=max(min(self.pack_size, max_limit - offset * self.pack_size), 0),
6064
offset=' OFFSET {}'.format(offset * self.pack_size) if offset else ''
6165
),
6266
use_labels=True
@@ -65,8 +69,8 @@ def __call__(self, context, *, engine):
6569
if not len(results):
6670
break
6771

68-
for i, row in enumerate(results):
69-
if not i:
72+
for row in results:
73+
if not context.output_type:
7074
context.set_output_fields(row.keys())
7175
yield tuple(row)
7276

‎examples/select_insert.py

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
import sqlalchemy
2+
3+
import bonobo
4+
import bonobo_sqlalchemy
5+
6+
7+
def get_services():
8+
return {'sqlalchemy.pgengine': sqlalchemy.create_engine('postgresql+psycopg2://@localhost:5432/example')}
9+
10+
11+
def get_graph(**options):
12+
"""This function builds the graph that needs to be executed. :return: bonobo.Graph """
13+
return bonobo.Graph(
14+
bonobo_sqlalchemy.Select('SELECT * FROM table', engine='sqlalchemy.pgengine'),
15+
bonobo_sqlalchemy.InsertOrUpdate(table_name='table_1', engine='sqlalchemy.pgengine'),
16+
)
17+
18+
19+
# The __main__ block actually execute the graph.
20+
if __name__ == '__main__':
21+
parser = bonobo.get_argument_parser()
22+
with bonobo.parse_args(parser) as options:
23+
bonobo.run(get_graph(**options), services=get_services(**options))

‎examples/sequel_extract.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
def get_graph(**options):
99
graph = bonobo.Graph()
1010
graph.add_chain(
11-
bonobo_sqlalchemy.Select('SELECT * FROM example', limit=100),
11+
bonobo_sqlalchemy.Select('SELECT * FROM example', limit=100, pack_size=9),
1212
bonobo.PrettyPrinter(),
1313
)
1414

‎requirements-dev.txt

Lines changed: 40 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,54 @@
11
-e .[dev]
2+
-r requirements.txt
3+
alabaster==0.7.10
24
appdirs==1.4.3
3-
attrs==17.4.0
4-
bonobo==0.6.0
5-
certifi==2017.11.5
5+
arrow==0.12.1
6+
atomicwrites==1.1.5
7+
attrs==18.1.0
8+
babel==2.6.0
9+
binaryornot==0.4.4
10+
bonobo[dev]==0.6.2
11+
certifi==2018.4.16
612
chardet==3.0.4
13+
click==6.7
714
colorama==0.3.9
8-
coverage==4.4.2
9-
fs==2.0.17
10-
graphviz==0.8.2
15+
cookiecutter==1.5.1
16+
coverage==4.5.1
17+
docutils==0.14
18+
fs==2.0.23
19+
future==0.16.0
20+
graphviz==0.8.3
1121
idna==2.6
22+
imagesize==1.0.0
23+
jinja2-time==0.2.0
1224
jinja2==2.10
1325
markupsafe==1.0
14-
mondrian==0.6.1
15-
packaging==16.8
16-
pbr==3.1.1
26+
mondrian==0.7.0
27+
more-itertools==4.2.0
28+
packaging==17.1
29+
pbr==4.0.4
1730
pluggy==0.6.0
18-
psutil==5.4.3
19-
py==1.5.2
31+
poyo==0.4.1
32+
psutil==5.4.6
33+
py==1.5.3
34+
pygments==2.2.0
2035
pyparsing==2.2.0
2136
pytest-cov==2.5.1
22-
pytest==3.3.1
23-
python-slugify==1.2.4
24-
pytz==2017.3
37+
pytest-timeout==1.2.1
38+
pytest==3.6.1
39+
python-dateutil==2.7.3
40+
python-slugify==1.2.5
41+
pytz==2018.4
2542
requests==2.18.4
2643
six==1.11.0
44+
snowballstemmer==1.2.1
45+
sphinx-sitemap==0.2
46+
sphinx==1.7.5
47+
sphinxcontrib-websupport==1.1.0
2748
stevedore==1.28.0
28-
unidecode==0.4.21
49+
typing==3.6.4
50+
unidecode==1.0.22
2951
urllib3==1.22
30-
whistle==1.0.0
31-
yapf==0.20.0
52+
whichcraft==0.4.1
53+
whistle==1.0.1
54+
yapf==0.22.0

‎requirements.txt

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,26 @@
11
-e .
22
appdirs==1.4.3
3-
bonobo==0.6.0
4-
certifi==2017.11.5
3+
bonobo==0.6.2
4+
certifi==2018.4.16
55
chardet==3.0.4
66
colorama==0.3.9
7-
fs==2.0.17
8-
graphviz==0.8.2
7+
fs==2.0.23
8+
graphviz==0.8.3
99
idna==2.6
1010
jinja2==2.10
1111
markupsafe==1.0
12-
mondrian==0.6.1
13-
packaging==16.8
14-
pbr==3.1.1
15-
psutil==5.4.3
12+
mondrian==0.7.0
13+
packaging==17.1
14+
pbr==4.0.4
15+
psutil==5.4.6
1616
pyparsing==2.2.0
17-
python-slugify==1.2.4
18-
pytz==2017.3
17+
python-slugify==1.2.5
18+
pytz==2018.4
1919
requests==2.18.4
2020
six==1.11.0
21-
sqlalchemy==1.2.0
21+
sqlalchemy==1.2.8
2222
stevedore==1.28.0
23-
unidecode==0.4.21
23+
typing==3.6.4
24+
unidecode==1.0.22
2425
urllib3==1.22
25-
whistle==1.0.0
26+
whistle==1.0.1

‎setup.py

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1-
# This file is autogenerated by medikit code generator.
2-
# All changes will be overwritten.
1+
# Generated by Medikit 0.6.3 on 2018-06-11.
2+
# All changes will be overriden.
3+
# Edit Projectfile and run “make update” (or “medikit update”) to regenerate.
34

45
from setuptools import setup, find_packages
56
from codecs import open
@@ -46,17 +47,15 @@ def execfile(fname, globs, locs=None):
4647
description='Bonobo SQLAlchemy Extension',
4748
license='Apache License, Version 2.0',
4849
name='bonobo_sqlalchemy',
50+
python_requires='>=3.5',
4951
version=version,
5052
long_description=long_description,
5153
classifiers=classifiers,
5254
packages=find_packages(exclude=['ez_setup', 'example', 'test']),
5355
include_package_data=True,
5456
install_requires=['SQLAlchemy (~= 1.2)', 'bonobo (~= 0.6.0)'],
5557
extras_require={
56-
'dev': [
57-
'bonobo (~= 0.6.0)', 'coverage (>= 4.4, < 5.0)', 'pytest (>= 3.1, < 4.0)', 'pytest-cov (>= 2.5, < 3.0)',
58-
'yapf'
59-
]
58+
'dev': ['bonobo[dev] (~= 0.6.0)', 'coverage (~= 4.4)', 'pytest (~= 3.4)', 'pytest-cov (~= 2.5)', 'yapf']
6059
},
6160
url='https://www.bonobo-project.org/with/sqlalchemy',
6261
download_url='https://github.com/python-bonobo/bonobo-sqlalchemy/tarball/{version}'.format(version=version),

0 commit comments

Comments
 (0)
0