8000 Add Composer samples that explicitly call python2 (#1711) · cevaris/python-docs-samples@b4098b6 · GitHub
[go: up one dir, main page]
Skip to content

Commit b4098b6

Browse files
tswasttswast
authored
Add Composer samples that explicitly call python2 (GoogleCloudPlatform#1711)
* Add Composer samples that explicitly call python2 * Use new unit_testing module for Python 2 DAG samples.
1 parent 90ab014 commit b4098b6

File tree

5 files changed

+179
-0
lines changed

5 files changed

+179
-0
lines changed

‎composer/data/python2_script.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Copyright 2018 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
print 'Output from Python 2.' # noqa

‎composer/workflows/bashoperator_python2.py

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# Copyright 2018 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# [START composer_bashoperator_python2]
16+
import datetime
17+
18+
from airflow import models
19+
from airflow.operators import bash_operator
20+
21+
22+
yesterday = datetime.datetime.combine(
23+
datetime.datetime.today() - datetime.timedelta(1),
24+
datetime.datetime.min.time())
25+
26+
27+
default_dag_args = {
28+
# Setting start date as yesterday starts the DAG immediately when it is
29+
# detected in the Cloud Storage bucket.
30+
'start_date': yesterday,
31+
}
32+
33+
with models.DAG(
34+
'composer_sample_bashoperator_python2',
35+
schedule_interval=datetime.timedelta(days=1),
36+
default_args=default_dag_args) as dag:
37+
38+
run_python2 = bash_operator.BashOperator(
39+
task_id='run_python2',
40+
# This example runs a Python script from the data folder to prevent
41+
# Airflow from attempting to parse the script as a DAG.
42+
bash_command='python2 /home/airflow/gcs/data/python2_script.py',
43+
)
44+
# [END composer_bashoperator_python2]

‎composer/workflows/bashoperator_python2_test.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Copyright 2018 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from . import unit_testing
16+
17+
18+
def test_dag_import():
19+
"""Test that the DAG file can be successfully imported.
20+
21+
This tests that the DAG can be parsed, but does not run it in an Airflow
22+
environment. This is a recommended sanity check by the official Airflow
23+
docs: https://airflow.incubator.apache.org/tutorial.html#testing
24+
"""
25+
from . import bashoperator_python2 as module
26+
unit_testing.assert_has_valid_dag(module)

‎composer/workflows/pythonvirtualenvoperator_python2.py

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# Copyright 2018 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# [START composer_pythonvirtualenvoperator_python2]
16+
import datetime
17+
18+
from airflow import models
19+
from airflow.operators import python_operator
20+
21+
22+
def python2_function():
23+
"""A function which has not been converted to Python 3."""
24+
# Use the global variable virtualenv_string_args to pass in values when the
25+
# Python version differs from that used by the Airflow process.
26+
global virtualenv_string_args
27+
28+
# Imports must happen within the function when run with the
29+
# PythonVirtualenvOperator.
30+
import cStringIO
31+
import logging
32+
33+
arg0 = virtualenv_string_args[0]
34+
buffer = cStringIO.StringIO()
35+
buffer.write('Wrote an ASCII string to buffer:\n')
36+
buffer.write(arg0)
37+
logging.info(buffer.getvalue())
38+
39+
40+
yesterday = datetime.datetime.combine(
41+
datetime.datetime.today() - datetime.timedelta(1),
42+
datetime.datetime.min.time())
43+
44+
45+
default_dag_args = {
46+
# Setting start date as yesterday starts the DAG immediately when it is
47+
# detected in the Cloud Storage bucket.
48+
'start_date': yesterday,
49+
}
50+
51+
with models.DAG(
52+
'composer_sample_pythonvirtualenvoperator_python2',
53+
schedule_interval=datetime.timedelta(days=1),
54+
default_args=default_dag_args) as dag:
55+
56+
# Use the PythonVirtualenvOperator to select an explicit python_version.
57+
run_python2 = python_operator.PythonVirtualenvOperator(
58+
task_id='run_python2',
59+
python_callable=python2_function,
60+
python_version='2',
61+
string_args=['An example input string'],
62+
)
63+
# [END composer_pythonvirtualenvoperator_python2]

‎composer/workflows/pythonvirtualenvoperator_python2_test.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Copyright 2018 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import sys
16+
17+
import pytest
18+
19+
from . import unit_testing
20+
21+
22+
@pytest.mark.skipif(sys.version_info >= (3, 0), reason="requires Python 2")
23+
def test_dag_import():
24+
"""Test that the DAG file can be successfully imported.
25+
26+
This tests that the DAG can be parsed, but does not run it in an Airflow
27+
environment. This is a recommended sanity check by the official Airflow
28+
docs: https://airflow.incubator.apache.org/tutorial.html#testing
29+
"""
30+
from . import pythonvirtualenvoperator_python2 as module
31+
unit_testing.assert_has_valid_dag(module)

0 commit comments

Comments
 (0)
0