8000 Remove hardcoded dependencies from install instruction (#969) · IntelPython/sdc@f9dd4d7 · GitHub
[go: up one dir, main page]
Skip to content
This repository was archived by the owner on Feb 2, 2024. It is now read-only.

Commit f9dd4d7

Browse files
kozlov-alexeykozlov-alexey
authored
Remove hardcoded dependencies from install instruction (#969)
* Remove hardcoded dependencies from install instruction Motivation: installation instruction is often updated at the same time when SDC moves to new version of dependencies, and while building SDC from source works, installing from pre-built binaries might fail since package is uploaded to intel/label/beta only after all validation checks. * Verify install steps in azure CI * Fixing PEP * Fixes pack of failed tests (#974) * Fixes pack of failed tests * Skipping floordiv test due to diff behavior in Stock vs Intel numpy * Fixing bugs test_conda_install run
1 parent 1574682 commit f9dd4d7

9 files changed

+100
-18
lines changed

‎README.rst

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,13 @@ Distribution includes Intel® SDC for Python 3.6 and Python 3.7 for Windows and
3737

3838
Intel® SDC conda package can be installed using the steps below::
3939

40-
> conda create -n sdc-env python=<3.7 or 3.6> pyarrow=2.0.0 pandas=1.2.0 -c anaconda -c conda-forge
40+
> conda create -n sdc-env python=<3.7 or 3.6> -c anaconda -c conda-forge
4141
> conda activate sdc-env
4242
> conda install sdc -c intel/label/beta -c intel -c defaults -c conda-forge --override-channels
4343

4444
Intel® SDC wheel package can be installed using the steps below::
4545

46-
> conda create -n sdc-env python=<3.7 or 3.6> pip pyarrow=2.0.0 pandas=1.2.0 -c anaconda -c conda-forge
46+
> conda create -n sdc-env python=<3.7 or 3.6> pip -c anaconda -c conda-forge
4747
> conda activate sdc-env
4848
> pip install --index-url https://pypi.anaconda.org/intel/label/beta/simple --extra-index-url https://pypi.anaconda.org/intel/simple --extra-index-url https://pypi.org/simple sdc
4949

‎buildscripts/azure/template-linux-macos.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,9 @@ jobs:
4343
python buildscripts/build_doc.py --python=$PYTHON_VER --sdc-channel=./sdc-build
4444
displayName: 'Build Intel SDC documentation'
4545
continueOnError: ${{ parameters.allowFailure }}
46+
47+
- script: |
48+
source $HOME/miniconda3/bin/activate
49+
python buildscripts/test_conda_install.py --python=$PYTHON_VER --channels="-c anaconda -c conda-forge"
50+
displayName: 'Test SDC installation from conda'
51+
continueOnError: ${{ parameters.allowFailure }}

‎buildscripts/azure/template-windows.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,9 @@ jobs:
3636
python buildscripts\\run_examples.py --python=%PYTHON_VER% --sdc-channel=.\\sdc-build
3737
displayName: 'Run Intel SDC examples'
3838
continueOnError: ${{ parameters.allowFailure }}
39+
40+
- script: |
41+
call C:\\Miniconda\\Scripts\activate.bat
42+
python buildscripts\\test_conda_install.py --python=%PYTHON_VER% --channels="-c anaconda -c conda-forge"
43+
displayName: 'Test SDC installation from conda'
44+
continueOnError: ${{ parameters.allowFailure }}

‎buildscripts/build_doc.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,12 +88,13 @@ def publish_doc(sdc_utils):
8888
parser = argparse.ArgumentParser()
8989
parser.add_argument('--python', default='3.7', choices=['3.6', '3.7', '3.8'],
9090
help='Python version, default = 3.7')
91+
parser.add_argument('--channels', default=None, help='Default env channels')
9192
parser.add_argument('--sdc-channel', default=None, help='Intel SDC channel')
9293
parser.add_argument('--publish', action='store_true', help='Publish documentation to sdc-doc')
9394

9495
args = parser.parse_args()
9596

96-
sdc_utils = SDC_Build_Utilities(args.python, args.sdc_channel)
97+
sdc_utils = SDC_Build_Utilities(args.python, args.channels, args.sdc_channel)
9798
sdc_utils.log_info('Build Intel(R) SDC documentation', separate=True)
9899
sdc_utils.log_info(sdc_utils.line_double)
99100
sdc_utils.create_environment(['sphinx', 'sphinxcontrib-programoutput'])

‎buildscripts/run_benchmarks.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,14 +47,15 @@ def run_benchmarks(sdc_utils, args_list, num_threads_list):
4747
parser = argparse.ArgumentParser()
4848
parser.add_argument('--python', default='3.7', choices=['3.6', '3.7', '3.8'],
4949
help='Python version, default = 3.7')
50+
parser.add_argument('--channels', default=None, help='Default env channels')
5051
parser.add_argument('--sdc-channel', default=None, help='Intel SDC channel')
5152
parser.add_argument('--args-list', required=True, nargs='+', help='List of arguments sets for benchmarks')
5253
parser.add_argument('--num-threads-list', required=True, nargs='+',
5354
help='List of values for NUMBA_NUM_THREADS env variable')
5455

5556
args = parser.parse_args()
5657

57-
sdc_utils = SDC_Build_Utilities(args.python, args.sdc_channel)
58+
sdc_utils = SDC_Build_Utilities(args.python, args.channels, args.sdc_channel)
5859
sdc_utils.log_info('Run Intel(R) SDC benchmarks', separate=True)
5960
sdc_utils.log_info(sdc_utils.line_double)
6061
sdc_utils.create_environment(['openpyxl', 'xlrd'])

‎buildscripts/run_examples.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,11 +119,12 @@ def run_examples(sdc_utils):
119119
parser = argparse.ArgumentParser()
120120
parser.add_argument('--python', default='3.7', choices=['3.6', '3.7', '3.8'],
121121
help='Python version, default = 3.7')
122+
parser.add_argument('--channels', default=None, help='Default env channels')
122123
parser.add_argument('--sdc-channel', default=None, help='Intel SDC channel')
123124

124125
args = parser.parse_args()
125126

126-
sdc_utils = SDC_Build_Utilities(args.python, args.sdc_channel)
127+
sdc_utils = SDC_Build_Utilities(args.python, args.channels, args.sdc_channel)
127128
sdc_utils.log_info('Run Intel(R) SDC examples', separate=True)
128129
sdc_utils.log_info(sdc_utils.line_double)
129130
sdc_utils.create_environment()

‎buildscripts/test.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,12 @@ def run_tests(sdc_utils):
4848
parser = argparse.ArgumentParser()
4949
parser.add_argument('--python', default='3.7', choices=['3.6', '3.7', '3.8'],
5050
help='Python version, default = 3.7')
51-
parser.add_argument('--sdc-channel', required=True, help='Local Intel SDC channel')
51+
parser.add_argument('--channels', default=None, help='Default env channels')
52+
parser.add_argument('--sdc-channel', default=None, help='Intel SDC channel')
5253

5354
args = parser.parse_args()
5455

55-
sdc_utils = SDC_Build_Utilities(args.python, args.sdc_channel)
56+
sdc_utils = SDC_Build_Utilities(args.python, args.channels, args.sdc_channel)
5657
sdc_utils.log_info('Run Intel(R) SDC tests', separate=True)
5758
sdc_utils.log_info(sdc_utils.line_double)
5859
sdc_utils.create_environment(['conda-build'])

‎buildscripts/test_conda_install.py

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# *****************************************************************************
2+
# Copyright (c) 2019-2021, Intel Corporation All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions are met:
6+
#
7+
# Redistributions of source code must retain the above copyright notice,
8+
# this list of conditions and the following disclaimer.
9+
#
10+
# Redistributions in binary form must reproduce the above copyright notice,
11+
# this list of conditions and the following disclaimer in the documentation
12+
# and/or other materials provided with the distribution.
13+
#
14+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15+
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16+
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
18+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21+
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22+
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23+
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24+
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25+
# *****************************************************************************
26+
27+
28+
import argparse
29+
import os
30+
import shutil
31+
import traceback
32+
import re
33+
34+
from pathlib import Path
35+
from utilities import SDC_Build_Utilities
36+
37+
38+
def check_sdc_installed(sdc_utils, sdc_package):
39+
cmd_output = sdc_utils.get_command_output('conda list sdc')
40+
pattern = sdc_package.replace('=', r'\s+')
41+
return re.search(pattern, cmd_output)
42+
43+
44+
if __name__ == '__main__':
45+
parser = argparse.ArgumentParser()
46+
parser.add_argument('--python', default='3.7', choices=['3.6', '3.7', '3.8'],
47+
help='Python version, default = 3.7')
48+
parser.add_argument('--channels', default=None, help='Default env channels')
49+
parser.add_argument('--sdc-channel', default=None, help='Intel SDC channel')
50+
51+
args = parser.parse_args()
52+
53+
sdc_utils = SDC_Build_Utilities(args.python, args.channels, args.sdc_channel)
54+
sdc_utils.log_info('Test Intel(R) SDC conda install', separate=True)
55+
sdc_utils.log_info(sdc_utils.line_double)
56+
sdc_utils.create_environment()
57+
sdc_package = f'sdc={sdc_utils.get_sdc_version_from_channel()}'
58+
59+
# channels list is aligned with install instruction in README.rst
60+
install_channels = "-c intel/label/beta -c intel -c defaults -c conda-forge"
61+
sdc_utils.install_conda_package([sdc_package], channels=install_channels)
62+
63+
assert check_sdc_installed(sdc_utils, sdc_package), "SDC package was not installed"

‎buildscripts/utilities.py

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
# *****************************************************************************
2626

2727

28-
2928
import json
3029
import os
3130
import platform
@@ -41,7 +40,7 @@
4140

4241

4342
class SDC_Build_Utilities:
44-
def __init__(self, python, sdc_local_channel=None):
43+
def __init__(self, python, channels=None, sdc_channel=None):
4544
self.src_path = Path(__file__).resolve().parent.parent
4645
self.env_name = 'sdc_env'
4746
self.python = python
@@ -52,10 +51,14 @@ def __init__(self, python, sdc_local_channel=None):
5251
self.line_single = '-'*80
5352

5453
# Set channels
55-
self.channel_list = ['-c', 'defaults', '-c', 'conda-forge']
56-
if sdc_local_channel:
57-
sdc_local_channel = Path(sdc_local_channel).resolve().as_uri()
58-
self.channel_list = ['-c', sdc_local_channel] + self.channel_list
54+
build_channels = ['-c', 'defaults', '-c', 'conda-forge']
55+
self.channel_list = build_channels if channels is None else channels.split()
56+
if sdc_channel:
57+
self.sdc_channel = Path(sdc_channel).resolve().as_uri()
58+
self.channel_list = ['-c', self.sdc_channel] + self.channel_list
59+
else:
60+
self.sdc_channel = 'intel/label/beta'
61+
# keep SDC channel but do not add it to env channels
5962
self.channels = ' '.join(self.channel_list)
6063

6164
# Conda activate command and conda croot (build) folder
@@ -92,12 +95,13 @@ def create_environment(self, packages_list=[]):
9295

9396
return
9497

95-
def install_conda_package(self, packages_list):
98+
def install_conda_package(self, packages_list, channels=None):
9699
assert type(packages_list) == list, 'Argument should be a list'
97100

98101
self.log_info(f'Install {" ".join(packages_list)} to {self.env_name} conda environment')
99102
install_args = ['-n', self.env_name]
100-
install_args += self.channel_list + ['--override-channels', '-q', '-y'] + packages_list
103+
replace_channels = channels.split() if channels else self.channel_list
104+
install_args += replace_channels + ['--override-channels', '-q', '-y'] + packages_list
101105
self.log_info(self.__run_conda_command(Conda_Commands.INSTALL, install_args))
102106

103107
return
@@ -140,12 +144,11 @@ def log_info(self, msg, separate=False):
140144
def get_sdc_version_from_channel(self):
141145
python_version = 'py' + self.python.replace('.', '')
142146

143-
# Get Intel SDC version from first channel in channel_list
144-
search_args = ['sdc', '-c', self.channel_list[1], '--override-channels', '--json']
147+
search_args = ['sdc', '-c', self.sdc_channel, '--override-channels', '--json']
145148
search_result = self.__run_conda_command(Conda_Commands.SEARCH, search_args)
146149

147150
repo_data = json.loads(search_result)
148-
for package_data in repo_data['sdc']:
151+
for package_data in reversed(repo_data['sdc']):
149152
sdc_version = package_data['version']
150153
sdc_build = package_data['build']
151154
if python_version in sdc_build:

0 commit comments

Comments
 (0)
0