8000 build simplification by stonebig · Pull Request #298 · winpython/winpython · GitHub
[go: up one dir, main page]
Skip to content

build simplification #298

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 19, 2016
Merged

build simplification #298

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
11a47c4
finish removing archis
Mar 19, 2016
a3c426a
remove rootdir usage
Mar 19, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 17 additions & 24 deletions diff.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,30 +61,24 @@ def upgrade_wiki(self, other):
other.version, self.version, self.description)


def get_basedir(version, rootdir=None):
"""Return basedir from WinPython version"""
rootdir = rootdir if rootdir is not None else utils.ROOT_DIR
assert rootdir is not None, "The *rootdir* directory must be specified"
return osp.join(rootdir, 'basedir%s' % version[::2][:2])


class PackageIndex(object):
WINPYTHON_PATTERN = r'\#\# WinPython ([0-9\.a-zA-Z]*)'
TOOLS_LINE = '### Tools'
PYTHON_PACKAGES_LINE = '### Python packages'
HEADER_LINE1 = 'Name | Version | Description'
HEADER_LINE2 = '-----|---------|------------'

def __init__(self, version, rootdir=None, flavor=''):
def __init__(self, version, basedir=None, flavor=''):
self.version = version
self.other_packages = {}
self.python_packages = {}
self.flavor = flavor
basedir = get_basedir(version, rootdir=rootdir)
self.basedir = basedir
self.from_file(basedir)

def from_file(self, basedir):
fname = osp.join(basedir, 'build%s' % self.flavor,
#fname = osp.join(basedir, 'build%s' % self.flavor,
fname = osp.join(CHANGELOGS_DIR,
'WinPython%s-%s.md' % (self.flavor, self.version))
with open(fname, 'r') as fdesc: # python3 doesn't like 'rb'
text = fdesc.read()
Expand Down Expand Up @@ -154,9 +148,9 @@ def diff_package_dicts(dict1_in, dict2_in):
return text


def find_closer_version(version1, rootdir=None, flavor=''):
def find_closer_version(version1, basedir=None, flavor=''):
"""Find version which is the closest to `version`"""
builddir = osp.join(get_basedir(version1, rootdir), 'build%s' % flavor)
builddir = osp.join(basedir, 'build%s' % flavor)
func = lambda name: re.match(r'WinPython%s-([0-9\.]*)\.(txt|md)' % flavor, name)
versions = [func(name).groups()[0]
for name in os.listdir(builddir) if func(name)]
Expand All @@ -170,20 +164,20 @@ def find_closer_version(version1, rootdir=None, flavor=''):
return versions[index-1]


def compare_package_indexes(version2, version1=None, rootdir=None, flavor='',
def compare_package_indexes(version2, version1=None, basedir=None, flavor='',
flavor1=None):
"""Compare two package index Wiki pages"""
if version1 is None:
version1 = find_closer_version(version2, rootdir=rootdir,
version1 = find_closer_version(version2, basedir=basedir,
flavor=flavor)
flavor1 = flavor1 if flavor1 is not None else flavor
text = '\r\n'.join(["## History of changes for WinPython %s" %
(version2+flavor),
"", "The following changes were made to WinPython "
"distribution since version %s." % (version1+flavor1),
"", ""])
pi1 = PackageIndex(version1, rootdir=rootdir, flavor=flavor1)
pi2 = PackageIndex(version2, rootdir=rootdir, flavor=flavor)
pi1 = PackageIndex(version1, basedir=basedir, flavor=flavor1)
pi2 = PackageIndex(version2, basedir=basedir, flavor=flavor)
tools_text = diff_package_dicts(pi1.other_packages, pi2.other_packages)
if tools_text:
text += PackageIndex.TOOLS_LINE + '\r\n\r\n' + tools_text
Expand All @@ -203,13 +197,12 @@ def _copy_all_changelogs(version, basedir, flavor=''):
osp.join(basedir, 'build%s' % flavor, name))


def write_changelog(version2, version1=None, rootdir=None, flavor='',
def write_changelog(version2, version1=None, basedir=None, flavor='',
release_level=''):
"""Write changelog between version1 and version2 of WinPython"""
basedir = get_basedir(version2, rootdir=rootdir)
_copy_all_changelogs(version2, basedir, flavor=flavor)
print ('comparing_package_indexes', version2, rootdir, flavor)
text = compare_package_indexes(version2, version1, rootdir=rootdir,
print ('comparing_package_indexes', version2, basedir, flavor)
text = compare_package_indexes(version2, version1, basedir=basedir,
flavor=flavor)
fname = osp.join(basedir, 'build%s' % flavor,
'WinPython%s-%s_History.md' % (flavor, version2))
Expand All @@ -219,9 +212,9 @@ def write_changelog(version2, version1=None, rootdir=None, flavor='',
shutil.copyfile(fname, osp.join(CHANGELOGS_DIR, osp.basename(fname)))


def test_parse_package_index_wiki(version, rootdir=None, flavor=''):
def test_parse_package_index_wiki(version, basedir=None, flavor=''):
"""Parse the package index Wiki page"""
pi = PackageIndex(version, rootdir=rootdir, flavor=flavor)
pi = PackageIndex(version, basedir=basedir, flavor=flavor)
utils.print_box("WinPython %s:" % pi.version)
utils.print_box("Tools:")
for package in pi.other_packages.values():
Expand All @@ -238,8 +231,8 @@ def test_compare(basedir, version2, version1):


if __name__ == '__main__':
print (compare_package_indexes('3.5.0.1', '3.4.3.6',
rootdir='D:\Winpython', flavor='Slim', flavor1=''))
print (compare_package_indexes('3.4.4.1', '3.4.3.6',
basedir='D:\Winpython\basedir34', flavor='Slim', flavor1=''))
# test_parse_package_index_wiki('2.7.3.3')
# print(compare_package_indexes('2.7.3.3', '2.7.3.1'))
# write_changelog('2.7.4.1', '2.7.4.0')
Expand Down
10 changes: 8 additions & 2 deletions generate_a_winpython_distro.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@ rem set my_python_target=34
rem set my_pyver=3.4
rem set my_release=84

rem *****************************
rem v2 2016-03-19 change
rem we don't use building rootdir (D:\winPython) anymore
rem we use only building basedir (D:\WinPython\basedir34Qt5)
set my_basedir=%my_root_dir_for_builds%\basedir%my_python_target%

rem ***********************************************************
rem Override other scripts (simpler maintenance)

Expand Down Expand Up @@ -102,8 +108,8 @@ call %my_buildenv%\scripts\env.bat

rem build with this
cd /D %~dp0
echo python.exe -c "from make import *;make_all(%my_release%, '%my_release_level%', pyver='%my_pyver%', rootdir=r'%my_root_dir_for_builds%', verbose=True, architecture=%my_arch%, flavor='%my_flavor%', requirements=r'%my_requirements%', install_options=r'%my_install_options%', find_links=r'%my_find_links%', source_dirs=r'%my_source_dirs%', toolsdirs=r'%my_toolsdirs%', docsdirs=r'%my_docsdirs%')">>%my_archive_log%
python.exe -c "from make import *;make_all(%my_release%, '%my_release_level%', pyver='%my_pyver%', rootdir=r'%my_root_dir_for_builds%', verbose=True, architecture=%my_arch%, flavor='%my_flavor%', requirements=r'%my_requirements%', install_options=r'%my_install_options%', find_links=r'%my_find_links%', source_dirs=r'%my_source_dirs%', toolsdirs=r'%my_toolsdirs%', docsdirs=r'%my_docsdirs%')">>%my_archive_log%
echo python.exe -c "from make import *;make_all(%my_release%, '%my_release_level%', pyver='%my_pyver%', basedir=r'%my_basedir%', verbose=True, architecture=%my_arch%, flavor='%my_flavor%', requirements=r'%my_requirements%', install_options=r'%my_install_options%', find_links=r'%my_find_links%', source_dirs=r'%my_source_dirs%', toolsdirs=r'%my_toolsdirs%', docsdirs=r'%my_docsdirs%')">>%my_archive_log%
python.exe -c "from make import *;make_all(%my_release%, '%my_release_level%', pyver='%my_pyver%', basedir=r'%my_basedir%', verbose=True, architecture=%my_arch%, flavor='%my_flavor%', requirements=r'%my_requirements%', install_options=r'%my_install_options%', find_links=r'%my_find_links%', source_dirs=r'%my_source_dirs%', toolsdirs=r'%my_toolsdirs%', docsdirs=r'%my_docsdirs%')">>%my_archive_log%

echo ===============>>%my_archive_log%
echo END OF creation>>%my_archive_log%
Expand Down
42 changes: 19 additions & 23 deletions make.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ class WinPythonDistribution(object):

def __init__(self, build_number, release_level, target, wheeldir,
toolsdirs=None, verbose=False, simulation=False,
rootdir=None, install_options=None, flavor='', docsdirs=None):
basedir=None, install_options=None, flavor='', docsdirs=None):
assert isinstance(build_number, int)
assert isinstance(release_level, str)
self.build_number = build_number
Expand All @@ -121,7 +121,7 @@ def __init__(self, build_number, release_level, target, wheeldir,
self.distribution = None
self.installed_packages = []
self.simulation = simulation
self.rootdir = rootdir # added to build from winpython
self.basedir = basedir # added to build from winpython
self.install_options = install_options
self.flavor = flavor

Expand Down Expand Up @@ -1011,25 +1011,24 @@ def make(self, remove_existing=True, requirements=None, my_winpydir=None): #, f

# Writing changelog
self._print("Writing changelog")
diff.write_changelog(self.winpyver2, rootdir=self.rootdir,
diff.write_changelog(self.winpyver2, basedir=self.basedir,
flavor=self.flavor, release_level=self.release_level)
self._print_done()


def rebuild_winpython(basedir=None, verbose=False, archis=(32, 64), targetdir=None):
def rebuild_winpython(basedir=None, verbose=False, architecture=64, targetdir=None):
"""Rebuild winpython package from source"""
basedir = basedir if basedir is not None else utils.BASE_DIR
for architecture in archis:
suffix = '.win32' if architecture == 32 else '.win-amd64'
if targetdir is not None:
packdir = targetdir
else:
packdir = osp.join(basedir, 'packages' + suffix)
for name in os.listdir(packdir):
if name.startswith('winpython-') and name.endswith(('.exe', '.whl')):
os.remove(osp.join(packdir, name))
utils.build_wininst(osp.dirname(osp.abspath(__file__)), copy_to=packdir,
architecture=architecture, verbose=verbose, installer='bdist_wheel')
suffix = '.win32' if architecture == 32 else '.win-amd64'
if targetdir is not None:
packdir = targetdir
else:
packdir = osp.join(basedir, 'packages' + suffix)
for name in os.listdir(packdir):
if name.startswith('winpython-') and name.endswith(('.exe', '.whl')):
os.remove(osp.join(packdir, name))
utils.build_wininst(osp.dirname(osp.abspath(__file__)), copy_to=packdir,
architecture=architecture, verbose=verbose, installer='bdist_wheel')


def transform_in_list(list_in, list_type=None):
Expand All @@ -1044,8 +1043,8 @@ def transform_in_list(list_in, list_type=None):


def make_all(build_number, release_level, pyver, architecture,
basedir=None, verbose=False, remove_existing=True,
create_installer=True, simulation=False, rootdir=None,
basedir, verbose=False, remove_existing=True,
create_installer=True, simulation=False,
install_options=['--no-index'], flavor='', requirements=None,
find_links=None, source_dirs=None, toolsdirs=None,
docsdirs=None):
Expand All @@ -1061,13 +1060,10 @@ def make_all(build_number, release_level, pyver, architecture,
`pyver`: python version ('3.4' or 3.5')
`architecture`: [int] (32 or 64)
`basedir`: where will be created tmp_wheel dir. and Winpython-xyz dir.
(rootdir: root directory containing 'basedir27', 'basedir33', etc.)
""" + utils.ROOTDIR_DOC

if basedir is None:
basedir = utils.BASE_DIR
if basedir is None:
basedir = utils.get_basedir(pyver, rootdir=rootdir)

assert basedir is not None, "The *basedir* directory must be specified"
assert architecture in (32, 64)
Expand All @@ -1085,7 +1081,7 @@ def make_all(build_number, release_level, pyver, architecture,
os.mkdir(wheeldir)

# Rebuild Winpython in this wheel dir
rebuild_winpython(basedir=basedir, archis=(architecture,), targetdir=wheeldir)
rebuild_winpython(basedir=basedir, architecture=architecture, targetdir=wheeldir)

# Copy Every package directory to the wheel directory

A93C Expand Down Expand Up @@ -1114,7 +1110,7 @@ def make_all(build_number, release_level, pyver, architecture,
dist = WinPythonDistribution(build_number, release_level,
builddir, wheeldir, toolsdirs,
verbose=verbose, simulation=simulation,
rootdir=rootdir,
basedir=basedir,
install_options=install_options + find_list,
flavor=flavor, docsdirs=docsdirs)
# define a pre-defined winpydir, instead of having to guess
Expand All @@ -1133,7 +1129,7 @@ def make_all(build_number, release_level, pyver, architecture,
# DO create only one version at a time
# You may have to manually delete previous build\winpython-.. directory

make_all(1, release_level='build3', pyver='3.4', rootdir=r'D:\Winpython', verbose=True,
make_all(1, release_level='build3', pyver='3.4', basedir=r'D:\Winpython\basedir34', verbose=True,
architecture=64, flavor='Barebone',
requirements=r'D:\Winpython\basedir34\barebone_requirements.txt',
install_options=r'--no-index --pre --trusted-host=None',
Expand Down
13 changes: 0 additions & 13 deletions winpython/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,19 +48,6 @@
* (optional) `tools.win-amd64`: contains 64-bit-specific tools"""


def get_basedir(pyver, rootdir=None):
"""Get basedir from Python version

`pyver`: Python version (X.Y format) [str]
`rootdir`: [str] if None, WINPYTHONROOTDIR env var must be set
(rootdir: root directory containing 'basedir27', 'basedir33', etc.)
""" + ROOTDIR_DOC
assert re.match(r'[0-9]+\.[0-9]+', pyver) is not None
rootdir = rootdir if rootdir is not None else ROOT_DIR
assert rootdir is not None, "The *rootdir* directory must be specified"
return osp.join(rootdir, 'basedir%s' % pyver[::2][:2])


def onerror(function, path, excinfo):
"""Error handler for `shutil.rmtree`.

Expand Down
0