8000 Hashsums2 2 by xloem · Pull Request #2 · xloem/python-for-android · GitHub
[go: up one dir, main page]
Skip to content

Hashsums2 2 #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: develop
Choose a base branch
from
Open

Hashsums2 2 #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
90000ca
address comments to #2351, bringing hashes in line with pypi
xloem Dec 24, 2020
bb07ed5
lint fixes
xloem Dec 24, 2020
3ff5cc1
fix location of verify_algsum
xloem Dec 24, 2020
bdc6fd4
lint fix
xloem Dec 24, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 38 additions & 39 deletions pythonforandroid/recipe.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,22 +53,21 @@ class Recipe(with_metaclass(RecipeMeta)):
'''A string giving the version of the software the recipe describes,
e.g. ``2.0.3`` or ``master``.'''

md5sum = None
'''The md5sum of the source from the :attr:`url`. Non-essential, but
you should try to include this, it is used to check that the download
finished correctly.
sha256sum = None
'''The sha256sum of the source from the :attr:`url`. As of 2020, pip
recommendes use of this hash. You should try to include this. It
is used to check that the download finished correctly.
'''

sha512sum = None
'''The sha512sum of the source from the :attr:`url`. Non-essential, but
you should try to include this, it is used to check that the download
finished correctly.
md5sum = None
'''The md5s 10000 um of the source from the :attr:`url`. Non-essential. It
is used to check that the download finished correctly.
'''

blake2bsum = None
'''The blake2bsum of the source from the :attr:`url`. Non-essential, but
you should try to include this, it is used to check that the download
finished correctly.
blake2b_256sum = None
'''The blake2b_256sum of the source from the :attr:`url`. Non-essential,
but you should try to include this, it is used to check that the
download finished correctly.
'''

depends = []
Expand Down Expand Up @@ -355,8 +354,8 @@ def download(self):

url = self.versioned_url
expected_digests = {}
for alg in set(hashlib.algorithms_guaranteed) | set(('md5', 'sha512', 'blake2b')):
expected_digest = getattr(self, alg + 'sum') if hasattr(self, alg + 'sum') else None
for alg in set(hashlib.algorithms_guaranteed) | set(('sha256', 'md5', 'blake2b_256')):
expected_digest = getattr(self, alg + 'sum', None)
ma = match(u'^(.+)#' + alg + u'=([0-9a-f]{32,})$', url)
if ma: # fragmented URL?
if expected_digest:
Expand All @@ -379,16 +378,7 @@ def download(self):
if not exists(marker_filename):
shprint(sh.rm, filename)
else:
for alg, expected_digest in expected_digests.items():
current_digest = algsum(alg, filename)
if current_digest != expected_digest:
debug('* Generated {}sum: {}'.format(alg,
current_digest))
debug('* Expected {}sum: {}'.format(alg,
expected_digest))
raise ValueError(
('Generated {0}sum does not match expected {0}sum '
'for {1} recipe').format(alg, self.name))
verify_algsum(self, expected_digests, filename)
do_download = False

# If we got this far, we will download
Expand All @@ -400,16 +390,7 @@ def download(self):
shprint(sh.touch, marker_filename)

if exists(filename) and isfile(filename):
for alg, expected_digest in expected_digests.items():
current_digest = algsum(alg, filename)
if current_digest != expected_digest:
debug('* Generated {}sum: {}'.format(alg,
current_digest))
debug('* Expected {}sum: {}'.format(alg,
expected_digest))
raise ValueError(
('Generated {0}sum does not match expected {0}sum '
'for {1} recipe').format(alg, self.name))
verify_algsum(self, expected_digests, filename)
else:
info('{} download already cached, skipping'.format(self.name))

Expand Down Expand Up @@ -1195,10 +1176,28 @@ def reduce_object_file_names(self, dirn):
shprint(sh.mv, filen, join(file_dirname, parts[0] + '.so'))


def algsum(alg, filen):
'''Calculate the digest of a file.
def verify_algsum(recipe, algs, filen):
'''Verify digest of a file.
'''
with open(filen, 'rb') as fileh:
digest = getattr(hashlib, alg)(fileh.read())

return digest.hexdigest()
for alg, expected_digest in algs.items():

with open(filen, 'rb') as fileh:
func = getattr(hashlib, alg, None)
if func is not None:
digest = func(fileh.read())
elif '_' in alg: # for custom digest_sizes, such as blake2b_256
offset = alg.rfind('_')
func = getattr(hashlib, alg[:offset])
digest_size = int(alg[offset + 1:])
digest = func(fileh.read(), digest_size=digest_size)
current_digest = digest.hexdigest()

if current_digest != expected_digest:
debug('* Generated {}sum: {}'.format(alg,
current_digest))
debug('* Expected {}sum: {}'.format(alg,
expected_digest))
raise ValueError(
('Generated {0}sum does not match expected {0}sum '
'for {1} recipe').format(alg, recipe.name))
32 changes: 31 additions & 1 deletion tests/test_recipe.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,13 +115,23 @@ def test_download_url_not_set(self):
mock.call('Skipping test_recipe download as no URL is set')]

@staticmethod
def get_dummy_python_recipe_for_download_tests():
def get_dummy_python_recipe_for_download_tests(good_digests=[], bad_digests=[]):
"""
Helper method for creating a test recipe used in download tests.
"""
recipe = DummyRecipe()
filename = 'Python-3.7.4.tgz'
url = 'https://www.python.org/ftp/python/3.7.4/{}'.format(filename)
for bad_digest in bad_digests:
setattr(recipe, bad_digest + 'sum', 'x')
if 'sha256' in good_digests:
recipe.sha256sum = 'd63e63e14e6d29e17490abbe6f7d17afb3db182dbd801229f14e55f4157c4ba3'
if 'md5' in good_digests:
recipe.md5sum = '68111671e5b2db4aef7b9ab01bf0f9be'
if 'blake2b_256' in good_digests:
recipe.blake2b_256sum = '24a28aeace25e4397edc62ede83790541257e7281cc33539cece84dd3371f640'
if 'sha512' in good_digests:
recipe.sha512sum = 'c25a72ad792f7c1b4c2f79faebbe9608d04b04b2fe58ab804cb4732cdaa75ea93d175f5e52b38e91cb6ae0559ea6b645d802c8b6a869584e8bb9b5018367ce3d'
recipe._url = url
recipe.ctx = Context()
return recipe, filename
Expand All @@ -146,6 +156,26 @@ def test_download_url_is_set(self):
'https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz')]
assert m_touch.call_count == 1

def test_hashes(self):
"""
Verifies digest hashes are enforced.
"""
with (
tempfile.TemporaryDirectory()) as temp_dir:
digests = set(('sha256', 'md5', 'blake2b_256', 'sha512'))
recipe, filename = self.get_dummy_python_recipe_for_download_tests(
good_digests=digests)
recipe.ctx.setup_dirs(temp_dir)
recipe.download()
for bad_digest in digests:
good_digests = digests - set([bad_digest])
recipe, filename = self.get_dummy_python_recipe_for_download_tests(
good_digests=good_digests,
bad_digests=[bad_digest])
recipe.ctx.setup_dirs(temp_dir)
with self.assertRaises(ValueError):
recipe.download()

def test_download_file_scheme_https(self):
"""
Verifies `urlretrieve()` is being called on https downloads.
Expand Down
0