8000 [MRG+2] MAINT: Refactor the converted-image cache by jkseppan · Pull Request #7764 · matplotlib/matplotlib · GitHub
[go: up one dir, main page]
Skip to content

[MRG+2] MAINT: Refactor the converted-image cache #7764

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 19 commits into from
Closed

[MRG+2] MAINT: Refactor the converted-image cache #7764

Changes from 1 commit
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
1b24a16
Factor the converted-image cache out of compare.py
jkseppan Jan 7, 2017
650ddf2
Import cbook again
jkseppan Jan 21, 2017
3b5f636
Lock the cache directory
jkseppan Jan 21, 2017
556fd82
Communicate node-specific cache reports to master if using xdist
jkseppan Jan 22, 2017
9f5a06d
Delete the nose plugin
jkseppan Jan 25, 2017
33f2528
Reinstate deleted functions
jkseppan Jan 25, 2017
2136a26
Make conversion_cache private
jkseppan Jan 25, 2017
6995a27
Make globals private
jkseppan Jan 25, 2017
9ea66dd
Improve wording
jkseppan Jan 25, 2017
3751f01
mkdirs throws OSError
jkseppan Jan 25, 2017
a70081e
Improve documentation in response to review
jkseppan Jan 30, 2017
85f54de
Use pytest facilities
jkseppan Jan 30, 2017
0d51e03
Simplify gs and inkscape version checking logic
jkseppan Jan 30, 2017
9de06c0
pep8 fixes
jkseppan Jan 30, 2017
6007b37
There is no nose plugin any more
jkseppan Feb 18, 2017
e52cdd2
Import the pytest hooks in the other conftest.py files
jkseppan Feb 20, 2017
e5184bb
Omit unnecessary tmpdir deletions
jkseppan Feb 22, 2017
f47cbe4
Move pytest_addoption call to top-level conftest.py
jkseppan Feb 22, 2017
996f22b
pep8 fix
jkseppan Feb 22, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Lock the cache directory 
In case multiple processes use it at the same time.
Don't delete the lock file in expire().
  • Loading branch information
@jkseppan
jkseppan committed Feb 20, 2017
commit 3b5f636d0b6f9acaac7fb84fd7e3c647beaae4db
46 changes: 25 additions & 21 deletions lib/matplotlib/testing/conversion_cache.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,12 +89,13 @@ def get(self, filename, newname):
self.gets.add(filename)
hash_value = self._get_file_hash(filename)
cached_file = os.path.join(self.cachedir, hash_value + self.cached_ext)
if os.path.exists(cached_file):
shutil.copyfile(cached_file, newname)
self.hits.add(filename)
return True
else:
return False
with cbook.Locked(self.cachedir):
if os.path.exists(cached_file):
shutil.copyfile(cached_file, newname)
self.hits.add(filename)
return True
else:
return False

def put(self, original, converted):
"""Insert a file into the cache.
Expand All @@ -108,7 +109,8 @@ def put(self, original, converted):
"""
hash_value = self._get_file_hash(original)
cached_file = os.path.join(self.cachedir, hash_value + self.cached_ext)
shutil.copyfile(converted, cached_file)
with cbook.Locked(self.cachedir):
shutil.copyfile(converted, cached_file)

def _get_file_hash(self, path, block_size=2 ** 20):
if path in self.hash_cache:
Expand Down Expand Up @@ -150,20 +152,22 @@ def expire(self):
Orders files by access time, so the least recently used files
get deleted first.
"""
stats = {filename: os.stat(os.path.join(self.cachedir, filename))
for filename in os.listdir(self.cachedir)}
usage = sum(f.st_size for f in stats.values())
to_free = usage - self.max_size
if to_free <= 0:
return

files = sorted(os.listdir(self.cachedir),
key=lambda f: stats[f].st_atime,
reverse=True)
while to_free > 0:
filename = files.pop()
os.remove(os.path.join(self.cachedir, filename))
to_free -= stats[filename].st_size
with cbook.Locked(self.cachedir):
stats = {filename: os.stat(os.path.join(self.cachedir, filename))
for filename in os.listdir(self.cachedir)
if filename.endswith(self.cached_ext)}
usage = sum(f.st_size for f in stats.values())
to_free = usage - self.max_size
if to_free <= 0:
return

files = sorted(stats.keys(),
key=lambda f: stats[f].st_atime,
reverse=True)
while to_free > 0:
filename = files.pop()
os.remove(os.path.join(self.cachedir, filename))
to_free -= stats[filename].st_size

@staticmethod
def get_cache_dir():
Expand Down
0