From 0b4a1a7098e0b4930e7a5d3ea2411f30ff6923a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Tue, 22 Mar 2016 10:41:35 +0100 Subject: [PATCH] Only download a file once per Blender session. --- blender_cloud/pillar.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/blender_cloud/pillar.py b/blender_cloud/pillar.py index 67b0687..7e33a4d 100644 --- a/blender_cloud/pillar.py +++ b/blender_cloud/pillar.py @@ -17,7 +17,7 @@ _pillar_api = None # will become a pillarsdk.Api object. log = logging.getLogger(__name__) uncached_session = requests.session() _testing_blender_id_profile = None # Just for testing, overrides what is returned by blender_id_profile. - +_downloaded_urls = set() # URLs we've downloaded this Blender session. class UserNotLoggedInError(RuntimeError): """Raised when the user should be logged in on Blender ID, but isn't. @@ -153,7 +153,14 @@ async def download_to_file(url, filename, *, # Check file length. expected_content_length = int(stored_headers['Content-Length']) statinfo = os.stat(filename) - if expected_content_length != statinfo.st_size: + if expected_content_length == statinfo.st_size: + # File exists, and is of the correct length. Don't bother downloading again + # if we already downloaded it this session. + if url in _downloaded_urls: + log.debug('Already downloaded %s this session, skipping this request.', + url) + return + else: log.debug('File size should be %i but is %i; ignoring cache.', expected_content_length, statinfo.st_size) stored_headers = {} @@ -202,6 +209,7 @@ async def download_to_file(url, filename, *, if response.status_code == 304: # The file we have cached is still good, just use that instead. + _downloaded_urls.add(url) return # After we performed the GET request, we should check whether we should start @@ -216,6 +224,7 @@ async def download_to_file(url, filename, *, # We're done downloading, now we have something cached we can use. log.debug('Saving header cache to %s', header_store) + _downloaded_urls.add(url) with open(header_store, 'w') as outfile: json.dump({ 'ETag': str(response.headers.get('etag', '')),