From 4f509951a7c3729719e4168962a74c7a16299705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Tue, 17 May 2016 09:51:33 +0200 Subject: [PATCH] =?UTF-8?q?Reduced=20the=20GCloud=20resumable=20upload=20t?= =?UTF-8?q?hreshold=205=20MiB=20=E2=86=92=20100=20KiB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Files larger than the threshold are streamed from disk, smaller than the threshold and they are loaded into memory first. --- pillar/application/modules/file_storage.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pillar/application/modules/file_storage.py b/pillar/application/modules/file_storage.py index 6b2ebdf2..ba71d7f9 100644 --- a/pillar/application/modules/file_storage.py +++ b/pillar/application/modules/file_storage.py @@ -546,6 +546,10 @@ def stream_to_gcs(project_id): stream_for_gcs = uploaded_file.stream # Upload the file to GCS. + from gcloud.streaming import transfer + # Files larger than this many bytes will be streamed directly from disk, smaller + # ones will be read into memory and then uploaded. + transfer.RESUMABLE_UPLOAD_THRESHOLD = 102400 try: gcs = GoogleCloudStorageBucket(project_id) blob = gcs.bucket.blob('_/' + internal_fname, chunk_size=256 * 1024 * 2)