upload/download cache in 64MB chunks (#224)

Author: ib-steffen

src/api/handlers/job_api.py

@@ -434,43 +434,39 @@ def get(self):
 
         return send_file(f)
 
-cache_upload_parser = api.parser()
-cache_upload_parser.add_argument('cache.tar.snappy', location='files',
-                                 type=FileStorage, required=True)
-
 
 @api.route("/api/job/cache", doc=False)
 class Cache(Resource):
 
     def get(self):
+        g.release_db()
         project_id = g.token['project']['id']
         job_name = g.token['job']['name']
-
-        template = 'project_%s_job_%s.tar.snappy'
-        key = template % (project_id, job_name)
+        filename = request.args.get('filename', None)
+        template = 'project_%s_job_%s_%s'
+        key = template % (project_id, job_name, filename)
         key = key.replace('/', '_')
 
-        g.release_db()
-
         f = storage.download_cache(key)
 
         if not f:
             abort(404)
 
         return send_file(f)
 
-    @api.expect(cache_upload_parser)
     def post(self):
+        g.release_db()
+
         project_id = g.token['project']['id']
         job_name = g.token['job']['name']
 
-        template = 'project_%s_job_%s.tar.snappy'
-        key = template % (project_id, job_name)
-        key = key.replace('/', '_')
-
-        g.release_db()
+        for f, _ in request.files.items():
+            template = 'project_%s_job_%s_%s'
+            key = template % (project_id, job_name, f)
+            key = key.replace('/', '_')
+            stream = request.files[f].stream
+            storage.upload_cache(stream, key)
 
-        storage.upload_cache(request.files['cache.tar.snappy'].stream, key)
         return jsonify({})
 
 

src/job/job.py

@@ -491,7 +491,7 @@ def main_run_job(self):
         if not self.job['definition'].get('cache', {}).get('data', True):
             c.collect("Not downloading cache, because cache.data has been set to false", show=True)
         else:
-            self.get_file_from_api_server("/cache", storage_cache_tar)
+            self.get_file_from_api_server("/cache", storage_cache_tar, split=True)
 
             if os.path.isfile(storage_cache_tar):
                 c.collect("Unpacking cache", show=True)
@@ -553,7 +553,7 @@ def main_run_job(self):
                 file_size = os.stat(storage_cache_tar).st_size
 
                 c.collect("Output size: %s kb" % (file_size / 1024), show=True)
-                self.post_file_to_api_server('/cache', storage_cache_tar)
+                self.post_file_to_api_server('/cache', storage_cache_tar, split=True)
             else:
                 c.collect("Cache is empty", show=True)
         c.collect("", show=True)