Merge remote-tracking branch 'upstream/master'

Author: sap-coutantm

deploy/infrabox/templates/metrics/service.yaml

@@ -3,6 +3,7 @@ kind: Service
 apiVersion: v1
 metadata:
   name: infrabox-metrics
+  namespace: {{ template "system_namespace" . }}
   labels:
     app: infrabox-metrics
 spec:

deploy/infrabox/templates/prometheus/metrics.yaml

@@ -3,6 +3,7 @@ apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
   name: infrabox-metrics
+  namespace: {{ template "system_namespace" . }}
   labels:
     app: infrabox
 spec:

deploy/infrabox/templates/prometheus/prometheus.yaml

@@ -3,6 +3,7 @@ apiVersion: monitoring.coreos.com/v1
 kind: Prometheus
 metadata:
   name: infrabox
+  namespace: {{ template "system_namespace" . }}
 spec:
   serviceAccountName: infrabox
   serviceMonitorSelector:

deploy/infrabox/templates/prometheus/service.yaml

@@ -3,6 +3,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: prometheus
+  namespace: {{ template "system_namespace" . }}
 spec:
   ports:
   - name: web

src/api/handlers/projects/jobs.py

@@ -1,13 +1,16 @@
 import json
 import os
 
+import requests
+
 from flask import g, abort, Response, send_file, request
 from flask_restplus import Resource
 
 from pyinfraboxutils import get_logger
 from pyinfraboxutils.ibflask import auth_required, OK
 from pyinfraboxutils.storage import storage
 from api.namespaces import project as ns
+from pyinfraboxutils.token import encode_user_token
 
 logger = get_logger('api')
 
@@ -299,20 +302,50 @@ class ArchiveDownload(Resource):
 
     @auth_required(['user'], allow_if_public=True)
     def get(self, project_id, job_id):
-        f = request.args.get('filename', None)
+        filename = request.args.get('filename', None)
 
-        if not f:
+        if not filename:
             abort(404)
 
-        key = '%s/%s' % (job_id, f)
-        f = storage.download_archive(key)
+        result = g.db.execute_one_dict('''
+                    SELECT cluster_name
+                    FROM job
+                    WHERE   id = %s
+                        AND project_id = %s
+                ''', [job_id, project_id])
+
+        if not result or not result['cluster_name']:
+            abort(404)
+
+        job_cluster = result['cluster_name']
+        key = '%s/%s' % (job_id, filename)
+
+        if os.environ['INFRABOX_CLUSTER_NAME'] == job_cluster:
+            f = storage.download_archive(key)
+        else:
+            c = g.db.execute_one_dict('''
+                                SELECT *
+                                FROM cluster
+                                WHERE name=%s
+                            ''', [job_cluster])
+            url = '%s/api/v1/projects/%s/jobs/%s/archive/download?filename=%s' % (c['root_url'], project_id, job_id, filename)
+            try:
+                token = encode_user_token(g.token['user']['id'])
+            except AttributeError:
+                #public project has no token here.
+                token = ""
+            headers = {'Authorization': 'bearer ' + token}
+            logger.info('get archive %s from %s', [filename, url])
+
+            # TODO(ib-steffen): allow custom ca bundles
+            r = requests.get(url,headers=headers, timeout=120, verify=False, stream=True)
+            f = r.raw
 
         if not f:
             logger.error(key)
             abort(404)
 
-        return send_file(f, as_attachment=True, attachment_filename=os.path.basename(f))
-
+        return send_file(f, as_attachment=True, attachment_filename=os.path.basename(filename))
 
 @ns.route('/<project_id>/jobs/<job_id>/archive')
 class Archive(Resource):

src/job/infrabox_job/job.py

@@ -193,6 +193,7 @@ def _get_file_from_api_server(self, url, path):
                         if chunk:
                             f.write(chunk)
 
+                return
             except Exception as e:
                 message = str(e)
                 self.console.collect('Failed to download file (%s), retrying' % message, show=True)

src/job/job.py

@@ -9,6 +9,7 @@
 import uuid
 import base64
 import traceback
+import urllib3
 
 from pyinfrabox.infrabox import validate_json
 from pyinfrabox.docker_compose import create_from
@@ -21,6 +22,8 @@
 from pyinfraboxutils.coverage import Parser as CoverageParser
 from pyinfraboxutils import get_env
 from pyinfraboxutils import get_logger
+
+urllib3.disable_warnings()
 logger = get_logger('scheduler')
 
 def makedirs(path):
@@ -247,6 +250,7 @@ def get_source(self):
             raise Exception('Unknown project type')
 
         c.execute(['chmod', '-R', 'a+rwX', self.mount_repo_dir])
+        c.execute(['ls', '-alh', self.mount_repo_dir], show=True)
 
     def main_create_jobs(self):
         c = self.console
@@ -752,6 +756,11 @@ def logout_docker_registry(self):
         c = self.console
         c.execute(['docker', 'logout', get_registry_name()], show=False)
 
+    def run_docker_pull(self, image_name):
+        c = self.console
+        cmd = ['docker', 'pull', image_name]
+        c.execute(cmd, show=True, show_cmd=False)
+
     def run_docker_container(self, image_name):
         c = self.console
         collector = StatsCollector()
@@ -868,7 +877,7 @@ def build_docker_image(self, image_name, cache_image, target=None):
 
         try:
             c.header("Build image", show=True)
-            self.get_cached_image(cache_image)
+            cache_from = self.get_cached_image(cache_image)
             docker_file = os.path.normpath(os.path.join(self._get_build_context_current_job(),
                                                         self.job['dockerfile']))
 
@@ -889,7 +898,7 @@ def build_docker_image(self, image_name, cache_image, target=None):
 
             if target:
                 cmd += ['--target', target]
-            else:
+            elif cache_from:
                 cmd += ['--cache-from', cache_image]
 
             cwd = self._get_build_context_current_job()
@@ -907,6 +916,10 @@ def run_job_docker_image(self, c):
             self._login_source_registries()
             self.run_docker_container(image_name)
             self._logout_source_registries()
+        else:
+            self._login_source_registries()
+            self.run_docker_pull(image_name)
+            self._logout_source_registries()
 
         self.deploy_images(image_name)
         c.header("Finalize", show=True)
@@ -985,13 +998,14 @@ def get_cached_image(self, image_name_latest):
 
         if not self.job['definition'].get('cache', {}).get('image', False):
             c.collect("Not pulling cached image, because cache.image has been set to false", show=True)
-            return
+            return False
 
         c.collect("Get cached image %s" % image_name_latest, show=True)
 
         self.login_docker_registry()
         c.execute(['docker', 'pull', image_name_latest], show=True, ignore_error=True)
         self.logout_docker_registry()
+        return True
 
     def cache_docker_image(self, image_name_build, image_name_latest):
         c = self.console