Merge remote-tracking branch 'upstream/master'

Author: sap-coutantm

deploy/infrabox/templates/api/deployment.yaml

@@ -51,20 +51,6 @@ spec:
                 -
                     name: INFRABOX_GERRIT_ENABLED
                     value: {{ .Values.gerrit.enabled | quote }}
-                livenessProbe:
-                  httpGet:
-                    path: /ping
-                    port: 8080
-                  initialDelaySeconds: 5
-                  periodSeconds: 5
-                  timeoutSeconds: 5
-                readinessProbe:
-                  httpGet:
-                    path: /ping
-                    port: 8080
-                  initialDelaySeconds: 5
-                  periodSeconds: 5
-                  timeoutSeconds: 5
             volumes:
                 {{ include "volumes_database" . | indent 16 }}
                 {{ include "volumes_rsa" . | indent 16 }}

docs/doc.md

@@ -35,7 +35,8 @@ The job type docker is one of the most important jobs. You can use it to run any
         "build_arguments": { ... },
         "deployments": [ ... ],
         "security_context": { ... },
-        "repository": { ... }
+        "repository": { ... },
+        "registries": []
     }]
 }
 ```
@@ -56,6 +57,7 @@ The job type docker is one of the most important jobs. You can use it to run any
 |deployments|false|[Deployment Configuration](#deployments)|[]|Push your images to a registry|
 |security_context|false|[Security Context](#security-context)|[]|Configure security related options|
 |repository|false|[Repository Configuration](#repository)|{}|Configure git repository options|
+|registries|false|[Source Registry Configuration](#image-source-registry)|[]|Configure the source registries|
 
 ### Deployments
 The docker images which have been build can also be pushed to a docker registry by InfraBox.

infrabox/generator/generator.py

@@ -63,12 +63,6 @@ def main():
                 'data': False
             }
 
-    if not branch:
-        for j in deployments['jobs']:
-            if 'deployments' in j:
-                del j['deployments']
-
-
     with open('/infrabox/output/deployments.json', 'w') as out:
         json.dump(deployments, out)
 

src/api/handlers/job_api.py

@@ -596,8 +596,6 @@ def get(self, parent_job_id):
 
         key = "%s/%s" % (parent_job_id, filename)
 
-        app.logger.error(key)
-
         g.release_db()
         f = storage.download_output(key)
 

src/api/handlers/projects/jobs.py

@@ -311,7 +311,7 @@ def get(self, project_id, job_id):
             logger.error(key)
             abort(404)
 
-        return send_file(f, attachment_filename=os.path.basename(f))
+        return send_file(f, as_attachment=True, attachment_filename=os.path.basename(f))
 
 
 @ns.route('/<project_id>/jobs/<job_id>/archive')

src/controller/pkg/stub/function.go

@@ -167,7 +167,7 @@ func (c *Controller) syncFunctionInvocation(cr *v1alpha1.IBFunctionInvocation, l
 			log.Info("Updating job status")
 
 			if cr.Status.State.Terminated != nil {
-				if cr.Status.State.Terminated.Message == "Error" {
+				if cr.Status.State.Terminated.Reason == "Error" {
 					log.Errorf("%+v\n", pod)
 				}
 			}

src/controller/pkg/stub/pipeline.go

@@ -47,7 +47,6 @@ func (c *Controller) deletePipelineInvocation(cr *v1alpha1.IBPipelineInvocation,
 	cr.SetFinalizers([]string{})
 	err = updateStatus(cr, log)
 	if err != nil && !errors.IsNotFound(err) {
-		logrus.Errorf("Failed to remove finalizers: %v", err)
 		return err
 	}
 

src/job/infrabox_job/job.py

@@ -93,18 +93,25 @@ def create_jobs(self, jobs):
             "jobs": jobs,
         }
 
-        r = requests.post("%s/create_jobs" % self.api_server,
-                          headers=self.get_headers(),
-                          json=payload, timeout=60, verify=self.verify)
+        while True:
+            r = requests.post("%s/create_jobs" % self.api_server,
+                              headers=self.get_headers(),
+                              json=payload, timeout=60, verify=self.verify)
 
-        if r.status_code != 200:
-            msg = r.text
-            try:
-                msg = r.json()['message']
-            except:
-                pass
+            if r.status_code == 200:
+                return
 
-            raise Failure(msg)
+            if r.status_code == 400:
+                msg = r.text
+                try:
+                    msg = r.json()['message']
+                except:
+                    pass
+
+                raise Failure(msg)
+
+            self.console.collect('Failed to connect to API, retrying.', show=True)
+            time.sleep(3)
 
     def post_api_server(self, endpoint, data=None):
         while True:
@@ -119,6 +126,7 @@ def post_api_server(self, endpoint, data=None):
             except Exception as e:
                 print e
 
+            self.console.collect('Failed to connect to API, retrying.', show=True)
             time.sleep(1)
 
     def post_stats(self, stat):
@@ -165,7 +173,7 @@ def _get_file_from_api_server(self, url, path):
         message = None
 
         r = None
-        for _ in xrange(0, 5):
+        for _ in xrange(0, 20):
             try:
                 message = None
                 r = requests.get("%s%s" % (self.api_server, url),
@@ -176,6 +184,8 @@ def _get_file_from_api_server(self, url, path):
                     return
 
                 if r.status_code != 200:
+                    self.console.collect('Failed to download file (%s), retrying' % r.status_code, show=True)
+                    time.sleep(10)
                     continue
 
                 with open(path, 'wb') as  f:
@@ -185,6 +195,7 @@ def _get_file_from_api_server(self, url, path):
 
             except Exception as e:
                 message = str(e)
+                self.console.collect('Failed to download file (%s), retrying' % message, show=True)
                 time.sleep(10)
                 continue
 
@@ -199,7 +210,7 @@ def _get_file_from_api_server(self, url, path):
             except:
                 pass
 
-            raise Failure('Failed to download file: %s' % msg)
+            raise Failure('Failed to download file(%s): %s' % (r.status_code, msg))
 
     def post_file_to_api_server(self, url, path, filename=None, split=False):
         if not filename:

src/job/job.py

@@ -597,6 +597,9 @@ def run_job_docker_compose(self, c):
             service_badge_dir = os.path.join(self.infrabox_badge_dir, service)
             makedirs(service_badge_dir)
 
+            service_archive_dir = os.path.join(self.infrabox_archive_dir, service)
+            makedirs(service_archive_dir)
+
             service_volumes = [
                 "%s:/infrabox/cache" % service_cache_dir,
                 "%s:/infrabox/inputs" % self.infrabox_inputs_dir,
@@ -606,6 +609,7 @@ def run_job_docker_compose(self, c):
                 "%s:/infrabox/upload/markup" % service_markup_dir,
                 "%s:/infrabox/upload/badge" % service_badge_dir,
                 "%s:/infrabox/upload/coverage" % service_coverage_dir,
+                "%s:/infrabox/upload/archive" % service_archive_dir,
             ]
 
             for v in compose_file_content['services'][service].get('volumes', []):
@@ -732,7 +736,10 @@ def deploy_images(self, image_name):
         c.header("Deploying", show=True)
 
         for dep in self.deployments:
-            self.deploy_image(image_name, dep)
+            target = dep.get('target', None)
+
+            if not target:
+                self.deploy_image(image_name, dep)
 
     def login_docker_registry(self):
         c = self.console
@@ -835,6 +842,13 @@ def run_docker_container(self, image_name):
                 logger.exception(ex)
                 raise Failure("Could not get exit code of container")
 
+            try:
+                c.execute(("docker", "commit", container_name, image_name))
+                c.header("Finalize", show=True)
+            except Exception as ex:
+                logger.exception(ex)
+                raise Failure("Could not commit and push container")
+
             logger.exception(e)
             raise Failure("Container run exited with error (exit code=%s)" % exit_code)
 
@@ -889,7 +903,7 @@ def build_docker_image(self, image_name, cache_image, target=None):
     def run_job_docker_image(self, c):
         image_name = self.job['definition']['image'].replace('$INFRABOX_BUILD_NUMBER', str(self.build['build_number']))
 
-        if self.job.get('run', True):
+        if self.job.get('definition', {}).get('run', True):
             self._login_source_registries()
             self.run_docker_container(image_name)
             self._logout_source_registries()
@@ -950,13 +964,19 @@ def run_job_docker(self, c):
 
         if self.deployments:
             for d in self.deployments:
-                self.build_docker_image(image_name_build, image_name_latest, d.get('target', None))
+                target = d.get('target', None)
+
+                if not target and not self.job.get('build_only', True):
+                    continue
+
+                self.build_docker_image(image_name_build, image_name_latest, target=target)
                 c.header("Deploying", show=True)
                 self.deploy_image(image_name_build, d)
 
         if not self.job.get('build_only', True):
             self.build_docker_image(image_name_build, image_name_latest)
             self.run_docker_container(image_name_build)
+            self.deploy_images(image_name_build)
 
         c.header("Finalize", show=True)
 

src/pyinfrabox/infrabox/__init__.py

@@ -247,7 +247,7 @@ def parse_docker(d, path):
     check_allowed_properties(d, path, ("type", "name", "docker_file", "depends_on", "resources",
                                        "build_only", "environment",
                                        "build_arguments", "deployments", "timeout", "security_context",
-                                       "build_context", "cache", "repository", "cluster", "services"))
+                                       "build_context", "cache", "repository", "cluster", "services", "registries"))
     check_required_properties(d, path, ("type", "name", "docker_file", "resources"))
     check_name(d['name'], path + ".name")
     check_text(d['docker_file'], path + ".docker_file")
@@ -289,6 +289,9 @@ def parse_docker(d, path):
     if 'build_context' in d:
         check_text(d['build_context'], path + ".build_context")
 
+    if 'registries' in d:
+        parse_registries(d['registries'], path + '.registries')
+
 def parse_docker_compose(d, path):
     check_allowed_properties(d, path, ("type", "name", "docker_compose_file", "depends_on",
                                        "environment", "resources", "cache", "timeout", "cluster", "repository"))