minor fixes

Author: ib-steffen

deploy/infrabox/templates/api/deployment.yaml

@@ -51,20 +51,6 @@ spec:
                 -
                     name: INFRABOX_GERRIT_ENABLED
                     value: {{ .Values.gerrit.enabled | quote }}
-                livenessProbe:
-                  httpGet:
-                    path: /ping
-                    port: 8080
-                  initialDelaySeconds: 5
-                  periodSeconds: 5
-                  timeoutSeconds: 5
-                readinessProbe:
-                  httpGet:
-                    path: /ping
-                    port: 8080
-                  initialDelaySeconds: 5
-                  periodSeconds: 5
-                  timeoutSeconds: 5
             volumes:
                 {{ include "volumes_database" . | indent 16 }}
                 {{ include "volumes_rsa" . | indent 16 }}

src/api/handlers/job_api.py

@@ -596,8 +596,6 @@ def get(self, parent_job_id):
 
         key = "%s/%s" % (parent_job_id, filename)
 
-        app.logger.error(key)
-
         g.release_db()
         f = storage.download_output(key)
 

src/controller/pkg/stub/pipeline.go

@@ -47,7 +47,6 @@ func (c *Controller) deletePipelineInvocation(cr *v1alpha1.IBPipelineInvocation,
 	cr.SetFinalizers([]string{})
 	err = updateStatus(cr, log)
 	if err != nil && !errors.IsNotFound(err) {
-		logrus.Errorf("Failed to remove finalizers: %v", err)
 		return err
 	}
 

src/job/infrabox_job/job.py

@@ -93,18 +93,25 @@ def create_jobs(self, jobs):
             "jobs": jobs,
         }
 
-        r = requests.post("%s/create_jobs" % self.api_server,
-                          headers=self.get_headers(),
-                          json=payload, timeout=60, verify=self.verify)
+        while True:
+            r = requests.post("%s/create_jobs" % self.api_server,
+                              headers=self.get_headers(),
+                              json=payload, timeout=60, verify=self.verify)
 
-        if r.status_code != 200:
-            msg = r.text
-            try:
-                msg = r.json()['message']
-            except:
-                pass
+            if r.status_code == 200:
+                return
 
-            raise Failure(msg)
+            if r.status_code == 400:
+                msg = r.text
+                try:
+                    msg = r.json()['message']
+                except:
+                    pass
+
+                raise Failure(msg)
+
+            self.console.collect('Failed to connect to API, retrying.', show=True)
+            time.sleep(3)
 
     def post_api_server(self, endpoint, data=None):
         while True:
@@ -119,6 +126,7 @@ def post_api_server(self, endpoint, data=None):
             except Exception as e:
                 print e
 
+            self.console.collect('Failed to connect to API, retrying.', show=True)
             time.sleep(1)
 
     def post_stats(self, stat):
@@ -165,7 +173,7 @@ def _get_file_from_api_server(self, url, path):
         message = None
 
         r = None
-        for _ in xrange(0, 5):
+        for _ in xrange(0, 20):
             try:
                 message = None
                 r = requests.get("%s%s" % (self.api_server, url),
@@ -176,6 +184,8 @@ def _get_file_from_api_server(self, url, path):
                     return
 
                 if r.status_code != 200:
+                    self.console.collect('Failed to download file (%s), retrying' % r.status_code, show=True)
+                    time.sleep(10)
                     continue
 
                 with open(path, 'wb') as  f:
@@ -185,6 +195,7 @@ def _get_file_from_api_server(self, url, path):
 
             except Exception as e:
                 message = str(e)
+                self.console.collect('Failed to download file (%s), retrying' % message, show=True)
                 time.sleep(10)
                 continue
 
@@ -199,7 +210,7 @@ def _get_file_from_api_server(self, url, path):
             except:
                 pass
 
-            raise Failure('Failed to download file: %s' % msg)
+            raise Failure('Failed to download file(%s): %s' % (r.status_code, msg))
 
     def post_file_to_api_server(self, url, path, filename=None, split=False):
         if not filename:

src/job/job.py

@@ -736,7 +736,10 @@ def deploy_images(self, image_name):
         c.header("Deploying", show=True)
 
         for dep in self.deployments:
-            self.deploy_image(image_name, dep)
+            target = dep.get('target', None)
+
+            if not target:
+                self.deploy_image(image_name, dep)
 
     def login_docker_registry(self):
         c = self.console
@@ -839,6 +842,13 @@ def run_docker_container(self, image_name):
                 logger.exception(ex)
                 raise Failure("Could not get exit code of container")
 
+            try:
+                c.execute(("docker", "commit", container_name, image_name))
+                c.header("Finalize", show=True)
+            except Exception as ex:
+                logger.exception(ex)
+                raise Failure("Could not commit and push container")
+
             logger.exception(e)
             raise Failure("Container run exited with error (exit code=%s)" % exit_code)
 
@@ -954,13 +964,19 @@ def run_job_docker(self, c):
 
         if self.deployments:
             for d in self.deployments:
-                self.build_docker_image(image_name_build, image_name_latest, d.get('target', None))
+                target = d.get('target', None)
+
+                if not target and not self.job.get('build_only', True):
+                    continue
+
+                self.build_docker_image(image_name_build, image_name_latest, target=target)
                 c.header("Deploying", show=True)
                 self.deploy_image(image_name_build, d)
 
         if not self.job.get('build_only', True):
             self.build_docker_image(image_name_build, image_name_latest)
             self.run_docker_container(image_name_build)
+            self.deploy_images(image_name_build)
 
         c.header("Finalize", show=True)
 

src/scheduler/kubernetes/scheduler.py

@@ -366,8 +366,16 @@ def handle_orphaned_jobs(self):
                             if not message:
                                 message = stepStatus['State']['terminated'].get('reason', 'Unknown Error')
 
+                    if not message and current_state != 'finished':
+                        import json
+                        self.logger.error(json.dumps(status, indent=4))
+
                     delete_job = True
 
+                if message == 'Error':
+                    import json
+                    self.logger.error(json.dumps(status, indent=4))
+
                 if s == "error":
                     current_state = 'error'
                     delete_job = True
@@ -376,7 +384,11 @@ def handle_orphaned_jobs(self):
 
                 if 'stepStatuses' in status and status['stepStatuses']:
                     stepStatus = status['stepStatuses'][-1]
-                    node_name = stepStatus.get('nodeName', None)
+                    nn = stepStatus.get('nodeName', None)
+
+                    if nn:
+                        # don't overwrite existing node name with none
+                        node_name = nn
 
                 start_date = status.get('startTime', None)
                 end_date = status.get('completionTime', None)
@@ -448,6 +460,7 @@ def assign_cluster(self):
             """, [cluster_name, j[0]])
             cursor.close()
 
+
     def update_cluster_state(self):
         cluster_name = os.environ['INFRABOX_CLUSTER_NAME']
         labels = []
@@ -470,6 +483,13 @@ def update_cluster_state(self):
         items = data.get('items', [])
 
         for i in items:
+            metadata = i.get('metadata', {})
+            l = metadata.get('labels', {})
+            master = l.get('node-role.kubernetes.io/master', "false")
+
+            if master == "true":
+                continue
+
             nodes += 1
             cpu += int(i['status']['capacity']['cpu'])
             mem = i['status']['capacity']['memory']