Merge remote-tracking branch 'upstream/master'

fix #71

Author: sap-coutantm

docs/doc.md

@@ -90,7 +90,7 @@ Deployments are currently only supported for `docker` job types.
 |type|true|string||Has to be "docker-registry"|
 |host|true|string||Host of the registry. I.e. my-registry.com:5000|
 |repository|true|string||Name of the repository|
-|tag|false|string|build\_<NUMBER>|The of the image|
+|tag|false|string|build\_\<NUMBER>|The tag of the image|
 |target|false|string||When building a Dockerfile with multiple build stages `target` can be used to specify an intermediate build stage by name as a final stage for the resulting image which should be deployed|
 |username|false|string||Username to be used with the registry|
 |password|false|[Secret](#secrets)||Secret containing the password|
@@ -122,7 +122,7 @@ Deployments are currently only supported for `docker` job types.
 |type|true|string||Has to be "ecr"|
 |host|true|string||ECR endpoint|
 |repository|true|string||Name of the repository|
-|tag|false|string|build\_<NUMBER>|The of the image|
+|tag|false|string|build\_\<NUMBER>|The tag of the image|
 |target|false|string||When building a Dockerfile with multiple build stages `target` can be used to specify an intermediate build stage by name as a final stage for the resulting image which should be deployed|
 |region|true|string||AWS Region|
 |access\_key\_id|true|[Secret](#secrets)||Secret containing the AWS `Access Key ID`|
@@ -153,7 +153,7 @@ Deployments are currently only supported for `docker` job types.
 |type|true|string||Has to be "gcr"|
 |host|true|string||GCR endpoint i.e. us.gcr.io|
 |repository|true|string||Name of the repository|
-|tag|false|string|build\_<NUMBER>|The of the image|
+|tag|false|string|build\_\<NUMBER>|The tag of the image|
 |target|false|string||When building a Dockerfile with multiple build stages `target` can be used to specify an intermediate build stage by name as a final stage for the resulting image which should be deployed|
 |region|true|string||AWS Region|
 |service\_account|true|[Secret](#secrets)||Secret containing the GCP `Service Account` with role `Storage Admin`|

src/api/handlers/job_api.py

@@ -1,7 +1,6 @@
 #pylint: disable=too-many-lines,too-few-public-methods,too-many-locals,too-many-statements,too-many-branches
 import os
 import json
-import time
 import uuid
 import copy
 import urllib
@@ -874,9 +873,6 @@ def post(self):
                                build_arguments, deployments, limits_cpu, limits_memory, timeout,
                                resources, json.dumps(job), job['cluster']['name']])
 
-            # to make sure the get picked up in the right order by the scheduler
-            time.sleep(0.1)
-
         g.db.commit()
         return "Successfully create jobs"
 

src/api/handlers/projects/build.py

@@ -124,7 +124,7 @@ def get(self, project_id, build_id):
         ''', [build_id, project_id])
 
         for j in jobs:
-            key = 'project_%s_branch_%s_job_%s.tar.gz' % (project_id, j['branch'], j['name'])
+            key = 'project_%s_job_%s.tar.snappy' % (project_id, j['name'])
             storage.delete_cache(key)
 
         return OK('Cleared cache')

src/api/handlers/projects/jobs.py

@@ -555,7 +555,7 @@ def get(self, project_id, job_id):
         if not job:
             abort(404)
 
-        key = 'project_%s_branch_%s_job_%s.tar.gz' % (project_id, job['branch'], job['name'])
+        key = 'project_%s_job_%s.tar.snappy' % (project_id, job['name'])
         storage.delete_cache(key)
 
         return OK('Cleared cache')

src/github/trigger/trigger.py

@@ -11,8 +11,11 @@
 from pyinfraboxutils.ibbottle import InfraBoxPostgresPlugin
 from pyinfraboxutils.db import connect_db
 
+import bottle
 from bottle import post, run, request, response, install, get
 
+bottle.BaseRequest.MEMFILE_MAX = 10 * 1024 * 1024
+
 logger = get_logger("github")
 
 def res(status, message):

src/job/Dockerfile

@@ -1,6 +1,6 @@
 FROM docker:17.12.1-ce-dind
 
-ENV CLOUD_SDK_VERSION 198.0.0
+ENV CLOUD_SDK_VERSION 210.0.0
 ENV PATH /google-cloud-sdk/bin:$PATH
 
 RUN apk add --no-cache \

src/job/entrypoint.sh

@@ -13,7 +13,7 @@ if [ ! -e /var/run/docker.sock ]; then
     echo "Waiting for docker daemon to start up"
 
     # Start docker daemon
-    dockerd-entrypoint.sh --storage-driver overlay --data-root /data/docker &
+    nohup dockerd-entrypoint.sh --storage-driver overlay --data-root /data/docker > /tmp/dockerd.log &
 
     # Wait until daemon is ready
     COUNTER=0
@@ -23,6 +23,7 @@ if [ ! -e /var/run/docker.sock ]; then
 
       if [ $COUNTER -gt 60 ]; then
         echo "Docker daemon not started" > '/dev/termination-log'
+        cat /tmp/dockerd.log >> /dev/termination-log
         exit 1
       fi
     done

src/job/infrabox_job/job.py

@@ -94,9 +94,14 @@ def create_jobs(self, jobs):
         }
 
         while True:
-            r = requests.post("%s/create_jobs" % self.api_server,
-                              headers=self.get_headers(),
-                              json=payload, timeout=60, verify=self.verify)
+            try:
+                r = requests.post("%s/create_jobs" % self.api_server,
+                                  headers=self.get_headers(),
+                                  json=payload, timeout=300, verify=self.verify)
+            except:
+                self.console.collect('Failed to connect to API, retrying.', show=True)
+                time.sleep(3)
+                continue
 
             if r.status_code == 200:
                 return

src/job/job.py

@@ -286,6 +286,9 @@ def check_container_crashed(self):
 
     def main(self):
         self.load_data()
+        # Date
+        self.console.collect("Date:\n", show=True)
+        self.console.execute(['date'], show=True, show_cmd=False)
 
         # Show environment
         self.console.collect("Environment:\n", show=True)
@@ -503,7 +506,7 @@ def main_run_job(self):
             if os.path.isfile(storage_cache_tar):
                 c.collect("Unpacking cache", show=True)
                 try:
-                    c.execute(['time', 'tar', '-zxf', storage_cache_tar, '-C', self.infrabox_cache_dir], show=True)
+                    self.uncompress(storage_cache_tar, self.infrabox_cache_dir)
                 except:
                     c.collect("Failed to unpack cache\n", show=True)
                 os.remove(storage_cache_tar)

src/pyinfraboxutils/ibflask.py

@@ -186,7 +186,6 @@ def decorated_function(*args, **kwargs):
 
         job_state = r[0]
         if job_state not in ('queued', 'running', 'scheduled'):
-            logger.warn('job not in an active state')
             abort(401, 'Unauthorized')
 
 
@@ -213,7 +212,6 @@ def validate_job_token(token):
 
     job_state = r[0]
     if job_state not in ('queued', 'running', 'scheduled'):
-        logger.warn('job not in an active state')
         abort(401, 'Unauthorized')