Merge pull request #171 from SAP/ssh-key

use ssh key to clone private github repos

Author: ib-steffen

deploy/infrabox/templates/_helpers.tpl

@@ -95,11 +95,13 @@ https://{{- required "host is required" .Values.host -}}:{{- .Values.port -}}
 {{ end }}
 
 {{ define "mounts_gerrit" }}
+{{ if .Values.gerrit.enabled }}
 -
     name: gerrit-ssh
     mountPath: /tmp/gerrit
     readOnly: true
 {{ end }}
+{{ end }}
 
 {{ define "volumes_gerrit" }}
 {{ if .Values.gerrit.enabled }}

deploy/infrabox/templates/function_crd.yaml

@@ -15,7 +15,6 @@ spec:
             memory: 1Gi
     env:
     {{ include "env_general" . | indent 4 }}
-    {{ include "env_gerrit" . | indent 4 }}
     -
         name: INFRABOX_JOB_STORAGE_DRIVER
         value: {{ .Values.job.storage_driver }}
@@ -34,11 +33,6 @@ spec:
     -
         mountPath: /local-cache
         name: local-cache
-{{ end }}
-{{ if .Values.gerrit.enabled }}
-    -
-        mountPath: /var/run/gerrit/
-        name: gerrit-ssh
 {{ end }}
     volumes:
     -
@@ -54,9 +48,3 @@ spec:
         hostPath:
             path: {{ default "/tmp/infrabox/local_cache" .Values.local_cache.host_path }}
 {{ end }}
-{{ if .Values.gerrit.enabled }}
-    -
-        name: gerrit-ssh
-        secret:
-            secretName: infrabox-gerrit-ssh
-{{ end }}

deploy/infrabox/templates/scheduler/deployment.yaml

@@ -35,6 +35,8 @@ spec:
                     value: "443"
                 volumeMounts:
                 {{ include "mounts_rsa_private" . | indent 16 }}
+                {{ include "mounts_gerrit" . | indent 16 }}
             volumes:
                 {{ include "volumes_database" . | indent 16 }}
                 {{ include "volumes_rsa" . | indent 16 }}
+                {{ include "volumes_gerrit" . | indent 16 }}

src/api/handlers/projects/projects.py

@@ -10,6 +10,8 @@
 from pyinfraboxutils.ibflask import OK
 from pyinfraboxutils.ibopa import opa_push_project_data, opa_push_collaborator_data
 
+from Crypto.PublicKey import RSA
+
 ns = api.namespace('Projects',
                    path='/api/v1/projects',
                    description='Project related operations')
@@ -156,11 +158,15 @@ def post(self):
             owner = split[0]
             repo_name = split[1]
 
+            clone_url = repo['clone_url']
+            if repo['private']:
+                clone_url = repo['ssh_url']
+
             g.db.execute('''
                 INSERT INTO repository (name, html_url, clone_url, github_id,
                                         private, project_id, github_owner)
                 VALUES (%s, %s, %s, %s, %s, %s, %s)
-            ''', [repo['name'], repo['html_url'], repo['clone_url'],
+            ''', [repo['name'], repo['html_url'], clone_url,
                   repo['id'], repo['private'], project_id, repo['owner']['login']])
 
             insecure_ssl = "0"
@@ -200,6 +206,31 @@ def post(self):
                 UPDATE repository SET github_hook_id = %s
                 WHERE github_id = %s
             ''', [hook['id'], repo['id']])
+
+            # deploy key
+            key = RSA.generate(2048)
+            private_key = key.exportKey('PEM')
+            public_key = key.publickey().exportKey('OpenSSH')
+            deploy_key_config = {
+                'title': "InfraBox",
+                'key': public_key,
+                'read_only': True
+            }
+
+            url = '%s/repos/%s/%s/keys' % (os.environ['INFRABOX_GITHUB_API_URL'],
+                                           owner, repo_name)
+
+            # TODO(ib-steffen): allow custom ca bundles
+            r = requests.post(url, headers=headers, json=deploy_key_config, verify=False)
+
+            if r.status_code != 201:
+                abort(400, 'Failed to create deploy key')
+
+            g.db.execute('''
+                UPDATE repository SET private_key = %s
+                WHERE github_id = %s
+            ''', [private_key, repo['id']])
+
         elif typ == 'gerrit':
             g.db.execute('''
                 INSERT INTO repository (name, private, project_id, html_url, clone_url, github_id)

src/db/migrations/00022.sql

@@ -0,0 +1 @@
+ALTER TABLE repository ADD COLUMN private_key text;

src/github/trigger/trigger.py

@@ -121,11 +121,10 @@ def create_build(self, commit_id, project_id):
         build_id = result[0][0]
         return build_id
 
-    def create_job(self, commit_id, clone_url, build_id, project_id, github_private_repo, branch, env=None, fork=False):
+    def create_job(self, commit_id, clone_url, build_id, project_id, branch, env=None, fork=False):
         git_repo = {
             "commit": commit_id,
             "clone_url": clone_url,
-            "github_private_repo": github_private_repo,
             "branch": branch,
             "fork": fork
         }
@@ -168,13 +167,12 @@ def create_push(self, c, repository, branch, tag):
             return
 
         result = self.execute('''
-            SELECT id, project_id, private
+            SELECT id, project_id
             FROM repository
             WHERE github_id = %s''', [repository['id']])[0]
 
         repo_id = result[0]
         project_id = result[1]
-        github_repo_private = result[2]
         commit_id = None
 
         result = self.execute('''
@@ -226,7 +224,7 @@ def create_push(self, c, repository, branch, tag):
 
         build_id = self.create_build(commit_id, project_id)
         self.create_job(c['id'], repository['clone_url'], build_id,
-                        project_id, github_repo_private, branch)
+                        project_id, branch)
 
     def handle_push(self, event):
         result = self.execute('''
@@ -275,7 +273,7 @@ def handle_pull_request(self, event):
             return res(200, 'action ignored')
 
         result = self.execute('''
-            SELECT id, project_id, private FROM repository WHERE github_id = %s;
+            SELECT id, project_id FROM repository WHERE github_id = %s;
         ''', [event['repository']['id']])
 
         if not result:
@@ -285,7 +283,6 @@ def handle_pull_request(self, event):
 
         repo_id = result[0]
         project_id = result[1]
-        github_repo_private = result[2]
 
         result = self.execute('''
             SELECT build_on_push FROM project WHERE id = %s;
@@ -398,7 +395,7 @@ def handle_pull_request(self, event):
         build_id = self.create_build(commit_id, project_id)
         self.create_job(event['pull_request']['head']['sha'],
                         event['pull_request']['head']['repo']['clone_url'],
-                        build_id, project_id, github_repo_private, branch, env=env, fork=is_fork)
+                        build_id, project_id, branch, env=env, fork=is_fork)
 
         self.conn.commit()
 

src/job/entrypoint.sh

@@ -31,14 +31,14 @@ else
     echo "Using host docker daemon socket"
 fi
 
-if [ -f /var/run/gerrit/id_rsa ]; then
+if [ ! -z "$INFRABOX_GIT_PRIVATE_KEY" ]; then
     echo "Setting private key"
     eval `ssh-agent -s`
-    cp /var/run/gerrit/id_rsa ~/.ssh/id_rsa
+    echo $INFRABOX_GIT_PRIVATE_KEY > ~/.ssh/id_rsa
     chmod 600 ~/.ssh/id_rsa
     echo "StrictHostKeyChecking no" > ~/.ssh/config
     ssh-add ~/.ssh/id_rsa
-    ssh-keyscan -p $INFRABOX_GERRIT_PORT $INFRABOX_GERRIT_HOSTNAME >> ~/.ssh/known_hosts
+    ssh-keyscan -p $INFRABOX_GIT_PORT $INFRABOX_GIT_HOSTNAME >> ~/.ssh/known_hosts
 else
     echo "No private key configured"
 fi

src/job/job.py

@@ -211,7 +211,6 @@ def get_source(self):
             repo = self.job['repo']
             clone_url = repo['clone_url']
             branch = repo.get('branch', None)
-            private = repo.get('github_private_repo', False)
             clone_all = repo.get('full_history', False)
             ref = repo.get('ref', None)
 
@@ -229,10 +228,6 @@ def get_source(self):
             if not repo_clone:
                 return
 
-            if private:
-                clone_url = clone_url.replace('github.com',
-                                              '%[email protected]' % self.repository['github_api_token'])
-
             self.clone_repo(commit, clone_url, branch, ref, clone_all, submodules=repo_submodules)
         elif self.project['type'] == 'upload':
             c.collect("Downloading Source")

src/scheduler/kubernetes/scheduler.py

@@ -570,6 +570,56 @@ def kube_job(self, job_id, cpu, mem, services=None):
             'value': str(cpu)
         }]
 
+        # Get ssh key for private repos
+        cursor = self.conn.cursor()
+        cursor.execute('''
+            SELECT p.type, p.id
+            FROM project p
+            JOIN job j
+            ON j.project_id = p.id
+            WHERE j.id = %s
+        ''', [job_id])
+        result = cursor.fetchone()
+        cursor.close()
+
+        project_type = result[0]
+        project_id = result[1]
+
+        private_key = None
+        if project_type == 'github':
+            cursor = self.conn.cursor()
+            cursor.execute('''
+                SELECT r.private_key
+                FROM repository r
+                WHERE r.project_id = %s
+            ''', [project_id])
+            result = cursor.fetchone()
+            cursor.close()
+            private_key = result[0]
+
+            env += [{
+                'name': 'INFRABOX_GIT_PORT',
+                'value': '443'
+            }, {
+                'name': 'INFRABOX_GIT_HOSTNAME',
+                'value': 'github.com'
+            }, {
+                'name': 'INFRABOX_GIT_PRIVATE_KEY',
+                'value': private_key
+            }]
+        elif project_type == 'gerrit':
+            with open(os.environ['INFRABOX_GERRIT_KEY_FILENAME']) as key:
+                env += [{
+                    'name': 'INFRABOX_GIT_PORT',
+                    'value': os.environ['INFRABOX_GERRIT_PORT']
+                }, {
+                    'name': 'INFRABOX_GIT_HOSTNAME',
+                    'value': os.environ['INFRABOX_GERRIT_HOSTNAME']
+                }, {
+                    'name': 'INFRABOX_GIT_PRIVATE_KEY',
+                    'value': key.read()
+                }]
+
         root_url = os.environ['INFRABOX_ROOT_URL']
 
         if services: