feat(DM01-5915): upload archive files on job timeout (#597)

* feat(DM01-5915): upload archive files on job timeout

* fix avg_duration

Author: liuwei08

src/api/handlers/job_api.py

@@ -1075,7 +1075,7 @@ def post(self):
                     AND j.state = 'finished'
             """, [project_id, project_id, build_number - 10, build_number, job['name']])[0]
 
-            job['avg_duration'] = avg_duration if avg_duration is not None else 0
+            job['avg_duration'] = float(avg_duration) if avg_duration is not None else 0
 
             if not job['env_vars']:
                 job['env_vars'] = {}

src/scheduler/kubernetes/scheduler.py

@@ -926,6 +926,12 @@ def handle_timeouts(self):
                 WHERE id = %s and state = 'running' """, (job_id,))
             cursor.close()
 
+            # Delete the k8s resource so the pod receives SIGTERM and has
+            # terminationGracePeriodSeconds (60s) to run finalize_upload(),
+            # which uploads /infrabox/upload/archive. kube_delete_job is
+            # idempotent — safe if the resource is already being deleted.
+            self.kube_delete_job(job_id)
+
     def upload_console(self, job_id):
         cursor = self.conn.cursor()
         cursor.execute("begin")