fix(DM01-5918): docker-compose containers before archive upload on timeout (#598)

* fix(DM01-5918): gracefully stop docker-compose containers before archive upload on timeout

* increase the priority of the upload_archive() function

* update upload archive func

Author: liuwei08

src/job/job.py

@@ -378,22 +378,12 @@ def upload_archive(self):
         testresult_exists = False
 
         if os.path.exists(self.infrabox_archive_dir):
-            files = self.get_files_in_dir(self.infrabox_archive_dir)
-
-            if files:
-                c.collect("Uploading /infrabox/upload/archive", show=True)
+            if self.get_files_in_dir(self.infrabox_archive_dir):
                 archive_exists = True
-                for f in files:
-                    c.collect("%s" % f, show=True)
-                    self.post_file_to_api_server("/archive", f, filename=f.replace(self.infrabox_upload_dir+'/', ''))
 
         if os.path.exists(self.infrabox_testresult_dir):
-            files = self.get_files_in_dir(self.infrabox_testresult_dir)
-
-            if files:
+            if self.get_files_in_dir(self.infrabox_testresult_dir):
                 testresult_exists = True
-                for f in files:
-                    c.collect("%s" % f, show=True)
 
         tar_file = os.path.join(self.infrabox_upload_dir, 'all_archives' + '.tar.gz')
         with tarfile.open(tar_file, mode='w:gz') as archive:
@@ -404,6 +394,12 @@ def upload_archive(self):
 
         self.post_file_to_api_server("/archive", tar_file)
 
+        if archive_exists:
+            c.collect("Uploading /infrabox/upload/archive", show=True)
+            for f in self.get_files_in_dir(self.infrabox_archive_dir):
+                c.collect("%s" % f, show=True)
+                self.post_file_to_api_server("/archive", f, filename=f.replace(self.infrabox_upload_dir+'/', ''))
+
 
     def upload_coverage_results(self):
         if not os.path.exists(self.infrabox_coverage_dir):
@@ -501,16 +497,24 @@ def _get_size(self, start_path):
         return total_size
 
     def finalize_upload(self):
+        self.upload_archive()
         self.upload_coverage_results()
         self.upload_test_results()
         self.upload_markup_files()
         self.upload_badge_files()
-        self.upload_archive()
 
     def handle_abort(self, signum, sigframe):
         if not self.aborted:
             self.aborted = True
             self.console.collect("##Aborted", show=True)
+            if hasattr(self, '_compose_file_new'):
+                try:
+                    subprocess.call(
+                        ['docker-compose', '-f', self._compose_file_new, 'stop', '--timeout', '30'],
+                        env=self.environment,
+                        timeout=35)
+                except Exception:
+                    pass
             self.finalize_upload()
 
     def main_run_job(self):
@@ -722,6 +726,8 @@ def run_job_docker_compose(self, c):
         with open(compose_file_new, "w+") as out:
             json.dump(compose_file_content, out)
 
+        self._compose_file_new = compose_file_new
+
         collector = StatsCollector()
 
         self._login_source_registries()

src/scheduler/kubernetes/scheduler.py

@@ -481,7 +481,7 @@ def _sync(self, fi):
                         'automountServiceAccountToken': False,
                         'containers': containers,
                         'restartPolicy': 'Never',
-                        'terminationGracePeriodSeconds': 60,
+                        'terminationGracePeriodSeconds': 180,
                         'volumes': f['spec']['volumes'],
                         'imagePullSecrets': f['spec'].get('imagePullSecrets', None)
                     },