Merge pull request #11 from OPPIDA/fix/various

Author: nolliv22

codesectools/cli.py

@@ -160,7 +160,10 @@ def get_downloadable() -> dict[str, DownloadableRequirement | Dataset]:
 
 
 @cli.command(hidden=download_hidden)
-def download(name: download_arg_type = download_arg_value, test: bool = False) -> None:
+def download(
+    name: download_arg_type = download_arg_value,
+    test: Annotated[bool, typer.Option(hidden=True)] = False,
+) -> None:
     """Download and install any missing resources that are available for download."""
     if name is None:
         print("All downloadable resources have been retrieved.")

codesectools/datasets/BenchmarkJava/dataset.py

@@ -22,17 +22,13 @@ class TestCode(File):
     Inherits from the base `File` class and adds a `vuln_type` attribute
     specific to this dataset.
 
-    Attributes:
-        vuln_type (str): The type of vulnerability present in the file.
-
     """
 
     def __init__(
         self,
         filepath: Path,
         content: str | bytes,
         cwes: list[CWE],
-        vuln_type: str,
         has_vuln: bool,
     ) -> None:
         """Initialize a TestCode instance.
@@ -41,16 +37,13 @@ def __init__(
             filepath: The path to the file.
             content: The content of the file, as a string or bytes.
             cwes: A list of CWEs associated with the file.
-            vuln_type: The type of vulnerability.
             has_vuln: A boolean indicating if the vulnerability is real or a false positive test case.
 
         """
         super().__init__(
             filepath=filepath, content=content, cwes=cwes, has_vuln=has_vuln
         )
 
-        self.vuln_type = vuln_type
-
 
 class BenchmarkJava(PrebuiltFileDataset):
     """Represents the BenchmarkJava dataset.
@@ -151,14 +144,12 @@ def load_dataset(self) -> list[TestCode]:
             filepath = testcode_dir / filename
             content = filepath.read_text()
             cwes = [CWEs.from_id(int(row[3]))]
-            vuln_type = row[1]
             has_vuln = True if row[2] == "true" else False
             files.append(
                 TestCode(
                     filepath.relative_to(self.directory),
                     content,
                     cwes,
-                    vuln_type,
                     has_vuln,
                 )
             )

codesectools/sasts/core/sast/__init__.py

@@ -35,6 +35,7 @@
     USER_OUTPUT_DIR,
     MissingFile,
     NonZeroExit,
+    render_command,
     run_command,
 )
 
@@ -52,7 +53,7 @@ class SAST(ABC):
         supported_datasets (list[Dataset]): A list of supported dataset classes.
         properties (SASTProperties): The properties of the SAST tool.
         requirements (SASTRequirements): The requirements for the SAST tool.
-        commands (list[list[Union[str, tuple[str]]]]): The list of commands templates to be rendred and executed.
+        commands (list[list[Union[str, tuple[str]]]]): The list of commands templates to be rendered and executed.
         valid_codes (list[int]): A list of exit codes indicating that the command did not fail.
         environ (dict[str, str]): Environment variables to set for commands.
         output_files (list[tuple[Path, bool]]): Expected output files and
@@ -89,46 +90,15 @@ def __init__(self) -> None:
         Set up supported datasets, the output directory, and requirement status.
         """
         self.supported_datasets = [
-            DATASETS_ALL[d] for d in self.supported_dataset_names
+            DATASETS_ALL[d]
+            for d in self.supported_dataset_names
+            if DATASETS_ALL[d].is_cached()
         ]
         self.output_dir = USER_OUTPUT_DIR / self.name
         self.requirements.name = self.name
         self.status = self.requirements.get_status()
         self.missing = self.requirements.get_missing()
 
-    def render_command(self, command: list[str], map: dict[str, str]) -> list[str]:
-        """Render a command template by replacing placeholders with values.
-
-        Args:
-            command: The command template as a list of strings.
-            map: A dictionary of placeholders to their replacement values.
-
-        Returns:
-            The rendered command as a list of strings.
-
-        """
-        _command = command.copy()
-        for pattern, value in map.items():
-            for i, arg in enumerate(_command):
-                # Check if optional argument can be used
-                if isinstance(arg, tuple):
-                    default_arg, optional_arg = arg
-                    if pattern in optional_arg:
-                        _command[i] = arg.replace(pattern, value)
-                    else:
-                        _command[i] = default_arg
-                else:
-                    if pattern in arg:
-                        _command[i] = arg.replace(pattern, value)
-
-        # Remove not rendered part of the command:
-        __command = []
-        for part in _command:
-            if not ("{" in part and "}" in part):
-                __command.append(part)
-
-        return __command
-
     def run_analysis(
         self, lang: str, project_dir: Path, output_dir: Path, **kwargs: Any
     ) -> None:
@@ -165,7 +135,7 @@ def run_analysis(
             command_output = ""
             start = time.time()
             for command in self.commands:
-                rendered_command = self.render_command(command, render_variables)
+                rendered_command = render_command(command, render_variables)
                 retcode, out = run_command(rendered_command, project_dir, self.environ)
                 command_output += out
                 if retcode not in self.valid_codes:

codesectools/sasts/tools/Bearer/sast.py

@@ -26,7 +26,7 @@ class BearerSAST(BuildlessSAST):
         supported_dataset_names (list[str]): A list of names of compatible datasets.
         properties (SASTProperties): The properties of the SAST tool.
         requirements (SASTRequirements): The requirements for the SAST tool.
-        commands (list[list[Union[str, tuple[str]]]]): The list of commands templates to be rendred and executed.
+        commands (list[list[Union[str, tuple[str]]]]): The list of commands templates to be rendered and executed.
         valid_codes (list[int]): A list of exit codes indicating that the command did not fail.
         output_files (list[tuple[Path, bool]]): A list of expected output files and
             whether they are required.

codesectools/sasts/tools/Coverity/sast.py

@@ -25,7 +25,7 @@ class CoveritySAST(BuildlessSAST):
         supported_dataset_names (list[str]): A list of names of compatible datasets.
         properties (SASTProperties): The properties of the SAST tool.
         requirements (SASTRequirements): The requirements for the SAST tool.
-        commands (list[list[Union[str, tuple[str]]]]): The list of commands templates to be rendred and executed.
+        commands (list[list[Union[str, tuple[str]]]]): The list of commands templates to be rendered and executed.
         valid_codes (list[int]): A list of exit codes indicating that the command did not fail.
         output_files (list[tuple[Path, bool]]): A list of expected output files and
             whether they are required.

codesectools/sasts/tools/Cppcheck/parser.py

@@ -79,7 +79,7 @@ def __init__(self, output_dir: Path, xml_tree: ElementTree, cmdout: dict) -> Non
                         filepath=Path(error.xpath("location")[0].get("file")),
                         checker=error.get("id"),
                         category=category,
-                        cwe=CWEs.from_id(error.get("cwe", -1)),
+                        cwe=CWEs.from_id(int(error.get("cwe", -1))),
                         message=error.get("msg"),
                         lines=[
                             int(location.get("line"))

codesectools/sasts/tools/Cppcheck/sast.py

@@ -24,7 +24,7 @@ class CppcheckSAST(PrebuiltBuildlessSAST):
         supported_dataset_names (list[str]): A list of names of compatible datasets.
         properties (SASTProperties): The properties of the SAST tool.
         requirements (SASTRequirements): The requirements for the SAST tool.
-        commands (list[list[Union[str, tuple[str]]]]): The list of commands templates to be rendred and executed.
+        commands (list[list[Union[str, tuple[str]]]]): The list of commands templates to be rendered and executed.
         valid_codes (list[int]): A list of exit codes indicating that the command did not fail.
         output_files (list[tuple[Path, bool]]): A list of expected output files and
             whether they are required.

codesectools/sasts/tools/SemgrepCE/sast.py

@@ -26,7 +26,7 @@ class SemgrepCESAST(BuildlessSAST):
         supported_dataset_names (list[str]): A list of names of compatible datasets.
         properties (SASTProperties): The properties of the SAST tool.
         requirements (SASTRequirements): The requirements for the SAST tool.
-        commands (list[list[Union[str, tuple[str]]]]): The list of commands templates to be rendred and executed.
+        commands (list[list[Union[str, tuple[str]]]]): The list of commands templates to be rendered and executed.
         valid_codes (list[int]): A list of exit codes indicating that the command did not fail.
         output_files (list[tuple[Path, bool]]): A list of expected output files and
             whether they are required.

codesectools/sasts/tools/SnykCode/parser.py

@@ -54,7 +54,7 @@ class SnykCodeAnalysisResult(AnalysisResult):
     metadata, including timings, file lists, and defects.
     """
 
-    normalize_lang_names = {"cpp": ["c", "cpp"]}
+    normalize_lang_names = {"java": ["java"], "cpp": ["c", "cpp"]}
 
     def __init__(self, output_dir: Path, result_data: dict, cmdout: dict) -> None:
         """Initialize a SnykCodeAnalysisResult instance.
@@ -83,7 +83,7 @@ def __init__(self, output_dir: Path, result_data: dict, cmdout: dict) -> None:
             for result in run["results"]:
                 rule_index = result["ruleIndex"]
                 lang, *_, checker = result["ruleId"].split("/")
-                if self.lang not in self.normalize_lang_names[lang]:
+                if self.lang not in self.normalize_lang_names.get(lang, []):
                     continue
 
                 start = (

codesectools/sasts/tools/SnykCode/sast.py

@@ -22,7 +22,7 @@ class SnykCodeSAST(BuildlessSAST):
         supported_dataset_names (list[str]): A list of names of compatible datasets.
         properties (SASTProperties): The properties of the SAST tool.
         requirements (SASTRequirements): The requirements for the SAST tool.
-        commands (list[list[Union[str, tuple[str]]]]): The list of commands templates to be rendred and executed.
+        commands (list[list[Union[str, tuple[str]]]]): The list of commands templates to be rendered and executed.
         valid_codes (list[int]): A list of exit codes indicating that the command did not fail.
         output_files (list[tuple[Path, bool]]): A list of expected output files and
             whether they are required.