Merge pull request #6 from OPPIDA/fix/sast-various

Author: nolliv22

codesectools/datasets/core/dataset.py

@@ -9,7 +9,6 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
-from pathlib import Path
 from typing import TYPE_CHECKING
 
 import git
@@ -22,6 +21,7 @@
 from codesectools.utils import USER_CACHE_DIR
 
 if TYPE_CHECKING:
+    from pathlib import Path
     from typing import Self
 
     from codesectools.sasts.core.parser import AnalysisResult, Defect
@@ -296,7 +296,7 @@ def validate(self, analysis_result: AnalysisResult) -> FileDatasetData:
             if not defect.cwe or defect.cwe.id == -1:
                 continue
 
-            file_cwe_pair = (Path(defect.file).name, defect.cwe)  # TODO: USE FULL PATH
+            file_cwe_pair = (defect.filename, defect.cwe)
             if file_cwe_pair not in unique_reported_defects:
                 unique_reported_defects[file_cwe_pair] = defect
 
@@ -563,18 +563,18 @@ def validate(self, analysis_results: list[AnalysisResult]) -> GitRepoDatasetData
 
             # 1. Process reported defects to get unique (file, cwe) pairs
             # and keep one original Defect object for each to retain metadata.
-            unique_reported_defects: dict[tuple[str, CWE], Defect] = {}
+            unique_reported_defects: dict[tuple[Path, CWE], Defect] = {}
             for defect in analysis_result.defects:
                 if not defect.cwe or defect.cwe.id == -1:
                     continue
 
-                file_cwe_pair = (defect.file_path, defect.cwe)
+                file_cwe_pair = (defect.filepath, defect.cwe)
                 if file_cwe_pair not in unique_reported_defects:
                     unique_reported_defects[file_cwe_pair] = defect
 
             # 2. Classify unique reported defects as TP or FP.
-            tp_defects_map: dict[tuple[str, CWE], Defect] = {}
-            fp_defects_map: dict[tuple[str, CWE], Defect] = {}
+            tp_defects_map: dict[tuple[Path, CWE], Defect] = {}
+            fp_defects_map: dict[tuple[Path, CWE], Defect] = {}
 
             if repo.has_vuln:
                 for (filename, cwe), defect in unique_reported_defects.items():

codesectools/sasts/all/cli.py

@@ -157,13 +157,21 @@ def list_() -> None:
             table.add_row(
                 dataset_full_name,
                 "Dataset",
-                ", ".join(f"[b]{sast.name}[/b]" for sast in all_sast.sasts),
+                ", ".join(
+                    f"[b]{sast.name}[/b]"
+                    for sast in all_sast.sasts
+                    if dataset_full_name in sast.list_results(dataset=True)
+                ),
             )
         for project in all_sast.list_results(project=True):
             table.add_row(
                 project,
                 "Project",
-                ", ".join(f"[b]{sast.name}[/b]" for sast in all_sast.sasts),
+                ", ".join(
+                    f"[b]{sast.name}[/b]"
+                    for sast in all_sast.sasts
+                    if project in sast.list_results(project=True)
+                ),
             )
 
         print(table)

codesectools/sasts/all/parser.py

@@ -76,11 +76,11 @@ def stats_by_files(self) -> dict:
         """Calculate statistics on defects, grouped by file."""
         stats = {}
         for defect in self.defects:
-            if defect.file_path not in stats.keys():
-                stats[defect.file_path] = {"count": 1, "sasts": [defect.sast]}
+            if defect.filepath_str not in stats.keys():
+                stats[defect.filepath_str] = {"count": 1, "sasts": [defect.sast]}
             else:
-                stats[defect.file_path]["sasts"].append(defect.sast)
-                stats[defect.file_path]["count"] += 1
+                stats[defect.filepath_str]["sasts"].append(defect.sast)
+                stats[defect.filepath_str]["count"] += 1
 
         return stats
 
@@ -117,13 +117,13 @@ def stats_by_cwes(self) -> dict:
             if defect.cwe not in stats:
                 stats[defect.cwe] = {
                     "count": 1,
-                    "files": [defect.file_path],
+                    "files": [defect.filepath_str],
                     "sast_counts": {defect.sast: 1},
                 }
             else:
                 stats[defect.cwe]["count"] += 1
-                if defect.file_path not in stats[defect.cwe]["files"]:
-                    stats[defect.cwe]["files"].append(defect.file_path)
+                if defect.filepath_str not in stats[defect.cwe]["files"]:
+                    stats[defect.cwe]["files"].append(defect.filepath_str)
                 stats[defect.cwe]["sast_counts"][defect.sast] = (
                     stats[defect.cwe]["sast_counts"].get(defect.sast, 0) + 1
                 )
@@ -133,9 +133,9 @@ def stats_by_scores(self) -> dict:
         """Calculate a risk score for each file based on defect data."""
         defect_files = {}
         for defect in self.defects:
-            if defect.file_path not in defect_files:
-                defect_files[defect.file_path] = []
-            defect_files[defect.file_path].append(defect)
+            if defect.filepath_str not in defect_files:
+                defect_files[defect.filepath_str] = []
+            defect_files[defect.filepath_str].append(defect)
 
         stats = {}
         for defect_file, defects in defect_files.items():
@@ -192,9 +192,9 @@ def prepare_report_data(self) -> dict:
 
         defect_files = {}
         for defect in self.defects:
-            if defect.file_path not in defect_files:
-                defect_files[defect.file_path] = []
-            defect_files[defect.file_path].append(defect)
+            if defect.filepath_str not in defect_files:
+                defect_files[defect.filepath_str] = []
+            defect_files[defect.filepath_str].append(defect)
 
         for defect_file, defects in defect_files.items():
             for k, v in scores[defect_file]["score"].items():
@@ -211,7 +211,7 @@ def prepare_report_data(self) -> dict:
 
             report["defects"][defect_file] = {
                 "score": scores[defect_file]["score"],
-                "source_path": str(self.source_path / defect.file),
+                "source_path": str(self.source_path / defect.filepath),
                 "locations": locations,
                 "raw": defects,
             }

codesectools/sasts/all/sast.py

@@ -62,7 +62,7 @@ def list_results(
                     sast.list_results(project=project, dataset=dataset, limit=limit)
                 )
             else:
-                output_dirs &= set(
+                output_dirs |= set(
                     sast.list_results(project=project, dataset=dataset, limit=limit)
                 )
         return output_dirs

codesectools/sasts/core/parser.py

@@ -18,8 +18,9 @@ class Defect:
 
     Attributes:
         sast (str): The name of the SAST tool that reported the defect.
-        file (Path): The path to the file where the defect was found.
-        file_path (str): The string representation of the file path.
+        filepath (Path): The path to the file where the defect was found.
+        filepath_str (str): The string representation of the file path.
+        filename (str): The name of the file.
         checker (str): The name of the checker or rule that reported the defect.
         category (str): The category of the checker (e.g., security, performance).
         cwe (CWE): The CWE associated with the defect.
@@ -33,7 +34,7 @@ class Defect:
 
     def __init__(
         self,
-        file: Path,
+        filepath: Path,
         checker: str,
         category: str,
         cwe: CWE,
@@ -44,7 +45,7 @@ def __init__(
         """Initialize a Defect instance.
 
         Args:
-            file: The file path of the defect.
+            filepath: The file path of the defect.
             checker: The name of the rule/checker.
             category: The category of the checker.
             cwe: The CWE associated with the defect.
@@ -53,8 +54,9 @@ def __init__(
             data: Raw data from the SAST tool for this defect.
 
         """
-        self.file = file
-        self.file_path = str(file)
+        self.filepath = filepath
+        self.filepath_str = str(filepath)
+        self.filename = filepath.name
         self.checker = checker
         self.category = category
         self.cwe = cwe
@@ -206,9 +208,9 @@ def stats_by_checkers(self) -> dict:
         stats = {}
         for defect in self.defects:
             if defect.checker not in stats.keys():
-                stats[defect.checker] = {"count": 1, "files": [defect.file_path]}
+                stats[defect.checker] = {"count": 1, "files": [defect.filepath_str]}
             else:
-                stats[defect.checker]["files"].append(defect.file_path)
+                stats[defect.checker]["files"].append(defect.filepath_str)
                 stats[defect.checker]["count"] += 1
 
         return stats
@@ -250,11 +252,11 @@ def stats_by_files(self) -> dict:
         """
         stats = {}
         for defect in self.defects:
-            if defect.file_path not in stats.keys():
-                stats[defect.file_path] = {"count": 1, "checkers": [defect.checker]}
+            if defect.filepath_str not in stats.keys():
+                stats[defect.filepath_str] = {"count": 1, "checkers": [defect.checker]}
             else:
-                stats[defect.file_path]["checkers"].append(defect.checker)
-                stats[defect.file_path]["count"] += 1
+                stats[defect.filepath_str]["checkers"].append(defect.checker)
+                stats[defect.filepath_str]["count"] += 1
 
         return stats
 
@@ -269,9 +271,9 @@ def stats_by_cwes(self) -> dict:
         stats = {}
         for defect in self.defects:
             if defect.cwe not in stats.keys():
-                stats[defect.cwe] = {"count": 1, "files": [defect.file_path]}
+                stats[defect.cwe] = {"count": 1, "files": [defect.filepath_str]}
             else:
-                stats[defect.cwe]["files"].append(defect.file_path)
+                stats[defect.cwe]["files"].append(defect.filepath_str)
                 stats[defect.cwe]["count"] += 1
 
         return stats

codesectools/sasts/core/sast/__init__.py

@@ -187,17 +187,19 @@ def save_results(self, project_dir: Path, output_dir: Path, extra: dict) -> None
             parent_dir = path_from_root.parent
             filename = path_from_root.name
             if "*" not in filename:
-                file_path = project_dir / parent_dir / filename
-                if file_path.is_file():
-                    shutil.copy2(file_path, output_dir / filename)
+                filepath = project_dir / parent_dir / filename
+                if filepath.is_file():
+                    if not filepath == output_dir / filename:
+                        shutil.copy2(filepath, output_dir / filename)
                 else:
                     if required:
                         missing_files.append(filename)
             else:
-                file_paths = (project_dir / parent_dir).glob(filename)
-                if file_paths:
-                    for file_path in file_paths:
-                        shutil.copy2(file_path, output_dir / file_path.name)
+                filepaths = (project_dir / parent_dir).glob(filename)
+                if filepaths:
+                    for filepath in filepaths:
+                        if not filepath == output_dir / filename:
+                            shutil.copy2(filepath, output_dir / filepath.name)
                 else:
                     if required:
                         missing_files.append(filename)

codesectools/sasts/tools/Bearer/parser.py

@@ -28,7 +28,7 @@ def __init__(self, defect_data: dict, severity: str) -> None:
 
         """
         super().__init__(
-            file=Path(defect_data["filename"]),
+            filepath=Path(defect_data["filename"]),
             checker=defect_data["id"],
             category=severity,
             cwe=CWEs.from_id(int(defect_data["cwe_ids"][0])),

codesectools/sasts/tools/Coverity/parser.py

@@ -78,7 +78,7 @@ def __init__(self, defect_data: dict) -> None:
 
         """
         super().__init__(
-            file=Path(defect_data["file"]),
+            filepath=Path(defect_data["file"]),
             checker=defect_data["checker"],
             category=None,
             cwe=CWEs.from_id(TYPE_TO_CWE.get(defect_data["type"], -1)),
@@ -158,7 +158,7 @@ def __init__(
 
         self.time = int(self.metrics["time"])
 
-        self.files = list(map(lambda line: Path(line).name, captured_list.splitlines()))
+        self.files = list(map(lambda line: str(Path(line)), captured_list.splitlines()))
 
         file_count = 0
         for lang, pattern in LANGUAGES.items():
@@ -202,16 +202,16 @@ def load_from_output_dir(cls, output_dir: Path) -> Self:
         cmdout = json.load((output_dir / "cstools_output.json").open())
 
         # Analysis metrics
-        file_path = output_dir / "ANALYSIS.metrics.xml"
-        if file_path.is_file():
-            analysis_data = xmltodict.parse(file_path.open("rb"))
+        filepath = output_dir / "ANALYSIS.metrics.xml"
+        if filepath.is_file():
+            analysis_data = xmltodict.parse(filepath.open("rb"))
         else:
             raise MissingFile(["ANALYSIS.metrics.xml"])
 
         # Config
-        file_path = output_dir / "coverity.yaml"
-        if file_path.is_file():
-            config_data = yaml.load(file_path.open("r"), Loader=yaml.Loader)
+        filepath = output_dir / "coverity.yaml"
+        if filepath.is_file():
+            config_data = yaml.load(filepath.open("r"), Loader=yaml.Loader)
         else:
             config_data = None
 

codesectools/sasts/tools/SemgrepCE/parser.py

@@ -37,7 +37,7 @@ def __init__(self, defect_data: dict) -> None:
 
         """
         super().__init__(
-            file=Path(defect_data["path"]),
+            filepath=Path(defect_data["path"]),
             checker=defect_data["check_id"].split(".")[-1],
             category=defect_data["extra"]["metadata"].get("impact", "NONE"),
             cwe=CWEs.from_string(defect_data["extra"]["metadata"].get("cwe", [""])[0]),

codesectools/sasts/tools/SnykCode/parser.py

@@ -24,7 +24,7 @@ class SnykCodeIssue(Defect):
 
     def __init__(
         self,
-        file: Path,
+        filepath: Path,
         checker: str,
         category: str,
         cwe: CWE,
@@ -35,7 +35,7 @@ def __init__(
         """Initialize a SnykCodeIssue instance.
 
         Args:
-            file: The file path of the defect.
+            filepath: The file path of the defect.
             checker: The name of the rule/checker.
             category: The category of the checker.
             cwe: The CWE associated with the defect.
@@ -44,7 +44,7 @@ def __init__(
             data: Raw data from the SAST tool for this defect.
 
         """
-        super().__init__(file, checker, category, cwe, message, location, data)
+        super().__init__(filepath, checker, category, cwe, message, location, data)
 
 
 class SnykCodeAnalysisResult(AnalysisResult):
@@ -85,7 +85,7 @@ def __init__(self, output_dir: Path, result_data: dict, cmdout: dict) -> None:
                     continue
 
                 defect = SnykCodeIssue(
-                    file=Path(
+                    filepath=Path(
                         result["locations"][0]["physicalLocation"]["artifactLocation"][
                             "uri"
                         ]
@@ -112,7 +112,7 @@ def __init__(self, output_dir: Path, result_data: dict, cmdout: dict) -> None:
                 )
                 self.defects.append(defect)
 
-        self.files = list(set(d.file for d in self.defects))
+        self.files = list(set(d.filepath_str for d in self.defects))
 
     @classmethod
     def load_from_output_dir(cls, output_dir: Path) -> Self: