feat: improve the source tracking to reduce Source.Unknown

Author: ibbem

src/main/java/org/variantsync/diffdetective/diff/git/GitDiffer.java

@@ -21,9 +21,11 @@
 import org.variantsync.diffdetective.variation.DiffLinesLabel;
 import org.variantsync.diffdetective.variation.diff.VariationDiff;
 import org.variantsync.diffdetective.variation.diff.parse.VariationDiffParser;
+import org.variantsync.diffdetective.variation.tree.source.GitSource;
 
 import java.io.*;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
@@ -252,6 +254,7 @@ private static CommitDiffResult getPatchDiffs(
 
                     final VariationDiff<DiffLinesLabel> variationDiff = VariationDiffParser.createVariationDiff(
                             fullDiff,
+                            new GitSource(repository, childCommit.getId().name(), Path.of(filename)),
                             repository.getParseOptions().variationDiffParseOptions()
                     );
 

src/main/java/org/variantsync/diffdetective/examplesearch/ExampleFinder.java

@@ -106,14 +106,10 @@ private boolean checkIfExample(Analysis analysis, String localDiff) {
         // We do not want a variationDiff for the entire file but only for the local change to have a small example.
         final VariationDiff<DiffLinesLabel> localTree;
         try {
-            localTree = VariationDiff.fromDiff(localDiff, new VariationDiffParseOptions(annotationParser, true, true));
+            localTree = VariationDiff.fromDiff(localDiff, Source.findFirst(variationDiff, GitPatch.class), new VariationDiffParseOptions(annotationParser, true, true));
             // Not every local diff can be parsed to a VariationDiff because diffs are unaware of the underlying language (i.e., CPP).
             // We want only running examples whose diffs describe entire diff trees for easier understanding.
-            if (isGoodExample.test(localTree)) {
-                GitPatch variationDiffSource = Source.findFirst(variationDiff, GitPatch.class);
-                Assert.assertNotNull(variationDiffSource);
-                localTree.setSource(variationDiffSource.shallowClone());
-            } else {
+            if (!isGoodExample.test(localTree)) {
                 return false;
             }
         } catch (DiffParseException e) {

src/main/java/org/variantsync/diffdetective/experiments/thesis_bm/ConstructionValidation.java

@@ -5,6 +5,7 @@
 import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.io.Writer;
+import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
@@ -42,6 +43,7 @@
 import org.variantsync.diffdetective.variation.diff.Time;
 import org.variantsync.diffdetective.variation.diff.filter.VariationDiffFilter;
 import org.variantsync.diffdetective.variation.diff.parse.VariationDiffParser;
+import org.variantsync.diffdetective.variation.tree.source.GitSource;
 import org.variantsync.functjonal.category.InplaceSemigroup;
 import org.variantsync.functjonal.map.MergeMap;
 
@@ -375,6 +377,7 @@ private void counts(VariationDiff<DiffLinesLabel> tree, VariationDiffStatistics
     }
 
     private VariationDiff<DiffLinesLabel> parseVariationTree(Analysis analysis, RevCommit commit) throws IOException, DiffParseException {
+        String fileName = analysis.getCurrentPatch().getFileName(AFTER);
         try (BufferedReader afterFile =
             new BufferedReader(
                 /*
@@ -386,10 +389,14 @@ private VariationDiff<DiffLinesLabel> parseVariationTree(Analysis analysis, RevC
                     GitDiffer.getBeforeFullFile(
                         analysis.getRepository(),
                         commit,
-                        analysis.getCurrentPatch().getFileName(AFTER)),
+                        fileName),
                     0xfeff)) // BOM, same as GitDiffer.BOM_PATTERN
         ) {
-            return VariationDiffParser.createVariationTree(afterFile, analysis.getRepository().getParseOptions().variationDiffParseOptions());
+            return VariationDiffParser.createVariationTree(
+                afterFile,
+                new GitSource(analysis.getRepository(), commit.getId().name(), Path.of(fileName)),
+                analysis.getRepository().getParseOptions().variationDiffParseOptions()
+            );
         }
     }
 

src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java

@@ -226,7 +226,7 @@ public static String parseUnparseTree(String text, VariationDiffParseOptions opt
     public static String parseUnparseDiff(String textDiff, VariationDiffParseOptions option) {
         String temp = "b";
         try {
-            VariationDiff<DiffLinesLabel> diff = VariationDiff.fromDiff(textDiff, option);
+            VariationDiff<DiffLinesLabel> diff = VariationDiff.fromDiff(textDiff, Source.Unknown, option);
             temp = VariationUnparser.unparseDiff(diff);
         } catch (Exception e) {
             e.printStackTrace();

src/main/java/org/variantsync/diffdetective/variation/diff/VariationDiff.java

@@ -20,7 +20,6 @@
 import org.variantsync.diffdetective.variation.diff.construction.JGitDiff;
 import org.variantsync.diffdetective.variation.diff.parse.VariationDiffParseOptions;
 import org.variantsync.diffdetective.variation.diff.parse.VariationDiffParser;
-import org.variantsync.diffdetective.variation.diff.source.PatchString;
 import org.variantsync.diffdetective.variation.diff.traverse.VariationDiffTraversal;
 import org.variantsync.diffdetective.variation.diff.traverse.VariationDiffVisitor;
 import org.variantsync.diffdetective.variation.tree.VariationTree;
@@ -91,9 +90,7 @@ public VariationDiff(DiffNode<L> root, Source source) {
      */
     public static VariationDiff<DiffLinesLabel> fromFile(final Path p, VariationDiffParseOptions parseOptions) throws IOException, DiffParseException {
         try (BufferedReader file = Files.newBufferedReader(p)) {
-            final VariationDiff<DiffLinesLabel> tree = VariationDiffParser.createVariationDiff(file, parseOptions);
-            tree.setSource(new FileSource(p));
-            return tree;
+            return VariationDiffParser.createVariationDiff(file, new FileSource(p), parseOptions);
         }
     }
 
@@ -103,13 +100,13 @@ public static VariationDiff<DiffLinesLabel> fromFile(final Path p, VariationDiff
      * So just lines preceded by "+", "-", or " " are expected.
      * @param diff The diff as text. Lines should be separated by a newline character. Each line should be preceded by either "+", "-", or " ".
      * @param parseOptions {@link VariationDiffParseOptions} for the parsing process.
+     * @param source the {@link Source} of {@code diff}
      * @return A result either containing the parsed VariationDiff or an error message in case of failure.
      * @throws DiffParseException if {@code diff} couldn't be parsed
      */
-    public static VariationDiff<DiffLinesLabel> fromDiff(final String diff, final VariationDiffParseOptions parseOptions) throws DiffParseException {
-        final VariationDiff<DiffLinesLabel> d;
+    public static VariationDiff<DiffLinesLabel> fromDiff(final String diff, final Source source, final VariationDiffParseOptions parseOptions) throws DiffParseException {
         try {
-            d = VariationDiffParser.createVariationDiff(diff, parseOptions);
+            return VariationDiffParser.createVariationDiff(diff, source, parseOptions);
         } catch (DiffParseException e) {
             Logger.error("""
                             Could not parse diff:
@@ -119,8 +116,6 @@ public static VariationDiff<DiffLinesLabel> fromDiff(final String diff, final Va
                     diff);
             throw e;
         }
-        d.setSource(new PatchString(diff));
-        return d;
     }
 
     /**
@@ -164,7 +159,7 @@ public static Result<VariationDiff<DiffLinesLabel>, List<DiffError>> fromPatch(f
 
     /**
      * Create a VariationDiff from two given text files.
-     * @see #fromLines(String, String, DiffAlgorithm.SupportedAlgorithm, VariationDiffParseOptions)
+     * @see #fromLines(String, String, Source, Source, DiffAlgorithm.SupportedAlgorithm, VariationDiffParseOptions)
      */
     public static VariationDiff<DiffLinesLabel> fromFiles(
             final Path beforeFile,
@@ -176,22 +171,24 @@ public static VariationDiff<DiffLinesLabel> fromFiles(
         try (BufferedReader b = Files.newBufferedReader(beforeFile);
              BufferedReader a = Files.newBufferedReader(afterFile)
         ) {
-            return fromLines(IOUtils.toString(b), IOUtils.toString(a), algorithm, options);
+            return fromLines(IOUtils.toString(b), IOUtils.toString(a), new FileSource(beforeFile), new FileSource(afterFile), algorithm, options);
         }
     }
 
     /**
      * Creates a variation diff from to line-based text inputs.
      * This method just forwards to:
-     * @see JGitDiff#diff(String, String, DiffAlgorithm.SupportedAlgorithm, VariationDiffParseOptions)
+     * @see JGitDiff#diff(String, String, Source, Source, DiffAlgorithm.SupportedAlgorithm, VariationDiffParseOptions)
      */
     public static VariationDiff<DiffLinesLabel> fromLines(
             String before,
             String after,
+            Source beforeSource,
+            Source afterSource,
             DiffAlgorithm.SupportedAlgorithm algorithm,
             VariationDiffParseOptions options) throws IOException, DiffParseException
     {
-        return JGitDiff.diff(before, after, algorithm, options);
+        return JGitDiff.diff(before, after, beforeSource, afterSource, algorithm, options);
     }
 
     /**

src/main/java/org/variantsync/diffdetective/variation/diff/construction/JGitDiff.java

@@ -4,6 +4,8 @@
 import org.eclipse.jgit.diff.*;
 import org.variantsync.diffdetective.diff.git.GitDiffer;
 import org.variantsync.diffdetective.diff.result.DiffParseException;
+import org.variantsync.diffdetective.util.CompositeSource;
+import org.variantsync.diffdetective.util.Source;
 import org.variantsync.diffdetective.variation.DiffLinesLabel;
 import org.variantsync.diffdetective.variation.diff.Time;
 import org.variantsync.diffdetective.variation.diff.VariationDiff;
@@ -122,9 +124,11 @@ Using our own formatter without diff headers (paired with a maximum context (?))
      * Uses JGit to diff the two files using the specified {@code options}, and afterwards, creates the variation diff.
      * Creates a variation diff from to line-based text inputs.
      * First creates a line-based diff with {@link #textDiff(String, String, DiffAlgorithm.SupportedAlgorithm)}
-     * and then parses that diff with {@link VariationDiff#fromDiff(String, VariationDiffParseOptions)}.
+     * and then parses that diff with {@link VariationDiff#fromDiff(String, Source, VariationDiffParseOptions)}.
      * @param linesBefore State of annotated lines before the change.
      * @param linesAfter State of annotated lines after the change.
+     * @param sourceBefore the {@link Source} of {@code linesBefore}
+     * @param sourceAfter the {@link Source} of {@code linesAfter}
      * @param algorithm Specification of which algorithm to use for diffing with JGit.
      * @param options various options for parsing
      * @return A variation diff comprising the changes.
@@ -134,9 +138,15 @@ Using our own formatter without diff headers (paired with a maximum context (?))
     public static VariationDiff<DiffLinesLabel> diff(
             String linesBefore,
             String linesAfter,
+            Source sourceBefore,
+            Source sourceAfter,
             DiffAlgorithm.SupportedAlgorithm algorithm,
             VariationDiffParseOptions options
     ) throws IOException, DiffParseException {
-        return VariationDiff.fromDiff(textDiff(linesBefore, linesAfter, algorithm), options);
+        return VariationDiff.fromDiff(
+            textDiff(linesBefore, linesAfter, algorithm),
+            new CompositeSource("JGitDiff.textDiff", sourceBefore, sourceAfter),
+            options
+        );
     }
 }

src/main/java/org/variantsync/diffdetective/variation/diff/parse/VariationDiffParser.java

@@ -17,6 +17,7 @@
 import org.variantsync.diffdetective.feature.Annotation;
 import org.variantsync.diffdetective.feature.AnnotationType;
 import org.variantsync.diffdetective.util.Assert;
+import org.variantsync.diffdetective.util.Source;
 import org.variantsync.diffdetective.variation.DiffLinesLabel;
 import org.variantsync.diffdetective.variation.NodeType;
 import org.variantsync.diffdetective.variation.diff.DiffNode;
@@ -117,17 +118,18 @@ public record DiffLine(DiffType diffType, String content) {
 
 
     /**
-     * The same as {@link VariationDiffParser#createVariationDiff(BufferedReader, VariationDiffParseOptions)}
+     * The same as {@link VariationDiffParser#createVariationDiff(BufferedReader, Source, VariationDiffParseOptions)}
      * but with the diff given as a single string with line breaks instead of a {@link BufferedReader}.
      *
      * @throws DiffParseException if {@code fullDiff} couldn't be parsed
      */
     public static VariationDiff<DiffLinesLabel> createVariationDiff(
             final String fullDiff,
+            final Source source,
             final VariationDiffParseOptions parseOptions
     ) throws DiffParseException {
         try {
-            return createVariationDiff(new BufferedReader(new StringReader(fullDiff)), parseOptions);
+            return createVariationDiff(new BufferedReader(new StringReader(fullDiff)), source, parseOptions);
         } catch (IOException e) {
             throw new AssertionError("No actual IO should be performed because only a StringReader is used");
         }
@@ -140,18 +142,20 @@ public static VariationDiff<DiffLinesLabel> createVariationDiff(
      * This parsing algorithm is described in detail in Sören Viegener's bachelor's thesis.
      *
      * @param fullDiff The full diff of a patch obtained from a buffered reader.
+     * @param source  the {@link Source} of {@code fullDiff}
      * @param options  {@link VariationDiffParseOptions} for the parsing process.
      * @return A parsed {@link VariationDiff} upon success or an error indicating why parsing failed.
      * @throws IOException        when reading from {@code fullDiff} fails.
      * @throws DiffParseException if an error in the diff or macro syntax is detected
      */
     public static VariationDiff<DiffLinesLabel> createVariationDiff(
             BufferedReader fullDiff,
+            Source source,
             final VariationDiffParseOptions options
     ) throws IOException, DiffParseException {
         return new VariationDiffParser(
                 options
-        ).parse(() -> {
+        ).parse(source, () -> {
             String line = fullDiff.readLine();
             if (line == null) {
                 return null;
@@ -162,22 +166,24 @@ public static VariationDiff<DiffLinesLabel> createVariationDiff(
 
     /**
      * Parses a variation tree from a source file.
-     * This method is similar to {@link #createVariationDiff(BufferedReader, VariationDiffParseOptions)}
+     * This method is similar to {@link #createVariationDiff(BufferedReader, Source, VariationDiffParseOptions)}
      * but acts as if all lines where unmodified.
      *
      * @param file    The source code file (not a diff) to be parsed.
+     * @param source the {@link Source} of {@code file}
      * @param options {@link VariationDiffParseOptions} for the parsing process.
      * @return A parsed {@link VariationDiff}.
      * @throws IOException        iff {@code file} throws an {@code IOException}
      * @throws DiffParseException if an error in the diff or macro syntax is detected
      */
     public static VariationDiff<DiffLinesLabel> createVariationTree(
             BufferedReader file,
+            Source source,
             VariationDiffParseOptions options
     ) throws IOException, DiffParseException {
         return new VariationDiffParser(
                 options
-        ).parse(() -> {
+        ).parse(source, () -> {
             String line = file.readLine();
             if (line == null) {
                 return null;
@@ -198,7 +204,7 @@ public static VariationDiff<DiffLinesLabel> createVariationTree(
     /**
      * Initializes the parse state.
      *
-     * @see #createVariationDiff(BufferedReader, VariationDiffParseOptions)
+     * @see #createVariationDiff(BufferedReader, Source, VariationDiffParseOptions)
      */
     private VariationDiffParser(
             VariationDiffParseOptions options
@@ -209,6 +215,7 @@ private VariationDiffParser(
     /**
      * Parses the line diff {@code fullDiff}.
      *
+     * @param source the {@link Source} of {@code lines}
      * @param lines should supply successive lines of the diff to be parsed, or {@code null} if
      *              there are no more lines to be parsed.
      * @return the parsed {@code VariationDiff}
@@ -217,6 +224,7 @@ private VariationDiffParser(
      *                            is detected
      */
     private VariationDiff<DiffLinesLabel> parse(
+            Source source,
             FailableSupplier<DiffLine, IOException> lines
     ) throws IOException, DiffParseException {
         DiffNode<DiffLinesLabel> root = DiffNode.createRoot(new DiffLinesLabel());
@@ -297,7 +305,7 @@ private VariationDiff<DiffLinesLabel> parse(
             );
         }
 
-        return new VariationDiff<>(root);
+        return new VariationDiff<>(root, source);
     }
 
     /**

src/main/java/org/variantsync/diffdetective/variation/diff/view/DiffView.java

@@ -6,6 +6,7 @@
 import org.variantsync.diffdetective.experiments.views.Main;
 import org.variantsync.diffdetective.util.Assert;
 import org.variantsync.diffdetective.util.CollectionUtils;
+import org.variantsync.diffdetective.util.Source;
 import org.variantsync.diffdetective.variation.DiffLinesLabel;
 import org.variantsync.diffdetective.variation.Label;
 import org.variantsync.diffdetective.variation.diff.*;
@@ -72,12 +73,19 @@ public static <L extends Label> BiPredicate<Time, Projection<L>> computeWhenNode
     private static <L extends Label> VariationDiff<DiffLinesLabel> naive(final VariationDiff<L> d, final Relevance rho, final String[] projectionViewText) throws IOException, DiffParseException {
         final VariationDiff<DiffLinesLabel> view;
         try {
-            view = JGitDiff.diff(projectionViewText[0], projectionViewText[1], DiffAlgorithm.SupportedAlgorithm.MYERS, Main.VARIATION_DIFF_PARSE_OPTIONS);
+            view = JGitDiff.diff(
+                projectionViewText[0],
+                projectionViewText[1],
+                Source.Unknown, // overridden below
+                Source.Unknown, // overridden below
+                DiffAlgorithm.SupportedAlgorithm.MYERS,
+                Main.VARIATION_DIFF_PARSE_OPTIONS
+            );
         } catch (DiffParseException e) {
             Logger.error("Could not parse diff obtained with query {} at {}", d.getSource(), rho);
             throw e;
         }
-        view.setSource(new ViewSource<>(d, rho));
+        view.setSource(new ViewSource<>(d, rho, "naive"));
 
         return view;
     }
@@ -153,6 +161,7 @@ public static <L extends Label> VariationDiff<L> badgood(final VariationDiff<L>
 
         // unify
         final VariationDiff<L> goodDiff = badDiff.toGood();
+        goodDiff.setSource(new ViewSource<>(d, rho, "badgood"));
         goodDiff.assertConsistency();
         return goodDiff;
     }
@@ -261,7 +270,7 @@ record Edge<L extends Label>(DiffNode<L> childCopy, DiffNode<L> parentInD, Time
 
         // Step 4: Build return value
         Assert.assertNotNull(rootCopy[0]);
-        return new VariationDiff<>(rootCopy[0], new ViewSource<>(d, rho));
+        return new VariationDiff<>(rootCopy[0], new ViewSource<>(d, rho, "optimized"));
     }
 
     /**

src/main/java/org/variantsync/diffdetective/variation/diff/view/ViewSource.java

@@ -13,10 +13,10 @@
  * @param diff The original variation diff on which the variation diff with this source is a view on.
  * @param relevance The relevance predicate that was used to create the view.
  */
-public record ViewSource<L extends Label>(VariationDiff<L> diff, Relevance relevance) implements Source {
+public record ViewSource<L extends Label>(VariationDiff<L> diff, Relevance relevance, String method) implements Source {
     @Override
     public String getSourceExplanation() {
-        return "View";
+        return "view";
     }
 
     @Override