Merge branch 'mining' of github.com:VariantSync/DiffDetective into mining

Author: Christof Tinnes

docs/datasets/emacs.md

@@ -0,0 +1,3 @@
+Project name | Domain                  | Source code available (**y**es/**n**o)? | Is it a git repository (**y**es/**n**o)? | Repository URL                                               | Clone URL                                                        | Estimated number of commits
+---|-------------------------|-----------------------------------------|-----------------------------------|--------------------------------------------------------------|------------------------------------------------------------------|---
+emacs | text editor             | y                                       | y                                 | https://github.com/emacs-mirror/emacs                        | https://github.com/emacs-mirror/emacs.git                        | 153,926

runPostprocessing.sh

@@ -0,0 +1,2 @@
+java -cp "target/diffdetective-1.0.0-jar-with-dependencies.jar" org.variantsync.diffdetective.mining.postprocessing.MiningPostprocessing "results/fsg/difftrees_pilot/all_results" "results/compositeCandidates"
+echo "runPostprocessing.sh DONE"

runPreprocessing.sh

@@ -1,2 +1,2 @@
-java -cp "mining/lib/DiffDetective-1.0.0-jar-with-dependencies.jar" mining.DiffTreeMiner
+java -cp "target/diffdetective-1.0.0-jar-with-dependencies.jar" org.variantsync.diffdetective.mining.DiffTreeMiner
 echo "runPreprocessing.sh DONE"

src/main/java/org/variantsync/diffdetective/analysis/AnalysisResult.java

@@ -83,7 +83,15 @@ public AnalysisResult() {
     }
 
     public AnalysisResult(final String repoName) {
-        this(repoName, 0, 0, 0, 0, 0, 0, 0, CommitProcessTime.Unknown(repoName, Long.MAX_VALUE), CommitProcessTime.Unknown(repoName, Long.MIN_VALUE), new DiffTreeSerializeDebugData(), new ExplainedFilterSummary());
+        this(
+                repoName,
+                0, 0, 0, 0,
+                0,
+                0, 0,
+                CommitProcessTime.Unknown(repoName, Long.MAX_VALUE),
+                CommitProcessTime.Unknown(repoName, Long.MIN_VALUE),
+                new DiffTreeSerializeDebugData(),
+                new ExplainedFilterSummary());
     }
 
     public AnalysisResult(

src/main/java/org/variantsync/diffdetective/analysis/HistoryAnalysis.java

@@ -33,7 +33,7 @@ public record HistoryAnalysis(
     public static final int COMMITS_TO_PROCESS_PER_THREAD_DEFAULT = 1000;
 
     @Deprecated
-    public static void mine(
+    public static void analyze(
             final Repository repo,
             final Path outputDir,
             final DiffTreeLineGraphExportOptions exportOptions,
@@ -73,7 +73,7 @@ public static void mine(
         exportMetadata(outputDir, totalResult);
     }
 
-    public static void mineAsync(
+    public static void analyzeAsync(
             final Repository repo,
             final Path outputDir,
             final CommitHistoryAnalysisTaskFactory taskFactory,
@@ -85,7 +85,7 @@ public static void mineAsync(
 
         // prepare tasks
         final int nThreads = Diagnostics.INSTANCE.run().getNumberOfAvailableProcessors();
-        Logger.info(">>> Scheduling asynchronous mining on " + nThreads + " threads.");
+        Logger.info(">>> Scheduling asynchronous analysis on " + nThreads + " threads.");
         clock.start();
         final InvocationCounter<RevCommit, RevCommit> numberOfTotalCommits = InvocationCounter.justCount();
         final Iterator<CommitHistoryAnalysisTask> tasks = new MappedIterator<>(
@@ -102,7 +102,7 @@ public static void mineAsync(
         Logger.info("<<< done in " + clock.printPassedSeconds());
 
         final TaskCompletionMonitor commitSpeedMonitor = new TaskCompletionMonitor(0, TaskCompletionMonitor.LogProgress("commits"));
-        Logger.info(">>> Run mining");
+        Logger.info(">>> Run Analysis");
         clock.start();
         commitSpeedMonitor.start();
         try (final ScheduledTasksIterator<AnalysisResult> threads = new ScheduledTasksIterator<>(tasks, nThreads)) {
@@ -144,7 +144,7 @@ public void runAsync() {
             final Path repoOutputDir = outputDir.resolve(repo.getRepositoryName());
             /// Don't repeat work we already did:
             if (!Files.exists(repoOutputDir.resolve(TOTAL_RESULTS_FILE_NAME))) {
-                mineAsync(repo, repoOutputDir, whatToDo, commitsToProcessPerThread);
+                analyzeAsync(repo, repoOutputDir, whatToDo, commitsToProcessPerThread);
 //                mine(repo, repoOutputDir, ExportOptions(repo), MiningStrategy());
 
                 postProcessingOnRepositoryOutputDir.accept(repoOutputDir);

src/main/java/org/variantsync/diffdetective/datasets/DefaultDatasets.java

@@ -9,6 +9,7 @@
 
 public class DefaultDatasets {
     public final static Path DEFAULT_DATASETS_FILE = Path.of("docs", "datasets.md");
+    public final static Path EMACS = Path.of("docs", "datasets", "emacs.md");
 
     public static List<DatasetDescription> loadDefaultDatasets() {
         return loadDatasets(DEFAULT_DATASETS_FILE);

src/main/java/org/variantsync/diffdetective/diff/difftree/DiffNode.java

@@ -335,6 +335,10 @@ public boolean insertAfterChild(final DiffNode child, int index) {
 
     public boolean addBeforeChild(final DiffNode child) {
         if (!child.isAdd()) {
+            if (child.beforeParent != null) {
+                throw new IllegalArgumentException("Given child " + child + " already has a before parent (" + child.beforeParent + ")!");
+            }
+
             addWithoutDuplicates(beforeChildren, child);
             addWithoutDuplicates(childOrder, child);
             child.setBeforeParent(this);
@@ -345,6 +349,10 @@ public boolean addBeforeChild(final DiffNode child) {
 
     public boolean addAfterChild(final DiffNode child) {
         if (!child.isRem()) {
+            if (child.afterParent != null) {
+                throw new IllegalArgumentException("Given child " + child + " already has an after parent (" + child.afterParent + ")!");
+            }
+
             addWithoutDuplicates(afterChildren, child);
             addWithoutDuplicates(childOrder, child);
             child.setAfterParent(this);

src/main/java/org/variantsync/diffdetective/diff/difftree/parse/MultiLineMacroParser.java

@@ -100,6 +100,7 @@ ParseResult consume(
 
             return ParseResult.SUCCESS;
         } else {
+            // We either found the ending line of a multiline macro or just a plain line outside of a macro.
             final boolean inBeforeMLMacro = beforeMLMacro != null;
             final boolean inAfterMLMacro = afterMLMacro != null;
 
@@ -110,7 +111,7 @@ ParseResult consume(
                         && inAfterMLMacro
                         && diffType == DiffType.NON
                         && beforeMLMacro.equals(afterMLMacro)) {
-                    // We have one single end line for to equal multi line macros -> Merge the nodes.
+                    // We have one single end line for two equal multi line macros -> Merge the nodes.
                     final DiffNode mlNode = finalizeMLMacro(
                             lineNo,
                             line,

src/main/java/org/variantsync/diffdetective/diff/difftree/parse/MultilineMacro.java

@@ -72,11 +72,22 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         MultilineMacro that = (MultilineMacro) o;
-        return startLine == that.startLine && lines.equals(that.lines) && diffType == that.diffType;
+        return startLine.equals(that.startLine) && lines.equals(that.lines) && diffType == that.diffType;
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(lines, diffType, startLine);
     }
+
+    @Override
+    public String toString() {
+        return "MultilineMacro{" +
+                "lines=" + lines +
+                ", beforeParent=" + beforeParent +
+                ", afterParent=" + afterParent +
+                ", startLine=" + startLine +
+                ", diffType=" + diffType +
+                '}';
+    }
 }

src/main/java/org/variantsync/diffdetective/diff/difftree/render/DiffTreeRenderer.java

@@ -73,36 +73,50 @@ public boolean render(PatchDiff patchDiff, final Path directory, final RenderOpt
         return render(patchDiff.getDiffTree(), patchDiff, directory, options);
     }
 
+    public boolean render(PatchDiff patchDiff, final Path directory, final RenderOptions options, final DiffTreeLineGraphExportOptions exportOptions) {
+        return render(patchDiff.getDiffTree(), patchDiff, directory, options, exportOptions);
+    }
+
     public boolean render(final DiffTree tree, final GitPatch patch, final Path directory, final RenderOptions options) {
         final String treeAndFileName =
                 patch.getFileName()
                         + LineGraphConstants.TREE_NAME_SEPARATOR
                         + patch.getCommitHash();
-        return render(tree, treeAndFileName, directory, options);
+        return render(tree, treeAndFileName, directory, options, options.toLineGraphOptions());
+    }
+
+    public boolean render(final DiffTree tree, final GitPatch patch, final Path directory, final RenderOptions options, final DiffTreeLineGraphExportOptions exportOptions) {
+        final String treeAndFileName =
+                patch.getFileName()
+                        + LineGraphConstants.TREE_NAME_SEPARATOR
+                        + patch.getCommitHash();
+        return render(tree, treeAndFileName, directory, options, exportOptions);
     }
 
     public boolean render(final DiffTree tree, final String treeAndFileName, final Path directory) {
         return render(tree, treeAndFileName, directory, RenderOptions.DEFAULT);
     }
 
     public boolean render(final DiffTree tree, final String treeAndFileName, final Path directory, RenderOptions options) {
-        return render(tree, treeAndFileName, directory, options,
+        return render(tree, treeAndFileName, directory, options, options.toLineGraphOptions());
+    }
+
+    public boolean render(final DiffTree tree, final String treeAndFileName, final Path directory, RenderOptions options, final DiffTreeLineGraphExportOptions exportOptions) {
+        return render(tree, treeAndFileName, directory, options, exportOptions,
                 (treeName, treeSource) -> LineGraphConstants.LG_TREE_HEADER + " " + treeAndFileName + LineGraphConstants.TREE_NAME_SEPARATOR + "0"
                 );
     }
 
     public boolean renderWithTreeFormat(final DiffTree tree, final String treeAndFileName, final Path directory, RenderOptions options) {
-        return render(tree, treeAndFileName, directory, options,
+        return render(tree, treeAndFileName, directory, options, options.toLineGraphOptions(),
                 (treeName, treeSource) -> options.treeFormat().toLineGraphLine(treeSource)
         );
     }
 
-    private boolean render(final DiffTree tree, final String treeAndFileName, final Path directory, RenderOptions options, BiFunction<String, DiffTreeSource, String> treeHeader) {
-        final DiffTreeLineGraphExportOptions lgoptions = new DiffTreeLineGraphExportOptions(options.format(), options.treeFormat(), options.nodeFormat(), options.edgeFormat());
-
+    private boolean render(final DiffTree tree, final String treeAndFileName, final Path directory, RenderOptions options, DiffTreeLineGraphExportOptions exportOptions, BiFunction<String, DiffTreeSource, String> treeHeader) {
         final Path tempFile = directory.resolve(treeAndFileName + ".lg");
 
-        final Pair<DiffTreeSerializeDebugData, String> result = LineGraphExport.toLineGraphFormat(tree, lgoptions);
+        final Pair<DiffTreeSerializeDebugData, String> result = LineGraphExport.toLineGraphFormat(tree, exportOptions);
         Assert.assertNotNull(result);
         final String lg = treeHeader.apply(treeAndFileName, tree.getSource()) + StringUtils.LINEBREAK + result.second();