Skip to content

Commit 5119f4c

Browse files
pmbittnerpmbittner
authored
Merge pull request #34 from VariantSync/benjamin/refactoring
Various refactorings and performance improvements
2 parents ca0e416 + 8ba61f0 commit 5119f4c

54 files changed

Lines changed: 608 additions & 585 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ buildNumber.properties
116116
*.ser
117117
patches/*
118118
.venv/
119+
/linegraph/__pycache__/
119120

120121
/input
121122
/error

‎pom.xml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
<version>1.0.0</version>
1010

1111
<properties>
12+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
13+
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
1214
<maven.compiler.source>16</maven.compiler.source>
1315
<maven.compiler.target>16</maven.compiler.target>
1416
</properties>

‎src/main/java/org/variantsync/diffdetective/analysis/AnalysisResult.java‎

Lines changed: 58 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,15 @@
66
import org.variantsync.diffdetective.metadata.ExplainedFilterSummary;
77
import org.variantsync.diffdetective.metadata.Metadata;
88
import org.variantsync.diffdetective.pattern.elementary.proposed.ProposedElementaryPatterns;
9-
import org.variantsync.diffdetective.util.IO;
109
import org.variantsync.functjonal.Functjonal;
1110
import org.variantsync.functjonal.category.InplaceMonoid;
1211
import org.variantsync.functjonal.category.InplaceSemigroup;
1312
import org.variantsync.functjonal.category.Semigroup;
1413
import org.variantsync.functjonal.map.MergeMap;
1514

15+
import java.io.BufferedReader;
1616
import java.io.IOException;
17+
import java.nio.file.Files;
1718
import java.nio.file.Path;
1819
import java.util.*;
1920
import java.util.function.BiConsumer;
@@ -137,76 +138,72 @@ public static AnalysisResult importFrom(final Path p, final Map<String, BiConsum
137138

138139
final List<String> filterHitsLines = new ArrayList<>();
139140
final List<String> elementaryPatternCountsLines = new ArrayList<>();
140-
141-
String fileInput = IO.readAsString(p); // read in metadata file
142-
fileInput = fileInput.replace("\r", ""); // remove carriage returns if present
143-
final String[] lines = fileInput.split("\n");
144-
String[] keyValuePair;
145-
String key;
146-
String value;
147-
148-
// examine each line of the metadata file separately
149-
for (/*final*/ String line : lines) {
150-
keyValuePair = line.split(": ");
151-
key = keyValuePair[0];
152-
value = keyValuePair[1];
153-
154-
switch (key) {
155-
case MetadataKeys.REPONAME -> result.repoName = value;
156-
case MetadataKeys.TREES -> result.exportedTrees = Integer.parseInt(value);
157-
case MetadataKeys.PROCESSED_COMMITS -> result.exportedCommits = Integer.parseInt(value);
158-
case MetadataKeys.TOTAL_COMMITS -> result.totalCommits = Integer.parseInt(value);
159-
case MetadataKeys.EMPTY_COMMITS -> result.emptyCommits = Integer.parseInt(value);
160-
case MetadataKeys.FAILED_COMMITS -> result.failedCommits = Integer.parseInt(value);
161-
case MetadataKeys.FILTERED_COMMITS -> { /* Do nothing because this value is derived. */ }
162-
case MetadataKeys.NON_NODE_COUNT -> result.debugData.numExportedNonNodes = Integer.parseInt(value);
163-
case MetadataKeys.ADD_NODE_COUNT -> result.debugData.numExportedAddNodes = Integer.parseInt(value);
164-
case MetadataKeys.REM_NODE_COUNT -> result.debugData.numExportedRemNodes = Integer.parseInt(value);
165-
case MetadataKeys.MINCOMMIT -> result.min.set(CommitProcessTime.fromString(value));
166-
case MetadataKeys.MAXCOMMIT -> result.max.set(CommitProcessTime.fromString(value));
167-
case MetadataKeys.RUNTIME -> {
168-
if (value.endsWith("s")) {
169-
value = value.substring(0, value.length() - 1);
141+
142+
try (BufferedReader input = Files.newBufferedReader(p)) {
143+
// examine each line of the metadata file separately
144+
String line;
145+
while ((line = input.readLine()) != null) {
146+
String[] keyValuePair = line.split(": ");
147+
String key = keyValuePair[0];
148+
String value = keyValuePair[1];
149+
150+
switch (key) {
151+
case MetadataKeys.REPONAME -> result.repoName = value;
152+
case MetadataKeys.TREES -> result.exportedTrees = Integer.parseInt(value);
153+
case MetadataKeys.PROCESSED_COMMITS -> result.exportedCommits = Integer.parseInt(value);
154+
case MetadataKeys.TOTAL_COMMITS -> result.totalCommits = Integer.parseInt(value);
155+
case MetadataKeys.EMPTY_COMMITS -> result.emptyCommits = Integer.parseInt(value);
156+
case MetadataKeys.FAILED_COMMITS -> result.failedCommits = Integer.parseInt(value);
157+
case MetadataKeys.FILTERED_COMMITS -> { /* Do nothing because this value is derived. */ }
158+
case MetadataKeys.NON_NODE_COUNT -> result.debugData.numExportedNonNodes = Integer.parseInt(value);
159+
case MetadataKeys.ADD_NODE_COUNT -> result.debugData.numExportedAddNodes = Integer.parseInt(value);
160+
case MetadataKeys.REM_NODE_COUNT -> result.debugData.numExportedRemNodes = Integer.parseInt(value);
161+
case MetadataKeys.MINCOMMIT -> result.min.set(CommitProcessTime.fromString(value));
162+
case MetadataKeys.MAXCOMMIT -> result.max.set(CommitProcessTime.fromString(value));
163+
case MetadataKeys.RUNTIME -> {
164+
if (value.endsWith("s")) {
165+
value = value.substring(0, value.length() - 1);
166+
}
167+
result.runtimeInSeconds = Double.parseDouble(value);
170168
}
171-
result.runtimeInSeconds = Double.parseDouble(value);
172-
}
173-
case MetadataKeys.RUNTIME_WITH_MULTITHREADING -> {
174-
if (value.endsWith("s")) {
175-
value = value.substring(0, value.length() - 1);
169+
case MetadataKeys.RUNTIME_WITH_MULTITHREADING -> {
170+
if (value.endsWith("s")) {
171+
value = value.substring(0, value.length() - 1);
172+
}
173+
result.runtimeWithMultithreadingInSeconds = Double.parseDouble(value);
176174
}
177-
result.runtimeWithMultithreadingInSeconds = Double.parseDouble(value);
178-
}
179-
default -> {
175+
default -> {
180176

181-
// temporary fix for renaming from Unchanged to Untouched
182-
final String unchanged = "Unchanged";
183-
if (key.startsWith(unchanged)) {
184-
key = ProposedElementaryPatterns.Untouched.getName();
185-
line = key + line.substring(unchanged.length());
186-
}
177+
// temporary fix for renaming from Unchanged to Untouched
178+
final String unchanged = "Unchanged";
179+
if (key.startsWith(unchanged)) {
180+
key = ProposedElementaryPatterns.Untouched.getName();
181+
line = key + line.substring(unchanged.length());
182+
}
187183

188-
final String finalKey = key;
189-
if (ProposedElementaryPatterns.All.stream().anyMatch(pattern -> pattern.getName().equals(finalKey))) {
190-
elementaryPatternCountsLines.add(line);
191-
} else if (key.startsWith(ExplainedFilterSummary.FILTERED_MESSAGE_BEGIN)) {
192-
filterHitsLines.add(line);
193-
} else if (key.startsWith(ERROR_BEGIN)) {
194-
DiffError e = new DiffError(key.substring(ERROR_BEGIN.length(), key.length() - ERROR_END.length()));
195-
// add DiffError
196-
result.diffErrors.put(e, Integer.parseInt(value));
197-
} else {
198-
final BiConsumer<AnalysisResult, String> customParser = customParsers.get(key);
199-
if (customParser == null) {
200-
final String errorMessage = "Unknown entry \"" + line + "\"!";
201-
throw new IOException(errorMessage);
184+
final String finalKey = key;
185+
if (ProposedElementaryPatterns.All.stream().anyMatch(pattern -> pattern.getName().equals(finalKey))) {
186+
elementaryPatternCountsLines.add(line);
187+
} else if (key.startsWith(ExplainedFilterSummary.FILTERED_MESSAGE_BEGIN)) {
188+
filterHitsLines.add(line);
189+
} else if (key.startsWith(ERROR_BEGIN)) {
190+
DiffError e = new DiffError(key.substring(ERROR_BEGIN.length(), key.length() - ERROR_END.length()));
191+
// add DiffError
192+
result.diffErrors.put(e, Integer.parseInt(value));
202193
} else {
203-
customParser.accept(result, value);
194+
final BiConsumer<AnalysisResult, String> customParser = customParsers.get(key);
195+
if (customParser == null) {
196+
final String errorMessage = "Unknown entry \"" + line + "\"!";
197+
throw new IOException(errorMessage);
198+
} else {
199+
customParser.accept(result, value);
200+
}
204201
}
205202
}
206203
}
207204
}
208205
}
209-
206+
210207
result.filterHits = ExplainedFilterSummary.parse(filterHitsLines);
211208
result.elementaryPatternCounts = ElementaryPatternCount.parse(elementaryPatternCountsLines, p.toString());
212209

‎src/main/java/org/variantsync/diffdetective/analysis/HistoryAnalysis.java‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -55,20 +55,20 @@ public static void mine(
5555
strategy,
5656
commitsToProcess
5757
));
58-
Logger.info("Scheduled " + commitsToProcess.size() + " commits.");
58+
Logger.info("Scheduled {} commits.", commitsToProcess.size());
5959
commitsToProcess = null; // free reference to enable garbage collection
60-
Logger.info("<<< done after " + clock.printPassedSeconds());
60+
Logger.info("<<< done after {}", clock.printPassedSeconds());
6161

6262
Logger.info(">>> Run mining");
6363
clock.start();
6464
try {
6565
totalResult = task.call();
6666
} catch (Exception e) {
6767
Logger.error(e);
68-
Logger.info("<<< aborted after " + clock.printPassedSeconds());
68+
Logger.info("<<< aborted after {}", clock.printPassedSeconds());
6969
return;
7070
}
71-
Logger.info("<<< done after " + clock.printPassedSeconds());
71+
Logger.info("<<< done after {}", clock.printPassedSeconds());
7272

7373
exportMetadata(outputDir, totalResult);
7474
}
@@ -85,7 +85,7 @@ public static void mineAsync(
8585

8686
// prepare tasks
8787
final int nThreads = Diagnostics.INSTANCE.run().getNumberOfAvailableProcessors();
88-
Logger.info(">>> Scheduling asynchronous mining on " + nThreads + " threads.");
88+
Logger.info(">>> Scheduling asynchronous mining on {} threads.", nThreads);
8989
clock.start();
9090
final InvocationCounter<RevCommit, RevCommit> numberOfTotalCommits = InvocationCounter.justCount();
9191
final Iterator<CommitHistoryAnalysisTask> tasks = new MappedIterator<>(
@@ -99,7 +99,7 @@ public static void mineAsync(
9999
outputDir.resolve(commitList.get(0).getId().getName() + ".lg"),
100100
commitList)
101101
);
102-
Logger.info("<<< done in " + clock.printPassedSeconds());
102+
Logger.info("<<< done in {}", clock.printPassedSeconds());
103103

104104
final TaskCompletionMonitor commitSpeedMonitor = new TaskCompletionMonitor(0, TaskCompletionMonitor.LogProgress("commits"));
105105
Logger.info(">>> Run mining");
@@ -112,13 +112,12 @@ public static void mineAsync(
112112
commitSpeedMonitor.addFinishedTasks(threadsResult.exportedCommits);
113113
}
114114
} catch (Exception e) {
115-
Logger.error("Failed to run all mining task!");
116-
Logger.error(e);
115+
Logger.error(e, "Failed to run all mining task");
117116
System.exit(0);
118117
}
119118

120119
final double runtime = clock.getPassedSeconds();
121-
Logger.info("<<< done in " + Clock.printPassedSeconds(runtime));
120+
Logger.info("<<< done in {}", Clock.printPassedSeconds(runtime));
122121

123122
totalResult.runtimeWithMultithreadingInSeconds = runtime;
124123
totalResult.totalCommits = numberOfTotalCommits.invocationCount().get();
@@ -132,12 +131,12 @@ public static <T> void exportMetadata(final Path outputDir, final Metadata<T> to
132131

133132
public static <T> void exportMetadataToFile(final Path outputFile, final Metadata<T> totalResult) {
134133
final String prettyMetadata = totalResult.exportTo(outputFile);
135-
Logger.info("Metadata:\n" + prettyMetadata);
134+
Logger.info("Metadata:\n{}", prettyMetadata);
136135
}
137136

138137
public void runAsync() {
139138
for (final Repository repo : repositoriesToAnalyze) {
140-
Logger.info(" === Begin Processing " + repo.getRepositoryName() + " ===");
139+
Logger.info(" === Begin Processing {} ===", repo.getRepositoryName());
141140
final Clock clock = new Clock();
142141
clock.start();
143142

@@ -149,10 +148,10 @@ public void runAsync() {
149148

150149
postProcessingOnRepositoryOutputDir.accept(repoOutputDir);
151150
} else {
152-
Logger.info(" Skipping repository " + repo.getRepositoryName() + " because it has already been processed.");
151+
Logger.info(" Skipping repository {} because it has already been processed.", repo.getRepositoryName());
153152
}
154153

155-
Logger.info(" === End Processing " + repo.getRepositoryName() + " after " + clock.printPassedSeconds() + " ===");
154+
Logger.info(" === End Processing {} after {} ===", repo.getRepositoryName(), clock.printPassedSeconds());
156155
}
157156
}
158157
}

‎src/main/java/org/variantsync/diffdetective/analysis/monitoring/TaskCompletionMonitor.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,11 @@ public void reportProgress() {
4545
}
4646

4747
public static Consumer<TimeInfo> LogProgress(final String tasksName) {
48-
return time -> Logger.info("Processed " + time.completedTasks + " " + tasksName
49-
+ " after " + String.format("%.2f", time.passedSeconds) + "s"
50-
+ " at about " + String.format("%.2f", time.tasksPerSecond) + tasksName + "/s.");
48+
return time -> Logger.info("Processed {} {} after {}s at about {}{}/s.",
49+
time.completedTasks,
50+
tasksName,
51+
String.format("%.2f", time.passedSeconds),
52+
String.format("%.2f", time.tasksPerSecond),
53+
tasksName);
5154
}
5255
}

‎src/main/java/org/variantsync/diffdetective/analysis/strategies/AnalyzeAllThenExport.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ public void onCommit(CommitDiff commit, String lineGraph) {
2424
@Override
2525
public void end() {
2626
final String lineGraph = waitForAll.toString();
27-
// Logger.info("Writing file " + outputPath);
27+
// Logger.info("Writing file {}", outputPath);
2828
IO.tryWrite(outputPath, lineGraph);
2929
}
3030
}

‎src/main/java/org/variantsync/diffdetective/analysis/strategies/AnalyzeAndExportIncrementally.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ public void end() {
5454

5555
public static void exportAppend(final Path outputPath, final String linegraph) {
5656
try {
57-
// Logger.info("Writing file " + outputPath);
57+
// Logger.info("Writing file {}", outputPath);
5858
IO.append(outputPath, linegraph);
5959
} catch (IOException exception) {
6060
Logger.error(exception);

‎src/main/java/org/variantsync/diffdetective/datasets/DatasetDescription.java‎

Lines changed: 16 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,14 @@
11
package org.variantsync.diffdetective.datasets;
22

3-
import org.variantsync.diffdetective.util.FileUtils;
43
import org.variantsync.diffdetective.util.LaTeX;
54
import org.variantsync.diffdetective.util.StringUtils;
65

76
import java.io.IOException;
7+
import java.nio.file.Files;
88
import java.nio.file.Path;
9-
import java.util.ArrayList;
109
import java.util.List;
10+
import java.util.stream.Collectors;
11+
import java.util.stream.Stream;
1112

1213
public record DatasetDescription(
1314
String name,
@@ -16,28 +17,20 @@ public record DatasetDescription(
1617
String commits
1718
) {
1819
public static List<DatasetDescription> fromMarkdown(final Path markdownFile) throws IOException {
19-
final String markdown = FileUtils.readUTF8(markdownFile);
20-
final String[] lines = markdown.split(StringUtils.LINEBREAK_REGEX);
21-
22-
final List<DatasetDescription> datasets = new ArrayList<>(lines.length - 2);
23-
// Start at 2 to skip header and separator line of table
24-
for (int i = 2; i < lines.length; ++i) {
25-
final String[] cells = lines[i].split("\\|");
26-
27-
final String hasCode = cells[2];
28-
final String isGitRepo = cells[3];
29-
30-
if (isYes(hasCode) && isYes(isGitRepo)) {
31-
datasets.add(new DatasetDescription(
32-
cells[0].trim(), // name
33-
cells[5].trim(), // clone URL,
34-
cells[1].trim(), // domain
35-
cells[6].trim() // #commits
36-
));
37-
}
20+
try (Stream<String> lines = Files.lines(markdownFile)) {
21+
return lines
22+
.skip(2) // Skip header
23+
.map(line -> line.split("\\|"))
24+
.filter(cells ->
25+
isYes(cells[2]) && // hasCode
26+
isYes(cells[3]) // isGitRepo
27+
).map(cells -> new DatasetDescription(
28+
cells[0].trim(), // name
29+
cells[5].trim(), // clone URL
30+
cells[1].trim(), // domain
31+
cells[6].trim()) // #commits
32+
).collect(Collectors.toList());
3833
}
39-
40-
return datasets;
4134
}
4235

4336
public static String asLaTeXTable(final List<DatasetDescription> datasets) {

‎src/main/java/org/variantsync/diffdetective/datasets/DatasetFactory.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ public List<Repository> createAll(final Collection<DatasetDescription> datasets,
8181
try {
8282
Assert.assertTrue(repo.getGitRepo().run().pull().call().isSuccessful());
8383
} catch (GitAPIException e) {
84-
Logger.error("Failed to pull repository \"" + repo.getRepositoryName() + "\"!", e);
84+
Logger.error(e, "Failed to pull repository '{}'", repo.getRepositoryName());
8585
}
8686
}
8787
}

‎src/main/java/org/variantsync/diffdetective/datasets/DefaultDatasets.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ public static List<DatasetDescription> loadDatasets(final Path datasetsFile) {
1818
try {
1919
return DatasetDescription.fromMarkdown(datasetsFile);
2020
} catch (IOException e) {
21-
Logger.error("Failed to load at least one dataset from " + datasetsFile + " because:", e);
21+
Logger.error(e, "Failed to load at least one dataset from {} because", datasetsFile);
2222
Logger.error("Aborting execution!");
2323
return new ArrayList<>();
2424
}

0 commit comments

Comments
 (0)