Skip to content

Commit c226aaf

Browse files
pmbittnerpmbittner
committed
benchmark benchmark
1 parent c90e400 commit c226aaf

1 file changed

Lines changed: 205 additions & 0 deletions

File tree

‎src/test/java/vevos/VEVOSBenchmark.java‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
package vevos;
2+
3+
import de.ovgu.featureide.fm.core.base.IFeatureModel;
4+
import org.junit.Test;
5+
import vevos.feature.Variant;
6+
import vevos.feature.sampling.FeatureIDESampler;
7+
import vevos.feature.sampling.Sample;
8+
import vevos.feature.sampling.Sampler;
9+
import vevos.functjonal.Cast;
10+
import vevos.functjonal.Lazy;
11+
import vevos.functjonal.Result;
12+
import vevos.io.Resources;
13+
import vevos.io.TextIO;
14+
import vevos.repository.BusyboxRepository;
15+
import vevos.repository.SPLRepository;
16+
import vevos.util.Clock;
17+
import vevos.util.Logger;
18+
import vevos.util.io.CaseSensitivePath;
19+
import vevos.variability.EvolutionStep;
20+
import vevos.variability.SPLCommit;
21+
import vevos.variability.VariabilityDataset;
22+
import vevos.variability.pc.Artefact;
23+
import vevos.variability.pc.SourceCodeFile;
24+
import vevos.variability.pc.groundtruth.GroundTruth;
25+
import vevos.variability.pc.options.ArtefactFilter;
26+
import vevos.variability.pc.options.VariantGenerationOptions;
27+
28+
import java.util.List;
29+
import java.util.Optional;
30+
import java.util.Set;
31+
import java.util.function.Consumer;
32+
import java.util.function.Function;
33+
34+
public class VEVOSBenchmark {
35+
private record Repo(
36+
CaseSensitivePath splRepositoryPath,
37+
CaseSensitivePath groundTruthDatasetPath,
38+
CaseSensitivePath variantsGenerationDir,
39+
Function<CaseSensitivePath, SPLRepository> createRepo,
40+
Consumer<SPLRepository> prepare,
41+
Consumer<SPLRepository> cleanup) {}
42+
43+
private static final CaseSensitivePath WIN_SUBMISSION_DIR = CaseSensitivePath.of("/mnt/c/Users/Paul Bittner/Documents/MyDocuments/Projects/VEVOS/Submission/Extraction/extraction-results/");
44+
private static final CaseSensitivePath WSL_VARIANT_EVOLUTION_DATASETS_DIR = CaseSensitivePath.of("../variantevolution_datasets/");
45+
46+
private static final Repo LINUX = new Repo(
47+
WSL_VARIANT_EVOLUTION_DATASETS_DIR.resolve("linux"),
48+
WIN_SUBMISSION_DIR.resolve("linux"),
49+
CaseSensitivePath.of("../dockerbrezel/linux"),
50+
path -> new SPLRepository(path.path()),
51+
s -> {},
52+
s -> {}
53+
);
54+
private static final Repo BUSYBOX = new Repo(
55+
WSL_VARIANT_EVOLUTION_DATASETS_DIR.resolve("busybox/busybox"),
56+
WSL_VARIANT_EVOLUTION_DATASETS_DIR.resolve("busybox/variability"),
57+
// WIN_SUBMISSION_DIR.resolve("busybox"),
58+
CaseSensitivePath.of("../dockerbrezel/busybox"),
59+
path -> new BusyboxRepository(path.path()),
60+
s -> { },
61+
s -> {
62+
BusyboxRepository b = Cast.unchecked(s);
63+
try {
64+
b.postprocess();
65+
} catch (Exception e) {
66+
Logger.error("Error in Busybox cleanup", e);
67+
}
68+
}
69+
);
70+
private final static int NUMBER_OF_VARIANTS_TO_GENERATE = 5;
71+
private final static int MAX_COMMITS_TO_ANALYZE = 20;
72+
73+
private static String logTime(final String task, final double seconds) {
74+
final String msg = task + " took " + seconds + "s.";
75+
Logger.info(msg);
76+
return msg;
77+
}
78+
79+
private static void resultEntry(final StringBuilder builder, String entry) {
80+
builder.append(entry).append("\r\n");
81+
}
82+
83+
public static void benchmark(final Repo repo) throws Exception {
84+
VEVOS.Initialize();
85+
// Logger.setLogLevel(LogLevel.INFO);
86+
87+
final StringBuilder timeData = new StringBuilder();
88+
89+
final SPLRepository splRepo = repo.createRepo().apply(repo.splRepositoryPath);
90+
91+
final Clock clock = new Clock();
92+
clock.start();
93+
final VariabilityDataset dataset =
94+
Resources.Instance().load(VariabilityDataset.class, repo.groundTruthDatasetPath().path());
95+
final double timeDatasetLoading = clock.getPassedSeconds();
96+
resultEntry(timeData, logTime("Loading dataset", timeDatasetLoading));
97+
98+
clock.start();
99+
final Set<EvolutionStep<SPLCommit>> evolutionSteps = dataset.getEvolutionSteps();
100+
final double timeEvolutionStepCreation = clock.getPassedSeconds();
101+
resultEntry(timeData, logTime("Creating evolution steps", timeEvolutionStepCreation));
102+
103+
Logger.info("The dataset contains " + dataset.getSuccessCommits().size() + " commits for which the variability extraction succeeded.");
104+
Logger.info("The dataset contains " + dataset.getErrorCommits().size() + " commits for which the variability extraction failed.");
105+
Logger.info("The dataset contains " + dataset.getPartialSuccessCommits().size() + " commits that for which the file presence conditions are missing.");
106+
Logger.info("The dataset contains " + evolutionSteps.size() + " usable pairs of commits.");
107+
108+
final List<SPLCommit> subhistory = dataset.getSuccessCommits();
109+
110+
final Sampler variantsSampler =
111+
FeatureIDESampler.CreateRandomSampler(NUMBER_OF_VARIANTS_TO_GENERATE);
112+
// new FeatureIDESampler(NUMBER_OF_VARIANTS_TO_GENERATE, cnf -> new VEVOSRandomSampling(cnf, NUMBER_OF_VARIANTS_TO_GENERATE));
113+
double timeLoadPCsAverage = 0;
114+
double timeSampleAverage = 0;
115+
double timeGenVariantsAverage = 0;
116+
117+
int analyzedCommits = 0;
118+
for (final SPLCommit splCommit : subhistory) {
119+
Logger.info("-- Processing commit " + splCommit.id() + " --");
120+
splRepo.checkoutCommit(splCommit);
121+
repo.prepare.accept(splRepo);
122+
123+
clock.start();
124+
final Lazy<Optional<IFeatureModel>> loadFeatureModel = splCommit.featureModel();
125+
final Lazy<Optional<Artefact>> loadPresenceConditions = splCommit.presenceConditions();
126+
127+
if (loadPresenceConditions.run().isEmpty()) {
128+
Logger.info("has no PCs");
129+
continue;
130+
}
131+
if (loadFeatureModel.run().isEmpty()) {
132+
Logger.info("has no FM");
133+
continue;
134+
}
135+
136+
final Artefact pcs = loadPresenceConditions.run().orElseThrow();
137+
final IFeatureModel featureModel = loadFeatureModel.run().orElseThrow();
138+
final double timeLoadPCs = clock.getPassedSeconds();
139+
timeLoadPCsAverage += timeLoadPCs;
140+
logTime("Loading PCs and FM", timeLoadPCs);
141+
142+
// Logger.info("#features = " + featureModel.getFeatures().size());
143+
// Logger.info("Feature model is valid = " + FeatureModelUtils.isValid(featureModel));
144+
145+
clock.start();
146+
final Sample variants = variantsSampler.sample(featureModel);
147+
final double timeSample = clock.getPassedSeconds();
148+
timeSampleAverage += timeSample;
149+
logTime("Sampling " + variants.size() + " variants", timeSample);
150+
151+
final ArtefactFilter<SourceCodeFile> artefactFilter = ArtefactFilter.KeepAll();
152+
final VariantGenerationOptions generationOptions = VariantGenerationOptions.ExitOnError(artefactFilter);
153+
154+
clock.start();
155+
for (final Variant variant : variants) {
156+
final CaseSensitivePath variantDir = repo.variantsGenerationDir().resolve(splCommit.id(), "_", analyzedCommits + "" , "_", variant.getName());
157+
final Result<GroundTruth, Exception> result = pcs.generateVariant(variant, repo.splRepositoryPath(), variantDir, generationOptions);
158+
159+
// final FeatureIDEConfiguration config = (FeatureIDEConfiguration) variant.getConfiguration();
160+
// Logger.info("Generating variant with configuration:\r\n" + config.toAssignment());
161+
162+
if (result.isSuccess()) {
163+
final GroundTruth groundTruth = result.getSuccess();
164+
final Artefact presenceConditionsOfVariant = groundTruth.variant();
165+
Resources.Instance().write(Artefact.class, presenceConditionsOfVariant, variantDir.resolve("pcs.variant.csv").path());
166+
} else {
167+
throw result.getFailure();
168+
}
169+
}
170+
final double timeGenVariants = clock.getPassedSeconds();
171+
timeGenVariantsAverage += timeGenVariants;
172+
logTime("Generating " + variants.size() + " variants", timeGenVariants);
173+
174+
repo.cleanup.accept(splRepo);
175+
176+
++analyzedCommits;
177+
if (analyzedCommits > MAX_COMMITS_TO_ANALYZE) {
178+
break;
179+
}
180+
}
181+
182+
timeLoadPCsAverage = timeLoadPCsAverage / ((double)analyzedCommits);
183+
timeSampleAverage = timeSampleAverage / ((double)analyzedCommits);
184+
timeGenVariantsAverage = timeGenVariantsAverage / (analyzedCommits * NUMBER_OF_VARIANTS_TO_GENERATE);
185+
186+
resultEntry(timeData, logTime("Loading PCs Average", timeLoadPCsAverage));
187+
resultEntry(timeData, logTime("Sampling " + NUMBER_OF_VARIANTS_TO_GENERATE + " variants Average", timeSampleAverage));
188+
resultEntry(timeData, logTime("Generating " + NUMBER_OF_VARIANTS_TO_GENERATE + " variants Average", timeGenVariantsAverage));
189+
resultEntry(timeData, "Commits: " + analyzedCommits);
190+
191+
final String result = timeData.toString();
192+
Logger.info("Done\r\n" + result);
193+
TextIO.write(repo.variantsGenerationDir.resolve("benchmarkdata.txt").path(), result);
194+
}
195+
196+
@Test
197+
public void benchmarkLinux() throws Exception {
198+
benchmark(LINUX);
199+
}
200+
201+
@Test
202+
public void benchmarkBusybox() throws Exception {
203+
benchmark(BUSYBOX);
204+
}
205+
}

0 commit comments

Comments
 (0)