Merge pull request #5 from VariantSync/macros_in_variants

Macros in Variants

Author: AlexanderSchultheiss

repo/org/variantsync/functjonal/1.0-SNAPSHOT/functjonal-1.0-SNAPSHOT.jar

@@ -7,17 +7,17 @@
     <snapshot>
       <localCopy>true</localCopy>
     </snapshot>
-    <lastUpdated>20220405113508</lastUpdated>
+    <lastUpdated>20220629093021</lastUpdated>
     <snapshotVersions>
       <snapshotVersion>
         <extension>jar</extension>
         <value>1.0-SNAPSHOT</value>
-        <updated>20220405113508</updated>
+        <updated>20220629093021</updated>
       </snapshotVersion>
       <snapshotVersion>
         <extension>pom</extension>
         <value>1.0-SNAPSHOT</value>
-        <updated>20220405113508</updated>
+        <updated>20220629093021</updated>
       </snapshotVersion>
     </snapshotVersions>
   </versioning>

repo/org/variantsync/functjonal/1.0-SNAPSHOT/maven-metadata-local.xml

@@ -6,6 +6,6 @@
     <versions>
       <version>1.0-SNAPSHOT</version>
     </versions>
-    <lastUpdated>20220405113508</lastUpdated>
+    <lastUpdated>20220629093021</lastUpdated>
   </versioning>
 </metadata>

repo/org/variantsync/functjonal/maven-metadata-local.xml

@@ -132,7 +132,7 @@ public static void main(final String[] args) throws Resources.ResourceIOExceptio
                 /// Moreover, VariantGenerationOptions allow to configure some parameters for the variant generation.
                 //Here, we just instruct the generation to exit in case an error happens but we could for example also instruct it to ignore errors and proceed.
                 final ArtefactFilter<SourceCodeFile> artefactFilter = ArtefactFilter.KeepAll();
-                final VariantGenerationOptions generationOptions = VariantGenerationOptions.ExitOnError(artefactFilter);
+                final VariantGenerationOptions generationOptions = VariantGenerationOptions.ExitOnError(false, artefactFilter);
 
                 /// Checkout the considered commit of the input SPL to access its source code.
                 try {

src/main/java/org/variantsync/vevos/simulation/examples/GenerationExample.java

@@ -146,7 +146,7 @@ public static void benchmark(final Repo repo) throws Exception {
             logTime("Sampling " + variants.size() + " variants", timeSample);
 
             final ArtefactFilter<SourceCodeFile> artefactFilter = ArtefactFilter.KeepAll();
-            final VariantGenerationOptions generationOptions = VariantGenerationOptions.ExitOnError(artefactFilter);
+            final VariantGenerationOptions generationOptions = VariantGenerationOptions.ExitOnError(false, artefactFilter);
 
             clock.start();
             for (final Variant variant : variants) {

src/main/java/org/variantsync/vevos/simulation/examples/VEVOSBenchmark.java

@@ -19,6 +19,9 @@
 // TODO: Implement readLinesAs(Path p, Function<> f) with which one can load a file into a desired format
 
 public class TextIO {
+    public final static String LINEBREAK = "\r\n";
+    public final static String LINEBREAK_REGEX = "\\r?\\n";
+
     public static String[] readLinesAsArray(final File file) throws IOException {
         final LinkedList<String> lines = new LinkedList<>();
         try (final BufferedReader reader = new BufferedReader(new FileReader(file))) {
@@ -50,60 +53,19 @@ public static String readLastLine(final File file) throws IOException {
      * @return The lines that were read
      */
     public static Result<List<String>, IOException> readLinesTrimmed(final Path p) {
-        return Result.Try(() -> Files.readAllLines(p).stream().map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList()));
+        return readLines(p).map(lines ->
+                lines.stream()
+                        .map(String::trim)
+                        .filter(s -> !s.isEmpty())
+                        .collect(Collectors.toList()));
     }
 
-    /**
-     * Extract all lines in [lineFrom, lineTo] = i from sourceFile and put them into targetFile for each interval i in linesToTake.
-     *
-     * @param sourceFile  File from which to read lines. Won't be altered.
-     * @param targetFile  File to write the desired lines to.
-     * @param linesToTake Intervals of lines to copy.
-     * @throws IOException May occur upon writing or creating files.
-     */
-    public static void copyTextLines(final Path sourceFile, final Path targetFile, final List<Integer> linesToTake) throws IOException {
-        /// Do not use Files.readAllLines(sourceFile) as it assumes the files to be in UTF-8 and crashes otherwise.
-        //  Do also not use try (final Stream<String> linesStream = new BufferedReader(new FileReader(sourceFile.toFile())).lines()) {
-        // Apparently, Java is stupid and the BufferedReader is not closed by the try-with-resources if it is anonymous.
-        try (BufferedReader br = new BufferedReader(new FileReader(sourceFile.toFile()));
-             Stream<String> linesStream = br.lines()) {
-            final List<String> read_lines = linesStream.collect(Collectors.toList());
-            final StringBuilder linesToWrite = new StringBuilder();
-
-            for (final Integer lineNo : linesToTake) {
-                int lineIndex = lineNo - 1;
-                // skip lines that exceed the content
-                if (lineIndex >= read_lines.size()) {
-                    String logMessage = "Skipped copying line "
-                            + lineNo
-                            + " from \""
-                            + sourceFile
-                            + "\" to \""
-                            + targetFile
-                            + "\" as it is out of bounds [1, "
-                            + read_lines.size()
-                            + "]!";
-
-                    if (lineIndex > read_lines.size()) {
-                        // This was logged frequently and is caused by https://bugs.openjdk.java.net/browse/JDK-8199413
-                        // Skipping the line really is the best solution, as the empty line is created by appending a line separator
-                        // to the previous line. I added the additional if-statement, to only catch cases in which more than one line 
-                        // is out of bounds, which indicates a serious problem. 
-                        Logger.error(logMessage);
-                    }
-                } else {
-                    // The list read_lines is 0-based.
-                    // Given lines are 1-based because line numbers are typically given 1-based.
-                    // Thus, we have to -1 here because line numbers are 1-indexed
-                    // but we want to look them up in read_lines, which is 0-based.
-                    linesToWrite.append(read_lines.get(lineIndex)).append(System.lineSeparator());
-                }
-            }
-
-            Files.write(
-                    targetFile,
-                    linesToWrite.toString().getBytes(),
-                    StandardOpenOption.APPEND);
+    public static Result<List<String>, IOException> readLines(final Path p) {
+        try (final BufferedReader br = new BufferedReader(new FileReader(p.toFile()));
+             final Stream<String> linesStream = br.lines()) {
+            return Result.Success(linesStream.toList());
+        } catch (final IOException e) {
+            return Result.Failure(e);
         }
     }
 
@@ -120,6 +82,18 @@ public static void write(final Path p, final String text) throws IOException {
         Files.writeString(p, text, StandardCharsets.UTF_8, StandardOpenOption.CREATE_NEW);
     }
 
+    /**
+     * Append the given text to the given file.
+     * Assumes that the given file already exists.
+     *
+     * @param p    Existing file to append text to.
+     * @param text Text to write to file.
+     * @throws IOException if an I/O error occurs while writing to the file, or the text cannot be encoded using the specified charset.
+     */
+    public static void append(final Path p, final String text) throws IOException {
+        Files.writeString(p, text, StandardCharsets.UTF_8, StandardOpenOption.APPEND);
+    }
+
     public static String readAsString(final Path p) throws IOException {
         try (final BufferedReader reader = new BufferedReader(new FileReader(p.toFile()))) {
             return reader.lines().collect(Collectors.joining());

src/main/java/org/variantsync/vevos/simulation/io/TextIO.java

@@ -8,6 +8,9 @@
 import org.variantsync.vevos.simulation.variability.pc.groundtruth.BlockMatching;
 import org.variantsync.vevos.simulation.variability.pc.groundtruth.GroundTruth;
 import org.variantsync.vevos.simulation.variability.pc.options.VariantGenerationOptions;
+import org.variantsync.vevos.simulation.variability.pc.variantlines.VariantAnnotation;
+import org.variantsync.vevos.simulation.variability.pc.variantlines.VariantLine;
+import org.variantsync.vevos.simulation.variability.pc.variantlines.VariantLineChunk;
 import org.variantsync.vevos.simulation.variability.pc.visitor.LineBasedAnnotationVisitorFocus;
 
 import java.util.ArrayList;
@@ -51,9 +54,9 @@ public LineBasedAnnotation(final LineBasedAnnotation other) {
         this.style = other.style;
     }
 
-    private static void addRange(final List<Integer> list, final int fromInclusive, final int toInclusive) {
+    private static void addRange(final List<VariantLineChunk> list, final int fromInclusive, final int toInclusive) {
         for (int i = fromInclusive; i <= toInclusive; ++i) {
-            list.add(i);
+            list.add(new VariantLine(i));
         }
     }
 
@@ -151,8 +154,9 @@ private Optional<LineBasedAnnotation> deriveForVariant(final Variant variant, in
      * @param isIncluded A predicate to select subtrees. A subtree will be considered if isIncluded returns true for it.
      * @return All line numbers that should be copied from the SPL file to the variant file. 1-based.
      */
-    public List<Integer> getAllLinesFor(final Predicate<LineBasedAnnotation> isIncluded) {
-        final List<Integer> chunksToWrite = new ArrayList<>();
+    public VariantAnnotation getLinesToCopy(final Predicate<LineBasedAnnotation> isIncluded) {
+        final List<VariantLineChunk> chunksToWrite = new ArrayList<>();
+//        final List<Integer> chunksToWrite = new ArrayList<>();
         final int firstCodeLine = getLineFrom() + style.offset; // ignore #if
         final int lastCodeLine = getLineTo() - style.offset; // ignore #endif
 
@@ -163,7 +167,7 @@ public List<Integer> getAllLinesFor(final Predicate<LineBasedAnnotation> isInclu
             }
 
             if (isIncluded.test(subtree)) {
-                chunksToWrite.addAll(subtree.getAllLinesFor(isIncluded));
+                chunksToWrite.add(subtree.getLinesToCopy(isIncluded));
             }
 
             currentLine = subtree.getLineTo() + 1;
@@ -173,7 +177,10 @@ public List<Integer> getAllLinesFor(final Predicate<LineBasedAnnotation> isInclu
             addRange(chunksToWrite, currentLine, lastCodeLine);
         }
 
-        return chunksToWrite;
+        return new VariantAnnotation(
+                this.getFeatureMapping(),
+                chunksToWrite
+        );
     }
 
     public void simplify() {

src/main/java/org/variantsync/vevos/simulation/variability/pc/LineBasedAnnotation.java

@@ -10,18 +10,15 @@
 import org.variantsync.vevos.simulation.util.fide.bugfix.FixTrueFalse;
 import org.variantsync.vevos.simulation.util.io.CaseSensitivePath;
 import org.variantsync.vevos.simulation.util.io.PathUtils;
-import org.variantsync.vevos.simulation.variability.pc.groundtruth.AnnotationGroundTruth;
 import org.variantsync.vevos.simulation.variability.pc.groundtruth.BlockMatching;
 import org.variantsync.vevos.simulation.variability.pc.groundtruth.GroundTruth;
 import org.variantsync.vevos.simulation.variability.pc.options.VariantGenerationOptions;
+import org.variantsync.vevos.simulation.variability.pc.variantlines.VariantAnnotation;
 import org.variantsync.vevos.simulation.variability.pc.visitor.SourceCodeFileVisitorFocus;
 
 import java.io.FileNotFoundException;
-import java.io.IOException;
 import java.nio.file.Files;
 import java.util.Collections;
-import java.util.List;
-import java.util.Optional;
 
 /**
  * Represents a variable source code file (e.g., because part of a plugin or only conditionally included).
@@ -57,36 +54,49 @@ public Result<GroundTruth, Exception> generateVariant(
             return Result.Failure(new FileNotFoundException("Source file " + sourceFile + " does not exist!"));
         }
 
-        final Result<Optional<AnnotationGroundTruth>, IOException> groundTruth =
+        return
                 // Create the target file.
                 PathUtils.createEmptyAsResult(targetFile.path())
                 // Write to target file.
-                .bind(unit -> Traversable.sequence(rootAnnotation.deriveForVariant(variant).map(splAnnotationGroundTruth -> {
+                .bind(unit -> Traversable.sequence(
+                        // Compute ground truth for our variant (i.e., make the variant feature-aware
+                        rootAnnotation
+                                .deriveForVariant(variant)
+                                .map(splAnnotationGroundTruth -> {
                     final BlockMatching lineMatching = splAnnotationGroundTruth.matching();
-                    // only write lines of blocks that are part of our variant
-                    final List<Integer> lines = rootAnnotation.getAllLinesFor(lineMatching::isPresentInVariant);
-                    return Result.Try(
-                            () -> {
-                                TextIO.copyTextLines(sourceFile.path(), targetFile.path(), lines);
-                                return splAnnotationGroundTruth;
-                            });
-                })));
-
-        return groundTruth.bimap(
-                // In case of success, return ground truth.
-                Functjonal.match(
-                        splAnnotationGroundTruth -> GroundTruth.forSourceCodeFile(
-                                new SourceCodeFile(getFeatureMapping(), getFile(), splAnnotationGroundTruth.variantArtefact()),
-                                splAnnotationGroundTruth
+                    // Retrieve all lines of code from the SPL file that should be included in the variant file.
+                    final VariantAnnotation variantCode = rootAnnotation.getLinesToCopy(lineMatching::isPresentInVariant);
+                    return TextIO
+                            // read all lines in the input SPL file
+                            .readLines(sourceFile.path())
+                            .bind(splLines -> Result.Try(() ->
+                                    // write all lines that should be included in the variant to the text file
+                                    TextIO.append(
+                                            targetFile.path(),
+                                            String.join(
+                                                    TextIO.LINEBREAK,
+                                                    //
+                                                    variantCode.project(strategy, splLines)
+                                            )
+                                    )
+                            ))
+                            .map(unit2 -> splAnnotationGroundTruth);
+                })))
+                .bimap(
+                        // In case of success, return ground truth.
+                        Functjonal.match(
+                                splAnnotationGroundTruth -> GroundTruth.forSourceCodeFile(
+                                        new SourceCodeFile(getFeatureMapping(), getFile(), splAnnotationGroundTruth.variantArtefact()),
+                                        splAnnotationGroundTruth
+                                ),
+                                () -> GroundTruth.withoutAnnotations(new SourceCodeFile(getFeatureMapping(), getFile()))
                         ),
-                        () -> GroundTruth.withoutAnnotations(new SourceCodeFile(getFeatureMapping(), getFile()))
-                ),
-                // In case of failure, log it (and implicitly transform IOException to Exception).
-                ioexception -> {
-                    Logger.error("Could not create variant file " + targetFile + " because ", ioexception);
-                    return ioexception;
-                }
-        );
+                        // In case of failure, log it (and implicitly transform IOException to Exception).
+                        ioexception -> {
+                            Logger.error("Could not create variant file " + targetFile + " because ", ioexception);
+                            return ioexception;
+                        }
+                );
     }
 
     public LineBasedAnnotation getRootAnnotation() {

src/main/java/org/variantsync/vevos/simulation/variability/pc/SourceCodeFile.java

@@ -5,18 +5,15 @@
 public record VariantGenerationOptions(
         boolean exitOnError,
         boolean ignoreNonExistentSPLFiles,
+        boolean withMacros,
         ArtefactFilter<SourceCodeFile> filter
 )
 {
-    public static VariantGenerationOptions ExitOnError(final ArtefactFilter<SourceCodeFile> filter) {
-        return new VariantGenerationOptions(true, false, filter);
+    public static VariantGenerationOptions ExitOnError(final boolean withMacros, final ArtefactFilter<SourceCodeFile> filter) {
+        return new VariantGenerationOptions(true, false, withMacros, filter);
     }
 
-    public static VariantGenerationOptions IgnoreErrors(final ArtefactFilter<SourceCodeFile> filter) {
-        return new VariantGenerationOptions(false, true, filter);
-    }
-
-    public static VariantGenerationOptions ExitOnErrorButAllowNonExistentFiles(final ArtefactFilter<SourceCodeFile> filter) {
-        return new VariantGenerationOptions(true, true, filter);
+    public static VariantGenerationOptions ExitOnErrorButAllowNonExistentFiles(final boolean withMacros, final ArtefactFilter<SourceCodeFile> filter) {
+        return new VariantGenerationOptions(true, true, withMacros, filter);
     }
 }

src/main/java/org/variantsync/vevos/simulation/variability/pc/options/VariantGenerationOptions.java

@@ -0,0 +1,32 @@
+package org.variantsync.vevos.simulation.variability.pc.variantlines;
+
+import org.prop4j.Node;
+import org.prop4j.NodeWriter;
+import org.variantsync.vevos.simulation.variability.pc.options.VariantGenerationOptions;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public record VariantAnnotation(
+        Node condition,
+        List<VariantLineChunk> lines
+) implements VariantLineChunk {
+    @Override
+    public List<String> project(final VariantGenerationOptions projectionOptions, final List<String> splFileLines) {
+        final List<String> result = new ArrayList<>();
+
+        if (projectionOptions.withMacros()) {
+            result.add("#if " + condition.toString(NodeWriter.javaSymbols));
+        }
+
+        for (final VariantLineChunk child : lines) {
+            result.addAll(child.project(projectionOptions, splFileLines));
+        }
+
+        if (projectionOptions.withMacros()) {
+            result.add("#endif");
+        }
+
+        return result;
+    }
+}