refactor: reduce overhead of AnnotationParser implementations

Author: AlexanderSchultheiss

src/main/java/org/variantsync/diffdetective/datasets/PatchDiffParseOptions.java

@@ -1,6 +1,6 @@
 package org.variantsync.diffdetective.datasets;
 
-import org.variantsync.diffdetective.feature.cpp.CPPAnnotationParser;
+import org.variantsync.diffdetective.feature.PreprocessorAnnotationParser;
 import org.variantsync.diffdetective.variation.diff.parse.VariationDiffParseOptions;
 
 /**
@@ -27,7 +27,7 @@ public enum DiffStoragePolicy {
     /**
      * Creates PatchDiffParseOptions with the given annotation parser.
      */
-    public PatchDiffParseOptions withAnnotationParser(CPPAnnotationParser annotationParser) {
+    public PatchDiffParseOptions withAnnotationParser(PreprocessorAnnotationParser annotationParser) {
         return new PatchDiffParseOptions(
                 this.diffStoragePolicy(),
                 this.variationDiffParseOptions().withAnnotationParser(annotationParser)

src/main/java/org/variantsync/diffdetective/datasets/predefined/Marlin.java

@@ -2,8 +2,8 @@
 
 import org.variantsync.diffdetective.datasets.PatchDiffParseOptions;
 import org.variantsync.diffdetective.datasets.Repository;
+import org.variantsync.diffdetective.feature.PreprocessorAnnotationParser;
 import org.variantsync.diffdetective.feature.PropositionalFormulaParser;
-import org.variantsync.diffdetective.feature.cpp.CPPAnnotationParser;
 
 import java.nio.file.Path;
 
@@ -13,8 +13,9 @@
  * @author Kevin Jedelhauser, Paul Maximilian Bittner
  */
 public class Marlin {
-    public static final CPPAnnotationParser ANNOTATION_PARSER =
-            new CPPAnnotationParser(
+    public static final PreprocessorAnnotationParser ANNOTATION_PARSER =
+            new PreprocessorAnnotationParser(
+                    PreprocessorAnnotationParser.CPP_PATTERN,
                     PropositionalFormulaParser.Default,
                     new MarlinCPPDiffLineFormulaExtractor()
             );

src/main/java/org/variantsync/diffdetective/feature/PreprocessorAnnotationParser.java

@@ -0,0 +1,99 @@
+package org.variantsync.diffdetective.feature;
+
+import org.prop4j.Node;
+import org.variantsync.diffdetective.error.UnparseableFormulaException;
+import org.variantsync.diffdetective.feature.cpp.CPPDiffLineFormulaExtractor;
+import org.variantsync.diffdetective.feature.jpp.JPPDiffLineFormulaExtractor;
+
+import java.util.regex.Pattern;
+
+/**
+ * A parser of preprocessor-like annotations.
+ *
+ * @author Paul Bittner, Alexander Schultheiß
+ */
+public class PreprocessorAnnotationParser implements AnnotationParser {
+    /**
+     * Matches the beginning or end of CPP conditional macros.
+     * It doesn't match the whole macro name, for example for {@code #ifdef} only {@code "#if"} is
+     * matched and only {@code "if"} is captured.
+     * <p>
+     * Note that this pattern doesn't handle comments between {@code #} and the macro name.
+     */
+    public final static Pattern CPP_PATTERN =
+            Pattern.compile("^[+-]?\\s*#\\s*(if|elif|else|endif)");
+
+    /**
+     * Matches the beginning or end of JPP conditional macros.
+     * It doesn't match the whole macro name, for example for {@code //#if defined(x)} only {@code "//#if"} is
+     * matched and only {@code "if"} is captured.
+     * <p>
+     */
+    public final static Pattern JPP_PATTERN =
+            Pattern.compile("^[+-]?\\s*//\\s*#\\s*(if|elif|else|endif)");
+
+    /**
+     * Default parser for C preprocessor annotations.
+     * Created by invoking {@link #PreprocessorAnnotationParser(Pattern, PropositionalFormulaParser, DiffLineFormulaExtractor)}.
+     */
+    public static final PreprocessorAnnotationParser CPPAnnotationParser =
+            new PreprocessorAnnotationParser(CPP_PATTERN, PropositionalFormulaParser.Default, new CPPDiffLineFormulaExtractor());
+
+    /**
+     * Default parser for <a href="https://www.slashdev.ca/javapp/">JavaPP (Java PreProcessor)</a> annotations.
+     * Created by invoking {@link #PreprocessorAnnotationParser(Pattern, PropositionalFormulaParser, DiffLineFormulaExtractor)}.
+     */
+    public static final PreprocessorAnnotationParser JPPAnnotationParser =
+            new PreprocessorAnnotationParser(JPP_PATTERN, PropositionalFormulaParser.Default, new JPPDiffLineFormulaExtractor());
+
+    // Pattern that is used to identify the AnnotationType of a given annotation.
+    private final Pattern annotationPattern;
+    private final PropositionalFormulaParser formulaParser;
+    private final DiffLineFormulaExtractor extractor;
+
+    /**
+     * Invokes {@link #PreprocessorAnnotationParser(Pattern, PropositionalFormulaParser, DiffLineFormulaExtractor)} with
+     * the {@link PropositionalFormulaParser#Default default formula parser} and a new {@link DiffLineFormulaExtractor}.
+     *
+     * @param annotationPattern Pattern that is used to identify the AnnotationType of a given annotation; {@link #CPP_PATTERN} provides an example
+     */
+    public PreprocessorAnnotationParser(final Pattern annotationPattern, final DiffLineFormulaExtractor formulaExtractor) {
+        this(annotationPattern, PropositionalFormulaParser.Default, formulaExtractor);
+    }
+
+    /**
+     * Creates a new preprocessor annotation parser.
+     *
+     * @param annotationPattern Pattern that is used to identify the AnnotationType of a given annotation; {@link #CPP_PATTERN} provides an example
+     * @param formulaParser     Parser that is used to parse propositional formulas in conditional annotations (e.g., the formula <code>f</code> in <code>#if f</code>).
+     * @param formulaExtractor  An extractor that extracts the formula part of a preprocessor annotation that is then given to the formulaParser.
+     */
+    public PreprocessorAnnotationParser(final Pattern annotationPattern, final PropositionalFormulaParser formulaParser, DiffLineFormulaExtractor formulaExtractor) {
+        this.annotationPattern = annotationPattern;
+        this.formulaParser = formulaParser;
+        this.extractor = formulaExtractor;
+    }
+
+    /**
+     * Parses the condition of the given line of source code that contains a preprocessor macro (i.e., IF, IFDEF, ELIF).
+     *
+     * @param line The line of code of a preprocessor annotation.
+     * @return The formula of the macro in the given line.
+     * If no such formula could be parsed, returns a Literal with the line's condition as name.
+     * @throws UnparseableFormulaException when {@link DiffLineFormulaExtractor#extractFormula(String)} throws.
+     */
+    public Node parseAnnotation(String line) throws UnparseableFormulaException {
+        return this.formulaParser.parse(extractor.extractFormula(line));
+    }
+
+    @Override
+    public AnnotationType determineAnnotationType(String text) {
+        var matcher = annotationPattern.matcher(text);
+        int nameId = 1;
+        if (matcher.find()) {
+            return AnnotationType.fromName(matcher.group(nameId));
+        } else {
+            return AnnotationType.None;
+        }
+    }
+}

src/main/java/org/variantsync/diffdetective/feature/PropositionalFormulaParser.java

@@ -1,5 +1,6 @@
 package org.variantsync.diffdetective.feature;
 
+import org.prop4j.Literal;
 import org.prop4j.Node;
 import org.prop4j.NodeReader;
 import org.variantsync.diffdetective.util.fide.FixTrueFalse;
@@ -36,6 +37,11 @@ public interface PropositionalFormulaParser {
             node = FixTrueFalse.EliminateTrueAndFalseInplace(node).get();
         }
 
+        if (node == null) {
+//            Logger.warn("Could not parse expression '{}' to feature mapping. Using it as literal.", fmString);
+            node = new Literal(text);
+        }
+
         return node;
     };
 }

src/main/java/org/variantsync/diffdetective/feature/cpp/CPPAnnotationParser.java

@@ -0,0 +1,50 @@
+package org.variantsync.diffdetective.feature.jpp;
+
+import org.antlr.v4.runtime.CharStreams;
+import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.tree.ParseTree;
+import org.variantsync.diffdetective.feature.AbstractingFormulaExtractor;
+import org.variantsync.diffdetective.feature.ParseErrorListener;
+import org.variantsync.diffdetective.feature.antlr.JPPExpressionLexer;
+import org.variantsync.diffdetective.feature.antlr.JPPExpressionParser;
+
+import java.util.regex.Pattern;
+
+/**
+ * Extracts the expression from a <a href="https://www.slashdev.ca/javapp/">JavaPP (Java PreProcessor)</a> statement .
+ * For example, given the annotation "//#if defined(A) || B()", the extractor would extract "A || B".
+ * The extractor detects if and elif annotations (other annotations do not have expressions).
+ * The given JPP statement might also be a line in a diff (i.e., preceeded by a - or +).
+ *
+ * @author Alexander Schultheiß
+ */
+public class JPPDiffLineFormulaExtractor extends AbstractingFormulaExtractor {
+    private static final String JPP_ANNOTATION_REGEX = "^[+-]?\\s*//\\s*#\\s*(if|elif)(\\s+(.*)|(\\(.*\\)))$";
+    private static final Pattern JPP_ANNOTATION_PATTERN = Pattern.compile(JPP_ANNOTATION_REGEX);
+
+    public JPPDiffLineFormulaExtractor() {
+        super(JPP_ANNOTATION_PATTERN);
+    }
+
+    /**
+     * Abstract the given formula.
+     * <p>
+     * First, the visitor uses ANTLR to parse the formula into a parse tree gives the tree to a {@link ControllingJPPExpressionVisitor}.
+     * The visitor traverses the tree starting from the root, searching for subtrees that must be abstracted.
+     * If such a subtree is found, the visitor calls an {@link AbstractingJPPExpressionVisitor} to abstract the part of
+     * the formula in the subtree.
+     * </p>
+     *
+     * @param formula that is to be abstracted
+     * @return the abstracted formula
+     */
+    @Override
+    protected String abstractFormula(String formula) {
+        JPPExpressionLexer lexer = new JPPExpressionLexer(CharStreams.fromString(formula));
+        CommonTokenStream tokens = new CommonTokenStream(lexer);
+        JPPExpressionParser parser = new JPPExpressionParser(tokens);
+        parser.addErrorListener(new ParseErrorListener(formula));
+        ParseTree tree = parser.expression();
+        return tree.accept(new ControllingJPPExpressionVisitor()).toString();
+    }
+}

src/main/java/org/variantsync/diffdetective/feature/jpp/JPPDiffLineFormulaExtractor.java

@@ -5,7 +5,7 @@
 import org.variantsync.diffdetective.datasets.Repository;
 import org.variantsync.diffdetective.diff.git.PatchDiff;
 import org.variantsync.diffdetective.diff.result.DiffParseException;
-import org.variantsync.diffdetective.feature.cpp.CPPAnnotationParser;
+import org.variantsync.diffdetective.feature.PreprocessorAnnotationParser;
 import org.variantsync.diffdetective.mining.RWCompositePatternNodeFormat;
 import org.variantsync.diffdetective.mining.RWCompositePatternTreeFormat;
 import org.variantsync.diffdetective.mining.VariationDiffMiner;
@@ -102,7 +102,7 @@ private static void render(final Path fileToRender) {
             try {
                 t = VariationDiff.fromFile(fileToRender,
                         new VariationDiffParseOptions(
-                                CPPAnnotationParser.Default, collapseMultipleCodeLines, ignoreEmptyLines
+                                PreprocessorAnnotationParser.CPPAnnotationParser, collapseMultipleCodeLines, ignoreEmptyLines
                         ));
             } catch (IOException | DiffParseException e) {
                 Logger.error(e, "Could not read given file '{}'", fileToRender);

src/main/java/org/variantsync/diffdetective/internal/SimpleRenderer.java

@@ -1,7 +1,7 @@
 package org.variantsync.diffdetective.variation.diff.parse;
 
 import org.variantsync.diffdetective.feature.AnnotationParser;
-import org.variantsync.diffdetective.feature.cpp.CPPAnnotationParser;
+import org.variantsync.diffdetective.feature.PreprocessorAnnotationParser;
 
 /**
  * Parse options that should be used when parsing {@link org.variantsync.diffdetective.variation.diff.VariationDiff}s.
@@ -37,7 +37,7 @@ public VariationDiffParseOptions(
     /**
      * Creates VariationDiffParseOptions with the given annotation parser.
      */
-    public VariationDiffParseOptions withAnnotationParser(CPPAnnotationParser annotationParser) {
+    public VariationDiffParseOptions withAnnotationParser(PreprocessorAnnotationParser annotationParser) {
         return new VariationDiffParseOptions(
                 annotationParser,
                 this.collapseMultipleCodeLines(),
@@ -47,10 +47,10 @@ public VariationDiffParseOptions withAnnotationParser(CPPAnnotationParser annota
 
     /**
      * Default value for VariationDiffParseOptions that does not remember parsed unix diffs
-     * and uses the default value for the parsing annotations ({@link CPPAnnotationParser#Default}).
+     * and uses the default value for the parsing annotations ({@link PreprocessorAnnotationParser#CPPAnnotationParser}).
      */
     public static final VariationDiffParseOptions Default = new VariationDiffParseOptions(
-            CPPAnnotationParser.Default,
+            PreprocessorAnnotationParser.CPPAnnotationParser,
             false,
             false
     );

src/main/java/org/variantsync/diffdetective/variation/diff/parse/VariationDiffParseOptions.java

@@ -1,9 +1,13 @@
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
 import org.variantsync.diffdetective.error.UnparseableFormulaException;
+import org.variantsync.diffdetective.feature.jpp.JPPDiffLineFormulaExtractor;
 
 import java.util.List;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
 // Test cases for a parser of https://www.slashdev.ca/javapp/
 public class JPPParserTest {
     private record TestCase(String formula, String expected) {
@@ -70,17 +74,18 @@ private static List<JPPParserTest.ThrowingTestCase> throwingTestCases() {
     @ParameterizedTest
     @MethodSource("testCases")
     public void testCase(JPPParserTest.TestCase testCase) throws UnparseableFormulaException {
-//        assertEquals(
-//                testCase.expected,
-//                // TODO:
-//        );
+        assertEquals(
+                testCase.expected,
+                new JPPDiffLineFormulaExtractor().extractFormula(testCase.formula())
+        );
     }
 
     @ParameterizedTest
     @MethodSource("throwingTestCases")
     public void throwingTestCase(JPPParserTest.ThrowingTestCase testCase) {
-        //  assertThrows(UnparseableFormulaException.class, () -> //TODO
-        //  );
+        assertThrows(UnparseableFormulaException.class, () ->
+                new JPPDiffLineFormulaExtractor().extractFormula(testCase.formula)
+        );
     }
 
 }