Mozilla bug 1540675 - Handle line breaks correctly in comment end bang state. r=alchen.

Author: hsivonen

src/nu/validator/htmlparser/impl/Tokenizer.java

@@ -2820,10 +2820,12 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 continue stateloop;
                             case '\r':
                                 appendStrBufCarriageReturn();
+                                state = transition(state, Tokenizer.COMMENT, reconsume, pos);
                                 break stateloop;
                             case '\n':
                                 appendStrBufLineFeed();
-                                continue;
+                                state = transition(state, Tokenizer.COMMENT, reconsume, pos);
+                                continue stateloop;
                             case '\u0000':
                                 c = '\uFFFD';
                                 // CPPONLY: MOZ_FALLTHROUGH;

src/nu/validator/htmlparser/impl/TreeBuilder.java

@@ -5231,7 +5231,7 @@ private void popForeign(int origPos, int eltPos) throws SAXException {
         StackNode<T> node = stack[currentPtr];
         if (origPos != currentPtr || eltPos != currentPtr) {
             markMalformedIfScript(node.node);
-    	}
+        }
         assert debugOnlyClearLastStackSlot();
         currentPtr--;
         elementPopped(node.ns, node.popName, node.node);

test-src/nu/validator/htmlparser/test/JSONArrayTokenHandler.java

@@ -145,8 +145,8 @@ public boolean wantsComments() throws SAXException {
     }
 
     public void error(SAXParseException exception) throws SAXException {
-        flushCharacters();
-        array.getValue().add(PARSE_ERROR);
+//        flushCharacters();
+//        array.getValue().add(PARSE_ERROR);
     }
 
     public void fatalError(SAXParseException exception) throws SAXException {

test-src/nu/validator/htmlparser/test/TokenizerTester.java

@@ -57,8 +57,12 @@ public class TokenizerTester {
 
     private static JSONString RCDATA = new JSONString("RCDATA state");
 
+    private static JSONString CDATA = new JSONString("CDATA section state");
+
     private static JSONString RAWTEXT = new JSONString("RAWTEXT state");
 
+    private static JSONString SCRIPT_DATA = new JSONString("Script data state");
+
     private static boolean jsonDeepEquals(JSONValue one, JSONValue other) {
         if (one.isSimple()) {
             return one.equals(other);
@@ -144,9 +148,15 @@ private void runTest(JSONObject test) throws SAXException, IOException {
                 } else if (RCDATA.equals(value)) {
                     runTestInner(inputString, expectedTokens, description,
                             Tokenizer.RCDATA, lastStartTag);
+                } else if (CDATA.equals(value)) {
+                    runTestInner(inputString, expectedTokens, description,
+                            Tokenizer.CDATA_SECTION, lastStartTag);
                 } else if (PLAINTEXT.equals(value)) {
                     runTestInner(inputString, expectedTokens, description,
                             Tokenizer.PLAINTEXT, lastStartTag);
+                } else if (SCRIPT_DATA.equals(value)) {
+                    runTestInner(inputString, expectedTokens, description,
+                            Tokenizer.SCRIPT_DATA, "script");
                 } else {
                     throw new RuntimeException("Broken test data.");
                 }

test-src/nu/validator/htmlparser/test/TreeTester.java

@@ -83,12 +83,25 @@ private boolean runTest() throws Throwable {
                 // spin
             }
 
+            boolean newErrors = false;
             StringBuilder sb = new StringBuilder();
             int c;
             while ((c = aggregateStream.read()) != '\n') {
                 sb.append((char) c);
             }
             String label = sb.toString();
+            if ("new-errors".equals(label)) {
+                newErrors = true;
+                stream = new UntilHashInputStream(aggregateStream);
+                while (stream.read() != -1) {
+                    // spin
+                }
+                sb.setLength(0);
+                while ((c = aggregateStream.read()) != '\n') {
+                    sb.append((char) c);
+                }
+                label = sb.toString();
+            }
             if ("document-fragment".equals(label)) {
                 sb.setLength(0);
                 while ((c = aggregateStream.read()) != '\n') {
@@ -160,6 +173,18 @@ private boolean runTest() throws Throwable {
                 expectedErrors.add(line);
             }
 
+            if (newErrors) {
+                if (skipLabel()) { // #new-errors
+                    System.err.println("Premature end of test data.");
+                    return false;
+                }
+                br = new BufferedReader(new InputStreamReader(
+                        new UntilHashInputStream(aggregateStream), "UTF-8"));
+                while ((line = br.readLine()) != null) {
+                    expectedErrors.add(line);
+                }
+            }
+
             if (context != null) {
                 if (skipLabel()) { // #document-fragment
                     System.err.println("Premature end of test data.");