Use separate Ragel machines for script/style tags

Previously a single Ragel machine was used for processing HTML
script and style tags. This had the unfortunate side-effect that the
following was not parsed correctly (while being valid HTML):

    <script>
    var foo = "</style>";
    </script>

The same applied to style tags:

    <style>
    /* </script> */
    </style>

By using separate machines we can work around the above issue. The
downside is that this can produce multiple T_TEXT nodes, which have to
be stitched back together in the parser.

ext/c/lexer.rl

@@ -19,11 +19,15 @@ on `ts` and `te`) so the macro ignores this argument.
 #define advance_line(amount) \
     rb_funcall(self, id_advance_line, 1, INT2NUM(amount));
 
-#define literal_html_element_p() \
-    rb_funcall(self, id_literal_html_element_p, 0) == Qtrue
+#define html_script_p() \
+    rb_funcall(self, id_html_script_p, 0) == Qtrue
+
+#define html_style_p() \
+    rb_funcall(self, id_html_style_p, 0) == Qtrue
 
 ID id_advance_line;
-ID id_literal_html_element_p;
+ID id_html_script_p;
+ID id_html_style_p;
 ID id_html;
 
 %%machine c_lexer;
@@ -183,9 +187,10 @@ void Init_liboga_xml_lexer()
     VALUE mXML   = rb_const_get(mOga, rb_intern("XML"));
     VALUE cLexer = rb_define_class_under(mXML, "Lexer", rb_cObject);
 
-    id_advance_line           = rb_intern("advance_line");
-    id_literal_html_element_p = rb_intern("literal_html_element?");
-    id_html                   = rb_intern("html");
+    id_advance_line  = rb_intern("advance_line");
+    id_html_script_p = rb_intern("html_script?");
+    id_html_style_p  = rb_intern("html_style?");
+    id_html          = rb_intern("html");
 
     rb_define_method(cLexer, "advance_native", oga_xml_lexer_advance, 1);
     rb_define_method(cLexer, "reset_native", oga_xml_lexer_reset, 0);

ext/java/org/liboga/xml/Lexer.rl

@@ -194,13 +194,23 @@ public class Lexer extends RubyObject
     }
 
     /**
-     * See * Oga::XML::Lexer#literal_html_element? for more information.
+     * @see Oga::XML::Lexer#html_script?
      */
-    public Boolean literal_html_element_p()
+    public Boolean html_script_p()
     {
         ThreadContext context = this.runtime.getCurrentContext();
 
-        return this.callMethod(context, "literal_html_element?").isTrue();
+        return this.callMethod(context, "html_script?").isTrue();
+    }
+
+    /**
+     * @see Oga::XML::Lexer#html_style?
+     */
+    public Boolean html_style_p()
+    {
+        ThreadContext context = this.runtime.getCurrentContext();
+
+        return this.callMethod(context, "html_style?").isTrue();
     }
 }
 

ext/ragel/base_lexer.rl

@@ -58,7 +58,7 @@
     }
 
     action advance_newline {
-        advance_line(1)
+        advance_line(1);
     }
 
     action hold_and_return {
@@ -376,6 +376,12 @@
         callback_simple(id_on_element_end);
     }
 
+    action close_element_fnext_main {
+        callback_simple(id_on_element_end);
+
+        fnext main;
+    }
+
     # Machine used for lexing the name/namespace of an element.
     element_name := |*
         identifier ':' => {
@@ -465,9 +471,13 @@
         '>' => {
             callback_simple(id_on_element_open_end);
 
-            if ( literal_html_element_p() )
+            if ( html_script_p() )
             {
-                fnext literal_html_element;
+                fnext html_script;
+            }
+            else if ( html_style_p() )
+            {
+                fnext html_style;
             }
             else
             {
@@ -506,6 +516,17 @@
     terminate_text = '</' | '<!' | '<?' | element_start;
     allowed_text   = (any* -- terminate_text) $count_newlines;
 
+    action emit_text {
+        callback(id_on_text, data, encoding, ts, te);
+
+        if ( lines > 0 )
+        {
+            advance_line(lines);
+
+            lines = 0;
+        }
+    }
+
     text := |*
         terminate_text | allowed_text => {
             callback(id_on_text, data, encoding, ts, te);
@@ -541,36 +562,17 @@
     # Certain tags in HTML can contain basically anything except for the literal
     # closing tag. Two examples are script and style tags.  As a result of this
     # we can't use the regular text machine.
-    literal_html_closing_tags = '</script>' | '</style>';
-    literal_html_allowed = (any* -- literal_html_closing_tags) $count_newlines;
 
-    literal_html_element := |*
-        literal_html_allowed => {
-            callback(id_on_text, data, encoding, ts, te);
+    literal_html_allowed = (^'<'+ | '<'+) $count_newlines;
 
-            if ( lines > 0 )
-            {
-                advance_line(lines);
+    html_script := |*
+        literal_html_allowed => emit_text;
+        '</script>'          => close_element_fnext_main;
+    *|;
 
-                lines = 0;
-            }
-        };
-
-        literal_html_allowed %{ mark = p; } literal_html_closing_tags => {
-            callback(id_on_text, data, encoding, ts, mark);
-
-            p    = mark - 1;
-            mark = 0;
-
-            if ( lines > 0 )
-            {
-                advance_line(lines);
-
-                lines = 0;
-            }
-
-            fnext main;
-        };
+    html_style := |*
+        literal_html_allowed => emit_text;
+        '</style>'           => close_element_fnext_main;
     *|;
 
     # The main machine aka the entry point of Ragel.