Lex comments in chunks

Similar to this being added for CDATA tags in
8acc7fc743 comments are now also lexed in
chunks.

Related issue: #93

lib/oga/xml/lexer.rb

@@ -284,13 +284,27 @@ module Oga
         add_token(:T_CDATA_BODY, value)
       end
 
+      ##
+      # Called on the open comment tag.
+      #
+      def on_comment_start
+        add_token(:T_COMMENT_START)
+      end
+
+      ##
+      # Called on the closing comment tag.
+      #
+      def on_comment_end
+        add_token(:T_COMMENT_END)
+      end
+
       ##
       # Called on a comment.
       #
       # @param [String] value
       #
-      def on_comment(value)
-        add_token(:T_COMMENT, value)
+      def on_comment_body(value)
+        add_token(:T_COMMENT_BODY, value)
       end
 
       ##

lib/oga/xml/parser.rll

@@ -24,7 +24,8 @@
 
 %terminals T_TEXT T_STRING_SQUOTE T_STRING_DQUOTE T_STRING_BODY;
 %terminals T_DOCTYPE_START T_DOCTYPE_END T_DOCTYPE_TYPE T_DOCTYPE_NAME;
-%terminals T_DOCTYPE_INLINE T_COMMENT;
+%terminals T_DOCTYPE_INLINE;
+%terminals T_COMMENT_START T_COMMENT_BODY T_COMMENT_END;
 %terminals T_CDATA_START T_CDATA_BODY T_CDATA_END;
 %terminals T_ELEM_START T_ELEM_NAME T_ELEM_NS T_ELEM_END T_ATTR T_ATTR_NS;
 %terminals T_XML_DECL_START T_XML_DECL_END;
@@ -105,7 +106,12 @@ cdata_body
 # Comments
 
 comment
-  = T_COMMENT { on_comment(val[0]) }
+  = T_COMMENT_START comment_body T_COMMENT_END { on_comment(val[1]) }
+  ;
+
+comment_body
+  = T_COMMENT_BODY comment_body { val[0] + val[1] }
+  | _                           { '' }
   ;
 
 # Processing Instructions