Don't use separate tokens/nodes for newlines.

Newlines are now lexed together with regular text. The line numbers are
advanced based on the amount of "\n" sequences in a text buffer.
This commit is contained in:
Yorick Peterse committed 2014-03-17 21:26:21 +01:00
1 parent 8898d08831
commit 274ab359ba
2 files changed
+16 -31

No files matched your search

+15 -22
View File
@@ -81,8 +81,8 @@ module Oga
private
def advance_line
@line += 1
def advance_line(amount = 1)
@line += amount
@column = 1
end
@@ -113,9 +113,19 @@ module Oga
add_token(:T_TEXT, @text_buffer)
lines = @text_buffer.count("\n")
advance_line(lines) if lines > 0
@text_buffer = ''
end
def buffer_text_until_eof(eof)
@text_buffer << text
emit_text_buffer if @te == eof
end
def emit_string_buffer
add_token(:T_STRING, @string_buffer)
advance_column
@@ -134,11 +144,6 @@ module Oga
newline = '\n' | '\r\n';
whitespace = [ \t];
action emit_newline {
t(:T_TEXT)
advance_line
}
# String processing
#
# These actions/definitions can be used to process single and/or double
@@ -300,16 +305,8 @@ module Oga
element_start = '<' element_name;
element_text := |*
newline => {
emit_text_buffer
t(:T_TEXT)
advance_line
};
^('<' | newline) => {
@text_buffer << text
emit_text_buffer if @te == eof
^'<' => {
buffer_text_until_eof(eof)
};
'<' => {
@@ -391,8 +388,6 @@ module Oga
*|;
main := |*
newline @emit_text_buffer => emit_newline;
doctype_start @emit_text_buffer => {
t(:T_DOCTYPE_START)
fcall doctype;
@@ -411,9 +406,7 @@ module Oga
element_start @emit_text_buffer => open_element;
any => {
@text_buffer << text
emit_text_buffer if @te == eof
buffer_text_until_eof(eof)
};
*|;
}%%