Lazy decoding of XML/HTML entities.

Instead of decoding entities in the lexer we'll do this whenever XML::Text#text
is called. This removes the overhead from the parsing phase and ensures the
process is only triggered when actually needed. Note that calling #to_xml and/or
the #inspect methods on a Text (or parent) instance will also trigger the entity
conversion process.

The new entity decoding API supports both regular entities (e.g. &) as well
as codepoint based entities (both regular and hexadecimal codepoints).

To allow safe read-only access to Text instances from multiple threads a mutex
is used. This mutex ensures that only 1 thread can trigger the conversion
process.

Fixes #68

lib/oga.rb

@@ -3,6 +3,7 @@ gem 'racc'
 require 'ast'
 require 'set'
 require 'stringio'
+require 'thread'
 
 require_relative 'oga/version'
 require_relative 'oga/oga'
@@ -43,6 +44,7 @@ require_relative 'oga/xml/pull_parser'
 
 require_relative 'oga/html/parser'
 require_relative 'oga/html/sax_parser'
+require_relative 'oga/html/entities'
 
 require_relative 'oga/xpath/lexer'
 require_relative 'oga/xpath/parser'

lib/oga/xml/entities.rb

@@ -1,5 +1,9 @@
 module Oga
   module XML
+    ##
+    # Module for encoding/decoding XML and HTML entities. The mapping of HTML
+    # entities can be found in {Oga::HTML::Entities::DECODE_MAPPING}.
+    #
     module Entities
       ##
       # Hash containing XML entities and the corresponding characters.
@@ -11,15 +15,10 @@ module Oga
       #
       DECODE_MAPPING = {
         '&lt;'   => '<',
-        '&#60;'  => '<',
         '&gt;'   => '>',
-        '&#62;'  => '>',
         '&apos;' => "'",
-        '&#39;'  => "'",
         '&quot;' => '"',
-        '&#34;'  => '"',
         '&amp;'  => '&',
-        '&#38;'  => '&',
       }
 
       ##
@@ -35,16 +34,46 @@ module Oga
         '<' => '&lt;',
       }
 
+      ##
+      # @return [String]
+      #
+      AMPERSAND = '&'.freeze
+
+      ##
+      # Regexp for matching XML/HTML entities such as "&nbsp;".
+      #
+      # @return [Regexp]
+      #
+      REGULAR_ENTITY = /&[a-zA-Z]+;/
+
+      ##
+      # Regexp for matching XML/HTML entities such as "&#38;".
+      #
+      # @return [Regexp]
+      #
+      CODEPOINT_ENTITY = /&#(x)?([a-zA-Z0-9]+);/
+
+      ##
+      # @return [Regexp]
+      #
+      ENCODE_REGEXP = Regexp.new(ENCODE_MAPPING.keys.join('|'))
+
       ##
       # Decodes XML entities.
       #
       # @param [String] input
+      # @param [Array] keys
+      # @param [Hash] mapping
       # @return [String]
       #
-      def self.decode(input)
-        if input.include?('&')
-          DECODE_MAPPING.each do |find, replace|
-            input = input.gsub(find, replace)
+      def self.decode(input, mapping = DECODE_MAPPING)
+        return input unless input.include?(AMPERSAND)
+
+        input = input.gsub(REGULAR_ENTITY, mapping)
+
+        if input.include?(AMPERSAND)
+          input = input.gsub(CODEPOINT_ENTITY) do |match|
+            [$1 ? Integer($2, 16) : Integer($2)].pack('U')
           end
         end
 
@@ -55,14 +84,11 @@ module Oga
       # Encodes special characters as XML entities.
       #
       # @param [String] input
+      # @param [Hash] mapping
       # @return [String]
       #
-      def self.encode(input)
-        ENCODE_MAPPING.each do |from, to|
-          input = input.gsub(from, to) if input.include?(from)
-        end
-
-        return input
+      def self.encode(input, mapping = ENCODE_MAPPING)
+        return input.gsub(ENCODE_REGEXP, mapping)
       end
     end # Entities
   end # XML

lib/oga/xml/lexer.rb

@@ -217,7 +217,7 @@ module Oga
       # @param [String] value The data between the quotes.
       #
       def on_string_body(value)
-        add_token(:T_STRING_BODY, Entities.decode(value))
+        add_token(:T_STRING_BODY, value)
       end
 
       ##
@@ -373,7 +373,7 @@ module Oga
       def on_text(value)
         return if value.empty?
 
-        add_token(:T_TEXT, Entities.decode(value))
+        add_token(:T_TEXT, value)
       end
 
       ##

lib/oga/xml/text.rb

@@ -5,19 +5,65 @@ module Oga
     # have any children, attributes and the likes; just text.
     #
     class Text < CharacterNode
+      def initialize(*args)
+        super
+
+        @mutex   = Mutex.new
+        @decoded = false
+      end
+
+      ##
+      # @param [String] value
+      #
+      def text=(value)
+        # In case of concurrent text/text= calls.
+        @mutex.synchronize do
+          @decoded = false
+          @text    = value
+        end
+      end
+
+      ##
+      # Returns the text as a String. Upon the first call any XML/HTML entities
+      # are decoded.
+      #
+      # @return [String]
+      #
+      def text
+        @mutex.synchronize do
+          unless @decoded
+            decoder  = html? ? HTML::Entities : Entities
+            @text    = decoder.decode(@text)
+            @decoded = true
+          end
+        end
+
+        return @text
+      end
+
       ##
       # @see [Oga::XML::CharacterNode#to_xml]
       #
       def to_xml
         node = parent
-        root = root_node
 
-        if root.is_a?(Document) and node.is_a?(Element) and root.html? \
+        if node.is_a?(Element) and html? \
         and Lexer::LITERAL_HTML_ELEMENTS.include?(node.name)
           return super
-        else
-          return Entities.encode(super)
         end
+
+        return Entities.encode(super)
+      end
+
+      private
+
+      ##
+      # @return [TrueClass|FalseClass]
+      #
+      def html?
+        root = root_node
+
+        return root.is_a?(Document) && root.html?
       end
     end # Text
   end # XML