rubychan
diff --git a/‎Changes.textile
Lines changed: 8 additions & 1 deletion b/‎Changes.textile
Lines changed: 8 additions & 1 deletion
diff --git a/‎lib/coderay/encoders/debug_lint.rb
Lines changed: 2 additions & 1 deletion b/‎lib/coderay/encoders/debug_lint.rb
Lines changed: 2 additions & 1 deletion
diff --git a/‎lib/coderay/encoders/html.rb
Lines changed: 14 additions & 9 deletions b/‎lib/coderay/encoders/html.rb
Lines changed: 14 additions & 9 deletions
diff --git a/‎lib/coderay/encoders/lint.rb
Lines changed: 3 additions & 1 deletion b/‎lib/coderay/encoders/lint.rb
Lines changed: 3 additions & 1 deletion
diff --git a/‎lib/coderay/helpers/file_type.rb
Lines changed: 2 additions & 2 deletions b/‎lib/coderay/helpers/file_type.rb
Lines changed: 2 additions & 2 deletions
diff --git a/‎lib/coderay/helpers/plugin.rb
Lines changed: 11 additions & 11 deletions b/‎lib/coderay/helpers/plugin.rb
Lines changed: 11 additions & 11 deletions
diff --git a/‎lib/coderay/scanners/debug.rb
Lines changed: 18 additions & 8 deletions b/‎lib/coderay/scanners/debug.rb
Lines changed: 18 additions & 8 deletions
diff --git a/‎lib/coderay/scanners/raydebug.rb
Lines changed: 20 additions & 11 deletions b/‎lib/coderay/scanners/raydebug.rb
Lines changed: 20 additions & 11 deletions
diff --git a/‎lib/coderay/scanners/ruby/patterns.rb
Lines changed: 4 additions & 1 deletion b/‎lib/coderay/scanners/ruby/patterns.rb
Lines changed: 4 additions & 1 deletion
diff --git a/‎lib/coderay/scanners/ruby/string_state.rb
Lines changed: 7 additions & 7 deletions b/‎lib/coderay/scanners/ruby/string_state.rb
Lines changed: 7 additions & 7 deletions
diff --git a/‎lib/coderay/styles/alpha.rb
Lines changed: 1 addition & 1 deletion b/‎lib/coderay/styles/alpha.rb
Lines changed: 1 addition & 1 deletion
diff --git a/‎lib/coderay/token_kinds.rb
Lines changed: 3 additions & 5 deletions b/‎lib/coderay/token_kinds.rb
Lines changed: 3 additions & 5 deletions
diff --git a/‎test/functional/basic.rb
Lines changed: 1 addition & 3 deletions b/‎test/functional/basic.rb
Lines changed: 1 addition & 3 deletions
@@ -10,6 +10,7 @@ h2. Changes in 1.1
 * New scanner: Taskpaper [#39, thanks to shimomura]
 * Diff scanner: Highlight inline changes in multi-line changes [#99]
 * JavaScript scanner: Highlight multi-line comments in diff correctly
+* Ruby scanner: Accept %i and %I symbol lists (Ruby 2.0) [thanks to Nathan Youngman]
 * Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126]
 * HTML scanner displays style tags and attributes now [#145]
 * Remove double-click toggle handler from HTML table output
@@ -22,11 +23,17 @@ h2. Changes in 1.1
 * @CodeRay::TokenKinds@ should not be frozen [#130, thanks to Gavin Kistner]
 * New token type @:id@ for CSS/Sass [#27]
 * New token type @:done@ for Taskpaper [#39]
-* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and nathany]
+* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and Nathan Youngman]
+* New token type @:unknown@ for Debug scanner
 * Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom]
 * Override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05]
 * Fixed @:docstring@ token type style
+* Fixed several problems related to Hash caches and dynamic Symbol creation that might have been exploited by an attacker [#148]
+* @PluginHost@ now works with Strings instead of Symbols internally (to avoid using @#to_sym@)
+* The @Debug@ scanner maps unknown token kinds to @:unknown@ (to avoid creating Symbols based on possibly unsafe input)
+* The @Raydebug@ scanner highlights unknown token kinds as @:plain@
 * @Plugin@ does not warn about fallback when default is defined
+* Fixed @HTML@ encoder when output is a StringIO (eg. when using @-HTML@)
 * @HTML@ encoder will not warn about unclosed token groups at the end of the stream
 * @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now
 * @Debug@ encoder will not warn about errors in the token stream
 
@@ -18,7 +18,8 @@ class DebugLint < Debug
     register_for :debug_lint
 
     def text_token text, kind
-      raise Lint::EmptyToken, 'empty token' if text.empty?
+      raise Lint::EmptyToken,       'empty token for %p' % [kind] if text.empty?
+      raise Lint::UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind
       super
     end
 
 
@@ -182,13 +182,15 @@ def finish options
         @last_opened = nil
       end
 
-      @out.extend Output
-      @out.css = @css
-      if options[:line_numbers]
-        Numbering.number! @out, options[:line_numbers], options
+      if @out.respond_to? :to_str
+        @out.extend Output
+        @out.css = @css
+        if options[:line_numbers]
+          Numbering.number! @out, options[:line_numbers], options
+        end
+        @out.wrap! options[:wrap]
+        @out.apply_title! options[:title]
       end
-      @out.wrap! options[:wrap]
-      @out.apply_title! options[:title]
 
       if defined?(@real_out) && @real_out
         @real_out << @out
@@ -276,7 +278,7 @@ def style_for_kinds kinds
 
     def make_span_for_kinds method, hint
       Hash.new do |h, kinds|
-        h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin
+        begin
           css_class = css_class_for_kinds(kinds)
           title     = HTML.token_path_to_hint hint, kinds if hint
 
@@ -288,6 +290,9 @@ def make_span_for_kinds method, hint
               "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
             end
           end
+        end.tap do |span|
+          h.clear if h.size >= 100
+          h[kinds] = span
         end
       end
     end
@@ -300,8 +305,8 @@ def check_group_nesting name, kind
 
     def break_lines text, style
       reopen = ''
-      @opened.each_with_index do |k, index|
-        reopen << (@span_for_kinds[index > 0 ? [k, *@opened[0...index]] : k] || '<span>')
+      @opened.each_with_index do |kind, index|
+        reopen << (@span_for_kinds[index > 0 ? [kind, *@opened[0...index]] : kind] || '<span>')
       end
       text.gsub("\n", "#{'</span>' * @opened.size}#{'</span>' if style}\n#{reopen}#{style}")
     end
 
@@ -17,10 +17,12 @@ class Lint < Debug
 
     InvalidTokenStream         = Class.new StandardError
     EmptyToken                 = Class.new InvalidTokenStream
+    UnknownTokenKind           = Class.new InvalidTokenStream
     IncorrectTokenGroupNesting = Class.new InvalidTokenStream
 
     def text_token text, kind
-      raise EmptyToken, 'empty token' if text.empty?
+      raise EmptyToken,       'empty token for %p' % [kind] if text.empty?
+      raise UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind
     end
 
     def begin_group kind
 
@@ -38,7 +38,7 @@ def [] filename, read_shebang = false
           (TypeFromExt[ext2.downcase] if ext2) ||
           TypeFromName[name] ||
           TypeFromName[name.downcase]
-        type ||= shebang(filename) if read_shebang
+        type ||= type_from_shebang(filename) if read_shebang
 
         type
       end
@@ -63,7 +63,7 @@ def fetch filename, default = nil, read_shebang = false
 
     protected
 
-      def shebang filename
+      def type_from_shebang filename
         return unless File.exist? filename
         File.open filename, 'r' do |f|
           if first_line = f.gets
 
@@ -30,7 +30,7 @@ module PluginHost
     # * a file could not be found
     # * the requested Plugin is not registered
     PluginNotFound = Class.new LoadError
-    HostNotFound = Class.new LoadError
+    HostNotFound   = Class.new LoadError
 
     PLUGIN_HOSTS = []
     PLUGIN_HOSTS_BY_ID = {}  # dummy hash
@@ -49,8 +49,8 @@ def load_all
     def [] id, *args, &blk
       plugin = validate_id(id)
       begin
-        plugin = plugin_hash.[] plugin, *args, &blk
-      end while plugin.is_a? Symbol
+        plugin = plugin_hash.[](plugin, *args, &blk)
+      end while plugin.is_a? String
       plugin
     end
 
@@ -95,7 +95,7 @@ def plugin_path *args
     def map hash
       for from, to in hash
         from = validate_id from
-        to = validate_id to
+        to   = validate_id to
         plugin_hash[from] = to unless plugin_hash.has_key? from
       end
     end
@@ -197,22 +197,22 @@ def path_to plugin_id
       File.join plugin_path, "#{plugin_id}.rb"
     end
 
-    # Converts +id+ to a Symbol if it is a String,
-    # or returns +id+ if it already is a Symbol.
+    # Converts +id+ to a valid plugin ID String, or returns +nil+.
     #
     # Raises +ArgumentError+ for all other objects, or if the
     # given String includes non-alphanumeric characters (\W).
     def validate_id id
-      if id.is_a? Symbol or id.nil?
-        id
-      elsif id.is_a? String
+      case id
+      when Symbol
+        id.to_s
+      when String
         if id[/\w+/] == id
-          id.downcase.to_sym
+          id.downcase
         else
           raise ArgumentError, "Invalid id given: #{id}"
         end
       else
-        raise ArgumentError, "String or Symbol expected, but #{id.class} given."
+        raise ArgumentError, "Symbol or String expected, but #{id.class} given."
       end
     end
 
 
@@ -1,16 +1,23 @@
+require 'set'
+
 module CodeRay
 module Scanners
 
   # = Debug Scanner
   # 
-  # Interprets the output of the Encoders::Debug encoder.
+  # Interprets the output of the Encoders::Debug encoder (basically the inverse function).
   class Debug < Scanner
 
     register_for :debug
     title 'CodeRay Token Dump Import'
 
   protected
 
+    def setup
+      super
+      @known_token_kinds = TokenKinds.keys.map(&:to_s).to_set
+    end
+    
     def scan_tokens encoder, options
 
       opened_tokens = []
@@ -21,16 +28,19 @@ def scan_tokens encoder, options
           encoder.text_token match, :space
 
         elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \)? /x)
-          kind = self[1].to_sym
-          match = self[2].gsub(/\\(.)/m, '\1')
-          unless TokenKinds.has_key? kind
-            kind = :error
-            match = matched
+          if @known_token_kinds.include? self[1]
+            encoder.text_token self[2].gsub(/\\(.)/m, '\1'), self[1].to_sym
+          else
+            encoder.text_token matched, :unknown
           end
-          encoder.text_token match, kind
 
         elsif match = scan(/ (\w+) ([<\[]) /x)
-          kind = self[1].to_sym
+          if @known_token_kinds.include? self[1]
+            kind = self[1].to_sym
+          else
+            kind = :unknown
+          end
+          
           opened_tokens << kind
           case self[2]
           when '<'
 
@@ -1,9 +1,11 @@
+require 'set'
+
 module CodeRay
 module Scanners
 
-  # = Debug Scanner
   # = Raydebug Scanner
   # 
-  # Parses the output of the Encoders::Debug encoder.
+  # Highlights the output of the Encoders::Debug encoder.
   class Raydebug < Scanner
 
     register_for :raydebug
@@ -12,6 +14,11 @@ class Raydebug < Scanner
 
   protected
 
+    def setup
+      super
+      @known_token_kinds = TokenKinds.keys.map(&:to_s).to_set
+    end
+    
     def scan_tokens encoder, options
 
       opened_tokens = []
@@ -26,20 +33,22 @@ def scan_tokens encoder, options
           encoder.text_token kind, :class
           encoder.text_token '(', :operator
           match = self[2]
-          encoder.text_token match, kind.to_sym unless match.empty?
+          unless match.empty?
+            if @known_token_kinds.include? kind
+              encoder.text_token match, kind.to_sym
+            else
+              encoder.text_token match, :plain
+            end
+          end
           encoder.text_token match, :operator if match = scan(/\)/)
 
         elsif match = scan(/ (\w+) ([<\[]) /x)
-          kind = self[1]
-          case self[2]
-          when '<'
-            encoder.text_token kind, :class
-          when '['
-            encoder.text_token kind, :class
+          encoder.text_token self[1], :class
+          if @known_token_kinds.include? self[1]
+            kind = self[1].to_sym
           else
-            raise 'CodeRay bug: This case should not be reached.'
+            kind = :unknown
           end
-          kind = kind.to_sym
           opened_tokens << kind
           encoder.begin_group kind
           encoder.text_token self[2], :operator
 
@@ -157,13 +157,16 @@ module Ruby::Patterns  # :nodoc: all
       yield
     ])
 
-    FANCY_STRING_START = / % ( [QqrsWwx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x
+    FANCY_STRING_START = / % ( [iIqQrswWx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x
     FANCY_STRING_KIND = Hash.new(:string).merge({
+      'i' => :symbol,
+      'I' => :symbol,
       'r' => :regexp,
       's' => :symbol,
       'x' => :shell,
     })
     FANCY_STRING_INTERPRETED = Hash.new(true).merge({
+      'i' => false,
       'q' => false,
       's' => false,
       'w' => false,
 
@@ -16,7 +16,6 @@ class StringState < Struct.new :type, :interpreted, :delim, :heredoc,
 
       STRING_PATTERN = Hash.new do |h, k|
         delim, interpreted = *k
-        # delim = delim.dup  # workaround for old Ruby
         delim_pattern = Regexp.escape(delim)
         if closing_paren = CLOSING_PAREN[delim]
           delim_pattern << Regexp.escape(closing_paren)
@@ -29,12 +28,13 @@ class StringState < Struct.new :type, :interpreted, :delim, :heredoc,
         #     '| [|?*+(){}\[\].^$]'
         #   end
 
-        h[k] =
-          if interpreted && delim != '#'
-            / (?= [#{delim_pattern}] | \# [{$@] ) /mx
-          else
-            / (?= [#{delim_pattern}] ) /mx
-          end
+        if interpreted && delim != '#'
+          / (?= [#{delim_pattern}] | \# [{$@] ) /mx
+        else
+          / (?= [#{delim_pattern}] ) /mx
+        end.tap do |pattern|
+          h[k] = pattern if (delim.respond_to?(:ord) ? delim.ord : delim[0]) < 256
+        end
       end
 
       def initialize kind, interpreted, delim, heredoc = false
 
@@ -125,7 +125,7 @@ class Alpha < Style
 .string .modifier { color: #E40 }
 .symbol { color:#A60 }
 .symbol .content { color:#A60 }
-.symbol .delimiter { color:#630 }
+.symbol .delimiter { color:#740 }
 .tag { color:#070; font-weight:bold }
 .type { color:#339; font-weight:bold }
 .value { color: #088 }
 
@@ -1,10 +1,7 @@
 module CodeRay
 
   # A Hash of all known token kinds and their associated CSS classes.
-  TokenKinds = Hash.new do |h, k|
-    warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG
-    false
-  end
+  TokenKinds = Hash.new(false)
 
   # speedup
   TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity
@@ -83,5 +80,6 @@ module CodeRay
     :plain               => false                 # almost all scanners
   )
 
-  TokenKinds[:method] = TokenKinds[:function]
+  TokenKinds[:method]  = TokenKinds[:function]
+  TokenKinds[:unknown] = TokenKinds[:plain]
 end
@@ -164,9 +164,7 @@ def test_token_kinds
       end
     end
     assert_equal 'reserved', CodeRay::TokenKinds[:reserved]
-    assert_warning 'Undefined Token kind: :shibboleet' do
-      assert_equal false, CodeRay::TokenKinds[:shibboleet]
-    end
+    assert_equal false,      CodeRay::TokenKinds[:shibboleet]
   end
 
   class Milk < CodeRay::Encoders::Encoder