8000 Merge branch 'master' into upstream · tricknotes/coderay@9651f0b · GitHub
[go: up one dir, main page]
Skip to content

Commit 9651f0b

Browse files
kornykorny
committed
Merge branch 'master' into upstream
2 parents 14f3a4f + d7f1669 commit 9651f0b

File tree

13 files changed

+112
-52
lines changed

13 files changed

+112
-52
lines changed

‎Changes.textile

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ h2. Changes in 1.1
1212
* Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126]
1313
* Remove double-click toggle handler from HTML table output
1414
* Fixes to CSS scanner (floats, pseudoclasses)
15+
* Fixed empty tokens and unclosed token groups in HTML, CSS, Diff, Goovy, PHP, Raydebug, Ruby, SQL, and YAML scanners [#144]
16+
* Added @:keep_state@ functionality to more scanners [#116]
1517
* CSS scanner uses @:id@ and @:tag@ now [#27]
1618
* Removed @Tokens#dump@, @Tokens.load@, @Tokens::Undumping@, and @zlib@ dependency. Nobody was using this, right?
1719
* Add .xaml file type [#121, thanks to Kozman Bálint]
@@ -20,10 +22,13 @@ h2. Changes in 1.1
2022
* New token type @:done@ for Taskpaper [#39]
2123
* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and nathany]
2224
* Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom]
23-
* Override Bootstrap's pre word-break setting for line numbers [#102, thanks to lightswitch05]
25+
* Override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05]
2426
* Fixed @:docstring@ token type style
2527
* @Plugin@ does not warn about fallback when default is defined
28+
* @HTML@ encoder will not warn about unclosed token groups at the end of the stream
2629
* @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now
30+
* @Debug@ encoder will not warn about errors in the token stream
31+
* New @DebugLint@ encoder that checks for empty tokens and correct nesting
2732

2833
h2. Changes in 1.0.9
2934

‎lib/coderay/encoders/debug_lint.rb

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,6 @@ class DebugLint < Debug
1919
EmptyToken = Class.new InvalidTokenStream
2020
IncorrectTokenGroupNesting = Class.new InvalidTokenStream
2121

22-
def initialize options = {}
23-
super
24-
@opened = []
25-
end
26-
2722
def text_token text, kind
2823
raise EmptyToken, 'empty token' if text.empty?
2924
super
@@ -35,7 +30,8 @@ def begin_group kind
3530
end
3631

3732
def end_group kind
38-
raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_group)" if @opened.pop != kind
33+
raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
34+
@opened.pop
3935
super
4036
end
4137

@@ -45,7 +41,20 @@ def begin_line kind
4541
end
4642

4743
def end_line kind
48-
raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_line)" if @opened.pop != kind
44+
raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
45+
@opened.pop
46+
super
47+
end
48+
49+
protected
50+
51+
def setup options
52+
super
53+
@opened = []
54+
end
55+
56+
def finish options
57+
raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty?
4958
super
5059
end
5160

‎lib/coderay/encoders/html.rb

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,6 @@ def setup options
193193

194194
def finish options
195195
unless @opened.empty?
196-
warn '%d tokens still open: %p' % [@opened.size, @opened] if $CODERAY_DEBUG
197196
@out << '</span>' while @opened.pop
198197
@last_opened = nil
199198
end

‎lib/coderay/scanners/css.rb

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -145,10 +145,10 @@ def scan_tokens encoder, options
145145
start = match[/^\w+\(/]
146146
encoder.text_token start, :delimiter
147147
if match[-1] == ?)
148-
encoder.text_token match[start.size..-2], :content
148+
encoder.text_token match[start.size..-2], :content if match.size > start.size + 1
149149
encoder.text_token ')', :delimiter
150150
else
151-
encoder.text_token match[start.size..-1], :content
151+
encoder.text_token match[start.size..-1], :content if match.size > start.size
152152
end
153153
encoder.end_group :function
154154

‎lib/coderay/scanners/diff.rb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ def scan_tokens encoder, options
6969
state = :added
7070
elsif match = scan(/\\ .*/)
7171
encoder.text_token match, :comment
72-
elsif match = scan(/@@(?>[^@\n]*)@@/)
72+
elsif match = scan(/@@(?>[^@\n]+)@@/)
7373
content_scanner.state = :initial unless match?(/\n\+/)
7474
content_scanner_entry_state = nil
7575
if check(/\n|$/)

‎lib/coderay/scanners/groovy.rb

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,12 @@ class Groovy < Java
3636

3737
protected
3838

39+
def setup
40+
@state = :initial
41+
end
42+
3943
def scan_tokens encoder, options
40-
41-
state = :initial
44+
state = options[:state] || @state
4245
inline_block_stack = []
4346
inline_block_paren_depth = nil
4447
string_delimiter = nil
@@ -223,7 +226,7 @@ def scan_tokens encoder, options
223226
encoder.text_token match, :content # TODO: Shouldn't this be :error?
224227

225228
elsif match = scan(/ \\ | \n /x)
226-
encoder.end_group state
229+
encoder.end_group state == :regexp ? :regexp : :string
227230
encoder.text_token match, :error
228231
after_def = value_expected = false
229232
state = :initial
@@ -243,7 +246,17 @@ def scan_tokens encoder, options
243246
end
244247

245248
if [:multiline_string, :string, :regexp].include? state
246-
encoder.end_group state
249+
encoder.end_group state == :regexp ? :regexp : :string
250+
end
251+
252+
if options[:keep_state]
253+
@state = state
254+
end
255+
256+
until inline_block_stack.empty?
257+
state, = *inline_block_stack.pop
258+
encoder.end_group :inline
259+
encoder.end_group state == :regexp ? :regexp : :string
247260
end
248261

249262
encoder

‎lib/coderay/scanners/lua.rb

Lines changed: 21 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,15 @@ def setup
5959
# CodeRay entry hook. Starts parsing.
6060
def scan_tokens(encoder, options)
6161
state = options[:state] || @state
62+
brace_depth = @brace_depth
63+
num_equals = nil
6264

6365
until eos?
6466
case state
6567

6668
when :initial
6769
if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]]
68-
@num_equals = match.count("=") # Number must match for comment end
70+
num_equals = match.count("=") # Number must match for comment end
6971
encoder.begin_group(:comment)
7072
encoder.text_token(match, :delimiter)
7173
state = :long_comment
@@ -74,7 +76,7 @@ def scan_tokens(encoder, options)
7476
encoder.text_token(match, :comment)
7577

7678
elsif match = scan(/\[=*\[/) # [[ long (possibly multiline) string ]]
77-
@num_equals = match.count("=") # Number must match for comment end
79+
num_equals = match.count("=") # Number must match for comment end
7880
encoder.begin_group(:string)
7981
encoder.text_token(match, :delimiter)
8082
state = :long_string
@@ -101,19 +103,19 @@ def scan_tokens(encoder, options)
101103

102104
elsif match = scan(/\{/) # Opening table brace {
103105
encoder.begin_group(:map)
104-
encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter)
105-
@brace_depth += 1
106+
encoder.text_token(match, brace_depth >= 1 ? :inline_delimiter : :delimiter)
107+
brace_depth += 1
106108
state = :map
107109

108110
elsif match = scan(/\}/) # Closing table brace }
109-
if @brace_depth == 1
110-
@brace_depth = 0
111+
if brace_depth == 1
112+
brace_depth = 0
111113
encoder.text_token(match, :delimiter)
112114
encoder.end_group(:map)
113-
elsif @brace_depth == 0 # Mismatched brace
115+
elsif brace_depth == 0 # Mismatched brace
114116
encoder.text_token(match, :error)
115117
else
116-
@brace_depth -= 1
118+
brace_depth -= 1
117119
encoder.text_token(match, :inline_delimiter)
118120
encoder.end_group(:map)
119121
state = :map
@@ -122,7 +124,7 @@ def scan_tokens(encoder, options)
122124
elsif match = scan(/["']/) # String delimiters " and '
123125
encoder.begin_group(:string)
124126
encoder.text_token(match, :delimiter)
125-
@start_delim = match
127+
start_delim = match
126128
state = :string
127129

128130
# ↓Prefix hex number ←|→ decimal number
@@ -146,7 +148,7 @@ def scan_tokens(encoder, options)
146148
# It may be that we’re scanning a full-blown subexpression of a table
147149
# (tables can contain full expressions in parts).
148150
# If this is the case, return to :map scanning state.
149-
state = :map if state == :initial && @brace_depth >= 1
151+
state = :map if state == :initial && brace_depth >= 1
150152

151153
when :function_expected
152154
if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name
@@ -198,10 +200,10 @@ def scan_tokens(encoder, options)
198200
end
199201

200202
when :long_comment
201-
if match = scan(/.*?(?=\]={#@num_equals}\])/m)
203+
if match = scan(/.*?(?=\]={#{num_equals}}\])/m)
202204
encoder.text_token(match, :content)
203205

204-
delim = scan(/\]={#@num_equals}\]/)
206+
delim = scan(/\]={#{num_equals}}\]/)
205207
encoder.text_token(delim, :delimiter)
206208
else # No terminator found till EOF
207209
encoder.text_token(rest, :error)
@@ -211,10 +213,10 @@ def scan_tokens(encoder, options)
211213
state = :initial
212214

213215
when :long_string
214-
if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences
216+
if match = scan(/.*?(?=\]={#{num_equals}}\])/m) # Long strings do not interpret any escape sequences
215217
encoder.text_token(match, :content)
216218

217-
delim = scan(/\]={#@num_equals}\]/)
219+
delim = scan(/\]={#{num_equals}}\]/)
218220
encoder.text_token(delim, :delimiter)
219221
else # No terminator found till EOF
220222
encoder.text_token(rest, :error)
@@ -224,11 +226,11 @@ def scan_tokens(encoder, options)
224226
state = :initial
225227

226228
when :string
227-
if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z)
229+
if match = scan(/[^\\#{start_delim}\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z)
228230
encoder.text_token(match, :content)
229231
elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m)
230232
encoder.text_token(match, :char)
231-
elsif match = scan(Regexp.compile(@start_delim))
233+
elsif match = scan(Regexp.compile(start_delim))
232234
encoder.text_token(match, :delimiter)
233235
encoder.end_group(:string)
234236
state = :initial
@@ -266,6 +268,9 @@ def scan_tokens(encoder, options)
266268
@state = state
267269
end
268270

271+
encoder.end_group :string if [:string].include? state
272+
brace_depth.times { encoder.end_group :map }
273+
269274
encoder
270275
end
271276

‎lib/coderay/scanners/php.rb

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,7 @@ def scan_tokens encoder, options
265265
@html_scanner.tokenize match unless match.empty?
266266
end
267267

268-
when :php
268+
when :php, :php_inline
269269
if match = scan(/\s+/)
270270
encoder.text_token match, :space
271271

@@ -332,14 +332,15 @@ def scan_tokens encoder, options
332332
if states.size == 1
333333
encoder.text_token match, :error
334334
else
335-
states.pop
335+
state = states.pop
336336
if states.last.is_a?(::Array)
337337
delimiter = states.last[1]
338338
states[-1] = states.last[0]
339339
encoder.text_token match, :delimiter
340340
encoder.end_group :inline
341341
else
342342
encoder.text_token match, :operator
343+
encoder.end_group :inline if state == :php_inline
343344
label_expected = true
344345
end
345346
end
@@ -350,7 +351,14 @@ def scan_tokens encoder, options
350351

351352
elsif match = scan(RE::PHP_END)
352353
encoder.text_token match, :inline_delimiter
353-
states = [:initial]
354+
while state = states.pop
355+
encoder.end_group :string if [:sqstring, :dqstring].include? state
356+
if state.is_a? Array
357+
encoder.end_group :inline
358+
encoder.end_group :string if [:sqstring, :dqstring].include? state.first
359+
end
360+
end
361+
states << :initial
354362

355363
elsif match = scan(/<<<(?:(#{RE::IDENTIFIER})|"(#{RE::IDENTIFIER})"|'(#{RE::IDENTIFIER})')/o)
356364
encoder.begin_group :string
@@ -400,6 +408,7 @@ def scan_tokens encoder, options
400408
elsif match = scan(/\\/)
401409
encoder.text_token match, :error
402410
else
411+
encoder.end_group :string
403412
states.pop
404413
end
405414

@@ -459,7 +468,7 @@ def scan_tokens encoder, options
459468
encoder.begin_group :inline
460469
states[-1] = [states.last, delimiter]
461470
delimiter = nil
462-
states.push :php
471+
states.push :php_inline
463472
encoder.text_token match, :delimiter
464473
else
465474
encoder.text_token match, :content
@@ -469,6 +478,7 @@ def scan_tokens encoder, options
469478
elsif match = scan(/\$/)
470479
encoder.text_token match, :content
471480
else
481+
encoder.end_group :string
472482
states.pop
473483
end
474484

@@ -500,6 +510,14 @@ 10000 def scan_tokens encoder, options
500510

501511
end
502512

513+
while state = states.pop
514+
encoder.end_group :string if [:sqstring, :dqstring].include? state
515+
if state.is_a? Array
516+
encoder.end_group :inline
517+
encoder.end_group :string if [:sqstring, :dqstring].include? state.first
518+
end
519+
end
520+
503521
encoder
504522
end
505523

‎lib/coderay/scanners/raydebug.rb

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,23 @@
11
module CodeRay
22
module Scanners
3-
3+
44
# = Debug Scanner
55
#
66
# Parses the output of the Encoders::Debug encoder.
77
class Raydebug < Scanner
8-
8+
99
register_for :raydebug
1010
file_extension 'raydebug'
1111
title 'CodeRay Token Dump'
1212

1313
protected
1414

1515
def scan_tokens encoder, options
16-
16+
1717
opened_tokens = []
18-
18+
1919
until eos?
20-
20+
2121
if match = scan(/\s+/)
2222
encoder.text_token match, :space
2323

@@ -26,7 +26,7 @@ def scan_tokens encoder, options
2626
encoder.text_token kind, :class
2727
encoder.text_token '(', :operator
2828
match = self[2]
29-
encoder.text_token match, kind.to_sym
29+
encoder.text_token match, kind.to_sym unless match.empty?
3030
encoder.text_token match, :operator if match = scan(/\)/)
3131

3232
elsif match = scan(/ (\w+) ([<\[]) /x)
@@ -59,8 +59,8 @@ def scan_tokens encoder, options
5959

6060
encoder
6161
end
62-
62+
6363
end
64-
64+
6565
end
6666
end

0 commit comments

Comments
 (0)
0