1
1
module CodeRay module Scanners
2
-
2
+
3
3
#A simple scanner for a simple language: Io
4
-
4
+
5
5
class Io < Scanner
6
6
7
7
register_for :io
8
-
8
+
9
9
RESERVED_WORDS = [ 'clone' , 'init' , 'method' , 'list' , 'vector' , 'block' , 'if' , 'ifTrue' , 'ifFalse' , 'ifTrueIfFalse' , 'then' , 'for' , 'loop' ,
10
10
'reverseForeach' , 'foreach' , 'map' , 'continue' , 'break' , 'while' , 'do' , 'return' ,
11
11
'self' , 'sender' , 'target' , 'proto' , 'parent' , 'protos' ]
12
12
13
13
PREDEFINED_TYPES = [ ]
14
14
15
- PREDEFINED_CONSTANTS = [ 'Object' , 'Lobby' ,
15
+ PREDEFINED_CONSTANTS = [ 'Object' , 'Lobby' ,
16
16
'TRUE' , 'true' , 'FALSE' , 'false' , 'NULL' , 'null' , 'Null' , 'Nil' , 'nil' , 'YES' , 'NO' ]
17
17
18
18
IDENT_KIND = WordList . new ( :ident ) .
19
19
add ( RESERVED_WORDS , :reserved ) .
20
20
add ( PREDEFINED_TYPES , :pre_type ) .
21
21
add ( PREDEFINED_CONSTANTS , :pre_constant )
22
22
23
- ESCAPE = / [rbfnrtv \n \\ '"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
23
+ ESCAPE = / [rbfntv \n \\ '"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
24
24
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
25
25
26
26
def scan_tokens tokens , options
@@ -33,24 +33,24 @@ def scan_tokens tokens, options
33
33
match = nil
34
34
35
35
if state == :initial
36
-
36
+
37
37
if scan ( / \s + | \\ \n /x )
38
38
kind = :space
39
-
39
+
40
40
elsif scan ( %r! // [^\n \\ ]* (?: \\ . [^\n \\ ]* )* | /\* (?: .*? \* / | .* ) !mx )
41
41
kind = :comment
42
42
43
-
43
+
44
44
elsif scan ( / [-+*\/ \$ \@ =<>?:;,!&^|()\[ \] {}~%]+ | \. (?!\d ) /x )
45
45
kind = :operator
46
-
46
+
47
47
elsif match = scan ( / [A-Za-z_][A-Za-z_0-9]* /x )
48
48
kind = IDENT_KIND [ match ]
49
49
if kind == :ident and check ( /:(?!:)/ )
50
50
match << scan ( /:/ )
51
51
kind = :label
52
52
end
53
-
53
+
54
54
elsif match = scan ( /L?"/ )
55
55
tokens << [ :open , :string ]
56
56
if match [ 0 ] == ?L
@@ -59,30 +59,30 @@ def scan_tokens tokens, options
59
59
end
60
60
state = :string
61
61
kind = :delimiter
62
-
62
+
63
63
elsif scan ( /#\s *(\w *)/ )
64
64
kind = :preprocessor # FIXME multiline preprocs
65
65
state = :include_expected if self [ 1 ] == 'include'
66
-
66
+
67
67
elsif scan ( / L?' (?: [^\' \n \\ ] | \\ #{ ESCAPE } )? '? /ox )
68
68
kind = :char
69
-
69
+
70
70
elsif scan ( /0[xX][0-9A-Fa-f]+/ )
71
71
kind = :hex
72
-
72
+
73
73
elsif scan ( /(?:0[0-7]+)(?![89.eEfF])/ )
74
74
kind = :oct
75
-
75
+
76
76
elsif scan ( /(?:\d +)(?![.eEfF])/ )
77
77
kind = :integer
78
-
78
+
79
79
elsif scan ( /\d [fF]?|\d *\. \d +(?:[eE][+-]?\d +)?[fF]?|\d +[eE][+-]?\d +[fF]?/ )
80
80
kind = :float
81
81
82
82
else
83
83
getch
84
84
end
85
-
85
+
86
86
elsif state == :string
87
87
if scan ( /[^\\ "]+/ )
88
88
kind = :content
@@ -99,33 +99,33 @@ def scan_tokens tokens, options
99
99
else
100
100
raise "else case \" reached; %p not handled." % peek ( 1 ) , tokens
101
101
end
102
-
102
+
103
103
elsif state == :include_expected
104
104
if scan ( /<[^>\n ]+>?|"[^"\n \\ ]*(?:\\ .[^"\n \\ ]*)*"?/ )
105
105
kind = :include
106
106
state = :initial
107
-
107
+
108
108
elsif match = scan ( /\s +/ )
109
109
kind = :space
110
110
state = :initial if match . index ?\n
111
-
111
+
112
112
else
113
113
getch
114
-
114
+
115
115
end
116
-
116
+
117
117
else
118
118
raise 'else-case reached' , tokens
119
-
119
+
120
120
end
121
-
121
+
122
122
match ||= matched
123
123
raise [ match , kind ] , tokens if kind == :error
124
124
125
125
tokens << [ match , kind ]
126
-
126
+
127
127
end
128
-
128
+
129
129
tokens
130
130
end
131
131
0 commit comments