8000 Update lexing and parsing · rune-rs/rune@67446ee · GitHub
[go: up one dir, main page]
Skip to content

Commit 67446ee

Browse files
udoprogudoprog
committed
Update lexing and parsing
1 parent 1428e6e commit 67446ee

File tree

8 files changed

+65
-16
lines changed

8 files changed

+65
-16
lines changed

‎assets/tokens.yaml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,7 @@
394394
variant: Whitespace
395395
doc: whitespace.
396396
# after this point comes high-level tokens used in grammar
397-
- {kind: "syntax", variant: "Root", doc: "the root node of a parse tree"}
397+
- {kind: "syntax", variant: "Root", doc: "a syntax root"}
398398
- {kind: "syntax", variant: "Local", doc: "a variable declaration"}
399399
- {kind: "syntax", variant: "Item", doc: "an item declaration"}
400400
- {kind: "syntax", variant: "ItemEnum", doc: "an enum declaration"}
@@ -464,6 +464,7 @@
464464
- {kind: "syntax", variant: "ClosureArguments", doc: "closure arguments"}
465465
- {kind: "syntax", variant: "AnonymousObjectKey", doc: "an `#{` anonymous object key"}
466466
- {kind: "syntax", variant: "Attribute", doc: "an attribute"}
467+
- {kind: "syntax", variant: "InnerAttribute", doc: "an inner attribute"}
467468
- {kind: "syntax", variant: "Modifiers", doc: "item modifiers, like `pub const`"}
468469
- {kind: "syntax", variant: "ModifierCrate", doc: "the `(crate)` modifier"}
469470
- {kind: "syntax", variant: "ModifierIn", doc: "the `(in <path>)` modifier"}

‎crates/rune/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ serde = { version = "1.0.163", default-features = false, features = ["derive", "
4848
musli = { version = "0.0.121", default-features = false, features = ["alloc"] }
4949
once_cell = { version = "1.18.0", default-features = false, features = ["critical-section"] }
5050

51-
syntree = { version = "0.17.0", optional = true }
51+
syntree = { version = "0.17.2", optional = true }
5252
anyhow = { version = "1.0.71", default-features = false, optional = true }
5353
bincode = { version = "1.3.3", optional = true }
5454
clap = { version = "4.2.7", features = ["derive"], optional = true }

‎crates/rune/src/ast/generated.rs

Lines changed: 5 additions & 5 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/rune/src/compile/options.rs

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,8 @@ pub struct Options {
112112
pub(crate) test_std: bool,
113113
/// Enable lowering optimizations.
114114
pub(crate) lowering: u8,
115+
/// Print source tree.
116+
pub(crate) print_tree: bool,
115117
/// Rune format options.
116118
pub(crate) fmt: FmtOptions,
117119
}
@@ -127,6 +129,7 @@ impl Options {
127129
function_body: false,
128130
test_std: false,
129131
lowering: 0,
132+
print_tree: false,
130133
fmt: FmtOptions::DEFAULT,
131134
};
132135

@@ -220,6 +223,16 @@ impl Options {
220223
default: "0",
221224
options: "0-3",
222225
},
226+
OptionMeta {
227+
key: "print-tree",
228+
unstable: false,
229+
doc: &docstring! {
230+
/// Print the parsed source tree when formatting to
231+
/// standard output.
232+
},
233+
default: "false",
234+
options: BOOL,
235+
},
223236
OptionMeta {
224237
key: "fmt.print-tree",
225238
unstable: false,
@@ -302,6 +315,9 @@ impl Options {
302315
}
303316
};
304317
}
318+
"print-tree" => {
319+
self.print_tree = tail.map_or(true, |s| s == "true");
320+
}
305321
other => {
306322
let Some((head, tail)) = other.split_once('.') else {
307323
return Err(ParseOptionError {

‎crates/rune/src/fmt/format.rs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,15 @@ fn expr_labels<'a>(fmt: &mut Formatter<'a>, p: &mut Stream<'a>) -> Result<()> {
6363
Ok(())
6464
}
6565

66+
fn inner_attributes<'a>(fmt: &mut Formatter<'a>, p: &mut Stream<'a>) -> Result<()> {
67+
while let Some(attr) = p.try_pump(InnerAttribute)? {
68+
attr.fmt(fmt)?;
69+
fmt.nl(1)?;
70+
}
71+
72+
Ok(())
73+
}
74+
6675
fn attributes<'a>(fmt: &mut Formatter<'a>, p: &mut Stream<'a>) -> Result<Attrs> {
6776
let mut attrs = Attrs::default();
6877

@@ -1611,6 +1620,8 @@ enum StmtKind {
16111620

16121621
/// The contents of a block.
16131622
fn block_content<'a>(fmt: &mut Formatter<'a>, p: &mut Stream<'a>) -> Result<()> {
1623+
inner_attributes(fmt, p)?;
1624+
16141625
let mut last_kind = StmtKind::None;
16151626

16161627
while !p.is_eof() {

‎crates/rune/src/grammar/grammar.rs

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,8 @@ fn stmt(p: &mut Parser<'_>) -> Result<()> {
120120
}
121121
}
122122

123+
10000 inner_attributes(p)?;
124+
123125
let c = p.checkpoint()?;
124126
attributes(p)?;
125127
let m = modifiers(p)?;
@@ -187,8 +189,8 @@ fn item_const(p: &mut Parser<'_>) -> Result<()> {
187189
}
188190

189191
#[tracing::instrument(skip_all)]
190-
fn attributes(p: &mut Parser<'_>) -> Result<()> {
191-
while matches!((p.peek()?, p.glued(1)?), (K![#], K![!]) | (K![#], K!['['])) {
192+
fn inner_attributes(p: &mut Parser<'_>) -> Result<()> {
193+
while matches!((p.peek()?, p.glued(1)?), (K![#], K![!])) {
192194
let c = p.checkpoint()?;
193195

194196
p.bump()?;
@@ -199,6 +201,24 @@ fn attributes(p: &mut Parser<'_>) -> Result<()> {
199201
p.bump()?;
200202
}
201203

204+
p.close_at(&c, InnerAttribute)?;
205+
}
206+
207+
Ok(())
208+
}
209+
210+
#[tracing::instrument(skip_all)]
211+
fn attributes(p: &mut Parser<'_>) -> Result<()> {
212+
while matches!((p.peek()?, p.glued(1)?), (K![#], K!['['])) {
213+
let c = p.checkpoint()?;
214+
215+
p.bump()?;
216+
217+
if p.bump_if(K!['['])? {
218+
token_stream(p, brackets)?;
219+
p.bump()?;
220+
}
221+
202222
p.close_at(&c, Attribute)?;
203223
}
204224

‎crates/rune/src/grammar/parser.rs

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -95,9 +95,9 @@ impl<'a> Parser<'a> {
9595
pub(super) fn bump(&mut self) -> Result<Token> {
9696
let tok = self.next()?;
9797

98-
self.tree
99-
.token(tok.kind, tok.span.range().len())
100-
.with_span(tok.span)?;
98+
let span = syntree::Span::new(tok.span.start.0, tok.span.end.0);
99+
100+
self.tree.token_with(tok.kind, span).with_span(tok.span)?;
101101

102102
Ok(tok)
103103
}
@@ -143,9 +143,11 @@ impl<'a> Parser<'a> {
143143
pub(super) fn push(&mut self, kind: Kind) -> Result<()> {
144144
let tok = self.next()?;
145145
self.tree.open(kind).with_span(tok.span)?;
146-
self.tree
147-
.token(tok.kind, tok.span.range().len())
148-
.with_span(tok.span)?;
146+
147+
let span = syntree::Span::new(tok.span.start.0, tok.span.end.0);
148+
149+
self.tree.token_with(tok.kind, span).with_span(tok.span)?;
150+
149151
self.tree.close().with_span(tok.span)?;
150152
Ok(())
151153
}

‎crates/rune/src/tests.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ pub(crate) mod prelude {
2727
};
2828
pub(crate) use futures_executor::block_on;
2929

30-
pub(crate) use ::rust_alloc::borrow::ToOwned;
3130
pub(crate) use ::rust_alloc::boxed::Box;
3231
pub(crate) use ::rust_alloc::string::{String, ToString};
3332
pub(crate) use ::rust_alloc::sync::Arc;

0 commit comments

Comments
 (0)
0