8000 Fix trailing newline issue #17 · rmliddle/RustPython@e922a6a · GitHub
[go: up one dir, main page]
Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit e922a6a

Browse files
windelbouwmanwindelbouwman
committed
Fix trailing newline issue RustPython#17 
1 parent 8c199ee commit e922a6a

File tree

7 files changed

+82
-80
lines changed

7 files changed

+82
-80
lines changed

‎parser/src/lexer.rs

Lines changed: 71 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -16,34 +16,45 @@ use unic_emoji_char::is_emoji_presentation;
1616
use unicode_xid::UnicodeXID;
1717
use wtf8;
1818

19-
#[derive(Clone, Copy, PartialEq, Debug)]
19+
#[derive(Clone, Copy, PartialEq, Debug, Default)]
2020
struct IndentationLevel {
2121
tabs: usize,
2222
spaces: usize,
2323
}
2424

2525
impl IndentationLevel {
26-
fn new() -> IndentationLevel {
27-
IndentationLevel { tabs: 0, spaces: 0 }
28-
}
29-
fn compare_strict(&self, other: &IndentationLevel) -> Option<Ordering> {
26+
fn compare_strict(
27+
&self,
28+
other: &IndentationLevel,
29+
location: Location,
30+
) -> Result<Ordering, LexicalError> {
3031
// We only know for sure that we're smaller or bigger if tabs
3132
// and spaces both differ in the same direction. Otherwise we're
3233
// dependent on the size of tabs.
3334
if self.tabs < other.tabs {
3435
if self.spaces <= other.spaces {
35-
Some(Ordering::Less)
36+
Ok(Ordering::Less)
3637
} else {
37-
None
38+
Err(LexicalError {
39+
location,
40+
error: LexicalErrorType::OtherError(
41+
"inconsistent use of tabs and spaces in indentation".to_string(),
42+
),
43+
})
3844
}
3945
} else if self.tabs > other.tabs {
4046
if self.spaces >= other.spaces {
41-
Some(Ordering::Greater)
47+
Ok(Ordering::Greater)
4248
} else {
43-
None
49+
Err(LexicalError {
50+
location,
51+
error: LexicalErrorType::OtherError(
52+
"inconsistent use of tabs and spaces in indentation".to_string(),
53+
),
54+
})
4455
}
4556
} else {
46-
Some(self.spaces.cmp(&other.spaces))
57+
Ok(self.spaces.cmp(&other.spaces))
4758
}
4859
}
4960
}
@@ -234,7 +245,7 @@ where
234245
chars: input,
235246
at_begin_of_line: true,
236247
nesting: 0,
237-
indentation_stack: vec![IndentationLevel::new()],
248+
indentation_stack: vec![Default::default()],
238249
pending: Vec::new(),
239250
chr0: None,
240251
location: Location::new(0, 0),
@@ -704,65 +715,49 @@ where
704715

705716
if self.nesting == 0 {
706717
// Determine indent or dedent:
707-
let current_indentation = *self.indentation_stack.last().unwrap();
708-
let ordering = indentation_level.compare_strict(&current_indentation);
718+
let current_indentation = self.indentation_stack.last().unwrap();
719+
let ordering = indentation_level.compare_strict(current_indentation, self.get_pos())?;
709720
match ordering {
710-
Some(Ordering::Equal) => {
721+
Ordering::Equal => {
711722
// Same same
712723
}
713-
Some(Ordering::Greater) => {
724+
Ordering::Greater => {
714725
// New indentation level:
715726
self.indentation_stack.push(indentation_level);
716727
let tok_start = self.get_pos();
717728
let tok_end = tok_start.clone();
718729
self.emit((tok_start, Tok::Indent, tok_end));
719730
}
720-
Some(Ordering::Less) => {
731+
Ordering::Less => {
721732
// One or more dedentations
722733
// Pop off other levels until col is found:
723734

724735
loop {
736+
let current_indentation = self.indentation_stack.last().unwrap();
725737
let ordering = indentation_level
726-
.compare_strict(self.indentation_stack.last().unwrap());
738+
.compare_strict(current_indentation, self.get_pos())?;
727739
match ordering {
728-
Some(Ordering::Less) => {
740+
Ordering::Less => {
729741
self.indentation_stack.pop();
730742
let tok_start = self.get_pos();
731743
let tok_end = tok_start.clone();
732744
self.emit((tok_start, Tok::Dedent, tok_end));
733745
}
734-
None => {
746+
Ordering::Equal => {
747+
// We arrived at proper level of indentation.
748+
break;
749+
}
750+
Ordering::Greater => {
751+
// TODO: handle wrong indentations
735752
return Err(LexicalError {
736753
error: LexicalErrorType::OtherError(
737-
"inconsistent use of tabs and spaces in indentation"
738-
.to_string(),
754+
"Non matching indentation levels!".to_string(),
739755
),
740756
location: self.get_pos(),
741757
});
742758
}
743-
_ => {
744-
break;
745-
}
746759
};
747760
}
748-
749-
if indentation_level != *self.indentation_stack.last().unwrap() {
750-
// TODO: handle wrong indentations
751-
return Err(LexicalError {
752-
error: LexicalErrorType::OtherError(
753-
"Non matching indentation levels!".to_string(),
754-
),
755-
location: self.get_pos(),
756-
});
757-
}
758-
}
759-
None => {
760-
return Err(LexicalError {
761-
error: LexicalErrorType::OtherError(
762-
"inconsistent use of tabs and spaces in indentation".to_string(),
763-
),
764-
location: self.get_pos(),
765-
});
766761
}
767762
}
768763
}
@@ -804,18 +799,18 @@ where
804799
});
805800
}
806801

807-
/*
808-
// Next, insert a trailing newline, if required.
809-
if !self.at_begin_of_line {
810-
self.emit((tok_pos.clone(), Tok::Newline, tok_pos.clone()));
811-
}
802+
// Next, insert a trailing newline, if required.
803+
if !self.at_begin_of_line {
804+
self.at_begin_of_line = true;
805+
self.emit((tok_pos.clone(), Tok::Newline, tok_pos.clone()));
806+
}
807+
808+
// Next, flush the indentation stack to zero.
809+
while self.indentation_stack.len() > 1 {
810+
self.indentation_stack.pop();
811+
self.emit((tok_pos.clone(), Tok::Dedent, tok_pos.clone()));
812+
}
812813

813-
// Next, flush the indentation stack to zero.
814-
while !self.indentation_stack.is_empty() {
815-
self.indentation_stack.pop();
816-
self.emit((tok_pos.clone(), Tok::Dedent, tok_pos.clone()));
817-
}
818-
*/
819814
self.emit((tok_pos.clone(), Tok::EndOfFile, tok_pos));
820815
}
821816

@@ -1325,7 +1320,8 @@ mod tests {
13251320
Tok::String {
13261321
value: "\\".to_string(),
13271322
is_fstring: false,
1328-
}
1323+
},
1324+
Tok::Newline,
13291325
]
13301326
);
13311327
}
@@ -1358,6 +1354,7 @@ mod tests {
13581354
real: 0.0,
13591355
imag: 2.2,
13601356
},
1357+
Tok::Newline,
13611358
]
13621359
);
13631360
}
@@ -1369,7 +1366,7 @@ mod tests {
13691366
fn $name() {
13701367
let source = String::from(format!(r"99232 # {}", $eol));
13711368
let tokens = lex_source(&source);
1372-
assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }]);
1369+
assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Newline]);
13731370
}
13741371
)*
13751372
}
@@ -1395,6 +1392,7 @@ mod tests {
13951392
Tok::Int { value: BigInt::from(123) },
13961393
Tok::Newline,
13971394
Tok::Int { value: BigInt::from(456) },
1395+
Tok::Newline,
13981396
]
13991397
)
14001398
}
@@ -1430,6 +1428,7 @@ mod tests {
14301428
Tok::Int {
14311429
value: BigInt::from(0)
14321430
},
1431+
Tok::Newline,
14331432
]
14341433
);
14351434
}
@@ -1457,6 +1456,7 @@ mod tests {
14571456
Tok::Int { value: BigInt::from(99) },
14581457
Tok::Newline,
14591458
Tok::Dedent,
1459+
Tok::Newline,
14601460
]
14611461
);
14621462
}
@@ -1501,6 +1501,7 @@ mod tests {
15011501
Tok::Newline,
15021502
Tok::Dedent,
15031503
Tok::Dedent,
1504+
Tok::Newline,
15041505
]
15051506
);
15061507
}
@@ -1539,6 +1540,7 @@ mod tests {
15391540
Tok::Newline,
15401541
Tok::Dedent,
15411542
Tok::Dedent,
1543+
Tok::Newline,
15421544
]
15431545
);
15441546
}
@@ -1578,6 +1580,7 @@ mod tests {
15781580
Tok::Int { value: BigInt::from(2) },
15791581
Tok::Rsqb,
15801582
Tok::Newline,
1583+
Tok::Newline,
15811584
]
15821585
);
15831586
}
@@ -1603,6 +1606,7 @@ mod tests {
16031606
Tok::DoubleSlashEqual,
16041607
Tok::Slash,
16051608
Tok::Slash,
1609+
Tok::Newline,
16061610
]
16071611
);
16081612
}
@@ -1642,6 +1646,7 @@ mod tests {
16421646
value: String::from("raw\'"),
16431647
is_fstring: false,
16441648
},
1649+
Tok::Newline,
16451650
]
16461651
);
16471652
}
@@ -1660,6 +1665,7 @@ mod tests {
16601665
value: String::from("abcdef"),
16611666
is_fstring: false,
16621667
},
1668+
Tok::Newline,
16631669
]
16641670
)
16651671
}
@@ -1674,26 +1680,32 @@ mod tests {
16741680
}
16751681

16761682
#[test]
1677-
fn test_byte() {
1683+
fn test_single_quoted_byte() {
16781684
// single quote
16791685
let all = r##"b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff'"##;
16801686
let source = String::from(all);
16811687
let tokens = lex_source(&source);
16821688
let res = (0..=255).collect::<Vec<u8>>();
1683-
assert_eq!(tokens, vec![Tok::Bytes { value: res }]);
1689+
assert_eq!(tokens, vec![Tok::Bytes { value: res }, Tok::Newline]);
1690+
}
16841691

1692+
#[test]
1693+
fn test_double_quoted_byte() {
16851694
// double quote
16861695
let all = r##"b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff""##;
16871696
let source = String::from(all);
16881697
let tokens = lex_source(&source);
16891698
let res = (0..=255).collect::<Vec<u8>>();
1690-
assert_eq!(tokens, vec![Tok::Bytes { value: res }]);
1699+
assert_eq!(tokens, vec![Tok::Bytes { value: res }, Tok::Newline]);
1700+
}
16911701

1702+
#[test]
1703+
fn test_escape_char_in_byte_literal() {
16921704
// backslash doesnt escape
16931705
let all = r##"b"omkmok\Xaa""##;
16941706
let source = String::from(all);
16951707
let tokens = lex_source(&source);
16961708
let res = vec![111, 109, 107, 109, 111, 107, 92, 88, 97, 97];
1697-
assert_eq!(tokens, vec![Tok::Bytes { value: res }]);
1709+
assert_eq!(tokens, vec![Tok::Bytes { value: res }, Tok::Newline]);
16981710
}
16991711
}

‎parser/src/parser.rs

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ mod tests {
127127

128128
#[test]
129129
fn test_parse_print_hello() {
130-
let source = String::from("print('Hello world')\n");
130+
let source = String::from("print('Hello world')");
131131
let parse_ast = parse_program(&source).unwrap();
132132
assert_eq!(
133133
parse_ast,
@@ -151,7 +151,7 @@ mod tests {
151151

152152
#[test]
153153
fn test_parse_print_2() {
154-
let source = String::from("print('Hello world', 2)\n");
154+
let source = String::from("print('Hello world', 2)");
155155
let parse_ast = parse_program(&source).unwrap();
156156
assert_eq!(
157157
parse_ast,
@@ -175,7 +175,7 @@ mod tests {
175175

176176
#[test]
177177
fn test_parse_kwargs() {
178-
let source = String::from("my_func('positional', keyword=2)\n");
178+
let source = String::from("my_func('positional', keyword=2)");
179179
let parse_ast = parse_program(&source).unwrap();
180180
assert_eq!(
181181
parse_ast,
@@ -202,7 +202,7 @@ mod tests {
202202

203203
#[test]
204204
fn test_parse_if_elif_else() {
205-
let source = String::from("if 1: 10\nelif 2: 20\nelse: 30\n");
205+
let source = String::from("if 1: 10\nelif 2: 20\nelse: 30");
206206
let parse_ast = parse_statement(&source).unwrap();
207207
assert_eq!(
208208
parse_ast,
@@ -226,7 +226,7 @@ mod tests {
226226

227227
#[test]
228228
fn test_parse_lambda() {
229-
let source = String::from("lambda x, y: x * y\n"); // lambda(x, y): x * y");
229+
let source = String::from("lambda x, y: x * y"); // lambda(x, y): x * y");
230230
let parse_ast = parse_statement(&source);
231231
assert_eq!(
232232
parse_ast,
@@ -265,7 +265,7 @@ mod tests {
265265

266266
#[test]
267267
fn test_parse_tuples() {
268-
let source = String::from("a, b = 4, 5\n");
268+
let source = String::from("a, b = 4, 5");
269269

270270
assert_eq!(
271271
parse_statement(&source),
@@ -292,7 +292,7 @@ mod tests {
292292
#[test]
293293
fn test_parse_class() {
294294
let source = String::from(
295-
"class Foo(A, B):\n def __init__(self):\n pass\n def method_with_default(self, arg='default'):\n pass\n",
295+
"class Foo(A, B):\n def __init__(self):\n pass\n def method_with_default(self, arg='default'):\n pass",
296296
);
297297
assert_eq!(
298298
parse_statement(&source),

‎parser/src/python.lalrpop

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ grammar;
2121
pub Top: ast::Top = {
2222
StartProgram <p:Program> => ast::Top::Program(p),
2323
StartStatement <s:Statement> => ast::Top::Statement(s),
24-
StartExpression <e:Test> => ast::Top::Expression(e),
24+
StartExpression <e:Test> ("\n")* => ast::Top::Expression(e),
2525
};
2626

2727
Program: ast::Program = {

‎src/main.rs

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -298,11 +298,9 @@ fn handle_exception<T>(vm: &VirtualMachine, result: PyResult<T>) {
298298
}
299299
}
300300

301-
fn run_command(vm: &VirtualMachine, mut source: String) -> PyResult<()> {
301+
fn run_command(vm: &VirtualMachine, source: String) -> PyResult<()> {
302302
debug!("Running command {}", source);
303303

304-
// This works around https://github.com/RustPython/RustPython/issues/17
305-
source.push('\n');
306304
_run_string(vm, &source, "<stdin>".to_string())?;
307305
Ok(())
308306
}

0 commit comments

Comments
 (0)
0