8000 implement linetable · RustPython/RustPython@7f683ca · GitHub
[go: up one dir, main page]
Skip to content

Commit 7f683ca

Browse files
youknowoneyouknowone
committed
implement linetable
1 parent 4a4fb2f commit 7f683ca

File tree

5 files changed

+614
-21
lines changed

5 files changed

+614
-21
lines changed

‎Lib/test/test_code.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -431,6 +431,7 @@ def test_co_positions_artificial_instructions(self):
431431
]
432432
)
433433

434+
# TODO: RUSTPYTHON
434435
@unittest.expectedFailure
435436
def test_endline_and_columntable_none_when_no_debug_ranges(self):
436437
# Make sure that if `-X no_debug_ranges` is used, there is
@@ -447,6 +448,7 @@ def f():
447448
""")
448449
assert_python_ok('-X', 'no_debug_ranges', '-c', code)
449450

451+
# TODO: RUSTPYTHON
450452
@unittest.expectedFailure
451453
def test_endline_and_columntable_none_when_no_debug_ranges_env(self):
452454
# Same as above but using the environment variable opt out.

‎compiler/codegen/src/ir.rs‎

Lines changed: 140 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use rustpython_compiler_core::{
55
OneIndexed, SourceLocation,
66
bytecode::{
77
CodeFlags, CodeObject, CodeUnit, ConstantData, InstrDisplayContext, Instruction, Label,
8-
OpArg,
8+
OpArg, PyCodeLocationInfoKind,
99
},
1010
};
1111

@@ -72,6 +72,7 @@ pub struct InstructionInfo {
7272
pub target: BlockIdx,
7373
// pub range: TextRange,
7474
pub location: SourceLocation,
75+
// TODO: end_location for debug ranges
7576
}
7677

7778
// spell-checker:ignore petgraph
@@ -199,6 +200,9 @@ impl CodeInfo {
199200
locations.clear()
200201
}
201202

203+
// Generate linetable from locations
204+
let linetable = generate_linetable(&locations, first_line_number.get() as i32);
205+
202206
Ok(CodeObject {
203207
flags,
204208
posonlyarg_count,
@@ -218,6 +222,8 @@ impl CodeInfo {
218222
cellvars: cellvar_cache.into_iter().collect(),
219223
freevars: freevar_cache.into_iter().collect(),
220224
cell2arg,
225+
linetable,
226+
exceptiontable: Box::new([]), // TODO: Generate actual exception table
221227
})
222228
}
223229

@@ -388,3 +394,136 @@ fn iter_blocks(blocks: &[Block]) -> impl Iterator<Item = (BlockIdx, &Block)> + '
388394
Some((idx, b))
389395
})
390396
}
397+
398+
/// Generate CPython 3.11+ format linetable from source locations
399+
fn generate_linetable(locations: &[SourceLocation], first_line: i32) -> Box<[u8]> {
400+
if locations.is_empty() {
401+
return Box::new([]);
402+
}
403+
404+
let mut linetable = Vec::new();
405+
// Initialize prev_line to first_line
406+
// The first entry's delta is relative to co_firstlineno
407+
let mut prev_line = first_line;
408+
let mut i = 0;
409+
410+
while i < locations.len() {
411+
let loc = &locations[i];
412+
413+
// Count consecutive instructions with the same location
414+
let mut length = 1;
415+
while i + length < locations.len() && locations[i + length] == locations[i] {
416+
length += 1;
417+
}
418+
419+
// Process in chunks of up to 8 instructions
420+
while length > 0 {
421+
let entry_length = length.min(8);
422+
423+
// Get line and column information
424+
// SourceLocation always has row and column (both are OneIndexed)
425+
let line = loc.row.get() as i32;
426+
let col = (loc.column.get() as i32) - 1; // Convert 1-based to 0-based
427+
let column = Some(col);
428+
429+
let line_delta = line - prev_line;
430+
431+
// Choose the appropriate encoding based on line delta and column info
432+
// Note: SourceLocation always has valid column, so we never get NO_COLUMNS case
433+
if line_delta == 0 {
434+
let col = column.unwrap();
435+
let end_col = col; // Use same column for end (no range info available)
436+
437+
if col < 80 && end_col - col < 16 && end_col >= col {
438+
// Short form (codes 0-9) for common cases
439+
let code = (col / 8).min(9) as u8; // Short0 to Short9
440+
linetable.push(0x80 | (code << 3) | ((entry_length - 1) as u8));
441+
let col_byte = (((col % 8) as u8) << 4) | ((end_col - col) as u8 & 0xf);
442+
linetable.push(col_byte);
443+
} else if col < 128 && end_col < 128 {
444+
// One-line form (code 10) for same line
445+
linetable.push(
446+
0x80 | ((PyCodeLocationInfoKind::OneLine0 as u8) << 3)
447+
| ((entry_length - 1) as u8),
448+
);
449+
linetable.push(col as u8);
450+
linetable.push(end_col as u8);
451+
} else {
452+
// Long form for columns >= 128
453+
linetable.push(
454+
0x80 | ((PyCodeLocationInfoKind::Long as u8) << 3)
455+
| ((entry_length - 1) as u8),
456+
);
457+
write_signed_varint(&mut linetable, 0); // line_delta = 0
458+
write_varint(&mut linetable, 0); // end_line delta = 0
459+
write_varint(&mut linetable, (col as u32) + 1); // column + 1 for encoding
460+
write_varint(&mut linetable, (end_col as u32) + 1); // end_col + 1
461+
}
462+
} else if line_delta > 0 && line_delta < 3 && column.is_some() {
463+
// One-line form (codes 11-12) for line deltas 1-2
464+
let col = column.unwrap();
465+
let end_col = col; // Use same column for end
466+
467+
if col < 128 && end_col < 128 {
468+
let code = (PyCodeLocationInfoKind::OneLine0 as u8) + (line_delta as u8); // 11 for delta=1, 12 for delta=2
469+
linetable.push(0x80 | (code << 3) | ((entry_length - 1) as u8));
470+
linetable.push(col as u8);
471+
linetable.push(end_col as u8);
472+
} else {
473+
// Long form for columns >= 128 or negative line delta
474+
linetable.push(
475+
0x80 | ((PyCodeLocationInfoKind::Long as u8) << 3)
476+
| ((entry_length - 1) as u8),
477+
);
478+
write_signed_varint(&mut linetable, line_delta);
479+
write_varint(&mut linetable, 0); // end_line delta = 0
480+
write_varint(&mut linetable, (col as u32) + 1); // column + 1 for encoding
481+
write_varint(&mut linetable, (end_col as u32) + 1); // end_col + 1
482+
}
483+
} else {
484+
// Long form (code 14) for all other cases
485+
// This handles: line_delta < 0, line_delta >= 3, or columns >= 128
486+
let col = column.unwrap_or(0);
487+
let end_col = col; // Use same column for end
488+
linetable.push(
489+
0x80 | ((PyCodeLocationInfoKind::Long as u8) << 3) | ((entry_length - 1) as u8),
490+
);
491+
write_signed_varint(&mut linetable, line_delta);
492+
write_varint(&mut linetable, 0); // end_line delta = 0
493+
write_varint(&mut linetable, (col as u32) + 1); // column + 1 for encoding
494+
write_varint(&mut linetable, (end_col as u32) + 1); // end_col + 1
495+
}
496+
497+
prev_line = line;
498+
length -= entry_length;
499+
i += entry_length;
500+
}
501+
}
502+
503+
linetable.into_boxed_slice()
504+
}
505+
506+
/// Write a variable-length unsigned integer (6-bit chunks)
507+
/// Returns the number of bytes written
508+
fn write_varint(buf: &mut Vec<u8>, mut val: u32) -> usize {
509+
let start_len = buf.len();
510+
while val >= 64 {
511+
buf.push(0x40 | (val & 0x3f) as u8);
512+
val >>= 6;
513+
}
514+
buf.push(val as u8);
515+
buf.len() - start_len
516+
}
517+
518+
/// Write a variable-length signed integer
519+
/// Returns the number of bytes written
520+
fn write_signed_varint(buf: &mut Vec<u8>, val: i32) -> usize {
521+
let uval = if val < 0 {
522+
// (unsigned int)(-val) has an undefined behavior for INT_MIN
523+
// So we use (0 - val as u32) to handle it correctly
524+
((0u32.wrapping_sub(val as u32)) << 1) | 1
525+
} else {
526+
(val as u32) << 1
527+
};
528+
write_varint(buf, uval)
529+
}

‎compiler/core/src/bytecode.rs‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,75 @@ pub enum ResumeType {
3333
AfterAwait = 3,
3434
}
3535

36+
/// CPython 3.11+ linetable location info codes
37+
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
38+
#[repr(u8)]
39+
pub enum PyCodeLocationInfoKind {
40+
// Short forms are 0 to 9
41+
Short0 = 0,
42+
Short1 = 1,
43+
Short2 = 2,
44+
Short3 = 3,
45+
Short4 = 4,
46+
Short5 = 5,
47+
Short6 = 6,
48+
Short7 = 7,
49+
Short8 = 8,
50+
Short9 = 9,
51+
// One line forms are 10 to 12
52+
OneLine0 = 10,
53+
OneLine1 = 11,
54+
OneLine2 = 12,
55+
NoColumns = 13,
56+
Long = 14,
57+
None = 15,
58+
}
59+
60+
impl PyCodeLocationInfoKind {
61+
pub fn from_code(code: u8) -> Option<Self> {
62+
match code {
63+
0 => Some(Self::Short0),
64+
1 => Some(Self::Short1),
65+
2 => Some(Self::Short2),
66+
3 => Some(Self::Short3),
67+
4 => Some(Self::Short4),
68+
5 => Some(Self::Short5),
69+
6 => Some(Self::Short6),
70+
7 => Some(Self::Short7),
71+
8 => Some(Self::Short8),
72+
9 => Some(Self::Short9),
73+
10 => Some(Self::OneLine0),
74+
11 => Some(Self::OneLine1),
75+
12 => Some(Self::OneLine2),
76+
13 => Some(Self::NoColumns),
77+
14 => Some(Self::Long),
78+
15 => Some(Self::None),
79+
_ => Option::None,
80+
}
81+
}
82+
83+
pub fn is_short(&self) -> bool {
84+
(*self as u8) <= 9
85+
}
86+
87+
pub fn short_column_group(&self) -> Option<u8> {
88+
if self.is_short() {
89+
Some(*self as u8)
90+
} else {
91+
Option::None
92+
}
93+
}
94+
95+
pub fn one_line_delta(&self) -> Option<i32> {
96+
match self {
97+
Self::OneLine0 => Some(0),
98+
Self::OneLine1 => Some(1),
99+
Self::OneLine2 => Some(2),
100+
_ => Option::None,
101+
}
102+
}
103+
}
104+
36105
pub trait Constant: Sized {
37106
type Name: AsRef<str>;
38107

@@ -146,6 +215,10 @@ pub struct CodeObject<C: Constant = ConstantData> {
146215
pub varnames: Box<[C::Name]>,
147216
pub cellvars: Box<[C::Name]>,
148217
pub freevars: Box<[C::Name]>,
218+
pub linetable: Box<[u8]>,
219+
// Line number table (CPython 3.11+ format)
220+
pub exceptiontable: Box<[u8]>,
221+
// Exception handling table
149222
}
150223

151224
bitflags! {
@@ -1202,6 +1275,8 @@ impl<C: Constant> CodeObject<C> {
12021275
first_line_number: self.first_line_number,
12031276
max_stackdepth: self.max_stackdepth,
12041277
cell2arg: self.cell2arg,
1278+
linetable: self.linetable,
1279+
exceptiontable: self.exceptiontable,
12051280
}
12061281
}
12071282

@@ -1232,6 +1307,8 @@ impl<C: Constant> CodeObject<C> {
12321307
first_line_number: self.first_line_number,
12331308
max_stackdepth: self.max_stackdepth,
12341309
cell2arg: self.cell2arg.clone(),
1310+
linetable: self.linetable.clone(),
1311+
exceptiontable: self.exceptiontable.clone(),
12351312
}
12361313
}
12371314
}

‎compiler/core/src/marshal.rs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,13 @@ pub fn deserialize_code<R: Read, Bag: ConstantBag>(
251251
let cellvars = read_names()?;
252252
let freevars = read_names()?;
253253

254+
// Read linetable and exceptiontable
255+
let linetable_len = rdr.read_u32()?;
256+
let linetable = rdr.read_slice(linetable_len)?.to_vec().into_boxed_slice();
257+
258+
let exceptiontable_len = rdr.read_u32()?;
259+
let exceptiontable = rdr.read_slice(exceptiontable_len)?.to_vec().into_boxed_slice();
260+
254261
Ok(CodeObject {
255262
instructions,
256263
locations,
@@ -269,6 +276,8 @@ pub fn deserialize_code<R: Read, Bag: ConstantBag>(
269276
varnames,
270277
cellvars,
271278
freevars,
279+
linetable,
280+
exceptiontable,
272281
})
273282
}
274283

@@ -684,4 +693,8 @@ pub fn serialize_code<W: Write, C: Constant>(buf: &mut W, code: &CodeObject<C>)
684693
write_names(&code.varnames);
685694
write_names(&code.cellvars);
686695
write_names(&code.freevars);
696+
697+
// Serialize linetable and exceptiontable
698+
write_vec(buf, &code.linetable);
699+
write_vec(buf, &code.exceptiontable);
687700
}

0 commit comments

Comments
 (0)
0