forked from RustPython/RustPython
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtokenize.rs
More file actions
32 lines (24 loc) · 938 Bytes
/
tokenize.rs
File metadata and controls
32 lines (24 loc) · 938 Bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
/*
* python tokenize module.
*/
extern crate rustpython_parser;
use std::iter::FromIterator;
use self::rustpython_parser::lexer;
use crate::obj::objstr;
use crate::pyobject::{PyContext, PyFuncArgs, PyObjectRef, PyResult, TypeProtocol};
use crate::VirtualMachine;
fn tokenize_tokenize(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult {
arg_check!(vm, args, required = [(readline, Some(vm.ctx.str_type()))]);
let source = objstr::get_value(readline);
// TODO: implement generator when the time has come.
let lexer1 = lexer::make_tokenizer(&source);
let tokens = lexer1.map(|st| vm.ctx.new_str(format!("{:?}", st.unwrap().1)));
let tokens = Vec::from_iter(tokens);
Ok(vm.ctx.new_list(tokens))
}
// TODO: create main function when called with -m
pub fn mk_module(ctx: &PyContext) -> PyObjectRef {
py_module!(ctx, "tokenize", {
"tokenize" => ctx.new_rustfunc(tokenize_tokenize)
})
}