@@ -59,7 +59,9 @@ tok_new(void)
59
59
sizeof (struct tok_state ));
60
60
if (tok == NULL )
61
61
return NULL ;
62
- tok -> buf = tok -> cur = tok -> end = tok -> inp = tok -> start = NULL ;
62
+ tok -> buf = tok -> cur = tok -> inp = NULL ;
63
+ tok -> start = NULL ;
64
+ tok -> end = NULL ;
63
65
tok -> done = E_OK ;
64
66
tok -> fp = NULL ;
65
67
tok -> input = NULL ;
@@ -111,7 +113,9 @@ error_ret(struct tok_state *tok) /* XXX */
111
113
tok -> decoding_erred = 1 ;
112
114
if (tok -> fp != NULL && tok -> buf != NULL ) /* see PyTokenizer_Free */
113
115
PyMem_FREE (tok -> buf );
114
- tok -> buf = tok -> cur = tok -> end = tok -> inp = tok -> start = NULL ;
116
+ tok -> buf = tok -> cur = tok -> inp = NULL ;
117
+ tok -> start = NULL ;
118
+ tok -> end = NULL ;
115
119
tok -> done = E_DECODE ;
116
120
return NULL ; /* as if it were EOF */
117
121
}
@@ -664,11 +668,11 @@ translate_newlines(const char *s, int exec_input, struct tok_state *tok) {
664
668
Look for encoding declarations inside STR, and record them
665
669
inside TOK. */
666
670
667
- static const char *
671
+ static char *
668
672
decode_str (const char * input , int single , struct tok_state * tok )
669
673
{
670
674
PyObject * utf8 = NULL ;
671
- const char * str ;
675
+ char * str ;
672
676
const char * s ;
673
677
const char * newl [2 ] = {NULL , NULL };
674
678
int lineno = 0 ;
@@ -726,43 +730,46 @@ struct tok_state *
726
730
PyTokenizer_FromString (const char * str , int exec_input )
727
731
{
728
732
struct tok_state * tok = tok_new ();
733
+ char * decoded ;
734
+
729
735
if (tok == NULL )
730
736
return NULL ;
731
- str = decode_str (str , exec_input , tok );
732
- if (str == NULL ) {
737
+ decoded = decode_str (str , exec_input , tok );
738
+ if (decoded == NULL ) {
733
739
PyTokenizer_Free (tok );
734
740
return NULL ;
735
741
}
736
742
737
- /* XXX: constify members. */
738
- tok -> buf = tok -> cur = tok -> end = tok -> inp = ( char * ) str ;
743
+ tok -> buf = tok -> cur = tok -> inp = decoded ;
744
+ tok -> end = decoded ;
739
745
return tok ;
740
746
}
741
747
742
748
struct tok_state *
743
749
PyTokenizer_FromUTF8 (const char * str , int exec_input )
744
750
{
745
751
struct tok_state * tok = tok_new ();
752
+ char * translated ;
746
753
if (tok == NULL )
747
754
return NULL ;
748
- tok -> input = str = translate_newlines (str , exec_input , tok );
749
- if (str == NULL ) {
755
+ tok -> input = translated = translate_newlines (str , exec_input , tok );
756
+ if (translated == NULL ) {
750
757
PyTokenizer_Free (tok );
751
758
return NULL ;
752
759
}
753
760
tok -> decoding_state = STATE_RAW ;
754
761
tok -> read_coding_spec = 1 ;
755
762
tok -> enc = NULL ;
756
- tok -> str = str ;
763
+ tok -> str = translated ;
757
764
tok -> encoding = (char * )PyMem_MALLOC (6 );
758
765
if (!tok -> encoding ) {
759
766
PyTokenizer_Free (tok );
760
767
return NULL ;
761
768
}
762
769
strcpy (tok -> encoding , "utf-8" );
763
770
764
- /* XXX: constify members. */
765
- tok -> buf = tok -> cur = tok -> end = tok -> inp = ( char * ) str ;
771
+ tok -> buf = tok -> cur = tok -> inp = translated ;
772
+ tok -> end = translated ;
766
773
return tok ;
767
774
}
768
775
@@ -812,7 +819,7 @@ PyTokenizer_Free(struct tok_state *tok)
812
819
if (tok -> fp != NULL && tok -> buf != NULL )
813
820
PyMem_FREE (tok -> buf );
814
821
if (tok -> input )
815
- PyMem_FREE (( char * ) tok -> input );
822
+ PyMem_FREE (tok -> input );
816
823
PyMem_FREE (tok );
817
824
}
818
825
@@ -1138,7 +1145,7 @@ tok_decimal_tail(struct tok_state *tok)
1138
1145
/* Get next token, after space stripping etc. */
1139
1146
1140
1147
static int
1141
- tok_get (struct tok_state * tok , char * * p_start , char * * p_end )
1148
+ tok_get (struct tok_state * tok , const char * * p_start , const char * * p_end )
1142
1149
{
1143
1150
int c ;
1144
1151
int blankline , nonascii ;
@@ -1321,7 +1328,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
1321
1328
&& ((unsigned char )ignore_end [0 ] >= 128 || Py_ISALNUM (ignore_end [0 ]))));
1322
1329
1323
1330
if (is_type_ignore ) {
1324
- * p_start = ( char * ) ignore_end ;
1331
+ * p_start = ignore_end ;
1325
1332
* p_end = tok -> cur ;
1326
1333
1327
1334
/* If this type ignore is the only thing on the line, consume the newline also. */
@@ -1331,7 +1338,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
1331
1338
}
1332
1339
return TYPE_IGNORE ;
1333
1340
} else {
1334
- * p_start = ( char * ) type_start ; /* after type_comment_prefix */
1341
+ * p_start = type_start ; /* after type_comment_prefix */
1335
1342
* p_end = tok -> cur ;
1336
1343
return TYPE_COMMENT ;
1337
1344
}
@@ -1410,7 +1417,8 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
1410
1417
Look ahead one token to see if that is 'def'. */
1411
1418
1412
1419
struct tok_state ahead_tok ;
1413
- char * ahead_tok_start = NULL , * ahead_tok_end = NULL ;
1420
+ const char * ahead_tok_start = NULL ;
1421
+ const char * ahead_tok_end = NULL ;
1414
1422
int ahead_tok_kind ;
1415
1423
1416
1424
memcpy (& ahead_tok , tok , sizeof (ahead_tok ));
@@ -1798,7 +1806,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
1798
1806
}
1799
1807
1800
1808
int
1801
- PyTokenizer_Get (struct tok_state * tok , char * * p_start , char * * p_end )
1809
+ PyTokenizer_Get (struct tok_state * tok , const char * * p_start , const char * * p_end )
1802
1810
{
1803
1811
int result = tok_get (tok , p_start , p_end );
1804
1812
if (tok -> decoding_erred ) {
@@ -1823,7 +1831,9 @@ PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
1823
1831
{
1824
1832
struct tok_state * tok ;
1825
1833
FILE * fp ;
1826
- char * p_start = NULL , * p_end = NULL , * encoding = NULL ;
1834
+ const char * p_start = NULL ;
1835
+ const char * p_end = NULL ;
1836
+ char * encoding = NULL ;
1827
1837
1828
1838
fd = _Py_dup (fd );
1829
1839
if (fd < 0 ) {
0 commit comments