8000 Add tokenizer_data_gen to build process · php/php-src@b424cd6 · GitHub
[go: up one dir, main page]

Skip to content

Commit b424cd6

Browse files
committed
Add tokenizer_data_gen to build process
1 parent dda0cea commit b424cd6

File tree

4 files changed

+97
-80
lines changed

4 files changed

+97
-80
lines changed

ext/tokenizer/Makefile.frag

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
$(top_srcdir)/Zend/zend_language_parser.c:
22
$(top_srcdir)/Zend/zend_language_scanner.c:
3-
$(top_srcdir)/ext/tokenizer/tokenizer_data.c: $(top_srcdir)/Zend/zend_language_parser.h
3+
$(top_srcdir)/ext/tokenizer/tokenizer_data.c: $(top_srcdir)/Zend/zend_language_parser.y $(top_srcdir)/Zend/zend_language_parser.h
4+
if test ! -z "$(PHP_EXECUTABLE)" && test -x "$(PHP_EXECUTABLE)"; then \
5+
$(PHP_EXECUTABLE) $(srcdir)/tokenizer_data_gen.php; \
6+
fi;
47
$(builddir)/tokenizer.lo: $(top_srcdir)/Zend/zend_language_parser.c $(top_srcdir)/Zend/zend_language_scanner.c

ext/tokenizer/tokenizer_data.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
/*
1818
DO NOT EDIT THIS FILE!
19-
This file is generated using tokenizer_data_gen.sh
19+
This file is generated using tokenizer_data_gen.php
2020
*/
2121

2222
#include "php.h"

ext/tokenizer/tokenizer_data_gen.php

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
<?php
2+
3+
$infile = 'Zend/zend_language_parser.h';
4+
$outfile = 'ext/tokenizer/tokenizer_data.c';
5+
6+
if (!file_exists($infile)) {
7+
fwrite(STDERR, <<<ERROR
8+
$infile is missing.
9+
10+
Please, generate the PHP parser files by scripts/dev/genfiles
11+
or by running the ./configure build step.
12+
ERROR);
13+
exit(1);
14+
}
15+
16+
$result = '';
17+
18+
$result .= <<<CODE
19+
/*
20+
+----------------------------------------------------------------------+
21+
| Copyright (c) The PHP Group |
22+
+----------------------------------------------------------------------+
23+
| This source file is subject to version 3.01 of the PHP license, |
24+
| that is bundled with this package in the file LICENSE, and is |
25+
| available through the world-wide-web at the following url: |
26+
| http://www.php.net/license/3_01.txt |
27+
| If you did not receive a copy of the PHP license and are unable to |
28+
| obtain it through the world-wide-web, please send a note to |
29+
| license@php.net so we can mail you a copy immediately. |
30+
+----------------------------------------------------------------------+
31+
| Author: Johannes Schlueter <johannes@php.net> |
32+
+----------------------------------------------------------------------+
33+
*/
34+
35+
/*
36+
DO NOT EDIT THIS FILE!
37+
This file is generated using tokenizer_data_gen.php
38+
*/
39+
40+
#include "php.h"
41+
#include "zend.h"
42+
#include <zend_language_parser.h>
43+
44+
45+
void tokenizer_register_constants(INIT_FUNC_ARGS) {
46+
47+
CODE;
48+
49+
$incontent = file_get_contents($infile);
50+
preg_match_all('(^ (?<token_name>T_.*?)\b)m', $incontent, $matches);
51+
52+
foreach ($matches['token_name'] as $tokenName) {
53+
if ($tokenName === 'T_NOELSE' || $tokenName === 'T_ERROR') {
54+
continue;
55+
}
56+
$result .= "\tREGISTER_LONG_CONSTANT(\"$tokenName\", $tokenName, CONST_CS | CONST_PERSISTENT);\n";
57+
}
58+
$result .= "\tREGISTER_LONG_CONSTANT(\"T_DOUBLE_COLON\", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT);\n";
59+
60+
$result .= <<<CODE
61+
}
62+
63+
char *get_token_type_name(int token_type)
64+
{
65+
\tswitch (token_type) {
66+
67+
68+
CODE;
69+
70+
foreach ($matches['token_name'] as $tokenName) {
71+
if ($tokenName === 'T_NOELSE' || $tokenName === 'T_ERROR') {
72+
continue;
73+
}
74+
if ($tokenName === 'T_PAAMAYIM_NEKUDOTAYIM') {
75+
$result .= "\t\tcase T_PAAMAYIM_NEKUDOTAYIM: return \"T_DOUBLE_COLON\";\n";
76+
} else {
77+
$result .= "\t\tcase $tokenName: return \"$tokenName\";\n";
78+
}
79+
}
80+
81+
$result .= <<<CODE
82+
83+
\t}
84+
\treturn NULL;
85+
}
86+
87+
88+
CODE;
89+
90+
file_put_contents($outfile, $result);
91+
92+
echo "Wrote $outfile\n";

ext/tokenizer/tokenizer_data_gen.sh

Lines changed: 0 additions & 78 deletions
This file was deleted.

0 commit comments

Comments
 (0)
0