8000 Add tokenizer_data_gen to build process by iluuu1994 · Pull Request #6723 · php/php-src · GitHub
[go: up one dir, main page]
Skip to content

Add tokenizer_data_gen to build process #6723

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Add tokenizer_data_gen to build process #6723

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion ext/tokenizer/Makefile.frag
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
$(top_srcdir)/Zend/zend_language_parser.c:
$(top_srcdir)/Zend/zend_language_scanner.c:
$(top_srcdir)/ext/tokenizer/tokenizer_data.c: $(top_srcdir)/Zend/zend_language_parser.h
$(top_srcdir)/ext/tokenizer/tokenizer_data.c: $(top_srcdir)/Zend/zend_language_parser.y $(top_srcdir)/Zend/zend_language_parser.h
if test ! -z "$(PHP_EXECUTABLE)" && test -x "$(PHP_EXECUTABLE)"; then \
$(PHP_EXECUTABLE) $(srcdir)/tokenizer_data_gen.php; \
fi;
$(builddir)/tokenizer.lo: $(top_srcdir)/Zend/zend_language_parser.c $(top_srcdir)/Zend/zend_language_scanner.c
2 changes: 1 addition & 1 deletion ext/tokenizer/tokenizer_data.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

/*
DO NOT EDIT THIS FILE!
This file is generated using tokenizer_data_gen.sh
This file is generated using tokenizer_data_gen.php
*/

#include "php.h"
Expand Down
92 changes: 92 additions & 0 deletions ext/tokenizer/tokenizer_data_gen.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
<?php

$infile = 'Zend/zend_language_parser.h';
$outfile = 'ext/tokenizer/tokenizer_data.c';

if (!file_exists($infile)) {
fwrite(STDERR, <<<ERROR
$infile is missing.

Please, generate the PHP parser files by scripts/dev/genfiles
or by running the ./configure build step.
ERROR);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove indentation here to keep this compatible with PHP < 7.3. We should probably bump our PHP version requirement, but this heredoc block probably isn't the right reason to do it :)

exit(1);
}

$result = '';

$result .= <<<CODE
/*
+----------------------------------------------------------------------+
| Copyright (c) The PHP Group |
+----------------------------------------------------------------------+
| This source file is subject to version 3.01 of the PHP license, |
| that is bundled with this package in the file LICENSE, and is |
| available through the world-wide-web at the following url: |
| http://www.php.net/license/3_01.txt |
| If you did not receive a copy of the PHP license and are unable to |
| obtain it through the world-wide-web, please send a note to |
| license@php.net so we can mail you a copy immediately. |
+----------------------------------------------------------------------+
| Author: Johannes Schlueter <johannes@php.net> |
+----------------------------------------------------------------------+
*/

/*
DO NOT EDIT THIS FILE!
This file is generated using tokenizer_data_gen.php
*/

#include "php.h"
#include "zend.h"
#include <zend_language_parser.h>


void tokenizer_register_constants(INIT_FUNC_ARGS) {

CODE;

$incontent = file_get_contents($infile);
preg_match_all('(^ (?<token_name>T_.*?)\b)m', $incontent, $matches);

foreach ($matches['token_name'] as $tokenName) {
if ($tokenName === 'T_NOELSE' || $tokenName === 'T_ERROR') {
continue;
}
$result .= "\tREGISTER_LONG_CONSTANT(\"$tokenName\", $tokenName, CONST_CS | CONST_PERSISTENT);\n&qu 8000 ot;;
}
$result .= "\tREGISTER_LONG_CONSTANT(\"T_DOUBLE_COLON\", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT);\n";

$result .= <<<CODE
}

char *get_token_type_name(int token_type)
{
\tswitch (token_type) {


CODE;

foreach ($matches['token_name'] as $tokenName) {
if ($tokenName === 'T_NOELSE' || $tokenName === 'T_ERROR') {
continue;
}
if ($tokenName === 'T_PAAMAYIM_NEKUDOTAYIM') {
$result .= "\t\tcase T_PAAMAYIM_NEKUDOTAYIM: return \"T_DOUBLE_COLON\";\n";
} else {
$result .= "\t\tcase $tokenName: return \"$tokenName\";\n";
}
}

$result .= <<<CODE

\t}
\treturn NULL;
}


CODE;

file_put_contents($outfile, $result);

echo "Wrote $outfile\n";
78 changes: 0 additions & 78 deletions ext/tokenizer/tokenizer_data_gen.sh
View file
Open in desktop

This file was deleted.

0