py/lexer: Add static assert that token enum values all fit in a byte.

dpgeorge · dpgeorge · commit 96007e7de558 · 2024-07-18T12:44:44.000+10:00
Signed-off-by: Damien George &lt;damien@micropython.org&gt;
diff --git a/py/lexer.c b/py/lexer.c
@@ -228,7 +228,6 @@ static const char *const tok_enc =
     "=e="         // = ==
     "!.";         // start of special cases: != . ...
 
-// TODO static assert that number of tokens is less than 256 so we can safely make this table with byte sized entries
 static const uint8_t tok_enc_kind[] = {
     MP_TOKEN_DEL_PAREN_OPEN, MP_TOKEN_DEL_PAREN_CLOSE,
     MP_TOKEN_DEL_BRACKET_OPEN, MP_TOKEN_DEL_BRACKET_CLOSE,
@@ -774,6 +773,9 @@ void mp_lexer_to_next(mp_lexer_t *lex) {
     } else {
         // search for encoded delimiter or operator
 
+        // assert that the token enum value fits in a byte, so they all fit in tok_enc_kind
+        MP_STATIC_ASSERT(MP_TOKEN_NUMBER_OF <= 256);
+
         const char *t = tok_enc;
         size_t tok_enc_index = 0;
         for (; *t != 0 && !is_char(lex, *t); t += 1) {
diff --git a/py/lexer.h b/py/lexer.h
@@ -152,6 +152,8 @@ typedef enum _mp_token_kind_t {
     MP_TOKEN_DEL_SEMICOLON,
     MP_TOKEN_DEL_EQUAL,
     MP_TOKEN_DEL_MINUS_MORE,
+
+    MP_TOKEN_NUMBER_OF,
 } mp_token_kind_t;
 
 // this data structure is exposed for efficiency
