Set generate_format_op as internal function

python · JukkaL · Aug 6, 2021 · Aug 3, 2021 · Aug 4, 2021 · Aug 4, 2021
commit 0ad827fb7cc3a4aaf54860e127140aab20f79156
diff --git a/mypyc/irbuild/format_str_tokenizer.py b/mypyc/irbuild/format_str_tokenizer.py
@@ -25,6 +25,20 @@ class FormatOp(Enum):
     INT = 'd'
 
 
+def generate_format_ops(specifiers: List[ConversionSpecifier]) -> Optional[List[FormatOp]]:
+    format_ops = []
+    for spec in specifiers:
+        # TODO: Match specifiers instead of using whole_seq
+        if spec.whole_seq == '%s' or spec.whole_seq == '{:{}}':
+            format_op = FormatOp.STR
+        elif spec.whole_seq:
+            return None
+        else:
+            format_op = FormatOp.STR
+        format_ops.append(format_op)
+    return format_ops
+
+
 def tokenizer_printf_style(format_str: str) -> Tuple[List[str], List[ConversionSpecifier]]:
     """Tokenize a printf-style format string using regex.
 
@@ -50,14 +64,16 @@ def tokenizer_printf_style(format_str: str) -> Tuple[List[str], List[ConversionS
 
 
 def tokenizer_format_call(
-        format_str: str) -> Optional[Tuple[List[str], List[ConversionSpecifier]]]:
+        format_str: str) -> Optional[Tuple[List[str], List[FormatOp]]]:
     """Tokenize a str.format() format string.
 
     The core function parse_format_value() is shared with mypy.
-    However, this function
+    With these specifiers, we then parse the literal substrings
+    of the original format string and convert `ConversionSpecifier`
+    to `FormatOp`.
 
     Return:
-        A list of string literals and a list of conversion operations.
+        A list of string literals and a list of FormatOps.
         Return None if it cannot parse the string.
     """
     # Creates an empty MessageBuilder here.
@@ -66,6 +82,9 @@ def tokenizer_format_call(
                                     MessageBuilder(Errors(), {}))
     if specifiers is None:
         return None
+    format_ops = generate_format_ops(specifiers)
+    if format_ops is None:
+        return None
 
     literals: List[str] = []
     last_end = 0
@@ -74,24 +93,10 @@ def tokenizer_format_call(
         literals.append(format_str[last_end:spec.start_pos - 1])
         last_end = spec.start_pos + len(spec.whole_seq) + 1
     literals.append(format_str[last_end:])
-
     # Deal with escaped {{
     literals = [x.replace('{{', '{').replace('}}', '}') for x in literals]
-    return literals, specifiers
 
-
-def generate_format_ops(specifiers: List[ConversionSpecifier]) -> Optional[List[FormatOp]]:
-    format_ops = []
-    for spec in specifiers:
-        # TODO: Match specifiers instead of using whole_seq
-        if spec.whole_seq == '%s' or spec.whole_seq == '{:{}}':
-            format_op = FormatOp.STR
-        elif spec.whole_seq:
-            return None
-        else:
-            format_op = FormatOp.STR
-        format_ops.append(format_op)
-    return format_ops
+    return literals, format_ops
 
 
 def convert_expr(builder: IRBuilder, format_ops: List[FormatOp],

diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py
@@ -28,7 +28,7 @@
     bool_rprimitive, c_int_rprimitive, c_pyssize_t_rprimitive, is_dict_rprimitive
 )
 from mypyc.irbuild.format_str_tokenizer import (
-    tokenizer_format_call, join_formatted_strings, generate_format_ops, convert_expr
+    tokenizer_format_call, join_formatted_strings, convert_expr
 )
 from mypyc.primitives.dict_ops import (
     dict_keys_op, dict_values_op, dict_items_op, dict_setdefault_spec_init_op
@@ -397,17 +397,11 @@ def translate_str_format(
         tokens = tokenizer_format_call(format_str)
         if tokens is None:
             return None
-        literals, specifiers = tokens
-
-        format_ops = generate_format_ops(specifiers)
-        if format_ops is None:
-            return None
-
+        literals, format_ops = tokens
         # Convert variables to strings
         substitutions = convert_expr(builder, format_ops, expr.args, expr.line)
         if substitutions is None:
             return None
-
         return join_formatted_strings(builder, literals, substitutions, expr.line)
     return None