upgrade_pythoncapi.py

#!/usr/bin/env python3
import argparse
import os
import re
import urllib.request
import sys


MIN_PYTHON = (2, 7)


PYTHONCAPI_COMPAT_URL = ('https://raw.githubusercontent.com/python/'
                         'pythoncapi-compat/main/pythoncapi_compat.h')
PYTHONCAPI_COMPAT_H = 'pythoncapi_compat.h'
INCLUDE_PYTHONCAPI_COMPAT = f'#include "{PYTHONCAPI_COMPAT_H}"'
INCLUDE_PYTHONCAPI_COMPAT2 = f'#include <{PYTHONCAPI_COMPAT_H}>'

C_FILE_EXT = (
    # C language
    ".c", ".h",
    # C++ language
    ".cc", ".cpp", ".cxx", ".hpp",
)
IGNORE_DIRS = (".git", ".tox")


# Match spaces but not newline characters.
# Similar to \s but exclude newline characters and only look for ASCII spaces
SPACE_REGEX = r'[ \t\f\v]'
# Match the end of a line: newline characters of a single line
NEWLINE_REGEX = r'(?:\n|\r|\r\n)'
# Match the indentation at the beginning of a line
INDENTATION_REGEX = fr'^{SPACE_REGEX}*'


# Match a C identifier: 'identifier', 'var_3', 'NameCamelCase', '_var'
# Use \b to only match a full word: match "a_b", but not just "b" in "a_b".
ID_REGEX = r'\b[a-zA-Z_][a-zA-Z0-9_]*\b'
# Match 'array[3]'
SUBEXPR_REGEX = fr'{ID_REGEX}(?:\[[^]]+\])*'
# Match a C expression like "frame", "frame.attr", "obj->attr" or "*obj".
# Don't match functions calls like "func()".
EXPR_REGEX = (fr"\*?"  # "*" prefix
              fr"{SUBEXPR_REGEX}"  # "var"
              fr"(?:(?:->|\.){SUBEXPR_REGEX})*")  # "->attr" or ".attr"

# # Match 'PyObject *var' and 'struct MyStruct* var'
TYPE_PTR_REGEX = fr'{ID_REGEX} *\*'

# Match '(PyObject*)' and nothing
OPT_CAST_REGEX = fr'(?:\({TYPE_PTR_REGEX} *\){SPACE_REGEX}*)?'


def same_indentation(group):
    # the regex must have re.MULTILINE flag
    return fr'{SPACE_REGEX}*(?:{NEWLINE_REGEX}{group})?'


def get_member_regex_str(member):
    # Match "var->member".
    return fr'\b({EXPR_REGEX}) *-> *{member}\b'


def get_member_regex(member):
    # Match "var->member" (get).
    # Don't match "var->member = value" (set).
    # Don't match "Py_CLEAR(var->member)".
    # Only "Py_CLEAR(" exact string is excluded.
    regex = (r'(?<!Py_CLEAR\()'
             + get_member_regex_str(member)
             + r'(?!\s*=\s*)')
    return re.compile(regex)


def assign_regex_str(var, expr):
    # Match "var = expr;".
    return fr'{var} *= *{expr}\s*;'


def set_member_regex(member):
    # Match "var->member = expr;".
    regex = assign_regex_str(get_member_regex_str(member), r'([^=].*)')
    return re.compile(regex)


def call_assign_regex(name):
    # Match "Py_TYPE(expr) = expr;".
    # Don't match "assert(Py_TYPE(expr) == expr);".
    # Tolerate spaces
    regex = fr'{name} *\( *(.+) *\) *= *([^=].*) *;'
    return re.compile(regex)


def is_c_filename(filename):
    return filename.endswith(C_FILE_EXT)


class Operation:
    NAME = "<name>"
    REPLACE = ()
    NEED_PYTHONCAPI_COMPAT = False

    def __init__(self, patcher):
        self.patcher = patcher

    def patch(self, content):
        old_content = content
        for regex, replace in self.REPLACE:
            content = regex.sub(replace, content)
        if content != old_content and self.NEED_PYTHONCAPI_COMPAT:
            content = self.patcher.add_pythoncapi_compat(content)
        return content


class Py_TYPE(Operation):
    NAME = "Py_TYPE"
    REPLACE = (
        (get_member_regex('ob_type'), r'Py_TYPE(\1)'),
    )
    # Py_TYPE() was added to Python 2.6.


class Py_SIZE(Operation):
    NAME = "Py_SIZE"
    REPLACE = (
        (get_member_regex('ob_size'), r'Py_SIZE(\1)'),
    )
    # Py_SIZE() was added to Python 2.6.


class Py_REFCNT(Operation):
    NAME = "Py_REFCNT"
    REPLACE = (
        (get_member_regex('ob_refcnt'), r'Py_REFCNT(\1)'),
    )
    # Py_REFCNT() was added to Python 2.6.


class Py_SET_TYPE(Operation):
    NAME = "Py_SET_TYPE"
    REPLACE = (
        (call_assign_regex('Py_TYPE'), r'Py_SET_TYPE(\1, \2);'),
        (set_member_regex('ob_type'), r'Py_SET_TYPE(\1, \2);'),
    )
    # Need Py_SET_TYPE(): new in Python 3.9.
    NEED_PYTHONCAPI_COMPAT = (MIN_PYTHON < (3, 9))


class Py_SET_SIZE(Operation):
    NAME = "Py_SET_SIZE"
    REPLACE = (
        (call_assign_regex('Py_SIZE'), r'Py_SET_SIZE(\1, \2);'),
        (set_member_regex('ob_size'), r'Py_SET_SIZE(\1, \2);'),
    )
    # Need Py_SET_SIZE(): new in Python 3.9.
    NEED_PYTHONCAPI_COMPAT = (MIN_PYTHON < (3, 9))


class Py_SET_REFCNT(Operation):
    NAME = "Py_SET_REFCNT"
    REPLACE = (
        (call_assign_regex('Py_REFCNT'), r'Py_SET_REFCNT(\1, \2);'),
        (set_member_regex('ob_refcnt'), r'Py_SET_REFCNT(\1, \2);'),
    )
    # Need Py_SET_REFCNT(): new in Python 3.9.
    NEED_PYTHONCAPI_COMPAT = (MIN_PYTHON < (3, 9))


class PyObject_NEW(Operation):
    NAME = "PyObject_NEW"
    # In Python 3.9, the PyObject_NEW() macro becomes an alias to the
    # PyObject_New() macro, and the PyObject_NEW_VAR() macro becomes an alias
    # to the PyObject_NewVar() macro.
    REPLACE = (
        (re.compile(r"\bPyObject_NEW\b( *\()"), r'PyObject_New\1'),
        (re.compile(r"\bPyObject_NEW_VAR\b( *\()"), r'PyObject_NewVar\1'),
    )


class PyMem_MALLOC(Operation):
    NAME = "PyMem_MALLOC"
    # In Python 3.9, the PyObject_NEW() macro becomes an alias to the
    # PyObject_New() macro, and the PyObject_NEW_VAR() macro becomes an alias
    # to the PyObject_NewVar() macro.

    REPLACE = (
        (re.compile(r"\bPyMem_MALLOC\b( *\()"), r'PyMem_Malloc\1'),
        (re.compile(r"\bPyMem_REALLOC\b( *\()"), r'PyMem_Realloc\1'),
        (re.compile(r"\bPyMem_FREE\b( *\()"), r'PyMem_Free\1'),
        (re.compile(r"\bPyMem_Del\b( *\()"), r'PyMem_Free\1'),
        (re.compile(r"\bPyMem_DEL\b( *\()"), r'PyMem_Free\1'),
    )


class PyObject_MALLOC(Operation):
    NAME = "PyObject_MALLOC"
    # In Python 3.9, the PyObject_NEW() macro becomes an alias to the
    # PyObject_New() macro, and the PyObject_NEW_VAR() macro becomes an alias
    # to the PyObject_NewVar() macro.

    REPLACE = (
        (re.compile(r"\bPyObject_MALLOC\b( *\()"), r'PyObject_Malloc\1'),
        (re.compile(r"\bPyObject_REALLOC\b( *\()"), r'PyObject_Realloc\1'),
        (re.compile(r"\bPyObject_FREE\b( *\()"), r'PyObject_Free\1'),
        (re.compile(r"\bPyObject_Del\b( *\()"), r'PyObject_Free\1'),
        (re.compile(r"\bPyObject_DEL\b( *\()"), r'PyObject_Free\1'),
    )


class PyFrame_GetBack(Operation):
    NAME = "PyFrame_GetBack"
    REPLACE = (
        (get_member_regex('f_back'), r'_PyFrame_GetBackBorrow(\1)'),
    )
    # Need _PyFrame_GetBackBorrow() (PyFrame_GetBack() is new in Python 3.9)
    NEED_PYTHONCAPI_COMPAT = (MIN_PYTHON < (3, 9))


class PyFrame_GetCode(Operation):
    NAME = "PyFrame_GetCode"

    REPLACE = (
        (get_member_regex('f_code'), r'_PyFrame_GetCodeBorrow(\1)'),
    )
    # Need _PyFrame_GetCodeBorrow() (PyFrame_GetCode() is new in Python 3.9)
    NEED_PYTHONCAPI_COMPAT = (MIN_PYTHON < (3, 9))


class PyThreadState_GetInterpreter(Operation):
    NAME = "PyThreadState_GetInterpreter"
    REPLACE = (
        (get_member_regex('interp'), r'PyThreadState_GetInterpreter(\1)'),
    )
    # Need PyThreadState_GetInterpreter() (new in Python 3.9)
    NEED_PYTHONCAPI_COMPAT = (MIN_PYTHON < (3, 9))


class PyThreadState_GetFrame(Operation):
    NAME = "PyThreadState_GetFrame"
    REPLACE = (
        (get_member_regex('frame'), r'_PyThreadState_GetFrameBorrow(\1)'),
    )
    # Need _PyThreadState_GetFrameBorrow()
    # (PyThreadState_GetFrame() is new in Python 3.9)
    NEED_PYTHONCAPI_COMPAT = (MIN_PYTHON < (3, 9))


class Py_NewRef(Operation):
    NAME = "Py_NewRef"
    REPLACE = (
        # "Py_INCREF(x); return x;" => "return Py_NewRef(x);"
        # "Py_XINCREF(x); return x;" => "return Py_XNewRef(x);"
        # The two statements must be at the same indentation, otherwise the
        # regex does not match.
        (re.compile(fr'({INDENTATION_REGEX})'
                    + fr'Py_(X?)INCREF\(({EXPR_REGEX})\)\s*;'
                    + same_indentation(r'\1')
                    + fr'return {OPT_CAST_REGEX}\3;',
                    re.MULTILINE),
         r'\1return Py_\2NewRef(\3);'),

        # Same regex than the previous one,
        # but the two statements are on the same line.
        (re.compile(fr'Py_(X?)INCREF\(({EXPR_REGEX})\)\s*;'
                    + fr'{SPACE_REGEX}*'
                    + fr'return {OPT_CAST_REGEX}\2;',
                    re.MULTILINE),
         r'return Py_\1NewRef(\2);'),

        # "Py_INCREF(x); y = x;" must be replaced before
        # "y = x; Py_INCREF(y);", to not miss consecutive
        # "Py_INCREF; assign; Py_INCREF; assign; ..." (see unit tests).

        # "Py_INCREF(x); y = x;" => "y = Py_NewRef(x)"
        # "Py_XINCREF(x); y = x;" => "y = Py_XNewRef(x)"
        # The two statements must have the same indentation, otherwise the
        # regex does not match.
        (re.compile(fr'({INDENTATION_REGEX})'
                    + fr'Py_(X?)INCREF\(({EXPR_REGEX})\);'
                    + same_indentation(r'\1')
                    + assign_regex_str(fr'({EXPR_REGEX})',
                                       fr'{OPT_CAST_REGEX}\3'),
                    re.MULTILINE),
         r'\1\4 = Py_\2NewRef(\3);'),

        # Same regex than the previous one,
        # but the two statements are on the same line.
        (re.compile(fr'Py_(X?)INCREF\(({EXPR_REGEX})\);'
                    + fr'{SPACE_REGEX}*'
                    + assign_regex_str(fr'({EXPR_REGEX})',
                                       fr'{OPT_CAST_REGEX}\2')),
         r'\3 = Py_\1NewRef(\2);'),

        # "y = x; Py_INCREF(x);" => "y = Py_NewRef(x);"
        # "y = x; Py_INCREF(y);" => "y = Py_NewRef(x);"
        # "y = x; Py_XINCREF(x);" => "y = Py_XNewRef(x);"
        # "y = x; Py_XINCREF(y);" => "y = Py_XNewRef(x);"
        # "y = (PyObject*)x; Py_XINCREF(y);" => "y = Py_XNewRef(x);"
        # The two statements must have the same indentation, otherwise the
        # regex does not match.
        (re.compile(fr'({INDENTATION_REGEX})'
                    + assign_regex_str(fr'({EXPR_REGEX})',
                                       fr'{OPT_CAST_REGEX}({EXPR_REGEX})')
                    + same_indentation(r'\1')
                    + r'Py_(X?)INCREF\((?:\2|\3)\);',
                    re.MULTILINE),
         r'\1\2 = Py_\4NewRef(\3);'),

        # Same regex than the previous one,
        # but the two statements are on the same line.
        (re.compile(assign_regex_str(fr'({EXPR_REGEX})',
                                     fr'{OPT_CAST_REGEX}({EXPR_REGEX})')
                    + fr'{SPACE_REGEX}*'
                    + r'Py_(X?)INCREF\((?:\1|\2)\);'),
         r'\1 = Py_\3NewRef(\2);'),

        # "PyObject *var = x; Py_INCREF(x);" => "PyObject *var = Py_NewRef(x);"
        # The two statements must have the same indentation, otherwise the
        # regex does not match.
        (re.compile(fr'({INDENTATION_REGEX})'
                      # "type* var = expr;"
                    + assign_regex_str(fr'({TYPE_PTR_REGEX} *)({EXPR_REGEX})',
                                       fr'({OPT_CAST_REGEX})({EXPR_REGEX})')
                    + same_indentation(r'\1')
                      # "Py_INCREF(var);"
                    + r'Py_(X?)INCREF\((?:\3|\5)\);',
                    re.MULTILINE),
         r'\1\2\3 = \4Py_\6NewRef(\5);'),

        # "Py_INCREF(x); PyObject *var = x;" => "PyObject *var = Py_NewRef(x);"
        # The two statements must have the same indentation, otherwise the
        # regex does not match.
        (re.compile(fr'({INDENTATION_REGEX})'
                      # "Py_INCREF(var);"
                    + fr'Py_(X?)INCREF\(({EXPR_REGEX})\);'
                    + same_indentation(r'\1')
                      # "type* var = expr;"
                    + assign_regex_str(fr'({TYPE_PTR_REGEX} *{EXPR_REGEX})',
                                       fr'({OPT_CAST_REGEX})\3'),
                    re.MULTILINE),
         r'\1\4 = \5Py_\2NewRef(\3);'),
    )
    # Need Py_NewRef(): new in Python 3.10
    NEED_PYTHONCAPI_COMPAT = (MIN_PYTHON < (3, 10))


class Py_CLEAR(Operation):
    NAME = "Py_CLEAR"
    REPLACE = (
        # "Py_XDECREF(x); x = NULL;" => "Py_CLEAR(x)";
        # The two statements must have the same indentation, otherwise the
        # regex does not match.
        (re.compile(fr'({INDENTATION_REGEX})'
                    + fr'Py_XDECREF\(({EXPR_REGEX})\) *;'
                    + same_indentation(r'\1')
                    + assign_regex_str(r'\2', r'NULL'),
                    re.MULTILINE),
         r'\1Py_CLEAR(\2);'),

        # "Py_XDECREF(x); x = NULL;" => "Py_CLEAR(x)";
        (re.compile(fr'Py_XDECREF\(({EXPR_REGEX})\) *;'
                    + fr'{SPACE_REGEX}*'
                    + assign_regex_str(r'\1', r'NULL')),
         r'Py_CLEAR(\1);'),
    )


SETREF_VALUE = fr'{OPT_CAST_REGEX}(?:{EXPR_REGEX}|Py_X?NewRef\({EXPR_REGEX}\))'


class Py_SETREF(Operation):
    NAME = "Py_SETREF"
    REPLACE = (
        # "Py_INCREF(y); Py_CLEAR(x); x = y;" => "Py_XSETREF(x, y)";
        # Statements must have the same indentation, otherwise the regex does
        # not match.
        (re.compile(fr'({INDENTATION_REGEX})'
                    + fr'Py_(X?)INCREF\(({EXPR_REGEX})\) *;'
                    + same_indentation(r'\1')
                    + fr'Py_CLEAR\(({EXPR_REGEX})\) *;'
                    + same_indentation(r'\1')
                    + assign_regex_str(r'\4', r'\3'),
                    re.MULTILINE),
         r'\1Py_XSETREF(\4, Py_\2NewRef(\3));'),

        # "Py_CLEAR(x); x = y;" => "Py_XSETREF(x, y)";
        # Statements must have the same indentation, otherwise the regex does
        # not match.
        (re.compile(fr'({INDENTATION_REGEX})'
                    + fr'Py_CLEAR\(({EXPR_REGEX})\) *;'
                    + same_indentation(r'\1')
                    + assign_regex_str(r'\2',
                                       fr'({SETREF_VALUE})'),
                    re.MULTILINE),
         r'\1Py_XSETREF(\2, \3);'),

        # "Py_INCREF(y); Py_DECREF(x); x = y;" => "Py_SETREF(x, y)";
        # Statements must have the same indentation, otherwise the regex does
        # not match.
        (re.compile(fr'({INDENTATION_REGEX})'
                    + fr'Py_(X?)INCREF\(({EXPR_REGEX})\) *;'
                    + same_indentation(r'\1')
                    + fr'Py_(X?)DECREF\(({EXPR_REGEX})\) *;'
                    + same_indentation(r'\1')
                    + assign_regex_str(r'\5', r'\3'),
                    re.MULTILINE),
         r'\1Py_\4SETREF(\5, Py_\2NewRef(\3));'),

        # "Py_DECREF(x); x = y;" => "Py_SETREF(x, y)";
        # "Py_DECREF(x); x = Py_NewRef(y);" => "Py_SETREF(x, Py_NewRef(y))";
        # Statements must have the same indentation, otherwise the regex does
        # not match.
        (re.compile(fr'({INDENTATION_REGEX})'
                    + fr'Py_(X?)DECREF\(({EXPR_REGEX})\) *;'
                    + same_indentation(r'\1')
                    + assign_regex_str(r'\3',
                                       fr'({SETREF_VALUE})'),
                    re.MULTILINE),
         r'\1Py_\2SETREF(\3, \4);'),

        # "old = var; var = new; Py_DECREF(old);" => "Py_SETREF(var, new);"
        # "PyObject *old = var; var = new; Py_DECREF(old);" => "Py_SETREF(var, new);"
        # Statements must have the same indentation, otherwise the regex does
        # not match.
        (re.compile(fr'({INDENTATION_REGEX})'
                    + fr'(?:{ID_REGEX} *\* *)?({ID_REGEX}) *= *{OPT_CAST_REGEX}({EXPR_REGEX}) *;'
                    + same_indentation(r'\1')
                    + assign_regex_str(r'\3',
                                       fr'({SETREF_VALUE})')
                    + same_indentation(r'\1')
                    + fr'Py_(X?)DECREF\(\2\) *;',
                    re.MULTILINE),
         r'\1Py_\5SETREF(\3, \4);'),
    )
    # Need Py_NewRef(): new in Python 3.5
    NEED_PYTHONCAPI_COMPAT = (MIN_PYTHON < (3, 5))


class Py_Is(Operation):
    NAME = "Py_Is"

    def replace2(regs):
        x = regs.group(1)
        y = regs.group(2)
        if y == 'NULL':
            return regs.group(0)
        return f'{x} = _Py_StealRef({y});'

    REPLACE = []
    expr = fr'({EXPR_REGEX})'
    for name in ('None', 'True', 'False'):
        REPLACE.extend((
            (re.compile(fr'{expr} == Py_{name}\b'),
             fr'Py_Is{name}(\1)'),
            (re.compile(fr'{expr} != Py_{name}\b'),
             fr'!Py_Is{name}(\1)'),
        ))

    # Need Py_IsNone(), Py_IsTrue(), Py_IsFalse(): new in Python 3.10
    NEED_PYTHONCAPI_COMPAT = (MIN_PYTHON < (3, 10))


OPERATIONS = (
    Py_SET_TYPE,
    Py_SET_SIZE,
    Py_SET_REFCNT,
    # Py_SET_xxx must be run before Py_xxx
    Py_TYPE,
    Py_SIZE,
    Py_REFCNT,

    Py_Is,

    PyObject_NEW,
    PyMem_MALLOC,
    PyObject_MALLOC,

    PyFrame_GetBack,
    PyFrame_GetCode,

    PyThreadState_GetInterpreter,
    PyThreadState_GetFrame,

    # Code style: excluded from "all"
    Py_NewRef,
    Py_CLEAR,
    Py_SETREF,
)

EXCLUDE_FROM_ALL = (
    Py_NewRef,
    Py_CLEAR,
    Py_SETREF,
)


def all_operations():
    return set(operation_class.NAME for operation_class in OPERATIONS
               if operation_class not in EXCLUDE_FROM_ALL)


class Patcher:
    def __init__(self, args=None):
        self.exitcode = 0
        self.pythoncapi_compat_added = 0
        self.want_pythoncapi_compat = False
        self.operations = None
        self.applied_operations = set()

        # Set temporariliy by patch()
        self._has_pythoncapi_compat = None
        self._applied_operations = None

        self._parse_options(args)

    def log(self, msg=''):
        print(msg, file=sys.stderr, flush=True)

    def warning(self, msg):
        self.log(f"WARNING: {msg}")

    def _get_operations(self, parser):
        args_names = self.args.operations.split(',')

        wanted = set()
        for name in args_names:
            name = name.strip()
            if not name:
                continue

            if name == "all":
                wanted |= all_operations()
            elif name.startswith("-"):
                name = name[1:]
                wanted.discard(name)
            else:
                wanted.add(name)

        operations = []
        for operation_class in OPERATIONS:
            name = operation_class.NAME
            if name not in wanted:
                continue
            wanted.discard(name)
            operation = operation_class(self)
            operations.append(operation)

        if wanted:
            print(f"invalid operations: {','.join(wanted)}")
            print()
            self.usage(parser)
            sys.exit(1)

        return operations

    def add_line(self, content, line):
        # Use the first matching newline
        match = re.search(r'(?:\r\n|\n|\r)', content)
        newline = match.group(0) if match else '\n'

        line = line + newline
        # FIXME: tolerate trailing spaces
        if line not in content:
            # FIXME: add macro after the first header comment
            # FIXME: add macro after includes
            # FIXME: add macro after: #define PY_SSIZE_T_CLEAN
            return line + newline + content
        else:
            return content

    def add_pythoncapi_compat(self, content):
        if self._has_pythoncapi_compat:
            return content
        content = self.add_line(content, INCLUDE_PYTHONCAPI_COMPAT)
        self._has_pythoncapi_compat = True
        self.pythoncapi_compat_added += 1
        return content

    def _patch(self, content):
        try:
            has = (self.args.no_compat
                   or INCLUDE_PYTHONCAPI_COMPAT in content
                   or INCLUDE_PYTHONCAPI_COMPAT2 in content)
            self._has_pythoncapi_compat = has
            self._applied_operations = []
            for operation in self.operations:
                new_content = operation.patch(content)
                if new_content != content:
                    self._applied_operations.append(operation.NAME)
                content = new_content
            applied_operations = self._applied_operations
        finally:
            self._has_pythoncapi_compat = None
            self._applied_operations = None
        return (content, applied_operations)

    def patch(self, content):
        return self._patch(content)[0]

    def patch_file(self, filename):
        if os.path.basename(filename) == PYTHONCAPI_COMPAT_H:
            self.log(f"Skip {filename}")
            return

        encoding = "utf-8"
        errors = "surrogateescape"

        with open(filename, encoding=encoding, errors=errors, newline="") as fp:
            old_contents = fp.read()

        new_contents, operations = self._patch(old_contents)

        if self.args.to_stdout:
            print(new_contents, end="")
            return (new_contents != old_contents)

        # Don't rewrite if the filename for in-place replacement,
        # to avoid changing the file modification time.
        if new_contents == old_contents:
            return False

        if not self.args.no_backup:
            old_filename = filename + ".old"
            # If old_filename already exists, replace it
            os.replace(filename, old_filename)

        with open(filename, "w", encoding=encoding, errors=errors, newline="") as fp:
            fp.write(new_contents)

        self.applied_operations |= set(operations)
        operations = ', '.join(operations)
        self.log(f"Patched file: {filename} ({operations})")
        return True

    def _walk_dir(self, path):
        empty = True

        for dirpath, dirnames, filenames in os.walk(path):
            # Don't walk into .tox
            for ignore_name in IGNORE_DIRS:
                try:
                    dirnames.remove(ignore_name)
                except ValueError:
                    pass
            for filename in filenames:
                if is_c_filename(filename):
                    yield os.path.join(dirpath, filename)
                    empty = False

        if empty:
            self.warning(f"Directory {path} doesn't contain any C file")
            self.exitcode = 1

    def walk(self, paths):
        for path in paths:
            if os.path.isdir(path):
                for filename in self._walk_dir(path):
                    yield filename
            elif os.path.exists(path):
                yield path
            else:
                self.warning(f"Path {path} does not exist")
                self.exitcode = 1

    def get_latest_header(self, base_dir):
        target = os.path.join(base_dir, PYTHONCAPI_COMPAT_H)
        self.log(f"Download the file from {PYTHONCAPI_COMPAT_URL} to {target}.")
        urllib.request.urlretrieve(PYTHONCAPI_COMPAT_URL, target)

    @staticmethod
    def usage(parser):
        parser.print_help()
        print()
        print("Operations:")
        print()
        for operation in sorted(OPERATIONS,
                                key=lambda operation: operation.NAME.lower()):
            print(f"- {operation.NAME}")
        print()
        print("If a directory is passed, search for .c and .h files "
              "in subdirectories.")

    def _parse_dir_path(self, path):
        if os.path.isdir(path):
            return path
        else:
            raise argparse.ArgumentTypeError(f"{path} is not a valid path")

    def _parse_options(self, args):
        parser = argparse.ArgumentParser(
            description="Upgrade C extension modules to newer Python C API")
        parser.add_argument(
            '-o', '--operations', action="store",
            default="all",
            help='Space separated list of operation names to apply')
        parser.add_argument(
            '-q', '--quiet', action="store_true",
            help='Quiet mode')
        parser.add_argument(
            '-c', '--to-stdout', action="store_true",
            help='Write output into stdout instead of modifying files '
                 'in-place (imply quiet mode)')
        parser.add_argument(
            '-B', '--no-backup', action="store_true",
            help="Don't create .old backup files")
        parser.add_argument(
            '-C', '--no-compat', action="store_true",
            help=f"Don't add: {INCLUDE_PYTHONCAPI_COMPAT}")
        parser.add_argument(
            '-d', '--download', metavar='PATH',
            help=f'Download latest pythoncapi_compat.h file to designated PATH',
            type=self._parse_dir_path)
        parser.add_argument(
            metavar='file_or_directory', dest="paths", nargs='*')

        args = parser.parse_args(args)
        if not args.paths and not args.download:
            self.usage(parser)
            sys.exit(1)

        if args.to_stdout:
            args.quiet = True

        self.args = args
        self.operations = self._get_operations(parser)

    def main(self):
        if self.args.paths:
            for filename in self.walk(self.args.paths):
                self.patch_file(filename)

        if self.applied_operations:
            nops = len(self.applied_operations)
            ops = ', '.join(sorted(self.applied_operations))
            self.log()
            self.log(f"Applied operations ({nops}): {ops}")

        if self.args.download:
            path = self.args.download
            self.get_latest_header(path)

        if self.pythoncapi_compat_added and not self.args.quiet:
            self.log()
            self.log(f"{INCLUDE_PYTHONCAPI_COMPAT} added: you may have "
                     f"to copy {PYTHONCAPI_COMPAT_H} to your project")
            self.log("Run 'python upgrade_pythoncapi.py --download <target_path>'")

        sys.exit(self.exitcode)


if __name__ == "__main__":
    Patcher().main()