From dbc44c6d527f09aa94d40319348c756ed02d64a7 Mon Sep 17 00:00:00 2001 From: James Tatum Date: Fri, 6 May 2016 23:57:51 -0700 Subject: [PATCH 1/4] Make waiter cross-platform Waiter depended on a couple of posix tricks. The first was a clever use of os.waitpid(-1, 0) to wait for any spawned processes to return. On Windows, this raises an exception - Windows has no concept of a process group. The second was the use of NamedTempFile(). On anything but Windows, a file can be open for writing and for reading at the same time. This trick actually isn't necessary the way NamedTempFile is used here. It exposes a file-like object pointing to a file already opened in binary mode. All we have to do is seek and read from it. --- mypy/waiter.py | 44 ++++++++++++++++---------------------------- 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/mypy/waiter.py b/mypy/waiter.py index 50a949fbed45..4bb66654ca92 100644 --- a/mypy/waiter.py +++ b/mypy/waiter.py @@ -31,28 +31,12 @@ def __init__(self, name: str, args: List[str], *, cwd: str = None, self.end_time = None # type: float def start(self) -> None: - self.outfile = tempfile.NamedTemporaryFile() + self.outfile = tempfile.TemporaryFile() self.start_time = time.time() self.process = Popen(self.args, cwd=self.cwd, env=self.env, stdout=self.outfile, stderr=STDOUT) self.pid = self.process.pid - def handle_exit_status(self, status: int) -> None: - """Update process exit status received via an external os.waitpid() call.""" - # Inlined subprocess._handle_exitstatus, it's not a public API. - # TODO(jukka): I'm not quite sure why this is implemented like this. - self.end_time = time.time() - process = self.process - assert process.returncode is None - if os.WIFSIGNALED(status): - process.returncode = -os.WTERMSIG(status) - elif os.WIFEXITED(status): - process.returncode = os.WEXITSTATUS(status) - else: - # Should never happen - raise RuntimeError("Unknown child exit status!") - assert process.returncode is not None - def wait(self) -> int: return self.process.wait() @@ -60,13 +44,10 @@ def status(self) -> Optional[int]: return self.process.returncode def read_output(self) -> str: - with open(self.outfile.name, 'rb') as file: - # Assume it's ascii to avoid unicode headaches (and portability issues). - return file.read().decode('ascii') - - def cleanup(self) -> None: - self.outfile.close() - assert not os.path.exists(self.outfile.name) + file = self.outfile + file.seek(0) + # Assume it's ascii to avoid unicode headaches (and portability issues). + return file.read().decode('ascii') @property def elapsed_time(self) -> float: @@ -178,17 +159,25 @@ def _record_time(self, name: str, elapsed_time: float) -> None: name2 = re.sub('( .*?) .*', r'\1', name) # First two words. self.times2[name2] = elapsed_time + self.times2.get(name2, 0) + def _poll_current(self) -> Tuple[int, int]: + while True: + time.sleep(.25) + for pid in self.current: + cmd = self.current[pid][1] + code = cmd.process.poll() + if code is not None: + cmd.end_time = time.time() + return pid, code + def _wait_next(self) -> Tuple[List[str], int, int]: """Wait for a single task to finish. Return tuple (list of failed tasks, number test cases, number of failed tests). """ - pid, status = os.waitpid(-1, 0) + pid, status = self._poll_current() num, cmd = self.current.pop(pid) name = cmd.name - cmd.handle_exit_status(status) - self._record_time(cmd.name, cmd.elapsed_time) rc = cmd.wait() @@ -223,7 +212,6 @@ def _wait_next(self) -> Tuple[List[str], int, int]: # Get task output. output = cmd.read_output() - cmd.cleanup() num_tests, num_tests_failed = parse_test_stats_from_output(output, fail_type) if fail_type is not None or self.verbosity >= 1: From 5b0e561caa9d1ff18c67686026cfe84df779856b Mon Sep 17 00:00:00 2001 From: James Tatum Date: Sat, 7 May 2016 00:06:06 -0700 Subject: [PATCH 2/4] Add appveyor config for Windows CI The only tricky bit of this is renaming python.exe to python2.exe. This is due to util.try_find_python2_interpreter(), which may well need work for Windows since the version symlinks don't exist on Windows. --- appveyor.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000000..4d1dc1161331 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,19 @@ +environment: + matrix: + - PYTHON: C:\\Python33 + - PYTHON: C:\\Python34 + - PYTHON: C:\\Python35 + - PYTHON: C:\\Python33-x64 + - PYTHON: C:\\Python34-x64 + - PYTHON: C:\\Python35-x64 +install: + - "git submodule update --init typeshed" + - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;C:\\Python27;%PATH%" + - "REN C:\\Python27\\python.exe python2.exe" + - "python --version" + - "python2 --version" +build_script: + - "pip install -r test-requirements.txt" + - "python setup.py install" +test_script: +- cmd: python runtests.py -v From 00afaa75b5579c92163f8902dc8de32ea2381d80 Mon Sep 17 00:00:00 2001 From: James Tatum Date: Sat, 7 May 2016 09:38:20 -0700 Subject: [PATCH 3/4] Fixing Windows tests Most of these fixes revolve around the path separator and the way Windows handles files and locking. There is one bug fix in here - build.write_cache() was using os.rename to replace a file, which fails on Windows. I was only able to fix that for Python 3.3 and up. --- mypy/build.py | 12 ++++++--- mypy/test/data.py | 2 ++ mypy/test/data/cmdline.test | 12 ++++----- mypy/test/data/pythoneval.test | 2 +- mypy/test/data/semanal-modules.test | 6 ++--- mypy/test/testcheck.py | 3 ++- mypy/test/teststubgen.py | 39 +++++++++++++++-------------- 7 files changed, 42 insertions(+), 34 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 5238767c7372..d50f9f1e6bf8 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -832,10 +832,14 @@ def write_cache(id: str, path: str, tree: MypyFile, with open(meta_json_tmp, 'w') as f: json.dump(meta, f, sort_keys=True) f.write('\n') - # TODO: On Windows, os.rename() may not be atomic, and we could - # use os.replace(). However that's new in Python 3.3. - os.rename(data_json_tmp, data_json) - os.rename(meta_json_tmp, meta_json) + # TODO: This is a temporary change until Python 3.2 support is dropped, see #1504 + # os.rename will raise an exception rather than replace files on Windows + try: + replace = os.replace + except AttributeError: + replace = os.rename + replace(data_json_tmp, data_json) + replace(meta_json_tmp, meta_json) """Dependency manager. diff --git a/mypy/test/data.py b/mypy/test/data.py index 07f6d00f3e5e..bbda65c11cd3 100644 --- a/mypy/test/data.py +++ b/mypy/test/data.py @@ -65,6 +65,8 @@ def parse_test_cases( tcout = [] # type: List[str] if i < len(p) and p[i].id == 'out': tcout = p[i].data + if p[i].arg == 'pathfix': + tcout = [s.replace('/', os.path.sep) for s in tcout] ok = True i += 1 elif optional_out: diff --git a/mypy/test/data/cmdline.test b/mypy/test/data/cmdline.test index 276300b49173..ed6d41a1f98b 100644 --- a/mypy/test/data/cmdline.test +++ b/mypy/test/data/cmdline.test @@ -18,7 +18,7 @@ undef [file pkg/subpkg/a.py] undef import pkg.subpkg.a -[out] +[out pathfix] pkg/a.py:1: error: Name 'undef' is not defined pkg/subpkg/a.py:1: error: Name 'undef' is not defined @@ -31,7 +31,7 @@ undef [file pkg/subpkg/a.py] undef import pkg.subpkg.a -[out] +[out pathfix] pkg/a.py:1: error: Name 'undef' is not defined pkg/subpkg/a.py:1: error: Name 'undef' is not defined @@ -41,7 +41,7 @@ pkg/subpkg/a.py:1: error: Name 'undef' is not defined undef [file dir/subdir/a.py] undef -[out] +[out pathfix] dir/a.py:1: error: Name 'undef' is not defined [case testCmdlineNonPackageSlash] @@ -50,7 +50,7 @@ dir/a.py:1: error: Name 'undef' is not defined undef [file dir/subdir/a.py] undef -[out] +[out pathfix] dir/a.py:1: error: Name 'undef' is not defined [case testCmdlinePackageContainingSubdir] @@ -60,7 +60,7 @@ dir/a.py:1: error: Name 'undef' is not defined undef [file pkg/subdir/a.py] undef -[out] +[out pathfix] pkg/a.py:1: error: Name 'undef' is not defined [case testCmdlineNonPackageContainingPackage] @@ -71,6 +71,6 @@ import subpkg.a [file dir/subpkg/__init__.py] [file dir/subpkg/a.py] undef -[out] +[out pathfix] dir/subpkg/a.py:1: error: Name 'undef' is not defined dir/a.py:1: error: Name 'undef' is not defined diff --git a/mypy/test/data/pythoneval.test b/mypy/test/data/pythoneval.test index 604befc6c211..2d36ab9ac62a 100644 --- a/mypy/test/data/pythoneval.test +++ b/mypy/test/data/pythoneval.test @@ -16,7 +16,7 @@ import re from typing import Sized, Sequence, Iterator, Iterable, Mapping, AbstractSet def check(o, t): - rep = re.sub('0x[0-9a-f]+', '0x...', repr(o)) + rep = re.sub('0x[0-9a-fA-F]+', '0x...', repr(o)) rep = rep.replace('sequenceiterator', 'str_iterator') trep = str(t).replace('_abcoll.Sized', 'collections.abc.Sized') print(rep, trep, isinstance(o, t)) diff --git a/mypy/test/data/semanal-modules.test b/mypy/test/data/semanal-modules.test index a7cbef578085..86131a345aed 100644 --- a/mypy/test/data/semanal-modules.test +++ b/mypy/test/data/semanal-modules.test @@ -770,7 +770,7 @@ import m.x [file m/__init__.py] [file m/x.py] from .x import nonexistent -[out] +[out pathfix] main:1: note: In module imported here: tmp/m/x.py:1: error: Module has no attribute 'nonexistent' @@ -779,7 +779,7 @@ import m.x [file m/__init__.py] [file m/x.py] from m.x import nonexistent -[out] +[out pathfix] main:1: note: In module imported here: tmp/m/x.py:1: error: Module has no attribute 'nonexistent' @@ -846,7 +846,7 @@ import m x [file m.py] y -[out] +[out pathfix] main:1: note: In module imported here: tmp/m.py:1: error: Name 'y' is not defined main:2: error: Name 'x' is not defined diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 412693a39acb..1ba35595c14e 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -153,7 +153,8 @@ def find_error_paths(self, a: List[str]) -> Set[str]: for line in a: m = re.match(r'([^\s:]+):\d+: error:', line) if m: - hits.add(m.group(1)) + p = m.group(1).replace('/', os.path.sep) + hits.add(p) return hits def find_module_files(self) -> Dict[str, str]: diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 893659d2f6cf..a74525f14787 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -108,7 +108,8 @@ def test_stubgen(testcase): sys.path.insert(0, 'stubgen-test-path') os.mkdir('stubgen-test-path') source = '\n'.join(testcase.input) - handle = tempfile.NamedTemporaryFile(prefix='prog_', suffix='.py', dir='stubgen-test-path') + handle = tempfile.NamedTemporaryFile(prefix='prog_', suffix='.py', dir='stubgen-test-path', + delete=False) assert os.path.isabs(handle.name) path = os.path.basename(handle.name) name = path[:-3] @@ -116,26 +117,26 @@ def test_stubgen(testcase): out_dir = '_out' os.mkdir(out_dir) try: - with open(path, 'w') as file: - file.write(source) - file.close() - # Without this we may sometimes be unable to import the module below, as importlib - # caches os.listdir() results in Python 3.3+ (Guido explained this to me). - reset_importlib_caches() - try: - if testcase.name.endswith('_import'): - generate_stub_for_module(name, out_dir, quiet=True) - else: - generate_stub(path, out_dir) - a = load_output(out_dir) - except CompileError as e: - a = e.messages - assert_string_arrays_equal(testcase.output, a, - 'Invalid output ({}, line {})'.format( - testcase.file, testcase.line)) + handle.write(bytes(source, 'ascii')) + handle.close() + # Without this we may sometimes be unable to import the module below, as importlib + # caches os.listdir() results in Python 3.3+ (Guido explained this to me). + reset_importlib_caches() + try: + if testcase.name.endswith('_import'): + generate_stub_for_module(name, out_dir, quiet=True) + else: + generate_stub(path, out_dir) + a = load_output(out_dir) + except CompileError as e: + a = e.messages + assert_string_arrays_equal(testcase.output, a, + 'Invalid output ({}, line {})'.format( + testcase.file, testcase.line)) finally: - shutil.rmtree(out_dir) handle.close() + os.unlink(handle.name) + shutil.rmtree(out_dir) def reset_importlib_caches(): From 48aaf7147c9030b97ec4c605732a0a99413d9dcc Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 28 May 2016 16:13:12 -0700 Subject: [PATCH 4/4] More subtle translation of / to \ in error messages for Windows. (The goal is to avoid having to mark up tests just because they need this kind of translation.) --- mypy/test/data.py | 23 +++++++++++++++++++---- mypy/test/data/cmdline.test | 12 ++++++------ mypy/test/data/semanal-modules.test | 6 +++--- mypy/test/testcmdline.py | 5 ++++- mypy/test/testsemanal.py | 5 ++++- mypy/test/testtransform.py | 4 +++- 6 files changed, 39 insertions(+), 16 deletions(-) diff --git a/mypy/test/data.py b/mypy/test/data.py index bbda65c11cd3..bc5b7c40fc0d 100644 --- a/mypy/test/data.py +++ b/mypy/test/data.py @@ -16,7 +16,8 @@ def parse_test_cases( perform: Callable[['DataDrivenTestCase'], None], base_path: str = '.', optional_out: bool = False, - include_path: str = None) -> List['DataDrivenTestCase']: + include_path: str = None, + native_sep: bool = False) -> List['DataDrivenTestCase']: """Parse a file with test case descriptions. Return an array of test cases. @@ -65,8 +66,8 @@ def parse_test_cases( tcout = [] # type: List[str] if i < len(p) and p[i].id == 'out': tcout = p[i].data - if p[i].arg == 'pathfix': - tcout = [s.replace('/', os.path.sep) for s in tcout] + if native_sep and os.path.sep == '\\': + tcout = [fix_win_path(line) for line in tcout] ok = True i += 1 elif optional_out: @@ -293,7 +294,7 @@ def expand_includes(a: List[str], base_path: str) -> List[str]: return res -def expand_errors(input, output, fnam): +def expand_errors(input: List[str], output: List[str], fnam: str) -> None: """Transform comments such as '# E: message' in input. The result is lines like 'fnam:line: error: message'. @@ -304,3 +305,17 @@ def expand_errors(input, output, fnam): if m: severity = 'error' if m.group(1) == 'E' else 'note' output.append('{}:{}: {}: {}'.format(fnam, i + 1, severity, m.group(2))) + + +def fix_win_path(line: str) -> str: + r"""Changes paths to Windows paths in error messages. + + E.g. foo/bar.py -> foo\bar.py. + """ + m = re.match(r'^([\S/]+):(\d+:)?(\s+.*)', line) + if not m: + return line + else: + filename, lineno, message = m.groups() + return '{}:{}{}'.format(filename.replace('/', '\\'), + lineno or '', message) diff --git a/mypy/test/data/cmdline.test b/mypy/test/data/cmdline.test index ed6d41a1f98b..276300b49173 100644 --- a/mypy/test/data/cmdline.test +++ b/mypy/test/data/cmdline.test @@ -18,7 +18,7 @@ undef [file pkg/subpkg/a.py] undef import pkg.subpkg.a -[out pathfix] +[out] pkg/a.py:1: error: Name 'undef' is not defined pkg/subpkg/a.py:1: error: Name 'undef' is not defined @@ -31,7 +31,7 @@ undef [file pkg/subpkg/a.py] undef import pkg.subpkg.a -[out pathfix] +[out] pkg/a.py:1: error: Name 'undef' is not defined pkg/subpkg/a.py:1: error: Name 'undef' is not defined @@ -41,7 +41,7 @@ pkg/subpkg/a.py:1: error: Name 'undef' is not defined undef [file dir/subdir/a.py] undef -[out pathfix] +[out] dir/a.py:1: error: Name 'undef' is not defined [case testCmdlineNonPackageSlash] @@ -50,7 +50,7 @@ dir/a.py:1: error: Name 'undef' is not defined undef [file dir/subdir/a.py] undef -[out pathfix] +[out] dir/a.py:1: error: Name 'undef' is not defined [case testCmdlinePackageContainingSubdir] @@ -60,7 +60,7 @@ dir/a.py:1: error: Name 'undef' is not defined undef [file pkg/subdir/a.py] undef -[out pathfix] +[out] pkg/a.py:1: error: Name 'undef' is not defined [case testCmdlineNonPackageContainingPackage] @@ -71,6 +71,6 @@ import subpkg.a [file dir/subpkg/__init__.py] [file dir/subpkg/a.py] undef -[out pathfix] +[out] dir/subpkg/a.py:1: error: Name 'undef' is not defined dir/a.py:1: error: Name 'undef' is not defined diff --git a/mypy/test/data/semanal-modules.test b/mypy/test/data/semanal-modules.test index 86131a345aed..a7cbef578085 100644 --- a/mypy/test/data/semanal-modules.test +++ b/mypy/test/data/semanal-modules.test @@ -770,7 +770,7 @@ import m.x [file m/__init__.py] [file m/x.py] from .x import nonexistent -[out pathfix] +[out] main:1: note: In module imported here: tmp/m/x.py:1: error: Module has no attribute 'nonexistent' @@ -779,7 +779,7 @@ import m.x [file m/__init__.py] [file m/x.py] from m.x import nonexistent -[out pathfix] +[out] main:1: note: In module imported here: tmp/m/x.py:1: error: Module has no attribute 'nonexistent' @@ -846,7 +846,7 @@ import m x [file m.py] y -[out pathfix] +[out] main:1: note: In module imported here: tmp/m.py:1: error: Name 'y' is not defined main:2: error: Name 'x' is not defined diff --git a/mypy/test/testcmdline.py b/mypy/test/testcmdline.py index d8e85a6da20b..a78cbe265a79 100644 --- a/mypy/test/testcmdline.py +++ b/mypy/test/testcmdline.py @@ -29,7 +29,10 @@ def cases(self) -> List[DataDrivenTestCase]: c = [] # type: List[DataDrivenTestCase] for f in cmdline_files: c += parse_test_cases(os.path.join(test_data_prefix, f), - test_python_evaluation, test_temp_dir, True) + test_python_evaluation, + base_path=test_temp_dir, + optional_out=True, + native_sep=True) return c diff --git a/mypy/test/testsemanal.py b/mypy/test/testsemanal.py index a1d6a5835ac6..9611284e793e 100644 --- a/mypy/test/testsemanal.py +++ b/mypy/test/testsemanal.py @@ -36,7 +36,10 @@ def cases(self): c = [] for f in semanal_files: c += parse_test_cases(os.path.join(test_data_prefix, f), - test_semanal, test_temp_dir, optional_out=True) + test_semanal, + base_path=test_temp_dir, + optional_out=True, + native_sep=True) return c diff --git a/mypy/test/testtransform.py b/mypy/test/testtransform.py index 1d9916ea2f5e..4b17d2ef8e59 100644 --- a/mypy/test/testtransform.py +++ b/mypy/test/testtransform.py @@ -30,7 +30,9 @@ def cases(self): c = [] for f in self.transform_files: c += parse_test_cases(os.path.join(test_data_prefix, f), - test_transform, test_temp_dir) + test_transform, + base_path=test_temp_dir, + native_sep=True) return c