Convention for test data which uses APIs defined by Python stubs

Currently we check only for syntax errors in stub files. The idea is to add test data for static analyzers similar to how it's done in DefinitelyTyped for TypeScript.
python · vlasovskikh · Jan 26, 2017 · Feb 5, 2017 · Feb 5, 2017 · Feb 5, 2017
commit 4e8d414dd14f7282ddbc85a9207dae06f68160d3
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -14,6 +14,8 @@ are important to the project's success.
     * [Contact us](#discussion) before starting significant work.
     * IMPORTANT: For new libraries, [get permission from the library owner first](#adding-a-new-library).
     * Create your stubs [conforming to the coding style](#stub-file-coding-style).
+    * Add tests for your stubs to `test_data`. These tests are not meant
+      to be executable. Type checkers analyze them statically.
     * Make sure `runtests.sh` passes cleanly on Mypy, pytype, and flake8.
 4. [Submit your changes](#submitting-changes):
     * Open a pull request

diff --git a/test_data/stdlib/2and3/collections_test.py b/test_data/stdlib/2and3/collections_test.py
@@ -0,0 +1,14 @@
+from collections import namedtuple
+
+
+def test_collections_namedtuple():
+    # type: () -> None
+    Point = namedtuple('Point', 'x y')
+    p = Point(1, 2)
+
+    p._replace(y=3.14)
+    p._asdict()
+    p.x, p.y
+    p[0] + p[1]
+    p.index(1)
+    Point._make([('x', 1), ('y', 3.14)])
diff --git a/tests/mypy_test.py b/tests/mypy_test.py
@@ -63,7 +63,7 @@ def libpath(major, minor):
     versions.append('2and3')
     paths = []
     for v in versions:
-        for top in ['stdlib', 'third_party']:
+        for top in ['stdlib', 'third_party', 'test_data/stdlib', 'test_data/third_party']:
             p = os.path.join(top, v)
             if os.path.isdir(p):
                 paths.append(p)

diff --git a/tests/pytype_test.py b/tests/pytype_test.py
@@ -70,7 +70,7 @@ def pytype_test(args):
         print("Cannot run pytd. Did you install pytype?")
         return 0, 0
 
-    wanted = re.compile(r"stdlib/(2|2\.7|2and3)/.*\.pyi$")
+    wanted = re.compile(r"stdlib/(2|2\.7|2and3)/.*\.pyi?$")
     skipped = re.compile("(%s)$" % "|".join(load_blacklist()))
     files = []
 
@@ -80,6 +80,12 @@ def pytype_test(args):
             if wanted.search(f) and not skipped.search(f):
                 files.append(f)
 
+    for root, _, filenames in os.walk("test_data/stdlib"):
+        for f in sorted(filenames):
+            f = os.path.join(root, f)
+            if wanted.search(f) and not skipped.search(f):
+                files.append(f)
+
     running_tests = collections.deque()
     max_code, runs, errors = 0, 0, 0
     print("Running pytype tests...")