Generate .pyi's using stubgen from mypy (#17521)

Add mypy (and dependencies) as new externals. Add a rule to generate
`.pyi`s for `pydrake` using the same and a tiny wrapping helper script.

For now, this must be manually invoked, and some functions have degraded
type information (see #17520). However, this will give us the ability to
manually test the generated `.pyi` files. Adding logic to install them
and/or otherwise bundle them can come later.

bindings/pydrake/BUILD.bazel

@@ -680,6 +680,26 @@ generate_pybind_coverage(
     xml_docstrings = ":documentation_pybind.xml",
 )
 
+drake_py_binary(
+    name = "stubgen",
+    srcs = ["stubgen.py"],
+    args = ["--package=pydrake"],
+    deps = [
+        ":all_py",
+        "@mypy_internal//:mypy",
+    ],
+)
+
+# TODO(mwoehlke-kitware): actually install the .pyi's.
+
+drake_py_unittest(
+    name = "stubgen_test",
+    deps = [
+        ":all_py",
+        "@mypy_internal//:mypy",
+    ],
+)
+
 add_lint_tests_pydrake(
     python_lint_extra_srcs = [
         ":test/all_install_test.py",

bindings/pydrake/stubgen.py

@@ -0,0 +1,8 @@
+"""Command-line tool to generate Drake's Python Interface files (.pyi)."""
+
+import sys
+
+from mypy import stubgen
+
+if __name__ == "__main__":
+    sys.exit(stubgen.main())

bindings/pydrake/test/stubgen_test.py

@@ -0,0 +1,31 @@
+"""Tests generation of Drake's Python Interface files (.pyi)."""
+
+import os
+import unittest
+
+from mypy import stubgen
+
+
+class TestStubgen(unittest.TestCase):
+    # TODO(mwoehlke-kitware): test the already-generated files instead.
+    def test_generation(self):
+        """Ensure that stubgen runs and generates output.
+
+        For now, this is more or less just a smoke test, with a very cursory
+        check that the output is 'reasonable'.
+        """
+        output_dir = os.environ['TEST_TMPDIR']
+        args = ['--package', 'pydrake', '--output', output_dir]
+
+        # Generate stubs.
+        result = stubgen.main(args)
+        self.assertTrue(result is None or result == 0)
+
+        # Find some of the expected output and look for an expected function.
+        expected = os.path.join(output_dir, 'pydrake', '__init__.pyi')
+        found_expected_decl = False
+        for line in open(expected, 'r'):
+            if line.startswith('def getDrakePath():'):
+                found_expected_decl = True
+                break
+        self.assertTrue(found_expected_decl)

tools/workspace/default.bzl

@@ -60,6 +60,8 @@ load("@drake//tools/workspace/models_internal:repository.bzl", "models_internal_
 load("@drake//tools/workspace/mosek:repository.bzl", "mosek_repository")
 load("@drake//tools/workspace/msgpack:repository.bzl", "msgpack_repository")
 load("@drake//tools/workspace/msgpack_lite_js:repository.bzl", "msgpack_lite_js_repository")  # noqa
+load("@drake//tools/workspace/mypy_extensions_internal:repository.bzl", "mypy_extensions_internal_repository")  # noqa
+load("@drake//tools/workspace/mypy_internal:repository.bzl", "mypy_internal_repository")  # noqa
 load("@drake//tools/workspace/net_sf_jchart2d:repository.bzl", "net_sf_jchart2d_repository")  # noqa
 load("@drake//tools/workspace/nlopt:repository.bzl", "nlopt_repository")
 load("@drake//tools/workspace/nlopt_internal:repository.bzl", "nlopt_internal_repository")  # noqa
@@ -91,6 +93,8 @@ load("@drake//tools/workspace/styleguide:repository.bzl", "styleguide_repository
 load("@drake//tools/workspace/suitesparse:repository.bzl", "suitesparse_repository")  # noqa
 load("@drake//tools/workspace/tinyobjloader:repository.bzl", "tinyobjloader_repository")  # noqa
 load("@drake//tools/workspace/tinyxml2:repository.bzl", "tinyxml2_repository")
+load("@drake//tools/workspace/tomli_internal:repository.bzl", "tomli_internal_repository")  # noqa
+load("@drake//tools/workspace/typing_extensions_internal:repository.bzl", "typing_extensions_internal_repository")  # noqa
 load("@drake//tools/workspace/uritemplate_py_internal:repository.bzl", "uritemplate_py_internal_repository")  # noqa
 load("@drake//tools/workspace/usockets:repository.bzl", "usockets_repository")  # noqa
 load("@drake//tools/workspace/uwebsockets:repository.bzl", "uwebsockets_repository")  # noqa
@@ -266,6 +270,10 @@ def add_default_repositories(excludes = [], mirrors = DEFAULT_MIRRORS):
         msgpack_repository(name = "msgpack")
     if "msgpack_lite_js" not in excludes:
         msgpack_lite_js_repository(name = "msgpack_lite_js", mirrors = mirrors)
+    if "mypy_extensions_internal" not in excludes:
+        mypy_extensions_internal_repository(name = "mypy_extensions_internal", mirrors = mirrors)  # noqa
+    if "mypy_internal" not in excludes:
+        mypy_internal_repository(name = "mypy_internal", mirrors = mirrors)
     if "net_sf_jchart2d" not in excludes:
         net_sf_jchart2d_repository(name = "net_sf_jchart2d", mirrors = mirrors)
     if "nlopt" not in excludes:
@@ -348,6 +356,10 @@ def add_default_repositories(excludes = [], mirrors = DEFAULT_MIRRORS):
         tinyobjloader_repository(name = "tinyobjloader", mirrors = mirrors)
     if "tinyxml2" not in excludes:
         tinyxml2_repository(name = "tinyxml2")
+    if "tomli_internal" not in excludes:
+        tomli_internal_repository(name = "tomli_internal", mirrors = mirrors)
+    if "typing_extensions_internal" not in excludes:
+        typing_extensions_internal_repository(name = "typing_extensions_internal", mirrors = mirrors)  # noqa
     if "uritemplate_py" not in excludes:
         add_deprecation(
             name = "uritemplate_py",

tools/workspace/mypy_extensions_internal/BUILD.bazel

@@ -0,0 +1,8 @@
+# -*- python -*-
+
+# This file exists to make our directory into a Bazel package, so that our
+# neighboring *.bzl file can be loaded elsewhere.
+
+load("//tools/lint:lint.bzl", "add_lint_tests")
+
+add_lint_tests()

tools/workspace/mypy_extensions_internal/package.BUILD.bazel

@@ -0,0 +1,12 @@
+# -*- python -*-
+
+load("@drake//tools/skylark:py.bzl", "py_library")
+
+licenses(["notice"])  # MIT
+
+py_library(
+    name = "mypy_extensions",
+    srcs = ["mypy_extensions.py"],
+    imports = ["."],
+    visibility = ["//visibility:public"],
+)

tools/workspace/mypy_extensions_internal/repository.bzl

@@ -0,0 +1,15 @@
+# -*- python -*-
+
+load("@drake//tools/workspace:github.bzl", "github_archive")
+
+def mypy_extensions_internal_repository(
+        name,
+        mirrors = None):
+    github_archive(
+        name = name,
+        repository = "python/mypy_extensions",
+        commit = "0.4.3",
+        sha256 = "21e830f4baf996d0a9bd7048a5b23722dbd3b88354b8bf0e22376f9a8d508c16",  # noqa
+        build_file = ":package.BUILD.bazel",
+        mirrors = mirrors,
+    )

tools/workspace/mypy_internal/BUILD.bazel

@@ -0,0 +1,8 @@
+# -*- python -*-
+
+# This file exists to make our directory into a Bazel package, so that our
+# neighboring *.bzl file can be loaded elsewhere.
+
+load("//tools/lint:lint.bzl", "add_lint_tests")
+
+add_lint_tests()

tools/workspace/mypy_internal/package.BUILD.bazel

@@ -0,0 +1,17 @@
+# -*- python -*-
+
+load("@drake//tools/skylark:py.bzl", "py_library")
+
+licenses(["notice"])  # Python-2.0
+
+py_library(
+    name = "mypy",
+    srcs = glob(["mypy/**/*.py"]),
+    data = glob(["mypy/typeshed/**"]),
+    visibility = ["//visibility:public"],
+    deps = [
+        "@mypy_extensions_internal//:mypy_extensions",
+        "@tomli_internal//:tomli",
+        "@typing_extensions_internal//:typing_extensions",
+    ],
+)

tools/workspace/mypy_internal/repository.bzl

@@ -0,0 +1,16 @@
+# -*- python -*-
+
+load("@drake//tools/workspace:github.bzl", "github_archive")
+
+def mypy_internal_repository(
+        name,
+        mirrors = None):
+    github_archive(
+        name = name,
+        repository = "python/mypy",
+        # TODO(mwoehlke-kitware): switch to a tag >= v0.980.
+        commit = "3ae19a25f0a39358ede1383e93d44ef9abf165e0",
+        sha256 = "866503aed58d7207c0fe4bca9a6a51c1eaa6668157b1c5ed61b40eda71af9175",  # noqa
+        build_file = ":package.BUILD.bazel",
+        mirrors = mirrors,
+    )

tools/workspace/tomli_internal/BUILD.bazel

@@ -0,0 +1,8 @@
+# -*- python -*-
+
+# This file exists to make our directory into a Bazel package, so that our
+# neighboring *.bzl file can be loaded elsewhere.
+
+load("//tools/lint:lint.bzl", "add_lint_tests")
+
+add_lint_tests()

tools/workspace/tomli_internal/package.BUILD.bazel

@@ -0,0 +1,12 @@
+# -*- python -*-
+
+load("@drake//tools/skylark:py.bzl", "py_library")
+
+licenses(["notice"])  # MIT
+
+py_library(
+    name = "tomli",
+    srcs = glob(["src/tomli/*.py"]),
+    imports = ["src"],
+    visibility = ["//visibility:public"],
+)

tools/workspace/tomli_internal/repository.bzl

@@ -0,0 +1,15 @@
+# -*- python -*-
+
+load("@drake//tools/workspace:github.bzl", "github_archive")
+
+def tomli_internal_repository(
+        name,
+        mirrors = None):
+    github_archive(
+        name = name,
+        repository = "hukkin/tomli",
+        commit = "2.0.1",
+        sha256 = "ad22dbc128623e0c156ffaff019f29f456eba8a5d5a05164dd34f63e560449df",  # noqa
+        build_file = ":package.BUILD.bazel",
+        mirrors = mirrors,
+    )

tools/workspace/typing_extensions_internal/BUILD.bazel

@@ -0,0 +1,8 @@
+# -*- python -*-
+
+# This file exists to make our directory into a Bazel package, so that our
+# neighboring *.bzl file can be loaded elsewhere.
+
+load("//tools/lint:lint.bzl", "add_lint_tests")
+
+add_lint_tests()

tools/workspace/typing_extensions_internal/package.BUILD.bazel

@@ -0,0 +1,12 @@
+# -*- python -*-
+
+load("@drake//tools/skylark:py.bzl", "py_library")
+
+licenses(["notice"])  # Python-2.0
+
+py_library(
+    name = "typing_extensions",
+    srcs = ["src/typing_extensions.py"],
+    imports = ["src"],
+    visibility = ["//visibility:public"],
+)

tools/workspace/typing_extensions_internal/repository.bzl

@@ -0,0 +1,15 @@
+# -*- python -*-
+
+load("@drake//tools/workspace:github.bzl", "github_archive")
+
+def typing_extensions_internal_repository(
+        name,
+        mirrors = None):
+    github_archive(
+        name = name,
+        repository = "python/typing_extensions",
+        commit = "4.3.0",
+        sha256 = "9dbc928aed2839a23d210726697700a1c4593ab3bbf82b981fcc44585a47ce30",  # noqa
+        build_file = ":package.BUILD.bazel",
+        mirrors = mirrors,
+    )