From 5e1c0c857e00e6da37257d0bc68c263008c702d0 Mon Sep 17 00:00:00 2001 From: Kyle Gentle Date: Thu, 4 Aug 2022 22:38:03 -0400 Subject: [PATCH] pyright type checking: Initial setup * Update imports to use fully qualified module name * Add `typecheck` target to Makefile * Add type annotations to appease pyright * Switch from importlib_resources backport library to stdlib import.resources --- Makefile | 5 ++++- etc/api/type-cli.yaml | 2 +- etc/bin/mako-render | 3 ++- requirements.txt | 3 ++- src/mako/Cargo.toml.mako | 2 +- src/mako/LICENSE.md.mako | 2 +- src/mako/api/README.md.mako | 4 ++-- src/mako/api/api.rs.mako | 2 +- src/mako/api/lib.rs.mako | 4 ++-- src/mako/api/lib/lib.mako | 2 +- src/mako/api/lib/mbuild.mako | 2 +- src/mako/api/lib/rbuild.mako | 2 +- src/mako/api/lib/schema.mako | 2 +- src/mako/cli/README.md.mako | 6 +++--- src/mako/cli/docs/commands.md.mako | 6 +++--- src/mako/cli/lib/argparse.mako | 4 ++-- src/mako/cli/lib/engine.mako | 4 ++-- src/mako/cli/main.rs.mako | 4 ++-- src/mako/cli/mkdocs.yml.mako | 4 ++-- src/mako/deps.mako | 4 ++-- src/mako/index.html.mako | 2 +- src/mako/lib/cli.py | 6 ++++-- src/mako/lib/util.mako | 4 ++-- src/mako/lib/util.py | 17 ++++++----------- src/mako/lib/util_test.py | 15 +++++++++------ 25 files changed, 58 insertions(+), 53 deletions(-) diff --git a/Makefile b/Makefile index ccd8f23409..7c27213f26 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ endif API_JSON_FILES = $(shell find etc -type f -name '*-api.json') MAKO_LIB_DIR = $(MAKO_SRC)/lib MAKO_LIB_FILES = $(shell find $(MAKO_LIB_DIR) -type f -name '*.*') -MAKO = export PREPROC=$(PREPROC); export PYTHONPATH=$(MAKO_LIB_DIR):$(PYTHONPATH); $(TPL) --template-dir '.' +MAKO = export PREPROC=$(PREPROC); export PYTHONPATH=$(MAKO_SRC):$(PYTHONPATH); $(TPL) --template-dir '.' MAKO_STANDARD_DEPENDENCIES = $(API_SHARED_INFO) $(MAKO_LIB_FILES) $(MAKO_RENDER) $(PREPROC) help: @@ -98,6 +98,9 @@ test-gen: $(PYTHON_BIN) test: test-gen +typecheck: $(PYTHON_BIN) + $(PYTHON) -m pyright $(MAKO_LIB_DIR) + clean: clean-all-api clean-all-cli docs-all-clean -rm -Rf $(VENV_DIR) -rm $(API_DEPS) $(CLI_DEPS) diff --git a/etc/api/type-cli.yaml b/etc/api/type-cli.yaml index 70752edb3f..29b50c212c 100644 --- a/etc/api/type-cli.yaml +++ b/etc/api/type-cli.yaml @@ -5,7 +5,7 @@ mkdocs: # if docs_dir changes, remember to update the sources as well. docs_dir: docs mako: - post_processor_module: cli + post_processor_module: "lib.cli" make: id: cli target_name: CLIs diff --git a/etc/bin/mako-render b/etc/bin/mako-render index 99a6eb342b..20ed422307 100644 --- a/etc/bin/mako-render +++ b/etc/bin/mako-render @@ -4,6 +4,7 @@ # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php from argparse import ArgumentParser +from importlib import import_module from os.path import isfile, dirname import os import sys @@ -289,7 +290,7 @@ def cmdline(argv=None): post_processor = lambda r, of: r if options.post_process_python_module: fn_name = 'process_template_result' - pm = __import__(options.post_process_python_module, globals(), locals(), []) + pm = import_module(options.post_process_python_module) post_processor = getattr(pm, fn_name, None) if post_processor is None: raise AssertionError("python module '%s' must have a function called '%s'" diff --git a/requirements.txt b/requirements.txt index 4474a85bc9..7ef6fdf2a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ pyyaml<6 mkdocs==0.11 pytest pytest-cov -importlib_resources codecov ghp-import +pyright +types-PyYAML diff --git a/src/mako/Cargo.toml.mako b/src/mako/Cargo.toml.mako index ee41cfe758..740398e684 100644 --- a/src/mako/Cargo.toml.mako +++ b/src/mako/Cargo.toml.mako @@ -1,4 +1,4 @@ -<%! from util import (estr, enclose_in, hash_comment, library_to_crate_name, to_extern_crate_name) %>\ +<%! from lib.util import (estr, enclose_in, hash_comment, library_to_crate_name, to_extern_crate_name) %>\ <%namespace name="util" file="lib/util.mako"/>\ <%block filter="hash_comment">\ <%util:gen_info source="${self.uri}" />\ diff --git a/src/mako/LICENSE.md.mako b/src/mako/LICENSE.md.mako index 7c51da3999..9f8bc47fe9 100644 --- a/src/mako/LICENSE.md.mako +++ b/src/mako/LICENSE.md.mako @@ -1,5 +1,5 @@ ## -*- coding: utf-8 -*- -<%! import util %>\ +<%! import lib.util as util %>\ <%namespace name="mutil" file="lib/util.mako"/>\ <%block filter="util.markdown_comment">\ <%mutil:gen_info source="${self.uri}" />\ diff --git a/src/mako/api/README.md.mako b/src/mako/api/README.md.mako index 88b6429e0a..2a14bb6f5d 100644 --- a/src/mako/api/README.md.mako +++ b/src/mako/api/README.md.mako @@ -1,5 +1,5 @@ <% - from util import (markdown_comment, new_context) + from lib.util import (markdown_comment, new_context) c = new_context(schemas, resources, context.get('methods')) %>\ <%namespace name="lib" file="lib/lib.mako"/>\ @@ -10,4 +10,4 @@ The `${util.crate_name()}` library allows access to all features of the *Google ${util.canonical_name()}* service. ${lib.docs(c, rust_doc=False)} -<%lib:license /> \ No newline at end of file +<%lib:license /> diff --git a/src/mako/api/api.rs.mako b/src/mako/api/api.rs.mako index 7d0d46c4de..750ded94bb 100644 --- a/src/mako/api/api.rs.mako +++ b/src/mako/api/api.rs.mako @@ -4,7 +4,7 @@ <%namespace name="mbuild" file="lib/mbuild.mako"/>\ <%namespace name="schema" file="lib/schema.mako"/>\ <% - from util import (new_context, rust_comment, rust_doc_comment, rust_module_doc_comment, + from lib.util import (new_context, rust_comment, rust_doc_comment, rust_module_doc_comment, rb_type, hub_type, mangle_ident, hub_type_params_s, rb_type_params_s, find_fattest_resource, HUB_TYPE_PARAMETERS, METHODS_RESOURCE, UNUSED_TYPE_MARKER, schema_markers) diff --git a/src/mako/api/lib.rs.mako b/src/mako/api/lib.rs.mako index 4365a23c98..38a4c1fb59 100644 --- a/src/mako/api/lib.rs.mako +++ b/src/mako/api/lib.rs.mako @@ -1,7 +1,7 @@ <%namespace name="lib" file="lib/lib.mako"/>\ <%namespace name="util" file="../lib/util.mako"/>\ <% - from util import (new_context, rust_comment, rust_module_doc_comment) + from lib.util import (new_context, rust_comment, rust_module_doc_comment) c = new_context(schemas, resources, context.get('methods')) %>\ @@ -25,7 +25,7 @@ ${lib.docs(c)} <%namespace name="mbuild" file="lib/mbuild.mako"/>\ <%namespace name="schema" file="lib/schema.mako"/>\ <% - from util import (new_context, rust_comment, rust_doc_comment, rust_module_doc_comment, + from lib.util import (new_context, rust_comment, rust_doc_comment, rust_module_doc_comment, rb_type, hub_type, mangle_ident, hub_type_params_s, rb_type_params_s, find_fattest_resource, HUB_TYPE_PARAMETERS, METHODS_RESOURCE, UNUSED_TYPE_MARKER, schema_markers) diff --git a/src/mako/api/lib/lib.mako b/src/mako/api/lib/lib.mako index 70de2bc7d6..fc35b8667f 100644 --- a/src/mako/api/lib/lib.mako +++ b/src/mako/api/lib/lib.mako @@ -1,5 +1,5 @@ <%! - from util import (activity_split, put_and, md_italic, split_camelcase_s, canonical_type_name, hub_type, + from lib.util import (activity_split, put_and, md_italic, split_camelcase_s, canonical_type_name, hub_type, rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment, markdown_rust_block, unindent_first_by, mangle_ident, mb_type, singular, scope_url_to_variant, PART_MARKER_TRAIT, RESOURCE_MARKER_TRAIT, CALL_BUILDER_MARKERT_TRAIT, diff --git a/src/mako/api/lib/mbuild.mako b/src/mako/api/lib/mbuild.mako index d0251df5fe..b904ea6327 100644 --- a/src/mako/api/lib/mbuild.mako +++ b/src/mako/api/lib/mbuild.mako @@ -1,5 +1,5 @@ <%! - from util import (put_and, rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment, + from lib.util import (put_and, rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment, rb_type, mb_type, singular, hub_type, to_fqan, indent_all_but_first_by, activity_rust_type, mangle_ident, activity_input_type, get_word, split_camelcase_s, property, is_pod_property, TREF, IO_REQUEST, diff --git a/src/mako/api/lib/rbuild.mako b/src/mako/api/lib/rbuild.mako index 8139db3286..05b6760a92 100644 --- a/src/mako/api/lib/rbuild.mako +++ b/src/mako/api/lib/rbuild.mako @@ -1,5 +1,5 @@ <%! - from util import (put_and, rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment, + from lib.util import (put_and, rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment, rb_type, singular, hub_type, mangle_ident, mb_type, property, to_fqan, indent_all_but_first_by, is_repeated_property, is_required_property, activity_input_type, TREF, IO_REQUEST, schema_to_required_property, diff --git a/src/mako/api/lib/schema.mako b/src/mako/api/lib/schema.mako index b8e461207f..51e24e13eb 100644 --- a/src/mako/api/lib/schema.mako +++ b/src/mako/api/lib/schema.mako @@ -1,5 +1,5 @@ <%! - from util import (schema_markers, rust_doc_comment, mangle_ident, to_rust_type, put_and, + from lib.util import (schema_markers, rust_doc_comment, mangle_ident, to_rust_type, put_and, IO_TYPES, activity_split, enclose_in, REQUEST_MARKER_TRAIT, mb_type, indent_all_but_first_by, NESTED_TYPE_SUFFIX, RESPONSE_MARKER_TRAIT, split_camelcase_s, METHODS_RESOURCE, PART_MARKER_TRAIT, canonical_type_name, TO_PARTS_MARKER, UNUSED_TYPE_MARKER, is_schema_with_optionals, diff --git a/src/mako/cli/README.md.mako b/src/mako/cli/README.md.mako index 05ce684201..b8022074da 100644 --- a/src/mako/cli/README.md.mako +++ b/src/mako/cli/README.md.mako @@ -1,6 +1,6 @@ <% - from util import (markdown_comment, new_context) - from cli import (CONFIG_DIR, CONFIG_DIR_FLAG, SCOPE_FLAG, application_secret_path, DEBUG_FLAG) + from lib.util import (markdown_comment, new_context) + from lib.cli import (CONFIG_DIR, CONFIG_DIR_FLAG, SCOPE_FLAG, application_secret_path, DEBUG_FLAG) c = new_context(schemas, resources, context.get('methods')) %>\ @@ -96,4 +96,4 @@ You may consider redirecting standard error into a file for ease of use, e.g. `$ [scopes]: https://developers.google.com/+/api/oauth#scopes [revoke-access]: http://webapps.stackexchange.com/a/30849 [google-dev-console]: https://console.developers.google.com/ -[google-project-new]: https://developers.google.com/console/help/new/ \ No newline at end of file +[google-project-new]: https://developers.google.com/console/help/new/ diff --git a/src/mako/cli/docs/commands.md.mako b/src/mako/cli/docs/commands.md.mako index 4ada130904..e656cdd47f 100644 --- a/src/mako/cli/docs/commands.md.mako +++ b/src/mako/cli/docs/commands.md.mako @@ -1,8 +1,8 @@ <%namespace name="util" file="../../lib/util.mako"/>\ <%! from mako.filters import xml_escape - from util import (hash_comment, new_context, method_default_scope, indent_all_but_first_by, is_repeated_property, custom_sorted) - from cli import (subcommand_md_filename, new_method_context, SPLIT_START, SPLIT_END, pretty, SCOPE_FLAG, + from lib.util import (hash_comment, new_context, method_default_scope, indent_all_but_first_by, is_repeated_property, custom_sorted) + from lib.cli import (subcommand_md_filename, new_method_context, SPLIT_START, SPLIT_END, pretty, SCOPE_FLAG, mangle_subcommand, is_request_value_property, FIELD_SEP, PARAM_FLAG, UPLOAD_FLAG, docopt_mode, FILE_ARG, MIME_ARG, OUT_ARG, OUTPUT_FLAG, to_cli_schema, cli_schema_to_yaml, SchemaEntry, STRUCT_FLAG, field_to_value, CTYPE_ARRAY, CTYPE_MAP, to_docopt_arg, FILE_FLAG, MIME_FLAG, @@ -220,4 +220,4 @@ ${self._list_schem_args(f, cursor_tokens, first_flag)} %>\ % endif % endfor - \ No newline at end of file + diff --git a/src/mako/cli/lib/argparse.mako b/src/mako/cli/lib/argparse.mako index 07a03ac6e6..f4ffb8b29e 100644 --- a/src/mako/cli/lib/argparse.mako +++ b/src/mako/cli/lib/argparse.mako @@ -2,8 +2,8 @@ <%! import os - from util import (put_and, supports_scopes, api_index, indent_by, enclose_in, put_and, escape_rust_string) - from cli import (mangle_subcommand, new_method_context, PARAM_FLAG, STRUCT_FLAG, UPLOAD_FLAG, OUTPUT_FLAG, VALUE_ARG, + from lib.util import (put_and, supports_scopes, api_index, indent_by, enclose_in, put_and, escape_rust_string) + from lib.cli import (mangle_subcommand, new_method_context, PARAM_FLAG, STRUCT_FLAG, UPLOAD_FLAG, OUTPUT_FLAG, VALUE_ARG, CONFIG_DIR, SCOPE_FLAG, is_request_value_property, FIELD_SEP, docopt_mode, FILE_ARG, MIME_ARG, OUT_ARG, CONFIG_DIR_FLAG, KEY_VALUE_ARG, to_docopt_arg, DEBUG_FLAG, MODE_ARG, SCOPE_ARG, CONFIG_DIR_ARG, FILE_FLAG, MIME_FLAG, subcommand_md_filename) diff --git a/src/mako/cli/lib/engine.mako b/src/mako/cli/lib/engine.mako index 11201ceaa7..dfdc68342b 100644 --- a/src/mako/cli/lib/engine.mako +++ b/src/mako/cli/lib/engine.mako @@ -1,9 +1,9 @@ <%namespace name="util" file="../../lib/util.mako"/>\ <%! - from util import (hub_type, mangle_ident, indent_all_but_first_by, activity_rust_type, setter_fn_name, ADD_PARAM_FN, + from lib.util import (hub_type, mangle_ident, indent_all_but_first_by, activity_rust_type, setter_fn_name, ADD_PARAM_FN, upload_action_fn, is_schema_with_optionals, schema_markers, indent_by, method_default_scope, ADD_SCOPE_FN, TREF, enclose_in) - from cli import (mangle_subcommand, new_method_context, PARAM_FLAG, STRUCT_FLAG, OUTPUT_FLAG, VALUE_ARG, + from lib.cli import (mangle_subcommand, new_method_context, PARAM_FLAG, STRUCT_FLAG, OUTPUT_FLAG, VALUE_ARG, CONFIG_DIR, SCOPE_FLAG, is_request_value_property, FIELD_SEP, docopt_mode, FILE_ARG, MIME_ARG, OUT_ARG, call_method_ident, POD_TYPES, opt_value, ident, JSON_TYPE_VALUE_MAP, KEY_VALUE_ARG, to_cli_schema, SchemaEntry, CTYPE_POD, actual_json_type, CTYPE_MAP, CTYPE_ARRAY, diff --git a/src/mako/cli/main.rs.mako b/src/mako/cli/main.rs.mako index 0cb4c58734..d407982e05 100644 --- a/src/mako/cli/main.rs.mako +++ b/src/mako/cli/main.rs.mako @@ -2,9 +2,9 @@ <%namespace name="engine" file="lib/engine.mako"/>\ <%namespace name="util" file="../lib/util.mako"/>\ <% - from util import (new_context, rust_comment, to_extern_crate_name, library_to_crate_name, library_name, + from lib.util import (new_context, rust_comment, to_extern_crate_name, library_to_crate_name, library_name, indent_all_but_first_by) - from cli import OUT_ARG, DEBUG_FLAG, opt_value + from lib.cli import OUT_ARG, DEBUG_FLAG, opt_value c = new_context(schemas, resources, context.get('methods')) default_user_agent = "google-cli-rust-client/" + cargo.build_version diff --git a/src/mako/cli/mkdocs.yml.mako b/src/mako/cli/mkdocs.yml.mako index 4c59257db6..d3692abbb3 100644 --- a/src/mako/cli/mkdocs.yml.mako +++ b/src/mako/cli/mkdocs.yml.mako @@ -1,6 +1,6 @@ <% - from util import (put_and, new_context) - from cli import (subcommand_md_filename, mangle_subcommand, pretty) + from lib.util import (put_and, new_context) + from lib.cli import (subcommand_md_filename, mangle_subcommand, pretty) c = new_context(schemas, resources, context.get('methods')) %>\ diff --git a/src/mako/deps.mako b/src/mako/deps.mako index dc627d71ee..e24bdb1a5d 100644 --- a/src/mako/deps.mako +++ b/src/mako/deps.mako @@ -58,7 +58,7 @@ <% continue %>\ % endif <% - import util + import lib.util as util import os import json @@ -191,7 +191,7 @@ help${agsuffix}: % for info in (apis.get('items') or []): <% - import util + import lib.util as util import os name = util.normalize_library_name(info['name']) target = util.api_json_path(directories.api_base, name, info['version']) diff --git a/src/mako/index.html.mako b/src/mako/index.html.mako index b07e32e66f..5d8058af61 100644 --- a/src/mako/index.html.mako +++ b/src/mako/index.html.mako @@ -2,7 +2,7 @@ import json import os import yaml - from util import (api_json_path, library_name, library_to_crate_name, + from lib.util import (api_json_path, library_name, library_to_crate_name, gen_crate_dir, api_index, crates_io_url, program_name, crate_version) diff --git a/src/mako/lib/cli.py b/src/mako/lib/cli.py index 215f0f1806..ee8b9c36ef 100644 --- a/src/mako/lib/cli.py +++ b/src/mako/lib/cli.py @@ -1,4 +1,4 @@ -import util +import lib.util as util import os import re @@ -239,7 +239,7 @@ def field_to_value(f): return v # split the result along split segments -def process_template_result(r, output_file): +def process_template_result(r, output_file: str): found = False dir = None if output_file: @@ -249,6 +249,8 @@ def process_template_result(r, output_file): # end handle output directory for m in re_splitters.finditer(r): + if not dir: + raise RuntimeError("Missing directory; was output_file specified?") found = True fh = open(os.path.join(dir, m.group(1)), 'wb') fh.write(m.group(2).encode('UTF-8')) diff --git a/src/mako/lib/util.mako b/src/mako/lib/util.mako index a0dff1b34f..8e58910650 100644 --- a/src/mako/lib/util.mako +++ b/src/mako/lib/util.mako @@ -1,4 +1,4 @@ -<%! import util %>\ +<%! import lib.util as util %>\ ## source should be ${self.uri} ## you need to escape the output, using a filter for example @@ -58,4 +58,4 @@ ${canonicalName}\ <%def name="program_name()" buffered="True">\ ${util.program_name(name, version)}\ - \ No newline at end of file + diff --git a/src/mako/lib/util.py b/src/mako/lib/util.py index 08df1d9aaf..c6c78a8bf5 100644 --- a/src/mako/lib/util.py +++ b/src/mako/lib/util.py @@ -1,6 +1,7 @@ import re import os from random import (randint, random, choice, seed) +from typing import Any, List, Mapping, Optional, Tuple import collections from copy import deepcopy import subprocess @@ -10,9 +11,9 @@ seed(1337) re_linestart = re.compile('^', flags=re.MULTILINE) re_spaces_after_newline = re.compile('^ {4}', flags=re.MULTILINE) re_first_4_spaces = re.compile('^ {1,4}', flags=re.MULTILINE) -re_desc_parts = re.compile("((the part (names|properties) that you can include in the parameter value are)|(supported values are ))(.*?)\.", flags=re.IGNORECASE|re.MULTILINE) +re_desc_parts = re.compile(r"((the part (names|properties) that you can include in the parameter value are)|(supported values are ))(.*?)\.", flags=re.IGNORECASE|re.MULTILINE) -re_find_replacements = re.compile("\{[/\+]?\w+\*?\}") +re_find_replacements = re.compile(r"\{[/\+]?\w+\*?\}") HTTP_METHODS = set(("OPTIONS", "GET", "POST", "PUT", "DELETE", "HEAD", "TRACE", "CONNECT", "PATCH" )) @@ -120,9 +121,7 @@ def items(p): else: return p._items() -def custom_sorted(p): - if not isinstance(p, list): - assert(false, p, "unexpected type") +def custom_sorted(p: List[Mapping[str, Any]]) -> List[Mapping[str, Any]]: return sorted(p, key = lambda p: p['name']) # ============================================================================== @@ -491,7 +490,7 @@ def is_schema_with_optionals(schema_markers): ## @name Activity Utilities # @{ # return (category, name|None, method) -def activity_split(fqan): +def activity_split(fqan: str) -> Tuple[str, Optional[str], str]: t = fqan.split('.') mt = t[2:] if not mt: @@ -513,10 +512,6 @@ def to_fqan(name, resource, method): def activity_name_to_type_name(an): return canonical_type_name(an)[:-1] -# yields (category, resource, activity, activity_data) -def iter_acitivities(c): - return ((activity_split(an) + [a]) for an, a in c.fqan_map.items()) - # return a list of parameter structures of all params of the given method dict # apply a prune filter to restrict the set of returned parameters. # The order will always be: partOrder + alpha @@ -810,7 +805,7 @@ def _is_special_version(v): return v.endswith('alpha') or v.endswith('beta') def to_api_version(v): - m = re.search("_?v(\d(\.\d)*)_?", v) + m = re.search(r"_?v(\d(\.\d)*)_?", v) if not m and _is_special_version(v): return v assert m, "Expected to find a version within '%s'" % v diff --git a/src/mako/lib/util_test.py b/src/mako/lib/util_test.py index 395f554e11..e0f6661959 100644 --- a/src/mako/lib/util_test.py +++ b/src/mako/lib/util_test.py @@ -1,15 +1,18 @@ #!/usr/bin/env python +import importlib.resources import unittest import json -import importlib_resources -from .util import to_api_version, library_name, re_find_replacements, to_rust_type, new_context -from . import test_data +from lib.util import to_api_version, library_name, re_find_replacements, to_rust_type, new_context +import lib.test_data as test_data -def read_test_json_file(resource): - data = importlib_resources.read_text(test_data, resource) +TEST_JSON_FILE = "photoslibrary-api.json" + + +def read_test_json_file(): + data = importlib.resources.read_text(test_data, TEST_JSON_FILE) return json.loads(data) class UtilsTest(unittest.TestCase): @@ -65,7 +68,7 @@ class UtilsTest(unittest.TestCase): self.assertEqual(ms[0], '{+project}') def test_to_rust_type(self): - full_api_schema = read_test_json_file('photoslibrary-api.json') + full_api_schema = read_test_json_file() schemas = full_api_schema['schemas']