From 5e1c0c857e00e6da37257d0bc68c263008c702d0 Mon Sep 17 00:00:00 2001 From: Kyle Gentle Date: Thu, 4 Aug 2022 22:38:03 -0400 Subject: [PATCH 1/5] pyright type checking: Initial setup * Update imports to use fully qualified module name * Add `typecheck` target to Makefile * Add type annotations to appease pyright * Switch from importlib_resources backport library to stdlib import.resources --- Makefile | 5 ++++- etc/api/type-cli.yaml | 2 +- etc/bin/mako-render | 3 ++- requirements.txt | 3 ++- src/mako/Cargo.toml.mako | 2 +- src/mako/LICENSE.md.mako | 2 +- src/mako/api/README.md.mako | 4 ++-- src/mako/api/api.rs.mako | 2 +- src/mako/api/lib.rs.mako | 4 ++-- src/mako/api/lib/lib.mako | 2 +- src/mako/api/lib/mbuild.mako | 2 +- src/mako/api/lib/rbuild.mako | 2 +- src/mako/api/lib/schema.mako | 2 +- src/mako/cli/README.md.mako | 6 +++--- src/mako/cli/docs/commands.md.mako | 6 +++--- src/mako/cli/lib/argparse.mako | 4 ++-- src/mako/cli/lib/engine.mako | 4 ++-- src/mako/cli/main.rs.mako | 4 ++-- src/mako/cli/mkdocs.yml.mako | 4 ++-- src/mako/deps.mako | 4 ++-- src/mako/index.html.mako | 2 +- src/mako/lib/cli.py | 6 ++++-- src/mako/lib/util.mako | 4 ++-- src/mako/lib/util.py | 17 ++++++----------- src/mako/lib/util_test.py | 15 +++++++++------ 25 files changed, 58 insertions(+), 53 deletions(-) diff --git a/Makefile b/Makefile index ccd8f23409..7c27213f26 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ endif API_JSON_FILES = $(shell find etc -type f -name '*-api.json') MAKO_LIB_DIR = $(MAKO_SRC)/lib MAKO_LIB_FILES = $(shell find $(MAKO_LIB_DIR) -type f -name '*.*') -MAKO = export PREPROC=$(PREPROC); export PYTHONPATH=$(MAKO_LIB_DIR):$(PYTHONPATH); $(TPL) --template-dir '.' +MAKO = export PREPROC=$(PREPROC); export PYTHONPATH=$(MAKO_SRC):$(PYTHONPATH); $(TPL) --template-dir '.' MAKO_STANDARD_DEPENDENCIES = $(API_SHARED_INFO) $(MAKO_LIB_FILES) $(MAKO_RENDER) $(PREPROC) help: @@ -98,6 +98,9 @@ test-gen: $(PYTHON_BIN) test: test-gen +typecheck: $(PYTHON_BIN) + $(PYTHON) -m pyright $(MAKO_LIB_DIR) + clean: clean-all-api clean-all-cli docs-all-clean -rm -Rf $(VENV_DIR) -rm $(API_DEPS) $(CLI_DEPS) diff --git a/etc/api/type-cli.yaml b/etc/api/type-cli.yaml index 70752edb3f..29b50c212c 100644 --- a/etc/api/type-cli.yaml +++ b/etc/api/type-cli.yaml @@ -5,7 +5,7 @@ mkdocs: # if docs_dir changes, remember to update the sources as well. docs_dir: docs mako: - post_processor_module: cli + post_processor_module: "lib.cli" make: id: cli target_name: CLIs diff --git a/etc/bin/mako-render b/etc/bin/mako-render index 99a6eb342b..20ed422307 100644 --- a/etc/bin/mako-render +++ b/etc/bin/mako-render @@ -4,6 +4,7 @@ # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php from argparse import ArgumentParser +from importlib import import_module from os.path import isfile, dirname import os import sys @@ -289,7 +290,7 @@ def cmdline(argv=None): post_processor = lambda r, of: r if options.post_process_python_module: fn_name = 'process_template_result' - pm = __import__(options.post_process_python_module, globals(), locals(), []) + pm = import_module(options.post_process_python_module) post_processor = getattr(pm, fn_name, None) if post_processor is None: raise AssertionError("python module '%s' must have a function called '%s'" diff --git a/requirements.txt b/requirements.txt index 4474a85bc9..7ef6fdf2a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ pyyaml<6 mkdocs==0.11 pytest pytest-cov -importlib_resources codecov ghp-import +pyright +types-PyYAML diff --git a/src/mako/Cargo.toml.mako b/src/mako/Cargo.toml.mako index ee41cfe758..740398e684 100644 --- a/src/mako/Cargo.toml.mako +++ b/src/mako/Cargo.toml.mako @@ -1,4 +1,4 @@ -<%! from util import (estr, enclose_in, hash_comment, library_to_crate_name, to_extern_crate_name) %>\ +<%! from lib.util import (estr, enclose_in, hash_comment, library_to_crate_name, to_extern_crate_name) %>\ <%namespace name="util" file="lib/util.mako"/>\ <%block filter="hash_comment">\ <%util:gen_info source="${self.uri}" />\ diff --git a/src/mako/LICENSE.md.mako b/src/mako/LICENSE.md.mako index 7c51da3999..9f8bc47fe9 100644 --- a/src/mako/LICENSE.md.mako +++ b/src/mako/LICENSE.md.mako @@ -1,5 +1,5 @@ ## -*- coding: utf-8 -*- -<%! import util %>\ +<%! import lib.util as util %>\ <%namespace name="mutil" file="lib/util.mako"/>\ <%block filter="util.markdown_comment">\ <%mutil:gen_info source="${self.uri}" />\ diff --git a/src/mako/api/README.md.mako b/src/mako/api/README.md.mako index 88b6429e0a..2a14bb6f5d 100644 --- a/src/mako/api/README.md.mako +++ b/src/mako/api/README.md.mako @@ -1,5 +1,5 @@ <% - from util import (markdown_comment, new_context) + from lib.util import (markdown_comment, new_context) c = new_context(schemas, resources, context.get('methods')) %>\ <%namespace name="lib" file="lib/lib.mako"/>\ @@ -10,4 +10,4 @@ The `${util.crate_name()}` library allows access to all features of the *Google ${util.canonical_name()}* service. ${lib.docs(c, rust_doc=False)} -<%lib:license /> \ No newline at end of file +<%lib:license /> diff --git a/src/mako/api/api.rs.mako b/src/mako/api/api.rs.mako index 7d0d46c4de..750ded94bb 100644 --- a/src/mako/api/api.rs.mako +++ b/src/mako/api/api.rs.mako @@ -4,7 +4,7 @@ <%namespace name="mbuild" file="lib/mbuild.mako"/>\ <%namespace name="schema" file="lib/schema.mako"/>\ <% - from util import (new_context, rust_comment, rust_doc_comment, rust_module_doc_comment, + from lib.util import (new_context, rust_comment, rust_doc_comment, rust_module_doc_comment, rb_type, hub_type, mangle_ident, hub_type_params_s, rb_type_params_s, find_fattest_resource, HUB_TYPE_PARAMETERS, METHODS_RESOURCE, UNUSED_TYPE_MARKER, schema_markers) diff --git a/src/mako/api/lib.rs.mako b/src/mako/api/lib.rs.mako index 4365a23c98..38a4c1fb59 100644 --- a/src/mako/api/lib.rs.mako +++ b/src/mako/api/lib.rs.mako @@ -1,7 +1,7 @@ <%namespace name="lib" file="lib/lib.mako"/>\ <%namespace name="util" file="../lib/util.mako"/>\ <% - from util import (new_context, rust_comment, rust_module_doc_comment) + from lib.util import (new_context, rust_comment, rust_module_doc_comment) c = new_context(schemas, resources, context.get('methods')) %>\ @@ -25,7 +25,7 @@ ${lib.docs(c)} <%namespace name="mbuild" file="lib/mbuild.mako"/>\ <%namespace name="schema" file="lib/schema.mako"/>\ <% - from util import (new_context, rust_comment, rust_doc_comment, rust_module_doc_comment, + from lib.util import (new_context, rust_comment, rust_doc_comment, rust_module_doc_comment, rb_type, hub_type, mangle_ident, hub_type_params_s, rb_type_params_s, find_fattest_resource, HUB_TYPE_PARAMETERS, METHODS_RESOURCE, UNUSED_TYPE_MARKER, schema_markers) diff --git a/src/mako/api/lib/lib.mako b/src/mako/api/lib/lib.mako index 70de2bc7d6..fc35b8667f 100644 --- a/src/mako/api/lib/lib.mako +++ b/src/mako/api/lib/lib.mako @@ -1,5 +1,5 @@ <%! - from util import (activity_split, put_and, md_italic, split_camelcase_s, canonical_type_name, hub_type, + from lib.util import (activity_split, put_and, md_italic, split_camelcase_s, canonical_type_name, hub_type, rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment, markdown_rust_block, unindent_first_by, mangle_ident, mb_type, singular, scope_url_to_variant, PART_MARKER_TRAIT, RESOURCE_MARKER_TRAIT, CALL_BUILDER_MARKERT_TRAIT, diff --git a/src/mako/api/lib/mbuild.mako b/src/mako/api/lib/mbuild.mako index d0251df5fe..b904ea6327 100644 --- a/src/mako/api/lib/mbuild.mako +++ b/src/mako/api/lib/mbuild.mako @@ -1,5 +1,5 @@ <%! - from util import (put_and, rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment, + from lib.util import (put_and, rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment, rb_type, mb_type, singular, hub_type, to_fqan, indent_all_but_first_by, activity_rust_type, mangle_ident, activity_input_type, get_word, split_camelcase_s, property, is_pod_property, TREF, IO_REQUEST, diff --git a/src/mako/api/lib/rbuild.mako b/src/mako/api/lib/rbuild.mako index 8139db3286..05b6760a92 100644 --- a/src/mako/api/lib/rbuild.mako +++ b/src/mako/api/lib/rbuild.mako @@ -1,5 +1,5 @@ <%! - from util import (put_and, rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment, + from lib.util import (put_and, rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment, rb_type, singular, hub_type, mangle_ident, mb_type, property, to_fqan, indent_all_but_first_by, is_repeated_property, is_required_property, activity_input_type, TREF, IO_REQUEST, schema_to_required_property, diff --git a/src/mako/api/lib/schema.mako b/src/mako/api/lib/schema.mako index b8e461207f..51e24e13eb 100644 --- a/src/mako/api/lib/schema.mako +++ b/src/mako/api/lib/schema.mako @@ -1,5 +1,5 @@ <%! - from util import (schema_markers, rust_doc_comment, mangle_ident, to_rust_type, put_and, + from lib.util import (schema_markers, rust_doc_comment, mangle_ident, to_rust_type, put_and, IO_TYPES, activity_split, enclose_in, REQUEST_MARKER_TRAIT, mb_type, indent_all_but_first_by, NESTED_TYPE_SUFFIX, RESPONSE_MARKER_TRAIT, split_camelcase_s, METHODS_RESOURCE, PART_MARKER_TRAIT, canonical_type_name, TO_PARTS_MARKER, UNUSED_TYPE_MARKER, is_schema_with_optionals, diff --git a/src/mako/cli/README.md.mako b/src/mako/cli/README.md.mako index 05ce684201..b8022074da 100644 --- a/src/mako/cli/README.md.mako +++ b/src/mako/cli/README.md.mako @@ -1,6 +1,6 @@ <% - from util import (markdown_comment, new_context) - from cli import (CONFIG_DIR, CONFIG_DIR_FLAG, SCOPE_FLAG, application_secret_path, DEBUG_FLAG) + from lib.util import (markdown_comment, new_context) + from lib.cli import (CONFIG_DIR, CONFIG_DIR_FLAG, SCOPE_FLAG, application_secret_path, DEBUG_FLAG) c = new_context(schemas, resources, context.get('methods')) %>\ @@ -96,4 +96,4 @@ You may consider redirecting standard error into a file for ease of use, e.g. `$ [scopes]: https://developers.google.com/+/api/oauth#scopes [revoke-access]: http://webapps.stackexchange.com/a/30849 [google-dev-console]: https://console.developers.google.com/ -[google-project-new]: https://developers.google.com/console/help/new/ \ No newline at end of file +[google-project-new]: https://developers.google.com/console/help/new/ diff --git a/src/mako/cli/docs/commands.md.mako b/src/mako/cli/docs/commands.md.mako index 4ada130904..e656cdd47f 100644 --- a/src/mako/cli/docs/commands.md.mako +++ b/src/mako/cli/docs/commands.md.mako @@ -1,8 +1,8 @@ <%namespace name="util" file="../../lib/util.mako"/>\ <%! from mako.filters import xml_escape - from util import (hash_comment, new_context, method_default_scope, indent_all_but_first_by, is_repeated_property, custom_sorted) - from cli import (subcommand_md_filename, new_method_context, SPLIT_START, SPLIT_END, pretty, SCOPE_FLAG, + from lib.util import (hash_comment, new_context, method_default_scope, indent_all_but_first_by, is_repeated_property, custom_sorted) + from lib.cli import (subcommand_md_filename, new_method_context, SPLIT_START, SPLIT_END, pretty, SCOPE_FLAG, mangle_subcommand, is_request_value_property, FIELD_SEP, PARAM_FLAG, UPLOAD_FLAG, docopt_mode, FILE_ARG, MIME_ARG, OUT_ARG, OUTPUT_FLAG, to_cli_schema, cli_schema_to_yaml, SchemaEntry, STRUCT_FLAG, field_to_value, CTYPE_ARRAY, CTYPE_MAP, to_docopt_arg, FILE_FLAG, MIME_FLAG, @@ -220,4 +220,4 @@ ${self._list_schem_args(f, cursor_tokens, first_flag)} %>\ % endif % endfor - \ No newline at end of file + diff --git a/src/mako/cli/lib/argparse.mako b/src/mako/cli/lib/argparse.mako index 07a03ac6e6..f4ffb8b29e 100644 --- a/src/mako/cli/lib/argparse.mako +++ b/src/mako/cli/lib/argparse.mako @@ -2,8 +2,8 @@ <%! import os - from util import (put_and, supports_scopes, api_index, indent_by, enclose_in, put_and, escape_rust_string) - from cli import (mangle_subcommand, new_method_context, PARAM_FLAG, STRUCT_FLAG, UPLOAD_FLAG, OUTPUT_FLAG, VALUE_ARG, + from lib.util import (put_and, supports_scopes, api_index, indent_by, enclose_in, put_and, escape_rust_string) + from lib.cli import (mangle_subcommand, new_method_context, PARAM_FLAG, STRUCT_FLAG, UPLOAD_FLAG, OUTPUT_FLAG, VALUE_ARG, CONFIG_DIR, SCOPE_FLAG, is_request_value_property, FIELD_SEP, docopt_mode, FILE_ARG, MIME_ARG, OUT_ARG, CONFIG_DIR_FLAG, KEY_VALUE_ARG, to_docopt_arg, DEBUG_FLAG, MODE_ARG, SCOPE_ARG, CONFIG_DIR_ARG, FILE_FLAG, MIME_FLAG, subcommand_md_filename) diff --git a/src/mako/cli/lib/engine.mako b/src/mako/cli/lib/engine.mako index 11201ceaa7..dfdc68342b 100644 --- a/src/mako/cli/lib/engine.mako +++ b/src/mako/cli/lib/engine.mako @@ -1,9 +1,9 @@ <%namespace name="util" file="../../lib/util.mako"/>\ <%! - from util import (hub_type, mangle_ident, indent_all_but_first_by, activity_rust_type, setter_fn_name, ADD_PARAM_FN, + from lib.util import (hub_type, mangle_ident, indent_all_but_first_by, activity_rust_type, setter_fn_name, ADD_PARAM_FN, upload_action_fn, is_schema_with_optionals, schema_markers, indent_by, method_default_scope, ADD_SCOPE_FN, TREF, enclose_in) - from cli import (mangle_subcommand, new_method_context, PARAM_FLAG, STRUCT_FLAG, OUTPUT_FLAG, VALUE_ARG, + from lib.cli import (mangle_subcommand, new_method_context, PARAM_FLAG, STRUCT_FLAG, OUTPUT_FLAG, VALUE_ARG, CONFIG_DIR, SCOPE_FLAG, is_request_value_property, FIELD_SEP, docopt_mode, FILE_ARG, MIME_ARG, OUT_ARG, call_method_ident, POD_TYPES, opt_value, ident, JSON_TYPE_VALUE_MAP, KEY_VALUE_ARG, to_cli_schema, SchemaEntry, CTYPE_POD, actual_json_type, CTYPE_MAP, CTYPE_ARRAY, diff --git a/src/mako/cli/main.rs.mako b/src/mako/cli/main.rs.mako index 0cb4c58734..d407982e05 100644 --- a/src/mako/cli/main.rs.mako +++ b/src/mako/cli/main.rs.mako @@ -2,9 +2,9 @@ <%namespace name="engine" file="lib/engine.mako"/>\ <%namespace name="util" file="../lib/util.mako"/>\ <% - from util import (new_context, rust_comment, to_extern_crate_name, library_to_crate_name, library_name, + from lib.util import (new_context, rust_comment, to_extern_crate_name, library_to_crate_name, library_name, indent_all_but_first_by) - from cli import OUT_ARG, DEBUG_FLAG, opt_value + from lib.cli import OUT_ARG, DEBUG_FLAG, opt_value c = new_context(schemas, resources, context.get('methods')) default_user_agent = "google-cli-rust-client/" + cargo.build_version diff --git a/src/mako/cli/mkdocs.yml.mako b/src/mako/cli/mkdocs.yml.mako index 4c59257db6..d3692abbb3 100644 --- a/src/mako/cli/mkdocs.yml.mako +++ b/src/mako/cli/mkdocs.yml.mako @@ -1,6 +1,6 @@ <% - from util import (put_and, new_context) - from cli import (subcommand_md_filename, mangle_subcommand, pretty) + from lib.util import (put_and, new_context) + from lib.cli import (subcommand_md_filename, mangle_subcommand, pretty) c = new_context(schemas, resources, context.get('methods')) %>\ diff --git a/src/mako/deps.mako b/src/mako/deps.mako index dc627d71ee..e24bdb1a5d 100644 --- a/src/mako/deps.mako +++ b/src/mako/deps.mako @@ -58,7 +58,7 @@ <% continue %>\ % endif <% - import util + import lib.util as util import os import json @@ -191,7 +191,7 @@ help${agsuffix}: % for info in (apis.get('items') or []): <% - import util + import lib.util as util import os name = util.normalize_library_name(info['name']) target = util.api_json_path(directories.api_base, name, info['version']) diff --git a/src/mako/index.html.mako b/src/mako/index.html.mako index b07e32e66f..5d8058af61 100644 --- a/src/mako/index.html.mako +++ b/src/mako/index.html.mako @@ -2,7 +2,7 @@ import json import os import yaml - from util import (api_json_path, library_name, library_to_crate_name, + from lib.util import (api_json_path, library_name, library_to_crate_name, gen_crate_dir, api_index, crates_io_url, program_name, crate_version) diff --git a/src/mako/lib/cli.py b/src/mako/lib/cli.py index 215f0f1806..ee8b9c36ef 100644 --- a/src/mako/lib/cli.py +++ b/src/mako/lib/cli.py @@ -1,4 +1,4 @@ -import util +import lib.util as util import os import re @@ -239,7 +239,7 @@ def field_to_value(f): return v # split the result along split segments -def process_template_result(r, output_file): +def process_template_result(r, output_file: str): found = False dir = None if output_file: @@ -249,6 +249,8 @@ def process_template_result(r, output_file): # end handle output directory for m in re_splitters.finditer(r): + if not dir: + raise RuntimeError("Missing directory; was output_file specified?") found = True fh = open(os.path.join(dir, m.group(1)), 'wb') fh.write(m.group(2).encode('UTF-8')) diff --git a/src/mako/lib/util.mako b/src/mako/lib/util.mako index a0dff1b34f..8e58910650 100644 --- a/src/mako/lib/util.mako +++ b/src/mako/lib/util.mako @@ -1,4 +1,4 @@ -<%! import util %>\ +<%! import lib.util as util %>\ ## source should be ${self.uri} ## you need to escape the output, using a filter for example @@ -58,4 +58,4 @@ ${canonicalName}\ <%def name="program_name()" buffered="True">\ ${util.program_name(name, version)}\ - \ No newline at end of file + diff --git a/src/mako/lib/util.py b/src/mako/lib/util.py index 08df1d9aaf..c6c78a8bf5 100644 --- a/src/mako/lib/util.py +++ b/src/mako/lib/util.py @@ -1,6 +1,7 @@ import re import os from random import (randint, random, choice, seed) +from typing import Any, List, Mapping, Optional, Tuple import collections from copy import deepcopy import subprocess @@ -10,9 +11,9 @@ seed(1337) re_linestart = re.compile('^', flags=re.MULTILINE) re_spaces_after_newline = re.compile('^ {4}', flags=re.MULTILINE) re_first_4_spaces = re.compile('^ {1,4}', flags=re.MULTILINE) -re_desc_parts = re.compile("((the part (names|properties) that you can include in the parameter value are)|(supported values are ))(.*?)\.", flags=re.IGNORECASE|re.MULTILINE) +re_desc_parts = re.compile(r"((the part (names|properties) that you can include in the parameter value are)|(supported values are ))(.*?)\.", flags=re.IGNORECASE|re.MULTILINE) -re_find_replacements = re.compile("\{[/\+]?\w+\*?\}") +re_find_replacements = re.compile(r"\{[/\+]?\w+\*?\}") HTTP_METHODS = set(("OPTIONS", "GET", "POST", "PUT", "DELETE", "HEAD", "TRACE", "CONNECT", "PATCH" )) @@ -120,9 +121,7 @@ def items(p): else: return p._items() -def custom_sorted(p): - if not isinstance(p, list): - assert(false, p, "unexpected type") +def custom_sorted(p: List[Mapping[str, Any]]) -> List[Mapping[str, Any]]: return sorted(p, key = lambda p: p['name']) # ============================================================================== @@ -491,7 +490,7 @@ def is_schema_with_optionals(schema_markers): ## @name Activity Utilities # @{ # return (category, name|None, method) -def activity_split(fqan): +def activity_split(fqan: str) -> Tuple[str, Optional[str], str]: t = fqan.split('.') mt = t[2:] if not mt: @@ -513,10 +512,6 @@ def to_fqan(name, resource, method): def activity_name_to_type_name(an): return canonical_type_name(an)[:-1] -# yields (category, resource, activity, activity_data) -def iter_acitivities(c): - return ((activity_split(an) + [a]) for an, a in c.fqan_map.items()) - # return a list of parameter structures of all params of the given method dict # apply a prune filter to restrict the set of returned parameters. # The order will always be: partOrder + alpha @@ -810,7 +805,7 @@ def _is_special_version(v): return v.endswith('alpha') or v.endswith('beta') def to_api_version(v): - m = re.search("_?v(\d(\.\d)*)_?", v) + m = re.search(r"_?v(\d(\.\d)*)_?", v) if not m and _is_special_version(v): return v assert m, "Expected to find a version within '%s'" % v diff --git a/src/mako/lib/util_test.py b/src/mako/lib/util_test.py index 395f554e11..e0f6661959 100644 --- a/src/mako/lib/util_test.py +++ b/src/mako/lib/util_test.py @@ -1,15 +1,18 @@ #!/usr/bin/env python +import importlib.resources import unittest import json -import importlib_resources -from .util import to_api_version, library_name, re_find_replacements, to_rust_type, new_context -from . import test_data +from lib.util import to_api_version, library_name, re_find_replacements, to_rust_type, new_context +import lib.test_data as test_data -def read_test_json_file(resource): - data = importlib_resources.read_text(test_data, resource) +TEST_JSON_FILE = "photoslibrary-api.json" + + +def read_test_json_file(): + data = importlib.resources.read_text(test_data, TEST_JSON_FILE) return json.loads(data) class UtilsTest(unittest.TestCase): @@ -65,7 +68,7 @@ class UtilsTest(unittest.TestCase): self.assertEqual(ms[0], '{+project}') def test_to_rust_type(self): - full_api_schema = read_test_json_file('photoslibrary-api.json') + full_api_schema = read_test_json_file() schemas = full_api_schema['schemas'] From 08552c43644ee953f1c69b4e5c018d35d1f865f6 Mon Sep 17 00:00:00 2001 From: Kyle Gentle Date: Sat, 6 Aug 2022 16:08:06 -0400 Subject: [PATCH 2/5] Restructure `src` dir Make a few changes in the toplevel src dir to help separate templates from code. Specifically, we rename `src/mako` to `src/generator`, and nest the mako templates inside a `src/generator/templates` dir. This isolates most Python code into the `src/generator/lib` dir. --- .gitignore | 2 +- Makefile | 13 +++++++------ etc/api/shared.yaml | 2 +- etc/api/type-cli.yaml | 2 +- src/{mako/lib => generator}/__init__.py | 0 .../lib/test_data => generator/lib}/__init__.py | 0 src/{mako => generator}/lib/cli.py | 2 +- src/generator/lib/test_data/__init__.py | 0 .../lib/test_data/photoslibrary-api.json | 0 src/{mako => generator}/lib/util.mako | 2 +- src/{mako => generator}/lib/util.py | 17 +++++++++++------ src/{mako => generator}/lib/util_test.py | 4 ++-- .../templates}/Cargo.toml.mako | 4 ++-- .../templates}/LICENSE.md.mako | 4 ++-- .../templates}/api/README.md.mako | 4 ++-- .../templates}/api/api.rs.mako | 4 ++-- .../templates}/api/lib.rs.mako | 8 ++++---- .../templates}/api/lib/lib.mako | 4 ++-- .../templates}/api/lib/mbuild.mako | 4 ++-- .../templates}/api/lib/rbuild.mako | 4 ++-- .../templates}/api/lib/schema.mako | 2 +- .../templates}/cli/README.md.mako | 6 +++--- .../templates}/cli/docs/commands.md.mako | 6 +++--- .../templates}/cli/lib/argparse.mako | 6 +++--- .../templates}/cli/lib/engine.mako | 6 +++--- .../templates}/cli/main.rs.mako | 6 +++--- .../templates}/cli/mkdocs.yml.mako | 6 +++--- src/{mako => generator/templates}/deps.mako | 4 ++-- .../templates}/index.html.mako | 2 +- .../templates}/rustfmt.toml.mako | 0 30 files changed, 65 insertions(+), 59 deletions(-) rename src/{mako/lib => generator}/__init__.py (100%) rename src/{mako/lib/test_data => generator/lib}/__init__.py (100%) rename src/{mako => generator}/lib/cli.py (99%) create mode 100644 src/generator/lib/test_data/__init__.py rename src/{mako => generator}/lib/test_data/photoslibrary-api.json (100%) rename src/{mako => generator}/lib/util.mako (97%) rename src/{mako => generator}/lib/util.py (99%) rename src/{mako => generator}/lib/util_test.py (96%) rename src/{mako => generator/templates}/Cargo.toml.mako (89%) rename src/{mako => generator/templates}/LICENSE.md.mako (93%) rename src/{mako => generator/templates}/api/README.md.mako (75%) rename src/{mako => generator/templates}/api/api.rs.mako (95%) rename src/{mako => generator/templates}/api/lib.rs.mako (86%) rename src/{mako => generator/templates}/api/lib/lib.mako (98%) rename src/{mako => generator/templates}/api/lib/mbuild.mako (99%) rename src/{mako => generator/templates}/api/lib/rbuild.mako (96%) rename src/{mako => generator/templates}/api/lib/schema.mako (98%) rename src/{mako => generator/templates}/cli/README.md.mako (95%) rename src/{mako => generator/templates}/cli/docs/commands.md.mako (96%) rename src/{mako => generator/templates}/cli/lib/argparse.mako (96%) rename src/{mako => generator/templates}/cli/lib/engine.mako (98%) rename src/{mako => generator/templates}/cli/main.rs.mako (89%) rename src/{mako => generator/templates}/cli/mkdocs.yml.mako (79%) rename src/{mako => generator/templates}/deps.mako (99%) rename src/{mako => generator/templates}/index.html.mako (98%) rename src/{mako => generator/templates}/rustfmt.toml.mako (100%) diff --git a/.gitignore b/.gitignore index 3d887dbf61..be911ee383 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ gen/doc *.go **/target/ -**/docs/ +gen/**/docs/ **/build_html/ .*.deps **/Cargo.lock diff --git a/Makefile b/Makefile index 7c27213f26..77783ffa4e 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,9 @@ TPL := $(PYTHON) $(MAKO_RENDER) MKDOCS := $(shell pwd)/$(VENV_DIR)/bin/mkdocs GHP_IMPORT := $(shell pwd)/$(VENV_DIR)/bin/ghp-import -MAKO_SRC = src/mako +GEN_SRC = src/generator +GEN_LIB_SRC = $(GEN_SRC)/lib +MAKO_SRC = src/generator/templates RUST_SRC = src/rust PREPROC_DIR = $(RUST_SRC)/preproc PREPROC = target/release/preproc @@ -36,9 +38,8 @@ else API_LIST := $(API_LIST)api-list.yaml endif API_JSON_FILES = $(shell find etc -type f -name '*-api.json') -MAKO_LIB_DIR = $(MAKO_SRC)/lib -MAKO_LIB_FILES = $(shell find $(MAKO_LIB_DIR) -type f -name '*.*') -MAKO = export PREPROC=$(PREPROC); export PYTHONPATH=$(MAKO_SRC):$(PYTHONPATH); $(TPL) --template-dir '.' +MAKO_LIB_FILES = $(shell find $(GEN_LIB_SRC) -type f -name '*.*') +MAKO = export PREPROC=$(PREPROC); export PYTHONPATH=src:$(PYTHONPATH); $(TPL) --template-dir '.' MAKO_STANDARD_DEPENDENCIES = $(API_SHARED_INFO) $(MAKO_LIB_FILES) $(MAKO_RENDER) $(PREPROC) help: @@ -71,7 +72,7 @@ $(PYTHON_BIN): $(VENV_BIN) requirements.txt python3 -m virtualenv -p python3 $(VENV_DIR) $@ -m pip install -r requirements.txt -$(MAKO_RENDER): $(PYTHON_BIN) $(wildcard $(MAKO_LIB_DIR)/*) +$(MAKO_RENDER): $(PYTHON_BIN) $(wildcard $(GEN_LIB_SRC)/*) # Explicitly NOT depending on $(MAKO_LIB_FILES), as it's quite stable and now takes 'too long' thanks # to a URL get call to the google discovery service @@ -99,7 +100,7 @@ test-gen: $(PYTHON_BIN) test: test-gen typecheck: $(PYTHON_BIN) - $(PYTHON) -m pyright $(MAKO_LIB_DIR) + $(PYTHON) -m pyright $(GEN_LIB_SRC) clean: clean-all-api clean-all-cli docs-all-clean -rm -Rf $(VENV_DIR) diff --git a/etc/api/shared.yaml b/etc/api/shared.yaml index cea2aa42f7..09e0d7c67b 100644 --- a/etc/api/shared.yaml +++ b/etc/api/shared.yaml @@ -83,7 +83,7 @@ directories: # where are all the API meta files api_base: etc/api # all mako source files - mako_src: src/mako + mako_src: src/generator/templates # The subdirectory to contain documentation from all APIs and related programs doc_subdir: doc cargo: diff --git a/etc/api/type-cli.yaml b/etc/api/type-cli.yaml index 29b50c212c..5083a1cf31 100644 --- a/etc/api/type-cli.yaml +++ b/etc/api/type-cli.yaml @@ -5,7 +5,7 @@ mkdocs: # if docs_dir changes, remember to update the sources as well. docs_dir: docs mako: - post_processor_module: "lib.cli" + post_processor_module: "generator.lib.cli" make: id: cli target_name: CLIs diff --git a/src/mako/lib/__init__.py b/src/generator/__init__.py similarity index 100% rename from src/mako/lib/__init__.py rename to src/generator/__init__.py diff --git a/src/mako/lib/test_data/__init__.py b/src/generator/lib/__init__.py similarity index 100% rename from src/mako/lib/test_data/__init__.py rename to src/generator/lib/__init__.py diff --git a/src/mako/lib/cli.py b/src/generator/lib/cli.py similarity index 99% rename from src/mako/lib/cli.py rename to src/generator/lib/cli.py index ee8b9c36ef..ef50b19185 100644 --- a/src/mako/lib/cli.py +++ b/src/generator/lib/cli.py @@ -1,4 +1,4 @@ -import lib.util as util +import generator.lib.util as util import os import re diff --git a/src/generator/lib/test_data/__init__.py b/src/generator/lib/test_data/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/mako/lib/test_data/photoslibrary-api.json b/src/generator/lib/test_data/photoslibrary-api.json similarity index 100% rename from src/mako/lib/test_data/photoslibrary-api.json rename to src/generator/lib/test_data/photoslibrary-api.json diff --git a/src/mako/lib/util.mako b/src/generator/lib/util.mako similarity index 97% rename from src/mako/lib/util.mako rename to src/generator/lib/util.mako index 8e58910650..0b541b79bd 100644 --- a/src/mako/lib/util.mako +++ b/src/generator/lib/util.mako @@ -1,4 +1,4 @@ -<%! import lib.util as util %>\ +<%! import generator.lib.util as util %>\ ## source should be ${self.uri} ## you need to escape the output, using a filter for example diff --git a/src/mako/lib/util.py b/src/generator/lib/util.py similarity index 99% rename from src/mako/lib/util.py rename to src/generator/lib/util.py index c6c78a8bf5..f91aeda622 100644 --- a/src/mako/lib/util.py +++ b/src/generator/lib/util.py @@ -341,11 +341,16 @@ def _assure_unique_type_name(schemas, tn): return tn # map a json type to an rust type -# sn = schema name -# pn = property name # t = type dict # NOTE: In case you don't understand how this algorithm really works ... me neither - THE AUTHOR -def to_rust_type(schemas, sn, pn, t, allow_optionals=True, _is_recursive=False): +def to_rust_type( + schemas, + schema_name, + property_name, + t, + allow_optionals=True, + _is_recursive=False +): def nested_type(nt): if 'items' in nt: nt = nt['items'] @@ -354,8 +359,8 @@ def to_rust_type(schemas, sn, pn, t, allow_optionals=True, _is_recursive=False): else: assert(is_nested_type_property(nt)) # It's a nested type - we take it literally like $ref, but generate a name for the type ourselves - return _assure_unique_type_name(schemas, nested_type_name(sn, pn)) - return to_rust_type(schemas, sn, pn, nt, allow_optionals=False, _is_recursive=True) + return _assure_unique_type_name(schemas, nested_type_name(schema_name, property_name)) + return to_rust_type(schemas, schema_name, property_name, nt, allow_optionals=False, _is_recursive=True) def wrap_type(tn): if allow_optionals: @@ -368,7 +373,7 @@ def to_rust_type(schemas, sn, pn, t, allow_optionals=True, _is_recursive=False): # which is fine for now. 'allow_optionals' implicitly restricts type boxing for simple types - it # usually is on on the first call, and off when recursion is involved. tn = t[TREF] - if not _is_recursive and tn == sn: + if not _is_recursive and tn == schema_name: tn = 'Option>' % tn return wrap_type(tn) try: diff --git a/src/mako/lib/util_test.py b/src/generator/lib/util_test.py similarity index 96% rename from src/mako/lib/util_test.py rename to src/generator/lib/util_test.py index e0f6661959..f824c8141d 100644 --- a/src/mako/lib/util_test.py +++ b/src/generator/lib/util_test.py @@ -4,8 +4,8 @@ import importlib.resources import unittest import json -from lib.util import to_api_version, library_name, re_find_replacements, to_rust_type, new_context -import lib.test_data as test_data +from generator.lib.util import to_api_version, library_name, re_find_replacements, to_rust_type, new_context +import generator.lib.test_data as test_data TEST_JSON_FILE = "photoslibrary-api.json" diff --git a/src/mako/Cargo.toml.mako b/src/generator/templates/Cargo.toml.mako similarity index 89% rename from src/mako/Cargo.toml.mako rename to src/generator/templates/Cargo.toml.mako index 740398e684..1d23f7138c 100644 --- a/src/mako/Cargo.toml.mako +++ b/src/generator/templates/Cargo.toml.mako @@ -1,5 +1,5 @@ -<%! from lib.util import (estr, enclose_in, hash_comment, library_to_crate_name, to_extern_crate_name) %>\ -<%namespace name="util" file="lib/util.mako"/>\ +<%! from generator.lib.util import (estr, enclose_in, hash_comment, library_to_crate_name, to_extern_crate_name) %>\ +<%namespace name="util" file="../lib/util.mako"/>\ <%block filter="hash_comment">\ <%util:gen_info source="${self.uri}" />\ diff --git a/src/mako/LICENSE.md.mako b/src/generator/templates/LICENSE.md.mako similarity index 93% rename from src/mako/LICENSE.md.mako rename to src/generator/templates/LICENSE.md.mako index 9f8bc47fe9..ffdcb034db 100644 --- a/src/mako/LICENSE.md.mako +++ b/src/generator/templates/LICENSE.md.mako @@ -1,6 +1,6 @@ ## -*- coding: utf-8 -*- -<%! import lib.util as util %>\ -<%namespace name="mutil" file="lib/util.mako"/>\ +<%! import generator.lib.util as util %>\ +<%namespace name="mutil" file="../lib/util.mako"/>\ <%block filter="util.markdown_comment">\ <%mutil:gen_info source="${self.uri}" />\ diff --git a/src/mako/api/README.md.mako b/src/generator/templates/api/README.md.mako similarity index 75% rename from src/mako/api/README.md.mako rename to src/generator/templates/api/README.md.mako index 2a14bb6f5d..8908256d02 100644 --- a/src/mako/api/README.md.mako +++ b/src/generator/templates/api/README.md.mako @@ -1,9 +1,9 @@ <% - from lib.util import (markdown_comment, new_context) + from generator.lib.util import (markdown_comment, new_context) c = new_context(schemas, resources, context.get('methods')) %>\ <%namespace name="lib" file="lib/lib.mako"/>\ -<%namespace name="util" file="../lib/util.mako"/>\ +<%namespace name="util" file="../../lib/util.mako"/>\ <%block filter="markdown_comment">\ <%util:gen_info source="${self.uri}" />\ diff --git a/src/mako/api/api.rs.mako b/src/generator/templates/api/api.rs.mako similarity index 95% rename from src/mako/api/api.rs.mako rename to src/generator/templates/api/api.rs.mako index 750ded94bb..f86ebd1dd0 100644 --- a/src/mako/api/api.rs.mako +++ b/src/generator/templates/api/api.rs.mako @@ -1,10 +1,10 @@ <%namespace name="lib" file="lib/lib.mako"/>\ -<%namespace name="util" file="../lib/util.mako"/>\ +<%namespace name="util" file="../../lib/util.mako"/>\ <%namespace name="rbuild" file="lib/rbuild.mako"/>\ <%namespace name="mbuild" file="lib/mbuild.mako"/>\ <%namespace name="schema" file="lib/schema.mako"/>\ <% - from lib.util import (new_context, rust_comment, rust_doc_comment, rust_module_doc_comment, + from generator.lib.util import (new_context, rust_comment, rust_doc_comment, rust_module_doc_comment, rb_type, hub_type, mangle_ident, hub_type_params_s, rb_type_params_s, find_fattest_resource, HUB_TYPE_PARAMETERS, METHODS_RESOURCE, UNUSED_TYPE_MARKER, schema_markers) diff --git a/src/mako/api/lib.rs.mako b/src/generator/templates/api/lib.rs.mako similarity index 86% rename from src/mako/api/lib.rs.mako rename to src/generator/templates/api/lib.rs.mako index 38a4c1fb59..a9ef504da6 100644 --- a/src/mako/api/lib.rs.mako +++ b/src/generator/templates/api/lib.rs.mako @@ -1,7 +1,7 @@ <%namespace name="lib" file="lib/lib.mako"/>\ -<%namespace name="util" file="../lib/util.mako"/>\ +<%namespace name="util" file="../../lib/util.mako"/>\ <% - from lib.util import (new_context, rust_comment, rust_module_doc_comment) + from generator.lib.util import (new_context, rust_comment, rust_module_doc_comment) c = new_context(schemas, resources, context.get('methods')) %>\ @@ -20,12 +20,12 @@ ${lib.docs(c)} #![allow(unused_imports, unused_mut, dead_code)] <%namespace name="lib" file="lib/lib.mako"/>\ -<%namespace name="util" file="../lib/util.mako"/>\ +<%namespace name="util" file="../../lib/util.mako"/>\ <%namespace name="rbuild" file="lib/rbuild.mako"/>\ <%namespace name="mbuild" file="lib/mbuild.mako"/>\ <%namespace name="schema" file="lib/schema.mako"/>\ <% - from lib.util import (new_context, rust_comment, rust_doc_comment, rust_module_doc_comment, + from generator.lib.util import (new_context, rust_comment, rust_doc_comment, rust_module_doc_comment, rb_type, hub_type, mangle_ident, hub_type_params_s, rb_type_params_s, find_fattest_resource, HUB_TYPE_PARAMETERS, METHODS_RESOURCE, UNUSED_TYPE_MARKER, schema_markers) diff --git a/src/mako/api/lib/lib.mako b/src/generator/templates/api/lib/lib.mako similarity index 98% rename from src/mako/api/lib/lib.mako rename to src/generator/templates/api/lib/lib.mako index fc35b8667f..1faed8fcd4 100644 --- a/src/mako/api/lib/lib.mako +++ b/src/generator/templates/api/lib/lib.mako @@ -1,5 +1,5 @@ <%! - from lib.util import (activity_split, put_and, md_italic, split_camelcase_s, canonical_type_name, hub_type, + from generator.lib.util import (activity_split, put_and, md_italic, split_camelcase_s, canonical_type_name, hub_type, rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment, markdown_rust_block, unindent_first_by, mangle_ident, mb_type, singular, scope_url_to_variant, PART_MARKER_TRAIT, RESOURCE_MARKER_TRAIT, CALL_BUILDER_MARKERT_TRAIT, @@ -12,7 +12,7 @@ def pretty_name(name): return ' '.join(split_camelcase_s(name).split('.')) %>\ -<%namespace name="util" file="../../lib/util.mako"/>\ +<%namespace name="util" file="../../../lib/util.mako"/>\ <%namespace name="mbuild" file="mbuild.mako"/>\ ## If rust-doc is True, examples will be made to work for rust doc tests. Otherwise they are set diff --git a/src/mako/api/lib/mbuild.mako b/src/generator/templates/api/lib/mbuild.mako similarity index 99% rename from src/mako/api/lib/mbuild.mako rename to src/generator/templates/api/lib/mbuild.mako index b904ea6327..18007e04fa 100644 --- a/src/mako/api/lib/mbuild.mako +++ b/src/generator/templates/api/lib/mbuild.mako @@ -1,5 +1,5 @@ <%! - from lib.util import (put_and, rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment, + from generator.lib.util import (put_and, rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment, rb_type, mb_type, singular, hub_type, to_fqan, indent_all_but_first_by, activity_rust_type, mangle_ident, activity_input_type, get_word, split_camelcase_s, property, is_pod_property, TREF, IO_REQUEST, @@ -28,7 +28,7 @@ part_desc = part_desc[:-1] return part_desc %>\ -<%namespace name="util" file="../../lib/util.mako"/>\ +<%namespace name="util" file="../../../lib/util.mako"/>\ <%namespace name="lib" file="lib.mako"/>\ ## Creates a method builder type diff --git a/src/mako/api/lib/rbuild.mako b/src/generator/templates/api/lib/rbuild.mako similarity index 96% rename from src/mako/api/lib/rbuild.mako rename to src/generator/templates/api/lib/rbuild.mako index 05b6760a92..a336c893c0 100644 --- a/src/mako/api/lib/rbuild.mako +++ b/src/generator/templates/api/lib/rbuild.mako @@ -1,5 +1,5 @@ <%! - from lib.util import (put_and, rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment, + from generator.lib.util import (put_and, rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment, rb_type, singular, hub_type, mangle_ident, mb_type, property, to_fqan, indent_all_but_first_by, is_repeated_property, is_required_property, activity_input_type, TREF, IO_REQUEST, schema_to_required_property, @@ -8,7 +8,7 @@ struct_type_bounds_s, METHODS_RESOURCE, SPACES_PER_TAB, prefix_all_but_first_with, METHODS_BUILDER_MARKER_TRAIT, remove_empty_lines, method_default_scope, rust_doc_sanitize) %>\ -<%namespace name="util" file="../../lib/util.mako"/>\ +<%namespace name="util" file="../../../lib/util.mako"/>\ <%namespace name="lib" file="lib.mako"/>\ ## Creates a Resource builder type diff --git a/src/mako/api/lib/schema.mako b/src/generator/templates/api/lib/schema.mako similarity index 98% rename from src/mako/api/lib/schema.mako rename to src/generator/templates/api/lib/schema.mako index 51e24e13eb..8a063b33af 100644 --- a/src/mako/api/lib/schema.mako +++ b/src/generator/templates/api/lib/schema.mako @@ -1,5 +1,5 @@ <%! - from lib.util import (schema_markers, rust_doc_comment, mangle_ident, to_rust_type, put_and, + from generator.lib.util import (schema_markers, rust_doc_comment, mangle_ident, to_rust_type, put_and, IO_TYPES, activity_split, enclose_in, REQUEST_MARKER_TRAIT, mb_type, indent_all_but_first_by, NESTED_TYPE_SUFFIX, RESPONSE_MARKER_TRAIT, split_camelcase_s, METHODS_RESOURCE, PART_MARKER_TRAIT, canonical_type_name, TO_PARTS_MARKER, UNUSED_TYPE_MARKER, is_schema_with_optionals, diff --git a/src/mako/cli/README.md.mako b/src/generator/templates/cli/README.md.mako similarity index 95% rename from src/mako/cli/README.md.mako rename to src/generator/templates/cli/README.md.mako index b8022074da..00b2ffa594 100644 --- a/src/mako/cli/README.md.mako +++ b/src/generator/templates/cli/README.md.mako @@ -1,10 +1,10 @@ <% - from lib.util import (markdown_comment, new_context) - from lib.cli import (CONFIG_DIR, CONFIG_DIR_FLAG, SCOPE_FLAG, application_secret_path, DEBUG_FLAG) + from generator.lib.util import (markdown_comment, new_context) + from generator.lib.cli import (CONFIG_DIR, CONFIG_DIR_FLAG, SCOPE_FLAG, application_secret_path, DEBUG_FLAG) c = new_context(schemas, resources, context.get('methods')) %>\ -<%namespace name="util" file="../lib/util.mako"/>\ +<%namespace name="util" file="../../lib/util.mako"/>\ <%namespace name="argparse" file="lib/argparse.mako"/>\ <%block filter="markdown_comment">\ <%util:gen_info source="${self.uri}" />\ diff --git a/src/mako/cli/docs/commands.md.mako b/src/generator/templates/cli/docs/commands.md.mako similarity index 96% rename from src/mako/cli/docs/commands.md.mako rename to src/generator/templates/cli/docs/commands.md.mako index e656cdd47f..455b0045cf 100644 --- a/src/mako/cli/docs/commands.md.mako +++ b/src/generator/templates/cli/docs/commands.md.mako @@ -1,8 +1,8 @@ -<%namespace name="util" file="../../lib/util.mako"/>\ +<%namespace name="util" file="../../../lib/util.mako"/>\ <%! from mako.filters import xml_escape - from lib.util import (hash_comment, new_context, method_default_scope, indent_all_but_first_by, is_repeated_property, custom_sorted) - from lib.cli import (subcommand_md_filename, new_method_context, SPLIT_START, SPLIT_END, pretty, SCOPE_FLAG, + from generator.lib.util import (hash_comment, new_context, method_default_scope, indent_all_but_first_by, is_repeated_property, custom_sorted) + from generator.lib.cli import (subcommand_md_filename, new_method_context, SPLIT_START, SPLIT_END, pretty, SCOPE_FLAG, mangle_subcommand, is_request_value_property, FIELD_SEP, PARAM_FLAG, UPLOAD_FLAG, docopt_mode, FILE_ARG, MIME_ARG, OUT_ARG, OUTPUT_FLAG, to_cli_schema, cli_schema_to_yaml, SchemaEntry, STRUCT_FLAG, field_to_value, CTYPE_ARRAY, CTYPE_MAP, to_docopt_arg, FILE_FLAG, MIME_FLAG, diff --git a/src/mako/cli/lib/argparse.mako b/src/generator/templates/cli/lib/argparse.mako similarity index 96% rename from src/mako/cli/lib/argparse.mako rename to src/generator/templates/cli/lib/argparse.mako index f4ffb8b29e..6988489f00 100644 --- a/src/mako/cli/lib/argparse.mako +++ b/src/generator/templates/cli/lib/argparse.mako @@ -1,9 +1,9 @@ -<%namespace name="util" file="../../lib/util.mako"/>\ +<%namespace name="util" file="../../../lib/util.mako"/>\ <%! import os - from lib.util import (put_and, supports_scopes, api_index, indent_by, enclose_in, put_and, escape_rust_string) - from lib.cli import (mangle_subcommand, new_method_context, PARAM_FLAG, STRUCT_FLAG, UPLOAD_FLAG, OUTPUT_FLAG, VALUE_ARG, + from generator.lib.util import (put_and, supports_scopes, api_index, indent_by, enclose_in, put_and, escape_rust_string) + from generator.lib.cli import (mangle_subcommand, new_method_context, PARAM_FLAG, STRUCT_FLAG, UPLOAD_FLAG, OUTPUT_FLAG, VALUE_ARG, CONFIG_DIR, SCOPE_FLAG, is_request_value_property, FIELD_SEP, docopt_mode, FILE_ARG, MIME_ARG, OUT_ARG, CONFIG_DIR_FLAG, KEY_VALUE_ARG, to_docopt_arg, DEBUG_FLAG, MODE_ARG, SCOPE_ARG, CONFIG_DIR_ARG, FILE_FLAG, MIME_FLAG, subcommand_md_filename) diff --git a/src/mako/cli/lib/engine.mako b/src/generator/templates/cli/lib/engine.mako similarity index 98% rename from src/mako/cli/lib/engine.mako rename to src/generator/templates/cli/lib/engine.mako index dfdc68342b..01cf023541 100644 --- a/src/mako/cli/lib/engine.mako +++ b/src/generator/templates/cli/lib/engine.mako @@ -1,9 +1,9 @@ -<%namespace name="util" file="../../lib/util.mako"/>\ +<%namespace name="util" file="../../../lib/util.mako"/>\ <%! - from lib.util import (hub_type, mangle_ident, indent_all_but_first_by, activity_rust_type, setter_fn_name, ADD_PARAM_FN, + from generator.lib.util import (hub_type, mangle_ident, indent_all_but_first_by, activity_rust_type, setter_fn_name, ADD_PARAM_FN, upload_action_fn, is_schema_with_optionals, schema_markers, indent_by, method_default_scope, ADD_SCOPE_FN, TREF, enclose_in) - from lib.cli import (mangle_subcommand, new_method_context, PARAM_FLAG, STRUCT_FLAG, OUTPUT_FLAG, VALUE_ARG, + from generator.lib.cli import (mangle_subcommand, new_method_context, PARAM_FLAG, STRUCT_FLAG, OUTPUT_FLAG, VALUE_ARG, CONFIG_DIR, SCOPE_FLAG, is_request_value_property, FIELD_SEP, docopt_mode, FILE_ARG, MIME_ARG, OUT_ARG, call_method_ident, POD_TYPES, opt_value, ident, JSON_TYPE_VALUE_MAP, KEY_VALUE_ARG, to_cli_schema, SchemaEntry, CTYPE_POD, actual_json_type, CTYPE_MAP, CTYPE_ARRAY, diff --git a/src/mako/cli/main.rs.mako b/src/generator/templates/cli/main.rs.mako similarity index 89% rename from src/mako/cli/main.rs.mako rename to src/generator/templates/cli/main.rs.mako index d407982e05..52cb8cc038 100644 --- a/src/mako/cli/main.rs.mako +++ b/src/generator/templates/cli/main.rs.mako @@ -1,10 +1,10 @@ <%namespace name="argparse" file="lib/argparse.mako"/>\ <%namespace name="engine" file="lib/engine.mako"/>\ -<%namespace name="util" file="../lib/util.mako"/>\ +<%namespace name="util" file="../../lib/util.mako"/>\ <% - from lib.util import (new_context, rust_comment, to_extern_crate_name, library_to_crate_name, library_name, + from generator.lib.util import (new_context, rust_comment, to_extern_crate_name, library_to_crate_name, library_name, indent_all_but_first_by) - from lib.cli import OUT_ARG, DEBUG_FLAG, opt_value + from generator.lib.cli import OUT_ARG, DEBUG_FLAG, opt_value c = new_context(schemas, resources, context.get('methods')) default_user_agent = "google-cli-rust-client/" + cargo.build_version diff --git a/src/mako/cli/mkdocs.yml.mako b/src/generator/templates/cli/mkdocs.yml.mako similarity index 79% rename from src/mako/cli/mkdocs.yml.mako rename to src/generator/templates/cli/mkdocs.yml.mako index d3692abbb3..8b915c37dc 100644 --- a/src/mako/cli/mkdocs.yml.mako +++ b/src/generator/templates/cli/mkdocs.yml.mako @@ -1,10 +1,10 @@ <% - from lib.util import (put_and, new_context) - from lib.cli import (subcommand_md_filename, mangle_subcommand, pretty) + from generator.lib.util import (put_and, new_context) + from generator.lib.cli import (subcommand_md_filename, mangle_subcommand, pretty) c = new_context(schemas, resources, context.get('methods')) %>\ -<%namespace name="util" file="../lib/util.mako"/>\ +<%namespace name="util" file="../../lib/util.mako"/>\ site_name: ${util.canonical_name()} v${util.crate_version()} site_url: ${cargo.doc_base_url}/${util.crate_name()} site_description: A complete library to interact with ${util.canonical_name()} (protocol ${version}) diff --git a/src/mako/deps.mako b/src/generator/templates/deps.mako similarity index 99% rename from src/mako/deps.mako rename to src/generator/templates/deps.mako index e24bdb1a5d..7185757a94 100644 --- a/src/mako/deps.mako +++ b/src/generator/templates/deps.mako @@ -58,7 +58,7 @@ <% continue %>\ % endif <% - import lib.util as util + import generator.lib.util as util import os import json @@ -191,7 +191,7 @@ help${agsuffix}: % for info in (apis.get('items') or []): <% - import lib.util as util + import generator.lib.util as util import os name = util.normalize_library_name(info['name']) target = util.api_json_path(directories.api_base, name, info['version']) diff --git a/src/mako/index.html.mako b/src/generator/templates/index.html.mako similarity index 98% rename from src/mako/index.html.mako rename to src/generator/templates/index.html.mako index 5d8058af61..cc978ac3da 100644 --- a/src/mako/index.html.mako +++ b/src/generator/templates/index.html.mako @@ -2,7 +2,7 @@ import json import os import yaml - from lib.util import (api_json_path, library_name, library_to_crate_name, + from generator.lib.util import (api_json_path, library_name, library_to_crate_name, gen_crate_dir, api_index, crates_io_url, program_name, crate_version) diff --git a/src/mako/rustfmt.toml.mako b/src/generator/templates/rustfmt.toml.mako similarity index 100% rename from src/mako/rustfmt.toml.mako rename to src/generator/templates/rustfmt.toml.mako From 8ba6acb88bf889d41560ccc2c16f5e884af68b9c Mon Sep 17 00:00:00 2001 From: Kyle Gentle Date: Mon, 8 Aug 2022 20:08:13 -0400 Subject: [PATCH 3/5] util.py: Make new_context dict-compatible This is an incremental change towards a strongly-typed util module, aimed at reducing dependency on the DictObject class. The rough idea is to annotate everything as Dict, add some tests to codify the existing behavior, and then start defining dataclasses for the dischovery schema. We also remove some unused logic & params. --- src/generator/lib/util.py | 100 +++++++++--------- src/generator/templates/api/README.md.mako | 2 +- src/generator/templates/api/api.rs.mako | 2 +- src/generator/templates/api/lib.rs.mako | 4 +- src/generator/templates/cli/README.md.mako | 2 +- .../templates/cli/docs/commands.md.mako | 2 +- src/generator/templates/cli/main.rs.mako | 2 +- src/generator/templates/cli/mkdocs.yml.mako | 2 +- 8 files changed, 57 insertions(+), 59 deletions(-) diff --git a/src/generator/lib/util.py b/src/generator/lib/util.py index f91aeda622..8b075b9fcb 100644 --- a/src/generator/lib/util.py +++ b/src/generator/lib/util.py @@ -1,11 +1,12 @@ -import re import os -from random import (randint, random, choice, seed) -from typing import Any, List, Mapping, Optional, Tuple -import collections -from copy import deepcopy +import re import subprocess +from dataclasses import dataclass +from random import (randint, random, choice, seed) +from typing import Any, Dict, List, Mapping, Tuple +from copy import deepcopy + seed(1337) re_linestart = re.compile('^', flags=re.MULTILINE) @@ -494,8 +495,7 @@ def is_schema_with_optionals(schema_markers): # ------------------------- ## @name Activity Utilities # @{ -# return (category, name|None, method) -def activity_split(fqan: str) -> Tuple[str, Optional[str], str]: +def activity_split(fqan: str) -> Tuple[str, str, str]: t = fqan.split('.') mt = t[2:] if not mt: @@ -663,31 +663,37 @@ It should be used to handle progress information, and to implement a certain lev ## -- End Activity Utilities -- @} -Context = collections.namedtuple('Context', ['sta_map', 'fqan_map', 'rta_map', 'rtc_map', 'schemas']) +@dataclass +class Context: + sta_map: Dict[str, Any] + fqan_map: Dict[str, Any] + rta_map: Dict[str, Any] + rtc_map: Dict[str, Any] + schemas: Dict[str, Any] # return a newly build context from the given data -def new_context(schemas, resources, methods): +def new_context(schemas: Dict[str, Dict[str, Any]], resources: Dict[str, Any]) -> Context: # Returns (A, B) where # A: { SchemaTypeName -> { fqan -> ['request'|'response', ...]} # B: { fqan -> activity_method_data } # fqan = fully qualified activity name - def build_activity_mappings(activities, res = None, fqan = None): + def build_activity_mappings(resources: Dict[str, Any], res = None, fqan = None) -> Tuple[Dict[str, Any], Dict[str, Any]]: if res is None: res = dict() if fqan is None: fqan = dict() - for k,a in activities.items(): + for k,a in resources.items(): if 'resources' in a: - build_activity_mappings(a.resources, res, fqan) + build_activity_mappings(a["resources"], res, fqan) if 'methods' not in a: continue - for mn, m in a.methods.items(): - assert m.id not in fqan - category, resource, method = activity_split(m.id) + for mn, m in a["methods"].items(): + assert m["id"] not in fqan + category, resource, method = activity_split(m["id"]) # This may be another name by which people try to find the method. # As it has no resource, we put in a 'fake resource' (METHODS_RESOURCE), which # needs some special treatment only in key-spots - fqan_key = m.id + fqan_key = m["id"] if resource == METHODS_RESOURCE: fqan_key = to_fqan(category, resource, method) fqan[fqan_key] = m @@ -697,7 +703,7 @@ def new_context(schemas, resources, methods): continue tn = to_rust_type(schemas, None, None, t, allow_optionals=False) info = res.setdefault(tn, dict()) - io_info = info.setdefault(m.id, []) + io_info = info.setdefault(m["id"], []) io_info.append(in_out_type_name) # end for each io type @@ -707,8 +713,8 @@ def new_context(schemas, resources, methods): # the latter is used to deduce the resource name tn = activity_name_to_type_name(resource) info = res.setdefault(tn, dict()) - if m.id not in info: - info.setdefault(m.id, []) + if m["id"] not in info: + info.setdefault(m["id"], []) # end handle other cases # end for each method # end for each activity @@ -717,7 +723,7 @@ def new_context(schemas, resources, methods): # A dict of {s.id -> schema} , with all schemas having the 'parents' key set with [s.id, ...] of all parents # in order of traversal, [-1] is first parent, [0] is the root of them all - def build_schema_map(): + def build_schema_map() -> Dict[str, Any]: # 'type' in t and t.type == 'object' and 'properties' in t or ('items' in t and 'properties' in t.items) PARENT = 'parents' USED_BY = 'used_by' @@ -729,8 +735,8 @@ def new_context(schemas, resources, methods): def link_used(s, rs): if TREF in s: l = assure_list(all_schemas[s[TREF]], USED_BY) - if rs.id not in l: - l.append(rs.id) + if rs["id"] not in l: + l.append(rs["id"]) def append_unique(l, s): if s not in l: @@ -738,14 +744,14 @@ def new_context(schemas, resources, methods): return l all_schemas = deepcopy(schemas) - def recurse_properties(prefix, rs, s, parent_ids): + def recurse_properties(prefix: str, rs: Any, s: Any, parent_ids: List[str]): assure_list(s, USED_BY) assure_list(s, PARENT).extend(parent_ids) link_used(s, rs) if is_nested_type_property(s) and 'id' not in s: s.id = prefix - all_schemas[s.id] = s + all_schemas[s["id"]] = s rs = s # end this is already a perfectly valid type @@ -756,54 +762,46 @@ def new_context(schemas, resources, methods): if is_nested_type_property(p): ns = deepcopy(p) ns.id = _assure_unique_type_name(schemas, nested_type_name(prefix, pn)) - all_schemas[ns.id] = ns + all_schemas[ns["id"]] = ns # To allow us recursing arrays, we simply put items one level up if 'items' in p: - ns.update((k, deepcopy(v)) for k, v in p.items.items()) + ns.update((k, deepcopy(v)) for k, v in p["items"].items()) - recurse_properties(ns.id, ns, ns, append_unique(parent_ids, rs.id)) + recurse_properties(ns.id, ns, ns, append_unique(parent_ids, rs["id"])) elif is_map_prop(p): recurse_properties(nested_type_name(prefix, pn), rs, - p.additionalProperties, append_unique(parent_ids, rs.id)) + p["additionalProperties"], append_unique(parent_ids, rs["id"])) elif 'items' in p: recurse_properties(nested_type_name(prefix, pn), rs, - p.items, append_unique(parent_ids, rs.id)) - elif 'variant' in p: - for enum in p.variant.map: - recurse_properties(prefix, rs, enum, parent_ids) + p["items"], append_unique(parent_ids, rs["id"])) # end handle prop itself # end for each property # end utility for s in all_schemas.values(): - recurse_properties(s.id, s, s, []) + recurse_properties(s["id"], s, s, []) # end for each schema return all_schemas # end utility all_schemas = schemas and build_schema_map() or dict() - if not (resources or methods): + if not resources: return Context(dict(), dict(), dict(), dict(), all_schemas) - rta_map, rtc_map, sta_map, fqan_map = dict(), dict(), dict(), dict() + rta_map: Dict[str, Any] = {} + rtc_map: Dict[str, Any] = {} + sta_map: Dict[str, Any] = {} + fqan_map: Dict[str, Any] = {} - sources = list() - if bool(resources): - sources.append(resources) - if bool(methods): - sources.append({None : type(methods)({'methods' : methods})}) - - for data_source in sources: - _sta_map, _fqan_map = build_activity_mappings(data_source) - for an in _fqan_map: - category, resource, activity = activity_split(an) - rta_map.setdefault(resource, list()).append(activity) - assert rtc_map.setdefault(resource, category) == category - # end for each fqan - sta_map.update(_sta_map) - fqan_map.update(_fqan_map) - # end for each data source + _sta_map, _fqan_map = build_activity_mappings(resources) + for an in _fqan_map: + category, resource, activity = activity_split(an) + rta_map.setdefault(resource, list()).append(activity) + assert rtc_map.setdefault(resource, category) == category + # end for each fqan + sta_map.update(_sta_map) + fqan_map.update(_fqan_map) return Context(sta_map, fqan_map, rta_map, rtc_map, all_schemas) def _is_special_version(v): diff --git a/src/generator/templates/api/README.md.mako b/src/generator/templates/api/README.md.mako index 8908256d02..8d96be6c4c 100644 --- a/src/generator/templates/api/README.md.mako +++ b/src/generator/templates/api/README.md.mako @@ -1,6 +1,6 @@ <% from generator.lib.util import (markdown_comment, new_context) - c = new_context(schemas, resources, context.get('methods')) + c = new_context(schemas, resources) %>\ <%namespace name="lib" file="lib/lib.mako"/>\ <%namespace name="util" file="../../lib/util.mako"/>\ diff --git a/src/generator/templates/api/api.rs.mako b/src/generator/templates/api/api.rs.mako index f86ebd1dd0..b5dfd40a1d 100644 --- a/src/generator/templates/api/api.rs.mako +++ b/src/generator/templates/api/api.rs.mako @@ -9,7 +9,7 @@ rb_type_params_s, find_fattest_resource, HUB_TYPE_PARAMETERS, METHODS_RESOURCE, UNUSED_TYPE_MARKER, schema_markers) - c = new_context(schemas, resources, context.get('methods')) + c = new_context(schemas, resources) hub_type = hub_type(c.schemas, util.canonical_name()) ht_params = hub_type_params_s() diff --git a/src/generator/templates/api/lib.rs.mako b/src/generator/templates/api/lib.rs.mako index a9ef504da6..b6062b2cdf 100644 --- a/src/generator/templates/api/lib.rs.mako +++ b/src/generator/templates/api/lib.rs.mako @@ -3,7 +3,7 @@ <% from generator.lib.util import (new_context, rust_comment, rust_module_doc_comment) - c = new_context(schemas, resources, context.get('methods')) + c = new_context(schemas, resources) %>\ <%block filter="rust_comment">\ <%util:gen_info source="${self.uri}" />\ @@ -30,7 +30,7 @@ ${lib.docs(c)} rb_type_params_s, find_fattest_resource, HUB_TYPE_PARAMETERS, METHODS_RESOURCE, UNUSED_TYPE_MARKER, schema_markers) - c = new_context(schemas, resources, context.get('methods')) + c = new_context(schemas, resources) hub_type = hub_type(c.schemas, util.canonical_name()) ht_params = hub_type_params_s() diff --git a/src/generator/templates/cli/README.md.mako b/src/generator/templates/cli/README.md.mako index 00b2ffa594..b8ef77cf2b 100644 --- a/src/generator/templates/cli/README.md.mako +++ b/src/generator/templates/cli/README.md.mako @@ -2,7 +2,7 @@ from generator.lib.util import (markdown_comment, new_context) from generator.lib.cli import (CONFIG_DIR, CONFIG_DIR_FLAG, SCOPE_FLAG, application_secret_path, DEBUG_FLAG) - c = new_context(schemas, resources, context.get('methods')) + c = new_context(schemas, resources) %>\ <%namespace name="util" file="../../lib/util.mako"/>\ <%namespace name="argparse" file="lib/argparse.mako"/>\ diff --git a/src/generator/templates/cli/docs/commands.md.mako b/src/generator/templates/cli/docs/commands.md.mako index 455b0045cf..fa41742579 100644 --- a/src/generator/templates/cli/docs/commands.md.mako +++ b/src/generator/templates/cli/docs/commands.md.mako @@ -15,7 +15,7 @@ NO_DESC = 'No description provided.' %>\ <% - c = new_context(schemas, resources, context.get('methods')) + c = new_context(schemas, resources) %>\ % for resource in sorted(c.rta_map.keys()): % for method in sorted(c.rta_map[resource]): diff --git a/src/generator/templates/cli/main.rs.mako b/src/generator/templates/cli/main.rs.mako index 52cb8cc038..c7efb11d65 100644 --- a/src/generator/templates/cli/main.rs.mako +++ b/src/generator/templates/cli/main.rs.mako @@ -6,7 +6,7 @@ indent_all_but_first_by) from generator.lib.cli import OUT_ARG, DEBUG_FLAG, opt_value - c = new_context(schemas, resources, context.get('methods')) + c = new_context(schemas, resources) default_user_agent = "google-cli-rust-client/" + cargo.build_version %>\ <%block filter="rust_comment">\ diff --git a/src/generator/templates/cli/mkdocs.yml.mako b/src/generator/templates/cli/mkdocs.yml.mako index 8b915c37dc..6566f5ad77 100644 --- a/src/generator/templates/cli/mkdocs.yml.mako +++ b/src/generator/templates/cli/mkdocs.yml.mako @@ -2,7 +2,7 @@ from generator.lib.util import (put_and, new_context) from generator.lib.cli import (subcommand_md_filename, mangle_subcommand, pretty) - c = new_context(schemas, resources, context.get('methods')) + c = new_context(schemas, resources) %>\ <%namespace name="util" file="../../lib/util.mako"/>\ site_name: ${util.canonical_name()} v${util.crate_version()} From a60caa2690ec45014898d866213020c973d02cee Mon Sep 17 00:00:00 2001 From: Kyle Gentle Date: Thu, 18 Aug 2022 21:02:57 -0400 Subject: [PATCH 4/5] generator lib: Initial tests Move existing tests into a new __tests__ module under generator/lib. Also define a new test for `util.new_context`, to prevent regressions as changes are made. --- .../lib/{test_data => __tests__}/__init__.py | 0 src/generator/lib/__tests__/context_test.py | 1369 +++++++++++++++++ .../lib/__tests__/test_data/__init__.py | 0 .../test_data/discovery_document.py} | 4 + .../lib/{ => __tests__}/util_test.py | 14 +- 5 files changed, 1376 insertions(+), 11 deletions(-) rename src/generator/lib/{test_data => __tests__}/__init__.py (100%) create mode 100644 src/generator/lib/__tests__/context_test.py create mode 100644 src/generator/lib/__tests__/test_data/__init__.py rename src/generator/lib/{test_data/photoslibrary-api.json => __tests__/test_data/discovery_document.py} (99%) rename src/generator/lib/{ => __tests__}/util_test.py (92%) diff --git a/src/generator/lib/test_data/__init__.py b/src/generator/lib/__tests__/__init__.py similarity index 100% rename from src/generator/lib/test_data/__init__.py rename to src/generator/lib/__tests__/__init__.py diff --git a/src/generator/lib/__tests__/context_test.py b/src/generator/lib/__tests__/context_test.py new file mode 100644 index 0000000000..37fb9b2fc4 --- /dev/null +++ b/src/generator/lib/__tests__/context_test.py @@ -0,0 +1,1369 @@ +import json +import unittest +from pprint import pprint + +from generator.lib.util import new_context +from .test_data.discovery_document import DISCOVERY_DOC + + +class ContextTest(unittest.TestCase): + def setUp(self): + self.discovery_doc = json.loads(DISCOVERY_DOC) + + def test_sta_map(self): + expected = { + "AddEnrichmentToAlbumRequest": { + "photoslibrary.albums.addEnrichment": ["request"] + }, + "AddEnrichmentToAlbumResponse": { + "photoslibrary.albums.addEnrichment": ["response"] + }, + "Album": { + "photoslibrary.albums.addEnrichment": [], + "photoslibrary.albums.create": ["response"], + "photoslibrary.albums.get": ["response"], + "photoslibrary.albums.list": [], + "photoslibrary.albums.share": [], + "photoslibrary.sharedAlbums.get": ["response"], + }, + "BatchCreateMediaItemsRequest": { + "photoslibrary.mediaItems.batchCreate": ["request"] + }, + "BatchCreateMediaItemsResponse": { + "photoslibrary.mediaItems.batchCreate": ["response"] + }, + "CreateAlbumRequest": {"photoslibrary.albums.create": ["request"]}, + "JoinSharedAlbumRequest": {"photoslibrary.sharedAlbums.join": ["request"]}, + "JoinSharedAlbumResponse": { + "photoslibrary.sharedAlbums.join": ["response"] + }, + "ListAlbumsResponse": {"photoslibrary.albums.list": ["response"]}, + "ListMediaItemsResponse": {"photoslibrary.mediaItems.list": ["response"]}, + "ListSharedAlbumsResponse": { + "photoslibrary.sharedAlbums.list": ["response"] + }, + "MediaItem": { + "photoslibrary.mediaItems.batchCreate": [], + "photoslibrary.mediaItems.get": ["response"], + "photoslibrary.mediaItems.list": [], + "photoslibrary.mediaItems.search": [], + }, + "SearchMediaItemsRequest": {"photoslibrary.mediaItems.search": ["request"]}, + "SearchMediaItemsResponse": { + "photoslibrary.mediaItems.search": ["response"] + }, + "ShareAlbumRequest": {"photoslibrary.albums.share": ["request"]}, + "ShareAlbumResponse": {"photoslibrary.albums.share": ["response"]}, + "SharedAlbum": { + "photoslibrary.sharedAlbums.get": [], + "photoslibrary.sharedAlbums.join": [], + "photoslibrary.sharedAlbums.list": [], + }, + } + + schemas = self.discovery_doc["schemas"] + resources = self.discovery_doc["resources"] + + actual = new_context(schemas, resources).sta_map + self.assertEqual(actual, expected) + + def test_fqan_map(self): + expected = { + "photoslibrary.mediaItems.batchCreate": { + "flatPath": "v1/mediaItems:batchCreate", + "path": "v1/mediaItems:batchCreate", + "id": "photoslibrary.mediaItems.batchCreate", + "request": {"$ref": "BatchCreateMediaItemsRequest"}, + "description": "Creates one or more media items in a user's Google Photos library.\n\nThis is the second step for creating a media item. For details regarding\nStep 1, uploading the raw bytes to a Google Server, see\nUploading media.\n\nThis call adds the media item to the library. If an album `id` is\nspecified, the call adds the media item to the album too. By default, the\nmedia item will be added to the end of the library or album.\n\nIf an album `id` and position are both defined, the media item is\nadded to the album at the specified position.\n\nIf the call contains multiple media items, they're added at the specified\nposition.\nIf you are creating a media item in a shared album where you are not the\nowner, you are not allowed to position the media item. Doing so will result\nin a `BAD REQUEST` error.", + "response": {"$ref": "BatchCreateMediaItemsResponse"}, + "parameterOrder": [], + "httpMethod": "POST", + "scopes": [ + "https://www.googleapis.com/auth/photoslibrary", + "https://www.googleapis.com/auth/photoslibrary.appendonly", + "https://www.googleapis.com/auth/photoslibrary.sharing", + ], + "parameters": {}, + }, + "photoslibrary.mediaItems.search": { + "flatPath": "v1/mediaItems:search", + "id": "photoslibrary.mediaItems.search", + "path": "v1/mediaItems:search", + "description": "Searches for media items in a user's Google Photos library.\nIf no filters are set, then all media items in the user's library are\nreturned.\nIf an album is set, all media items in the specified album are returned.\nIf filters are specified, media items that match the filters from the user's\nlibrary are listed.\nIf you set both the album and the filters, the request results in an error.", + "request": {"$ref": "SearchMediaItemsRequest"}, + "httpMethod": "POST", + "parameterOrder": [], + "response": {"$ref": "SearchMediaItemsResponse"}, + "parameters": {}, + "scopes": [ + "https://www.googleapis.com/auth/photoslibrary", + "https://www.googleapis.com/auth/photoslibrary.readonly", + "https://www.googleapis.com/auth/photoslibrary.readonly.appcreateddata", + ], + }, + "photoslibrary.mediaItems.list": { + "flatPath": "v1/mediaItems", + "id": "photoslibrary.mediaItems.list", + "path": "v1/mediaItems", + "description": "List all media items from a user's Google Photos library.", + "httpMethod": "GET", + "response": {"$ref": "ListMediaItemsResponse"}, + "parameterOrder": [], + "parameters": { + "pageToken": { + "location": "query", + "description": "A continuation token to get the next page of the results. Adding this to\nthe request returns the rows after the `pageToken`. The `pageToken` should\nbe the value returned in the `nextPageToken` parameter in the response to\nthe `listMediaItems` request.", + "type": "string", + }, + "pageSize": { + "location": "query", + "description": "Maximum number of media items to return in the response. The default number\nof media items to return at a time is 25. The maximum `pageSize` is 100.", + "format": "int32", + "type": "integer", + }, + }, + "scopes": [ + "https://www.googleapis.com/auth/photoslibrary", + "https://www.googleapis.com/auth/photoslibrary.readonly", + "https://www.googleapis.com/auth/photoslibrary.readonly.appcreateddata", + ], + }, + "photoslibrary.mediaItems.get": { + "description": "Returns the media item for the specified media item `id`.", + "response": {"$ref": "MediaItem"}, + "parameterOrder": ["mediaItemId"], + "httpMethod": "GET", + "parameters": { + "mediaItemId": { + "description": "Identifier of media item to be requested.", + "required": True, + "type": "string", + "pattern": "^[^/]+$", + "location": "path", + } + }, + "scopes": [ + "https://www.googleapis.com/auth/photoslibrary", + "https://www.googleapis.com/auth/photoslibrary.readonly", + "https://www.googleapis.com/auth/photoslibrary.readonly.appcreateddata", + ], + "flatPath": "v1/mediaItems/{mediaItemsId}", + "path": "v1/mediaItems/{+mediaItemId}", + "id": "photoslibrary.mediaItems.get", + }, + "photoslibrary.sharedAlbums.get": { + "httpMethod": "GET", + "response": {"$ref": "Album"}, + "parameterOrder": ["shareToken"], + "parameters": { + "albumId": { + "location": "query", + "description": "Identifier of the album to be requested. Must not be set if `shareToken` is\nset.", + "type": "string", + }, + "shareToken": { + "location": "path", + "description": "Share token of the album to be request. Must not be set if `albumId` is\nset.", + "required": True, + "type": "string", + "pattern": "^[^/]+$", + }, + }, + "scopes": [ + "https://www.googleapis.com/auth/photoslibrary", + "https://www.googleapis.com/auth/photoslibrary.readonly", + "https://www.googleapis.com/auth/photoslibrary.readonly.appcreateddata", + "https://www.googleapis.com/auth/photoslibrary.sharing", + ], + "flatPath": "v1/sharedAlbums/{sharedAlbumsId}", + "id": "photoslibrary.sharedAlbums.get", + "path": "v1/sharedAlbums/{+shareToken}", + "description": "Returns the album based on the specified `albumId` or `shareToken`.\nExactly one of `albumId` and `shareToken` must be set.\nThe `albumId` should be the ID of an album owned by the user or a shared\nalbum that the user has joined.", + }, + "photoslibrary.sharedAlbums.list": { + "response": {"$ref": "ListSharedAlbumsResponse"}, + "parameterOrder": [], + "httpMethod": "GET", + "scopes": [ + "https://www.googleapis.com/auth/photoslibrary", + "https://www.googleapis.com/auth/photoslibrary.readonly", + "https://www.googleapis.com/auth/photoslibrary.readonly.appcreateddata", + ], + "parameters": { + "pageSize": { + "description": "Maximum number of albums to return in the response. The default number of\nalbums to return at a time is 20. The maximum `pageSize` is 50.", + "format": "int32", + "type": "integer", + "location": "query", + }, + "excludeNonAppCreatedData": { + "description": "If set, the results exclude media items that were not created by this app.\nDefaults to false (all albums are returned). This field is ignored if the\nphotoslibrary.readonly.appcreateddata scope is used.", + "type": "boolean", + "location": "query", + }, + "pageToken": { + "location": "query", + "description": "A continuation token to get the next page of the results. Adding this to\nthe request returns the rows after the `pageToken`. The `pageToken` should\nbe the value returned in the `nextPageToken` parameter in the response to\nthe `listSharedAlbums` request.", + "type": "string", + }, + }, + "flatPath": "v1/sharedAlbums", + "path": "v1/sharedAlbums", + "id": "photoslibrary.sharedAlbums.list", + "description": "Lists all shared albums available in the Sharing tab of the\nuser's Google Photos app.", + }, + "photoslibrary.sharedAlbums.join": { + "description": "Joins a shared album on behalf of the Google Photos user.", + "request": {"$ref": "JoinSharedAlbumRequest"}, + "httpMethod": "POST", + "parameterOrder": [], + "response": {"$ref": "JoinSharedAlbumResponse"}, + "parameters": {}, + "scopes": ["https://www.googleapis.com/auth/photoslibrary.sharing"], + "flatPath": "v1/sharedAlbums:join", + "id": "photoslibrary.sharedAlbums.join", + "path": "v1/sharedAlbums:join", + }, + "photoslibrary.albums.list": { + "flatPath": "v1/albums", + "id": "photoslibrary.albums.list", + "path": "v1/albums", + "description": "Lists all albums shown to a user in the Albums tab of the Google\nPhotos app.", + "httpMethod": "GET", + "response": {"$ref": "ListAlbumsResponse"}, + "parameterOrder": [], + "parameters": { + "pageToken": { + "location": "query", + "description": "A continuation token to get the next page of the results. Adding this to\nthe request returns the rows after the `pageToken`. The `pageToken` should\nbe the value returned in the `nextPageToken` parameter in the response to\nthe `listAlbums` request.", + "type": "string", + }, + "pageSize": { + "location": "query", + "description": "Maximum number of albums to return in the response. The default number of\nalbums to return at a time is 20. The maximum `pageSize` is 50.", + "format": "int32", + "type": "integer", + }, + "excludeNonAppCreatedData": { + "description": "If set, the results exclude media items that were not created by this app.\nDefaults to false (all albums are returned). This field is ignored if the\nphotoslibrary.readonly.appcreateddata scope is used.", + "type": "boolean", + "location": "query", + }, + }, + "scopes": [ + "https://www.googleapis.com/auth/photoslibrary", + "https://www.googleapis.com/auth/photoslibrary.readonly", + "https://www.googleapis.com/auth/photoslibrary.readonly.appcreateddata", + ], + }, + "photoslibrary.albums.get": { + "flatPath": "v1/albums/{albumsId}", + "id": "photoslibrary.albums.get", + "path": "v1/albums/{+albumId}", + "description": "Returns the album based on the specified `albumId` or `shareToken`.\nExactly one of `albumId` and `shareToken` must be set.\nThe `albumId` should be the ID of an album owned by the user or a shared\nalbum that the user has joined.", + "httpMethod": "GET", + "response": {"$ref": "Album"}, + "parameterOrder": ["albumId"], + "parameters": { + "albumId": { + "location": "path", + "description": "Identifier of the album to be requested. Must not be set if `shareToken` is\nset.", + "required": True, + "type": "string", + "pattern": "^[^/]+$", + }, + "shareToken": { + "location": "query", + "description": "Share token of the album to be request. Must not be set if `albumId` is\nset.", + "type": "string", + }, + }, + "scopes": [ + "https://www.googleapis.com/auth/photoslibrary", + "https://www.googleapis.com/auth/photoslibrary.readonly", + "https://www.googleapis.com/auth/photoslibrary.readonly.appcreateddata", + "https://www.googleapis.com/auth/photoslibrary.sharing", + ], + }, + "photoslibrary.albums.addEnrichment": { + "response": {"$ref": "AddEnrichmentToAlbumResponse"}, + "parameterOrder": ["albumId"], + "httpMethod": "POST", + "scopes": [ + "https://www.googleapis.com/auth/photoslibrary", + "https://www.googleapis.com/auth/photoslibrary.appendonly", + "https://www.googleapis.com/auth/photoslibrary.sharing", + ], + "parameters": { + "albumId": { + "description": "Identifier of the album where the enrichment is to be added.", + "required": True, + "type": "string", + "pattern": "^[^/]+$", + "location": "path", + } + }, + "flatPath": "v1/albums/{albumsId}:addEnrichment", + "path": "v1/albums/{+albumId}:addEnrichment", + "id": "photoslibrary.albums.addEnrichment", + "request": {"$ref": "AddEnrichmentToAlbumRequest"}, + "description": "Adds an enrichment at a specified position in a defined album.", + }, + "photoslibrary.albums.create": { + "response": {"$ref": "Album"}, + "parameterOrder": [], + "httpMethod": "POST", + "scopes": [ + "https://www.googleapis.com/auth/photoslibrary", + "https://www.googleapis.com/auth/photoslibrary.appendonly", + "https://www.googleapis.com/auth/photoslibrary.sharing", + ], + "parameters": {}, + "flatPath": "v1/albums", + "path": "v1/albums", + "id": "photoslibrary.albums.create", + "request": {"$ref": "CreateAlbumRequest"}, + "description": "Creates an album in a user's Google Photos library.", + }, + "photoslibrary.albums.share": { + "request": {"$ref": "ShareAlbumRequest"}, + "description": "Marks an album as shared and accessible to other users. This action can\nonly be performed on albums which were created by the developer via the\nAPI.", + "response": {"$ref": "ShareAlbumResponse"}, + "parameterOrder": ["albumId"], + "httpMethod": "POST", + "scopes": ["https://www.googleapis.com/auth/photoslibrary.sharing"], + "parameters": { + "albumId": { + "location": "path", + "description": "Identifier of the album to be shared. This `albumId` must belong to an\nalbum created by the developer.", + "required": True, + "type": "string", + "pattern": "^[^/]+$", + } + }, + "flatPath": "v1/albums/{albumsId}:share", + "path": "v1/albums/{+albumId}:share", + "id": "photoslibrary.albums.share", + }, + } + + schemas = self.discovery_doc["schemas"] + resources = self.discovery_doc["resources"] + + actual = new_context(schemas, resources).fqan_map + self.assertEqual(actual, expected) + + def test_rta_map(self): + expected = { + "mediaItems": ["batchCreate", "search", "list", "get"], + "sharedAlbums": ["get", "list", "join"], + "albums": ["list", "get", "addEnrichment", "create", "share"], + } + + schemas = self.discovery_doc["schemas"] + resources = self.discovery_doc["resources"] + + actual = new_context(schemas, resources).rta_map + self.assertEqual(actual, expected) + + def test_rtc_map(self): + expected = { + "mediaItems": "photoslibrary", + "sharedAlbums": "photoslibrary", + "albums": "photoslibrary", + } + + schemas = self.discovery_doc["schemas"] + resources = self.discovery_doc["resources"] + + actual = new_context(schemas, resources).rtc_map + with open("actual.json", "w") as aj: + json.dump(actual, aj, indent=4) + self.assertEqual(actual, expected) + + def test_schemas(self): + expected = { + "EnrichmentItem": { + "description": "An enrichment item.", + "type": "object", + "properties": { + "id": { + "description": "Identifier of the enrichment item.", + "type": "string", + } + }, + "id": "EnrichmentItem", + "used_by": ["AddEnrichmentToAlbumResponse"], + "parents": [], + }, + "BatchCreateMediaItemsRequest": { + "description": "Request to create one or more media items in a user's Google Photos library.\nIf an `albumid` is specified, the media items are also added to that album.\n`albumPosition` is optional and can only be specified if an `albumId` is set.", + "type": "object", + "properties": { + "albumId": { + "description": "Identifier of the album where the media items are added. The media items\nare also added to the user's library. This is an optional field.", + "type": "string", + }, + "newMediaItems": { + "description": "List of media items to be created.", + "type": "array", + "items": { + "$ref": "NewMediaItem", + "used_by": [], + "parents": ["BatchCreateMediaItemsRequest"], + }, + }, + "albumPosition": { + "$ref": "AlbumPosition", + "description": "Position in the album where the media items are added. If not\nspecified, the media items are added to the end of the album (as per\nthe default value, that is, `LAST_IN_ALBUM`). The request fails if this\nfield is set and the `albumId` is not specified. The request will also fail\nif you set the field and are not the owner of the shared album.", + }, + }, + "id": "BatchCreateMediaItemsRequest", + "used_by": [], + "parents": [], + }, + "MapEnrichment": { + "description": "An enrichment containing a map, showing origin and destination locations.", + "type": "object", + "properties": { + "destination": { + "$ref": "Location", + "description": "Destination location for this enrichemt item.", + }, + "origin": { + "description": "Origin location for this enrichment item.", + "$ref": "Location", + }, + }, + "id": "MapEnrichment", + "used_by": ["NewEnrichmentItem"], + "parents": [], + }, + "AddEnrichmentToAlbumRequest": { + "description": "Request to add an enrichment to a specific album at a specific position.", + "type": "object", + "properties": { + "newEnrichmentItem": { + "$ref": "NewEnrichmentItem", + "description": "The enrichment to be added.", + }, + "albumPosition": { + "description": "The position in the album where the enrichment is to be inserted.", + "$ref": "AlbumPosition", + }, + }, + "id": "AddEnrichmentToAlbumRequest", + "used_by": [], + "parents": [], + }, + "DateRange": { + "description": 'Defines a range of dates. Both dates must be of the same format. For more\ninformation, see Date', + "type": "object", + "properties": { + "startDate": { + "description": "The start date (included as part of the range) in one of the formats\ndescribed.", + "$ref": "Date", + }, + "endDate": { + "description": "The end date (included as part of the range). It must be specified in the\nsame format as the start date.", + "$ref": "Date", + }, + }, + "id": "DateRange", + "used_by": ["DateFilter"], + "parents": [], + }, + "NewMediaItem": { + "description": "New media item that's created in a user's Google Photos account.", + "type": "object", + "properties": { + "description": { + "description": "Description of the media item. This will be shown to the user in the item's\ninfo section in the Google Photos app.\nThis string shouldn't be more than 1000 characters.", + "type": "string", + }, + "simpleMediaItem": { + "description": "A new media item that has been uploaded via the included `uploadToken`.", + "$ref": "SimpleMediaItem", + }, + }, + "id": "NewMediaItem", + "used_by": ["BatchCreateMediaItemsRequest"], + "parents": [], + }, + "Status": { + "description": "The `Status` type defines a logical error model that is suitable for different\nprogramming environments, including REST APIs and RPC APIs. It is used by\n[gRPC](https://github.com/grpc). The error model is designed to be:\n\n- Simple to use and understand for most users\n- Flexible enough to meet unexpected needs\n\n# Overview\n\nThe `Status` message contains three pieces of data: error code, error message,\nand error details. The error code should be an enum value of\ngoogle.rpc.Code, but it may accept additional error codes if needed. The\nerror message should be a developer-facing English message that helps\ndevelopers *understand* and *resolve* the error. If a localized user-facing\nerror message is needed, put the localized message in the error details or\nlocalize it in the client. The optional error details may contain arbitrary\ninformation about the error. There is a predefined set of error detail types\nin the package `google.rpc` that can be used for common error conditions.\n\n# Language mapping\n\nThe `Status` message is the logical representation of the error model, but it\nis not necessarily the actual wire format. When the `Status` message is\nexposed in different client libraries and different wire protocols, it can be\nmapped differently. For example, it will likely be mapped to some exceptions\nin Java, but more likely mapped to some error codes in C.\n\n# Other uses\n\nThe error model and the `Status` message can be used in a variety of\nenvironments, either with or without APIs, to provide a\nconsistent developer experience across different environments.\n\nExample uses of this error model include:\n\n- Partial errors. If a service needs to return partial errors to the client,\n it may embed the `Status` in the normal response to indicate the partial\n errors.\n\n- Workflow errors. A typical workflow has multiple steps. Each step may\n have a `Status` message for error reporting.\n\n- Batch operations. If a client uses batch request and batch response, the\n `Status` message should be used directly inside batch response, one for\n each error sub-response.\n\n- Asynchronous operations. If an API call embeds asynchronous operation\n results in its response, the status of those operations should be\n represented directly using the `Status` message.\n\n- Logging. If some API errors are stored in logs, the message `Status` could\n be used directly after any stripping needed for security/privacy reasons.", + "type": "object", + "properties": { + "code": { + "description": "The status code, which should be an enum value of google.rpc.Code.", + "format": "int32", + "type": "integer", + }, + "message": { + "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\ngoogle.rpc.Status.details field, or localized by the client.", + "type": "string", + }, + "details": { + "description": "A list of messages that carry the error details. There is a common set of\nmessage types for APIs to use.", + "type": "array", + "items": { + "type": "object", + "additionalProperties": { + "description": "Properties of the object. Contains field @type with type URL.", + "type": "any", + "used_by": [], + "parents": ["Status"], + }, + "used_by": [], + "parents": ["Status"], + }, + }, + }, + "id": "Status", + "used_by": ["NewMediaItemResult"], + "parents": [], + }, + "MediaTypeFilter": { + "description": "This filter defines the type of media items to be returned, for example,\nvideos or photos. All the specified media types are treated as an OR when\nused together.", + "type": "object", + "properties": { + "mediaTypes": { + "description": "The types of media items to be included. This field should be populated\nwith only one media type. If you specify multiple media types, it results\nin an error.", + "type": "array", + "items": { + "type": "string", + "enum": ["ALL_MEDIA", "VIDEO", "PHOTO"], + "used_by": [], + "parents": ["MediaTypeFilter"], + }, + "enumDescriptions": [ + "Treated as if no filters are applied. All media types are included.", + "All media items that are considered videos.\nThis also includes movies the user has created using the Google Photos\napp.", + "All media items that are considered photos. This includes .bmp, .gif,\n.ico, .jpg (and other spellings), .tiff, .webp and special photo types\nsuch as iOS live photos, Android motion photos, panoramas, photospheres.", + ], + } + }, + "id": "MediaTypeFilter", + "used_by": ["Filters"], + "parents": [], + }, + "SearchMediaItemsResponse": { + "description": "List of media items that match the search parameters.", + "type": "object", + "properties": { + "nextPageToken": { + "description": "[Output only] Use this token to get the next set of media items. Its\npresence is the only reliable indicator of more media items being available\nin the next request.", + "type": "string", + }, + "mediaItems": { + "description": "[Output only] List of media items that match the search parameters.", + "type": "array", + "items": { + "$ref": "MediaItem", + "used_by": [], + "parents": ["SearchMediaItemsResponse"], + }, + }, + }, + "id": "SearchMediaItemsResponse", + "used_by": [], + "parents": [], + }, + "LocationEnrichment": { + "description": "An enrichment containing a single location.", + "type": "object", + "properties": { + "location": { + "$ref": "Location", + "description": "Location for this enrichment item.", + } + }, + "id": "LocationEnrichment", + "used_by": ["NewEnrichmentItem"], + "parents": [], + }, + "NewEnrichmentItem": { + "description": "A new enrichment item to be added to an album, used by the\n`albums.addEnrichment` call.", + "type": "object", + "properties": { + "mapEnrichment": { + "$ref": "MapEnrichment", + "description": "Map to be added to the album.", + }, + "textEnrichment": { + "$ref": "TextEnrichment", + "description": "Text to be added to the album.", + }, + "locationEnrichment": { + "description": "Location to be added to the album.", + "$ref": "LocationEnrichment", + }, + }, + "id": "NewEnrichmentItem", + "used_by": ["AddEnrichmentToAlbumRequest"], + "parents": [], + }, + "ListMediaItemsResponse": { + "description": "List of all media items from the user's Google Photos library.", + "type": "object", + "properties": { + "nextPageToken": { + "description": "[Output only] Token to use to get the next set of media items. Its presence\nis the only reliable indicator of more media items being available in the\nnext request.", + "type": "string", + }, + "mediaItems": { + "description": "[Output only] List of media items in the user's library.", + "type": "array", + "items": { + "$ref": "MediaItem", + "used_by": [], + "parents": ["ListMediaItemsResponse"], + }, + }, + }, + "id": "ListMediaItemsResponse", + "used_by": [], + "parents": [], + }, + "AlbumPosition": { + "description": "Specifies a position in an album.", + "type": "object", + "properties": { + "relativeMediaItemId": { + "description": "The media item to which the position is relative to.\nOnly used when position type is AFTER_MEDIA_ITEM.", + "type": "string", + }, + "position": { + "enumDescriptions": [ + "Default value if this enum isn't set.", + "At the beginning of the album.", + "At the end of the album.", + "After a media item.", + "After an enrichment item.", + ], + "enum": [ + "POSITION_TYPE_UNSPECIFIED", + "FIRST_IN_ALBUM", + "LAST_IN_ALBUM", + "AFTER_MEDIA_ITEM", + "AFTER_ENRICHMENT_ITEM", + ], + "description": "Type of position, for a media or enrichment item.", + "type": "string", + }, + "relativeEnrichmentItemId": { + "description": "The enrichment item to which the position is relative to.\nOnly used when position type is AFTER_ENRICHMENT_ITEM.", + "type": "string", + }, + }, + "id": "AlbumPosition", + "used_by": [ + "BatchCreateMediaItemsRequest", + "AddEnrichmentToAlbumRequest", + ], + "parents": [], + }, + "LatLng": { + "description": 'An object representing a latitude/longitude pair. This is expressed as a pair\nof doubles representing degrees latitude and degrees longitude. Unless\nspecified otherwise, this must conform to the\nWGS84\nstandard. Values must be within normalized ranges.', + "type": "object", + "properties": { + "latitude": { + "description": "The latitude in degrees. It must be in the range [-90.0, +90.0].", + "format": "double", + "type": "number", + }, + "longitude": { + "description": "The longitude in degrees. It must be in the range [-180.0, +180.0].", + "format": "double", + "type": "number", + }, + }, + "id": "LatLng", + "used_by": ["Location"], + "parents": [], + }, + "BatchCreateMediaItemsResponse": { + "description": "List of media items created.", + "type": "object", + "properties": { + "newMediaItemResults": { + "description": "[Output only] List of media items created.", + "type": "array", + "items": { + "$ref": "NewMediaItemResult", + "used_by": [], + "parents": ["BatchCreateMediaItemsResponse"], + }, + } + }, + "id": "BatchCreateMediaItemsResponse", + "used_by": [], + "parents": [], + }, + "ShareInfo": { + "description": "Information about albums that are shared. This information is only included\nif you created the album, it is shared and you have the sharing scope.", + "type": "object", + "properties": { + "isJoined": { + "description": "True if the user has joined the album. This is always true for the owner\nof the shared album.", + "type": "boolean", + }, + "sharedAlbumOptions": { + "$ref": "SharedAlbumOptions", + "description": "Options that control the sharing of an album.", + }, + "shareableUrl": { + "description": "A link to the album that's now shared on the Google Photos website and app.\nAnyone with the link can access this shared album and see all of the items\npresent in the album.", + "type": "string", + }, + "shareToken": { + "description": "A token that can be used by other users to join this shared album via the\nAPI.", + "type": "string", + }, + }, + "id": "ShareInfo", + "used_by": ["Album", "ShareAlbumResponse"], + "parents": [], + }, + "MediaItem": { + "description": "Representation of a media item (such as a photo or video) in Google Photos.", + "type": "object", + "properties": { + "baseUrl": { + "description": "A URL to the media item's bytes. This shouldn't be used directly to access\nthe media item. For example, `'=w2048-h1024'` will set the dimensions of a\nmedia item of type photo to have a width of 2048 px and height of 1024 px.", + "type": "string", + }, + "mimeType": { + "description": "MIME type of the media item. For example, `image/jpeg`.", + "type": "string", + }, + "contributorInfo": { + "$ref": "ContributorInfo", + "description": "Information about the user who created this media item.", + }, + "description": { + "description": "Description of the media item. This is shown to the user in the item's\ninfo section in the Google Photos app.", + "type": "string", + }, + "mediaMetadata": { + "$ref": "MediaMetadata", + "description": "Metadata related to the media item, such as, height, width, or\ncreation time.", + }, + "filename": { + "description": "Filename of the media item. This is shown to the user in the item's info\nsection in the Google Photos app.", + "type": "string", + }, + "id": { + "description": "Identifier for the media item. This is a persistent identifier that can be\nused between sessions to identify this media item.", + "type": "string", + }, + "productUrl": { + "description": "Google Photos URL for the media item. This link is available to\nthe user only if they're signed in.", + "type": "string", + }, + }, + "id": "MediaItem", + "used_by": [ + "SearchMediaItemsResponse", + "ListMediaItemsResponse", + "NewMediaItemResult", + ], + "parents": [], + }, + "Album": { + "description": "Representation of an album in Google Photos.\nAlbums are containers for media items. If an album has been shared by the\napplication, it contains an extra `shareInfo` property.", + "type": "object", + "properties": { + "mediaItemsCount": { + "description": "[Output only] The number of media items in the album.", + "format": "int64", + "type": "string", + }, + "title": { + "description": "Name of the album displayed to the user in their Google Photos account.\nThis string shouldn't be more than 500 characters.", + "type": "string", + }, + "coverPhotoBaseUrl": { + "description": "[Output only] A URL to the cover photo's bytes. This shouldn't be used as\nis. Parameters should be appended to this URL before use. For example,\n`'=w2048-h1024'` sets the dimensions of\nthe cover photo to have a width of 2048 px and height of 1024 px.", + "type": "string", + }, + "isWriteable": { + "description": "[Output only] True if you can create media items in this album.\nThis field is based on the scopes granted and permissions of the album. If\nthe scopes are changed or permissions of the album are changed, this field\nis updated.", + "type": "boolean", + }, + "coverPhotoMediaItemId": { + "description": "[Output only] Identifier for the media item associated with the cover\nphoto.", + "type": "string", + }, + "id": { + "description": "[Ouput only] Identifier for the album. This is a persistent identifier that\ncan be used between sessions to identify this album.", + "type": "string", + }, + "productUrl": { + "description": "[Output only] Google Photos URL for the album. The user needs to be signed\nin to their Google Photos account to access this link.", + "type": "string", + }, + "shareInfo": { + "description": "[Output only] Information related to shared albums.\nThis field is only populated if the album is a shared album, the\ndeveloper created the album and the user has granted the\n`photoslibrary.sharing` scope.", + "$ref": "ShareInfo", + }, + }, + "id": "Album", + "used_by": [ + "ListAlbumsResponse", + "ListSharedAlbumsResponse", + "CreateAlbumRequest", + "JoinSharedAlbumResponse", + ], + "parents": [], + }, + "JoinSharedAlbumRequest": { + "description": "Request to join a shared album on behalf of the user. This uses a shareToken\nwhich can be acquired via the shareAlbum or listSharedAlbums calls.", + "type": "object", + "properties": { + "shareToken": { + "description": "Token to join the shared album on behalf of the user.", + "type": "string", + } + }, + "id": "JoinSharedAlbumRequest", + "used_by": [], + "parents": [], + }, + "SharedAlbumOptions": { + "description": "Options that control the sharing of an album.", + "type": "object", + "properties": { + "isCollaborative": { + "description": "True if the shared album allows collaborators (users who have joined\nthe album) to add media items to it. Defaults to false.", + "type": "boolean", + }, + "isCommentable": { + "description": "True if the shared album allows the owner and the collaborators (users\nwho have joined the album) to add comments to the album. Defaults to false.", + "type": "boolean", + }, + }, + "id": "SharedAlbumOptions", + "used_by": ["ShareInfo", "ShareAlbumRequest"], + "parents": [], + }, + "TextEnrichment": { + "description": "An enrichment containing text.", + "type": "object", + "properties": { + "text": { + "description": "Text for this enrichment item.", + "type": "string", + } + }, + "id": "TextEnrichment", + "used_by": ["NewEnrichmentItem"], + "parents": [], + }, + "ContentFilter": { + "description": "This filter allows you to return media items based on the content type.\n\nIt's possible to specify a list of categories to include, and/or a list of\ncategories to exclude. Within each list, the categories are combined with an\nOR.

\nThe content filter `includedContentCategories`: [c1, c2, c3] would get media\nitems that contain (c1 OR c2 OR c3).

\nThe content filter `excludedContentCategories`: [c1, c2, c3] would NOT get\nmedia items that contain (c1 OR c2 OR c3).

\nYou can also include some categories while excluding others, as in this\nexample: `includedContentCategories`: [c1, c2], `excludedContentCategories`:\n[c3, c4]

\nThe previous example would get media items that contain (c1 OR c2) AND NOT\n(c3 OR c4). A category that appears in `includedContentategories` must not\nappear in `excludedContentCategories`.", + "type": "object", + "properties": { + "includedContentCategories": { + "enumDescriptions": [ + "Default content category. This category is ignored when any other category\nis used in the filter.", + "Media items containing landscapes.", + "Media items containing receipts.", + "Media items containing cityscapes.", + "Media items containing landmarks.", + "Media items that are selfies.", + "Media items containing people.", + "Media items containing pets.", + "Media items from weddings.", + "Media items from birthdays.", + "Media items containing documents.", + "Media items taken during travel.", + "Media items containing animals.", + "Media items containing food.", + "Media items from sporting events.", + "Media items taken at night.", + "Media items from performances.", + "Media items containing whiteboards.", + "Media items that are screenshots.", + "Media items that are considered to be utility. These include, but aren't\nlimited to documents, screenshots, whiteboards etc.", + ], + "description": "The set of categories to be included in the media item search results.\nThe items in the set are ORed. There's a maximum of 10\n`includedContentCategories` per request.", + "type": "array", + "items": { + "type": "string", + "enum": [ + "NONE", + "LANDSCAPES", + "RECEIPTS", + "CITYSCAPES", + "LANDMARKS", + "SELFIES", + "PEOPLE", + "PETS", + "WEDDINGS", + "BIRTHDAYS", + "DOCUMENTS", + "TRAVEL", + "ANIMALS", + "FOOD", + "SPORT", + "NIGHT", + "PERFORMANCES", + "WHITEBOARDS", + "SCREENSHOTS", + "UTILITY", + ], + "used_by": [], + "parents": ["ContentFilter"], + }, + }, + "excludedContentCategories": { + "enumDescriptions": [ + "Default content category. This category is ignored when any other category\nis used in the filter.", + "Media items containing landscapes.", + "Media items containing receipts.", + "Media items containing cityscapes.", + "Media items containing landmarks.", + "Media items that are selfies.", + "Media items containing people.", + "Media items containing pets.", + "Media items from weddings.", + "Media items from birthdays.", + "Media items containing documents.", + "Media items taken during travel.", + "Media items containing animals.", + "Media items containing food.", + "Media items from sporting events.", + "Media items taken at night.", + "Media items from performances.", + "Media items containing whiteboards.", + "Media items that are screenshots.", + "Media items that are considered to be utility. These include, but aren't\nlimited to documents, screenshots, whiteboards etc.", + ], + "description": "The set of categories which are not to be included in the media item search\nresults. The items in the set are ORed. There's a maximum of 10\n`excludedContentCategories` per request.", + "type": "array", + "items": { + "enum": [ + "NONE", + "LANDSCAPES", + "RECEIPTS", + "CITYSCAPES", + "LANDMARKS", + "SELFIES", + "PEOPLE", + "PETS", + "WEDDINGS", + "BIRTHDAYS", + "DOCUMENTS", + "TRAVEL", + "ANIMALS", + "FOOD", + "SPORT", + "NIGHT", + "PERFORMANCES", + "WHITEBOARDS", + "SCREENSHOTS", + "UTILITY", + ], + "type": "string", + "used_by": [], + "parents": ["ContentFilter"], + }, + }, + }, + "id": "ContentFilter", + "used_by": ["Filters"], + "parents": [], + }, + "ShareAlbumResponse": { + "description": "Response to successfully sharing an album.", + "type": "object", + "properties": { + "shareInfo": { + "$ref": "ShareInfo", + "description": "[Output only] Information about the shared album.", + } + }, + "id": "ShareAlbumResponse", + "used_by": [], + "parents": [], + }, + "Date": { + "description": "Represents a whole calendar date. The day may be 0 to represent a year and month where the day isn't significant, such as a whole calendar month. The month may be 0 to represent a a day and a year where the month isn't signficant, like when you want to specify the same day in every month of a year or a specific year. The year may be 0 to represent a month and day independent of year, like an anniversary date.", + "type": "object", + "properties": { + "year": { + "description": "Year of date. Must be from 1 to 9999, or 0 if specifying a date without\na year.", + "format": "int32", + "type": "integer", + }, + "day": { + "description": "Day of month. Must be from 1 to 31 and valid for the year and month, or 0 if specifying a year/month where the day isn't significant.", + "format": "int32", + "type": "integer", + }, + "month": { + "description": "Month of year. Must be from 1 to 12, or 0 if specifying a year without a\nmonth and day.", + "format": "int32", + "type": "integer", + }, + }, + "id": "Date", + "used_by": ["DateRange", "DateFilter"], + "parents": [], + }, + "Filters": { + "description": "Filters that can be applied to a media item search.\nIf multiple filter options are specified, they're treated as AND with each\nother.", + "type": "object", + "properties": { + "mediaTypeFilter": { + "description": "Filters the media items based on the type of media.", + "$ref": "MediaTypeFilter", + }, + "excludeNonAppCreatedData": { + "description": "If set, the results exclude media items that were not created by this app.\nDefaults to false (all media items are returned). This field is ignored if\nthe photoslibrary.readonly.appcreateddata scope is used.", + "type": "boolean", + }, + "dateFilter": { + "description": "Filters the media items based on their creation date.", + "$ref": "DateFilter", + }, + "contentFilter": { + "description": "Filters the media items based on their content.", + "$ref": "ContentFilter", + }, + "includeArchivedMedia": { + "description": "If set, the results include media items that the user has archived.\nDefaults to false (archived media items aren't included).", + "type": "boolean", + }, + }, + "id": "Filters", + "used_by": ["SearchMediaItemsRequest"], + "parents": [], + }, + "MediaMetadata": { + "description": "Metadata for a media item.", + "type": "object", + "properties": { + "video": { + "description": "Metadata for a video media type.", + "$ref": "Video", + }, + "width": { + "description": "Original width (in pixels) of the media item.", + "format": "int64", + "type": "string", + }, + "creationTime": { + "description": "Time when the media item was first created (not when it was uploaded to\nGoogle Photos).", + "format": "google-datetime", + "type": "string", + }, + "height": { + "description": "Original height (in pixels) of the media item.", + "format": "int64", + "type": "string", + }, + "photo": { + "description": "Metadata for a photo media type.", + "$ref": "Photo", + }, + }, + "id": "MediaMetadata", + "used_by": ["MediaItem"], + "parents": [], + }, + "SearchMediaItemsRequest": { + "description": "Request to search for media items in a user's library.\n\nIf the album id is specified, this call will return the list of media items\nin the album. If neither filters nor album id are\nspecified, this call will return all media items in a user's Google Photos\nlibrary.\n\nIf filters are specified, this call will return all media items in\nthe user's library that fulfill the filter criteria.\n\nFilters and album id must not both be set, as this will result in an\ninvalid request.", + "type": "object", + "properties": { + "albumId": { + "description": "Identifier of an album. If populated, lists all media items in\nspecified album. Can't set in conjunction with any filters.", + "type": "string", + }, + "pageSize": { + "description": "Maximum number of media items to return in the response. The default number\nof media items to return at a time is 25. The maximum\n`pageSize` is 100.", + "format": "int32", + "type": "integer", + }, + "filters": { + "$ref": "Filters", + "description": "Filters to apply to the request. Can't be set in conjunction with an\n`albumId`.", + }, + "pageToken": { + "description": "A continuation token to get the next page of the results. Adding this to\nthe request returns the rows after the `pageToken`. The `pageToken` should\nbe the value returned in the `nextPageToken` parameter in the response to\nthe `searchMediaItems` request.", + "type": "string", + }, + }, + "id": "SearchMediaItemsRequest", + "used_by": [], + "parents": [], + }, + "Location": { + "description": "Represents a physical location.", + "type": "object", + "properties": { + "latlng": { + "description": "Position of the location on the map.", + "$ref": "LatLng", + }, + "locationName": { + "description": "Name of the location to be displayed.", + "type": "string", + }, + }, + "id": "Location", + "used_by": ["MapEnrichment", "LocationEnrichment"], + "parents": [], + }, + "Video": { + "description": "Metadata that is specific to a video, for example, fps and processing status.\nSome of these fields may be null or not included.", + "type": "object", + "properties": { + "fps": { + "description": "Frame rate of the video.", + "format": "double", + "type": "number", + }, + "cameraModel": { + "description": "Model of the camera with which the video was taken.", + "type": "string", + }, + "status": { + "description": "Processing status of the video.", + "type": "string", + "enumDescriptions": [ + "Video processing status is unknown.", + "Video is being processed. The user sees an icon for this\nvideo in the Google Photos app; however, it isn't playable yet.", + "Video processing is complete and it is now ready for viewing.", + "Something has gone wrong and the video has failed to process.", + ], + "enum": ["UNSPECIFIED", "PROCESSING", "READY", "FAILED"], + }, + "cameraMake": { + "description": "Brand of the camera with which the video was taken.", + "type": "string", + }, + }, + "id": "Video", + "used_by": ["MediaMetadata"], + "parents": [], + }, + "DateFilter": { + "description": "This filter defines the allowed dates or date ranges for the media returned.\nIt's possible to pick a set of specific dates and a set of date ranges.", + "type": "object", + "properties": { + "dates": { + "description": "List of dates that match the media items' creation date. A maximum of\n5 dates can be included per request.", + "type": "array", + "items": { + "$ref": "Date", + "used_by": [], + "parents": ["DateFilter"], + }, + }, + "ranges": { + "description": "List of dates ranges that match the media items' creation date. A\nmaximum of 5 dates ranges can be included per request.", + "type": "array", + "items": { + "$ref": "DateRange", + "used_by": [], + "parents": ["DateFilter"], + }, + }, + }, + "id": "DateFilter", + "used_by": ["Filters"], + "parents": [], + }, + "NewMediaItemResult": { + "description": "Result of creating a new media item.", + "type": "object", + "properties": { + "uploadToken": { + "description": "The upload token used to create this new media item.", + "type": "string", + }, + "status": { + "$ref": "Status", + "description": 'If an error occurred during the creation of this media item, this field\nis populated with information related to the error. For details regarding\nthis field, see Status.', + }, + "mediaItem": { + "$ref": "MediaItem", + "description": "Media item created with the upload token. It's populated if no errors\noccurred and the media item was created successfully.", + }, + }, + "id": "NewMediaItemResult", + "used_by": ["BatchCreateMediaItemsResponse"], + "parents": [], + }, + "ListAlbumsResponse": { + "description": "List of albums requested.", + "type": "object", + "properties": { + "nextPageToken": { + "description": "[Output only] Token to use to get the next set of albums. Populated if\nthere are more albums to retrieve for this request.", + "type": "string", + }, + "albums": { + "description": "[Output only] List of albums shown in the Albums tab of the user's Google\nPhotos app.", + "type": "array", + "items": { + "$ref": "Album", + "used_by": [], + "parents": ["ListAlbumsResponse"], + }, + }, + }, + "id": "ListAlbumsResponse", + "used_by": [], + "parents": [], + }, + "ListSharedAlbumsResponse": { + "description": "List of shared albums requested.", + "type": "object", + "properties": { + "nextPageToken": { + "description": "[Output only] Token to use to get the next set of shared albums. Populated\nif there are more shared albums to retrieve for this request.", + "type": "string", + }, + "sharedAlbums": { + "description": "[Output only] List of shared albums.", + "type": "array", + "items": { + "$ref": "Album", + "used_by": [], + "parents": ["ListSharedAlbumsResponse"], + }, + }, + }, + "id": "ListSharedAlbumsResponse", + "used_by": [], + "parents": [], + }, + "CreateAlbumRequest": { + "description": "Request to create an album in Google Photos.", + "type": "object", + "properties": { + "album": { + "description": "The album to be created.", + "$ref": "Album", + } + }, + "id": "CreateAlbumRequest", + "used_by": [], + "parents": [], + }, + "SimpleMediaItem": { + "description": "A simple media item to be created in Google Photos via an upload token.", + "type": "object", + "properties": { + "uploadToken": { + "description": "Token identifying the media bytes that have been uploaded to Google.", + "type": "string", + } + }, + "id": "SimpleMediaItem", + "used_by": ["NewMediaItem"], + "parents": [], + }, + "AddEnrichmentToAlbumResponse": { + "description": "The enrichment item that's created.", + "type": "object", + "properties": { + "enrichmentItem": { + "description": "[Output only] Enrichment which was added.", + "$ref": "EnrichmentItem", + } + }, + "id": "AddEnrichmentToAlbumResponse", + "used_by": [], + "parents": [], + }, + "ContributorInfo": { + "description": "Information about the user who added the media item. Note that this\ninformation is included only if the media item is within a shared album\ncreated by your app and you have the sharing scope.", + "type": "object", + "properties": { + "profilePictureBaseUrl": { + "description": "URL to the profile picture of the contributor.", + "type": "string", + }, + "displayName": { + "description": "Display name of the contributor.", + "type": "string", + }, + }, + "id": "ContributorInfo", + "used_by": ["MediaItem"], + "parents": [], + }, + "JoinSharedAlbumResponse": { + "description": "Response to successfully joining the shared album on behalf of the user.", + "type": "object", + "properties": { + "album": { + "$ref": "Album", + "description": "Shared album that the user has joined.", + } + }, + "id": "JoinSharedAlbumResponse", + "used_by": [], + "parents": [], + }, + "ShareAlbumRequest": { + "description": "Request to make an album shared in Google Photos.", + "type": "object", + "properties": { + "sharedAlbumOptions": { + "description": "Options to be set when converting the album to a shared album.", + "$ref": "SharedAlbumOptions", + } + }, + "id": "ShareAlbumRequest", + "used_by": [], + "parents": [], + }, + "Photo": { + "description": "Metadata that is specific to a photo, such as, ISO, focal length and\nexposure time. Some of these fields may be null or not included.", + "type": "object", + "properties": { + "cameraModel": { + "description": "Model of the camera with which the photo was taken.", + "type": "string", + }, + "cameraMake": { + "description": "Brand of the camera with which the photo was taken.", + "type": "string", + }, + "focalLength": { + "description": "Focal length of the camera lens with which the photo was taken.", + "format": "float", + "type": "number", + }, + "isoEquivalent": { + "description": "ISO of the camera with which the photo was taken.", + "format": "int32", + "type": "integer", + }, + "apertureFNumber": { + "description": "Aperture f number of the camera lens with which the photo was taken.", + "format": "float", + "type": "number", + }, + "exposureTime": { + "description": "Exposure time of the camera aperture when the photo was taken.", + "format": "google-duration", + "type": "string", + }, + }, + "id": "Photo", + "used_by": ["MediaMetadata"], + "parents": [], + }, + } + + schemas = self.discovery_doc["schemas"] + resources = self.discovery_doc["resources"] + + actual = new_context(schemas, resources).schemas + self.assertEqual(actual, expected) diff --git a/src/generator/lib/__tests__/test_data/__init__.py b/src/generator/lib/__tests__/test_data/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/generator/lib/test_data/photoslibrary-api.json b/src/generator/lib/__tests__/test_data/discovery_document.py similarity index 99% rename from src/generator/lib/test_data/photoslibrary-api.json rename to src/generator/lib/__tests__/test_data/discovery_document.py index 7d74bb7917..2e12bc7394 100644 --- a/src/generator/lib/test_data/photoslibrary-api.json +++ b/src/generator/lib/__tests__/test_data/discovery_document.py @@ -1,3 +1,6 @@ +# Discovery document for photoslibrary, retrieved from: +# https://photoslibrary.googleapis.com/$discovery/rest?version=v1 +DISCOVERY_DOC = r""" { "documentationLink": "https://developers.google.com/photos/", "id": "photoslibrary:v1", @@ -1332,3 +1335,4 @@ "kind": "discovery#restDescription", "basePath": "" } +""" \ No newline at end of file diff --git a/src/generator/lib/util_test.py b/src/generator/lib/__tests__/util_test.py similarity index 92% rename from src/generator/lib/util_test.py rename to src/generator/lib/__tests__/util_test.py index f824c8141d..9e1393aefe 100644 --- a/src/generator/lib/util_test.py +++ b/src/generator/lib/__tests__/util_test.py @@ -1,20 +1,12 @@ #!/usr/bin/env python -import importlib.resources import unittest import json -from generator.lib.util import to_api_version, library_name, re_find_replacements, to_rust_type, new_context -import generator.lib.test_data as test_data +from generator.lib.util import to_api_version, library_name, re_find_replacements, to_rust_type +from .test_data.discovery_document import DISCOVERY_DOC -TEST_JSON_FILE = "photoslibrary-api.json" - - -def read_test_json_file(): - data = importlib.resources.read_text(test_data, TEST_JSON_FILE) - return json.loads(data) - class UtilsTest(unittest.TestCase): def test_to_version_ok(self): @@ -68,7 +60,7 @@ class UtilsTest(unittest.TestCase): self.assertEqual(ms[0], '{+project}') def test_to_rust_type(self): - full_api_schema = read_test_json_file() + full_api_schema = json.loads(DISCOVERY_DOC) schemas = full_api_schema['schemas'] From 3483ab0fb6630f9aa9cd8d6d1fe7a6fee2a0896b Mon Sep 17 00:00:00 2001 From: Kyle Gentle Date: Thu, 18 Aug 2022 21:10:07 -0400 Subject: [PATCH 5/5] Remove dynamic imports in mako-render It's cheap to `import json`, and with recent versions of python the `simplejson` lib isn't required. --- etc/bin/mako-render | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/etc/bin/mako-render b/etc/bin/mako-render index 20ed422307..f10190eba3 100644 --- a/etc/bin/mako-render +++ b/etc/bin/mako-render @@ -4,10 +4,13 @@ # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php from argparse import ArgumentParser +from copy import deepcopy from importlib import import_module from os.path import isfile, dirname +import json import os import sys +import yaml from mako.template import Template from mako.lookup import TemplateLookup from mako import exceptions @@ -213,27 +216,17 @@ def load_data(datafiles): :Returns: data (dict) :Raises: ImportError, ValueError """ - imported_json = False imported_yaml = False mydata = {} for filename, namespace in datafiles: data = None if filename[-5:].lower() == ".json": - if not imported_json: - try: - import simplejson as json - except ImportError: - import json - imported_json = True try: data = json.load(open(filename, 'r')) except ValueError as err: raise ValueError("Invalid JSON in file '%s'. (%s)" % (filename, str(err))) elif filename[-5:].lower() in (".yaml", ".yml"): - if not imported_yaml: - import yaml - imported_yaml = True data = yaml.load_all(open(filename, 'r'), Loader=yaml.loader.FullLoader) data = list(data)[0] else: