Restructure src dir

Make a few changes in the toplevel src dir to help separate
templates from code. Specifically, we rename `src/mako` to
`src/generator`, and nest the mako templates inside a
`src/generator/templates` dir.

This isolates most Python code into the `src/generator/lib` dir.

.gitignore

@@ -5,7 +5,7 @@
 gen/doc
 *.go    
 **/target/
-**/docs/
+gen/**/docs/
 **/build_html/
 .*.deps
 **/Cargo.lock

Makefile

@@ -18,7 +18,9 @@ TPL := $(PYTHON) $(MAKO_RENDER)
 MKDOCS := $(shell pwd)/$(VENV_DIR)/bin/mkdocs
 GHP_IMPORT  := $(shell pwd)/$(VENV_DIR)/bin/ghp-import
 
-MAKO_SRC = src/mako
+GEN_SRC = src/generator
+GEN_LIB_SRC = $(GEN_SRC)/lib
+MAKO_SRC = src/generator/templates
 RUST_SRC = src/rust
 PREPROC_DIR = $(RUST_SRC)/preproc
 PREPROC = target/release/preproc
@@ -36,9 +38,8 @@ else
 API_LIST := $(API_LIST)api-list.yaml
 endif
 API_JSON_FILES = $(shell find etc -type f -name '*-api.json')
-MAKO_LIB_DIR = $(MAKO_SRC)/lib
-MAKO_LIB_FILES = $(shell find $(MAKO_LIB_DIR) -type f -name '*.*')
-MAKO = export PREPROC=$(PREPROC); export PYTHONPATH=$(MAKO_SRC):$(PYTHONPATH); $(TPL) --template-dir '.'
+MAKO_LIB_FILES = $(shell find $(GEN_LIB_SRC) -type f -name '*.*')
+MAKO = export PREPROC=$(PREPROC); export PYTHONPATH=src:$(PYTHONPATH); $(TPL) --template-dir '.'
 MAKO_STANDARD_DEPENDENCIES = $(API_SHARED_INFO) $(MAKO_LIB_FILES) $(MAKO_RENDER) $(PREPROC)
 
 help:
@@ -71,7 +72,7 @@ $(PYTHON_BIN): $(VENV_BIN) requirements.txt
 	python3 -m virtualenv -p python3 $(VENV_DIR)
 	$@ -m pip install -r requirements.txt
 
-$(MAKO_RENDER): $(PYTHON_BIN) $(wildcard $(MAKO_LIB_DIR)/*)
+$(MAKO_RENDER): $(PYTHON_BIN) $(wildcard $(GEN_LIB_SRC)/*)
 
 # Explicitly NOT depending on $(MAKO_LIB_FILES), as it's quite stable and now takes 'too long' thanks
 # to a URL get call to the google discovery service
@@ -99,7 +100,7 @@ test-gen: $(PYTHON_BIN)
 test: test-gen
 
 typecheck: $(PYTHON_BIN)
-	$(PYTHON) -m pyright $(MAKO_LIB_DIR)
+	$(PYTHON) -m pyright $(GEN_LIB_SRC)
 
 clean: clean-all-api clean-all-cli docs-all-clean
 	-rm -Rf $(VENV_DIR)

etc/api/shared.yaml

@@ -83,7 +83,7 @@ directories:
   # where are all the API meta files
   api_base: etc/api
   # all mako source files
-  mako_src: src/mako
+  mako_src: src/generator/templates
   # The subdirectory to contain documentation from all APIs and related programs
   doc_subdir: doc
 cargo:

etc/api/type-cli.yaml

@@ -5,7 +5,7 @@ mkdocs:
   # if docs_dir changes, remember to update the sources as well.
   docs_dir: docs
 mako:
-  post_processor_module: "lib.cli"
+  post_processor_module: "generator.lib.cli"
 make:
   id: cli
   target_name: CLIs

src/generator/lib/cli.py

@@ -1,4 +1,4 @@
-import lib.util as util
+import generator.lib.util as util
 
 import os
 import re

src/generator/lib/util.mako

@@ -1,4 +1,4 @@
-<%! import lib.util as util %>\
+<%! import generator.lib.util as util %>\
 
 ## source should be ${self.uri}
 ## you need to escape the output, using a filter for example

src/generator/lib/util.py

@@ -341,11 +341,16 @@ def _assure_unique_type_name(schemas, tn):
     return tn
 
 # map a json type to an rust type
-# sn = schema name
-# pn = property name
 # t = type dict
 # NOTE: In case you don't understand how this algorithm really works ... me neither - THE AUTHOR
-def to_rust_type(schemas, sn, pn, t, allow_optionals=True, _is_recursive=False):
+def to_rust_type(
+    schemas,
+    schema_name, 
+    property_name, 
+    t, 
+    allow_optionals=True, 
+    _is_recursive=False
+):
     def nested_type(nt):
         if 'items' in nt:
             nt = nt['items']
@@ -354,8 +359,8 @@ def to_rust_type(schemas, sn, pn, t, allow_optionals=True, _is_recursive=False):
         else:
             assert(is_nested_type_property(nt))
             # It's a nested type - we take it literally like $ref, but generate a name for the type ourselves
-            return _assure_unique_type_name(schemas, nested_type_name(sn, pn))
-        return to_rust_type(schemas, sn, pn, nt, allow_optionals=False, _is_recursive=True)
+            return _assure_unique_type_name(schemas, nested_type_name(schema_name, property_name))
+        return to_rust_type(schemas, schema_name, property_name, nt, allow_optionals=False, _is_recursive=True)
 
     def wrap_type(tn):
         if allow_optionals:
@@ -368,7 +373,7 @@ def to_rust_type(schemas, sn, pn, t, allow_optionals=True, _is_recursive=False):
         # which is fine for now. 'allow_optionals' implicitly restricts type boxing for simple types - it
         # usually is on on the first call, and off when recursion is involved.
         tn = t[TREF]
-        if not _is_recursive and tn == sn:
+        if not _is_recursive and tn == schema_name:
             tn = 'Option<Box<%s>>' % tn
         return wrap_type(tn)
     try:

src/generator/lib/util_test.py

@@ -4,8 +4,8 @@ import importlib.resources
 import unittest
 import json
 
-from lib.util import to_api_version, library_name, re_find_replacements, to_rust_type, new_context
-import lib.test_data as test_data
+from generator.lib.util import to_api_version, library_name, re_find_replacements, to_rust_type, new_context
+import generator.lib.test_data as test_data
 
 
 TEST_JSON_FILE = "photoslibrary-api.json"

src/generator/templates/Cargo.toml.mako

@@ -1,5 +1,5 @@
-<%! from lib.util import (estr, enclose_in, hash_comment, library_to_crate_name, to_extern_crate_name) %>\
-<%namespace name="util" file="lib/util.mako"/>\
+<%! from generator.lib.util import (estr, enclose_in, hash_comment, library_to_crate_name, to_extern_crate_name) %>\
+<%namespace name="util" file="../lib/util.mako"/>\
 <%block filter="hash_comment">\
 <%util:gen_info source="${self.uri}" />\
 </%block>

src/generator/templates/LICENSE.md.mako

@@ -1,6 +1,6 @@
 ## -*- coding: utf-8 -*-
-<%! import lib.util as util %>\
-<%namespace name="mutil" file="lib/util.mako"/>\
+<%! import generator.lib.util as util %>\
+<%namespace name="mutil" file="../lib/util.mako"/>\
 <%block filter="util.markdown_comment">\
 <%mutil:gen_info source="${self.uri}" />\
 </%block>

src/generator/templates/api/README.md.mako

@@ -1,9 +1,9 @@
 <%
-	from lib.util import (markdown_comment, new_context)
+	from generator.lib.util import (markdown_comment, new_context)
 	c = new_context(schemas, resources, context.get('methods'))
 %>\
 <%namespace name="lib" file="lib/lib.mako"/>\
-<%namespace name="util" file="../lib/util.mako"/>\
+<%namespace name="util" file="../../lib/util.mako"/>\
 <%block filter="markdown_comment">\
 <%util:gen_info source="${self.uri}" />\
 </%block>

src/generator/templates/api/api.rs.mako

@@ -1,10 +1,10 @@
 <%namespace name="lib" file="lib/lib.mako"/>\
-<%namespace name="util" file="../lib/util.mako"/>\
+<%namespace name="util" file="../../lib/util.mako"/>\
 <%namespace name="rbuild" file="lib/rbuild.mako"/>\
 <%namespace name="mbuild" file="lib/mbuild.mako"/>\
 <%namespace name="schema" file="lib/schema.mako"/>\
 <%
-    from lib.util import (new_context, rust_comment, rust_doc_comment, rust_module_doc_comment,
+    from generator.lib.util import (new_context, rust_comment, rust_doc_comment, rust_module_doc_comment,
                       rb_type, hub_type, mangle_ident, hub_type_params_s, 
                       rb_type_params_s, find_fattest_resource, HUB_TYPE_PARAMETERS, METHODS_RESOURCE,
                       UNUSED_TYPE_MARKER, schema_markers)

src/generator/templates/api/lib.rs.mako

@@ -1,7 +1,7 @@
 <%namespace name="lib" file="lib/lib.mako"/>\
-<%namespace name="util" file="../lib/util.mako"/>\
+<%namespace name="util" file="../../lib/util.mako"/>\
 <%
-    from lib.util import (new_context, rust_comment, rust_module_doc_comment)
+    from generator.lib.util import (new_context, rust_comment, rust_module_doc_comment)
 
     c = new_context(schemas, resources, context.get('methods'))
 %>\
@@ -20,12 +20,12 @@ ${lib.docs(c)}
 #![allow(unused_imports, unused_mut, dead_code)]
 
 <%namespace name="lib" file="lib/lib.mako"/>\
-<%namespace name="util" file="../lib/util.mako"/>\
+<%namespace name="util" file="../../lib/util.mako"/>\
 <%namespace name="rbuild" file="lib/rbuild.mako"/>\
 <%namespace name="mbuild" file="lib/mbuild.mako"/>\
 <%namespace name="schema" file="lib/schema.mako"/>\
 <%
-    from lib.util import (new_context, rust_comment, rust_doc_comment, rust_module_doc_comment,
+    from generator.lib.util import (new_context, rust_comment, rust_doc_comment, rust_module_doc_comment,
                       rb_type, hub_type, mangle_ident, hub_type_params_s,
                       rb_type_params_s, find_fattest_resource, HUB_TYPE_PARAMETERS, METHODS_RESOURCE,
                       UNUSED_TYPE_MARKER, schema_markers)

src/generator/templates/api/lib/lib.mako

@@ -1,5 +1,5 @@
 <%!
-    from lib.util import (activity_split, put_and, md_italic, split_camelcase_s, canonical_type_name, hub_type,
+    from generator.lib.util import (activity_split, put_and, md_italic, split_camelcase_s, canonical_type_name, hub_type,
                       rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment, markdown_rust_block,
                       unindent_first_by, mangle_ident, mb_type, singular, scope_url_to_variant,
                       PART_MARKER_TRAIT, RESOURCE_MARKER_TRAIT, CALL_BUILDER_MARKERT_TRAIT, 
@@ -12,7 +12,7 @@
     def pretty_name(name):
         return ' '.join(split_camelcase_s(name).split('.'))
 %>\
-<%namespace name="util" file="../../lib/util.mako"/>\
+<%namespace name="util" file="../../../lib/util.mako"/>\
 <%namespace name="mbuild" file="mbuild.mako"/>\
 
 ## If rust-doc is True, examples will be made to work for rust doc tests. Otherwise they are set 

src/generator/templates/api/lib/mbuild.mako

@@ -1,5 +1,5 @@
 <%!
-    from lib.util import (put_and, rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment,
+    from generator.lib.util import (put_and, rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment,
                       rb_type, mb_type, singular, hub_type, to_fqan, indent_all_but_first_by,
                       activity_rust_type, mangle_ident, activity_input_type, get_word,
                       split_camelcase_s, property, is_pod_property, TREF, IO_REQUEST,
@@ -28,7 +28,7 @@
         part_desc = part_desc[:-1]
         return part_desc
 %>\
-<%namespace name="util" file="../../lib/util.mako"/>\
+<%namespace name="util" file="../../../lib/util.mako"/>\
 <%namespace name="lib" file="lib.mako"/>\
 
 ## Creates a method builder type

src/generator/templates/api/lib/rbuild.mako

@@ -1,5 +1,5 @@
 <%!
-    from lib.util import (put_and, rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment,
+    from generator.lib.util import (put_and, rust_test_fn_invisible, rust_doc_test_norun, rust_doc_comment,
                       rb_type, singular, hub_type, mangle_ident, mb_type, property,
                       to_fqan, indent_all_but_first_by, is_repeated_property, is_required_property,
                       activity_input_type, TREF, IO_REQUEST, schema_to_required_property, 
@@ -8,7 +8,7 @@
                       struct_type_bounds_s, METHODS_RESOURCE, SPACES_PER_TAB, prefix_all_but_first_with,
                       METHODS_BUILDER_MARKER_TRAIT, remove_empty_lines, method_default_scope, rust_doc_sanitize)
 %>\
-<%namespace name="util" file="../../lib/util.mako"/>\
+<%namespace name="util" file="../../../lib/util.mako"/>\
 <%namespace name="lib" file="lib.mako"/>\
 
 ## Creates a Resource builder type

src/generator/templates/api/lib/schema.mako

@@ -1,5 +1,5 @@
 <%!
-    from lib.util import (schema_markers, rust_doc_comment, mangle_ident, to_rust_type, put_and,
+    from generator.lib.util import (schema_markers, rust_doc_comment, mangle_ident, to_rust_type, put_and,
                       IO_TYPES, activity_split, enclose_in, REQUEST_MARKER_TRAIT, mb_type, indent_all_but_first_by,
                       NESTED_TYPE_SUFFIX, RESPONSE_MARKER_TRAIT, split_camelcase_s, METHODS_RESOURCE,
                       PART_MARKER_TRAIT, canonical_type_name, TO_PARTS_MARKER, UNUSED_TYPE_MARKER, is_schema_with_optionals,

src/generator/templates/cli/README.md.mako

@@ -1,10 +1,10 @@
 <%
-    from lib.util import (markdown_comment, new_context)
-    from lib.cli import (CONFIG_DIR, CONFIG_DIR_FLAG, SCOPE_FLAG, application_secret_path, DEBUG_FLAG)
+    from generator.lib.util import (markdown_comment, new_context)
+    from generator.lib.cli import (CONFIG_DIR, CONFIG_DIR_FLAG, SCOPE_FLAG, application_secret_path, DEBUG_FLAG)
 
     c = new_context(schemas, resources, context.get('methods'))
 %>\
-<%namespace name="util" file="../lib/util.mako"/>\
+<%namespace name="util" file="../../lib/util.mako"/>\
 <%namespace name="argparse" file="lib/argparse.mako"/>\
 <%block filter="markdown_comment">\
 <%util:gen_info source="${self.uri}" />\

src/generator/templates/cli/docs/commands.md.mako

@@ -1,8 +1,8 @@
-<%namespace name="util" file="../../lib/util.mako"/>\
+<%namespace name="util" file="../../../lib/util.mako"/>\
 <%!
     from mako.filters import xml_escape
-    from lib.util import (hash_comment, new_context, method_default_scope, indent_all_but_first_by, is_repeated_property, custom_sorted)
-    from lib.cli import (subcommand_md_filename, new_method_context, SPLIT_START, SPLIT_END, pretty, SCOPE_FLAG,
+    from generator.lib.util import (hash_comment, new_context, method_default_scope, indent_all_but_first_by, is_repeated_property, custom_sorted)
+    from generator.lib.cli import (subcommand_md_filename, new_method_context, SPLIT_START, SPLIT_END, pretty, SCOPE_FLAG,
                      mangle_subcommand, is_request_value_property, FIELD_SEP, PARAM_FLAG, UPLOAD_FLAG, docopt_mode,
                      FILE_ARG, MIME_ARG, OUT_ARG, OUTPUT_FLAG, to_cli_schema, cli_schema_to_yaml, SchemaEntry,
                      STRUCT_FLAG, field_to_value, CTYPE_ARRAY, CTYPE_MAP, to_docopt_arg, FILE_FLAG, MIME_FLAG, 

src/generator/templates/cli/lib/argparse.mako

@@ -1,9 +1,9 @@
-<%namespace name="util" file="../../lib/util.mako"/>\
+<%namespace name="util" file="../../../lib/util.mako"/>\
 <%!
     import os
 
-    from lib.util import (put_and, supports_scopes, api_index, indent_by, enclose_in, put_and, escape_rust_string)
-    from lib.cli import (mangle_subcommand, new_method_context, PARAM_FLAG, STRUCT_FLAG, UPLOAD_FLAG, OUTPUT_FLAG, VALUE_ARG,
+    from generator.lib.util import (put_and, supports_scopes, api_index, indent_by, enclose_in, put_and, escape_rust_string)
+    from generator.lib.cli import (mangle_subcommand, new_method_context, PARAM_FLAG, STRUCT_FLAG, UPLOAD_FLAG, OUTPUT_FLAG, VALUE_ARG,
                      CONFIG_DIR, SCOPE_FLAG, is_request_value_property, FIELD_SEP, docopt_mode, FILE_ARG, MIME_ARG, OUT_ARG,
                      CONFIG_DIR_FLAG, KEY_VALUE_ARG, to_docopt_arg, DEBUG_FLAG, MODE_ARG, SCOPE_ARG,
                      CONFIG_DIR_ARG, FILE_FLAG, MIME_FLAG, subcommand_md_filename)

src/generator/templates/cli/lib/engine.mako

@@ -1,9 +1,9 @@
-<%namespace name="util" file="../../lib/util.mako"/>\
+<%namespace name="util" file="../../../lib/util.mako"/>\
 <%!
-    from lib.util import (hub_type, mangle_ident, indent_all_but_first_by, activity_rust_type, setter_fn_name, ADD_PARAM_FN,
+    from generator.lib.util import (hub_type, mangle_ident, indent_all_but_first_by, activity_rust_type, setter_fn_name, ADD_PARAM_FN,
                       upload_action_fn, is_schema_with_optionals, schema_markers, indent_by, method_default_scope,
                       ADD_SCOPE_FN, TREF, enclose_in)
-    from lib.cli import (mangle_subcommand, new_method_context, PARAM_FLAG, STRUCT_FLAG, OUTPUT_FLAG, VALUE_ARG,
+    from generator.lib.cli import (mangle_subcommand, new_method_context, PARAM_FLAG, STRUCT_FLAG, OUTPUT_FLAG, VALUE_ARG,
                      CONFIG_DIR, SCOPE_FLAG, is_request_value_property, FIELD_SEP, docopt_mode, FILE_ARG, MIME_ARG, OUT_ARG,
                      call_method_ident, POD_TYPES, opt_value, ident, JSON_TYPE_VALUE_MAP,
                      KEY_VALUE_ARG, to_cli_schema, SchemaEntry, CTYPE_POD, actual_json_type, CTYPE_MAP, CTYPE_ARRAY,

src/generator/templates/cli/main.rs.mako

@@ -1,10 +1,10 @@
 <%namespace name="argparse" file="lib/argparse.mako"/>\
 <%namespace name="engine" file="lib/engine.mako"/>\
-<%namespace name="util" file="../lib/util.mako"/>\
+<%namespace name="util" file="../../lib/util.mako"/>\
 <%  
-    from lib.util import (new_context, rust_comment, to_extern_crate_name, library_to_crate_name, library_name,
+    from generator.lib.util import (new_context, rust_comment, to_extern_crate_name, library_to_crate_name, library_name,
                       indent_all_but_first_by)
-    from lib.cli import OUT_ARG, DEBUG_FLAG, opt_value
+    from generator.lib.cli import OUT_ARG, DEBUG_FLAG, opt_value
 
     c = new_context(schemas, resources, context.get('methods'))
     default_user_agent = "google-cli-rust-client/" + cargo.build_version

src/generator/templates/cli/mkdocs.yml.mako

@@ -1,10 +1,10 @@
 <%
-    from lib.util import (put_and, new_context)
-    from lib.cli import (subcommand_md_filename, mangle_subcommand, pretty)
+    from generator.lib.util import (put_and, new_context)
+    from generator.lib.cli import (subcommand_md_filename, mangle_subcommand, pretty)
 
     c = new_context(schemas, resources, context.get('methods'))
 %>\
-<%namespace name="util" file="../lib/util.mako"/>\
+<%namespace name="util" file="../../lib/util.mako"/>\
 site_name: ${util.canonical_name()} v${util.crate_version()}
 site_url: ${cargo.doc_base_url}/${util.crate_name()}
 site_description: A complete library to interact with ${util.canonical_name()} (protocol ${version})

src/generator/templates/deps.mako

@@ -58,7 +58,7 @@
 <% continue %>\
 % endif
 <%
-	import lib.util as util
+	import generator.lib.util as util
 	import os
 	import json
 
@@ -191,7 +191,7 @@ help${agsuffix}:
 
 % for info in (apis.get('items') or []):
 <%
-	import lib.util as util
+	import generator.lib.util as util
 	import os
 	name = util.normalize_library_name(info['name'])
 	target = util.api_json_path(directories.api_base, name, info['version'])

src/generator/templates/index.html.mako

@@ -2,7 +2,7 @@
     import json
     import os
     import yaml
-    from lib.util import (api_json_path, library_name, library_to_crate_name,
+    from generator.lib.util import (api_json_path, library_name, library_to_crate_name,
                       gen_crate_dir, api_index, crates_io_url, program_name,
                       crate_version)