diff --git a/.gitignore b/.gitignore index 8d7c70a3da..3d887dbf61 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ gen/doc # Python: Makefile lead dirs .pyenv-* -.virtualenv +.virtualenv.marker # Sublime *.sublime-workspace diff --git a/Makefile b/Makefile index c14c8ca3a8..063e4fcac9 100644 --- a/Makefile +++ b/Makefile @@ -2,13 +2,13 @@ .SUFFIXES: VIRTUALENV_VERSION = 16.0.0 -VENV_BIN = .virtualenv/virtualenv.py +VENV_BIN = .virtualenv.marker +VENV_VERSION = 20.2.2 VENV_DIR := .pyenv-$(shell uname) PYTHON_BIN := $(VENV_DIR)/bin/python PYTHON := . $(VENV_DIR)/bin/activate; python -PIP := $(PYTHON) -m pip PYTEST := $(PYTHON) -m pytest MAKO_RENDER := etc/bin/mako-render @@ -63,13 +63,12 @@ $(PREPROC): $(PREPROC_DIR)/src/main.rs cd "$(PREPROC_DIR)" && cargo build --release $(VENV_BIN): - wget -nv https://pypi.python.org/packages/source/v/virtualenv/virtualenv-$(VIRTUALENV_VERSION).tar.gz -O virtualenv-$(VIRTUALENV_VERSION).tar.gz - tar -xzf virtualenv-$(VIRTUALENV_VERSION).tar.gz && mv virtualenv-$(VIRTUALENV_VERSION) ./.virtualenv && rm -f virtualenv-$(VIRTUALENV_VERSION).tar.gz - chmod +x $@ + python3 -m pip install --user virtualenv==$(VENV_VERSION) + touch $@ $(PYTHON_BIN): $(VENV_BIN) requirements.txt - $(VENV_BIN) -p python2.7 $(VENV_DIR) - $(PIP) install -r requirements.txt + python3 -m virtualenv -p python3 $(VENV_DIR) + $@ -m pip install -r requirements.txt $(MAKO_RENDER): $(PYTHON_BIN) $(wildcard $(MAKO_LIB_DIR)/*) diff --git a/etc/bin/mako-render b/etc/bin/mako-render index 89c1a2fb56..69d6ceddbc 100644 --- a/etc/bin/mako-render +++ b/etc/bin/mako-render @@ -70,9 +70,6 @@ class DictObject(object): def __repr__(self): return repr(self.__dict__) - def __getattr__(self, name): - return object.__getattribute__(self, name) - def __getitem__(self, name): try: return getattr(self, name) @@ -191,9 +188,9 @@ class DictObject(object): """as dict.items""" return list(self.__dict__.items()) - def iteritems(self): - """as dict.iteritems""" - return iter(self.__dict__.items()) + def _items(self): + """as dict.items, avoiding name clashes""" + return list(self.__dict__.items()) def pop(self, key, default=sys): """as dict.pop""" diff --git a/requirements.txt b/requirements.txt index d56e7627fb..6755433c5f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -mako +mako==1.1.3 pyyaml mkdocs==0.16.3 pytest diff --git a/src/mako/api/lib.rs.mako b/src/mako/api/lib.rs.mako index 734433a757..c68bf3f5dc 100644 --- a/src/mako/api/lib.rs.mako +++ b/src/mako/api/lib.rs.mako @@ -168,7 +168,7 @@ ${rbuild.new(resource, c)} // CallBuilders ### // ################# -% for resource, methods in c.rta_map.iteritems(): +% for resource, methods in c.rta_map.items(): % for method in methods: ${mbuild.new(resource, method, c)} diff --git a/src/mako/api/lib/lib.mako b/src/mako/api/lib/lib.mako index 94bc3d51dd..f3cca20bfe 100644 --- a/src/mako/api/lib/lib.mako +++ b/src/mako/api/lib/lib.mako @@ -156,7 +156,7 @@ let r = hub.resource().activity(...).${api.terms.action}() Or specifically ... ```ignore -% for an, a in c.sta_map[fr.id].iteritems(): +% for an, a in c.sta_map[fr.id].items(): <% category, resource, activity = activity_split(an) %>\ let r = hub.${mangle_ident(resource)}().${mangle_ident(activity)}(...).${api.terms.action}() % endfor diff --git a/src/mako/api/lib/mbuild.mako b/src/mako/api/lib/mbuild.mako index 07d41532e5..79916b5da8 100644 --- a/src/mako/api/lib/mbuild.mako +++ b/src/mako/api/lib/mbuild.mako @@ -11,7 +11,7 @@ DELEGATE_PROPERTY_NAME, struct_type_bounds_s, scope_url_to_variant, re_find_replacements, ADD_PARAM_FN, ADD_PARAM_MEDIA_EXAMPLE, upload_action_fn, METHODS_RESOURCE, method_name_to_variant, unique_type_name, size_to_bytes, method_default_scope, - is_repeated_property, setter_fn_name, ADD_SCOPE_FN, rust_doc_sanitize) + is_repeated_property, setter_fn_name, ADD_SCOPE_FN, rust_doc_sanitize, items) def get_parts(part_prop): if not part_prop: @@ -154,7 +154,7 @@ ${self._setter_fn(resource, method, m, p, part_prop, ThisType, c)}\ /// /// # Additional Parameters /// - % for opn, op in list((opn, op) for (opn, op) in parameters.iteritems() if opn not in [p.name for p in params]): + % for opn, op in list((opn, op) for (opn, op) in parameters.items() if opn not in [p.name for p in params]): /// * *${opn}* (${op.location}-${op.type}) - ${op.description} % endfor % endif @@ -336,7 +336,7 @@ ${capture(lib.test_hub, hub_type_name, comments=show_all) | hide_filter} // into the respective structure. Some of the parts shown here might not be applicable ! // ${random_value_warning} let mut ${rb_name} = ${request_value_type}::default(); -% for spn, sp in request_value.get('properties', dict()).iteritems(): +% for spn, sp in items(request_value.get('properties', dict())): % if parts is not None and spn not in parts: <% continue %> % endif @@ -896,7 +896,7 @@ if enable_resource_parsing \ % for p in media_params: ${p.description | rust_doc_sanitize, rust_doc_comment, indent_all_but_first_by(1)} /// - % for item_name, item in p.info.iteritems(): + % for item_name, item in p.info.items(): /// * *${split_camelcase_s(item_name)}*: ${isinstance(item, (list, tuple)) and put_and(enclose_in("'", item)) or str(item)} % endfor pub fn ${upload_action_fn(api.terms.upload_action, p.type.suffix)}<${mtype_param}>(self, ${p.type.arg_name}: ${mtype_param}, mime_type: mime::Mime) -> ${rtype} diff --git a/src/mako/api/lib/rbuild.mako b/src/mako/api/lib/rbuild.mako index f87d3378ae..feff10e5c8 100644 --- a/src/mako/api/lib/rbuild.mako +++ b/src/mako/api/lib/rbuild.mako @@ -110,7 +110,7 @@ impl${rb_params} ${ThisType} { % for p in optional_props: ${property(p.name)}: Default::default(), % endfor -% for prop_key, custom_name in api.properties.iteritems(): +% for prop_key, custom_name in api.properties.items(): % if prop_key == 'scopes' and not method_default_scope(m): <% continue %>\ % endif diff --git a/src/mako/api/lib/schema.mako b/src/mako/api/lib/schema.mako index 0484b5c40b..0f123af431 100644 --- a/src/mako/api/lib/schema.mako +++ b/src/mako/api/lib/schema.mako @@ -3,7 +3,7 @@ IO_TYPES, activity_split, enclose_in, REQUEST_MARKER_TRAIT, mb_type, indent_all_but_first_by, NESTED_TYPE_SUFFIX, RESPONSE_MARKER_TRAIT, split_camelcase_s, METHODS_RESOURCE, unique_type_name, PART_MARKER_TRAIT, canonical_type_name, TO_PARTS_MARKER, UNUSED_TYPE_MARKER, is_schema_with_optionals, - rust_doc_sanitize) + rust_doc_sanitize, items) %>\ ## Build a schema which must be an object ################################################################################################################### @@ -12,7 +12,7 @@ <% struct = 'pub struct ' + unique_type_name(s.id) %>\ % if properties: ${struct} { -% for pn, p in properties.iteritems(): +% for pn, p in items(properties): ${p.get('description', 'no description provided') | rust_doc_sanitize, rust_doc_comment, indent_all_but_first_by(1)} % if pn != mangle_ident(pn): #[serde(rename="${pn}")] @@ -111,7 +111,7 @@ impl ${TO_PARTS_MARKER} for ${s_type} { /// the parts you want to see in the server response. fn to_parts(&self) -> String { let mut r = String::new(); - % for pn, p in s.properties.iteritems(): + % for pn, p in items(s.properties): <% mn = 'self.' + mangle_ident(pn) rt = to_rust_type(schemas, s.id, pn, p, allow_optionals=allow_optionals) @@ -140,7 +140,7 @@ ${s.get('description', 'There is no detailed description.')} This type is used in activities, which are methods you may call on this type or where this type is involved in. The list links the activity name, along with information about where it is used (one of ${put_and(enclose_in('*', IO_TYPES))}). -% for a, iot in c.sta_map[s.id].iteritems(): +% for a, iot in c.sta_map[s.id].items(): <% category, name, method = activity_split(a) name_suffix = ' ' + split_camelcase_s(name) diff --git a/src/mako/cli/docs/commands.md.mako b/src/mako/cli/docs/commands.md.mako index 79f2c79397..4ada130904 100644 --- a/src/mako/cli/docs/commands.md.mako +++ b/src/mako/cli/docs/commands.md.mako @@ -1,7 +1,7 @@ <%namespace name="util" file="../../lib/util.mako"/>\ <%! from mako.filters import xml_escape - from util import (hash_comment, new_context, method_default_scope, indent_all_but_first_by, is_repeated_property) + from util import (hash_comment, new_context, method_default_scope, indent_all_but_first_by, is_repeated_property, custom_sorted) from cli import (subcommand_md_filename, new_method_context, SPLIT_START, SPLIT_END, pretty, SCOPE_FLAG, mangle_subcommand, is_request_value_property, FIELD_SEP, PARAM_FLAG, UPLOAD_FLAG, docopt_mode, FILE_ARG, MIME_ARG, OUT_ARG, OUTPUT_FLAG, to_cli_schema, cli_schema_to_yaml, SchemaEntry, @@ -143,7 +143,7 @@ You may set the following properties to further configure the call. Please note or more key-value-pairs, and is called like this `-${PARAM_FLAG} k1=v1 k2=v2` even though the listing below repeats the `-${PARAM_FLAG}` for completeness. -% for p in sorted(oprops): +% for p in custom_sorted(oprops): ${self._md_property(p)} % endfor % endif # optional method properties diff --git a/src/mako/cli/lib/argparse.mako b/src/mako/cli/lib/argparse.mako index e538bf61bc..7bdacd3db5 100644 --- a/src/mako/cli/lib/argparse.mako +++ b/src/mako/cli/lib/argparse.mako @@ -16,7 +16,7 @@ return 'None' if isinstance(v, bool): v = v and 'true' or 'false' - elif isinstance(v, basestring): + elif isinstance(v, str): v = 'r##"%s"##' % v elif isinstance(v, list): v = 'vec![%s]' % ','.join('UploadProtocol::%s' % p.capitalize() for p in v) diff --git a/src/mako/cli/lib/engine.mako b/src/mako/cli/lib/engine.mako index 6319331f78..193a9b30e1 100644 --- a/src/mako/cli/lib/engine.mako +++ b/src/mako/cli/lib/engine.mako @@ -357,7 +357,7 @@ if dry_run { allow_optionals = allow_optionals_fn(schema) if not allow_optionals: opt_access = '' - for fn, f in schema.fields.iteritems(): + for fn, f in schema.fields.items(): cur.append(['%s%s' % (mangle_ident(fn), opt_access), fn]) fields.add(fn) if isinstance(f, SchemaEntry): diff --git a/src/mako/deps.mako b/src/mako/deps.mako index 9c5bd2f332..d53bc7a73b 100644 --- a/src/mako/deps.mako +++ b/src/mako/deps.mako @@ -4,7 +4,7 @@ <% import os - import urllib2 + import urllib import json apis = {} @@ -23,7 +23,7 @@ if os.environ.get('FETCH_APIS') is not None: discovery_url = 'https://www.googleapis.com/discovery/v1/' - apis = json.loads(urllib2.urlopen(discovery_url + "apis").read()) + apis = json.loads(urllib.request.urlopen(discovery_url + "apis").read()) print('Loaded {} apis from Google'.format(len(apis['items']))) @@ -46,7 +46,7 @@ if mako is not UNDEFINED: post_processor_arg = '--post-process-python-module=%s' % mako.post_processor_module %>\ -% for an, versions in api.list.iteritems(): +% for an, versions in api.list.items(): % if an in api.get('blacklist', list()): <% continue %>\ % endif @@ -98,8 +98,9 @@ api_info.append((api_target, api_clean, api_cargo, api_doc, api_crate_publish_file, gen_root)) space_join = lambda i: ' '.join(a[i] for a in api_info) - except: + except Exception as e: print('Could not open JSON file at {}'.format(api_json)) + print(e) %>\ ${api_common}: $(RUST_SRC)/${make.id}/cmn.rs $(lastword $(MAKEFILE_LIST)) ${gen_root_stamp} @ echo "// COPY OF '$<'" > $@ diff --git a/src/mako/lib/cli.py b/src/mako/lib/cli.py index 3c73494b93..5efc7689e3 100644 --- a/src/mako/lib/cli.py +++ b/src/mako/lib/cli.py @@ -172,7 +172,7 @@ def to_cli_schema(c, schema): properties[e.type_value] = e # end handle enumerations - for pn, p in properties.iteritems(): + for pn, p in util.items(properties): def set_nested_schema(ns): if ns.fields: fd[pn] = ns diff --git a/src/mako/lib/util.py b/src/mako/lib/util.py index 6d03c89732..862e062c54 100644 --- a/src/mako/lib/util.py +++ b/src/mako/lib/util.py @@ -114,6 +114,17 @@ data_unit_multipliers = { HUB_TYPE_PARAMETERS = ('C', 'A') +def items(p): + if isinstance(p, dict): + return p.items() + else: + return p._items() + +def custom_sorted(p): + if not isinstance(p, list): + assert(false, p, "unexpected type") + return sorted(p, key = lambda p: p['name']) + # ============================================================================== ## @name Filters # ------------------------------------------------------------------------------ @@ -440,7 +451,7 @@ def schema_markers(s, c, transitive=True): continue has_activity = True # it should have at least one activity that matches it's type to qualify for the Resource trait - for fqan, iot in activities.iteritems(): + for fqan, iot in activities.items(): _, resource, _ = activity_split(fqan) if resource and activity_name_to_type_name(resource).lower() == sid.lower(): res.add(RESOURCE_MARKER_TRAIT) @@ -504,7 +515,7 @@ def activity_name_to_type_name(an): # yields (category, resource, activity, activity_data) def iter_acitivities(c): - return ((activity_split(an) + [a]) for an, a in c.fqan_map.iteritems()) + return ((activity_split(an) + [a]) for an, a in c.fqan_map.items()) # return a list of parameter structures of all params of the given method dict # apply a prune filter to restrict the set of returned parameters. @@ -512,7 +523,7 @@ def iter_acitivities(c): def _method_params(m, required=None, location=None): res = list() po = m.get('parameterOrder', []) - for pn, p in m.get('parameters', dict()).iteritems(): + for pn, p in m.get('parameters', dict()).items(): if required is not None and p.get('required', False) != required: continue if location is not None and p.get('location', '') != location: @@ -613,7 +624,7 @@ def method_media_params(m): # actually, one of them is required, but we can't encode that ... # runtime will have to check res = list() - for pn, proto in mu.protocols.iteritems(): + for pn, proto in mu.protocols.items(): # the pi (proto-info) dict can be shown to the user pi = {'multipart': proto.multipart and 'yes' or 'no', 'maxSize': mu.get('maxSize', '0kb'), 'validMimeTypes': mu.accept} try: @@ -672,12 +683,12 @@ def new_context(schemas, resources, methods): res = dict() if fqan is None: fqan = dict() - for k,a in activities.iteritems(): + for k,a in activities.items(): if 'resources' in a: build_activity_mappings(a.resources, res, fqan) if 'methods' not in a: continue - for mn, m in a.methods.iteritems(): + for mn, m in a.methods.items(): assert m.id not in fqan category, resource, method = activity_split(m.id) # This may be another name by which people try to find the method. @@ -746,7 +757,8 @@ def new_context(schemas, resources, methods): # end this is already a perfectly valid type properties = s.get('properties', {'': s}) - for pn, p in properties.iteritems(): + + for pn, p in items(properties): link_used(p, rs) if is_nested_type_property(p): ns = deepcopy(p) @@ -755,7 +767,7 @@ def new_context(schemas, resources, methods): # To allow us recursing arrays, we simply put items one level up if 'items' in p: - ns.update((k, deepcopy(v)) for k, v in p.items.iteritems()) + ns.update((k, deepcopy(v)) for k, v in p.items.items()) recurse_properties(ns.id, ns, ns, append_unique(parent_ids, rs.id)) elif is_map_prop(p): @@ -847,7 +859,7 @@ def library_to_crate_name(name, suffix=''): # return version like 0.1.0+2014031421 def crate_version(build_version, revision): - return '%s+%s' % (build_version, isinstance(revision, basestring) and revision or '00000000') + return '%s+%s' % (build_version, isinstance(revision, str) and revision or '00000000') # return a crate name for us in extern crate statements def to_extern_crate_name(crate_name):