Merge branch 'toolchain-update'

.gitignore

@@ -12,7 +12,7 @@ gen/doc
 
 # Python: Makefile lead dirs
 .pyenv-*
-.virtualenv
+.virtualenv.marker
 
 # Sublime
 *.sublime-workspace

Makefile

@@ -2,13 +2,13 @@
 .SUFFIXES:
 
 VIRTUALENV_VERSION = 16.0.0
-VENV_BIN = .virtualenv/virtualenv.py
+VENV_BIN = .virtualenv.marker
+VENV_VERSION = 20.2.2
 
 VENV_DIR := .pyenv-$(shell uname)
 PYTHON_BIN := $(VENV_DIR)/bin/python
 
 PYTHON := . $(VENV_DIR)/bin/activate; python
-PIP := $(PYTHON) -m pip
 PYTEST := $(PYTHON) -m pytest
 
 MAKO_RENDER := etc/bin/mako-render
@@ -63,13 +63,12 @@ $(PREPROC): $(PREPROC_DIR)/src/main.rs
 	cd "$(PREPROC_DIR)" && cargo build --release 
 
 $(VENV_BIN):
-	wget -nv https://pypi.python.org/packages/source/v/virtualenv/virtualenv-$(VIRTUALENV_VERSION).tar.gz -O virtualenv-$(VIRTUALENV_VERSION).tar.gz
-	tar -xzf virtualenv-$(VIRTUALENV_VERSION).tar.gz && mv virtualenv-$(VIRTUALENV_VERSION) ./.virtualenv && rm -f virtualenv-$(VIRTUALENV_VERSION).tar.gz
-	chmod +x $@
+	python3 -m pip install --user virtualenv==$(VENV_VERSION)
+	touch $@
 
 $(PYTHON_BIN): $(VENV_BIN) requirements.txt
-	$(VENV_BIN) -p python2.7 $(VENV_DIR)
-	$(PIP) install -r requirements.txt
+	python3 -m virtualenv -p python3 $(VENV_DIR)
+	$@ -m pip install -r requirements.txt
 
 $(MAKO_RENDER): $(PYTHON_BIN) $(wildcard $(MAKO_LIB_DIR)/*)
 

etc/bin/mako-render

@@ -70,9 +70,6 @@ class DictObject(object):
     def __repr__(self):
         return repr(self.__dict__)
 
-    def __getattr__(self, name):
-        return object.__getattribute__(self, name)
-
     def __getitem__(self, name):
         try:
             return getattr(self, name)
@@ -191,9 +188,9 @@ class DictObject(object):
         """as dict.items"""
         return list(self.__dict__.items())
 
-    def iteritems(self):
-        """as dict.iteritems"""
-        return iter(self.__dict__.items())
+    def _items(self):
+        """as dict.items, avoiding name clashes"""
+        return list(self.__dict__.items())
 
     def pop(self, key, default=sys):
         """as dict.pop"""

requirements.txt

@@ -1,4 +1,4 @@
-mako
+mako==1.1.3
 pyyaml
 mkdocs==0.16.3
 pytest

src/mako/api/lib.rs.mako

@@ -168,7 +168,7 @@ ${rbuild.new(resource, c)}
 // CallBuilders   ###
 // #################
 
-% for resource, methods in c.rta_map.iteritems():
+% for resource, methods in c.rta_map.items():
 % for method in methods:
 ${mbuild.new(resource, method, c)}
 

src/mako/api/lib/lib.mako

@@ -156,7 +156,7 @@ let r = hub.resource().activity(...).${api.terms.action}()
 Or specifically ...
 
 ```ignore
-% for an, a in c.sta_map[fr.id].iteritems():
+% for an, a in c.sta_map[fr.id].items():
 <% category, resource, activity = activity_split(an) %>\
 let r = hub.${mangle_ident(resource)}().${mangle_ident(activity)}(...).${api.terms.action}()
 % endfor

src/mako/api/lib/mbuild.mako

@@ -11,7 +11,7 @@
                       DELEGATE_PROPERTY_NAME, struct_type_bounds_s, scope_url_to_variant,
                       re_find_replacements, ADD_PARAM_FN, ADD_PARAM_MEDIA_EXAMPLE, upload_action_fn, METHODS_RESOURCE,
                       method_name_to_variant, unique_type_name, size_to_bytes, method_default_scope,
-                      is_repeated_property, setter_fn_name, ADD_SCOPE_FN, rust_doc_sanitize)
+                      is_repeated_property, setter_fn_name, ADD_SCOPE_FN, rust_doc_sanitize, items)
 
     def get_parts(part_prop):
         if not part_prop:
@@ -154,7 +154,7 @@ ${self._setter_fn(resource, method, m, p, part_prop, ThisType, c)}\
     ///
     /// # Additional Parameters
     ///
-    % for opn, op in list((opn, op) for (opn, op) in parameters.iteritems() if opn not in [p.name for p in params]):
+    % for opn, op in list((opn, op) for (opn, op) in parameters.items() if opn not in [p.name for p in params]):
     /// * *${opn}* (${op.location}-${op.type}) - ${op.description}
     % endfor
     % endif
@@ -336,7 +336,7 @@ ${capture(lib.test_hub, hub_type_name, comments=show_all) | hide_filter}
 // into the respective structure. Some of the parts shown here might not be applicable !
 // ${random_value_warning}
 let mut ${rb_name} = ${request_value_type}::default();
-% for spn, sp in request_value.get('properties', dict()).iteritems():
+% for spn, sp in items(request_value.get('properties', dict())):
 % if parts is not None and spn not in parts:
 <% continue %>
 % endif
@@ -896,7 +896,7 @@ if enable_resource_parsing \
     % for p in media_params:
     ${p.description | rust_doc_sanitize, rust_doc_comment, indent_all_but_first_by(1)}
     ///
-    % for item_name, item in p.info.iteritems():
+    % for item_name, item in p.info.items():
     /// * *${split_camelcase_s(item_name)}*: ${isinstance(item, (list, tuple)) and put_and(enclose_in("'", item)) or str(item)}
     % endfor
     pub fn ${upload_action_fn(api.terms.upload_action, p.type.suffix)}<${mtype_param}>(self, ${p.type.arg_name}: ${mtype_param}, mime_type: mime::Mime) -> ${rtype}

src/mako/api/lib/rbuild.mako

@@ -110,7 +110,7 @@ impl${rb_params} ${ThisType} {
             % for p in optional_props:
             ${property(p.name)}: Default::default(),
             % endfor
-% for prop_key, custom_name in api.properties.iteritems():
+% for prop_key, custom_name in api.properties.items():
             % if prop_key == 'scopes' and not method_default_scope(m):
 <% continue %>\
             % endif

src/mako/api/lib/schema.mako

@@ -3,7 +3,7 @@
                       IO_TYPES, activity_split, enclose_in, REQUEST_MARKER_TRAIT, mb_type, indent_all_but_first_by,
                       NESTED_TYPE_SUFFIX, RESPONSE_MARKER_TRAIT, split_camelcase_s, METHODS_RESOURCE, unique_type_name, 
                       PART_MARKER_TRAIT, canonical_type_name, TO_PARTS_MARKER, UNUSED_TYPE_MARKER, is_schema_with_optionals,
-                      rust_doc_sanitize)
+                      rust_doc_sanitize, items)
 %>\
 ## Build a schema which must be an object
 ###################################################################################################################
@@ -12,7 +12,7 @@
 <% struct = 'pub struct ' + unique_type_name(s.id) %>\
 % if properties:
 ${struct} {
-% for pn, p in properties.iteritems():
+% for pn, p in items(properties):
     ${p.get('description', 'no description provided') | rust_doc_sanitize, rust_doc_comment, indent_all_but_first_by(1)}
     % if pn != mangle_ident(pn):
     #[serde(rename="${pn}")]
@@ -111,7 +111,7 @@ impl ${TO_PARTS_MARKER} for ${s_type} {
     /// the parts you want to see in the server response.
     fn to_parts(&self) -> String {
         let mut r = String::new();
-        % for pn, p in s.properties.iteritems():
+        % for pn, p in items(s.properties):
 <%
             mn = 'self.' + mangle_ident(pn)
             rt = to_rust_type(schemas, s.id, pn, p, allow_optionals=allow_optionals)
@@ -140,7 +140,7 @@ ${s.get('description', 'There is no detailed description.')}
 This type is used in activities, which are methods you may call on this type or where this type is involved in. 
 The list links the activity name, along with information about where it is used (one of ${put_and(enclose_in('*', IO_TYPES))}).
 
-% for a, iot in c.sta_map[s.id].iteritems():
+% for a, iot in c.sta_map[s.id].items():
 <%
     category, name, method = activity_split(a)
     name_suffix = ' ' + split_camelcase_s(name)

src/mako/cli/docs/commands.md.mako

@@ -1,7 +1,7 @@
 <%namespace name="util" file="../../lib/util.mako"/>\
 <%!
     from mako.filters import xml_escape
-    from util import (hash_comment, new_context, method_default_scope, indent_all_but_first_by, is_repeated_property)
+    from util import (hash_comment, new_context, method_default_scope, indent_all_but_first_by, is_repeated_property, custom_sorted)
     from cli import (subcommand_md_filename, new_method_context, SPLIT_START, SPLIT_END, pretty, SCOPE_FLAG,
                      mangle_subcommand, is_request_value_property, FIELD_SEP, PARAM_FLAG, UPLOAD_FLAG, docopt_mode,
                      FILE_ARG, MIME_ARG, OUT_ARG, OUTPUT_FLAG, to_cli_schema, cli_schema_to_yaml, SchemaEntry,
@@ -143,7 +143,7 @@ You may set the following properties to further configure the call. Please note
 or more key-value-pairs, and is called like this `-${PARAM_FLAG} k1=v1 k2=v2` even though the listing below repeats the
 `-${PARAM_FLAG}` for completeness.
 
-% for p in sorted(oprops):
+% for p in custom_sorted(oprops):
 ${self._md_property(p)}
 % endfor 
 % endif # optional method properties

src/mako/cli/lib/argparse.mako

@@ -16,7 +16,7 @@
             return 'None'
         if isinstance(v, bool):
             v = v and 'true' or 'false'
-        elif isinstance(v, basestring):
+        elif isinstance(v, str):
             v = 'r##"%s"##' % v
         elif isinstance(v, list):
             v = 'vec![%s]' % ','.join('UploadProtocol::%s' % p.capitalize() for p in v)

src/mako/cli/lib/engine.mako

@@ -357,7 +357,7 @@ if dry_run {
         allow_optionals = allow_optionals_fn(schema)
         if not allow_optionals:
             opt_access = ''
-        for fn, f in schema.fields.iteritems():
+        for fn, f in schema.fields.items():
             cur.append(['%s%s' % (mangle_ident(fn), opt_access), fn])
             fields.add(fn)
             if isinstance(f, SchemaEntry):

src/mako/deps.mako

@@ -4,7 +4,7 @@
 
 <%
 	import os
-	import urllib2
+	import urllib
 	import json
 
 	apis = {}
@@ -23,7 +23,7 @@
 
 	if os.environ.get('FETCH_APIS') is not None:
 		discovery_url = 'https://www.googleapis.com/discovery/v1/'
-		apis = json.loads(urllib2.urlopen(discovery_url + "apis").read())
+		apis = json.loads(urllib.request.urlopen(discovery_url + "apis").read())
 
 		print('Loaded {} apis from Google'.format(len(apis['items'])))
 
@@ -46,7 +46,7 @@
 	if mako is not UNDEFINED:
 		post_processor_arg = '--post-process-python-module=%s' % mako.post_processor_module
 %>\
-% for an, versions in api.list.iteritems():
+% for an, versions in api.list.items():
 % if an in api.get('blacklist', list()):
 <% continue %>\
 % endif
@@ -98,8 +98,9 @@
 			api_info.append((api_target, api_clean, api_cargo, api_doc, api_crate_publish_file, gen_root))
 
 			space_join = lambda i: ' '.join(a[i] for a in api_info)
-	except:
+	except Exception as e:
 		print('Could not open JSON file at {}'.format(api_json))
+		print(e)
 %>\
 ${api_common}: $(RUST_SRC)/${make.id}/cmn.rs $(lastword $(MAKEFILE_LIST)) ${gen_root_stamp}
 	@ echo "// COPY OF '$<'"  > $@

src/mako/lib/cli.py

@@ -172,7 +172,7 @@ def to_cli_schema(c, schema):
             properties[e.type_value] = e
     # end handle enumerations
 
-    for pn, p in properties.iteritems():
+    for pn, p in util.items(properties):
         def set_nested_schema(ns):
             if ns.fields:
                 fd[pn] = ns

src/mako/lib/util.py

@@ -114,6 +114,17 @@ data_unit_multipliers = {
 
 HUB_TYPE_PARAMETERS = ('C', 'A')
 
+def items(p):
+    if isinstance(p, dict):
+        return p.items()
+    else:
+        return p._items()
+
+def custom_sorted(p):
+    if not isinstance(p, list):
+        assert(false, p, "unexpected type")
+    return sorted(p, key = lambda p: p['name'])
+
 # ==============================================================================
 ## @name Filters
 # ------------------------------------------------------------------------------
@@ -440,7 +451,7 @@ def schema_markers(s, c, transitive=True):
             continue
         has_activity = True
         # it should have at least one activity that matches it's type to qualify for the Resource trait
-        for fqan, iot in activities.iteritems():
+        for fqan, iot in activities.items():
             _, resource, _ = activity_split(fqan)
             if resource and activity_name_to_type_name(resource).lower() == sid.lower():
                 res.add(RESOURCE_MARKER_TRAIT)
@@ -504,7 +515,7 @@ def activity_name_to_type_name(an):
 
 # yields (category, resource, activity, activity_data)
 def iter_acitivities(c):
-    return ((activity_split(an) + [a]) for an, a in c.fqan_map.iteritems())
+    return ((activity_split(an) + [a]) for an, a in c.fqan_map.items())
 
 # return a list of parameter structures of all params of the given method dict
 # apply a prune filter to restrict the set of returned parameters.
@@ -512,7 +523,7 @@ def iter_acitivities(c):
 def _method_params(m, required=None, location=None):
     res = list()
     po = m.get('parameterOrder', [])
-    for pn, p in m.get('parameters', dict()).iteritems():
+    for pn, p in m.get('parameters', dict()).items():
         if required is not None and p.get('required', False) != required:
             continue
         if location is not None and p.get('location', '') != location:
@@ -613,7 +624,7 @@ def method_media_params(m):
     # actually, one of them is required, but we can't encode that ...
     # runtime will have to check
     res = list()
-    for pn, proto in mu.protocols.iteritems():
+    for pn, proto in mu.protocols.items():
         # the pi (proto-info) dict can be shown to the user
         pi = {'multipart': proto.multipart and 'yes' or 'no', 'maxSize': mu.get('maxSize', '0kb'), 'validMimeTypes': mu.accept}
         try:
@@ -672,12 +683,12 @@ def new_context(schemas, resources, methods):
             res = dict()
         if fqan is None:
             fqan = dict()
-        for k,a in activities.iteritems():
+        for k,a in activities.items():
             if 'resources' in a:
                 build_activity_mappings(a.resources, res, fqan)
             if 'methods' not in a:
                 continue
-            for mn, m in a.methods.iteritems():
+            for mn, m in a.methods.items():
                 assert m.id not in fqan
                 category, resource, method = activity_split(m.id)
                 # This may be another name by which people try to find the method.
@@ -746,7 +757,8 @@ def new_context(schemas, resources, methods):
             # end this is already a perfectly valid type
 
             properties = s.get('properties', {'': s})
-            for pn, p in properties.iteritems():
+
+            for pn, p in items(properties):
                 link_used(p, rs)
                 if is_nested_type_property(p):
                     ns = deepcopy(p)
@@ -755,7 +767,7 @@ def new_context(schemas, resources, methods):
 
                     # To allow us recursing arrays, we simply put items one level up
                     if 'items' in p:
-                        ns.update((k, deepcopy(v)) for k, v in p.items.iteritems())
+                        ns.update((k, deepcopy(v)) for k, v in p.items.items())
 
                     recurse_properties(ns.id, ns, ns, append_unique(parent_ids, rs.id))
                 elif is_map_prop(p):
@@ -847,7 +859,7 @@ def library_to_crate_name(name, suffix=''):
 
 # return version like 0.1.0+2014031421
 def crate_version(build_version, revision):
-    return '%s+%s' % (build_version, isinstance(revision, basestring) and revision or '00000000')
+    return '%s+%s' % (build_version, isinstance(revision, str) and revision or '00000000')
 
 # return a crate name for us in extern crate statements
 def to_extern_crate_name(crate_name):