From 5b4f18d341cbd8f87d3e3792b1dfa803f7849015 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Apr 2015 10:50:19 +0200 Subject: [PATCH] fix(api+cli): improved scope handling; fix CLI * in APIs, scopes will now be per-method, and if no scope is given, we will assume only the API key has to be set. Previously there was a wild mix between globally mentioned scopes and method scopes. * assure CLI generation works so far, for all avaialable APIs Related to #48 --- README.md | 2 +- src/mako/api/lib/mbuild.mako | 16 +++--- src/mako/cli/docs/commands.md.mako | 25 ++++----- src/mako/cli/lib/cli.py | 85 +++++++++++++++++++++++++++++- src/mako/lib/util.py | 9 ++-- 5 files changed, 109 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index d5f718b01d..a6ca0cb007 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Click the image below to see the playlist with all project related content: Each episode sums up one major step in project development: -* [Episode 1](http://youtu.be/2U3SpepKaBE): How to write 78 APIs in 5s +* [Episode 1](http://youtu.be/2U3SpepKaBE): How to write 78 APIs in 5 seconds # Build Instructions diff --git a/src/mako/api/lib/mbuild.mako b/src/mako/api/lib/mbuild.mako index e1af5ac7ed..49c16d0579 100644 --- a/src/mako/api/lib/mbuild.mako +++ b/src/mako/api/lib/mbuild.mako @@ -8,7 +8,7 @@ indent_by, to_rust_type, rnd_arg_val_for_type, extract_parts, mb_type_params_s, hub_type_params_s, method_media_params, enclose_in, mb_type_bounds, method_response, CALL_BUILDER_MARKERT_TRAIT, pass_through, markdown_rust_block, parts_from_params, - DELEGATE_PROPERTY_NAME, struct_type_bounds_s, supports_scopes, scope_url_to_variant, + DELEGATE_PROPERTY_NAME, struct_type_bounds_s, scope_url_to_variant, re_find_replacements, ADD_PARAM_FN, ADD_PARAM_MEDIA_EXAMPLE, upload_action_fn, METHODS_RESOURCE, method_name_to_variant, unique_type_name, size_to_bytes, method_default_scope, is_repeated_property) @@ -119,7 +119,7 @@ pub struct ${ThisType} % endfor ## A generic map for additinal parameters. Sometimes you can set some that are documented online only ${api.properties.params}: HashMap, - % if supports_scopes(auth): + % if method_default_scope(m): ## We need the scopes sorted, to not unnecessarily query new tokens ${api.properties.scopes}: BTreeMap % endif @@ -156,7 +156,7 @@ ${self._setter_fn(resource, method, m, p, part_prop, ThisType, c)}\ self } - % if supports_scopes(auth): + % if method_default_scope(m): /// Identifies the authorization scope for the method you are building. /// /// Use this method to actively specify which scope should be used, instead the default `Scope` variant @@ -428,9 +428,7 @@ match result { delegate_finish = 'dlg.finished' auth_call = 'self.hub.auth.borrow_mut()' - if supports_scopes(auth): - default_scope = method_default_scope(m) - # end handle default scope + default_scope = method_default_scope(m) # s = '{foo}' -> ('{foo}', 'foo') -> (find_this, replace_with) seen = set() @@ -573,7 +571,7 @@ else { % else: let mut url = "${baseUrl}${m.path}".to_string(); % endif - % if not supports_scopes(auth): + % if not default_scope: <% assert 'key' in parameters, "Expected 'key' parameter if there are no scopes" %> @@ -657,7 +655,7 @@ else { % endif loop { - % if supports_scopes(auth): + % if default_scope: let mut token = ${auth_call}.token(self.${api.properties.scopes}.keys()); if token.is_none() { token = dlg.token(); @@ -706,7 +704,7 @@ else { let mut client = &mut *self.hub.client.borrow_mut(); let mut req = client.borrow_mut().request(${method_name_to_variant(m.httpMethod)}, url.as_ref()) .header(UserAgent(self.hub._user_agent.clone()))\ - % if supports_scopes(auth): + % if default_scope: .header(auth_header.clone())\ % endif diff --git a/src/mako/cli/docs/commands.md.mako b/src/mako/cli/docs/commands.md.mako index dc4ec2b2b2..5955240b53 100644 --- a/src/mako/cli/docs/commands.md.mako +++ b/src/mako/cli/docs/commands.md.mako @@ -3,11 +3,13 @@ from util import (hash_comment, new_context, method_default_scope, indent_all_but_first_by, is_repeated_property) from cli import (subcommand_md_filename, new_method_context, SPLIT_START, SPLIT_END, pretty, SCOPE_FLAG, mangle_subcommand, is_request_value_property, FIELD_SEP, PARAM_FLAG, UPLOAD_FLAG, docopt_mode, - FILE_ARG, MIME_ARG, OUT_ARG, OUTPUT_FLAG) + FILE_ARG, MIME_ARG, OUT_ARG, OUTPUT_FLAG, to_cli_schema, cli_schema_to_yaml) from copy import deepcopy escape_html = lambda n: n.replace('>', r'\>') + + NO_DESC = 'No description provided.' %>\ <% c = new_context(schemas, resources, context.get('methods')) @@ -18,7 +20,7 @@ mc = new_method_context(resource, method, c) %>\ ${SPLIT_START} ${subcommand_md_filename(resource, method)} -% if mc.m.description: +% if 'description' in mc.m: ${mc.m.description} % endif # show method description % if mc.m.get('scopes'): @@ -48,10 +50,13 @@ You can set the scope for this method like this: `${util.program_name()} --${SCO # Required Scalar ${len(rprops) > 1 and 'Arguments' or 'Argument'} % for p in rprops: * **<${mangle_subcommand(p.name)}\>** - - ${p.get('description') or 'No description provided' | indent_all_but_first_by(2)} + - ${p.get('description') or NO_DESC | indent_all_but_first_by(2)} % endfor # each required property (which is not the request value) % endif # have required properties % if mc.request_value: +<% + request_cli_schema = to_cli_schema(c, mc.request_value) +%>\ # Required Request Value The request value is a data-structure with various fields. Each field may be a simple scalar or another data-structure. @@ -59,15 +64,7 @@ In the latter case it is advised to set the field-cursor to the data-structure's For example, a structure like this: ``` -"scalar_int": 5, -"struct": { - "scalar_float": 2.4 - "sub_struct": { - "strings": ["baz", "bar"], - "mapping": HashMap, - } -} -"scalar_str": "foo", +${cli_schema_to_yaml(request_cli_schema)} ``` can be set completely with the following arguments. Note how the cursor position is adjusted the respective fields: @@ -91,7 +88,7 @@ This method supports the upload of data, using the following protocol${len(mc.me * **-${UPLOAD_FLAG} ${docopt_mode(protocols)} ${escape_html(FILE_ARG)} ${escape_html(MIME_ARG)}** % for mp in mc.media_params: - - **${mp.protocol}** - ${mp.description.split('\n')[0]} + - **${mp.protocol}** - ${mp.get('description', NO_DESC).split('\n')[0]} % endfor # each media param - **${escape_html(FILE_ARG)}** + Path to file to upload. It must be seekable. @@ -153,5 +150,5 @@ ${SPLIT_END} <%def name="_md_property(p)">\ * **-${PARAM_FLAG} ${mangle_subcommand(p.name)}=${p.type}** - - ${p.get('description') or "No description provided" | indent_all_but_first_by(2)} + - ${p.get('description') or NO_DESC | indent_all_but_first_by(2)} \ No newline at end of file diff --git a/src/mako/cli/lib/cli.py b/src/mako/cli/lib/cli.py index e6a97125db..93ae0f7b81 100644 --- a/src/mako/cli/lib/cli.py +++ b/src/mako/cli/lib/cli.py @@ -3,6 +3,7 @@ import util import os import re import collections +from copy import deepcopy SPLIT_START = '>>>>>>>' SPLIT_END = '<<<<<<<' @@ -22,12 +23,19 @@ FIELD_SEP = '.' CONFIG_DIR = '~/.google-service-cli' +POD_TYPES = set(('boolean', 'integer', 'number', 'uint32', 'double', 'float', 'int32', 'int64', 'uint64', 'string')) + re_splitters = re.compile(r"%s ([\w\-\.]+)\n(.*?)\n%s" % (SPLIT_START, SPLIT_END), re.MULTILINE|re.DOTALL) MethodContext = collections.namedtuple('MethodContext', ['m', 'response_schema', 'params', 'request_value', 'media_params' ,'required_props', 'optional_props', 'part_prop']) +CTYPE_POD = 'pod' +CTYPE_ARRAY = 'list' +CTYPE_MAP = 'map' +SchemaEntry = collections.namedtuple('SchemaEntry', ['container_type', 'actual_property', 'property']) + def new_method_context(resource, method, c): m = c.fqan_map[util.to_fqan(c.rtc_map[resource], resource, method)] response_schema = util.method_response(c, m) @@ -46,6 +54,7 @@ def pretty(n): def is_request_value_property(mc, p): return mc.request_value and mc.request_value.id == p.get(util.TREF) + # transform name to be a suitable subcommand def mangle_subcommand(name): return util.camel_to_under(name).replace('_', '-').replace('.', '-') @@ -55,12 +64,86 @@ def mangle_subcommand(name): def subcommand_md_filename(resource, method): return mangle_subcommand(resource) + '_' + mangle_subcommand(method) + '.md' + def docopt_mode(protocols): mode = '|'.join(protocols) if len(protocols) > 1: mode = '(%s)' % mode return mode + +# Return schema' with fields dict: { 'field1' : SchemaField(...), 'SubSchema': schema' } +def to_cli_schema(c, schema): + res = deepcopy(schema) + fd = dict() + res['fields'] = fd + + # util.nested_type_name + properties = schema.get('properties', dict()) + if not properties and 'variant' in schema and 'map' in schema.variant: + for e in schema.variant.map: + assert util.TREF in e + properties[e.type_value] = e + # end handle enumerations + + for pn, p in properties.iteritems(): + def set_nested_schema(ns): + if ns.fields: + fd[pn] = ns + # end utility + + def dup_property(): + pc = deepcopy(p) + if 'type' in pc and pc.type == 'string' and 'Count' in pn: + pc.type = 'int64' + return pc + # end + + if util.TREF in p: + if p[util.TREF] != schema.id: # prevent recursion (in case of self-referential schemas) + set_nested_schema(to_cli_schema(c, c.schemas[p[util.TREF]])) + elif p.type == 'array' and 'items' in p and 'type' in p.get('items') and p.get('items').type in POD_TYPES: + pc = dup_property() + fd[pn] = SchemaEntry(CTYPE_ARRAY, pc.get('items'), pc) + elif p.type == 'object': + if util.is_map_prop(p): + if 'type' in p.additionalProperties and p.additionalProperties.type in POD_TYPES: + pc = dup_property() + fd[pn] = SchemaEntry(CTYPE_MAP, pc.additionalProperties, pc) + else: + set_nested_schema(to_cli_schema(c, c.schemas[util.nested_type_name(schema.id, pn)])) + elif p.type in POD_TYPES: + pc = dup_property() + fd[pn] = SchemaEntry(CTYPE_POD, pc, pc) + # end handle property type + # end + + return res + + +# Convert the given cli-schema (result from to_cli_schema(schema)) to a yaml-like string. It's suitable for +# documentation only +def cli_schema_to_yaml(schema, prefix=''): + if not prefix: + o = '%s%s:\n' % (prefix, util.unique_type_name(schema.id)) + else: + o = '' + prefix += ' ' + for fn, f in schema.fields.iteritems(): + o += '%s%s:' % (prefix, mangle_subcommand(fn)) + if not isinstance(f, SchemaEntry): + o += '\n' + cli_schema_to_yaml(f, prefix) + else: + t = f.actual_property.type + if f.container_type == CTYPE_ARRAY: + t = '[%s]' % t + elif f.container_type == CTYPE_MAP: + t = '{ string: %s }' % t + o += ' %s\n' % t + # end for each field + return o + + # split the result along split segments def process_template_result(r, output_file): found = False @@ -74,7 +157,7 @@ def process_template_result(r, output_file): for m in re_splitters.finditer(r): found = True fh = open(os.path.join(dir, m.group(1)), 'wb') - fh.write(m.group(2)) + fh.write(m.group(2).encode('UTF-8')) fh.close() # end for each match diff --git a/src/mako/lib/util.py b/src/mako/lib/util.py index 17835024d2..b4e0da3a7e 100644 --- a/src/mako/lib/util.py +++ b/src/mako/lib/util.py @@ -297,7 +297,7 @@ def mangle_ident(n): return n + '_' return n -def _is_map_prop(p): +def is_map_prop(p): return 'additionalProperties' in p def _assure_unique_type_name(schemas, tn): @@ -342,7 +342,7 @@ def to_rust_type(schemas, sn, pn, t, allow_optionals=True, _is_recursive=False): if t.type == 'array': return wrap_type("%s<%s>" % (rust_type, unique_type_name((nested_type(t))))) elif t.type == 'object': - if _is_map_prop(t): + if is_map_prop(t): return wrap_type("%s" % (rust_type, nested_type(t))) else: return wrap_type(nested_type(t)) @@ -725,7 +725,7 @@ def new_context(schemas, resources, methods): ns.update((k, deepcopy(v)) for k, v in p.items.iteritems()) recurse_properties(ns.id, ns, ns, append_unique(parent_ids, rs.id)) - elif _is_map_prop(p): + elif is_map_prop(p): recurse_properties(nested_type_name(prefix, pn), rs, p.additionalProperties, append_unique(parent_ids, rs.id)) elif 'items' in p: @@ -838,7 +838,10 @@ def supports_scopes(auth): return bool(auth) and bool(auth.oauth2) # Returns th desired scope for the given method. It will use read-only scopes for read-only methods +# May be None no scope-based authentication is required def method_default_scope(m): + if 'scopes' not in m: + return None default_scope = sorted(m.scopes)[0] if m.httpMethod in ('HEAD', 'GET', 'OPTIONS', 'TRACE'): for scope in m.scopes: