From 3d831bcdc6d2601a553a66b738edbb9b91e02984 Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Thu, 6 Oct 2022 22:57:48 -0700 Subject: [PATCH 01/27] Use type specified by format key where possible --- src/generator/lib/util.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/generator/lib/util.py b/src/generator/lib/util.py index 8b075b9fcb..f33bc5f8f4 100644 --- a/src/generator/lib/util.py +++ b/src/generator/lib/util.py @@ -33,6 +33,17 @@ TYPE_MAP = {'boolean' : 'bool', 'string' : 'String', 'object' : 'HashMap'} +# TODO: Provide support for these as well +# Default to using string type for now +UNSUPPORTED_TYPES = { + "google-duration", + "byte", + "google-datetime", + "date-time", + "google-fieldmask", + "date", +} + RESERVED_WORDS = set(('abstract', 'alignof', 'as', 'become', 'box', 'break', 'const', 'continue', 'crate', 'do', 'else', 'enum', 'extern', 'false', 'final', 'fn', 'for', 'if', 'impl', 'in', 'let', 'loop', 'macro', 'match', 'mod', 'move', 'mut', 'offsetof', 'override', 'priv', 'pub', 'pure', 'ref', @@ -372,28 +383,29 @@ def to_rust_type( if TREF in t: # simple, non-recursive fix for some recursive types. This only works on the first depth level # which is fine for now. 'allow_optionals' implicitly restricts type boxing for simple types - it - # usually is on on the first call, and off when recursion is involved. + # is usually on the first call, and off when recursion is involved. tn = t[TREF] if not _is_recursive and tn == schema_name: tn = 'Option>' % tn return wrap_type(tn) try: - rust_type = TYPE_MAP[t['type']] + # TODO: add support for all types and remove this check + # rust_type = TYPE_MAP[t.get("format", t["type"])] + # prefer format if present - provides support for i64 + if "format" in t and t["format"] in TYPE_MAP: + rust_type = TYPE_MAP[t["format"]] + else: + rust_type = TYPE_MAP[t["type"]] if t['type'] == 'array': - return wrap_type("%s<%s>" % (rust_type, (nested_type(t)))) + return wrap_type("%s<%s>" % (rust_type, nested_type(t))) elif t['type'] == 'object': if is_map_prop(t): return wrap_type("%s" % (rust_type, nested_type(t))) - else: - return wrap_type(nested_type(t)) - elif rust_type == USE_FORMAT: - rust_type = TYPE_MAP[t['format']] + return wrap_type(nested_type(t)) if t.get('repeated', False): - rust_type = 'Vec<%s>' % rust_type - else: - rust_type = wrap_type(rust_type) - return rust_type + return 'Vec<%s>' % rust_type + return wrap_type(rust_type) except KeyError as err: raise AssertionError("%s: Property type '%s' unknown - add new type mapping: %s" % (str(err), t['type'], str(t))) except AttributeError as err: From 66c535e4d67b2ebcb726eaad4cb6daf24c650ecc Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Fri, 7 Oct 2022 02:14:26 -0700 Subject: [PATCH 02/27] Add support for duration and base64 serde --- google-apis-common/Cargo.toml | 3 +- google-apis-common/src/lib.rs | 97 +++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/google-apis-common/Cargo.toml b/google-apis-common/Cargo.toml index 2edda545ef..c0cc13475a 100644 --- a/google-apis-common/Cargo.toml +++ b/google-apis-common/Cargo.toml @@ -17,7 +17,8 @@ doctest = false [dependencies] mime = "^ 0.2.0" -serde = "^ 1.0" +serde = { version = "^ 1.0", features = ["derive"] } +base64 = "0.13.0" serde_json = "^ 1.0" ## TODO: Make yup-oauth2 optional ## yup-oauth2 = { version = "^ 7.0", optional = true } diff --git a/google-apis-common/src/lib.rs b/google-apis-common/src/lib.rs index 0b1900f07f..37eb28e56c 100644 --- a/google-apis-common/src/lib.rs +++ b/google-apis-common/src/lib.rs @@ -843,6 +843,103 @@ mod yup_oauth2_impl { } } +pub mod types { + use std::str::FromStr; + use serde::{Deserialize, Deserializer, Serializer}; + // https://github.com/protocolbuffers/protobuf-go/blob/6875c3d7242d1a3db910ce8a504f124cb840c23a/types/known/durationpb/duration.pb.go#L148 + #[derive(Deserialize)] + #[serde(try_from = "IntermediateDuration")] + pub struct Duration { + pub seconds: i64, + pub nanoseconds: i32, + } + + #[derive(Deserialize)] + struct IntermediateDuration<'a>(&'a str); + + impl serde::Serialize for Duration { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if self.nanoseconds != 0 { + if self.seconds == 0 && self.nanoseconds.is_negative() { + serializer.serialize_str(&format!("-0.{}s", self.nanoseconds.abs())) + } else { + serializer.serialize_str(&format!("{}.{}s", self.seconds, self.nanoseconds.abs())) + } + } else { + serializer.serialize_str(&format!("{}s", self.seconds)) + } + } + } + + impl <'a> TryFrom> for Duration { + type Error = std::num::ParseIntError; + + fn try_from(value: IntermediateDuration<'a>) -> Result { + let abs_duration = 315576000000i64; + // TODO: Test strings like -.s, -0.0s + if !value.0.ends_with('s') { + todo!(); + } + let (seconds, nanoseconds) = if let Some((seconds, nanos)) = value.0.split_once('.') { + let seconds = i64::from_str(seconds)?; + let nano_len = nanos.len() - 1; + // up . 000_000_000 + let nano_digits = nanos.chars().filter(|c| c.is_digit(10)).count() as u32; + if nano_digits > 9 { + todo!() + } + // 2 digits: 120_000_000 + // 9 digits: 123_456_789 + // pad to 9 digits + let nanos = i32::from_str(&nanos[..nanos.len() - 1])? * 10_i32.pow(9 - nano_digits); + (seconds, nanos) + } else { + // TODO: handle negative number + (i64::from_str(&value.0[..value.0.len().saturating_sub(1)])?, 0) + }; + if (seconds > 0 && nanoseconds < 0) || (seconds < 0 && nanoseconds > 0) { + todo!(); + } + + if seconds >= abs_duration || seconds <= -abs_duration { + todo!(); + } + if nanoseconds >= 1_000_000_000 || nanoseconds <= -1_000_000_000 { + todo!(); + } + + Ok(Duration { seconds, nanoseconds }) + } + } + + + // #[serde(serialize_with = "path")] + fn to_urlsafe_base64(x: &str, s: S) -> Result + where + S: Serializer, + { + s.serialize_str(&base64::encode_config(x, base64::URL_SAFE)) + } + // #[serde(deserialize_with = "path")] + fn from_urlsafe_base64<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let s: &str = Deserialize::deserialize(deserializer)?; + Ok(base64::decode_config(s, base64::URL_SAFE).unwrap()) + } + // TODO: + // "google-datetime", + // "date-time", + // "date", + + // TODO: https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/field-mask + // "google-fieldmask", +} + #[cfg(test)] mod test_api { use super::*; From 29aa8df15b0ee08b7dea83f9a25cd9d6c2304b99 Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Fri, 7 Oct 2022 13:36:03 -0700 Subject: [PATCH 03/27] Use appropriate types for date-time, duration, bytes --- google-apis-common/Cargo.toml | 1 + google-apis-common/src/lib.rs | 26 +- src/generator/lib/util.py | 282 ++++++++++++++------ src/generator/templates/api/lib/schema.mako | 3 + 4 files changed, 221 insertions(+), 91 deletions(-) diff --git a/google-apis-common/Cargo.toml b/google-apis-common/Cargo.toml index c0cc13475a..4bbe0e0b40 100644 --- a/google-apis-common/Cargo.toml +++ b/google-apis-common/Cargo.toml @@ -20,6 +20,7 @@ mime = "^ 0.2.0" serde = { version = "^ 1.0", features = ["derive"] } base64 = "0.13.0" serde_json = "^ 1.0" +chrono = { version = "0.4.22", features = ["serde"] } ## TODO: Make yup-oauth2 optional ## yup-oauth2 = { version = "^ 7.0", optional = true } yup-oauth2 = "^ 7.0" diff --git a/google-apis-common/src/lib.rs b/google-apis-common/src/lib.rs index 37eb28e56c..80fc5954a6 100644 --- a/google-apis-common/src/lib.rs +++ b/google-apis-common/src/lib.rs @@ -24,6 +24,7 @@ use tokio::time::sleep; use tower_service; pub use yup_oauth2 as oauth2; +pub use chrono; const LINE_ENDING: &str = "\r\n"; @@ -854,9 +855,16 @@ pub mod types { pub nanoseconds: i32, } + impl From for chrono::Duration { + fn from(duration: Duration) -> chrono::Duration { + chrono::Duration::seconds(duration.seconds) + chrono::Duration::nanoseconds(duration.nanoseconds as i64) + } + } + #[derive(Deserialize)] struct IntermediateDuration<'a>(&'a str); + impl serde::Serialize for Duration { fn serialize(&self, serializer: S) -> Result where @@ -917,24 +925,24 @@ pub mod types { // #[serde(serialize_with = "path")] - fn to_urlsafe_base64(x: &str, s: S) -> Result + pub fn to_urlsafe_base64(x: Option<&str>, s: S) -> Result where S: Serializer, { - s.serialize_str(&base64::encode_config(x, base64::URL_SAFE)) + match x { + None => s.serialize_none(), + Some(x) => s.serialize_some(&base64::encode_config(x, base64::URL_SAFE)) + } } // #[serde(deserialize_with = "path")] - fn from_urlsafe_base64<'de, D>(deserializer: D) -> Result, D::Error> + pub fn from_urlsafe_base64<'de, D>(deserializer: D) -> Result>, D::Error> where D: Deserializer<'de>, { - let s: &str = Deserialize::deserialize(deserializer)?; - Ok(base64::decode_config(s, base64::URL_SAFE).unwrap()) + let s: Option<&str> = Deserialize::deserialize(deserializer)?; + // TODO: Map error + Ok(s.map(|s| base64::decode_config(s, base64::URL_SAFE).unwrap())) } - // TODO: - // "google-datetime", - // "date-time", - // "date", // TODO: https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/field-mask // "google-fieldmask", diff --git a/src/generator/lib/util.py b/src/generator/lib/util.py index f33bc5f8f4..5fa6bff15c 100644 --- a/src/generator/lib/util.py +++ b/src/generator/lib/util.py @@ -12,36 +12,40 @@ seed(1337) re_linestart = re.compile('^', flags=re.MULTILINE) re_spaces_after_newline = re.compile('^ {4}', flags=re.MULTILINE) re_first_4_spaces = re.compile('^ {1,4}', flags=re.MULTILINE) -re_desc_parts = re.compile(r"((the part (names|properties) that you can include in the parameter value are)|(supported values are ))(.*?)\.", flags=re.IGNORECASE|re.MULTILINE) +re_desc_parts = re.compile( + r"((the part (names|properties) that you can include in the parameter value are)|(supported values are ))(.*?)\.", + flags=re.IGNORECASE | re.MULTILINE) re_find_replacements = re.compile(r"\{[/\+]?\w+\*?\}") -HTTP_METHODS = set(("OPTIONS", "GET", "POST", "PUT", "DELETE", "HEAD", "TRACE", "CONNECT", "PATCH" )) - +HTTP_METHODS = set(("OPTIONS", "GET", "POST", "PUT", "DELETE", "HEAD", "TRACE", "CONNECT", "PATCH")) +CHRONO_DATETIME = 'client::chrono::DateTime' USE_FORMAT = 'use_format_field' -TYPE_MAP = {'boolean' : 'bool', - 'integer' : USE_FORMAT, - 'number' : USE_FORMAT, - 'uint32' : 'u32', - 'double' : 'f64', - 'float' : 'f32', - 'int32' : 'i32', - 'any' : 'String', # TODO: Figure out how to handle it. It's 'interface' in Go ... - 'int64' : 'i64', - 'uint64' : 'u64', - 'array' : 'Vec', - 'string' : 'String', - 'object' : 'HashMap'} - -# TODO: Provide support for these as well -# Default to using string type for now -UNSUPPORTED_TYPES = { - "google-duration", - "byte", - "google-datetime", - "date-time", - "google-fieldmask", - "date", +TYPE_MAP = { + 'boolean': 'bool', + 'integer': USE_FORMAT, + 'number': USE_FORMAT, + 'uint32': 'u32', + 'double': 'f64', + 'float': 'f32', + 'int32': 'i32', + 'any': 'String', # TODO: Figure out how to handle it. It's 'interface' in Go ... + 'int64': 'i64', + 'uint64': 'u64', + 'array': 'Vec', + 'string': 'String', + 'object': 'HashMap', + # should be correct + 'google-datetime': CHRONO_DATETIME, + # assumption + 'date-time': CHRONO_DATETIME, + 'date': CHRONO_DATETIME, + # custom impl + 'google-duration': 'client::types::Duration', + # guessing bytes is universally url-safe b64 + "byte": "Vec", + # TODO: Provide support for these as well + "google-fieldmask": 'String' } RESERVED_WORDS = set(('abstract', 'alignof', 'as', 'become', 'box', 'break', 'const', 'continue', 'crate', 'do', @@ -50,17 +54,21 @@ RESERVED_WORDS = set(('abstract', 'alignof', 'as', 'become', 'box', 'break', 'co 'return', 'sizeof', 'static', 'self', 'struct', 'super', 'true', 'trait', 'type', 'typeof', 'unsafe', 'unsized', 'use', 'virtual', 'where', 'while', 'yield')) -words = [w.strip(',') for w in "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.".split(' ')] -RUST_TYPE_RND_MAP = {'bool': lambda: str(bool(randint(0, 1))).lower(), - 'u32' : lambda: randint(0, 100), - 'u64' : lambda: randint(0, 100), - 'f64' : lambda: random(), - 'f32' : lambda: random(), - 'i32' : lambda: randint(-101, -1), - 'i64' : lambda: randint(-101, -1), - 'String': lambda: '"%s"' % choice(words), - '&str': lambda: '"%s"' % choice(words), - '&Vec': lambda: '&vec!["%s".into()]' % choice(words), # why a reference to Vec? Because it works. Should be slice, but who knows how typing works here. +words = [w.strip(',') for w in + "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.".split( + ' ')] +RUST_TYPE_RND_MAP = { + 'bool': lambda: str(bool(randint(0, 1))).lower(), + 'u32': lambda: randint(0, 100), + 'u64': lambda: randint(0, 100), + 'f64': lambda: random(), + 'f32': lambda: random(), + 'i32': lambda: randint(-101, -1), + 'i64': lambda: randint(-101, -1), + 'String': lambda: '"%s"' % choice(words), + '&str': lambda: '"%s"' % choice(words), + '&Vec': lambda: '&vec!["%s".into()]' % choice(words), + # why a reference to Vec? Because it works. Should be slice, but who knows how typing works here. } TREF = '$ref' IO_RESPONSE = 'response' @@ -92,7 +100,7 @@ TO_PARTS_MARKER = 'client::ToParts' UNUSED_TYPE_MARKER = 'client::UnusedType' PROTOCOL_TYPE_INFO = { - 'simple' : { + 'simple': { 'arg_name': 'stream', 'description': """Upload media all at once. If the upload fails for whichever reason, all progress is lost.""", @@ -100,7 +108,7 @@ If the upload fails for whichever reason, all progress is lost.""", 'suffix': '', 'example_value': 'fs::File::open("file.ext").unwrap(), "application/octet-stream".parse().unwrap()' }, - 'resumable' : { + 'resumable': { 'arg_name': 'resumeable_stream', 'description': """Upload media in a resumable fashion. Even if the upload fails or is interrupted, it can be resumed for a @@ -127,14 +135,17 @@ data_unit_multipliers = { HUB_TYPE_PARAMETERS = ('S',) + def items(p): if isinstance(p, dict): return p.items() else: return p._items() + def custom_sorted(p: List[Mapping[str, Any]]) -> List[Mapping[str, Any]]: - return sorted(p, key = lambda p: p['name']) + return sorted(p, key=lambda p: p['name']) + # ============================================================================== ## @name Filters @@ -145,19 +156,23 @@ def custom_sorted(p: List[Mapping[str, Any]]) -> List[Mapping[str, Any]]: def rust_module_doc_comment(s): return re_linestart.sub('//! ', s) + # rust doc comment filter def rust_doc_comment(s): return re_linestart.sub('/// ', s) + # returns true if there is an indication for something that is interpreted as doc comment by rustdoc def has_markdown_codeblock_with_indentation(s): return re_spaces_after_newline.search(s) != None + def preprocess(s): p = subprocess.Popen([os.environ['PREPROC']], close_fds=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) res = p.communicate(s.encode('utf-8')) return res[0].decode('utf-8') + # runs the preprocessor in case there is evidence for code blocks using indentation def rust_doc_sanitize(s): if has_markdown_codeblock_with_indentation(s): @@ -165,36 +180,45 @@ def rust_doc_sanitize(s): else: return s + # rust comment filter def rust_comment(s): return re_linestart.sub('// ', s) + # hash-based comment filter def hash_comment(s): return re_linestart.sub('# ', s) + # hides lines in rust examples, if not already hidden, or empty. def hide_rust_doc_test(s): return re.sub('^[^#\n]', lambda m: '# ' + m.group(), s, flags=re.MULTILINE) + # remove the first indentation (must be spaces !) def unindent(s): return re_first_4_spaces.sub('', s) + # don't do anything with the passed in string def pass_through(s): return s + # tabs: 1 tabs is 4 spaces def unindent_first_by(tabs): def unindent_inner(s): return re_linestart.sub(' ' * tabs * SPACES_PER_TAB, s) + return unindent_inner + # filter to remove empty lines from a string def remove_empty_lines(s): return re.sub("^\n", '', s, flags=re.MULTILINE) + # Prepend prefix to each line but the first def prefix_all_but_first_with(prefix): def indent_inner(s): @@ -204,11 +228,12 @@ def prefix_all_but_first_with(prefix): f = s p = None else: - f = s[:i+1] - p = s[i+1:] + f = s[:i + 1] + p = s[i + 1:] if p is None: return f return f + re_linestart.sub(prefix, p) + return indent_inner @@ -219,48 +244,59 @@ def indent_all_but_first_by(indent, indent_in_tabs=True): spaces = ' ' * indent return prefix_all_but_first_with(spaces) + # add 4 spaces to the beginning of a line. # useful if you have defs embedded in an unindent block - they need to counteract. # It's a bit itchy, but logical def indent(s): return re_linestart.sub(' ' * SPACES_PER_TAB, s) + # indent by given amount of spaces def indent_by(n): def indent_inner(s): return re_linestart.sub(' ' * n, s) + return indent_inner + # return s, with trailing newline def trailing_newline(s): if not s.endswith('\n'): return s + '\n' return s + # a rust test that doesn't run though def rust_doc_test_norun(s): return "```test_harness,no_run\n%s```" % trailing_newline(s) + # a rust code block in (github) markdown def markdown_rust_block(s): return "```Rust\n%s```" % trailing_newline(s) + # wraps s into an invisible doc test function. def rust_test_fn_invisible(s): return "# async fn dox() {\n%s# }" % trailing_newline(s) + # markdown comments def markdown_comment(s): return "" % trailing_newline(s) + # escape each string in l with "s" and return the new list def estr(l): return ['"%s"' % i for i in l] + # escape all '"' with '\"' def escape_rust_string(s): return s.replace('"', '\\"') + ## -- End Filters -- @} # ============================================================================== @@ -276,28 +312,34 @@ def put_and(l): return l[0] return ', '.join(l[:-1]) + ' and ' + l[-1] + # ['foo', ...] with e == '*' -> ['*foo*', ...] def enclose_in(e, l): return ['%s%s%s' % (e, s, e) for s in l] + def md_italic(l): return enclose_in('*', l) + def singular(s): if s.endswith('ies'): - return s[:-3]+'y' + return s[:-3] + 'y' if s[-1] == 's': return s[:-1] return s + def split_camelcase_s(s): s1 = re.sub('(.)([A-Z][a-z]+)', r'\1 \2', s) return re.sub('([a-z0-9])([A-Z])', r'\1 \2', s1).lower() + def camel_to_under(s): s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', s) return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() + # there are property descriptions from which parts can be extracted. Regex is based on youtube ... it's sufficiently # easy enough to add more cases ... # return ['part', ...] or [] @@ -313,6 +355,7 @@ def extract_parts(desc): res.append(part) return res + ## -- End Natural Language Utilities -- @} @@ -324,6 +367,7 @@ def extract_parts(desc): def capitalize(s): return s[:1].upper() + s[1:] + # Return transformed string that could make a good type name def canonical_type_name(s): # can't use s.capitalize() as it will lower-case the remainder of the string @@ -332,10 +376,12 @@ def canonical_type_name(s): s = ''.join(capitalize(t) for t in s.split('-')) return capitalize(s) + def nested_type_name(sn, pn): suffix = canonical_type_name(pn) return sn + suffix + # Make properties which are reserved keywords usable def mangle_ident(n): n = camel_to_under(n).replace('-', '.').replace('.', '_').replace('$', '') @@ -343,33 +389,36 @@ def mangle_ident(n): return n + '_' return n + def is_map_prop(p): return 'additionalProperties' in p + def _assure_unique_type_name(schemas, tn): if tn in schemas: tn += 'Nested' assert tn not in schemas return tn + # map a json type to an rust type # t = type dict # NOTE: In case you don't understand how this algorithm really works ... me neither - THE AUTHOR def to_rust_type( - schemas, - schema_name, - property_name, - t, - allow_optionals=True, - _is_recursive=False -): + schemas, + schema_name, + property_name, + t, + allow_optionals=True, + _is_recursive=False +) -> str: def nested_type(nt): if 'items' in nt: nt = nt['items'] elif 'additionalProperties' in nt: nt = nt['additionalProperties'] else: - assert(is_nested_type_property(nt)) + assert (is_nested_type_property(nt)) # It's a nested type - we take it literally like $ref, but generate a name for the type ourselves return _assure_unique_type_name(schemas, nested_type_name(schema_name, property_name)) return to_rust_type(schemas, schema_name, property_name, nt, allow_optionals=False, _is_recursive=True) @@ -407,18 +456,22 @@ def to_rust_type( return 'Vec<%s>' % rust_type return wrap_type(rust_type) except KeyError as err: - raise AssertionError("%s: Property type '%s' unknown - add new type mapping: %s" % (str(err), t['type'], str(t))) + raise AssertionError( + "%s: Property type '%s' unknown - add new type mapping: %s" % (str(err), t['type'], str(t))) except AttributeError as err: raise AssertionError("%s: unknown dict layout: %s" % (str(err), t)) + # return True if this property is actually a nested type def is_nested_type_property(t): return 'type' in t and t['type'] == 'object' and 'properties' in t or ('items' in t and 'properties' in t['items']) + # Return True if the schema is nested def is_nested_type(s): return len(s.parents) > 0 + # convert a rust-type to something that would be taken as input of a function # even though our storage type is different def activity_input_type(schemas, p): @@ -432,15 +485,16 @@ def activity_input_type(schemas, p): return n return '&%s' % n + def is_pod_property(p): - return 'format' in p or p.get('type','') == 'boolean' + return 'format' in p or p.get('type', '') == 'boolean' def _traverse_schema_ids(s, c): ids = [s.id] used_by = s.used_by + s.parents - seen = set() # protect against loops, just to be sure ... + seen = set() # protect against loops, just to be sure ... while used_by: id = used_by.pop() if id in seen: @@ -454,6 +508,7 @@ def _traverse_schema_ids(s, c): # end gather usages return ids + # Return sorted type names of all markers applicable to the given schema # This list is transitive. Thus, if the schema is used as child of someone with a trait, it # inherits this trait @@ -496,6 +551,7 @@ def schema_markers(s, c, transitive=True): return sorted(res) + ## -- End Rust TypeSystem -- @} # NOTE: unfortunately, it turned out that sometimes fields are missing. The only way to handle this is to @@ -504,6 +560,7 @@ def schema_markers(s, c, transitive=True): def is_schema_with_optionals(schema_markers): return True + # ------------------------- ## @name Activity Utilities # @{ @@ -517,18 +574,22 @@ def activity_split(fqan: str) -> Tuple[str, str, str]: # end return t[0], t[1], '.'.join(mt) + # Shorthand to get a type from parameters of activities def activity_rust_type(schemas, p, allow_optionals=True): return to_rust_type(schemas, None, p.name, p, allow_optionals=allow_optionals) + # the inverse of activity-split, but needs to know the 'name' of the API def to_fqan(name, resource, method): return '%s.%s.%s' % (name, resource, method) + # videos -> Video def activity_name_to_type_name(an): return canonical_type_name(an)[:-1] + # return a list of parameter structures of all params of the given method dict # apply a prune filter to restrict the set of returned parameters. # The order will always be: partOrder + alpha @@ -555,6 +616,7 @@ def _method_params(m, required=None, location=None): # end for each parameter return sorted(res, key=lambda p: (p.priority, p.name), reverse=True) + def _method_io(type_name, c, m, marker=None): s = c.schemas.get(m.get(type_name, dict()).get(TREF)) if s is None: @@ -563,15 +625,18 @@ def _method_io(type_name, c, m, marker=None): return None return s + # return the given method's request or response schema (dict), or None. # optionally return only schemas with the given marker trait def method_request(c, m, marker=None): return _method_io('request', c, m, marker) + # As method request, but returns response instead def method_response(c, m, marker=None): return _method_io('response', c, m, marker) + # return string like 'n.clone()', but depending on the type name of tn (e.g. &str -> n.to_string()) def rust_copy_value_s(n, tn, p): if 'clone_value' in p: @@ -583,23 +648,28 @@ def rust_copy_value_s(n, tn, p): nc = n return nc + # convert a schema into a property (for use with rust type generation). # n = name of the property def schema_to_required_property(s, n): return type(s)({'name': n, TREF: s.id, 'priority': REQUEST_PRIORITY, 'is_query_param': False}) + def is_required_property(p): return p.get('required', False) or p.get('priority', 0) > 0 + def is_repeated_property(p): return p.get('repeated', False) + def setter_fn_name(p): fn_name = p.name if is_repeated_property(p): fn_name = 'add_' + fn_name return fn_name + # _method_params(...), request_value|None -> (required_properties, optional_properties, part_prop|None) def organize_params(params, request_value): part_prop = None @@ -617,6 +687,7 @@ def organize_params(params, request_value): # end for each property return required_props, optional_props, part_prop + # returns method parameters based on whether we can make uploads, and which protocols are supported # or empty list if there is no media upload def method_media_params(m): @@ -631,23 +702,25 @@ def method_media_params(m): res = list() for pn, proto in mu.protocols.items(): # the pi (proto-info) dict can be shown to the user - pi = {'multipart': proto.multipart and 'yes' or 'no', 'maxSize': mu.get('maxSize', '0kb'), 'validMimeTypes': mu.accept} + pi = {'multipart': proto.multipart and 'yes' or 'no', 'maxSize': mu.get('maxSize', '0kb'), + 'validMimeTypes': mu.accept} try: ti = type(m)(PROTOCOL_TYPE_INFO[pn]) except KeyError: raise AssertionError("media upload protocol '%s' is not implemented" % pn) p = type(m)({'name': 'media_%s', - 'info': pi, - 'protocol': pn, - 'path': proto.path, - 'type': ti, - 'description': ti.description, - 'max_size': size_to_bytes(mu.get('maxSize', '0kb'))}) + 'info': pi, + 'protocol': pn, + 'path': proto.path, + 'type': ti, + 'description': ti.description, + 'max_size': size_to_bytes(mu.get('maxSize', '0kb'))}) res.append(p) # end for each proto return res + # Build all parameters used in a given method ! # schemas, context, method(dict), 'request'|'response', request_prop_name -> (params, request_value|None) def build_all_params(c, m): @@ -656,18 +729,18 @@ def build_all_params(c, m): if request_value: params.insert(0, schema_to_required_property(request_value, REQUEST_VALUE_PROPERTY_NAME)) # add the delegate. It's a type parameter, which has to remain in sync with the type-parameters we actually build. - dp = type(m)({ 'name': DELEGATE_PROPERTY_NAME, - TREF: "&'a mut dyn %s" % DELEGATE_TYPE, - 'input_type': "&'a mut dyn %s" % DELEGATE_TYPE, - 'clone_value': '{}', - 'skip_example' : True, - 'priority': 0, - 'is_query_param': False, - 'description': -"""The delegate implementation is consulted whenever there is an intermediate result, or if something goes wrong -while executing the actual API request. - -It should be used to handle progress information, and to implement a certain level of resilience."""}) + dp = type(m)({'name': DELEGATE_PROPERTY_NAME, + TREF: "&'a mut dyn %s" % DELEGATE_TYPE, + 'input_type': "&'a mut dyn %s" % DELEGATE_TYPE, + 'clone_value': '{}', + 'skip_example': True, + 'priority': 0, + 'is_query_param': False, + 'description': + """The delegate implementation is consulted whenever there is an intermediate result, or if something goes wrong + while executing the actual API request. + + It should be used to handle progress information, and to implement a certain level of resilience."""}) params.append(dp) return params, request_value @@ -683,18 +756,20 @@ class Context: rtc_map: Dict[str, Any] schemas: Dict[str, Any] + # return a newly build context from the given data def new_context(schemas: Dict[str, Dict[str, Any]], resources: Dict[str, Any]) -> Context: # Returns (A, B) where # A: { SchemaTypeName -> { fqan -> ['request'|'response', ...]} # B: { fqan -> activity_method_data } # fqan = fully qualified activity name - def build_activity_mappings(resources: Dict[str, Any], res = None, fqan = None) -> Tuple[Dict[str, Any], Dict[str, Any]]: + def build_activity_mappings(resources: Dict[str, Any], res=None, fqan=None) -> Tuple[ + Dict[str, Any], Dict[str, Any]]: if res is None: res = dict() if fqan is None: fqan = dict() - for k,a in resources.items(): + for k, a in resources.items(): if 'resources' in a: build_activity_mappings(a["resources"], res, fqan) if 'methods' not in a: @@ -731,6 +806,7 @@ def new_context(schemas: Dict[str, Dict[str, Any]], resources: Dict[str, Any]) - # end for each method # end for each activity return res, fqan + # end utility # A dict of {s.id -> schema} , with all schemas having the 'parents' key set with [s.id, ...] of all parents @@ -739,12 +815,14 @@ def new_context(schemas: Dict[str, Dict[str, Any]], resources: Dict[str, Any]) - # 'type' in t and t.type == 'object' and 'properties' in t or ('items' in t and 'properties' in t.items) PARENT = 'parents' USED_BY = 'used_by' - def assure_list(s, k): + + def assure_list(s: Dict[str, Any], k: str): if k not in s: s[k] = list() return s[k] + # end - def link_used(s, rs): + def link_used(s: Dict[str, Any], rs): if TREF in s: l = assure_list(all_schemas[s[TREF]], USED_BY) if rs["id"] not in l: @@ -756,6 +834,7 @@ def new_context(schemas: Dict[str, Dict[str, Any]], resources: Dict[str, Any]) - return l all_schemas = deepcopy(schemas) + def recurse_properties(prefix: str, rs: Any, s: Any, parent_ids: List[str]): assure_list(s, USED_BY) assure_list(s, PARENT).extend(parent_ids) @@ -783,18 +862,20 @@ def new_context(schemas: Dict[str, Dict[str, Any]], resources: Dict[str, Any]) - recurse_properties(ns.id, ns, ns, append_unique(parent_ids, rs["id"])) elif is_map_prop(p): recurse_properties(nested_type_name(prefix, pn), rs, - p["additionalProperties"], append_unique(parent_ids, rs["id"])) + p["additionalProperties"], append_unique(parent_ids, rs["id"])) elif 'items' in p: recurse_properties(nested_type_name(prefix, pn), rs, - p["items"], append_unique(parent_ids, rs["id"])) + p["items"], append_unique(parent_ids, rs["id"])) # end handle prop itself # end for each property + # end utility for s in all_schemas.values(): recurse_properties(s["id"], s, s, []) # end for each schema return all_schemas + # end utility all_schemas = schemas and build_schema_map() or dict() @@ -816,9 +897,11 @@ def new_context(schemas: Dict[str, Dict[str, Any]], resources: Dict[str, Any]) - fqan_map.update(_fqan_map) return Context(sta_map, fqan_map, rta_map, rtc_map, all_schemas) + def _is_special_version(v): return v.endswith('alpha') or v.endswith('beta') + def to_api_version(v): m = re.search(r"_?v(\d(\.\d)*)_?", v) if not m and _is_special_version(v): @@ -840,9 +923,11 @@ def to_api_version(v): version = version + '_' + remainder return version + def normalize_library_name(name): return name.lower() + # build a full library name (non-canonical) def library_name(name, version): version = to_api_version(version) @@ -853,39 +938,50 @@ def library_name(name, version): version = 'v' + version return normalize_library_name(name) + version + def target_directory_name(name, version, suffix): return library_name(name, version) + suffix + # return crate name for given result of `library_name()` def library_to_crate_name(name, suffix=''): return 'google-' + name + suffix + # return version like 0.1.0+2014031421 def crate_version(build_version, revision): return '%s+%s' % (build_version, isinstance(revision, str) and revision or '00000000') + # return a crate name for us in extern crate statements def to_extern_crate_name(crate_name): return crate_name.replace('-', '_') + def docs_rs_url(base_url, crate_name, version): return base_url + '/' + crate_name + '/' + version + def crate_name(name, version, make): return library_to_crate_name(library_name(name, version), make.target_suffix) + def gen_crate_dir(name, version, ti): return to_extern_crate_name(library_to_crate_name(library_name(name, version), ti.target_suffix)) + def crates_io_url(name, version): return "https://crates.io/crates/%s" % library_to_crate_name(library_name(name, version)) + def program_name(name, version): return library_name(name, version).replace('_', '-') + def api_json_path(api_base, name, version): return api_base + '/' + name + '/' + version + '/' + name + '-api.json' + def api_index(DOC_ROOT, name, version, ti, cargo, revision, check_exists=True): crate_dir = gen_crate_dir(name, version, ti) if ti.documentation_engine == 'rustdoc': @@ -898,21 +994,26 @@ def api_index(DOC_ROOT, name, version, ti, cargo, revision, check_exists=True): return index_file_path return None + # return type name of a resource method builder, from a resource name def rb_type(r): return "%sMethods" % singular(canonical_type_name(r)) + def _to_type_params_s(p): return '<%s>' % ', '.join(p) + # return type parameters of a the hub, ready for use in Rust code def hub_type_params_s(): return _to_type_params_s(HUB_TYPE_PARAMETERS) + # Returns True if this API has particular authentication scopes to choose from def supports_scopes(auth): return bool(auth) and bool(auth.oauth2) + # Returns th desired scope for the given method. It will use read-only scopes for read-only methods # May be None no scope-based authentication is required def method_default_scope(m): @@ -928,29 +1029,35 @@ def method_default_scope(m): # end try to find read-only default scope return default_scope -_rb_type_params = ("'a", ) + HUB_TYPE_PARAMETERS + +_rb_type_params = ("'a",) + HUB_TYPE_PARAMETERS # type parameters for a resource builder - keeps hub as borrow def rb_type_params_s(resource, c): return _to_type_params_s(_rb_type_params) + # type bounds for resource and method builder def struct_type_bounds_s(): return ', '.join(tp + ": 'a" for tp in HUB_TYPE_PARAMETERS) + # type params for the given method builder, as string suitable for Rust code def mb_type_params_s(m): return _to_type_params_s(_rb_type_params) + # as rb_additional_type_params, but for an individual method, as seen from a resource builder ! def mb_additional_type_params(m): return [] + # return type name for a method on the given resource def mb_type(r, m): return "%s%sCall" % (singular(canonical_type_name(r)), dot_sep_to_canonical_type_name(m)) + # canonicalName = util.canonical_name() def hub_type(schemas, canonicalName): name = canonical_type_name(canonicalName) @@ -958,8 +1065,9 @@ def hub_type(schemas, canonicalName): name += 'Hub' return name + # return e + d[n] + e + ' ' or '' -def get_word(d, n, e = ''): +def get_word(d, n, e=''): if n in d: v = e + d[n] + e if not v.endswith(' '): @@ -968,22 +1076,27 @@ def get_word(d, n, e = ''): else: return '' + # n = 'FooBar' -> _foo_bar def property(n): return '_' + mangle_ident(n) + def upload_action_fn(upload_action_term, suffix): return upload_action_term + suffix + # n = 'foo.bar.Baz' -> 'FooBarBaz' def dot_sep_to_canonical_type_name(n): return ''.join(canonical_type_name(singular(t)) for t in n.split('.')) + def find_fattest_resource(c): fr = None if c.schemas: for candidate in sorted(c.schemas.values(), - key=lambda s: (len(c.sta_map.get(s.id, [])), len(s.get('properties', []))), reverse=True): + key=lambda s: (len(c.sta_map.get(s.id, [])), len(s.get('properties', []))), + reverse=True): if candidate.id in c.sta_map: fr = candidate break @@ -991,6 +1104,7 @@ def find_fattest_resource(c): # end if there are schemas return fr + # Extract valid parts from the description of the parts prop contained within the given parameter list # can be an empty list. def parts_from_params(params): @@ -1004,6 +1118,7 @@ def parts_from_params(params): return part_prop, extract_parts(part_prop.get('description', '')) return part_prop, list() + # Convert a scope url to a nice enum variant identifier, ready for use in code # name = name of the api, without version, non-normalized (!) def scope_url_to_variant(name, url, fully_qualified=True): @@ -1027,6 +1142,7 @@ def scope_url_to_variant(name, url, fully_qualified=True): return fqvn(FULL) return fqvn(dot_sep_to_canonical_type_name(repl(base))) + def method_name_to_variant(name): name = name.upper() fmt = 'hyper::Method.from_str("%s")' @@ -1034,6 +1150,7 @@ def method_name_to_variant(name): fmt = 'hyper::Method::%s' return fmt % name + # given a rust type-name (no optional, as from to_rust_type), you will get a suitable random default value # as string suitable to be passed as reference (or copy, where applicable) def rnd_arg_val_for_type(tn): @@ -1042,6 +1159,7 @@ def rnd_arg_val_for_type(tn): except KeyError: return '&Default::default()' + # Converts a size to the respective integer # size string like 1MB or 2TB, or 35.5KB def size_to_bytes(size): diff --git a/src/generator/templates/api/lib/schema.mako b/src/generator/templates/api/lib/schema.mako index 8a063b33af..25fc6d13f4 100644 --- a/src/generator/templates/api/lib/schema.mako +++ b/src/generator/templates/api/lib/schema.mako @@ -17,6 +17,9 @@ ${struct} { % if pn != mangle_ident(pn): #[serde(rename="${pn}")] % endif + % if p.get("format", None) == "byte": + #[serde(serialize_with = "client::types::to_urlsafe_base64", deserialize_with = "client::types::from_urlsafe_base64")] + % endif pub ${mangle_ident(pn)}: ${to_rust_type(schemas, s.id, pn, p, allow_optionals=allow_optionals)}, % endfor } From fc780014d4e94b7a01c5237feb21999c86474cc1 Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Fri, 7 Oct 2022 13:49:22 -0700 Subject: [PATCH 04/27] Clean up duration parsing code --- google-apis-common/src/lib.rs | 39 +++++++++++++++++------------------ 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/google-apis-common/src/lib.rs b/google-apis-common/src/lib.rs index 80fc5954a6..6cb3f3fcc0 100644 --- a/google-apis-common/src/lib.rs +++ b/google-apis-common/src/lib.rs @@ -888,36 +888,35 @@ pub mod types { fn try_from(value: IntermediateDuration<'a>) -> Result { let abs_duration = 315576000000i64; // TODO: Test strings like -.s, -0.0s - if !value.0.ends_with('s') { - todo!(); - } - let (seconds, nanoseconds) = if let Some((seconds, nanos)) = value.0.split_once('.') { + let value = match value.0.strip_suffix('s') { + None => todo!("Missing 's' suffix case not handled"), + Some(v) => v + }; + + let (seconds, nanoseconds) = if let Some((seconds, nanos)) = value.split_once('.') { + let is_neg = seconds.starts_with("-"); let seconds = i64::from_str(seconds)?; - let nano_len = nanos.len() - 1; // up . 000_000_000 - let nano_digits = nanos.chars().filter(|c| c.is_digit(10)).count() as u32; - if nano_digits > 9 { - todo!() + let nano_magnitude = nanos.chars().filter(|c| c.is_digit(10)).count() as u32; + if nano_magnitude > 9 { + // catches numbers larger than 999_999_999 + todo!("Numeric overflow case not handled") + } + + // u32::from_str prevents negative nanos (eg '0.-12s) -> lossless conversion to i32 + // 10_u32.pow(...) scales number to appropriate # of nanoseconds + let mut nanos = (u32::from_str(&nanos).expect("negative nanos not handled") * 10_u32.pow(9 - nano_magnitude)) as i32; + if is_neg { + nanos = -nanos; } - // 2 digits: 120_000_000 - // 9 digits: 123_456_789 - // pad to 9 digits - let nanos = i32::from_str(&nanos[..nanos.len() - 1])? * 10_i32.pow(9 - nano_digits); (seconds, nanos) } else { - // TODO: handle negative number - (i64::from_str(&value.0[..value.0.len().saturating_sub(1)])?, 0) + (i64::from_str(value), 0) }; - if (seconds > 0 && nanoseconds < 0) || (seconds < 0 && nanoseconds > 0) { - todo!(); - } if seconds >= abs_duration || seconds <= -abs_duration { todo!(); } - if nanoseconds >= 1_000_000_000 || nanoseconds <= -1_000_000_000 { - todo!(); - } Ok(Duration { seconds, nanoseconds }) } From 444b610ddc62af31c9c95e6865b8e58452afb10f Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Fri, 7 Oct 2022 14:06:04 -0700 Subject: [PATCH 05/27] Add proper error handling for parsing Duration --- google-apis-common/src/lib.rs | 57 +++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/google-apis-common/src/lib.rs b/google-apis-common/src/lib.rs index 6cb3f3fcc0..4adca87d51 100644 --- a/google-apis-common/src/lib.rs +++ b/google-apis-common/src/lib.rs @@ -415,7 +415,7 @@ impl<'a> Read for MultiPartReader<'a> { (n, true, _) if n > 0 => { let (headers, reader) = self.raw_parts.remove(0); let mut c = Cursor::new(Vec::::new()); - // TODO: The first line ending should be omitted for the first part, + //TODO: The first line ending should be omitted for the first part, // fortunately Google's API serves don't seem to mind. (write!( &mut c, @@ -845,6 +845,7 @@ mod yup_oauth2_impl { } pub mod types { + use std::fmt::Formatter; use std::str::FromStr; use serde::{Deserialize, Deserializer, Serializer}; // https://github.com/protocolbuffers/protobuf-go/blob/6875c3d7242d1a3db910ce8a504f124cb840c23a/types/known/durationpb/duration.pb.go#L148 @@ -882,43 +883,75 @@ pub mod types { } } + #[derive(Debug)] + enum ParseDurationError { + MissingSecondSuffix, + NanosTooSmall, + ParseIntError(std::num::ParseIntError), + SecondOverflow { seconds: i64, max_seconds: i64 }, + SecondUnderflow { seconds: i64, min_seconds: i64 } + } + + impl From for ParseDurationError { + fn from(pie: std::num::ParseIntError) -> Self { + ParseDurationError::ParseIntError(pie) + } + } + + impl std::fmt::Display for ParseDurationError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + ParseDurationError::MissingSecondSuffix => write!(f, "'s' suffix was not present"), + ParseDurationError::NanosTooSmall => write!(f, "more than 9 digits of second precision required"), + ParseDurationError::ParseIntError(pie) => write!(f, "{}", pie), + ParseDurationError::SecondOverflow { seconds, max_seconds } => write!(f, "seconds overflow (got {}, maximum seconds possible {})", seconds, max_seconds), + ParseDurationError::SecondUnderflow { seconds, min_seconds } => write!(f, "seconds underflow (got {}, minimum seconds possible {})", seconds, min_seconds) + } + } + } + + impl std::error::Error for ParseDurationError {} + impl <'a> TryFrom> for Duration { - type Error = std::num::ParseIntError; + type Error = ParseDurationError; fn try_from(value: IntermediateDuration<'a>) -> Result { let abs_duration = 315576000000i64; // TODO: Test strings like -.s, -0.0s let value = match value.0.strip_suffix('s') { - None => todo!("Missing 's' suffix case not handled"), + None => return Err(ParseDurationError::MissingSecondSuffix), Some(v) => v }; let (seconds, nanoseconds) = if let Some((seconds, nanos)) = value.split_once('.') { let is_neg = seconds.starts_with("-"); let seconds = i64::from_str(seconds)?; - // up . 000_000_000 let nano_magnitude = nanos.chars().filter(|c| c.is_digit(10)).count() as u32; if nano_magnitude > 9 { - // catches numbers larger than 999_999_999 - todo!("Numeric overflow case not handled") + // not enough precision to model the remaining digits + return Err(ParseDurationError::NanosTooSmall); } // u32::from_str prevents negative nanos (eg '0.-12s) -> lossless conversion to i32 // 10_u32.pow(...) scales number to appropriate # of nanoseconds - let mut nanos = (u32::from_str(&nanos).expect("negative nanos not handled") * 10_u32.pow(9 - nano_magnitude)) as i32; + let nanos = u32::from_str(nanos)? as i32; + + let mut nanos = nanos * 10_i32.pow(9 - nano_magnitude); if is_neg { nanos = -nanos; } (seconds, nanos) } else { - (i64::from_str(value), 0) + (i64::from_str(value)?, 0) }; - if seconds >= abs_duration || seconds <= -abs_duration { - todo!(); + if seconds >= abs_duration { + Err(ParseDurationError::SecondOverflow { seconds, max_seconds: abs_duration }) + } else if seconds <= -abs_duration { + Err(ParseDurationError::SecondUnderflow { seconds, min_seconds: -abs_duration }) + } else { + Ok(Duration { seconds, nanoseconds}) } - - Ok(Duration { seconds, nanoseconds }) } } From 05df68de324561773b9867f143bf7dbb4381f603 Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Fri, 7 Oct 2022 20:34:40 -0700 Subject: [PATCH 06/27] Use chrono::Duration directly with serde attributes --- google-apis-common/src/lib.rs | 146 ++++++++++---------- src/generator/templates/api/lib/schema.mako | 2 + 2 files changed, 73 insertions(+), 75 deletions(-) diff --git a/google-apis-common/src/lib.rs b/google-apis-common/src/lib.rs index 4adca87d51..2a7115a83a 100644 --- a/google-apis-common/src/lib.rs +++ b/google-apis-common/src/lib.rs @@ -849,39 +849,6 @@ pub mod types { use std::str::FromStr; use serde::{Deserialize, Deserializer, Serializer}; // https://github.com/protocolbuffers/protobuf-go/blob/6875c3d7242d1a3db910ce8a504f124cb840c23a/types/known/durationpb/duration.pb.go#L148 - #[derive(Deserialize)] - #[serde(try_from = "IntermediateDuration")] - pub struct Duration { - pub seconds: i64, - pub nanoseconds: i32, - } - - impl From for chrono::Duration { - fn from(duration: Duration) -> chrono::Duration { - chrono::Duration::seconds(duration.seconds) + chrono::Duration::nanoseconds(duration.nanoseconds as i64) - } - } - - #[derive(Deserialize)] - struct IntermediateDuration<'a>(&'a str); - - - impl serde::Serialize for Duration { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - if self.nanoseconds != 0 { - if self.seconds == 0 && self.nanoseconds.is_negative() { - serializer.serialize_str(&format!("-0.{}s", self.nanoseconds.abs())) - } else { - serializer.serialize_str(&format!("{}.{}s", self.seconds, self.nanoseconds.abs())) - } - } else { - serializer.serialize_str(&format!("{}s", self.seconds)) - } - } - } #[derive(Debug)] enum ParseDurationError { @@ -892,6 +859,45 @@ pub mod types { SecondUnderflow { seconds: i64, min_seconds: i64 } } + fn parse_duration_from_str(s: &str) -> Result { + let abs_duration = 315576000000i64; + // TODO: Test strings like -.s, -0.0s + let value = match s.strip_suffix('s') { + None => return Err(ParseDurationError::MissingSecondSuffix), + Some(v) => v + }; + + let (seconds, nanoseconds) = if let Some((seconds, nanos)) = value.split_once('.') { + let is_neg = seconds.starts_with("-"); + let seconds = i64::from_str(seconds)?; + let nano_magnitude = nanos.chars().filter(|c| c.is_digit(10)).count() as u32; + if nano_magnitude > 9 { + // not enough precision to model the remaining digits + return Err(ParseDurationError::NanosTooSmall); + } + + // u32::from_str prevents negative nanos (eg '0.-12s) -> lossless conversion to i32 + // 10_u32.pow(...) scales number to appropriate # of nanoseconds + let nanos = u32::from_str(nanos)? as i32; + + let mut nanos = nanos * 10_i32.pow(9 - nano_magnitude); + if is_neg { + nanos = -nanos; + } + (seconds, nanos) + } else { + (i64::from_str(value)?, 0) + }; + + if seconds >= abs_duration { + Err(ParseDurationError::SecondOverflow { seconds, max_seconds: abs_duration }) + } else if seconds <= -abs_duration { + Err(ParseDurationError::SecondUnderflow { seconds, min_seconds: -abs_duration }) + } else { + Ok(chrono::Duration::seconds(seconds) + chrono::Duration::nanoseconds(nanoseconds.into())) + } + } + impl From for ParseDurationError { fn from(pie: std::num::ParseIntError) -> Self { ParseDurationError::ParseIntError(pie) @@ -903,7 +909,7 @@ pub mod types { match self { ParseDurationError::MissingSecondSuffix => write!(f, "'s' suffix was not present"), ParseDurationError::NanosTooSmall => write!(f, "more than 9 digits of second precision required"), - ParseDurationError::ParseIntError(pie) => write!(f, "{}", pie), + ParseDurationError::ParseIntError(pie) => write!(f, "{:?}", pie), ParseDurationError::SecondOverflow { seconds, max_seconds } => write!(f, "seconds overflow (got {}, maximum seconds possible {})", seconds, max_seconds), ParseDurationError::SecondUnderflow { seconds, min_seconds } => write!(f, "seconds underflow (got {}, minimum seconds possible {})", seconds, min_seconds) } @@ -912,52 +918,41 @@ pub mod types { impl std::error::Error for ParseDurationError {} - impl <'a> TryFrom> for Duration { - type Error = ParseDurationError; - - fn try_from(value: IntermediateDuration<'a>) -> Result { - let abs_duration = 315576000000i64; - // TODO: Test strings like -.s, -0.0s - let value = match value.0.strip_suffix('s') { - None => return Err(ParseDurationError::MissingSecondSuffix), - Some(v) => v - }; - - let (seconds, nanoseconds) = if let Some((seconds, nanos)) = value.split_once('.') { - let is_neg = seconds.starts_with("-"); - let seconds = i64::from_str(seconds)?; - let nano_magnitude = nanos.chars().filter(|c| c.is_digit(10)).count() as u32; - if nano_magnitude > 9 { - // not enough precision to model the remaining digits - return Err(ParseDurationError::NanosTooSmall); + pub fn to_duration_str(x: Option<&chrono::Duration>, s: S) -> Result + where + S: Serializer, + { + match x { + None => s.serialize_none(), + Some(x) => { + let seconds = x.num_seconds(); + let nanoseconds = (*x - chrono::Duration::seconds(seconds)) + .num_nanoseconds() + .expect("number of nanoseconds is less than or equal to 1 billion") as i32; + // might be left with -1 + non-zero nanos + if nanoseconds != 0 { + if seconds == 0 && nanoseconds.is_negative() { + s.serialize_str(&format!("-0.{}s", nanoseconds.abs())) + } else { + s.serialize_str(&format!("{}.{}s", seconds, nanoseconds.abs())) + } + } else { + s.serialize_str(&format!("{}s", seconds)) } - - // u32::from_str prevents negative nanos (eg '0.-12s) -> lossless conversion to i32 - // 10_u32.pow(...) scales number to appropriate # of nanoseconds - let nanos = u32::from_str(nanos)? as i32; - - let mut nanos = nanos * 10_i32.pow(9 - nano_magnitude); - if is_neg { - nanos = -nanos; - } - (seconds, nanos) - } else { - (i64::from_str(value)?, 0) - }; - - if seconds >= abs_duration { - Err(ParseDurationError::SecondOverflow { seconds, max_seconds: abs_duration }) - } else if seconds <= -abs_duration { - Err(ParseDurationError::SecondUnderflow { seconds, min_seconds: -abs_duration }) - } else { - Ok(Duration { seconds, nanoseconds}) } } } + // #[serde(deserialize_with = "path")] + pub fn from_duration_str<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let s: Option<&str> = Deserialize::deserialize(deserializer)?; + // TODO: Map error + Ok(s.map(|s| parse_duration_from_str(s).unwrap())) + } - - // #[serde(serialize_with = "path")] - pub fn to_urlsafe_base64(x: Option<&str>, s: S) -> Result + pub fn to_urlsafe_base64(x: Option<&[u8]>, s: S) -> Result where S: Serializer, { @@ -966,6 +961,7 @@ pub mod types { Some(x) => s.serialize_some(&base64::encode_config(x, base64::URL_SAFE)) } } + // #[serde(deserialize_with = "path")] pub fn from_urlsafe_base64<'de, D>(deserializer: D) -> Result>, D::Error> where diff --git a/src/generator/templates/api/lib/schema.mako b/src/generator/templates/api/lib/schema.mako index 25fc6d13f4..55590f8a76 100644 --- a/src/generator/templates/api/lib/schema.mako +++ b/src/generator/templates/api/lib/schema.mako @@ -19,6 +19,8 @@ ${struct} { % endif % if p.get("format", None) == "byte": #[serde(serialize_with = "client::types::to_urlsafe_base64", deserialize_with = "client::types::from_urlsafe_base64")] + % elif p.get("format", None) == "google-duration": + #[serde(serialize_with = "client::types::to_duration_str", deserialize_with = "client::types::from_duration_str")] % endif pub ${mangle_ident(pn)}: ${to_rust_type(schemas, s.id, pn, p, allow_optionals=allow_optionals)}, % endfor From 6ced748cb1d4729688423d06f21da015b43aeba4 Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Fri, 7 Oct 2022 20:38:41 -0700 Subject: [PATCH 07/27] Fix tested type --- src/generator/lib/__tests__/util_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/generator/lib/__tests__/util_test.py b/src/generator/lib/__tests__/util_test.py index 9e1393aefe..eb34be7e1a 100644 --- a/src/generator/lib/__tests__/util_test.py +++ b/src/generator/lib/__tests__/util_test.py @@ -82,7 +82,7 @@ class UtilsTest(unittest.TestCase): test_properties = ( ('Album', 'title', 'String'), # string ('Status', 'code', 'i32'), # numeric - ('Album', 'mediaItemsCount', 'String'), # numeric via "count" keyword + ('Album', 'mediaItemsCount', 'i64'), # numeric via "count" keyword ('Album', 'isWriteable', 'bool'), # boolean ('Album', 'shareInfo', 'ShareInfo'), # reference type ('SearchMediaItemsResponse', 'mediaItems', 'Vec'), # array @@ -90,7 +90,7 @@ class UtilsTest(unittest.TestCase): for (class_name, property_name, expected) in test_properties: property_value = schemas[class_name]['properties'][property_name] rust_type = to_rust_type(schemas, class_name, property_name, property_value, allow_optionals=False) - self.assertEqual(rust_type, expected) + self.assertEqual(rust_type, expected, f"Parsed class: {class_name}, property: {property_name}") # items reference class_name = 'SearchMediaItemsResponse' From 477be5d76c1f33a1df6796684b4a29b59bd0ccfd Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Fri, 7 Oct 2022 20:40:47 -0700 Subject: [PATCH 08/27] Fix type signatures --- google-apis-common/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-apis-common/src/lib.rs b/google-apis-common/src/lib.rs index 2a7115a83a..5396398932 100644 --- a/google-apis-common/src/lib.rs +++ b/google-apis-common/src/lib.rs @@ -918,7 +918,7 @@ pub mod types { impl std::error::Error for ParseDurationError {} - pub fn to_duration_str(x: Option<&chrono::Duration>, s: S) -> Result + pub fn to_duration_str(x: &Option, s: S) -> Result where S: Serializer, { @@ -952,7 +952,7 @@ pub mod types { Ok(s.map(|s| parse_duration_from_str(s).unwrap())) } - pub fn to_urlsafe_base64(x: Option<&[u8]>, s: S) -> Result + pub fn to_urlsafe_base64(x: &Option>, s: S) -> Result where S: Serializer, { From 44882a3c449181ce9e6184fec3e00c8e6ee27bd6 Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Fri, 7 Oct 2022 20:44:31 -0700 Subject: [PATCH 09/27] use chrono::Duration instead of custom client type --- src/generator/lib/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/generator/lib/util.py b/src/generator/lib/util.py index 5fa6bff15c..2508797d88 100644 --- a/src/generator/lib/util.py +++ b/src/generator/lib/util.py @@ -41,7 +41,7 @@ TYPE_MAP = { 'date-time': CHRONO_DATETIME, 'date': CHRONO_DATETIME, # custom impl - 'google-duration': 'client::types::Duration', + 'google-duration': 'client::chrono::Duration', # guessing bytes is universally url-safe b64 "byte": "Vec", # TODO: Provide support for these as well From 1f10077e445cf990d86689c0024ead671c02c92a Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Fri, 7 Oct 2022 20:59:51 -0700 Subject: [PATCH 10/27] Find sources for appropriate types --- src/generator/lib/util.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/generator/lib/util.py b/src/generator/lib/util.py index 2508797d88..50c1306a87 100644 --- a/src/generator/lib/util.py +++ b/src/generator/lib/util.py @@ -19,7 +19,9 @@ re_desc_parts = re.compile( re_find_replacements = re.compile(r"\{[/\+]?\w+\*?\}") HTTP_METHODS = set(("OPTIONS", "GET", "POST", "PUT", "DELETE", "HEAD", "TRACE", "CONNECT", "PATCH")) -CHRONO_DATETIME = 'client::chrono::DateTime' +CHRONO_PATH = "client::chrono" +CHRONO_DATETIME = f"{CHRONO_PATH}::DateTime<{CHRONO_PATH}::offset::FixedOffset>" +CHRONO_DATE = f"{CHRONO_PATH}::NaiveDate" USE_FORMAT = 'use_format_field' TYPE_MAP = { 'boolean': 'bool', @@ -35,13 +37,16 @@ TYPE_MAP = { 'array': 'Vec', 'string': 'String', 'object': 'HashMap', - # should be correct + # https://developers.google.com/protocol-buffers/docs/reference/java/com/google/protobuf/Timestamp + # In JSON format, the Timestamp type is encoded as a string in the [RFC 3339] 'google-datetime': CHRONO_DATETIME, - # assumption + # RFC 3339 date-time value 'date-time': CHRONO_DATETIME, - 'date': CHRONO_DATETIME, - # custom impl - 'google-duration': 'client::chrono::Duration', + # A date in RFC 3339 format with only the date part + # e.g. "2013-01-15" + 'date': CHRONO_DATE, + # custom serde impl - {seconds}.{nanoseconds}s + 'google-duration': f"{CHRONO_PATH}::Duration", # guessing bytes is universally url-safe b64 "byte": "Vec", # TODO: Provide support for these as well From 66db5c892d3eca733ec6ec797b62fef29a31a455 Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Fri, 7 Oct 2022 21:27:13 -0700 Subject: [PATCH 11/27] Use type constructors for examples --- src/generator/lib/util.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/generator/lib/util.py b/src/generator/lib/util.py index 50c1306a87..208f72f2ff 100644 --- a/src/generator/lib/util.py +++ b/src/generator/lib/util.py @@ -20,7 +20,7 @@ re_find_replacements = re.compile(r"\{[/\+]?\w+\*?\}") HTTP_METHODS = set(("OPTIONS", "GET", "POST", "PUT", "DELETE", "HEAD", "TRACE", "CONNECT", "PATCH")) CHRONO_PATH = "client::chrono" -CHRONO_DATETIME = f"{CHRONO_PATH}::DateTime<{CHRONO_PATH}::offset::FixedOffset>" +CHRONO_DATETIME = f"{CHRONO_PATH}::NaiveDateTime" CHRONO_DATE = f"{CHRONO_PATH}::NaiveDate" USE_FORMAT = 'use_format_field' TYPE_MAP = { @@ -38,7 +38,7 @@ TYPE_MAP = { 'string': 'String', 'object': 'HashMap', # https://developers.google.com/protocol-buffers/docs/reference/java/com/google/protobuf/Timestamp - # In JSON format, the Timestamp type is encoded as a string in the [RFC 3339] + # In JSON format, the Timestamp type is encoded as a string in the [RFC 3339] format 'google-datetime': CHRONO_DATETIME, # RFC 3339 date-time value 'date-time': CHRONO_DATETIME, @@ -62,6 +62,12 @@ RESERVED_WORDS = set(('abstract', 'alignof', 'as', 'become', 'box', 'break', 'co words = [w.strip(',') for w in "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.".split( ' ')] + + +def chrono_date(): + return f"{CHRONO_DATE}::from_ymd({randint(1, 9999)}, {randint(1, 12)}, {randint(1, 31)})" + + RUST_TYPE_RND_MAP = { 'bool': lambda: str(bool(randint(0, 1))).lower(), 'u32': lambda: randint(0, 100), @@ -73,6 +79,15 @@ RUST_TYPE_RND_MAP = { 'String': lambda: '"%s"' % choice(words), '&str': lambda: '"%s"' % choice(words), '&Vec': lambda: '&vec!["%s".into()]' % choice(words), + "Vec": lambda: f"vec![0, 1, 2, 3]", + "&Vec": lambda: f"&vec![0, 1, 2, 3]", + # TODO: styling this + f"{CHRONO_PATH}::Duration": lambda: f"{CHRONO_PATH}::Duration::seconds({randint(0, 9999999)})", + CHRONO_DATE: chrono_date, + CHRONO_DATETIME: lambda: f"{chrono_date()}.with_hms({randint(0, 23)}, {randint(0, 59)}, {randint(0, 59)})", + f"&{CHRONO_PATH}::Duration": lambda: f"&{CHRONO_PATH}::Duration::seconds({randint(0, 9999999)})", + f"&{CHRONO_DATE}": lambda: f"&{chrono_date()}", + f"&{CHRONO_DATETIME}": lambda: f"&{chrono_date()}.with_hms({randint(0, 23)}, {randint(0, 59)}, {randint(0, 59)})" # why a reference to Vec? Because it works. Should be slice, but who knows how typing works here. } TREF = '$ref' From 158e52399e649955c8b0e86a44e24caf9d224898 Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Fri, 7 Oct 2022 21:35:05 -0700 Subject: [PATCH 12/27] Use UTC DateTime --- src/generator/lib/util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/generator/lib/util.py b/src/generator/lib/util.py index 208f72f2ff..604bd9f046 100644 --- a/src/generator/lib/util.py +++ b/src/generator/lib/util.py @@ -20,7 +20,7 @@ re_find_replacements = re.compile(r"\{[/\+]?\w+\*?\}") HTTP_METHODS = set(("OPTIONS", "GET", "POST", "PUT", "DELETE", "HEAD", "TRACE", "CONNECT", "PATCH")) CHRONO_PATH = "client::chrono" -CHRONO_DATETIME = f"{CHRONO_PATH}::NaiveDateTime" +CHRONO_DATETIME = f"{CHRONO_PATH}::DateTime<{CHRONO_PATH}::offset::Utc>" CHRONO_DATE = f"{CHRONO_PATH}::NaiveDate" USE_FORMAT = 'use_format_field' TYPE_MAP = { @@ -84,10 +84,10 @@ RUST_TYPE_RND_MAP = { # TODO: styling this f"{CHRONO_PATH}::Duration": lambda: f"{CHRONO_PATH}::Duration::seconds({randint(0, 9999999)})", CHRONO_DATE: chrono_date, - CHRONO_DATETIME: lambda: f"{chrono_date()}.with_hms({randint(0, 23)}, {randint(0, 59)}, {randint(0, 59)})", + CHRONO_DATETIME: lambda: f"{CHRONO_PATH}::Utc::now()", f"&{CHRONO_PATH}::Duration": lambda: f"&{CHRONO_PATH}::Duration::seconds({randint(0, 9999999)})", f"&{CHRONO_DATE}": lambda: f"&{chrono_date()}", - f"&{CHRONO_DATETIME}": lambda: f"&{chrono_date()}.with_hms({randint(0, 23)}, {randint(0, 59)}, {randint(0, 59)})" + f"&{CHRONO_DATETIME}": lambda: f"&{CHRONO_PATH}::Utc::now()", # why a reference to Vec? Because it works. Should be slice, but who knows how typing works here. } TREF = '$ref' From 23dd5d7c24dd80de2902c569a8f73efa6ae30e94 Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Fri, 7 Oct 2022 21:46:03 -0700 Subject: [PATCH 13/27] chrono example types --- src/generator/lib/util.py | 10 +++++----- src/generator/templates/api/lib.rs.mako | 2 +- src/generator/templates/api/lib/lib.mako | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/generator/lib/util.py b/src/generator/lib/util.py index 604bd9f046..0827bf1bf0 100644 --- a/src/generator/lib/util.py +++ b/src/generator/lib/util.py @@ -65,7 +65,7 @@ words = [w.strip(',') for w in def chrono_date(): - return f"{CHRONO_DATE}::from_ymd({randint(1, 9999)}, {randint(1, 12)}, {randint(1, 31)})" + return f"chrono::NaiveDate::from_ymd({randint(1, 9999)}, {randint(1, 12)}, {randint(1, 31)})" RUST_TYPE_RND_MAP = { @@ -82,12 +82,12 @@ RUST_TYPE_RND_MAP = { "Vec": lambda: f"vec![0, 1, 2, 3]", "&Vec": lambda: f"&vec![0, 1, 2, 3]", # TODO: styling this - f"{CHRONO_PATH}::Duration": lambda: f"{CHRONO_PATH}::Duration::seconds({randint(0, 9999999)})", + f"{CHRONO_PATH}::Duration": lambda: f"chrono::Duration::seconds({randint(0, 9999999)})", CHRONO_DATE: chrono_date, - CHRONO_DATETIME: lambda: f"{CHRONO_PATH}::Utc::now()", - f"&{CHRONO_PATH}::Duration": lambda: f"&{CHRONO_PATH}::Duration::seconds({randint(0, 9999999)})", + CHRONO_DATETIME: lambda: f"chrono::Utc::now()", + f"&{CHRONO_PATH}::Duration": lambda: f"&chrono::Duration::seconds({randint(0, 9999999)})", f"&{CHRONO_DATE}": lambda: f"&{chrono_date()}", - f"&{CHRONO_DATETIME}": lambda: f"&{CHRONO_PATH}::Utc::now()", + f"&{CHRONO_DATETIME}": lambda: f"&chrono::Utc::now()", # why a reference to Vec? Because it works. Should be slice, but who knows how typing works here. } TREF = '$ref' diff --git a/src/generator/templates/api/lib.rs.mako b/src/generator/templates/api/lib.rs.mako index 1bf1ef0faa..a3adabaa6c 100644 --- a/src/generator/templates/api/lib.rs.mako +++ b/src/generator/templates/api/lib.rs.mako @@ -44,7 +44,7 @@ ${lib.docs(c)} pub use hyper; pub use hyper_rustls; pub extern crate google_apis_common as client; - +pub use client::chrono; pub mod api; // Re-export the hub type and some basic client structs diff --git a/src/generator/templates/api/lib/lib.mako b/src/generator/templates/api/lib/lib.mako index 1faed8fcd4..3d5ccd98cd 100644 --- a/src/generator/templates/api/lib/lib.mako +++ b/src/generator/templates/api/lib/lib.mako @@ -243,7 +243,7 @@ Arguments will always be copied or cloned into the builder, to make them indepen ############################################################################################### <%def name="test_hub(hub_type, comments=True)">\ use std::default::Default; -use ${util.library_name()}::{${hub_type}, oauth2, hyper, hyper_rustls}; +use ${util.library_name()}::{${hub_type}, oauth2, hyper, hyper_rustls, chrono}; % if comments: // Get an ApplicationSecret instance by some means. It contains the `client_id` and From 5398dc6f7945ee08071c8131ed3fb02163342a60 Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Sat, 8 Oct 2022 00:51:59 -0700 Subject: [PATCH 14/27] refactor serde functionality into separate module --- google-apis-common/Cargo.toml | 3 - google-apis-common/src/lib.rs | 136 +------------------ google-apis-common/src/serde.rs | 140 ++++++++++++++++++++ src/generator/templates/api/lib/schema.mako | 4 +- 4 files changed, 145 insertions(+), 138 deletions(-) create mode 100644 google-apis-common/src/serde.rs diff --git a/google-apis-common/Cargo.toml b/google-apis-common/Cargo.toml index 4bbe0e0b40..ac27adbe23 100644 --- a/google-apis-common/Cargo.toml +++ b/google-apis-common/Cargo.toml @@ -29,6 +29,3 @@ hyper = "^ 0.14" http = "^0.2" tokio = "^1.0" tower-service = "^0.3.1" - -[dev-dependencies] -serde = { version = "^ 1.0", features = ["derive"] } diff --git a/google-apis-common/src/lib.rs b/google-apis-common/src/lib.rs index 5396398932..cbd7bb20a5 100644 --- a/google-apis-common/src/lib.rs +++ b/google-apis-common/src/lib.rs @@ -1,3 +1,5 @@ +pub mod serde; + use std::error; use std::error::Error as StdError; use std::fmt::{self, Display}; @@ -844,138 +846,6 @@ mod yup_oauth2_impl { } } -pub mod types { - use std::fmt::Formatter; - use std::str::FromStr; - use serde::{Deserialize, Deserializer, Serializer}; - // https://github.com/protocolbuffers/protobuf-go/blob/6875c3d7242d1a3db910ce8a504f124cb840c23a/types/known/durationpb/duration.pb.go#L148 - - #[derive(Debug)] - enum ParseDurationError { - MissingSecondSuffix, - NanosTooSmall, - ParseIntError(std::num::ParseIntError), - SecondOverflow { seconds: i64, max_seconds: i64 }, - SecondUnderflow { seconds: i64, min_seconds: i64 } - } - - fn parse_duration_from_str(s: &str) -> Result { - let abs_duration = 315576000000i64; - // TODO: Test strings like -.s, -0.0s - let value = match s.strip_suffix('s') { - None => return Err(ParseDurationError::MissingSecondSuffix), - Some(v) => v - }; - - let (seconds, nanoseconds) = if let Some((seconds, nanos)) = value.split_once('.') { - let is_neg = seconds.starts_with("-"); - let seconds = i64::from_str(seconds)?; - let nano_magnitude = nanos.chars().filter(|c| c.is_digit(10)).count() as u32; - if nano_magnitude > 9 { - // not enough precision to model the remaining digits - return Err(ParseDurationError::NanosTooSmall); - } - - // u32::from_str prevents negative nanos (eg '0.-12s) -> lossless conversion to i32 - // 10_u32.pow(...) scales number to appropriate # of nanoseconds - let nanos = u32::from_str(nanos)? as i32; - - let mut nanos = nanos * 10_i32.pow(9 - nano_magnitude); - if is_neg { - nanos = -nanos; - } - (seconds, nanos) - } else { - (i64::from_str(value)?, 0) - }; - - if seconds >= abs_duration { - Err(ParseDurationError::SecondOverflow { seconds, max_seconds: abs_duration }) - } else if seconds <= -abs_duration { - Err(ParseDurationError::SecondUnderflow { seconds, min_seconds: -abs_duration }) - } else { - Ok(chrono::Duration::seconds(seconds) + chrono::Duration::nanoseconds(nanoseconds.into())) - } - } - - impl From for ParseDurationError { - fn from(pie: std::num::ParseIntError) -> Self { - ParseDurationError::ParseIntError(pie) - } - } - - impl std::fmt::Display for ParseDurationError { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - ParseDurationError::MissingSecondSuffix => write!(f, "'s' suffix was not present"), - ParseDurationError::NanosTooSmall => write!(f, "more than 9 digits of second precision required"), - ParseDurationError::ParseIntError(pie) => write!(f, "{:?}", pie), - ParseDurationError::SecondOverflow { seconds, max_seconds } => write!(f, "seconds overflow (got {}, maximum seconds possible {})", seconds, max_seconds), - ParseDurationError::SecondUnderflow { seconds, min_seconds } => write!(f, "seconds underflow (got {}, minimum seconds possible {})", seconds, min_seconds) - } - } - } - - impl std::error::Error for ParseDurationError {} - - pub fn to_duration_str(x: &Option, s: S) -> Result - where - S: Serializer, - { - match x { - None => s.serialize_none(), - Some(x) => { - let seconds = x.num_seconds(); - let nanoseconds = (*x - chrono::Duration::seconds(seconds)) - .num_nanoseconds() - .expect("number of nanoseconds is less than or equal to 1 billion") as i32; - // might be left with -1 + non-zero nanos - if nanoseconds != 0 { - if seconds == 0 && nanoseconds.is_negative() { - s.serialize_str(&format!("-0.{}s", nanoseconds.abs())) - } else { - s.serialize_str(&format!("{}.{}s", seconds, nanoseconds.abs())) - } - } else { - s.serialize_str(&format!("{}s", seconds)) - } - } - } - } - // #[serde(deserialize_with = "path")] - pub fn from_duration_str<'de, D>(deserializer: D) -> Result, D::Error> - where - D: Deserializer<'de>, - { - let s: Option<&str> = Deserialize::deserialize(deserializer)?; - // TODO: Map error - Ok(s.map(|s| parse_duration_from_str(s).unwrap())) - } - - pub fn to_urlsafe_base64(x: &Option>, s: S) -> Result - where - S: Serializer, - { - match x { - None => s.serialize_none(), - Some(x) => s.serialize_some(&base64::encode_config(x, base64::URL_SAFE)) - } - } - - // #[serde(deserialize_with = "path")] - pub fn from_urlsafe_base64<'de, D>(deserializer: D) -> Result>, D::Error> - where - D: Deserializer<'de>, - { - let s: Option<&str> = Deserialize::deserialize(deserializer)?; - // TODO: Map error - Ok(s.map(|s| base64::decode_config(s, base64::URL_SAFE).unwrap())) - } - - // TODO: https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/field-mask - // "google-fieldmask", -} - #[cfg(test)] mod test_api { use super::*; @@ -983,7 +853,7 @@ mod test_api { use std::str::FromStr; use serde_json as json; - use serde::{Serialize, Deserialize}; + use ::serde::{Serialize, Deserialize}; #[test] fn serde() { diff --git a/google-apis-common/src/serde.rs b/google-apis-common/src/serde.rs new file mode 100644 index 0000000000..4208ba0140 --- /dev/null +++ b/google-apis-common/src/serde.rs @@ -0,0 +1,140 @@ +pub mod duration { + use std::fmt::Formatter; + use std::str::FromStr; + use serde::{Deserialize, Deserializer, Serializer}; + + #[derive(Debug)] + enum ParseDurationError { + MissingSecondSuffix, + NanosTooSmall, + ParseIntError(std::num::ParseIntError), + SecondOverflow { seconds: i64, max_seconds: i64 }, + SecondUnderflow { seconds: i64, min_seconds: i64 } + } + + impl From for ParseDurationError { + fn from(pie: std::num::ParseIntError) -> Self { + ParseDurationError::ParseIntError(pie) + } + } + + impl std::fmt::Display for ParseDurationError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + ParseDurationError::MissingSecondSuffix => write!(f, "'s' suffix was not present"), + ParseDurationError::NanosTooSmall => write!(f, "more than 9 digits of second precision required"), + ParseDurationError::ParseIntError(pie) => write!(f, "{:?}", pie), + ParseDurationError::SecondOverflow { seconds, max_seconds } => write!(f, "seconds overflow (got {}, maximum seconds possible {})", seconds, max_seconds), + ParseDurationError::SecondUnderflow { seconds, min_seconds } => write!(f, "seconds underflow (got {}, minimum seconds possible {})", seconds, min_seconds) + } + } + } + + impl std::error::Error for ParseDurationError {} + + fn parse_duration(s: &str) -> Result { + let abs_duration = 315576000000i64; + // TODO: Test strings like -.s, -0.0s + let value = match s.strip_suffix('s') { + None => return Err(ParseDurationError::MissingSecondSuffix), + Some(v) => v + }; + + let (seconds, nanoseconds) = if let Some((seconds, nanos)) = value.split_once('.') { + let is_neg = seconds.starts_with("-"); + let seconds = i64::from_str(seconds)?; + let nano_magnitude = nanos.chars().filter(|c| c.is_digit(10)).count() as u32; + if nano_magnitude > 9 { + // not enough precision to model the remaining digits + return Err(ParseDurationError::NanosTooSmall); + } + + // u32::from_str prevents negative nanos (eg '0.-12s) -> lossless conversion to i32 + // 10_u32.pow(...) scales number to appropriate # of nanoseconds + let nanos = u32::from_str(nanos)? as i32; + + let mut nanos = nanos * 10_i32.pow(9 - nano_magnitude); + if is_neg { + nanos = -nanos; + } + (seconds, nanos) + } else { + (i64::from_str(value)?, 0) + }; + + if seconds >= abs_duration { + Err(ParseDurationError::SecondOverflow { seconds, max_seconds: abs_duration }) + } else if seconds <= -abs_duration { + Err(ParseDurationError::SecondUnderflow { seconds, min_seconds: -abs_duration }) + } else { + Ok(chrono::Duration::seconds(seconds) + chrono::Duration::nanoseconds(nanoseconds.into())) + } + } + + pub fn serialize(x: &Option, s: S) -> Result + where + S: Serializer, + { + match x { + None => s.serialize_none(), + Some(x) => { + let seconds = x.num_seconds(); + let nanoseconds = (*x - chrono::Duration::seconds(seconds)) + .num_nanoseconds() + .expect("number of nanoseconds is less than or equal to 1 billion") as i32; + // might be left with -1 + non-zero nanos + if nanoseconds != 0 { + if seconds == 0 && nanoseconds.is_negative() { + s.serialize_str(&format!("-0.{}s", nanoseconds.abs())) + } else { + s.serialize_str(&format!("{}.{}s", seconds, nanoseconds.abs())) + } + } else { + s.serialize_str(&format!("{}s", seconds)) + } + } + } + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let s: Option<&str> = Deserialize::deserialize(deserializer)?; + match s.map(|s| parse_duration(s)) { + None => Ok(None), + Some(Ok(d)) => Ok(Some(d)), + Some(Err(e)) => Err(serde::de::Error::custom(e)), + } + } +} + +pub mod urlsafe_base64 { + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(x: &Option>, s: S) -> Result + where + S: Serializer, + { + match x { + None => s.serialize_none(), + Some(x) => s.serialize_some(&base64::encode_config(x, base64::URL_SAFE)) + } + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> + where + D: Deserializer<'de>, + { + let s: Option<&str> = Deserialize::deserialize(deserializer)?; + // TODO: Map error + match s.map(|s| base64::decode_config(s, base64::URL_SAFE)) { + None => Ok(None), + Some(Ok(d)) => Ok(Some(d)), + Some(Err(e)) => Err(serde::de::Error::custom(e)), + } + } + + // TODO: https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/field-mask + // "google-fieldmask" +} diff --git a/src/generator/templates/api/lib/schema.mako b/src/generator/templates/api/lib/schema.mako index 55590f8a76..9f9cb5bfd9 100644 --- a/src/generator/templates/api/lib/schema.mako +++ b/src/generator/templates/api/lib/schema.mako @@ -18,9 +18,9 @@ ${struct} { #[serde(rename="${pn}")] % endif % if p.get("format", None) == "byte": - #[serde(serialize_with = "client::types::to_urlsafe_base64", deserialize_with = "client::types::from_urlsafe_base64")] + #[serde(with = "client::serde::urlsafe_base64")] % elif p.get("format", None) == "google-duration": - #[serde(serialize_with = "client::types::to_duration_str", deserialize_with = "client::types::from_duration_str")] + #[serde(with = "client::serde::duration")] % endif pub ${mangle_ident(pn)}: ${to_rust_type(schemas, s.id, pn, p, allow_optionals=allow_optionals)}, % endfor From 76627413a34ff335a74750ba3f2cbc8999d4c6f3 Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Sat, 8 Oct 2022 00:58:37 -0700 Subject: [PATCH 15/27] serde cleanup --- google-apis-common/src/serde.rs | 83 ++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 32 deletions(-) diff --git a/google-apis-common/src/serde.rs b/google-apis-common/src/serde.rs index 4208ba0140..e775b5040d 100644 --- a/google-apis-common/src/serde.rs +++ b/google-apis-common/src/serde.rs @@ -1,15 +1,18 @@ pub mod duration { use std::fmt::Formatter; use std::str::FromStr; + use serde::{Deserialize, Deserializer, Serializer}; + use chrono::Duration; + #[derive(Debug)] enum ParseDurationError { MissingSecondSuffix, NanosTooSmall, ParseIntError(std::num::ParseIntError), SecondOverflow { seconds: i64, max_seconds: i64 }, - SecondUnderflow { seconds: i64, min_seconds: i64 } + SecondUnderflow { seconds: i64, min_seconds: i64 }, } impl From for ParseDurationError { @@ -22,22 +25,38 @@ pub mod duration { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { ParseDurationError::MissingSecondSuffix => write!(f, "'s' suffix was not present"), - ParseDurationError::NanosTooSmall => write!(f, "more than 9 digits of second precision required"), + ParseDurationError::NanosTooSmall => { + write!(f, "more than 9 digits of second precision required") + } ParseDurationError::ParseIntError(pie) => write!(f, "{:?}", pie), - ParseDurationError::SecondOverflow { seconds, max_seconds } => write!(f, "seconds overflow (got {}, maximum seconds possible {})", seconds, max_seconds), - ParseDurationError::SecondUnderflow { seconds, min_seconds } => write!(f, "seconds underflow (got {}, minimum seconds possible {})", seconds, min_seconds) + ParseDurationError::SecondOverflow { + seconds, + max_seconds, + } => write!( + f, + "seconds overflow (got {}, maximum seconds possible {})", + seconds, max_seconds + ), + ParseDurationError::SecondUnderflow { + seconds, + min_seconds, + } => write!( + f, + "seconds underflow (got {}, minimum seconds possible {})", + seconds, min_seconds + ), } } } impl std::error::Error for ParseDurationError {} - fn parse_duration(s: &str) -> Result { + fn parse_duration(s: &str) -> Result { let abs_duration = 315576000000i64; // TODO: Test strings like -.s, -0.0s let value = match s.strip_suffix('s') { None => return Err(ParseDurationError::MissingSecondSuffix), - Some(v) => v + Some(v) => v, }; let (seconds, nanoseconds) = if let Some((seconds, nanos)) = value.split_once('.') { @@ -63,25 +82,32 @@ pub mod duration { }; if seconds >= abs_duration { - Err(ParseDurationError::SecondOverflow { seconds, max_seconds: abs_duration }) + Err(ParseDurationError::SecondOverflow { + seconds, + max_seconds: abs_duration, + }) } else if seconds <= -abs_duration { - Err(ParseDurationError::SecondUnderflow { seconds, min_seconds: -abs_duration }) + Err(ParseDurationError::SecondUnderflow { + seconds, + min_seconds: -abs_duration, + }) } else { - Ok(chrono::Duration::seconds(seconds) + chrono::Duration::nanoseconds(nanoseconds.into())) + Ok(Duration::seconds(seconds) + Duration::nanoseconds(nanoseconds.into())) } } - pub fn serialize(x: &Option, s: S) -> Result - where - S: Serializer, + pub fn serialize(x: &Option, s: S) -> Result + where + S: Serializer, { match x { None => s.serialize_none(), Some(x) => { let seconds = x.num_seconds(); - let nanoseconds = (*x - chrono::Duration::seconds(seconds)) + let nanoseconds = (*x - Duration::seconds(seconds)) .num_nanoseconds() - .expect("number of nanoseconds is less than or equal to 1 billion") as i32; + .expect("number of nanoseconds is less than or equal to 1 billion") + as i32; // might be left with -1 + non-zero nanos if nanoseconds != 0 { if seconds == 0 && nanoseconds.is_negative() { @@ -96,16 +122,13 @@ pub mod duration { } } - pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> - where - D: Deserializer<'de>, + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, { let s: Option<&str> = Deserialize::deserialize(deserializer)?; - match s.map(|s| parse_duration(s)) { - None => Ok(None), - Some(Ok(d)) => Ok(Some(d)), - Some(Err(e)) => Err(serde::de::Error::custom(e)), - } + s.map(|s| parse_duration(s).map_err(serde::de::Error::custom)) + .transpose() } } @@ -118,7 +141,7 @@ pub mod urlsafe_base64 { { match x { None => s.serialize_none(), - Some(x) => s.serialize_some(&base64::encode_config(x, base64::URL_SAFE)) + Some(x) => s.serialize_some(&base64::encode_config(x, base64::URL_SAFE)), } } @@ -127,14 +150,10 @@ pub mod urlsafe_base64 { D: Deserializer<'de>, { let s: Option<&str> = Deserialize::deserialize(deserializer)?; - // TODO: Map error - match s.map(|s| base64::decode_config(s, base64::URL_SAFE)) { - None => Ok(None), - Some(Ok(d)) => Ok(Some(d)), - Some(Err(e)) => Err(serde::de::Error::custom(e)), - } + s.map(|s| base64::decode_config(s, base64::URL_SAFE).map_err(serde::de::Error::custom)) + .transpose() } - - // TODO: https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/field-mask - // "google-fieldmask" } + +// TODO: https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/field-mask +// "google-fieldmask" From 928c6027e65aed1fb71269571fecdf20e5b0a57c Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Sat, 8 Oct 2022 01:55:54 -0700 Subject: [PATCH 16/27] Add serde test cases --- google-apis-common/src/serde.rs | 110 +++++++++++++++++++++++++++++--- 1 file changed, 101 insertions(+), 9 deletions(-) diff --git a/google-apis-common/src/serde.rs b/google-apis-common/src/serde.rs index e775b5040d..af67212726 100644 --- a/google-apis-common/src/serde.rs +++ b/google-apis-common/src/serde.rs @@ -6,6 +6,8 @@ pub mod duration { use chrono::Duration; + const MAX_SECONDS: i64 = 315576000000i64; + #[derive(Debug)] enum ParseDurationError { MissingSecondSuffix, @@ -52,7 +54,6 @@ pub mod duration { impl std::error::Error for ParseDurationError {} fn parse_duration(s: &str) -> Result { - let abs_duration = 315576000000i64; // TODO: Test strings like -.s, -0.0s let value = match s.strip_suffix('s') { None => return Err(ParseDurationError::MissingSecondSuffix), @@ -81,15 +82,15 @@ pub mod duration { (i64::from_str(value)?, 0) }; - if seconds >= abs_duration { + if seconds >= MAX_SECONDS { Err(ParseDurationError::SecondOverflow { seconds, - max_seconds: abs_duration, + max_seconds: MAX_SECONDS, }) - } else if seconds <= -abs_duration { + } else if seconds <= -MAX_SECONDS { Err(ParseDurationError::SecondUnderflow { seconds, - min_seconds: -abs_duration, + min_seconds: -MAX_SECONDS, }) } else { Ok(Duration::seconds(seconds) + Duration::nanoseconds(nanoseconds.into())) @@ -106,14 +107,13 @@ pub mod duration { let seconds = x.num_seconds(); let nanoseconds = (*x - Duration::seconds(seconds)) .num_nanoseconds() - .expect("number of nanoseconds is less than or equal to 1 billion") + .expect("absolute number of nanoseconds is less than 1 billion") as i32; - // might be left with -1 + non-zero nanos if nanoseconds != 0 { if seconds == 0 && nanoseconds.is_negative() { - s.serialize_str(&format!("-0.{}s", nanoseconds.abs())) + s.serialize_str(&format!("-0.{:0>9}s", nanoseconds.abs())) } else { - s.serialize_str(&format!("{}.{}s", seconds, nanoseconds.abs())) + s.serialize_str(&format!("{}.{:0>9}s", seconds, nanoseconds.abs())) } } else { s.serialize_str(&format!("{}s", seconds)) @@ -157,3 +157,95 @@ pub mod urlsafe_base64 { // TODO: https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/field-mask // "google-fieldmask" +#[cfg(test)] +mod test { + use super::{duration, urlsafe_base64}; + use serde::{Deserialize, Serialize}; + + #[derive(Serialize, Deserialize, Debug, PartialEq)] + struct DurationWrapper { + #[serde(with = "duration")] + duration: Option, + } + + #[derive(Serialize, Deserialize)] + struct Base64Wrapper { + #[serde(with = "urlsafe_base64")] + bytes: Option>, + } + + #[test] + fn test_duration_de_success_cases() { + let durations = [ + ("-0.2s", -200_000_000), + ("0.000000001s", 1), + ("999.999999999s", 999_999_999_999), + ("129s", 129_000_000_000), + ("0.123456789s", 123_456_789), + ]; + for (repr, nanos) in durations.into_iter() { + let wrapper: DurationWrapper = + serde_json::from_str(&format!("{{\"duration\": \"{}\"}}", repr)).unwrap(); + assert_eq!( + Some(nanos), + wrapper.duration.unwrap().num_nanoseconds(), + "parsed \"{}\" expecting Duration with {}ns", + repr, + nanos + ); + } + } + + #[test] + fn test_duration_de_failure_cases() { + let durations = ["1.-3s", "1.1111111111s", "1.2"]; + for repr in durations.into_iter() { + assert!( + serde_json::from_str::(&format!("{{\"duration\": \"{}\"}}", repr)) + .is_err(), + "parsed \"{}\" expecting err", + repr + ); + } + } + + #[test] + fn test_duration_ser_success_cases() { + let durations = [ + -200_000_000, + 1, + 999_999_999_999, + 129_000_000_000, + 123_456_789, + ]; + + for nanos in durations.into_iter() { + let wrapper = DurationWrapper { + duration: Some(chrono::Duration::nanoseconds(nanos)), + }; + let s = serde_json::to_string(&wrapper); + assert!(s.is_ok(), "Could not serialize {}ns", nanos); + let s = s.unwrap(); + assert_eq!( + wrapper, + serde_json::from_str(&s).unwrap(), + "round trip should return same duration" + ); + } + } + + #[test] + fn urlsafe_base64_de_success_cases() { + let wrapper: Base64Wrapper = + serde_json::from_str(r#"{"bytes": "aGVsbG8gd29ybGQ="}"#).unwrap(); + assert_eq!( + Some(b"hello world".as_slice()), + wrapper.bytes.as_ref().map(Vec::as_slice) + ); + } + + #[test] + fn urlsafe_base64_de_failure_cases() { + assert!(serde_json::from_str::(r#"{"bytes": "aGVsbG8gd29ybG+Q"}"#).is_err()); + } +} From afb96bd264c91372b9900360d9d060ea38c7a31e Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Sat, 8 Oct 2022 03:22:08 -0700 Subject: [PATCH 17/27] Add FieldMask and serde impl --- google-apis-common/src/lib.rs | 31 ++++++++ google-apis-common/src/serde.rs | 81 ++++++++++++++++++++- src/generator/lib/util.py | 7 +- src/generator/templates/api/lib.rs.mako | 2 +- src/generator/templates/api/lib/lib.mako | 2 +- src/generator/templates/api/lib/schema.mako | 6 +- 6 files changed, 118 insertions(+), 11 deletions(-) diff --git a/google-apis-common/src/lib.rs b/google-apis-common/src/lib.rs index cbd7bb20a5..73b068eb63 100644 --- a/google-apis-common/src/lib.rs +++ b/google-apis-common/src/lib.rs @@ -846,6 +846,37 @@ mod yup_oauth2_impl { } } +fn titlecase(source: &str, dest: &mut String) { + let mut underscore = false; + for c in source.chars() { + if c == '_' { + underscore = true; + } else if underscore { + dest.push(c.to_ascii_uppercase()); + underscore = false; + } else { + dest.push(c); + } + } +} + + +/// A `FieldMask` as defined in `https://github.com/protocolbuffers/protobuf/blob/ec1a70913e5793a7d0a7b5fbf7e0e4f75409dd41/src/google/protobuf/field_mask.proto#L180` +#[derive(Debug, PartialEq, Default)] +pub struct FieldMask(Vec); + +impl FieldMask { + pub fn to_string(&self) -> String { + let mut repr = String::new(); + for path in &self.0 { + titlecase(path, &mut repr); + repr.push(','); + } + repr.pop(); + repr + } +} + #[cfg(test)] mod test_api { use super::*; diff --git a/google-apis-common/src/serde.rs b/google-apis-common/src/serde.rs index af67212726..c1728495a0 100644 --- a/google-apis-common/src/serde.rs +++ b/google-apis-common/src/serde.rs @@ -155,12 +155,68 @@ pub mod urlsafe_base64 { } } -// TODO: https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/field-mask -// "google-fieldmask" +pub mod field_mask { + /// Implementation based on `https://chromium.googlesource.com/infra/luci/luci-go/+/23ea7a05c6a5/common/proto/fieldmasks.go#184` + use serde::{Deserialize, Deserializer, Serializer}; + use crate::FieldMask; + + fn snakecase(source: &str) -> String { + let mut dest = String::with_capacity(source.len() + 5); + for c in source.chars() { + if c.is_ascii_uppercase() { + dest.push('_'); + dest.push(c.to_ascii_lowercase()); + } else { + dest.push(c); + } + } + dest + } + + fn parse_field_mask(s: &str) -> FieldMask { + let mut in_quotes = false; + let mut prev_ind = 0; + let mut paths = Vec::new(); + for (i, c) in s.chars().enumerate() { + if c == '`' { + in_quotes = !in_quotes; + } else if in_quotes { + continue; + } else if c == ',' { + paths.push(snakecase(&s[prev_ind..i])); + prev_ind = i + 1; + } + } + paths.push(snakecase(&s[prev_ind..])); + FieldMask(paths) + } + + pub fn serialize(x: &Option, s: S) -> Result + where + S: Serializer, + { + match x { + None => s.serialize_none(), + Some(fieldmask) => { + s.serialize_some(fieldmask.to_string().as_str()) + } + } + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let s: Option<&str> = Deserialize::deserialize(deserializer)?; + Ok(s.map(parse_field_mask)) + } +} + #[cfg(test)] mod test { - use super::{duration, urlsafe_base64}; + use super::{duration, urlsafe_base64, field_mask}; use serde::{Deserialize, Serialize}; + use crate::FieldMask; #[derive(Serialize, Deserialize, Debug, PartialEq)] struct DurationWrapper { @@ -174,6 +230,12 @@ mod test { bytes: Option>, } + #[derive(Serialize, Deserialize, Debug, PartialEq)] + struct FieldMaskWrapper { + #[serde(with = "field_mask")] + fields: Option, + } + #[test] fn test_duration_de_success_cases() { let durations = [ @@ -244,8 +306,19 @@ mod test { ); } - #[test] + #[test] fn urlsafe_base64_de_failure_cases() { assert!(serde_json::from_str::(r#"{"bytes": "aGVsbG8gd29ybG+Q"}"#).is_err()); } + + #[test] + fn field_mask_roundtrip() { + let wrapper = FieldMaskWrapper { + fields: Some(FieldMask(vec!["user.display_name".to_string(), "photo".to_string()])) + }; + let json_repr = &serde_json::to_string(&wrapper); + assert!(json_repr.is_ok(), "serialization should succeed"); + assert_eq!(wrapper, serde_json::from_str(r#"{"fields": "user.displayName,photo"}"#).unwrap()); + assert_eq!(wrapper, serde_json::from_str(json_repr.as_ref().unwrap()).unwrap(), "round trip should succeed"); + } } diff --git a/src/generator/lib/util.py b/src/generator/lib/util.py index 0827bf1bf0..9f1e8f13fe 100644 --- a/src/generator/lib/util.py +++ b/src/generator/lib/util.py @@ -49,8 +49,7 @@ TYPE_MAP = { 'google-duration': f"{CHRONO_PATH}::Duration", # guessing bytes is universally url-safe b64 "byte": "Vec", - # TODO: Provide support for these as well - "google-fieldmask": 'String' + "google-fieldmask": "client::FieldMask" } RESERVED_WORDS = set(('abstract', 'alignof', 'as', 'become', 'box', 'break', 'const', 'continue', 'crate', 'do', @@ -80,15 +79,17 @@ RUST_TYPE_RND_MAP = { '&str': lambda: '"%s"' % choice(words), '&Vec': lambda: '&vec!["%s".into()]' % choice(words), "Vec": lambda: f"vec![0, 1, 2, 3]", + # why a reference to Vec? Because it works. Should be slice, but who knows how typing works here. "&Vec": lambda: f"&vec![0, 1, 2, 3]", # TODO: styling this f"{CHRONO_PATH}::Duration": lambda: f"chrono::Duration::seconds({randint(0, 9999999)})", CHRONO_DATE: chrono_date, CHRONO_DATETIME: lambda: f"chrono::Utc::now()", + "FieldMask": lambda: f"FieldMask(vec![{choice(words)}])", f"&{CHRONO_PATH}::Duration": lambda: f"&chrono::Duration::seconds({randint(0, 9999999)})", f"&{CHRONO_DATE}": lambda: f"&{chrono_date()}", f"&{CHRONO_DATETIME}": lambda: f"&chrono::Utc::now()", - # why a reference to Vec? Because it works. Should be slice, but who knows how typing works here. + f"&FieldMask": lambda: f"&FieldMask(vec![{choice(words)}])", } TREF = '$ref' IO_RESPONSE = 'response' diff --git a/src/generator/templates/api/lib.rs.mako b/src/generator/templates/api/lib.rs.mako index a3adabaa6c..f1042232ef 100644 --- a/src/generator/templates/api/lib.rs.mako +++ b/src/generator/templates/api/lib.rs.mako @@ -50,4 +50,4 @@ pub mod api; // Re-export the hub type and some basic client structs pub use api::${hub_type}; // Re-export the yup_oauth2 crate, that is required to call some methods of the hub and the client -pub use client::{Result, Error, Delegate, oauth2}; +pub use client::{Result, Error, Delegate, oauth2, FieldMask}; diff --git a/src/generator/templates/api/lib/lib.mako b/src/generator/templates/api/lib/lib.mako index 3d5ccd98cd..ee82edb98c 100644 --- a/src/generator/templates/api/lib/lib.mako +++ b/src/generator/templates/api/lib/lib.mako @@ -243,7 +243,7 @@ Arguments will always be copied or cloned into the builder, to make them indepen ############################################################################################### <%def name="test_hub(hub_type, comments=True)">\ use std::default::Default; -use ${util.library_name()}::{${hub_type}, oauth2, hyper, hyper_rustls, chrono}; +use ${util.library_name()}::{${hub_type}, oauth2, hyper, hyper_rustls, chrono, FieldMask}; % if comments: // Get an ApplicationSecret instance by some means. It contains the `client_id` and diff --git a/src/generator/templates/api/lib/schema.mako b/src/generator/templates/api/lib/schema.mako index 9f9cb5bfd9..34f6cb6769 100644 --- a/src/generator/templates/api/lib/schema.mako +++ b/src/generator/templates/api/lib/schema.mako @@ -17,10 +17,12 @@ ${struct} { % if pn != mangle_ident(pn): #[serde(rename="${pn}")] % endif - % if p.get("format", None) == "byte": + % if p.get("format") == "byte": #[serde(with = "client::serde::urlsafe_base64")] - % elif p.get("format", None) == "google-duration": + % elif p.get("format") == "google-duration": #[serde(with = "client::serde::duration")] + % elif p.get("format") == "google-fieldmask": + #[serde(with = "client::serde::field_mask")] % endif pub ${mangle_ident(pn)}: ${to_rust_type(schemas, s.id, pn, p, allow_optionals=allow_optionals)}, % endfor From 2f3972036c01f3c69cd4bc51c3f9bbe9b832e9be Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Sat, 8 Oct 2022 03:24:47 -0700 Subject: [PATCH 18/27] Remove unnecessary examples for ref types --- src/generator/lib/util.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/generator/lib/util.py b/src/generator/lib/util.py index 9f1e8f13fe..35fb586068 100644 --- a/src/generator/lib/util.py +++ b/src/generator/lib/util.py @@ -86,10 +86,6 @@ RUST_TYPE_RND_MAP = { CHRONO_DATE: chrono_date, CHRONO_DATETIME: lambda: f"chrono::Utc::now()", "FieldMask": lambda: f"FieldMask(vec![{choice(words)}])", - f"&{CHRONO_PATH}::Duration": lambda: f"&chrono::Duration::seconds({randint(0, 9999999)})", - f"&{CHRONO_DATE}": lambda: f"&{chrono_date()}", - f"&{CHRONO_DATETIME}": lambda: f"&chrono::Utc::now()", - f"&FieldMask": lambda: f"&FieldMask(vec![{choice(words)}])", } TREF = '$ref' IO_RESPONSE = 'response' From a2d16944cd5a32b6b0a10ea0a30d5cd48d40f740 Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Sat, 8 Oct 2022 12:36:59 -0700 Subject: [PATCH 19/27] Make format fully supported --- src/generator/lib/util.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/generator/lib/util.py b/src/generator/lib/util.py index 35fb586068..df82a1a111 100644 --- a/src/generator/lib/util.py +++ b/src/generator/lib/util.py @@ -455,13 +455,9 @@ def to_rust_type( tn = 'Option>' % tn return wrap_type(tn) try: - # TODO: add support for all types and remove this check - # rust_type = TYPE_MAP[t.get("format", t["type"])] - # prefer format if present - provides support for i64 - if "format" in t and t["format"] in TYPE_MAP: - rust_type = TYPE_MAP[t["format"]] - else: - rust_type = TYPE_MAP[t["type"]] + # prefer format if present + rust_type = TYPE_MAP[t.get("format", t["type"])] + if t['type'] == 'array': return wrap_type("%s<%s>" % (rust_type, nested_type(t))) elif t['type'] == 'object': From 8809ec4807385e2c90fa2093885e67567f21a94f Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Sat, 8 Oct 2022 12:56:30 -0700 Subject: [PATCH 20/27] Add base64 round trip test --- google-apis-common/src/serde.rs | 41 +++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/google-apis-common/src/serde.rs b/google-apis-common/src/serde.rs index c1728495a0..e933776c7a 100644 --- a/google-apis-common/src/serde.rs +++ b/google-apis-common/src/serde.rs @@ -156,9 +156,9 @@ pub mod urlsafe_base64 { } pub mod field_mask { + use crate::FieldMask; /// Implementation based on `https://chromium.googlesource.com/infra/luci/luci-go/+/23ea7a05c6a5/common/proto/fieldmasks.go#184` use serde::{Deserialize, Deserializer, Serializer}; - use crate::FieldMask; fn snakecase(source: &str) -> String { let mut dest = String::with_capacity(source.len() + 5); @@ -178,7 +178,7 @@ pub mod field_mask { let mut prev_ind = 0; let mut paths = Vec::new(); for (i, c) in s.chars().enumerate() { - if c == '`' { + if c == '`' { in_quotes = !in_quotes; } else if in_quotes { continue; @@ -197,9 +197,7 @@ pub mod field_mask { { match x { None => s.serialize_none(), - Some(fieldmask) => { - s.serialize_some(fieldmask.to_string().as_str()) - } + Some(fieldmask) => s.serialize_some(fieldmask.to_string().as_str()), } } @@ -214,9 +212,9 @@ pub mod field_mask { #[cfg(test)] mod test { - use super::{duration, urlsafe_base64, field_mask}; - use serde::{Deserialize, Serialize}; + use super::{duration, field_mask, urlsafe_base64}; use crate::FieldMask; + use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Debug, PartialEq)] struct DurationWrapper { @@ -224,7 +222,7 @@ mod test { duration: Option, } - #[derive(Serialize, Deserialize)] + #[derive(Serialize, Deserialize, Debug, PartialEq)] struct Base64Wrapper { #[serde(with = "urlsafe_base64")] bytes: Option>, @@ -306,19 +304,38 @@ mod test { ); } - #[test] + #[test] fn urlsafe_base64_de_failure_cases() { assert!(serde_json::from_str::(r#"{"bytes": "aGVsbG8gd29ybG+Q"}"#).is_err()); } + #[test] + fn urlsafe_base64_roundtrip() { + let wrapper = Base64Wrapper { + bytes: Some(b"Hello world!".to_vec()), + }; + let s = serde_json::to_string(&wrapper).expect("serialization of bytes infallible"); + assert_eq!(wrapper, serde_json::from_str::(&s).unwrap()); + } + #[test] fn field_mask_roundtrip() { let wrapper = FieldMaskWrapper { - fields: Some(FieldMask(vec!["user.display_name".to_string(), "photo".to_string()])) + fields: Some(FieldMask(vec![ + "user.display_name".to_string(), + "photo".to_string(), + ])), }; let json_repr = &serde_json::to_string(&wrapper); assert!(json_repr.is_ok(), "serialization should succeed"); - assert_eq!(wrapper, serde_json::from_str(r#"{"fields": "user.displayName,photo"}"#).unwrap()); - assert_eq!(wrapper, serde_json::from_str(json_repr.as_ref().unwrap()).unwrap(), "round trip should succeed"); + assert_eq!( + wrapper, + serde_json::from_str(r#"{"fields": "user.displayName,photo"}"#).unwrap() + ); + assert_eq!( + wrapper, + serde_json::from_str(json_repr.as_ref().unwrap()).unwrap(), + "round trip should succeed" + ); } } From d043fd67b914fc2223007c8969ae76ca77b356d4 Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Sat, 8 Oct 2022 12:56:54 -0700 Subject: [PATCH 21/27] Include references to definitions of types --- src/generator/lib/util.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/generator/lib/util.py b/src/generator/lib/util.py index df82a1a111..b17d508bc1 100644 --- a/src/generator/lib/util.py +++ b/src/generator/lib/util.py @@ -37,18 +37,19 @@ TYPE_MAP = { 'array': 'Vec', 'string': 'String', 'object': 'HashMap', - # https://developers.google.com/protocol-buffers/docs/reference/java/com/google/protobuf/Timestamp + # https://github.com/protocolbuffers/protobuf/blob/ec1a70913e5793a7d0a7b5fbf7e0e4f75409dd41/src/google/protobuf/timestamp.proto # In JSON format, the Timestamp type is encoded as a string in the [RFC 3339] format 'google-datetime': CHRONO_DATETIME, - # RFC 3339 date-time value + # Per .json files: RFC 3339 timestamp 'date-time': CHRONO_DATETIME, - # A date in RFC 3339 format with only the date part + # Per .json files: A date in RFC 3339 format with only the date part # e.g. "2013-01-15" 'date': CHRONO_DATE, - # custom serde impl - {seconds}.{nanoseconds}s + # https://github.com/protocolbuffers/protobuf/blob/ec1a70913e5793a7d0a7b5fbf7e0e4f75409dd41/src/google/protobuf/duration.proto 'google-duration': f"{CHRONO_PATH}::Duration", # guessing bytes is universally url-safe b64 "byte": "Vec", + # https://github.com/protocolbuffers/protobuf/blob/ec1a70913e5793a7d0a7b5fbf7e0e4f75409dd41/src/google/protobuf/field_mask.proto "google-fieldmask": "client::FieldMask" } From ddac761e06dd2bd6c544bfafc8f058871aa3ec04 Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Sat, 8 Oct 2022 15:48:28 -0700 Subject: [PATCH 22/27] Add #[serde(default)] for Option parsing If using #[serde(with = ...)] with an Option type, serde will expect all marked fields to be present. Adding #[serde(default)] restores expected behaviour - if no Option value is present, None will be used. --- google-apis-common/src/serde.rs | 76 +++++++++++++++++++-- src/generator/templates/api/lib/schema.mako | 8 ++- 2 files changed, 75 insertions(+), 9 deletions(-) diff --git a/google-apis-common/src/serde.rs b/google-apis-common/src/serde.rs index e933776c7a..f0fb840e5b 100644 --- a/google-apis-common/src/serde.rs +++ b/google-apis-common/src/serde.rs @@ -127,8 +127,9 @@ pub mod duration { D: Deserializer<'de>, { let s: Option<&str> = Deserialize::deserialize(deserializer)?; - s.map(|s| parse_duration(s).map_err(serde::de::Error::custom)) + s.map(parse_duration) .transpose() + .map_err(serde::de::Error::custom) } } @@ -150,8 +151,9 @@ pub mod urlsafe_base64 { D: Deserializer<'de>, { let s: Option<&str> = Deserialize::deserialize(deserializer)?; - s.map(|s| base64::decode_config(s, base64::URL_SAFE).map_err(serde::de::Error::custom)) + s.map(|s| base64::decode_config(s, base64::URL_SAFE)) .transpose() + .map_err(serde::de::Error::custom) } } @@ -210,30 +212,65 @@ pub mod field_mask { } } +pub mod str_like { + /// Implementation based on `https://chromium.googlesource.com/infra/luci/luci-go/+/23ea7a05c6a5/common/proto/fieldmasks.go#184` + use serde::{Deserialize, Deserializer, Serializer}; + use std::str::FromStr; + + pub fn serialize(x: &Option, s: S) -> Result + where + S: Serializer, + T: std::fmt::Display, + { + match x { + None => s.serialize_none(), + Some(num) => s.serialize_some(num.to_string().as_str()), + } + } + + pub fn deserialize<'de, D, T>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + T: FromStr, + ::Err: std::fmt::Display, + { + let s: Option<&str> = Deserialize::deserialize(deserializer)?; + s.map(T::from_str) + .transpose() + .map_err(serde::de::Error::custom) + } +} + #[cfg(test)] mod test { - use super::{duration, field_mask, urlsafe_base64}; + use super::{duration, field_mask, str_like, urlsafe_base64}; use crate::FieldMask; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Debug, PartialEq)] struct DurationWrapper { - #[serde(with = "duration")] + #[serde(default, with = "duration")] duration: Option, } #[derive(Serialize, Deserialize, Debug, PartialEq)] struct Base64Wrapper { - #[serde(with = "urlsafe_base64")] + #[serde(default, with = "urlsafe_base64")] bytes: Option>, } #[derive(Serialize, Deserialize, Debug, PartialEq)] struct FieldMaskWrapper { - #[serde(with = "field_mask")] + #[serde(default, with = "field_mask")] fields: Option, } + #[derive(Serialize, Deserialize, Debug, PartialEq)] + struct I64Wrapper { + #[serde(default, with = "str_like")] + num: Option, + } + #[test] fn test_duration_de_success_cases() { let durations = [ @@ -338,4 +375,31 @@ mod test { "round trip should succeed" ); } + + #[test] + fn num_roundtrip() { + let wrapper = I64Wrapper { + num: Some(i64::MAX), + }; + + let json_repr = &serde_json::to_string(&wrapper); + assert!(json_repr.is_ok(), "serialization should succeed"); + assert_eq!( + wrapper, + serde_json::from_str(&format!("{{\"num\": \"{}\"}}", i64::MAX)).unwrap() + ); + assert_eq!( + wrapper, + serde_json::from_str(json_repr.as_ref().unwrap()).unwrap(), + "round trip should succeed" + ); + } + + #[test] + fn test_empty_wrapper() { + assert_eq!(DurationWrapper { duration: None }, serde_json::from_str("{}").unwrap()); + assert_eq!(Base64Wrapper { bytes: None }, serde_json::from_str("{}").unwrap()); + assert_eq!(FieldMaskWrapper { fields: None }, serde_json::from_str("{}").unwrap()); + assert_eq!(I64Wrapper { num: None }, serde_json::from_str("{}").unwrap()); + } } diff --git a/src/generator/templates/api/lib/schema.mako b/src/generator/templates/api/lib/schema.mako index 34f6cb6769..d4b22f941b 100644 --- a/src/generator/templates/api/lib/schema.mako +++ b/src/generator/templates/api/lib/schema.mako @@ -18,11 +18,13 @@ ${struct} { #[serde(rename="${pn}")] % endif % if p.get("format") == "byte": - #[serde(with = "client::serde::urlsafe_base64")] + #[serde(default, with = "client::serde::urlsafe_base64")] % elif p.get("format") == "google-duration": - #[serde(with = "client::serde::duration")] + #[serde(default, with = "client::serde::duration")] % elif p.get("format") == "google-fieldmask": - #[serde(with = "client::serde::field_mask")] + #[serde(default, with = "client::serde::field_mask")] + % elif p.get("format") in {"uint64", "int64"}: + #[serde(default, with = "client::serde::str_like")] % endif pub ${mangle_ident(pn)}: ${to_rust_type(schemas, s.id, pn, p, allow_optionals=allow_optionals)}, % endfor From 8cc27075638e005448defd31fbf8ec3e445dd7b0 Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Sat, 8 Oct 2022 19:46:57 -0700 Subject: [PATCH 23/27] Fix cargo check w.r.t. FieldMask The serde traits are now directly implemented for FieldMask - this helps address potential serde issues with wrapper types, and simplifies the serde process somewhat. --- google-apis-common/src/field_mask.rs | 121 ++++++++++++++++++++ google-apis-common/src/lib.rs | 35 +----- google-apis-common/src/serde.rs | 101 +++------------- src/generator/templates/api/lib/schema.mako | 2 - 4 files changed, 137 insertions(+), 122 deletions(-) create mode 100644 google-apis-common/src/field_mask.rs diff --git a/google-apis-common/src/field_mask.rs b/google-apis-common/src/field_mask.rs new file mode 100644 index 0000000000..c20c8b32a2 --- /dev/null +++ b/google-apis-common/src/field_mask.rs @@ -0,0 +1,121 @@ +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +fn titlecase(source: &str, dest: &mut String) { + let mut underscore = false; + for c in source.chars() { + if c == '_' { + underscore = true; + } else if underscore { + dest.push(c.to_ascii_uppercase()); + underscore = false; + } else { + dest.push(c); + } + } +} + +fn snakecase(source: &str) -> String { + let mut dest = String::with_capacity(source.len() + 5); + for c in source.chars() { + if c.is_ascii_uppercase() { + dest.push('_'); + dest.push(c.to_ascii_lowercase()); + } else { + dest.push(c); + } + } + dest +} + +/// A `FieldMask` as defined in `https://github.com/protocolbuffers/protobuf/blob/ec1a70913e5793a7d0a7b5fbf7e0e4f75409dd41/src/google/protobuf/field_mask.proto#L180` +#[derive(Clone, Debug, Default, PartialEq)] +pub struct FieldMask(Vec); + +impl Serialize for FieldMask { + fn serialize(&self, s: S) -> Result + where + S: Serializer, + { + s.serialize_str(self.to_string().as_str()) + } +} + +impl<'de> Deserialize<'de> for FieldMask { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s: &str = Deserialize::deserialize(deserializer)?; + Ok(FieldMask::from_str(s)) + } +} + +impl FieldMask { + fn from_str(s: &str) -> FieldMask { + let mut in_quotes = false; + let mut prev_ind = 0; + let mut paths = Vec::new(); + for (i, c) in s.chars().enumerate() { + if c == '`' { + in_quotes = !in_quotes; + } else if in_quotes { + continue; + } else if c == ',' { + paths.push(snakecase(&s[prev_ind..i])); + prev_ind = i + 1; + } + } + paths.push(snakecase(&s[prev_ind..])); + FieldMask(paths) + } + + pub fn to_string(&self) -> String { + let mut repr = String::new(); + for path in &self.0 { + titlecase(path, &mut repr); + repr.push(','); + } + repr.pop(); + repr + } +} + +#[cfg(test)] +mod test { + use crate::field_mask::FieldMask; + use serde::{Deserialize, Serialize}; + + #[derive(Serialize, Deserialize, Debug, PartialEq)] + struct FieldMaskWrapper { + fields: Option, + } + + #[test] + fn field_mask_roundtrip() { + let wrapper = FieldMaskWrapper { + fields: Some(FieldMask(vec![ + "user.display_name".to_string(), + "photo".to_string(), + ])), + }; + let json_repr = &serde_json::to_string(&wrapper); + assert!(json_repr.is_ok(), "serialization should succeed"); + assert_eq!( + wrapper, + serde_json::from_str(r#"{"fields": "user.displayName,photo"}"#).unwrap() + ); + assert_eq!( + wrapper, + serde_json::from_str(json_repr.as_ref().unwrap()).unwrap(), + "round trip should succeed" + ); + } + + #[test] + fn test_empty_wrapper() { + assert_eq!( + FieldMaskWrapper { fields: None }, + serde_json::from_str("{}").unwrap() + ); + } +} diff --git a/google-apis-common/src/lib.rs b/google-apis-common/src/lib.rs index 73b068eb63..80f4ed565f 100644 --- a/google-apis-common/src/lib.rs +++ b/google-apis-common/src/lib.rs @@ -1,4 +1,5 @@ pub mod serde; +pub mod field_mask; use std::error; use std::error::Error as StdError; @@ -25,8 +26,9 @@ use tokio::io::{AsyncRead, AsyncWrite}; use tokio::time::sleep; use tower_service; -pub use yup_oauth2 as oauth2; pub use chrono; +pub use field_mask::FieldMask; +pub use yup_oauth2 as oauth2; const LINE_ENDING: &str = "\r\n"; @@ -846,37 +848,6 @@ mod yup_oauth2_impl { } } -fn titlecase(source: &str, dest: &mut String) { - let mut underscore = false; - for c in source.chars() { - if c == '_' { - underscore = true; - } else if underscore { - dest.push(c.to_ascii_uppercase()); - underscore = false; - } else { - dest.push(c); - } - } -} - - -/// A `FieldMask` as defined in `https://github.com/protocolbuffers/protobuf/blob/ec1a70913e5793a7d0a7b5fbf7e0e4f75409dd41/src/google/protobuf/field_mask.proto#L180` -#[derive(Debug, PartialEq, Default)] -pub struct FieldMask(Vec); - -impl FieldMask { - pub fn to_string(&self) -> String { - let mut repr = String::new(); - for path in &self.0 { - titlecase(path, &mut repr); - repr.push(','); - } - repr.pop(); - repr - } -} - #[cfg(test)] mod test_api { use super::*; diff --git a/google-apis-common/src/serde.rs b/google-apis-common/src/serde.rs index f0fb840e5b..d5a0942aa6 100644 --- a/google-apis-common/src/serde.rs +++ b/google-apis-common/src/serde.rs @@ -157,61 +157,6 @@ pub mod urlsafe_base64 { } } -pub mod field_mask { - use crate::FieldMask; - /// Implementation based on `https://chromium.googlesource.com/infra/luci/luci-go/+/23ea7a05c6a5/common/proto/fieldmasks.go#184` - use serde::{Deserialize, Deserializer, Serializer}; - - fn snakecase(source: &str) -> String { - let mut dest = String::with_capacity(source.len() + 5); - for c in source.chars() { - if c.is_ascii_uppercase() { - dest.push('_'); - dest.push(c.to_ascii_lowercase()); - } else { - dest.push(c); - } - } - dest - } - - fn parse_field_mask(s: &str) -> FieldMask { - let mut in_quotes = false; - let mut prev_ind = 0; - let mut paths = Vec::new(); - for (i, c) in s.chars().enumerate() { - if c == '`' { - in_quotes = !in_quotes; - } else if in_quotes { - continue; - } else if c == ',' { - paths.push(snakecase(&s[prev_ind..i])); - prev_ind = i + 1; - } - } - paths.push(snakecase(&s[prev_ind..])); - FieldMask(paths) - } - - pub fn serialize(x: &Option, s: S) -> Result - where - S: Serializer, - { - match x { - None => s.serialize_none(), - Some(fieldmask) => s.serialize_some(fieldmask.to_string().as_str()), - } - } - - pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> - where - D: Deserializer<'de>, - { - let s: Option<&str> = Deserialize::deserialize(deserializer)?; - Ok(s.map(parse_field_mask)) - } -} - pub mod str_like { /// Implementation based on `https://chromium.googlesource.com/infra/luci/luci-go/+/23ea7a05c6a5/common/proto/fieldmasks.go#184` use serde::{Deserialize, Deserializer, Serializer}; @@ -243,8 +188,7 @@ pub mod str_like { #[cfg(test)] mod test { - use super::{duration, field_mask, str_like, urlsafe_base64}; - use crate::FieldMask; + use super::{duration, str_like, urlsafe_base64}; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Debug, PartialEq)] @@ -259,12 +203,6 @@ mod test { bytes: Option>, } - #[derive(Serialize, Deserialize, Debug, PartialEq)] - struct FieldMaskWrapper { - #[serde(default, with = "field_mask")] - fields: Option, - } - #[derive(Serialize, Deserialize, Debug, PartialEq)] struct I64Wrapper { #[serde(default, with = "str_like")] @@ -355,27 +293,6 @@ mod test { assert_eq!(wrapper, serde_json::from_str::(&s).unwrap()); } - #[test] - fn field_mask_roundtrip() { - let wrapper = FieldMaskWrapper { - fields: Some(FieldMask(vec![ - "user.display_name".to_string(), - "photo".to_string(), - ])), - }; - let json_repr = &serde_json::to_string(&wrapper); - assert!(json_repr.is_ok(), "serialization should succeed"); - assert_eq!( - wrapper, - serde_json::from_str(r#"{"fields": "user.displayName,photo"}"#).unwrap() - ); - assert_eq!( - wrapper, - serde_json::from_str(json_repr.as_ref().unwrap()).unwrap(), - "round trip should succeed" - ); - } - #[test] fn num_roundtrip() { let wrapper = I64Wrapper { @@ -397,9 +314,17 @@ mod test { #[test] fn test_empty_wrapper() { - assert_eq!(DurationWrapper { duration: None }, serde_json::from_str("{}").unwrap()); - assert_eq!(Base64Wrapper { bytes: None }, serde_json::from_str("{}").unwrap()); - assert_eq!(FieldMaskWrapper { fields: None }, serde_json::from_str("{}").unwrap()); - assert_eq!(I64Wrapper { num: None }, serde_json::from_str("{}").unwrap()); + assert_eq!( + DurationWrapper { duration: None }, + serde_json::from_str("{}").unwrap() + ); + assert_eq!( + Base64Wrapper { bytes: None }, + serde_json::from_str("{}").unwrap() + ); + assert_eq!( + I64Wrapper { num: None }, + serde_json::from_str("{}").unwrap() + ); } } diff --git a/src/generator/templates/api/lib/schema.mako b/src/generator/templates/api/lib/schema.mako index d4b22f941b..e10d88fbce 100644 --- a/src/generator/templates/api/lib/schema.mako +++ b/src/generator/templates/api/lib/schema.mako @@ -21,8 +21,6 @@ ${struct} { #[serde(default, with = "client::serde::urlsafe_base64")] % elif p.get("format") == "google-duration": #[serde(default, with = "client::serde::duration")] - % elif p.get("format") == "google-fieldmask": - #[serde(default, with = "client::serde::field_mask")] % elif p.get("format") in {"uint64", "int64"}: #[serde(default, with = "client::serde::str_like")] % endif From f6cced960594d680c2335c5740c331b99e46bdcc Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Sat, 8 Oct 2022 23:01:30 -0700 Subject: [PATCH 24/27] Support serde for arbitrary field types This introduces the `serde_with` dependency and `rust_type.py`, to allow supporting arbitrary types for serialization. Since fields may have arbitrary types (eg. `HashMap<_, chrono::Duration>`) which need deserialization, it is necessary to use type-based serialization to avoid implementing (de)serialization for every permutation of types that require special serialization. However, `serde` does not let you (de)serialize one type as another (eg. `chrono::Duration` as `Wrapper`) - thus necessitating `serde_with`, which does. `rust_type.py` introduces the `RustType` class, which makes it easy to describe the (de)serialization type used by `serde_with` --- google-apis-common/Cargo.toml | 5 +- google-apis-common/src/lib.rs | 3 +- google-apis-common/src/serde.rs | 138 +++++++++----------- src/generator/lib/rust_type.py | 76 +++++++++++ src/generator/lib/util.py | 86 +++++++++--- src/generator/templates/api/api.rs.mako | 2 +- src/generator/templates/api/lib/schema.mako | 19 +-- 7 files changed, 221 insertions(+), 108 deletions(-) create mode 100644 src/generator/lib/rust_type.py diff --git a/google-apis-common/Cargo.toml b/google-apis-common/Cargo.toml index ac27adbe23..ca375dbd0d 100644 --- a/google-apis-common/Cargo.toml +++ b/google-apis-common/Cargo.toml @@ -18,9 +18,12 @@ doctest = false [dependencies] mime = "^ 0.2.0" serde = { version = "^ 1.0", features = ["derive"] } -base64 = "0.13.0" +serde_with = "2.0.1" serde_json = "^ 1.0" + +base64 = "0.13.0" chrono = { version = "0.4.22", features = ["serde"] } + ## TODO: Make yup-oauth2 optional ## yup-oauth2 = { version = "^ 7.0", optional = true } yup-oauth2 = "^ 7.0" diff --git a/google-apis-common/src/lib.rs b/google-apis-common/src/lib.rs index 80f4ed565f..022b9a22c0 100644 --- a/google-apis-common/src/lib.rs +++ b/google-apis-common/src/lib.rs @@ -1,5 +1,5 @@ -pub mod serde; pub mod field_mask; +pub mod serde; use std::error; use std::error::Error as StdError; @@ -28,6 +28,7 @@ use tower_service; pub use chrono; pub use field_mask::FieldMask; +pub use serde_with; pub use yup_oauth2 as oauth2; const LINE_ENDING: &str = "\r\n"; diff --git a/google-apis-common/src/serde.rs b/google-apis-common/src/serde.rs index d5a0942aa6..a34d4318ad 100644 --- a/google-apis-common/src/serde.rs +++ b/google-apis-common/src/serde.rs @@ -1,9 +1,9 @@ pub mod duration { + use serde::{Deserialize, Deserializer}; + use serde_with::{DeserializeAs, SerializeAs}; use std::fmt::Formatter; use std::str::FromStr; - use serde::{Deserialize, Deserializer, Serializer}; - use chrono::Duration; const MAX_SECONDS: i64 = 315576000000i64; @@ -53,7 +53,7 @@ pub mod duration { impl std::error::Error for ParseDurationError {} - fn parse_duration(s: &str) -> Result { + fn duration_from_str(s: &str) -> Result { // TODO: Test strings like -.s, -0.0s let value = match s.strip_suffix('s') { None => return Err(ParseDurationError::MissingSecondSuffix), @@ -97,115 +97,95 @@ pub mod duration { } } - pub fn serialize(x: &Option, s: S) -> Result - where - S: Serializer, - { - match x { - None => s.serialize_none(), - Some(x) => { - let seconds = x.num_seconds(); - let nanoseconds = (*x - Duration::seconds(seconds)) - .num_nanoseconds() - .expect("absolute number of nanoseconds is less than 1 billion") - as i32; - if nanoseconds != 0 { - if seconds == 0 && nanoseconds.is_negative() { - s.serialize_str(&format!("-0.{:0>9}s", nanoseconds.abs())) - } else { - s.serialize_str(&format!("{}.{:0>9}s", seconds, nanoseconds.abs())) - } - } else { - s.serialize_str(&format!("{}s", seconds)) - } + fn duration_to_string(duration: &Duration) -> String { + let seconds = duration.num_seconds(); + let nanoseconds = (*duration - Duration::seconds(seconds)) + .num_nanoseconds() + .expect("absolute number of nanoseconds is less than 1 billion") + as i32; + if nanoseconds != 0 { + if seconds == 0 && nanoseconds.is_negative() { + format!("-0.{:0>9}s", nanoseconds.abs()) + } else { + format!("{}.{:0>9}s", seconds, nanoseconds.abs()) } + } else { + format!("{}s", seconds) } } - pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> - where - D: Deserializer<'de>, - { - let s: Option<&str> = Deserialize::deserialize(deserializer)?; - s.map(parse_duration) - .transpose() - .map_err(serde::de::Error::custom) + pub struct Wrapper; + + impl SerializeAs for Wrapper { + fn serialize_as(value: &Duration, s: S) -> Result + where + S: serde::Serializer, + { + s.serialize_str(&duration_to_string(value)) + } + } + + impl<'de> DeserializeAs<'de, Duration> for Wrapper { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = Deserialize::deserialize(deserializer)?; + duration_from_str(s).map_err(serde::de::Error::custom) + } } } pub mod urlsafe_base64 { use serde::{Deserialize, Deserializer, Serializer}; + use serde_with::{DeserializeAs, SerializeAs}; - pub fn serialize(x: &Option>, s: S) -> Result - where - S: Serializer, - { - match x { - None => s.serialize_none(), - Some(x) => s.serialize_some(&base64::encode_config(x, base64::URL_SAFE)), + pub struct Wrapper; + + impl SerializeAs> for Wrapper { + fn serialize_as(value: &Vec, s: S) -> Result + where + S: Serializer, + { + s.serialize_str(&base64::encode_config(value, base64::URL_SAFE)) } } - pub fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> - where - D: Deserializer<'de>, - { - let s: Option<&str> = Deserialize::deserialize(deserializer)?; - s.map(|s| base64::decode_config(s, base64::URL_SAFE)) - .transpose() - .map_err(serde::de::Error::custom) - } -} - -pub mod str_like { - /// Implementation based on `https://chromium.googlesource.com/infra/luci/luci-go/+/23ea7a05c6a5/common/proto/fieldmasks.go#184` - use serde::{Deserialize, Deserializer, Serializer}; - use std::str::FromStr; - - pub fn serialize(x: &Option, s: S) -> Result - where - S: Serializer, - T: std::fmt::Display, - { - match x { - None => s.serialize_none(), - Some(num) => s.serialize_some(num.to_string().as_str()), + impl<'de> DeserializeAs<'de, Vec> for Wrapper { + fn deserialize_as(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let s: &str = Deserialize::deserialize(deserializer)?; + base64::decode_config(s, base64::URL_SAFE).map_err(serde::de::Error::custom) } } - - pub fn deserialize<'de, D, T>(deserializer: D) -> Result, D::Error> - where - D: Deserializer<'de>, - T: FromStr, - ::Err: std::fmt::Display, - { - let s: Option<&str> = Deserialize::deserialize(deserializer)?; - s.map(T::from_str) - .transpose() - .map_err(serde::de::Error::custom) - } } #[cfg(test)] mod test { - use super::{duration, str_like, urlsafe_base64}; + use super::{duration, urlsafe_base64}; use serde::{Deserialize, Serialize}; + use serde_with::{serde_as, DisplayFromStr}; + #[serde_as] #[derive(Serialize, Deserialize, Debug, PartialEq)] struct DurationWrapper { - #[serde(default, with = "duration")] + #[serde_as(as = "Option")] duration: Option, } + #[serde_as] #[derive(Serialize, Deserialize, Debug, PartialEq)] struct Base64Wrapper { - #[serde(default, with = "urlsafe_base64")] + #[serde_as(as = "Option")] bytes: Option>, } + #[serde_as] #[derive(Serialize, Deserialize, Debug, PartialEq)] struct I64Wrapper { - #[serde(default, with = "str_like")] + #[serde_as(as = "Option")] num: Option, } diff --git a/src/generator/lib/rust_type.py b/src/generator/lib/rust_type.py new file mode 100644 index 0000000000..5a07765672 --- /dev/null +++ b/src/generator/lib/rust_type.py @@ -0,0 +1,76 @@ +from typing import Optional, List +from copy import deepcopy + + +class RustType: + def __init__(self, name: str, members: Optional[List["RustType"]] = None): + self.name = name + self.members = members + + def serde_replace_inner_ty(self, from_to): + if self.members is None: + return False + + changed = False + for i, member in enumerate(self.members): + if member in from_to: + self.members[i] = from_to[member] + changed = True + else: + # serde_as fails to compile if type definition includes + # types without custom serialization + if not member.serde_replace_inner_ty(from_to): + self.members[i] = Base("_") + return changed + + def serde_as(self) -> "RustType": + copied = deepcopy(self) + from_to = { + Vec(Base("u8")): Base("::client::serde::urlsafe_base64::Wrapper"), + Base("client::chrono::Duration"): Base("::client::serde::duration::Wrapper"), + Base("i64"): Base("::client::serde_with::DisplayFromStr"), + Base("u64"): Base("::client::serde_with::DisplayFromStr"), + } + + changed = copied.serde_replace_inner_ty(from_to) + + return copied, changed + + def __str__(self): + if self.members: + return f"{self.name}<{', '.join(str(m) for m in self.members)}>" + return self.name + + def __eq__(self, other): + if not isinstance(other, RustType): + return False + return self.name == other.name and self.members == other.members + + def __hash__(self): + if self.members: + return hash((self.name, *[(i, v) for i, v in enumerate(self.members)])) + return hash((self.name, None)) + +class Option(RustType): + def __init__(self, member): + super().__init__("Option", [member]) + + +class Box(RustType): + def __init__(self, member): + super().__init__("Box", [member]) + + +class Vec(RustType): + def __init__(self, member): + super().__init__("Vec", [member]) + + +class HashMap(RustType): + def __init__(self, key, value): + super().__init__("HashMap", [key, value]) + + +class Base(RustType): + def __init__(self, name): + super().__init__(name) \ No newline at end of file diff --git a/src/generator/lib/util.py b/src/generator/lib/util.py index b17d508bc1..5379c04a55 100644 --- a/src/generator/lib/util.py +++ b/src/generator/lib/util.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from random import (randint, random, choice, seed) from typing import Any, Dict, List, Mapping, Tuple from copy import deepcopy +from .rust_type import Base, Box, HashMap, Vec, Option, RustType seed(1337) @@ -53,6 +54,36 @@ TYPE_MAP = { "google-fieldmask": "client::FieldMask" } +RUST_TYPE_MAP = { + 'boolean': Base("bool"), + 'integer': USE_FORMAT, + 'number': USE_FORMAT, + 'uint32': Base("u32"), + 'double': Base("f64"), + 'float': Base("f32"), + 'int32': Base("i32"), + 'any': Base("String"), # TODO: Figure out how to handle it. It's 'interface' in Go ... + 'int64': Base("i64"), + 'uint64': Base("u64"), + 'array': Vec(None), + 'string': Base("String"), + 'object': HashMap(None, None), + # https://github.com/protocolbuffers/protobuf/blob/ec1a70913e5793a7d0a7b5fbf7e0e4f75409dd41/src/google/protobuf/timestamp.proto + # In JSON format, the Timestamp type is encoded as a string in the [RFC 3339] format + 'google-datetime': Base(CHRONO_DATETIME), + # Per .json files: RFC 3339 timestamp + 'date-time': Base(CHRONO_DATETIME), + # Per .json files: A date in RFC 3339 format with only the date part + # e.g. "2013-01-15" + 'date': Base(CHRONO_DATE), + # https://github.com/protocolbuffers/protobuf/blob/ec1a70913e5793a7d0a7b5fbf7e0e4f75409dd41/src/google/protobuf/duration.proto + 'google-duration': Base(f"{CHRONO_PATH}::Duration"), + # guessing bytes is universally url-safe b64 + "byte": Vec(Base("u8")), + # https://github.com/protocolbuffers/protobuf/blob/ec1a70913e5793a7d0a7b5fbf7e0e4f75409dd41/src/google/protobuf/field_mask.proto + "google-fieldmask": Base("client::FieldMask") +} + RESERVED_WORDS = set(('abstract', 'alignof', 'as', 'become', 'box', 'break', 'const', 'continue', 'crate', 'do', 'else', 'enum', 'extern', 'false', 'final', 'fn', 'for', 'if', 'impl', 'in', 'let', 'loop', 'macro', 'match', 'mod', 'move', 'mut', 'offsetof', 'override', 'priv', 'pub', 'pure', 'ref', @@ -430,21 +461,43 @@ def to_rust_type( allow_optionals=True, _is_recursive=False ) -> str: - def nested_type(nt): + return str(to_rust_type_inner(schemas, schema_name, property_name, t, allow_optionals, _is_recursive)) + + +def to_serde_type( + schemas, + schema_name, + property_name, + t, + allow_optionals=True, + _is_recursive=False +) -> Tuple[RustType, bool]: + return to_rust_type_inner(schemas, schema_name, property_name, t, allow_optionals, _is_recursive).serde_as() + + +def to_rust_type_inner( + schemas, + schema_name, + property_name, + t, + allow_optionals=True, + _is_recursive=False +) -> RustType: + def nested_type(nt) -> RustType: if 'items' in nt: nt = nt['items'] elif 'additionalProperties' in nt: nt = nt['additionalProperties'] else: - assert (is_nested_type_property(nt)) + assert is_nested_type_property(nt) # It's a nested type - we take it literally like $ref, but generate a name for the type ourselves - return _assure_unique_type_name(schemas, nested_type_name(schema_name, property_name)) - return to_rust_type(schemas, schema_name, property_name, nt, allow_optionals=False, _is_recursive=True) + return Base(_assure_unique_type_name(schemas, nested_type_name(schema_name, property_name))) + return to_rust_type_inner(schemas, schema_name, property_name, nt, allow_optionals=False, _is_recursive=True) - def wrap_type(tn): + def wrap_type(rt) -> RustType: if allow_optionals: - tn = "Option<%s>" % tn - return tn + return Option(rt) + return rt # unconditionally handle $ref types, which should point to another schema. if TREF in t: @@ -452,22 +505,21 @@ def to_rust_type( # which is fine for now. 'allow_optionals' implicitly restricts type boxing for simple types - it # is usually on the first call, and off when recursion is involved. tn = t[TREF] + rt = Base(tn) if not _is_recursive and tn == schema_name: - tn = 'Option>' % tn - return wrap_type(tn) + rt = Option(Box(rt)) + return wrap_type(rt) try: # prefer format if present - rust_type = TYPE_MAP[t.get("format", t["type"])] - - if t['type'] == 'array': - return wrap_type("%s<%s>" % (rust_type, nested_type(t))) - elif t['type'] == 'object': + rust_type = RUST_TYPE_MAP[t.get("format", t["type"])] + if rust_type == Vec(None): + return wrap_type(Vec(nested_type(t))) + if rust_type == HashMap(None, None): if is_map_prop(t): - return wrap_type("%s" % (rust_type, nested_type(t))) + return wrap_type(HashMap(Base("String"), nested_type(t))) return wrap_type(nested_type(t)) - if t.get('repeated', False): - return 'Vec<%s>' % rust_type + return Vec(rust_type) return wrap_type(rust_type) except KeyError as err: raise AssertionError( diff --git a/src/generator/templates/api/api.rs.mako b/src/generator/templates/api/api.rs.mako index 7bc840be33..cf15ee2aff 100644 --- a/src/generator/templates/api/api.rs.mako +++ b/src/generator/templates/api/api.rs.mako @@ -31,7 +31,7 @@ use tokio::time::sleep; use tower_service; use serde::{Serialize, Deserialize}; -use crate::{client, client::GetToken, client::oauth2}; +use crate::{client, client::GetToken, client::oauth2, client::serde_with}; // ############## // UTILITIES ### diff --git a/src/generator/templates/api/lib/schema.mako b/src/generator/templates/api/lib/schema.mako index e10d88fbce..0a5b87ca05 100644 --- a/src/generator/templates/api/lib/schema.mako +++ b/src/generator/templates/api/lib/schema.mako @@ -1,5 +1,5 @@ <%! - from generator.lib.util import (schema_markers, rust_doc_comment, mangle_ident, to_rust_type, put_and, + from generator.lib.util import (schema_markers, rust_doc_comment, mangle_ident, to_serde_type, to_rust_type, put_and, IO_TYPES, activity_split, enclose_in, REQUEST_MARKER_TRAIT, mb_type, indent_all_but_first_by, NESTED_TYPE_SUFFIX, RESPONSE_MARKER_TRAIT, split_camelcase_s, METHODS_RESOURCE, PART_MARKER_TRAIT, canonical_type_name, TO_PARTS_MARKER, UNUSED_TYPE_MARKER, is_schema_with_optionals, @@ -17,14 +17,14 @@ ${struct} { % if pn != mangle_ident(pn): #[serde(rename="${pn}")] % endif - % if p.get("format") == "byte": - #[serde(default, with = "client::serde::urlsafe_base64")] - % elif p.get("format") == "google-duration": - #[serde(default, with = "client::serde::duration")] - % elif p.get("format") in {"uint64", "int64"}: - #[serde(default, with = "client::serde::str_like")] + <% + rust_ty = to_rust_type(schemas, s.id, pn, p, allow_optionals=allow_optionals) + serde_ty, use_custom_serde = to_serde_type(schemas, s.id, pn, p, allow_optionals=allow_optionals) + %> + % if use_custom_serde: + #[serde_as(as = "${serde_ty}")] % endif - pub ${mangle_ident(pn)}: ${to_rust_type(schemas, s.id, pn, p, allow_optionals=allow_optionals)}, + pub ${mangle_ident(pn)}: ${rust_ty}, % endfor } % elif 'additionalProperties' in s: @@ -83,7 +83,8 @@ ${struct} { _never_set: Option } <%block filter="rust_doc_sanitize, rust_doc_comment">\ ${doc(s, c)}\ -#[derive(${', '.join(traits)})] + #[serde_with::serde_as(crate = "::client::serde_with")] + #[derive(${', '.join(traits)})] % if s.type == 'object': ${_new_object(s, s.get('properties'), c, allow_optionals)}\ % elif s.type == 'array': From a1041d6e1651e3a550971b0e8173190924aa1411 Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Sat, 8 Oct 2022 23:12:36 -0700 Subject: [PATCH 25/27] Fix serde_as not reporting changed member types --- src/generator/lib/rust_type.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/generator/lib/rust_type.py b/src/generator/lib/rust_type.py index 5a07765672..e86a747fcc 100644 --- a/src/generator/lib/rust_type.py +++ b/src/generator/lib/rust_type.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import Optional, List, Tuple from copy import deepcopy @@ -8,6 +8,13 @@ class RustType: self.members = members def serde_replace_inner_ty(self, from_to): + """Create a type which can be used by serde_with::serde_as to (de)serialize this type. + Substitutions described by from_to are used for types which require special (de)serialization support. + Returns true if the type changes (and thus, requires special (de)serialization). + + :param from_to: + :return: + """ if self.members is None: return False @@ -19,11 +26,13 @@ class RustType: else: # serde_as fails to compile if type definition includes # types without custom serialization - if not member.serde_replace_inner_ty(from_to): + if member.serde_replace_inner_ty(from_to): + changed = True + else: self.members[i] = Base("_") return changed - def serde_as(self) -> "RustType": + def serde_as(self) -> Tuple["RustType", bool]: copied = deepcopy(self) from_to = { Vec(Base("u8")): Base("::client::serde::urlsafe_base64::Wrapper"), @@ -73,4 +82,4 @@ class HashMap(RustType): class Base(RustType): def __init__(self, name): - super().__init__(name) \ No newline at end of file + super().__init__(name) From 1c04f662d17fd131c9342bb4ccccb728e03b9376 Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Sun, 9 Oct 2022 00:04:30 -0700 Subject: [PATCH 26/27] Use correct string impls for http headers --- google-apis-common/src/serde.rs | 14 +++++++++++--- src/generator/lib/util.py | 10 ++++++++++ src/generator/templates/api/lib/mbuild.mako | 11 ++++++----- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/google-apis-common/src/serde.rs b/google-apis-common/src/serde.rs index a34d4318ad..e7de0b1c4e 100644 --- a/google-apis-common/src/serde.rs +++ b/google-apis-common/src/serde.rs @@ -97,7 +97,7 @@ pub mod duration { } } - fn duration_to_string(duration: &Duration) -> String { + pub fn to_string(duration: &Duration) -> String { let seconds = duration.num_seconds(); let nanoseconds = (*duration - Duration::seconds(seconds)) .num_nanoseconds() @@ -121,7 +121,7 @@ pub mod duration { where S: serde::Serializer, { - s.serialize_str(&duration_to_string(value)) + s.serialize_str(&to_string(value)) } } @@ -142,12 +142,16 @@ pub mod urlsafe_base64 { pub struct Wrapper; + pub fn to_string(bytes: &Vec) -> String { + base64::encode_config(bytes, base64::URL_SAFE) + } + impl SerializeAs> for Wrapper { fn serialize_as(value: &Vec, s: S) -> Result where S: Serializer, { - s.serialize_str(&base64::encode_config(value, base64::URL_SAFE)) + s.serialize_str(&to_string(value)) } } @@ -162,6 +166,10 @@ pub mod urlsafe_base64 { } } +pub fn datetime_to_string(datetime: &chrono::DateTime) -> String { + datetime.to_rfc3339_opts(chrono::SecondsFormat::Millis, true) +} + #[cfg(test)] mod test { use super::{duration, urlsafe_base64}; diff --git a/src/generator/lib/util.py b/src/generator/lib/util.py index 5379c04a55..a9f789c3e7 100644 --- a/src/generator/lib/util.py +++ b/src/generator/lib/util.py @@ -1241,5 +1241,15 @@ def size_to_bytes(size): # end handle errors gracefully +def string_impl(p): + return { + "google-duration": "::client::serde::duration::to_string", + "byte": "::client::serde::urlsafe_base64::to_string", + "google-datetime": "::client::serde::datetime_to_string", + "date-time": "::client::serde::datetime_to_string", + "google-fieldmask": "(|x: &client::FieldMask| x.to_string())" + }.get(p.get("format"), "(|x: &dyn std::fmt::Display| x.to_string())") + + if __name__ == '__main__': raise AssertionError('For import only') diff --git a/src/generator/templates/api/lib/mbuild.mako b/src/generator/templates/api/lib/mbuild.mako index 1c970b828e..e6db099470 100644 --- a/src/generator/templates/api/lib/mbuild.mako +++ b/src/generator/templates/api/lib/mbuild.mako @@ -11,7 +11,7 @@ DELEGATE_PROPERTY_NAME, struct_type_bounds_s, scope_url_to_variant, re_find_replacements, ADD_PARAM_FN, ADD_PARAM_MEDIA_EXAMPLE, upload_action_fn, METHODS_RESOURCE, method_name_to_variant, size_to_bytes, method_default_scope, - is_repeated_property, setter_fn_name, ADD_SCOPE_FN, rust_doc_sanitize, items) + is_repeated_property, setter_fn_name, ADD_SCOPE_FN, rust_doc_sanitize, items, string_impl) def get_parts(part_prop): if not part_prop: @@ -534,6 +534,7 @@ match result { % for p in field_params: <% pname = 'self.' + property(p.name) # property identifier + to_string_impl = string_impl(p) %>\ ## parts can also be derived from the request, but we do that only if it's not set % if p.name == 'part' and request_value: @@ -556,15 +557,15 @@ match result { % if p.get('repeated', False): if ${pname}.len() > 0 { for f in ${pname}.iter() { - params.push(("${p.name}", f.to_string())); + params.push(("${p.name}", ${to_string_impl}(f))); } } % elif not is_required_property(p): - if let Some(value) = ${pname} { - params.push(("${p.name}", value.to_string())); + if let Some(value) = ${pname}.as_ref() { + params.push(("${p.name}", ${to_string_impl}(value))); } % else: - params.push(("${p.name}", ${pname}.to_string())); + params.push(("${p.name}", ${to_string_impl}(&${pname}))); % endif % endfor ## Additional params - may not overlap with optional params From 59874c9c98775fe5f48c9ca403efaef91392e320 Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Mon, 10 Oct 2022 00:53:24 +0000 Subject: [PATCH 27/27] Remove indentation --- src/generator/templates/api/lib/schema.mako | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/generator/templates/api/lib/schema.mako b/src/generator/templates/api/lib/schema.mako index 0a5b87ca05..55e2c5fdcf 100644 --- a/src/generator/templates/api/lib/schema.mako +++ b/src/generator/templates/api/lib/schema.mako @@ -83,8 +83,8 @@ ${struct} { _never_set: Option } <%block filter="rust_doc_sanitize, rust_doc_comment">\ ${doc(s, c)}\ - #[serde_with::serde_as(crate = "::client::serde_with")] - #[derive(${', '.join(traits)})] +#[serde_with::serde_as(crate = "::client::serde_with")] +#[derive(${', '.join(traits)})] % if s.type == 'object': ${_new_object(s, s.get('properties'), c, allow_optionals)}\ % elif s.type == 'array':