From 9377220c59cf9a8a29720b0528b9263e9e947580 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 10 Mar 2015 15:00:43 +0100 Subject: [PATCH] chore(data-merge): api-list is now in separte file This file is completely generated, and allows us to easily bring in new versions after each json update. To make that work, we simple merge all data handed to mako-render, inside of it. That way, we can put 'api/list' data in any yaml. --- Makefile | 10 ++- etc/api/api-list.yaml | 147 +++++++++++++++++++++++++++++++++ etc/api/shared.yaml | 145 -------------------------------- etc/bin/api_version_to_yaml.py | 18 ++-- etc/bin/mako-render | 23 +++++- 5 files changed, 185 insertions(+), 158 deletions(-) create mode 100644 etc/api/api-list.yaml diff --git a/Makefile b/Makefile index ad3427355b..8ba21e25d0 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ VENV_DIR := .pyenv PYTHON := $(VENV_DIR)/bin/python PIP := $(VENV_DIR)/bin/pip MAKO_RENDER := etc/bin/mako-render +API_VERSION_GEN := etc/bin/api_version_to_yaml.py TPL := $(PYTHON) $(MAKO_RENDER) MAKO_SRC = src/mako @@ -15,6 +16,7 @@ RUST_SRC = src/rust API_DEPS_TPL = $(MAKO_SRC)/deps.mako API_DEPS = .api.deps API_SHARED_INFO = etc/api/shared.yaml +API_LIST = etc/api/api-list.yaml API_JSON_FILES = $(shell find etc -type f -name '*-api.json') MAKO_LIB_DIR = $(MAKO_SRC)/lib MAKO_LIB_FILES = $(shell find $(MAKO_LIB_DIR) -type f -name '*.*') @@ -39,11 +41,14 @@ $(PYTHON): $(MAKO_RENDER): $(PYTHON) -$(API_DEPS): $(API_SHARED_INFO) $(API_DEPS_TPL) $(MAKO_LIB_FILES) $(MAKO_RENDER) - PYTHONPATH=$(MAKO_LIB_DIR) $(TPL) --template-dir '.' -io $(API_DEPS_TPL)=$@ --data-files $(API_SHARED_INFO) +$(API_DEPS): $(API_SHARED_INFO) $(API_DEPS_TPL) $(MAKO_LIB_FILES) $(MAKO_RENDER) $(API_LIST) + PYTHONPATH=$(MAKO_LIB_DIR) $(TPL) --template-dir '.' -io $(API_DEPS_TPL)=$@ --data-files $(API_SHARED_INFO) $(API_LIST) api-deps: $(API_DEPS) +$(API_LIST): $(API_VERSION_GEN) + $(API_VERSION_GEN) etc/api $@ $@ + include $(API_DEPS) LICENSE.md: $(MAKO_SRC)/LICENSE.md.mako $(API_SHARED_INFO) @@ -56,6 +61,7 @@ regen-apis: clean-apis apis license clean: clean-apis -rm -Rf $(VENV_DIR) -rm $(API_DEPS) + -rm $(API_LIST) update-json: etc/bin/update-json.sh $(GOOGLE_GO_APIS_REPO) etc/api diff --git a/etc/api/api-list.yaml b/etc/api/api-list.yaml new file mode 100644 index 0000000000..247fcd7cc8 --- /dev/null +++ b/etc/api/api-list.yaml @@ -0,0 +1,147 @@ +# DO NOT EDIT !!! +# File created from 'etc/bin/api_version_to_yaml.py etc/api etc/api/api-list.yaml etc/api/api-list.yaml' +# DO NOT EDIT !!! +api: + list: + adexchangebuyer: + - v1.2 + adexchangeseller: + - v1 + admin: + - directory_v1 + adsense: + - v1.2 + adsensehost: + - v4.1 + analytics: + - v2.4 + androidpublisher: + - v1 + appsactivity: + - v1 + appstate: + - v1 + audit: + - v1 + autoscaler: + - v1beta2 + bigquery: + - v2 + blogger: + - v2 + books: + - v1 + calendar: + - v3 + civicinfo: + - us_v1 + cloudlatencytest: + - v2 + cloudmonitoring: + - v2beta2 + compute: + - v1 + container: + - v1beta1 + content: + - v2 + coordinate: + - v1 + customsearch: + - v1 + dataflow: + - v1b4 + datastore: + - v1beta1 + deploymentmanager: + - v2beta1 + dfareporting: + - v1 + discovery: + - v1 + dns: + - v1beta1 + doubleclickbidmanager: + - v1 + doubleclicksearch: + - v2 + drive: + - v1 + fitness: + - v1 + freebase: + - v1 + games: + - v1 + gamesconfiguration: + - v1configuration + gamesmanagement: + - v1management + gan: + - v1beta1 + genomics: + - v1beta + gmail: + - v1 + groupsmigration: + - v1 + groupssettings: + - v1 + identitytoolkit: + - v3 + licensing: + - v1 + manager: + - v1beta2 + mapsengine: + - exp2 + mirror: + - v1 + oauth2: + - v1 + pagespeedonline: + - v1 + plus: + - v1 + plusdomains: + - v1 + prediction: + - v1.2 + pubsub: + - v1beta1 + qpxexpress: + - v1 + replicapool: + - v1beta1 + replicapoolupdater: + - v1beta1 + reseller: + - v1 + resourceviews: + - v1beta1 + siteverification: + - v1 + spectrum: + - v1explorer + sqladmin: + - v1beta1 + storage: + - v1 + tagmanager: + - v1 + taskqueue: + - v1beta1 + tasks: + - v1 + translate: + - v2 + urlshortener: + - v1 + webfonts: + - v1 + webmasters: + - v3 + youtube: + - v3 + youtubeanalytics: + - v1 diff --git a/etc/api/shared.yaml b/etc/api/shared.yaml index 2c05a969a5..9cc75f942a 100644 --- a/etc/api/shared.yaml +++ b/etc/api/shared.yaml @@ -31,151 +31,6 @@ api: - admin # ERROR - webfonts # no oauth2 - customsearch # no oauth2 - list: - # use etc/bin/api_version_to_yaml.py to update with latest available versions - # It's a semi-automated process ... for now - adexchangebuyer: - - v1.2 - adexchangeseller: - - v1 - admin: - - directory_v1 - adsense: - - v1.2 - adsensehost: - - v4.1 - analytics: - - v2.4 - androidpublisher: - - v1 - appsactivity: - - v1 - appstate: - - v1 - audit: - - v1 - autoscaler: - - v1beta2 - bigquery: - - v2 - blogger: - - v2 - books: - - v1 - calendar: - - v3 - civicinfo: - - us_v1 - cloudlatencytest: - - v2 - cloudmonitoring: - - v2beta2 - compute: - - v1 - container: - - v1beta1 - content: - - v2 - coordinate: - - v1 - customsearch: - - v1 - dataflow: - - v1b4 - datastore: - - v1beta1 - deploymentmanager: - - v2beta1 - dfareporting: - - v1 - discovery: - - v1 - dns: - - v1beta1 - doubleclickbidmanager: - - v1 - doubleclicksearch: - - v2 - drive: - - v1 - fitness: - - v1 - freebase: - - v1 - games: - - v1 - gamesconfiguration: - - v1configuration - gamesmanagement: - - v1management - gan: - - v1beta1 - genomics: - - v1beta - gmail: - - v1 - groupsmigration: - - v1 - groupssettings: - - v1 - identitytoolkit: - - v3 - licensing: - - v1 - manager: - - v1beta2 - mapsengine: - - exp2 - mirror: - - v1 - oauth2: - - v1 - pagespeedonline: - - v1 - plus: - - v1 - plusdomains: - - v1 - prediction: - - v1.2 - pubsub: - - v1beta1 - qpxexpress: - - v1 - replicapool: - - v1beta1 - replicapoolupdater: - - v1beta1 - reseller: - - v1 - resourceviews: - - v1beta1 - siteverification: - - v1 - spectrum: - - v1explorer - sqladmin: - - v1beta1 - storage: - - v1 - tagmanager: - - v1 - taskqueue: - - v1beta1 - tasks: - - v1 - translate: - - v2 - urlshortener: - - v1 - webfonts: - - v1 - webmasters: - - v3 - youtube: - - v3 - youtubeanalytics: - - v1 base_path: "etc/api" terms: # how to actually do something with the API diff --git a/etc/bin/api_version_to_yaml.py b/etc/bin/api_version_to_yaml.py index 7c0afc3840..47196be404 100755 --- a/etc/bin/api_version_to_yaml.py +++ b/etc/bin/api_version_to_yaml.py @@ -14,19 +14,19 @@ join = os.path.join if __name__ != '__main__': raise AssertionError("Not for import") -if len(sys.argv) != 2: - sys.stderr.write("USAGE: , i.e. etc/api\n") +if len(sys.argv) != 4: + sys.stderr.write("USAGE: , i.e. etc/api etc/api/api-list.yaml out.yaml\n") sys.exit(1) api_base = sys.argv[1] if not isdir(api_base): raise ValueError("Directory '%s' not accessible" % api_base) -yaml_path = join(api_base, 'shared.yaml') +yaml_path = sys.argv[2] if not isfile(yaml_path): - raise AssertionError("Didn't find yaml data at '%s'" % yaml_path) + raise ValueError("Didn't find yaml data at '%s'" % yaml_path) -api_data = list(yaml.load_all(open(yaml_path, 'r')))[0]['api']['list'] +api_data = yaml.load(open(yaml_path, 'r'))['api']['list'] for api_name in sorted(os.listdir(api_base)): api_path = join(api_base, api_name) if not isdir(api_path): @@ -41,6 +41,10 @@ for api_name in sorted(os.listdir(api_base)): api_data[api_name] = list(sorted(versions)) # end for each item in api-base -yaml.dump(dict(api=dict(list=api_data)), sys.stdout, default_flow_style=False) - +fp = open(sys.argv[3], 'wb') +fp.write("# DO NOT EDIT !!!\n") +fp.write("# Created by '%s'\n" % ' '.join(sys.argv)) +fp.write("# DO NOT EDIT !!!\n") +yaml.dump(dict(api=dict(list=api_data)), fp, default_flow_style=False) +fp.close() diff --git a/etc/bin/mako-render b/etc/bin/mako-render index 84b3a99c67..422de6fc03 100644 --- a/etc/bin/mako-render +++ b/etc/bin/mako-render @@ -12,6 +12,7 @@ from mako.lookup import TemplateLookup from mako import exceptions +# From https://github.com/Byron/bcore class DictObject(object): """An object which wraps a dictionary to allow object.key access. @@ -240,10 +241,9 @@ def load_data(datafiles): else: raise ValueError("Invalid data-file '%s', must be .json, .yaml or .yml" % filename) assert data is not None - if not namespace: - mydata.update(data) - else: - mydata.update({namespace: data}) + if namespace: + data = {namespace: data} + mydata = merge(mydata, data) return mydata @@ -320,6 +320,21 @@ def cmdline(argv=None): _exit() # end for each input file +# From http://stackoverflow.com/questions/7204805/dictionaries-of-dictionaries-merge +def merge(a, b, path=None): + if path is None: path = [] + for key in b: + if key in a: + if isinstance(a[key], dict) and isinstance(b[key], dict): + merge(a[key], b[key], path + [str(key)]) + elif a[key] == b[key]: + pass # same leaf value + else: + raise Exception('Conflict at %s' % '.'.join(path + [str(key)])) + else: + a[key] = b[key] + return a + if __name__ == "__main__": cmdline()