diff --git a/Makefile b/Makefile index ad3427355b..8ba21e25d0 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ VENV_DIR := .pyenv PYTHON := $(VENV_DIR)/bin/python PIP := $(VENV_DIR)/bin/pip MAKO_RENDER := etc/bin/mako-render +API_VERSION_GEN := etc/bin/api_version_to_yaml.py TPL := $(PYTHON) $(MAKO_RENDER) MAKO_SRC = src/mako @@ -15,6 +16,7 @@ RUST_SRC = src/rust API_DEPS_TPL = $(MAKO_SRC)/deps.mako API_DEPS = .api.deps API_SHARED_INFO = etc/api/shared.yaml +API_LIST = etc/api/api-list.yaml API_JSON_FILES = $(shell find etc -type f -name '*-api.json') MAKO_LIB_DIR = $(MAKO_SRC)/lib MAKO_LIB_FILES = $(shell find $(MAKO_LIB_DIR) -type f -name '*.*') @@ -39,11 +41,14 @@ $(PYTHON): $(MAKO_RENDER): $(PYTHON) -$(API_DEPS): $(API_SHARED_INFO) $(API_DEPS_TPL) $(MAKO_LIB_FILES) $(MAKO_RENDER) - PYTHONPATH=$(MAKO_LIB_DIR) $(TPL) --template-dir '.' -io $(API_DEPS_TPL)=$@ --data-files $(API_SHARED_INFO) +$(API_DEPS): $(API_SHARED_INFO) $(API_DEPS_TPL) $(MAKO_LIB_FILES) $(MAKO_RENDER) $(API_LIST) + PYTHONPATH=$(MAKO_LIB_DIR) $(TPL) --template-dir '.' -io $(API_DEPS_TPL)=$@ --data-files $(API_SHARED_INFO) $(API_LIST) api-deps: $(API_DEPS) +$(API_LIST): $(API_VERSION_GEN) + $(API_VERSION_GEN) etc/api $@ $@ + include $(API_DEPS) LICENSE.md: $(MAKO_SRC)/LICENSE.md.mako $(API_SHARED_INFO) @@ -56,6 +61,7 @@ regen-apis: clean-apis apis license clean: clean-apis -rm -Rf $(VENV_DIR) -rm $(API_DEPS) + -rm $(API_LIST) update-json: etc/bin/update-json.sh $(GOOGLE_GO_APIS_REPO) etc/api diff --git a/etc/api/api-list.yaml b/etc/api/api-list.yaml new file mode 100644 index 0000000000..247fcd7cc8 --- /dev/null +++ b/etc/api/api-list.yaml @@ -0,0 +1,147 @@ +# DO NOT EDIT !!! +# File created from 'etc/bin/api_version_to_yaml.py etc/api etc/api/api-list.yaml etc/api/api-list.yaml' +# DO NOT EDIT !!! +api: + list: + adexchangebuyer: + - v1.2 + adexchangeseller: + - v1 + admin: + - directory_v1 + adsense: + - v1.2 + adsensehost: + - v4.1 + analytics: + - v2.4 + androidpublisher: + - v1 + appsactivity: + - v1 + appstate: + - v1 + audit: + - v1 + autoscaler: + - v1beta2 + bigquery: + - v2 + blogger: + - v2 + books: + - v1 + calendar: + - v3 + civicinfo: + - us_v1 + cloudlatencytest: + - v2 + cloudmonitoring: + - v2beta2 + compute: + - v1 + container: + - v1beta1 + content: + - v2 + coordinate: + - v1 + customsearch: + - v1 + dataflow: + - v1b4 + datastore: + - v1beta1 + deploymentmanager: + - v2beta1 + dfareporting: + - v1 + discovery: + - v1 + dns: + - v1beta1 + doubleclickbidmanager: + - v1 + doubleclicksearch: + - v2 + drive: + - v1 + fitness: + - v1 + freebase: + - v1 + games: + - v1 + gamesconfiguration: + - v1configuration + gamesmanagement: + - v1management + gan: + - v1beta1 + genomics: + - v1beta + gmail: + - v1 + groupsmigration: + - v1 + groupssettings: + - v1 + identitytoolkit: + - v3 + licensing: + - v1 + manager: + - v1beta2 + mapsengine: + - exp2 + mirror: + - v1 + oauth2: + - v1 + pagespeedonline: + - v1 + plus: + - v1 + plusdomains: + - v1 + prediction: + - v1.2 + pubsub: + - v1beta1 + qpxexpress: + - v1 + replicapool: + - v1beta1 + replicapoolupdater: + - v1beta1 + reseller: + - v1 + resourceviews: + - v1beta1 + siteverification: + - v1 + spectrum: + - v1explorer + sqladmin: + - v1beta1 + storage: + - v1 + tagmanager: + - v1 + taskqueue: + - v1beta1 + tasks: + - v1 + translate: + - v2 + urlshortener: + - v1 + webfonts: + - v1 + webmasters: + - v3 + youtube: + - v3 + youtubeanalytics: + - v1 diff --git a/etc/api/shared.yaml b/etc/api/shared.yaml index 2c05a969a5..9cc75f942a 100644 --- a/etc/api/shared.yaml +++ b/etc/api/shared.yaml @@ -31,151 +31,6 @@ api: - admin # ERROR - webfonts # no oauth2 - customsearch # no oauth2 - list: - # use etc/bin/api_version_to_yaml.py to update with latest available versions - # It's a semi-automated process ... for now - adexchangebuyer: - - v1.2 - adexchangeseller: - - v1 - admin: - - directory_v1 - adsense: - - v1.2 - adsensehost: - - v4.1 - analytics: - - v2.4 - androidpublisher: - - v1 - appsactivity: - - v1 - appstate: - - v1 - audit: - - v1 - autoscaler: - - v1beta2 - bigquery: - - v2 - blogger: - - v2 - books: - - v1 - calendar: - - v3 - civicinfo: - - us_v1 - cloudlatencytest: - - v2 - cloudmonitoring: - - v2beta2 - compute: - - v1 - container: - - v1beta1 - content: - - v2 - coordinate: - - v1 - customsearch: - - v1 - dataflow: - - v1b4 - datastore: - - v1beta1 - deploymentmanager: - - v2beta1 - dfareporting: - - v1 - discovery: - - v1 - dns: - - v1beta1 - doubleclickbidmanager: - - v1 - doubleclicksearch: - - v2 - drive: - - v1 - fitness: - - v1 - freebase: - - v1 - games: - - v1 - gamesconfiguration: - - v1configuration - gamesmanagement: - - v1management - gan: - - v1beta1 - genomics: - - v1beta - gmail: - - v1 - groupsmigration: - - v1 - groupssettings: - - v1 - identitytoolkit: - - v3 - licensing: - - v1 - manager: - - v1beta2 - mapsengine: - - exp2 - mirror: - - v1 - oauth2: - - v1 - pagespeedonline: - - v1 - plus: - - v1 - plusdomains: - - v1 - prediction: - - v1.2 - pubsub: - - v1beta1 - qpxexpress: - - v1 - replicapool: - - v1beta1 - replicapoolupdater: - - v1beta1 - reseller: - - v1 - resourceviews: - - v1beta1 - siteverification: - - v1 - spectrum: - - v1explorer - sqladmin: - - v1beta1 - storage: - - v1 - tagmanager: - - v1 - taskqueue: - - v1beta1 - tasks: - - v1 - translate: - - v2 - urlshortener: - - v1 - webfonts: - - v1 - webmasters: - - v3 - youtube: - - v3 - youtubeanalytics: - - v1 base_path: "etc/api" terms: # how to actually do something with the API diff --git a/etc/bin/api_version_to_yaml.py b/etc/bin/api_version_to_yaml.py index 7c0afc3840..47196be404 100755 --- a/etc/bin/api_version_to_yaml.py +++ b/etc/bin/api_version_to_yaml.py @@ -14,19 +14,19 @@ join = os.path.join if __name__ != '__main__': raise AssertionError("Not for import") -if len(sys.argv) != 2: - sys.stderr.write("USAGE: , i.e. etc/api\n") +if len(sys.argv) != 4: + sys.stderr.write("USAGE: , i.e. etc/api etc/api/api-list.yaml out.yaml\n") sys.exit(1) api_base = sys.argv[1] if not isdir(api_base): raise ValueError("Directory '%s' not accessible" % api_base) -yaml_path = join(api_base, 'shared.yaml') +yaml_path = sys.argv[2] if not isfile(yaml_path): - raise AssertionError("Didn't find yaml data at '%s'" % yaml_path) + raise ValueError("Didn't find yaml data at '%s'" % yaml_path) -api_data = list(yaml.load_all(open(yaml_path, 'r')))[0]['api']['list'] +api_data = yaml.load(open(yaml_path, 'r'))['api']['list'] for api_name in sorted(os.listdir(api_base)): api_path = join(api_base, api_name) if not isdir(api_path): @@ -41,6 +41,10 @@ for api_name in sorted(os.listdir(api_base)): api_data[api_name] = list(sorted(versions)) # end for each item in api-base -yaml.dump(dict(api=dict(list=api_data)), sys.stdout, default_flow_style=False) - +fp = open(sys.argv[3], 'wb') +fp.write("# DO NOT EDIT !!!\n") +fp.write("# Created by '%s'\n" % ' '.join(sys.argv)) +fp.write("# DO NOT EDIT !!!\n") +yaml.dump(dict(api=dict(list=api_data)), fp, default_flow_style=False) +fp.close() diff --git a/etc/bin/mako-render b/etc/bin/mako-render index 84b3a99c67..422de6fc03 100644 --- a/etc/bin/mako-render +++ b/etc/bin/mako-render @@ -12,6 +12,7 @@ from mako.lookup import TemplateLookup from mako import exceptions +# From https://github.com/Byron/bcore class DictObject(object): """An object which wraps a dictionary to allow object.key access. @@ -240,10 +241,9 @@ def load_data(datafiles): else: raise ValueError("Invalid data-file '%s', must be .json, .yaml or .yml" % filename) assert data is not None - if not namespace: - mydata.update(data) - else: - mydata.update({namespace: data}) + if namespace: + data = {namespace: data} + mydata = merge(mydata, data) return mydata @@ -320,6 +320,21 @@ def cmdline(argv=None): _exit() # end for each input file +# From http://stackoverflow.com/questions/7204805/dictionaries-of-dictionaries-merge +def merge(a, b, path=None): + if path is None: path = [] + for key in b: + if key in a: + if isinstance(a[key], dict) and isinstance(b[key], dict): + merge(a[key], b[key], path + [str(key)]) + elif a[key] == b[key]: + pass # same leaf value + else: + raise Exception('Conflict at %s' % '.'.join(path + [str(key)])) + else: + a[key] = b[key] + return a + if __name__ == "__main__": cmdline()