From c482fb21281f3884fe5b09608dddb38cf4d774e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20=C3=81lvarez?= Date: Fri, 14 May 2021 04:16:23 +0200 Subject: [PATCH] only US --- README.md | 8 +++++- blizzard.py | 3 +++ cron.sh | 11 +++++++++ spec.py | 6 ++--- updater.py | 71 +++++++++++++++++++++++++---------------------------- 5 files changed, 58 insertions(+), 41 deletions(-) create mode 100755 cron.sh diff --git a/README.md b/README.md index a68625f..30c7dea 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,13 @@ Create a `.credentials` file with `client_id:secret` from https://develop.battle.net/access +With no crontab execute `python3 updater.py`. + Crontab, every day at 12:30h: ``` -30 12 * * * time python3 updater.py +30 12 * * * cron.sh ``` + +Only `US` region is explored because all API are the same and this is the default API for Blizzard. + +Without `locale` we get all the translations, so I don't iterate through locales. If you ask for a locale, you get the value right in the field. diff --git a/blizzard.py b/blizzard.py index 288c817..5b09a8c 100644 --- a/blizzard.py +++ b/blizzard.py @@ -11,3 +11,6 @@ def get_bnet_host(region: str) -> str: def get_credentials(path: str) -> Tuple[str, str]: return tuple(open(os.path.join(path, '.credentials')).read().strip().split(':')) + +def get_by_key(values: list, key: str, value): + return next(x for x in values if x[key] == value) diff --git a/cron.sh b/cron.sh new file mode 100755 index 0000000..63412b3 --- /dev/null +++ b/cron.sh @@ -0,0 +1,11 @@ +#! /bin/bash + +TIME=`(time python3 updater.py ) 2>&1` + +# add every new file +git add raw +git add craft +git add last-modified.db + +git commit -m 'daily cron' -m "$TIME" +git push diff --git a/spec.py b/spec.py index a804473..f0884e3 100644 --- a/spec.py +++ b/spec.py @@ -58,9 +58,9 @@ apis = [ {'group': 'Covenant', 'path': '/data/wow/covenant/index', 'namespaces': ['static'], 'index': True}, {'group': 'Covenant', 'path': '/data/wow/covenant/soulbind/index', 'namespaces': ['static'], 'index': True}, {'group': 'Covenant', 'path': '/data/wow/covenant/conduit/index', 'namespaces': ['static'], 'index': True}, - {'group': 'Creature', 'path': '/data/wow/creature-family/index', 'namespaces': ['static'], 'index': True}, - {'group': 'Creature', 'path': '/data/wow/creature-type/index', 'namespaces': ['static'], 'index': True}, - {'group': 'Guild Crest', 'path': '/data/wow/guild-crest/index', 'namespaces': ['static'], 'index': True}, + {'group': 'Creature', 'path': '/data/wow/creature-family/index', 'namespaces': ['static', 'static-classic'], 'index': True}, + {'group': 'Creature', 'path': '/data/wow/creature-type/index', 'namespaces': ['static', 'static-classic'], 'index': True}, + {'group': 'Guild Crest', 'path': '/data/wow/guild-crest/index', 'namespaces': ['static', 'static-classic'], 'index': True}, {'group': '', 'path': '/data/wow/item-class/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/item-set/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/journal-expansion/index', 'namespaces': ['static'], 'index': True}, diff --git a/updater.py b/updater.py index 5414714..45476d6 100644 --- a/updater.py +++ b/updater.py @@ -31,9 +31,9 @@ class Updater(object): return oauth - def api_call(self, region: str, path: str, namespace: str, locale: str, access_token: str, headers=None) -> requests.Response: + def api_call(self, region: str, path: str, namespace: str, access_token: str, headers=None) -> requests.Response: url = f"{blizzard.get_api_host(region)}{path}" - qs = {'namespace': f"{namespace}-{region}", 'local': locale, 'access_token': access_token} + qs = {'namespace': f"{namespace}-{region}", 'access_token': access_token} api = self.http.get(url, params=qs, headers=headers) #, headers={'Authorization': f"Authorization: Token {access_token}"}) api.raise_for_status() @@ -48,8 +48,8 @@ class Updater(object): raise - def save_raw(self, region: str, path: str, namespace: str, locale: str, raw: requests.Response): - dst = f"{config.raw}/{region}/{locale}/{path.replace('/', '_')}.{namespace}.json" + def save_raw(self, region: str, path: str, namespace: str, raw: requests.Response): + dst = f"{config.raw}/{path.replace('/', '_')}.{namespace}.json" self.create_dst(dst) @@ -57,8 +57,8 @@ class Updater(object): f.write(json.dumps(raw.json(), indent=2)) - def get_last_modified(self, region: str, path: str, namespace: str, locale: str): - key = f"{region}.{locale}.{path}.{namespace}" + def get_last_modified(self, region: str, path: str, namespace: str): + key = f"{path}.{namespace}" # cached! if self.last_modified is not None and key in self.last_modified: @@ -77,8 +77,8 @@ class Updater(object): return None - def save_last_modified(self, region: str, path: str, namespace: str, locale: str, modified: str): - key = f"{region}.{locale}.{path}.{namespace}" + def save_last_modified(self, region: str, path: str, namespace: str, modified: str): + key = f"{path}.{namespace}" db = os.path.join(config.pwd, 'last-modified.db') # never trust the cache, open, modify and save updated data for safety @@ -98,40 +98,37 @@ class Updater(object): os.replace(tmp_path, db) - def iterate_index(self): - for region in spec.regions: - # access token for region - try: - oauth = self.region_oauth(region['code']) - access_token = oauth['access_token'] - except (requests.exceptions.HTTPError, KeyError) as e: - log(region, type(e), e) + def iterate_index(self, region: dict): + # access token for region + try: + oauth = self.region_oauth(region) + access_token = oauth['access_token'] + except (requests.exceptions.HTTPError, KeyError) as e: + log(region, type(e), e) + return + + # loop every api + for api in spec.apis: + if not 'index' in api or not api['index']: continue - # loop every api - for api in spec.apis: - if not 'index' in api or not api['index']: + for namespace in api['namespaces']: + try: + last_modified = self.get_last_modified(region, api['path'], namespace) + headers = {'If-Modified-Since': last_modified} if last_modified is not None else None + + response = self.api_call(region, api['path'], namespace, access_token, headers=headers) + + if response.status_code == 200: + self.save_last_modified(region, api['path'], namespace, response.headers['Last-Modified']) + self.save_raw(region, api['path'], namespace, response) + + except requests.exceptions.HTTPError as e: + log(e) continue - for locale in region['locales']: - # retail or classic - for namespace in api['namespaces']: - try: - last_modified = self.get_last_modified(region['code'], api['path'], namespace, locale) - headers = {'If-Modified-Since': last_modified} if last_modified is not None else None - - response = self.api_call(region['code'], api['path'], namespace, locale, access_token, headers=headers) - - if response.status_code == 200: - self.save_last_modified(region['code'], api['path'], namespace, locale, response.headers['Last-Modified']) - self.save_raw(region['code'], api['path'], namespace, locale, response) - - except requests.exceptions.HTTPError as e: - log(e) - continue - if __name__ == "__main__": updater = Updater() - updater.iterate_index() + updater.iterate_index(blizzard.get_by_key(spec.regions, 'code', 'us')['code']) #updater.iterate_links()