From 1e3c2750e5a87dbea7a81ddae37fc1786db778db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20=C3=81lvarez?= Date: Fri, 14 May 2021 03:05:14 +0200 Subject: [PATCH] missing files --- README.md | 6 +++ spec.py | 123 ++++++++++++++++++++++++++++++++++------------------- updater.py | 79 +++++++++++++++++++++++++++++----- 3 files changed, 155 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 57c7111..a68625f 100644 --- a/README.md +++ b/README.md @@ -1 +1,7 @@ Create a `.credentials` file with `client_id:secret` from https://develop.battle.net/access + +Crontab, every day at 12:30h: + +``` +30 12 * * * time python3 updater.py +``` diff --git a/spec.py b/spec.py index 60922bc..a804473 100644 --- a/spec.py +++ b/spec.py @@ -12,51 +12,88 @@ regions = [ #{'code': 'cn', 'locales': ['zh_CN']} ] +groups = [ + 'Achievement', + 'Auction House', + 'Azerite Essence', + 'Connected Realm', + 'Covenant', + 'Creature', + 'Guild Crest', + 'Item', + 'Journal', + 'Media Search', + 'Modified Crafting', + 'Mount', + 'Mythic Keystone Affix', + 'Mythic Keystone Dungeon', + 'Mythic Keystone Leaderboard', + 'Mythic Raid Leaderboard', + 'Pet', + 'Playable Class', + 'Playable Race', + 'Playable Specialization', + 'Power Type', + 'Profession', + 'PvP Season', + 'PvP Tier', + 'Quest', + 'Realm', + 'Region', + 'Reputations', + 'Spell', + 'Talent', + 'Tech Talent', + 'Title', + 'WoW Token', +] + """ https://develop.battle.net/documentation/world-of-warcraft/game-data-apis """ apis = [ - {'path': '/data/wow/achievement-category/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/achievement/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/connected-realm/index', 'namespaces': ['dynamic', 'dynamic-classic'], 'index': True}, - {'path': '/data/wow/covenant/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/covenant/soulbind/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/covenant/conduit/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/creature-family/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/creature-type/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/item-class/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/item-set/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/journal-expansion/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/journal-encounter/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/journal-instance/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/modified-crafting/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/modified-crafting/category/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/modified-crafting/reagent-slot-type/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/mount/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/keystone-affix/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/mythic-keystone/dungeon/index', 'namespaces': ['dynamic'], 'index': True}, - {'path': '/data/wow/mythic-keystone/index', 'namespaces': ['dynamic'], 'index': True}, - {'path': '/data/wow/mythic-keystone/period/index', 'namespaces': ['dynamic'], 'index': True}, - {'path': '/data/wow/pet/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/pet-ability/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/playable-class/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/playable-race/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/playable-specialization/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/power-type/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/profession/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/pvp-season/index', 'namespaces': ['dynamic'], 'index': True}, - {'path': '/data/wow/pvp-tier/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/quest/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/quest/category/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/quest/area/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/quest/type/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/realm/index', 'namespaces': ['dynamic'], 'index': True}, - {'path': '/data/wow/region/index', 'namespaces': ['dynamic'], 'index': True}, - {'path': '/data/wow/reputation-faction/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/reputation-tiers/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/talent/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/pvp-talent/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/tech-talent-tree/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/tech-talent/index', 'namespaces': ['static'], 'index': True}, - {'path': '/data/wow/title/index', 'namespaces': ['static'], 'index': True}, + {'group': 'Achievement', 'path': '/data/wow/achievement-category/index', 'namespaces': ['static'], 'index': True}, + {'group': 'Achievement', 'path': '/data/wow/achievement/index', 'namespaces': ['static'], 'index': True}, + {'group': 'Connected Realm', 'path': '/data/wow/connected-realm/index', 'namespaces': ['dynamic', 'dynamic-classic'], 'index': True}, + {'group': 'Covenant', 'path': '/data/wow/covenant/index', 'namespaces': ['static'], 'index': True}, + {'group': 'Covenant', 'path': '/data/wow/covenant/soulbind/index', 'namespaces': ['static'], 'index': True}, + {'group': 'Covenant', 'path': '/data/wow/covenant/conduit/index', 'namespaces': ['static'], 'index': True}, + {'group': 'Creature', 'path': '/data/wow/creature-family/index', 'namespaces': ['static'], 'index': True}, + {'group': 'Creature', 'path': '/data/wow/creature-type/index', 'namespaces': ['static'], 'index': True}, + {'group': 'Guild Crest', 'path': '/data/wow/guild-crest/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/item-class/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/item-set/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/journal-expansion/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/journal-encounter/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/journal-instance/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/modified-crafting/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/modified-crafting/category/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/modified-crafting/reagent-slot-type/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/mount/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/keystone-affix/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/mythic-keystone/dungeon/index', 'namespaces': ['dynamic'], 'index': True}, + {'group': '', 'path': '/data/wow/mythic-keystone/index', 'namespaces': ['dynamic'], 'index': True}, + {'group': '', 'path': '/data/wow/mythic-keystone/period/index', 'namespaces': ['dynamic'], 'index': True}, + {'group': '', 'path': '/data/wow/pet/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/pet-ability/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/playable-class/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/playable-race/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/playable-specialization/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/power-type/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/profession/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/pvp-season/index', 'namespaces': ['dynamic'], 'index': True}, + {'group': '', 'path': '/data/wow/pvp-tier/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/quest/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/quest/category/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/quest/area/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/quest/type/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/realm/index', 'namespaces': ['dynamic'], 'index': True}, + {'group': '', 'path': '/data/wow/region/index', 'namespaces': ['dynamic'], 'index': True}, + {'group': '', 'path': '/data/wow/reputation-faction/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/reputation-tiers/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/talent/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/pvp-talent/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/tech-talent-tree/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/tech-talent/index', 'namespaces': ['static'], 'index': True}, + {'group': '', 'path': '/data/wow/title/index', 'namespaces': ['static'], 'index': True}, ] diff --git a/updater.py b/updater.py index 5d0d38f..5414714 100644 --- a/updater.py +++ b/updater.py @@ -4,18 +4,24 @@ import errno import requests import json import datetime +import tempfile import config import blizzard import spec + def log(*data): print(datetime.datetime.now(), '|', *data) + class Updater(object): + def __init__(self): self.http = requests.Session() self.credentials = blizzard.get_credentials(config.pwd) + self.last_modified = None + def region_oauth(self, region: str) -> dict: api = self.http.post(f"{blizzard.get_bnet_host(region)}/oauth/token", data={'grant_type': 'client_credentials'}, auth=self.credentials) @@ -24,11 +30,16 @@ class Updater(object): api.raise_for_status() return oauth - def api_call(self, url: str) -> requests.Response: - api = self.http.get(url) #, headers={'Authorization': f"Authorization: Token {access_token}"}) + + def api_call(self, region: str, path: str, namespace: str, locale: str, access_token: str, headers=None) -> requests.Response: + url = f"{blizzard.get_api_host(region)}{path}" + qs = {'namespace': f"{namespace}-{region}", 'local': locale, 'access_token': access_token} + + api = self.http.get(url, params=qs, headers=headers) #, headers={'Authorization': f"Authorization: Token {access_token}"}) api.raise_for_status() return api + def create_dst(self, dst: str): try: os.makedirs(os.path.dirname(dst)) @@ -36,9 +47,56 @@ class Updater(object): if e.errno != errno.EEXIST: raise - def save(self, dst, content): + + def save_raw(self, region: str, path: str, namespace: str, locale: str, raw: requests.Response): + dst = f"{config.raw}/{region}/{locale}/{path.replace('/', '_')}.{namespace}.json" + + self.create_dst(dst) + with open(dst, 'w+') as f: - f.write(json.dumps(content.json(), indent=4)) + f.write(json.dumps(raw.json(), indent=2)) + + + def get_last_modified(self, region: str, path: str, namespace: str, locale: str): + key = f"{region}.{locale}.{path}.{namespace}" + + # cached! + if self.last_modified is not None and key in self.last_modified: + return self.last_modified[key] + + # not cached, check db again + db = os.path.join(config.pwd, 'last-modified.db') + + with open(db, 'r') as f: + data = json.load(f) + self.last_modified = data + + try: + return data[key] + except KeyError: + return None + + + def save_last_modified(self, region: str, path: str, namespace: str, locale: str, modified: str): + key = f"{region}.{locale}.{path}.{namespace}" + db = os.path.join(config.pwd, 'last-modified.db') + + # never trust the cache, open, modify and save updated data for safety + with open(db, 'r') as f: + data = json.load(f) + + data[key] = modified + # update cache with new data + self.last_modified = data + + # safe write: tmp write and then mv + fd, tmp_path = tempfile.mkstemp(dir=config.pwd) + try: + with os.fdopen(fd, 'w') as tmp: + json.dump(data, tmp, indent=2) + finally: + os.replace(tmp_path, db) + def iterate_index(self): for region in spec.regions: @@ -59,19 +117,20 @@ class Updater(object): # retail or classic for namespace in api['namespaces']: try: - dst = f"{config.raw}/{region['code']}/{locale}/{api['path'].replace('/', '_')}.{namespace}.json" + last_modified = self.get_last_modified(region['code'], api['path'], namespace, locale) + headers = {'If-Modified-Since': last_modified} if last_modified is not None else None - url = f"{blizzard.get_api_host(region['code'])}{api['path']}" - url += f"?namespace={namespace}-{region['code']}&locale={locale}&access_token={access_token}" - response = self.api_call(url) + response = self.api_call(region['code'], api['path'], namespace, locale, access_token, headers=headers) - self.create_dst(dst) - self.save(dst, response) + if response.status_code == 200: + self.save_last_modified(region['code'], api['path'], namespace, locale, response.headers['Last-Modified']) + self.save_raw(region['code'], api['path'], namespace, locale, response) except requests.exceptions.HTTPError as e: log(e) continue + if __name__ == "__main__": updater = Updater() updater.iterate_index()