code/blizzard-wow-game-data-apis
code
/
blizzard-wow-game-data-apis
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

missing files

This commit is contained in:
Sergio Álvarez 2021-05-14 03:05:14 +02:00
parent ae1f72b032
commit 1e3c2750e5
3 changed files with 155 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -1 +1,7 @@
Create a `.credentials` file with `client_id:secret` from https://develop.battle.net/access Create a `.credentials` file with `client_id:secret` from https://develop.battle.net/access
Crontab, every day at 12:30h:
```
30 12 * * * time python3 updater.py
```

123
spec.py
View File

@ -12,51 +12,88 @@ regions = [
#{'code': 'cn', 'locales': ['zh_CN']} #{'code': 'cn', 'locales': ['zh_CN']}
] ]
groups = [
'Achievement',
'Auction House',
'Azerite Essence',
'Connected Realm',
'Covenant',
'Creature',
'Guild Crest',
'Item',
'Journal',
'Media Search',
'Modified Crafting',
'Mount',
'Mythic Keystone Affix',
'Mythic Keystone Dungeon',
'Mythic Keystone Leaderboard',
'Mythic Raid Leaderboard',
'Pet',
'Playable Class',
'Playable Race',
'Playable Specialization',
'Power Type',
'Profession',
'PvP Season',
'PvP Tier',
'Quest',
'Realm',
'Region',
'Reputations',
'Spell',
'Talent',
'Tech Talent',
'Title',
'WoW Token',
]
""" """
https://develop.battle.net/documentation/world-of-warcraft/game-data-apis https://develop.battle.net/documentation/world-of-warcraft/game-data-apis
""" """
apis = [ apis = [
{'path': '/data/wow/achievement-category/index', 'namespaces': ['static'], 'index': True}, {'group': 'Achievement', 'path': '/data/wow/achievement-category/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/achievement/index', 'namespaces': ['static'], 'index': True}, {'group': 'Achievement', 'path': '/data/wow/achievement/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/connected-realm/index', 'namespaces': ['dynamic', 'dynamic-classic'], 'index': True}, {'group': 'Connected Realm', 'path': '/data/wow/connected-realm/index', 'namespaces': ['dynamic', 'dynamic-classic'], 'index': True},
{'path': '/data/wow/covenant/index', 'namespaces': ['static'], 'index': True}, {'group': 'Covenant', 'path': '/data/wow/covenant/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/covenant/soulbind/index', 'namespaces': ['static'], 'index': True}, {'group': 'Covenant', 'path': '/data/wow/covenant/soulbind/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/covenant/conduit/index', 'namespaces': ['static'], 'index': True}, {'group': 'Covenant', 'path': '/data/wow/covenant/conduit/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/creature-family/index', 'namespaces': ['static'], 'index': True}, {'group': 'Creature', 'path': '/data/wow/creature-family/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/creature-type/index', 'namespaces': ['static'], 'index': True}, {'group': 'Creature', 'path': '/data/wow/creature-type/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/item-class/index', 'namespaces': ['static'], 'index': True}, {'group': 'Guild Crest', 'path': '/data/wow/guild-crest/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/item-set/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/item-class/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/journal-expansion/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/item-set/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/journal-encounter/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/journal-expansion/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/journal-instance/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/journal-encounter/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/modified-crafting/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/journal-instance/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/modified-crafting/category/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/modified-crafting/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/modified-crafting/reagent-slot-type/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/modified-crafting/category/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/mount/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/modified-crafting/reagent-slot-type/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/keystone-affix/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/mount/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/mythic-keystone/dungeon/index', 'namespaces': ['dynamic'], 'index': True}, {'group': '', 'path': '/data/wow/keystone-affix/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/mythic-keystone/index', 'namespaces': ['dynamic'], 'index': True}, {'group': '', 'path': '/data/wow/mythic-keystone/dungeon/index', 'namespaces': ['dynamic'], 'index': True},
{'path': '/data/wow/mythic-keystone/period/index', 'namespaces': ['dynamic'], 'index': True}, {'group': '', 'path': '/data/wow/mythic-keystone/index', 'namespaces': ['dynamic'], 'index': True},
{'path': '/data/wow/pet/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/mythic-keystone/period/index', 'namespaces': ['dynamic'], 'index': True},
{'path': '/data/wow/pet-ability/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/pet/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/playable-class/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/pet-ability/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/playable-race/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/playable-class/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/playable-specialization/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/playable-race/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/power-type/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/playable-specialization/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/profession/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/power-type/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/pvp-season/index', 'namespaces': ['dynamic'], 'index': True}, {'group': '', 'path': '/data/wow/profession/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/pvp-tier/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/pvp-season/index', 'namespaces': ['dynamic'], 'index': True},
{'path': '/data/wow/quest/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/pvp-tier/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/quest/category/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/quest/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/quest/area/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/quest/category/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/quest/type/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/quest/area/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/realm/index', 'namespaces': ['dynamic'], 'index': True}, {'group': '', 'path': '/data/wow/quest/type/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/region/index', 'namespaces': ['dynamic'], 'index': True}, {'group': '', 'path': '/data/wow/realm/index', 'namespaces': ['dynamic'], 'index': True},
{'path': '/data/wow/reputation-faction/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/region/index', 'namespaces': ['dynamic'], 'index': True},
{'path': '/data/wow/reputation-tiers/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/reputation-faction/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/talent/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/reputation-tiers/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/pvp-talent/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/talent/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/tech-talent-tree/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/pvp-talent/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/tech-talent/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/tech-talent-tree/index', 'namespaces': ['static'], 'index': True},
{'path': '/data/wow/title/index', 'namespaces': ['static'], 'index': True}, {'group': '', 'path': '/data/wow/tech-talent/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/title/index', 'namespaces': ['static'], 'index': True},
] ]

79
updater.py
View File

@ -4,18 +4,24 @@ import errno
import requests import requests
import json import json
import datetime import datetime
import tempfile
import config import config
import blizzard import blizzard
import spec import spec
def log(*data): def log(*data):
print(datetime.datetime.now(), '|', *data) print(datetime.datetime.now(), '|', *data)
class Updater(object): class Updater(object):
def __init__(self): def __init__(self):
self.http = requests.Session() self.http = requests.Session()
self.credentials = blizzard.get_credentials(config.pwd) self.credentials = blizzard.get_credentials(config.pwd)
self.last_modified = None
def region_oauth(self, region: str) -> dict: def region_oauth(self, region: str) -> dict:
api = self.http.post(f"{blizzard.get_bnet_host(region)}/oauth/token", data={'grant_type': 'client_credentials'}, auth=self.credentials) api = self.http.post(f"{blizzard.get_bnet_host(region)}/oauth/token", data={'grant_type': 'client_credentials'}, auth=self.credentials)
@ -24,11 +30,16 @@ class Updater(object):
api.raise_for_status() api.raise_for_status()
return oauth return oauth
def api_call(self, url: str) -> requests.Response:
api = self.http.get(url) #, headers={'Authorization': f"Authorization: Token {access_token}"}) def api_call(self, region: str, path: str, namespace: str, locale: str, access_token: str, headers=None) -> requests.Response:
url = f"{blizzard.get_api_host(region)}{path}"
qs = {'namespace': f"{namespace}-{region}", 'local': locale, 'access_token': access_token}
api = self.http.get(url, params=qs, headers=headers) #, headers={'Authorization': f"Authorization: Token {access_token}"})
api.raise_for_status() api.raise_for_status()
return api return api
def create_dst(self, dst: str): def create_dst(self, dst: str):
try: try:
os.makedirs(os.path.dirname(dst)) os.makedirs(os.path.dirname(dst))
@ -36,9 +47,56 @@ class Updater(object):
if e.errno != errno.EEXIST: if e.errno != errno.EEXIST:
raise raise
def save(self, dst, content):
def save_raw(self, region: str, path: str, namespace: str, locale: str, raw: requests.Response):
dst = f"{config.raw}/{region}/{locale}/{path.replace('/', '_')}.{namespace}.json"
self.create_dst(dst)
with open(dst, 'w+') as f: with open(dst, 'w+') as f:
f.write(json.dumps(content.json(), indent=4)) f.write(json.dumps(raw.json(), indent=2))
def get_last_modified(self, region: str, path: str, namespace: str, locale: str):
key = f"{region}.{locale}.{path}.{namespace}"
# cached!
if self.last_modified is not None and key in self.last_modified:
return self.last_modified[key]
# not cached, check db again
db = os.path.join(config.pwd, 'last-modified.db')
with open(db, 'r') as f:
data = json.load(f)
self.last_modified = data
try:
return data[key]
except KeyError:
return None
def save_last_modified(self, region: str, path: str, namespace: str, locale: str, modified: str):
key = f"{region}.{locale}.{path}.{namespace}"
db = os.path.join(config.pwd, 'last-modified.db')
# never trust the cache, open, modify and save updated data for safety
with open(db, 'r') as f:
data = json.load(f)
data[key] = modified
# update cache with new data
self.last_modified = data
# safe write: tmp write and then mv
fd, tmp_path = tempfile.mkstemp(dir=config.pwd)
try:
with os.fdopen(fd, 'w') as tmp:
json.dump(data, tmp, indent=2)
finally:
os.replace(tmp_path, db)
def iterate_index(self): def iterate_index(self):
for region in spec.regions: for region in spec.regions:
@ -59,19 +117,20 @@ class Updater(object):
# retail or classic # retail or classic
for namespace in api['namespaces']: for namespace in api['namespaces']:
try: try:
dst = f"{config.raw}/{region['code']}/{locale}/{api['path'].replace('/', '_')}.{namespace}.json" last_modified = self.get_last_modified(region['code'], api['path'], namespace, locale)
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
url = f"{blizzard.get_api_host(region['code'])}{api['path']}" response = self.api_call(region['code'], api['path'], namespace, locale, access_token, headers=headers)
url += f"?namespace={namespace}-{region['code']}&locale={locale}&access_token={access_token}"
response = self.api_call(url)
self.create_dst(dst) if response.status_code == 200:
self.save(dst, response) self.save_last_modified(region['code'], api['path'], namespace, locale, response.headers['Last-Modified'])
self.save_raw(region['code'], api['path'], namespace, locale, response)
except requests.exceptions.HTTPError as e: except requests.exceptions.HTTPError as e:
log(e) log(e)
continue continue
if __name__ == "__main__": if __name__ == "__main__":
updater = Updater() updater = Updater()
updater.iterate_index() updater.iterate_index()