code/blizzard-wow-game-data-apis
code
/
blizzard-wow-game-data-apis
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

no more db, now using mtime from cached files

This commit is contained in:
Sergio Álvarez 2021-05-14 23:11:40 +02:00
parent 56a4518ccf
commit c27861622e
2 changed files with 381 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

363
spec.py
View File

@ -12,88 +12,291 @@ regions = [
#{'code': 'cn', 'locales': ['zh_CN']} #{'code': 'cn', 'locales': ['zh_CN']}
] ]
groups = [
'Achievement',
'Auction House',
'Azerite Essence',
'Connected Realm',
'Covenant',
'Creature',
'Guild Crest',
'Item',
'Journal',
'Media Search',
'Modified Crafting',
'Mount',
'Mythic Keystone Affix',
'Mythic Keystone Dungeon',
'Mythic Keystone Leaderboard',
'Mythic Raid Leaderboard',
'Pet',
'Playable Class',
'Playable Race',
'Playable Specialization',
'Power Type',
'Profession',
'PvP Season',
'PvP Tier',
'Quest',
'Realm',
'Region',
'Reputations',
'Spell',
'Talent',
'Tech Talent',
'Title',
'WoW Token',
]
""" """
https://develop.battle.net/documentation/world-of-warcraft/game-data-apis https://develop.battle.net/documentation/world-of-warcraft/game-data-apis
""" """
apis = [ apis = [
{'group': 'Achievement', 'path': '/data/wow/achievement-category/index', 'namespaces': ['static'], 'index': True}, {
{'group': 'Achievement', 'path': '/data/wow/achievement/index', 'namespaces': ['static'], 'index': True}, 'group': 'Achievement',
{'group': 'Connected Realm', 'path': '/data/wow/connected-realm/index', 'namespaces': ['dynamic', 'dynamic-classic'], 'index': True}, 'path': '/data/wow/achievement-category/index',
{'group': 'Covenant', 'path': '/data/wow/covenant/index', 'namespaces': ['static'], 'index': True}, 'namespace': 'static',
{'group': 'Covenant', 'path': '/data/wow/covenant/soulbind/index', 'namespaces': ['static'], 'index': True}, 'index': True
{'group': 'Covenant', 'path': '/data/wow/covenant/conduit/index', 'namespaces': ['static'], 'index': True}, },
{'group': 'Creature', 'path': '/data/wow/creature-family/index', 'namespaces': ['static', 'static-classic'], 'index': True}, {
{'group': 'Creature', 'path': '/data/wow/creature-type/index', 'namespaces': ['static', 'static-classic'], 'index': True}, 'group': 'Achievement',
{'group': 'Guild Crest', 'path': '/data/wow/guild-crest/index', 'namespaces': ['static', 'static-classic'], 'index': True}, 'path': '/data/wow/achievement/index',
{'group': '', 'path': '/data/wow/item-class/index', 'namespaces': ['static'], 'index': True}, 'namespace': 'static',
{'group': '', 'path': '/data/wow/item-set/index', 'namespaces': ['static'], 'index': True}, 'index': True
{'group': '', 'path': '/data/wow/journal-expansion/index', 'namespaces': ['static'], 'index': True}, },
{'group': '', 'path': '/data/wow/journal-encounter/index', 'namespaces': ['static'], 'index': True}, {
{'group': '', 'path': '/data/wow/journal-instance/index', 'namespaces': ['static'], 'index': True}, 'group': 'Connected Realm',
{'group': '', 'path': '/data/wow/modified-crafting/index', 'namespaces': ['static'], 'index': True}, 'path': '/data/wow/connected-realm/index',
{'group': '', 'path': '/data/wow/modified-crafting/category/index', 'namespaces': ['static'], 'index': True}, 'namespace': 'dynamic',
{'group': '', 'path': '/data/wow/modified-crafting/reagent-slot-type/index', 'namespaces': ['static'], 'index': True}, 'index': True
{'group': '', 'path': '/data/wow/mount/index', 'namespaces': ['static'], 'index': True}, },
{'group': '', 'path': '/data/wow/keystone-affix/index', 'namespaces': ['static'], 'index': True}, #{
{'group': '', 'path': '/data/wow/mythic-keystone/dungeon/index', 'namespaces': ['dynamic'], 'index': True}, # 'group': 'Connected Realm',
{'group': '', 'path': '/data/wow/mythic-keystone/index', 'namespaces': ['dynamic'], 'index': True}, # 'path': '/data/wow/connected-realm/{connectedRealmId}',
{'group': '', 'path': '/data/wow/mythic-keystone/period/index', 'namespaces': ['dynamic'], 'index': True}, # 'namespace': 'dynamic',
{'group': '', 'path': '/data/wow/pet/index', 'namespaces': ['static'], 'index': True}, # 'connectedRealmId': {
{'group': '', 'path': '/data/wow/pet-ability/index', 'namespaces': ['static'], 'index': True}, # 'source': '/data/wow/connected-realm/index',
{'group': '', 'path': '/data/wow/playable-class/index', 'namespaces': ['static'], 'index': True}, # 'loop': lambda l: l
{'group': '', 'path': '/data/wow/playable-race/index', 'namespaces': ['static'], 'index': True}, # }
{'group': '', 'path': '/data/wow/playable-specialization/index', 'namespaces': ['static'], 'index': True}, #},
{'group': '', 'path': '/data/wow/power-type/index', 'namespaces': ['static'], 'index': True}, {
{'group': '', 'path': '/data/wow/profession/index', 'namespaces': ['static'], 'index': True}, 'group': 'Covenant',
{'group': '', 'path': '/data/wow/pvp-season/index', 'namespaces': ['dynamic'], 'index': True}, 'path': '/data/wow/covenant/index',
{'group': '', 'path': '/data/wow/pvp-tier/index', 'namespaces': ['static'], 'index': True}, 'namespace': 'static',
{'group': '', 'path': '/data/wow/quest/index', 'namespaces': ['static'], 'index': True}, 'index': True
{'group': '', 'path': '/data/wow/quest/category/index', 'namespaces': ['static'], 'index': True}, },
{'group': '', 'path': '/data/wow/quest/area/index', 'namespaces': ['static'], 'index': True}, {
{'group': '', 'path': '/data/wow/quest/type/index', 'namespaces': ['static'], 'index': True}, 'group': 'Covenant',
{'group': '', 'path': '/data/wow/realm/index', 'namespaces': ['dynamic'], 'index': True}, 'path': '/data/wow/covenant/soulbind/index',
{'group': '', 'path': '/data/wow/region/index', 'namespaces': ['dynamic'], 'index': True}, 'namespace': 'static',
{'group': '', 'path': '/data/wow/reputation-faction/index', 'namespaces': ['static'], 'index': True}, 'index': True
{'group': '', 'path': '/data/wow/reputation-tiers/index', 'namespaces': ['static'], 'index': True}, },
{'group': '', 'path': '/data/wow/talent/index', 'namespaces': ['static'], 'index': True}, {
{'group': '', 'path': '/data/wow/pvp-talent/index', 'namespaces': ['static'], 'index': True}, 'group': 'Covenant',
{'group': '', 'path': '/data/wow/tech-talent-tree/index', 'namespaces': ['static'], 'index': True}, 'path': '/data/wow/covenant/conduit/index',
{'group': '', 'path': '/data/wow/tech-talent/index', 'namespaces': ['static'], 'index': True}, 'namespace': 'static',
{'group': '', 'path': '/data/wow/title/index', 'namespaces': ['static'], 'index': True}, 'index': True
},
{
'group': 'Creature',
'path': '/data/wow/creature-family/index',
'namespace': 'static',
'index': True
},
{
'group': 'Creature',
'path': '/data/wow/creature-type/index',
'namespace': 'static',
'index': True
},
{
'group': 'Guild Crest',
'path': '/data/wow/guild-crest/index',
'namespace': 'static',
'index': True
},
{
'group': 'Item',
'path': '/data/wow/item-class/index',
'namespace': 'static',
'index': True
},
{
'group': 'Item',
'path': '/data/wow/item-set/index',
'namespace': 'static',
'index': True
},
{
'group': 'Journal',
'path': '/data/wow/journal-expansion/index',
'namespace': 'static',
'index': True
},
{
'group': 'Journal',
'path': '/data/wow/journal-encounter/index',
'namespace': 'static',
'index': True
},
{
'group': 'Journal',
'path': '/data/wow/journal-instance/index',
'namespace': 'static',
'index': True
},
{
'group': 'Modified Crafting',
'path': '/data/wow/modified-crafting/index',
'namespace': 'static',
'index': True
},
{
'group': 'Modified Crafting',
'path': '/data/wow/modified-crafting/category/index',
'namespace': 'static',
'index': True
},
{
'group': 'Modified Crafting',
'path': '/data/wow/modified-crafting/reagent-slot-type/index',
'namespace': 'static',
'index': True
},
{
'group': 'Mount',
'path': '/data/wow/mount/index',
'namespace': 'static',
'index': True
},
{
'group': 'Mythic Keystone Affix',
'path': '/data/wow/keystone-affix/index',
'namespace': 'static',
'index': True
},
{
'group': 'Mythic Keystone Dungeon',
'path': '/data/wow/mythic-keystone/dungeon/index',
'namespace': 'dynamic',
'index': True
},
{
'group': 'Mythic Keystone Dungeon',
'path': '/data/wow/mythic-keystone/index',
'namespace': 'dynamic',
'index': True
},
{
'group': 'Mythic Keystone Dungeon',
'path': '/data/wow/mythic-keystone/period/index',
'namespace': 'dynamic',
'index': True
},
{
'group': 'Pet',
'path': '/data/wow/pet/index',
'namespace': 'static',
'index': True
},
{
'group': 'Pet',
'path': '/data/wow/pet-ability/index',
'namespace': 'static',
'index': True
},
{
'group': 'Playable Class',
'path': '/data/wow/playable-class/index',
'namespace': 'static',
'index': True
},
{
'group': 'Playable Race',
'path': '/data/wow/playable-race/index',
'namespace': 'static',
'index': True
},
{
'group': 'Playable Specialization',
'path': '/data/wow/playable-specialization/index',
'namespace': 'static',
'index': True
},
{
'group': 'Power Type',
'path': '/data/wow/power-type/index',
'namespace': 'static',
'index': True
},
{
'group': 'Profession',
'path': '/data/wow/profession/index',
'namespace': 'static',
'index': True
},
{
'group': 'PvP Season',
'path': '/data/wow/pvp-season/index',
'namespace': 'dynamic',
'index': True
},
{
'group': 'PvP Tier',
'path': '/data/wow/pvp-tier/index',
'namespace': 'static',
'index': True
},
{
'group': 'Quest',
'path': '/data/wow/quest/index',
'namespace': 'static',
'index': True
},
{
'group': 'Quest',
'path': '/data/wow/quest/category/index',
'namespace': 'static',
'index': True
},
{
'group': 'Quest',
'path': '/data/wow/quest/area/index',
'namespace': 'static',
'index': True
},
{
'group': 'Quest',
'path': '/data/wow/quest/type/index',
'namespace': 'static',
'index': True
},
{
'group': 'Realm',
'path': '/data/wow/realm/index',
'namespace': 'dynamic',
'index': True
},
{
'group': 'Region',
'path': '/data/wow/region/index',
'namespace': 'dynamic',
'index': True
},
{
'group': 'Reputations',
'path': '/data/wow/reputation-faction/index',
'namespace': 'static',
'index': True
},
{
'group': 'Reputations',
'path': '/data/wow/reputation-tiers/index',
'namespace': 'static',
'index': True
},
{
'group': 'Talent',
'path': '/data/wow/talent/index',
'namespace': 'static',
'index': True
},
{
'group': 'Talent',
'path': '/data/wow/pvp-talent/index',
'namespace': 'static',
'index': True
},
{
'group': 'Tech Talent',
'path': '/data/wow/tech-talent-tree/index',
'namespace': 'static',
'index': True
},
{
'group': 'Tech Talent',
'path': '/data/wow/tech-talent/index',
'namespace': 'static',
'index': True
},
{
'group': 'Title',
'path': '/data/wow/title/index',
'namespace': 'static',
'index': True
},
{
'group': 'Title',
'path': '/data/wow/title/{titleId}',
'namespace': 'static',
'titleId': {
'source': '/data/wow/title/index',
'list': 'titles',
'value': lambda item: item['id']
}
},
] ]

152
updater.py
View File

@ -4,7 +4,9 @@ import errno
import requests import requests
import json import json
import datetime import datetime
import time
import tempfile import tempfile
import pytz
import config import config
import blizzard import blizzard
@ -17,23 +19,32 @@ def log(*data):
class Updater(object): class Updater(object):
def __init__(self): RFC2616 = '%a, %d %b %Y %H:%M:%S %Z'
def __init__(self, region: str):
self.region = region
self.http = requests.Session() self.http = requests.Session()
self.credentials = blizzard.get_credentials(config.pwd) self.credentials = blizzard.get_credentials(config.pwd)
self.last_modified = None self.last_modified = {}
self.oauth = None
def region_oauth(self, region: str) -> dict: def get_oauth(self) -> dict:
api = self.http.post(f"{blizzard.get_bnet_host(region)}/oauth/token", data={'grant_type': 'client_credentials'}, auth=self.credentials) if self.oauth is not None:
return self.oauth
api = self.http.post(f"{blizzard.get_bnet_host(self.region)}/oauth/token", data={'grant_type': 'client_credentials'}, auth=self.credentials)
oauth = api.json() oauth = api.json()
#log(region, oauth) #log(region, oauth)
api.raise_for_status() api.raise_for_status()
self.oauth = oauth
return oauth return oauth
def api_call(self, region: str, path: str, namespace: str, access_token: str, headers=None) -> requests.Response: def api_call(self, path: str, namespace: str, access_token: str, headers=None) -> requests.Response:
url = f"{blizzard.get_api_host(region)}{path}" url = f"{blizzard.get_api_host(self.region)}{path}"
qs = {'namespace': f"{namespace}-{region}", 'access_token': access_token} qs = {'namespace': f"{namespace}-{self.region}", 'access_token': access_token}
api = self.http.get(url, params=qs, headers=headers) #, headers={'Authorization': f"Authorization: Token {access_token}"}) api = self.http.get(url, params=qs, headers=headers) #, headers={'Authorization': f"Authorization: Token {access_token}"})
api.raise_for_status() api.raise_for_status()
@ -48,80 +59,99 @@ class Updater(object):
raise raise
def save_raw(self, region: str, path: str, namespace: str, raw: requests.Response): def save_raw(self, path: str, raw: requests.Response):
dst = f"{config.raw}/{path.replace('/', '_')}.{namespace}.json" dst = f"{config.raw}/{path.replace('/', '_')}.json"
self.create_dst(dst) self.create_dst(dst)
with open(dst, 'w+') as f:
f.write(json.dumps(raw.json(), indent=2))
def get_last_modified(self, region: str, path: str, namespace: str):
key = f"{path}.{namespace}"
# cached!
if self.last_modified is not None and key in self.last_modified:
return self.last_modified[key]
# not cached, check db again
db = os.path.join(config.pwd, 'last-modified.db')
with open(db, 'r') as f:
data = json.load(f)
self.last_modified = data
try:
return data[key]
except KeyError:
return None
def save_last_modified(self, region: str, path: str, namespace: str, modified: str):
key = f"{path}.{namespace}"
db = os.path.join(config.pwd, 'last-modified.db')
# never trust the cache, open, modify and save updated data for safety
with open(db, 'r') as f:
data = json.load(f)
data[key] = modified
# update cache with new data
self.last_modified = data
# safe write: tmp write and then mv # safe write: tmp write and then mv
fd, tmp_path = tempfile.mkstemp(dir=config.pwd) fd, tmp_path = tempfile.mkstemp(dir=config.pwd)
try: try:
with os.fdopen(fd, 'w') as tmp: with os.fdopen(fd, 'w') as tmp:
json.dump(data, tmp, indent=2) json.dump(raw.json(), tmp, indent=2)
finally: finally:
os.replace(tmp_path, db) os.replace(tmp_path, dst)
def iterate_index(self, region: dict): def get_last_modified(self, path: str):
# access token for region # cached!
if path in self.last_modified:
return self.last_modified[path]
dst = f"{config.raw}/{path.replace('/', '_')}.json"
# mtime to datetime from timestamp, default tz, replace to GMT, format
modified = datetime.datetime.fromtimestamp(os.path.getmtime(dst)).astimezone().replace(tzinfo=pytz.timezone('GMT')).strftime(self.RFC2616)
self.last_modified[path] = modified
return self.last_modified[path]
def set_last_modified(self, path: str, modified: str):
dst = f"{config.raw}/{path.replace('/', '_')}.json"
self.last_modified[path] = modified
# same Last-Modified for raw file
epoch = time.mktime(time.strptime(modified, self.RFC2616)) # Tue, 11 May 2021 14:06:37 GMT
os.utime(dst, (epoch, epoch))
def iterate_index(self):
try: try:
oauth = self.region_oauth(region) oauth = self.get_oauth()
access_token = oauth['access_token'] access_token = oauth['access_token']
except (requests.exceptions.HTTPError, KeyError) as e: except (requests.exceptions.HTTPError, KeyError) as e:
log(region, type(e), e) log(type(e), e)
return return
# loop every api
for api in spec.apis: for api in spec.apis:
# only indexes
if not 'index' in api or not api['index']: if not 'index' in api or not api['index']:
continue continue
for namespace in api['namespaces']:
try: try:
last_modified = self.get_last_modified(region, api['path'], namespace) last_modified = self.get_last_modified(api['path'])
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
response = self.api_call(region, api['path'], namespace, access_token, headers=headers) response = self.api_call(api['path'], api['namespace'], access_token, headers=headers)
log(response.status_code, api)
log(response.request.headers)
log(response.headers)
if response.status_code == 200: if response.status_code == 200:
self.save_last_modified(region, api['path'], namespace, response.headers['Last-Modified']) self.save_raw(api['path'], response)
self.save_raw(region, api['path'], namespace, response) self.set_last_modified(api['path'], response.headers['Last-Modified'])
except requests.exceptions.HTTPError as e:
log(e)
continue
def iterate_links(self):
try:
oauth = self.get_oauth()
access_token = oauth['access_token']
except (requests.exceptions.HTTPError, KeyError) as e:
log(type(e), e)
return
for api in spec.apis:
# no indexes
if 'index' in api and api['index']:
continue
log(api)
continue
try:
last_modified = self.get_last_modified(api['path'])
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
response = self.api_call(api['path'], api['namespace'], access_token, headers=headers)
log(response.status_code, api)
if response.status_code == 200:
self.save_last_modified(api['path'], response.headers['Last-Modified'])
self.save_raw(api['path'], response)
except requests.exceptions.HTTPError as e: except requests.exceptions.HTTPError as e:
log(e) log(e)
@ -129,6 +159,8 @@ class Updater(object):
if __name__ == "__main__": if __name__ == "__main__":
updater = Updater() region = blizzard.get_by_key(spec.regions, 'code', 'us')['code']
updater.iterate_index(blizzard.get_by_key(spec.regions, 'code', 'us')['code'])
#updater.iterate_links() updater = Updater(region)
updater.iterate_index()
updater.iterate_links()