code
/
blizzard-wow-game-data-apis
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

no more db, now using mtime from cached files

main
Sergio Álvarez 4 months ago
parent
56a4518ccf
commit
c27861622e
2 changed files with 375 additions and 140 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 363
      spec.py
  2. 152
      updater.py

363
spec.py
View File

@ -12,88 +12,291 @@ regions = [
#{'code': 'cn', 'locales': ['zh_CN']}
]
groups = [
'Achievement',
'Auction House',
'Azerite Essence',
'Connected Realm',
'Covenant',
'Creature',
'Guild Crest',
'Item',
'Journal',
'Media Search',
'Modified Crafting',
'Mount',
'Mythic Keystone Affix',
'Mythic Keystone Dungeon',
'Mythic Keystone Leaderboard',
'Mythic Raid Leaderboard',
'Pet',
'Playable Class',
'Playable Race',
'Playable Specialization',
'Power Type',
'Profession',
'PvP Season',
'PvP Tier',
'Quest',
'Realm',
'Region',
'Reputations',
'Spell',
'Talent',
'Tech Talent',
'Title',
'WoW Token',
]
"""
https://develop.battle.net/documentation/world-of-warcraft/game-data-apis
"""
apis = [
{'group': 'Achievement', 'path': '/data/wow/achievement-category/index', 'namespaces': ['static'], 'index': True},
{'group': 'Achievement', 'path': '/data/wow/achievement/index', 'namespaces': ['static'], 'index': True},
{'group': 'Connected Realm', 'path': '/data/wow/connected-realm/index', 'namespaces': ['dynamic', 'dynamic-classic'], 'index': True},
{'group': 'Covenant', 'path': '/data/wow/covenant/index', 'namespaces': ['static'], 'index': True},
{'group': 'Covenant', 'path': '/data/wow/covenant/soulbind/index', 'namespaces': ['static'], 'index': True},
{'group': 'Covenant', 'path': '/data/wow/covenant/conduit/index', 'namespaces': ['static'], 'index': True},
{'group': 'Creature', 'path': '/data/wow/creature-family/index', 'namespaces': ['static', 'static-classic'], 'index': True},
{'group': 'Creature', 'path': '/data/wow/creature-type/index', 'namespaces': ['static', 'static-classic'], 'index': True},
{'group': 'Guild Crest', 'path': '/data/wow/guild-crest/index', 'namespaces': ['static', 'static-classic'], 'index': True},
{'group': '', 'path': '/data/wow/item-class/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/item-set/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/journal-expansion/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/journal-encounter/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/journal-instance/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/modified-crafting/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/modified-crafting/category/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/modified-crafting/reagent-slot-type/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/mount/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/keystone-affix/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/mythic-keystone/dungeon/index', 'namespaces': ['dynamic'], 'index': True},
{'group': '', 'path': '/data/wow/mythic-keystone/index', 'namespaces': ['dynamic'], 'index': True},
{'group': '', 'path': '/data/wow/mythic-keystone/period/index', 'namespaces': ['dynamic'], 'index': True},
{'group': '', 'path': '/data/wow/pet/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/pet-ability/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/playable-class/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/playable-race/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/playable-specialization/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/power-type/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/profession/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/pvp-season/index', 'namespaces': ['dynamic'], 'index': True},
{'group': '', 'path': '/data/wow/pvp-tier/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/quest/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/quest/category/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/quest/area/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/quest/type/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/realm/index', 'namespaces': ['dynamic'], 'index': True},
{'group': '', 'path': '/data/wow/region/index', 'namespaces': ['dynamic'], 'index': True},
{'group': '', 'path': '/data/wow/reputation-faction/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/reputation-tiers/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/talent/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/pvp-talent/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/tech-talent-tree/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/tech-talent/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/title/index', 'namespaces': ['static'], 'index': True},
{
'group': 'Achievement',
'path': '/data/wow/achievement-category/index',
'namespace': 'static',
'index': True
},
{
'group': 'Achievement',
'path': '/data/wow/achievement/index',
'namespace': 'static',
'index': True
},
{
'group': 'Connected Realm',
'path': '/data/wow/connected-realm/index',
'namespace': 'dynamic',
'index': True
},
#{
# 'group': 'Connected Realm',
# 'path': '/data/wow/connected-realm/{connectedRealmId}',
# 'namespace': 'dynamic',
# 'connectedRealmId': {
# 'source': '/data/wow/connected-realm/index',
# 'loop': lambda l: l
# }
#},
{
'group': 'Covenant',
'path': '/data/wow/covenant/index',
'namespace': 'static',
'index': True
},
{
'group': 'Covenant',
'path': '/data/wow/covenant/soulbind/index',
'namespace': 'static',
'index': True
},
{
'group': 'Covenant',
'path': '/data/wow/covenant/conduit/index',
'namespace': 'static',
'index': True
},
{
'group': 'Creature',
'path': '/data/wow/creature-family/index',
'namespace': 'static',
'index': True
},
{
'group': 'Creature',
'path': '/data/wow/creature-type/index',
'namespace': 'static',
'index': True
},
{
'group': 'Guild Crest',
'path': '/data/wow/guild-crest/index',
'namespace': 'static',
'index': True
},
{
'group': 'Item',
'path': '/data/wow/item-class/index',
'namespace': 'static',
'index': True
},
{
'group': 'Item',
'path': '/data/wow/item-set/index',
'namespace': 'static',
'index': True
},
{
'group': 'Journal',
'path': '/data/wow/journal-expansion/index',
'namespace': 'static',
'index': True
},
{
'group': 'Journal',
'path': '/data/wow/journal-encounter/index',
'namespace': 'static',
'index': True
},
{
'group': 'Journal',
'path': '/data/wow/journal-instance/index',
'namespace': 'static',
'index': True
},
{
'group': 'Modified Crafting',
'path': '/data/wow/modified-crafting/index',
'namespace': 'static',
'index': True
},
{
'group': 'Modified Crafting',
'path': '/data/wow/modified-crafting/category/index',
'namespace': 'static',
'index': True
},
{
'group': 'Modified Crafting',
'path': '/data/wow/modified-crafting/reagent-slot-type/index',
'namespace': 'static',
'index': True
},
{
'group': 'Mount',
'path': '/data/wow/mount/index',
'namespace': 'static',
'index': True
},
{
'group': 'Mythic Keystone Affix',
'path': '/data/wow/keystone-affix/index',
'namespace': 'static',
'index': True
},
{
'group': 'Mythic Keystone Dungeon',
'path': '/data/wow/mythic-keystone/dungeon/index',
'namespace': 'dynamic',
'index': True
},
{
'group': 'Mythic Keystone Dungeon',
'path': '/data/wow/mythic-keystone/index',
'namespace': 'dynamic',
'index': True
},
{
'group': 'Mythic Keystone Dungeon',
'path': '/data/wow/mythic-keystone/period/index',
'namespace': 'dynamic',
'index': True
},
{
'group': 'Pet',
'path': '/data/wow/pet/index',
'namespace': 'static',
'index': True
},
{
'group': 'Pet',
'path': '/data/wow/pet-ability/index',
'namespace': 'static',
'index': True
},
{
'group': 'Playable Class',
'path': '/data/wow/playable-class/index',
'namespace': 'static',
'index': True
},
{
'group': 'Playable Race',
'path': '/data/wow/playable-race/index',
'namespace': 'static',
'index': True
},
{
'group': 'Playable Specialization',
'path': '/data/wow/playable-specialization/index',
'namespace': 'static',
'index': True
},
{
'group': 'Power Type',
'path': '/data/wow/power-type/index',
'namespace': 'static',
'index': True
},
{
'group': 'Profession',
'path': '/data/wow/profession/index',
'namespace': 'static',
'index': True
},
{
'group': 'PvP Season',
'path': '/data/wow/pvp-season/index',
'namespace': 'dynamic',
'index': True
},
{
'group': 'PvP Tier',
'path': '/data/wow/pvp-tier/index',
'namespace': 'static',
'index': True
},
{
'group': 'Quest',
'path': '/data/wow/quest/index',
'namespace': 'static',
'index': True
},
{
'group': 'Quest',
'path': '/data/wow/quest/category/index',
'namespace': 'static',
'index': True
},
{
'group': 'Quest',
'path': '/data/wow/quest/area/index',
'namespace': 'static',
'index': True
},
{
'group': 'Quest',
'path': '/data/wow/quest/type/index',
'namespace': 'static',
'index': True
},
{
'group': 'Realm',
'path': '/data/wow/realm/index',
'namespace': 'dynamic',
'index': True
},
{
'group': 'Region',
'path': '/data/wow/region/index',
'namespace': 'dynamic',
'index': True
},
{
'group': 'Reputations',
'path': '/data/wow/reputation-faction/index',
'namespace': 'static',
'index': True
},
{
'group': 'Reputations',
'path': '/data/wow/reputation-tiers/index',
'namespace': 'static',
'index': True
},
{
'group': 'Talent',
'path': '/data/wow/talent/index',
'namespace': 'static',
'index': True
},
{
'group': 'Talent',
'path': '/data/wow/pvp-talent/index',
'namespace': 'static',
'index': True
},
{
'group': 'Tech Talent',
'path': '/data/wow/tech-talent-tree/index',
'namespace': 'static',
'index': True
},
{
'group': 'Tech Talent',
'path': '/data/wow/tech-talent/index',
'namespace': 'static',
'index': True
},
{
'group': 'Title',
'path': '/data/wow/title/index',
'namespace': 'static',
'index': True
},
{
'group': 'Title',
'path': '/data/wow/title/{titleId}',
'namespace': 'static',
'titleId': {
'source': '/data/wow/title/index',
'list': 'titles',
'value': lambda item: item['id']
}
},
]

152
updater.py
View File

@ -4,7 +4,9 @@ import errno
import requests
import json
import datetime
import time
import tempfile
import pytz
import config
import blizzard
@ -17,23 +19,32 @@ def log(*data):
class Updater(object):
def __init__(self):
RFC2616 = '%a, %d %b %Y %H:%M:%S %Z'
def __init__(self, region: str):
self.region = region
self.http = requests.Session()
self.credentials = blizzard.get_credentials(config.pwd)
self.last_modified = None
self.last_modified = {}
self.oauth = None
def get_oauth(self) -> dict:
if self.oauth is not None:
return self.oauth
def region_oauth(self, region: str) -> dict:
api = self.http.post(f"{blizzard.get_bnet_host(region)}/oauth/token", data={'grant_type': 'client_credentials'}, auth=self.credentials)
api = self.http.post(f"{blizzard.get_bnet_host(self.region)}/oauth/token", data={'grant_type': 'client_credentials'}, auth=self.credentials)
oauth = api.json()
#log(region, oauth)
api.raise_for_status()
self.oauth = oauth
return oauth
def api_call(self, region: str, path: str, namespace: str, access_token: str, headers=None) -> requests.Response:
url = f"{blizzard.get_api_host(region)}{path}"
qs = {'namespace': f"{namespace}-{region}", 'access_token': access_token}
def api_call(self, path: str, namespace: str, access_token: str, headers=None) -> requests.Response:
url = f"{blizzard.get_api_host(self.region)}{path}"
qs = {'namespace': f"{namespace}-{self.region}", 'access_token': access_token}
api = self.http.get(url, params=qs, headers=headers) #, headers={'Authorization': f"Authorization: Token {access_token}"})
api.raise_for_status()
@ -48,87 +59,108 @@ class Updater(object):
raise
def save_raw(self, region: str, path: str, namespace: str, raw: requests.Response):
dst = f"{config.raw}/{path.replace('/', '_')}.{namespace}.json"
def save_raw(self, path: str, raw: requests.Response):
dst = f"{config.raw}/{path.replace('/', '_')}.json"
self.create_dst(dst)
with open(dst, 'w+') as f:
f.write(json.dumps(raw.json(), indent=2))
# safe write: tmp write and then mv
fd, tmp_path = tempfile.mkstemp(dir=config.pwd)
try:
with os.fdopen(fd, 'w') as tmp:
json.dump(raw.json(), tmp, indent=2)
finally:
os.replace(tmp_path, dst)
def get_last_modified(self, region: str, path: str, namespace: str):
key = f"{path}.{namespace}"
def get_last_modified(self, path: str):
# cached!
if self.last_modified is not None and key in self.last_modified:
return self.last_modified[key]
if path in self.last_modified:
return self.last_modified[path]
# not cached, check db again
db = os.path.join(config.pwd, 'last-modified.db')
dst = f"{config.raw}/{path.replace('/', '_')}.json"
with open(db, 'r') as f:
data = json.load(f)
self.last_modified = data
# mtime to datetime from timestamp, default tz, replace to GMT, format
modified = datetime.datetime.fromtimestamp(os.path.getmtime(dst)).astimezone().replace(tzinfo=pytz.timezone('GMT')).strftime(self.RFC2616)
self.last_modified[path] = modified
try:
return data[key]
except KeyError:
return None
return self.last_modified[path]
def save_last_modified(self, region: str, path: str, namespace: str, modified: str):
key = f"{path}.{namespace}"
db = os.path.join(config.pwd, 'last-modified.db')
def set_last_modified(self, path: str, modified: str):
dst = f"{config.raw}/{path.replace('/', '_')}.json"
self.last_modified[path] = modified
# never trust the cache, open, modify and save updated data for safety
with open(db, 'r') as f:
data = json.load(f)
# same Last-Modified for raw file
epoch = time.mktime(time.strptime(modified, self.RFC2616)) # Tue, 11 May 2021 14:06:37 GMT
os.utime(dst, (epoch, epoch))
data[key] = modified
# update cache with new data
self.last_modified = data
# safe write: tmp write and then mv
fd, tmp_path = tempfile.mkstemp(dir=config.pwd)
def iterate_index(self):
try:
with os.fdopen(fd, 'w') as tmp:
json.dump(data, tmp, indent=2)
finally:
os.replace(tmp_path, db)
oauth = self.get_oauth()
access_token = oauth['access_token']
except (requests.exceptions.HTTPError, KeyError) as e:
log(type(e), e)
return
for api in spec.apis:
# only indexes
if not 'index' in api or not api['index']:
continue
try:
last_modified = self.get_last_modified(api['path'])
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
response = self.api_call(api['path'], api['namespace'], access_token, headers=headers)
def iterate_index(self, region: dict):
# access token for region
log(response.status_code, api)
log(response.request.headers)
log(response.headers)
if response.status_code == 200:
self.save_raw(api['path'], response)
self.set_last_modified(api['path'], response.headers['Last-Modified'])
except requests.exceptions.HTTPError as e:
log(e)
continue
def iterate_links(self):
try:
oauth = self.region_oauth(region)
oauth = self.get_oauth()
access_token = oauth['access_token']
except (requests.exceptions.HTTPError, KeyError) as e:
log(region, type(e), e)
log(type(e), e)
return
# loop every api
for api in spec.apis:
if not 'index' in api or not api['index']:
# no indexes
if 'index' in api and api['index']:
continue
for namespace in api['namespaces']:
try:
last_modified = self.get_last_modified(region, api['path'], namespace)
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
log(api)
continue
try:
last_modified = self.get_last_modified(api['path'])
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
response = self.api_call(region, api['path'], namespace, access_token, headers=headers)
response = self.api_call(api['path'], api['namespace'], access_token, headers=headers)
if response.status_code == 200:
self.save_last_modified(region, api['path'], namespace, response.headers['Last-Modified'])
self.save_raw(region, api['path'], namespace, response)
log(response.status_code, api)
if response.status_code == 200:
self.save_last_modified(api['path'], response.headers['Last-Modified'])
self.save_raw(api['path'], response)
except requests.exceptions.HTTPError as e:
log(e)
continue
except requests.exceptions.HTTPError as e:
log(e)
continue
if __name__ == "__main__":
updater = Updater()
updater.iterate_index(blizzard.get_by_key(spec.regions, 'code', 'us')['code'])
#updater.iterate_links()
region = blizzard.get_by_key(spec.regions, 'code', 'us')['code']
updater = Updater(region)
updater.iterate_index()
updater.iterate_links()
Write Preview
Loading…
Cancel
Save