missing files
This commit is contained in:
parent
ae1f72b032
commit
1e3c2750e5
|
|
@ -1 +1,7 @@
|
|||
Create a `.credentials` file with `client_id:secret` from https://develop.battle.net/access
|
||||
|
||||
Crontab, every day at 12:30h:
|
||||
|
||||
```
|
||||
30 12 * * * time python3 updater.py
|
||||
```
|
||||
|
|
|
|||
123
spec.py
123
spec.py
|
|
@ -12,51 +12,88 @@ regions = [
|
|||
#{'code': 'cn', 'locales': ['zh_CN']}
|
||||
]
|
||||
|
||||
groups = [
|
||||
'Achievement',
|
||||
'Auction House',
|
||||
'Azerite Essence',
|
||||
'Connected Realm',
|
||||
'Covenant',
|
||||
'Creature',
|
||||
'Guild Crest',
|
||||
'Item',
|
||||
'Journal',
|
||||
'Media Search',
|
||||
'Modified Crafting',
|
||||
'Mount',
|
||||
'Mythic Keystone Affix',
|
||||
'Mythic Keystone Dungeon',
|
||||
'Mythic Keystone Leaderboard',
|
||||
'Mythic Raid Leaderboard',
|
||||
'Pet',
|
||||
'Playable Class',
|
||||
'Playable Race',
|
||||
'Playable Specialization',
|
||||
'Power Type',
|
||||
'Profession',
|
||||
'PvP Season',
|
||||
'PvP Tier',
|
||||
'Quest',
|
||||
'Realm',
|
||||
'Region',
|
||||
'Reputations',
|
||||
'Spell',
|
||||
'Talent',
|
||||
'Tech Talent',
|
||||
'Title',
|
||||
'WoW Token',
|
||||
]
|
||||
|
||||
"""
|
||||
https://develop.battle.net/documentation/world-of-warcraft/game-data-apis
|
||||
"""
|
||||
apis = [
|
||||
{'path': '/data/wow/achievement-category/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/achievement/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/connected-realm/index', 'namespaces': ['dynamic', 'dynamic-classic'], 'index': True},
|
||||
{'path': '/data/wow/covenant/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/covenant/soulbind/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/covenant/conduit/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/creature-family/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/creature-type/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/item-class/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/item-set/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/journal-expansion/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/journal-encounter/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/journal-instance/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/modified-crafting/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/modified-crafting/category/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/modified-crafting/reagent-slot-type/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/mount/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/keystone-affix/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/mythic-keystone/dungeon/index', 'namespaces': ['dynamic'], 'index': True},
|
||||
{'path': '/data/wow/mythic-keystone/index', 'namespaces': ['dynamic'], 'index': True},
|
||||
{'path': '/data/wow/mythic-keystone/period/index', 'namespaces': ['dynamic'], 'index': True},
|
||||
{'path': '/data/wow/pet/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/pet-ability/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/playable-class/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/playable-race/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/playable-specialization/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/power-type/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/profession/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/pvp-season/index', 'namespaces': ['dynamic'], 'index': True},
|
||||
{'path': '/data/wow/pvp-tier/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/quest/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/quest/category/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/quest/area/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/quest/type/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/realm/index', 'namespaces': ['dynamic'], 'index': True},
|
||||
{'path': '/data/wow/region/index', 'namespaces': ['dynamic'], 'index': True},
|
||||
{'path': '/data/wow/reputation-faction/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/reputation-tiers/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/talent/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/pvp-talent/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/tech-talent-tree/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/tech-talent/index', 'namespaces': ['static'], 'index': True},
|
||||
{'path': '/data/wow/title/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': 'Achievement', 'path': '/data/wow/achievement-category/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': 'Achievement', 'path': '/data/wow/achievement/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': 'Connected Realm', 'path': '/data/wow/connected-realm/index', 'namespaces': ['dynamic', 'dynamic-classic'], 'index': True},
|
||||
{'group': 'Covenant', 'path': '/data/wow/covenant/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': 'Covenant', 'path': '/data/wow/covenant/soulbind/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': 'Covenant', 'path': '/data/wow/covenant/conduit/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': 'Creature', 'path': '/data/wow/creature-family/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': 'Creature', 'path': '/data/wow/creature-type/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': 'Guild Crest', 'path': '/data/wow/guild-crest/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/item-class/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/item-set/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/journal-expansion/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/journal-encounter/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/journal-instance/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/modified-crafting/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/modified-crafting/category/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/modified-crafting/reagent-slot-type/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/mount/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/keystone-affix/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/mythic-keystone/dungeon/index', 'namespaces': ['dynamic'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/mythic-keystone/index', 'namespaces': ['dynamic'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/mythic-keystone/period/index', 'namespaces': ['dynamic'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/pet/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/pet-ability/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/playable-class/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/playable-race/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/playable-specialization/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/power-type/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/profession/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/pvp-season/index', 'namespaces': ['dynamic'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/pvp-tier/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/quest/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/quest/category/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/quest/area/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/quest/type/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/realm/index', 'namespaces': ['dynamic'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/region/index', 'namespaces': ['dynamic'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/reputation-faction/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/reputation-tiers/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/talent/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/pvp-talent/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/tech-talent-tree/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/tech-talent/index', 'namespaces': ['static'], 'index': True},
|
||||
{'group': '', 'path': '/data/wow/title/index', 'namespaces': ['static'], 'index': True},
|
||||
]
|
||||
|
|
|
|||
79
updater.py
79
updater.py
|
|
@ -4,18 +4,24 @@ import errno
|
|||
import requests
|
||||
import json
|
||||
import datetime
|
||||
import tempfile
|
||||
|
||||
import config
|
||||
import blizzard
|
||||
import spec
|
||||
|
||||
|
||||
def log(*data):
|
||||
print(datetime.datetime.now(), '|', *data)
|
||||
|
||||
|
||||
class Updater(object):
|
||||
|
||||
def __init__(self):
|
||||
self.http = requests.Session()
|
||||
self.credentials = blizzard.get_credentials(config.pwd)
|
||||
self.last_modified = None
|
||||
|
||||
|
||||
def region_oauth(self, region: str) -> dict:
|
||||
api = self.http.post(f"{blizzard.get_bnet_host(region)}/oauth/token", data={'grant_type': 'client_credentials'}, auth=self.credentials)
|
||||
|
|
@ -24,11 +30,16 @@ class Updater(object):
|
|||
api.raise_for_status()
|
||||
return oauth
|
||||
|
||||
def api_call(self, url: str) -> requests.Response:
|
||||
api = self.http.get(url) #, headers={'Authorization': f"Authorization: Token {access_token}"})
|
||||
|
||||
def api_call(self, region: str, path: str, namespace: str, locale: str, access_token: str, headers=None) -> requests.Response:
|
||||
url = f"{blizzard.get_api_host(region)}{path}"
|
||||
qs = {'namespace': f"{namespace}-{region}", 'local': locale, 'access_token': access_token}
|
||||
|
||||
api = self.http.get(url, params=qs, headers=headers) #, headers={'Authorization': f"Authorization: Token {access_token}"})
|
||||
api.raise_for_status()
|
||||
return api
|
||||
|
||||
|
||||
def create_dst(self, dst: str):
|
||||
try:
|
||||
os.makedirs(os.path.dirname(dst))
|
||||
|
|
@ -36,9 +47,56 @@ class Updater(object):
|
|||
if e.errno != errno.EEXIST:
|
||||
raise
|
||||
|
||||
def save(self, dst, content):
|
||||
|
||||
def save_raw(self, region: str, path: str, namespace: str, locale: str, raw: requests.Response):
|
||||
dst = f"{config.raw}/{region}/{locale}/{path.replace('/', '_')}.{namespace}.json"
|
||||
|
||||
self.create_dst(dst)
|
||||
|
||||
with open(dst, 'w+') as f:
|
||||
f.write(json.dumps(content.json(), indent=4))
|
||||
f.write(json.dumps(raw.json(), indent=2))
|
||||
|
||||
|
||||
def get_last_modified(self, region: str, path: str, namespace: str, locale: str):
|
||||
key = f"{region}.{locale}.{path}.{namespace}"
|
||||
|
||||
# cached!
|
||||
if self.last_modified is not None and key in self.last_modified:
|
||||
return self.last_modified[key]
|
||||
|
||||
# not cached, check db again
|
||||
db = os.path.join(config.pwd, 'last-modified.db')
|
||||
|
||||
with open(db, 'r') as f:
|
||||
data = json.load(f)
|
||||
self.last_modified = data
|
||||
|
||||
try:
|
||||
return data[key]
|
||||
except KeyError:
|
||||
return None
|
||||
|
||||
|
||||
def save_last_modified(self, region: str, path: str, namespace: str, locale: str, modified: str):
|
||||
key = f"{region}.{locale}.{path}.{namespace}"
|
||||
db = os.path.join(config.pwd, 'last-modified.db')
|
||||
|
||||
# never trust the cache, open, modify and save updated data for safety
|
||||
with open(db, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
data[key] = modified
|
||||
# update cache with new data
|
||||
self.last_modified = data
|
||||
|
||||
# safe write: tmp write and then mv
|
||||
fd, tmp_path = tempfile.mkstemp(dir=config.pwd)
|
||||
try:
|
||||
with os.fdopen(fd, 'w') as tmp:
|
||||
json.dump(data, tmp, indent=2)
|
||||
finally:
|
||||
os.replace(tmp_path, db)
|
||||
|
||||
|
||||
def iterate_index(self):
|
||||
for region in spec.regions:
|
||||
|
|
@ -59,19 +117,20 @@ class Updater(object):
|
|||
# retail or classic
|
||||
for namespace in api['namespaces']:
|
||||
try:
|
||||
dst = f"{config.raw}/{region['code']}/{locale}/{api['path'].replace('/', '_')}.{namespace}.json"
|
||||
last_modified = self.get_last_modified(region['code'], api['path'], namespace, locale)
|
||||
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
|
||||
|
||||
url = f"{blizzard.get_api_host(region['code'])}{api['path']}"
|
||||
url += f"?namespace={namespace}-{region['code']}&locale={locale}&access_token={access_token}"
|
||||
response = self.api_call(url)
|
||||
response = self.api_call(region['code'], api['path'], namespace, locale, access_token, headers=headers)
|
||||
|
||||
self.create_dst(dst)
|
||||
self.save(dst, response)
|
||||
if response.status_code == 200:
|
||||
self.save_last_modified(region['code'], api['path'], namespace, locale, response.headers['Last-Modified'])
|
||||
self.save_raw(region['code'], api['path'], namespace, locale, response)
|
||||
|
||||
except requests.exceptions.HTTPError as e:
|
||||
log(e)
|
||||
continue
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
updater = Updater()
|
||||
updater.iterate_index()
|
||||
|
|
|
|||
Loading…
Reference in New Issue