code/blizzard-wow-game-data-apis
code
/
blizzard-wow-game-data-apis
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

only US

This commit is contained in:
Sergio Álvarez 2021-05-14 04:16:23 +02:00
parent c0e363a5bd
commit c482fb2128
5 changed files with 58 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@ -1,7 +1,13 @@
Create a `.credentials` file with `client_id:secret` from https://develop.battle.net/access
With no crontab execute `python3 updater.py`.
Crontab, every day at 12:30h:
```
30 12 * * * time python3 updater.py
30 12 * * * cron.sh
```
Only `US` region is explored because all API are the same and this is the default API for Blizzard.
Without `locale` we get all the translations, so I don't iterate through locales. If you ask for a locale, you get the value right in the field.

3
blizzard.py
View File

@ -11,3 +11,6 @@ def get_bnet_host(region: str) -> str:
def get_credentials(path: str) -> Tuple[str, str]:
return tuple(open(os.path.join(path, '.credentials')).read().strip().split(':'))
def get_by_key(values: list, key: str, value):
return next(x for x in values if x[key] == value)

11
cron.sh Executable file
View File

@ -0,0 +1,11 @@
#! /bin/bash
TIME=`(time python3 updater.py ) 2>&1`
# add every new file
git add raw
git add craft
git add last-modified.db
git commit -m 'daily cron' -m "$TIME"
git push

6
spec.py
View File

@ -58,9 +58,9 @@ apis = [
{'group': 'Covenant', 'path': '/data/wow/covenant/index', 'namespaces': ['static'], 'index': True},
{'group': 'Covenant', 'path': '/data/wow/covenant/soulbind/index', 'namespaces': ['static'], 'index': True},
{'group': 'Covenant', 'path': '/data/wow/covenant/conduit/index', 'namespaces': ['static'], 'index': True},
{'group': 'Creature', 'path': '/data/wow/creature-family/index', 'namespaces': ['static'], 'index': True},
{'group': 'Creature', 'path': '/data/wow/creature-type/index', 'namespaces': ['static'], 'index': True},
{'group': 'Guild Crest', 'path': '/data/wow/guild-crest/index', 'namespaces': ['static'], 'index': True},
{'group': 'Creature', 'path': '/data/wow/creature-family/index', 'namespaces': ['static', 'static-classic'], 'index': True},
{'group': 'Creature', 'path': '/data/wow/creature-type/index', 'namespaces': ['static', 'static-classic'], 'index': True},
{'group': 'Guild Crest', 'path': '/data/wow/guild-crest/index', 'namespaces': ['static', 'static-classic'], 'index': True},
{'group': '', 'path': '/data/wow/item-class/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/item-set/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/journal-expansion/index', 'namespaces': ['static'], 'index': True},

71
updater.py
View File

@ -31,9 +31,9 @@ class Updater(object):
return oauth
def api_call(self, region: str, path: str, namespace: str, locale: str, access_token: str, headers=None) -> requests.Response:
def api_call(self, region: str, path: str, namespace: str, access_token: str, headers=None) -> requests.Response:
url = f"{blizzard.get_api_host(region)}{path}"
qs = {'namespace': f"{namespace}-{region}", 'local': locale, 'access_token': access_token}
qs = {'namespace': f"{namespace}-{region}", 'access_token': access_token}
api = self.http.get(url, params=qs, headers=headers) #, headers={'Authorization': f"Authorization: Token {access_token}"})
api.raise_for_status()
@ -48,8 +48,8 @@ class Updater(object):
raise
def save_raw(self, region: str, path: str, namespace: str, locale: str, raw: requests.Response):
dst = f"{config.raw}/{region}/{locale}/{path.replace('/', '_')}.{namespace}.json"
def save_raw(self, region: str, path: str, namespace: str, raw: requests.Response):
dst = f"{config.raw}/{path.replace('/', '_')}.{namespace}.json"
self.create_dst(dst)
@ -57,8 +57,8 @@ class Updater(object):
f.write(json.dumps(raw.json(), indent=2))
def get_last_modified(self, region: str, path: str, namespace: str, locale: str):
key = f"{region}.{locale}.{path}.{namespace}"
def get_last_modified(self, region: str, path: str, namespace: str):
key = f"{path}.{namespace}"
# cached!
if self.last_modified is not None and key in self.last_modified:
@ -77,8 +77,8 @@ class Updater(object):
return None
def save_last_modified(self, region: str, path: str, namespace: str, locale: str, modified: str):
key = f"{region}.{locale}.{path}.{namespace}"
def save_last_modified(self, region: str, path: str, namespace: str, modified: str):
key = f"{path}.{namespace}"
db = os.path.join(config.pwd, 'last-modified.db')
# never trust the cache, open, modify and save updated data for safety
@ -98,40 +98,37 @@ class Updater(object):
os.replace(tmp_path, db)
def iterate_index(self):
for region in spec.regions:
# access token for region
try:
oauth = self.region_oauth(region['code'])
access_token = oauth['access_token']
except (requests.exceptions.HTTPError, KeyError) as e:
log(region, type(e), e)
def iterate_index(self, region: dict):
# access token for region
try:
oauth = self.region_oauth(region)
access_token = oauth['access_token']
except (requests.exceptions.HTTPError, KeyError) as e:
log(region, type(e), e)
return
# loop every api
for api in spec.apis:
if not 'index' in api or not api['index']:
continue
# loop every api
for api in spec.apis:
if not 'index' in api or not api['index']:
for namespace in api['namespaces']:
try:
last_modified = self.get_last_modified(region, api['path'], namespace)
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
response = self.api_call(region, api['path'], namespace, access_token, headers=headers)
if response.status_code == 200:
self.save_last_modified(region, api['path'], namespace, response.headers['Last-Modified'])
self.save_raw(region, api['path'], namespace, response)
except requests.exceptions.HTTPError as e:
log(e)
continue
for locale in region['locales']:
# retail or classic
for namespace in api['namespaces']:
try:
last_modified = self.get_last_modified(region['code'], api['path'], namespace, locale)
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
response = self.api_call(region['code'], api['path'], namespace, locale, access_token, headers=headers)
if response.status_code == 200:
self.save_last_modified(region['code'], api['path'], namespace, locale, response.headers['Last-Modified'])
self.save_raw(region['code'], api['path'], namespace, locale, response)
except requests.exceptions.HTTPError as e:
log(e)
continue
if __name__ == "__main__":
updater = Updater()
updater.iterate_index()
updater.iterate_index(blizzard.get_by_key(spec.regions, 'code', 'us')['code'])
#updater.iterate_links()