code
/
blizzard-wow-game-data-apis
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

only US

main
Sergio Álvarez 5 months ago
parent
c0e363a5bd
commit
c482fb2128
5 changed files with 56 additions and 39 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 8
      README.md
  2. 3
      blizzard.py
  3. 11
      cron.sh
  4. 6
      spec.py
  5. 67
      updater.py

8
README.md
View File

@ -1,7 +1,13 @@
Create a `.credentials` file with `client_id:secret` from https://develop.battle.net/access
With no crontab execute `python3 updater.py`.
Crontab, every day at 12:30h:
```
30 12 * * * time python3 updater.py
30 12 * * * cron.sh
```
Only `US` region is explored because all API are the same and this is the default API for Blizzard.
Without `locale` we get all the translations, so I don't iterate through locales. If you ask for a locale, you get the value right in the field.

3
blizzard.py
View File

@ -11,3 +11,6 @@ def get_bnet_host(region: str) -> str:
def get_credentials(path: str) -> Tuple[str, str]:
return tuple(open(os.path.join(path, '.credentials')).read().strip().split(':'))
def get_by_key(values: list, key: str, value):
return next(x for x in values if x[key] == value)

11
cron.sh
View File

@ -0,0 +1,11 @@
#! /bin/bash
TIME=`(time python3 updater.py ) 2>&1`
# add every new file
git add raw
git add craft
git add last-modified.db
git commit -m 'daily cron' -m "$TIME"
git push

6
spec.py
View File

@ -58,9 +58,9 @@ apis = [
{'group': 'Covenant', 'path': '/data/wow/covenant/index', 'namespaces': ['static'], 'index': True},
{'group': 'Covenant', 'path': '/data/wow/covenant/soulbind/index', 'namespaces': ['static'], 'index': True},
{'group': 'Covenant', 'path': '/data/wow/covenant/conduit/index', 'namespaces': ['static'], 'index': True},
{'group': 'Creature', 'path': '/data/wow/creature-family/index', 'namespaces': ['static'], 'index': True},
{'group': 'Creature', 'path': '/data/wow/creature-type/index', 'namespaces': ['static'], 'index': True},
{'group': 'Guild Crest', 'path': '/data/wow/guild-crest/index', 'namespaces': ['static'], 'index': True},
{'group': 'Creature', 'path': '/data/wow/creature-family/index', 'namespaces': ['static', 'static-classic'], 'index': True},
{'group': 'Creature', 'path': '/data/wow/creature-type/index', 'namespaces': ['static', 'static-classic'], 'index': True},
{'group': 'Guild Crest', 'path': '/data/wow/guild-crest/index', 'namespaces': ['static', 'static-classic'], 'index': True},
{'group': '', 'path': '/data/wow/item-class/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/item-set/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/journal-expansion/index', 'namespaces': ['static'], 'index': True},

67
updater.py
View File

@ -31,9 +31,9 @@ class Updater(object):
return oauth
def api_call(self, region: str, path: str, namespace: str, locale: str, access_token: str, headers=None) -> requests.Response:
def api_call(self, region: str, path: str, namespace: str, access_token: str, headers=None) -> requests.Response:
url = f"{blizzard.get_api_host(region)}{path}"
qs = {'namespace': f"{namespace}-{region}", 'local': locale, 'access_token': access_token}
qs = {'namespace': f"{namespace}-{region}", 'access_token': access_token}
api = self.http.get(url, params=qs, headers=headers) #, headers={'Authorization': f"Authorization: Token {access_token}"})
api.raise_for_status()
@ -48,8 +48,8 @@ class Updater(object):
raise
def save_raw(self, region: str, path: str, namespace: str, locale: str, raw: requests.Response):
dst = f"{config.raw}/{region}/{locale}/{path.replace('/', '_')}.{namespace}.json"
def save_raw(self, region: str, path: str, namespace: str, raw: requests.Response):
dst = f"{config.raw}/{path.replace('/', '_')}.{namespace}.json"
self.create_dst(dst)
@ -57,8 +57,8 @@ class Updater(object):
f.write(json.dumps(raw.json(), indent=2))
def get_last_modified(self, region: str, path: str, namespace: str, locale: str):
key = f"{region}.{locale}.{path}.{namespace}"
def get_last_modified(self, region: str, path: str, namespace: str):
key = f"{path}.{namespace}"
# cached!
if self.last_modified is not None and key in self.last_modified:
@ -77,8 +77,8 @@ class Updater(object):
return None
def save_last_modified(self, region: str, path: str, namespace: str, locale: str, modified: str):
key = f"{region}.{locale}.{path}.{namespace}"
def save_last_modified(self, region: str, path: str, namespace: str, modified: str):
key = f"{path}.{namespace}"
db = os.path.join(config.pwd, 'last-modified.db')
# never trust the cache, open, modify and save updated data for safety
@ -98,40 +98,37 @@ class Updater(object):
os.replace(tmp_path, db)
def iterate_index(self):
for region in spec.regions:
# access token for region
try:
oauth = self.region_oauth(region['code'])
access_token = oauth['access_token']
except (requests.exceptions.HTTPError, KeyError) as e:
log(region, type(e), e)
def iterate_index(self, region: dict):
# access token for region
try:
oauth = self.region_oauth(region)
access_token = oauth['access_token']
except (requests.exceptions.HTTPError, KeyError) as e:
log(region, type(e), e)
return
# loop every api
for api in spec.apis:
if not 'index' in api or not api['index']:
continue
# loop every api
for api in spec.apis:
if not 'index' in api or not api['index']:
continue
for locale in region['locales']:
# retail or classic
for namespace in api['namespaces']:
try:
last_modified = self.get_last_modified(region['code'], api['path'], namespace, locale)
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
for namespace in api['namespaces']:
try:
last_modified = self.get_last_modified(region, api['path'], namespace)
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
response = self.api_call(region['code'], api['path'], namespace, locale, access_token, headers=headers)
response = self.api_call(region, api['path'], namespace, access_token, headers=headers)
if response.status_code == 200:
self.save_last_modified(region['code'], api['path'], namespace, locale, response.headers['Last-Modified'])
self.save_raw(region['code'], api['path'], namespace, locale, response)
if response.status_code == 200:
self.save_last_modified(region, api['path'], namespace, response.headers['Last-Modified'])
self.save_raw(region, api['path'], namespace, response)
except requests.exceptions.HTTPError as e:
log(e)
continue
except requests.exceptions.HTTPError as e:
log(e)
continue
if __name__ == "__main__":
updater = Updater()
updater.iterate_index()
updater.iterate_index(blizzard.get_by_key(spec.regions, 'code', 'us')['code'])
#updater.iterate_links()
Write Preview
Loading…
Cancel
Save