blizzard-wow-game-data-apis/updater.py

138 lines
4.4 KiB
Python
Raw Normal View History

2021-05-12 06:13:54 +00:00
import os
import errno
import requests
import json
2021-05-13 22:34:22 +00:00
import datetime
2021-05-14 01:05:14 +00:00
import tempfile
2021-05-12 06:13:54 +00:00
2021-05-13 22:34:22 +00:00
import config
import blizzard
import spec
2021-05-12 06:13:54 +00:00
2021-05-14 01:05:14 +00:00
2021-05-13 22:34:22 +00:00
def log(*data):
print(datetime.datetime.now(), '|', *data)
2021-05-12 06:13:54 +00:00
2021-05-14 01:05:14 +00:00
2021-05-13 22:34:22 +00:00
class Updater(object):
2021-05-14 01:05:14 +00:00
2021-05-13 22:34:22 +00:00
def __init__(self):
self.http = requests.Session()
self.credentials = blizzard.get_credentials(config.pwd)
2021-05-14 01:05:14 +00:00
self.last_modified = None
2021-05-12 06:13:54 +00:00
2021-05-13 22:34:22 +00:00
def region_oauth(self, region: str) -> dict:
api = self.http.post(f"{blizzard.get_bnet_host(region)}/oauth/token", data={'grant_type': 'client_credentials'}, auth=self.credentials)
oauth = api.json()
#log(region, oauth)
api.raise_for_status()
return oauth
2021-05-12 06:13:54 +00:00
2021-05-14 01:05:14 +00:00
def api_call(self, region: str, path: str, namespace: str, locale: str, access_token: str, headers=None) -> requests.Response:
url = f"{blizzard.get_api_host(region)}{path}"
qs = {'namespace': f"{namespace}-{region}", 'local': locale, 'access_token': access_token}
api = self.http.get(url, params=qs, headers=headers) #, headers={'Authorization': f"Authorization: Token {access_token}"})
2021-05-13 22:34:22 +00:00
api.raise_for_status()
return api
2021-05-12 06:13:54 +00:00
2021-05-14 01:05:14 +00:00
2021-05-13 22:34:22 +00:00
def create_dst(self, dst: str):
try:
os.makedirs(os.path.dirname(dst))
except OSError as e:
if e.errno != errno.EEXIST:
raise
2021-05-12 06:13:54 +00:00
2021-05-14 01:05:14 +00:00
def save_raw(self, region: str, path: str, namespace: str, locale: str, raw: requests.Response):
dst = f"{config.raw}/{region}/{locale}/{path.replace('/', '_')}.{namespace}.json"
self.create_dst(dst)
2021-05-13 22:34:22 +00:00
with open(dst, 'w+') as f:
2021-05-14 01:05:14 +00:00
f.write(json.dumps(raw.json(), indent=2))
def get_last_modified(self, region: str, path: str, namespace: str, locale: str):
key = f"{region}.{locale}.{path}.{namespace}"
# cached!
if self.last_modified is not None and key in self.last_modified:
return self.last_modified[key]
# not cached, check db again
db = os.path.join(config.pwd, 'last-modified.db')
with open(db, 'r') as f:
data = json.load(f)
self.last_modified = data
try:
return data[key]
except KeyError:
return None
def save_last_modified(self, region: str, path: str, namespace: str, locale: str, modified: str):
key = f"{region}.{locale}.{path}.{namespace}"
db = os.path.join(config.pwd, 'last-modified.db')
# never trust the cache, open, modify and save updated data for safety
with open(db, 'r') as f:
data = json.load(f)
data[key] = modified
# update cache with new data
self.last_modified = data
# safe write: tmp write and then mv
fd, tmp_path = tempfile.mkstemp(dir=config.pwd)
try:
with os.fdopen(fd, 'w') as tmp:
json.dump(data, tmp, indent=2)
finally:
os.replace(tmp_path, db)
2021-05-12 06:13:54 +00:00
2021-05-13 22:34:22 +00:00
def iterate_index(self):
for region in spec.regions:
# access token for region
try:
oauth = self.region_oauth(region['code'])
access_token = oauth['access_token']
except (requests.exceptions.HTTPError, KeyError) as e:
log(region, type(e), e)
continue
2021-05-12 06:13:54 +00:00
2021-05-13 22:34:22 +00:00
# loop every api
for api in spec.apis:
if not 'index' in api or not api['index']:
2021-05-12 06:13:54 +00:00
continue
2021-05-13 22:34:22 +00:00
for locale in region['locales']:
# retail or classic
for namespace in api['namespaces']:
try:
2021-05-14 01:05:14 +00:00
last_modified = self.get_last_modified(region['code'], api['path'], namespace, locale)
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
2021-05-13 22:34:22 +00:00
2021-05-14 01:05:14 +00:00
response = self.api_call(region['code'], api['path'], namespace, locale, access_token, headers=headers)
2021-05-13 22:34:22 +00:00
2021-05-14 01:05:14 +00:00
if response.status_code == 200:
self.save_last_modified(region['code'], api['path'], namespace, locale, response.headers['Last-Modified'])
self.save_raw(region['code'], api['path'], namespace, locale, response)
2021-05-13 22:34:22 +00:00
except requests.exceptions.HTTPError as e:
log(e)
continue
2021-05-14 01:05:14 +00:00
2021-05-13 22:34:22 +00:00
if __name__ == "__main__":
updater = Updater()
updater.iterate_index()
#updater.iterate_links()