blizzard-wow-game-data-apis/updater.py

166 lines
5.0 KiB
Python
Raw Normal View History

2021-05-12 06:13:54 +00:00
import os
import errno
import requests
import json
2021-05-13 22:34:22 +00:00
import datetime
import time
2021-05-14 01:05:14 +00:00
import tempfile
import pytz
2021-05-12 06:13:54 +00:00
2021-05-13 22:34:22 +00:00
import config
import blizzard
import spec
2021-05-12 06:13:54 +00:00
2021-05-14 01:05:14 +00:00
2021-05-13 22:34:22 +00:00
def log(*data):
print(datetime.datetime.now(), '|', *data)
2021-05-12 06:13:54 +00:00
2021-05-14 01:05:14 +00:00
2021-05-13 22:34:22 +00:00
class Updater(object):
2021-05-14 01:05:14 +00:00
RFC2616 = '%a, %d %b %Y %H:%M:%S %Z'
def __init__(self, region: str):
self.region = region
2021-05-13 22:34:22 +00:00
self.http = requests.Session()
self.credentials = blizzard.get_credentials(config.pwd)
self.last_modified = {}
self.oauth = None
2021-05-14 01:05:14 +00:00
def get_oauth(self) -> dict:
if self.oauth is not None:
return self.oauth
2021-05-12 06:13:54 +00:00
api = self.http.post(f"{blizzard.get_bnet_host(self.region)}/oauth/token", data={'grant_type': 'client_credentials'}, auth=self.credentials)
2021-05-13 22:34:22 +00:00
oauth = api.json()
#log(region, oauth)
api.raise_for_status()
self.oauth = oauth
2021-05-13 22:34:22 +00:00
return oauth
2021-05-12 06:13:54 +00:00
2021-05-14 01:05:14 +00:00
def api_call(self, path: str, namespace: str, access_token: str, headers=None) -> requests.Response:
url = f"{blizzard.get_api_host(self.region)}{path}"
qs = {'namespace': f"{namespace}-{self.region}", 'access_token': access_token}
2021-05-14 01:05:14 +00:00
api = self.http.get(url, params=qs, headers=headers) #, headers={'Authorization': f"Authorization: Token {access_token}"})
2021-05-13 22:34:22 +00:00
api.raise_for_status()
return api
2021-05-12 06:13:54 +00:00
2021-05-14 01:05:14 +00:00
2021-05-13 22:34:22 +00:00
def create_dst(self, dst: str):
try:
os.makedirs(os.path.dirname(dst))
except OSError as e:
if e.errno != errno.EEXIST:
raise
2021-05-12 06:13:54 +00:00
2021-05-14 01:05:14 +00:00
def save_raw(self, path: str, raw: requests.Response):
dst = f"{config.raw}/{path.replace('/', '_')}.json"
2021-05-14 01:05:14 +00:00
self.create_dst(dst)
# safe write: tmp write and then mv
fd, tmp_path = tempfile.mkstemp(dir=config.pwd)
try:
with os.fdopen(fd, 'w') as tmp:
json.dump(raw.json(), tmp, indent=2)
finally:
os.replace(tmp_path, dst)
2021-05-14 01:05:14 +00:00
def get_last_modified(self, path: str):
2021-05-14 01:05:14 +00:00
# cached!
if path in self.last_modified:
return self.last_modified[path]
2021-05-14 01:05:14 +00:00
dst = f"{config.raw}/{path.replace('/', '_')}.json"
2021-05-14 01:05:14 +00:00
2021-05-15 08:28:08 +00:00
try:
# mtime to datetime from timestamp, default tz, replace to GMT, format
modified = datetime.datetime.fromtimestamp(os.path.getmtime(dst)).astimezone().replace(tzinfo=pytz.timezone('GMT')).strftime(self.RFC2616)
self.last_modified[path] = modified
return self.last_modified[path]
except FileNotFoundError:
return None
2021-05-14 01:05:14 +00:00
def set_last_modified(self, path: str, modified: str):
dst = f"{config.raw}/{path.replace('/', '_')}.json"
self.last_modified[path] = modified
2021-05-14 01:05:14 +00:00
# same Last-Modified for raw file
epoch = time.mktime(time.strptime(modified, self.RFC2616)) # Tue, 11 May 2021 14:06:37 GMT
os.utime(dst, (epoch, epoch))
2021-05-14 01:05:14 +00:00
def iterate_index(self):
2021-05-14 01:05:14 +00:00
try:
oauth = self.get_oauth()
access_token = oauth['access_token']
except (requests.exceptions.HTTPError, KeyError) as e:
log(type(e), e)
return
for api in spec.apis:
# only indexes
if not 'index' in api or not api['index']:
continue
try:
last_modified = self.get_last_modified(api['path'])
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
2021-05-14 01:05:14 +00:00
response = self.api_call(api['path'], api['namespace'], access_token, headers=headers)
2021-05-12 06:13:54 +00:00
if response.status_code == 200:
self.save_raw(api['path'], response)
self.set_last_modified(api['path'], response.headers['Last-Modified'])
except requests.exceptions.HTTPError as e:
log(e)
continue
def iterate_links(self):
2021-05-14 02:16:23 +00:00
try:
oauth = self.get_oauth()
2021-05-14 02:16:23 +00:00
access_token = oauth['access_token']
except (requests.exceptions.HTTPError, KeyError) as e:
log(type(e), e)
2021-05-14 02:16:23 +00:00
return
for api in spec.apis:
# no indexes
if 'index' in api and api['index']:
2021-05-13 22:34:22 +00:00
continue
2021-05-12 06:13:54 +00:00
log(api)
continue
try:
last_modified = self.get_last_modified(api['path'])
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
2021-05-13 22:34:22 +00:00
response = self.api_call(api['path'], api['namespace'], access_token, headers=headers)
2021-05-13 22:34:22 +00:00
log(response.status_code, api)
if response.status_code == 200:
self.save_last_modified(api['path'], response.headers['Last-Modified'])
self.save_raw(api['path'], response)
2021-05-13 22:34:22 +00:00
except requests.exceptions.HTTPError as e:
log(e)
continue
2021-05-13 22:34:22 +00:00
2021-05-14 01:05:14 +00:00
2021-05-13 22:34:22 +00:00
if __name__ == "__main__":
region = blizzard.get_by_key(spec.regions, 'code', 'us')['code']
updater = Updater(region)
updater.iterate_index()
2021-05-15 08:28:08 +00:00
#updater.iterate_links()