135 lines
4.1 KiB
Python
135 lines
4.1 KiB
Python
|
|
import os
|
|
import errno
|
|
import requests
|
|
import json
|
|
import datetime
|
|
import tempfile
|
|
|
|
import config
|
|
import blizzard
|
|
import spec
|
|
|
|
|
|
def log(*data):
|
|
print(datetime.datetime.now(), '|', *data)
|
|
|
|
|
|
class Updater(object):
|
|
|
|
def __init__(self):
|
|
self.http = requests.Session()
|
|
self.credentials = blizzard.get_credentials(config.pwd)
|
|
self.last_modified = None
|
|
|
|
|
|
def region_oauth(self, region: str) -> dict:
|
|
api = self.http.post(f"{blizzard.get_bnet_host(region)}/oauth/token", data={'grant_type': 'client_credentials'}, auth=self.credentials)
|
|
oauth = api.json()
|
|
#log(region, oauth)
|
|
api.raise_for_status()
|
|
return oauth
|
|
|
|
|
|
def api_call(self, region: str, path: str, namespace: str, access_token: str, headers=None) -> requests.Response:
|
|
url = f"{blizzard.get_api_host(region)}{path}"
|
|
qs = {'namespace': f"{namespace}-{region}", 'access_token': access_token}
|
|
|
|
api = self.http.get(url, params=qs, headers=headers) #, headers={'Authorization': f"Authorization: Token {access_token}"})
|
|
api.raise_for_status()
|
|
return api
|
|
|
|
|
|
def create_dst(self, dst: str):
|
|
try:
|
|
os.makedirs(os.path.dirname(dst))
|
|
except OSError as e:
|
|
if e.errno != errno.EEXIST:
|
|
raise
|
|
|
|
|
|
def save_raw(self, region: str, path: str, namespace: str, raw: requests.Response):
|
|
dst = f"{config.raw}/{path.replace('/', '_')}.{namespace}.json"
|
|
|
|
self.create_dst(dst)
|
|
|
|
with open(dst, 'w+') as f:
|
|
f.write(json.dumps(raw.json(), indent=2))
|
|
|
|
|
|
def get_last_modified(self, region: str, path: str, namespace: str):
|
|
key = f"{path}.{namespace}"
|
|
|
|
# cached!
|
|
if self.last_modified is not None and key in self.last_modified:
|
|
return self.last_modified[key]
|
|
|
|
# not cached, check db again
|
|
db = os.path.join(config.pwd, 'last-modified.db')
|
|
|
|
with open(db, 'r') as f:
|
|
data = json.load(f)
|
|
self.last_modified = data
|
|
|
|
try:
|
|
return data[key]
|
|
except KeyError:
|
|
return None
|
|
|
|
|
|
def save_last_modified(self, region: str, path: str, namespace: str, modified: str):
|
|
key = f"{path}.{namespace}"
|
|
db = os.path.join(config.pwd, 'last-modified.db')
|
|
|
|
# never trust the cache, open, modify and save updated data for safety
|
|
with open(db, 'r') as f:
|
|
data = json.load(f)
|
|
|
|
data[key] = modified
|
|
# update cache with new data
|
|
self.last_modified = data
|
|
|
|
# safe write: tmp write and then mv
|
|
fd, tmp_path = tempfile.mkstemp(dir=config.pwd)
|
|
try:
|
|
with os.fdopen(fd, 'w') as tmp:
|
|
json.dump(data, tmp, indent=2)
|
|
finally:
|
|
os.replace(tmp_path, db)
|
|
|
|
|
|
def iterate_index(self, region: dict):
|
|
# access token for region
|
|
try:
|
|
oauth = self.region_oauth(region)
|
|
access_token = oauth['access_token']
|
|
except (requests.exceptions.HTTPError, KeyError) as e:
|
|
log(region, type(e), e)
|
|
return
|
|
|
|
# loop every api
|
|
for api in spec.apis:
|
|
if not 'index' in api or not api['index']:
|
|
continue
|
|
|
|
for namespace in api['namespaces']:
|
|
try:
|
|
last_modified = self.get_last_modified(region, api['path'], namespace)
|
|
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
|
|
|
|
response = self.api_call(region, api['path'], namespace, access_token, headers=headers)
|
|
|
|
if response.status_code == 200:
|
|
self.save_last_modified(region, api['path'], namespace, response.headers['Last-Modified'])
|
|
self.save_raw(region, api['path'], namespace, response)
|
|
|
|
except requests.exceptions.HTTPError as e:
|
|
log(e)
|
|
continue
|
|
|
|
|
|
if __name__ == "__main__":
|
|
updater = Updater()
|
|
updater.iterate_index(blizzard.get_by_key(spec.regions, 'code', 'us')['code'])
|
|
#updater.iterate_links()
|