import os import errno import requests import json import datetime import tempfile import config import blizzard import spec def log(*data): print(datetime.datetime.now(), '|', *data) class Updater(object): def __init__(self): self.http = requests.Session() self.credentials = blizzard.get_credentials(config.pwd) self.last_modified = None def region_oauth(self, region: str) -> dict: api = self.http.post(f"{blizzard.get_bnet_host(region)}/oauth/token", data={'grant_type': 'client_credentials'}, auth=self.credentials) oauth = api.json() #log(region, oauth) api.raise_for_status() return oauth def api_call(self, region: str, path: str, namespace: str, access_token: str, headers=None) -> requests.Response: url = f"{blizzard.get_api_host(region)}{path}" qs = {'namespace': f"{namespace}-{region}", 'access_token': access_token} api = self.http.get(url, params=qs, headers=headers) #, headers={'Authorization': f"Authorization: Token {access_token}"}) api.raise_for_status() return api def create_dst(self, dst: str): try: os.makedirs(os.path.dirname(dst)) except OSError as e: if e.errno != errno.EEXIST: raise def save_raw(self, region: str, path: str, namespace: str, raw: requests.Response): dst = f"{config.raw}/{path.replace('/', '_')}.{namespace}.json" self.create_dst(dst) with open(dst, 'w+') as f: f.write(json.dumps(raw.json(), indent=2)) def get_last_modified(self, region: str, path: str, namespace: str): key = f"{path}.{namespace}" # cached! if self.last_modified is not None and key in self.last_modified: return self.last_modified[key] # not cached, check db again db = os.path.join(config.pwd, 'last-modified.db') with open(db, 'r') as f: data = json.load(f) self.last_modified = data try: return data[key] except KeyError: return None def save_last_modified(self, region: str, path: str, namespace: str, modified: str): key = f"{path}.{namespace}" db = os.path.join(config.pwd, 'last-modified.db') # never trust the cache, open, modify and save updated data for safety with open(db, 'r') as f: data = json.load(f) data[key] = modified # update cache with new data self.last_modified = data # safe write: tmp write and then mv fd, tmp_path = tempfile.mkstemp(dir=config.pwd) try: with os.fdopen(fd, 'w') as tmp: json.dump(data, tmp, indent=2) finally: os.replace(tmp_path, db) def iterate_index(self, region: dict): # access token for region try: oauth = self.region_oauth(region) access_token = oauth['access_token'] except (requests.exceptions.HTTPError, KeyError) as e: log(region, type(e), e) return # loop every api for api in spec.apis: if not 'index' in api or not api['index']: continue for namespace in api['namespaces']: try: last_modified = self.get_last_modified(region, api['path'], namespace) headers = {'If-Modified-Since': last_modified} if last_modified is not None else None response = self.api_call(region, api['path'], namespace, access_token, headers=headers) if response.status_code == 200: self.save_last_modified(region, api['path'], namespace, response.headers['Last-Modified']) self.save_raw(region, api['path'], namespace, response) except requests.exceptions.HTTPError as e: log(e) continue if __name__ == "__main__": updater = Updater() updater.iterate_index(blizzard.get_by_key(spec.regions, 'code', 'us')['code']) #updater.iterate_links()