blizzard-wow-game-data-apis/updater.py

import os
import errno
import requests
import json
import datetime
import tempfile

import config
import blizzard
import spec


def log(*data):
    print(datetime.datetime.now(), '|', *data)


class Updater(object):

    def __init__(self):
        self.http = requests.Session()
        self.credentials = blizzard.get_credentials(config.pwd)
        self.last_modified = None


    def region_oauth(self, region: str) -> dict:
        api = self.http.post(f"{blizzard.get_bnet_host(region)}/oauth/token", data={'grant_type': 'client_credentials'}, auth=self.credentials)
        oauth = api.json()
        #log(region, oauth)
        api.raise_for_status()
        return oauth


    def api_call(self, region: str, path: str, namespace: str, locale: str, access_token: str, headers=None) -> requests.Response:
        url = f"{blizzard.get_api_host(region)}{path}"
        qs = {'namespace': f"{namespace}-{region}", 'local': locale, 'access_token': access_token}

        api = self.http.get(url, params=qs, headers=headers) #, headers={'Authorization': f"Authorization: Token {access_token}"})
        api.raise_for_status()
        return api


    def create_dst(self, dst: str):
        try:
            os.makedirs(os.path.dirname(dst))
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise


    def save_raw(self, region: str, path: str, namespace: str, locale: str, raw: requests.Response):
        dst = f"{config.raw}/{region}/{locale}/{path.replace('/', '_')}.{namespace}.json"

        self.create_dst(dst)

        with open(dst, 'w+') as f:
            f.write(json.dumps(raw.json(), indent=2))


    def get_last_modified(self, region: str, path: str, namespace: str, locale: str):
        key = f"{region}.{locale}.{path}.{namespace}"

        # cached!
        if self.last_modified is not None and key in self.last_modified:
            return self.last_modified[key]

        # not cached, check db again
        db = os.path.join(config.pwd, 'last-modified.db')

        with open(db, 'r') as f:
            data = json.load(f)
        self.last_modified = data

        try:
            return data[key]
        except KeyError:
            return None


    def save_last_modified(self, region: str, path: str, namespace: str, locale: str, modified: str):
        key = f"{region}.{locale}.{path}.{namespace}"
        db = os.path.join(config.pwd, 'last-modified.db')

        # never trust the cache, open, modify and save updated data for safety
        with open(db, 'r') as f:
            data = json.load(f)

        data[key] = modified
        # update cache with new data
        self.last_modified = data

        # safe write: tmp write and then mv
        fd, tmp_path = tempfile.mkstemp(dir=config.pwd)
        try:
            with os.fdopen(fd, 'w') as tmp:
                json.dump(data, tmp, indent=2)
        finally:
            os.replace(tmp_path, db)


    def iterate_index(self):
        for region in spec.regions:
            # access token for region
            try:
                oauth = self.region_oauth(region['code'])
                access_token = oauth['access_token']
            except (requests.exceptions.HTTPError, KeyError) as e:
                log(region, type(e), e)
                continue

            # loop every api
            for api in spec.apis:
                if not 'index' in api or not api['index']:
                    continue

                for locale in region['locales']:
                    # retail or classic
                    for namespace in api['namespaces']:
                        try:
                            last_modified = self.get_last_modified(region['code'], api['path'], namespace, locale)
                            headers = {'If-Modified-Since': last_modified} if last_modified is not None else None

                            response = self.api_call(region['code'], api['path'], namespace, locale, access_token, headers=headers)

                            if response.status_code == 200:
                                self.save_last_modified(region['code'], api['path'], namespace, locale, response.headers['Last-Modified'])
                                self.save_raw(region['code'], api['path'], namespace, locale, response)

                        except requests.exceptions.HTTPError as e:
                            log(e)
                            continue


if __name__ == "__main__":
    updater = Updater()
    updater.iterate_index()
    #updater.iterate_links()