|
|
|
@ -31,9 +31,9 @@ class Updater(object):
@@ -31,9 +31,9 @@ class Updater(object):
|
|
|
|
|
return oauth |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def api_call(self, region: str, path: str, namespace: str, locale: str, access_token: str, headers=None) -> requests.Response: |
|
|
|
|
def api_call(self, region: str, path: str, namespace: str, access_token: str, headers=None) -> requests.Response: |
|
|
|
|
url = f"{blizzard.get_api_host(region)}{path}" |
|
|
|
|
qs = {'namespace': f"{namespace}-{region}", 'local': locale, 'access_token': access_token} |
|
|
|
|
qs = {'namespace': f"{namespace}-{region}", 'access_token': access_token} |
|
|
|
|
|
|
|
|
|
api = self.http.get(url, params=qs, headers=headers) #, headers={'Authorization': f"Authorization: Token {access_token}"}) |
|
|
|
|
api.raise_for_status() |
|
|
|
@ -48,8 +48,8 @@ class Updater(object):
@@ -48,8 +48,8 @@ class Updater(object):
|
|
|
|
|
raise |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def save_raw(self, region: str, path: str, namespace: str, locale: str, raw: requests.Response): |
|
|
|
|
dst = f"{config.raw}/{region}/{locale}/{path.replace('/', '_')}.{namespace}.json" |
|
|
|
|
def save_raw(self, region: str, path: str, namespace: str, raw: requests.Response): |
|
|
|
|
dst = f"{config.raw}/{path.replace('/', '_')}.{namespace}.json" |
|
|
|
|
|
|
|
|
|
self.create_dst(dst) |
|
|
|
|
|
|
|
|
@ -57,8 +57,8 @@ class Updater(object):
@@ -57,8 +57,8 @@ class Updater(object):
|
|
|
|
|
f.write(json.dumps(raw.json(), indent=2)) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_last_modified(self, region: str, path: str, namespace: str, locale: str): |
|
|
|
|
key = f"{region}.{locale}.{path}.{namespace}" |
|
|
|
|
def get_last_modified(self, region: str, path: str, namespace: str): |
|
|
|
|
key = f"{path}.{namespace}" |
|
|
|
|
|
|
|
|
|
# cached! |
|
|
|
|
if self.last_modified is not None and key in self.last_modified: |
|
|
|
@ -77,8 +77,8 @@ class Updater(object):
@@ -77,8 +77,8 @@ class Updater(object):
|
|
|
|
|
return None |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def save_last_modified(self, region: str, path: str, namespace: str, locale: str, modified: str): |
|
|
|
|
key = f"{region}.{locale}.{path}.{namespace}" |
|
|
|
|
def save_last_modified(self, region: str, path: str, namespace: str, modified: str): |
|
|
|
|
key = f"{path}.{namespace}" |
|
|
|
|
db = os.path.join(config.pwd, 'last-modified.db') |
|
|
|
|
|
|
|
|
|
# never trust the cache, open, modify and save updated data for safety |
|
|
|
@ -98,40 +98,37 @@ class Updater(object):
@@ -98,40 +98,37 @@ class Updater(object):
|
|
|
|
|
os.replace(tmp_path, db) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def iterate_index(self): |
|
|
|
|
for region in spec.regions: |
|
|
|
|
# access token for region |
|
|
|
|
try: |
|
|
|
|
oauth = self.region_oauth(region['code']) |
|
|
|
|
access_token = oauth['access_token'] |
|
|
|
|
except (requests.exceptions.HTTPError, KeyError) as e: |
|
|
|
|
log(region, type(e), e) |
|
|
|
|
def iterate_index(self, region: dict): |
|
|
|
|
# access token for region |
|
|
|
|
try: |
|
|
|
|
oauth = self.region_oauth(region) |
|
|
|
|
access_token = oauth['access_token'] |
|
|
|
|
except (requests.exceptions.HTTPError, KeyError) as e: |
|
|
|
|
log(region, type(e), e) |
|
|
|
|
return |
|
|
|
|
|
|
|
|
|
# loop every api |
|
|
|
|
for api in spec.apis: |
|
|
|
|
if not 'index' in api or not api['index']: |
|
|
|
|
continue |
|
|
|
|
|
|
|
|
|
# loop every api |
|
|
|
|
for api in spec.apis: |
|
|
|
|
if not 'index' in api or not api['index']: |
|
|
|
|
continue |
|
|
|
|
|
|
|
|
|
for locale in region['locales']: |
|
|
|
|
# retail or classic |
|
|
|
|
for namespace in api['namespaces']: |
|
|
|
|
try: |
|
|
|
|
last_modified = self.get_last_modified(region['code'], api['path'], namespace, locale) |
|
|
|
|
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None |
|
|
|
|
for namespace in api['namespaces']: |
|
|
|
|
try: |
|
|
|
|
last_modified = self.get_last_modified(region, api['path'], namespace) |
|
|
|
|
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None |
|
|
|
|
|
|
|
|
|
response = self.api_call(region['code'], api['path'], namespace, locale, access_token, headers=headers) |
|
|
|
|
response = self.api_call(region, api['path'], namespace, access_token, headers=headers) |
|
|
|
|
|
|
|
|
|
if response.status_code == 200: |
|
|
|
|
self.save_last_modified(region['code'], api['path'], namespace, locale, response.headers['Last-Modified']) |
|
|
|
|
self.save_raw(region['code'], api['path'], namespace, locale, response) |
|
|
|
|
if response.status_code == 200: |
|
|
|
|
self.save_last_modified(region, api['path'], namespace, response.headers['Last-Modified']) |
|
|
|
|
self.save_raw(region, api['path'], namespace, response) |
|
|
|
|
|
|
|
|
|
except requests.exceptions.HTTPError as e: |
|
|
|
|
log(e) |
|
|
|
|
continue |
|
|
|
|
except requests.exceptions.HTTPError as e: |
|
|
|
|
log(e) |
|
|
|
|
continue |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__": |
|
|
|
|
updater = Updater() |
|
|
|
|
updater.iterate_index() |
|
|
|
|
updater.iterate_index(blizzard.get_by_key(spec.regions, 'code', 'us')['code']) |
|
|
|
|
#updater.iterate_links() |
|
|
|
|