no more db, now using mtime from cached files

This commit is contained in:
Sergio Álvarez 2021-05-14 23:11:40 +02:00
parent 56a4518ccf
commit c27861622e
2 changed files with 381 additions and 146 deletions

363
spec.py
View File

@ -12,88 +12,291 @@ regions = [
#{'code': 'cn', 'locales': ['zh_CN']}
]
groups = [
'Achievement',
'Auction House',
'Azerite Essence',
'Connected Realm',
'Covenant',
'Creature',
'Guild Crest',
'Item',
'Journal',
'Media Search',
'Modified Crafting',
'Mount',
'Mythic Keystone Affix',
'Mythic Keystone Dungeon',
'Mythic Keystone Leaderboard',
'Mythic Raid Leaderboard',
'Pet',
'Playable Class',
'Playable Race',
'Playable Specialization',
'Power Type',
'Profession',
'PvP Season',
'PvP Tier',
'Quest',
'Realm',
'Region',
'Reputations',
'Spell',
'Talent',
'Tech Talent',
'Title',
'WoW Token',
]
"""
https://develop.battle.net/documentation/world-of-warcraft/game-data-apis
"""
apis = [
{'group': 'Achievement', 'path': '/data/wow/achievement-category/index', 'namespaces': ['static'], 'index': True},
{'group': 'Achievement', 'path': '/data/wow/achievement/index', 'namespaces': ['static'], 'index': True},
{'group': 'Connected Realm', 'path': '/data/wow/connected-realm/index', 'namespaces': ['dynamic', 'dynamic-classic'], 'index': True},
{'group': 'Covenant', 'path': '/data/wow/covenant/index', 'namespaces': ['static'], 'index': True},
{'group': 'Covenant', 'path': '/data/wow/covenant/soulbind/index', 'namespaces': ['static'], 'index': True},
{'group': 'Covenant', 'path': '/data/wow/covenant/conduit/index', 'namespaces': ['static'], 'index': True},
{'group': 'Creature', 'path': '/data/wow/creature-family/index', 'namespaces': ['static', 'static-classic'], 'index': True},
{'group': 'Creature', 'path': '/data/wow/creature-type/index', 'namespaces': ['static', 'static-classic'], 'index': True},
{'group': 'Guild Crest', 'path': '/data/wow/guild-crest/index', 'namespaces': ['static', 'static-classic'], 'index': True},
{'group': '', 'path': '/data/wow/item-class/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/item-set/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/journal-expansion/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/journal-encounter/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/journal-instance/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/modified-crafting/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/modified-crafting/category/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/modified-crafting/reagent-slot-type/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/mount/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/keystone-affix/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/mythic-keystone/dungeon/index', 'namespaces': ['dynamic'], 'index': True},
{'group': '', 'path': '/data/wow/mythic-keystone/index', 'namespaces': ['dynamic'], 'index': True},
{'group': '', 'path': '/data/wow/mythic-keystone/period/index', 'namespaces': ['dynamic'], 'index': True},
{'group': '', 'path': '/data/wow/pet/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/pet-ability/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/playable-class/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/playable-race/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/playable-specialization/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/power-type/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/profession/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/pvp-season/index', 'namespaces': ['dynamic'], 'index': True},
{'group': '', 'path': '/data/wow/pvp-tier/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/quest/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/quest/category/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/quest/area/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/quest/type/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/realm/index', 'namespaces': ['dynamic'], 'index': True},
{'group': '', 'path': '/data/wow/region/index', 'namespaces': ['dynamic'], 'index': True},
{'group': '', 'path': '/data/wow/reputation-faction/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/reputation-tiers/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/talent/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/pvp-talent/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/tech-talent-tree/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/tech-talent/index', 'namespaces': ['static'], 'index': True},
{'group': '', 'path': '/data/wow/title/index', 'namespaces': ['static'], 'index': True},
{
'group': 'Achievement',
'path': '/data/wow/achievement-category/index',
'namespace': 'static',
'index': True
},
{
'group': 'Achievement',
'path': '/data/wow/achievement/index',
'namespace': 'static',
'index': True
},
{
'group': 'Connected Realm',
'path': '/data/wow/connected-realm/index',
'namespace': 'dynamic',
'index': True
},
#{
# 'group': 'Connected Realm',
# 'path': '/data/wow/connected-realm/{connectedRealmId}',
# 'namespace': 'dynamic',
# 'connectedRealmId': {
# 'source': '/data/wow/connected-realm/index',
# 'loop': lambda l: l
# }
#},
{
'group': 'Covenant',
'path': '/data/wow/covenant/index',
'namespace': 'static',
'index': True
},
{
'group': 'Covenant',
'path': '/data/wow/covenant/soulbind/index',
'namespace': 'static',
'index': True
},
{
'group': 'Covenant',
'path': '/data/wow/covenant/conduit/index',
'namespace': 'static',
'index': True
},
{
'group': 'Creature',
'path': '/data/wow/creature-family/index',
'namespace': 'static',
'index': True
},
{
'group': 'Creature',
'path': '/data/wow/creature-type/index',
'namespace': 'static',
'index': True
},
{
'group': 'Guild Crest',
'path': '/data/wow/guild-crest/index',
'namespace': 'static',
'index': True
},
{
'group': 'Item',
'path': '/data/wow/item-class/index',
'namespace': 'static',
'index': True
},
{
'group': 'Item',
'path': '/data/wow/item-set/index',
'namespace': 'static',
'index': True
},
{
'group': 'Journal',
'path': '/data/wow/journal-expansion/index',
'namespace': 'static',
'index': True
},
{
'group': 'Journal',
'path': '/data/wow/journal-encounter/index',
'namespace': 'static',
'index': True
},
{
'group': 'Journal',
'path': '/data/wow/journal-instance/index',
'namespace': 'static',
'index': True
},
{
'group': 'Modified Crafting',
'path': '/data/wow/modified-crafting/index',
'namespace': 'static',
'index': True
},
{
'group': 'Modified Crafting',
'path': '/data/wow/modified-crafting/category/index',
'namespace': 'static',
'index': True
},
{
'group': 'Modified Crafting',
'path': '/data/wow/modified-crafting/reagent-slot-type/index',
'namespace': 'static',
'index': True
},
{
'group': 'Mount',
'path': '/data/wow/mount/index',
'namespace': 'static',
'index': True
},
{
'group': 'Mythic Keystone Affix',
'path': '/data/wow/keystone-affix/index',
'namespace': 'static',
'index': True
},
{
'group': 'Mythic Keystone Dungeon',
'path': '/data/wow/mythic-keystone/dungeon/index',
'namespace': 'dynamic',
'index': True
},
{
'group': 'Mythic Keystone Dungeon',
'path': '/data/wow/mythic-keystone/index',
'namespace': 'dynamic',
'index': True
},
{
'group': 'Mythic Keystone Dungeon',
'path': '/data/wow/mythic-keystone/period/index',
'namespace': 'dynamic',
'index': True
},
{
'group': 'Pet',
'path': '/data/wow/pet/index',
'namespace': 'static',
'index': True
},
{
'group': 'Pet',
'path': '/data/wow/pet-ability/index',
'namespace': 'static',
'index': True
},
{
'group': 'Playable Class',
'path': '/data/wow/playable-class/index',
'namespace': 'static',
'index': True
},
{
'group': 'Playable Race',
'path': '/data/wow/playable-race/index',
'namespace': 'static',
'index': True
},
{
'group': 'Playable Specialization',
'path': '/data/wow/playable-specialization/index',
'namespace': 'static',
'index': True
},
{
'group': 'Power Type',
'path': '/data/wow/power-type/index',
'namespace': 'static',
'index': True
},
{
'group': 'Profession',
'path': '/data/wow/profession/index',
'namespace': 'static',
'index': True
},
{
'group': 'PvP Season',
'path': '/data/wow/pvp-season/index',
'namespace': 'dynamic',
'index': True
},
{
'group': 'PvP Tier',
'path': '/data/wow/pvp-tier/index',
'namespace': 'static',
'index': True
},
{
'group': 'Quest',
'path': '/data/wow/quest/index',
'namespace': 'static',
'index': True
},
{
'group': 'Quest',
'path': '/data/wow/quest/category/index',
'namespace': 'static',
'index': True
},
{
'group': 'Quest',
'path': '/data/wow/quest/area/index',
'namespace': 'static',
'index': True
},
{
'group': 'Quest',
'path': '/data/wow/quest/type/index',
'namespace': 'static',
'index': True
},
{
'group': 'Realm',
'path': '/data/wow/realm/index',
'namespace': 'dynamic',
'index': True
},
{
'group': 'Region',
'path': '/data/wow/region/index',
'namespace': 'dynamic',
'index': True
},
{
'group': 'Reputations',
'path': '/data/wow/reputation-faction/index',
'namespace': 'static',
'index': True
},
{
'group': 'Reputations',
'path': '/data/wow/reputation-tiers/index',
'namespace': 'static',
'index': True
},
{
'group': 'Talent',
'path': '/data/wow/talent/index',
'namespace': 'static',
'index': True
},
{
'group': 'Talent',
'path': '/data/wow/pvp-talent/index',
'namespace': 'static',
'index': True
},
{
'group': 'Tech Talent',
'path': '/data/wow/tech-talent-tree/index',
'namespace': 'static',
'index': True
},
{
'group': 'Tech Talent',
'path': '/data/wow/tech-talent/index',
'namespace': 'static',
'index': True
},
{
'group': 'Title',
'path': '/data/wow/title/index',
'namespace': 'static',
'index': True
},
{
'group': 'Title',
'path': '/data/wow/title/{titleId}',
'namespace': 'static',
'titleId': {
'source': '/data/wow/title/index',
'list': 'titles',
'value': lambda item: item['id']
}
},
]

View File

@ -4,7 +4,9 @@ import errno
import requests
import json
import datetime
import time
import tempfile
import pytz
import config
import blizzard
@ -17,23 +19,32 @@ def log(*data):
class Updater(object):
def __init__(self):
RFC2616 = '%a, %d %b %Y %H:%M:%S %Z'
def __init__(self, region: str):
self.region = region
self.http = requests.Session()
self.credentials = blizzard.get_credentials(config.pwd)
self.last_modified = None
self.last_modified = {}
self.oauth = None
def region_oauth(self, region: str) -> dict:
api = self.http.post(f"{blizzard.get_bnet_host(region)}/oauth/token", data={'grant_type': 'client_credentials'}, auth=self.credentials)
def get_oauth(self) -> dict:
if self.oauth is not None:
return self.oauth
api = self.http.post(f"{blizzard.get_bnet_host(self.region)}/oauth/token", data={'grant_type': 'client_credentials'}, auth=self.credentials)
oauth = api.json()
#log(region, oauth)
api.raise_for_status()
self.oauth = oauth
return oauth
def api_call(self, region: str, path: str, namespace: str, access_token: str, headers=None) -> requests.Response:
url = f"{blizzard.get_api_host(region)}{path}"
qs = {'namespace': f"{namespace}-{region}", 'access_token': access_token}
def api_call(self, path: str, namespace: str, access_token: str, headers=None) -> requests.Response:
url = f"{blizzard.get_api_host(self.region)}{path}"
qs = {'namespace': f"{namespace}-{self.region}", 'access_token': access_token}
api = self.http.get(url, params=qs, headers=headers) #, headers={'Authorization': f"Authorization: Token {access_token}"})
api.raise_for_status()
@ -48,87 +59,108 @@ class Updater(object):
raise
def save_raw(self, region: str, path: str, namespace: str, raw: requests.Response):
dst = f"{config.raw}/{path.replace('/', '_')}.{namespace}.json"
def save_raw(self, path: str, raw: requests.Response):
dst = f"{config.raw}/{path.replace('/', '_')}.json"
self.create_dst(dst)
with open(dst, 'w+') as f:
f.write(json.dumps(raw.json(), indent=2))
def get_last_modified(self, region: str, path: str, namespace: str):
key = f"{path}.{namespace}"
# cached!
if self.last_modified is not None and key in self.last_modified:
return self.last_modified[key]
# not cached, check db again
db = os.path.join(config.pwd, 'last-modified.db')
with open(db, 'r') as f:
data = json.load(f)
self.last_modified = data
try:
return data[key]
except KeyError:
return None
def save_last_modified(self, region: str, path: str, namespace: str, modified: str):
key = f"{path}.{namespace}"
db = os.path.join(config.pwd, 'last-modified.db')
# never trust the cache, open, modify and save updated data for safety
with open(db, 'r') as f:
data = json.load(f)
data[key] = modified
# update cache with new data
self.last_modified = data
# safe write: tmp write and then mv
fd, tmp_path = tempfile.mkstemp(dir=config.pwd)
try:
with os.fdopen(fd, 'w') as tmp:
json.dump(data, tmp, indent=2)
json.dump(raw.json(), tmp, indent=2)
finally:
os.replace(tmp_path, db)
os.replace(tmp_path, dst)
def iterate_index(self, region: dict):
# access token for region
def get_last_modified(self, path: str):
# cached!
if path in self.last_modified:
return self.last_modified[path]
dst = f"{config.raw}/{path.replace('/', '_')}.json"
# mtime to datetime from timestamp, default tz, replace to GMT, format
modified = datetime.datetime.fromtimestamp(os.path.getmtime(dst)).astimezone().replace(tzinfo=pytz.timezone('GMT')).strftime(self.RFC2616)
self.last_modified[path] = modified
return self.last_modified[path]
def set_last_modified(self, path: str, modified: str):
dst = f"{config.raw}/{path.replace('/', '_')}.json"
self.last_modified[path] = modified
# same Last-Modified for raw file
epoch = time.mktime(time.strptime(modified, self.RFC2616)) # Tue, 11 May 2021 14:06:37 GMT
os.utime(dst, (epoch, epoch))
def iterate_index(self):
try:
oauth = self.region_oauth(region)
oauth = self.get_oauth()
access_token = oauth['access_token']
except (requests.exceptions.HTTPError, KeyError) as e:
log(region, type(e), e)
log(type(e), e)
return
# loop every api
for api in spec.apis:
# only indexes
if not 'index' in api or not api['index']:
continue
for namespace in api['namespaces']:
try:
last_modified = self.get_last_modified(region, api['path'], namespace)
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
try:
last_modified = self.get_last_modified(api['path'])
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
response = self.api_call(region, api['path'], namespace, access_token, headers=headers)
response = self.api_call(api['path'], api['namespace'], access_token, headers=headers)
if response.status_code == 200:
self.save_last_modified(region, api['path'], namespace, response.headers['Last-Modified'])
self.save_raw(region, api['path'], namespace, response)
log(response.status_code, api)
log(response.request.headers)
log(response.headers)
if response.status_code == 200:
self.save_raw(api['path'], response)
self.set_last_modified(api['path'], response.headers['Last-Modified'])
except requests.exceptions.HTTPError as e:
log(e)
continue
except requests.exceptions.HTTPError as e:
log(e)
continue
def iterate_links(self):
try:
oauth = self.get_oauth()
access_token = oauth['access_token']
except (requests.exceptions.HTTPError, KeyError) as e:
log(type(e), e)
return
for api in spec.apis:
# no indexes
if 'index' in api and api['index']:
continue
log(api)
continue
try:
last_modified = self.get_last_modified(api['path'])
headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
response = self.api_call(api['path'], api['namespace'], access_token, headers=headers)
log(response.status_code, api)
if response.status_code == 200:
self.save_last_modified(api['path'], response.headers['Last-Modified'])
self.save_raw(api['path'], response)
except requests.exceptions.HTTPError as e:
log(e)
continue
if __name__ == "__main__":
updater = Updater()
updater.iterate_index(blizzard.get_by_key(spec.regions, 'code', 'us')['code'])
#updater.iterate_links()
region = blizzard.get_by_key(spec.regions, 'code', 'us')['code']
updater = Updater(region)
updater.iterate_index()
updater.iterate_links()