no more db, now using mtime from cached files

spec.py

@@ -12,88 +12,291 @@ regions = [
     #{'code': 'cn', 'locales': ['zh_CN']}
 ]
 
-groups = [
-    'Achievement',
-    'Auction House',
-    'Azerite Essence',
-    'Connected Realm',
-    'Covenant',
-    'Creature',
-    'Guild Crest',
-    'Item',
-    'Journal',
-    'Media Search',
-    'Modified Crafting',
-    'Mount',
-    'Mythic Keystone Affix',
-    'Mythic Keystone Dungeon',
-    'Mythic Keystone Leaderboard',
-    'Mythic Raid Leaderboard',
-    'Pet',
-    'Playable Class',
-    'Playable Race',
-    'Playable Specialization',
-    'Power Type',
-    'Profession',
-    'PvP Season',
-    'PvP Tier',
-    'Quest',
-    'Realm',
-    'Region',
-    'Reputations',
-    'Spell',
-    'Talent',
-    'Tech Talent',
-    'Title',
-    'WoW Token',
-]
-
 """
 https://develop.battle.net/documentation/world-of-warcraft/game-data-apis
 """
 apis = [
-    {'group': 'Achievement', 'path': '/data/wow/achievement-category/index', 'namespaces': ['static'], 'index': True},
+    {
-    {'group': 'Achievement', 'path': '/data/wow/achievement/index', 'namespaces': ['static'], 'index': True},
+        'group': 'Achievement', 
-    {'group': 'Connected Realm', 'path': '/data/wow/connected-realm/index', 'namespaces': ['dynamic', 'dynamic-classic'], 'index': True},
+        'path': '/data/wow/achievement-category/index', 
-    {'group': 'Covenant', 'path': '/data/wow/covenant/index', 'namespaces': ['static'], 'index': True},
+        'namespace': 'static', 
-    {'group': 'Covenant', 'path': '/data/wow/covenant/soulbind/index', 'namespaces': ['static'], 'index': True},
+        'index': True
-    {'group': 'Covenant', 'path': '/data/wow/covenant/conduit/index', 'namespaces': ['static'], 'index': True},
+    },
-    {'group': 'Creature', 'path': '/data/wow/creature-family/index', 'namespaces': ['static', 'static-classic'], 'index': True},
+    {
-    {'group': 'Creature', 'path': '/data/wow/creature-type/index', 'namespaces': ['static', 'static-classic'], 'index': True},
+        'group': 'Achievement', 
-    {'group': 'Guild Crest', 'path': '/data/wow/guild-crest/index', 'namespaces': ['static', 'static-classic'], 'index': True},
+        'path': '/data/wow/achievement/index', 
-    {'group': '', 'path': '/data/wow/item-class/index', 'namespaces': ['static'], 'index': True},
+        'namespace': 'static', 
-    {'group': '', 'path': '/data/wow/item-set/index', 'namespaces': ['static'], 'index': True},
+        'index': True
-    {'group': '', 'path': '/data/wow/journal-expansion/index', 'namespaces': ['static'], 'index': True},
+    },
-    {'group': '', 'path': '/data/wow/journal-encounter/index', 'namespaces': ['static'], 'index': True},
+    {
-    {'group': '', 'path': '/data/wow/journal-instance/index', 'namespaces': ['static'], 'index': True},
+        'group': 'Connected Realm', 
-    {'group': '', 'path': '/data/wow/modified-crafting/index', 'namespaces': ['static'], 'index': True},
+        'path': '/data/wow/connected-realm/index', 
-    {'group': '', 'path': '/data/wow/modified-crafting/category/index', 'namespaces': ['static'], 'index': True},
+        'namespace': 'dynamic', 
-    {'group': '', 'path': '/data/wow/modified-crafting/reagent-slot-type/index', 'namespaces': ['static'], 'index': True},
+        'index': True
-    {'group': '', 'path': '/data/wow/mount/index', 'namespaces': ['static'], 'index': True},
+    },
-    {'group': '', 'path': '/data/wow/keystone-affix/index', 'namespaces': ['static'], 'index': True},
+    #{
-    {'group': '', 'path': '/data/wow/mythic-keystone/dungeon/index', 'namespaces': ['dynamic'], 'index': True},
+    #    'group': 'Connected Realm', 
-    {'group': '', 'path': '/data/wow/mythic-keystone/index', 'namespaces': ['dynamic'], 'index': True},
+    #    'path': '/data/wow/connected-realm/{connectedRealmId}', 
-    {'group': '', 'path': '/data/wow/mythic-keystone/period/index', 'namespaces': ['dynamic'], 'index': True},
+    #    'namespace': 'dynamic', 
-    {'group': '', 'path': '/data/wow/pet/index', 'namespaces': ['static'], 'index': True},
+    #    'connectedRealmId': {
-    {'group': '', 'path': '/data/wow/pet-ability/index', 'namespaces': ['static'], 'index': True},
+    #        'source': '/data/wow/connected-realm/index',
-    {'group': '', 'path': '/data/wow/playable-class/index', 'namespaces': ['static'], 'index': True},
+    #        'loop': lambda l: l
-    {'group': '', 'path': '/data/wow/playable-race/index', 'namespaces': ['static'], 'index': True},
+    #    }
-    {'group': '', 'path': '/data/wow/playable-specialization/index', 'namespaces': ['static'], 'index': True},
+    #},
-    {'group': '', 'path': '/data/wow/power-type/index', 'namespaces': ['static'], 'index': True},
+    {
-    {'group': '', 'path': '/data/wow/profession/index', 'namespaces': ['static'], 'index': True},
+        'group': 'Covenant', 
-    {'group': '', 'path': '/data/wow/pvp-season/index', 'namespaces': ['dynamic'], 'index': True},
+        'path': '/data/wow/covenant/index', 
-    {'group': '', 'path': '/data/wow/pvp-tier/index', 'namespaces': ['static'], 'index': True},
+        'namespace': 'static', 
-    {'group': '', 'path': '/data/wow/quest/index', 'namespaces': ['static'], 'index': True},
+        'index': True
-    {'group': '', 'path': '/data/wow/quest/category/index', 'namespaces': ['static'], 'index': True},
+    },
-    {'group': '', 'path': '/data/wow/quest/area/index', 'namespaces': ['static'], 'index': True},
+    {
-    {'group': '', 'path': '/data/wow/quest/type/index', 'namespaces': ['static'], 'index': True},
+        'group': 'Covenant', 
-    {'group': '', 'path': '/data/wow/realm/index', 'namespaces': ['dynamic'], 'index': True},
+        'path': '/data/wow/covenant/soulbind/index', 
-    {'group': '', 'path': '/data/wow/region/index', 'namespaces': ['dynamic'], 'index': True},
+        'namespace': 'static', 
-    {'group': '', 'path': '/data/wow/reputation-faction/index', 'namespaces': ['static'], 'index': True},
+        'index': True
-    {'group': '', 'path': '/data/wow/reputation-tiers/index', 'namespaces': ['static'], 'index': True},
+    },
-    {'group': '', 'path': '/data/wow/talent/index', 'namespaces': ['static'], 'index': True},
+    {
-    {'group': '', 'path': '/data/wow/pvp-talent/index', 'namespaces': ['static'], 'index': True},
+        'group': 'Covenant', 
-    {'group': '', 'path': '/data/wow/tech-talent-tree/index', 'namespaces': ['static'], 'index': True},
+        'path': '/data/wow/covenant/conduit/index', 
-    {'group': '', 'path': '/data/wow/tech-talent/index', 'namespaces': ['static'], 'index': True},
+        'namespace': 'static', 
-    {'group': '', 'path': '/data/wow/title/index', 'namespaces': ['static'], 'index': True},
+        'index': True
+    },
+    {
+        'group': 'Creature', 
+        'path': '/data/wow/creature-family/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Creature', 
+        'path': '/data/wow/creature-type/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Guild Crest', 
+        'path': '/data/wow/guild-crest/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Item', 
+        'path': '/data/wow/item-class/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Item', 
+        'path': '/data/wow/item-set/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Journal', 
+        'path': '/data/wow/journal-expansion/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Journal', 
+        'path': '/data/wow/journal-encounter/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Journal', 
+        'path': '/data/wow/journal-instance/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Modified Crafting', 
+        'path': '/data/wow/modified-crafting/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Modified Crafting', 
+        'path': '/data/wow/modified-crafting/category/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Modified Crafting', 
+        'path': '/data/wow/modified-crafting/reagent-slot-type/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Mount', 
+        'path': '/data/wow/mount/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Mythic Keystone Affix', 
+        'path': '/data/wow/keystone-affix/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Mythic Keystone Dungeon', 
+        'path': '/data/wow/mythic-keystone/dungeon/index', 
+        'namespace': 'dynamic', 
+        'index': True
+    },
+    {
+        'group': 'Mythic Keystone Dungeon', 
+        'path': '/data/wow/mythic-keystone/index', 
+        'namespace': 'dynamic', 
+        'index': True
+    },
+    {
+        'group': 'Mythic Keystone Dungeon', 
+        'path': '/data/wow/mythic-keystone/period/index', 
+        'namespace': 'dynamic', 
+        'index': True
+    },
+    {
+        'group': 'Pet', 
+        'path': '/data/wow/pet/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Pet', 
+        'path': '/data/wow/pet-ability/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Playable Class', 
+        'path': '/data/wow/playable-class/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Playable Race', 
+        'path': '/data/wow/playable-race/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Playable Specialization', 
+        'path': '/data/wow/playable-specialization/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Power Type', 
+        'path': '/data/wow/power-type/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Profession', 
+        'path': '/data/wow/profession/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'PvP Season', 
+        'path': '/data/wow/pvp-season/index', 
+        'namespace': 'dynamic', 
+        'index': True
+    },
+    {
+        'group': 'PvP Tier', 
+        'path': '/data/wow/pvp-tier/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Quest', 
+        'path': '/data/wow/quest/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Quest', 
+        'path': '/data/wow/quest/category/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Quest', 
+        'path': '/data/wow/quest/area/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Quest', 
+        'path': '/data/wow/quest/type/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Realm', 
+        'path': '/data/wow/realm/index', 
+        'namespace': 'dynamic', 
+        'index': True
+    },
+    {
+        'group': 'Region', 
+        'path': '/data/wow/region/index', 
+        'namespace': 'dynamic', 
+        'index': True
+    },
+    {
+        'group': 'Reputations', 
+        'path': '/data/wow/reputation-faction/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Reputations', 
+        'path': '/data/wow/reputation-tiers/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Talent', 
+        'path': '/data/wow/talent/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Talent', 
+        'path': '/data/wow/pvp-talent/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Tech Talent', 
+        'path': '/data/wow/tech-talent-tree/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Tech Talent', 
+        'path': '/data/wow/tech-talent/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Title', 
+        'path': '/data/wow/title/index', 
+        'namespace': 'static', 
+        'index': True
+    },
+    {
+        'group': 'Title', 
+        'path': '/data/wow/title/{titleId}', 
+        'namespace': 'static', 
+        'titleId': {
+            'source': '/data/wow/title/index',
+            'list': 'titles',
+            'value': lambda item: item['id']
+        }
+    },
 ]

updater.py

@@ -4,7 +4,9 @@ import errno
 import requests
 import json
 import datetime
+import time
 import tempfile
+import pytz
 
 import config
 import blizzard
@@ -17,23 +19,32 @@ def log(*data):
 
 class Updater(object):
 
-    def __init__(self):
+    RFC2616 = '%a, %d %b %Y %H:%M:%S %Z'
+
+    def __init__(self, region: str):
+        self.region = region
         self.http = requests.Session()
         self.credentials = blizzard.get_credentials(config.pwd)
-        self.last_modified = None
+        self.last_modified = {}
+        self.oauth = None
 
 
-    def region_oauth(self, region: str) -> dict:
+    def get_oauth(self) -> dict:
-        api = self.http.post(f"{blizzard.get_bnet_host(region)}/oauth/token", data={'grant_type': 'client_credentials'}, auth=self.credentials)
+        if self.oauth is not None:
+            return self.oauth
+
+        api = self.http.post(f"{blizzard.get_bnet_host(self.region)}/oauth/token", data={'grant_type': 'client_credentials'}, auth=self.credentials)
         oauth = api.json()
         #log(region, oauth)
         api.raise_for_status()
+
+        self.oauth = oauth
         return oauth
 
 
-    def api_call(self, region: str, path: str, namespace: str, access_token: str, headers=None) -> requests.Response:
+    def api_call(self, path: str, namespace: str, access_token: str, headers=None) -> requests.Response:
-        url = f"{blizzard.get_api_host(region)}{path}"
+        url = f"{blizzard.get_api_host(self.region)}{path}"
-        qs = {'namespace': f"{namespace}-{region}", 'access_token': access_token}
+        qs = {'namespace': f"{namespace}-{self.region}", 'access_token': access_token}
 
         api = self.http.get(url, params=qs, headers=headers) #, headers={'Authorization': f"Authorization: Token {access_token}"})
         api.raise_for_status()
@@ -48,87 +59,108 @@ class Updater(object):
                 raise
 
 
-    def save_raw(self, region: str, path: str, namespace: str, raw: requests.Response):
+    def save_raw(self, path: str, raw: requests.Response):
-        dst = f"{config.raw}/{path.replace('/', '_')}.{namespace}.json"
+        dst = f"{config.raw}/{path.replace('/', '_')}.json"
 
         self.create_dst(dst)
 
-        with open(dst, 'w+') as f:
-            f.write(json.dumps(raw.json(), indent=2))
-
-
-    def get_last_modified(self, region: str, path: str, namespace: str):
-        key = f"{path}.{namespace}"
-
-        # cached!
-        if self.last_modified is not None and key in self.last_modified:
-            return self.last_modified[key]
-
-        # not cached, check db again
-        db = os.path.join(config.pwd, 'last-modified.db')
-
-        with open(db, 'r') as f:
-            data = json.load(f)
-        self.last_modified = data
-
-        try:
-            return data[key]
-        except KeyError:
-            return None
-
-
-    def save_last_modified(self, region: str, path: str, namespace: str, modified: str):
-        key = f"{path}.{namespace}"
-        db = os.path.join(config.pwd, 'last-modified.db')
-
-        # never trust the cache, open, modify and save updated data for safety
-        with open(db, 'r') as f:
-            data = json.load(f)
-
-        data[key] = modified
-        # update cache with new data
-        self.last_modified = data
-
         # safe write: tmp write and then mv
         fd, tmp_path = tempfile.mkstemp(dir=config.pwd)
         try:
             with os.fdopen(fd, 'w') as tmp:
-                json.dump(data, tmp, indent=2)
+                json.dump(raw.json(), tmp, indent=2)
         finally:
-            os.replace(tmp_path, db)
+            os.replace(tmp_path, dst)
 
 
-    def iterate_index(self, region: dict):
+    def get_last_modified(self, path: str):
-        # access token for region
+        # cached!
+        if path in self.last_modified:
+            return self.last_modified[path]
+
+        dst = f"{config.raw}/{path.replace('/', '_')}.json"
+
+        # mtime to datetime from timestamp, default tz, replace to GMT, format
+        modified = datetime.datetime.fromtimestamp(os.path.getmtime(dst)).astimezone().replace(tzinfo=pytz.timezone('GMT')).strftime(self.RFC2616)
+        self.last_modified[path] = modified
+
+        return self.last_modified[path]
+
+
+    def set_last_modified(self, path: str, modified: str):
+        dst = f"{config.raw}/{path.replace('/', '_')}.json"
+        self.last_modified[path] = modified
+
+        # same Last-Modified for raw file
+        epoch = time.mktime(time.strptime(modified, self.RFC2616)) # Tue, 11 May 2021 14:06:37 GMT
+        os.utime(dst, (epoch, epoch))
+
+
+    def iterate_index(self):
         try:
-            oauth = self.region_oauth(region)
+            oauth = self.get_oauth()
             access_token = oauth['access_token']
         except (requests.exceptions.HTTPError, KeyError) as e:
-            log(region, type(e), e)
+            log(type(e), e)
             return
 
-        # loop every api
         for api in spec.apis:
+            # only indexes
             if not 'index' in api or not api['index']:
                 continue
 
-            for namespace in api['namespaces']:
+            try:
-                try:
+                last_modified = self.get_last_modified(api['path'])
-                    last_modified = self.get_last_modified(region, api['path'], namespace)
+                headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
-                    headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
 
-                    response = self.api_call(region, api['path'], namespace, access_token, headers=headers)
+                response = self.api_call(api['path'], api['namespace'], access_token, headers=headers)
 
-                    if response.status_code == 200:
+                log(response.status_code, api)
-                        self.save_last_modified(region, api['path'], namespace, response.headers['Last-Modified'])
+                log(response.request.headers)
-                        self.save_raw(region, api['path'], namespace, response)
+                log(response.headers)
+                if response.status_code == 200:
+                    self.save_raw(api['path'], response)
+                    self.set_last_modified(api['path'], response.headers['Last-Modified'])
 
-                except requests.exceptions.HTTPError as e:
+            except requests.exceptions.HTTPError as e:
-                    log(e)
+                log(e)
-                    continue
+                continue
+
+
+    def iterate_links(self):
+        try:
+            oauth = self.get_oauth()
+            access_token = oauth['access_token']
+        except (requests.exceptions.HTTPError, KeyError) as e:
+            log(type(e), e)
+            return
+
+        for api in spec.apis:
+            # no indexes
+            if 'index' in api and api['index']:
+                continue
+
+            log(api)
+            continue
+            try:
+                last_modified = self.get_last_modified(api['path'])
+                headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
+
+                response = self.api_call(api['path'], api['namespace'], access_token, headers=headers)
+
+                log(response.status_code, api)
+                if response.status_code == 200:
+                    self.save_last_modified(api['path'], response.headers['Last-Modified'])
+                    self.save_raw(api['path'], response)
+
+            except requests.exceptions.HTTPError as e:
+                log(e)
+                continue
 
 
 if __name__ == "__main__":
-    updater = Updater()
+    region = blizzard.get_by_key(spec.regions, 'code', 'us')['code']
-    updater.iterate_index(blizzard.get_by_key(spec.regions, 'code', 'us')['code'])
+
-    #updater.iterate_links()
+    updater = Updater(region)
+    updater.iterate_index()
+    updater.iterate_links()