only US

README.md

@@ -1,7 +1,13 @@
 Create a `.credentials` file with `client_id:secret` from https://develop.battle.net/access
 
+With no crontab execute `python3 updater.py`.
+
 Crontab, every day at 12:30h:
 
 ```
-30 12 * * * time python3 updater.py
+30 12 * * * cron.sh
 ```
+
+Only `US` region is explored because all API are the same and this is the default API for Blizzard.
+
+Without `locale` we get all the translations, so I don't iterate through locales. If you ask for a locale, you get the value right in the field.

blizzard.py

@@ -11,3 +11,6 @@ def get_bnet_host(region: str) -> str:
 
 def get_credentials(path: str) -> Tuple[str, str]:
     return tuple(open(os.path.join(path, '.credentials')).read().strip().split(':'))
+
+def get_by_key(values: list, key: str, value):
+    return next(x for x in values if x[key] == value)

cron.sh

@@ -0,0 +1,11 @@
+#! /bin/bash
+
+TIME=`(time python3 updater.py ) 2>&1`
+
+# add every new file
+git add raw
+git add craft
+git add last-modified.db
+
+git commit -m 'daily cron' -m "$TIME"
+git push

spec.py

@@ -58,9 +58,9 @@ apis = [
     {'group': 'Covenant', 'path': '/data/wow/covenant/index', 'namespaces': ['static'], 'index': True},
     {'group': 'Covenant', 'path': '/data/wow/covenant/soulbind/index', 'namespaces': ['static'], 'index': True},
     {'group': 'Covenant', 'path': '/data/wow/covenant/conduit/index', 'namespaces': ['static'], 'index': True},
-    {'group': 'Creature', 'path': '/data/wow/creature-family/index', 'namespaces': ['static'], 'index': True},
+    {'group': 'Creature', 'path': '/data/wow/creature-family/index', 'namespaces': ['static', 'static-classic'], 'index': True},
-    {'group': 'Creature', 'path': '/data/wow/creature-type/index', 'namespaces': ['static'], 'index': True},
+    {'group': 'Creature', 'path': '/data/wow/creature-type/index', 'namespaces': ['static', 'static-classic'], 'index': True},
-    {'group': 'Guild Crest', 'path': '/data/wow/guild-crest/index', 'namespaces': ['static'], 'index': True},
+    {'group': 'Guild Crest', 'path': '/data/wow/guild-crest/index', 'namespaces': ['static', 'static-classic'], 'index': True},
     {'group': '', 'path': '/data/wow/item-class/index', 'namespaces': ['static'], 'index': True},
     {'group': '', 'path': '/data/wow/item-set/index', 'namespaces': ['static'], 'index': True},
     {'group': '', 'path': '/data/wow/journal-expansion/index', 'namespaces': ['static'], 'index': True},

updater.py

@@ -31,9 +31,9 @@ class Updater(object):
         return oauth
 
 
-    def api_call(self, region: str, path: str, namespace: str, locale: str, access_token: str, headers=None) -> requests.Response:
+    def api_call(self, region: str, path: str, namespace: str, access_token: str, headers=None) -> requests.Response:
         url = f"{blizzard.get_api_host(region)}{path}"
-        qs = {'namespace': f"{namespace}-{region}", 'local': locale, 'access_token': access_token}
+        qs = {'namespace': f"{namespace}-{region}", 'access_token': access_token}
 
         api = self.http.get(url, params=qs, headers=headers) #, headers={'Authorization': f"Authorization: Token {access_token}"})
         api.raise_for_status()
@@ -48,8 +48,8 @@ class Updater(object):
                 raise
 
 
-    def save_raw(self, region: str, path: str, namespace: str, locale: str, raw: requests.Response):
+    def save_raw(self, region: str, path: str, namespace: str, raw: requests.Response):
-        dst = f"{config.raw}/{region}/{locale}/{path.replace('/', '_')}.{namespace}.json"
+        dst = f"{config.raw}/{path.replace('/', '_')}.{namespace}.json"
 
         self.create_dst(dst)
 
@@ -57,8 +57,8 @@ class Updater(object):
             f.write(json.dumps(raw.json(), indent=2))
 
 
-    def get_last_modified(self, region: str, path: str, namespace: str, locale: str):
+    def get_last_modified(self, region: str, path: str, namespace: str):
-        key = f"{region}.{locale}.{path}.{namespace}"
+        key = f"{path}.{namespace}"
 
         # cached!
         if self.last_modified is not None and key in self.last_modified:
@@ -77,8 +77,8 @@ class Updater(object):
             return None
 
 
-    def save_last_modified(self, region: str, path: str, namespace: str, locale: str, modified: str):
+    def save_last_modified(self, region: str, path: str, namespace: str, modified: str):
-        key = f"{region}.{locale}.{path}.{namespace}"
+        key = f"{path}.{namespace}"
         db = os.path.join(config.pwd, 'last-modified.db')
 
         # never trust the cache, open, modify and save updated data for safety
@@ -98,40 +98,37 @@ class Updater(object):
             os.replace(tmp_path, db)
 
 
-    def iterate_index(self):
+    def iterate_index(self, region: dict):
-        for region in spec.regions:
+        # access token for region
-            # access token for region
+        try:
-            try:
+            oauth = self.region_oauth(region)
-                oauth = self.region_oauth(region['code'])
+            access_token = oauth['access_token']
-                access_token = oauth['access_token']
+        except (requests.exceptions.HTTPError, KeyError) as e:
-            except (requests.exceptions.HTTPError, KeyError) as e:
+            log(region, type(e), e)
-                log(region, type(e), e)
+            return
+
+        # loop every api
+        for api in spec.apis:
+            if not 'index' in api or not api['index']:
                 continue
 
-            # loop every api
+            for namespace in api['namespaces']:
-            for api in spec.apis:
+                try:
-                if not 'index' in api or not api['index']:
+                    last_modified = self.get_last_modified(region, api['path'], namespace)
+                    headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
+
+                    response = self.api_call(region, api['path'], namespace, access_token, headers=headers)
+
+                    if response.status_code == 200:
+                        self.save_last_modified(region, api['path'], namespace, response.headers['Last-Modified'])
+                        self.save_raw(region, api['path'], namespace, response)
+
+                except requests.exceptions.HTTPError as e:
+                    log(e)
                     continue
 
-                for locale in region['locales']:
-                    # retail or classic
-                    for namespace in api['namespaces']:
-                        try:
-                            last_modified = self.get_last_modified(region['code'], api['path'], namespace, locale)
-                            headers = {'If-Modified-Since': last_modified} if last_modified is not None else None
-
-                            response = self.api_call(region['code'], api['path'], namespace, locale, access_token, headers=headers)
-
-                            if response.status_code == 200:
-                                self.save_last_modified(region['code'], api['path'], namespace, locale, response.headers['Last-Modified'])
-                                self.save_raw(region['code'], api['path'], namespace, locale, response)
-
-                        except requests.exceptions.HTTPError as e:
-                            log(e)
-                            continue
-
 
 if __name__ == "__main__":
     updater = Updater()
-    updater.iterate_index()
+    updater.iterate_index(blizzard.get_by_key(spec.regions, 'code', 'us')['code'])
     #updater.iterate_links()