Updating generate-json.py and index.json

- Updated generate-json.py to support addons_contrib (previously ran into an unhandled UnicodeDecodeError).
- 'download_url' key now works; used with http://localhost:8000 for now.
- index.json generated with updated generate-json.py, now contains addons and addons_contrib.

tools/generate-json.py

@@ -34,7 +34,7 @@ RECOMMENDED_KEYS = ('author', 'description', 'location', 'wiki_url', 'category')
 CURRENT_SCHEMA_VERSION = 1
 
 
-def iter_addons(addons_dir: str) -> (str, str):
+def iter_addons(addons_dir: str) -> (str, str, str):
     """Generator, yields IDs and filenames of addons.
 
     If the addon is a package, yields its __init__.py as filename.
@@ -52,9 +52,9 @@ def iter_addons(addons_dir: str) -> (str, str):
                 log.info('Skipping %s, it does not seem to be a Python package', item.path)
                 continue
 
-            yield (base, fname)
+            yield (base, fname, '.zip')
         else:
-            yield (base, item.path)
+            yield (base, item.path, '.py')
 
 
 def parse_blinfo(addon_fname: str) -> dict:
@@ -66,7 +66,11 @@ def parse_blinfo(addon_fname: str) -> dict:
     log.debug('Parsing %s', addon_fname)
 
     with open(addon_fname) as infile:
+        try:
             source = infile.read()
+        except UnicodeDecodeError as ex:
+            log.warning('Skipping addon: UnicodeDecodeError in %s: %s', addon_fname, ex)
+            return None
 
     try:
         tree = ast.parse(source, addon_fname)
@@ -125,13 +129,13 @@ def parse_addons(addons_dir: str, addons_source: str, addons_base_url: str) -> d
 
     json_data = {}
 
-    for (addon_id, addon_fname) in iter_addons(addons_dir):
+    for (addon_id, addon_fname, addon_ext) in iter_addons(addons_dir):
         bl_info = parse_blinfo(addon_fname)
         if bl_info is None:
             # The reason why has already been logged.
             continue
 
-        url = urllib.parse.urljoin(addons_base_url, addon_id)  # TODO: construct the proper URL (zip/py/whl).
+        url = urllib.parse.urljoin(addons_base_url, addon_id + addon_ext)
         as_json = blinfo_to_json(bl_info, addon_id, addons_source, url)
         if as_json is None:
             # The reason why has already been logged.