From 1db0d0dec44180bba7d6ec9f999d5c18f0bcc304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Wed, 22 Jun 2016 00:10:51 +0200 Subject: [PATCH] Using 'ast' and 'json' modules to parse & write addon data. --- tools/generate-json.py | 375 +++++++++++++++++++---------------------- 1 file changed, 175 insertions(+), 200 deletions(-) diff --git a/tools/generate-json.py b/tools/generate-json.py index 744dbc9..fc71d9d 100644 --- a/tools/generate-json.py +++ b/tools/generate-json.py @@ -19,212 +19,187 @@ # for use in the package manager add-on. import os -import shlex +import ast +import argparse +import logging +import urllib.parse +import json -addons_directory = 'C:\\Users\\Peter\\Documents\\blender-git\\blender\\release\\scripts\\addons\\' -contrib_directory = 'C:\\Users\\Peter\\Documents\\blender-git\\blender\\release\\scripts\\addons_contrib\\' -index_directory = 'C:\\Users\\Peter\\Documents\\blender-package-manager-addon\\addons\\' +logging.basicConfig(format='%(asctime)-15s %(levelname)8s %(name)s %(message)s', + level=logging.INFO) +log = logging.getLogger('generate-json') -# This script is functional for now, but rather "hacky" and much of it -# will be replaced by a modified version of the code which Blender uses -# to parse add-on's bl_info dictionary. +REQUIRED_KEYS = ('name', 'blender') +RECOMMENDED_KEYS = ('author', 'description', 'location', 'wiki_url', 'category') +CURRENT_SCHEMA_VERSION = 1 -def list_addons (dir, addon=""): - json_text = "" - for item in os.scandir(dir): + +def iter_addons(addons_dir: str) -> (str, str): + """Generator, yields IDs and filenames of addons. + + If the addon is a package, yields its __init__.py as filename. + """ + + for item in os.scandir(addons_dir): if item.name[0] == '.': continue - if '.' not in item.name: - if addon not in item.name and addon != "": - continue - if item.is_dir() and (item.name == addon or addon == ""): - filename = os.path.join(item.path, '__init__.py') - if item.is_file() and '.py' in item and (item.name[:-3] == addon or addon == ""): - filename = item.path - - try: - file_lines = open(filename, mode='r').readlines() - except FileNotFoundError: - continue - new_lines = [] - for line in file_lines: - if not line.startswith('#'): - #line = line[:line.find('#')] FIXME did not take into - # account '#' symbols inside quotes. - if line.__len__() > 0: - new_lines.append(line) - file_text = ''.join(new_lines) - - bl_info_at = file_text.find('bl_info') - file_text = file_text[bl_info_at:] - open_bracket_at = file_text.find('{') - offset = 0 - quote_type = "" - in_quote = False - current_quote = "key" - escape_next = False - - bl_info = {} - key = "" - value = "" - tuple_item = 0 - - for char in file_text[open_bracket_at+1:]: - if char == '\\': - escape_next = not escape_next - - was_in_quote = in_quote - if (not escape_next and (char == "'" or char == '"') and (not in_quote or (in_quote and char == quote_type))): - in_quote = not in_quote - quote_type = char - - if in_quote and quote_type == ')' and (char == "'" or char == '"'): - quote_type = char - value = "" - - if not in_quote and char == '(': - in_quote = True - value = () - quote_type = ')' - elif in_quote and char == quote_type and quote_type == ')': - in_quote = False - elif in_quote and char != quote_type: - if quote_type != ')': - if current_quote == "key": - key += char - else: - value += char - else: - if char == ',': - value = value + (tuple_item, ) - tuple_item = 0 - elif char.isdigit(): - tuple_item = (tuple_item * 10) + int(char) - - if not in_quote: - if char == ',' or char == '}': - current_quote = "key" - if key != "": - if value != "" or value != None: - bl_info[key] = value - else: - bl_info[key] = "" - key = "" - value = "" - tuple_item = 0 - elif was_in_quote: - if current_quote == "value" and quote_type == ')': - current_quote = "key" - value = value + (tuple_item, ) - bl_info[key] = value - key = "" - value = "" - tuple_item = 0 - else: - current_quote = "value" - - offset += 1 - - if not in_quote and char == '}': - break - - #print(file_text[:offset+open_bracket_at+1]) - #for k, v in bl_info.items(): - # print('"' + k + '": ' + repr(v)) - if item.is_dir(): - json_text += bl_info_to_json(bl_info, item.name) - else: - json_text += bl_info_to_json(bl_info, item.name, is_zip = False) - - return json_text -def bl_info_to_json (bl_info, addon_id, source = 'internal', is_zip = True): - required_keys = ['name', 'blender'] - recommended_keys = ['author', 'description', 'location', 'wiki_url', 'category'] - - for key in required_keys: - if key not in bl_info: - print("Error: missing key \"" + key + "\" in add-on " + addon_id - + "'s bl_info, or bl_info dict may be malformed") - return "" - - for key in recommended_keys: - if key not in bl_info: - print("Warning: missing key \"" + key + "\" in add-on " + addon_id - + "'s bl_info, or bl_info dict may be malformed") - - author = "" - if 'author' in bl_info: - author = bl_info['author'] - - description = "" - if 'description' in bl_info: - description = bl_info['description'] - - tracker_url = "" - if 'tracker_url' in bl_info: - tracker_url = bl_info['tracker_url'] - - wiki_url = "" - if 'wiki_url' in bl_info: - wiki_url = bl_info['wiki_url'] - - location = "" - if 'location' in bl_info: - location = bl_info['location'] - - category = "" - if 'category' in bl_info: - category = bl_info['category'] - - warning = "" - if 'warning' in bl_info: - warning = bl_info['warning'] - - version = "unversioned" - if 'version' in bl_info: - version = '.'.join(map(str, bl_info['version'])) - - support = 'community'.lower() - if 'support' in bl_info: - support = bl_info['support'].lower() - - json_text = "" - - json_text += "\t\t\"" + addon_id + "\": {\n" - json_text += "\t\t\t\"source\": \"" + source + "\"\n" - json_text += "\t\t\t\"name\": \"" + bl_info['name'] + "\"\n" - json_text += "\t\t\t\"description\": \"" + description + "\"\n" - json_text += "\t\t\t\"author\": \"" + author + "\"\n" - json_text += "\t\t\t\"wiki_url\": \"" + wiki_url + "\"\n" - json_text += "\t\t\t\"tracker_url\": \"" + tracker_url + "\"\n" - json_text += "\t\t\t\"location\": \"" + location + "\"\n" - json_text += "\t\t\t\"category\": \"" + category + "\"\n" - json_text += "\t\t\t\"version\": {\n" - json_text += "\t\t\t\t\"" + version + "\": {\n" - json_text += "\t\t\t\t\t\"blender\": \"" + '.'.join(map(str, bl_info['blender'])) + "\"\n" - if warning != "": - json_text += "\t\t\t\t\t\"warning\": \"" + warning + "\"\n" - json_text += "\t\t\t\t\t\"support\": \"" + support + "\"\n" - if is_zip: - json_text += "\t\t\t\t\t\"filename\": \"" + version + ".zip\"\n" + if item.is_dir(): + fname = os.path.join(item.path, '__init__.py') + if not os.path.exists(fname): + log.info('Skipping %s, it does not seem to be a Python package', item.path) + continue + + yield (item.name, fname) + else: + yield (item.name, item.path) + + +def parse_blinfo(addon_fname: str) -> dict: + """Parses a Python file, returning its bl_info dict. + + Returns None if the file doesn't contain a bl_info dict. + """ + + log.debug('Parsing %s', addon_fname) + + with open(addon_fname) as infile: + source = infile.read() + + try: + tree = ast.parse(source, addon_fname) + except SyntaxError as ex: + log.warning('Skipping addon: SyntaxError in %s: %s', addon_fname, ex) + return None + + for body in tree.body: + if body.__class__ != ast.Assign: + continue + + if len(body.targets) != 1: + continue + + if getattr(body.targets[0], 'id', '') != 'bl_info': + continue + + return ast.literal_eval(body.value) + + log.warning('Unable to find bl_info dict in %s', addon_fname) + return None + + +def blinfo_to_json(bl_info, addon_id, source, url) -> dict: + """Augments the bl_info dict with information for the package manager. + + Also checks for missing required/recommended keys. + + :returns: the augmented dict, or None if there were missing required keys. + """ + + missing_req_keys = [key for key in REQUIRED_KEYS + if key not in bl_info] + if missing_req_keys: + log.warning('Addon %s misses required key(s) %s; skipping this addon.', + addon_id, ', '.join(missing_req_keys)) + return None + + missing_rec_keys = [key for key in RECOMMENDED_KEYS + if key not in bl_info] + if missing_rec_keys: + log.info('Addon %s misses recommended key(s) %s', + addon_id, ', '.join(missing_rec_keys)) + + json_data = bl_info.copy() + json_data.update({ + 'download_url': url, + 'source': source, + }) + + return json_data + + +def parse_addons(addons_dir: str, addons_source: str, addons_base_url: str) -> dict: + """Parses info of all addons in the given directory.""" + + json_data = {} + + for (addon_id, addon_fname) in iter_addons(addons_dir): + bl_info = parse_blinfo(addon_fname) + if bl_info is None: + # The reason why has already been logged. + continue + + url = urllib.parse.urljoin(addons_base_url, addon_id) # TODO: construct the proper URL (zip/py/whl). + as_json = blinfo_to_json(bl_info, addon_id, addons_source, url) + if as_json is None: + # The reason why has already been logged. + continue + + json_data[addon_id] = as_json + + return json_data + + +def parse_existing_index(index_fname: str) -> dict: + """Parses an existing index JSON file, returning its 'addons' dict. + + Raises a ValueError if the schema version is unsupported. + """ + + log.info('Reading existing %s', index_fname) + + with open(index_fname, 'r', encoding='utf8') as infile: + existing_data = json.load(infile) + + # Check the schema version. + schema_version = existing_data.get('schema-version', '-missing-') + if schema_version != CURRENT_SCHEMA_VERSION: + log.fatal('Unable to load existing data, wrong schema version: %s', + schema_version) + raise ValueError('Unsupported schema %s' % schema_version) + + addon_data = existing_data['addons'] + return addon_data + + +def write_index_file(index_fname: str, addon_data: dict): + """Writes the index JSON file.""" + + log.info('Writing addon index to %s', index_fname) + with open(index_fname, 'w', encoding='utf8') as outfile: + json.dump(addon_data, outfile, indent=4, sort_keys=True) + + +def main(): + parser = argparse.ArgumentParser(description='Generate index.json from addons dir.') + + parser.add_argument('--merge', action='store_true', default=False, + help='merge with any existing index.json file') + parser.add_argument('--source', nargs='?', type=str, default='internal', + help='set the source of the addons') + parser.add_argument('--base', nargs='?', type=str, default='https://packages.blender.org/', + help='set the base download URL of the addons') + parser.add_argument('dir', metavar='DIR', type=str, + help='addons directory') + args = parser.parse_args() + + # Load the existing index.json if requested. + if args.merge: + addon_data = parse_existing_index('index.json') else: - json_text += "\t\t\t\t\t\"filename\": \"" + version + ".py\"\n" - json_text += "\t\t\t\t}\n" - json_text += "\t\t\t}\n" - json_text += "\t\t}\n" - - return json_text + addon_data = {} -addon = "" -json_text = "{\n" -json_text += "\t\"schema-version\": \"1\"\n" -json_text += "\t\"internal-url\": \"https://git.blender.org/gitweb/gitweb.cgi/blender-package-manager-addon.git/blob_plain/HEAD:/addons/\"\n" -json_text += "\t\"addons\": {\n" -json_text += list_addons(addons_directory, addon) -json_text += list_addons(contrib_directory, addon) -json_text += "\t}\n" -json_text += "}\n" + new_addon_data = parse_addons(args.dir, args.source, args.base) + addon_data.update(new_addon_data) -index_file = open(os.path.join(index_directory, 'index.json'), mode='w') -index_file.write(json_text) \ No newline at end of file + final_json = { + 'schema-version': CURRENT_SCHEMA_VERSION, + 'addons': addon_data, + } + + write_index_file('index.json', final_json) + log.info('Done!') + + +if __name__ == '__main__': + main()