Using 'ast' and 'json' modules to parse & write addon data.
This commit is contained in:
@@ -19,212 +19,187 @@
|
||||
# for use in the package manager add-on.
|
||||
|
||||
import os
|
||||
import shlex
|
||||
import ast
|
||||
import argparse
|
||||
import logging
|
||||
import urllib.parse
|
||||
import json
|
||||
|
||||
addons_directory = 'C:\\Users\\Peter\\Documents\\blender-git\\blender\\release\\scripts\\addons\\'
|
||||
contrib_directory = 'C:\\Users\\Peter\\Documents\\blender-git\\blender\\release\\scripts\\addons_contrib\\'
|
||||
index_directory = 'C:\\Users\\Peter\\Documents\\blender-package-manager-addon\\addons\\'
|
||||
logging.basicConfig(format='%(asctime)-15s %(levelname)8s %(name)s %(message)s',
|
||||
level=logging.INFO)
|
||||
log = logging.getLogger('generate-json')
|
||||
|
||||
# This script is functional for now, but rather "hacky" and much of it
|
||||
# will be replaced by a modified version of the code which Blender uses
|
||||
# to parse add-on's bl_info dictionary.
|
||||
REQUIRED_KEYS = ('name', 'blender')
|
||||
RECOMMENDED_KEYS = ('author', 'description', 'location', 'wiki_url', 'category')
|
||||
CURRENT_SCHEMA_VERSION = 1
|
||||
|
||||
def list_addons (dir, addon=""):
|
||||
json_text = ""
|
||||
for item in os.scandir(dir):
|
||||
|
||||
def iter_addons(addons_dir: str) -> (str, str):
|
||||
"""Generator, yields IDs and filenames of addons.
|
||||
|
||||
If the addon is a package, yields its __init__.py as filename.
|
||||
"""
|
||||
|
||||
for item in os.scandir(addons_dir):
|
||||
if item.name[0] == '.':
|
||||
continue
|
||||
if '.' not in item.name:
|
||||
if addon not in item.name and addon != "":
|
||||
continue
|
||||
if item.is_dir() and (item.name == addon or addon == ""):
|
||||
filename = os.path.join(item.path, '__init__.py')
|
||||
if item.is_file() and '.py' in item and (item.name[:-3] == addon or addon == ""):
|
||||
filename = item.path
|
||||
|
||||
try:
|
||||
file_lines = open(filename, mode='r').readlines()
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
new_lines = []
|
||||
for line in file_lines:
|
||||
if not line.startswith('#'):
|
||||
#line = line[:line.find('#')] FIXME did not take into
|
||||
# account '#' symbols inside quotes.
|
||||
if line.__len__() > 0:
|
||||
new_lines.append(line)
|
||||
file_text = ''.join(new_lines)
|
||||
|
||||
bl_info_at = file_text.find('bl_info')
|
||||
file_text = file_text[bl_info_at:]
|
||||
open_bracket_at = file_text.find('{')
|
||||
offset = 0
|
||||
quote_type = ""
|
||||
in_quote = False
|
||||
current_quote = "key"
|
||||
escape_next = False
|
||||
|
||||
bl_info = {}
|
||||
key = ""
|
||||
value = ""
|
||||
tuple_item = 0
|
||||
|
||||
for char in file_text[open_bracket_at+1:]:
|
||||
if char == '\\':
|
||||
escape_next = not escape_next
|
||||
|
||||
was_in_quote = in_quote
|
||||
if (not escape_next and (char == "'" or char == '"') and (not in_quote or (in_quote and char == quote_type))):
|
||||
in_quote = not in_quote
|
||||
quote_type = char
|
||||
|
||||
if in_quote and quote_type == ')' and (char == "'" or char == '"'):
|
||||
quote_type = char
|
||||
value = ""
|
||||
|
||||
if not in_quote and char == '(':
|
||||
in_quote = True
|
||||
value = ()
|
||||
quote_type = ')'
|
||||
elif in_quote and char == quote_type and quote_type == ')':
|
||||
in_quote = False
|
||||
elif in_quote and char != quote_type:
|
||||
if quote_type != ')':
|
||||
if current_quote == "key":
|
||||
key += char
|
||||
else:
|
||||
value += char
|
||||
else:
|
||||
if char == ',':
|
||||
value = value + (tuple_item, )
|
||||
tuple_item = 0
|
||||
elif char.isdigit():
|
||||
tuple_item = (tuple_item * 10) + int(char)
|
||||
|
||||
if not in_quote:
|
||||
if char == ',' or char == '}':
|
||||
current_quote = "key"
|
||||
if key != "":
|
||||
if value != "" or value != None:
|
||||
bl_info[key] = value
|
||||
else:
|
||||
bl_info[key] = ""
|
||||
key = ""
|
||||
value = ""
|
||||
tuple_item = 0
|
||||
elif was_in_quote:
|
||||
if current_quote == "value" and quote_type == ')':
|
||||
current_quote = "key"
|
||||
value = value + (tuple_item, )
|
||||
bl_info[key] = value
|
||||
key = ""
|
||||
value = ""
|
||||
tuple_item = 0
|
||||
else:
|
||||
current_quote = "value"
|
||||
|
||||
offset += 1
|
||||
|
||||
if not in_quote and char == '}':
|
||||
break
|
||||
|
||||
#print(file_text[:offset+open_bracket_at+1])
|
||||
#for k, v in bl_info.items():
|
||||
# print('"' + k + '": ' + repr(v))
|
||||
if item.is_dir():
|
||||
json_text += bl_info_to_json(bl_info, item.name)
|
||||
else:
|
||||
json_text += bl_info_to_json(bl_info, item.name, is_zip = False)
|
||||
|
||||
return json_text
|
||||
|
||||
def bl_info_to_json (bl_info, addon_id, source = 'internal', is_zip = True):
|
||||
required_keys = ['name', 'blender']
|
||||
recommended_keys = ['author', 'description', 'location', 'wiki_url', 'category']
|
||||
|
||||
for key in required_keys:
|
||||
if key not in bl_info:
|
||||
print("Error: missing key \"" + key + "\" in add-on " + addon_id
|
||||
+ "'s bl_info, or bl_info dict may be malformed")
|
||||
return ""
|
||||
|
||||
for key in recommended_keys:
|
||||
if key not in bl_info:
|
||||
print("Warning: missing key \"" + key + "\" in add-on " + addon_id
|
||||
+ "'s bl_info, or bl_info dict may be malformed")
|
||||
|
||||
author = ""
|
||||
if 'author' in bl_info:
|
||||
author = bl_info['author']
|
||||
|
||||
description = ""
|
||||
if 'description' in bl_info:
|
||||
description = bl_info['description']
|
||||
|
||||
tracker_url = ""
|
||||
if 'tracker_url' in bl_info:
|
||||
tracker_url = bl_info['tracker_url']
|
||||
|
||||
wiki_url = ""
|
||||
if 'wiki_url' in bl_info:
|
||||
wiki_url = bl_info['wiki_url']
|
||||
|
||||
location = ""
|
||||
if 'location' in bl_info:
|
||||
location = bl_info['location']
|
||||
|
||||
category = ""
|
||||
if 'category' in bl_info:
|
||||
category = bl_info['category']
|
||||
|
||||
warning = ""
|
||||
if 'warning' in bl_info:
|
||||
warning = bl_info['warning']
|
||||
|
||||
version = "unversioned"
|
||||
if 'version' in bl_info:
|
||||
version = '.'.join(map(str, bl_info['version']))
|
||||
|
||||
support = 'community'.lower()
|
||||
if 'support' in bl_info:
|
||||
support = bl_info['support'].lower()
|
||||
|
||||
json_text = ""
|
||||
|
||||
json_text += "\t\t\"" + addon_id + "\": {\n"
|
||||
json_text += "\t\t\t\"source\": \"" + source + "\"\n"
|
||||
json_text += "\t\t\t\"name\": \"" + bl_info['name'] + "\"\n"
|
||||
json_text += "\t\t\t\"description\": \"" + description + "\"\n"
|
||||
json_text += "\t\t\t\"author\": \"" + author + "\"\n"
|
||||
json_text += "\t\t\t\"wiki_url\": \"" + wiki_url + "\"\n"
|
||||
json_text += "\t\t\t\"tracker_url\": \"" + tracker_url + "\"\n"
|
||||
json_text += "\t\t\t\"location\": \"" + location + "\"\n"
|
||||
json_text += "\t\t\t\"category\": \"" + category + "\"\n"
|
||||
json_text += "\t\t\t\"version\": {\n"
|
||||
json_text += "\t\t\t\t\"" + version + "\": {\n"
|
||||
json_text += "\t\t\t\t\t\"blender\": \"" + '.'.join(map(str, bl_info['blender'])) + "\"\n"
|
||||
if warning != "":
|
||||
json_text += "\t\t\t\t\t\"warning\": \"" + warning + "\"\n"
|
||||
json_text += "\t\t\t\t\t\"support\": \"" + support + "\"\n"
|
||||
if is_zip:
|
||||
json_text += "\t\t\t\t\t\"filename\": \"" + version + ".zip\"\n"
|
||||
if item.is_dir():
|
||||
fname = os.path.join(item.path, '__init__.py')
|
||||
if not os.path.exists(fname):
|
||||
log.info('Skipping %s, it does not seem to be a Python package', item.path)
|
||||
continue
|
||||
|
||||
yield (item.name, fname)
|
||||
else:
|
||||
yield (item.name, item.path)
|
||||
|
||||
|
||||
def parse_blinfo(addon_fname: str) -> dict:
|
||||
"""Parses a Python file, returning its bl_info dict.
|
||||
|
||||
Returns None if the file doesn't contain a bl_info dict.
|
||||
"""
|
||||
|
||||
log.debug('Parsing %s', addon_fname)
|
||||
|
||||
with open(addon_fname) as infile:
|
||||
source = infile.read()
|
||||
|
||||
try:
|
||||
tree = ast.parse(source, addon_fname)
|
||||
except SyntaxError as ex:
|
||||
log.warning('Skipping addon: SyntaxError in %s: %s', addon_fname, ex)
|
||||
return None
|
||||
|
||||
for body in tree.body:
|
||||
if body.__class__ != ast.Assign:
|
||||
continue
|
||||
|
||||
if len(body.targets) != 1:
|
||||
continue
|
||||
|
||||
if getattr(body.targets[0], 'id', '') != 'bl_info':
|
||||
continue
|
||||
|
||||
return ast.literal_eval(body.value)
|
||||
|
||||
log.warning('Unable to find bl_info dict in %s', addon_fname)
|
||||
return None
|
||||
|
||||
|
||||
def blinfo_to_json(bl_info, addon_id, source, url) -> dict:
|
||||
"""Augments the bl_info dict with information for the package manager.
|
||||
|
||||
Also checks for missing required/recommended keys.
|
||||
|
||||
:returns: the augmented dict, or None if there were missing required keys.
|
||||
"""
|
||||
|
||||
missing_req_keys = [key for key in REQUIRED_KEYS
|
||||
if key not in bl_info]
|
||||
if missing_req_keys:
|
||||
log.warning('Addon %s misses required key(s) %s; skipping this addon.',
|
||||
addon_id, ', '.join(missing_req_keys))
|
||||
return None
|
||||
|
||||
missing_rec_keys = [key for key in RECOMMENDED_KEYS
|
||||
if key not in bl_info]
|
||||
if missing_rec_keys:
|
||||
log.info('Addon %s misses recommended key(s) %s',
|
||||
addon_id, ', '.join(missing_rec_keys))
|
||||
|
||||
json_data = bl_info.copy()
|
||||
json_data.update({
|
||||
'download_url': url,
|
||||
'source': source,
|
||||
})
|
||||
|
||||
return json_data
|
||||
|
||||
|
||||
def parse_addons(addons_dir: str, addons_source: str, addons_base_url: str) -> dict:
|
||||
"""Parses info of all addons in the given directory."""
|
||||
|
||||
json_data = {}
|
||||
|
||||
for (addon_id, addon_fname) in iter_addons(addons_dir):
|
||||
bl_info = parse_blinfo(addon_fname)
|
||||
if bl_info is None:
|
||||
# The reason why has already been logged.
|
||||
continue
|
||||
|
||||
url = urllib.parse.urljoin(addons_base_url, addon_id) # TODO: construct the proper URL (zip/py/whl).
|
||||
as_json = blinfo_to_json(bl_info, addon_id, addons_source, url)
|
||||
if as_json is None:
|
||||
# The reason why has already been logged.
|
||||
continue
|
||||
|
||||
json_data[addon_id] = as_json
|
||||
|
||||
return json_data
|
||||
|
||||
|
||||
def parse_existing_index(index_fname: str) -> dict:
|
||||
"""Parses an existing index JSON file, returning its 'addons' dict.
|
||||
|
||||
Raises a ValueError if the schema version is unsupported.
|
||||
"""
|
||||
|
||||
log.info('Reading existing %s', index_fname)
|
||||
|
||||
with open(index_fname, 'r', encoding='utf8') as infile:
|
||||
existing_data = json.load(infile)
|
||||
|
||||
# Check the schema version.
|
||||
schema_version = existing_data.get('schema-version', '-missing-')
|
||||
if schema_version != CURRENT_SCHEMA_VERSION:
|
||||
log.fatal('Unable to load existing data, wrong schema version: %s',
|
||||
schema_version)
|
||||
raise ValueError('Unsupported schema %s' % schema_version)
|
||||
|
||||
addon_data = existing_data['addons']
|
||||
return addon_data
|
||||
|
||||
|
||||
def write_index_file(index_fname: str, addon_data: dict):
|
||||
"""Writes the index JSON file."""
|
||||
|
||||
log.info('Writing addon index to %s', index_fname)
|
||||
with open(index_fname, 'w', encoding='utf8') as outfile:
|
||||
json.dump(addon_data, outfile, indent=4, sort_keys=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Generate index.json from addons dir.')
|
||||
|
||||
parser.add_argument('--merge', action='store_true', default=False,
|
||||
help='merge with any existing index.json file')
|
||||
parser.add_argument('--source', nargs='?', type=str, default='internal',
|
||||
help='set the source of the addons')
|
||||
parser.add_argument('--base', nargs='?', type=str, default='https://packages.blender.org/',
|
||||
help='set the base download URL of the addons')
|
||||
parser.add_argument('dir', metavar='DIR', type=str,
|
||||
help='addons directory')
|
||||
args = parser.parse_args()
|
||||
|
||||
# Load the existing index.json if requested.
|
||||
if args.merge:
|
||||
addon_data = parse_existing_index('index.json')
|
||||
else:
|
||||
json_text += "\t\t\t\t\t\"filename\": \"" + version + ".py\"\n"
|
||||
json_text += "\t\t\t\t}\n"
|
||||
json_text += "\t\t\t}\n"
|
||||
json_text += "\t\t}\n"
|
||||
|
||||
return json_text
|
||||
addon_data = {}
|
||||
|
||||
addon = ""
|
||||
json_text = "{\n"
|
||||
json_text += "\t\"schema-version\": \"1\"\n"
|
||||
json_text += "\t\"internal-url\": \"https://git.blender.org/gitweb/gitweb.cgi/blender-package-manager-addon.git/blob_plain/HEAD:/addons/\"\n"
|
||||
json_text += "\t\"addons\": {\n"
|
||||
json_text += list_addons(addons_directory, addon)
|
||||
json_text += list_addons(contrib_directory, addon)
|
||||
json_text += "\t}\n"
|
||||
json_text += "}\n"
|
||||
new_addon_data = parse_addons(args.dir, args.source, args.base)
|
||||
addon_data.update(new_addon_data)
|
||||
|
||||
index_file = open(os.path.join(index_directory, 'index.json'), mode='w')
|
||||
index_file.write(json_text)
|
||||
final_json = {
|
||||
'schema-version': CURRENT_SCHEMA_VERSION,
|
||||
'addons': addon_data,
|
||||
}
|
||||
|
||||
write_index_file('index.json', final_json)
|
||||
log.info('Done!')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user