Using 'ast' and 'json' modules to parse & write addon data.
This commit is contained in:
@@ -19,212 +19,187 @@
|
|||||||
# for use in the package manager add-on.
|
# for use in the package manager add-on.
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import shlex
|
import ast
|
||||||
|
import argparse
|
||||||
|
import logging
|
||||||
|
import urllib.parse
|
||||||
|
import json
|
||||||
|
|
||||||
addons_directory = 'C:\\Users\\Peter\\Documents\\blender-git\\blender\\release\\scripts\\addons\\'
|
logging.basicConfig(format='%(asctime)-15s %(levelname)8s %(name)s %(message)s',
|
||||||
contrib_directory = 'C:\\Users\\Peter\\Documents\\blender-git\\blender\\release\\scripts\\addons_contrib\\'
|
level=logging.INFO)
|
||||||
index_directory = 'C:\\Users\\Peter\\Documents\\blender-package-manager-addon\\addons\\'
|
log = logging.getLogger('generate-json')
|
||||||
|
|
||||||
# This script is functional for now, but rather "hacky" and much of it
|
REQUIRED_KEYS = ('name', 'blender')
|
||||||
# will be replaced by a modified version of the code which Blender uses
|
RECOMMENDED_KEYS = ('author', 'description', 'location', 'wiki_url', 'category')
|
||||||
# to parse add-on's bl_info dictionary.
|
CURRENT_SCHEMA_VERSION = 1
|
||||||
|
|
||||||
def list_addons (dir, addon=""):
|
|
||||||
json_text = ""
|
def iter_addons(addons_dir: str) -> (str, str):
|
||||||
for item in os.scandir(dir):
|
"""Generator, yields IDs and filenames of addons.
|
||||||
|
|
||||||
|
If the addon is a package, yields its __init__.py as filename.
|
||||||
|
"""
|
||||||
|
|
||||||
|
for item in os.scandir(addons_dir):
|
||||||
if item.name[0] == '.':
|
if item.name[0] == '.':
|
||||||
continue
|
continue
|
||||||
if '.' not in item.name:
|
|
||||||
if addon not in item.name and addon != "":
|
if item.is_dir():
|
||||||
|
fname = os.path.join(item.path, '__init__.py')
|
||||||
|
if not os.path.exists(fname):
|
||||||
|
log.info('Skipping %s, it does not seem to be a Python package', item.path)
|
||||||
continue
|
continue
|
||||||
if item.is_dir() and (item.name == addon or addon == ""):
|
|
||||||
filename = os.path.join(item.path, '__init__.py')
|
|
||||||
if item.is_file() and '.py' in item and (item.name[:-3] == addon or addon == ""):
|
|
||||||
filename = item.path
|
|
||||||
|
|
||||||
try:
|
yield (item.name, fname)
|
||||||
file_lines = open(filename, mode='r').readlines()
|
else:
|
||||||
except FileNotFoundError:
|
yield (item.name, item.path)
|
||||||
continue
|
|
||||||
new_lines = []
|
|
||||||
for line in file_lines:
|
|
||||||
if not line.startswith('#'):
|
|
||||||
#line = line[:line.find('#')] FIXME did not take into
|
|
||||||
# account '#' symbols inside quotes.
|
|
||||||
if line.__len__() > 0:
|
|
||||||
new_lines.append(line)
|
|
||||||
file_text = ''.join(new_lines)
|
|
||||||
|
|
||||||
bl_info_at = file_text.find('bl_info')
|
|
||||||
file_text = file_text[bl_info_at:]
|
|
||||||
open_bracket_at = file_text.find('{')
|
|
||||||
offset = 0
|
|
||||||
quote_type = ""
|
|
||||||
in_quote = False
|
|
||||||
current_quote = "key"
|
|
||||||
escape_next = False
|
|
||||||
|
|
||||||
bl_info = {}
|
def parse_blinfo(addon_fname: str) -> dict:
|
||||||
key = ""
|
"""Parses a Python file, returning its bl_info dict.
|
||||||
value = ""
|
|
||||||
tuple_item = 0
|
|
||||||
|
|
||||||
for char in file_text[open_bracket_at+1:]:
|
Returns None if the file doesn't contain a bl_info dict.
|
||||||
if char == '\\':
|
"""
|
||||||
escape_next = not escape_next
|
|
||||||
|
|
||||||
was_in_quote = in_quote
|
log.debug('Parsing %s', addon_fname)
|
||||||
if (not escape_next and (char == "'" or char == '"') and (not in_quote or (in_quote and char == quote_type))):
|
|
||||||
in_quote = not in_quote
|
|
||||||
quote_type = char
|
|
||||||
|
|
||||||
if in_quote and quote_type == ')' and (char == "'" or char == '"'):
|
with open(addon_fname) as infile:
|
||||||
quote_type = char
|
source = infile.read()
|
||||||
value = ""
|
|
||||||
|
|
||||||
if not in_quote and char == '(':
|
try:
|
||||||
in_quote = True
|
tree = ast.parse(source, addon_fname)
|
||||||
value = ()
|
except SyntaxError as ex:
|
||||||
quote_type = ')'
|
log.warning('Skipping addon: SyntaxError in %s: %s', addon_fname, ex)
|
||||||
elif in_quote and char == quote_type and quote_type == ')':
|
return None
|
||||||
in_quote = False
|
|
||||||
elif in_quote and char != quote_type:
|
|
||||||
if quote_type != ')':
|
|
||||||
if current_quote == "key":
|
|
||||||
key += char
|
|
||||||
else:
|
|
||||||
value += char
|
|
||||||
else:
|
|
||||||
if char == ',':
|
|
||||||
value = value + (tuple_item, )
|
|
||||||
tuple_item = 0
|
|
||||||
elif char.isdigit():
|
|
||||||
tuple_item = (tuple_item * 10) + int(char)
|
|
||||||
|
|
||||||
if not in_quote:
|
for body in tree.body:
|
||||||
if char == ',' or char == '}':
|
if body.__class__ != ast.Assign:
|
||||||
current_quote = "key"
|
continue
|
||||||
if key != "":
|
|
||||||
if value != "" or value != None:
|
|
||||||
bl_info[key] = value
|
|
||||||
else:
|
|
||||||
bl_info[key] = ""
|
|
||||||
key = ""
|
|
||||||
value = ""
|
|
||||||
tuple_item = 0
|
|
||||||
elif was_in_quote:
|
|
||||||
if current_quote == "value" and quote_type == ')':
|
|
||||||
current_quote = "key"
|
|
||||||
value = value + (tuple_item, )
|
|
||||||
bl_info[key] = value
|
|
||||||
key = ""
|
|
||||||
value = ""
|
|
||||||
tuple_item = 0
|
|
||||||
else:
|
|
||||||
current_quote = "value"
|
|
||||||
|
|
||||||
offset += 1
|
if len(body.targets) != 1:
|
||||||
|
continue
|
||||||
|
|
||||||
if not in_quote and char == '}':
|
if getattr(body.targets[0], 'id', '') != 'bl_info':
|
||||||
break
|
continue
|
||||||
|
|
||||||
#print(file_text[:offset+open_bracket_at+1])
|
return ast.literal_eval(body.value)
|
||||||
#for k, v in bl_info.items():
|
|
||||||
# print('"' + k + '": ' + repr(v))
|
|
||||||
if item.is_dir():
|
|
||||||
json_text += bl_info_to_json(bl_info, item.name)
|
|
||||||
else:
|
|
||||||
json_text += bl_info_to_json(bl_info, item.name, is_zip = False)
|
|
||||||
|
|
||||||
return json_text
|
log.warning('Unable to find bl_info dict in %s', addon_fname)
|
||||||
|
return None
|
||||||
|
|
||||||
def bl_info_to_json (bl_info, addon_id, source = 'internal', is_zip = True):
|
|
||||||
required_keys = ['name', 'blender']
|
|
||||||
recommended_keys = ['author', 'description', 'location', 'wiki_url', 'category']
|
|
||||||
|
|
||||||
for key in required_keys:
|
def blinfo_to_json(bl_info, addon_id, source, url) -> dict:
|
||||||
if key not in bl_info:
|
"""Augments the bl_info dict with information for the package manager.
|
||||||
print("Error: missing key \"" + key + "\" in add-on " + addon_id
|
|
||||||
+ "'s bl_info, or bl_info dict may be malformed")
|
|
||||||
return ""
|
|
||||||
|
|
||||||
for key in recommended_keys:
|
Also checks for missing required/recommended keys.
|
||||||
if key not in bl_info:
|
|
||||||
print("Warning: missing key \"" + key + "\" in add-on " + addon_id
|
|
||||||
+ "'s bl_info, or bl_info dict may be malformed")
|
|
||||||
|
|
||||||
author = ""
|
:returns: the augmented dict, or None if there were missing required keys.
|
||||||
if 'author' in bl_info:
|
"""
|
||||||
author = bl_info['author']
|
|
||||||
|
|
||||||
description = ""
|
missing_req_keys = [key for key in REQUIRED_KEYS
|
||||||
if 'description' in bl_info:
|
if key not in bl_info]
|
||||||
description = bl_info['description']
|
if missing_req_keys:
|
||||||
|
log.warning('Addon %s misses required key(s) %s; skipping this addon.',
|
||||||
|
addon_id, ', '.join(missing_req_keys))
|
||||||
|
return None
|
||||||
|
|
||||||
tracker_url = ""
|
missing_rec_keys = [key for key in RECOMMENDED_KEYS
|
||||||
if 'tracker_url' in bl_info:
|
if key not in bl_info]
|
||||||
tracker_url = bl_info['tracker_url']
|
if missing_rec_keys:
|
||||||
|
log.info('Addon %s misses recommended key(s) %s',
|
||||||
|
addon_id, ', '.join(missing_rec_keys))
|
||||||
|
|
||||||
wiki_url = ""
|
json_data = bl_info.copy()
|
||||||
if 'wiki_url' in bl_info:
|
json_data.update({
|
||||||
wiki_url = bl_info['wiki_url']
|
'download_url': url,
|
||||||
|
'source': source,
|
||||||
|
})
|
||||||
|
|
||||||
location = ""
|
return json_data
|
||||||
if 'location' in bl_info:
|
|
||||||
location = bl_info['location']
|
|
||||||
|
|
||||||
category = ""
|
|
||||||
if 'category' in bl_info:
|
|
||||||
category = bl_info['category']
|
|
||||||
|
|
||||||
warning = ""
|
def parse_addons(addons_dir: str, addons_source: str, addons_base_url: str) -> dict:
|
||||||
if 'warning' in bl_info:
|
"""Parses info of all addons in the given directory."""
|
||||||
warning = bl_info['warning']
|
|
||||||
|
|
||||||
version = "unversioned"
|
json_data = {}
|
||||||
if 'version' in bl_info:
|
|
||||||
version = '.'.join(map(str, bl_info['version']))
|
|
||||||
|
|
||||||
support = 'community'.lower()
|
for (addon_id, addon_fname) in iter_addons(addons_dir):
|
||||||
if 'support' in bl_info:
|
bl_info = parse_blinfo(addon_fname)
|
||||||
support = bl_info['support'].lower()
|
if bl_info is None:
|
||||||
|
# The reason why has already been logged.
|
||||||
|
continue
|
||||||
|
|
||||||
json_text = ""
|
url = urllib.parse.urljoin(addons_base_url, addon_id) # TODO: construct the proper URL (zip/py/whl).
|
||||||
|
as_json = blinfo_to_json(bl_info, addon_id, addons_source, url)
|
||||||
|
if as_json is None:
|
||||||
|
# The reason why has already been logged.
|
||||||
|
continue
|
||||||
|
|
||||||
json_text += "\t\t\"" + addon_id + "\": {\n"
|
json_data[addon_id] = as_json
|
||||||
json_text += "\t\t\t\"source\": \"" + source + "\"\n"
|
|
||||||
json_text += "\t\t\t\"name\": \"" + bl_info['name'] + "\"\n"
|
return json_data
|
||||||
json_text += "\t\t\t\"description\": \"" + description + "\"\n"
|
|
||||||
json_text += "\t\t\t\"author\": \"" + author + "\"\n"
|
|
||||||
json_text += "\t\t\t\"wiki_url\": \"" + wiki_url + "\"\n"
|
def parse_existing_index(index_fname: str) -> dict:
|
||||||
json_text += "\t\t\t\"tracker_url\": \"" + tracker_url + "\"\n"
|
"""Parses an existing index JSON file, returning its 'addons' dict.
|
||||||
json_text += "\t\t\t\"location\": \"" + location + "\"\n"
|
|
||||||
json_text += "\t\t\t\"category\": \"" + category + "\"\n"
|
Raises a ValueError if the schema version is unsupported.
|
||||||
json_text += "\t\t\t\"version\": {\n"
|
"""
|
||||||
json_text += "\t\t\t\t\"" + version + "\": {\n"
|
|
||||||
json_text += "\t\t\t\t\t\"blender\": \"" + '.'.join(map(str, bl_info['blender'])) + "\"\n"
|
log.info('Reading existing %s', index_fname)
|
||||||
if warning != "":
|
|
||||||
json_text += "\t\t\t\t\t\"warning\": \"" + warning + "\"\n"
|
with open(index_fname, 'r', encoding='utf8') as infile:
|
||||||
json_text += "\t\t\t\t\t\"support\": \"" + support + "\"\n"
|
existing_data = json.load(infile)
|
||||||
if is_zip:
|
|
||||||
json_text += "\t\t\t\t\t\"filename\": \"" + version + ".zip\"\n"
|
# Check the schema version.
|
||||||
|
schema_version = existing_data.get('schema-version', '-missing-')
|
||||||
|
if schema_version != CURRENT_SCHEMA_VERSION:
|
||||||
|
log.fatal('Unable to load existing data, wrong schema version: %s',
|
||||||
|
schema_version)
|
||||||
|
raise ValueError('Unsupported schema %s' % schema_version)
|
||||||
|
|
||||||
|
addon_data = existing_data['addons']
|
||||||
|
return addon_data
|
||||||
|
|
||||||
|
|
||||||
|
def write_index_file(index_fname: str, addon_data: dict):
|
||||||
|
"""Writes the index JSON file."""
|
||||||
|
|
||||||
|
log.info('Writing addon index to %s', index_fname)
|
||||||
|
with open(index_fname, 'w', encoding='utf8') as outfile:
|
||||||
|
json.dump(addon_data, outfile, indent=4, sort_keys=True)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description='Generate index.json from addons dir.')
|
||||||
|
|
||||||
|
parser.add_argument('--merge', action='store_true', default=False,
|
||||||
|
help='merge with any existing index.json file')
|
||||||
|
parser.add_argument('--source', nargs='?', type=str, default='internal',
|
||||||
|
help='set the source of the addons')
|
||||||
|
parser.add_argument('--base', nargs='?', type=str, default='https://packages.blender.org/',
|
||||||
|
help='set the base download URL of the addons')
|
||||||
|
parser.add_argument('dir', metavar='DIR', type=str,
|
||||||
|
help='addons directory')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
# Load the existing index.json if requested.
|
||||||
|
if args.merge:
|
||||||
|
addon_data = parse_existing_index('index.json')
|
||||||
else:
|
else:
|
||||||
json_text += "\t\t\t\t\t\"filename\": \"" + version + ".py\"\n"
|
addon_data = {}
|
||||||
json_text += "\t\t\t\t}\n"
|
|
||||||
json_text += "\t\t\t}\n"
|
|
||||||
json_text += "\t\t}\n"
|
|
||||||
|
|
||||||
return json_text
|
new_addon_data = parse_addons(args.dir, args.source, args.base)
|
||||||
|
addon_data.update(new_addon_data)
|
||||||
|
|
||||||
addon = ""
|
final_json = {
|
||||||
json_text = "{\n"
|
'schema-version': CURRENT_SCHEMA_VERSION,
|
||||||
json_text += "\t\"schema-version\": \"1\"\n"
|
'addons': addon_data,
|
||||||
json_text += "\t\"internal-url\": \"https://git.blender.org/gitweb/gitweb.cgi/blender-package-manager-addon.git/blob_plain/HEAD:/addons/\"\n"
|
}
|
||||||
json_text += "\t\"addons\": {\n"
|
|
||||||
json_text += list_addons(addons_directory, addon)
|
|
||||||
json_text += list_addons(contrib_directory, addon)
|
|
||||||
json_text += "\t}\n"
|
|
||||||
json_text += "}\n"
|
|
||||||
|
|
||||||
index_file = open(os.path.join(index_directory, 'index.json'), mode='w')
|
write_index_file('index.json', final_json)
|
||||||
index_file.write(json_text)
|
log.info('Done!')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
|
|||||||
Reference in New Issue
Block a user