WIP: MaterialX addon #104594

Closed
Bogdan Nagirniak wants to merge 34 commits from BogdanNagirniak/blender-addons:materialx-addon into main

When changing the target branch, be careful to rebase the branch in your fork to match. See documentation.
7 changed files with 816 additions and 1 deletions
Showing only changes of commit a83e948bb8 - Show all commits

View File

@ -26,6 +26,7 @@ import bpy
from . import preferences
from . import node_tree
from . import nodes
from . import matlib
from . import logging
log = logging.Log("")
@ -36,10 +37,12 @@ def register():
bpy.utils.register_class(preferences.AddonPreferences)
bpy.utils.register_class(node_tree.MxNodeTree)
nodes.register()
matlib.register()
def unregister():
log("unregister")
matlib.unregister()
nodes.unregister()
bpy.utils.unregister_class(node_tree.MxNodeTree)
bpy.utils.unregister_class(preferences.AddonPreferences)

View File

@ -0,0 +1,34 @@
# SPDX-License-Identifier: GPL-2.0-or-later
# Copyright 2022, AMD
import bpy
from . import ui, properties
register_properties, unregister_properties = bpy.utils.register_classes_factory(
[
properties.MatlibProperties,
properties.WindowManagerProperties,
]
)
register_ui, unregister_ui = bpy.utils.register_classes_factory(
[
ui.MATLIB_PT_matlib,
ui.MATLIB_PT_matlib_tools,
ui.MATLIB_OP_load_materials,
ui.MATLIB_OP_load_package,
ui.MATLIB_OP_import_material,
ui.MATERIAL_OP_matlib_clear_search,
]
)
def register():
register_properties()
register_ui()
def unregister():
unregister_ui()
unregister_properties()

435
materialx/matlib/manager.py Normal file
View File

@ -0,0 +1,435 @@
# SPDX-License-Identifier: GPL-2.0-or-later
# Copyright 2022, AMD
import requests
import weakref
from dataclasses import dataclass, field
import shutil
from pathlib import Path
import zipfile
import json
import threading
from concurrent import futures
import bpy.utils.previews
from ..utils import logging, update_ui, MATLIB_DIR
log = logging.Log('matlib.manager')
URL = "https://api.matlib.gpuopen.com/api"
def download_file(url, path, cache_check=True):
if cache_check and path.is_file():
return path
log("download_file", f"{url=}, {path=}")
path.parent.mkdir(parents=True, exist_ok=True)
with requests.get(url, stream=True) as response:
with open(path, 'wb') as f:
shutil.copyfileobj(response.raw, f)
log("download_file", "done")
return path
def download_file_callback(url, path, update_callback, cache_check=True):
if cache_check and path.is_file():
return None
log("download_file_callback", f"{url=}, {path=}")
path.parent.mkdir(parents=True, exist_ok=True)
path_raw = path.with_suffix(".raw")
size = 0
with requests.get(url, stream=True) as response:
with open(path_raw, 'wb') as f:
if update_callback:
for chunk in response.iter_content(chunk_size=8192):
size += len(chunk)
update_callback(size)
f.write(chunk)
path_raw.rename(path)
log("download_file_callback", "done")
return path
def request_json(url, params, path, cache_check=True):
if cache_check and path and path.is_file():
with open(path) as json_file:
return json.load(json_file)
log("request_json", f"{url=}, {params=}, {path=}")
response = requests.get(url, params=params)
res_json = response.json()
if path:
save_json(res_json, path)
log("request_json", "done")
return res_json
def save_json(json_obj, path):
path.parent.mkdir(parents=True, exist_ok=True)
with open(path, 'w') as outfile:
json.dump(json_obj, outfile)
@dataclass(init=False)
class Render:
id: str
author: str = field(init=False, default=None)
image: str = field(init=False, default=None)
image_url: str = field(init=False, default=None)
image_path: Path = field(init=False, default=None)
thumbnail: str = field(init=False, default=None)
thumbnail_url: str = field(init=False, default=None)
thumbnail_path: Path = field(init=False, default=None)
thumbnail_icon_id: int = field(init=False, default=None)
def __init__(self, id, material):
self.id = id
self.material = weakref.ref(material)
@property
def cache_dir(self):
return self.material().cache_dir
def get_info(self, cache_chek=True):
json_data = request_json(f"{URL}/renders/{self.id}", None,
self.cache_dir / f"R-{self.id[:8]}.json", cache_chek)
self.author = json_data['author']
self.image = json_data['image']
self.image_url = json_data['image_url']
self.thumbnail = json_data['thumbnail']
self.thumbnail_url = json_data['thumbnail_url']
def get_image(self, cache_check=True):
self.image_path = download_file(self.image_url,
self.cache_dir / self.image, cache_check)
def get_thumbnail(self, cache_check=True):
self.thumbnail_path = download_file(self.thumbnail_url,
self.cache_dir / self.thumbnail, cache_check)
def thumbnail_load(self, pcoll):
thumb = pcoll.get(self.thumbnail)
if not thumb:
thumb = pcoll.load(self.thumbnail, str(self.thumbnail_path), 'IMAGE')
self.thumbnail_icon_id = thumb.icon_id
@dataclass(init=False)
class Package:
id: str
author: str = field(init=False, default=None)
label: str = field(init=False, default=None)
file: str = field(init=False, default=None)
file_url: str = field(init=False, default=None)
size_str: str = field(init=False, default=None)
def __init__(self, id, material):
self.id = id
self.material = weakref.ref(material)
self.size_load = None
@property
def cache_dir(self):
return self.material().cache_dir / f"P-{self.id[:8]}"
@property
def file_path(self):
return self.cache_dir / self.file
@property
def has_file(self):
return self.file_path.is_file()
def get_info(self, cache_check=True):
json_data = request_json(f"{URL}/packages/{self.id}", None,
self.cache_dir / "info.json", cache_check)
self.author = json_data['author']
self.file = json_data['file']
self.file_url = json_data['file_url']
self.label = json_data['label']
self.size_str = json_data['size']
def download(self, cache_check=True):
def callback(size):
self.size_load = size
update_ui()
download_file_callback(self.file_url, self.file_path, callback, cache_check)
def unzip(self, path=None, cache_check=True):
if not path:
path = self.cache_dir / "package"
if path.is_dir() and not cache_check:
shutil.rmtree(path, ignore_errors=True)
if not path.is_dir():
with zipfile.ZipFile(self.file_path) as z:
z.extractall(path=path)
mtlx_file = next(path.glob("**/*.mtlx"))
return mtlx_file
@property
def size(self):
n, b = self.size_str.split(" ")
size = float(n)
if b == "MB":
size *= 1048576 # 2 ** 20
elif b == "KB":
size *= 1024 # 2 ** 10
elif b == "GB":
size *= 2 ** 30
return int(size)
def __lt__(self, other):
return self.size < other.size
@dataclass
class Category:
id: str
title: str = field(init=False, default=None)
@property
def cache_dir(self):
return MATLIB_DIR
def get_info(self, use_cache=True):
if not self.id:
return
json_data = request_json(f"{URL}/categories/{self.id}", None,
self.cache_dir / f"C-{self.id[:8]}.json", use_cache)
self.title = json_data['title']
def __lt__(self, other):
return self.title < other.title
@dataclass(init=False)
class Material:
id: str
author: str
title: str
description: str
category: Category
status: str
renders: list[Render]
packages: list[Package]
def __init__(self, mat_json):
self.id = mat_json['id']
self.author = mat_json['author']
self.title = mat_json['title']
self.description = mat_json['description']
self.category = Category(mat_json['category'])
self.status = mat_json['status']
self.renders = []
for id in mat_json['renders_order']:
self.renders.append(Render(id, self))
self.packages = []
for id in mat_json['packages']:
self.packages.append(Package(id, self))
save_json(mat_json, self.cache_dir / "info.json")
def __lt__(self, other):
return self.title.lower() < other.title.lower()
@property
def cache_dir(self):
return MATLIB_DIR / f"M-{self.id[:8]}"
@classmethod
def get_materials(cls):
offset = 0
limit = 500
while True:
res_json = request_json(f"{URL}/materials", {'limit': limit, 'offset': offset}, None)
count = res_json['count']
for mat_json in res_json['results']:
mat = Material(mat_json)
if not mat.packages or not mat.category.id:
continue
yield mat
offset += limit
if offset >= count:
break
@classmethod
def get_materials_cache(cls):
for f in MATLIB_DIR.glob("M-*/info.json"):
with open(f) as json_file:
mat_json = json.load(json_file)
yield Material(mat_json)
class Manager:
def __init__(self):
self.materials = None
self.categories = None
self.pcoll = None
self.load_thread = None
self.package_executor = None
self.status = ""
self.is_synced = None
def __del__(self):
# bpy.utils.previews.remove(self.pcoll)
pass
def set_status(self, msg):
self.status = msg
update_ui()
@property
def materials_list(self):
# required for thread safe purposes
return list(manager.materials.values())
@property
def categories_list(self):
# required for thread safe purposes
return list(manager.categories.values())
def check_load_materials(self, reset=False):
# required is not None condition to prevent further update if no material is found at first time.
if self.materials is not None and not reset:
return True
if reset and self.pcoll:
bpy.utils.previews.remove(self.pcoll)
self.materials = {}
self.categories = {}
self.pcoll = bpy.utils.previews.new()
def category_load(cat):
cat.get_info()
self.categories[cat.id] = cat
def material_load(mat, is_cached):
for render in mat.renders:
render.get_info()
render.get_thumbnail()
render.thumbnail_load(self.pcoll)
for package in mat.packages:
package.get_info()
self.materials[mat.id] = mat
self.set_status(f"Syncing {len(self.materials)} {'cached' if is_cached else 'online'} materials...")
def load():
self.is_synced = False
self.set_status("Start syncing...")
with futures.ThreadPoolExecutor() as executor:
try:
#
# getting cached materials
#
materials = {mat.id: mat for mat in Material.get_materials_cache()}
categories = {mat.category.id: mat.category for mat in materials.values()}
# loading categories
category_loaders = [executor.submit(category_load, cat)
for cat in categories.values()]
for future in futures.as_completed(category_loaders):
future.result()
# updating category for cached materials
for mat in materials.values():
mat.category.get_info()
# loading cached materials
material_loaders = [executor.submit(material_load, mat, True)
for mat in materials.values()]
for future in futures.as_completed(material_loaders):
future.result()
#
# getting and syncing with online materials
#
online_materials = {mat.id: mat for mat in Material.get_materials()}
# loading new categories
new_categories = {}
for mat in online_materials.values():
cat = mat.category
if cat.id not in categories and cat.id not in new_categories:
new_categories[cat.id] = cat
category_loaders = [executor.submit(category_load, cat)
for cat in new_categories.values()]
for future in futures.as_completed(category_loaders):
future.result()
# updating categories for online materials
for mat in online_materials.values():
mat.category.get_info()
# loading online materials
material_loaders = [executor.submit(material_load, mat, False)
for mat in online_materials.values()]
for future in futures.as_completed(material_loaders):
future.result()
self.set_status(f"Syncing {len(self.materials)} materials completed")
except requests.exceptions.RequestException as err:
executor.shutdown(wait=True, cancel_futures=True)
self.set_status(f"Connection error. Synced {len(self.materials)} materials")
log.error(err)
finally:
self.is_synced = True
self.load_thread = threading.Thread(target=load, daemon=True)
self.load_thread.start()
return False
def load_package(self, package):
package.size_load = 0
def package_load():
try:
package.download()
except requests.exceptions.RequestException as err:
log.error(err)
package.size_load = None
update_ui()
if not self.package_executor:
self.package_executor = futures.ThreadPoolExecutor()
self.package_executor.submit(package_load)
manager = Manager()

View File

@ -0,0 +1,130 @@
# SPDX-License-Identifier: GPL-2.0-or-later
# Copyright 2022, AMD
import bpy
from ..matlib.manager import manager
from ..utils import MaterialXProperties
class MatlibProperties(bpy.types.PropertyGroup):
def get_materials(self) -> dict:
materials = {}
search_str = self.search.strip().lower()
materials_list = manager.materials_list
for mat in materials_list:
if search_str not in mat.title.lower():
continue
if not (mat.category.id == self.category_id or self.category_id == 'ALL'):
continue
materials[mat.id] = mat
return materials
def get_materials_prop(self, context):
materials = []
for i, mat in enumerate(sorted(self.get_materials().values())):
description = mat.title
if mat.description:
description += f"\n{mat.description}"
description += f"\nCategory: {mat.category.title}\nAuthor: {mat.author}"
icon_id = mat.renders[0].thumbnail_icon_id if mat.renders else 'MATERIAL'
materials.append((mat.id, mat.title, description, icon_id, i))
return materials
def get_categories_prop(self, context):
categories = []
if manager.categories is None:
return categories
categories += [('ALL', "All Categories", "Show materials for all categories")]
categories_list = manager.categories_list
categories += ((cat.id, cat.title, f"Show materials with category {cat.title}")
for cat in sorted(categories_list))
return categories
def get_packages_prop(self, context):
packages = []
mat = self.material
if not mat:
return packages
for i, p in enumerate(sorted(mat.packages)):
description = f"Package: {p.label} ({p.size_str})\nAuthor: {p.author}"
if p.has_file:
description += "\nReady to import"
icon_id = 'RADIOBUT_ON' if p.has_file else 'RADIOBUT_OFF'
packages.append((p.id, f"{p.label} ({p.size_str})", description, icon_id, i))
return packages
def update_material(self, context):
mat = self.material
if mat:
self.package_id = min(mat.packages).id
def update_category(self, context):
materials = self.get_materials()
if not materials:
return
mat = min(materials.values())
self.material_id = mat.id
self.package_id = min(mat.packages).id
def update_search(self, context):
materials = self.get_materials()
if not materials or self.material_id in materials:
return
mat = min(materials.values())
self.material_id = mat.id
self.package_id = min(mat.packages).id
material_id: bpy.props.EnumProperty(
name="Material",
description="Select material",
items=get_materials_prop,
update=update_material,
)
category_id: bpy.props.EnumProperty(
name="Category",
description="Select materials category",
items=get_categories_prop,
update=update_category,
)
search: bpy.props.StringProperty(
name="Search",
description="Search materials by title",
update=update_search,
)
package_id: bpy.props.EnumProperty(
name="Package",
description="Selected material package",
items=get_packages_prop,
)
@property
def material(self):
return manager.materials.get(self.material_id)
@property
def package(self):
mat = self.material
if not mat:
return None
return next((p for p in mat.packages if p.id == self.package_id), None)
class WindowManagerProperties(MaterialXProperties):
bl_type = bpy.types.WindowManager
matlib: bpy.props.PointerProperty(type=MatlibProperties)

187
materialx/matlib/ui.py Normal file
View File

@ -0,0 +1,187 @@
# SPDX-License-Identifier: GPL-2.0-or-later
# Copyright 2022, AMD
import traceback
import textwrap
import MaterialX as mx
import bpy
from .. import utils
from ..node_tree import MxNodeTree
from ..utils import mx as mx_utils
from .manager import manager
#from .. import config
from ..utils import logging
log = logging.Log('matlib.ui')
class MATERIAL_OP_matlib_clear_search(bpy.types.Operator):
"""Create new MaterialX node tree for selected material"""
bl_idname = utils.with_prefix("matlib_clear_search")
bl_label = ""
def execute(self, context):
context.window_manager.materialx.matlib.search = ''
return {"FINISHED"}
class MATLIB_OP_load_materials(bpy.types.Operator):
"""Load materials"""
bl_idname = utils.with_prefix("matlib_load")
bl_label = "Reload Library"
def execute(self, context):
manager.check_load_materials(reset=True)
return {"FINISHED"}
class MATLIB_OP_import_material(bpy.types.Operator):
"""Import Material Package to material"""
bl_idname = utils.with_prefix("matlib_import_material")
bl_label = "Import Material Package"
def execute(self, context):
matlib_prop = context.window_manager.materialx.matlib
package = matlib_prop.package
mtlx_file = package.unzip()
# getting/creating MxNodeTree
bl_material = context.material
mx_node_tree = bl_material.materialx.mx_node_tree
if not bl_material.materialx.mx_node_tree:
mx_node_tree = bpy.data.node_groups.new(f"MX_{bl_material.name}",
type=MxNodeTree.bl_idname)
bl_material.materialx.mx_node_tree = mx_node_tree
log(f"Reading: {mtlx_file}")
doc = mx.createDocument()
search_path = mx.FileSearchPath(str(mtlx_file.parent))
search_path.append(str(mx_utils.MX_LIBS_DIR))
try:
mx.readFromXmlFile(doc, str(mtlx_file), searchPath=search_path)
mx_node_tree.import_(doc, mtlx_file)
except Exception as e:
log.error(traceback.format_exc(), mtlx_file)
return {'CANCELLED'}
return {"FINISHED"}
class MATLIB_OP_load_package(bpy.types.Operator):
"""Download material package"""
bl_idname = utils.with_prefix("matlib_load_package")
bl_label = "Download Package"
def execute(self, context):
matlib_prop = context.window_manager.materialx.matlib
manager.load_package(matlib_prop.package)
return {"FINISHED"}
class MATLIB_PT_matlib(bpy.types.Panel):
bl_idname = utils.with_prefix("MATLIB_PT_matlib", '_', True)
bl_label = "Material Library"
bl_context = "material"
bl_region_type = 'WINDOW'
bl_space_type = 'PROPERTIES'
bl_options = {'DEFAULT_CLOSED'}
def draw(self, context):
layout = self.layout
matlib_prop = context.window_manager.materialx.matlib
manager.check_load_materials()
# category
layout.prop(matlib_prop, 'category_id')
# search
row = layout.row(align=True)
row.prop(matlib_prop, 'search', text="", icon='VIEWZOOM')
if matlib_prop.search:
row.operator(MATERIAL_OP_matlib_clear_search.bl_idname, icon='X')
# materials
col = layout.column(align=True)
materials = matlib_prop.get_materials()
if not materials:
col.label(text="Start syncing..." if not manager.materials else "No materials found")
return
row = col.row()
row.alignment = 'RIGHT'
row.label(text=f"{len(materials)} materials")
col.template_icon_view(matlib_prop, 'material_id', show_labels=True)
mat = matlib_prop.material
if not mat:
return
# other material renders
if len(mat.renders) > 1:
grid = col.grid_flow(align=True)
for i, render in enumerate(mat.renders):
if i % 6 == 0:
row = grid.row()
row.alignment = 'CENTER'
row.template_icon(render.thumbnail_icon_id, scale=5)
# material title
row = col.row()
row.alignment = 'CENTER'
row.label(text=mat.title)
# material description
col = layout.column(align=True)
if mat.description:
for line in textwrap.wrap(mat.description, 60):
col.label(text=line)
col = layout.column(align=True)
col.label(text=f"Category: {mat.category.title}")
col.label(text=f"Author: {mat.author}")
# packages
package = matlib_prop.package
if not package:
return
layout.prop(matlib_prop, 'package_id', icon='DOCUMENTS')
row = layout.row()
if package.has_file:
row.operator(MATLIB_OP_import_material.bl_idname, icon='IMPORT')
else:
if package.size_load is None:
row.operator(MATLIB_OP_load_package.bl_idname, icon='IMPORT')
else:
percent = min(100, int(package.size_load * 100 / package.size))
row.operator(MATLIB_OP_load_package.bl_idname, icon='IMPORT',
text=f"Downloading Package...{percent}%")
row.enabled = False
class MATLIB_PT_matlib_tools(bpy.types.Panel):
bl_label = "Tools"
bl_context = "material"
bl_region_type = 'WINDOW'
bl_space_type = 'PROPERTIES'
bl_parent_id = utils.with_prefix('MATLIB_PT_matlib', '_', True)
bl_options = {'DEFAULT_CLOSED'}
def draw(self, context):
layout = self.layout
col = layout.column()
col.label(text=manager.status)
row = col.row()
row.enabled = bool(manager.is_synced)
row.operator(MATLIB_OP_load_materials.bl_idname, icon='FILE_REFRESH')

View File

@ -1 +0,0 @@
gen_*.py

View File

@ -24,6 +24,8 @@ MX_LIBS_DIR = ADDON_ROOT_DIR / MX_LIBS_FOLDER
NODE_CLASSES_FOLDER = "materialx_nodes"
NODE_CLASSES_DIR = ADDON_DATA_DIR / NODE_CLASSES_FOLDER
MATLIB_FOLDER = "matlib"
MATLIB_DIR = ADDON_DATA_DIR / MATLIB_FOLDER
os.environ['MATERIALX_SEARCH_PATH'] = str(MX_LIBS_DIR)
@ -378,3 +380,28 @@ def pass_node_reroute(link):
link = link.from_node.inputs[0].links[0]
return link if link.is_valid else None
def update_ui(area_type='PROPERTIES', region_type='WINDOW'):
for window in bpy.context.window_manager.windows:
for area in window.screen.areas:
if area.type == area_type:
for region in area.regions:
if region.type == region_type:
region.tag_redraw()
class MaterialXProperties(bpy.types.PropertyGroup):
bl_type = None
@classmethod
def register(cls):
setattr(cls.bl_type, "materialx", bpy.props.PointerProperty(
name="MaterialX properties",
description="MaterialX properties",
type=cls,
))
@classmethod
def unregister(cls):
delattr(cls.bl_type, "materialx")