From 7966137b3e9575a7c2a63b30a2206addf9579dc4 Mon Sep 17 00:00:00 2001 From: Omar Emara Date: Tue, 14 Nov 2023 16:58:10 +0200 Subject: [PATCH 1/4] Realtime Compositor: Rewrite inpaint node This patch rewrites the Inpaint node in the Realtime Compositor. The old method suffered from discontinuities and singularities in the inpainting regions. Furthermore, it ignored semi-transparent areas. The new method is inspired by a two pass method described by the paper: Rosner, Jakub, et al. "Fast GPU-based image warping and inpainting for frame interpolation." International Conferences on Computer Graphics, Vision and Mathematics. 2010. In particular, we first fill the inpainting region using jump flooding, then we apply a variable size blur pass whose size is proportional to the distance to the inpainting boundary. The smoothed region is then mixed with the input using its alpha. The aforementioned method requires variable size blur, which is quite expensive for this use case, so a new implementation was added that approximates the method using a separable implementation, which provides a visually pleasing result assuming a sufficiently smooth radius field, which is true for our case since the field is an SDF. Fixes: #114422 --- .../realtime_compositor/CMakeLists.txt | 5 + ..._symmetric_separable_blur_variable_size.hh | 33 +++++ .../symmetric_separable_blur_variable_size.cc | 139 ++++++++++++++++++ .../compositor_inpaint_compute_region.glsl | 55 ++----- .../compositor_inpaint_fill_region.glsl | 48 ++++++ ...ymmetric_separable_blur_variable_size.glsl | 40 +++++ .../shaders/infos/compositor_inpaint_info.hh | 15 +- ...etric_separable_blur_variable_size_info.hh | 28 ++++ .../composite/nodes/node_composite_inpaint.cc | 90 +++++++++--- 9 files changed, 386 insertions(+), 67 deletions(-) create mode 100644 source/blender/compositor/realtime_compositor/algorithms/COM_algorithm_symmetric_separable_blur_variable_size.hh create mode 100644 source/blender/compositor/realtime_compositor/algorithms/intern/symmetric_separable_blur_variable_size.cc create mode 100644 source/blender/compositor/realtime_compositor/shaders/compositor_inpaint_fill_region.glsl create mode 100644 source/blender/compositor/realtime_compositor/shaders/compositor_symmetric_separable_blur_variable_size.glsl create mode 100644 source/blender/compositor/realtime_compositor/shaders/infos/compositor_symmetric_separable_blur_variable_size_info.hh diff --git a/source/blender/compositor/realtime_compositor/CMakeLists.txt b/source/blender/compositor/realtime_compositor/CMakeLists.txt index a16a85bd533..904b1891046 100644 --- a/source/blender/compositor/realtime_compositor/CMakeLists.txt +++ b/source/blender/compositor/realtime_compositor/CMakeLists.txt @@ -69,6 +69,7 @@ set(SRC algorithms/intern/smaa.cc algorithms/intern/summed_area_table.cc algorithms/intern/symmetric_separable_blur.cc + algorithms/intern/symmetric_separable_blur_variable_size.cc algorithms/intern/transform.cc algorithms/COM_algorithm_jump_flooding.hh @@ -79,6 +80,7 @@ set(SRC algorithms/COM_algorithm_smaa.hh algorithms/COM_algorithm_summed_area_table.hh algorithms/COM_algorithm_symmetric_separable_blur.hh + algorithms/COM_algorithm_symmetric_separable_blur_variable_size.hh algorithms/COM_algorithm_transform.hh cached_resources/intern/cached_mask.cc @@ -150,6 +152,7 @@ set(GLSL_SRC shaders/compositor_image_crop.glsl shaders/compositor_inpaint_compute_boundary.glsl shaders/compositor_inpaint_compute_region.glsl + shaders/compositor_inpaint_fill_region.glsl shaders/compositor_jump_flooding.glsl shaders/compositor_keying_compute_image.glsl shaders/compositor_keying_compute_matte.glsl @@ -186,6 +189,7 @@ set(GLSL_SRC shaders/compositor_symmetric_blur.glsl shaders/compositor_symmetric_blur_variable_size.glsl shaders/compositor_symmetric_separable_blur.glsl + shaders/compositor_symmetric_separable_blur_variable_size.glsl shaders/compositor_tone_map_photoreceptor.glsl shaders/compositor_tone_map_simple.glsl shaders/compositor_write_output.glsl @@ -297,6 +301,7 @@ set(SRC_SHADER_CREATE_INFOS shaders/infos/compositor_symmetric_blur_info.hh shaders/infos/compositor_symmetric_blur_variable_size_info.hh shaders/infos/compositor_symmetric_separable_blur_info.hh + shaders/infos/compositor_symmetric_separable_blur_variable_size_info.hh shaders/infos/compositor_tone_map_photoreceptor_info.hh shaders/infos/compositor_tone_map_simple_info.hh shaders/infos/compositor_write_output_info.hh diff --git a/source/blender/compositor/realtime_compositor/algorithms/COM_algorithm_symmetric_separable_blur_variable_size.hh b/source/blender/compositor/realtime_compositor/algorithms/COM_algorithm_symmetric_separable_blur_variable_size.hh new file mode 100644 index 00000000000..bb3e0b59d55 --- /dev/null +++ b/source/blender/compositor/realtime_compositor/algorithms/COM_algorithm_symmetric_separable_blur_variable_size.hh @@ -0,0 +1,33 @@ +/* SPDX-FileCopyrightText: 2023 Blender Authors + * + * SPDX-License-Identifier: GPL-2.0-or-later */ + +#pragma once + +#include "DNA_scene_types.h" + +#include "COM_context.hh" +#include "COM_result.hh" + +namespace blender::realtime_compositor { + +/* Blur the input using a horizontal and a vertical separable blur passes given the filter type + * using SymmetricSeparableBlurWeights, where the number of weights is equal to weights_resolution. + * Since the radius can be variable, the number of weights can be less than or more than the number + * of pixels actually getting accumulated during blurring, so the weights are interpolated in the + * shader as needed, the resolution is typically set to the maximum possible radius if known. The + * radius of the blur can be variable and is defined using the given radius float image. The output + * is written to the given output result, which will be allocated internally and is thus expected + * not to be previously allocated. + * + * Technically, variable size blur can't be computed separably, however, assuming a sufficiently + * smooth radius field, the results can be visually pleasing, so this can be used a more performant + * variable size blur if the quality is satisfactory. */ +void symmetric_separable_blur_variable_size(Context &context, + Result &input, + Result &output, + Result &radius, + int filter_type = R_FILTER_GAUSS, + int weights_resolution = 128); + +} // namespace blender::realtime_compositor diff --git a/source/blender/compositor/realtime_compositor/algorithms/intern/symmetric_separable_blur_variable_size.cc b/source/blender/compositor/realtime_compositor/algorithms/intern/symmetric_separable_blur_variable_size.cc new file mode 100644 index 00000000000..607d83062d9 --- /dev/null +++ b/source/blender/compositor/realtime_compositor/algorithms/intern/symmetric_separable_blur_variable_size.cc @@ -0,0 +1,139 @@ +/* SPDX-FileCopyrightText: 2023 Blender Authors + * + * SPDX-License-Identifier: GPL-2.0-or-later */ + +#include "BLI_assert.h" +#include "BLI_math_base.hh" +#include "BLI_math_vector.hh" +#include "BLI_math_vector_types.hh" + +#include "GPU_shader.h" +#include "GPU_texture.h" + +#include "COM_context.hh" +#include "COM_result.hh" +#include "COM_utilities.hh" + +#include "COM_algorithm_symmetric_separable_blur_variable_size.hh" + +#include "COM_symmetric_separable_blur_weights.hh" + +namespace blender::realtime_compositor { + +static const char *get_blur_shader(ResultType type) +{ + switch (type) { + case ResultType::Float: + return "compositor_symmetric_separable_blur_variable_size_float"; + case ResultType::Float2: + return "compositor_symmetric_separable_blur_variable_size_float2"; + case ResultType::Vector: + case ResultType::Color: + return "compositor_symmetric_separable_blur_variable_size_float4"; + case ResultType::Float3: + /* GPU module does not support float3 outputs. */ + break; + case ResultType::Int2: + /* Blur does not support integer types. */ + break; + } + + BLI_assert_unreachable(); + return nullptr; +} + +static Result horizontal_pass( + Context &context, Result &input, Result &radius, int filter_type, int weights_resolution) +{ + GPUShader *shader = context.get_shader(get_blur_shader(input.type())); + GPU_shader_bind(shader); + + GPU_shader_uniform_1b(shader, "is_vertical_pass", false); + + input.bind_as_texture(shader, "input_tx"); + + const SymmetricSeparableBlurWeights &weights = + context.cache_manager().symmetric_separable_blur_weights.get( + context, filter_type, weights_resolution); + weights.bind_as_texture(shader, "weights_tx"); + + radius.bind_as_texture(shader, "radius_tx"); + + /* We allocate an output image of a transposed size, that is, with a height equivalent to the + * width of the input and vice versa. This is done as a performance optimization. The shader + * will blur the image horizontally and write it to the intermediate output transposed. Then + * the vertical pass will execute the same horizontal blur shader, but since its input is + * transposed, it will effectively do a vertical blur and write to the output transposed, + * effectively undoing the transposition in the horizontal pass. This is done to improve + * spatial cache locality in the shader and to avoid having two separate shaders for each blur + * pass. */ + Domain domain = input.domain(); + const int2 transposed_domain = int2(domain.size.y, domain.size.x); + + Result output = context.create_temporary_result(input.type()); + output.allocate_texture(transposed_domain); + output.bind_as_image(shader, "output_img"); + + compute_dispatch_threads_at_least(shader, domain.size); + + GPU_shader_unbind(); + input.unbind_as_texture(); + weights.unbind_as_texture(); + radius.unbind_as_texture(); + output.unbind_as_image(); + + return output; +} + +static void vertical_pass(Context &context, + Result &original_input, + Result &horizontal_pass_result, + Result &output, + Result &radius, + int filter_type, + int weights_resolution) +{ + GPUShader *shader = context.get_shader(get_blur_shader(original_input.type())); + GPU_shader_bind(shader); + + GPU_shader_uniform_1b(shader, "is_vertical_pass", true); + + horizontal_pass_result.bind_as_texture(shader, "input_tx"); + + const SymmetricSeparableBlurWeights &weights = + context.cache_manager().symmetric_separable_blur_weights.get( + context, filter_type, weights_resolution); + weights.bind_as_texture(shader, "weights_tx"); + + radius.bind_as_texture(shader, "radius_tx"); + + Domain domain = original_input.domain(); + output.allocate_texture(domain); + output.bind_as_image(shader, "output_img"); + + /* Notice that the domain is transposed, see the note on the horizontal pass method for more + * information on the reasoning behind this. */ + compute_dispatch_threads_at_least(shader, int2(domain.size.y, domain.size.x)); + + GPU_shader_unbind(); + horizontal_pass_result.unbind_as_texture(); + output.unbind_as_image(); + weights.unbind_as_texture(); + radius.unbind_as_texture(); +} + +void symmetric_separable_blur_variable_size(Context &context, + Result &input, + Result &output, + Result &radius, + int filter_type, + int weights_resolution) +{ + Result horizontal_pass_result = horizontal_pass( + context, input, radius, filter_type, weights_resolution); + vertical_pass( + context, input, horizontal_pass_result, output, radius, filter_type, weights_resolution); + horizontal_pass_result.release(); +} + +} // namespace blender::realtime_compositor diff --git a/source/blender/compositor/realtime_compositor/shaders/compositor_inpaint_compute_region.glsl b/source/blender/compositor/realtime_compositor/shaders/compositor_inpaint_compute_region.glsl index 452af949078..4ce17df487a 100644 --- a/source/blender/compositor/realtime_compositor/shaders/compositor_inpaint_compute_region.glsl +++ b/source/blender/compositor/realtime_compositor/shaders/compositor_inpaint_compute_region.glsl @@ -2,12 +2,7 @@ * * SPDX-License-Identifier: GPL-2.0-or-later */ -/* Fill the inpainting region by sampling the color of the nearest boundary pixel if it is not - * further than the user supplied distance. Additionally, apply a lateral blur in the tangential - * path to the inpainting boundary to smooth out the inpainted region. */ - #pragma BLENDER_REQUIRE(gpu_shader_compositor_texture_utilities.glsl) -#pragma BLENDER_REQUIRE(gpu_shader_compositor_jump_flooding_lib.glsl) void main() { @@ -15,55 +10,23 @@ void main() vec4 color = texture_load(input_tx, texel); - /* An opaque pixel, no inpainting needed. */ + /* An opaque pixel, not part of the inpainting region, write the original color. */ if (color.a == 1.0) { imageStore(output_img, texel, color); return; } - ivec2 closest_boundary_texel = texture_load(flooded_boundary_tx, texel).xy; - float distance_to_boundary = distance(vec2(texel), vec2(closest_boundary_texel)); + float distance_to_boundary = texture_load(distance_to_boundary_tx, texel).x; - /* Further than the user supplied distance, write a transparent color. */ + /* Further than the inpainting distance, not part of the inpainting region, write the original + * color. */ if (distance_to_boundary > max_distance) { - imageStore(output_img, texel, vec4(0.0)); + imageStore(output_img, texel, color); return; } - /* We set the blur radius to be proportional to the distance to the boundary. */ - int blur_radius = int(ceil(distance_to_boundary)); - - /* Laterally blur by accumulate the boundary pixels nearest to the pixels along the tangential - * path in both directions starting from the current pixel, noting that the weights texture only - * stores the weights for the left half, but since the Gaussian is symmetric, the same weight is - * used for the right half and we add both of their contributions. */ - vec2 left_texel = vec2(texel); - vec2 right_texel = vec2(texel); - float accumulated_weight = 0.0; - vec4 accumulated_color = vec4(0.0); - for (int i = 0; i < blur_radius; i++) { - float weight = texture(gaussian_weights_tx, float(i / (blur_radius - 1))).x; - - { - ivec2 boundary_texel = texture_load(flooded_boundary_tx, ivec2(left_texel)).xy; - accumulated_color += texture_load(input_tx, boundary_texel) * weight; - accumulated_weight += weight; - - /* Move the left texel one pixel in the clockwise tangent to the boundary. */ - left_texel += normalize((left_texel - vec2(boundary_texel)).yx * vec2(-1.0, 1.0)); - } - - /* When i is zero, we are accumulating the center pixel, which was already accumulated as the - * left texel above, so no need to accumulate it again. */ - if (i != 0) { - ivec2 boundary_texel = texture_load(flooded_boundary_tx, ivec2(right_texel)).xy; - accumulated_color += texture_load(input_tx, boundary_texel) * weight; - accumulated_weight += weight; - - /* Move the left texel one pixel in the anti-clockwise tangent to the boundary. */ - right_texel += normalize((right_texel - vec2(boundary_texel)).yx * vec2(1.0, -1.0)); - } - } - - imageStore(output_img, texel, accumulated_color / accumulated_weight); + /* Mix the inpainted color with the original color using its alpha because semi-transparent areas + * are considered to be partially inpainted. */ + vec4 inpainted_color = texture_load(inpainted_region_tx, texel); + imageStore(output_img, texel, vec4(mix(inpainted_color.rgb, color.rgb, color.a), 1.0)); } diff --git a/source/blender/compositor/realtime_compositor/shaders/compositor_inpaint_fill_region.glsl b/source/blender/compositor/realtime_compositor/shaders/compositor_inpaint_fill_region.glsl new file mode 100644 index 00000000000..29923f44de3 --- /dev/null +++ b/source/blender/compositor/realtime_compositor/shaders/compositor_inpaint_fill_region.glsl @@ -0,0 +1,48 @@ +/* SPDX-FileCopyrightText: 2023 Blender Authors + * + * SPDX-License-Identifier: GPL-2.0-or-later */ + +/* Fill the inpainting region by sampling the color of the nearest boundary pixel. Additionally, + * compute some information about the inpainting region, like the distance to the boundary, as well + * as the blur radius to use to smooth out that region. */ + +#pragma BLENDER_REQUIRE(gpu_shader_math_base_lib.glsl) +#pragma BLENDER_REQUIRE(gpu_shader_compositor_texture_utilities.glsl) +#pragma BLENDER_REQUIRE(gpu_shader_compositor_jump_flooding_lib.glsl) + +void main() +{ + ivec2 texel = ivec2(gl_GlobalInvocationID.xy); + + vec4 color = texture_load(input_tx, texel); + + /* An opaque pixel, not part of the inpainting region. */ + if (color.a == 1.0) { + imageStore(filled_region_img, texel, color); + imageStore(smoothing_radius_img, texel, vec4(0.0)); + imageStore(distance_to_boundary_img, texel, vec4(0.0)); + return; + } + + ivec2 closest_boundary_texel = texture_load(flooded_boundary_tx, texel).xy; + float distance_to_boundary = distance(vec2(texel), vec2(closest_boundary_texel)); + imageStore(distance_to_boundary_img, texel, vec4(distance_to_boundary)); + + /* We follow this shader by a blur shader that smoothes out the inpainting region, where the blur + * radius is the radius of the circle that touches the boundary. We can imagine the blur window + * to be inscribed in that circle and thus the blur radius is the distance to the boundary + * divided by square root two. As a performance optimization, we limit the blurring to areas that + * will affect the inpainting region, that is, whose distance to boundary is less than double the + * inpainting distance. Additionally, we clamp to the distance to the inpainting distance since + * areas outside of the clamp range only indirectly affect the inpainting region due to blurring + * and thus needn't use higher blur radii. */ + float blur_window_size = min(max_distance, distance_to_boundary) / M_SQRT2; + bool skip_smoothing = distance_to_boundary > (max_distance * 2.0); + float smoothing_radius = skip_smoothing ? 0.0 : blur_window_size; + imageStore(smoothing_radius_img, texel, vec4(smoothing_radius)); + + /* Mix the boundary color with the original color using its alpha because semi-transparent areas + * are considered to be partially inpainted. */ + vec4 boundary_color = texture_load(input_tx, closest_boundary_texel); + imageStore(filled_region_img, texel, mix(boundary_color, color, color.a)); +} diff --git a/source/blender/compositor/realtime_compositor/shaders/compositor_symmetric_separable_blur_variable_size.glsl b/source/blender/compositor/realtime_compositor/shaders/compositor_symmetric_separable_blur_variable_size.glsl new file mode 100644 index 00000000000..63daa32360c --- /dev/null +++ b/source/blender/compositor/realtime_compositor/shaders/compositor_symmetric_separable_blur_variable_size.glsl @@ -0,0 +1,40 @@ +/* SPDX-FileCopyrightText: 2022 Blender Authors + * + * SPDX-License-Identifier: GPL-2.0-or-later */ + +#pragma BLENDER_REQUIRE(gpu_shader_compositor_blur_common.glsl) +#pragma BLENDER_REQUIRE(gpu_shader_compositor_texture_utilities.glsl) + +void main() +{ + ivec2 texel = ivec2(gl_GlobalInvocationID.xy); + + float accumulated_weight = 0.0; + vec4 accumulated_color = vec4(0.0); + + /* First, compute the contribution of the center pixel. */ + vec4 center_color = texture_load(input_tx, texel); + float center_weight = texture_load(weights_tx, 0).x; + accumulated_color += center_color * center_weight; + accumulated_weight += center_weight; + + /* The dispatch domain is transposed in the vertical pass, so make sure to reverse transpose the + * texel coordinates when loading the radius. See the horizontal_pass function in the + * symmetric_separable_blur_variable_size.cc file for more information. */ + int radius = int(texture_load(radius_tx, is_vertical_pass ? texel.yx : texel).x); + + /* Then, compute the contributions of the pixel to the right and left, noting that the + * weights texture only stores the weights for the positive half, but since the filter is + * symmetric, the same weight is used for the negative half and we add both of their + * contributions. */ + for (int i = 1; i < radius; i++) { + float weight = texture(weights_tx, i / float(radius)).x; + accumulated_color += texture_load(input_tx, texel + ivec2(i, 0)) * weight; + accumulated_color += texture_load(input_tx, texel + ivec2(-i, 0)) * weight; + accumulated_weight += weight * 2.0; + } + + /* Write the color using the transposed texel. See the horizontal_pass function mentioned above + * for more information on the rational behind this. */ + imageStore(output_img, texel.yx, accumulated_color / accumulated_weight); +} diff --git a/source/blender/compositor/realtime_compositor/shaders/infos/compositor_inpaint_info.hh b/source/blender/compositor/realtime_compositor/shaders/infos/compositor_inpaint_info.hh index 4144cd3ad8c..bb2719992fb 100644 --- a/source/blender/compositor/realtime_compositor/shaders/infos/compositor_inpaint_info.hh +++ b/source/blender/compositor/realtime_compositor/shaders/infos/compositor_inpaint_info.hh @@ -11,12 +11,23 @@ GPU_SHADER_CREATE_INFO(compositor_inpaint_compute_boundary) .compute_source("compositor_inpaint_compute_boundary.glsl") .do_static_compilation(true); -GPU_SHADER_CREATE_INFO(compositor_inpaint_compute_region) +GPU_SHADER_CREATE_INFO(compositor_inpaint_fill_region) .local_group_size(16, 16) .push_constant(Type::INT, "max_distance") .sampler(0, ImageType::FLOAT_2D, "input_tx") .sampler(1, ImageType::INT_2D, "flooded_boundary_tx") - .sampler(2, ImageType::INT_1D, "gaussian_weights_tx") + .image(0, GPU_RGBA16F, Qualifier::WRITE, ImageType::FLOAT_2D, "filled_region_img") + .image(1, GPU_R16F, Qualifier::WRITE, ImageType::FLOAT_2D, "distance_to_boundary_img") + .image(2, GPU_R16F, Qualifier::WRITE, ImageType::FLOAT_2D, "smoothing_radius_img") + .compute_source("compositor_inpaint_fill_region.glsl") + .do_static_compilation(true); + +GPU_SHADER_CREATE_INFO(compositor_inpaint_compute_region) + .local_group_size(16, 16) + .push_constant(Type::INT, "max_distance") + .sampler(0, ImageType::FLOAT_2D, "input_tx") + .sampler(1, ImageType::FLOAT_2D, "inpainted_region_tx") + .sampler(2, ImageType::FLOAT_2D, "distance_to_boundary_tx") .image(0, GPU_RGBA16F, Qualifier::WRITE, ImageType::FLOAT_2D, "output_img") .compute_source("compositor_inpaint_compute_region.glsl") .do_static_compilation(true); diff --git a/source/blender/compositor/realtime_compositor/shaders/infos/compositor_symmetric_separable_blur_variable_size_info.hh b/source/blender/compositor/realtime_compositor/shaders/infos/compositor_symmetric_separable_blur_variable_size_info.hh new file mode 100644 index 00000000000..1528c111b0e --- /dev/null +++ b/source/blender/compositor/realtime_compositor/shaders/infos/compositor_symmetric_separable_blur_variable_size_info.hh @@ -0,0 +1,28 @@ +/* SPDX-FileCopyrightText: 2023 Blender Authors + * + * SPDX-License-Identifier: GPL-2.0-or-later */ + +#include "gpu_shader_create_info.hh" + +GPU_SHADER_CREATE_INFO(compositor_symmetric_separable_blur_variable_size_shared) + .local_group_size(16, 16) + .push_constant(Type::BOOL, "is_vertical_pass") + .sampler(0, ImageType::FLOAT_2D, "input_tx") + .sampler(1, ImageType::FLOAT_1D, "weights_tx") + .sampler(2, ImageType::FLOAT_2D, "radius_tx") + .compute_source("compositor_symmetric_separable_blur_variable_size.glsl"); + +GPU_SHADER_CREATE_INFO(compositor_symmetric_separable_blur_variable_size_float) + .additional_info("compositor_symmetric_separable_blur_variable_size_shared") + .image(0, GPU_R16F, Qualifier::WRITE, ImageType::FLOAT_2D, "output_img") + .do_static_compilation(true); + +GPU_SHADER_CREATE_INFO(compositor_symmetric_separable_blur_variable_size_float2) + .additional_info("compositor_symmetric_separable_blur_variable_size_shared") + .image(0, GPU_RG16F, Qualifier::WRITE, ImageType::FLOAT_2D, "output_img") + .do_static_compilation(true); + +GPU_SHADER_CREATE_INFO(compositor_symmetric_separable_blur_variable_size_float4) + .additional_info("compositor_symmetric_separable_blur_variable_size_shared") + .image(0, GPU_RGBA16F, Qualifier::WRITE, ImageType::FLOAT_2D, "output_img") + .do_static_compilation(true); diff --git a/source/blender/nodes/composite/nodes/node_composite_inpaint.cc b/source/blender/nodes/composite/nodes/node_composite_inpaint.cc index 68bf0cbea6c..cfdaac5edc0 100644 --- a/source/blender/nodes/composite/nodes/node_composite_inpaint.cc +++ b/source/blender/nodes/composite/nodes/node_composite_inpaint.cc @@ -12,8 +12,8 @@ #include "DNA_scene_types.h" #include "COM_algorithm_jump_flooding.hh" +#include "COM_algorithm_symmetric_separable_blur_variable_size.hh" #include "COM_node_operation.hh" -#include "COM_symmetric_separable_blur_weights.hh" #include "COM_utilities.hh" #include "node_composite_util.hh" @@ -45,14 +45,11 @@ class InpaintOperation : public NodeOperation { { Result &input = get_input("Image"); Result &output = get_result("Image"); - if (input.is_single_value() || get_distance() == 0) { + if (input.is_single_value() || get_max_distance() == 0) { input.pass_through(output); return; } - /* Compute an image that marks the boundary pixels of the inpainting region as seed pixels in - * the format expected by the jump flooding algorithm. The inpainting region is the region - * composed of pixels that are not opaque. */ Result inpainting_boundary = compute_inpainting_boundary(); /* Compute a jump flooding table to get the closest boundary pixel to each pixel. */ @@ -61,11 +58,33 @@ class InpaintOperation : public NodeOperation { jump_flooding(context(), inpainting_boundary, flooded_boundary); inpainting_boundary.release(); - /* Fill the inpainting region based on the jump flooding table. */ - compute_inpainting_region(flooded_boundary); + Result filled_region = context().create_temporary_result(ResultType::Color); + Result distance_to_boundary = context().create_temporary_result(ResultType::Float, + ResultPrecision::Half); + Result smoothing_radius = context().create_temporary_result(ResultType::Float, + ResultPrecision::Half); + fill_inpainting_region( + flooded_boundary, filled_region, distance_to_boundary, smoothing_radius); flooded_boundary.release(); + + Result smoothed_region = context().create_temporary_result(ResultType::Color); + symmetric_separable_blur_variable_size(context(), + filled_region, + smoothed_region, + smoothing_radius, + R_FILTER_GAUSS, + get_max_distance()); + filled_region.release(); + smoothing_radius.release(); + + compute_inpainting_region(smoothed_region, distance_to_boundary); + distance_to_boundary.release(); + smoothed_region.release(); } + /* Compute an image that marks the boundary pixels of the inpainting region as seed pixels for + * the jump flooding algorithm. The inpainting region is the region composed of pixels that are + * not opaque. */ Result compute_inpainting_boundary() { GPUShader *shader = context().get_shader("compositor_inpaint_compute_boundary", @@ -90,25 +109,57 @@ class InpaintOperation : public NodeOperation { return inpainting_boundary; } - void compute_inpainting_region(Result &flooded_boundary) + /* Fill the inpainting region based on the jump flooding table and write the distance to the + * closest boundary pixel to an intermediate buffer. */ + void fill_inpainting_region(Result &flooded_boundary, + Result &filled_region, + Result &distance_to_boundary, + Result &smoothing_radius) { - GPUShader *shader = context().get_shader("compositor_inpaint_compute_region"); + GPUShader *shader = context().get_shader("compositor_inpaint_fill_region"); GPU_shader_bind(shader); - GPU_shader_uniform_1i(shader, "max_distance", get_distance()); + GPU_shader_uniform_1i(shader, "max_distance", get_max_distance()); const Result &input = get_input("Image"); input.bind_as_texture(shader, "input_tx"); flooded_boundary.bind_as_texture(shader, "flooded_boundary_tx"); - /* The lateral blur in the shader is proportional to the distance each pixel makes with the - * inpainting boundary. So the maximum possible blur radius is the user supplied distance. */ - const float max_radius = float(get_distance()); - const SymmetricSeparableBlurWeights &gaussian_weights = - context().cache_manager().symmetric_separable_blur_weights.get( - context(), R_FILTER_GAUSS, max_radius); - gaussian_weights.bind_as_texture(shader, "gaussian_weights_tx"); + const Domain domain = compute_domain(); + filled_region.allocate_texture(domain); + filled_region.bind_as_image(shader, "filled_region_img"); + + distance_to_boundary.allocate_texture(domain); + distance_to_boundary.bind_as_image(shader, "distance_to_boundary_img"); + + smoothing_radius.allocate_texture(domain); + smoothing_radius.bind_as_image(shader, "smoothing_radius_img"); + + compute_dispatch_threads_at_least(shader, domain.size); + + input.unbind_as_texture(); + flooded_boundary.unbind_as_texture(); + filled_region.unbind_as_image(); + distance_to_boundary.unbind_as_image(); + smoothing_radius.unbind_as_image(); + GPU_shader_unbind(); + } + + /* Compute the inpainting region by mixing the smoothed inpainted region with the original input + * up to the inpainting distance. */ + void compute_inpainting_region(Result &inpainted_region, Result &distance_to_boundary) + { + GPUShader *shader = context().get_shader("compositor_inpaint_compute_region"); + GPU_shader_bind(shader); + + GPU_shader_uniform_1i(shader, "max_distance", get_max_distance()); + + const Result &input = get_input("Image"); + input.bind_as_texture(shader, "input_tx"); + + inpainted_region.bind_as_texture(shader, "inpainted_region_tx"); + distance_to_boundary.bind_as_texture(shader, "distance_to_boundary_tx"); const Domain domain = compute_domain(); Result &output = get_result("Image"); @@ -118,12 +169,13 @@ class InpaintOperation : public NodeOperation { compute_dispatch_threads_at_least(shader, domain.size); input.unbind_as_texture(); - gaussian_weights.unbind_as_texture(); + inpainted_region.unbind_as_texture(); + distance_to_boundary.unbind_as_texture(); output.unbind_as_image(); GPU_shader_unbind(); } - int get_distance() + int get_max_distance() { return bnode().custom2; } -- 2.30.2 From 39e2d297617a4f8085dbd74eb91b25f648307e40 Mon Sep 17 00:00:00 2001 From: Omar Emara Date: Wed, 22 Nov 2023 14:20:08 +0200 Subject: [PATCH 2/4] Fix Metal compile error --- .../shaders/compositor_inpaint_fill_region.glsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/blender/compositor/realtime_compositor/shaders/compositor_inpaint_fill_region.glsl b/source/blender/compositor/realtime_compositor/shaders/compositor_inpaint_fill_region.glsl index 29923f44de3..829eabcdc1d 100644 --- a/source/blender/compositor/realtime_compositor/shaders/compositor_inpaint_fill_region.glsl +++ b/source/blender/compositor/realtime_compositor/shaders/compositor_inpaint_fill_region.glsl @@ -36,7 +36,7 @@ void main() * inpainting distance. Additionally, we clamp to the distance to the inpainting distance since * areas outside of the clamp range only indirectly affect the inpainting region due to blurring * and thus needn't use higher blur radii. */ - float blur_window_size = min(max_distance, distance_to_boundary) / M_SQRT2; + float blur_window_size = min(float(max_distance), distance_to_boundary) / M_SQRT2; bool skip_smoothing = distance_to_boundary > (max_distance * 2.0); float smoothing_radius = skip_smoothing ? 0.0 : blur_window_size; imageStore(smoothing_radius_img, texel, vec4(smoothing_radius)); -- 2.30.2 From f75ba0c47128eb2f62e11f468ee4bb16c522282e Mon Sep 17 00:00:00 2001 From: Omar Emara Date: Wed, 22 Nov 2023 14:20:34 +0200 Subject: [PATCH 3/4] Enable filtering for blur weights --- .../cached_resources/intern/symmetric_separable_blur_weights.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/blender/compositor/realtime_compositor/cached_resources/intern/symmetric_separable_blur_weights.cc b/source/blender/compositor/realtime_compositor/cached_resources/intern/symmetric_separable_blur_weights.cc index 22dd4a521d0..55b41042803 100644 --- a/source/blender/compositor/realtime_compositor/cached_resources/intern/symmetric_separable_blur_weights.cc +++ b/source/blender/compositor/realtime_compositor/cached_resources/intern/symmetric_separable_blur_weights.cc @@ -84,6 +84,8 @@ SymmetricSeparableBlurWeights::SymmetricSeparableBlurWeights(Context &context, Result::texture_format(ResultType::Float, context.get_precision()), GPU_TEXTURE_USAGE_GENERAL, weights.data()); + GPU_texture_filter_mode(texture_, true); + GPU_texture_extend_mode(texture_, GPU_SAMPLER_EXTEND_MODE_EXTEND); } SymmetricSeparableBlurWeights::~SymmetricSeparableBlurWeights() -- 2.30.2 From 89c84f27005ac6f536260c8d1c634a74b9bd66cb Mon Sep 17 00:00:00 2001 From: Omar Emara Date: Wed, 22 Nov 2023 14:21:03 +0200 Subject: [PATCH 4/4] Properly sample blur weights --- .../compositor_symmetric_separable_blur_variable_size.glsl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/blender/compositor/realtime_compositor/shaders/compositor_symmetric_separable_blur_variable_size.glsl b/source/blender/compositor/realtime_compositor/shaders/compositor_symmetric_separable_blur_variable_size.glsl index 63daa32360c..e72b211f592 100644 --- a/source/blender/compositor/realtime_compositor/shaders/compositor_symmetric_separable_blur_variable_size.glsl +++ b/source/blender/compositor/realtime_compositor/shaders/compositor_symmetric_separable_blur_variable_size.glsl @@ -27,8 +27,9 @@ void main() * weights texture only stores the weights for the positive half, but since the filter is * symmetric, the same weight is used for the negative half and we add both of their * contributions. */ - for (int i = 1; i < radius; i++) { - float weight = texture(weights_tx, i / float(radius)).x; + for (int i = 1; i <= radius; i++) { + /* Add 0.5 to evaluate at the center of the pixels. */ + float weight = texture(weights_tx, (float(i) + 0.5) / float(radius + 1)).x; accumulated_color += texture_load(input_tx, texel + ivec2(i, 0)) * weight; accumulated_color += texture_load(input_tx, texel + ivec2(-i, 0)) * weight; accumulated_weight += weight * 2.0; -- 2.30.2