Merge branch 'master' into xr-dev

Updated version of https://developer.blender.org/D13153.

This enables practical use of more Blender operators in VR by
converting 3D controller positions to 2D mouse positions as well as
simulating the event type (LEFTMOUSE, MOUSEMOVE...) and value
(PRESS, CLICK...) based on VR controller inputs. In this way, the
view3d.select operator can be used in VR without any modifications to
the operator logic, enabling selection of armatures, empties, etc. in
VR.

If an XR action's simulate_mouse property is set, the 3D position of
the corresponding controller aim pose will be projected to a 2D mouse
position using the perspective of the user's preferred eye
(XrSessionSettings.projection_eye) and the result stored in
wmEvent.xy/mval. In this way, operators can make use of these mouse
values as usual, without needing to know that the event is an XR event.

Along with mouse coordinate simulation, the XR event type and value
can be simulated as a different wmEvent type/value during handling,
where this mapping is configurable via the XrActionMapItem.simulate
(XrSimulateMouseParams) property of an XR action map item. The
simulated type/value can be individually set for press, hold (modal
operators only), and release VR button interactions, depending on the
operator execution mode (XrActionMapItem.op_mode).

build_files/config/pipeline_config.yaml

@@ -5,7 +5,7 @@
 update-code:
     git:
         submodules:
-        -   branch: master
+        -   branch: xr-dev
             commit_id: HEAD
             path: release/scripts/addons
         -   branch: master

intern/ghost/GHOST_Types.h

@@ -689,6 +689,10 @@ enum {
    * (use DirectX fallback instead). */
   GHOST_kXrContextGpuNVIDIA = (1 << 2),
 #  endif
+  /* Needed to selectively enable the #XR_HTCX_vive_tracker_interaction extension, as it causes
+     incorrect controller poses with SteamVR (1.21). Although not ideal, either trackers or
+     controllers (but not both) can be used during a given session. */
+  GHOST_kXrContextEnableViveTrackerExtension = (1 << 3),
 };
 
 typedef struct {

intern/ghost/intern/GHOST_XrContext.cpp

@@ -82,7 +82,7 @@ void GHOST_XrContext::initialize(const GHOST_XrContextCreateInfo *create_info)
       determineGraphicsBindingTypesToEnable(create_info);
 
   assert(m_oxr->instance == XR_NULL_HANDLE);
-  createOpenXRInstance(graphics_binding_types);
+  createOpenXRInstance(graphics_binding_types, create_info);
   storeInstanceProperties();
 
   /* Multiple bindings may be enabled. Now that we know the runtime in use, settle for one. */
@@ -95,7 +95,8 @@ void GHOST_XrContext::initialize(const GHOST_XrContextCreateInfo *create_info)
 }
 
 void GHOST_XrContext::createOpenXRInstance(
-    const std::vector<GHOST_TXrGraphicsBinding> &graphics_binding_types)
+    const std::vector<GHOST_TXrGraphicsBinding> &graphics_binding_types,
+    const GHOST_XrContextCreateInfo *ctx_create_info)
 {
   XrInstanceCreateInfo create_info = {XR_TYPE_INSTANCE_CREATE_INFO};
 
@@ -104,7 +105,7 @@ void GHOST_XrContext::createOpenXRInstance(
   create_info.applicationInfo.apiVersion = XR_CURRENT_API_VERSION;
 
   getAPILayersToEnable(m_enabled_layers);
-  getExtensionsToEnable(graphics_binding_types, m_enabled_extensions);
+  getExtensionsToEnable(graphics_binding_types, ctx_create_info, m_enabled_extensions);
   create_info.enabledApiLayerCount = m_enabled_layers.size();
   create_info.enabledApiLayerNames = m_enabled_layers.data();
   create_info.enabledExtensionCount = m_enabled_extensions.size();
@@ -400,6 +401,7 @@ static const char *openxr_ext_name_from_wm_gpu_binding(GHOST_TXrGraphicsBinding
  */
 void GHOST_XrContext::getExtensionsToEnable(
     const std::vector<GHOST_TXrGraphicsBinding> &graphics_binding_types,
+    const GHOST_XrContextCreateInfo *create_info,
     std::vector<const char *> &r_ext_names)
 {
   std::vector<std::string_view> try_ext;
@@ -415,6 +417,11 @@ void GHOST_XrContext::getExtensionsToEnable(
 #ifdef XR_HTC_VIVE_FOCUS3_CONTROLLER_INTERACTION_EXTENSION_NAME
   try_ext.push_back(XR_HTC_VIVE_FOCUS3_CONTROLLER_INTERACTION_EXTENSION_NAME);
 #endif
+  if ((create_info->context_flag & GHOST_kXrContextEnableViveTrackerExtension) != 0) {
+#ifdef XR_HTCX_VIVE_TRACKER_INTERACTION_EXTENSION_NAME
+    try_ext.push_back(XR_HTCX_VIVE_TRACKER_INTERACTION_EXTENSION_NAME);
+#endif
+  }
   try_ext.push_back(XR_HUAWEI_CONTROLLER_INTERACTION_EXTENSION_NAME);
 
   /* Controller model extension. */

intern/ghost/intern/GHOST_XrContext.h

@@ -108,7 +108,8 @@ class GHOST_XrContext : public GHOST_IXrContext {
   bool m_debug = false;
   bool m_debug_time = false;
 
-  void createOpenXRInstance(const std::vector<GHOST_TXrGraphicsBinding> &graphics_binding_types);
+  void createOpenXRInstance(const std::vector<GHOST_TXrGraphicsBinding> &graphics_binding_types,
+                            const GHOST_XrContextCreateInfo *create_info);
   void storeInstanceProperties();
   void initDebugMessenger();
 
@@ -122,6 +123,7 @@ class GHOST_XrContext : public GHOST_IXrContext {
   void initExtensionsEx(std::vector<XrExtensionProperties> &extensions, const char *layer_name);
   void getAPILayersToEnable(std::vector<const char *> &r_ext_names);
   void getExtensionsToEnable(const std::vector<GHOST_TXrGraphicsBinding> &graphics_binding_types,
+                             const GHOST_XrContextCreateInfo *create_info,
                              std::vector<const char *> &r_ext_names);
   std::vector<GHOST_TXrGraphicsBinding> determineGraphicsBindingTypesToEnable(
       const GHOST_XrContextCreateInfo *create_info);

source/blender/blenkernel/BKE_node.h

@@ -1223,6 +1223,7 @@ void BKE_nodetree_remove_layer_n(struct bNodeTree *ntree, struct Scene *scene, i
 #define SH_NODE_COMBINE_COLOR 711
 #define SH_NODE_SEPARATE_COLOR 712
 #define SH_NODE_MIX 713
+#define SH_NODE_VIRTUAL_MONITOR 714
 
 /** \} */
 

source/blender/blenkernel/intern/camera.c

@@ -41,6 +41,8 @@
 
 #include "DEG_depsgraph_query.h"
 
+#include "GPU_viewport.h"
+
 #include "MEM_guardedalloc.h"
 
 #include "BLO_read_write.h"
@@ -57,6 +59,20 @@ static void camera_init_data(ID *id)
   MEMCPY_STRUCT_AFTER(cam, DNA_struct_default_get(Camera), id);
 }
 
+/** Free (or release) any data used by this camera (does not free the camera itself). */
+static void camera_free_runtime_data(Camera *cam, const Camera *cam_src)
+{
+  if (cam->runtime.virtual_monitor_offscreen) {
+    if (!cam_src ||
+        (cam_src->runtime.virtual_monitor_offscreen != cam->runtime.virtual_monitor_offscreen)) {
+      GPU_offscreen_free(cam->runtime.virtual_monitor_offscreen);
+    }
+    cam->runtime.virtual_monitor_offscreen = NULL;
+  }
+  /* GPU texture is owned by the GPUOffscreen instance. */
+  cam->runtime.offscreen_color_texture = NULL;
+}
+
 /**
  * Only copy internal data of Camera ID from source
  * to already allocated/initialized destination.
@@ -80,13 +96,17 @@ static void camera_copy_data(Main *UNUSED(bmain), ID *id_dst, const ID *id_src,
     CameraBGImage *bgpic_dst = BKE_camera_background_image_copy(bgpic_src, flag_subdata);
     BLI_addtail(&cam_dst->bg_images, bgpic_dst);
   }
+
+  /* Free runtime data to prevent virtual monitor offscreen from being shared across data-blocks.
+   */
+  camera_free_runtime_data(cam_dst, cam_src);
 }
 
-/** Free (or release) any data used by this camera (does not free the camera itself). */
 static void camera_free_data(ID *id)
 {
   Camera *cam = (Camera *)id;
   BLI_freelistN(&cam->bg_images);
+  camera_free_runtime_data(cam, NULL);
 }
 
 static void camera_foreach_id(ID *id, LibraryForeachIDData *data)
@@ -137,6 +157,8 @@ static void camera_blend_read_data(BlendDataReader *reader, ID *id)
       bgpic->flag &= ~CAM_BGIMG_FLAG_OVERRIDE_LIBRARY_LOCAL;
     }
   }
+  ca->runtime.virtual_monitor_offscreen = NULL;
+  ca->runtime.offscreen_color_texture = NULL;
 }
 
 static void camera_blend_read_lib(BlendLibReader *reader, ID *id)

source/blender/blenloader/intern/readfile.cc

@@ -2333,6 +2333,10 @@ static void lib_link_wm_xr_data_restore(IDNameLib_Map *id_map, wmXrData *xr_data
 {
   xr_data->session_settings.base_pose_object = static_cast<Object *>(restore_pointer_by_name(
       id_map, (ID *)xr_data->session_settings.base_pose_object, USER_REAL));
+
+  LISTBASE_FOREACH (XrMotionCaptureObject *, mocap_ob, &xr_data->session_settings.mocap_objects) {
+    mocap_ob->ob = (Object *)restore_pointer_by_name(id_map, (ID *)mocap_ob->ob, USER_REAL);
+  }
 }
 
 static void lib_link_window_scene_data_restore(wmWindow *win, Scene *scene, ViewLayer *view_layer)

source/blender/draw/DRW_engine.h

@@ -99,6 +99,7 @@ void DRW_draw_render_loop_offscreen(struct Depsgraph *depsgraph,
                                     bool is_image_render,
                                     bool draw_background,
                                     bool do_color_management,
+                                    bool is_virtual_camera,
                                     struct GPUOffScreen *ofs,
                                     struct GPUViewport *viewport);
 void DRW_draw_render_loop_2d_ex(struct Depsgraph *depsgraph,

source/blender/draw/engines/eevee/eevee_materials.c

@@ -861,6 +861,12 @@ void EEVEE_materials_cache_populate(EEVEE_Data *vedata,
               continue;
             }
 
+            /* Virtual cameras can only be used in the main scene. */
+            if (DRW_state_is_virtual_camera() &&
+                GPU_material_flag_get(gpumat_array[i], GPU_MATFLAG_VIRTUAL_MONITOR)) {
+              continue;
+            }
+
             /* Do not render surface if we are rendering a volume object
              * and do not have a surface closure. */
             if (use_volume_material &&

source/blender/draw/intern/DRW_render.h

@@ -943,6 +943,10 @@ bool DRW_state_is_scene_render(void);
  */
 bool DRW_state_is_opengl_render(void);
 bool DRW_state_is_playback(void);
+/**
+ * Whether we are rendering a virtual camera, false when rendering the main camera/viewport.
+ */
+bool DRW_state_is_virtual_camera(void);
 /**
  * Is the user navigating the region.
  */

source/blender/draw/intern/draw_manager.c

@@ -1809,6 +1809,7 @@ void DRW_draw_render_loop_offscreen(struct Depsgraph *depsgraph,
                                     const bool is_image_render,
                                     const bool draw_background,
                                     const bool do_color_management,
+                                    const bool is_virtual_camera,
                                     GPUOffScreen *ofs,
                                     GPUViewport *viewport)
 {
@@ -1832,6 +1833,7 @@ void DRW_draw_render_loop_offscreen(struct Depsgraph *depsgraph,
   drw_state_prepare_clean_for_draw(&DST);
   DST.options.is_image_render = is_image_render;
   DST.options.draw_background = draw_background;
+  DST.options.is_virtual_camera = is_virtual_camera;
   DRW_draw_render_loop_ex(depsgraph, engine_type, region, v3d, render_viewport, NULL);
 
   if (draw_background) {
@@ -2924,6 +2926,11 @@ bool DRW_state_is_opengl_render(void)
   return DST.options.is_image_render && !DST.options.is_scene_render;
 }
 
+bool DRW_state_is_virtual_camera(void)
+{
+  return DST.options.is_virtual_camera;
+}
+
 bool DRW_state_is_playback(void)
 {
   if (DST.draw_ctx.evil_C != NULL) {

source/blender/draw/intern/draw_manager.h

@@ -612,6 +612,7 @@ typedef struct DRWManager {
     uint is_depth : 1;
     uint is_image_render : 1;
     uint is_scene_render : 1;
+    uint is_virtual_camera : 1;
     uint draw_background : 1;
     uint draw_text : 1;
   } options;

source/blender/draw/intern/draw_manager_data.cc

@@ -1835,6 +1835,11 @@ void DRW_shgroup_add_material_resources(DRWShadingGroup *grp, GPUMaterial *mater
       DRW_shgroup_uniform_texture_ex(
           grp, tex->sampler_name, *tex->sky, eGPUSamplerState(tex->sampler_state));
     }
+    else if (tex->camera && tex->camera[0]) {
+      /* VirtualCamera */
+      DRW_shgroup_uniform_texture_ex(
+          grp, tex->sampler_name, *tex->camera, eGPUSamplerState(tex->sampler_state));
+    }
   }
 
   GPUUniformBuf *ubo = GPU_material_uniform_buffer_get(material);

source/blender/editors/include/ED_transform.h

@@ -13,10 +13,13 @@ extern "C" {
 
 /* ******************* Registration Function ********************** */
 
+struct ID;
 struct Object;
 struct bContext;
+struct ViewLayer;
 struct wmKeyConfig;
 struct wmOperatorType;
+struct wmTimer;
 
 void ED_keymap_transform(struct wmKeyConfig *keyconf);
 void transform_operatortypes(void);
@@ -203,6 +206,19 @@ int ED_transform_calc_gizmo_stats(const struct bContext *C,
                                   const struct TransformCalcParams *params,
                                   struct TransformBounds *tbounds);
 
+/* transform_convert.c */
+void ED_transform_animrecord_check_state(struct Scene *scene,
+                                         struct wmTimer *animtimer,
+                                         struct ID *id);
+
+/* transform_convert_object.c */
+void ED_transform_autokeyframe_object(struct bContext *C,
+                                      struct Scene *scene,
+                                      struct ViewLayer *view_layer,
+                                      struct Object *ob,
+                                      int tmode);
+bool ED_transform_motionpath_need_update_object(struct Scene *scene, struct Object *ob);
+
 /**
  * Iterates over all the strips and finds the closest snapping candidate of either \a frame_1 or \a
  * frame_2. The closest snapping candidate will be the closest start or end frame of an existing

source/blender/editors/include/ED_view3d.h

@@ -976,6 +976,12 @@ void ED_view3d_viewcontext_init(struct bContext *C,
  * So object-mode specific values should remain cleared when initialized with another object.
  */
 void ED_view3d_viewcontext_init_object(struct ViewContext *vc, struct Object *obact);
+/**
+ * Deselect all except b.
+ */
+bool ED_view3d_object_deselect_all_except(const struct Scene *scene,
+                                          struct ViewLayer *view_layer,
+                                          struct Base *b);
 /**
  * Use this call when executing an operator,
  * event system doesn't set for each event the OpenGL drawing context.
@@ -1340,7 +1346,7 @@ void ED_view3d_local_collections_reset(struct bContext *C, bool reset_all);
 #ifdef WITH_XR_OPENXR
 void ED_view3d_xr_mirror_update(const struct ScrArea *area, const struct View3D *v3d, bool enable);
 void ED_view3d_xr_shading_update(struct wmWindowManager *wm,
-                                 const View3D *v3d,
+                                 const struct View3D *v3d,
                                  const struct Scene *scene);
 bool ED_view3d_is_region_xr_mirror_active(const struct wmWindowManager *wm,
                                           const struct View3D *v3d,

source/blender/editors/include/ED_view3d_offscreen.h

@@ -39,6 +39,7 @@ void ED_view3d_draw_offscreen(struct Depsgraph *depsgraph,
                               const char *viewname,
                               bool do_color_management,
                               bool restore_rv3d_mats,
+                              bool is_virtual_camera,
                               struct GPUOffScreen *ofs,
                               struct GPUViewport *viewport);
 /**

source/blender/editors/include/UI_interface.h

@@ -2417,6 +2417,7 @@ void uiTemplateEditModeSelection(uiLayout *layout, struct bContext *C);
 void uiTemplateReportsBanner(uiLayout *layout, struct bContext *C);
 void uiTemplateInputStatus(uiLayout *layout, struct bContext *C);
 void uiTemplateKeymapItemProperties(uiLayout *layout, struct PointerRNA *ptr);
+void uiTemplateXrActionmapItemProperties(uiLayout *layout, struct PointerRNA *ptr);
 
 bool uiTemplateEventFromKeymapItem(struct uiLayout *layout,
                                    const char *text,

source/blender/editors/interface/CMakeLists.txt

@@ -110,6 +110,9 @@ if(WIN32 OR APPLE)
   endif()
 endif()
 
+if(WITH_XR_OPENXR)
+  add_definitions(-DWITH_XR_OPENXR)
+endif()
 
 blender_add_lib(bf_editor_interface "${SRC}" "${INC}" "${INC_SYS}" "${LIB}")
 

source/blender/editors/interface/interface_templates.cc

@@ -6567,6 +6567,44 @@ void uiTemplateKeymapItemProperties(uiLayout *layout, PointerRNA *ptr)
 
 /** \} */
 
+/* -------------------------------------------------------------------- */
+/** \name XR Actionmap Template
+ * \{ */
+
+#ifdef WITH_XR_OPENXR
+static void xr_actionmap_item_modified(bContext *UNUSED(C),
+                                       void *UNUSED(ami_p),
+                                       void *UNUSED(unused))
+{
+}
+#endif
+
+void uiTemplateXrActionmapItemProperties(uiLayout *layout, PointerRNA *ptr)
+{
+#ifdef WITH_XR_OPENXR
+  PointerRNA propptr = RNA_pointer_get(ptr, "op_properties");
+
+  if (propptr.data) {
+    uiBut *but = static_cast<uiBut *>(uiLayoutGetBlock(layout)->buttons.last);
+
+    WM_operator_properties_sanitize(&propptr, false);
+    /* Use same template as keymap item properties. */
+    template_keymap_item_properties(layout, nullptr, &propptr);
+
+    for (; but; but = but->next) {
+      if (but->rnaprop) {
+        UI_but_func_set(but, xr_actionmap_item_modified, ptr->data, nullptr);
+        UI_but_flag_enable(but, UI_BUT_UPDATE_DELAY);
+      }
+    }
+  }
+#else
+  UNUSED_VARS(layout, ptr);
+#endif
+}
+
+/** \} */
+
 /* -------------------------------------------------------------------- */
 /** \name Event Icon Template
  * \{ */

source/blender/editors/space_node/drawnode.cc

@@ -432,6 +432,20 @@ static void node_shader_buts_tex_environment(uiLayout *layout, bContext *C, Poin
   node_buts_image_user(layout, C, &iuserptr, &imaptr, &iuserptr, false, true);
 }
 
+static void node_shader_buts_virtual_monitor(uiLayout *layout, bContext *C, PointerRNA *ptr)
+{
+  uiTemplateID(layout,
+               C,
+               ptr,
+               "camera",
+               nullptr,
+               nullptr,
+               nullptr,
+               UI_TEMPLATE_ID_FILTER_ALL,
+               false,
+               nullptr);
+}
+
 static void node_shader_buts_tex_environment_ex(uiLayout *layout, bContext *C, PointerRNA *ptr)
 {
   PointerRNA iuserptr = RNA_pointer_get(ptr, "image_user");
@@ -499,6 +513,9 @@ static void node_shader_set_butfunc(bNodeType *ntype)
       ntype->draw_buttons = node_shader_buts_tex_environment;
       ntype->draw_buttons_ex = node_shader_buts_tex_environment_ex;
       break;
+    case SH_NODE_VIRTUAL_MONITOR:
+      ntype->draw_buttons = node_shader_buts_virtual_monitor;
+      break;
     case SH_NODE_DISPLACEMENT:
     case SH_NODE_VECTOR_DISPLACEMENT:
       ntype->draw_buttons = node_shader_buts_displacement;

source/blender/editors/space_view3d/view3d_draw.cc

@@ -7,6 +7,7 @@
 
 #include <cmath>
 
+#include "BLI_float4x4.hh"
 #include "BLI_jitter_2d.h"
 #include "BLI_listbase.h"
 #include "BLI_math.h"
@@ -15,6 +16,7 @@
 #include "BLI_string.h"
 #include "BLI_string_utils.h"
 #include "BLI_threads.h"
+#include "BLI_vector.hh"
 
 #include "BKE_armature.h"
 #include "BKE_camera.h"
@@ -26,6 +28,7 @@
 #include "BKE_key.h"
 #include "BKE_layer.h"
 #include "BKE_main.h"
+#include "BKE_node.h"
 #include "BKE_object.h"
 #include "BKE_paint.h"
 #include "BKE_scene.h"
@@ -40,6 +43,7 @@
 #include "DNA_brush_types.h"
 #include "DNA_camera_types.h"
 #include "DNA_key_types.h"
+#include "DNA_material_types.h"
 #include "DNA_mesh_types.h"
 #include "DNA_object_types.h"
 #include "DNA_view3d_types.h"
@@ -61,6 +65,7 @@
 
 #include "GPU_batch.h"
 #include "GPU_batch_presets.h"
+#include "GPU_debug.h"
 #include "GPU_framebuffer.h"
 #include "GPU_immediate.h"
 #include "GPU_immediate_util.h"
@@ -87,6 +92,9 @@
 #include "view3d_intern.h" /* own include */
 
 using blender::float4;
+using blender::float4x4;
+using blender::int2;
+using blender::Vector;
 
 #define M_GOLDEN_RATIO_CONJUGATE 0.618033988749895f
 
@@ -1535,14 +1543,120 @@ void view3d_draw_region_info(const bContext *C, ARegion *region)
 /** \name Draw Viewport Contents
  * \{ */
 
+static void view3d_virtual_camera_update(
+    Scene *scene, Depsgraph *depsgraph, View3D *v3d, ARegion *region, Object *object)
+{
+  BLI_assert(object->type == OB_CAMERA);
+  int2 resolution(1920 / 2, 1080 / 2);
+
+  RegionView3D *old_rv3d = static_cast<RegionView3D *>(region->regiondata);
+  RegionView3D rv3d;
+  memcpy(&rv3d, old_rv3d, sizeof(RegionView3D));
+  region->regiondata = &rv3d;
+
+  Object *old_camera = v3d->camera;
+  v3d->camera = object;
+  rv3d.persp = RV3D_CAMOB;
+
+  Camera *camera = static_cast<Camera *>(object->data);
+  if (camera->runtime.virtual_monitor_offscreen == nullptr) {
+    camera->runtime.virtual_monitor_offscreen = GPU_offscreen_create(
+        UNPACK2(resolution), true, GPU_RGBA16F, nullptr);
+  }
+
+  float4x4 winmat;
+
+  // TODO: Multi view support?
+  CameraParams params;
+  BKE_camera_params_init(&params);
+  /* fallback for non camera objects */
+  params.clip_start = v3d->clip_start;
+  params.clip_end = v3d->clip_end;
+  BKE_camera_params_from_object(&params, object);
+  BKE_camera_params_compute_viewplane(&params, UNPACK2(resolution), scene->r.xasp, scene->r.yasp);
+  BKE_camera_params_compute_matrix(&params);
+  copy_m4_m4(winmat.ptr(), params.winmat);
+
+  GPUOffScreen *offscreen = camera->runtime.virtual_monitor_offscreen;
+
+  GPU_offscreen_bind(offscreen, true);
+  ED_view3d_draw_offscreen(depsgraph,
+                           scene,
+                           OB_MATERIAL,
+                           v3d,
+                           region,
+                           UNPACK2(resolution),
+                           nullptr,
+                           winmat.ptr(),
+                           false,
+                           true,
+                           nullptr,
+                           false,
+                           true,
+                           true,
+                           offscreen,
+                           nullptr);
+  GPU_offscreen_unbind(offscreen, true);
+  camera->runtime.offscreen_color_texture = GPU_offscreen_color_texture(
+      camera->runtime.virtual_monitor_offscreen);
+
+  v3d->camera = old_camera;
+  region->regiondata = old_rv3d;
+}
+
+static void view3d_draw_virtual_camera(Scene *scene,
+                                       Depsgraph *depsgraph,
+                                       View3D *v3d,
+                                       ARegion *region)
+{
+  /* TODO: Bad call! */
+  Main *bmain = DEG_get_bmain(depsgraph);
+
+  /* Collect all cameras in the scene that is used inside a virtual monitor. This should be
+   * optimized by a tagging system. There are far more materials then cameras in a typical scene.
+   */
+  Vector<Object *> virtual_cameras;
+  LISTBASE_FOREACH (Material *, material, &bmain->materials) {
+    if (!material->nodetree) {
+      continue;
+    }
+
+    LISTBASE_FOREACH (bNode *, node, &material->nodetree->nodes) {
+      if (node->type != SH_NODE_VIRTUAL_MONITOR) {
+        continue;
+      }
+      Object *ob = static_cast<Object *>(static_cast<void *>(node->id));
+      if (ob == nullptr || ob->type != OB_CAMERA) {
+        continue;
+      }
+      virtual_cameras.append(ob);
+    }
+  }
+
+  if (virtual_cameras.is_empty()) {
+    /* No cameras used as virtual monitor, so skip updating. */
+    return;
+  }
+
+  GPU_debug_group_begin("VirtualCameras");
+  for (Object *object : virtual_cameras) {
+    view3d_virtual_camera_update(scene, depsgraph, v3d, region, object);
+  }
+  GPU_debug_group_end();
+}
+
 static void view3d_draw_view(const bContext *C, ARegion *region)
 {
+  Scene *scene = CTX_data_scene(C);
+  Depsgraph *depsgraph = CTX_data_expect_evaluated_depsgraph(C);
+  View3D *v3d = CTX_wm_view3d(C);
+  view3d_draw_virtual_camera(scene, depsgraph, v3d, region);
   ED_view3d_draw_setup_view(CTX_wm_manager(C),
                             CTX_wm_window(C),
-                            CTX_data_expect_evaluated_depsgraph(C),
-                            CTX_data_scene(C),
+                            depsgraph,
+                            scene,
                             region,
-                            CTX_wm_view3d(C),
+                            v3d,
                             nullptr,
                             nullptr,
                             nullptr);
@@ -1640,6 +1754,7 @@ void ED_view3d_draw_offscreen(Depsgraph *depsgraph,
                               const char *viewname,
                               const bool do_color_management,
                               const bool restore_rv3d_mats,
+                              const bool is_virtual_camera,
                               GPUOffScreen *ofs,
                               GPUViewport *viewport)
 {
@@ -1715,6 +1830,7 @@ void ED_view3d_draw_offscreen(Depsgraph *depsgraph,
                                  is_image_render,
                                  draw_background,
                                  do_color_management,
+                                 is_virtual_camera,
                                  ofs,
                                  viewport);
   DRW_cache_free_old_subdiv();
@@ -1830,6 +1946,8 @@ void ED_view3d_draw_offscreen_simple(Depsgraph *depsgraph,
   /* Actually not used since we pass in the projection matrix. */
   v3d.lens = 0;
 
+  view3d_draw_virtual_camera(scene, depsgraph, &v3d, &ar);
+
   ED_view3d_draw_offscreen(depsgraph,
                            scene,
                            drawtype,
@@ -1844,6 +1962,7 @@ void ED_view3d_draw_offscreen_simple(Depsgraph *depsgraph,
                            viewname,
                            do_color_management,
                            true,
+                           false,
                            ofs,
                            viewport);
 }
@@ -1967,6 +2086,7 @@ ImBuf *ED_view3d_draw_offscreen_imbuf(Depsgraph *depsgraph,
                            viewname,
                            do_color_management,
                            restore_rv3d_mats,
+                           false,
                            ofs,
                            nullptr);
 

source/blender/editors/space_view3d/view3d_select.cc

@@ -136,6 +136,21 @@ void ED_view3d_viewcontext_init_object(ViewContext *vc, Object *obact)
   }
 }
 
+bool ED_view3d_object_deselect_all_except(const Scene *scene, ViewLayer *view_layer, Base *b)
+{
+  bool changed = false;
+  BKE_view_layer_synced_ensure(scene, view_layer);
+  LISTBASE_FOREACH (Base *, base, BKE_view_layer_object_bases_get(view_layer)) {
+    if (base->flag & BASE_SELECTED) {
+      if (b != base) {
+        ED_object_base_select(base, BA_DESELECT);
+        changed = true;
+      }
+    }
+  }
+  return changed;
+}
+
 /** \} */
 
 /* -------------------------------------------------------------------- */
@@ -157,22 +172,6 @@ static bool object_deselect_all_visible(const Scene *scene, ViewLayer *view_laye
   return changed;
 }
 
-/* deselect all except b */
-static bool object_deselect_all_except(const Scene *scene, ViewLayer *view_layer, Base *b)
-{
-  bool changed = false;
-  BKE_view_layer_synced_ensure(scene, view_layer);
-  LISTBASE_FOREACH (Base *, base, BKE_view_layer_object_bases_get(view_layer)) {
-    if (base->flag & BASE_SELECTED) {
-      if (b != base) {
-        ED_object_base_select(base, BA_DESELECT);
-        changed = true;
-      }
-    }
-  }
-  return changed;
-}
-
 /** \} */
 
 /* -------------------------------------------------------------------- */
@@ -1529,7 +1528,7 @@ static int object_select_menu_exec(bContext *C, wmOperator *op)
     }
   }
   else {
-    object_deselect_all_except(scene, view_layer, basact);
+    ED_view3d_object_deselect_all_except(scene, view_layer, basact);
     ED_object_base_select(basact, BA_SELECT);
     changed = true;
   }
@@ -2797,7 +2796,7 @@ static bool ed_object_select_pick(bContext *C,
       else if (found || params->deselect_all) {
         /* Deselect everything. */
         /* `basact` may be nullptr. */
-        if (object_deselect_all_except(scene, view_layer, basact)) {
+        if (ED_view3d_object_deselect_all_except(scene, view_layer, basact)) {
           changed_object = true;
         }
       }
@@ -2809,7 +2808,7 @@ static bool ed_object_select_pick(bContext *C,
     if (vc.obedit) {
       /* Only do the select (use for setting vertex parents & hooks).
        * In edit-mode do not activate. */
-      object_deselect_all_except(scene, view_layer, basact);
+      ED_view3d_object_deselect_all_except(scene, view_layer, basact);
       ED_object_base_select(basact, BA_SELECT);
 
       changed_object = true;
@@ -2840,7 +2839,7 @@ static bool ed_object_select_pick(bContext *C,
           break;
         }
         case SEL_OP_SET: {
-          object_deselect_all_except(scene, view_layer, basact);
+          ED_view3d_object_deselect_all_except(scene, view_layer, basact);
           ED_object_base_select(basact, BA_SELECT);
           break;
         }

source/blender/editors/transform/transform_convert.c

@@ -1157,10 +1157,8 @@ void transform_convert_clip_mirror_modifier_apply(TransDataContainer *tc)
   }
 }
 
-void animrecord_check_state(TransInfo *t, struct ID *id)
+void ED_transform_animrecord_check_state(Scene *scene, wmTimer *animtimer, struct ID *id)
 {
-  Scene *scene = t->scene;
-  wmTimer *animtimer = t->animtimer;
   ScreenAnimData *sad = (animtimer) ? animtimer->customdata : NULL;
 
   /* sanity checks */

source/blender/editors/transform/transform_convert.h

@@ -105,10 +105,6 @@ char transform_convert_frame_side_dir_get(TransInfo *t, float cframe);
  */
 bool FrameOnMouseSide(char side, float frame, float cframe);
 void transform_convert_clip_mirror_modifier_apply(TransDataContainer *tc);
-/**
- * For the realtime animation recording feature, handle overlapping data.
- */
-void animrecord_check_state(TransInfo *t, struct ID *id);
 
 /* transform_convert_action.c */
 

source/blender/editors/transform/transform_convert_armature.c

@@ -1472,7 +1472,7 @@ static void recalcData_pose(TransInfo *t)
         /* XXX: this currently doesn't work, since flags aren't set yet! */
         int targetless_ik = (t->flag & T_AUTOIK);
 
-        animrecord_check_state(t, &ob->id);
+        ED_transform_animrecord_check_state(t->scene, t->animtimer, &ob->id);
         autokeyframe_pose(t->context, t->scene, ob, t->mode, targetless_ik);
       }
 

source/blender/editors/transform/transform_convert_object.c

@@ -729,7 +729,7 @@ static void createTransObject(bContext *C, TransInfo *t)
  * \note Context may not always be available,
  * so must check before using it as it's a luxury for a few cases.
  */
-static void autokeyframe_object(
+void ED_transform_autokeyframe_object(
     bContext *C, Scene *scene, ViewLayer *view_layer, Object *ob, int tmode)
 {
   Main *bmain = CTX_data_main(C);
@@ -853,7 +853,7 @@ static void autokeyframe_object(
 
 /* Return if we need to update motion paths, only if they already exist,
  * and we will insert a keyframe at the end of transform. */
-static bool motionpath_need_update_object(Scene *scene, Object *ob)
+bool ED_transform_motionpath_need_update_object(Scene *scene, Object *ob)
 {
   /* XXX: there's potential here for problems with unkeyed rotations/scale,
    *      but for now (until proper data-locality for baking operations),
@@ -897,11 +897,11 @@ static void recalcData_objects(TransInfo *t)
       /* TODO: autokeyframe calls need some setting to specify to add samples
        * (FPoints) instead of keyframes? */
       if ((t->animtimer) && IS_AUTOKEY_ON(t->scene)) {
-        animrecord_check_state(t, &ob->id);
-        autokeyframe_object(t->context, t->scene, t->view_layer, ob, t->mode);
+        ED_transform_animrecord_check_state(t->scene, t->animtimer, &ob->id);
+        ED_transform_autokeyframe_object(t->context, t->scene, t->view_layer, ob, t->mode);
       }
 
-      motionpath_update |= motionpath_need_update_object(t->scene, ob);
+      motionpath_update |= ED_transform_motionpath_need_update_object(t->scene, ob);
 
       /* sets recalc flags fully, instead of flushing existing ones
        * otherwise proxies don't function correctly
@@ -974,10 +974,10 @@ static void special_aftertrans_update__object(bContext *C, TransInfo *t)
 
     /* Set auto-key if necessary. */
     if (!canceled) {
-      autokeyframe_object(C, t->scene, t->view_layer, ob, t->mode);
+      ED_transform_autokeyframe_object(C, t->scene, t->view_layer, ob, t->mode);
     }
 
-    motionpath_update |= motionpath_need_update_object(t->scene, ob);
+    motionpath_update |= ED_transform_motionpath_need_update_object(t->scene, ob);
 
     /* Restore rigid body transform. */
     if (ob->rigidbody_object && canceled) {

source/blender/editors/transform/transform_convert_sequencer_image.c

@@ -239,7 +239,7 @@ static void recalcData_sequencer_image(TransInfo *t)
     }
 
     if ((t->animtimer) && IS_AUTOKEY_ON(t->scene)) {
-      animrecord_check_state(t, &t->scene->id);
+      ED_transform_animrecord_check_state(t->scene, t->animtimer, &t->scene->id);
       autokeyframe_sequencer_image(t->context, t->scene, transform, t->mode);
     }
 

source/blender/gpu/CMakeLists.txt

@@ -471,6 +471,7 @@ set(GLSL_SRC
   shaders/material/gpu_shader_material_vector_rotate.glsl
   shaders/material/gpu_shader_material_velvet.glsl
   shaders/material/gpu_shader_material_vertex_color.glsl
+  shaders/material/gpu_shader_material_virtual_camera.glsl
   shaders/material/gpu_shader_material_volume_absorption.glsl
   shaders/material/gpu_shader_material_volume_principled.glsl
   shaders/material/gpu_shader_material_volume_scatter.glsl

source/blender/gpu/GPU_material.h

@@ -35,6 +35,7 @@ struct Material;
 struct Scene;
 struct bNode;
 struct bNodeTree;
+struct Camera;
 
 typedef struct GPUMaterial GPUMaterial;
 typedef struct GPUNode GPUNode;
@@ -82,6 +83,8 @@ typedef enum eGPUMaterialFlag {
   GPU_MATFLAG_OBJECT_INFO = (1 << 10),
   GPU_MATFLAG_AOV = (1 << 11),
 
+  GPU_MATFLAG_VIRTUAL_MONITOR = (1 << 12),
+
   GPU_MATFLAG_BARYCENTRIC = (1 << 20),
 
   /* Optimization to only add the branches of the principled shader that are necessary. */
@@ -178,6 +181,9 @@ GPUNodeLink *GPU_image_sky(GPUMaterial *mat,
                            const float *pixels,
                            float *layer,
                            eGPUSamplerState sampler_state);
+GPUNodeLink *GPU_image_camera(GPUMaterial *mat,
+                              struct Camera *camera,
+                              eGPUSamplerState sampler_state);
 GPUNodeLink *GPU_color_band(GPUMaterial *mat, int size, float *pixels, float *row);
 
 /**
@@ -322,6 +328,7 @@ typedef struct GPUMaterialTexture {
   bool iuser_available;
   struct GPUTexture **colorband;
   struct GPUTexture **sky;
+  struct GPUTexture **camera;
   char sampler_name[32];       /* Name of sampler in GLSL. */
   char tiled_mapping_name[32]; /* Name of tile mapping sampler in GLSL. */
   int users;

source/blender/gpu/intern/gpu_codegen.cc

@@ -400,6 +400,10 @@ void GPUCodegen::generate_resources()
       const char *name = info.name_buffer.append_sampler_name(tex->sampler_name);
       info.sampler(0, ImageType::FLOAT_2D_ARRAY, name, Frequency::BATCH);
     }
+    else if (tex->camera && tex->camera[0]) {
+      const char *name = info.name_buffer.append_sampler_name(tex->sampler_name);
+      info.sampler(slot++, ImageType::FLOAT_2D, name, Frequency::BATCH);
+    }
     else if (tex->tiled_mapping_name[0] != '\0') {
       const char *name = info.name_buffer.append_sampler_name(tex->sampler_name);
       info.sampler(slot++, ImageType::FLOAT_2D_ARRAY, name, Frequency::BATCH);

source/blender/gpu/intern/gpu_node_graph.cc

@@ -12,6 +12,7 @@
 
 #include "MEM_guardedalloc.h"
 
+#include "DNA_camera_types.h"
 #include "DNA_node_types.h"
 
 #include "BLI_ghash.h"
@@ -19,6 +20,7 @@
 #include "BLI_string.h"
 #include "BLI_utildefines.h"
 
+#include "GPU_framebuffer.h"
 #include "GPU_texture.h"
 #include "GPU_vertex_format.h"
 
@@ -114,6 +116,7 @@ static void gpu_node_input_link(GPUNode *node, GPUNodeLink *link, const eGPUType
       link->users++;
       break;
     case GPU_NODE_LINK_IMAGE:
+    case GPU_NODE_LINK_IMAGE_CAMERA:
     case GPU_NODE_LINK_IMAGE_TILED:
     case GPU_NODE_LINK_IMAGE_SKY:
     case GPU_NODE_LINK_COLORBAND:
@@ -476,6 +479,7 @@ static GPUMaterialTexture *gpu_node_graph_add_texture(GPUNodeGraph *graph,
                                                       ImageUser *iuser,
                                                       struct GPUTexture **colorband,
                                                       struct GPUTexture **sky,
+                                                      struct GPUTexture **camera,
                                                       GPUNodeLinkType link_type,
                                                       eGPUSamplerState sampler_state)
 {
@@ -484,7 +488,7 @@ static GPUMaterialTexture *gpu_node_graph_add_texture(GPUNodeGraph *graph,
   GPUMaterialTexture *tex = static_cast<GPUMaterialTexture *>(graph->textures.first);
   for (; tex; tex = tex->next) {
     if (tex->ima == ima && tex->colorband == colorband && tex->sky == sky &&
-        tex->sampler_state == sampler_state) {
+        tex->camera == camera && tex->sampler_state == sampler_state) {
       break;
     }
     num_textures++;
@@ -500,6 +504,7 @@ static GPUMaterialTexture *gpu_node_graph_add_texture(GPUNodeGraph *graph,
     }
     tex->colorband = colorband;
     tex->sky = sky;
+    tex->camera = camera;
     tex->sampler_state = sampler_state;
     BLI_snprintf(tex->sampler_name, sizeof(tex->sampler_name), "samp%d", num_textures);
     if (ELEM(link_type, GPU_NODE_LINK_IMAGE_TILED, GPU_NODE_LINK_IMAGE_TILED_MAPPING)) {
@@ -631,7 +636,24 @@ GPUNodeLink *GPU_image(GPUMaterial *mat,
   GPUNodeLink *link = gpu_node_link_create();
   link->link_type = GPU_NODE_LINK_IMAGE;
   link->texture = gpu_node_graph_add_texture(
-      graph, ima, iuser, nullptr, nullptr, link->link_type, sampler_state);
+      graph, ima, iuser, nullptr, nullptr, nullptr, link->link_type, sampler_state);
+  return link;
+}
+
+GPUNodeLink *GPU_image_camera(GPUMaterial *mat, Camera *camera, eGPUSamplerState sampler_state)
+{
+  GPUNodeGraph *graph = gpu_material_node_graph(mat);
+  GPUNodeLink *link = gpu_node_link_create();
+  link->link_type = GPU_NODE_LINK_IMAGE_CAMERA;
+  link->texture = gpu_node_graph_add_texture(graph,
+                                             nullptr,
+                                             nullptr,
+                                             nullptr,
+                                             nullptr,
+                                             &camera->runtime.offscreen_color_texture,
+                                             link->link_type,
+                                             sampler_state);
+  GPU_material_flag_set(mat, GPU_MATFLAG_VIRTUAL_MONITOR);
   return link;
 }
 
@@ -648,7 +670,7 @@ GPUNodeLink *GPU_image_sky(GPUMaterial *mat,
   GPUNodeLink *link = gpu_node_link_create();
   link->link_type = GPU_NODE_LINK_IMAGE_SKY;
   link->texture = gpu_node_graph_add_texture(
-      graph, nullptr, nullptr, nullptr, sky, link->link_type, sampler_state);
+      graph, nullptr, nullptr, nullptr, sky, nullptr, link->link_type, sampler_state);
   return link;
 }
 
@@ -661,7 +683,7 @@ GPUNodeLink *GPU_image_tiled(GPUMaterial *mat,
   GPUNodeLink *link = gpu_node_link_create();
   link->link_type = GPU_NODE_LINK_IMAGE_TILED;
   link->texture = gpu_node_graph_add_texture(
-      graph, ima, iuser, nullptr, nullptr, link->link_type, sampler_state);
+      graph, ima, iuser, nullptr, nullptr, nullptr, link->link_type, sampler_state);
   return link;
 }
 
@@ -671,7 +693,7 @@ GPUNodeLink *GPU_image_tiled_mapping(GPUMaterial *mat, Image *ima, ImageUser *iu
   GPUNodeLink *link = gpu_node_link_create();
   link->link_type = GPU_NODE_LINK_IMAGE_TILED_MAPPING;
   link->texture = gpu_node_graph_add_texture(
-      graph, ima, iuser, nullptr, nullptr, link->link_type, GPU_SAMPLER_MAX);
+      graph, ima, iuser, nullptr, nullptr, nullptr, link->link_type, GPU_SAMPLER_MAX);
   return link;
 }
 
@@ -684,7 +706,7 @@ GPUNodeLink *GPU_color_band(GPUMaterial *mat, int size, float *pixels, float *ro
   GPUNodeLink *link = gpu_node_link_create();
   link->link_type = GPU_NODE_LINK_COLORBAND;
   link->texture = gpu_node_graph_add_texture(
-      graph, nullptr, nullptr, colorband, nullptr, link->link_type, GPU_SAMPLER_MAX);
+      graph, nullptr, nullptr, colorband, nullptr, nullptr, link->link_type, GPU_SAMPLER_MAX);
   return link;
 }
 

source/blender/gpu/intern/gpu_node_graph.h

@@ -50,6 +50,7 @@ typedef enum {
   GPU_NODE_LINK_IMAGE_TILED,
   GPU_NODE_LINK_IMAGE_TILED_MAPPING,
   GPU_NODE_LINK_IMAGE_SKY,
+  GPU_NODE_LINK_IMAGE_CAMERA,
   GPU_NODE_LINK_OUTPUT,
   GPU_NODE_LINK_UNIFORM,
   GPU_NODE_LINK_DIFFERENTIATE_FLOAT_FN,

source/blender/gpu/shaders/material/gpu_shader_material_virtual_camera.glsl

@@ -0,0 +1,11 @@
+void node_virtual_camera_empty(vec3 co, out vec4 color, out float alpha)
+{
+  color = vec4(0.0);
+  alpha = 0.0;
+}
+
+void node_virtual_camera(vec3 co, sampler2D ima, out vec4 color, out float alpha)
+{
+  color = texture(ima, co.xy);
+  alpha = color.a;
+}

source/blender/makesdna/DNA_camera_types.h

@@ -71,6 +71,10 @@ typedef struct Camera_Runtime {
   float drw_depth[2];
   float drw_focusmat[4][4];
   float drw_normalmat[4][4];
+
+  struct GPUOffScreen *virtual_monitor_offscreen;
+  /* Local reference to not owning gpu texture. Used to have a reliable pointer to the texture. */
+  struct GPUTexture *offscreen_color_texture;
 } Camera_Runtime;
 
 typedef struct Camera {

source/blender/makesdna/DNA_xr_types.h

@@ -30,7 +30,9 @@ typedef struct XrSessionSettings {
   char draw_flags;
   /** Draw style for controller visualization. */
   char controller_draw_style;
-  char _pad2[2];
+  /** The eye (view) used for 3D->2D projection when simulating mouse. */
+  char projection_eye;
+  char _pad2;
 
   /** Clipping distance. */
   float clip_start, clip_end;
@@ -40,11 +42,22 @@ typedef struct XrSessionSettings {
   /** Object type settings to apply to VR view (unlike shading, not shared with window 3D-View). */
   int object_type_exclude_viewport;
   int object_type_exclude_select;
+
+  ListBase actionmaps; /* XrActionMap */
+  short act_actionmap;
+  short sel_actionmap;
+  char _pad3[2];
+
+  /** Objects to bind to headset/controller poses. */
+  short sel_mocap_object;
+  ListBase mocap_objects; /* #XrMotionCaptureObject */
 } XrSessionSettings;
 
 typedef enum eXrSessionFlag {
   XR_SESSION_USE_POSITION_TRACKING = (1 << 0),
   XR_SESSION_USE_ABSOLUTE_TRACKING = (1 << 1),
+  XR_SESSION_ENABLE_VIVE_TRACKER_EXTENSION =
+      (1 << 2), /* See #GHOST_kXrContextEnableViveTrackerExtension. */
 } eXrSessionFlag;
 
 typedef enum eXRSessionBasePoseType {
@@ -60,6 +73,11 @@ typedef enum eXrSessionControllerDrawStyle {
   XR_CONTROLLER_DRAW_LIGHT_RAY = 3,
 } eXrSessionControllerDrawStyle;
 
+typedef enum eXrSessionEye {
+  XR_EYE_LEFT = 0,
+  XR_EYE_RIGHT = 1,
+} eXrSessionEye;
+
 /** XR action type. Enum values match those in GHOST_XrActionType enum for consistency. */
 typedef enum eXrActionType {
   XR_BOOLEAN_INPUT = 1,
@@ -79,6 +97,8 @@ typedef enum eXrOpFlag {
 typedef enum eXrActionFlag {
   /** Action depends on two sub-action paths (i.e. two-handed/bi-manual action). */
   XR_ACTION_BIMANUAL = (1 << 0),
+  /** Whether to use controller inputs to simulate mouse inputs. */
+  XR_ACTION_SIMULATE_MOUSE = (1 << 1),
 } eXrActionFlag;
 
 typedef enum eXrHapticFlag {
@@ -108,8 +128,15 @@ typedef enum eXrPoseFlag {
   /* Pose represents controller grip/aim. */
   XR_POSE_GRIP = (1 << 0),
   XR_POSE_AIM = (1 << 1),
+  /* Pose represents VR tracker. */
+  XR_POSE_TRACKER = (1 << 2),
 } eXrPoseFlag;
 
+typedef enum eXrMotionCaptureFlag {
+  XR_MOCAP_OBJECT_ENABLE = (1 << 0),
+  XR_MOCAP_OBJECT_AUTOKEY = (1 << 1),
+} eXrMotionCaptureFlag;
+
 /**
  * The following user and component path lengths are dependent on OpenXR's XR_MAX_PATH_LENGTH
  * (256). A user path will be combined with a component path to identify an action binding, and
@@ -141,7 +168,8 @@ typedef struct XrActionMapBinding {
   /** Input threshold/region. */
   float float_threshold;
   short axis_flag; /* eXrAxisFlag */
-  char _pad[2];
+
+  short sel_component_path;
 
   /** Pose action properties. */
   float pose_location[3];
@@ -155,6 +183,16 @@ typedef struct XrUserPath {
   char path[64]; /* XR_MAX_USER_PATH_LENGTH */
 } XrUserPath;
 
+typedef struct XrSimulateMouseParams {
+  short press_type; /* wmEvent.type */
+  short press_val;  /* wmEvent.val */
+  short hold_type;
+  short hold_val;
+  short release_type;
+  short release_val;
+  short _pad[2];
+} XrSimulateMouseParams;
+
 typedef struct XrActionMapItem {
   struct XrActionMapItem *next, *prev;
 
@@ -174,6 +212,9 @@ typedef struct XrActionMapItem {
   /** RNA pointer to access properties. */
   struct PointerRNA *op_properties_ptr;
 
+  /** Mouse simulation. */
+  XrSimulateMouseParams simulate;
+
   short op_flag;     /* eXrOpFlag */
   short action_flag; /* eXrActionFlag */
   short haptic_flag; /* eXrHapticFlag */
@@ -187,8 +228,8 @@ typedef struct XrActionMapItem {
   float haptic_frequency;
   float haptic_amplitude;
 
-  short selbinding;
-  char _pad3[2];
+  short sel_user_path;
+  short sel_binding;
   ListBase bindings; /* XrActionMapBinding */
 } XrActionMapItem;
 
@@ -201,12 +242,30 @@ typedef struct XrActionMap {
   char name[64]; /* MAX_NAME */
 
   ListBase items; /* XrActionMapItem */
-  short selitem;
+  short sel_item;
   char _pad[6];
 } XrActionMap;
 
 /* -------------------------------------------------------------------- */
 
+typedef struct XrMotionCaptureObject {
+  struct XrMotionCaptureObject *next, *prev;
+
+  /** Object to bind to a VR device. Used as struct identifier. */
+  Object *ob;
+  /** OpenXR user path, identifies the target headset/controller. */
+  char user_path[64]; /* XR_MAX_USER_PATH_LENGTH */
+
+  /** Location/rotation offsets. */
+  float location_offset[3];
+  float rotation_offset[3];
+
+  short flag; /* eXrMotionCaptureFlag */
+  char _pad[6];
+} XrMotionCaptureObject;
+
+/* -------------------------------------------------------------------- */
+
 #ifdef __cplusplus
 }
 #endif

source/blender/makesrna/intern/rna_nodetree.c

@@ -5857,6 +5857,22 @@ static void def_sh_tex_coord(StructRNA *srna)
   RNA_def_property_update(prop, NC_NODE | NA_EDITED, "rna_Node_update");
 }
 
+static void def_sh_virtual_monitor(StructRNA *srna)
+{
+  PropertyRNA *prop;
+
+  prop = RNA_def_property(srna, "camera", PROP_POINTER, PROP_NONE);
+  RNA_def_property_flag(prop, PROP_EDITABLE);
+  RNA_def_property_override_flag(prop, PROPOVERRIDE_OVERRIDABLE_LIBRARY);
+  RNA_def_property_pointer_sdna(prop, NULL, "id");
+  RNA_def_property_pointer_funcs(prop, NULL, NULL, NULL, "rna_Camera_object_poll");
+  RNA_def_property_struct_type(prop, "Object");
+  RNA_def_property_ui_text(prop, "Camera", "");
+  RNA_def_property_update(prop, NC_NODE | NA_EDITED, "rna_Node_update");
+  /* TODO: add poll function show all cameras in active scene and view layer, excluding scene
+   * camera. */
+}
+
 static void def_sh_vect_transform(StructRNA *srna)
 {
   static const EnumPropertyItem prop_vect_type_items[] = {

source/blender/makesrna/intern/rna_ui_api.c

@@ -1797,6 +1797,11 @@ void RNA_api_ui_layout(StructRNA *srna)
   parm = RNA_def_pointer(func, "item", "KeyMapItem", "", "");
   RNA_def_parameter_flags(parm, PROP_NEVER_NULL, PARM_REQUIRED | PARM_RNAPTR);
 
+  func = RNA_def_function(
+      srna, "template_xr_actionmap_item_properties", "uiTemplateXrActionmapItemProperties");
+  parm = RNA_def_pointer(func, "item", "XrActionMapItem", "", "");
+  RNA_def_parameter_flags(parm, PROP_NEVER_NULL, PARM_REQUIRED | PARM_RNAPTR);
+
   func = RNA_def_function(srna, "template_component_menu", "uiTemplateComponentMenu");
   RNA_def_function_ui_description(func, "Item. Display expanded property in a popup menu");
   parm = RNA_def_pointer(func, "data", "AnyType", "", "Data from which to take property");

source/blender/makesrna/intern/rna_wm.c

@@ -2836,6 +2836,53 @@ static void rna_def_keyconfig(BlenderRNA *brna)
   RNA_api_keymapitem(srna);
 }
 
+#  ifdef WITH_XR_OPENXR
+/* Defined here instead of rna_xr.c to access event type/value enums. */
+static void rna_def_xr_simulate_mouse_params(BlenderRNA *brna)
+{
+  StructRNA *srna;
+  PropertyRNA *prop;
+
+  srna = RNA_def_struct(brna, "XrSimulateMouseParams", NULL);
+  RNA_def_struct_ui_text(srna,
+                         "XR Mouse Simulation Parameters",
+                         "Parameters for simulating mouse with VR controllers");
+
+  prop = RNA_def_property(srna, "press_type", PROP_ENUM, PROP_NONE);
+  RNA_def_property_enum_sdna(prop, NULL, "press_type");
+  RNA_def_property_enum_items(prop, rna_enum_event_type_items);
+  RNA_def_property_translation_context(prop, BLT_I18NCONTEXT_UI_EVENTS);
+  RNA_def_property_ui_text(prop, "Press Type", "Event type to simulate on VR action press");
+
+  prop = RNA_def_property(srna, "press_value", PROP_ENUM, PROP_NONE);
+  RNA_def_property_enum_sdna(prop, NULL, "press_val");
+  RNA_def_property_enum_items(prop, rna_enum_event_value_items);
+  RNA_def_property_ui_text(prop, "Press Value", "Event value to simulate on VR action press");
+
+  prop = RNA_def_property(srna, "hold_type", PROP_ENUM, PROP_NONE);
+  RNA_def_property_enum_sdna(prop, NULL, "hold_type");
+  RNA_def_property_enum_items(prop, rna_enum_event_type_items);
+  RNA_def_property_translation_context(prop, BLT_I18NCONTEXT_UI_EVENTS);
+  RNA_def_property_ui_text(prop, "Hold Type", "Event type to simulate on VR action hold");
+
+  prop = RNA_def_property(srna, "hold_value", PROP_ENUM, PROP_NONE);
+  RNA_def_property_enum_sdna(prop, NULL, "hold_val");
+  RNA_def_property_enum_items(prop, rna_enum_event_value_items);
+  RNA_def_property_ui_text(prop, "Hold Value", "Event value to simulate on VR action hold");
+
+  prop = RNA_def_property(srna, "release_type", PROP_ENUM, PROP_NONE);
+  RNA_def_property_enum_sdna(prop, NULL, "release_type");
+  RNA_def_property_enum_items(prop, rna_enum_event_type_items);
+  RNA_def_property_translation_context(prop, BLT_I18NCONTEXT_UI_EVENTS);
+  RNA_def_property_ui_text(prop, "Release Type", "Event type to simulate on VR action release");
+
+  prop = RNA_def_property(srna, "release_value", PROP_ENUM, PROP_NONE);
+  RNA_def_property_enum_sdna(prop, NULL, "release_val");
+  RNA_def_property_enum_items(prop, rna_enum_event_value_items);
+  RNA_def_property_ui_text(prop, "Release Value", "Event value to simulate on VR action release");
+}
+#  endif
+
 void RNA_def_wm(BlenderRNA *brna)
 {
   rna_def_operator(brna);
@@ -2853,6 +2900,9 @@ void RNA_def_wm(BlenderRNA *brna)
   rna_def_windowmanager(brna);
   rna_def_keyconfig_prefs(brna);
   rna_def_keyconfig(brna);
+#  ifdef WITH_XR_OPENXR
+  rna_def_xr_simulate_mouse_params(brna);
+#  endif
 }
 
 #endif /* RNA_RUNTIME */

source/blender/nodes/NOD_shader.h

@@ -15,6 +15,8 @@ extern "C" {
 
 extern struct bNodeTreeType *ntreeType_Shader;
 
+void register_node_type_sh_virtual_camera(void);
+
 void register_node_type_sh_custom_group(bNodeType *ntype);
 
 struct bNodeTreeExec *ntreeShaderBeginExecTree(struct bNodeTree *ntree);

source/blender/nodes/NOD_static_types.h

@@ -123,6 +123,7 @@ DefNode(ShaderNode,     SH_NODE_CURVE_FLOAT,        def_float_curve,        "CUR
 DefNode(ShaderNode,     SH_NODE_COMBINE_COLOR,      def_sh_combsep_color,   "COMBINE_COLOR",      CombineColor,     "Combine Color",     "Create a color from individual components using multiple models")
 DefNode(ShaderNode,     SH_NODE_SEPARATE_COLOR,     def_sh_combsep_color,   "SEPARATE_COLOR",     SeparateColor,    "Separate Color",    "Split a color into its individual components using multiple models")
 DefNode(ShaderNode,     SH_NODE_MIX,                def_sh_mix,             "MIX",                Mix,              "Mix",               "Mix values by a factor")
+DefNode(ShaderNode,     SH_NODE_VIRTUAL_MONITOR,    def_sh_virtual_monitor, "VIRTUAL_MONITOR",    VirtualMonitor,   "Virtual Monitor",   "Use the feed from a virtual camera (has to be in the same scene and view layer)")
 
 DefNode(CompositorNode, CMP_NODE_VIEWER,         def_cmp_viewer,         "VIEWER",         Viewer,           "Viewer",            ""              )
 DefNode(CompositorNode, CMP_NODE_RGB,            0,                      "RGB",            RGB,              "RGB",               ""              )

source/blender/nodes/shader/CMakeLists.txt

@@ -113,6 +113,7 @@ set(SRC
   nodes/node_shader_vector_rotate.cc
   nodes/node_shader_vector_transform.cc
   nodes/node_shader_vertex_color.cc
+  nodes/node_shader_virtual_monitor.cc
   nodes/node_shader_volume_absorption.cc
   nodes/node_shader_volume_info.cc
   nodes/node_shader_volume_principled.cc

source/blender/nodes/shader/node_shader_register.cc

@@ -103,6 +103,7 @@ void register_shader_nodes()
   register_node_type_sh_vector_displacement();
   register_node_type_sh_vector_rotate();
   register_node_type_sh_vertex_color();
+  register_node_type_sh_virtual_camera();
   register_node_type_sh_volume_absorption();
   register_node_type_sh_volume_info();
   register_node_type_sh_volume_principled();

source/blender/nodes/shader/node_shader_register.hh

@@ -102,6 +102,7 @@ void register_node_type_sh_vect_transform();
 void register_node_type_sh_vector_displacement();
 void register_node_type_sh_vector_rotate();
 void register_node_type_sh_vertex_color();
+void register_node_type_sh_virtual_camera();
 void register_node_type_sh_volume_absorption();
 void register_node_type_sh_volume_info();
 void register_node_type_sh_volume_principled();

source/blender/nodes/shader/nodes/node_shader_virtual_monitor.cc

@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later
+ * Copyright 2005 Blender Foundation. All rights reserved. */
+
+#include "node_shader_util.hh"
+
+#include "DNA_camera_types.h"
+
+#include "DEG_depsgraph_query.h"
+
+namespace blender::nodes::node_shader_virtual_monitor_cc {
+
+static void sh_node_virtual_monitor_declare(NodeDeclarationBuilder &b)
+{
+  b.add_input<decl::Vector>(N_("Vector")).implicit_field(implicit_field_inputs::position);
+  b.add_output<decl::Color>(N_("Color")).no_muted_links();
+  b.add_output<decl::Float>(N_("Alpha")).no_muted_links();
+}
+
+static int node_shader_gpu_virtual_monitor(GPUMaterial *mat,
+                                           bNode *node,
+                                           bNodeExecData * /*execdata*/,
+                                           GPUNodeStack *in,
+                                           GPUNodeStack *out)
+{
+  Object *object = (Object *)node->id;
+  if (object == nullptr || object->type != OB_CAMERA) {
+    return GPU_stack_link(mat, node, "node_virtual_camera_empty", in, out);
+  }
+
+  Object *orig_object = DEG_get_original_object(object);
+
+  Camera *cam = static_cast<Camera *>(orig_object->data);
+  node_shader_gpu_default_tex_coord(mat, node, &in[0].link);
+
+  return GPU_stack_link(
+      mat, node, "node_virtual_camera", in, out, GPU_image_camera(mat, cam, GPU_SAMPLER_DEFAULT));
+}
+
+}  // namespace blender::nodes::node_shader_virtual_monitor_cc
+
+void register_node_type_sh_virtual_camera()
+{
+  namespace file_ns = blender::nodes::node_shader_virtual_monitor_cc;
+
+  static bNodeType ntype;
+
+  sh_node_type_base(&ntype, SH_NODE_VIRTUAL_MONITOR, "Virtual Monitor", NODE_CLASS_INPUT);
+  ntype.declare = file_ns::sh_node_virtual_monitor_declare;
+  ntype.gpu_fn = file_ns::node_shader_gpu_virtual_monitor;
+
+  nodeRegisterType(&ntype);
+}

source/blender/python/gpu/gpu_py_offscreen.c

@@ -396,6 +396,7 @@ static PyObject *pygpu_offscreen_draw_view3d(BPyGPUOffScreen *self, PyObject *ar
                            "",
                            do_color_management,
                            true,
+                           false,
                            self->ofs,
                            self->viewport);
 

source/blender/windowmanager/CMakeLists.txt

@@ -188,6 +188,7 @@ if(WITH_XR_OPENXR)
     xr/intern/wm_xr_action.c
     xr/intern/wm_xr_actionmap.c
     xr/intern/wm_xr_draw.c
+    xr/intern/wm_xr_mocap.c
     xr/intern/wm_xr_operators.c
     xr/intern/wm_xr_session.c
 

source/blender/windowmanager/WM_api.h

@@ -1701,6 +1701,12 @@ bool WM_xr_session_state_controller_aim_location_get(const wmXrData *xr,
 bool WM_xr_session_state_controller_aim_rotation_get(const wmXrData *xr,
                                                      unsigned int subaction_idx,
                                                      float r_rotation[4]);
+bool WM_xr_session_state_tracker_location_get(const wmXrData *xr,
+                                              const char *subaction_path,
+                                              float r_location[3]);
+bool WM_xr_session_state_tracker_rotation_get(const wmXrData *xr,
+                                              const char *subaction_path,
+                                              float r_rotation[4]);
 bool WM_xr_session_state_nav_location_get(const wmXrData *xr, float r_location[3]);
 void WM_xr_session_state_nav_location_set(wmXrData *xr, const float location[3]);
 bool WM_xr_session_state_nav_rotation_get(const wmXrData *xr, float r_rotation[4]);
@@ -1725,6 +1731,7 @@ bool WM_xr_action_create(wmXrData *xr,
                          const ListBase *user_paths,
                          struct wmOperatorType *ot,
                          struct IDProperty *op_properties,
+                         const XrSimulateMouseParams *simulate,
                          const char *haptic_name,
                          const int64_t *haptic_duration,
                          const float *haptic_frequency,
@@ -1756,6 +1763,12 @@ bool WM_xr_controller_pose_actions_set(wmXrData *xr,
                                        const char *action_set_name,
                                        const char *grip_action_name,
                                        const char *aim_action_name);
+bool WM_xr_tracker_pose_action_add(wmXrData *xr,
+                                   const char *action_set_name,
+                                   const char *tracker_action_name);
+bool WM_xr_tracker_pose_action_remove(wmXrData *xr,
+                                      const char *action_set_name,
+                                      const char *tracker_action_name);
 
 /**
  * XR action functions to be called post-XR session start.
@@ -1779,23 +1792,17 @@ void WM_xr_haptic_action_stop(wmXrData *xr,
 
 /* wm_xr_actionmap.c */
 
-XrActionMap *WM_xr_actionmap_new(struct wmXrRuntimeData *runtime,
+XrActionMap *WM_xr_actionmap_new(XrSessionSettings *settings,
                                  const char *name,
                                  bool replace_existing);
 /**
  * Ensure unique name among all action maps.
  */
-void WM_xr_actionmap_ensure_unique(struct wmXrRuntimeData *runtime, XrActionMap *actionmap);
-XrActionMap *WM_xr_actionmap_add_copy(struct wmXrRuntimeData *runtime, XrActionMap *am_src);
-bool WM_xr_actionmap_remove(struct wmXrRuntimeData *runtime, XrActionMap *actionmap);
-XrActionMap *WM_xr_actionmap_find(struct wmXrRuntimeData *runtime, const char *name);
-void WM_xr_actionmap_clear(XrActionMap *actionmap);
-void WM_xr_actionmaps_clear(struct wmXrRuntimeData *runtime);
-ListBase *WM_xr_actionmaps_get(struct wmXrRuntimeData *runtime);
-short WM_xr_actionmap_active_index_get(const struct wmXrRuntimeData *runtime);
-void WM_xr_actionmap_active_index_set(struct wmXrRuntimeData *runtime, short idx);
-short WM_xr_actionmap_selected_index_get(const struct wmXrRuntimeData *runtime);
-void WM_xr_actionmap_selected_index_set(struct wmXrRuntimeData *runtime, short idx);
+void WM_xr_actionmap_ensure_unique(XrSessionSettings *settings, XrActionMap *actionmap);
+XrActionMap *WM_xr_actionmap_add_copy(XrSessionSettings *settings, XrActionMap *am_src);
+bool WM_xr_actionmap_remove(XrSessionSettings *settings, XrActionMap *actionmap);
+XrActionMap *WM_xr_actionmap_find(XrSessionSettings *settings, const char *name);
+void WM_xr_actionmaps_free(XrSessionSettings *settings);
 
 XrActionMapItem *WM_xr_actionmap_item_new(XrActionMap *actionmap,
                                           const char *name,
@@ -1824,6 +1831,18 @@ XrActionMapBinding *WM_xr_actionmap_binding_add_copy(XrActionMapItem *ami,
                                                      XrActionMapBinding *amb_src);
 bool WM_xr_actionmap_binding_remove(XrActionMapItem *ami, XrActionMapBinding *amb);
 XrActionMapBinding *WM_xr_actionmap_binding_find(XrActionMapItem *ami, const char *name);
+
+/* wm_xr_mocap.c */
+
+XrMotionCaptureObject *WM_xr_mocap_object_new(XrSessionSettings *settings, Object *ob);
+void WM_xr_mocap_object_ensure_unique(XrSessionSettings *settings,
+                                      XrMotionCaptureObject *mocap_ob);
+void WM_xr_mocap_object_remove(XrSessionSettings *settings, XrMotionCaptureObject *mocap_ob);
+XrMotionCaptureObject *WM_xr_mocap_object_find(const XrSessionSettings *settings,
+                                               const Object *ob);
+
+bool WM_xr_session_state_mocap_pose_get(const wmXrData *xr, XrMotionCaptureObject *mocap_ob);
+void WM_xr_session_state_mocap_pose_set(wmXrData *xr, const XrMotionCaptureObject *mocap_ob);
 #endif /* WITH_XR_OPENXR */
 
 #ifdef __cplusplus

source/blender/windowmanager/WM_types.h

@@ -841,6 +841,11 @@ typedef struct wmXrActionData {
 
   /** Whether bimanual interaction is occurring. */
   bool bimanual;
+  /** Whether the action is simulating mouse inputs. */
+  bool simulate_mouse;
+  /** Type and value for simulated mouse event. */
+  short simulate_type;
+  short simulate_val;
 } wmXrActionData;
 #endif
 

source/blender/windowmanager/gizmo/intern/wm_gizmo_map.c

@@ -936,11 +936,16 @@ void wm_gizmomap_handler_context_op(bContext *C, wmEventHandler_Op *handler)
   bScreen *screen = CTX_wm_screen(C);
 
   if (screen) {
-    ScrArea *area;
+    ScrArea *area = NULL;
 
-    for (area = screen->areabase.first; area; area = area->next) {
-      if (area == handler->context.area) {
-        break;
+    if ((handler->context.area->flag & AREA_FLAG_OFFSCREEN) != 0) {
+      area = handler->context.area;
+    }
+    else {
+      for (area = screen->areabase.first; area; area = area->next) {
+        if (area == handler->context.area) {
+          break;
+        }
       }
     }
     if (area == NULL) {

source/blender/windowmanager/intern/wm.c

@@ -98,6 +98,33 @@ static void window_manager_foreach_id(ID *id, LibraryForeachIDData *data)
 static void write_wm_xr_data(BlendWriter *writer, wmXrData *xr_data)
 {
   BKE_screen_view3d_shading_blend_write(writer, &xr_data->session_settings.shading);
+
+  LISTBASE_FOREACH (XrActionMap *, am, &xr_data->session_settings.actionmaps) {
+    BLO_write_struct(writer, XrActionMap, am);
+
+    LISTBASE_FOREACH (XrActionMapItem *, ami, &am->items) {
+      BLO_write_struct(writer, XrActionMapItem, ami);
+      if (ami->op[0] && ami->op_properties) {
+        IDP_BlendWrite(writer, ami->op_properties);
+      }
+
+      LISTBASE_FOREACH (XrUserPath *, user_path, &ami->user_paths) {
+        BLO_write_struct(writer, XrUserPath, user_path);
+      }
+
+      LISTBASE_FOREACH (XrActionMapBinding *, amb, &ami->bindings) {
+        BLO_write_struct(writer, XrActionMapBinding, amb);
+
+        LISTBASE_FOREACH (XrComponentPath *, component_path, &amb->component_paths) {
+          BLO_write_struct(writer, XrComponentPath, component_path);
+        }
+      }
+    }
+  }
+
+  LISTBASE_FOREACH (XrMotionCaptureObject *, mocap_ob, &xr_data->session_settings.mocap_objects) {
+    BLO_write_struct(writer, XrMotionCaptureObject, mocap_ob);
+  }
 }
 
 static void window_manager_blend_write(BlendWriter *writer, ID *id, const void *id_address)
@@ -126,6 +153,36 @@ static void window_manager_blend_write(BlendWriter *writer, ID *id, const void *
 static void direct_link_wm_xr_data(BlendDataReader *reader, wmXrData *xr_data)
 {
   BKE_screen_view3d_shading_blend_read_data(reader, &xr_data->session_settings.shading);
+
+  BLO_read_list(reader, &xr_data->session_settings.actionmaps);
+
+  LISTBASE_FOREACH (XrActionMap *, am, &xr_data->session_settings.actionmaps) {
+    BLO_read_list(reader, &am->items);
+
+    LISTBASE_FOREACH (XrActionMapItem *, ami, &am->items) {
+      if (ami->op[0] && ami->op_properties) {
+        BLO_read_data_address(reader, &ami->op_properties);
+        IDP_BlendDataRead(reader, &ami->op_properties);
+
+        ami->op_properties_ptr = MEM_callocN(sizeof(PointerRNA), "wmOpItemPtr");
+        WM_operator_properties_create(ami->op_properties_ptr, ami->op);
+        ami->op_properties_ptr->data = ami->op_properties;
+      }
+      else {
+        ami->op_properties = NULL;
+        ami->op_properties_ptr = NULL;
+      }
+
+      BLO_read_list(reader, &ami->user_paths);
+      BLO_read_list(reader, &ami->bindings);
+
+      LISTBASE_FOREACH (XrActionMapBinding *, amb, &ami->bindings) {
+        BLO_read_list(reader, &amb->component_paths);
+      }
+    }
+  }
+
+  BLO_read_list(reader, &xr_data->session_settings.mocap_objects);
 }
 
 static void window_manager_blend_read_data(BlendDataReader *reader, ID *id)
@@ -220,6 +277,10 @@ static void window_manager_blend_read_data(BlendDataReader *reader, ID *id)
 static void lib_link_wm_xr_data(BlendLibReader *reader, ID *parent_id, wmXrData *xr_data)
 {
   BLO_read_id_address(reader, parent_id->lib, &xr_data->session_settings.base_pose_object);
+
+  LISTBASE_FOREACH (XrMotionCaptureObject *, mocap_ob, &xr_data->session_settings.mocap_objects) {
+    BLO_read_id_address(reader, parent_id->lib, &mocap_ob->ob);
+  }
 }
 
 static void lib_link_workspace_instance_hook(BlendLibReader *reader,

source/blender/windowmanager/intern/wm_event_system.cc

@@ -143,14 +143,8 @@ wmEvent *wm_event_add(wmWindow *win, const wmEvent *event_to_add)
   return wm_event_add_ex(win, event_to_add, nullptr);
 }
 
-wmEvent *WM_event_add_simulate(wmWindow *win, const wmEvent *event_to_add)
+static void wm_event_simulate_eventstate(wmWindow *win, wmEvent *event)
 {
-  if ((G.f & G_FLAG_EVENT_SIMULATE) == 0) {
-    BLI_assert_unreachable();
-    return nullptr;
-  }
-  wmEvent *event = wm_event_add(win, event_to_add);
-
   /* Logic for setting previous value is documented on the #wmEvent struct,
    * see #wm_event_add_ghostevent for the implementation of logic this follows. */
   copy_v2_v2_int(win->eventstate->xy, event->xy);
@@ -159,9 +153,21 @@ wmEvent *WM_event_add_simulate(wmWindow *win, const wmEvent *event_to_add)
     copy_v2_v2_int(win->eventstate->prev_xy, win->eventstate->xy);
     copy_v2_v2_int(event->prev_xy, win->eventstate->xy);
   }
-  else if (ISKEYBOARD_OR_BUTTON(event->type)) {
+  else if (ISKEYBOARD_OR_BUTTON(event->type) && ELEM(event->val, KM_PRESS, KM_RELEASE)) {
     wm_event_state_update_and_click_set_ex(event, win->eventstate, ISKEYBOARD(event->type), false);
   }
+}
+
+wmEvent *WM_event_add_simulate(wmWindow *win, const wmEvent *event_to_add)
+{
+  if ((G.f & G_FLAG_EVENT_SIMULATE) == 0) {
+    BLI_assert_unreachable();
+    return nullptr;
+  }
+  wmEvent *event = wm_event_add(win, event_to_add);
+
+  wm_event_simulate_eventstate(win, event);
+
   return event;
 }
 
@@ -2017,10 +2023,15 @@ static void wm_handler_op_context_get_if_valid(bContext *C,
   else {
     ScrArea *area = nullptr;
 
-    ED_screen_areas_iter (win, screen, area_iter) {
-      if (area_iter == handler->context.area) {
-        area = area_iter;
-        break;
+    if ((handler->context.area->flag & AREA_FLAG_OFFSCREEN) != 0) {
+      area = handler->context.area;
+    }
+    else {
+      ED_screen_areas_iter (win, screen, area_iter) {
+        if (area_iter == handler->context.area) {
+          area = area_iter;
+          break;
+        }
       }
     }
 
@@ -3728,13 +3739,22 @@ static void wm_event_handle_xrevent(bContext *C,
   }
   BLI_assert(WM_region_use_viewport(area, region)); /* For operators using GPU-based selection. */
 
+  /* Check for simulated mouse event. */
+  wmXrActionData *actiondata = static_cast<wmXrActionData *>(event->customdata);
+  const bool simulate_event = (actiondata->simulate_mouse &&
+                               (actiondata->simulate_type != EVENT_NONE));
+  if (simulate_event) {
+    event->type = actiondata->simulate_type;
+    event->val = actiondata->simulate_val;
+    wm_event_simulate_eventstate(win, event);
+  }
+
   CTX_wm_area_set(C, area);
   CTX_wm_region_set(C, region);
 
   int action = wm_handlers_do(C, event, &win->modalhandlers);
 
   if ((action & WM_HANDLER_BREAK) == 0) {
-    wmXrActionData *actiondata = static_cast<wmXrActionData *>(event->customdata);
     if (actiondata->ot->modal && event->val == KM_RELEASE) {
       /* Don't execute modal operators on release. */
     }
@@ -5732,7 +5752,7 @@ void wm_event_add_ghostevent(wmWindowManager *wm, wmWindow *win, int type, void
 }
 
 #ifdef WITH_XR_OPENXR
-void wm_event_add_xrevent(wmWindow *win, wmXrActionData *actiondata, short val)
+void wm_event_add_xrevent(wmWindow *win, wmXrActionData *actiondata, short val, int mval[2])
 {
   BLI_assert(ELEM(val, KM_PRESS, KM_RELEASE));
 
@@ -5744,6 +5764,11 @@ void wm_event_add_xrevent(wmWindow *win, wmXrActionData *actiondata, short val)
   event.customdata = actiondata;
   event.customdata_free = true;
 
+  if (mval) {
+    copy_v2_v2_int(event.xy, mval);
+    copy_v2_v2_int(event.mval, mval);
+  }
+
   wm_event_add(win, &event);
 }
 #endif /* WITH_XR_OPENXR */

source/blender/windowmanager/intern/wm_files.c

@@ -203,6 +203,12 @@ static void wm_window_match_init(bContext *C, ListBase *wmlist)
       WM_msgbus_destroy(wm->message_bus);
       wm->message_bus = NULL;
     }
+
+#ifdef WITH_XR_OPENXR
+    /* Free XR action maps and motion capture objects. */
+    WM_xr_actionmaps_free(&wm->xr.session_settings);
+    BLI_freelistN(&wm->xr.session_settings.mocap_objects);
+#endif
   }
 
   BLI_listbase_clear(&G_MAIN->wm);

source/blender/windowmanager/intern/wm_window.c

@@ -1517,7 +1517,7 @@ void wm_window_process_events(const bContext *C)
 #ifdef WITH_XR_OPENXR
   /* XR events don't use the regular window queues. So here we don't only trigger
    * processing/dispatching but also handling. */
-  has_event |= wm_xr_events_handle(CTX_wm_manager(C));
+  has_event |= wm_xr_events_handle(C);
 #endif
   GPU_render_end();
 

source/blender/windowmanager/wm_event_system.h

@@ -145,7 +145,10 @@ void wm_event_do_handlers(bContext *C);
  */
 void wm_event_add_ghostevent(wmWindowManager *wm, wmWindow *win, int type, void *customdata);
 #ifdef WITH_XR_OPENXR
-void wm_event_add_xrevent(wmWindow *win, struct wmXrActionData *actiondata, short val);
+void wm_event_add_xrevent(wmWindow *win,
+                          struct wmXrActionData *actiondata,
+                          short val,
+                          int mval[2]);
 #endif
 
 void wm_event_do_depsgraph(bContext *C, bool is_after_open_file);

source/blender/windowmanager/xr/intern/wm_xr.c

@@ -8,6 +8,9 @@
  * representation of the OpenXR runtime connection within the application.
  */
 
+#include "BLI_listbase.h"
+
+#include "BKE_context.h"
 #include "BKE_global.h"
 #include "BKE_idprop.h"
 #include "BKE_main.h"
@@ -87,6 +90,9 @@ bool wm_xr_init(wmWindowManager *wm)
       create_info.context_flag |= GHOST_kXrContextGpuNVIDIA;
     }
 #endif
+    if ((wm->xr.session_settings.flag & XR_SESSION_ENABLE_VIVE_TRACKER_EXTENSION) != 0) {
+      create_info.context_flag |= GHOST_kXrContextEnableViveTrackerExtension;
+    }
 
     if (!(context = GHOST_XrContextCreate(&create_info))) {
       return false;
@@ -117,16 +123,20 @@ void wm_xr_exit(wmWindowManager *wm)
     IDP_FreeProperty(wm->xr.session_settings.shading.prop);
     wm->xr.session_settings.shading.prop = NULL;
   }
+  WM_xr_actionmaps_free(&wm->xr.session_settings);
+  BLI_freelistN(&wm->xr.session_settings.mocap_objects);
 }
 
-bool wm_xr_events_handle(wmWindowManager *wm)
+bool wm_xr_events_handle(const bContext *C)
 {
+  wmWindowManager *wm = CTX_wm_manager(C);
+
   if (wm->xr.runtime && wm->xr.runtime->context) {
     GHOST_XrEventsHandle(wm->xr.runtime->context);
 
     /* Process OpenXR action events. */
     if (WM_xr_session_is_ready(&wm->xr)) {
-      wm_xr_session_actions_update(wm);
+      wm_xr_session_actions_update(C);
     }
 
     /* wm_window_process_events() uses the return value to determine if it can put the main thread
@@ -168,7 +178,6 @@ void wm_xr_runtime_data_free(wmXrRuntimeData **runtime)
       (*runtime)->area = NULL;
     }
     wm_xr_session_data_free(&(*runtime)->session_state);
-    WM_xr_actionmaps_clear(*runtime);
 
     GHOST_XrContextDestroy(context);
   }

source/blender/windowmanager/xr/intern/wm_xr_action.c

@@ -43,6 +43,7 @@ static void action_set_destroy(void *val)
 
   MEM_SAFE_FREE(action_set->name);
 
+  BLI_freelistN(&action_set->tracker_actions);
   BLI_freelistN(&action_set->active_modal_actions);
   BLI_freelistN(&action_set->active_haptic_actions);
 
@@ -59,6 +60,7 @@ static wmXrAction *action_create(const char *action_name,
                                  const ListBase *user_paths,
                                  wmOperatorType *ot,
                                  IDProperty *op_properties,
+                                 const XrSimulateMouseParams *simulate,
                                  const char *haptic_name,
                                  const int64_t *haptic_duration,
                                  const float *haptic_frequency,
@@ -118,6 +120,11 @@ static wmXrAction *action_create(const char *action_name,
   action->ot = ot;
   action->op_properties = op_properties;
 
+  if (simulate) {
+    BLI_assert(is_button_action);
+    memcpy(&action->simulate, simulate, sizeof(action->simulate));
+  }
+
   if (haptic_name) {
     BLI_assert(is_button_action);
     action->haptic_name = MEM_mallocN(strlen(haptic_name) + 1, "XrAction_HapticName");
@@ -200,6 +207,11 @@ void WM_xr_action_set_destroy(wmXrData *xr, const char *action_set_name)
       action_set->controller_grip_action = action_set->controller_aim_action = NULL;
     }
 
+    if (!BLI_listbase_is_empty(&action_set->tracker_actions)) {
+      wm_xr_session_tracker_data_clear(session_state);
+      BLI_freelistN(&action_set->tracker_actions);
+    }
+
     BLI_freelistN(&action_set->active_modal_actions);
     BLI_freelistN(&action_set->active_haptic_actions);
 
@@ -216,6 +228,7 @@ bool WM_xr_action_create(wmXrData *xr,
                          const ListBase *user_paths,
                          wmOperatorType *ot,
                          IDProperty *op_properties,
+                         const XrSimulateMouseParams *simulate,
                          const char *haptic_name,
                          const int64_t *haptic_duration,
                          const float *haptic_frequency,
@@ -233,6 +246,7 @@ bool WM_xr_action_create(wmXrData *xr,
                                      user_paths,
                                      ot,
                                      op_properties,
+                                     simulate,
                                      haptic_name,
                                      haptic_duration,
                                      haptic_frequency,
@@ -309,6 +323,14 @@ void WM_xr_action_destroy(wmXrData *xr, const char *action_set_name, const char
     action_set->controller_grip_action = action_set->controller_aim_action = NULL;
   }
 
+  LISTBASE_FOREACH (LinkData *, ld, &action_set->tracker_actions) {
+    wmXrAction *tracker_action = ld->data;
+    if (STREQ(tracker_action->name, action_name)) {
+      WM_xr_tracker_pose_action_remove(xr, action_set_name, action_name);
+      break;
+    }
+  }
+
   LISTBASE_FOREACH (LinkData *, ld, &action_set->active_modal_actions) {
     wmXrAction *active_modal_action = ld->data;
     if (STREQ(active_modal_action->name, action_name)) {
@@ -416,6 +438,7 @@ bool WM_xr_active_action_set_set(wmXrData *xr, const char *action_set_name, bool
 
   xr->runtime->session_state.active_action_set = action_set;
 
+  /* Update controller/tracker data.  */
   if (action_set->controller_grip_action && action_set->controller_aim_action) {
     wm_xr_session_controller_data_populate(
         action_set->controller_grip_action, action_set->controller_aim_action, xr);
@@ -424,6 +447,13 @@ bool WM_xr_active_action_set_set(wmXrData *xr, const char *action_set_name, bool
     wm_xr_session_controller_data_clear(&xr->runtime->session_state);
   }
 
+  if (!BLI_listbase_is_empty(&action_set->tracker_actions)) {
+    wm_xr_session_tracker_data_populate(&action_set->tracker_actions, xr);
+  }
+  else {
+    wm_xr_session_tracker_data_clear(&xr->runtime->session_state);
+  }
+
   return true;
 }
 
@@ -469,6 +499,66 @@ bool WM_xr_controller_pose_actions_set(wmXrData *xr,
   return true;
 }
 
+bool WM_xr_tracker_pose_action_add(wmXrData *xr,
+                                   const char *action_set_name,
+                                   const char *tracker_action_name)
+{
+  wmXrActionSet *action_set = action_set_find(xr, action_set_name);
+  if (!action_set) {
+    return false;
+  }
+
+  wmXrAction *tracker_action = action_find(xr, action_set_name, tracker_action_name);
+  if (!tracker_action) {
+    return false;
+  }
+
+  LinkData *ld = MEM_callocN(sizeof(*ld), __func__);
+  ld->data = tracker_action;
+  BLI_addtail(&action_set->tracker_actions, ld);
+
+  if (action_set == xr->runtime->session_state.active_action_set) {
+    wm_xr_session_tracker_data_populate(&action_set->tracker_actions, xr);
+  }
+
+  return true;
+}
+
+bool WM_xr_tracker_pose_action_remove(wmXrData *xr,
+                                      const char *action_set_name,
+                                      const char *tracker_action_name)
+{
+  wmXrActionSet *action_set = action_set_find(xr, action_set_name);
+  if (!action_set) {
+    return false;
+  }
+
+  wmXrAction *tracker_action = action_find(xr, action_set_name, tracker_action_name);
+  if (!tracker_action) {
+    return false;
+  }
+
+  {
+    LinkData *ld = BLI_findptr(
+        &action_set->tracker_actions, tracker_action, offsetof(LinkData, data));
+    if (!ld) {
+      return false;
+    }
+    BLI_freelinkN(&action_set->tracker_actions, ld);
+  }
+
+  if (action_set == xr->runtime->session_state.active_action_set) {
+    if (BLI_listbase_is_empty(&action_set->tracker_actions)) {
+      wm_xr_session_tracker_data_clear(&xr->runtime->session_state);
+    }
+    else {
+      wm_xr_session_tracker_data_populate(&action_set->tracker_actions, xr);
+    }
+  }
+
+  return true;
+}
+
 bool WM_xr_action_state_get(const wmXrData *xr,
                             const char *action_set_name,
                             const char *action_name,

source/blender/windowmanager/xr/intern/wm_xr_actionmap.c

@@ -127,6 +127,7 @@ XrActionMapBinding *WM_xr_actionmap_binding_add_copy(XrActionMapItem *ami,
 static void wm_xr_actionmap_binding_clear(XrActionMapBinding *amb)
 {
   BLI_freelistN(&amb->component_paths);
+  amb->sel_component_path = 0;
 }
 
 bool WM_xr_actionmap_binding_remove(XrActionMapItem *ami, XrActionMapBinding *amb)
@@ -137,9 +138,9 @@ bool WM_xr_actionmap_binding_remove(XrActionMapItem *ami, XrActionMapBinding *am
     wm_xr_actionmap_binding_clear(amb);
     BLI_freelinkN(&ami->bindings, amb);
 
-    if (idx <= ami->selbinding) {
-      if (--ami->selbinding < 0) {
-        ami->selbinding = 0;
+    if (idx <= ami->sel_binding) {
+      if (--ami->sel_binding < 0) {
+        ami->sel_binding = 0;
       }
     }
 
@@ -192,11 +193,12 @@ static void wm_xr_actionmap_item_clear(XrActionMapItem *ami)
     wm_xr_actionmap_binding_clear(amb);
   }
   BLI_freelistN(&ami->bindings);
-  ami->selbinding = 0;
-
-  wm_xr_actionmap_item_properties_free(ami);
+  ami->sel_binding = 0;
 
   BLI_freelistN(&ami->user_paths);
+  ami->sel_user_path = 0;
+
+  wm_xr_actionmap_item_properties_free(ami);
 }
 
 void WM_xr_actionmap_item_properties_update_ot(XrActionMapItem *ami)
@@ -313,6 +315,12 @@ static XrActionMapItem *wm_xr_actionmap_item_copy(XrActionMapItem *ami_src)
     BLI_addtail(&ami_dst->bindings, amb_new);
   }
 
+  BLI_listbase_clear(&ami_dst->user_paths);
+  LISTBASE_FOREACH (XrUserPath *, path, &ami_src->user_paths) {
+    XrUserPath *path_new = MEM_dupallocN(path);
+    BLI_addtail(&ami_dst->user_paths, path_new);
+  }
+
   if (ami_dst->op_properties) {
     ami_dst->op_properties_ptr = MEM_callocN(sizeof(PointerRNA), "wmOpItemPtr");
     WM_operator_properties_create(ami_dst->op_properties_ptr, ami_dst->op);
@@ -324,12 +332,6 @@ static XrActionMapItem *wm_xr_actionmap_item_copy(XrActionMapItem *ami_src)
     ami_dst->op_properties_ptr = NULL;
   }
 
-  BLI_listbase_clear(&ami_dst->user_paths);
-  LISTBASE_FOREACH (XrUserPath *, path, &ami_src->user_paths) {
-    XrUserPath *path_new = MEM_dupallocN(path);
-    BLI_addtail(&ami_dst->user_paths, path_new);
-  }
-
   return ami_dst;
 }
 
@@ -352,9 +354,9 @@ bool WM_xr_actionmap_item_remove(XrActionMap *actionmap, XrActionMapItem *ami)
     wm_xr_actionmap_item_clear(ami);
     BLI_freelinkN(&actionmap->items, ami);
 
-    if (idx <= actionmap->selitem) {
-      if (--actionmap->selitem < 0) {
-        actionmap->selitem = 0;
+    if (idx <= actionmap->sel_item) {
+      if (--actionmap->sel_item < 0) {
+        actionmap->sel_item = 0;
       }
     }
 
@@ -382,39 +384,49 @@ XrActionMapItem *WM_xr_actionmap_item_find(XrActionMap *actionmap, const char *n
  * List of XR action map items.
  * \{ */
 
-XrActionMap *WM_xr_actionmap_new(wmXrRuntimeData *runtime, const char *name, bool replace_existing)
+static void wm_xr_actionmap_clear(XrActionMap *actionmap)
 {
-  XrActionMap *am_prev = WM_xr_actionmap_find(runtime, name);
+  LISTBASE_FOREACH (XrActionMapItem *, ami, &actionmap->items) {
+    wm_xr_actionmap_item_clear(ami);
+  }
+  BLI_freelistN(&actionmap->items);
+  actionmap->sel_item = 0;
+}
+
+XrActionMap *WM_xr_actionmap_new(XrSessionSettings *settings,
+                                 const char *name,
+                                 bool replace_existing)
+{
+  XrActionMap *am_prev = WM_xr_actionmap_find(settings, name);
   if (am_prev && replace_existing) {
-    WM_xr_actionmap_clear(am_prev);
+    wm_xr_actionmap_clear(am_prev);
     return am_prev;
   }
 
   XrActionMap *am = MEM_callocN(sizeof(struct XrActionMap), __func__);
   BLI_strncpy(am->name, name, MAX_NAME);
   if (am_prev) {
-    WM_xr_actionmap_ensure_unique(runtime, am);
+    WM_xr_actionmap_ensure_unique(settings, am);
   }
 
-  BLI_addtail(&runtime->actionmaps, am);
+  BLI_addtail(&settings->actionmaps, am);
 
   return am;
 }
 
-static XrActionMap *wm_xr_actionmap_find_except(wmXrRuntimeData *runtime,
+static XrActionMap *wm_xr_actionmap_find_except(XrSessionSettings *settings,
                                                 const char *name,
                                                 const XrActionMap *am_except)
 {
-  LISTBASE_FOREACH (XrActionMap *, am, &runtime->actionmaps) {
+  LISTBASE_FOREACH (XrActionMap *, am, &settings->actionmaps) {
     if (STREQLEN(name, am->name, MAX_NAME) && (am != am_except)) {
       return am;
     }
   }
-
   return NULL;
 }
 
-void WM_xr_actionmap_ensure_unique(wmXrRuntimeData *runtime, XrActionMap *actionmap)
+void WM_xr_actionmap_ensure_unique(XrSessionSettings *settings, XrActionMap *actionmap)
 {
   char name[MAX_NAME];
   char *suffix;
@@ -425,7 +437,7 @@ void WM_xr_actionmap_ensure_unique(wmXrRuntimeData *runtime, XrActionMap *action
   baselen = BLI_strnlen(name, MAX_NAME);
   suffix = &name[baselen];
 
-  while (wm_xr_actionmap_find_except(runtime, name, actionmap)) {
+  while (wm_xr_actionmap_find_except(settings, name, actionmap)) {
     if ((baselen + 1) + (log10(++idx) + 1) > MAX_NAME) {
       /* Use default base name. */
       BLI_strncpy(name, WM_XR_ACTIONMAP_STR_DEFAULT, MAX_NAME);
@@ -455,33 +467,33 @@ static XrActionMap *wm_xr_actionmap_copy(XrActionMap *am_src)
   return am_dst;
 }
 
-XrActionMap *WM_xr_actionmap_add_copy(wmXrRuntimeData *runtime, XrActionMap *am_src)
+XrActionMap *WM_xr_actionmap_add_copy(XrSessionSettings *settings, XrActionMap *am_src)
 {
   XrActionMap *am_dst = wm_xr_actionmap_copy(am_src);
 
-  WM_xr_actionmap_ensure_unique(runtime, am_dst);
+  WM_xr_actionmap_ensure_unique(settings, am_dst);
 
-  BLI_addtail(&runtime->actionmaps, am_dst);
+  BLI_addtail(&settings->actionmaps, am_dst);
 
   return am_dst;
 }
 
-bool WM_xr_actionmap_remove(wmXrRuntimeData *runtime, XrActionMap *actionmap)
+bool WM_xr_actionmap_remove(XrSessionSettings *settings, XrActionMap *actionmap)
 {
-  int idx = BLI_findindex(&runtime->actionmaps, actionmap);
+  int idx = BLI_findindex(&settings->actionmaps, actionmap);
 
   if (idx != -1) {
-    WM_xr_actionmap_clear(actionmap);
-    BLI_freelinkN(&runtime->actionmaps, actionmap);
+    wm_xr_actionmap_clear(actionmap);
+    BLI_freelinkN(&settings->actionmaps, actionmap);
 
-    if (idx <= runtime->actactionmap) {
-      if (--runtime->actactionmap < 0) {
-        runtime->actactionmap = 0;
+    if (idx <= settings->act_actionmap) {
+      if (--settings->act_actionmap < 0) {
+        settings->act_actionmap = 0;
       }
     }
-    if (idx <= runtime->selactionmap) {
-      if (--runtime->selactionmap < 0) {
-        runtime->selactionmap = 0;
+    if (idx <= settings->sel_actionmap) {
+      if (--settings->sel_actionmap < 0) {
+        settings->sel_actionmap = 0;
       }
     }
 
@@ -491,9 +503,9 @@ bool WM_xr_actionmap_remove(wmXrRuntimeData *runtime, XrActionMap *actionmap)
   return false;
 }
 
-XrActionMap *WM_xr_actionmap_find(wmXrRuntimeData *runtime, const char *name)
+XrActionMap *WM_xr_actionmap_find(XrSessionSettings *settings, const char *name)
 {
-  LISTBASE_FOREACH (XrActionMap *, am, &runtime->actionmaps) {
+  LISTBASE_FOREACH (XrActionMap *, am, &settings->actionmaps) {
     if (STREQLEN(name, am->name, MAX_NAME)) {
       return am;
     }
@@ -501,47 +513,13 @@ XrActionMap *WM_xr_actionmap_find(wmXrRuntimeData *runtime, const char *name)
   return NULL;
 }
 
-void WM_xr_actionmap_clear(XrActionMap *actionmap)
+void WM_xr_actionmaps_free(XrSessionSettings *settings)
 {
-  LISTBASE_FOREACH (XrActionMapItem *, ami, &actionmap->items) {
-    wm_xr_actionmap_item_clear(ami);
+  LISTBASE_FOREACH (XrActionMap *, am, &settings->actionmaps) {
+    wm_xr_actionmap_clear(am);
   }
-  BLI_freelistN(&actionmap->items);
-  actionmap->selitem = 0;
-}
-
-void WM_xr_actionmaps_clear(wmXrRuntimeData *runtime)
-{
-  LISTBASE_FOREACH (XrActionMap *, am, &runtime->actionmaps) {
-    WM_xr_actionmap_clear(am);
-  }
-  BLI_freelistN(&runtime->actionmaps);
-  runtime->actactionmap = runtime->selactionmap = 0;
-}
-
-ListBase *WM_xr_actionmaps_get(wmXrRuntimeData *runtime)
-{
-  return &runtime->actionmaps;
-}
-
-short WM_xr_actionmap_active_index_get(const wmXrRuntimeData *runtime)
-{
-  return runtime->actactionmap;
-}
-
-void WM_xr_actionmap_active_index_set(wmXrRuntimeData *runtime, short idx)
-{
-  runtime->actactionmap = idx;
-}
-
-short WM_xr_actionmap_selected_index_get(const wmXrRuntimeData *runtime)
-{
-  return runtime->selactionmap;
-}
-
-void WM_xr_actionmap_selected_index_set(wmXrRuntimeData *runtime, short idx)
-{
-  runtime->selactionmap = idx;
+  BLI_freelistN(&settings->actionmaps);
+  settings->act_actionmap = settings->sel_actionmap = 0;
 }
 
 /** \} */

source/blender/windowmanager/xr/intern/wm_xr_draw.c

@@ -73,10 +73,11 @@ static void wm_xr_draw_matrices_create(const wmXrDrawData *draw_data,
                                        const XrSessionSettings *session_settings,
                                        const wmXrSessionState *session_state,
                                        float r_viewmat[4][4],
+                                       float r_viewmat_base[4][4],
                                        float r_projmat[4][4])
 {
   GHOST_XrPose eye_pose;
-  float eye_inv[4][4], base_inv[4][4], nav_inv[4][4], m[4][4];
+  float eye_inv[4][4], base_inv[4][4], nav_inv[4][4];
 
   /* Calculate inverse eye matrix. */
   copy_qt_qt(eye_pose.orientation_quat, draw_view->eye_pose.orientation_quat);
@@ -93,8 +94,8 @@ static void wm_xr_draw_matrices_create(const wmXrDrawData *draw_data,
   /* Apply base pose and navigation. */
   wm_xr_pose_scale_to_imat(&draw_data->base_pose, draw_data->base_scale, base_inv);
   wm_xr_pose_scale_to_imat(&session_state->nav_pose_prev, session_state->nav_scale_prev, nav_inv);
-  mul_m4_m4m4(m, eye_inv, base_inv);
-  mul_m4_m4m4(r_viewmat, m, nav_inv);
+  mul_m4_m4m4(r_viewmat_base, eye_inv, base_inv);
+  mul_m4_m4m4(r_viewmat, r_viewmat_base, nav_inv);
 
   perspective_m4_fov(r_projmat,
                      draw_view->fov.angle_left,
@@ -135,13 +136,14 @@ void wm_xr_draw_view(const GHOST_XrDrawViewInfo *draw_view, void *customdata)
 
   const int display_flags = V3D_OFSDRAW_OVERRIDE_SCENE_SETTINGS | settings->draw_flags;
 
-  float viewmat[4][4], winmat[4][4];
+  float viewmat[4][4], viewmat_base[4][4], winmat[4][4];
 
   BLI_assert(WM_xr_session_is_ready(xr_data));
 
   wm_xr_session_draw_data_update(session_state, settings, draw_view, draw_data);
-  wm_xr_draw_matrices_create(draw_data, draw_view, settings, session_state, viewmat, winmat);
-  wm_xr_session_state_update(settings, draw_data, draw_view, session_state);
+  wm_xr_draw_matrices_create(
+      draw_data, draw_view, settings, session_state, viewmat, viewmat_base, winmat);
+  wm_xr_session_state_update(settings, draw_data, draw_view, viewmat, viewmat_base, session_state);
 
   if (!wm_xr_session_surface_offscreen_ensure(surface_data, draw_view)) {
     return;
@@ -227,7 +229,7 @@ static GPUBatch *wm_xr_controller_model_batch_create(GHOST_XrContextHandle xr_co
 
 static void wm_xr_controller_model_draw(const XrSessionSettings *settings,
                                         GHOST_XrContextHandle xr_context,
-                                        wmXrSessionState *state)
+                                        ListBase *controllers)
 {
   GHOST_XrControllerModelData model_data;
 
@@ -246,7 +248,7 @@ static void wm_xr_controller_model_draw(const XrSessionSettings *settings,
   GPU_depth_test(GPU_DEPTH_NONE);
   GPU_blend(GPU_BLEND_ALPHA);
 
-  LISTBASE_FOREACH (wmXrController *, controller, &state->controllers) {
+  LISTBASE_FOREACH (wmXrController *, controller, controllers) {
     GPUBatch *model = controller->model;
     if (!model) {
       model = controller->model = wm_xr_controller_model_batch_create(xr_context,
@@ -288,7 +290,7 @@ static void wm_xr_controller_model_draw(const XrSessionSettings *settings,
   }
 }
 
-static void wm_xr_controller_aim_draw(const XrSessionSettings *settings, wmXrSessionState *state)
+static void wm_xr_controller_aim_draw(const XrSessionSettings *settings, ListBase *controllers)
 {
   bool draw_ray;
   switch (settings->controller_draw_style) {
@@ -321,9 +323,9 @@ static void wm_xr_controller_aim_draw(const XrSessionSettings *settings, wmXrSes
     GPU_depth_test(GPU_DEPTH_LESS_EQUAL);
     GPU_blend(GPU_BLEND_ALPHA);
 
-    immBegin(GPU_PRIM_LINES, (uint)BLI_listbase_count(&state->controllers) * 2);
+    immBegin(GPU_PRIM_LINES, (uint)BLI_listbase_count(controllers) * 2);
 
-    LISTBASE_FOREACH (wmXrController *, controller, &state->controllers) {
+    LISTBASE_FOREACH (wmXrController *, controller, controllers) {
       const float(*mat)[4] = controller->aim_mat;
       madd_v3_v3v3fl(ray, mat[3], mat[2], -scale);
 
@@ -345,9 +347,9 @@ static void wm_xr_controller_aim_draw(const XrSessionSettings *settings, wmXrSes
     GPU_depth_test(GPU_DEPTH_NONE);
     GPU_blend(GPU_BLEND_NONE);
 
-    immBegin(GPU_PRIM_LINES, (uint)BLI_listbase_count(&state->controllers) * 6);
+    immBegin(GPU_PRIM_LINES, (uint)BLI_listbase_count(controllers) * 6);
 
-    LISTBASE_FOREACH (wmXrController *, controller, &state->controllers) {
+    LISTBASE_FOREACH (wmXrController *, controller, controllers) {
       const float(*mat)[4] = controller->aim_mat;
       madd_v3_v3v3fl(x_axis, mat[3], mat[0], scale);
       madd_v3_v3v3fl(y_axis, mat[3], mat[1], scale);
@@ -380,8 +382,19 @@ void wm_xr_draw_controllers(const bContext *UNUSED(C), ARegion *UNUSED(region),
   wmXrData *xr = customdata;
   const XrSessionSettings *settings = &xr->session_settings;
   GHOST_XrContextHandle xr_context = xr->runtime->context;
-  wmXrSessionState *state = &xr->runtime->session_state;
+  ListBase *controllers = &xr->runtime->session_state.controllers;
 
-  wm_xr_controller_model_draw(settings, xr_context, state);
-  wm_xr_controller_aim_draw(settings, state);
+  wm_xr_controller_model_draw(settings, xr_context, controllers);
+  wm_xr_controller_aim_draw(settings, controllers);
+}
+
+void wm_xr_draw_trackers(const bContext *UNUSED(C), ARegion *UNUSED(region), void *customdata)
+{
+  wmXrData *xr = customdata;
+  const XrSessionSettings *settings = &xr->session_settings;
+  GHOST_XrContextHandle xr_context = xr->runtime->context;
+  ListBase *trackers = &xr->runtime->session_state.trackers;
+
+  wm_xr_controller_model_draw(settings, xr_context, trackers);
+  wm_xr_controller_aim_draw(settings, trackers);
 }

source/blender/windowmanager/xr/intern/wm_xr_intern.h

@@ -21,7 +21,6 @@ typedef struct wmXrSessionState {
   float viewer_viewmat[4][4];
   /** The last known viewer matrix, without navigation applied. */
   float viewer_mat_base[4][4];
-  float focal_len;
 
   /** Copy of XrSessionSettings.base_pose_ data to detect changes that need
    * resetting to base pose. */
@@ -50,8 +49,12 @@ typedef struct wmXrSessionState {
   float nav_scale_prev;
   bool is_navigation_dirty;
 
+  /** Last known eye data. */
+  ListBase eyes; /* #wmXrEye */
   /** Last known controller data. */
   ListBase controllers; /* #wmXrController */
+  /** Last known tracker data. */
+  ListBase trackers; /* #wmXrController */
 
   /** The currently active action set that will be updated on calls to
    * #wm_xr_session_actions_update(). If NULL, all action sets will be treated as active and
@@ -59,6 +62,10 @@ typedef struct wmXrSessionState {
   struct wmXrActionSet *active_action_set;
   /* Name of the action set (if any) to activate before the next actions sync. */
   char active_action_set_next[64]; /* MAX_NAME */
+
+  /* Original poses for motion capture objects. Used to restore object transforms on session end.
+   */
+  ListBase mocap_orig_poses; /* #wmXrMotionCapturePose */
 } wmXrSessionState;
 
 typedef struct wmXrRuntimeData {
@@ -74,10 +81,6 @@ typedef struct wmXrRuntimeData {
   /** Although this struct is internal, RNA gets a handle to this for state information queries. */
   wmXrSessionState session_state;
   wmXrSessionExitFn exit_fn;
-
-  ListBase actionmaps; /* #XrActionMap */
-  short actactionmap;
-  short selactionmap;
 } wmXrRuntimeData;
 
 typedef struct wmXrViewportPair {
@@ -94,6 +97,8 @@ typedef struct {
   struct ARegionType *controller_art;
   /** Controller draw callback handle. */
   void *controller_draw_handle;
+  /** Tracker draw callback handle. */
+  void *tracker_draw_handle;
 } wmXrSurfaceData;
 
 typedef struct wmXrDrawData {
@@ -114,6 +119,14 @@ typedef struct wmXrDrawData {
   float eye_position_ofs[3]; /* Local/view space. */
 } wmXrDrawData;
 
+typedef struct wmXrEye {
+  struct wmXrEye *next, *prev;
+  float focal_len;
+  float viewmat[4][4];
+  /** Viewmat without navigation applied. */
+  float viewmat_base[4][4];
+} wmXrEye;
+
 typedef struct wmXrController {
   struct wmXrController *next, *prev;
   /** OpenXR user path identifier. */
@@ -153,6 +166,9 @@ typedef struct wmXrAction {
   struct wmOperatorType *ot;
   IDProperty *op_properties;
 
+  /** Mouse simulation. */
+  XrSimulateMouseParams simulate;
+
   /** Haptics. */
   char *haptic_name;
   int64_t haptic_duration;
@@ -178,20 +194,26 @@ typedef struct wmXrActionSet {
   /** XR pose actions that determine the controller grip/aim transforms. */
   wmXrAction *controller_grip_action;
   wmXrAction *controller_aim_action;
+  /** XR pose actions that determine tracker transforms. */
+  ListBase tracker_actions; /* #LinkData */
 
   /** Currently active modal actions. */
-  ListBase active_modal_actions;
+  ListBase active_modal_actions; /* #LinkData */
   /** Currently active haptic actions. */
-  ListBase active_haptic_actions;
+  ListBase active_haptic_actions; /* wmXrHapticAction */
 } wmXrActionSet;
 
-/* wm_xr.c */
+typedef struct wmXrMotionCapturePose {
+  struct wmXrMotionCapturePose *next, *prev;
+  const Object *ob;
+  GHOST_XrPose pose;
+} wmXrMotionCapturePose;
 
+/* wm_xr.c */
 wmXrRuntimeData *wm_xr_runtime_data_create(void);
 void wm_xr_runtime_data_free(wmXrRuntimeData **runtime);
 
 /* wm_xr_session.c */
-
 void wm_xr_session_data_free(wmXrSessionState *state);
 wmWindow *wm_xr_session_root_window_or_fallback_get(const wmWindowManager *wm,
                                                     const wmXrRuntimeData *runtime_data);
@@ -207,6 +229,8 @@ void wm_xr_session_draw_data_update(wmXrSessionState *state,
 void wm_xr_session_state_update(const XrSessionSettings *settings,
                                 const wmXrDrawData *draw_data,
                                 const GHOST_XrDrawViewInfo *draw_view,
+                                const float viewmat[4][4],
+                                const float viewmat_base[4][4],
                                 wmXrSessionState *state);
 bool wm_xr_session_surface_offscreen_ensure(wmXrSurfaceData *surface_data,
                                             const GHOST_XrDrawViewInfo *draw_view);
@@ -214,14 +238,15 @@ void *wm_xr_session_gpu_binding_context_create(void);
 void wm_xr_session_gpu_binding_context_destroy(GHOST_ContextHandle context);
 
 void wm_xr_session_actions_init(wmXrData *xr);
-void wm_xr_session_actions_update(wmWindowManager *wm);
+void wm_xr_session_actions_update(const struct bContext *C);
 void wm_xr_session_controller_data_populate(const wmXrAction *grip_action,
                                             const wmXrAction *aim_action,
                                             wmXrData *xr);
 void wm_xr_session_controller_data_clear(wmXrSessionState *state);
+void wm_xr_session_tracker_data_populate(const ListBase *tracker_actions, wmXrData *xr);
+void wm_xr_session_tracker_data_clear(wmXrSessionState *state);
 
 /* wm_xr_draw.c */
-
 void wm_xr_pose_to_mat(const GHOST_XrPose *pose, float r_mat[4][4]);
 void wm_xr_pose_scale_to_mat(const GHOST_XrPose *pose, float scale, float r_mat[4][4]);
 void wm_xr_pose_to_imat(const GHOST_XrPose *pose, float r_imat[4][4]);
@@ -234,3 +259,23 @@ void wm_xr_pose_scale_to_imat(const GHOST_XrPose *pose, float scale, float r_ima
  */
 void wm_xr_draw_view(const GHOST_XrDrawViewInfo *draw_view, void *customdata);
 void wm_xr_draw_controllers(const struct bContext *C, struct ARegion *region, void *customdata);
+void wm_xr_draw_trackers(const struct bContext *C, struct ARegion *region, void *customdata);
+
+/* wm_xr_mocap.c */
+void wm_xr_mocap_orig_poses_store(const XrSessionSettings *settings, wmXrSessionState *state);
+void wm_xr_mocap_orig_poses_restore(const wmXrSessionState *state, XrSessionSettings *settings);
+void wm_xr_mocap_object_autokey(struct bContext *C,
+                                struct Scene *scene,
+                                struct ViewLayer *view_layer,
+                                wmWindow *win,
+                                Object *ob,
+                                bool notify);
+void wm_xr_mocap_objects_update(const char *user_path,
+                                const GHOST_XrPose *pose,
+                                struct bContext *C,
+                                XrSessionSettings *settings,
+                                struct Scene *scene,
+                                struct ViewLayer *view_layer,
+                                wmWindow *win,
+                                struct bScreen *screen_anim,
+                                bool *notify);

source/blender/windowmanager/xr/intern/wm_xr_mocap.c

@@ -0,0 +1,272 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/** \file
+ * \ingroup wm
+ *
+ * \name Window-Manager XR Motion Capture
+ *
+ * API for XR motion capture objects.
+ */
+
+#include "BKE_context.h"
+#include "BKE_layer.h"
+
+#include "BLI_listbase.h"
+#include "BLI_math.h"
+
+#include "DEG_depsgraph.h"
+
+#include "DNA_object_types.h"
+
+#include "ED_keyframing.h"
+#include "ED_object.h"
+#include "ED_screen.h"
+#include "ED_transform.h"
+
+#include "GHOST_C-api.h"
+
+#include "MEM_guardedalloc.h"
+
+#include "WM_api.h"
+
+#include "wm_xr_intern.h"
+
+/* -------------------------------------------------------------------- */
+/** \name XR Motion Capture Objects
+ *
+ * List of XR motion capture objects. Stored in session settings and can be written to files.
+ * \{ */
+
+XrMotionCaptureObject *WM_xr_mocap_object_new(XrSessionSettings *settings, Object *ob)
+{
+  XrMotionCaptureObject *mocap_ob = WM_xr_mocap_object_find(settings, ob);
+
+  if (!mocap_ob) {
+    mocap_ob = MEM_callocN(sizeof(XrMotionCaptureObject), __func__);
+    mocap_ob->ob = ob;
+    BLI_addtail(&settings->mocap_objects, mocap_ob);
+  }
+
+  return mocap_ob;
+}
+
+void WM_xr_mocap_object_ensure_unique(XrSessionSettings *settings, XrMotionCaptureObject *mocap_ob)
+{
+  LISTBASE_FOREACH (XrMotionCaptureObject *, mocap_ob_other, &settings->mocap_objects) {
+    if ((mocap_ob_other != mocap_ob) && (mocap_ob_other->ob == mocap_ob->ob)) {
+      mocap_ob->ob = NULL;
+      return;
+    }
+  }
+}
+
+void WM_xr_mocap_object_remove(XrSessionSettings *settings, XrMotionCaptureObject *mocap_ob)
+{
+  int idx = BLI_findindex(&settings->mocap_objects, mocap_ob);
+
+  if (idx != -1) {
+    BLI_freelinkN(&settings->mocap_objects, mocap_ob);
+
+    if (idx <= settings->sel_mocap_object) {
+      if (--settings->sel_mocap_object < 0) {
+        settings->sel_mocap_object = 0;
+      }
+    }
+  }
+}
+
+XrMotionCaptureObject *WM_xr_mocap_object_find(const XrSessionSettings *settings, const Object *ob)
+{
+  return ob ? BLI_findptr(&settings->mocap_objects, ob, offsetof(XrMotionCaptureObject, ob)) :
+              NULL;
+}
+
+/** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name Motion Capture Runtime
+ *
+ * Runtime functions for motion capture object poses and autokeying.
+ * \{ */
+
+static void wm_xr_mocap_object_pose_get(const XrMotionCaptureObject *mocap_ob, GHOST_XrPose *pose)
+{
+  BLI_assert(mocap_ob->ob);
+  copy_v3_v3(pose->position, mocap_ob->ob->loc);
+  eul_to_quat(pose->orientation_quat, mocap_ob->ob->rot);
+}
+
+static void wm_xr_mocap_object_pose_set(const GHOST_XrPose *pose,
+                                        XrMotionCaptureObject *mocap_ob,
+                                        bool apply_offset)
+{
+  BLI_assert(mocap_ob->ob);
+
+  if (apply_offset) {
+    /* Convert offsets to pose (device) space. */
+    float loc_ofs[3], rot_ofs[4];
+    copy_v3_v3(loc_ofs, mocap_ob->location_offset);
+    mul_qt_v3(pose->orientation_quat, loc_ofs);
+
+    eul_to_quat(rot_ofs, mocap_ob->rotation_offset);
+    normalize_qt(rot_ofs);
+    invert_qt_normalized(rot_ofs);
+    mul_qt_qtqt(rot_ofs, pose->orientation_quat, rot_ofs);
+    normalize_qt(rot_ofs);
+
+    add_v3_v3v3(mocap_ob->ob->loc, pose->position, loc_ofs);
+    quat_to_eul(mocap_ob->ob->rot, rot_ofs);
+  }
+  else {
+    copy_v3_v3(mocap_ob->ob->loc, pose->position);
+    quat_to_eul(mocap_ob->ob->rot, pose->orientation_quat);
+  }
+
+  DEG_id_tag_update(&mocap_ob->ob->id, ID_RECALC_TRANSFORM);
+}
+
+void wm_xr_mocap_orig_poses_store(const XrSessionSettings *settings, wmXrSessionState *state)
+{
+  ListBase *mocap_orig_poses = &state->mocap_orig_poses;
+
+  LISTBASE_FOREACH (XrMotionCaptureObject *, mocap_ob, &settings->mocap_objects) {
+    wmXrMotionCapturePose *mocap_pose = MEM_callocN(sizeof(wmXrMotionCapturePose), __func__);
+    mocap_pose->ob = mocap_ob->ob;
+    if (mocap_ob->ob) {
+      wm_xr_mocap_object_pose_get(mocap_ob, &mocap_pose->pose);
+    }
+    BLI_addtail(mocap_orig_poses, mocap_pose);
+  }
+}
+
+void wm_xr_mocap_orig_poses_restore(const wmXrSessionState *state, XrSessionSettings *settings)
+{
+  ListBase *mocap_objects = &settings->mocap_objects;
+
+  LISTBASE_FOREACH (wmXrMotionCapturePose *, mocap_pose, &state->mocap_orig_poses) {
+    XrMotionCaptureObject *mocap_ob = BLI_findptr(
+        mocap_objects, mocap_pose->ob, offsetof(XrMotionCaptureObject, ob));
+    if (mocap_ob && mocap_ob->ob) {
+      wm_xr_mocap_object_pose_set(&mocap_pose->pose, mocap_ob, false);
+    }
+  }
+}
+
+void wm_xr_mocap_object_autokey(
+    bContext *C, Scene *scene, ViewLayer *view_layer, wmWindow *win, Object *ob, bool notify)
+{
+  /* Poll functions in keyingsets_utils.py require an active window and object. */
+  wmWindow *win_prev = win ? CTX_wm_window(C) : NULL;
+  if (win) {
+    CTX_wm_window_set(C, win);
+  }
+
+  Object *obact = CTX_data_active_object(C);
+  if (!obact) {
+    Base *base = BKE_view_layer_base_find(view_layer, ob);
+    if (base) {
+      ED_object_base_select(base, BA_SELECT);
+      ED_object_base_activate(C, base);
+    }
+  }
+
+  bScreen *screen = CTX_wm_screen(C);
+  if (screen && screen->animtimer && (IS_AUTOKEY_FLAG(scene, INSERTAVAIL) == 0) &&
+      ((scene->toolsettings->autokey_flag & ANIMRECORD_FLAG_WITHNLA) != 0)) {
+    ED_transform_animrecord_check_state(scene, screen->animtimer, &ob->id);
+  }
+
+  ED_transform_autokeyframe_object(C, scene, view_layer, ob, TFM_TRANSLATION);
+  if (IS_AUTOKEY_FLAG(scene, INSERTNEEDED)) {
+    ED_transform_autokeyframe_object(C, scene, view_layer, ob, TFM_ROTATION);
+  }
+
+  if (ED_transform_motionpath_need_update_object(scene, ob)) {
+    ListBase lb = {NULL, NULL};
+    BLI_addtail(&lb, BLI_genericNodeN(ob));
+
+    ED_objects_recalculate_paths(C, scene, OBJECT_PATH_CALC_RANGE_CURRENT_FRAME, &lb);
+
+    BLI_freelistN(&lb);
+  }
+
+  if (notify) {
+    WM_event_add_notifier(C, NC_OBJECT | ND_KEYS, NULL);
+    WM_main_add_notifier(NC_ANIMATION | ND_KEYFRAME | NA_EDITED, NULL);
+  }
+
+  if (win) {
+    CTX_wm_window_set(C, win_prev);
+  }
+}
+
+void wm_xr_mocap_objects_update(const char *user_path,
+                                const GHOST_XrPose *pose,
+                                bContext *C,
+                                XrSessionSettings *settings,
+                                Scene *scene,
+                                ViewLayer *view_layer,
+                                wmWindow *win,
+                                bScreen *screen_anim,
+                                bool *notify)
+{
+  LISTBASE_FOREACH (XrMotionCaptureObject *, mocap_ob, &settings->mocap_objects) {
+    if (mocap_ob->ob && ((mocap_ob->flag & XR_MOCAP_OBJECT_ENABLE) != 0) &&
+        STREQ(mocap_ob->user_path, user_path)) {
+      wm_xr_mocap_object_pose_set(pose, mocap_ob, true);
+
+      if (((mocap_ob->flag & XR_MOCAP_OBJECT_AUTOKEY) != 0) && screen_anim &&
+          autokeyframe_cfra_can_key(scene, &mocap_ob->ob->id)) {
+        wm_xr_mocap_object_autokey(C, scene, view_layer, win, mocap_ob->ob, *notify);
+        *notify = false;
+      }
+    }
+  }
+}
+
+bool WM_xr_session_state_mocap_pose_get(const wmXrData *xr, XrMotionCaptureObject *mocap_ob)
+{
+  if (!WM_xr_session_exists(xr)) {
+    return false;
+  }
+
+  if (mocap_ob->ob) {
+    wmXrMotionCapturePose *mocap_pose = BLI_findptr(&xr->runtime->session_state.mocap_orig_poses,
+                                                    mocap_ob->ob,
+                                                    offsetof(wmXrMotionCapturePose, ob));
+    if (!mocap_pose) {
+      return false;
+    }
+    wm_xr_mocap_object_pose_set(&mocap_pose->pose, mocap_ob, false);
+  }
+
+  return true;
+}
+
+void WM_xr_session_state_mocap_pose_set(wmXrData *xr, const XrMotionCaptureObject *mocap_ob)
+{
+  if (WM_xr_session_exists(xr) && mocap_ob->ob) {
+    wmXrMotionCapturePose *mocap_pose = BLI_findptr(&xr->runtime->session_state.mocap_orig_poses,
+                                                    mocap_ob->ob,
+                                                    offsetof(wmXrMotionCapturePose, ob));
+    if (mocap_pose) {
+      wm_xr_mocap_object_pose_get(mocap_ob, &mocap_pose->pose);
+    }
+  }
+}
+
+/** \} */

source/blender/windowmanager/xr/intern/wm_xr_session.c

@@ -25,6 +25,7 @@
 
 #include "ED_screen.h"
 #include "ED_space_api.h"
+#include "ED_view3d.h"
 
 #include "GHOST_C-api.h"
 
@@ -43,6 +44,9 @@
 #include "wm_window.h"
 #include "wm_xr_intern.h"
 
+/* OpenXR user path identifying the headset. Used for motion capture objects. */
+#define XR_HEADSET_PATH "/user/head"
+
 static wmSurface *g_xr_surface = NULL;
 static CLG_LogRef LOG = {"wm.xr"};
 
@@ -67,24 +71,28 @@ static void wm_xr_session_create_cb(void)
     settings->base_scale = 1.0f;
   }
   state->prev_base_scale = settings->base_scale;
+
+  /* Store motion capture object poses. */
+  wm_xr_mocap_orig_poses_store(settings, state);
 }
 
-static void wm_xr_session_controller_data_free(wmXrSessionState *state)
+static void wm_xr_session_controller_data_free(ListBase *controllers)
 {
-  ListBase *lb = &state->controllers;
   wmXrController *c;
-
-  while ((c = BLI_pophead(lb))) {
+  while ((c = BLI_pophead(controllers))) {
     if (c->model) {
       GPU_batch_discard(c->model);
     }
-    BLI_freelinkN(lb, c);
+    BLI_freelinkN(controllers, c);
   }
 }
 
 void wm_xr_session_data_free(wmXrSessionState *state)
 {
-  wm_xr_session_controller_data_free(state);
+  BLI_freelistN(&state->eyes);
+  wm_xr_session_controller_data_free(&state->controllers);
+  wm_xr_session_controller_data_free(&state->trackers);
+  BLI_freelistN(&state->mocap_orig_poses);
 }
 
 static void wm_xr_session_exit_cb(void *customdata)
@@ -96,6 +104,9 @@ static void wm_xr_session_exit_cb(void *customdata)
 
   xr_data->runtime->session_state.is_started = false;
 
+  /* Restore motion capture object poses. */
+  wm_xr_mocap_orig_poses_restore(&xr_data->runtime->session_state, &xr_data->session_settings);
+
   if (xr_data->runtime->exit_fn) {
     xr_data->runtime->exit_fn(xr_data);
   }
@@ -335,11 +346,23 @@ void wm_xr_session_draw_data_update(wmXrSessionState *state,
 void wm_xr_session_state_update(const XrSessionSettings *settings,
                                 const wmXrDrawData *draw_data,
                                 const GHOST_XrDrawViewInfo *draw_view,
+                                const float viewmat[4][4],
+                                const float viewmat_base[4][4],
                                 wmXrSessionState *state)
 {
   GHOST_XrPose viewer_pose;
   float viewer_mat[4][4], base_mat[4][4], nav_mat[4][4];
 
+  wmXrEye *eye = NULL;
+  if (draw_view->view_idx >= BLI_listbase_count(&state->eyes)) {
+    eye = MEM_callocN(sizeof(*eye), __func__);
+    BLI_addtail(&state->eyes, eye);
+  }
+  else {
+    eye = BLI_findlink(&state->eyes, draw_view->view_idx);
+  }
+  BLI_assert(eye);
+
   /* Calculate viewer matrix. */
   copy_qt_qt(viewer_pose.orientation_quat, draw_view->local_pose.orientation_quat);
   if ((settings->flag & XR_SESSION_USE_POSITION_TRACKING) == 0) {
@@ -365,9 +388,11 @@ void wm_xr_session_state_update(const XrSessionSettings *settings,
       &state->viewer_pose, draw_data->base_scale * state->nav_scale_prev, state->viewer_viewmat);
 
   /* No idea why, but multiplying by two seems to make it match the VR view more. */
-  state->focal_len = 2.0f *
-                     fov_to_focallength(draw_view->fov.angle_right - draw_view->fov.angle_left,
-                                        DEFAULT_SENSOR_WIDTH);
+  eye->focal_len = 2.0f *
+                   fov_to_focallength(draw_view->fov.angle_right - draw_view->fov.angle_left,
+                                      DEFAULT_SENSOR_WIDTH);
+  copy_m4_m4(eye->viewmat, viewmat);
+  copy_m4_m4(eye->viewmat_base, viewmat_base);
 
   copy_v3_v3(state->prev_eye_position_ofs, draw_data->eye_position_ofs);
   memcpy(&state->prev_base_pose, &draw_data->base_pose, sizeof(state->prev_base_pose));
@@ -426,7 +451,16 @@ bool WM_xr_session_state_viewer_pose_matrix_info_get(const wmXrData *xr,
   }
 
   copy_m4_m4(r_viewmat, xr->runtime->session_state.viewer_viewmat);
-  *r_focal_len = xr->runtime->session_state.focal_len;
+
+  /* Since viewer (eye centroid) does not have a focal length, just take it from projection eye. */
+  const wmXrEye *eye = BLI_findlink(&xr->runtime->session_state.eyes,
+                                    xr->session_settings.projection_eye);
+  if (!eye) {
+    /* Fall back to first XR view. */
+    eye = xr->runtime->session_state.eyes.first;
+    BLI_assert(eye);
+  }
+  *r_focal_len = eye->focal_len;
 
   return true;
 }
@@ -499,6 +533,48 @@ bool WM_xr_session_state_controller_aim_rotation_get(const wmXrData *xr,
   return true;
 }
 
+bool WM_xr_session_state_tracker_location_get(const wmXrData *xr,
+                                              const char *subaction_path,
+                                              float r_location[3])
+{
+  if (!WM_xr_session_is_ready(xr) || !xr->runtime->session_state.is_view_data_set) {
+    zero_v3(r_location);
+    return false;
+  }
+
+  const wmXrController *tracker = BLI_findstring(&xr->runtime->session_state.trackers,
+                                                 subaction_path,
+                                                 offsetof(wmXrController, subaction_path));
+  if (!tracker) {
+    zero_v3(r_location);
+    return false;
+  }
+
+  copy_v3_v3(r_location, tracker->grip_pose.position);
+  return true;
+}
+
+bool WM_xr_session_state_tracker_rotation_get(const wmXrData *xr,
+                                              const char *subaction_path,
+                                              float r_rotation[4])
+{
+  if (!WM_xr_session_is_ready(xr) || !xr->runtime->session_state.is_view_data_set) {
+    unit_qt(r_rotation);
+    return false;
+  }
+
+  const wmXrController *tracker = BLI_findstring(&xr->runtime->session_state.trackers,
+                                                 subaction_path,
+                                                 offsetof(wmXrController, subaction_path));
+  if (!tracker) {
+    unit_qt(r_rotation);
+    return false;
+  }
+
+  copy_qt_qt(r_rotation, tracker->grip_pose.orientation_quat);
+  return true;
+}
+
 bool WM_xr_session_state_nav_location_get(const wmXrData *xr, float r_location[3])
 {
   if (!WM_xr_session_is_ready(xr) || !xr->runtime->session_state.is_view_data_set) {
@@ -606,15 +682,79 @@ static void wm_xr_session_controller_pose_calc(const GHOST_XrPose *raw_pose,
   mat4_to_loc_quat(r_pose->position, r_pose->orientation_quat, r_mat);
 }
 
-static void wm_xr_session_controller_data_update(const XrSessionSettings *settings,
-                                                 const wmXrAction *grip_action,
+static void wm_xr_session_controller_data_update(const wmXrAction *grip_action,
                                                  const wmXrAction *aim_action,
+                                                 unsigned int subaction_idx,
+                                                 const float view_ofs[3],
+                                                 const float base_mat[4][4],
+                                                 const float nav_mat[4][4],
+                                                 bContext *C,
+                                                 XrSessionSettings *settings,
                                                  GHOST_XrContextHandle xr_context,
-                                                 wmXrSessionState *state)
+                                                 Scene *scene,
+                                                 ViewLayer *view_layer,
+                                                 wmWindow *win,
+                                                 bScreen *screen_anim,
+                                                 bool *notify,
+                                                 wmXrController *controller)
+{
+  wm_xr_session_controller_pose_calc(&((GHOST_XrPose *)grip_action->states)[subaction_idx],
+                                     view_ofs,
+                                     base_mat,
+                                     nav_mat,
+                                     &controller->grip_pose,
+                                     controller->grip_mat,
+                                     controller->grip_mat_base);
+  wm_xr_session_controller_pose_calc(&((GHOST_XrPose *)aim_action->states)[subaction_idx],
+                                     view_ofs,
+                                     base_mat,
+                                     nav_mat,
+                                     &controller->aim_pose,
+                                     controller->aim_mat,
+                                     controller->aim_mat_base);
+
+  /* Update motion capture objects. */
+  wm_xr_mocap_objects_update(controller->subaction_path,
+                             &controller->grip_pose,
+                             C,
+                             settings,
+                             scene,
+                             view_layer,
+                             win,
+                             screen_anim,
+                             notify);
+
+  /* Update controller model. */
+  if (!controller->model) {
+    /* Notify GHOST to load/continue loading the controller model data. This can be called more
+     * than once since the model may not be available from the runtime yet. The batch itself will
+     * be created in wm_xr_draw_controllers(). */
+    GHOST_XrLoadControllerModel(xr_context, controller->subaction_path);
+  }
+  else {
+    GHOST_XrUpdateControllerModelComponents(xr_context, controller->subaction_path);
+  }
+}
+
+static void wm_xr_session_controller_and_tracker_data_update(const wmXrAction *grip_action,
+                                                             const wmXrAction *aim_action,
+                                                             const ListBase *tracker_actions,
+                                                             bContext *C,
+                                                             XrSessionSettings *settings,
+                                                             GHOST_XrContextHandle xr_context,
+                                                             wmXrSessionState *state,
+                                                             wmWindow *win)
 {
   BLI_assert(grip_action->count_subaction_paths == aim_action->count_subaction_paths);
   BLI_assert(grip_action->count_subaction_paths == BLI_listbase_count(&state->controllers));
+  BLI_assert(BLI_listbase_count(tracker_actions) <= BLI_listbase_count(&state->trackers));
 
+  Scene *scene = CTX_data_scene(C);
+  ViewLayer *view_layer = CTX_data_view_layer(C);
+  wmWindowManager *wm = CTX_wm_manager(C);
+  bScreen *screen_anim = ED_screen_animation_playing(wm);
+  bool notify = true;
+  wmXrController *tracker = state->trackers.first;
   uint subaction_idx = 0;
   float view_ofs[3], base_mat[4][4], nav_mat[4][4];
 
@@ -631,30 +771,47 @@ static void wm_xr_session_controller_data_update(const XrSessionSettings *settin
   wm_xr_pose_scale_to_mat(&state->prev_base_pose, state->prev_base_scale, base_mat);
   wm_xr_pose_scale_to_mat(&state->nav_pose, state->nav_scale, nav_mat);
 
+  /* Update controllers. */
   LISTBASE_FOREACH_INDEX (wmXrController *, controller, &state->controllers, subaction_idx) {
-    wm_xr_session_controller_pose_calc(&((GHOST_XrPose *)grip_action->states)[subaction_idx],
-                                       view_ofs,
-                                       base_mat,
-                                       nav_mat,
-                                       &controller->grip_pose,
-                                       controller->grip_mat,
-                                       controller->grip_mat_base);
-    wm_xr_session_controller_pose_calc(&((GHOST_XrPose *)aim_action->states)[subaction_idx],
-                                       view_ofs,
-                                       base_mat,
-                                       nav_mat,
-                                       &controller->aim_pose,
-                                       controller->aim_mat,
-                                       controller->aim_mat_base);
+    wm_xr_session_controller_data_update(grip_action,
+                                         aim_action,
+                                         subaction_idx,
+                                         view_ofs,
+                                         base_mat,
+                                         nav_mat,
+                                         C,
+                                         settings,
+                                         xr_context,
+                                         scene,
+                                         view_layer,
+                                         win,
+                                         screen_anim,
+                                         &notify,
+                                         controller);
+  }
 
-    if (!controller->model) {
-      /* Notify GHOST to load/continue loading the controller model data. This can be called more
-       * than once since the model may not be available from the runtime yet. The batch itself will
-       * be created in wm_xr_draw_controllers(). */
-      GHOST_XrLoadControllerModel(xr_context, controller->subaction_path);
-    }
-    else {
-      GHOST_XrUpdateControllerModelComponents(xr_context, controller->subaction_path);
+  /* Update trackers. */
+  LISTBASE_FOREACH (LinkData *, ld, tracker_actions) {
+    const wmXrAction *tracker_action = ld->data;
+
+    for (subaction_idx = 0; subaction_idx < tracker_action->count_subaction_paths;
+         ++subaction_idx) {
+      wm_xr_session_controller_data_update(tracker_action,
+                                           tracker_action,
+                                           subaction_idx,
+                                           view_ofs,
+                                           base_mat,
+                                           nav_mat,
+                                           C,
+                                           settings,
+                                           xr_context,
+                                           scene,
+                                           view_layer,
+                                           win,
+                                           screen_anim,
+                                           &notify,
+                                           tracker);
+      tracker = tracker->next;
     }
   }
 }
@@ -822,7 +979,8 @@ static void wm_xr_session_action_states_interpret(wmXrData *xr,
                                                   int64_t time_now,
                                                   bool modal,
                                                   bool haptic,
-                                                  short *r_val)
+                                                  short *r_val,
+                                                  bool *r_press_start)
 {
   const char *haptic_subaction_path = ((action->haptic_flag & XR_HAPTIC_MATCHUSERPATHS) != 0) ?
                                           action->subaction_paths[subaction_idx] :
@@ -884,6 +1042,7 @@ static void wm_xr_session_action_states_interpret(wmXrData *xr,
     if (!prev) {
       if (modal || (action->op_flag == XR_OP_PRESS)) {
         *r_val = KM_PRESS;
+        *r_press_start = true;
       }
       if (haptic && (action->haptic_flag & (XR_HAPTIC_PRESS | XR_HAPTIC_REPEAT)) != 0) {
         /* Apply haptics. */
@@ -901,6 +1060,7 @@ static void wm_xr_session_action_states_interpret(wmXrData *xr,
     }
     else if (modal) {
       *r_val = KM_PRESS;
+      *r_press_start = false;
     }
     if (modal && !action->active_modal_path) {
       /* Set active modal path. */
@@ -928,6 +1088,7 @@ static void wm_xr_session_action_states_interpret(wmXrData *xr,
   else if (prev) {
     if (modal || (action->op_flag == XR_OP_RELEASE)) {
       *r_val = KM_RELEASE;
+      *r_press_start = false;
       if (modal && (action->subaction_paths[subaction_idx] == action->active_modal_path)) {
         /* Unset active modal path. */
         action->active_modal_path = NULL;
@@ -1020,7 +1181,9 @@ static wmXrActionData *wm_xr_session_event_create(const char *action_set_name,
                                                   const GHOST_XrPose *controller_aim_pose_other,
                                                   uint subaction_idx,
                                                   uint subaction_idx_other,
-                                                  bool bimanual)
+                                                  bool bimanual,
+                                                  short val,
+                                                  bool press_start)
 {
   wmXrActionData *data = MEM_callocN(sizeof(wmXrActionData), __func__);
   strcpy(data->action_set, action_set_name);
@@ -1079,10 +1242,56 @@ static wmXrActionData *wm_xr_session_event_create(const char *action_set_name,
   data->op_properties = action->op_properties;
 
   data->bimanual = bimanual;
+  data->simulate_mouse = (((action->action_flag & XR_ACTION_SIMULATE_MOUSE) != 0) &&
+                          controller_aim_pose);
+  if (data->simulate_mouse) {
+    switch (val) {
+      case KM_PRESS:
+        if (press_start) {
+          data->simulate_type = action->simulate.press_type;
+          data->simulate_val = action->simulate.press_val;
+        }
+        else {
+          data->simulate_type = action->simulate.hold_type;
+          data->simulate_val = action->simulate.hold_val;
+        }
+        break;
+      case KM_RELEASE:
+        data->simulate_type = action->simulate.release_type;
+        data->simulate_val = action->simulate.release_val;
+        break;
+      default:
+        BLI_assert_unreachable();
+        break;
+    }
+  }
 
   return data;
 }
 
+/**
+ * Convert 3D controller coordinates to 2D mouse (pixel) coordinates.
+ */
+static void map_to_pixel(
+    const float co[3], const float persmat[4][4], int winx, int winy, int r_co[2])
+{
+  float vec[3];
+  mul_v3_project_m4_v3(vec, persmat, co);
+  r_co[0] = (int)(((float)winx / 2.0f) * (1.0f + vec[0]));
+  r_co[1] = (int)(((float)winy / 2.0f) * (1.0f + vec[1]));
+}
+
+static void wm_xr_session_event_mval_calc(const wmXrRuntimeData *runtime,
+                                          const GHOST_XrPose *controller_aim_pose,
+                                          int r_mval[2])
+{
+  const ARegion *region = BKE_area_find_region_type(runtime->area, RGN_TYPE_WINDOW);
+  BLI_assert(region);
+  const RegionView3D *rv3d = region->regiondata;
+  BLI_assert(rv3d);
+  map_to_pixel(controller_aim_pose->position, rv3d->persmat, region->winx, region->winy, r_mval);
+}
+
 /* Dispatch events to window queues. */
 static void wm_xr_session_events_dispatch(wmXrData *xr,
                                           GHOST_XrContextHandle xr_context,
@@ -1091,19 +1300,16 @@ static void wm_xr_session_events_dispatch(wmXrData *xr,
                                           wmWindow *win)
 {
   const char *action_set_name = action_set->name;
-
   const uint count = GHOST_XrGetActionCount(xr_context, action_set_name);
   if (count < 1) {
     return;
   }
 
   const int64_t time_now = (int64_t)(PIL_check_seconds_timer() * 1000);
-
   ListBase *active_modal_actions = &action_set->active_modal_actions;
   ListBase *active_haptic_actions = &action_set->active_haptic_actions;
 
   wmXrAction **actions = MEM_calloc_arrayN(count, sizeof(*actions), __func__);
-
   GHOST_XrGetActionCustomdataArray(xr_context, action_set_name, (void **)actions);
 
   /* Check haptic action timers. */
@@ -1111,62 +1317,148 @@ static void wm_xr_session_events_dispatch(wmXrData *xr,
 
   for (uint action_idx = 0; action_idx < count; ++action_idx) {
     wmXrAction *action = actions[action_idx];
-    if (action && action->ot) {
-      const bool modal = action->ot->modal;
-      const bool haptic = (GHOST_XrGetActionCustomdata(
-                               xr_context, action_set_name, action->haptic_name) != NULL);
+    if (!action || !action->ot) {
+      continue;
+    }
 
-      for (uint subaction_idx = 0; subaction_idx < action->count_subaction_paths;
-           ++subaction_idx) {
-        short val = KM_NOTHING;
+    const bool modal = action->ot->modal;
+    const bool haptic = (GHOST_XrGetActionCustomdata(
+                             xr_context, action_set_name, action->haptic_name) != NULL);
 
-        /* Interpret action states (update modal/haptic action lists, apply haptics, etc). */
-        wm_xr_session_action_states_interpret(xr,
-                                              action_set_name,
-                                              action,
-                                              subaction_idx,
-                                              active_modal_actions,
-                                              active_haptic_actions,
-                                              time_now,
-                                              modal,
-                                              haptic,
-                                              &val);
+    for (uint subaction_idx = 0; subaction_idx < action->count_subaction_paths; ++subaction_idx) {
+      short val = KM_NOTHING;
+      bool press_start = false;
 
+      /* Interpret action states (update modal/haptic action lists, apply haptics, etc). */
+      wm_xr_session_action_states_interpret(xr,
+                                            action_set_name,
+                                            action,
+                                            subaction_idx,
+                                            active_modal_actions,
+                                            active_haptic_actions,
+                                            time_now,
+                                            modal,
+                                            haptic,
+                                            &val,
+                                            &press_start);
+      if (val == KM_NOTHING) {
+        continue;
+      }
+
+      if (modal) {
         const bool is_active_modal_action = wm_xr_session_modal_action_test(
             active_modal_actions, action, NULL);
+        if (!is_active_modal_action) {
+          continue;
+        }
         const bool is_active_modal_subaction = (!action->active_modal_path ||
                                                 (action->subaction_paths[subaction_idx] ==
                                                  action->active_modal_path));
-
-        if ((val != KM_NOTHING) &&
-            (!modal || (is_active_modal_action && is_active_modal_subaction))) {
-          const GHOST_XrPose *aim_pose = wm_xr_session_controller_aim_pose_find(
-              session_state, action->subaction_paths[subaction_idx]);
-          const GHOST_XrPose *aim_pose_other = NULL;
-          uint subaction_idx_other = 0;
-
-          /* Test for bimanual interaction. */
-          const bool bimanual = wm_xr_session_action_test_bimanual(
-              session_state, action, subaction_idx, &subaction_idx_other, &aim_pose_other);
-
-          wmXrActionData *actiondata = wm_xr_session_event_create(action_set_name,
-                                                                  action,
-                                                                  aim_pose,
-                                                                  aim_pose_other,
-                                                                  subaction_idx,
-                                                                  subaction_idx_other,
-                                                                  bimanual);
-          wm_event_add_xrevent(win, actiondata, val);
+        if (!is_active_modal_subaction) {
+          continue;
         }
       }
+
+      const GHOST_XrPose *aim_pose = wm_xr_session_controller_aim_pose_find(
+          session_state, action->subaction_paths[subaction_idx]);
+      const GHOST_XrPose *aim_pose_other = NULL;
+      unsigned int subaction_idx_other = 0;
+      int mval[2];
+
+      /* Test for bimanual interaction. */
+      const bool bimanual = wm_xr_session_action_test_bimanual(
+          session_state, action, subaction_idx, &subaction_idx_other, &aim_pose_other);
+
+      wmXrActionData *actiondata = wm_xr_session_event_create(action_set_name,
+                                                              action,
+                                                              aim_pose,
+                                                              aim_pose_other,
+                                                              subaction_idx,
+                                                              subaction_idx_other,
+                                                              bimanual,
+                                                              val,
+                                                              press_start);
+      /* Calculate simulated mouse coordinates. */
+      if (actiondata->simulate_mouse) {
+        wm_xr_session_event_mval_calc(xr->runtime, aim_pose, mval);
+      }
+
+      wm_event_add_xrevent(win, actiondata, val, actiondata->simulate_mouse ? mval : NULL);
     }
   }
 
   MEM_freeN(actions);
 }
 
-void wm_xr_session_actions_update(wmWindowManager *wm)
+static bool wm_xr_session_area_ensure(const XrSessionSettings *settings,
+                                      const wmXrSessionState *state,
+                                      const wmXrSurfaceData *surface_data,
+                                      wmWindowManager *wm,
+                                      Scene *scene,
+                                      Depsgraph *depsgraph,
+                                      wmWindow *win)
 {
+  if (settings->projection_eye >= BLI_listbase_count(&state->eyes) ||
+      settings->projection_eye >= BLI_listbase_count(&surface_data->viewports)) {
+    return false;
+  }
+
+  const wmXrEye *eye = BLI_findlink(&state->eyes, settings->projection_eye);
+  BLI_assert(eye);
+  const wmXrViewportPair *vp = BLI_findlink(&surface_data->viewports, settings->projection_eye);
+  BLI_assert(vp && vp->offscreen);
+  const int width = GPU_offscreen_width(vp->offscreen);
+  const int height = GPU_offscreen_height(vp->offscreen);
+
+  wmXrRuntimeData *runtime = wm->xr.runtime;
+  wmWindow *win_prev = wm->windrawable;
+
+  /* Ensure an XR area exists for events. */
+  if (!runtime->area) {
+    runtime->area = ED_area_offscreen_create(win, SPACE_VIEW3D);
+  }
+
+  ARegion *region = BKE_area_find_region_type(runtime->area, RGN_TYPE_WINDOW);
+  BLI_assert(region);
+  View3D *v3d = (View3D *)runtime->area->spacedata.first;
+  BLI_assert(v3d);
+  RegionView3D *rv3d = region->regiondata;
+  BLI_assert(rv3d);
+
+  /* Ensure region dimensions and view params match the XR (projection eye) view.*/
+  runtime->area->winx = width;
+  runtime->area->winy = height;
+
+  region->winx = width;
+  region->winy = height;
+  region->winrct.xmin = 0;
+  region->winrct.xmax = width - 1;
+  region->winrct.ymin = 0;
+  region->winrct.ymax = height - 1;
+
+  v3d->clip_start = settings->clip_start;
+  v3d->clip_end = settings->clip_end;
+  v3d->object_type_exclude_viewport = settings->object_type_exclude_viewport;
+  v3d->object_type_exclude_select = settings->object_type_exclude_select;
+  rv3d->persp = RV3D_PERSP;
+
+  wm_window_make_drawable(wm, win);
+
+  ED_view3d_update_viewmat(depsgraph, scene, v3d, region, eye->viewmat, NULL, NULL, true);
+
+  if (win_prev) {
+    wm_window_make_drawable(wm, win_prev);
+  }
+  else {
+    wm_window_clear_drawable(wm);
+  }
+
+  return true;
+}
+
+void wm_xr_session_actions_update(const bContext *C)
+{
+  wmWindowManager *wm = CTX_wm_manager(C);
   wmXrData *xr = &wm->xr;
   if (!xr->runtime) {
     return;
@@ -1175,6 +1467,7 @@ void wm_xr_session_actions_update(wmWindowManager *wm)
   XrSessionSettings *settings = &xr->session_settings;
   GHOST_XrContextHandle xr_context = xr->runtime->context;
   wmXrSessionState *state = &xr->runtime->session_state;
+  wmWindow *win = wm_xr_session_root_window_or_fallback_get(wm, xr->runtime);
 
   if (state->is_navigation_dirty) {
     memcpy(&state->nav_pose_prev, &state->nav_pose, sizeof(state->nav_pose_prev));
@@ -1189,6 +1482,11 @@ void wm_xr_session_actions_update(wmWindowManager *wm)
     mat4_to_loc_quat(state->viewer_pose.position, state->viewer_pose.orientation_quat, viewer_mat);
     wm_xr_pose_scale_to_imat(
         &state->viewer_pose, settings->base_scale * state->nav_scale, state->viewer_viewmat);
+
+    wm_xr_pose_scale_to_imat(&state->nav_pose, state->nav_scale, m);
+    LISTBASE_FOREACH (wmXrEye *, eye, &state->eyes) {
+      mul_m4_m4m4(eye->viewmat, eye->viewmat_base, m);
+    }
   }
 
   /* Set active action set if requested previously. */
@@ -1198,6 +1496,23 @@ void wm_xr_session_actions_update(wmWindowManager *wm)
   }
   wmXrActionSet *active_action_set = state->active_action_set;
 
+  /* Update headset motion capture objects. */
+  {
+    Scene *scene = CTX_data_scene(C);
+    ViewLayer *view_layer = CTX_data_view_layer(C);
+    bScreen *screen_anim = ED_screen_animation_playing(wm);
+    bool notify = true;
+    wm_xr_mocap_objects_update(XR_HEADSET_PATH,
+                               &state->viewer_pose,
+                               (bContext *)C,
+                               settings,
+                               scene,
+                               view_layer,
+                               win,
+                               screen_anim,
+                               &notify);
+  }
+
   const bool synced = GHOST_XrSyncActions(xr_context,
                                           active_action_set ? active_action_set->name : NULL);
   if (!synced) {
@@ -1206,28 +1521,27 @@ void wm_xr_session_actions_update(wmWindowManager *wm)
 
   /* Only update controller data and dispatch events for active action set. */
   if (active_action_set) {
-    wmWindow *win = wm_xr_session_root_window_or_fallback_get(wm, xr->runtime);
-
-    if (active_action_set->controller_grip_action && active_action_set->controller_aim_action) {
-      wm_xr_session_controller_data_update(settings,
-                                           active_action_set->controller_grip_action,
-                                           active_action_set->controller_aim_action,
-                                           xr_context,
-                                           state);
+    if ((active_action_set->controller_grip_action && active_action_set->controller_aim_action) ||
+        !BLI_listbase_is_empty(&active_action_set->tracker_actions)) {
+      wm_xr_session_controller_and_tracker_data_update(active_action_set->controller_grip_action,
+                                                       active_action_set->controller_aim_action,
+                                                       &active_action_set->tracker_actions,
+                                                       (bContext *)C,
+                                                       settings,
+                                                       xr_context,
+                                                       state,
+                                                       win);
     }
 
-    if (win) {
-      /* Ensure an XR area exists for events. */
-      if (!xr->runtime->area) {
-        xr->runtime->area = ED_area_offscreen_create(win, SPACE_VIEW3D);
+    if (win && g_xr_surface) {
+      Scene *scene;
+      Depsgraph *depsgraph;
+      wm_xr_session_scene_and_depsgraph_get(wm, &scene, &depsgraph);
+
+      if (wm_xr_session_area_ensure(
+              settings, state, g_xr_surface->customdata, wm, scene, depsgraph, win)) {
+        wm_xr_session_events_dispatch(xr, xr_context, active_action_set, state, win);
       }
-
-      /* Set XR area object type flags for operators. */
-      View3D *v3d = xr->runtime->area->spacedata.first;
-      v3d->object_type_exclude_viewport = settings->object_type_exclude_viewport;
-      v3d->object_type_exclude_select = settings->object_type_exclude_select;
-
-      wm_xr_session_events_dispatch(xr, xr_context, active_action_set, state, win);
     }
   }
 }
@@ -1240,15 +1554,13 @@ void wm_xr_session_controller_data_populate(const wmXrAction *grip_action,
 
   wmXrSessionState *state = &xr->runtime->session_state;
   ListBase *controllers = &state->controllers;
-
   BLI_assert(grip_action->count_subaction_paths == aim_action->count_subaction_paths);
   const uint count = grip_action->count_subaction_paths;
 
-  wm_xr_session_controller_data_free(state);
+  wm_xr_session_controller_data_free(controllers);
 
   for (uint i = 0; i < count; ++i) {
     wmXrController *controller = MEM_callocN(sizeof(*controller), __func__);
-
     BLI_assert(STREQ(grip_action->subaction_paths[i], aim_action->subaction_paths[i]));
     strcpy(controller->subaction_path, grip_action->subaction_paths[i]);
 
@@ -1269,7 +1581,7 @@ void wm_xr_session_controller_data_populate(const wmXrAction *grip_action,
 
 void wm_xr_session_controller_data_clear(wmXrSessionState *state)
 {
-  wm_xr_session_controller_data_free(state);
+  wm_xr_session_controller_data_free(&state->controllers);
 
   /* Deactivate draw callback. */
   if (g_xr_surface) {
@@ -1283,6 +1595,53 @@ void wm_xr_session_controller_data_clear(wmXrSessionState *state)
   }
 }
 
+void wm_xr_session_tracker_data_populate(const ListBase *tracker_actions, wmXrData *xr)
+{
+  wmXrSessionState *state = &xr->runtime->session_state;
+  ListBase *trackers = &state->trackers;
+
+  wm_xr_session_controller_data_free(trackers);
+
+  LISTBASE_FOREACH (const LinkData *, ld, tracker_actions) {
+    const wmXrAction *tracker_action = ld->data;
+
+    for (unsigned int subaction_idx = 0; subaction_idx < tracker_action->count_subaction_paths;
+         ++subaction_idx) {
+      wmXrController *tracker = MEM_callocN(sizeof(*tracker), __func__);
+      strcpy(tracker->subaction_path, tracker_action->subaction_paths[subaction_idx]);
+
+      BLI_addtail(trackers, tracker);
+    }
+  }
+
+  /* Activate draw callback. */
+  if (g_xr_surface) {
+    wmXrSurfaceData *surface_data = g_xr_surface->customdata;
+    if (surface_data && !surface_data->tracker_draw_handle) {
+      if (surface_data->controller_art) {
+        surface_data->tracker_draw_handle = ED_region_draw_cb_activate(
+            surface_data->controller_art, wm_xr_draw_trackers, xr, REGION_DRAW_POST_VIEW);
+      }
+    }
+  }
+}
+
+void wm_xr_session_tracker_data_clear(wmXrSessionState *state)
+{
+  wm_xr_session_controller_data_free(&state->trackers);
+
+  /* Deactivate draw callback. */
+  if (g_xr_surface) {
+    wmXrSurfaceData *surface_data = g_xr_surface->customdata;
+    if (surface_data && surface_data->tracker_draw_handle) {
+      if (surface_data->controller_art) {
+        ED_region_draw_cb_exit(surface_data->controller_art, surface_data->tracker_draw_handle);
+      }
+      surface_data->tracker_draw_handle = NULL;
+    }
+  }
+}
+
 /** \} */ /* XR-Session Actions */
 
 /* -------------------------------------------------------------------- */

source/blender/windowmanager/xr/wm_xr.h

@@ -6,6 +6,7 @@
 
 #pragma once
 
+struct bContext;
 struct wmWindowManager;
 struct wmXrData;
 
@@ -16,7 +17,7 @@ typedef void (*wmXrSessionExitFn)(const wmXrData *xr_data);
 bool wm_xr_init(wmWindowManager *wm);
 void wm_xr_exit(wmWindowManager *wm);
 void wm_xr_session_toggle(wmWindowManager *wm, wmWindow *win, wmXrSessionExitFn session_exit_fn);
-bool wm_xr_events_handle(wmWindowManager *wm);
+bool wm_xr_events_handle(const struct bContext *C);
 
 /* wm_xr_operators.c */