clamp mask resolution, the occasional crash would happen failing to alloc when adding feather points very close together.

source/blender/blenkernel/BKE_mask.h

@@ -214,6 +214,8 @@ void BKE_mask_init_layers(Mask *mask, struct layer_init_data *mlayer_data, int w
 #define MASKPOINT_SEL_HANDLE(p)     { (p)->bezt.f1 |=  SELECT; (p)->bezt.f3 |=  SELECT; } (void)0
 #define MASKPOINT_DESEL_HANDLE(p)   { (p)->bezt.f1 &= ~SELECT; (p)->bezt.f3 &= ~SELECT; } (void)0
 
+#define MASK_RESOL_MAX 128
+
 /* disable to test alternate rasterizer */
 /* #define USE_RASKTER */
 

source/blender/blenkernel/intern/mask.c

@@ -301,6 +301,12 @@ int BKE_mask_spline_resolution(MaskSpline *spline, int width, int height)
 		resol = MAX2(resol, cur_resol);
 	}
 
+	BLI_assert(resol > 0);
+
+	if (resol > MASK_RESOL_MAX) {
+		resol = MASK_RESOL_MAX;
+	}
+
 	return resol;
 }
 
@@ -331,6 +337,12 @@ int BKE_mask_spline_feather_resolution(MaskSpline *spline, int width, int height
 
 	resol += max_jump / max_segment;
 
+	BLI_assert(resol > 0);
+
+	if (resol > MASK_RESOL_MAX) {
+		resol = MASK_RESOL_MAX;
+	}
+
 	return resol;
 }