Experiment: DNA: makesdna parser #118532

Open
Guillermo Venegas wants to merge 29 commits from guishe/blender:dna-parser into main

When changing the target branch, be careful to rebase the branch in your fork to match. See documentation.
Contributor

Experimental initial DNA Parser. This attempts to remove some
limitations in DNA files, like using #define int values,
or for c++ files to allow use enums with fixed type, or eventually namespaces.

This initial state attempts to parse same as makesdna has been doing,
but allowing to easily extend the parser, so in the future we can parse more
declarations like namespaces, o default value initialization in structs members.

Instead of preprocessing the file, now the text remains as read from the file
and a lexer gather all tokens, ignoring white spaces or comments and parser tries
to match a sequence of tokens to build definitions that will be used by makesdna.

Using the plain text as it is read from the file, this can be used now too to print
where errors have occurred, like alignment or unsupported types.

Notes:

  1. #if/ifdef/ifndef .... #endif blocks are ignored except
    #ifdef DNA_DEPRECATED_ALLOW ... #endif blocks in the main scope.

In action:

It currently parses everything like makedna has been doing, but can now also parse enums with fixed types or const int defines, even structs defined within structs (although these definitions remain unused for the time being).

Plain code Parser debug result.

/*----------------------------------------------------------------*/
/** #pragma once and includes */

#pragma once

#include "...."
#include "DNA_ID.h"
#include "DNA_armature_types.h"
#include "DNA_listBase.h"
#include "DNA_session_uid_types.h"
#include "DNA_userdef_types.h"
#include "DNA_vec_types.h"

// Forward declarations
struct Collection;
struct GHash;
struct Object, SpaceLink;

#if 0
typedef enum class UnusedEnum: uint8_t {
  /* vert is selected */
  UNUSED_1 = (1 + 0*FILE_MAX),
  UNUSED_2 = (1 << 1),
} UnusedEnum;
ENUM_OPERATORS(UnusedEnum, UNUSED_2);
#endif


/* Const int define */
#define FILE_MAX 1024

/* define non const int*/
#define DNA_DEFINE_CXX_METHODS() ....

/** bMotionPathVert, taken from DNA_action_Types.h
 * modified with a child struct. */
typedef struct bMotionPathVert {
  DNA_DEFINE_CXX_METHODS(bMotionPathVert);
  /* Child struct */
  struct bMotionPathVertItem {
    DNA_DEFINE_CXX_METHODS(bMotionPathVert);
    float co[3], pre[235];
    struct Link *next, *pre;
    char path[FILE_MAX];
    bool (*poll)(bContext *, ARegion *);
    float (*data_src)[256];
  };
  /** Coordinates of point in 3D-space. */
  float co[3];
  /** Quick settings. */
  int flag;
} bMotionPathVert;

/** eMotionPathVert_Flag, taken from DNA_action_Types.h
 * modified with a fixed size. */
typedef enum class eMotionPathVert_Flag : uint8_t {
  /* vert is selected */
  VERT_SEL = (1 << 0),
  VERT_KEY = (1 << 1),
} eMotionPathVert_Flag;
ENUM_OPERATORS(eMotionPathVert_Flag, VERT_KEY);


#define FILE_MAX 1024
struct bMotionPathVert {
    struct bMotionPathVertItem {
        float co[3],pre[235];
        Link *next,*pre;
        char path[FILE_MAX];
        bool (*poll)(...);
        float (*data_src)[256];
    };
    float co[3];
    int flag;
};
enum eMotionPathVert_Flag: uint8_t {...};
Experimental initial DNA Parser. This attempts to remove some limitations in DNA files, like using `#define` int values, or for c++ files to allow use enums with fixed type, or eventually namespaces. This initial state attempts to parse same as makesdna has been doing, but allowing to easily extend the parser, so in the future we can parse more declarations like namespaces, o default value initialization in structs members. Instead of `preprocessing` the file, now the text remains as read from the file and a lexer gather all tokens, ignoring white spaces or comments and parser tries to match a sequence of tokens to build definitions that will be used by makesdna. Using the plain text as it is read from the file, this can be used now too to print where errors have occurred, like alignment or unsupported types. Notes: 1. `#if/ifdef/ifndef .... #endif` blocks are ignored except `#ifdef DNA_DEPRECATED_ALLOW ... #endif` blocks in the main scope. <hr> ### In action: It currently parses everything like makedna has been doing, but can now also parse enums with fixed types or const int defines, even structs defined within structs (although these definitions remain unused for the time being). <table> <tr > <td >Plain code</td> <td>Parser debug result.</td> </tr> <tr> <td> ```cpp /*----------------------------------------------------------------*/ /** #pragma once and includes */ #pragma once #include "...." #include "DNA_ID.h" #include "DNA_armature_types.h" #include "DNA_listBase.h" #include "DNA_session_uid_types.h" #include "DNA_userdef_types.h" #include "DNA_vec_types.h" // Forward declarations struct Collection; struct GHash; struct Object, SpaceLink; #if 0 typedef enum class UnusedEnum: uint8_t { /* vert is selected */ UNUSED_1 = (1 + 0*FILE_MAX), UNUSED_2 = (1 << 1), } UnusedEnum; ENUM_OPERATORS(UnusedEnum, UNUSED_2); #endif /* Const int define */ #define FILE_MAX 1024 /* define non const int*/ #define DNA_DEFINE_CXX_METHODS() .... /** bMotionPathVert, taken from DNA_action_Types.h * modified with a child struct. */ typedef struct bMotionPathVert { DNA_DEFINE_CXX_METHODS(bMotionPathVert); /* Child struct */ struct bMotionPathVertItem { DNA_DEFINE_CXX_METHODS(bMotionPathVert); float co[3], pre[235]; struct Link *next, *pre; char path[FILE_MAX]; bool (*poll)(bContext *, ARegion *); float (*data_src)[256]; }; /** Coordinates of point in 3D-space. */ float co[3]; /** Quick settings. */ int flag; } bMotionPathVert; /** eMotionPathVert_Flag, taken from DNA_action_Types.h * modified with a fixed size. */ typedef enum class eMotionPathVert_Flag : uint8_t { /* vert is selected */ VERT_SEL = (1 << 0), VERT_KEY = (1 << 1), } eMotionPathVert_Flag; ENUM_OPERATORS(eMotionPathVert_Flag, VERT_KEY); ``` </td> <td> ```cpp #define FILE_MAX 1024 struct bMotionPathVert { struct bMotionPathVertItem { float co[3],pre[235]; Link *next,*pre; char path[FILE_MAX]; bool (*poll)(...); float (*data_src)[256]; }; float co[3]; int flag; }; enum eMotionPathVert_Flag: uint8_t {...}; ``` </td> </tr> </table>
Guillermo Venegas added 1 commit 2024-02-20 20:04:51 +01:00
Guillermo Venegas changed title from Experiment: WIP: DNA: makedna parser to Experiment: WIP: DNA: makesdna parser 2024-02-20 20:19:59 +01:00
Guillermo Venegas added 1 commit 2024-02-20 20:32:00 +01:00
Guillermo Venegas added 2 commits 2024-02-29 16:09:55 +01:00
Iliya Katushenock added this to the Core project 2024-02-29 16:53:45 +01:00
Guillermo Venegas added 5 commits 2024-03-02 17:27:00 +01:00
Guillermo Venegas added 4 commits 2024-03-05 15:03:09 +01:00
Guillermo Venegas added 3 commits 2024-03-05 20:03:45 +01:00
Guillermo Venegas added 3 commits 2024-03-07 15:02:37 +01:00
Guillermo Venegas added 1 commit 2024-03-08 15:40:30 +01:00
Guillermo Venegas added 1 commit 2024-03-08 18:44:11 +01:00
Guillermo Venegas added 2 commits 2024-03-08 19:38:45 +01:00
Merge remote-tracking branch 'origin/main' into dna-parser
Some checks failed
buildbot/vexp-code-patch-darwin-x86_64 Build done.
buildbot/vexp-code-patch-lint Build done.
buildbot/vexp-code-patch-windows-amd64 Build done.
buildbot/vexp-code-patch-linux-x86_64 Build done.
buildbot/vexp-code-patch-darwin-arm64 Build done.
buildbot/vexp-code-patch-coordinator Build done.
7badaa70d5
Guillermo Venegas changed title from Experiment: WIP: DNA: makesdna parser to Experiment: DNA: makesdna parser 2024-03-08 20:01:44 +01:00
Guillermo Venegas requested review from Jacques Lucke 2024-03-11 14:49:44 +01:00
Guillermo Venegas reviewed 2024-03-11 14:52:41 +01:00
@ -1140,3 +1140,3 @@
/* Temp caching of UI-generated strings. */
char size_str[16];
char datetime_str[16 + 8];
char datetime_str[DATETIME_STR_SIZE];
Author
Contributor

Although it is ignored by the parser, even the ignored structures are parsed then discarded, expressions like sums would not be supported to parse

Although it is ignored by the parser, even the ignored structures are parsed then discarded, expressions like sums would not be supported to parse
Member

Thanks for the patch. I haven't looked at it in detail yet. I was wondering whether this is really a better approach than trying to use clang to parse the files as was tried by @LazyDodo afaik.

@blender-bot build

Thanks for the patch. I haven't looked at it in detail yet. I was wondering whether this is really a better approach than trying to use clang to parse the files as was tried by @LazyDodo afaik. @blender-bot build
Guillermo Venegas added 2 commits 2024-03-11 16:03:20 +01:00
Member

I abandoned it because clang/llvm is an optional dep for us, we can't rely on it fore a core component like makesdna , it would be a much much nicer solution though.

I abandoned it because clang/llvm is an optional dep for us, we can't rely on it fore a core component like makesdna , it would be a _much_ _much_ nicer solution though.
Guillermo Venegas added 2 commits 2024-03-12 03:01:27 +01:00
Guillermo Venegas added 2 commits 2024-03-12 15:52:08 +01:00
Merge remote-tracking branch 'origin/main' into dna-parser
Some checks failed
buildbot/vexp-code-patch-lint Build done.
buildbot/vexp-code-patch-linux-x86_64 Build done.
buildbot/vexp-code-patch-windows-amd64 Build done.
buildbot/vexp-code-patch-darwin-arm64 Build done.
buildbot/vexp-code-patch-darwin-x86_64 Build done.
buildbot/vexp-code-patch-coordinator Build done.
993e4da650
Member

@blender-bot build

@blender-bot build
Some checks failed
buildbot/vexp-code-patch-lint Build done.
buildbot/vexp-code-patch-linux-x86_64 Build done.
buildbot/vexp-code-patch-windows-amd64 Build done.
buildbot/vexp-code-patch-darwin-arm64 Build done.
buildbot/vexp-code-patch-darwin-x86_64 Build done.
buildbot/vexp-code-patch-coordinator Build done.
This pull request has changes conflicting with the target branch.
  • source/blender/makesdna/intern/CMakeLists.txt

Checkout

From your project repository, check out a new branch and test the changes.
git fetch -u dna-parser:guishe-dna-parser
git checkout guishe-dna-parser
Sign in to join this conversation.
No reviewers
No Label
Interest
Alembic
Interest
Animation & Rigging
Interest
Asset System
Interest
Audio
Interest
Automated Testing
Interest
Blender Asset Bundle
Interest
BlendFile
Interest
Collada
Interest
Compatibility
Interest
Compositing
Interest
Core
Interest
Cycles
Interest
Dependency Graph
Interest
Development Management
Interest
EEVEE
Interest
EEVEE & Viewport
Interest
Freestyle
Interest
Geometry Nodes
Interest
Grease Pencil
Interest
ID Management
Interest
Images & Movies
Interest
Import Export
Interest
Line Art
Interest
Masking
Interest
Metal
Interest
Modeling
Interest
Modifiers
Interest
Motion Tracking
Interest
Nodes & Physics
Interest
OpenGL
Interest
Overlay
Interest
Overrides
Interest
Performance
Interest
Physics
Interest
Pipeline, Assets & IO
Interest
Platforms, Builds & Tests
Interest
Python API
Interest
Render & Cycles
Interest
Render Pipeline
Interest
Sculpt, Paint & Texture
Interest
Text Editor
Interest
Translations
Interest
Triaging
Interest
Undo
Interest
USD
Interest
User Interface
Interest
UV Editing
Interest
VFX & Video
Interest
Video Sequencer
Interest
Virtual Reality
Interest
Vulkan
Interest
Wayland
Interest
Workbench
Interest: X11
Legacy
Asset Browser Project
Legacy
Blender 2.8 Project
Legacy
Milestone 1: Basic, Local Asset Browser
Legacy
OpenGL Error
Meta
Good First Issue
Meta
Papercut
Meta
Retrospective
Meta
Security
Module
Animation & Rigging
Module
Core
Module
Development Management
Module
EEVEE & Viewport
Module
Grease Pencil
Module
Modeling
Module
Nodes & Physics
Module
Pipeline, Assets & IO
Module
Platforms, Builds & Tests
Module
Python API
Module
Render & Cycles
Module
Sculpt, Paint & Texture
Module
Triaging
Module
User Interface
Module
VFX & Video
Platform
FreeBSD
Platform
Linux
Platform
macOS
Platform
Windows
Priority
High
Priority
Low
Priority
Normal
Priority
Unbreak Now!
Status
Archived
Status
Confirmed
Status
Duplicate
Status
Needs Info from Developers
Status
Needs Information from User
Status
Needs Triage
Status
Resolved
Type
Bug
Type
Design
Type
Known Issue
Type
Patch
Type
Report
Type
To Do
No Milestone
No project
No Assignees
3 Participants
Notifications
Due Date
The due date is invalid or out of range. Please use the format 'yyyy-mm-dd'.

No due date set.

Dependencies

No dependencies set.

Reference: blender/blender#118532
No description provided.