Experiment: DNA: makesdna parser #118532

Guillermo Venegas · 2024-02-20T20:04:41+01:00

Guillermo Venegas commented

2024-02-20 20:04:41 +01:00

Experimental initial DNA Parser. This attempts to remove some
limitations in DNA files, like using #define int values,
or for c++ files to allow use enums with fixed type, or eventually namespaces.

This initial state attempts to parse same as makesdna has been doing,
but allowing to easily extend the parser, so in the future we can parse more
declarations like namespaces, o default value initialization in structs members.

Instead of preprocessing the file, now the text remains as read from the file
and a lexer gather all tokens, ignoring white spaces or comments and parser tries
to match a sequence of tokens to build definitions that will be used by makesdna.

Using the plain text as it is read from the file, this can be used now too to print
where errors have occurred, like alignment or unsupported types.

Notes:

#if/ifdef/ifndef .... #endif blocks are ignored except
#ifdef DNA_DEPRECATED_ALLOW ... #endif blocks in the main scope.

In action:

It currently parses everything like makedna has been doing, but can now also parse enums with fixed types or const int defines, even structs defined within structs (although these definitions remain unused for the time being).

Plain code Parser debug result.


/*----------------------------------------------------------------*/
/** #pragma once and includes */

#pragma once

#include "...."
#include "DNA_ID.h"
#include "DNA_armature_types.h"
#include "DNA_listBase.h"
#include "DNA_session_uid_types.h"
#include "DNA_userdef_types.h"
#include "DNA_vec_types.h"

// Forward declarations
struct Collection;
struct GHash;
struct Object, SpaceLink;

#if 0
typedef enum class UnusedEnum: uint8_t {
  /* vert is selected */
  UNUSED_1 = (1 + 0*FILE_MAX),
  UNUSED_2 = (1 << 1),
} UnusedEnum;
ENUM_OPERATORS(UnusedEnum, UNUSED_2);
#endif


/* Const int define */
#define FILE_MAX 1024

/* define non const int*/
#define DNA_DEFINE_CXX_METHODS() ....

/** bMotionPathVert, taken from DNA_action_Types.h
 * modified with a child struct. */
typedef struct bMotionPathVert {
  DNA_DEFINE_CXX_METHODS(bMotionPathVert);
  /* Child struct */
  struct bMotionPathVertItem {
    DNA_DEFINE_CXX_METHODS(bMotionPathVert);
    float co[3], pre[235];
    struct Link *next, *pre;
    char path[FILE_MAX];
    bool (*poll)(bContext *, ARegion *);
    float (*data_src)[256];
  };
  /** Coordinates of point in 3D-space. */
  float co[3];
  /** Quick settings. */
  int flag;
} bMotionPathVert;

/** eMotionPathVert_Flag, taken from DNA_action_Types.h
 * modified with a fixed size. */
typedef enum class eMotionPathVert_Flag : uint8_t {
  /* vert is selected */
  VERT_SEL = (1 << 0),
  VERT_KEY = (1 << 1),
} eMotionPathVert_Flag;
ENUM_OPERATORS(eMotionPathVert_Flag, VERT_KEY);

#define FILE_MAX 1024
struct bMotionPathVert {
    struct bMotionPathVertItem {
        float co[3],pre[235];
        Link *next,*pre;
        char path[FILE_MAX];
        bool (*poll)(...);
        float (*data_src)[256];
    };
    float co[3];
    int flag;
};
enum eMotionPathVert_Flag: uint8_t {...};

Experimental initial DNA Parser. This attempts to remove some limitations in DNA files, like using `#define` int values, or for c++ files to allow use enums with fixed type, or eventually namespaces. This initial state attempts to parse same as makesdna has been doing, but allowing to easily extend the parser, so in the future we can parse more declarations like namespaces, o default value initialization in structs members. Instead of `preprocessing` the file, now the text remains as read from the file and a lexer gather all tokens, ignoring white spaces or comments and parser tries to match a sequence of tokens to build definitions that will be used by makesdna. Using the plain text as it is read from the file, this can be used now too to print where errors have occurred, like alignment or unsupported types. Notes: 1. `#if/ifdef/ifndef .... #endif` blocks are ignored except `#ifdef DNA_DEPRECATED_ALLOW ... #endif` blocks in the main scope. <hr> ### In action: It currently parses everything like makedna has been doing, but can now also parse enums with fixed types or const int defines, even structs defined within structs (although these definitions remain unused for the time being). <table> <tr > <td >Plain code</td> <td>Parser debug result.</td> </tr> <tr> <td> ```cpp /*----------------------------------------------------------------*/ /** #pragma once and includes */ #pragma once #include "...." #include "DNA_ID.h" #include "DNA_armature_types.h" #include "DNA_listBase.h" #include "DNA_session_uid_types.h" #include "DNA_userdef_types.h" #include "DNA_vec_types.h" // Forward declarations struct Collection; struct GHash; struct Object, SpaceLink; #if 0 typedef enum class UnusedEnum: uint8_t { /* vert is selected */ UNUSED_1 = (1 + 0*FILE_MAX), UNUSED_2 = (1 << 1), } UnusedEnum; ENUM_OPERATORS(UnusedEnum, UNUSED_2); #endif /* Const int define */ #define FILE_MAX 1024 /* define non const int*/ #define DNA_DEFINE_CXX_METHODS() .... /** bMotionPathVert, taken from DNA_action_Types.h * modified with a child struct. */ typedef struct bMotionPathVert { DNA_DEFINE_CXX_METHODS(bMotionPathVert); /* Child struct */ struct bMotionPathVertItem { DNA_DEFINE_CXX_METHODS(bMotionPathVert); float co[3], pre[235]; struct Link *next, *pre; char path[FILE_MAX]; bool (*poll)(bContext *, ARegion *); float (*data_src)[256]; }; /** Coordinates of point in 3D-space. */ float co[3]; /** Quick settings. */ int flag; } bMotionPathVert; /** eMotionPathVert_Flag, taken from DNA_action_Types.h * modified with a fixed size. */ typedef enum class eMotionPathVert_Flag : uint8_t { /* vert is selected */ VERT_SEL = (1 << 0), VERT_KEY = (1 << 1), } eMotionPathVert_Flag; ENUM_OPERATORS(eMotionPathVert_Flag, VERT_KEY); ``` </td> <td> ```cpp #define FILE_MAX 1024 struct bMotionPathVert { struct bMotionPathVertItem { float co[3],pre[235]; Link *next,*pre; char path[FILE_MAX]; bool (*poll)(...); float (*data_src)[256]; }; float co[3]; int flag; }; enum eMotionPathVert_Flag: uint8_t {...}; ``` </td> </tr> </table>

Guillermo Venegas added 1 commit 2024-02-20 20:04:51 +01:00

5e0d3f7128 experiment

Guillermo Venegas changed title from ~~Experiment: WIP: DNA: makedna parser~~ to Experiment: WIP: DNA: makesdna parser

2024-02-20 20:19:59 +01:00

Guillermo Venegas added 1 commit 2024-02-20 20:32:00 +01:00

a1b076c41c typo

Guillermo Venegas added 2 commits 2024-02-29 16:09:55 +01:00

8315b0a37e Merge remote-tracking branch 'origin/main' into dna-parser

e3551cd446 progress

Iliya Katushenock added this to the Core project 2024-02-29 16:53:45 +01:00

Guillermo Venegas added 5 commits 2024-03-02 17:27:00 +01:00

34cdd0e6a1 progress

f87bc4ea66 skip externs vars

ebce05b925 pointer to arrays

cb533d87bb dynamic arrays

c7f61638ce printf,format,deduplication

Guillermo Venegas added 4 commits 2024-03-05 15:03:09 +01:00

bde5ef051e fix erro in prev commit

c2031d5bed recursive structs and macro calls

91a6e5e6c1 cleanup

9d2e73889f use the parser in makesdna, formating and a lot of comments

Guillermo Venegas added 3 commits 2024-03-05 20:03:45 +01:00

7675a3a6b9 Merge remote-tracking branch 'origin/main' into dna-parser

de95f155d2 cleanup, add back a warning

a4a4fb9aba Merge remote-tracking branch 'origin/main' into dna-parser

Guillermo Venegas added 3 commits 2024-03-07 15:02:37 +01:00

2854ccd703 unbreak linux build

899896f4ea allow dna deprecated

84327ea635 Merge remote-tracking branch 'origin/main' into dna-parser

Guillermo Venegas added 1 commit 2024-03-08 15:40:30 +01:00

51e298e33c fix bool to int conversion

Guillermo Venegas added 1 commit 2024-03-08 18:44:11 +01:00

49d1a6d6cf cleanup

Guillermo Venegas added 2 commits 2024-03-08 19:38:45 +01:00

3d35ebcde9 fix error printf

buildbot/vexp-code-patch-darwin-x86_64 Build done. Details

buildbot/vexp-code-patch-lint Build done. Details

buildbot/vexp-code-patch-windows-amd64 Build done. Details

buildbot/vexp-code-patch-linux-x86_64 Build done. Details

buildbot/vexp-code-patch-darwin-arm64 Build done. Details

buildbot/vexp-code-patch-coordinator Build done. Details

7badaa70d5 Merge remote-tracking branch 'origin/main' into dna-parser

Guillermo Venegas changed title from ~~Experiment: WIP: DNA: makesdna parser~~ to Experiment: DNA: makesdna parser

2024-03-08 20:01:44 +01:00

Guillermo Venegas requested review from Jacques Lucke 2024-03-11 14:49:44 +01:00

Guillermo Venegas reviewed 2024-03-11 14:52:41 +01:00

source/blender/makesdna/DNA_space_types.h

						
				@ -1140,3 +1140,3 @@

				    /* Temp caching of UI-generated strings. */

				    char size_str[16];

				    char datetime_str[16 + 8];

				    char datetime_str[DATETIME_STR_SIZE];

Guillermo Venegas commented

2024-03-11 14:52:41 +01:00

Although it is ignored by the parser, even the ignored structures are parsed then discarded, expressions like sums would not be supported to parse

Jacques Lucke commented

2024-03-11 14:57:44 +01:00

Thanks for the patch. I haven't looked at it in detail yet. I was wondering whether this is really a better approach than trying to use clang to parse the files as was tried by @LazyDodo afaik.

@blender-bot build

Thanks for the patch. I haven't looked at it in detail yet. I was wondering whether this is really a better approach than trying to use clang to parse the files as was tried by @LazyDodo afaik. @blender-bot build

Guillermo Venegas added 2 commits 2024-03-11 16:03:20 +01:00

32a4c88f9b Merge remote-tracking branch 'origin/main' into dna-parser

5a80f6a6f8 add updated test

Ray molenkamp commented

2024-03-11 19:51:48 +01:00

I abandoned it because clang/llvm is an optional dep for us, we can't rely on it fore a core component like makesdna , it would be a much much nicer solution though.

I abandoned it because clang/llvm is an optional dep for us, we can't rely on it fore a core component like makesdna , it would be a _much_ _much_ nicer solution though.

Guillermo Venegas added 2 commits 2024-03-12 03:01:27 +01:00

2bbb3d5583 Merge remote-tracking branch 'origin/main' into dna-parser

45254092c5 cleanup

Guillermo Venegas added 2 commits 2024-03-12 15:52:08 +01:00

4476424153 missing include in darwing

buildbot/vexp-code-patch-lint Build done. Details

buildbot/vexp-code-patch-linux-x86_64 Build done. Details

buildbot/vexp-code-patch-windows-amd64 Build done. Details

buildbot/vexp-code-patch-darwin-arm64 Build done. Details

buildbot/vexp-code-patch-darwin-x86_64 Build done. Details

buildbot/vexp-code-patch-coordinator Build done. Details

993e4da650 Merge remote-tracking branch 'origin/main' into dna-parser

Jacques Lucke commented

2024-03-12 17:30:30 +01:00

@blender-bot build

buildbot/vexp-code-patch-lint Build done.

Details

buildbot/vexp-code-patch-linux-x86_64 Build done.

Details

buildbot/vexp-code-patch-windows-amd64 Build done.

Details

buildbot/vexp-code-patch-darwin-arm64 Build done.

Details

buildbot/vexp-code-patch-darwin-x86_64 Build done.

Details

buildbot/vexp-code-patch-coordinator Build done.

Details

Merge conflict checking is in progress. Try again in few moments.

View command line instructions.

Checkout

From your project repository, check out a new branch and test the changes.

git fetch -u dna-parser:guishe-dna-parser

git checkout guishe-dna-parser

Sign in to join this conversation.

No reviewers

No Label

Download

What's New

Blender Studio

Manual

Developers Blog

Documentation

Benchmark

Blender Conference

Development Fund

One-time Donations

Experiment: DNA: makesdna parser #118532

In action:

Checkout