diff options
| author | Rémi Verschelde | 2017-03-05 16:44:50 +0100 |
|---|---|---|
| committer | Rémi Verschelde | 2017-03-05 16:44:50 +0100 |
| commit | 5dbf1809c6e3e905b94b8764e99491e608122261 (patch) | |
| tree | 5e5a5360db15d86d59ec8c6e4f7eb511388c5a9a /modules/gdscript/gd_tokenizer.cpp | |
| parent | 45438e9918d421b244bfd7776a30e67dc7f2d3e3 (diff) | |
| download | godot-5dbf1809c6e3e905b94b8764e99491e608122261.tar.gz godot-5dbf1809c6e3e905b94b8764e99491e608122261.tar.zst godot-5dbf1809c6e3e905b94b8764e99491e608122261.zip | |
Diffstat (limited to 'modules/gdscript/gd_tokenizer.cpp')
| -rw-r--r-- | modules/gdscript/gd_tokenizer.cpp | 1159 |
1 files changed, 561 insertions, 598 deletions
diff --git a/modules/gdscript/gd_tokenizer.cpp b/modules/gdscript/gd_tokenizer.cpp index 54b9624e8..981924191 100644 --- a/modules/gdscript/gd_tokenizer.cpp +++ b/modules/gdscript/gd_tokenizer.cpp @@ -27,212 +27,205 @@ /* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /*************************************************************************/ #include "gd_tokenizer.h" -#include "print_string.h" #include "gd_functions.h" #include "io/marshalls.h" #include "map.h" +#include "print_string.h" -const char* GDTokenizer::token_names[TK_MAX]={ -"Empty", -"Identifier", -"Constant", -"Self", -"Built-In Type", -"Built-In Func", -"In", -"'=='", -"'!='", -"'<'", -"'<='", -"'>'", -"'>='", -"'and'", -"'or'", -"'not'", -"'+'", -"'-'", -"'*'", -"'/'", -"'%'", -"'<<'", -"'>>'", -"'='", -"'+='", -"'-='", -"'*='", -"'/='", -"'%='", -"'<<='", -"'>>='", -"'&='", -"'|='", -"'^='", -"'&'", -"'|'", -"'^'", -"'~'", -//"Plus Plus", -//"Minus Minus", -"if", -"elif", -"else", -"for", -"do", -"while", -"switch (reserved)", -"case (reserved)", -"break", -"continue", -"pass", -"return", -"match", -"func", -"class", -"extends", -"onready", -"tool", -"static", -"export", -"setget", -"const", -"var", -"enum", -"preload", -"assert", -"yield", -"signal", -"breakpoint", -"rpc", -"sync", -"master", -"slave", -"'['", -"']'", -"'{'", -"'}'", -"'('", -"')'", -"','", -"';'", -"'.'", -"'?'", -"':'", -"'\\n'", -"PI", -"_", -"INF", -"NAN", -"Error", -"EOF", -"Cursor"}; +const char *GDTokenizer::token_names[TK_MAX] = { + "Empty", + "Identifier", + "Constant", + "Self", + "Built-In Type", + "Built-In Func", + "In", + "'=='", + "'!='", + "'<'", + "'<='", + "'>'", + "'>='", + "'and'", + "'or'", + "'not'", + "'+'", + "'-'", + "'*'", + "'/'", + "'%'", + "'<<'", + "'>>'", + "'='", + "'+='", + "'-='", + "'*='", + "'/='", + "'%='", + "'<<='", + "'>>='", + "'&='", + "'|='", + "'^='", + "'&'", + "'|'", + "'^'", + "'~'", + //"Plus Plus", + //"Minus Minus", + "if", + "elif", + "else", + "for", + "do", + "while", + "switch (reserved)", + "case (reserved)", + "break", + "continue", + "pass", + "return", + "match", + "func", + "class", + "extends", + "onready", + "tool", + "static", + "export", + "setget", + "const", + "var", + "enum", + "preload", + "assert", + "yield", + "signal", + "breakpoint", + "rpc", + "sync", + "master", + "slave", + "'['", + "']'", + "'{'", + "'}'", + "'('", + "')'", + "','", + "';'", + "'.'", + "'?'", + "':'", + "'\\n'", + "PI", + "_", + "INF", + "NAN", + "Error", + "EOF", + "Cursor" +}; const char *GDTokenizer::get_token_name(Token p_token) { - ERR_FAIL_INDEX_V(p_token,TK_MAX,"<error>"); + ERR_FAIL_INDEX_V(p_token, TK_MAX, "<error>"); return token_names[p_token]; } static bool _is_text_char(CharType c) { - return (c>='a' && c<='z') || (c>='A' && c<='Z') || (c>='0' && c<='9') || c=='_'; + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; } static bool _is_number(CharType c) { - return (c>='0' && c<='9'); + return (c >= '0' && c <= '9'); } static bool _is_hex(CharType c) { - return (c>='0' && c<='9') || (c>='a' && c<='f') || (c>='A' && c<='F'); + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } -void GDTokenizerText::_make_token(Token p_type) { +void GDTokenizerText::_make_token(Token p_type) { - TokenData &tk=tk_rb[tk_rb_pos]; + TokenData &tk = tk_rb[tk_rb_pos]; - tk.type=p_type; - tk.line=line; - tk.col=column; + tk.type = p_type; + tk.line = line; + tk.col = column; - tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE; + tk_rb_pos = (tk_rb_pos + 1) % TK_RB_SIZE; } -void GDTokenizerText::_make_identifier(const StringName& p_identifier) { +void GDTokenizerText::_make_identifier(const StringName &p_identifier) { - TokenData &tk=tk_rb[tk_rb_pos]; + TokenData &tk = tk_rb[tk_rb_pos]; - tk.type=TK_IDENTIFIER; - tk.identifier=p_identifier; - tk.line=line; - tk.col=column; - - tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE; + tk.type = TK_IDENTIFIER; + tk.identifier = p_identifier; + tk.line = line; + tk.col = column; + tk_rb_pos = (tk_rb_pos + 1) % TK_RB_SIZE; } -void GDTokenizerText::_make_built_in_func(GDFunctions::Function p_func) { - - TokenData &tk=tk_rb[tk_rb_pos]; +void GDTokenizerText::_make_built_in_func(GDFunctions::Function p_func) { - tk.type=TK_BUILT_IN_FUNC; - tk.func=p_func; - tk.line=line; - tk.col=column; + TokenData &tk = tk_rb[tk_rb_pos]; - tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE; + tk.type = TK_BUILT_IN_FUNC; + tk.func = p_func; + tk.line = line; + tk.col = column; + tk_rb_pos = (tk_rb_pos + 1) % TK_RB_SIZE; } -void GDTokenizerText::_make_constant(const Variant& p_constant) { +void GDTokenizerText::_make_constant(const Variant &p_constant) { - TokenData &tk=tk_rb[tk_rb_pos]; + TokenData &tk = tk_rb[tk_rb_pos]; - tk.type=TK_CONSTANT; - tk.constant=p_constant; - tk.line=line; - tk.col=column; - - tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE; + tk.type = TK_CONSTANT; + tk.constant = p_constant; + tk.line = line; + tk.col = column; + tk_rb_pos = (tk_rb_pos + 1) % TK_RB_SIZE; } -void GDTokenizerText::_make_type(const Variant::Type& p_type) { - - - TokenData &tk=tk_rb[tk_rb_pos]; +void GDTokenizerText::_make_type(const Variant::Type &p_type) { - tk.type=TK_BUILT_IN_TYPE; - tk.vtype=p_type; - tk.line=line; - tk.col=column; + TokenData &tk = tk_rb[tk_rb_pos]; - tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE; + tk.type = TK_BUILT_IN_TYPE; + tk.vtype = p_type; + tk.line = line; + tk.col = column; + tk_rb_pos = (tk_rb_pos + 1) % TK_RB_SIZE; } +void GDTokenizerText::_make_error(const String &p_error) { -void GDTokenizerText::_make_error(const String& p_error) { - - error_flag=true; - last_error=p_error; - - TokenData &tk=tk_rb[tk_rb_pos]; - tk.type=TK_ERROR; - tk.constant=p_error; - tk.line=line; - tk.col=column; - tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE; + error_flag = true; + last_error = p_error; + TokenData &tk = tk_rb[tk_rb_pos]; + tk.type = TK_ERROR; + tk.constant = p_error; + tk.line = line; + tk.col = column; + tk_rb_pos = (tk_rb_pos + 1) % TK_RB_SIZE; } - void GDTokenizerText::_make_newline(int p_spaces) { - TokenData &tk=tk_rb[tk_rb_pos]; - tk.type=TK_NEWLINE; - tk.constant=p_spaces; - tk.line=line; - tk.col=column; - tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE; + TokenData &tk = tk_rb[tk_rb_pos]; + tk.type = TK_NEWLINE; + tk.constant = p_spaces; + tk.line = line; + tk.col = column; + tk_rb_pos = (tk_rb_pos + 1) % TK_RB_SIZE; } void GDTokenizerText::_advance() { @@ -243,29 +236,32 @@ void GDTokenizerText::_advance() { return; } - if (code_pos>=len) { + if (code_pos >= len) { _make_token(TK_EOF); return; } -#define GETCHAR(m_ofs) ((m_ofs+code_pos)>=len?0:_code[m_ofs+code_pos]) -#define INCPOS(m_amount) { code_pos+=m_amount; column+=m_amount; } +#define GETCHAR(m_ofs) ((m_ofs + code_pos) >= len ? 0 : _code[m_ofs + code_pos]) +#define INCPOS(m_amount) \ + { \ + code_pos += m_amount; \ + column += m_amount; \ + } while (true) { + bool is_node_path = false; + StringMode string_mode = STRING_DOUBLE_QUOTE; - bool is_node_path = false; - StringMode string_mode=STRING_DOUBLE_QUOTE; - - switch(GETCHAR(0)) { + switch (GETCHAR(0)) { case 0: _make_token(TK_EOF); break; case '\\': INCPOS(1); - if (GETCHAR(0)=='\r') { + if (GETCHAR(0) == '\r') { INCPOS(1); } - if (GETCHAR(0)!='\n') { + if (GETCHAR(0) != '\n') { _make_error("Expected newline after '\\'."); return; } @@ -273,7 +269,7 @@ void GDTokenizerText::_advance() { INCPOS(1); line++; - while(GETCHAR(0)==' ' || GETCHAR(0)=='\t') { + while (GETCHAR(0) == ' ' || GETCHAR(0) == '\t') { INCPOS(1); } @@ -286,9 +282,9 @@ void GDTokenizerText::_advance() { case '\n': { line++; INCPOS(1); - column=1; - int i=0; - while(GETCHAR(i)==' ' || GETCHAR(i)=='\t') { + column = 1; + int i = 0; + while (GETCHAR(i) == ' ' || GETCHAR(i) == '\t') { i++; } @@ -298,19 +294,19 @@ void GDTokenizerText::_advance() { #if 1 //py style tokenizer case '#': { // line comment skip - while(GETCHAR(0)!='\n') { + while (GETCHAR(0) != '\n') { code_pos++; - if (GETCHAR(0)==0) { //end of file + if (GETCHAR(0) == 0) { //end of file //_make_error("Unterminated Comment"); _make_token(TK_EOF); return; } } INCPOS(1); - column=1; + column = 1; line++; - int i=0; - while(GETCHAR(i)==' ' || GETCHAR(i)=='\t') { + int i = 0; + while (GETCHAR(i) == ' ' || GETCHAR(i) == '\t') { i++; } _make_newline(i); @@ -320,7 +316,7 @@ void GDTokenizerText::_advance() { #endif case '/': { - switch(GETCHAR(1)) { + switch (GETCHAR(1)) { #if 0 // c style tokenizer case '*': { // block comment int pos = code_pos+2; @@ -376,11 +372,10 @@ void GDTokenizerText::_advance() { } break; default: _make_token(TK_OP_DIV); - } } break; case '=': { - if (GETCHAR(1)=='=') { + if (GETCHAR(1) == '=') { _make_token(TK_OP_EQUAL); INCPOS(1); @@ -389,12 +384,12 @@ void GDTokenizerText::_advance() { } break; case '<': { - if (GETCHAR(1)=='=') { + if (GETCHAR(1) == '=') { _make_token(TK_OP_LESS_EQUAL); INCPOS(1); - } else if (GETCHAR(1)=='<') { - if (GETCHAR(2)=='=') { + } else if (GETCHAR(1) == '<') { + if (GETCHAR(2) == '=') { _make_token(TK_OP_ASSIGN_SHIFT_LEFT); INCPOS(1); } else { @@ -406,11 +401,11 @@ void GDTokenizerText::_advance() { } break; case '>': { - if (GETCHAR(1)=='=') { + if (GETCHAR(1) == '=') { _make_token(TK_OP_GREATER_EQUAL); INCPOS(1); - } else if (GETCHAR(1)=='>') { - if (GETCHAR(2)=='=') { + } else if (GETCHAR(1) == '>') { + if (GETCHAR(2) == '=') { _make_token(TK_OP_ASSIGN_SHIFT_RIGHT); INCPOS(1); @@ -424,7 +419,7 @@ void GDTokenizerText::_advance() { } break; case '!': { - if (GETCHAR(1)=='=') { + if (GETCHAR(1) == '=') { _make_token(TK_OP_NOT_EQUAL); INCPOS(1); } else { @@ -468,7 +463,7 @@ void GDTokenizerText::_advance() { _make_token(TK_DOLLAR); //for the get_node() shortener break; case '^': { - if (GETCHAR(1)=='=') { + if (GETCHAR(1) == '=') { _make_token(TK_OP_ASSIGN_BIT_XOR); INCPOS(1); } else { @@ -480,11 +475,11 @@ void GDTokenizerText::_advance() { _make_token(TK_OP_BIT_INVERT); break; case '&': { - if (GETCHAR(1)=='&') { + if (GETCHAR(1) == '&') { _make_token(TK_OP_AND); INCPOS(1); - } else if (GETCHAR(1)=='=') { + } else if (GETCHAR(1) == '=') { _make_token(TK_OP_ASSIGN_BIT_AND); INCPOS(1); } else { @@ -492,11 +487,11 @@ void GDTokenizerText::_advance() { } } break; case '|': { - if (GETCHAR(1)=='|') { + if (GETCHAR(1) == '|') { _make_token(TK_OP_OR); INCPOS(1); - } else if (GETCHAR(1)=='=') { + } else if (GETCHAR(1) == '=') { _make_token(TK_OP_ASSIGN_BIT_OR); INCPOS(1); } else { @@ -505,7 +500,7 @@ void GDTokenizerText::_advance() { } break; case '*': { - if (GETCHAR(1)=='=') { + if (GETCHAR(1) == '=') { _make_token(TK_OP_ASSIGN_MUL); INCPOS(1); } else { @@ -514,10 +509,10 @@ void GDTokenizerText::_advance() { } break; case '+': { - if (GETCHAR(1)=='=') { + if (GETCHAR(1) == '=') { _make_token(TK_OP_ASSIGN_ADD); INCPOS(1); - /* + /* } else if (GETCHAR(1)=='+') { _make_token(TK_OP_PLUS_PLUS); INCPOS(1); @@ -529,10 +524,10 @@ void GDTokenizerText::_advance() { } break; case '-': { - if (GETCHAR(1)=='=') { + if (GETCHAR(1) == '=') { _make_token(TK_OP_ASSIGN_SUB); INCPOS(1); - /* + /* } else if (GETCHAR(1)=='-') { _make_token(TK_OP_MINUS_MINUS); INCPOS(1); @@ -543,7 +538,7 @@ void GDTokenizerText::_advance() { } break; case '%': { - if (GETCHAR(1)=='=') { + if (GETCHAR(1) == '=') { _make_token(TK_OP_ASSIGN_MOD); INCPOS(1); } else { @@ -551,106 +546,103 @@ void GDTokenizerText::_advance() { } } break; case '@': - if( CharType(GETCHAR(1))!='"' && CharType(GETCHAR(1))!='\'' ) { + if (CharType(GETCHAR(1)) != '"' && CharType(GETCHAR(1)) != '\'') { _make_error("Unexpected '@'"); return; } INCPOS(1); - is_node_path=true; + is_node_path = true; case '\'': case '"': { - if (GETCHAR(0)=='\'') - string_mode=STRING_SINGLE_QUOTE; - - - int i=1; - if (string_mode==STRING_DOUBLE_QUOTE && GETCHAR(i)=='"' && GETCHAR(i+1)=='"') { - i+=2; - string_mode=STRING_MULTILINE; + if (GETCHAR(0) == '\'') + string_mode = STRING_SINGLE_QUOTE; + int i = 1; + if (string_mode == STRING_DOUBLE_QUOTE && GETCHAR(i) == '"' && GETCHAR(i + 1) == '"') { + i += 2; + string_mode = STRING_MULTILINE; } - String str; - while(true) { - if (CharType(GETCHAR(i))==0) { + while (true) { + if (CharType(GETCHAR(i)) == 0) { _make_error("Unterminated String"); return; - } else if( string_mode==STRING_DOUBLE_QUOTE && CharType(GETCHAR(i))=='"' ) { + } else if (string_mode == STRING_DOUBLE_QUOTE && CharType(GETCHAR(i)) == '"') { break; - } else if( string_mode==STRING_SINGLE_QUOTE && CharType(GETCHAR(i))=='\'' ) { + } else if (string_mode == STRING_SINGLE_QUOTE && CharType(GETCHAR(i)) == '\'') { break; - } else if( string_mode==STRING_MULTILINE && CharType(GETCHAR(i))=='\"' && CharType(GETCHAR(i+1))=='\"' && CharType(GETCHAR(i+2))=='\"') { - i+=2; + } else if (string_mode == STRING_MULTILINE && CharType(GETCHAR(i)) == '\"' && CharType(GETCHAR(i + 1)) == '\"' && CharType(GETCHAR(i + 2)) == '\"') { + i += 2; break; - } else if( string_mode!=STRING_MULTILINE && CharType(GETCHAR(i))=='\n') { + } else if (string_mode != STRING_MULTILINE && CharType(GETCHAR(i)) == '\n') { _make_error("Unexpected EOL at String."); return; - } else if( CharType(GETCHAR(i))==0xFFFF) { + } else if (CharType(GETCHAR(i)) == 0xFFFF) { //string ends here, next will be TK i--; break; - } else if (CharType(GETCHAR(i))=='\\') { + } else if (CharType(GETCHAR(i)) == '\\') { //escaped characters... i++; CharType next = GETCHAR(i); - if (next==0) { + if (next == 0) { _make_error("Unterminated String"); return; } - CharType res=0; + CharType res = 0; - switch(next) { + switch (next) { - case 'a': res=7; break; - case 'b': res=8; break; - case 't': res=9; break; - case 'n': res=10; break; - case 'v': res=11; break; - case 'f': res=12; break; - case 'r': res=13; break; - case '\'': res='\''; break; - case '\"': res='\"'; break; - case '\\': res='\\'; break; - case '/': res='/'; break; //wtf + case 'a': res = 7; break; + case 'b': res = 8; break; + case 't': res = 9; break; + case 'n': res = 10; break; + case 'v': res = 11; break; + case 'f': res = 12; break; + case 'r': res = 13; break; + case '\'': res = '\''; break; + case '\"': res = '\"'; break; + case '\\': res = '\\'; break; + case '/': + res = '/'; + break; //wtf case 'u': { //hexnumbarh - oct is deprecated - i+=1; - for(int j=0;j<4;j++) { - CharType c = GETCHAR(i+j); - if (c==0) { + i += 1; + for (int j = 0; j < 4; j++) { + CharType c = GETCHAR(i + j); + if (c == 0) { _make_error("Unterminated String"); return; } - if (!((c>='0' && c<='9') || (c>='a' && c<='f') || (c>='A' && c<='F'))) { + if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'))) { _make_error("Malformed hex constant in string"); return; } CharType v; - if (c>='0' && c<='9') { - v=c-'0'; - } else if (c>='a' && c<='f') { - v=c-'a'; - v+=10; - } else if (c>='A' && c<='F') { - v=c-'A'; - v+=10; + if (c >= '0' && c <= '9') { + v = c - '0'; + } else if (c >= 'a' && c <= 'f') { + v = c - 'a'; + v += 10; + } else if (c >= 'A' && c <= 'F') { + v = c - 'A'; + v += 10; } else { ERR_PRINT("BUG"); - v=0; + v = 0; } - res<<=4; - res|=v; - - + res <<= 4; + res |= v; } - i+=3; + i += 3; } break; default: { @@ -660,15 +652,15 @@ void GDTokenizerText::_advance() { } break; } - str+=res; + str += res; } else { - if (CharType(GETCHAR(i))=='\n') { + if (CharType(GETCHAR(i)) == '\n') { line++; - column=1; + column = 1; } - str+=CharType(GETCHAR(i)); + str += CharType(GETCHAR(i)); } i++; } @@ -686,54 +678,54 @@ void GDTokenizerText::_advance() { } break; default: { - if (_is_number(GETCHAR(0)) || (GETCHAR(0)=='.' && _is_number(GETCHAR(1)))) { + if (_is_number(GETCHAR(0)) || (GETCHAR(0) == '.' && _is_number(GETCHAR(1)))) { // parse number - bool period_found=false; - bool exponent_found=false; - bool hexa_found=false; - bool sign_found=false; + bool period_found = false; + bool exponent_found = false; + bool hexa_found = false; + bool sign_found = false; String str; - int i=0; + int i = 0; - while(true) { - if (GETCHAR(i)=='.') { + while (true) { + if (GETCHAR(i) == '.') { if (period_found || exponent_found) { _make_error("Invalid numeric constant at '.'"); return; } - period_found=true; - } else if (GETCHAR(i)=='x') { - if (hexa_found || str.length()!=1 || !( (i==1 && str[0]=='0') || (i==2 && str[1]=='0' && str[0]=='-') ) ) { + period_found = true; + } else if (GETCHAR(i) == 'x') { + if (hexa_found || str.length() != 1 || !((i == 1 && str[0] == '0') || (i == 2 && str[1] == '0' && str[0] == '-'))) { _make_error("Invalid numeric constant at 'x'"); return; } - hexa_found=true; - } else if (!hexa_found && GETCHAR(i)=='e') { + hexa_found = true; + } else if (!hexa_found && GETCHAR(i) == 'e') { if (hexa_found || exponent_found) { _make_error("Invalid numeric constant at 'e'"); return; } - exponent_found=true; + exponent_found = true; } else if (_is_number(GETCHAR(i))) { //all ok } else if (hexa_found && _is_hex(GETCHAR(i))) { - } else if ((GETCHAR(i)=='-' || GETCHAR(i)=='+') && exponent_found) { + } else if ((GETCHAR(i) == '-' || GETCHAR(i) == '+') && exponent_found) { if (sign_found) { _make_error("Invalid numeric constant at '-'"); return; } - sign_found=true; + sign_found = true; } else break; - str+=CharType(GETCHAR(i)); + str += CharType(GETCHAR(i)); i++; } - if (!( _is_number(str[str.length()-1]) || (hexa_found && _is_hex(str[str.length()-1])))) { - _make_error("Invalid numeric constant: "+str); + if (!(_is_number(str[str.length() - 1]) || (hexa_found && _is_hex(str[str.length() - 1])))) { + _make_error("Invalid numeric constant: " + str); return; } @@ -748,13 +740,12 @@ void GDTokenizerText::_advance() { } else { int64_t val = str.to_int64(); _make_constant(val); - } return; } - if (GETCHAR(0)=='.') { + if (GETCHAR(0) == '.') { //parse period _make_token(TK_PERIOD); break; @@ -763,74 +754,76 @@ void GDTokenizerText::_advance() { if (_is_text_char(GETCHAR(0))) { // parse identifier String str; - str+=CharType(GETCHAR(0)); + str += CharType(GETCHAR(0)); - int i=1; - while(_is_text_char(GETCHAR(i))) { - str+=CharType(GETCHAR(i)); + int i = 1; + while (_is_text_char(GETCHAR(i))) { + str += CharType(GETCHAR(i)); i++; } - bool identifier=false; + bool identifier = false; - if (str=="null") { + if (str == "null") { _make_constant(Variant()); - } else if (str=="true") { + } else if (str == "true") { _make_constant(true); - } else if (str=="false") { + } else if (str == "false") { _make_constant(false); } else { - bool found=false; + bool found = false; - struct _bit { Variant::Type type; const char *text;}; + struct _bit { + Variant::Type type; + const char *text; + }; //built in types - static const _bit type_list[]={ + static const _bit type_list[] = { //types - {Variant::BOOL,"bool"}, - {Variant::INT,"int"}, - {Variant::REAL,"float"}, - {Variant::STRING,"String"}, - {Variant::VECTOR2,"Vector2"}, - {Variant::RECT2,"Rect2"}, - {Variant::TRANSFORM2D,"Transform2D"}, - {Variant::VECTOR3,"Vector3"}, - {Variant::RECT3,"Rect3"}, - {Variant::PLANE,"Plane"}, - {Variant::QUAT,"Quat"}, - {Variant::BASIS,"Basis"}, - {Variant::TRANSFORM,"Transform"}, - {Variant::COLOR,"Color"}, - {Variant::IMAGE,"Image"}, - {Variant::_RID,"RID"}, - {Variant::OBJECT,"Object"}, - {Variant::INPUT_EVENT,"InputEvent"}, - {Variant::NODE_PATH,"NodePath"}, - {Variant::DICTIONARY,"Dictionary"}, - {Variant::ARRAY,"Array"}, - {Variant::POOL_BYTE_ARRAY,"PoolByteArray"}, - {Variant::POOL_INT_ARRAY,"PoolIntArray"}, - {Variant::POOL_REAL_ARRAY,"PoolFloatArray"}, - {Variant::POOL_STRING_ARRAY,"PoolStringArray"}, - {Variant::POOL_VECTOR2_ARRAY,"PoolVector2Array"}, - {Variant::POOL_VECTOR3_ARRAY,"PoolVector3Array"}, - {Variant::POOL_COLOR_ARRAY,"PoolColorArray"}, - {Variant::VARIANT_MAX,NULL}, + { Variant::BOOL, "bool" }, + { Variant::INT, "int" }, + { Variant::REAL, "float" }, + { Variant::STRING, "String" }, + { Variant::VECTOR2, "Vector2" }, + { Variant::RECT2, "Rect2" }, + { Variant::TRANSFORM2D, "Transform2D" }, + { Variant::VECTOR3, "Vector3" }, + { Variant::RECT3, "Rect3" }, + { Variant::PLANE, "Plane" }, + { Variant::QUAT, "Quat" }, + { Variant::BASIS, "Basis" }, + { Variant::TRANSFORM, "Transform" }, + { Variant::COLOR, "Color" }, + { Variant::IMAGE, "Image" }, + { Variant::_RID, "RID" }, + { Variant::OBJECT, "Object" }, + { Variant::INPUT_EVENT, "InputEvent" }, + { Variant::NODE_PATH, "NodePath" }, + { Variant::DICTIONARY, "Dictionary" }, + { Variant::ARRAY, "Array" }, + { Variant::POOL_BYTE_ARRAY, "PoolByteArray" }, + { Variant::POOL_INT_ARRAY, "PoolIntArray" }, + { Variant::POOL_REAL_ARRAY, "PoolFloatArray" }, + { Variant::POOL_STRING_ARRAY, "PoolStringArray" }, + { Variant::POOL_VECTOR2_ARRAY, "PoolVector2Array" }, + { Variant::POOL_VECTOR3_ARRAY, "PoolVector3Array" }, + { Variant::POOL_COLOR_ARRAY, "PoolColorArray" }, + { Variant::VARIANT_MAX, NULL }, }; { + int idx = 0; - int idx=0; - - while(type_list[idx].text) { + while (type_list[idx].text) { - if (str==type_list[idx].text) { + if (str == type_list[idx].text) { _make_type(type_list[idx].type); - found=true; + found = true; break; } idx++; @@ -841,12 +834,12 @@ void GDTokenizerText::_advance() { //built in func? - for(int i=0;i<GDFunctions::FUNC_MAX;i++) { + for (int i = 0; i < GDFunctions::FUNC_MAX; i++) { - if (str==GDFunctions::get_func_name(GDFunctions::Function(i))) { + if (str == GDFunctions::get_func_name(GDFunctions::Function(i))) { _make_built_in_func(GDFunctions::Function(i)); - found=true; + found = true; break; } } @@ -856,66 +849,68 @@ void GDTokenizerText::_advance() { if (!found) { + struct _kws { + Token token; + const char *text; + }; - struct _kws { Token token; const char *text;}; - - static const _kws keyword_list[]={ + static const _kws keyword_list[] = { //ops - {TK_OP_IN,"in"}, - {TK_OP_NOT,"not"}, - {TK_OP_OR,"or"}, - {TK_OP_AND,"and"}, + { TK_OP_IN, "in" }, + { TK_OP_NOT, "not" }, + { TK_OP_OR, "or" }, + { TK_OP_AND, "and" }, //func - {TK_PR_FUNCTION,"func"}, - {TK_PR_CLASS,"class"}, - {TK_PR_EXTENDS,"extends"}, - {TK_PR_ONREADY,"onready"}, - {TK_PR_TOOL,"tool"}, - {TK_PR_STATIC,"static"}, - {TK_PR_EXPORT,"export"}, - {TK_PR_SETGET,"setget"}, - {TK_PR_VAR,"var"}, - {TK_PR_PRELOAD,"preload"}, - {TK_PR_ASSERT,"assert"}, - {TK_PR_YIELD,"yield"}, - {TK_PR_SIGNAL,"signal"}, - {TK_PR_BREAKPOINT,"breakpoint"}, - {TK_PR_REMOTE,"remote"}, - {TK_PR_MASTER,"master"}, - {TK_PR_SLAVE,"slave"}, - {TK_PR_SYNC,"sync"}, - {TK_PR_CONST,"const"}, - {TK_PR_ENUM,"enum"}, + { TK_PR_FUNCTION, "func" }, + { TK_PR_CLASS, "class" }, + { TK_PR_EXTENDS, "extends" }, + { TK_PR_ONREADY, "onready" }, + { TK_PR_TOOL, "tool" }, + { TK_PR_STATIC, "static" }, + { TK_PR_EXPORT, "export" }, + { TK_PR_SETGET, "setget" }, + { TK_PR_VAR, "var" }, + { TK_PR_PRELOAD, "preload" }, + { TK_PR_ASSERT, "assert" }, + { TK_PR_YIELD, "yield" }, + { TK_PR_SIGNAL, "signal" }, + { TK_PR_BREAKPOINT, "breakpoint" }, + { TK_PR_REMOTE, "remote" }, + { TK_PR_MASTER, "master" }, + { TK_PR_SLAVE, "slave" }, + { TK_PR_SYNC, "sync" }, + { TK_PR_CONST, "const" }, + { TK_PR_ENUM, "enum" }, //controlflow - {TK_CF_IF,"if"}, - {TK_CF_ELIF,"elif"}, - {TK_CF_ELSE,"else"}, - {TK_CF_FOR,"for"}, - {TK_CF_WHILE,"while"}, - {TK_CF_DO,"do"}, - {TK_CF_SWITCH,"switch"}, - {TK_CF_CASE,"case"}, - {TK_CF_BREAK,"break"}, - {TK_CF_CONTINUE,"continue"}, - {TK_CF_RETURN,"return"}, - {TK_CF_MATCH, "match"}, - {TK_CF_PASS,"pass"}, - {TK_SELF,"self"}, - {TK_CONST_PI,"PI"}, - {TK_WILDCARD,"_"}, - {TK_CONST_INF,"INF"}, - {TK_CONST_NAN,"NAN"}, - {TK_ERROR,NULL} + { TK_CF_IF, "if" }, + { TK_CF_ELIF, "elif" }, + { TK_CF_ELSE, "else" }, + { TK_CF_FOR, "for" }, + { TK_CF_WHILE, "while" }, + { TK_CF_DO, "do" }, + { TK_CF_SWITCH, "switch" }, + { TK_CF_CASE, "case" }, + { TK_CF_BREAK, "break" }, + { TK_CF_CONTINUE, "continue" }, + { TK_CF_RETURN, "return" }, + { TK_CF_MATCH, "match" }, + { TK_CF_PASS, "pass" }, + { TK_SELF, "self" }, + { TK_CONST_PI, "PI" }, + { TK_WILDCARD, "_" }, + { TK_CONST_INF, "INF" }, + { TK_CONST_NAN, "NAN" }, + { TK_ERROR, NULL } }; - int idx=0; - found=false; + int idx = 0; + found = false; - while(keyword_list[idx].text) { + while (keyword_list[idx].text) { - if (str==keyword_list[idx].text) { + if (str == keyword_list[idx].text) { _make_token(keyword_list[idx].token); - found=true; + found = true; break; } idx++; @@ -923,10 +918,9 @@ void GDTokenizerText::_advance() { } if (!found) - identifier=true; + identifier = true; } - if (identifier) { _make_identifier(str); } @@ -943,381 +937,358 @@ void GDTokenizerText::_advance() { INCPOS(1); break; } - } -void GDTokenizerText::set_code(const String& p_code) { +void GDTokenizerText::set_code(const String &p_code) { - code=p_code; + code = p_code; len = p_code.length(); if (len) { - _code=&code[0]; + _code = &code[0]; } else { - _code=NULL; + _code = NULL; } - code_pos=0; - line=1; //it is stand-ar-ized that lines begin in 1 in code.. - column=1; //the same holds for columns - tk_rb_pos=0; - error_flag=false; - last_error=""; - for(int i=0;i<MAX_LOOKAHEAD+1;i++) + code_pos = 0; + line = 1; //it is stand-ar-ized that lines begin in 1 in code.. + column = 1; //the same holds for columns + tk_rb_pos = 0; + error_flag = false; + last_error = ""; + for (int i = 0; i < MAX_LOOKAHEAD + 1; i++) _advance(); } GDTokenizerText::Token GDTokenizerText::get_token(int p_offset) const { - ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, TK_ERROR); - ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, TK_ERROR); + ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, TK_ERROR); + ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, TK_ERROR); - int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE; + int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE; return tk_rb[ofs].type; } int GDTokenizerText::get_token_line(int p_offset) const { - ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, -1); - ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, -1); + ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, -1); + ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, -1); - int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE; + int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE; return tk_rb[ofs].line; } int GDTokenizerText::get_token_column(int p_offset) const { - ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, -1); - ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, -1); + ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, -1); + ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, -1); - int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE; + int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE; return tk_rb[ofs].col; } -const Variant& GDTokenizerText::get_token_constant(int p_offset) const { - ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, tk_rb[0].constant); - ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, tk_rb[0].constant); +const Variant &GDTokenizerText::get_token_constant(int p_offset) const { + ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, tk_rb[0].constant); + ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, tk_rb[0].constant); - int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE; - ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_CONSTANT,tk_rb[0].constant); + int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE; + ERR_FAIL_COND_V(tk_rb[ofs].type != TK_CONSTANT, tk_rb[0].constant); return tk_rb[ofs].constant; } StringName GDTokenizerText::get_token_identifier(int p_offset) const { - ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, StringName()); - ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, StringName()); + ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, StringName()); + ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, StringName()); - int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE; - ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_IDENTIFIER,StringName()); + int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE; + ERR_FAIL_COND_V(tk_rb[ofs].type != TK_IDENTIFIER, StringName()); return tk_rb[ofs].identifier; - } GDFunctions::Function GDTokenizerText::get_token_built_in_func(int p_offset) const { - ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, GDFunctions::FUNC_MAX); - ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, GDFunctions::FUNC_MAX); + ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, GDFunctions::FUNC_MAX); + ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, GDFunctions::FUNC_MAX); - int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE; - ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_BUILT_IN_FUNC,GDFunctions::FUNC_MAX); + int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE; + ERR_FAIL_COND_V(tk_rb[ofs].type != TK_BUILT_IN_FUNC, GDFunctions::FUNC_MAX); return tk_rb[ofs].func; - } Variant::Type GDTokenizerText::get_token_type(int p_offset) const { - ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, Variant::NIL); - ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, Variant::NIL); + ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, Variant::NIL); + ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, Variant::NIL); - int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE; - ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_BUILT_IN_TYPE,Variant::NIL); + int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE; + ERR_FAIL_COND_V(tk_rb[ofs].type != TK_BUILT_IN_TYPE, Variant::NIL); return tk_rb[ofs].vtype; - } - int GDTokenizerText::get_token_line_indent(int p_offset) const { - ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, 0); - ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, 0); + ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, 0); + ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, 0); - int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE; - ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_NEWLINE,0); + int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE; + ERR_FAIL_COND_V(tk_rb[ofs].type != TK_NEWLINE, 0); return tk_rb[ofs].constant; - } String GDTokenizerText::get_token_error(int p_offset) const { - ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, String()); - ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, String()); + ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, String()); + ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, String()); - int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE; - ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_ERROR,String()); + int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE; + ERR_FAIL_COND_V(tk_rb[ofs].type != TK_ERROR, String()); return tk_rb[ofs].constant; } void GDTokenizerText::advance(int p_amount) { - ERR_FAIL_COND( p_amount <=0 ); - for(int i=0;i<p_amount;i++) + ERR_FAIL_COND(p_amount <= 0); + for (int i = 0; i < p_amount; i++) _advance(); } - - - - - - - - - - ////////////////////////////////////////////////////////////////////////////////////////////////////// #define BYTECODE_VERSION 12 -Error GDTokenizerBuffer::set_code_buffer(const Vector<uint8_t> & p_buffer) { - +Error GDTokenizerBuffer::set_code_buffer(const Vector<uint8_t> &p_buffer) { - const uint8_t *buf=p_buffer.ptr(); - int total_len=p_buffer.size(); - ERR_FAIL_COND_V( p_buffer.size()<24 || p_buffer[0]!='G' || p_buffer[1]!='D' || p_buffer[2]!='S' || p_buffer[3]!='C',ERR_INVALID_DATA); + const uint8_t *buf = p_buffer.ptr(); + int total_len = p_buffer.size(); + ERR_FAIL_COND_V(p_buffer.size() < 24 || p_buffer[0] != 'G' || p_buffer[1] != 'D' || p_buffer[2] != 'S' || p_buffer[3] != 'C', ERR_INVALID_DATA); int version = decode_uint32(&buf[4]); - if (version>BYTECODE_VERSION) { + if (version > BYTECODE_VERSION) { ERR_EXPLAIN("Bytecode is too New! Please use a newer engine version."); - ERR_FAIL_COND_V(version>BYTECODE_VERSION,ERR_INVALID_DATA); + ERR_FAIL_COND_V(version > BYTECODE_VERSION, ERR_INVALID_DATA); } int identifier_count = decode_uint32(&buf[8]); int constant_count = decode_uint32(&buf[12]); int line_count = decode_uint32(&buf[16]); int token_count = decode_uint32(&buf[20]); - const uint8_t *b=buf; + const uint8_t *b = buf; - b=&buf[24]; - total_len-=24; + b = &buf[24]; + total_len -= 24; identifiers.resize(identifier_count); - for(int i=0;i<identifier_count;i++) { + for (int i = 0; i < identifier_count; i++) { int len = decode_uint32(b); - ERR_FAIL_COND_V(len>total_len,ERR_INVALID_DATA); - b+=4; + ERR_FAIL_COND_V(len > total_len, ERR_INVALID_DATA); + b += 4; Vector<uint8_t> cs; cs.resize(len); - for(int j=0;j<len;j++) { - cs[j]=b[j]^0xb6; + for (int j = 0; j < len; j++) { + cs[j] = b[j] ^ 0xb6; } - cs[cs.size()-1]=0; + cs[cs.size() - 1] = 0; String s; - s.parse_utf8((const char*)cs.ptr()); - b+=len; - total_len-=len+4; - identifiers[i]=s; + s.parse_utf8((const char *)cs.ptr()); + b += len; + total_len -= len + 4; + identifiers[i] = s; } constants.resize(constant_count); - for(int i=0;i<constant_count;i++) { + for (int i = 0; i < constant_count; i++) { Variant v; int len; - Error err = decode_variant(v,b,total_len,&len); + Error err = decode_variant(v, b, total_len, &len); if (err) return err; - b+=len; - total_len-=len; - constants[i]=v; - + b += len; + total_len -= len; + constants[i] = v; } - ERR_FAIL_COND_V(line_count*8>total_len,ERR_INVALID_DATA); + ERR_FAIL_COND_V(line_count * 8 > total_len, ERR_INVALID_DATA); - for(int i=0;i<line_count;i++) { + for (int i = 0; i < line_count; i++) { - uint32_t token=decode_uint32(b); - b+=4; - uint32_t linecol=decode_uint32(b); - b+=4; + uint32_t token = decode_uint32(b); + b += 4; + uint32_t linecol = decode_uint32(b); + b += 4; - lines.insert(token,linecol); - total_len-=8; + lines.insert(token, linecol); + total_len -= 8; } tokens.resize(token_count); - for(int i=0;i<token_count;i++) { + for (int i = 0; i < token_count; i++) { - ERR_FAIL_COND_V( total_len < 1, ERR_INVALID_DATA); + ERR_FAIL_COND_V(total_len < 1, ERR_INVALID_DATA); - if ((*b)&TOKEN_BYTE_MASK) { //little endian always - ERR_FAIL_COND_V( total_len < 4, ERR_INVALID_DATA); + if ((*b) & TOKEN_BYTE_MASK) { //little endian always + ERR_FAIL_COND_V(total_len < 4, ERR_INVALID_DATA); - tokens[i]=decode_uint32(b)&~TOKEN_BYTE_MASK; - b+=4; + tokens[i] = decode_uint32(b) & ~TOKEN_BYTE_MASK; + b += 4; } else { - tokens[i]=*b; - b+=1; + tokens[i] = *b; + b += 1; total_len--; } } - token=0; + token = 0; return OK; - } - -Vector<uint8_t> GDTokenizerBuffer::parse_code_string(const String& p_code) { +Vector<uint8_t> GDTokenizerBuffer::parse_code_string(const String &p_code) { Vector<uint8_t> buf; - - Map<StringName,int> identifier_map; - HashMap<Variant,int,VariantHasher,VariantComparator> constant_map; - Map<uint32_t,int> line_map; + Map<StringName, int> identifier_map; + HashMap<Variant, int, VariantHasher, VariantComparator> constant_map; + Map<uint32_t, int> line_map; Vector<uint32_t> token_array; GDTokenizerText tt; tt.set_code(p_code); - int line=-1; + int line = -1; - while(true) { + while (true) { - if (tt.get_token_line()!=line) { + if (tt.get_token_line() != line) { - line=tt.get_token_line(); - line_map[line]=token_array.size(); + line = tt.get_token_line(); + line_map[line] = token_array.size(); } - uint32_t token=tt.get_token(); - switch(tt.get_token()) { + uint32_t token = tt.get_token(); + switch (tt.get_token()) { case TK_IDENTIFIER: { StringName id = tt.get_token_identifier(); if (!identifier_map.has(id)) { int idx = identifier_map.size(); - identifier_map[id]=idx; + identifier_map[id] = idx; } - token|=identifier_map[id]<<TOKEN_BITS; + token |= identifier_map[id] << TOKEN_BITS; } break; case TK_CONSTANT: { Variant c = tt.get_token_constant(); if (!constant_map.has(c)) { int idx = constant_map.size(); - constant_map[c]=idx; + constant_map[c] = idx; } - token|=constant_map[c]<<TOKEN_BITS; + token |= constant_map[c] << TOKEN_BITS; } break; case TK_BUILT_IN_TYPE: { - token|=tt.get_token_type()<<TOKEN_BITS; + token |= tt.get_token_type() << TOKEN_BITS; } break; case TK_BUILT_IN_FUNC: { - token|=tt.get_token_built_in_func()<<TOKEN_BITS; + token |= tt.get_token_built_in_func() << TOKEN_BITS; } break; case TK_NEWLINE: { - token|=tt.get_token_line_indent()<<TOKEN_BITS; + token |= tt.get_token_line_indent() << TOKEN_BITS; } break; case TK_ERROR: { ERR_FAIL_V(Vector<uint8_t>()); } break; default: {} - }; token_array.push_back(token); - if (tt.get_token()==TK_EOF) + if (tt.get_token() == TK_EOF) break; tt.advance(); - } //reverse maps - Map<int,StringName> rev_identifier_map; - for(Map<StringName,int>::Element *E=identifier_map.front();E;E=E->next()) { - rev_identifier_map[E->get()]=E->key(); + Map<int, StringName> rev_identifier_map; + for (Map<StringName, int>::Element *E = identifier_map.front(); E; E = E->next()) { + rev_identifier_map[E->get()] = E->key(); } - Map<int,Variant> rev_constant_map; - const Variant *K =NULL; - while((K=constant_map.next(K))) { - rev_constant_map[constant_map[*K]]=*K; + Map<int, Variant> rev_constant_map; + const Variant *K = NULL; + while ((K = constant_map.next(K))) { + rev_constant_map[constant_map[*K]] = *K; } - Map<int,uint32_t> rev_line_map; - for(Map<uint32_t,int>::Element *E=line_map.front();E;E=E->next()) { - rev_line_map[E->get()]=E->key(); + Map<int, uint32_t> rev_line_map; + for (Map<uint32_t, int>::Element *E = line_map.front(); E; E = E->next()) { + rev_line_map[E->get()] = E->key(); } //save header buf.resize(24); - buf[0]='G'; - buf[1]='D'; - buf[2]='S'; - buf[3]='C'; - encode_uint32(BYTECODE_VERSION,&buf[4]); - encode_uint32(identifier_map.size(),&buf[8]); - encode_uint32(constant_map.size(),&buf[12]); - encode_uint32(line_map.size(),&buf[16]); - encode_uint32(token_array.size(),&buf[20]); + buf[0] = 'G'; + buf[1] = 'D'; + buf[2] = 'S'; + buf[3] = 'C'; + encode_uint32(BYTECODE_VERSION, &buf[4]); + encode_uint32(identifier_map.size(), &buf[8]); + encode_uint32(constant_map.size(), &buf[12]); + encode_uint32(line_map.size(), &buf[16]); + encode_uint32(token_array.size(), &buf[20]); //save identifiers - for(Map<int,StringName>::Element *E=rev_identifier_map.front();E;E=E->next()) { + for (Map<int, StringName>::Element *E = rev_identifier_map.front(); E; E = E->next()) { CharString cs = String(E->get()).utf8(); - int len = cs.length()+1; - int extra = 4-(len%4); - if (extra==4) - extra=0; + int len = cs.length() + 1; + int extra = 4 - (len % 4); + if (extra == 4) + extra = 0; uint8_t ibuf[4]; - encode_uint32(len+extra,ibuf); - for(int i=0;i<4;i++) { + encode_uint32(len + extra, ibuf); + for (int i = 0; i < 4; i++) { buf.push_back(ibuf[i]); } - for(int i=0;i<len;i++) { - buf.push_back(cs[i]^0xb6); + for (int i = 0; i < len; i++) { + buf.push_back(cs[i] ^ 0xb6); } - for(int i=0;i<extra;i++) { - buf.push_back(0^0xb6); + for (int i = 0; i < extra; i++) { + buf.push_back(0 ^ 0xb6); } } - for(Map<int,Variant>::Element *E=rev_constant_map.front();E;E=E->next()) { + for (Map<int, Variant>::Element *E = rev_constant_map.front(); E; E = E->next()) { int len; - Error err = encode_variant(E->get(),NULL,len); - ERR_FAIL_COND_V(err!=OK,Vector<uint8_t>()); - int pos=buf.size(); - buf.resize(pos+len); - encode_variant(E->get(),&buf[pos],len); + Error err = encode_variant(E->get(), NULL, len); + ERR_FAIL_COND_V(err != OK, Vector<uint8_t>()); + int pos = buf.size(); + buf.resize(pos + len); + encode_variant(E->get(), &buf[pos], len); } - for(Map<int,uint32_t>::Element *E=rev_line_map.front();E;E=E->next()) { + for (Map<int, uint32_t>::Element *E = rev_line_map.front(); E; E = E->next()) { uint8_t ibuf[8]; - encode_uint32(E->key(),&ibuf[0]); - encode_uint32(E->get(),&ibuf[4]); - for(int i=0;i<8;i++) + encode_uint32(E->key(), &ibuf[0]); + encode_uint32(E->get(), &ibuf[4]); + for (int i = 0; i < 8; i++) buf.push_back(ibuf[i]); } - for(int i=0;i<token_array.size();i++) { + for (int i = 0; i < token_array.size(); i++) { uint32_t token = token_array[i]; - if (token&~TOKEN_MASK) { + if (token & ~TOKEN_MASK) { uint8_t buf4[4]; - encode_uint32(token_array[i]|TOKEN_BYTE_MASK,&buf4[0]); - for(int j=0;j<4;j++) { + encode_uint32(token_array[i] | TOKEN_BYTE_MASK, &buf4[0]); + for (int j = 0; j < 4; j++) { buf.push_back(buf4[j]); } } else { @@ -1326,102 +1297,94 @@ Vector<uint8_t> GDTokenizerBuffer::parse_code_string(const String& p_code) { } return buf; - } GDTokenizerBuffer::Token GDTokenizerBuffer::get_token(int p_offset) const { - int offset = token+p_offset; + int offset = token + p_offset; - if (offset<0 || offset>=tokens.size()) + if (offset < 0 || offset >= tokens.size()) return TK_EOF; - return GDTokenizerBuffer::Token(tokens[offset]&TOKEN_MASK); + return GDTokenizerBuffer::Token(tokens[offset] & TOKEN_MASK); } +StringName GDTokenizerBuffer::get_token_identifier(int p_offset) const { -StringName GDTokenizerBuffer::get_token_identifier(int p_offset) const{ + int offset = token + p_offset; - int offset = token+p_offset; - - ERR_FAIL_INDEX_V(offset,tokens.size(),StringName()); - uint32_t identifier = tokens[offset]>>TOKEN_BITS; - ERR_FAIL_INDEX_V(identifier,(uint32_t)identifiers.size(),StringName()); + ERR_FAIL_INDEX_V(offset, tokens.size(), StringName()); + uint32_t identifier = tokens[offset] >> TOKEN_BITS; + ERR_FAIL_INDEX_V(identifier, (uint32_t)identifiers.size(), StringName()); return identifiers[identifier]; } -GDFunctions::Function GDTokenizerBuffer::get_token_built_in_func(int p_offset) const{ +GDFunctions::Function GDTokenizerBuffer::get_token_built_in_func(int p_offset) const { - int offset = token+p_offset; - ERR_FAIL_INDEX_V(offset,tokens.size(),GDFunctions::FUNC_MAX); - return GDFunctions::Function(tokens[offset]>>TOKEN_BITS); + int offset = token + p_offset; + ERR_FAIL_INDEX_V(offset, tokens.size(), GDFunctions::FUNC_MAX); + return GDFunctions::Function(tokens[offset] >> TOKEN_BITS); } -Variant::Type GDTokenizerBuffer::get_token_type(int p_offset) const{ +Variant::Type GDTokenizerBuffer::get_token_type(int p_offset) const { - int offset = token+p_offset; - ERR_FAIL_INDEX_V(offset,tokens.size(),Variant::NIL); + int offset = token + p_offset; + ERR_FAIL_INDEX_V(offset, tokens.size(), Variant::NIL); - return Variant::Type(tokens[offset]>>TOKEN_BITS); + return Variant::Type(tokens[offset] >> TOKEN_BITS); } -int GDTokenizerBuffer::get_token_line(int p_offset) const{ +int GDTokenizerBuffer::get_token_line(int p_offset) const { - int offset = token+p_offset; + int offset = token + p_offset; int pos = lines.find_nearest(offset); - if (pos<0) + if (pos < 0) return -1; - if (pos>=lines.size()) - pos=lines.size()-1; + if (pos >= lines.size()) + pos = lines.size() - 1; uint32_t l = lines.getv(pos); - return l&TOKEN_LINE_MASK; - + return l & TOKEN_LINE_MASK; } -int GDTokenizerBuffer::get_token_column(int p_offset) const{ +int GDTokenizerBuffer::get_token_column(int p_offset) const { - int offset = token+p_offset; + int offset = token + p_offset; int pos = lines.find_nearest(offset); - if (pos<0) + if (pos < 0) return -1; - if (pos>=lines.size()) - pos=lines.size()-1; + if (pos >= lines.size()) + pos = lines.size() - 1; uint32_t l = lines.getv(pos); - return l>>TOKEN_LINE_BITS; - + return l >> TOKEN_LINE_BITS; } -int GDTokenizerBuffer::get_token_line_indent(int p_offset) const{ +int GDTokenizerBuffer::get_token_line_indent(int p_offset) const { - int offset = token+p_offset; - ERR_FAIL_INDEX_V(offset,tokens.size(),0); - return tokens[offset]>>TOKEN_BITS; + int offset = token + p_offset; + ERR_FAIL_INDEX_V(offset, tokens.size(), 0); + return tokens[offset] >> TOKEN_BITS; } -const Variant& GDTokenizerBuffer::get_token_constant(int p_offset) const{ - +const Variant &GDTokenizerBuffer::get_token_constant(int p_offset) const { - int offset = token+p_offset; - ERR_FAIL_INDEX_V(offset,tokens.size(),nil); - uint32_t constant = tokens[offset]>>TOKEN_BITS; - ERR_FAIL_INDEX_V(constant,(uint32_t)constants.size(),nil); + int offset = token + p_offset; + ERR_FAIL_INDEX_V(offset, tokens.size(), nil); + uint32_t constant = tokens[offset] >> TOKEN_BITS; + ERR_FAIL_INDEX_V(constant, (uint32_t)constants.size(), nil); return constants[constant]; - } -String GDTokenizerBuffer::get_token_error(int p_offset) const{ +String GDTokenizerBuffer::get_token_error(int p_offset) const { ERR_FAIL_V(String()); } -void GDTokenizerBuffer::advance(int p_amount){ +void GDTokenizerBuffer::advance(int p_amount) { - ERR_FAIL_INDEX(p_amount+token,tokens.size()); - token+=p_amount; + ERR_FAIL_INDEX(p_amount + token, tokens.size()); + token += p_amount; } -GDTokenizerBuffer::GDTokenizerBuffer(){ - - token=0; +GDTokenizerBuffer::GDTokenizerBuffer() { + token = 0; } - |
