feat: Major grammar improvements and refactoring

- Refactor statement parsing with proper precedence handling
- Improve block structure parsing with indent/dedent support
- Enhance control flow parsing (conditionals, loops)
- Add print command support
- Improve function declaration parsing
- Update scanner for better string and comment handling
- Add comprehensive test corpus
- Better handling of newlines and statement boundaries
This commit is contained in:
2025-11-26 22:19:38 +01:00
parent b7942e9f79
commit 4d61f91e06
9 changed files with 15323 additions and 18695 deletions

740
src/grammar.json generated
View File

@@ -1,5 +1,4 @@
{
"$schema": "https://tree-sitter.github.io/tree-sitter/assets/schemas/grammar.schema.json",
"name": "stonescript",
"word": "identifier",
"rules": {
@@ -11,69 +10,90 @@
}
},
"_statement": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "comment"
},
{
"type": "SYMBOL",
"name": "block_comment"
},
{
"type": "SYMBOL",
"name": "variable_declaration"
},
{
"type": "SYMBOL",
"name": "function_declaration"
},
{
"type": "SYMBOL",
"name": "for_loop"
},
{
"type": "SYMBOL",
"name": "return_statement"
},
{
"type": "SYMBOL",
"name": "break_statement"
},
{
"type": "SYMBOL",
"name": "continue_statement"
},
{
"type": "SYMBOL",
"name": "import_statement"
},
{
"type": "SYMBOL",
"name": "new_expression"
},
{
"type": "SYMBOL",
"name": "conditional"
},
{
"type": "SYMBOL",
"name": "else_if_clause"
},
{
"type": "SYMBOL",
"name": "else_clause"
},
{
"type": "SYMBOL",
"name": "command_statement"
},
{
"type": "SYMBOL",
"name": "expression_statement"
}
]
"type": "PREC_RIGHT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "comment"
},
{
"type": "SYMBOL",
"name": "block_comment"
},
{
"type": "SYMBOL",
"name": "variable_declaration"
},
{
"type": "SYMBOL",
"name": "function_declaration"
},
{
"type": "SYMBOL",
"name": "for_loop"
},
{
"type": "SYMBOL",
"name": "return_statement"
},
{
"type": "SYMBOL",
"name": "break_statement"
},
{
"type": "SYMBOL",
"name": "continue_statement"
},
{
"type": "SYMBOL",
"name": "import_statement"
},
{
"type": "SYMBOL",
"name": "conditional"
},
{
"type": "SYMBOL",
"name": "else_clause"
},
{
"type": "PREC_DYNAMIC",
"value": 1,
"content": {
"type": "SYMBOL",
"name": "command"
}
},
{
"type": "SYMBOL",
"name": "print_command"
},
{
"type": "SYMBOL",
"name": "expression_statement"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_newline"
},
{
"type": "BLANK"
}
]
}
]
}
},
"comment": {
"type": "TOKEN",
@@ -188,37 +208,9 @@
"type": "STRING",
"value": ")"
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "function_body"
},
{
"type": "BLANK"
}
]
}
]
},
"function_body": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_indent"
},
{
"type": "REPEAT1",
"content": {
"type": "SYMBOL",
"name": "_statement"
}
},
{
"type": "SYMBOL",
"name": "_dedent"
"name": "block"
}
]
},
@@ -278,16 +270,8 @@
"name": "_expression"
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "block"
},
{
"type": "BLANK"
}
]
"type": "SYMBOL",
"name": "block"
}
]
},
@@ -311,16 +295,8 @@
"name": "_expression"
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "block"
},
{
"type": "BLANK"
}
]
"type": "SYMBOL",
"name": "block"
}
]
}
@@ -339,7 +315,7 @@
}
]
},
"new_expression": {
"new_statement": {
"type": "SEQ",
"members": [
{
@@ -354,7 +330,7 @@
},
"module_path": {
"type": "PATTERN",
"value": "[a-zA-Z_][a-zA-Z0-9_\\/]*"
"value": "[a-zA-Z_][a-zA-Z0-9_\\\\/]*"
},
"return_statement": {
"type": "PREC_RIGHT",
@@ -400,61 +376,42 @@
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "block"
},
{
"type": "BLANK"
}
]
}
]
},
"else_if_clause": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": ":?"
},
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "block"
},
{
"type": "BLANK"
}
]
"name": "block"
}
]
},
"else_clause": {
"type": "SEQ",
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": ":"
},
{
"type": "CHOICE",
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": ":?"
},
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "SYMBOL",
"name": "block"
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": ":"
},
{
"type": "BLANK"
"type": "SYMBOL",
"name": "block"
}
]
}
@@ -480,93 +437,55 @@
}
]
},
"command_statement": {
"command": {
"type": "PREC_DYNAMIC",
"value": 1,
"content": {
"type": "PREC_RIGHT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "REPEAT1",
"content": {
"type": "SYMBOL",
"name": "_command_arg"
}
}
]
}
}
},
"_command_arg": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "equip_command"
"name": "identifier"
},
{
"type": "SYMBOL",
"name": "activate_command"
"name": "number"
},
{
"type": "SYMBOL",
"name": "loadout_command"
"name": "string"
},
{
"type": "SYMBOL",
"name": "brew_command"
"name": "star_level"
},
{
"type": "SYMBOL",
"name": "disable_enable_command"
},
{
"type": "SYMBOL",
"name": "play_command"
},
{
"type": "SYMBOL",
"name": "print_command"
"name": "enchantment_level"
}
]
},
"equip_command": {
"type": "PREC_LEFT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "equip"
},
{
"type": "STRING",
"value": "equipL"
},
{
"type": "STRING",
"value": "equipR"
}
]
},
{
"type": "REPEAT1",
"content": {
"type": "SYMBOL",
"name": "item_criteria"
}
}
]
}
},
"item_criteria": {
"type": "PREC_LEFT",
"value": 0,
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "SYMBOL",
"name": "star_level"
},
{
"type": "SYMBOL",
"name": "enchantment_level"
}
]
}
},
"star_level": {
"type": "SEQ",
"members": [
@@ -593,162 +512,6 @@
}
]
},
"activate_command": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "activate"
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "STRING",
"value": "P"
},
{
"type": "STRING",
"value": "L"
},
{
"type": "STRING",
"value": "R"
}
]
}
]
},
"loadout_command": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "loadout"
},
{
"type": "SYMBOL",
"name": "number"
}
]
},
"brew_command": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "brew"
},
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "REPEAT",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "+"
},
{
"type": "SYMBOL",
"name": "identifier"
}
]
}
}
]
},
"disable_enable_command": {
"type": "PREC_LEFT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "disable"
},
{
"type": "STRING",
"value": "enable"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "abilities"
},
{
"type": "STRING",
"value": "hud"
},
{
"type": "STRING",
"value": "banner"
},
{
"type": "STRING",
"value": "loadout"
},
{
"type": "STRING",
"value": "npcDialog"
},
{
"type": "STRING",
"value": "pause"
},
{
"type": "STRING",
"value": "player"
}
]
}
]
}
},
"play_command": {
"type": "PREC_LEFT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "play"
},
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "number"
},
{
"type": "BLANK"
}
]
}
]
}
},
"print_command": {
"type": "PREC_RIGHT",
"value": 0,
@@ -784,30 +547,124 @@
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "print_args"
},
{
"type": "BLANK"
}
]
},
{
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "print_continuation"
}
}
]
}
},
"print_args": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "print_argument"
},
{
"type": "REPEAT",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": ","
},
{
"type": "SYMBOL",
"name": "print_argument"
}
]
}
}
]
},
"print_argument": {
"type": "PREC_LEFT",
"value": 0,
"content": {
"type": "REPEAT1",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "interpolation"
},
{
"type": "SYMBOL",
"name": "string"
},
{
"type": "SYMBOL",
"name": "color_code"
},
{
"type": "SYMBOL",
"name": "print_text"
}
]
}
}
},
"print_text": {
"type": "PATTERN",
"value": "[^,@\\r\\n\"]+"
},
"interpolation": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "@"
},
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "STRING",
"value": "@"
}
]
},
"print_continuation": {
"type": "PREC_RIGHT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "^"
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "identifier"
"type": "PATTERN",
"value": "[^@\\r\\n]+"
},
{
"type": "SYMBOL",
"name": "string"
},
{
"type": "SYMBOL",
"name": "number"
},
{
"type": "SYMBOL",
"name": "color_code"
},
{
"type": "STRING",
"value": ","
"name": "interpolation"
}
]
}
@@ -888,7 +745,11 @@
},
{
"type": "SYMBOL",
"name": "new_expression"
"name": "new_statement"
},
{
"type": "SYMBOL",
"name": "color_code"
}
]
},
@@ -1443,35 +1304,43 @@
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "REPEAT",
"content": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "comma_sep"
},
{
"type": "SYMBOL",
"name": "_expression"
}
]
}
}
]
"type": "SYMBOL",
"name": "array_elements"
},
{
"type": "BLANK"
}
]
},
{
"type": "STRING",
"value": "]"
}
]
},
"array_elements": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "REPEAT",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": ","
},
{
"type": "SYMBOL",
"name": "_expression"
}
]
}
},
{
"type": "CHOICE",
"members": [
@@ -1483,10 +1352,6 @@
"type": "BLANK"
}
]
},
{
"type": "STRING",
"value": "]"
}
]
},
@@ -1552,7 +1417,19 @@
"extras": [
{
"type": "PATTERN",
"value": "\\s"
"value": "[ \\t\\r\\f]"
},
{
"type": "PATTERN",
"value": "[\\r\\n]\\^"
},
{
"type": "SYMBOL",
"name": "comment"
},
{
"type": "SYMBOL",
"name": "block_comment"
}
],
"conflicts": [
@@ -1564,18 +1441,19 @@
"_expression"
],
[
"command_statement"
"command"
],
[
"_statement",
"_expression"
],
[
"equip_command"
],
[
"binary_expression",
"assignment_expression"
],
[
"command",
"_expression"
]
],
"precedences": [],
@@ -1594,6 +1472,6 @@
}
],
"inline": [],
"supertypes": [],
"reserved": {}
}
"supertypes": []
}

483
src/node-types.json generated
View File

@@ -1,19 +1,4 @@
[
{
"type": "activate_command",
"named": true,
"fields": {},
"children": {
"multiple": false,
"required": false,
"types": [
{
"type": "identifier",
"named": true
}
]
}
},
{
"type": "argument_list",
"named": true,
@@ -42,6 +27,10 @@
"type": "call_expression",
"named": true
},
{
"type": "color_code",
"named": true
},
{
"type": "comma_sep",
"named": true
@@ -63,7 +52,7 @@
"named": true
},
{
"type": "new_expression",
"type": "new_statement",
"named": true
},
{
@@ -98,8 +87,23 @@
"named": true,
"fields": {},
"children": {
"multiple": true,
"multiple": false,
"required": false,
"types": [
{
"type": "array_elements",
"named": true
}
]
}
},
{
"type": "array_elements",
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": true,
"types": [
{
"type": "array",
@@ -122,7 +126,7 @@
"named": true
},
{
"type": "comma_sep",
"type": "color_code",
"named": true
},
{
@@ -142,7 +146,7 @@
"named": true
},
{
"type": "new_expression",
"type": "new_statement",
"named": true
},
{
@@ -200,6 +204,10 @@
"type": "call_expression",
"named": true
},
{
"type": "color_code",
"named": true
},
{
"type": "float",
"named": true
@@ -217,7 +225,7 @@
"named": true
},
{
"type": "new_expression",
"type": "new_statement",
"named": true
},
{
@@ -275,6 +283,10 @@
"type": "call_expression",
"named": true
},
{
"type": "color_code",
"named": true
},
{
"type": "float",
"named": true
@@ -292,7 +304,7 @@
"named": true
},
{
"type": "new_expression",
"type": "new_statement",
"named": true
},
{
@@ -339,7 +351,7 @@
"named": true
},
{
"type": "command_statement",
"type": "command",
"named": true
},
{
@@ -358,10 +370,6 @@
"type": "else_clause",
"named": true
},
{
"type": "else_if_clause",
"named": true
},
{
"type": "expression_statement",
"named": true
@@ -379,7 +387,7 @@
"named": true
},
{
"type": "new_expression",
"type": "print_command",
"named": true
},
{
@@ -398,21 +406,6 @@
"named": true,
"fields": {}
},
{
"type": "brew_command",
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": true,
"types": [
{
"type": "identifier",
"named": true
}
]
}
},
{
"type": "call_expression",
"named": true,
@@ -441,6 +434,10 @@
"type": "call_expression",
"named": true
},
{
"type": "color_code",
"named": true
},
{
"type": "float",
"named": true
@@ -458,7 +455,7 @@
"named": true
},
{
"type": "new_expression",
"type": "new_statement",
"named": true
},
{
@@ -505,39 +502,31 @@
"fields": {}
},
{
"type": "command_statement",
"type": "command",
"named": true,
"fields": {},
"children": {
"multiple": false,
"multiple": true,
"required": true,
"types": [
{
"type": "activate_command",
"type": "enchantment_level",
"named": true
},
{
"type": "brew_command",
"type": "identifier",
"named": true
},
{
"type": "disable_enable_command",
"type": "number",
"named": true
},
{
"type": "equip_command",
"type": "star_level",
"named": true
},
{
"type": "loadout_command",
"named": true
},
{
"type": "play_command",
"named": true
},
{
"type": "print_command",
"type": "string",
"named": true
}
]
@@ -575,6 +564,10 @@
"type": "call_expression",
"named": true
},
{
"type": "color_code",
"named": true
},
{
"type": "float",
"named": true
@@ -592,7 +585,7 @@
"named": true
},
{
"type": "new_expression",
"type": "new_statement",
"named": true
},
{
@@ -622,30 +615,10 @@
]
}
},
{
"type": "disable_enable_command",
"named": true,
"fields": {}
},
{
"type": "else_clause",
"named": true,
"fields": {},
"children": {
"multiple": false,
"required": false,
"types": [
{
"type": "block",
"named": true
}
]
}
},
{
"type": "else_if_clause",
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": true,
@@ -674,6 +647,10 @@
"type": "call_expression",
"named": true
},
{
"type": "color_code",
"named": true
},
{
"type": "float",
"named": true
@@ -691,7 +668,7 @@
"named": true
},
{
"type": "new_expression",
"type": "new_statement",
"named": true
},
{
@@ -736,21 +713,6 @@
]
}
},
{
"type": "equip_command",
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": true,
"types": [
{
"type": "item_criteria",
"named": true
}
]
}
},
{
"type": "expression_statement",
"named": true,
@@ -779,6 +741,10 @@
"type": "call_expression",
"named": true
},
{
"type": "color_code",
"named": true
},
{
"type": "float",
"named": true
@@ -796,7 +762,7 @@
"named": true
},
{
"type": "new_expression",
"type": "new_statement",
"named": true
},
{
@@ -858,6 +824,10 @@
"type": "call_expression",
"named": true
},
{
"type": "color_code",
"named": true
},
{
"type": "float",
"named": true
@@ -875,7 +845,7 @@
"named": true
},
{
"type": "new_expression",
"type": "new_statement",
"named": true
},
{
@@ -905,77 +875,6 @@
]
}
},
{
"type": "function_body",
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": true,
"types": [
{
"type": "block_comment",
"named": true
},
{
"type": "break_statement",
"named": true
},
{
"type": "command_statement",
"named": true
},
{
"type": "comment",
"named": true
},
{
"type": "conditional",
"named": true
},
{
"type": "continue_statement",
"named": true
},
{
"type": "else_clause",
"named": true
},
{
"type": "else_if_clause",
"named": true
},
{
"type": "expression_statement",
"named": true
},
{
"type": "for_loop",
"named": true
},
{
"type": "function_declaration",
"named": true
},
{
"type": "import_statement",
"named": true
},
{
"type": "new_expression",
"named": true
},
{
"type": "return_statement",
"named": true
},
{
"type": "variable_declaration",
"named": true
}
]
}
},
{
"type": "function_declaration",
"named": true,
@@ -993,10 +892,10 @@
},
"children": {
"multiple": true,
"required": false,
"required": true,
"types": [
{
"type": "function_body",
"type": "block",
"named": true
},
{
@@ -1049,6 +948,10 @@
"type": "call_expression",
"named": true
},
{
"type": "color_code",
"named": true
},
{
"type": "float",
"named": true
@@ -1066,7 +969,7 @@
"named": true
},
{
"type": "new_expression",
"type": "new_statement",
"named": true
},
{
@@ -1097,7 +1000,7 @@
}
},
{
"type": "item_criteria",
"type": "interpolation",
"named": true,
"fields": {},
"children": {
@@ -1105,7 +1008,31 @@
"required": true,
"types": [
{
"type": "enchantment_level",
"type": "array",
"named": true
},
{
"type": "assignment_expression",
"named": true
},
{
"type": "binary_expression",
"named": true
},
{
"type": "boolean",
"named": true
},
{
"type": "call_expression",
"named": true
},
{
"type": "color_code",
"named": true
},
{
"type": "float",
"named": true
},
{
@@ -1113,23 +1040,40 @@
"named": true
},
{
"type": "star_level",
"type": "index_expression",
"named": true
}
]
}
},
{
"type": "loadout_command",
"named": true,
"fields": {},
"children": {
"multiple": false,
"required": true,
"types": [
},
{
"type": "member_expression",
"named": true
},
{
"type": "new_statement",
"named": true
},
{
"type": "null",
"named": true
},
{
"type": "number",
"named": true
},
{
"type": "parenthesized_expression",
"named": true
},
{
"type": "string",
"named": true
},
{
"type": "unary_expression",
"named": true
},
{
"type": "update_expression",
"named": true
}
]
}
@@ -1162,6 +1106,10 @@
"type": "call_expression",
"named": true
},
{
"type": "color_code",
"named": true
},
{
"type": "float",
"named": true
@@ -1179,7 +1127,7 @@
"named": true
},
{
"type": "new_expression",
"type": "new_statement",
"named": true
},
{
@@ -1221,7 +1169,7 @@
}
},
{
"type": "new_expression",
"type": "new_statement",
"named": true,
"fields": {},
"children": {
@@ -1278,6 +1226,10 @@
"type": "call_expression",
"named": true
},
{
"type": "color_code",
"named": true
},
{
"type": "float",
"named": true
@@ -1295,7 +1247,7 @@
"named": true
},
{
"type": "new_expression",
"type": "new_statement",
"named": true
},
{
@@ -1326,7 +1278,7 @@
}
},
{
"type": "play_command",
"type": "print_args",
"named": true,
"fields": {},
"children": {
@@ -1334,11 +1286,34 @@
"required": true,
"types": [
{
"type": "identifier",
"type": "print_argument",
"named": true
}
]
}
},
{
"type": "print_argument",
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": true,
"types": [
{
"type": "color_code",
"named": true
},
{
"type": "number",
"type": "interpolation",
"named": true
},
{
"type": "print_text",
"named": true
},
{
"type": "string",
"named": true
}
]
@@ -1353,19 +1328,26 @@
"required": false,
"types": [
{
"type": "color_code",
"type": "print_args",
"named": true
},
{
"type": "identifier",
"type": "print_continuation",
"named": true
},
}
]
}
},
{
"type": "print_continuation",
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": false,
"types": [
{
"type": "number",
"named": true
},
{
"type": "string",
"type": "interpolation",
"named": true
}
]
@@ -1399,6 +1381,10 @@
"type": "call_expression",
"named": true
},
{
"type": "color_code",
"named": true
},
{
"type": "float",
"named": true
@@ -1416,7 +1402,7 @@
"named": true
},
{
"type": "new_expression",
"type": "new_statement",
"named": true
},
{
@@ -1449,7 +1435,6 @@
{
"type": "source_file",
"named": true,
"root": true,
"fields": {},
"children": {
"multiple": true,
@@ -1464,7 +1449,7 @@
"named": true
},
{
"type": "command_statement",
"type": "command",
"named": true
},
{
@@ -1483,10 +1468,6 @@
"type": "else_clause",
"named": true
},
{
"type": "else_if_clause",
"named": true
},
{
"type": "expression_statement",
"named": true
@@ -1504,7 +1485,7 @@
"named": true
},
{
"type": "new_expression",
"type": "print_command",
"named": true
},
{
@@ -1566,6 +1547,10 @@
"type": "call_expression",
"named": true
},
{
"type": "color_code",
"named": true
},
{
"type": "float",
"named": true
@@ -1583,7 +1568,7 @@
"named": true
},
{
"type": "new_expression",
"type": "new_statement",
"named": true
},
{
@@ -1641,6 +1626,10 @@
"type": "call_expression",
"named": true
},
{
"type": "color_code",
"named": true
},
{
"type": "float",
"named": true
@@ -1658,7 +1647,7 @@
"named": true
},
{
"type": "new_expression",
"type": "new_statement",
"named": true
},
{
@@ -1726,6 +1715,10 @@
"type": "call_expression",
"named": true
},
{
"type": "color_code",
"named": true
},
{
"type": "float",
"named": true
@@ -1743,7 +1736,7 @@
"named": true
},
{
"type": "new_expression",
"type": "new_statement",
"named": true
},
{
@@ -1903,15 +1896,7 @@
"named": false
},
{
"type": "L",
"named": false
},
{
"type": "P",
"named": false
},
{
"type": "R",
"type": "@",
"named": false
},
{
@@ -1923,15 +1908,7 @@
"named": false
},
{
"type": "abilities",
"named": false
},
{
"type": "activate",
"named": false
},
{
"type": "banner",
"type": "^",
"named": false
},
{
@@ -1942,10 +1919,6 @@
"type": "break_statement",
"named": true
},
{
"type": "brew",
"named": false
},
{
"type": "color_code",
"named": true
@@ -1958,26 +1931,6 @@
"type": "continue_statement",
"named": true
},
{
"type": "disable",
"named": false
},
{
"type": "enable",
"named": false
},
{
"type": "equip",
"named": false
},
{
"type": "equipL",
"named": false
},
{
"type": "equipR",
"named": false
},
{
"type": "false",
"named": false
@@ -1994,10 +1947,6 @@
"type": "func",
"named": false
},
{
"type": "hud",
"named": false
},
{
"type": "identifier",
"named": true
@@ -2006,10 +1955,6 @@
"type": "import",
"named": false
},
{
"type": "loadout",
"named": false
},
{
"type": "module_path",
"named": true
@@ -2018,10 +1963,6 @@
"type": "new",
"named": false
},
{
"type": "npcDialog",
"named": false
},
{
"type": "null",
"named": true
@@ -2031,16 +1972,8 @@
"named": true
},
{
"type": "pause",
"named": false
},
{
"type": "play",
"named": false
},
{
"type": "player",
"named": false
"type": "print_text",
"named": true
},
{
"type": "return",

32344
src/parser.c generated

File diff suppressed because it is too large Load Diff

View File

@@ -6,8 +6,13 @@ enum TokenType {
NEWLINE,
INDENT,
DEDENT,
// ASCII_CONTENT,
};
// ... (skipping to logic)
typedef struct {
uint16_t *indent_stack;
size_t indent_stack_size;
@@ -63,10 +68,15 @@ unsigned tree_sitter_stonescript_external_scanner_serialize(void *payload, char
void tree_sitter_stonescript_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
Scanner *scanner = (Scanner *)payload;
size_t size = 0;
scanner->indent_stack_size = 1;
scanner->indent_stack[0] = 0;
scanner->queued_tokens_size = 0;
if (length < sizeof(uint32_t)) return;
uint32_t indent_stack_size = 0;
if (length == 0) return;
size_t i = 0;
@@ -91,6 +101,65 @@ void tree_sitter_stonescript_external_scanner_deserialize(void *payload, const c
bool tree_sitter_stonescript_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) {
Scanner *scanner = (Scanner *)payload;
// if (valid_symbols[ASCII_CONTENT]) {
// bool has_content = false;
//
// for (;;) {
// if (lexer->eof(lexer)) {
// break;
// }
//
// // Check if we're at the start of a line with 'asciiend'
// if (lexer->lookahead == '\n' || lexer->lookahead == '\r') {
// lexer->advance(lexer, false);
// if (lexer->lookahead == '\r' || lexer->lookahead == '\n') {
// lexer->advance(lexer, false);
// }
// lexer->mark_end(lexer);
// has_content = true;
//
// // Skip whitespace at the start of the line
// while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
// lexer->advance(lexer, false);
// }
//
// // Check if this line starts with 'asciiend'
// if (lexer->lookahead == 'a') {
// const char *keyword = "asciiend";
// bool match = true;
//
// for (int k = 0; k < 8; k++) {
// if (lexer->lookahead == keyword[k]) {
// lexer->advance(lexer, false);
// } else {
// match = false;
// break;
// }
// }
//
// // Check that asciiend is followed by whitespace or EOL
// if (match && (lexer->lookahead == '\n' || lexer->lookahead == '\r' ||
// lexer->lookahead == ' ' || lexer->lookahead == '\t' ||
// lexer->lookahead == ',' ||
// lexer->eof(lexer))) {
// lexer->result_symbol = ASCII_CONTENT;
// return has_content;
// }
//
// // Failed to match asciiend, mark the current position
// lexer->mark_end(lexer);
// }
// } else {
// lexer->advance(lexer, false);
// lexer->mark_end(lexer);
// has_content = true;
// }
// }
//
// lexer->result_symbol = ASCII_CONTENT;
// return has_content;
// }
if (scanner->queued_tokens_size > 0) {
enum TokenType token = scanner->queued_tokens[0];
for (size_t i = 1; i < scanner->queued_tokens_size; i++) {
@@ -128,6 +197,8 @@ bool tree_sitter_stonescript_external_scanner_scan(void *payload, TSLexer *lexer
}
}
if (found_end_of_line) {
uint16_t current_indent = scanner->indent_stack[scanner->indent_stack_size - 1];
@@ -141,7 +212,9 @@ bool tree_sitter_stonescript_external_scanner_scan(void *payload, TSLexer *lexer
return true;
}
if (valid_symbols[DEDENT] && indent_length < current_indent && scanner->indent_stack_size > 1) {
if (valid_symbols[DEDENT] && (indent_length < current_indent || (lexer->eof(lexer) && current_indent == 0)) && scanner->indent_stack_size > 1) {
scanner->indent_stack_size--;
while (scanner->indent_stack_size > 1 &&
@@ -157,6 +230,7 @@ bool tree_sitter_stonescript_external_scanner_scan(void *payload, TSLexer *lexer
}
if (valid_symbols[NEWLINE] && !lexer->eof(lexer)) {
lexer->mark_end(lexer);
lexer->result_symbol = NEWLINE;
return true;
}

View File

@@ -13,17 +13,12 @@ extern "C" {
#define ts_builtin_sym_end 0
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
#ifndef TREE_SITTER_API_H_
typedef uint16_t TSStateId;
#ifndef TREE_SITTER_API_H_
typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage;
typedef struct TSLanguageMetadata TSLanguageMetadata;
typedef struct TSLanguageMetadata {
uint8_t major_version;
uint8_t minor_version;
uint8_t patch_version;
} TSLanguageMetadata;
#endif
typedef struct {
@@ -32,11 +27,10 @@ typedef struct {
bool inherited;
} TSFieldMapEntry;
// Used to index the field and supertype maps.
typedef struct {
uint16_t index;
uint16_t length;
} TSMapSlice;
} TSFieldMapSlice;
typedef struct {
bool visible;
@@ -54,7 +48,6 @@ struct TSLexer {
uint32_t (*get_column)(TSLexer *);
bool (*is_at_included_range_start)(const TSLexer *);
bool (*eof)(const TSLexer *);
void (*log)(const TSLexer *, const char *, ...);
};
typedef enum {
@@ -86,12 +79,6 @@ typedef struct {
uint16_t external_lex_state;
} TSLexMode;
typedef struct {
uint16_t lex_state;
uint16_t external_lex_state;
uint16_t reserved_word_set_id;
} TSLexerMode;
typedef union {
TSParseAction action;
struct {
@@ -100,13 +87,8 @@ typedef union {
} entry;
} TSParseActionEntry;
typedef struct {
int32_t start;
int32_t end;
} TSCharacterRange;
struct TSLanguage {
uint32_t abi_version;
uint32_t version;
uint32_t symbol_count;
uint32_t alias_count;
uint32_t token_count;
@@ -122,13 +104,13 @@ struct TSLanguage {
const TSParseActionEntry *parse_actions;
const char * const *symbol_names;
const char * const *field_names;
const TSMapSlice *field_map_slices;
const TSFieldMapSlice *field_map_slices;
const TSFieldMapEntry *field_map_entries;
const TSSymbolMetadata *symbol_metadata;
const TSSymbol *public_symbol_map;
const uint16_t *alias_map;
const TSSymbol *alias_sequences;
const TSLexerMode *lex_modes;
const TSLexMode *lex_modes;
bool (*lex_fn)(TSLexer *, TSStateId);
bool (*keyword_lex_fn)(TSLexer *, TSStateId);
TSSymbol keyword_capture_token;
@@ -142,48 +124,15 @@ struct TSLanguage {
void (*deserialize)(void *, const char *, unsigned);
} external_scanner;
const TSStateId *primary_state_ids;
const char *name;
const TSSymbol *reserved_words;
uint16_t max_reserved_word_set_size;
uint32_t supertype_count;
const TSSymbol *supertype_symbols;
const TSMapSlice *supertype_map_slices;
const TSSymbol *supertype_map_entries;
TSLanguageMetadata metadata;
};
static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
uint32_t index = 0;
uint32_t size = len - index;
while (size > 1) {
uint32_t half_size = size / 2;
uint32_t mid_index = index + half_size;
const TSCharacterRange *range = &ranges[mid_index];
if (lookahead >= range->start && lookahead <= range->end) {
return true;
} else if (lookahead > range->end) {
index = mid_index;
}
size -= half_size;
}
const TSCharacterRange *range = &ranges[index];
return (lookahead >= range->start && lookahead <= range->end);
}
/*
* Lexer Macros
*/
#ifdef _MSC_VER
#define UNUSED __pragma(warning(suppress : 4101))
#else
#define UNUSED __attribute__((unused))
#endif
#define START_LEXER() \
bool result = false; \
bool skip = false; \
UNUSED \
bool eof = false; \
int32_t lookahead; \
goto start; \
@@ -199,17 +148,6 @@ static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, in
goto next_state; \
}
#define ADVANCE_MAP(...) \
{ \
static const uint16_t map[] = { __VA_ARGS__ }; \
for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \
if (map[i] == lookahead) { \
state = map[i + 1]; \
goto next_state; \
} \
} \
}
#define SKIP(state_value) \
{ \
skip = true; \
@@ -228,7 +166,7 @@ static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, in
* Parse Table Macros
*/
#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
#define SMALL_STATE(id) id - LARGE_STATE_COUNT
#define STATE(id) id
@@ -238,7 +176,7 @@ static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, in
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = (state_value) \
.state = state_value \
} \
}}
@@ -246,7 +184,7 @@ static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, in
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = (state_value), \
.state = state_value, \
.repetition = true \
} \
}}
@@ -259,15 +197,14 @@ static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, in
} \
}}
#define REDUCE(symbol_name, children, precedence, prod_id) \
{{ \
.reduce = { \
.type = TSParseActionTypeReduce, \
.symbol = symbol_name, \
.child_count = children, \
.dynamic_precedence = precedence, \
.production_id = prod_id \
}, \
#define REDUCE(symbol_val, child_count_val, ...) \
{{ \
.reduce = { \
.type = TSParseActionTypeReduce, \
.symbol = symbol_val, \
.child_count = child_count_val, \
__VA_ARGS__ \
}, \
}}
#define RECOVER() \