kurbezz/tree-sitter-stonescript
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add support for fullwidth Unicode, multiline arrays, and ASCII blocks

Browse Source 
- Add fullwidth brackets ［］ (U+FF3B, U+FF3D) support
- Add fullwidth quotes ＂ (U+FF02) support
- Fix multiline arrays with newlines between elements
- Fix line continuation with CRLF (^)
- Enable ASCII block syntax (ascii...asciiend and ［ascii...asciiend］)
- Update conflicts to resolve ambiguities

Fixed 51 parsing errors (253 -> 202 errors)
This commit is contained in:
Bulat Kurbanov
2025-11-26 23:04:03 +01:00
parent b746fcec44
commit 99dadd9ca7
5 changed files with 16668 additions and 12483 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
grammar.js
View File

@@ -142,8 +142,7 @@ module.exports = grammar({
print_command: $ => prec.right(seq(
choice('>', '>o', '>h', '>`', '>c', '>f'),
optional($.print_args),
repeat($.print_continuation)
optional($.print_args)
)),
// Print specific helpers
@@ -165,14 +164,6 @@ module.exports = grammar({
'@'
),
print_continuation: $ => prec.right(seq(
'^',
repeat(choice(
/[^@\r\n]+/,
$.interpolation
))
)),
color_code: $ => /#[a-zA-Z0-9]+/,
// Expressions
@@ -195,7 +186,7 @@ module.exports = grammar({
$.assignment_expression,
$.parenthesized_expression,
$.new_statement,
// $.ascii_string,
$.ascii_string,
$.color_code
),
@@ -218,9 +209,9 @@ module.exports = grammar({
index_expression: $ => prec.left(13, seq(
$._expression,
'[',
choice('[', '［'),
$._expression,
']'
choice(']', '］')
)),
unary_expression: $ => prec.right(12, seq(
@@ -264,14 +255,22 @@ module.exports = grammar({
// Arrays
array: $ => seq(
'[',
optional($.array_elements),
']'
choice('[', '［'),
repeat($._newline),
optional(seq(
$.array_elements,
repeat($._newline)
)),
choice(']', '］')
),
array_elements: $ => seq(
$._expression,
repeat(seq(',', $._expression)),
repeat(seq(
',',
repeat($._newline),
$._expression
)),
optional(',')
),
@@ -282,21 +281,24 @@ module.exports = grammar({
float: $ => /\d+\.\d+/,
string: $ => seq('"', repeat(choice(/[^"\\]/, /\\./)), '"'),
string: $ => choice(
seq('"', repeat(choice(/[^"\\]/, /\\./)), '"'),
seq('＂', repeat(choice(/[^＂\\]/, /\\./)), '＂')
),
boolean: $ => choice('true', 'false'),
null: $ => 'null',
// ascii_string: $ => seq(
// 'ascii',
// $.ascii_content,
// 'asciiend'
// )
ascii_string: $ => choice(
seq('ascii', $.ascii_content, 'asciiend'),
seq(choice('[', '［'), 'ascii', $.ascii_content, 'asciiend', choice(']', '］'))
)
},
extras: $ => [
/[ \t\r\f]/,
/[\r\n]\^/,
/\r?\n[ \t]*\^/,
$.comment,
$.block_comment
],
@@ -304,7 +306,8 @@ module.exports = grammar({
externals: $ => [
$._newline,
$._indent,
$._dedent
$._dedent,
$.ascii_content
],
word: $ => $.identifier,
@@ -315,7 +318,9 @@ module.exports = grammar({
[$.command],
[$._statement, $._expression], // new_statement can be both
[$.binary_expression, $.assignment_expression], // = operator ambiguity
[$.command, $._expression] // * operator ambiguity
[$.command, $._expression], // * operator ambiguity
[$.array_elements],
[$.ascii_string]
]
});

278
src/grammar.json generated
View File

@@ -1,4 +1,5 @@
{
"$schema": "https://tree-sitter.github.io/tree-sitter/assets/schemas/grammar.schema.json",
"name": "stonescript",
"word": "identifier",
"rules": {
@@ -558,13 +559,6 @@
"type": "BLANK"
}
]
},
{
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "print_continuation"
}
}
]
}
@@ -643,35 +637,6 @@
}
]
},
"print_continuation": {
"type": "PREC_RIGHT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "^"
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "PATTERN",
"value": "[^@\\r\\n]+"
},
{
"type": "SYMBOL",
"name": "interpolation"
}
]
}
}
]
}
},
"color_code": {
"type": "PATTERN",
"value": "#[a-zA-Z0-9]+"
@@ -747,6 +712,10 @@
"type": "SYMBOL",
"name": "new_statement"
},
{
"type": "SYMBOL",
"name": "ascii_string"
},
{
"type": "SYMBOL",
"name": "color_code"
@@ -859,16 +828,34 @@
"name": "_expression"
},
{
"type": "STRING",
"value": "["
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "["
},
{
"type": "STRING",
"value": "［"
}
]
},
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "STRING",
"value": "]"
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "]"
},
{
"type": "STRING",
"value": "］"
}
]
}
]
}
@@ -1297,15 +1284,43 @@
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "["
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "["
},
{
"type": "STRING",
"value": "［"
}
]
},
{
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "_newline"
}
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "array_elements"
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "array_elements"
},
{
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "_newline"
}
}
]
},
{
"type": "BLANK"
@@ -1313,8 +1328,17 @@
]
},
{
"type": "STRING",
"value": "]"
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "]"
},
{
"type": "STRING",
"value": "］"
}
]
}
]
},
@@ -1334,6 +1358,13 @@
"type": "STRING",
"value": ","
},
{
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "_newline"
}
},
{
"type": "SYMBOL",
"name": "_expression"
@@ -1368,31 +1399,65 @@
"value": "\\d+\\.\\d+"
},
"string": {
"type": "SEQ",
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "\""
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "PATTERN",
"value": "[^\"\\\\]"
},
{
"type": "PATTERN",
"value": "\\\\."
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "\""
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "PATTERN",
"value": "[^\"\\\\]"
},
{
"type": "PATTERN",
"value": "\\\\."
}
]
}
]
}
},
{
"type": "STRING",
"value": "\""
}
]
},
{
"type": "STRING",
"value": "\""
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "＂"
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "PATTERN",
"value": "[^＂\\\\]"
},
{
"type": "PATTERN",
"value": "\\\\."
}
]
}
},
{
"type": "STRING",
"value": "＂"
}
]
}
]
},
@@ -1412,6 +1477,71 @@
"null": {
"type": "STRING",
"value": "null"
},
"ascii_string": {
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "ascii"
},
{
"type": "SYMBOL",
"name": "ascii_content"
},
{
"type": "STRING",
"value": "asciiend"
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "["
},
{
"type": "STRING",
"value": "［"
}
]
},
{
"type": "STRING",
"value": "ascii"
},
{
"type": "SYMBOL",
"name": "ascii_content"
},
{
"type": "STRING",
"value": "asciiend"
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "]"
},
{
"type": "STRING",
"value": "］"
}
]
}
]
}
]
}
},
"extras": [
@@ -1421,7 +1551,7 @@
},
{
"type": "PATTERN",
"value": "[\\r\\n]\\^"
"value": "\\r?\\n[ \\t]*\\^"
},
{
"type": "SYMBOL",
@@ -1454,6 +1584,12 @@
[
"command",
"_expression"
],
[
"array_elements"
],
[
"ascii_string"
]
],
"precedences": [],
@@ -1469,9 +1605,13 @@
{
"type": "SYMBOL",
"name": "_dedent"
},
{
"type": "SYMBOL",
"name": "ascii_content"
}
],
"inline": [],
"supertypes": []
"supertypes": [],
"reserved": {}
}

133
src/node-types.json generated
View File

@@ -11,6 +11,10 @@
"type": "array",
"named": true
},
{
"type": "ascii_string",
"named": true
},
{
"type": "assignment_expression",
"named": true
@@ -109,6 +113,10 @@
"type": "array",
"named": true
},
{
"type": "ascii_string",
"named": true
},
{
"type": "assignment_expression",
"named": true
@@ -176,6 +184,21 @@
]
}
},
{
"type": "ascii_string",
"named": true,
"fields": {},
"children": {
"multiple": false,
"required": true,
"types": [
{
"type": "ascii_content",
"named": true
}
]
}
},
{
"type": "assignment_expression",
"named": true,
@@ -188,6 +211,10 @@
"type": "array",
"named": true
},
{
"type": "ascii_string",
"named": true
},
{
"type": "assignment_expression",
"named": true
@@ -267,6 +294,10 @@
"type": "array",
"named": true
},
{
"type": "ascii_string",
"named": true
},
{
"type": "assignment_expression",
"named": true
@@ -418,6 +449,10 @@
"type": "array",
"named": true
},
{
"type": "ascii_string",
"named": true
},
{
"type": "assignment_expression",
"named": true
@@ -544,6 +579,10 @@
"type": "array",
"named": true
},
{
"type": "ascii_string",
"named": true
},
{
"type": "assignment_expression",
"named": true
@@ -627,6 +666,10 @@
"type": "array",
"named": true
},
{
"type": "ascii_string",
"named": true
},
{
"type": "assignment_expression",
"named": true
@@ -725,6 +768,10 @@
"type": "array",
"named": true
},
{
"type": "ascii_string",
"named": true
},
{
"type": "assignment_expression",
"named": true
@@ -804,6 +851,10 @@
"type": "array",
"named": true
},
{
"type": "ascii_string",
"named": true
},
{
"type": "assignment_expression",
"named": true
@@ -932,6 +983,10 @@
"type": "array",
"named": true
},
{
"type": "ascii_string",
"named": true
},
{
"type": "assignment_expression",
"named": true
@@ -1011,6 +1066,10 @@
"type": "array",
"named": true
},
{
"type": "ascii_string",
"named": true
},
{
"type": "assignment_expression",
"named": true
@@ -1090,6 +1149,10 @@
"type": "array",
"named": true
},
{
"type": "ascii_string",
"named": true
},
{
"type": "assignment_expression",
"named": true
@@ -1210,6 +1273,10 @@
"type": "array",
"named": true
},
{
"type": "ascii_string",
"named": true
},
{
"type": "assignment_expression",
"named": true
@@ -1324,31 +1391,12 @@
"named": true,
"fields": {},
"children": {
"multiple": true,
"multiple": false,
"required": false,
"types": [
{
"type": "print_args",
"named": true
},
{
"type": "print_continuation",
"named": true
}
]
}
},
{
"type": "print_continuation",
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": false,
"types": [
{
"type": "interpolation",
"named": true
}
]
}
@@ -1365,6 +1413,10 @@
"type": "array",
"named": true
},
{
"type": "ascii_string",
"named": true
},
{
"type": "assignment_expression",
"named": true
@@ -1435,6 +1487,7 @@
{
"type": "source_file",
"named": true,
"root": true,
"fields": {},
"children": {
"multiple": true,
@@ -1531,6 +1584,10 @@
"type": "array",
"named": true
},
{
"type": "ascii_string",
"named": true
},
{
"type": "assignment_expression",
"named": true
@@ -1610,6 +1667,10 @@
"type": "array",
"named": true
},
{
"type": "ascii_string",
"named": true
},
{
"type": "assignment_expression",
"named": true
@@ -1699,6 +1760,10 @@
"type": "array",
"named": true
},
{
"type": "ascii_string",
"named": true
},
{
"type": "assignment_expression",
"named": true
@@ -1908,12 +1973,21 @@
"named": false
},
{
"type": "^",
"type": "ascii",
"named": false
},
{
"type": "ascii_content",
"named": true
},
{
"type": "asciiend",
"named": false
},
{
"type": "block_comment",
"named": true
"named": true,
"extra": true
},
{
"type": "break_statement",
@@ -1925,7 +1999,8 @@
},
{
"type": "comment",
"named": true
"named": true,
"extra": true
},
{
"type": "continue_statement",
@@ -1990,5 +2065,17 @@
{
"type": "|",
"named": false
},
{
"type": "＂",
"named": false
},
{
"type": "［",
"named": false
},
{
"type": "］",
"named": false
}
]

28563
src/parser.c generated
View File

File diff suppressed because it is too large Load Diff

118
src/scanner.c
View File

@@ -6,7 +6,7 @@ enum TokenType {
NEWLINE,
INDENT,
DEDENT,
// ASCII_CONTENT,
ASCII_CONTENT,
};
// ... (skipping to logic)
@@ -101,64 +101,64 @@ void tree_sitter_stonescript_external_scanner_deserialize(void *payload, const c
bool tree_sitter_stonescript_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) {
Scanner *scanner = (Scanner *)payload;
// if (valid_symbols[ASCII_CONTENT]) {
// bool has_content = false;
//
// for (;;) {
// if (lexer->eof(lexer)) {
// break;
// }
//
// // Check if we're at the start of a line with 'asciiend'
// if (lexer->lookahead == '\n' || lexer->lookahead == '\r') {
// lexer->advance(lexer, false);
// if (lexer->lookahead == '\r' || lexer->lookahead == '\n') {
// lexer->advance(lexer, false);
// }
// lexer->mark_end(lexer);
// has_content = true;
//
// // Skip whitespace at the start of the line
// while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
// lexer->advance(lexer, false);
// }
//
// // Check if this line starts with 'asciiend'
// if (lexer->lookahead == 'a') {
// const char *keyword = "asciiend";
// bool match = true;
//
// for (int k = 0; k < 8; k++) {
// if (lexer->lookahead == keyword[k]) {
// lexer->advance(lexer, false);
// } else {
// match = false;
// break;
// }
// }
//
// // Check that asciiend is followed by whitespace or EOL
// if (match && (lexer->lookahead == '\n' || lexer->lookahead == '\r' ||
// lexer->lookahead == ' ' || lexer->lookahead == '\t' ||
// lexer->lookahead == ',' ||
// lexer->eof(lexer))) {
// lexer->result_symbol = ASCII_CONTENT;
// return has_content;
// }
//
// // Failed to match asciiend, mark the current position
// lexer->mark_end(lexer);
// }
// } else {
// lexer->advance(lexer, false);
// lexer->mark_end(lexer);
// has_content = true;
// }
// }
//
// lexer->result_symbol = ASCII_CONTENT;
// return has_content;
// }
if (valid_symbols[ASCII_CONTENT]) {
bool has_content = false;
for (;;) {
if (lexer->eof(lexer)) {
break;
}
// Check if we're at the start of a line with 'asciiend'
if (lexer->lookahead == '\n' || lexer->lookahead == '\r') {
lexer->advance(lexer, false);
if (lexer->lookahead == '\r' || lexer->lookahead == '\n') {
lexer->advance(lexer, false);
}
lexer->mark_end(lexer);
has_content = true;
// Skip whitespace at the start of the line
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
lexer->advance(lexer, false);
}
// Check if this line starts with 'asciiend'
if (lexer->lookahead == 'a') {
const char *keyword = "asciiend";
bool match = true;
for (int k = 0; k < 8; k++) {
if (lexer->lookahead == keyword[k]) {
lexer->advance(lexer, false);
} else {
match = false;
break;
}
}
// Check that asciiend is followed by whitespace or EOL
if (match && (lexer->lookahead == '\n' || lexer->lookahead == '\r' ||
lexer->lookahead == ' ' || lexer->lookahead == '\t' ||
lexer->lookahead == ',' ||
lexer->eof(lexer))) {
lexer->result_symbol = ASCII_CONTENT;
return has_content;
}
// Failed to match asciiend, mark the current position
lexer->mark_end(lexer);
}
} else {
lexer->advance(lexer, false);
lexer->mark_end(lexer);
has_content = true;
}
}
lexer->result_symbol = ASCII_CONTENT;
return has_content;
}
if (scanner->queued_tokens_size > 0) {
enum TokenType token = scanner->queued_tokens[0];