update

corpus/test_ascii.txt

@@ -0,0 +1,22 @@
+==================
+ASCII Array
+==================
+var x = [ascii
+foo
+asciiend
+, ascii
+bar
+asciiend
+]
+------------------
+(source_file
+  (variable_declaration
+    (identifier)
+    (array
+      (array_elements
+        (ascii_string (ascii_content))
+        (ascii_string (ascii_content))
+      )
+    )
+  )
+)

grammar.js

@@ -8,6 +8,7 @@ module.exports = grammar({
             choice(
                 $._newline,
                 $.comment,
+                $.block_comment,
                 // Keyword-based statements (must come before generic command)
                 $.variable_declaration,      // 'var'
                 $.function_declaration,      // 'func'

src/grammar.json

@@ -1,5 +1,4 @@
 {
-  "$schema": "https://tree-sitter.github.io/tree-sitter/assets/schemas/grammar.schema.json",
   "name": "stonescript",
   "word": "identifier",
   "rules": {
@@ -27,6 +26,10 @@
                 "type": "SYMBOL",
                 "name": "comment"
               },
+              {
+                "type": "SYMBOL",
+                "name": "block_comment"
+              },
               {
                 "type": "SYMBOL",
                 "name": "variable_declaration"
@@ -1431,6 +1434,6 @@
     }
   ],
   "inline": [],
-  "supertypes": [],
+  "supertypes": []
-  "reserved": {}
 }
+

src/node-types.json

@@ -389,6 +389,10 @@
       "multiple": true,
       "required": false,
       "types": [
+        {
+          "type": "block_comment",
+          "named": true
+        },
         {
           "type": "break_statement",
           "named": true
@@ -1539,12 +1543,15 @@
   {
     "type": "source_file",
     "named": true,
-    "root": true,
     "fields": {},
     "children": {
       "multiple": true,
       "required": false,
       "types": [
+        {
+          "type": "block_comment",
+          "named": true
+        },
         {
           "type": "break_statement",
           "named": true
@@ -2042,8 +2049,7 @@
   },
   {
     "type": "block_comment",
-    "named": true,
+    "named": true
-    "extra": true
   },
   {
     "type": "break_statement",
@@ -2055,8 +2061,7 @@
   },
   {
     "type": "comment",
-    "named": true,
+    "named": true
-    "extra": true
   },
   {
     "type": "continue_statement",

src/scanner.c

@@ -102,10 +102,9 @@ void tree_sitter_stonescript_external_scanner_deserialize(void *payload, const c
 bool tree_sitter_stonescript_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) {
   Scanner *scanner = (Scanner *)payload;
 
-  // Try to handle block comments whenever we see /* 
+  // Try to handle block comments when parser expects them
-  // This needs to run early before other checks
+  // Only check if valid_symbols allows BLOCK_COMMENT
-  if (lexer->lookahead == '/') {
+  if (valid_symbols[BLOCK_COMMENT] && lexer->lookahead == '/') {
-    lexer->mark_end(lexer);
     lexer->advance(lexer, false);
     if (lexer->lookahead == '*') {
       lexer->advance(lexer, false);

src/tree_sitter/parser.h

@@ -13,17 +13,12 @@ extern "C" {
 #define ts_builtin_sym_end 0
 #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
 
-#ifndef TREE_SITTER_API_H_
 typedef uint16_t TSStateId;
+
+#ifndef TREE_SITTER_API_H_
 typedef uint16_t TSSymbol;
 typedef uint16_t TSFieldId;
 typedef struct TSLanguage TSLanguage;
-typedef struct TSLanguageMetadata TSLanguageMetadata;
-typedef struct TSLanguageMetadata {
-  uint8_t major_version;
-  uint8_t minor_version;
-  uint8_t patch_version;
-} TSLanguageMetadata;
 #endif
 
 typedef struct {
@@ -32,11 +27,10 @@ typedef struct {
   bool inherited;
 } TSFieldMapEntry;
 
-// Used to index the field and supertype maps.
 typedef struct {
   uint16_t index;
   uint16_t length;
-} TSMapSlice;
+} TSFieldMapSlice;
 
 typedef struct {
   bool visible;
@@ -54,7 +48,6 @@ struct TSLexer {
   uint32_t (*get_column)(TSLexer *);
   bool (*is_at_included_range_start)(const TSLexer *);
   bool (*eof)(const TSLexer *);
-  void (*log)(const TSLexer *, const char *, ...);
 };
 
 typedef enum {
@@ -86,12 +79,6 @@ typedef struct {
   uint16_t external_lex_state;
 } TSLexMode;
 
-typedef struct {
-  uint16_t lex_state;
-  uint16_t external_lex_state;
-  uint16_t reserved_word_set_id;
-} TSLexerMode;
-
 typedef union {
   TSParseAction action;
   struct {
@@ -100,13 +87,8 @@ typedef union {
   } entry;
 } TSParseActionEntry;
 
-typedef struct {
-  int32_t start;
-  int32_t end;
-} TSCharacterRange;
-
 struct TSLanguage {
-  uint32_t abi_version;
+  uint32_t version;
   uint32_t symbol_count;
   uint32_t alias_count;
   uint32_t token_count;
@@ -122,13 +104,13 @@ struct TSLanguage {
   const TSParseActionEntry *parse_actions;
   const char * const *symbol_names;
   const char * const *field_names;
-  const TSMapSlice *field_map_slices;
+  const TSFieldMapSlice *field_map_slices;
   const TSFieldMapEntry *field_map_entries;
   const TSSymbolMetadata *symbol_metadata;
   const TSSymbol *public_symbol_map;
   const uint16_t *alias_map;
   const TSSymbol *alias_sequences;
-  const TSLexerMode *lex_modes;
+  const TSLexMode *lex_modes;
   bool (*lex_fn)(TSLexer *, TSStateId);
   bool (*keyword_lex_fn)(TSLexer *, TSStateId);
   TSSymbol keyword_capture_token;
@@ -142,48 +124,15 @@ struct TSLanguage {
     void (*deserialize)(void *, const char *, unsigned);
   } external_scanner;
   const TSStateId *primary_state_ids;
-  const char *name;
-  const TSSymbol *reserved_words;
-  uint16_t max_reserved_word_set_size;
-  uint32_t supertype_count;
-  const TSSymbol *supertype_symbols;
-  const TSMapSlice *supertype_map_slices;
-  const TSSymbol *supertype_map_entries;
-  TSLanguageMetadata metadata;
 };
 
-static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
-  uint32_t index = 0;
-  uint32_t size = len - index;
-  while (size > 1) {
-    uint32_t half_size = size / 2;
-    uint32_t mid_index = index + half_size;
-    const TSCharacterRange *range = &ranges[mid_index];
-    if (lookahead >= range->start && lookahead <= range->end) {
-      return true;
-    } else if (lookahead > range->end) {
-      index = mid_index;
-    }
-    size -= half_size;
-  }
-  const TSCharacterRange *range = &ranges[index];
-  return (lookahead >= range->start && lookahead <= range->end);
-}
-
 /*
  *  Lexer Macros
  */
 
-#ifdef _MSC_VER
-#define UNUSED __pragma(warning(suppress : 4101))
-#else
-#define UNUSED __attribute__((unused))
-#endif
-
 #define START_LEXER()           \
   bool result = false;          \
   bool skip = false;            \
-  UNUSED                        \
   bool eof = false;             \
   int32_t lookahead;            \
   goto start;                   \
@@ -199,17 +148,6 @@ static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, in
     goto next_state;         \
   }
 
-#define ADVANCE_MAP(...)                                              \
-  {                                                                   \
-    static const uint16_t map[] = { __VA_ARGS__ };                    \
-    for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) {  \
-      if (map[i] == lookahead) {                                      \
-        state = map[i + 1];                                           \
-        goto next_state;                                              \
-      }                                                               \
-    }                                                                 \
-  }
-
 #define SKIP(state_value) \
   {                       \
     skip = true;          \
@@ -228,7 +166,7 @@ static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, in
  *  Parse Table Macros
  */
 
-#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
+#define SMALL_STATE(id) id - LARGE_STATE_COUNT
 
 #define STATE(id) id
 
@@ -238,7 +176,7 @@ static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, in
   {{                                  \
     .shift = {                        \
       .type = TSParseActionTypeShift, \
-      .state = (state_value)          \
+      .state = state_value            \
     }                                 \
   }}
 
@@ -246,7 +184,7 @@ static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, in
   {{                                  \
     .shift = {                        \
       .type = TSParseActionTypeShift, \
-      .state = (state_value),         \
+      .state = state_value,           \
       .repetition = true              \
     }                                 \
   }}
@@ -259,15 +197,14 @@ static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, in
     }                                 \
   }}
 
-#define REDUCE(symbol_name, children, precedence, prod_id) \
+#define REDUCE(symbol_val, child_count_val, ...) \
-  {{                                                       \
+  {{                                             \
-    .reduce = {                                            \
+    .reduce = {                                  \
-      .type = TSParseActionTypeReduce,                     \
+      .type = TSParseActionTypeReduce,           \
-      .symbol = symbol_name,                               \
+      .symbol = symbol_val,                      \
-      .child_count = children,                             \
+      .child_count = child_count_val,            \
-      .dynamic_precedence = precedence,                    \
+      __VA_ARGS__                                \
-      .production_id = prod_id                             \
+    },                                           \
-    },                                                     \
   }}
 
 #define RECOVER()                    \