Rework Symbol Classification

Author: sergeyklay

NEWS

@@ -9,20 +9,45 @@ This file is about changes in BNF Mode.
 ** Incompatible changes.
 *** Dropped support for GNU Emacs < 27.1.
 BNF Mode no longer support GNU Emacs versions older than 27.1.  This
-change allows to use modern APIs, like 'rx-define', and simplifies CI
+change allows to use modern APIs, like `rx-define', and simplifies CI
 workflows by not having to accommodate outdated Emacs versions.  If
 you are still using an older version of Emacs or other flavors (e.g.,
 XEmacs), please stick with an earlier version of BNF Mode.
 
 *** Removed the 'bnf-rx' macro.
-The 'bnf-rx' macro is no longer part of BNF Mode as it's no longer
-necessary.  BNF Mode now fully utilizes the modern 'rx' API, relying
-directly on 'rx-define' for pattern definitions.
+The `bnf-rx' macro is no longer part of BNF Mode as it's no longer
+necessary.  BNF Mode now fully utilizes the modern `rx' API, relying
+directly on `rx-define' for pattern definitions.
 
 *** Removed the 'bnf-rx-constituents' constant.
-BNF Mode no longer uses the 'bnf-rx-constituents' constant.  Instead,
-it now relies on 'rx-define' to define patterns directly, utilizing
-the modern 'rx' API to simplify and streamline pattern definitions.
+BNF Mode no longer uses the `bnf-rx-constituents' constant.  Instead,
+it now relies on `rx-define' to define patterns directly, utilizing
+the modern `rx' API to simplify and streamline pattern definitions.
+
+** Syntax Table changes.
+*** Improved Syntax Table Handling.
+Changed the syntax classification of :, =, and | from symbol
+constituents to punctuation characters.  This adjustment ensures that
+navigation commands like C-M-f and C-M-b work as expected, treating
+these characters as separate tokens rather than part of a symbol.
+
+*** Improved Handling of Angle Brackets.
+Changed the default syntax classification of < and > to punctuation
+characters.  Introduced the `bnf--syntax-propertize' constant that
+performs a detailed examination of the buffer content.  It selectively
+reclassifies < and > as angle brackets (delimiter characters) only
+when they enclose nonterminal symbols.  This approach allows for more
+accurate syntax highlighting and parsing, especially in lines where
+angle brackets are not part of nonterminals.  By defaulting to
+punctuation and then dynamically adjusting the syntax properties as
+needed, we ensure that angle brackets are correctly interpreted in
+various contexts within BNF grammar files.
+
+*** Fixed Incorrect Symbol Classification.
+Adjusted the syntax entries for characters like ', ", (, ), {, }, [,
+and ] to be treated as punctuation rather than symbols.  This change
+prevents these characters from being incorrectly grouped with adjacent
+symbols, enhancing text manipulation and editing commands.
 
 * BNF Mode 0.4.5
 ** Tests were migrated from ert-runner to buttercup.

bnf-mode.el

@@ -37,7 +37,7 @@
 ;;;; Requirements
 
 (eval-when-compile
-  (require 'rx))    ; `rx'
+  (require 'rx))    ; `rx', `rx-define'
 
 
 ;;;; Customization
@@ -69,8 +69,7 @@
 ;;;; Font Locking
 
 (defvar bnf-font-lock-keywords
-  `(
-    ;; LHS nonterminals may be preceded
+  `(;; LHS nonterminals may be preceded
     ;; by an unlimited number of spaces
     (,(rx (and line-start
                (0+ space)
@@ -88,15 +87,14 @@
                (0+ space)))
      1 font-lock-builtin-face)
     ;; “may expand into” symbol
-    (,(rx (and symbol-start
+    (,(rx (and ">"
+               (0+ (in " \t\n"))
                (group "::=")
-               symbol-end))
+               (0+ space)))
      1 font-lock-constant-face)
     ;; Alternatives
     (,(rx (and (0+ space)
-               symbol-start
                (group "|")
-               symbol-end
                (0+ space)))
      1 font-lock-warning-face))
   "Font lock BNF keywords for BNF Mode.")
@@ -106,35 +104,29 @@
 
 (defvar bnf-mode-syntax-table
   (let ((table (make-syntax-table)))
-    ;; FIXME: "_" doesn't mean "symbol" but "symbol constituent".
-    ;; I.e. the settings below mean that Emacs will consider "a:b=(c" as one
-    ;; symbol (aka "identifier") which can be seen if you try to C-M-f and
-    ;; C-M-b to move by sexps.
-
-    ;; Treat ::= as sequence of symbols
-    (modify-syntax-entry ?\: "_" table)
-    (modify-syntax-entry ?\= "_" table)
-
-    ;; Treat | as a symbol
-    (modify-syntax-entry ?\| "_" table)
+    ;; Treat :, =, and | as punctuation
+    (modify-syntax-entry ?\: "." table)
+    (modify-syntax-entry ?\= "." table)
+    (modify-syntax-entry ?\| "." table)
 
     ;; In BNF there are no strings
-    ;; so treat ' and " as symbols
-    (modify-syntax-entry ?\" "_" table)
-    (modify-syntax-entry ?\' "_" table)
+    ;; so treat ' and " as punctuation
+    (modify-syntax-entry ?\" "." table)
+    (modify-syntax-entry ?\' "." table)
 
     ;; In BNF there are no grouping
     ;; brackets except angle ones
-    (modify-syntax-entry ?\( "_" table)
-    (modify-syntax-entry ?\) "_" table)
-    (modify-syntax-entry ?\{ "_" table)
-    (modify-syntax-entry ?\} "_" table)
-    (modify-syntax-entry ?\[ "_" table)
-    (modify-syntax-entry ?\] "_" table)
-
-    ;; Group angle brackets
-    (modify-syntax-entry ?\< "(>" table)
-    (modify-syntax-entry ?\> ")<" table)
+    (modify-syntax-entry ?\( "." table)
+    (modify-syntax-entry ?\) "." table)
+    (modify-syntax-entry ?\{ "." table)
+    (modify-syntax-entry ?\} "." table)
+    (modify-syntax-entry ?\[ "." table)
+    (modify-syntax-entry ?\] "." table)
+
+    ;; Treat angle brackets as punctuation by default.
+    ;; We'll ajust them later, in `bnf--syntax-propertize'.
+    (modify-syntax-entry ?\< "." table)
+    (modify-syntax-entry ?\> "." table)
 
     ;; Comments are begins with “;” and ends with “\n”
     (modify-syntax-entry ?\; "<" table)
@@ -143,6 +135,18 @@
     table)
   "Syntax table in use in `bnf-mode' buffers.")
 
+(defconst bnf--syntax-propertize
+  (syntax-propertize-rules
+   ;; Group angle brackets
+   ("\\(<\\)\\([^<>]*\\)\\(>\\)"
+    (1 "(>")
+    (3 ")<")))
+  "Syntax propertization rules for `bnf-mode'.
+
+These rules assign appropriate syntax properties to specific
+sequences in BNF grammar files, ensuring correct syntax
+highlighting and code navigation in `bnf-mode' buffers.")
+
 
 ;;;; Initialization
 
@@ -162,6 +166,9 @@ Turning on BNF Mode calls the value of `prog-mode-hook' and then of
   (setq-local comment-end "")
   (setq-local comment-start-skip "\\(?:\\(\\W\\|^\\);+\\)\\s-+")
 
+  ;; Enable dynamic syntax properties for accurate parsing
+  (setq-local syntax-propertize-function bnf--syntax-propertize)
+
   ;; Font locking
   (setq font-lock-defaults
         '(

test/test-bnf-mode-font-lock.el

@@ -60,11 +60,9 @@
                "; foo" comment))))
 
   (it "does not mix terminals and nonterminals"
-    (expect "<stm> ::= <decl>
-     angle-brackets ::= are-optional"
+    (expect "<stm> ::= <decl>"
             :to-be-fontified-as
-            '(("stm" function-name "::=" constant "decl" builtin)
-              ("::=" constant))))
+            '(("stm" function-name "::=" constant "decl" builtin))))
 
   (it "fontifies nonterminals despite the case"
     (expect "<RULE> ::= <foo>