refactor: migrate syntax kind to codegen

Author: psteinroe

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "libpg_query"]
+	path = libpg_query
+	url = git@github.com:pganalyze/libpg_query.git

Cargo.toml

@@ -5,6 +5,7 @@ members = [
 
 [workspace.dependencies]
 parser = { path = "./crates/parser", version = "0.0.0" }
+codegen = { path = "./crates/codegen", version = "0.0.0" }
 sourcegen = { path = "./crates/sourcegen", version = "0.0.0" }
 pg_query_proto_parser = { path = "./crates/pg_query_proto_parser", version = "0.0.0" }
 triomphe = { version = "0.1.8", default-features = false, features = ["std"] }

crates/codegen/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "codegen"
+version = "0.0.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+proc-macro2 = "1.0.66"
+quote = "1.0.33"
+pg_query_proto_parser.workspace = true
+
+[lib]
+proc-macro = true

crates/codegen/src/lib.rs

@@ -0,0 +1,8 @@
+mod syntax_kind;
+
+use syntax_kind::syntax_kind_mod;
+
+#[proc_macro]
+pub fn syntax_kind(item: proc_macro::TokenStream) -> proc_macro::TokenStream {
+    syntax_kind_mod(item.into()).into()
+}

crates/codegen/src/syntax_kind.rs

@@ -0,0 +1,136 @@
+use std::collections::HashSet;
+
+use pg_query_proto_parser::{Node, ProtoParser, Token};
+use proc_macro2::{Ident, Literal};
+use quote::{format_ident, quote};
+
+pub fn syntax_kind_mod(_item: proc_macro2::TokenStream) -> proc_macro2::TokenStream {
+    // let parser = ProtoParser::new(
+    //     "/Users/raminder.singh/src/rust/postgres_lsp/crates/parser/proto/source.proto",
+    // );
+    let parser = ProtoParser::new("./crates/parser/proto/source.proto");
+    let proto_file = parser.parse();
+
+    let mut current_enum_names: HashSet<&str> = HashSet::new();
+
+    let custom_node_names = custom_node_names();
+    let custom_node_identifiers = custom_node_identifiers(&custom_node_names);
+    current_enum_names.extend(&custom_node_names);
+
+    let node_identifiers = node_identifiers(&proto_file.nodes, &current_enum_names);
+    current_enum_names.extend(node_names(&proto_file.nodes));
+
+    let token_identifiers = token_identifiers(&proto_file.tokens, &current_enum_names);
+    let token_value_literals = token_value_literals(&proto_file.tokens, &current_enum_names);
+
+    let syntax_kind_impl =
+        syntax_kind_impl(&node_identifiers, &token_identifiers, &token_value_literals);
+
+    quote! {
+        use cstree::Syntax;
+        use pg_query::{protobuf::ScanToken, NodeEnum, NodeRef};
+
+        /// An u32 enum of all valid syntax elements (nodes and tokens) of the postgres
+        /// sql dialect, and a few custom ones that are not parsed by pg_query.rs, such
+        /// as `Whitespace`.
+        #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Syntax)]
+        #[repr(u32)]
+        pub enum SyntaxKind {
+            // custom nodes, which are not parsed by pg_query.rs
+            #(#custom_node_identifiers),*,
+            #(#node_identifiers),*,
+            #(#token_identifiers),*,
+        }
+
+        #syntax_kind_impl
+    }
+}
+
+fn custom_node_names() -> Vec<&'static str> {
+    vec![
+        "SourceFile",
+        "Comment",
+        "Whitespace",
+        "Newline",
+        "Tab",
+        "Stmt",
+    ]
+}
+
+fn node_names(nodes: &[Node]) -> impl Iterator<Item = &str> {
+    nodes.iter().map(|node| node.name.as_str())
+}
+
+fn custom_node_identifiers(custom_node_names: &[&str]) -> Vec<Ident> {
+    custom_node_names
+        .iter()
+        .map(|&node_name| format_ident!("{}", node_name))
+        .collect()
+}
+
+fn node_identifiers(nodes: &[Node], existing_enum_names: &HashSet<&str>) -> Vec<Ident> {
+    nodes
+        .iter()
+        .filter(|&token| !existing_enum_names.contains(token.name.as_str()))
+        .map(|node| format_ident!("{}", &node.name))
+        .collect()
+}
+
+fn token_identifiers(tokens: &[Token], existing_enum_names: &HashSet<&str>) -> Vec<Ident> {
+    tokens
+        .iter()
+        .filter(|&token| !existing_enum_names.contains(token.name.as_str()))
+        .map(|token| format_ident!("{}", &token.name))
+        .collect()
+}
+
+fn token_value_literals(tokens: &[Token], existing_enum_names: &HashSet<&str>) -> Vec<Literal> {
+    tokens
+        .iter()
+        .filter(|&token| !existing_enum_names.contains(token.name.as_str()))
+        .map(|token| Literal::i32_unsuffixed(token.value))
+        .collect()
+}
+
+fn syntax_kind_impl(
+    node_identifiers: &[Ident],
+    token_identifiers: &[Ident],
+    token_value_literals: &[Literal],
+) -> proc_macro2::TokenStream {
+    let new_from_pg_query_node_fn = new_from_pg_query_node_fn(node_identifiers);
+    let new_from_pg_query_token_fn =
+        new_from_pg_query_token_fn(token_identifiers, token_value_literals);
+    quote! {
+        impl SyntaxKind {
+            #new_from_pg_query_node_fn
+
+            #new_from_pg_query_token_fn
+        }
+    }
+}
+
+fn new_from_pg_query_node_fn(node_identifiers: &[Ident]) -> proc_macro2::TokenStream {
+    quote! {
+        /// Converts a `pg_query` node to a `SyntaxKind`
+        pub fn new_from_pg_query_node(node: &NodeEnum) -> Self {
+            match node {
+                #(NodeEnum::#node_identifiers(_) => SyntaxKind::#node_identifiers),*
+            }
+        }
+    }
+}
+
+fn new_from_pg_query_token_fn(
+    token_identifiers: &[Ident],
+    token_value_literals: &[Literal],
+) -> proc_macro2::TokenStream {
+    quote! {
+        /// Converts a `pg_query` token to a `SyntaxKind`
+        pub fn new_from_pg_query_token(token: &ScanToken) -> Self {
+            match token.token {
+                #(#token_value_literals => SyntaxKind::#token_identifiers),*,
+                _ => panic!("Unknown token"),
+            }
+        }
+    }
+}

crates/parser/Cargo.toml

@@ -15,5 +15,5 @@ serde = { version = "1.0", features = ["derive"] }
 env_logger = { version = "0.9.1" }
 log = { version = "0.4.20" }
 
-sourcegen.workspace = true
+codegen.workspace = true
 pg_query_proto_parser.workspace = true

crates/parser/src/lib.rs

@@ -24,10 +24,10 @@ mod sibling_token;
 mod source_file;
 mod statement;
 mod syntax_error;
-mod syntax_kind_generated;
+mod syntax_kind_codegen;
 mod syntax_node;
 mod token_type;
 
 pub use crate::parser::{Parse, Parser};
-pub use crate::syntax_kind_generated::SyntaxKind;
+pub use crate::syntax_kind_codegen::SyntaxKind;
 pub use crate::syntax_node::{SyntaxElement, SyntaxNode, SyntaxToken};

crates/parser/src/parser.rs

@@ -6,7 +6,7 @@ use pg_query::NodeEnum;
 
 use crate::ast_node::RawStmt;
 use crate::syntax_error::SyntaxError;
-use crate::syntax_kind_generated::SyntaxKind;
+use crate::syntax_kind_codegen::SyntaxKind;
 use crate::syntax_node::SyntaxNode;
 use crate::token_type::TokenType;
 

crates/parser/src/sibling_token.rs

@@ -1,4 +1,4 @@
-use crate::SyntaxKind;
+use crate::syntax_kind_codegen::SyntaxKind;
 
 impl SyntaxKind {
     pub fn is_opening_sibling(&self) -> bool {

crates/parser/src/source_file.rs

@@ -1,6 +1,6 @@
 use logos::Logos;
 
-use crate::{parser::Parser, syntax_kind_generated::SyntaxKind};
+use crate::{parser::Parser, syntax_kind_codegen::SyntaxKind};
 
 /// A super simple lexer for sql files that splits the input into indivudual statements and
 /// comments.

crates/parser/src/statement.rs

@@ -4,7 +4,7 @@ use logos::{Logos, Span};
 use crate::{
     parser::Parser,
     pg_query_utils_generated::get_children,
-    syntax_kind_generated::SyntaxKind,
+    syntax_kind_codegen::SyntaxKind,
     token_type::{
         get_token_type_from_pg_query_token, get_token_type_from_statement_token, TokenType,
     },

crates/parser/src/syntax_kind_codegen.rs

@@ -0,0 +1,3 @@
+use codegen::syntax_kind;
+
+syntax_kind!();