Skip to content

Commit c0d4900

Browse files
psteinroepsteinroe
committedJul 9, 2023
fix: make it work
·
0.6.00.1.0
1 parent b4a2fef commit c0d4900

File tree

8 files changed

+225
-100
lines changed

8 files changed

+225
-100
lines changed
 

‎crates/parser/src/parser.rs

+20-7
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ pub struct Parser {
1414
curr_depth: i32,
1515
errors: Vec<SyntaxError>,
1616
stmts: Vec<RawStmt>,
17+
is_parsing_erronous_node: bool,
1718
}
1819

1920
#[derive(Debug)]
@@ -26,27 +27,33 @@ pub struct Parse {
2627
/// Main parser that controls the cst building process, and collects errors and statements
2728
impl Parser {
2829
pub fn close_until_depth(&mut self, depth: i32) {
29-
while self.curr_depth >= depth {
30-
self.inner.finish_node();
30+
while self.curr_depth >= depth && depth > 0 {
31+
self.finish_node();
3132
self.curr_depth -= 1;
3233
}
3334
}
3435

3536
/// start a new node of `SyntaxKind` at `depth`
3637
/// handles closing previous nodes if necessary
3738
/// and consumes token buffer before starting new node
38-
pub fn start_node(&mut self, kind: SyntaxKind, depth: &i32) {
39+
///
40+
/// if `SyntaxKind` is `SyntaxKind::AnyStatement`, sets `is_parsing_erronous_node` to true
41+
pub fn start_node(&mut self, kind: SyntaxKind, depth: i32) {
3942
// close until target depth
40-
self.close_until_depth(*depth);
43+
self.close_until_depth(depth);
4144

4245
self.consume_token_buffer();
4346

44-
self.curr_depth = *depth;
47+
self.curr_depth = depth;
4548
self.inner.start_node(kind);
49+
if kind == SyntaxKind::AnyStatement {
50+
self.is_parsing_erronous_node = true;
51+
}
4652
}
4753

4854
pub fn finish_node(&mut self) {
4955
self.inner.finish_node();
56+
self.is_parsing_erronous_node = false;
5057
}
5158

5259
/// Drains the token buffer and applies all tokens
@@ -58,10 +65,16 @@ impl Parser {
5865

5966
/// applies token based on its `SyntaxKindType`
6067
/// if `SyntaxKindType::Close`, closes all nodes until depth 1
61-
/// if `SyntaxKindType::Follow`, add token to buffer and wait until next node to apply token at
62-
/// same depth
68+
/// if `SyntaxKindType::Follow`, add token to buffer and wait until next node to apply token at same depth
6369
/// otherwise, applies token immediately
70+
///
71+
/// if `is_parsing_erronous_node` is true, applies token immediately
6472
pub fn token(&mut self, kind: SyntaxKind, text: &str) {
73+
if self.is_parsing_erronous_node {
74+
self.inner.token(kind, text);
75+
return;
76+
}
77+
6578
match kind.get_type() {
6679
Some(SyntaxKindType::Close) => {
6780
// move up to depth 2 and consume buffered tokens before applying closing token

‎crates/parser/src/source_file.rs

+27-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ impl Parser {
2626
pub fn parse_source_file(&mut self, text: &str) {
2727
let mut lexer = SourceFileToken::lexer(text);
2828

29-
self.start_node(SyntaxKind::SourceFile, &0);
29+
self.start_node(SyntaxKind::SourceFile, 0);
3030
while let Some(token) = lexer.next() {
3131
match token {
3232
Ok(token) => {
@@ -77,4 +77,30 @@ mod tests {
7777
assert_eq!(lex.next(), Some(Ok(SourceFileToken::Statement)));
7878
assert_eq!(lex.slice(), "select id,username from contact\n\nselect id,name\nfrom contact -- test inline comment\nwhere id = '123';");
7979
}
80+
81+
#[test]
82+
fn test_source_file_parser() {
83+
let input = "select id, name from users where id = '1224';
84+
85+
select select;
86+
87+
88+
89+
90+
91+
select 1;
92+
93+
";
94+
95+
let mut parser = Parser::default();
96+
println!("input {:?}", input);
97+
parser.parse_source_file(input);
98+
let parsed = parser.finish();
99+
100+
dbg!(parsed.errors);
101+
102+
dbg!(&parsed.cst);
103+
104+
assert_eq!(parsed.cst.text(), input);
105+
}
80106
}

‎crates/parser/src/statement.rs

+23-2
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ impl StatementToken {
103103

104104
impl Parser {
105105
pub fn parse_statement(&mut self, text: &str, at_offset: Option<u32>) {
106+
println!("#### parse_statement: {}", text);
106107
let offset = at_offset.unwrap_or(0);
107108
let range = TextRange::new(
108109
TextSize::from(offset),
@@ -116,6 +117,7 @@ impl Parser {
116117
Vec::new().into_iter().peekable()
117118
}
118119
};
120+
println!("pg_query_tokens: {:?}", pg_query_tokens);
119121

120122
let parsed = pg_query::parse(text);
121123
let proto;
@@ -137,14 +139,18 @@ impl Parser {
137139
Vec::new().into_iter().peekable()
138140
}
139141
};
142+
println!("pg_query_nodes: {:?}", pg_query_nodes);
140143

141144
let mut lexer = StatementToken::lexer(&text);
142145

143146
// parse root node if no syntax errors
144147
if pg_query_nodes.peek().is_some() {
145148
let (node, depth, _) = pg_query_nodes.next().unwrap();
146149
self.stmt(node.to_enum(), range);
147-
self.start_node(SyntaxKind::from_pg_query_node(&node), &depth);
150+
self.start_node(SyntaxKind::from_pg_query_node(&node), depth);
151+
} else {
152+
// fallback to generic node as root
153+
self.start_node(SyntaxKind::AnyStatement, 1);
148154
}
149155

150156
while let Some(token) = lexer.next() {
@@ -160,7 +166,7 @@ impl Parser {
160166
} else {
161167
// node is within span
162168
let (node, depth, _) = pg_query_nodes.next().unwrap();
163-
self.start_node(SyntaxKind::from_pg_query_node(&node), &depth);
169+
self.start_node(SyntaxKind::from_pg_query_node(&node), depth);
164170
}
165171
}
166172

@@ -249,6 +255,21 @@ mod tests {
249255
parser.parse_statement(input, None);
250256
let parsed = parser.finish();
251257

258+
dbg!(&parsed.cst);
259+
260+
assert_eq!(parsed.cst.text(), input);
261+
}
262+
263+
#[test]
264+
fn test_invalid_statement() {
265+
let input = "select select;";
266+
267+
let mut parser = Parser::default();
268+
parser.parse_statement(input, None);
269+
let parsed = parser.finish();
270+
271+
dbg!(&parsed.cst);
272+
252273
assert_eq!(parsed.cst.text(), input);
253274
}
254275
}

‎crates/postgres_lsp/src/main.rs

+117-87
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,25 @@
11
mod semantic_token;
2+
mod utils;
3+
24
use dashmap::DashMap;
3-
use log::{debug, error, info, log_enabled, Level};
4-
use parser::{Parse, Parser, SyntaxKind, SyntaxNode, SyntaxToken};
5+
use log::debug;
6+
use parser::{Parse, Parser};
57
use ropey::Rope;
6-
use semantic_token::LEGEND_TYPE;
8+
use semantic_token::{ImCompleteSemanticToken, LEGEND_TYPE};
79
use serde_json::Value;
810
use tower_lsp::jsonrpc::Result;
911
use tower_lsp::lsp_types::*;
1012
use tower_lsp::{Client, LanguageServer, LspService, Server};
1113

1214
use crate::semantic_token::semantic_token_from_syntax_kind;
15+
use crate::utils::offset_to_position;
1316

1417
#[derive(Debug)]
1518
struct Backend {
1619
client: Client,
1720
parse_map: DashMap<String, Parse>,
18-
// ast_map: DashMap<String, HashMap<String, Func>>,
1921
document_map: DashMap<String, Rope>,
20-
// semantic_token_map: DashMap<String, Vec<ImCompleteSemanticToken>>,
22+
semantic_token_map: DashMap<String, Vec<ImCompleteSemanticToken>>,
2123
}
2224

2325
#[tower_lsp::async_trait]
@@ -31,9 +33,9 @@ impl LanguageServer for Backend {
3133
offset_encoding: None,
3234
capabilities: ServerCapabilities {
3335
// inlay_hint_provider: Some(OneOf::Left(true)),
34-
// text_document_sync: Some(TextDocumentSyncCapability::Kind(
35-
// TextDocumentSyncKind::FULL,
36-
// )),
36+
text_document_sync: Some(TextDocumentSyncCapability::Kind(
37+
TextDocumentSyncKind::FULL,
38+
)),
3739
// completion_provider: Some(CompletionOptions {
3840
// resolve_provider: Some(false),
3941
// trigger_characters: Some(vec![".".to_string()]),
@@ -45,13 +47,13 @@ impl LanguageServer for Backend {
4547
// commands: vec!["dummy.do_something".to_string()],
4648
// work_done_progress_options: Default::default(),
4749
// }),
48-
// workspace: Some(WorkspaceServerCapabilities {
49-
// workspace_folders: Some(WorkspaceFoldersServerCapabilities {
50-
// supported: Some(true),
51-
// change_notifications: Some(OneOf::Left(true)),
52-
// }),
53-
// file_operations: None,
54-
// }),
50+
workspace: Some(WorkspaceServerCapabilities {
51+
workspace_folders: Some(WorkspaceFoldersServerCapabilities {
52+
supported: Some(true),
53+
change_notifications: Some(OneOf::Left(true)),
54+
}),
55+
file_operations: None,
56+
}),
5557
semantic_tokens_provider: Some(
5658
SemanticTokensServerCapabilities::SemanticTokensRegistrationOptions(
5759
SemanticTokensRegistrationOptions {
@@ -110,6 +112,9 @@ impl LanguageServer for Backend {
110112
}
111113

112114
async fn did_change(&self, mut params: DidChangeTextDocumentParams) {
115+
self.client
116+
.log_message(MessageType::INFO, "file changed!")
117+
.await;
113118
self.on_change(TextDocumentItem {
114119
uri: params.text_document.uri,
115120
text: std::mem::take(&mut params.content_changes[0].text),
@@ -133,8 +138,55 @@ impl LanguageServer for Backend {
133138
&self,
134139
params: SemanticTokensParams,
135140
) -> Result<Option<SemanticTokensResult>> {
136-
println!("semantic_tokens_full");
137-
return Ok(None);
141+
let uri = params.text_document.uri.to_string();
142+
self.client
143+
.log_message(MessageType::LOG, "semantic_token_full")
144+
.await;
145+
let semantic_tokens = || -> Option<Vec<SemanticToken>> {
146+
let mut im_complete_tokens = self.semantic_token_map.get_mut(&uri)?;
147+
let rope = self.document_map.get(&uri)?;
148+
im_complete_tokens.sort_by(|a, b| a.start.cmp(&b.start));
149+
let mut pre_line = 0;
150+
let mut pre_start = 0;
151+
let semantic_tokens = im_complete_tokens
152+
.iter()
153+
.filter_map(|token| {
154+
let line = rope.try_byte_to_line(token.start).ok()? as u32;
155+
let first = rope.try_line_to_char(line as usize).ok()? as u32;
156+
let start = rope.try_byte_to_char(token.start).ok()? as u32 - first;
157+
let delta_line = line - pre_line;
158+
let delta_start = if delta_line == 0 {
159+
start - pre_start
160+
} else {
161+
start
162+
};
163+
let ret = Some(SemanticToken {
164+
delta_line,
165+
delta_start,
166+
length: token.length as u32,
167+
token_type: token.token_type as u32,
168+
token_modifiers_bitset: 0,
169+
});
170+
pre_line = line;
171+
pre_start = start;
172+
ret
173+
})
174+
.collect::<Vec<_>>();
175+
Some(semantic_tokens)
176+
}();
177+
self.client
178+
.log_message(
179+
MessageType::LOG,
180+
format!("semantic_tokens: {:?}", semantic_tokens),
181+
)
182+
.await;
183+
if let Some(semantic_token) = semantic_tokens {
184+
return Ok(Some(SemanticTokensResult::Tokens(SemanticTokens {
185+
result_id: None,
186+
data: semantic_token,
187+
})));
188+
}
189+
Ok(None)
138190
}
139191

140192
async fn semantic_tokens_range(
@@ -185,83 +237,60 @@ struct TextDocumentItem {
185237
}
186238
impl Backend {
187239
async fn on_change(&self, params: TextDocumentItem) {
188-
debug!("on_change {:?}", params.uri);
240+
self.client
241+
.log_message(MessageType::INFO, format!("on_change {:?}", params.uri))
242+
.await;
189243
let rope = ropey::Rope::from_str(&params.text);
190244
self.document_map
191245
.insert(params.uri.to_string(), rope.clone());
246+
247+
let rope = ropey::Rope::from_str(&params.text);
192248
let mut parser = Parser::default();
249+
193250
parser.parse_source_file(&params.text);
251+
194252
let result = parser.finish();
195253

196-
let semantic_tokens = result.cst.descendants_with_tokens().filter_map(|item| {
197-
match SyntaxKind::try_from(item.syntax_kind()) {
198-
_ => panic!("unexpected syntax kind"),
199-
}
200-
});
201-
202-
// let (ast, errors, semantic_tokens) = parse(&params.text);
203-
204-
// let diagnostics = errors
205-
// .into_iter()
206-
// .filter_map(|item| {
207-
// let (message, span) = match item.reason() {
208-
// chumsky::error::SimpleReason::Unclosed { span, delimiter } => {
209-
// (format!("Unclosed delimiter {}", delimiter), span.clone())
210-
// }
211-
// chumsky::error::SimpleReason::Unexpected => (
212-
// format!(
213-
// "{}, expected {}",
214-
// if item.found().is_some() {
215-
// "Unexpected token in input"
216-
// } else {
217-
// "Unexpected end of input"
218-
// },
219-
// if item.expected().len() == 0 {
220-
// "something else".to_string()
221-
// } else {
222-
// item.expected()
223-
// .map(|expected| match expected {
224-
// Some(expected) => expected.to_string(),
225-
// None => "end of input".to_string(),
226-
// })
227-
// .collect::<Vec<_>>()
228-
// .join(", ")
229-
// }
230-
// ),
231-
// item.span(),
232-
// ),
233-
// chumsky::error::SimpleReason::Custom(msg) => (msg.to_string(), item.span()),
234-
// };
235-
//
236-
// || -> Option<Diagnostic> {
237-
// // let start_line = rope.try_char_to_line(span.start)?;
238-
// // let first_char = rope.try_line_to_char(start_line)?;
239-
// // let start_column = span.start - first_char;
240-
// let start_position = offset_to_position(span.start, &rope)?;
241-
// let end_position = offset_to_position(span.end, &rope)?;
242-
// // let end_line = rope.try_char_to_line(span.end)?;
243-
// // let first_char = rope.try_line_to_char(end_line)?;
244-
// // let end_column = span.end - first_char;
245-
// Some(Diagnostic::new_simple(
246-
// Range::new(start_position, end_position),
247-
// message,
248-
// ))
249-
// }()
250-
// })
251-
// .collect::<Vec<_>>();
254+
dbg!(&result.cst);
255+
256+
// update semantic tokens
257+
let semantic_tokens = result
258+
.cst
259+
.descendants_with_tokens()
260+
.filter_map(|item| match semantic_token_from_syntax_kind(item.kind()) {
261+
Some(token_type) => Some(ImCompleteSemanticToken {
262+
start: item.text_range().start().into(),
263+
token_type,
264+
length: item.text_range().len().into(),
265+
}),
266+
None => None,
267+
})
268+
.collect::<Vec<_>>();
269+
270+
// publish diagnostics
252271
//
253-
// self.client
254-
// .publish_diagnostics(params.uri.clone(), diagnostics, Some(params.version))
255-
// .await;
256-
257-
// if let Some(ast) = ast {
258-
// self.ast_map.insert(params.uri.to_string(), ast);
259-
// }
260-
// self.client
261-
// .log_message(MessageType::INFO, &format!("{:?}", semantic_tokens))
262-
// .await;
263-
// self.semantic_token_map
264-
// .insert(params.uri.to_string(), semantic_tokens);
272+
let diagnostics = result
273+
.errors
274+
.iter()
275+
.map(|error| {
276+
Diagnostic::new_simple(
277+
Range {
278+
start: offset_to_position(error.range().start().into(), &rope).unwrap(),
279+
end: offset_to_position(error.range().start().into(), &rope).unwrap(),
280+
},
281+
error.to_string(),
282+
)
283+
})
284+
.collect::<Vec<_>>();
285+
286+
self.client
287+
.publish_diagnostics(params.uri.clone(), diagnostics, Some(params.version))
288+
.await;
289+
290+
self.semantic_token_map
291+
.insert(params.uri.to_string(), semantic_tokens);
292+
293+
self.parse_map.insert(params.uri.to_string(), result);
265294
}
266295
}
267296

@@ -278,7 +307,8 @@ async fn main() {
278307
client,
279308
// ast_map: DashMap::new(),
280309
document_map: DashMap::new(),
281-
// semantic_token_map: DashMap::new(),
310+
parse_map: DashMap::new(),
311+
semantic_token_map: DashMap::new(),
282312
})
283313
.finish();
284314

‎crates/postgres_lsp/src/semantic_token.rs

+17-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use parser::{SyntaxKind, SyntaxToken};
1+
use parser::SyntaxKind;
22
use tower_lsp::lsp_types::SemanticTokenType;
33

44
/// Semantic token types that are used for highlighting
@@ -49,11 +49,19 @@ pub const LEGEND_TYPE: &[SemanticTokenType] = &[
4949
SemanticTokenType::OPERATOR,
5050
];
5151

52-
pub fn semantic_token_from_syntax_kind(syntax: SyntaxKind) -> Option<SemanticTokenType> {
53-
match syntax {
52+
#[derive(Debug, Clone)]
53+
pub struct ImCompleteSemanticToken {
54+
pub start: usize,
55+
pub length: usize,
56+
pub token_type: usize,
57+
}
58+
59+
pub fn semantic_token_from_syntax_kind(syntax: SyntaxKind) -> Option<usize> {
60+
let token_type = match syntax {
5461
SyntaxKind::Ascii37 => Some(SemanticTokenType::OPERATOR),
5562
SyntaxKind::Ascii42 => Some(SemanticTokenType::OPERATOR),
5663
SyntaxKind::Ascii43 => Some(SemanticTokenType::OPERATOR),
64+
SyntaxKind::Ascii44 => Some(SemanticTokenType::PROPERTY),
5765
SyntaxKind::Ascii45 => Some(SemanticTokenType::OPERATOR),
5866
SyntaxKind::Ascii47 => Some(SemanticTokenType::OPERATOR),
5967
SyntaxKind::Ascii60 => Some(SemanticTokenType::OPERATOR),
@@ -64,6 +72,12 @@ pub fn semantic_token_from_syntax_kind(syntax: SyntaxKind) -> Option<SemanticTok
6472
SyntaxKind::Select => Some(SemanticTokenType::KEYWORD),
6573
SyntaxKind::From => Some(SemanticTokenType::KEYWORD),
6674
SyntaxKind::Where => Some(SemanticTokenType::KEYWORD),
75+
SyntaxKind::ColumnRef => Some(SemanticTokenType::PROPERTY),
76+
SyntaxKind::RangeVar => Some(SemanticTokenType::CLASS),
6777
_ => None,
78+
};
79+
if let Some(token_type) = token_type {
80+
return LEGEND_TYPE.iter().position(|item| item == &token_type);
6881
}
82+
None
6983
}

‎crates/postgres_lsp/src/utils.rs

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
use ropey::Rope;
2+
use tower_lsp::lsp_types::Position;
3+
4+
pub fn offset_to_position(offset: usize, rope: &Rope) -> Option<Position> {
5+
let line = rope.try_char_to_line(offset).ok()?;
6+
let first_char_of_line = rope.try_line_to_char(line).ok()?;
7+
let column = offset - first_char_of_line;
8+
Some(Position::new(line as u32, column as u32))
9+
}

‎example/file.sql

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
2+
select id, name from users where id = '1224';
3+
4+
5+
select select;
6+
7+
8+
9+
select 1;
10+
11+
12+

‎example/.postgres_lsp.toml renamed to ‎root-file.txt

File renamed without changes.

0 commit comments

Comments
 (0)
Please sign in to comment.