Skip to content

Commit e531665

Browse files
tjdevriestjdevries
committed
wip: still working on putting back all the pieces
1 parent 01c4737 commit e531665

22 files changed

Lines changed: 3336 additions & 3645 deletions

‎crates/vim9-lexer/src/lib.rs‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ pub struct Span {
1818
}
1919

2020
impl Span {
21-
fn empty() -> Self {
21+
pub fn empty() -> Self {
2222
Self {
2323
start_row: 0,
2424
start_col: 0,
@@ -130,8 +130,7 @@ impl Token<'_> {
130130
pub fn fake() -> Token<'static> {
131131
Token {
132132
kind: TokenKind::Virtual,
133-
// text: TokenText::Ref(""),
134-
text: todo!(),
133+
text: TokenText::Empty,
135134
span: Span::empty(),
136135
}
137136
}
@@ -311,6 +310,8 @@ impl TokenKind {
311310
pub struct LexerState {
312311
position: usize,
313312
read_position: usize,
313+
314+
sublexer: Option<Box<dyn Fn(&Lexer) -> Result<Token>>>,
314315
}
315316

316317
pub struct Lexer {
@@ -336,14 +337,6 @@ impl Debug for Lexer {
336337
}
337338
}
338339

339-
// impl Iterator for Lexer {
340-
// type Item = Token;
341-
//
342-
// fn next(&self) -> Option<Self::Item> {
343-
// Some(self.next_token().expect("todo: should this just expect?"))
344-
// }
345-
// }
346-
347340
impl Lexer {
348341
fn ch(&self) -> Option<&char> {
349342
self.chars.get(self.state.borrow().position)
@@ -364,6 +357,7 @@ impl Lexer {
364357
state: RefCell::new(LexerState {
365358
position: 0,
366359
read_position: 1,
360+
sublexer: None,
367361
}),
368362
chars,
369363
lines,
@@ -679,6 +673,10 @@ impl Lexer {
679673
}
680674

681675
pub fn next_token(&self) -> Result<Token> {
676+
if let Some(sublexer) = &self.state.borrow().sublexer {
677+
return sublexer(&self);
678+
}
679+
682680
use TokenKind::*;
683681

684682
self.skip_whitespace();
@@ -1076,11 +1074,11 @@ mod test {
10761074
snapshot!(test_lambda, "../testdata/snapshots/lambda.vim");
10771075
snapshot!(test_types, "../testdata/snapshots/types.vim");
10781076
snapshot!(test_methods, "../testdata/snapshots/methods.vim");
1079-
snapshot!(test_execute, "../testdata/snapshots/execute.vim");
10801077

10811078
// snapshot!(test_cfilter, "../testdata/snapshots/cfilter.vim");
10821079

10831080
// TODO: Check more thoroughly
10841081
snapshot!(test_matchparen, "../../shared/snapshots/matchparen.vim");
10851082
snapshot!(test_handlers, "../../shared/snapshots/lsp_handlers.vim");
1083+
snapshot!(test_selection, "../../shared/snapshots/lsp_selection.vim");
10861084
}

‎crates/vim9-parser/src/cmds/cmd_auto.rs‎

Lines changed: 26 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,39 @@
11
use anyhow::Result;
2-
use vim9_lexer::{Token, TokenKind};
2+
use vim9_lexer::{Span, Token, TokenKind};
33

4-
use crate::{Block, Body, ExCommand, Literal, Parser};
4+
use crate::{Block, Body, ExCommand, Literal, Parser, TokenMeta};
55

66
#[derive(Debug, PartialEq, Clone)]
77
pub struct AugroupCommand {
8-
augroup: Token,
8+
augroup: TokenMeta,
99
pub augroup_name: Literal,
10-
augroup_eol: Token,
10+
augroup_eol: TokenMeta,
1111
pub body: Body,
12-
augroup_end: Token,
13-
augroup_end_name: Token,
14-
augroup_end_eol: Token,
12+
augroup_end: TokenMeta,
13+
augroup_end_name: TokenMeta,
14+
augroup_end_eol: TokenMeta,
1515
}
1616

1717
impl AugroupCommand {
18-
pub fn parse(parser: &mut Parser) -> Result<ExCommand> {
18+
pub fn parse(parser: &Parser) -> Result<ExCommand> {
1919
Ok(ExCommand::Augroup(AugroupCommand {
20-
augroup: parser.expect_identifier_with_text("augroup")?,
20+
augroup: parser.expect_identifier_with_text("augroup")?.into(),
2121
augroup_name: parser
2222
.expect_token(TokenKind::Identifier)?
2323
.try_into()?,
2424
augroup_eol: parser.expect_eol()?,
2525
// TODO: This should be until augroup END, unless you can't have nested ones legally
2626
body: Body::parse_until(parser, "augroup")?,
27-
augroup_end: parser.expect_identifier_with_text("augroup")?,
28-
augroup_end_name: parser.expect_identifier_with_text("END")?,
27+
augroup_end: parser.expect_identifier_with_text("augroup")?.into(),
28+
augroup_end_name: parser.expect_identifier_with_text("END")?.into(),
2929
augroup_end_eol: parser.expect_eol()?,
3030
}))
3131
}
3232
}
3333

3434
#[derive(Debug, PartialEq, Clone)]
3535
pub struct AutocmdCommand {
36-
autocmd: Token,
36+
autocmd: TokenMeta,
3737
pub bang: bool,
3838
pub events: Vec<Literal>,
3939
pub pattern: AutocmdPattern,
@@ -46,17 +46,16 @@ pub enum AutocmdPattern {
4646
Buffer,
4747
}
4848

49-
fn tokens_are_neighbors(left: &Token, right: &Token) -> bool {
50-
left.span.end_row == right.span.start_row
51-
&& left.span.end_col == right.span.start_col
49+
fn tokens_are_neighbors(left: &Span, right: &Span) -> bool {
50+
left.end_row == right.start_row && left.end_col == right.start_col
5251
}
5352

5453
impl AutocmdCommand {
55-
pub fn parse(parser: &mut Parser) -> Result<ExCommand> {
54+
pub fn parse(parser: &Parser) -> Result<ExCommand> {
5655
Ok(ExCommand::Autocmd(AutocmdCommand {
5756
// TODO: Accept au! for example
58-
autocmd: parser.expect_identifier_with_text("autocmd")?,
59-
bang: if parser.current_token.kind == TokenKind::Bang {
57+
autocmd: parser.expect_identifier_with_text("autocmd")?.into(),
58+
bang: if parser.front_kind() == TokenKind::Bang {
6059
parser.next_token();
6160
true
6261
} else {
@@ -67,7 +66,7 @@ impl AutocmdCommand {
6766

6867
loop {
6968
events.push(parser.pop().try_into()?);
70-
if parser.current_token.kind != TokenKind::Comma {
69+
if parser.front_kind() != TokenKind::Comma {
7170
break;
7271
}
7372

@@ -89,16 +88,16 @@ pub enum AutocmdBlock {
8988
}
9089

9190
impl AutocmdBlock {
92-
pub fn parse(parser: &mut Parser) -> Result<AutocmdBlock> {
93-
Ok(match parser.current_token.kind {
91+
pub fn parse(parser: &Parser) -> Result<AutocmdBlock> {
92+
Ok(match parser.front_kind() {
9493
TokenKind::LeftBrace => AutocmdBlock::Block(Block::parse(parser)?),
9594
_ => AutocmdBlock::Command(parser.parse_command()?.into()),
9695
})
9796
}
9897
}
9998
impl AutocmdPattern {
100-
fn parse(parser: &mut Parser) -> Result<Self> {
101-
Ok(if parser.current_token.kind == TokenKind::AngleLeft {
99+
fn parse(parser: &Parser) -> Result<Self> {
100+
Ok(if parser.front_kind() == TokenKind::AngleLeft {
102101
parser.read_until(|t| {
103102
matches!(t.kind, TokenKind::AngleRight | TokenKind::GreaterThan)
104103
});
@@ -121,7 +120,10 @@ impl AutocmdPattern {
121120
// Append the text of the token.
122121
text += tok.text.as_str();
123122

124-
if !tokens_are_neighbors(&tok, &parser.current_token) {
123+
if !tokens_are_neighbors(
124+
&tok.span,
125+
&parser.front_ref().span,
126+
) {
125127
pattern.push(std::mem::take(&mut text));
126128
break pattern;
127129
}

‎crates/vim9-parser/src/cmds/cmd_try.rs‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,22 @@
11
use std::collections::HashSet;
22

33
use anyhow::Result;
4-
use vim9_lexer::Token;
54

6-
use crate::{Body, ExCommand, Expression, Parser};
5+
use crate::{Body, ExCommand, Expression, Parser, TokenMeta};
76

87
#[derive(Debug, PartialEq, Clone)]
98
pub struct TryCommand {
10-
try_: Token,
11-
try_eol: Token,
9+
try_: TokenMeta,
10+
try_eol: TokenMeta,
1211
pub body: Body,
1312
pub catch: Option<CatchCommand>,
1413
pub finally: Option<FinallyCommand>,
15-
endtry_: Token,
16-
endtry_eol: Token,
14+
endtry_: TokenMeta,
15+
endtry_eol: TokenMeta,
1716
}
1817

1918
impl TryCommand {
20-
pub fn parse(parser: &mut Parser) -> Result<ExCommand> {
19+
pub fn parse(parser: &Parser) -> Result<ExCommand> {
2120
let try_endings: HashSet<String> = HashSet::from_iter(
2221
vec![
2322
"catch".to_string(),
@@ -28,13 +27,13 @@ impl TryCommand {
2827
);
2928

3029
Ok(ExCommand::Try(TryCommand {
31-
try_: parser.expect_identifier_with_text("try")?,
30+
try_: parser.expect_identifier_with_text("try")?.into(),
3231
try_eol: parser.expect_eol()?,
3332
body: Body::parse_until_any(parser, &try_endings)?,
3433
catch: {
35-
if parser.current_token.text.as_str() == "catch" {
34+
if parser.front_text().eq("catch") {
3635
Some(CatchCommand {
37-
catch_: parser.pop(),
36+
catch_: parser.pop().into(),
3837
expr: None,
3938
catch_eol: parser.expect_eol()?,
4039
body: Body::parse_until_any(parser, &try_endings)?,
@@ -44,17 +43,17 @@ impl TryCommand {
4443
}
4544
},
4645
finally: None,
47-
endtry_: parser.expect_identifier_with_text("endtry")?,
46+
endtry_: parser.expect_identifier_with_text("endtry")?.into(),
4847
endtry_eol: parser.expect_eol()?,
4948
}))
5049
}
5150
}
5251

5352
#[derive(Debug, PartialEq, Clone)]
5453
pub struct CatchCommand {
55-
catch_: Token,
54+
catch_: TokenMeta,
5655
pub expr: Option<Expression>,
57-
catch_eol: Token,
56+
catch_eol: TokenMeta,
5857
pub body: Body,
5958
}
6059

‎crates/vim9-parser/src/cmds/cmd_user.rs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
use anyhow::Result;
2-
use vim9_lexer::{Token, TokenKind};
2+
use vim9_lexer::TokenKind;
33

4-
use crate::{ExCommand, Parser};
4+
use crate::{ExCommand, Parser, TokenMeta};
55

66
// TODO:
77
// - consider how script vars are supposed to work in this context
88

99
#[derive(Debug, PartialEq, Clone)]
1010
pub struct UserCommand {
11-
tok: Token,
11+
tok: TokenMeta,
1212
pub bang: bool,
1313
pub command_bang: bool,
1414
pub command_bar: bool,
@@ -24,16 +24,16 @@ pub struct UserCommand {
2424
}
2525

2626
impl UserCommand {
27-
pub fn parse(parser: &mut Parser) -> Result<ExCommand> {
28-
let tok = parser.expect_identifier_with_text("command")?;
27+
pub fn parse(parser: &Parser) -> Result<ExCommand> {
28+
let tok = parser.expect_identifier_with_text("command")?.into();
2929
let bang = parser.consume_if_kind(TokenKind::Bang).is_some();
3030

3131
let mut command_bang = false;
3232
let mut command_bar = false;
3333
let mut command_nargs = None;
3434
let mut command_complete = None;
3535
let mut command_range = None;
36-
while parser.current_token.kind == TokenKind::Minus {
36+
while parser.front_kind() == TokenKind::Minus {
3737
parser.next_token();
3838
parser.ensure_token(TokenKind::Identifier)?;
3939

0 commit comments

Comments
 (0)