fix: more lenient parsing for non-vim9script code

Author: tjdevries

crates/shared/snapshots/mapleader.vim

@@ -0,0 +1 @@
+let mapleader = "\\<Space>"

crates/vim9-gen/src/lib.rs

@@ -1883,6 +1883,9 @@ mod test {
     snapshot!(test_export, "../testdata/snapshots/export.vim");
     // snapshot!(test_matchparen, "../../shared/snapshots/matchparen.vim");
 
+    // Issues #41
+    snapshot!(test_mapleader, "../../shared/snapshots/mapleader.vim");
+
     #[test]
     fn test_simple_def() {
         let contents = r#"

crates/vim9-gen/testdata/output/vim9_gen__test__mapleader.snap

@@ -0,0 +1,18 @@
+---
+source: crates/vim9-gen/src/lib.rs
+expression: "generate(contents, ParserOpts { mode: ParserMode::Standalone }).unwrap().lua"
+---
+----------------------------------------
+-- This file is generated via github.com/tjdevries/vim9jit
+-- For any bugs, please first consider reporting there.
+----------------------------------------
+
+-- Ignore "value assigned to a local variable is unused" because
+--  we can't guarantee that local variables will be used by plugins
+-- luacheck: ignore 311
+
+local vim9 = require('_vim9script')
+local M = {}
+pcall(vim.cmd, [[ let mapleader =  "\<Space>" ]])
+return M
+

crates/vim9-parser/src/lib.rs

@@ -80,6 +80,16 @@ pub struct TokenOwned {
     pub span: Span,
 }
 
+impl TokenOwned {
+    pub fn literal(&self) -> String {
+        match self.kind {
+            TokenKind::SingleQuote => format!("'{}'", self.text),
+            TokenKind::DoubleQuoteString => format!("\"{}\"", self.text).replace("\\\\", "\\"),
+            _ => self.text.clone(),
+        }
+    }
+}
+
 impl Debug for TokenOwned {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(
@@ -418,11 +428,12 @@ impl SharedCommand {
             let tok = parser.pop();
 
             if prev_end > tok.span.start_col {
-                panic!("failed to make shared command: {parser:#?}");
+                // panic!("failed to make shared command: {parser:#?}");
+                continue;
             }
 
             contents += " ".repeat(tok.span.start_col - prev_end).as_str();
-            contents += tok.text.as_str();
+            contents += &tok.literal();
 
             prev_end = tok.span.end_col;
         }
@@ -1928,10 +1939,18 @@ impl PeekInfo {
     }
 }
 
+#[derive(Debug, PartialEq, Eq)]
+pub enum ParserMode {
+    PreVim9Script,
+    PostVim9Script,
+}
+
 #[derive(Debug)]
 pub struct Parser<'a> {
     lexer: &'a Lexer,
     token_buffer: RefCell<VecDeque<Token<'a>>>,
+
+    mode: RefCell<ParserMode>,
 }
 
 impl<'a> Parser<'a> {
@@ -1941,6 +1960,7 @@ impl<'a> Parser<'a> {
         tokens.push_back(lexer.next_token().unwrap());
 
         Self {
+            mode: RefCell::new(ParserMode::PreVim9Script),
             token_buffer: RefCell::new(tokens),
             lexer,
         }
@@ -2418,6 +2438,15 @@ impl<'a> Parser<'a> {
     pub fn parse_command(&self) -> Result<ExCommand> {
         use TokenKind::*;
 
+        if *self.mode.borrow() == ParserMode::PreVim9Script
+            && !self.command_match("vim9script")
+            && !self.command_match("def")
+        {
+            if self.command_match("let") {
+                return SharedCommand::parse(&self);
+            }
+        }
+
         // If the line starts with a colon, then just skip over it.
         if self.front_kind() == Colon {
             self.next_token();
@@ -2453,6 +2482,9 @@ impl<'a> Parser<'a> {
 
             Identifier => {
                 if self.command_match("vim9script") {
+                    // We've now seen vim9script, go ahead and update
+                    *self.mode.borrow_mut() = ParserMode::PostVim9Script;
+
                     ExCommand::Vim9Script(Vim9ScriptCommand::parse(self)?)
                 } else if self.command_match("execute") {
                     ExecuteCommand::parse(self)?
@@ -2810,6 +2842,9 @@ mod test {
     snap!(test_fileselect, "../../shared/snapshots/lsp_fileselect.vim");
     snap!(test_startup, "../../shared/snapshots/startup9.vim");
 
+    // Issues #41
+    snap!(test_mapleader, "../../shared/snapshots/mapleader.vim");
+
     #[test]
     fn test_peek_n() {
         let input = "vim9script\nvar x = true\n";

crates/vim9-parser/testdata/output/vim9_parser__test__mapleader.snap

@@ -0,0 +1,12 @@
+---
+source: crates/vim9-parser/src/lib.rs
+expression: snapshot_parsing(contents)
+---
+[
+    SharedCommand(
+        SharedCommand {
+            contents: "let mapleader =  \\\\<Space>",
+            eol: Token(EndOfLine, (0,27)->(0,27)),
+        },
+    ),
+]

scratch/vimcmd.lua

@@ -0,0 +1,5 @@
+vim.cmd([[
+  if 1
+    finish
+  endif
+]])