📦 Extract lexer into standalone ndc_lexer crate

timfennis · timfennis · commit f6a90495a58a · 2026-02-27T17:27:55.000+01:00
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [workspace]
 resolver = "3"
-members = ["ndc_macros", "ndc_bin", "ndc_lib", "ndc_lsp", "benches", "tests"]
+members = ["ndc_macros", "ndc_bin", "ndc_lib", "ndc_lsp", "ndc_lexer", "benches", "tests"]
 
 [workspace.package]
 edition = "2024"
@@ -17,6 +17,7 @@ derive_builder = { version = "0.20.2" }
 derive_more = { version = "2.1.1", features = ["deref", "deref_mut", "from", "display", "constructor"] }
 factorial = "0.4.0"
 itertools = "0.14.0"
+ndc_lexer = { path = "ndc_lexer" }
 ndc_lib = { path = "ndc_lib" }
 ndc_lsp = { path = "ndc_lsp" }
 ndc_macros = { path = "ndc_macros" }
diff --git a/ndc_bin/Cargo.toml b/ndc_bin/Cargo.toml
@@ -14,6 +14,7 @@ clap.workspace = true
 itertools.workspace = true
 strsim.workspace = true
 miette = { version = "7.6.0", features = ["fancy"] }
+ndc_lexer.workspace = true
 ndc_lib.workspace = true
 ndc_lsp.workspace = true
 owo-colors.workspace = true
diff --git a/ndc_bin/src/diagnostic.rs b/ndc_bin/src/diagnostic.rs
@@ -1,6 +1,6 @@
 use miette::{Diagnostic, LabeledSpan, SourceSpan};
 use ndc_lib::interpreter::InterpreterError;
-use ndc_lib::lexer::Span;
+use ndc_lexer::Span;
 use std::fmt;
 
 fn span_to_source_span(span: Span) -> SourceSpan {
diff --git a/ndc_bin/src/highlighter.rs b/ndc_bin/src/highlighter.rs
@@ -3,7 +3,7 @@ use miette::{
     SpanContents,
     highlighters::{Highlighter, HighlighterState},
 };
-use ndc_lib::lexer::{Lexer, Token};
+use ndc_lexer::{Lexer, Token};
 use owo_colors::{Rgb, Style, Styled};
 
 #[derive(Default)]
diff --git a/ndc_lexer/Cargo.toml b/ndc_lexer/Cargo.toml
@@ -0,0 +1,9 @@
+[package]
+name = "ndc_lexer"
+edition.workspace = true
+version.workspace = true
+
+[dependencies]
+num.workspace = true
+ryu.workspace = true
+thiserror.workspace = true
diff --git a/ndc_lexer/src/lib.rs b/ndc_lexer/src/lib.rs
diff --git a/ndc_lexer/src/number.rs b/ndc_lexer/src/number.rs
diff --git a/ndc_lexer/src/span.rs b/ndc_lexer/src/span.rs
diff --git a/ndc_lexer/src/string.rs b/ndc_lexer/src/string.rs
diff --git a/ndc_lexer/src/token.rs b/ndc_lexer/src/token.rs
@@ -331,7 +331,7 @@ impl From<String> for Token {
 
 #[cfg(test)]
 mod test {
-    use crate::lexer::Span;
+    use crate::Span;
 
     #[test]
     fn test_merge() {
diff --git a/ndc_lib/Cargo.toml b/ndc_lib/Cargo.toml
@@ -11,6 +11,7 @@ derive_more.workspace = true
 derive_builder.workspace = true
 factorial.workspace = true
 itertools.workspace = true
+ndc_lexer.workspace = true
 ndc_macros.workspace = true
 num.workspace = true
 once_cell.workspace = true
diff --git a/ndc_lib/src/ast/expression.rs b/ndc_lib/src/ast/expression.rs
@@ -2,7 +2,7 @@ use crate::ast::operator::LogicalOperator;
 use crate::ast::parser::Error as ParseError;
 use crate::interpreter::evaluate::EvaluationError;
 use crate::interpreter::function::StaticType;
-use crate::lexer::Span;
+use ndc_lexer::Span;
 use num::BigInt;
 use num::complex::Complex64;
 
diff --git a/ndc_lib/src/ast/operator.rs b/ndc_lib/src/ast/operator.rs
@@ -1,5 +1,5 @@
 use crate::ast::Error as ParseError;
-use crate::lexer::{Token, TokenLocation};
+use ndc_lexer::{Token, TokenLocation};
 use std::fmt;
 use std::fmt::Formatter;
 
diff --git a/ndc_lib/src/ast/parser.rs b/ndc_lib/src/ast/parser.rs
@@ -3,7 +3,7 @@ use std::fmt::Write;
 use crate::ast::Expression;
 use crate::ast::expression::{Binding, ExpressionLocation, ForBody, ForIteration, Lvalue};
 use crate::ast::operator::{BinaryOperator, LogicalOperator, UnaryOperator};
-use crate::lexer::{Span, Token, TokenLocation};
+use ndc_lexer::{Span, Token, TokenLocation};
 
 pub struct Parser {
     tokens: Vec<TokenLocation>,
diff --git a/ndc_lib/src/interpreter/evaluate/index.rs b/ndc_lib/src/interpreter/evaluate/index.rs
@@ -14,8 +14,8 @@ use crate::interpreter::environment::Environment;
 use crate::{
     ast::{Expression, ExpressionLocation},
     interpreter::{function::FunctionCarrier, sequence::Sequence, value::Value},
-    lexer::Span,
 };
+use ndc_lexer::Span;
 use itertools::Itertools;
 use std::cell::RefCell;
 use std::cmp::min;
diff --git a/ndc_lib/src/interpreter/evaluate/mod.rs b/ndc_lib/src/interpreter/evaluate/mod.rs
@@ -9,7 +9,7 @@ use crate::interpreter::iterator::mut_value_to_iterator;
 use crate::interpreter::num::Number;
 use crate::interpreter::sequence::Sequence;
 use crate::interpreter::value::Value;
-use crate::lexer::Span;
+use ndc_lexer::Span;
 use index::{Offset, evaluate_as_index, get_at_index, set_at_index};
 use itertools::Itertools;
 use std::cell::RefCell;
diff --git a/ndc_lib/src/interpreter/function.rs b/ndc_lib/src/interpreter/function.rs
@@ -7,7 +7,7 @@ use crate::interpreter::evaluate::{
 use crate::interpreter::num::{BinaryOperatorError, Number};
 use crate::interpreter::sequence::Sequence;
 use crate::interpreter::value::Value;
-use crate::lexer::Span;
+use ndc_lexer::Span;
 use derive_builder::Builder;
 use itertools::Itertools;
 use std::cell::{BorrowError, BorrowMutError, RefCell};
diff --git a/ndc_lib/src/interpreter/mod.rs b/ndc_lib/src/interpreter/mod.rs
@@ -7,7 +7,7 @@ use crate::interpreter::evaluate::{EvaluationError, evaluate_expression};
 use crate::interpreter::function::FunctionCarrier;
 use crate::interpreter::semantic::analyser::{Analyser, ScopeTree};
 use crate::interpreter::value::Value;
-use crate::lexer::{Lexer, TokenLocation};
+use ndc_lexer::{Lexer, TokenLocation};
 pub mod environment;
 pub mod evaluate;
 pub mod function;
@@ -118,7 +118,7 @@ pub enum InterpreterError {
     #[error("Error while lexing source")]
     Lexer {
         #[from]
-        cause: crate::lexer::Error,
+        cause: ndc_lexer::Error,
     },
     #[error("Error while parsing source")]
     Parser {
diff --git a/ndc_lib/src/interpreter/num.rs b/ndc_lib/src/interpreter/num.rs
@@ -8,7 +8,7 @@ use crate::ast::BinaryOperator;
 use crate::interpreter::evaluate::EvaluationError;
 use crate::interpreter::function::StaticType;
 use crate::interpreter::int::Int;
-use crate::lexer::Span;
+use ndc_lexer::Span;
 use num::bigint::TryFromBigIntError;
 use num::complex::{Complex64, ComplexFloat};
 use num::{BigInt, BigRational, Complex, FromPrimitive, Signed, ToPrimitive, Zero};
diff --git a/ndc_lib/src/interpreter/semantic/analyser.rs b/ndc_lib/src/interpreter/semantic/analyser.rs
@@ -2,7 +2,7 @@ use crate::ast::{
     Binding, Expression, ExpressionLocation, ForBody, ForIteration, Lvalue, ResolvedVar,
 };
 use crate::interpreter::function::StaticType;
-use crate::lexer::Span;
+use ndc_lexer::Span;
 use itertools::Itertools;
 use std::fmt::{Debug, Formatter};
 
diff --git a/ndc_lib/src/lib.rs b/ndc_lib/src/lib.rs
@@ -2,5 +2,4 @@ pub mod ast;
 mod compare;
 mod hash_map;
 pub mod interpreter;
-pub mod lexer;
 pub mod stdlib;
diff --git a/ndc_lsp/Cargo.toml b/ndc_lsp/Cargo.toml
@@ -6,6 +6,7 @@ version.workspace = true
 
 [dependencies]
 tokio = { version = "1.49.0", features = ["full"] }
+ndc_lexer.workspace = true
 ndc_lib.workspace = true
 tower-lsp.workspace = true
 
diff --git a/ndc_lsp/src/backend.rs b/ndc_lsp/src/backend.rs
@@ -2,7 +2,7 @@ use std::collections::HashMap;
 
 use ndc_lib::ast::{Expression, ExpressionLocation, ForBody, ForIteration, Lvalue};
 use ndc_lib::interpreter::Interpreter;
-use ndc_lib::lexer::{Lexer, Span, TokenLocation};
+use ndc_lexer::{Lexer, Span, TokenLocation};
 use tokio::sync::Mutex;
 use tower_lsp::jsonrpc::Result as JsonRPCResult;
 use tower_lsp::lsp_types::{
@@ -31,7 +31,7 @@ impl Backend {
     async fn validate(&self, uri: &Url, text: &str) {
         let scanner = Lexer::new(text);
         let tokens = scanner
-            .collect::<Result<Vec<TokenLocation>, ndc_lib::lexer::Error>>()
+            .collect::<Result<Vec<TokenLocation>, ndc_lexer::Error>>()
             .map_err(|err| {
                 let span: Span = err.location();
 
