From ed3c1b36a69e0c7818906f6a3548e4e180783082 Mon Sep 17 00:00:00 2001 From: Charles Corbett Date: Fri, 12 Oct 2018 19:59:45 -0700 Subject: [PATCH 1/6] Bootstrapping the lexer --- Makefile | 0 src/lexer/javesource.go | 10 ++++++++++ src/lexer/token.go | 11 +++++++++++ 3 files changed, 21 insertions(+) create mode 100644 Makefile create mode 100644 src/lexer/javesource.go create mode 100644 src/lexer/token.go diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e69de29 diff --git a/src/lexer/javesource.go b/src/lexer/javesource.go new file mode 100644 index 0000000..c076d46 --- /dev/null +++ b/src/lexer/javesource.go @@ -0,0 +1,10 @@ +package lexer + +// JaveSource defines the input jave file +type JaveSource struct { + Path string + Name string + Contents []rune + NewLines []int + Tokens []*Token +} diff --git a/src/lexer/token.go b/src/lexer/token.go new file mode 100644 index 0000000..406961b --- /dev/null +++ b/src/lexer/token.go @@ -0,0 +1,11 @@ +package lexer + +// TokenType maps token to type +type TokenType int + +// Token defines individual token +type Token struct { + Type TokenType + Contents string + //Where Span +} From 8b5595b01fdbfa6cf2b5bfc1ca25c8fa25652ce5 Mon Sep 17 00:00:00 2001 From: Charles Corbett Date: Fri, 12 Oct 2018 20:13:13 -0700 Subject: [PATCH 2/6] Resetting --- Makefile | 0 cmd/root.go | 84 +++++++++++++++++++++++++++++++++++++++++ main.go | 11 ++++++ src/lexer/javesource.go | 10 ----- src/lexer/token.go | 11 ------ 5 files changed, 95 insertions(+), 21 deletions(-) delete mode 100644 Makefile create mode 100644 cmd/root.go create mode 100644 main.go delete mode 100644 src/lexer/javesource.go delete mode 100644 src/lexer/token.go diff --git a/Makefile b/Makefile deleted file mode 100644 index e69de29..0000000 diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..1fb13bc --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,84 @@ +// Copyright © 2018 NAME HERE +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "os" + + homedir "github.com/mitchellh/go-homedir" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var cfgFile string + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "jave", + Short: "Jave is what jave does", + Long: `Jave is a programming language written in idiocy.`, + // Uncomment the following line if your bare application + // has an action associated with it: + // Run: func(cmd *cobra.Command, args []string) { }, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func init() { + cobra.OnInitialize(initConfig) + + // Here you will define your flags and configuration settings. + // Cobra supports persistent flags, which, if defined here, + // will be global for your application. + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.jave.yaml)") + + // Cobra also supports local flags, which will only run + // when this action is called directly. + rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} + +// initConfig reads in config file and ENV variables if set. +func initConfig() { + if cfgFile != "" { + // Use config file from the flag. + viper.SetConfigFile(cfgFile) + } else { + // Find home directory. + home, err := homedir.Dir() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + // Search config in home directory with name ".jave" (without extension). + viper.AddConfigPath(home) + viper.SetConfigName(".jave") + } + + viper.AutomaticEnv() // read in environment variables that match + + // If a config file is found, read it in. + if err := viper.ReadInConfig(); err == nil { + fmt.Println("Using config file:", viper.ConfigFileUsed()) + } +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..e374546 --- /dev/null +++ b/main.go @@ -0,0 +1,11 @@ +// Copyright © 2018 Chedder +// +// MIT License + +package main + +import "github.com/OrcaLLC/jave/cmd" + +func main() { + cmd.Execute() +} diff --git a/src/lexer/javesource.go b/src/lexer/javesource.go deleted file mode 100644 index c076d46..0000000 --- a/src/lexer/javesource.go +++ /dev/null @@ -1,10 +0,0 @@ -package lexer - -// JaveSource defines the input jave file -type JaveSource struct { - Path string - Name string - Contents []rune - NewLines []int - Tokens []*Token -} diff --git a/src/lexer/token.go b/src/lexer/token.go deleted file mode 100644 index 406961b..0000000 --- a/src/lexer/token.go +++ /dev/null @@ -1,11 +0,0 @@ -package lexer - -// TokenType maps token to type -type TokenType int - -// Token defines individual token -type Token struct { - Type TokenType - Contents string - //Where Span -} From 171d9e97cfedbcae5b43b082c143998629dd42e2 Mon Sep 17 00:00:00 2001 From: Charles Corbett Date: Fri, 12 Oct 2018 23:45:52 -0700 Subject: [PATCH 3/6] Again again --- cmd/root.go | 84 ------------------------------------------- main.go | 4 +-- src/lexer/javefile.go | 1 + src/lexer/lexer.go | 1 + 4 files changed, 3 insertions(+), 87 deletions(-) delete mode 100644 cmd/root.go create mode 100644 src/lexer/javefile.go create mode 100644 src/lexer/lexer.go diff --git a/cmd/root.go b/cmd/root.go deleted file mode 100644 index 1fb13bc..0000000 --- a/cmd/root.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright © 2018 NAME HERE -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "fmt" - "os" - - homedir "github.com/mitchellh/go-homedir" - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -var cfgFile string - -// rootCmd represents the base command when called without any subcommands -var rootCmd = &cobra.Command{ - Use: "jave", - Short: "Jave is what jave does", - Long: `Jave is a programming language written in idiocy.`, - // Uncomment the following line if your bare application - // has an action associated with it: - // Run: func(cmd *cobra.Command, args []string) { }, -} - -// Execute adds all child commands to the root command and sets flags appropriately. -// This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - if err := rootCmd.Execute(); err != nil { - fmt.Println(err) - os.Exit(1) - } -} - -func init() { - cobra.OnInitialize(initConfig) - - // Here you will define your flags and configuration settings. - // Cobra supports persistent flags, which, if defined here, - // will be global for your application. - rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.jave.yaml)") - - // Cobra also supports local flags, which will only run - // when this action is called directly. - rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") -} - -// initConfig reads in config file and ENV variables if set. -func initConfig() { - if cfgFile != "" { - // Use config file from the flag. - viper.SetConfigFile(cfgFile) - } else { - // Find home directory. - home, err := homedir.Dir() - if err != nil { - fmt.Println(err) - os.Exit(1) - } - - // Search config in home directory with name ".jave" (without extension). - viper.AddConfigPath(home) - viper.SetConfigName(".jave") - } - - viper.AutomaticEnv() // read in environment variables that match - - // If a config file is found, read it in. - if err := viper.ReadInConfig(); err == nil { - fmt.Println("Using config file:", viper.ConfigFileUsed()) - } -} diff --git a/main.go b/main.go index e374546..e0f00a6 100644 --- a/main.go +++ b/main.go @@ -4,8 +4,6 @@ package main -import "github.com/OrcaLLC/jave/cmd" - func main() { - cmd.Execute() + // Load what we need then load either the file handler or input getter } diff --git a/src/lexer/javefile.go b/src/lexer/javefile.go new file mode 100644 index 0000000..9beaa38 --- /dev/null +++ b/src/lexer/javefile.go @@ -0,0 +1 @@ +package lexer diff --git a/src/lexer/lexer.go b/src/lexer/lexer.go new file mode 100644 index 0000000..9beaa38 --- /dev/null +++ b/src/lexer/lexer.go @@ -0,0 +1 @@ +package lexer From 33c8d5c7af0f2cf036c39c67aa0a28dd0ec829a3 Mon Sep 17 00:00:00 2001 From: Charles Corbett Date: Fri, 12 Oct 2018 23:46:32 -0700 Subject: [PATCH 4/6] Dep init --- Gopkg.lock | 9 +++++++++ Gopkg.toml | 30 ++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 Gopkg.lock create mode 100644 Gopkg.toml diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 0000000..bef2d00 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,9 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "ab4fef131ee828e96ba67d31a7d690bd5f2f42040c6766b1b12fe856f87e0ff7" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..24f0f17 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,30 @@ +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true + + +[prune] + go-tests = true + unused-packages = true From 9f382e29159892c889f4513b24cba59624b87547 Mon Sep 17 00:00:00 2001 From: Charles Corbett Date: Sat, 13 Oct 2018 00:04:56 -0700 Subject: [PATCH 5/6] Loads files and packs them into a shit struct --- Test.jave | 10 ++++++++++ main.go | 22 ++++++++++++++++++++++ src/lexer/javefile.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 Test.jave diff --git a/Test.jave b/Test.jave new file mode 100644 index 0000000..6f5538b --- /dev/null +++ b/Test.jave @@ -0,0 +1,10 @@ +/\ THIS IS A COMMENT +\/ +operation ()<> () <{ + allow newName 2b=2 name + << surname>>;; + give newName up;; +}> + +operation ()<<>>(<>) <{ + Test :< <>;; +}> \ No newline at end of file diff --git a/main.go b/main.go index e0f00a6..adb83f4 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,28 @@ package main +import ( + "fmt" + "log" + "os" + + "github.com/OrcaLLC/jave/src/lexer" +) + func main() { + // Fetch arguments to get the filename + runArgs := os.Args[1:] + filePath := runArgs[0] + + // Jave can only run one file at a time, dingus + if len(runArgs) > 1 { + log.Printf("Additional arguments ignored. Using: [%s] but found %v\n", filePath, runArgs) + } + // Load what we need then load either the file handler or input getter + thing, err := lexer.NewJaveFile(filePath) + if err != nil { + log.Fatalf("[JAVE FAIL]: %v\n", err) + } + fmt.Printf("%v", thing) } diff --git a/src/lexer/javefile.go b/src/lexer/javefile.go index 9beaa38..06126a9 100644 --- a/src/lexer/javefile.go +++ b/src/lexer/javefile.go @@ -1 +1,31 @@ package lexer + +import ( + "io/ioutil" + "path/filepath" +) + +// JaveFile defines incoming jave source +type JaveFile struct { + Path string + Name string + Contents []rune + NewLines []int +} + +// NewJaveFile ingests jave source file +func NewJaveFile(path string) (*JaveFile, error) { + // Get the filename + filename := filepath.Base(path) + + jf := &JaveFile{Name: filename, Path: path} + jf.NewLines = append(jf.NewLines, -1) + + contents, err := ioutil.ReadFile(jf.Path) + if err != nil { + return nil, err + } + + jf.Contents = []rune(string(contents)) + return jf, nil +} From 1147ca6aeaff385860787704b6b4bd9804cc95ff Mon Sep 17 00:00:00 2001 From: Charles Corbett Date: Mon, 15 Oct 2018 04:54:43 -0700 Subject: [PATCH 6/6] I got it to lex strangs --- .gitignore | 2 + Makefile | 14 ++ Test.jave | 11 +- Test2.jave | 6 + main.go | 14 +- src/lexer/javefile.go | 105 +++++++++++++ src/lexer/lexer.go | 319 ++++++++++++++++++++++++++++++++++++++++ src/lexer/token.go | 73 +++++++++ src/util/timed/timed.go | 37 +++++ 9 files changed, 579 insertions(+), 2 deletions(-) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 Test2.jave create mode 100644 src/lexer/token.go create mode 100644 src/util/timed/timed.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e3df679 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +target/* +target \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c7717ae --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +.PHONY: test build cover + +test: + echo "Not implemented" + +cover: + echo "Not implemented" + +build: + go build -o target/jave + +clean: + go clean + rm -rf target \ No newline at end of file diff --git a/Test.jave b/Test.jave index 6f5538b..e436828 100644 --- a/Test.jave +++ b/Test.jave @@ -1,5 +1,14 @@ -/\ THIS IS A COMMENT +/\ + Jave script to test lexing \/ +/\ + Basic function + layout: + operation ()<> () <{ + + }> +\/ +//\\ inline comment operation ()<> () <{ allow newName 2b=2 name + << surname>>;; give newName up;; diff --git a/Test2.jave b/Test2.jave new file mode 100644 index 0000000..3ecfde4 --- /dev/null +++ b/Test2.jave @@ -0,0 +1,6 @@ +/\ + Jave script to test v0.1 lexing +\/ +//\\ Print something +insist <> +insist <\> as a string!>> \ No newline at end of file diff --git a/main.go b/main.go index adb83f4..1161dea 100644 --- a/main.go +++ b/main.go @@ -15,6 +15,12 @@ import ( func main() { // Fetch arguments to get the filename runArgs := os.Args[1:] + + // If no arguments exit quietly because we don't have interactive yet + if len(runArgs) < 1 { + log.Fatal("No interactive jave yet.") + } + filePath := runArgs[0] // Jave can only run one file at a time, dingus @@ -27,5 +33,11 @@ func main() { if err != nil { log.Fatalf("[JAVE FAIL]: %v\n", err) } - fmt.Printf("%v", thing) + //fmt.Printf("%v", thing) + //fmt.Print("\n") + //fmt.Print("\n") + + stuff := lexer.Lex(thing) + fmt.Printf("%v", stuff) + fmt.Print("\n") } diff --git a/src/lexer/javefile.go b/src/lexer/javefile.go index 06126a9..5e26fc4 100644 --- a/src/lexer/javefile.go +++ b/src/lexer/javefile.go @@ -1,8 +1,10 @@ package lexer import ( + "bytes" "io/ioutil" "path/filepath" + "strings" ) // JaveFile defines incoming jave source @@ -11,6 +13,7 @@ type JaveFile struct { Name string Contents []rune NewLines []int + Tokens []*Token } // NewJaveFile ingests jave source file @@ -20,6 +23,7 @@ func NewJaveFile(path string) (*JaveFile, error) { jf := &JaveFile{Name: filename, Path: path} jf.NewLines = append(jf.NewLines, -1) + jf.NewLines = append(jf.NewLines, -1) contents, err := ioutil.ReadFile(jf.Path) if err != nil { @@ -29,3 +33,104 @@ func NewJaveFile(path string) (*JaveFile, error) { jf.Contents = []rune(string(contents)) return jf, nil } + +// GetLine gets a line of coke +func (s *JaveFile) GetLine(line int) string { + return string(s.Contents[s.NewLines[line]+1 : s.NewLines[line+1]]) +} + +// TabWidth idk +//const TabWidth = 4 + +// MarkPos because mark is a pos +func (s *JaveFile) MarkPos(pos Position) string { + buf := new(bytes.Buffer) + + lineString := s.GetLine(pos.Line) + lineStringRunes := []rune(lineString) + pad := pos.Char - 1 + + buf.WriteString(strings.Replace(strings.Replace(lineString, "%", "%%", -1), "\t", " ", -1)) + buf.WriteRune('\n') + for i := 0; i < pad; i++ { + spaces := 1 + + if lineStringRunes[i] == ' ' { + spaces = 1 + } + + for t := 0; t < spaces; t++ { + buf.WriteRune(' ') + } + } + buf.WriteString("^") + buf.WriteRune('\n') + + return buf.String() + +} + +// MarkSpan ohai mark +func (s *JaveFile) MarkSpan(span Span) string { + // if the span is just one character, use MarkPos instead + spanEnd := span.End() + spanEnd.Char-- + if span.Start() == spanEnd { + return s.MarkPos(span.Start()) + } + + // mark the span + buf := new(bytes.Buffer) + + for line := span.StartLine; line <= span.EndLine; line++ { + lineString := s.GetLine(line) + lineStringRunes := []rune(lineString) + + var pad int + if line == span.StartLine { + pad = span.StartChar - 1 + } else { + pad = 0 + } + + var length int + if line == span.EndLine { + length = span.EndChar - span.StartChar + } else { + length = len(lineStringRunes) + } + + buf.WriteString(strings.Replace(strings.Replace(lineString, "%", "%%", -1), "\t", " ", -1)) + buf.WriteRune('\n') + + for i := 0; i < pad; i++ { + spaces := 1 + + if lineStringRunes[i] == ' ' { + spaces = 1 + } + + for t := 0; t < spaces; t++ { + buf.WriteRune(' ') + } + } + + buf.WriteString("[DERP]") + for i := 0; i < length; i++ { + // there must be a less repetitive way to do this but oh well + spaces := 1 + + if lineStringRunes[i+pad] == ' ' { + spaces = 1 + } + + for t := 0; t < spaces; t++ { + buf.WriteRune('~') + } + } + //buf.WriteString(util.TEXT_RESET) + buf.WriteRune('\n') + } + + return buf.String() +} diff --git a/src/lexer/lexer.go b/src/lexer/lexer.go index 9beaa38..31ef5f8 100644 --- a/src/lexer/lexer.go +++ b/src/lexer/lexer.go @@ -1 +1,320 @@ package lexer + +import ( + "fmt" + "os" + "strings" + "unicode" + + "github.com/OrcaLLC/jave/src/util/timed" +) + +type lexer struct { + input *JaveFile + startPos, endPos int + curPos Position + tokStart Position +} + +// this is almost entirely stolen because Jave believes uniqueness +// is a flaw + +func (v *lexer) errPos(pos Position, err string, stuff ...interface{}) { + fmt.Printf("[sexer error]: [%s:%d:%d] %s", + pos.Filename, pos.Line, pos.Char, fmt.Sprintf(err, stuff...)) + + fmt.Printf("[lexer] %v", v.input.MarkPos(pos)) + + os.Exit(1) +} + +func (v *lexer) err(err string, stuff ...interface{}) { + v.errPos(v.curPos, err, stuff...) +} + +func (v *lexer) peek(ahead int) rune { + if ahead < 0 { + panic(fmt.Sprintf("[pervert] Tried to peek a negative number: %d", ahead)) + } + + if v.endPos+ahead >= len(v.input.Contents) { + return 0 + } + return v.input.Contents[v.endPos+ahead] +} + +func (v *lexer) consume() { + v.curPos.Char++ + if v.peek(0) == '\n' { + v.curPos.Char = 1 + v.curPos.Line++ + v.input.NewLines = append(v.input.NewLines, v.endPos) + } + + v.endPos++ +} + +func (v *lexer) expect(r rune) { + if v.peek(0) == r { + v.consume() + } else { + v.err("Expected `%c`, found `%c`", r, v.peek(0)) + } +} + +func (v *lexer) discardBuffer() { + v.startPos = v.endPos + + v.tokStart = v.curPos +} + +func (v *lexer) pushToken(t TokenType) { + tok := &Token{ + Type: t, + Contents: string(v.input.Contents[v.startPos:v.endPos]), + Where: NewSpan(v.tokStart, v.curPos), + } + + v.input.Tokens = append(v.input.Tokens, tok) + + fmt.Printf("lexer sexer [%4d:%4d:% 11s] `%s`\n", v.startPos, v.endPos, tok.Type, tok.Contents) + + v.discardBuffer() +} + +// Lex the sex up +func Lex(input *JaveFile) []*Token { + v := &lexer{ + input: input, + startPos: 0, + endPos: 0, + curPos: Position{Filename: input.Name, Line: 1, Char: 1}, + tokStart: Position{Filename: input.Name, Line: 1, Char: 1}, + } + + timed.Timed("lexing", input.Name, func() { + v.lex() + }) + //fmt.Println("Sexing up the code... ahem I mean lexing") + //v.lex() + + return v.input.Tokens +} + +func (v *lexer) lex() { + for { + v.skipLayoutAndComments() + + if isEOF(v.peek(0)) { + v.input.NewLines = append(v.input.NewLines, v.endPos) + return + } + + fmt.Printf("Peeking: %v\n", v.peek(0)) + + if isLetter(v.peek(0)) || v.peek(0) == '_' { + v.recognizeIdentifierToken() + } else if v.peek(0) == '<' && v.peek(1) == '<' { + v.recognizeStringToken() + } else if isSeparator(v.peek(0)) { + v.recognizeSeparatorToken() + } else { + v.err("I don't know this token.\n") + } + + // + // if isEOF(v.peek(0)) { + // v.input.NewLines = append(v.input.NewLines, v.endPos) + // return + // } + // + // if isDecimalDigit(v.peek(0)) { + // v.recognizeNumberToken() + // } else if isLetter(v.peek(0)) || v.peek(0) == '_' { + // v.recognizeIdentifierToken() + // } else if v.peek(0) == '"' { + // v.recognizeStringToken() + // } else if v.peek(0) == '\'' { + // v.recognizeCharacterToken() + // } else if isOperator(v.peek(0)) { + // v.recognizeOperatorToken() + // } else if isSeparator(v.peek(0)) { + // v.recognizeSeparatorToken() + // } else { + // v.err("Unrecognised token") + // } + //fmt.Printf("No sexing defined.") + } +} + +// returns true if a comment was skipped +func (v *lexer) skipComment() bool { + if v.skipBlockComment() { + return true + } else if v.skipLineComment() { + return true + } else { + return false + } +} + +func (v *lexer) skipBlockComment() bool { + pos := v.curPos + if v.peek(0) != '/' || v.peek(1) != '\\' { + return false + } + + v.consume() + v.consume() + isDoc := v.peek(0) == '\\' + + depth := 1 + for depth > 0 { + if isEOF(v.peek(0)) { + v.errPos(pos, "Unterminated block comment") + } + + if v.peek(0) == '/' && v.peek(1) == '\\' { + v.consume() + v.consume() + depth++ + } + + if v.peek(0) == '\\' && v.peek(1) == '/' { + v.consume() + v.consume() + depth-- + } + + v.consume() + } + + if isDoc { + v.pushToken(Doccomment) + } else { + v.discardBuffer() + } + fmt.Println("Recognized comment") + return true +} + +func (v *lexer) skipLineComment() bool { + if v.peek(0) != '/' || v.peek(1) != '/' || v.peek(2) != '\\' || v.peek(3) != '\\' { + return false + } + + v.consume() + v.consume() + isDoc := v.peek(0) == '/' + + for { + if isEOL(v.peek(0)) || isEOF(v.peek(0)) { + if isDoc { + v.pushToken(Doccomment) + } else { + v.discardBuffer() + } + v.consume() + fmt.Println("Recognized line comment") + return true + } + v.consume() + } +} + +func (v *lexer) skipLayoutAndComments() { + for { + for isLayout(v.peek(0)) { + v.consume() + } + v.discardBuffer() + + if !v.skipComment() { + break + } + + } + v.discardBuffer() +} + +// RECOGNIZE ME +func (v *lexer) recognizeIdentifierToken() { + v.consume() + + for isLetter(v.peek(0)) || isDecimalDigit(v.peek(0)) || v.peek(0) == '_' { + v.consume() + } + + v.pushToken(Identifier) +} + +func (v *lexer) recognizeStringToken() { + pos := v.curPos + + v.expect('<') + v.expect('<') + v.discardBuffer() + + for { + // escape to use > + if v.peek(0) == '\\' && v.peek(1) == '>' { + v.consume() + v.consume() + } else if v.peek(0) == '>' && v.peek(1) == '>' { + v.pushToken(String) + v.consume() + v.consume() + return + } else if isEOF(v.peek(0)) { + v.errPos(pos, "Unterminated string literal") + } else { + v.consume() + } + } +} + +func (v *lexer) recognizeSeparatorToken() { + v.consume() + v.pushToken(Separator) +} + +// is things +func isDecimalDigit(r rune) bool { + return r >= '0' && r <= '9' +} + +func isHexDigit(r rune) bool { + return isDecimalDigit(r) || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F') +} + +func isBinaryDigit(r rune) bool { + return r == '0' || r == '1' +} + +func isOctalDigit(r rune) bool { + return r >= '0' && r <= '7' +} + +func isLetter(r rune) bool { + return unicode.IsLetter(r) +} + +func isOperator(r rune) bool { + return strings.ContainsRune("+-*/=>