diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e3df679 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +target/* +target \ No newline at end of file diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 0000000..bef2d00 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,9 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "ab4fef131ee828e96ba67d31a7d690bd5f2f42040c6766b1b12fe856f87e0ff7" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..24f0f17 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,30 @@ +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true + + +[prune] + go-tests = true + unused-packages = true diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c7717ae --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +.PHONY: test build cover + +test: + echo "Not implemented" + +cover: + echo "Not implemented" + +build: + go build -o target/jave + +clean: + go clean + rm -rf target \ No newline at end of file diff --git a/Test.jave b/Test.jave new file mode 100644 index 0000000..e436828 --- /dev/null +++ b/Test.jave @@ -0,0 +1,19 @@ +/\ + Jave script to test lexing +\/ +/\ + Basic function + layout: + operation ()<> () <{ + + }> +\/ +//\\ inline comment +operation ()<> () <{ + allow newName 2b=2 name + << surname>>;; + give newName up;; +}> + +operation ()<<>>(<>) <{ + Test :< <>;; +}> \ No newline at end of file diff --git a/Test2.jave b/Test2.jave new file mode 100644 index 0000000..3ecfde4 --- /dev/null +++ b/Test2.jave @@ -0,0 +1,6 @@ +/\ + Jave script to test v0.1 lexing +\/ +//\\ Print something +insist <> +insist <\> as a string!>> \ No newline at end of file diff --git a/main.go b/main.go new file mode 100644 index 0000000..1161dea --- /dev/null +++ b/main.go @@ -0,0 +1,43 @@ +// Copyright © 2018 Chedder +// +// MIT License + +package main + +import ( + "fmt" + "log" + "os" + + "github.com/OrcaLLC/jave/src/lexer" +) + +func main() { + // Fetch arguments to get the filename + runArgs := os.Args[1:] + + // If no arguments exit quietly because we don't have interactive yet + if len(runArgs) < 1 { + log.Fatal("No interactive jave yet.") + } + + filePath := runArgs[0] + + // Jave can only run one file at a time, dingus + if len(runArgs) > 1 { + log.Printf("Additional arguments ignored. Using: [%s] but found %v\n", filePath, runArgs) + } + + // Load what we need then load either the file handler or input getter + thing, err := lexer.NewJaveFile(filePath) + if err != nil { + log.Fatalf("[JAVE FAIL]: %v\n", err) + } + //fmt.Printf("%v", thing) + //fmt.Print("\n") + //fmt.Print("\n") + + stuff := lexer.Lex(thing) + fmt.Printf("%v", stuff) + fmt.Print("\n") +} diff --git a/src/lexer/javefile.go b/src/lexer/javefile.go new file mode 100644 index 0000000..5e26fc4 --- /dev/null +++ b/src/lexer/javefile.go @@ -0,0 +1,136 @@ +package lexer + +import ( + "bytes" + "io/ioutil" + "path/filepath" + "strings" +) + +// JaveFile defines incoming jave source +type JaveFile struct { + Path string + Name string + Contents []rune + NewLines []int + Tokens []*Token +} + +// NewJaveFile ingests jave source file +func NewJaveFile(path string) (*JaveFile, error) { + // Get the filename + filename := filepath.Base(path) + + jf := &JaveFile{Name: filename, Path: path} + jf.NewLines = append(jf.NewLines, -1) + jf.NewLines = append(jf.NewLines, -1) + + contents, err := ioutil.ReadFile(jf.Path) + if err != nil { + return nil, err + } + + jf.Contents = []rune(string(contents)) + return jf, nil +} + +// GetLine gets a line of coke +func (s *JaveFile) GetLine(line int) string { + return string(s.Contents[s.NewLines[line]+1 : s.NewLines[line+1]]) +} + +// TabWidth idk +//const TabWidth = 4 + +// MarkPos because mark is a pos +func (s *JaveFile) MarkPos(pos Position) string { + buf := new(bytes.Buffer) + + lineString := s.GetLine(pos.Line) + lineStringRunes := []rune(lineString) + pad := pos.Char - 1 + + buf.WriteString(strings.Replace(strings.Replace(lineString, "%", "%%", -1), "\t", " ", -1)) + buf.WriteRune('\n') + for i := 0; i < pad; i++ { + spaces := 1 + + if lineStringRunes[i] == ' ' { + spaces = 1 + } + + for t := 0; t < spaces; t++ { + buf.WriteRune(' ') + } + } + buf.WriteString("^") + buf.WriteRune('\n') + + return buf.String() + +} + +// MarkSpan ohai mark +func (s *JaveFile) MarkSpan(span Span) string { + // if the span is just one character, use MarkPos instead + spanEnd := span.End() + spanEnd.Char-- + if span.Start() == spanEnd { + return s.MarkPos(span.Start()) + } + + // mark the span + buf := new(bytes.Buffer) + + for line := span.StartLine; line <= span.EndLine; line++ { + lineString := s.GetLine(line) + lineStringRunes := []rune(lineString) + + var pad int + if line == span.StartLine { + pad = span.StartChar - 1 + } else { + pad = 0 + } + + var length int + if line == span.EndLine { + length = span.EndChar - span.StartChar + } else { + length = len(lineStringRunes) + } + + buf.WriteString(strings.Replace(strings.Replace(lineString, "%", "%%", -1), "\t", " ", -1)) + buf.WriteRune('\n') + + for i := 0; i < pad; i++ { + spaces := 1 + + if lineStringRunes[i] == ' ' { + spaces = 1 + } + + for t := 0; t < spaces; t++ { + buf.WriteRune(' ') + } + } + + buf.WriteString("[DERP]") + for i := 0; i < length; i++ { + // there must be a less repetitive way to do this but oh well + spaces := 1 + + if lineStringRunes[i+pad] == ' ' { + spaces = 1 + } + + for t := 0; t < spaces; t++ { + buf.WriteRune('~') + } + } + //buf.WriteString(util.TEXT_RESET) + buf.WriteRune('\n') + } + + return buf.String() +} diff --git a/src/lexer/lexer.go b/src/lexer/lexer.go new file mode 100644 index 0000000..31ef5f8 --- /dev/null +++ b/src/lexer/lexer.go @@ -0,0 +1,320 @@ +package lexer + +import ( + "fmt" + "os" + "strings" + "unicode" + + "github.com/OrcaLLC/jave/src/util/timed" +) + +type lexer struct { + input *JaveFile + startPos, endPos int + curPos Position + tokStart Position +} + +// this is almost entirely stolen because Jave believes uniqueness +// is a flaw + +func (v *lexer) errPos(pos Position, err string, stuff ...interface{}) { + fmt.Printf("[sexer error]: [%s:%d:%d] %s", + pos.Filename, pos.Line, pos.Char, fmt.Sprintf(err, stuff...)) + + fmt.Printf("[lexer] %v", v.input.MarkPos(pos)) + + os.Exit(1) +} + +func (v *lexer) err(err string, stuff ...interface{}) { + v.errPos(v.curPos, err, stuff...) +} + +func (v *lexer) peek(ahead int) rune { + if ahead < 0 { + panic(fmt.Sprintf("[pervert] Tried to peek a negative number: %d", ahead)) + } + + if v.endPos+ahead >= len(v.input.Contents) { + return 0 + } + return v.input.Contents[v.endPos+ahead] +} + +func (v *lexer) consume() { + v.curPos.Char++ + if v.peek(0) == '\n' { + v.curPos.Char = 1 + v.curPos.Line++ + v.input.NewLines = append(v.input.NewLines, v.endPos) + } + + v.endPos++ +} + +func (v *lexer) expect(r rune) { + if v.peek(0) == r { + v.consume() + } else { + v.err("Expected `%c`, found `%c`", r, v.peek(0)) + } +} + +func (v *lexer) discardBuffer() { + v.startPos = v.endPos + + v.tokStart = v.curPos +} + +func (v *lexer) pushToken(t TokenType) { + tok := &Token{ + Type: t, + Contents: string(v.input.Contents[v.startPos:v.endPos]), + Where: NewSpan(v.tokStart, v.curPos), + } + + v.input.Tokens = append(v.input.Tokens, tok) + + fmt.Printf("lexer sexer [%4d:%4d:% 11s] `%s`\n", v.startPos, v.endPos, tok.Type, tok.Contents) + + v.discardBuffer() +} + +// Lex the sex up +func Lex(input *JaveFile) []*Token { + v := &lexer{ + input: input, + startPos: 0, + endPos: 0, + curPos: Position{Filename: input.Name, Line: 1, Char: 1}, + tokStart: Position{Filename: input.Name, Line: 1, Char: 1}, + } + + timed.Timed("lexing", input.Name, func() { + v.lex() + }) + //fmt.Println("Sexing up the code... ahem I mean lexing") + //v.lex() + + return v.input.Tokens +} + +func (v *lexer) lex() { + for { + v.skipLayoutAndComments() + + if isEOF(v.peek(0)) { + v.input.NewLines = append(v.input.NewLines, v.endPos) + return + } + + fmt.Printf("Peeking: %v\n", v.peek(0)) + + if isLetter(v.peek(0)) || v.peek(0) == '_' { + v.recognizeIdentifierToken() + } else if v.peek(0) == '<' && v.peek(1) == '<' { + v.recognizeStringToken() + } else if isSeparator(v.peek(0)) { + v.recognizeSeparatorToken() + } else { + v.err("I don't know this token.\n") + } + + // + // if isEOF(v.peek(0)) { + // v.input.NewLines = append(v.input.NewLines, v.endPos) + // return + // } + // + // if isDecimalDigit(v.peek(0)) { + // v.recognizeNumberToken() + // } else if isLetter(v.peek(0)) || v.peek(0) == '_' { + // v.recognizeIdentifierToken() + // } else if v.peek(0) == '"' { + // v.recognizeStringToken() + // } else if v.peek(0) == '\'' { + // v.recognizeCharacterToken() + // } else if isOperator(v.peek(0)) { + // v.recognizeOperatorToken() + // } else if isSeparator(v.peek(0)) { + // v.recognizeSeparatorToken() + // } else { + // v.err("Unrecognised token") + // } + //fmt.Printf("No sexing defined.") + } +} + +// returns true if a comment was skipped +func (v *lexer) skipComment() bool { + if v.skipBlockComment() { + return true + } else if v.skipLineComment() { + return true + } else { + return false + } +} + +func (v *lexer) skipBlockComment() bool { + pos := v.curPos + if v.peek(0) != '/' || v.peek(1) != '\\' { + return false + } + + v.consume() + v.consume() + isDoc := v.peek(0) == '\\' + + depth := 1 + for depth > 0 { + if isEOF(v.peek(0)) { + v.errPos(pos, "Unterminated block comment") + } + + if v.peek(0) == '/' && v.peek(1) == '\\' { + v.consume() + v.consume() + depth++ + } + + if v.peek(0) == '\\' && v.peek(1) == '/' { + v.consume() + v.consume() + depth-- + } + + v.consume() + } + + if isDoc { + v.pushToken(Doccomment) + } else { + v.discardBuffer() + } + fmt.Println("Recognized comment") + return true +} + +func (v *lexer) skipLineComment() bool { + if v.peek(0) != '/' || v.peek(1) != '/' || v.peek(2) != '\\' || v.peek(3) != '\\' { + return false + } + + v.consume() + v.consume() + isDoc := v.peek(0) == '/' + + for { + if isEOL(v.peek(0)) || isEOF(v.peek(0)) { + if isDoc { + v.pushToken(Doccomment) + } else { + v.discardBuffer() + } + v.consume() + fmt.Println("Recognized line comment") + return true + } + v.consume() + } +} + +func (v *lexer) skipLayoutAndComments() { + for { + for isLayout(v.peek(0)) { + v.consume() + } + v.discardBuffer() + + if !v.skipComment() { + break + } + + } + v.discardBuffer() +} + +// RECOGNIZE ME +func (v *lexer) recognizeIdentifierToken() { + v.consume() + + for isLetter(v.peek(0)) || isDecimalDigit(v.peek(0)) || v.peek(0) == '_' { + v.consume() + } + + v.pushToken(Identifier) +} + +func (v *lexer) recognizeStringToken() { + pos := v.curPos + + v.expect('<') + v.expect('<') + v.discardBuffer() + + for { + // escape to use > + if v.peek(0) == '\\' && v.peek(1) == '>' { + v.consume() + v.consume() + } else if v.peek(0) == '>' && v.peek(1) == '>' { + v.pushToken(String) + v.consume() + v.consume() + return + } else if isEOF(v.peek(0)) { + v.errPos(pos, "Unterminated string literal") + } else { + v.consume() + } + } +} + +func (v *lexer) recognizeSeparatorToken() { + v.consume() + v.pushToken(Separator) +} + +// is things +func isDecimalDigit(r rune) bool { + return r >= '0' && r <= '9' +} + +func isHexDigit(r rune) bool { + return isDecimalDigit(r) || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F') +} + +func isBinaryDigit(r rune) bool { + return r == '0' || r == '1' +} + +func isOctalDigit(r rune) bool { + return r >= '0' && r <= '7' +} + +func isLetter(r rune) bool { + return unicode.IsLetter(r) +} + +func isOperator(r rune) bool { + return strings.ContainsRune("+-*/=>