added parser support for multi-expression function bodies

Author: rsms

examples/program1.txt

@@ -11,6 +11,20 @@ X MUTABLE = F * A 4.5 2.3
 E = func (x Float, y Float) Float -> C 77.0 88.0
 F2 Float = E 9.1 X
 
+# Next up: Allow more than one expression in a function body
+G = func (x Int) Int ->
+	y = x * 9 \    # ... Do not stop at the next NewLine
+	+ 5            # expr: y = x * 9 + 5
+	y / 4 - 92     # return: expr: y / 4 - 92
+
+H = G 45
+
+
+# Todo: Control flow: if
+# Todo: Control flow: for
+# Todo: Control flow: while
+
+
 #E Float = D 3.1 # Should fail since D is not a callable
 
 ## A function definition

examples/program2-multi-line-func-body.txt

@@ -0,0 +1,12 @@
+# Next up: Allow more than one expression in a function body
+#A = func (x Int) Int -> 12
+G = func (x Int) Int ->
+  y = x * 9 \    # ... Do not stop at the next NewLine
+  + 5            # expr: y = x * 9 + 5
+  y / 4 - 92     # return: expr: y / 4 - 92
+
+H = G 45
+
+# Todo: Control flow: if
+# Todo: Control flow: for
+# Todo: Control flow: while

src/codegen/_VisitorImplHeader.h

@@ -20,8 +20,8 @@ using namespace llvm;
 // String formatting helper.
 //  std::string R_FMT(any << ..)
 //
-static std::ostringstream* _r_fmt_begin() { return new std::ostringstream; }
-static std::string _r_fmt_end(std::ostringstream* s) {
+static std::ostringstream* __attribute__((unused)) _r_fmt_begin() { return new std::ostringstream; }
+static std::string __attribute__((unused)) _r_fmt_end(std::ostringstream* s) {
   std::string str = s->str();
   delete s;
   return str;

src/main.cc

@@ -41,6 +41,7 @@ int main(int argc, char **argv) {
     return 1;
   }
   std::cout << "Parsed module: " << moduleFunc->body()->toString() << std::endl;
+  return 0; // xxx
 
   // Generate code
   codegen::Visitor codegenVisitor;

src/parse/Parser.h

@@ -3,14 +3,15 @@
 #define RSMS_PARSER_H
 
 #include "TokenBuffer.h"
+#include "../Logger.h"
 #include "../ast/Node.h"
 #include "../ast/Block.h"
 #include "../ast/Expression.h"
 #include "../ast/Function.h"
 
 #include <vector>
 
-#define DEBUG_PARSER 0
+#define DEBUG_PARSER 1
 #if DEBUG_PARSER
   #include "../DebugTrace.h"
   #define DEBUG_TRACE_PARSER DEBUG_TRACE
@@ -29,9 +30,11 @@ static int BinaryOperatorPrecedence(const Token& token) {
     return -1;
   switch (token.stringValue[0]) {
     case '*': return 40;
+    case '/': return 40;
     case '-': return 20;
     case '+': return 20;
     case '<': return 10;
+    case '>': return 10;
     default: return -1;
   }
 }
@@ -60,6 +63,7 @@ class Parser {
   uint32_t previousLineIndentation_ = 0;
   uint32_t currentLineIndentation_ = 0;
 
+  std::vector<Token> recentComments_;
   std::vector<std::string> errors_;
   //std::vector<std::string> warnings_;
   //std::vector<std::string> notices_;
@@ -429,12 +433,42 @@ class Parser {
       return (Function*)error("Expected '->' after function interface");
     }
     nextToken();  // eat '->'
+    
+    // Is this potentially a multi-expression function body?
+    uint32_t bodyStartedAtLineIndentation = UINT32_MAX;
+    if (token_.type == Token::NewLine) {
+      rlog("Yup, potentially a multi-expression func body ahead. line ind (prev/curr): "
+           << previousLineIndentation_ << '/' << currentLineIndentation_);
+      bodyStartedAtLineIndentation = currentLineIndentation_;
+    }
 
     // Parse function body
-    Expression *body = parseExpression();
-    if (body == 0) {
-      delete interface;
-      return 0;
+    Block* body = new Block;
+    
+    while (1) {
+      // Read one expression
+      Expression *expr = parseExpression();
+      if (expr == 0) {
+        delete interface;
+        return 0;
+      }
+      
+      // Add the expression to the function body
+      body->addNode(expr);
+      
+      // If we know this is a single-expression body, break after the first expression
+      if (bodyStartedAtLineIndentation == UINT32_MAX) {
+        rlog("Body ended (single-expression body)");
+        break;
+      } else {
+        
+        // Body ends when we either get a non-newline (e.g. a terminating) token, or the
+        // line indentation drops below the first line of the body
+        if (token_.type != Token::NewLine || currentLineIndentation_ < bodyStartedAtLineIndentation) {
+          rlog("Body ended (" << (token_.type != Token::NewLine ? "line indent drop" : "terminating token") << ")");
+          break;
+        }
+      }
     }
 
     return new Function(interface, body);
@@ -491,6 +525,14 @@ class Parser {
     Expression *lhs = parsePrimary();
     if (!lhs) return 0;
 
+    //// Backslash means "ignore the following sequence of linebreaks and comments
+    ////                  and treat whatever is after it as the same line"
+    //if (token_.type == Token::Backslash) {
+    //  // Eat <comment>*<LF><comment>* and continue
+    //  while (nextToken(/* newLineUpdatesLineIndentation = */false).type == Token::NewLine
+    //                                                     || token_.type == Token::Comment ) {}
+    //}
+    
     if (token_.type == Token::BinaryOperator) {
       // LHS binop RHS
       return parseBinOpRHS(0, lhs);
@@ -501,12 +543,11 @@ class Parser {
       if (!assignment) delete lhs;
       return assignment;
     } else if (token_.type == Token::Unexpected) {
-      error("Unexpected token when expecting a left-hand side expression");
+      error("Unexpected token when expecting a left-hand-side expression");
       nextToken(); // Skip token for error recovery.
       delete lhs;
       return 0;
     } else {
-      // LHS
       return lhs;
     }
   }
@@ -561,12 +602,23 @@ class Parser {
         nextToken();
         goto entry;
       }
-      default: return error("Unexpected token when expecting an expression");
+      default: return error("Unexpected token when expecting: id|intlit|floatlit|func|external|comment|newline");
     }
   }
 
   // ------------------------------------------------------------------------
 
+  inline const Token& _nextToken() {
+    if (token_.isNull()) {
+      token_ = const_cast<Token&>(tokens_.next());
+      futureToken_ = const_cast<Token&>(tokens_.next());
+    } else {
+      token_ = futureToken_;
+      futureToken_ = const_cast<Token&>(tokens_.next());
+    }
+    return token_;
+  }
+  
   // nextToken() -- Advances the token stream one token forward and returns
   //                the new token (a reference to the token_ instance variable).
   //
@@ -587,14 +639,37 @@ class Parser {
   //     The previous line number can be aquired from token_.line-1
   //
   const Token& nextToken() {
-    if (token_.isNull()) {
-      token_ = const_cast<Token&>(tokens_.next());
-      futureToken_ = const_cast<Token&>(tokens_.next());
+    _nextToken();
+    
+    recentComments_.clear();
+    
+    // Backslash means "ignore the following sequence of linebreaks and comments
+    //                  and treat whatever is after it as the same line"
+    if (token_.type == Token::Backslash) {
+      // Eat <comment>*<LF><comment>*
+      while (1) {
+        _nextToken();
+        if (token_.type == Token::Comment) {
+          recentComments_.push_back(token_);
+        } else if (token_.type != Token::NewLine) {
+          break;
+        }
+      }
     } else {
-      token_ = futureToken_;
-      futureToken_ = const_cast<Token&>(tokens_.next());
+      // Eat <comment>*
+      while (token_.type == Token::Comment) {
+        recentComments_.push_back(token_);
+        _nextToken();
+      }
     }
 
+    // if (recentComments_.size()) {
+    //   std::vector<Token>::const_iterator it = recentComments_.begin();
+    //   for (; it != recentComments_.end(); ++it) {
+    //     rlog("recentComments_[" << "] = " << (*it).toString());
+    //   }
+    // }
+    
     #if DEBUG_PARSER
     fprintf(stderr, "\e[34;1m>> %s\e[0m\n", token_.toString().c_str());
     #endif

src/parse/Token.h

@@ -32,6 +32,8 @@ class Token {
     Comma, // ','
     Stop, // '.'
     Assignment, // '='
+    Backslash, // '\'
+    
     NewLine,
 
     // Literals
@@ -129,6 +131,7 @@ const TokenTypeInfo Token::TypeInfo[] = {
   {"Comma",               0,0,0},
   {"Stop",                0,0,0},
   {"Assignment",          0,0,0},
+  {"Backslash",           0,0,0},
   {"NewLine",             0,0,0},
   {"IntLiteral",          .hasStringValue = 1,0, .hasIntValue = 1}, // intValue = radix
   {"FloatLiteral",        .hasStringValue = 1,0,0},

src/parse/Tokenizer.h

@@ -227,14 +227,14 @@ class Tokenizer {
       else {
 
         switch (input_->current()) {
-          case '<': {
+          case '<': { // '<-'?
             if (input_->futureCount() > 0 && input_->future(0) == '-') {
               token_.type = Token::LeftArrow;
               nextByte(); // consume '-'
               break;
             }
           }
-          case '-': {
+          case '-': { // '->'?
             if (input_->futureCount() > 0 && input_->future(0) == '>') {
               token_.type = Token::RightArrow;
               nextByte(); // consume '>'
@@ -246,6 +246,7 @@ class Tokenizer {
           case '*':
           case '/': token_.type = Token::BinaryOperator; break;
 
+          case '\\':token_.type = Token::Backslash; break;
           case '=': token_.type = Token::Assignment; break;
           case '(': token_.type = Token::LeftParen; break;
           case ')': token_.type = Token::RightParen; break;