partial cleanup of the macro pkg and move it from /core/router to the root because it may be used by the end-developers now to ammend the available macros per application

Former-commit-id: 951a5e7a401af25ecaa904ff6463b0def2c87afb

macro/interpreter/lexer/lexer.go

@@ -0,0 +1,202 @@
+package lexer
+
+import (
+	"github.com/kataras/iris/macro/interpreter/token"
+)
+
+// Lexer helps us to read/scan characters of a source and resolve their token types.
+type Lexer struct {
+	input   string
+	pos     int  // current pos in input, current char
+	readPos int  // current reading pos in input, after current char
+	ch      byte // current char under examination
+}
+
+// New takes a source, series of chars, and returns
+// a new, ready to read from the first letter, lexer.
+func New(src string) *Lexer {
+	l := &Lexer{
+		input: src,
+	}
+	// step to the first character in order to be ready
+	l.readChar()
+	return l
+}
+
+func (l *Lexer) readChar() {
+	if l.readPos >= len(l.input) {
+		l.ch = 0
+	} else {
+		l.ch = l.input[l.readPos]
+	}
+	l.pos = l.readPos
+	l.readPos++
+}
+
+const (
+	// Begin is the symbol which lexer should scan forward to.
+	Begin = '{' // token.LBRACE
+	// End is the symbol which lexer should stop scanning.
+	End = '}' // token.RBRACE
+)
+
+func resolveTokenType(ch byte) token.Type {
+	switch ch {
+	case Begin:
+		return token.LBRACE
+	case End:
+		return token.RBRACE
+	// Let's keep it simple, no evaluation for logical operators, we are not making a new programming language, keep it simple makis.
+	// ||
+	// case '|':
+	// 	if l.peekChar() == '|' {
+	// 		ch := l.ch
+	// 		l.readChar()
+	// 		t = token.Token{Type: token.OR, Literal: string(ch) + string(l.ch)}
+	// 	}
+	// ==
+	case ':':
+		return token.COLON
+	case '(':
+		return token.LPAREN
+	case ')':
+		return token.RPAREN
+	case ',':
+		return token.COMMA
+		// literals
+	case 0:
+		return token.EOF
+	default:
+		return token.IDENT //
+	}
+
+}
+
+// NextToken returns the next token in the series of characters.
+// It can be a single symbol, a token type or a literal.
+// It's able to return an EOF token too.
+//
+// It moves the cursor forward.
+func (l *Lexer) NextToken() (t token.Token) {
+	l.skipWhitespace()
+	typ := resolveTokenType(l.ch)
+	t.Type = typ
+	switch typ {
+	case token.EOF:
+		t.Literal = ""
+	case token.IDENT:
+		if isLetter(l.ch) {
+			// letters
+			lit := l.readIdentifier()
+			typ := token.LookupIdent(lit)
+			t = l.newToken(typ, lit)
+			return
+		}
+		if isDigit(l.ch) {
+			// numbers
+			lit := l.readNumber()
+			t = l.newToken(token.INT, lit)
+			return
+		}
+
+		t = l.newTokenRune(token.ILLEGAL, l.ch)
+	default:
+		t = l.newTokenRune(typ, l.ch)
+	}
+	l.readChar() // set the pos to the next
+	return
+}
+
+// NextDynamicToken doesn't cares about the grammar.
+// It reads numbers or any unknown symbol,
+// it's being used by parser to skip all characters
+// between parameter function's arguments inside parenthesis,
+// in order to allow custom regexp on the end-language too.
+//
+// It moves the cursor forward.
+func (l *Lexer) NextDynamicToken() (t token.Token) {
+	// calculate anything, even spaces.
+
+	// numbers
+	lit := l.readNumber()
+	if lit != "" {
+		return l.newToken(token.INT, lit)
+	}
+
+	lit = l.readIdentifierFuncArgument()
+	return l.newToken(token.IDENT, lit)
+}
+
+// used to skip any illegal token if inside parenthesis, used to be able to set custom regexp inside a func.
+func (l *Lexer) readIdentifierFuncArgument() string {
+	pos := l.pos
+	for resolveTokenType(l.ch) != token.RPAREN {
+		l.readChar()
+	}
+
+	return l.input[pos:l.pos]
+}
+
+// PeekNextTokenType returns only the token type
+// of the next character and it does not move forward the cursor.
+// It's being used by parser to recognise empty functions, i.e `even()`
+// as valid functions with zero input arguments.
+func (l *Lexer) PeekNextTokenType() token.Type {
+	if len(l.input)-1 > l.pos {
+		ch := l.input[l.pos]
+		return resolveTokenType(ch)
+	}
+	return resolveTokenType(0) // EOF
+}
+
+func (l *Lexer) newToken(tokenType token.Type, lit string) token.Token {
+	t := token.Token{
+		Type:    tokenType,
+		Literal: lit,
+		Start:   l.pos,
+		End:     l.pos,
+	}
+	// remember, l.pos is the last char
+	// and we want to include both start and end
+	// in order to be easy to the user to see by just marking the expression
+	if l.pos > 1 && len(lit) > 1 {
+		t.End = l.pos - 1
+		t.Start = t.End - len(lit) + 1
+	}
+
+	return t
+}
+
+func (l *Lexer) newTokenRune(tokenType token.Type, ch byte) token.Token {
+	return l.newToken(tokenType, string(ch))
+}
+
+func (l *Lexer) skipWhitespace() {
+	for l.ch == ' ' || l.ch == '\t' || l.ch == '\n' || l.ch == '\r' {
+		l.readChar()
+	}
+}
+
+func (l *Lexer) readIdentifier() string {
+	pos := l.pos
+	for isLetter(l.ch) || isDigit(l.ch) {
+		l.readChar()
+	}
+	return l.input[pos:l.pos]
+}
+
+func isLetter(ch byte) bool {
+	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_'
+}
+
+func (l *Lexer) readNumber() string {
+	pos := l.pos
+	for isDigit(l.ch) {
+		l.readChar()
+	}
+	return l.input[pos:l.pos]
+}
+
+func isDigit(ch byte) bool {
+	return '0' <= ch && ch <= '9'
+}