Made token values pointers in lexer/parser

parse/error.go

@@ -47,7 +47,7 @@ func (e *parseError) sprintf(format string, a ...interface{}) string {
 // is not of the expected type.
 type UnexpectedTokenError struct {
 	baseError
-	actual   token
+	actual   *token
 	expected []tokenType
 }
 
@@ -71,7 +71,7 @@ func (e *UnexpectedTokenError) Error() string {
 }
 
 // newUnexpectedTokenError returns a new UnexpectedTokenError
-func newUnexpectedTokenError(actual token, expected ...tokenType) error {
+func newUnexpectedTokenError(actual *token, expected ...tokenType) error {
 	return &UnexpectedTokenError{newBaseError(actual.Pos), actual, expected}
 }
 
@@ -100,14 +100,14 @@ func (e *UnexpectedEOFError) Error() string {
 }
 
 // newUnexpectedEOFError returns a new UnexpectedEOFError
-func newUnexpectedEOFError(tok token) error {
+func newUnexpectedEOFError(tok *token) error {
 	return &UnexpectedEOFError{newBaseError(tok.Pos)}
 }
 
 // UnexpectedValueError describes an invalid or unexpected value inside a token.
 type UnexpectedValueError struct {
 	baseError
-	tok token  // The actual token.
+	tok *token // The actual token.
 	val string // The expected value.
 }
 
@@ -116,7 +116,7 @@ func (e *UnexpectedValueError) Error() string {
 }
 
 // newUnexpectedValueError returns a new UnexpectedPunctuationError
-func newUnexpectedValueError(tok token, expected string) error {
+func newUnexpectedValueError(tok *token, expected string) error {
 	return &UnexpectedValueError{newBaseError(tok.Pos), tok, expected}
 }
 

parse/lex.go

@@ -112,15 +112,15 @@ type lexer struct {
 	line   int // The current line number
 	offset int // The current character offset on the current line
 	input  string
-	tokens chan token
+	tokens chan *token
 	state  stateFn
 	mode   mode
-	last   token // The last emitted token
-	parens int   // Number of open parenthesis
+	last   *token // The last emitted token
+	parens int    // Number of open parenthesis
 }
 
 // nextToken returns the next token emitted by the lexer.
-func (l *lexer) nextToken() token {
+func (l *lexer) nextToken() *token {
 	for v, ok := <-l.tokens; ok; {
 		l.last = v
 		return v
@@ -140,7 +140,7 @@ func (l *lexer) tokenize() {
 func newLexer(input io.Reader) *lexer {
 	// TODO: lexer should use the reader.
 	i, _ := ioutil.ReadAll(input)
-	return &lexer{0, 0, 1, 0, string(i), make(chan token), nil, modeNormal, token{}, 0}
+	return &lexer{0, 0, 1, 0, string(i), make(chan *token), nil, modeNormal, &token{}, 0}
 }
 
 func (l *lexer) next() (val string) {
@@ -184,7 +184,7 @@ func (l *lexer) emit(t tokenType) {
 		l.offset += len(val)
 	}
 
-	l.tokens <- tok
+	l.tokens <- &tok
 	l.start = l.pos
 	if tok.tokenType == tokenEOF {
 		close(l.tokens)
@@ -194,7 +194,7 @@ func (l *lexer) emit(t tokenType) {
 
 func (l *lexer) errorf(format string, args ...interface{}) stateFn {
 	tok := token{fmt.Sprintf(format, args...), tokenError, Pos{l.line, l.offset}}
-	l.tokens <- tok
+	l.tokens <- &tok
 
 	return nil
 }

parse/parse.go

@@ -21,8 +21,8 @@ type Tree struct {
 	blocks []map[string]*BlockNode // Contains each block available to this template.
 	macros map[string]*MacroNode   // All macros defined on this template.
 
-	unread []token // Any tokens received by the lexer but not yet read.
-	read   []token // Tokens that have already been read.
+	unread []*token // Any tokens received by the lexer but not yet read.
+	read   []*token // Tokens that have already been read.
 
 	Name string // A name identifying this tree; the template name.
 
@@ -43,8 +43,8 @@ func NewNamedTree(name string, input io.Reader) *Tree {
 		blocks: []map[string]*BlockNode{make(map[string]*BlockNode)},
 		macros: make(map[string]*MacroNode),
 
-		unread: make([]token, 0),
-		read:   make([]token, 0),
+		unread: make([]*token, 0),
+		read:   make([]*token, 0),
 
 		Name:     name,
 		Visitors: make([]NodeVisitor, 0),
@@ -88,7 +88,7 @@ func (t *Tree) enrichError(err error) error {
 }
 
 // peek returns the next unread token without advancing the internal cursor.
-func (t *Tree) peek() token {
+func (t *Tree) peek() *token {
 	tok := t.next()
 	t.backup()
 
@@ -96,8 +96,8 @@ func (t *Tree) peek() token {
 }
 
 // peek returns the next unread, non-space token without advancing the internal cursor.
-func (t *Tree) peekNonSpace() token {
-	var next token
+func (t *Tree) peekNonSpace() *token {
+	var next *token
 	for {
 		next = t.next()
 		if next.tokenType != tokenWhitespace {
@@ -109,7 +109,7 @@ func (t *Tree) peekNonSpace() token {
 
 // backup pushes the last read token back onto the unread stack and reduces the internal cursor by one.
 func (t *Tree) backup() {
-	var tok token
+	var tok *token
 	tok, t.read = t.read[len(t.read)-1], t.read[:len(t.read)-1]
 	t.unread = append(t.unread, tok)
 }
@@ -126,8 +126,8 @@ func (t *Tree) backup3() {
 }
 
 // next returns the next unread token and advances the internal cursor by one.
-func (t *Tree) next() token {
-	var tok token
+func (t *Tree) next() *token {
+	var tok *token
 	if len(t.unread) > 0 {
 		tok, t.unread = t.unread[len(t.unread)-1], t.unread[:len(t.unread)-1]
 	} else {
@@ -140,8 +140,8 @@ func (t *Tree) next() token {
 }
 
 // nextNonSpace returns the next non-whitespace token.
-func (t *Tree) nextNonSpace() token {
-	var next token
+func (t *Tree) nextNonSpace() *token {
+	var next *token
 	for {
 		next = t.next()
 		if next.tokenType != tokenWhitespace {
@@ -152,7 +152,7 @@ func (t *Tree) nextNonSpace() token {
 
 // expect returns the next non-space token. Additionally, if the token is not of one of the expected types,
 // an UnexpectedTokenError is returned.
-func (t *Tree) expect(typs ...tokenType) (token, error) {
+func (t *Tree) expect(typs ...tokenType) (*token, error) {
 	tok := t.nextNonSpace()
 	for _, typ := range typs {
 		if tok.tokenType == typ {
@@ -166,7 +166,7 @@ func (t *Tree) expect(typs ...tokenType) (token, error) {
 // expectValue returns the next non-space token, with additional checks on the value of the token.
 // If the token is not of the expected type, an UnexpectedTokenError is returned. If the token is not the
 // expected value, an UnexpectedValueError is returned.
-func (t *Tree) expectValue(typ tokenType, val string) (token, error) {
+func (t *Tree) expectValue(typ tokenType, val string) (*token, error) {
 	tok, err := t.expect(typ)
 	if err != nil {
 		return tok, err