gfxapi/gles/glsl/preprocessor/preprocessorImpl.go - platform/tools/gpu - Git at Google

 // Copyright (C) 2014 The Android Open Source Project
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package preprocessor

 import (
 	"android.googlesource.com/platform/tools/gpu/gfxapi/gles/glsl/ast"
 	"android.googlesource.com/platform/tools/gpu/parse"
 	"bytes"
 	"fmt"
 )

 // ifEntry is a structure containing the data necessary for proper evaluation of #if*
 // preprocessor directives.
 type ifEntry struct {
 	WasTrue bool // Whether one of the #if expressions in this chain was true
 	HadElse bool // Whether we already encountered an #else block
 	Copying bool // Whether the current block should be output
 }

 type macroDefinition struct {
 	name       string          // macro name
 	function   bool            // Whether this is a function macro.
 	argCount   int             // The number of arguments of the macro.
 	definition []macroExpander // The macro definition as a list of macroExpanders.
 }

 // preprocessorImpl stores the internal state of a preprocessor instance.
 type preprocessorImpl struct {
 	err   ast.ErrorCollector
 	lexer *lexer

 	macros       map[string]macroDefinition // All currently defined macros
 	ifStack      []ifEntry                  // The stack of all encountered #if directives.
 	line         int                        // The current line.
 	currentToken *tokenExpansion
 	evaluator    ExpressionEvaluator
 }

 func (p *preprocessorImpl) GetErrors() []error {
 	return ast.ConcatErrors(p.lexer.err.GetErrors(), p.err.GetErrors())
 }

 // skipping returnes true if we should skip this token. We skip if any of the #if directives in
 // the stack says we should skip.
 func (p *preprocessorImpl) skipping() (skip bool) {
 	skip = false
 	for _, entry := range p.ifStack {
 		skip = skip || !entry.Copying
 	}
 	return
 }

 // tokenReader is an internal interface encapsulating a stream of tokens.
 type tokenReader interface {
 	Next() tokenExpansion
 	Peek() tokenExpansion
 }

 // listReader is an implementation of tokenReader which reads tokens from a list. It is used to
 // rescan a macro expansion to expand macros recursively. It contains a nested tokenReader, which
 // is read from after the own token list. This happens in case of recursive function macros with
 // unbalanced parenthesis.
 type listReader struct {
 	list []tokenExpansion
 	next tokenReader
 }

 func (r *listReader) Next() (t tokenExpansion) {
 	if len(r.list) > 0 {
 		t = r.list[0]
 		r.list = r.list[1:]
 	} else if r.next != nil {
 		t = r.next.Next()
 	}
 	return
 }
 func (r *listReader) Peek() (t tokenExpansion) {
 	if len(r.list) > 0 {
 		t = r.list[0]
 	} else if r.next != nil {
 		t = r.next.Peek()
 	}
 	return
 }

 // processList is a helper function for processMacro. It calls processMacro on all tokens in the
 // list.
 func (p *preprocessorImpl) processList(r *listReader) (result []tokenExpansion) {

 	for len(r.list) > 0 {
 		token := r.Next()
 		result = append(result, p.processMacro(token, r)...)
 	}
 	return
 }

 // readMacroArgs reads macro arguments. It returns the arguments as a list of lists of tokens.
 // Failure is reported by the second return value.
 func (p *preprocessorImpl) readMacroArgs(reader tokenReader) (args [][]tokenExpansion, ok bool) {
 	var arg []tokenExpansion // currently processed argument
 	level := 0               // number of nested parenthesis
 	for {
 		if reader.Peek().Info.Token == nil {
 			p.err.Errorf("Unexpected end of file while processing a macro.")
 			return args, false
 		}
 		if level == 0 {
 			switch reader.Peek().Info.Token {
 			case OpRParen:
 				args = append(args, arg)
 				return args, true
 			case OpLParen:
 				level++
 				arg = append(arg, reader.Next())
 				continue
 			case ast.BoComma:
 				reader.Next()
 				args = append(args, arg)
 				arg = nil
 				continue
 			}
 		}
 		switch reader.Peek().Info.Token {
 		case OpRParen:
 			level--
 			arg = append(arg, reader.Next())
 		case OpLParen:
 			level++
 			arg = append(arg, reader.Next())
 		default:
 			arg = append(arg, reader.Next())
 		}
 	}
 }

 // parseMacroCallArgs reads arguments to a function macro, pre-expands them and computes the
 // intersection of their hide sets. It reads the argument from the specified token reader. In
 // case of errors the hide set is nil.
 func (p *preprocessorImpl) parseMacroCallArgs(reader tokenReader, macro tokenExpansion,
 	argCount int) ([][]tokenExpansion, hideSet) {

 	if reader.Peek().Info.Token != OpLParen {
 		// Function macros are not expanded if the next token is not '('.
 		return nil, nil
 	}

 	reader.Next()
 	args, ok := p.readMacroArgs(reader)
 	if !ok {
 		return nil, nil
 	}

 	lastTok := reader.Next()
 	if len(args) != argCount {
 		p.err.Errorf("Incorrect number of arguments to macro '%v': expected %d, got %d.",
 			macro.Info.Token, argCount, len(args))
 		// Try to recover by padding args
 		for len(args) < argCount {
 			args = append(args, nil)
 		}
 	}

 	// Macro argument pre-expansion
 	for i := range args {
 		args[i] = p.processList(&listReader{args[i], nil})
 	}

 	set := intersect(macro.HideSet, lastTok.HideSet)

 	return args, set
 }

 // processMacro checks t for macro definitions and fully expands it. reader is an interface to
 // the following tokens, needed for processing function macro invocations.
 func (p *preprocessorImpl) processMacro(t tokenExpansion, reader tokenReader) []tokenExpansion {
 	// eof pseudo-token
 	if t.Info.Token == nil {
 		return []tokenExpansion{t}
 	}

 	name := t.Info.Token.String()
 	def, present := p.macros[name]
 	if !present {
 		// no expansion needed
 		return []tokenExpansion{t}
 	}

 	set := t.HideSet

 	if _, present := set[name]; present {
 		// This macro should not be expanded.
 		return []tokenExpansion{t}
 	}

 	var args [][]tokenExpansion
 	if def.function {
 		args, set = p.parseMacroCallArgs(reader, t, def.argCount)
 		if set == nil {
 			return []tokenExpansion{t}
 		}
 	}

 	list := make([]tokenExpansion, 0, len(def.definition))
 	// Substitute arguments into macro definition
 	for _, expander := range def.definition {
 		list = append(list, expander(args)...)
 	}

 	// Extend the hide sets
 	for _, e := range list {
 		e.HideSet.AddAll(set)
 		e.HideSet[name] = struct{}{}
 	}

 	// Expand macros in the definition recursively
 	return p.processList(&listReader{list, reader})
 }

 // getDirectiveArguments is a helper function used to read the arguments of a preprocessor
 // directive. It consumes all tokens until the newline and returns them. If emptyOk is false, it
 // will raise an error in the case of an empty argument list.
 func (p *preprocessorImpl) getDirectiveArguments(info TokenInfo, emptyOk bool) []TokenInfo {
 	dir := info.Token
 	var ret []TokenInfo
 	for info = p.lexer.Peek(); info.Token != nil && !info.Newline; info = p.lexer.Peek() {
 		ret = append(ret, p.lexer.Next())
 	}

 	if len(ret) == 0 && !emptyOk {
 		p.err.Errorf("%s needs an argument.", dir)
 	}

 	return ret
 }

 func isIdentOrKeyword(t TokenInfo) bool {
 	switch t.Token.(type) {
 	case Identifier, Keyword, ast.BareType:
 		return true
 	default:
 		return false
 	}
 }

 // Given a list of tokens following `#define FOO(`, consume the tokens that make up the macro
 // argument list. Returns the list of unconsumed tokens and a `macro_name->position` map.
 func (p *preprocessorImpl) parseDefMacroArgs(macro Token,
 	args []TokenInfo) (rest []TokenInfo, argMap map[string]int) {

 	argMap = make(map[string]int)
 	for {
 		if len(args) <= 1 {
 			p.err.Errorf("Macro definition ended unexpectedly.")
 			return nil, nil
 		}

 		if !isIdentOrKeyword(args[0]) {
 			p.err.Errorf("Invalid function macro definition. "+
 				"Expected an identifier, got '%s'.", args[0].Token)
 			return
 		}
 		name := args[0].Token.String()

 		if _, ok := argMap[name]; ok {
 			p.err.Errorf("Macro '%s' contains two arguments named '%s'.", macro, name)
 		}

 		argMap[name] = len(argMap)

 		switch args[1].Token {
 		case OpRParen:
 			return args[2:], argMap
 		case ast.BoComma:
 			args = args[2:]
 			continue
 		default:
 			p.err.Errorf("Invalid function macro definition. "+
 				"Expected ',', ')', got '%s'.", args[1].Token)
 			return nil, nil
 		}
 	}
 }

 // process a #define directive
 func (p *preprocessorImpl) processDefine(args []TokenInfo) {
 	macro := args[0]

 	if _, ok := p.macros[macro.Token.String()]; ok {
 		p.err.Errorf("Macro '%s' already defined.", args[0].Token)
 	}

 	args = args[1:]

 	if len(args) == 0 || args[0].Whitespace || args[0].Token != OpLParen {
 		// Just an object macro, we're done.
 		expansion := make([]macroExpander, len(args))
 		for i := range args {
 			expansion[i] = args[i].expand
 		}

 		name := macro.Token.String()
 		p.macros[name] = macroDefinition{name, false, 0, expansion}
 		return
 	}

 	args, argMap := p.parseDefMacroArgs(macro.Token, args[1:])
 	if argMap == nil {
 		return
 	}

 	expansion := make([]macroExpander, len(args))
 	for i := range args {
 		if arg, ok := argMap[args[i].Token.String()]; ok {
 			expansion[i] = argumentExpander(arg).expand
 		} else {
 			expansion[i] = args[i].expand
 		}
 	}

 	name := macro.Token.String()
 	p.macros[name] = macroDefinition{name, true, len(argMap), expansion}
 }

 // processDirectives reads any preprocessor directives from the input stream and processes them.
 func (p *preprocessorImpl) processDirectives() {
 	for {
 		if _, ok := p.lexer.Peek().Token.(ppKeyword); !ok {
 			break
 		}
 		p.processDirective(p.lexer.Next())
 	}
 }

 func (p *preprocessorImpl) evaluateDefined(arg TokenInfo) tokenExpansion {
 	var ic ast.IntValue
 	if _, present := p.macros[arg.Token.String()]; present {
 		ic = ast.IntValue(1)
 	} else {
 		ic = ast.IntValue(0)
 	}
 	return newTokenExpansion(TokenInfo{Token: ic})
 }

 func (p *preprocessorImpl) evaluateIf(args []TokenInfo) bool {
 	// append fake EOF
 	lastToken := args[len(args)-1].Cst.Token()
 	eof := &parse.Leaf{}
 	eof.SetToken(parse.Token{Source: lastToken.Source, Start: lastToken.End, End: lastToken.End})
 	args = append(args, TokenInfo{Token: nil, Cst: eof})

 	var list []tokenExpansion
 	// convert args to tokenExpansions and evaluate defined(X)
 	for i := 0; i < len(args); i++ {
 		if args[i].Token == Identifier("defined") {
 			if i+1 < len(args) && isIdentOrKeyword(args[i+1]) {
 				list = append(list, p.evaluateDefined(args[i+1]))
 				i++
 			} else if i+3 < len(args) && args[i+1].Token == OpLParen &&
 				isIdentOrKeyword(args[i+2]) && args[i+3].Token == OpRParen {

 				list = append(list, p.evaluateDefined(args[i+2]))
 				i += 3
 			} else {
 				p.err.Errorf("Operator 'defined' used incorrectly.")
 			}
 		} else {
 			list = append(list, newTokenExpansion(args[i]))
 		}
 	}

 	reader := &listReader{list: list} // reader will read the arguments
 	worker := &listWorker{reader, p}  // worker will expand them
 	pp := &Preprocessor{impl: worker} // pp will provide the lookahead
 	val, err := p.evaluator(pp)       // and evaluator will evalate them

 	p.err.Error(err...)
 	return val != 0
 }

 func (p *preprocessorImpl) processDirective(info TokenInfo) {
 	switch info.Token {
 	case ppDefine:
 		args := p.getDirectiveArguments(info, false)
 		if p.skipping() || args == nil {
 			return
 		}
 		p.processDefine(args)

 	case ppUndef:
 		args := p.getDirectiveArguments(info, false)
 		if p.skipping() || args == nil {
 			return
 		}

 		if _, ok := p.macros[args[0].Token.String()]; !ok {
 			p.err.Errorf("Macro '%s' not defined.", args[0].Token)
 			return
 		}
 		delete(p.macros, args[0].Token.String())

 	case ppIf:
 		args := p.getDirectiveArguments(info, false)
 		val := p.evaluateIf(args)
 		p.ifStack = append(p.ifStack, ifEntry{Copying: val, WasTrue: val})

 	case ppElif:
 		args := p.getDirectiveArguments(info, true)

 		if len(p.ifStack) == 0 {
 			p.err.Errorf("Unmatched #elif.")
 			return
 		}

 		entry := &p.ifStack[len(p.ifStack)-1]
 		if entry.HadElse {
 			p.err.Errorf("#elif after #else.")
 			entry.Copying = false
 			return
 		}
 		if !entry.Copying {
 			val := p.evaluateIf(args)
 			entry.Copying = val
 			if val {
 				entry.WasTrue = true
 			}
 		} else {
 			entry.Copying = false
 		}
 		return

 	// TODO: support #version, #pragma and #extension instead of silently ignoring them.
 	case ppVersion, ppPragma, ppExtension:
 		_ = p.getDirectiveArguments(info, false)
 		return

 	case ppIfdef, ppIfndef:
 		args := p.getDirectiveArguments(info, false)

 		var defined bool
 		if args == nil {
 			defined = false
 		} else {
 			_, defined = p.macros[args[0].Token.String()]
 		}

 		copying := defined == (info.Token == ppIfdef)

 		p.ifStack = append(p.ifStack, ifEntry{Copying: copying, WasTrue: copying})

 	case ppElse:
 		_ = p.getDirectiveArguments(info, true)

 		if len(p.ifStack) == 0 {
 			p.err.Errorf("Unmatched #else.")
 			return
 		}
 		entry := &p.ifStack[len(p.ifStack)-1]
 		if entry.HadElse {
 			p.err.Errorf("#if directive has multiple #else directives.")
 			entry.Copying = false
 			return
 		}
 		entry.HadElse = true
 		entry.Copying = !entry.WasTrue

 	case ppEndif:
 		_ = p.getDirectiveArguments(info, true)

 		if len(p.ifStack) == 0 {
 			p.err.Errorf("Unmatched #endif.")
 			return
 		}
 		p.ifStack = p.ifStack[:len(p.ifStack)-1]

 	case ppError:
 		args := p.getDirectiveArguments(info, true)
 		if p.skipping() {
 			return
 		}

 		var msg bytes.Buffer
 		for _, i := range args {
 			i.Cst.Prefix().WriteTo(&msg)
 			msg.Write([]byte(i.Token.String()))
 			i.Cst.Suffix().WriteTo(&msg)
 		}
 		p.err.Errorf(msg.String())
 	}
 }

 func addBuiltinMacro(macros map[string]macroDefinition, name string, expander macroExpander) {
 	macros[name] = macroDefinition{
 		name:       name,
 		definition: []macroExpander{expander},
 	}
 }

 func newPreprocessorImpl(data string, eval ExpressionEvaluator, file int) *preprocessorImpl {
 	p := &preprocessorImpl{
 		lexer:     newLexer(fmt.Sprintf("File %v", file), data),
 		macros:    make(map[string]macroDefinition),
 		evaluator: eval,
 	}
 	addBuiltinMacro(p.macros, "__LINE__", p.expandLine)
 	addBuiltinMacro(p.macros, "__FILE__", TokenInfo{Token: ast.IntValue(file)}.expand)
 	addBuiltinMacro(p.macros, "__VERSION__", TokenInfo{Token: ast.IntValue(300)}.expand)
 	addBuiltinMacro(p.macros, "GL_ES", TokenInfo{Token: ast.IntValue(1)}.expand)
 	return p
 }

 //////////////////////////// tokenReader interface /////////////////////////

 func (p *preprocessorImpl) Peek() tokenExpansion {
 	for p.currentToken == nil {
 		// process any preprocessor directives
 		p.processDirectives()

 		tok := newTokenExpansion(p.lexer.Next())
 		p.line, _ = tok.Info.Cst.Token().Cursor()

 		if tok.Info.Token == nil {
 			if len(p.ifStack) > 0 {
 				p.err.Errorf("Unterminated #if directive at the end of file.")
 			}
 			p.currentToken = &tok
 		} else if !p.skipping() {
 			p.currentToken = &tok
 		}
 	}
 	return *p.currentToken
 }

 func (p *preprocessorImpl) Next() tokenExpansion {
 	ret := p.Peek()
 	p.currentToken = nil
 	return ret
 }

 //////////////////////////// worker interface /////////////////////////

 func (p *preprocessorImpl) Work() []tokenExpansion {
 	return p.processMacro(p.Next(), p)
 }
	// Copyright (C) 2014 The Android Open Source Project
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package preprocessor

	import (
	"android.googlesource.com/platform/tools/gpu/gfxapi/gles/glsl/ast"
	"android.googlesource.com/platform/tools/gpu/parse"
	"bytes"
	"fmt"
	)

	// ifEntry is a structure containing the data necessary for proper evaluation of #if*
	// preprocessor directives.
	type ifEntry struct {
	WasTrue bool // Whether one of the #if expressions in this chain was true
	HadElse bool // Whether we already encountered an #else block
	Copying bool // Whether the current block should be output
	}

	type macroDefinition struct {
	name string // macro name
	function bool // Whether this is a function macro.
	argCount int // The number of arguments of the macro.
	definition []macroExpander // The macro definition as a list of macroExpanders.
	}

	// preprocessorImpl stores the internal state of a preprocessor instance.
	type preprocessorImpl struct {
	err ast.ErrorCollector
	lexer *lexer

	macros map[string]macroDefinition // All currently defined macros
	ifStack []ifEntry // The stack of all encountered #if directives.
	line int // The current line.
	currentToken *tokenExpansion
	evaluator ExpressionEvaluator
	}

	func (p *preprocessorImpl) GetErrors() []error {
	return ast.ConcatErrors(p.lexer.err.GetErrors(), p.err.GetErrors())
	}

	// skipping returnes true if we should skip this token. We skip if any of the #if directives in
	// the stack says we should skip.
	func (p *preprocessorImpl) skipping() (skip bool) {
	skip = false
	for _, entry := range p.ifStack {
	skip = skip \|\| !entry.Copying
	}
	return
	}

	// tokenReader is an internal interface encapsulating a stream of tokens.
	type tokenReader interface {
	Next() tokenExpansion
	Peek() tokenExpansion
	}

	// listReader is an implementation of tokenReader which reads tokens from a list. It is used to
	// rescan a macro expansion to expand macros recursively. It contains a nested tokenReader, which
	// is read from after the own token list. This happens in case of recursive function macros with
	// unbalanced parenthesis.
	type listReader struct {
	list []tokenExpansion
	next tokenReader
	}

	func (r *listReader) Next() (t tokenExpansion) {
	if len(r.list) > 0 {
	t = r.list[0]
	r.list = r.list[1:]
	} else if r.next != nil {
	t = r.next.Next()
	}
	return
	}
	func (r *listReader) Peek() (t tokenExpansion) {
	if len(r.list) > 0 {
	t = r.list[0]
	} else if r.next != nil {
	t = r.next.Peek()
	}
	return
	}

	// processList is a helper function for processMacro. It calls processMacro on all tokens in the
	// list.
	func (p preprocessorImpl) processList(r listReader) (result []tokenExpansion) {

	for len(r.list) > 0 {
	token := r.Next()
	result = append(result, p.processMacro(token, r)...)
	}
	return
	}

	// readMacroArgs reads macro arguments. It returns the arguments as a list of lists of tokens.
	// Failure is reported by the second return value.
	func (p *preprocessorImpl) readMacroArgs(reader tokenReader) (args [][]tokenExpansion, ok bool) {
	var arg []tokenExpansion // currently processed argument
	level := 0 // number of nested parenthesis
	for {
	if reader.Peek().Info.Token == nil {
	p.err.Errorf("Unexpected end of file while processing a macro.")
	return args, false
	}
	if level == 0 {
	switch reader.Peek().Info.Token {
	case OpRParen:
	args = append(args, arg)
	return args, true
	case OpLParen:
	level++
	arg = append(arg, reader.Next())
	continue
	case ast.BoComma:
	reader.Next()
	args = append(args, arg)
	arg = nil
	continue
	}
	}
	switch reader.Peek().Info.Token {
	case OpRParen:
	level--
	arg = append(arg, reader.Next())
	case OpLParen:
	level++
	arg = append(arg, reader.Next())
	default:
	arg = append(arg, reader.Next())
	}
	}
	}

	// parseMacroCallArgs reads arguments to a function macro, pre-expands them and computes the
	// intersection of their hide sets. It reads the argument from the specified token reader. In
	// case of errors the hide set is nil.
	func (p *preprocessorImpl) parseMacroCallArgs(reader tokenReader, macro tokenExpansion,
	argCount int) ([][]tokenExpansion, hideSet) {

	if reader.Peek().Info.Token != OpLParen {
	// Function macros are not expanded if the next token is not '('.
	return nil, nil
	}

	reader.Next()
	args, ok := p.readMacroArgs(reader)
	if !ok {
	return nil, nil
	}

	lastTok := reader.Next()
	if len(args) != argCount {
	p.err.Errorf("Incorrect number of arguments to macro '%v': expected %d, got %d.",
	macro.Info.Token, argCount, len(args))
	// Try to recover by padding args
	for len(args) < argCount {
	args = append(args, nil)
	}
	}

	// Macro argument pre-expansion
	for i := range args {
	args[i] = p.processList(&listReader{args[i], nil})
	}

	set := intersect(macro.HideSet, lastTok.HideSet)

	return args, set
	}

	// processMacro checks t for macro definitions and fully expands it. reader is an interface to
	// the following tokens, needed for processing function macro invocations.
	func (p *preprocessorImpl) processMacro(t tokenExpansion, reader tokenReader) []tokenExpansion {
	// eof pseudo-token
	if t.Info.Token == nil {
	return []tokenExpansion{t}
	}

	name := t.Info.Token.String()
	def, present := p.macros[name]
	if !present {
	// no expansion needed
	return []tokenExpansion{t}
	}

	set := t.HideSet

	if _, present := set[name]; present {
	// This macro should not be expanded.
	return []tokenExpansion{t}
	}

	var args [][]tokenExpansion
	if def.function {
	args, set = p.parseMacroCallArgs(reader, t, def.argCount)
	if set == nil {
	return []tokenExpansion{t}
	}
	}

	list := make([]tokenExpansion, 0, len(def.definition))
	// Substitute arguments into macro definition
	for _, expander := range def.definition {
	list = append(list, expander(args)...)
	}

	// Extend the hide sets
	for _, e := range list {
	e.HideSet.AddAll(set)
	e.HideSet[name] = struct{}{}
	}

	// Expand macros in the definition recursively
	return p.processList(&listReader{list, reader})
	}

	// getDirectiveArguments is a helper function used to read the arguments of a preprocessor
	// directive. It consumes all tokens until the newline and returns them. If emptyOk is false, it
	// will raise an error in the case of an empty argument list.
	func (p *preprocessorImpl) getDirectiveArguments(info TokenInfo, emptyOk bool) []TokenInfo {
	dir := info.Token
	var ret []TokenInfo
	for info = p.lexer.Peek(); info.Token != nil && !info.Newline; info = p.lexer.Peek() {
	ret = append(ret, p.lexer.Next())
	}

	if len(ret) == 0 && !emptyOk {
	p.err.Errorf("%s needs an argument.", dir)
	}

	return ret
	}

	func isIdentOrKeyword(t TokenInfo) bool {
	switch t.Token.(type) {
	case Identifier, Keyword, ast.BareType:
	return true
	default:
	return false
	}
	}

	// Given a list of tokens following `#define FOO(`, consume the tokens that make up the macro
	// argument list. Returns the list of unconsumed tokens and a `macro_name->position` map.
	func (p *preprocessorImpl) parseDefMacroArgs(macro Token,
	args []TokenInfo) (rest []TokenInfo, argMap map[string]int) {

	argMap = make(map[string]int)
	for {
	if len(args) <= 1 {
	p.err.Errorf("Macro definition ended unexpectedly.")
	return nil, nil
	}

	if !isIdentOrKeyword(args[0]) {
	p.err.Errorf("Invalid function macro definition. "+
	"Expected an identifier, got '%s'.", args[0].Token)
	return
	}
	name := args[0].Token.String()

	if _, ok := argMap[name]; ok {
	p.err.Errorf("Macro '%s' contains two arguments named '%s'.", macro, name)
	}

	argMap[name] = len(argMap)

	switch args[1].Token {
	case OpRParen:
	return args[2:], argMap
	case ast.BoComma:
	args = args[2:]
	continue
	default:
	p.err.Errorf("Invalid function macro definition. "+
	"Expected ',', ')', got '%s'.", args[1].Token)
	return nil, nil
	}
	}
	}

	// process a #define directive
	func (p *preprocessorImpl) processDefine(args []TokenInfo) {
	macro := args[0]

	if _, ok := p.macros[macro.Token.String()]; ok {
	p.err.Errorf("Macro '%s' already defined.", args[0].Token)
	}

	args = args[1:]

	if len(args) == 0 \|\| args[0].Whitespace \|\| args[0].Token != OpLParen {
	// Just an object macro, we're done.
	expansion := make([]macroExpander, len(args))
	for i := range args {
	expansion[i] = args[i].expand
	}

	name := macro.Token.String()
	p.macros[name] = macroDefinition{name, false, 0, expansion}
	return
	}

	args, argMap := p.parseDefMacroArgs(macro.Token, args[1:])
	if argMap == nil {
	return
	}

	expansion := make([]macroExpander, len(args))
	for i := range args {
	if arg, ok := argMap[args[i].Token.String()]; ok {
	expansion[i] = argumentExpander(arg).expand
	} else {
	expansion[i] = args[i].expand
	}
	}

	name := macro.Token.String()
	p.macros[name] = macroDefinition{name, true, len(argMap), expansion}
	}

	// processDirectives reads any preprocessor directives from the input stream and processes them.
	func (p *preprocessorImpl) processDirectives() {
	for {
	if _, ok := p.lexer.Peek().Token.(ppKeyword); !ok {
	break
	}
	p.processDirective(p.lexer.Next())
	}
	}

	func (p *preprocessorImpl) evaluateDefined(arg TokenInfo) tokenExpansion {
	var ic ast.IntValue
	if _, present := p.macros[arg.Token.String()]; present {
	ic = ast.IntValue(1)
	} else {
	ic = ast.IntValue(0)
	}
	return newTokenExpansion(TokenInfo{Token: ic})
	}

	func (p *preprocessorImpl) evaluateIf(args []TokenInfo) bool {
	// append fake EOF
	lastToken := args[len(args)-1].Cst.Token()
	eof := &parse.Leaf{}
	eof.SetToken(parse.Token{Source: lastToken.Source, Start: lastToken.End, End: lastToken.End})
	args = append(args, TokenInfo{Token: nil, Cst: eof})

	var list []tokenExpansion
	// convert args to tokenExpansions and evaluate defined(X)
	for i := 0; i < len(args); i++ {
	if args[i].Token == Identifier("defined") {
	if i+1 < len(args) && isIdentOrKeyword(args[i+1]) {
	list = append(list, p.evaluateDefined(args[i+1]))
	i++
	} else if i+3 < len(args) && args[i+1].Token == OpLParen &&
	isIdentOrKeyword(args[i+2]) && args[i+3].Token == OpRParen {

	list = append(list, p.evaluateDefined(args[i+2]))
	i += 3
	} else {
	p.err.Errorf("Operator 'defined' used incorrectly.")
	}
	} else {
	list = append(list, newTokenExpansion(args[i]))
	}
	}

	reader := &listReader{list: list} // reader will read the arguments
	worker := &listWorker{reader, p} // worker will expand them
	pp := &Preprocessor{impl: worker} // pp will provide the lookahead
	val, err := p.evaluator(pp) // and evaluator will evalate them

	p.err.Error(err...)
	return val != 0
	}

	func (p *preprocessorImpl) processDirective(info TokenInfo) {
	switch info.Token {
	case ppDefine:
	args := p.getDirectiveArguments(info, false)
	if p.skipping() \|\| args == nil {
	return
	}
	p.processDefine(args)

	case ppUndef:
	args := p.getDirectiveArguments(info, false)
	if p.skipping() \|\| args == nil {
	return
	}

	if _, ok := p.macros[args[0].Token.String()]; !ok {
	p.err.Errorf("Macro '%s' not defined.", args[0].Token)
	return
	}
	delete(p.macros, args[0].Token.String())

	case ppIf:
	args := p.getDirectiveArguments(info, false)
	val := p.evaluateIf(args)
	p.ifStack = append(p.ifStack, ifEntry{Copying: val, WasTrue: val})

	case ppElif:
	args := p.getDirectiveArguments(info, true)

	if len(p.ifStack) == 0 {
	p.err.Errorf("Unmatched #elif.")
	return
	}

	entry := &p.ifStack[len(p.ifStack)-1]
	if entry.HadElse {
	p.err.Errorf("#elif after #else.")
	entry.Copying = false
	return
	}
	if !entry.Copying {
	val := p.evaluateIf(args)
	entry.Copying = val
	if val {
	entry.WasTrue = true
	}
	} else {
	entry.Copying = false
	}
	return

	// TODO: support #version, #pragma and #extension instead of silently ignoring them.
	case ppVersion, ppPragma, ppExtension:
	_ = p.getDirectiveArguments(info, false)
	return

	case ppIfdef, ppIfndef:
	args := p.getDirectiveArguments(info, false)

	var defined bool
	if args == nil {
	defined = false
	} else {
	_, defined = p.macros[args[0].Token.String()]
	}

	copying := defined == (info.Token == ppIfdef)

	p.ifStack = append(p.ifStack, ifEntry{Copying: copying, WasTrue: copying})

	case ppElse:
	_ = p.getDirectiveArguments(info, true)

	if len(p.ifStack) == 0 {
	p.err.Errorf("Unmatched #else.")
	return
	}
	entry := &p.ifStack[len(p.ifStack)-1]
	if entry.HadElse {
	p.err.Errorf("#if directive has multiple #else directives.")
	entry.Copying = false
	return
	}
	entry.HadElse = true
	entry.Copying = !entry.WasTrue

	case ppEndif:
	_ = p.getDirectiveArguments(info, true)

	if len(p.ifStack) == 0 {
	p.err.Errorf("Unmatched #endif.")
	return
	}
	p.ifStack = p.ifStack[:len(p.ifStack)-1]

	case ppError:
	args := p.getDirectiveArguments(info, true)
	if p.skipping() {
	return
	}

	var msg bytes.Buffer
	for _, i := range args {
	i.Cst.Prefix().WriteTo(&msg)
	msg.Write([]byte(i.Token.String()))
	i.Cst.Suffix().WriteTo(&msg)
	}
	p.err.Errorf(msg.String())
	}
	}

	func addBuiltinMacro(macros map[string]macroDefinition, name string, expander macroExpander) {
	macros[name] = macroDefinition{
	name: name,
	definition: []macroExpander{expander},
	}
	}

	func newPreprocessorImpl(data string, eval ExpressionEvaluator, file int) *preprocessorImpl {
	p := &preprocessorImpl{
	lexer: newLexer(fmt.Sprintf("File %v", file), data),
	macros: make(map[string]macroDefinition),
	evaluator: eval,
	}
	addBuiltinMacro(p.macros, "__LINE__", p.expandLine)
	addBuiltinMacro(p.macros, "__FILE__", TokenInfo{Token: ast.IntValue(file)}.expand)
	addBuiltinMacro(p.macros, "__VERSION__", TokenInfo{Token: ast.IntValue(300)}.expand)
	addBuiltinMacro(p.macros, "GL_ES", TokenInfo{Token: ast.IntValue(1)}.expand)
	return p
	}

	//////////////////////////// tokenReader interface /////////////////////////

	func (p *preprocessorImpl) Peek() tokenExpansion {
	for p.currentToken == nil {
	// process any preprocessor directives
	p.processDirectives()

	tok := newTokenExpansion(p.lexer.Next())
	p.line, _ = tok.Info.Cst.Token().Cursor()

	if tok.Info.Token == nil {
	if len(p.ifStack) > 0 {
	p.err.Errorf("Unterminated #if directive at the end of file.")
	}
	p.currentToken = &tok
	} else if !p.skipping() {
	p.currentToken = &tok
	}
	}
	return *p.currentToken
	}

	func (p *preprocessorImpl) Next() tokenExpansion {
	ret := p.Peek()
	p.currentToken = nil
	return ret
	}

	//////////////////////////// worker interface /////////////////////////

	func (p *preprocessorImpl) Work() []tokenExpansion {
	return p.processMacro(p.Next(), p)
	}