tree: b39eaa000a30dac1f20d405e3537e113895f24f5 [path history] [tgz]
  1. ast_test.go
  2. branch.go
  3. cst_test.go
  4. error.go
  5. fragment.go
  6. leaf.go
  7. node.go
  8. parse_binary.go
  9. parser.go
  10. parser_test.go
  11. reader.go
  12. reader_test.go
  13. README.md
  14. skip.go
  15. token.go
parse/README.md

parse


import "android.googlesource.com/platform/tools/gpu/parse"

Package parse provides support functionality for writing scannerless parsers.

The main entry point is the Parse function. It works by setting up a parser on the supplied content, and then invoking the supplied root parsing function. The CST is built for you inside the ParseLeaf and ParseBranch methods of the parser, but it is up to the supplied parsing functions to hold on to the CST if you want it, and also to build the AST.

Usage

const RuneEOL = '\n'

RuneEOL is the rune that marks the end of a line.

var (
	// ParseErrorLimit is the maximum number of errors before a parse is aborted.
	ParseErrorLimit = 10
	// AbortParse is paniced when a parse cannot continue. It is recovered at the
	// top level, to allow the errors to be cleanly returned to the caller.
	AbortParse = errors.New("abort")
)

func Parse

func Parse(root BranchParser, data string, skip Skip) []Error

Parse is the main entry point to the parse library. Given a root parse function, the input string and the Skip controller, it builds a and initializes a Parser, runs the root using it, verifies it worked correctly and then returns the errors generated if any.

type Branch

type Branch struct {

	// Children is the slice of child nodes for this Branch.
	Children []Node
}

Branch is a CST node that can have children.

func (*Branch) AddPrefix

func (n *Branch) AddPrefix(s Separator)

func (*Branch) AddSuffix

func (n *Branch) AddSuffix(s Separator)

func (*Branch) Prefix

func (n *Branch) Prefix() Separator

func (*Branch) Suffix

func (n *Branch) Suffix() Separator

func (*Branch) Token

func (n *Branch) Token() Token

func (*Branch) WriteTo

func (n *Branch) WriteTo(w io.Writer) error

type BranchParser

type BranchParser func(p *Parser, cst *Branch)

BranchParser is a function that is passed to ParseBranch. It is handed the Branch to fill in and the Parser to fill it from, and must either succeed or add an error to the parser.

type Error

type Error struct {
	// At is the parse fragment that was being processed when the error was encountered.
	At Fragment
	// Message is the message associated with the error.
	Message string
	// Stack is the captured stack trace at the point the error was noticed.
	Stack []byte
}

Error represents the information that us useful in debugging a parse failure.

func (Error) Error

func (err Error) Error() string

func (Error) Format

func (err Error) Format(f fmt.State, c rune)

type ErrorList

type ErrorList []Error

ErrorList is a convenience type for managing lists of errors.

func (*ErrorList) Add

func (l *ErrorList) Add(r *Reader, at Fragment, message string, args ...interface{})

type Fragment

type Fragment interface {
	// Token returns the underlying token of this node.
	Token() Token
	// Write is used to write the underlying token out to the writer.
	WriteTo(io.Writer) error
}

Fragment is a component of a cst that is backed by a token. This includes Nodes and all forms of space and comment.

func NewFragment

func NewFragment(token Token) Fragment

type Leaf

type Leaf struct {
}

Leaf nodes are part of the cst that cannot have child nodes, they represent a single token from the input.

func (*Leaf) AddPrefix

func (n *Leaf) AddPrefix(s Separator)

func (*Leaf) AddSuffix

func (n *Leaf) AddSuffix(s Separator)

func (*Leaf) Prefix

func (n *Leaf) Prefix() Separator

func (*Leaf) SetToken

func (n *Leaf) SetToken(token Token)

func (*Leaf) Suffix

func (n *Leaf) Suffix() Separator

func (*Leaf) Token

func (n *Leaf) Token() Token

func (*Leaf) WriteTo

func (n *Leaf) WriteTo(w io.Writer) error

type LeafParser

type LeafParser func(p *Parser, cst *Leaf)

LeafParser is a function that is passed to ParseLeaf. It is handed the Leaf to fill in and the Parser to fill it from, and must either succeed or add an error to the parser.

type Node

type Node interface {
	Fragment
	// Prefix returns the set of skippable fragments associated with this Node
	// that precede it in the stream. Association is defined by the Skip function
	// in use.
	Prefix() Separator
	// AddPrefix adds more fragments to the Prefix list.
	AddPrefix(Separator)
	// Suffix returns the set of skippable fragments associated with this Node
	// that follow it in the stream. Association is defined by the Skip function
	// in use, the default is until the end of the line.
	Suffix() Separator
	// AddSuffix adds more fragments to the Suffix list.
	AddSuffix(Separator)
}

Node is a Fragment in a cst that represents unskipped tokens.

type NumberKind

type NumberKind uint8

NumberKind is a type used by Reader.Numeric for identifying various kinds of numbers.

const (
	// No number was found.
	NotNumeric NumberKind = iota
	// A decimal number.
	Decimal
	// An octal number, starting with "0". PS: A lone "0" is classified as octal.
	Octal
	// A hexadecimal number, starting with "0x".
	Hexadecimal
	// A floating point number: "123.456". Whole and the fractional parts are optional (but
	// not both at the same time).
	Floating
	// A floating point number in scientific notation: "123.456e±789". The fractional part,
	// the dot and the exponent sign are all optional.
	Scientific
)

type Parser

type Parser struct {
	Reader           // The token reader for this parser.
	Errors ErrorList // The set of errors generated during the parse.
}

Parser contains all the context needed while parsing. They are built for you by the Parse function.

func (*Parser) Error

func (p *Parser) Error(message string, args ...interface{})

Error adds a new error to the parser error list. It will attempt to consume a token from the reader to act as a place holder, and also to ensure progress is made in the presence of errors.

func (*Parser) ErrorAt

func (p *Parser) ErrorAt(at Fragment, message string, args ...interface{})

ErrorAt is like Error, except because it is handed a fragment, it will not try to consume anything itself.

func (*Parser) Expected

func (p *Parser) Expected(value string)

Expected is a wrapper around p.ErrorAt for the very common case of an unexpected input. It uses value as the expected input, and parses a token of the stream for the unexpected actual input.

func (*Parser) ParseBranch

func (p *Parser) ParseBranch(cst *Branch, do BranchParser)

ParseLeaf adds a new Branch to cst and then calls the do function to parse the branch. This is called recursively to build the node tree.

func (*Parser) ParseLeaf

func (p *Parser) ParseLeaf(cst *Branch, do LeafParser)

ParseLeaf adds a new Leaf to cst and then calls the do function to parse the Leaf. If do is nil, a leaf will be built with the current unconsumed input.

type Reader

type Reader struct {
}

Reader is the interface to an object that converts a rune array into tokens.

func NewReader

func NewReader(data string) *Reader

NewReader creates a new reader which reads from the supplied string.

func (*Reader) Advance

func (r *Reader) Advance()

Advance moves the cursor one rune forward.

func (*Reader) AlphaNumeric

func (r *Reader) AlphaNumeric() bool

AlphaNumeric moves past anything that starts with a letter or underscore, and consists of letters, numbers or underscores. It returns true if the pattern was matched, false otherwise.

func (*Reader) Consume

func (r *Reader) Consume() Token

Consume consumes the current token.

func (*Reader) GuessNextToken

func (r *Reader) GuessNextToken() Token

GuessNextToken attempts to do a general purpose consume of a single arbitrary token from the stream. It is used by error handlers to indicate where the error occurred. It guarantees that if the stream is not finished, it will consume at least one character.

func (*Reader) IsEOF

func (r *Reader) IsEOF() bool

IsEOF returns true when the cursor is at the end of the input.

func (*Reader) NotSpace

func (r *Reader) NotSpace() bool

Space skips over any non whitespace, returning true if it advanced the cursor.

func (*Reader) Numeric

func (r *Reader) Numeric() NumberKind

Numeric tries to move past the common number pattern. It returns a constant of type NumberKind describing the kind of number it found.

func (*Reader) Peek

func (r *Reader) Peek() rune

Peek returns the next rune without advancing the cursor.

func (*Reader) Rollback

func (r *Reader) Rollback()

Rollback sets the cursor back to the last consume point.

func (*Reader) Rune

func (r *Reader) Rune(value rune) bool

Rune advances and returns true if the next rune after the cursor matches value.

func (*Reader) SeekRune

func (r *Reader) SeekRune(value rune) bool

SeekRune advances the cursor until either the value is found or the end of stream is reached. It returns true if it found value, false otherwise.

func (*Reader) Space

func (r *Reader) Space() bool

Space skips over any whitespace, returning true if it advanced the cursor.

func (*Reader) String

func (r *Reader) String(value string) bool

String checks to see if value occurs at cursor, if it does, it advances the cursor past it and returns true.

func (*Reader) Token

func (r *Reader) Token() Token

Token peeks at the current scanned token value. It does not consume anything.

type Separator

type Separator []Fragment

Separator is a list type to manage fragments that were skipped.

func (Separator) WriteTo

func (s Separator) WriteTo(w io.Writer) error

type Skip

type Skip func(parser *Parser, mode SkipMode) Separator

Skip is the function used to skip separating tokens. A separating token is one where, as far as the parser is concerned, the tokens do not exist, even though the tokens may have been necessary to separate the lexical tokens (whitespace), or carry useful information (comments).

func NewSkip

func NewSkip(line, blockstart, blockend string) Skip

NewSkip builds a Skip function for the common case of a parser that has one type of line comment, one type of block comment, and want to treat all unicode space characters as skippable.

type SkipMode

type SkipMode int
const (
	// SkipPrefix is the skip mode that skips tokens that are associated with
	// the following lexically relevant token. This is mostly important for
	// comment association.
	SkipPrefix SkipMode = iota
	// SkipSuffix is the skip mode that skips tokens that are associated with
	// the preceding lexically relevant token. This is mostly important for
	// comment association.
	SkipSuffix
)

type Token

type Token struct {
	Runes []rune // The full rune array for the string this token is from.
	Start int    // The start of the token in the full rune array.
	End   int    // One past the end of the token.
}

A Token represents the smallest consumed unit input.

func (Token) Cursor

func (t Token) Cursor() (line int, column int)

Cursor is used to calculate the line and column of the start of the token. It may be very expensive to call, and is intended to be used sparingly in producing human readable error messages only.

func (Token) Format

func (t Token) Format(f fmt.State, c rune)

Format implements fmt.Formatter writing the start end and value of the token.

func (Token) Len

func (t Token) Len() int

Len returns the length of the token in runes.

func (Token) String

func (t Token) String() string

String returns the string form of the rune range the token represents.