src/compiler/preprocessor/Tokenizer.l

/*
//
// Copyright (c) 2002-2011 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//

This file contains the Lex specification for GLSL ES preprocessor.
Based on Microsoft Visual Studio 2010 Preprocessor Grammar:
http://msdn.microsoft.com/en-us/library/2scxys89.aspx

IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
*/

%top{
//
// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//

// This file is auto-generated by generate_parser.sh. DO NOT EDIT!
}

%{
#include "Tokenizer.h"

#include "DiagnosticsBase.h"
#include "Token.h"

#if defined(__GNUC__)
// Triggered by the auto-generated yy_fatal_error function.
#pragma GCC diagnostic ignored "-Wmissing-noreturn"
#endif

typedef std::string YYSTYPE;
typedef pp::SourceLocation YYLTYPE;

// Use the unused yycolumn variable to track file (string) number.
#define yyfileno yycolumn

#define YY_USER_INIT                   \
    do {                               \
        yyfileno = 0;                  \
        yylineno = 1;                  \
        yyextra->leadingSpace = false; \
        yyextra->lineStart = true;     \
    } while(0);

#define YY_USER_ACTION                                              \
    do                                                              \
    {                                                               \
        pp::Input* input = &yyextra->input;                         \
        pp::Input::Location* scanLoc = &yyextra->scanLoc;           \
        while ((scanLoc->sIndex < input->count()) &&                \
               (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \
        {                                                           \
            scanLoc->cIndex -= input->length(scanLoc->sIndex++);    \
            ++yyfileno; yylineno = 1;                               \
        }                                                           \
        yylloc->file = yyfileno;                                    \
        yylloc->line = yylineno;                                    \
        scanLoc->cIndex += yyleng;                                  \
    } while(0);

#define YY_INPUT(buf, result, maxSize) \
    result = yyextra->input.read(buf, maxSize);

%}

%option noyywrap nounput never-interactive
%option reentrant bison-bridge bison-locations
%option prefix="pp"
%option extra-type="pp::Tokenizer::Context*"
%x COMMENT

NEWLINE     \n|\r|\r\n
IDENTIFIER  [_a-zA-Z][_a-zA-Z0-9]*
PUNCTUATOR  [][<>(){}.+-/*%^|&~=!:;,?]

DECIMAL_CONSTANT      [1-9][0-9]*
OCTAL_CONSTANT        0[0-7]*
HEXADECIMAL_CONSTANT  0[xX][0-9a-fA-F]+

DIGIT                [0-9]
EXPONENT_PART        [eE][+-]?{DIGIT}+
FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")

%%

    /* Line comment */
"//"[^\r\n]*

    /* Block comment */
    /* Line breaks are just counted - not returned. */
    /* The comment is replaced by a single space. */ 
"/*" { BEGIN(COMMENT); }
<COMMENT>[^*\r\n]+
<COMMENT>"*"
<COMMENT>{NEWLINE} { ++yylineno; }
<COMMENT>"*/" {
    yyextra->leadingSpace = true;
    BEGIN(INITIAL);
}

# {
    // # is only valid at start of line for preprocessor directives.
    yylval->assign(1, yytext[0]);
    return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER;
}

{IDENTIFIER} {
    yylval->assign(yytext, yyleng);
    return pp::Token::IDENTIFIER;
}

{DECIMAL_CONSTANT}|{OCTAL_CONSTANT}|{HEXADECIMAL_CONSTANT} {
    yylval->assign(yytext, yyleng);
    return pp::Token::CONST_INT;
}

({DIGIT}+{EXPONENT_PART})|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?) {
    yylval->assign(yytext, yyleng);
    return pp::Token::CONST_FLOAT;
}

    /* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */
    /* Rule to catch all invalid integers and floats. */
({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) {
    yylval->assign(yytext, yyleng);
    return pp::Token::PP_NUMBER;
}

"++" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_INC;
}
"--" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_DEC;
}
"<<" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_LEFT;
}
">>" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_RIGHT;
}
"<=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_LE;
}
">=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_GE;
}
"==" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_EQ;
}
"!=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_NE;
}
"&&" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_AND;
}
"^^" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_XOR;
}
"||" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_OR;
}
"+=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_ADD_ASSIGN;
}
"-=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_SUB_ASSIGN;
}
"*=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_MUL_ASSIGN;
}
"/=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_DIV_ASSIGN;
}
"%=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_MOD_ASSIGN;
}
"<<=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_LEFT_ASSIGN;
}
">>=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_RIGHT_ASSIGN;
}
"&=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_AND_ASSIGN;
}
"^=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_XOR_ASSIGN;
}
"|=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_OR_ASSIGN;
}

{PUNCTUATOR} {
    yylval->assign(1, yytext[0]);
    return yytext[0];
}

[ \t\v\f]+   { yyextra->leadingSpace = true; }

{NEWLINE} {
    ++yylineno;
    yylval->assign(1, '\n');
    return '\n';
}

. {
    yylval->assign(1, yytext[0]);
    return pp::Token::PP_OTHER;
}

<*><<EOF>> {
    // YY_USER_ACTION is not invoked for handling EOF.
    // Set the location for EOF token manually.
    pp::Input* input = &yyextra->input;
    pp::Input::Location* scanLoc = &yyextra->scanLoc;
    yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0;
    if (scanLoc->sIndex != sIndexMax)
    {
        // We can only reach here if there are empty strings at the
        // end of the input.
        scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;
        // FIXME: this is not 64-bit clean.
        yyfileno = static_cast<int>(sIndexMax); yylineno = 1;
    }
    yylloc->file = yyfileno;
    yylloc->line = yylineno;
    yylval->clear();

    if (YY_START == COMMENT)
    {
        yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
                                     pp::SourceLocation(yyfileno, yylineno),
                                     "");
    }
    yyterminate();
}

%%

namespace pp {

// TODO(alokp): Maximum token length should ideally be specified by
// the preprocessor client, i.e., the compiler.
const size_t Tokenizer::kMaxTokenLength = 256;

Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
{
    mContext.diagnostics = diagnostics;
}

Tokenizer::~Tokenizer()
{
    destroyScanner();
}

bool Tokenizer::init(size_t count, const char* const string[], const int length[])
{
    if ((count > 0) && (string == 0)) return false;

    mContext.input = Input(count, string, length);
    return initScanner();
}

void Tokenizer::setFileNumber(int file)
{
    // We use column number as file number.
    // See macro yyfileno.
    yyset_column(file, mHandle);
}

void Tokenizer::setLineNumber(int line)
{
    yyset_lineno(line, mHandle);
}

void Tokenizer::lex(Token* token)
{
    token->type = yylex(&token->text, &token->location, mHandle);
    if (token->text.size() > kMaxTokenLength)
    {
        mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG,
                                     token->location, token->text);
        token->text.erase(kMaxTokenLength);
    }

    token->flags = 0;

    token->setAtStartOfLine(mContext.lineStart);
    mContext.lineStart = token->type == '\n';

    token->setHasLeadingSpace(mContext.leadingSpace);
    mContext.leadingSpace = false;
}

bool Tokenizer::initScanner()
{
    if ((mHandle == NULL) && yylex_init_extra(&mContext, &mHandle))
        return false;

    yyrestart(0, mHandle);
    return true;
}

void Tokenizer::destroyScanner()
{
    if (mHandle == NULL)
        return;

    yylex_destroy(mHandle);
    mHandle = NULL;
}

}  // namespace pp