Source/devtools/front_end/SourceHTMLTokenizer.re2js - platform/external/chromium_org/third_party/WebKit - Git at Google

 /*
  * Copyright (C) 2009 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
  * copyright notice, this list of conditions and the following disclaimer
  * in the documentation and/or other materials provided with the
  * distribution.
  *     * Neither the name of Google Inc. nor the names of its
  * contributors may be used to endorse or promote products derived from
  * this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */

 // Generate js file as follows:
 //
 // re2c -isc devtools/front_end/SourceHTMLTokenizer.re2js \
 // | sed 's|^yy\([^:]*\)*\:|case \1:|' \
 // | sed 's|[*]cursor[+][+]|this._charAt(cursor++)|' \
 // | sed 's|[[*][+][+]cursor|this._charAt(++cursor)|' \
 // | sed 's|[*]cursor|this._charAt(cursor)|' \
 // | sed 's|yych = \*\([^;]*\)|yych = this._charAt\1|' \
 // | sed 's|goto case \([^;]*\)|{ gotoCase = \1; continue; }|' \
 // | sed 's|unsigned\ int|var|' \
 // | sed 's|var\ yych|case 1: var yych|' > devtools/front_end/SourceHTMLTokenizer.js

 /**
  * @constructor
  * @extends {WebInspector.SourceTokenizer}
  */
 WebInspector.SourceHTMLTokenizer = function()
 {
     WebInspector.SourceTokenizer.call(this);

     // The order is determined by the generated code.
     this._lexConditions = {
         INITIAL: 0,
         COMMENT: 1,
         DOCTYPE: 2,
         TAG: 3,
         DSTRING: 4,
         SSTRING: 5
     };
     this.case_INITIAL = 1000;
     this.case_COMMENT = 1001;
     this.case_DOCTYPE = 1002;
     this.case_TAG = 1003;
     this.case_DSTRING = 1004;
     this.case_SSTRING = 1005;

     this._parseConditions = {
         INITIAL: 0,
         ATTRIBUTE: 1,
         ATTRIBUTE_VALUE: 2,
         LINKIFY: 4,
         A_NODE: 8,
         SCRIPT: 16,
         STYLE: 32
     };

     this.condition = this.createInitialCondition();
 }

 WebInspector.SourceHTMLTokenizer.prototype = {
     createInitialCondition: function()
     {
         return { lexCondition: this._lexConditions.INITIAL, parseCondition: this._parseConditions.INITIAL };
     },

     set line(line) {
         if (this._condition.internalJavaScriptTokenizerCondition) {
             var match = /<\/script/i.exec(line);
             if (match) {
                 this._internalJavaScriptTokenizer.line = line.substring(0, match.index);
             } else
                 this._internalJavaScriptTokenizer.line = line;
         } else if (this._condition.internalCSSTokenizerCondition) {
             var match = /<\/style/i.exec(line);
             if (match) {
                 this._internalCSSTokenizer.line = line.substring(0, match.index);
             } else
                 this._internalCSSTokenizer.line = line;
         }
         this._line = line;
     },

     _isExpectingAttribute: function()
     {
         return this._condition.parseCondition & this._parseConditions.ATTRIBUTE;
     },

     _isExpectingAttributeValue: function()
     {
         return this._condition.parseCondition & this._parseConditions.ATTRIBUTE_VALUE;
     },

     _setExpectingAttribute: function()
     {
         if (this._isExpectingAttributeValue())
             this._condition.parseCondition ^= this._parseConditions.ATTRIBUTE_VALUE;
         this._condition.parseCondition |= this._parseConditions.ATTRIBUTE;
     },

     _setExpectingAttributeValue: function()
     {
         if (this._isExpectingAttribute())
             this._condition.parseCondition ^= this._parseConditions.ATTRIBUTE;
         this._condition.parseCondition |= this._parseConditions.ATTRIBUTE_VALUE;
     },

     /**
      * @param {boolean=} stringEnds
      */
     _stringToken: function(cursor, stringEnds)
     {
         if (!this._isExpectingAttributeValue()) {
             this.tokenType = null;
             return cursor;
         }
         this.tokenType = this._attrValueTokenType();
         if (stringEnds)
             this._setExpectingAttribute();
         return cursor;
     },

     _attrValueTokenType: function()
     {
         if (this._condition.parseCondition & this._parseConditions.LINKIFY) {
             if (this._condition.parseCondition & this._parseConditions.A_NODE)
                 return "html-external-link";
             return "html-resource-link";
         }
         return "html-attribute-value";
     },

     get _internalJavaScriptTokenizer()
     {
         return WebInspector.SourceTokenizer.Registry.getInstance().getTokenizer("text/javascript");
     },

     get _internalCSSTokenizer()
     {
         return WebInspector.SourceTokenizer.Registry.getInstance().getTokenizer("text/css");
     },

     scriptStarted: function(cursor)
     {
         this._condition.internalJavaScriptTokenizerCondition = this._internalJavaScriptTokenizer.createInitialCondition();
     },

     scriptEnded: function(cursor)
     {
     },

     styleSheetStarted: function(cursor)
     {
         this._condition.internalCSSTokenizerCondition = this._internalCSSTokenizer.createInitialCondition();
     },

     styleSheetEnded: function(cursor)
     {
     },

     nextToken: function(cursor)
     {
         if (this._condition.internalJavaScriptTokenizerCondition) {
             // Re-set line to force </script> detection first.
             this.line = this._line;
             if (cursor !== this._internalJavaScriptTokenizer._line.length) {
                 // Tokenizer is stateless, so restore its condition before tokenizing and save it after.
                 this._internalJavaScriptTokenizer.condition = this._condition.internalJavaScriptTokenizerCondition;
                 var result = this._internalJavaScriptTokenizer.nextToken(cursor);
                 this.tokenType = this._internalJavaScriptTokenizer.tokenType;
                 this._condition.internalJavaScriptTokenizerCondition = this._internalJavaScriptTokenizer.condition;
                 return result;
             } else if (cursor !== this._line.length)
                 delete this._condition.internalJavaScriptTokenizerCondition;
         } else if (this._condition.internalCSSTokenizerCondition) {
             // Re-set line to force </style> detection first.
             this.line = this._line;
             if (cursor !== this._internalCSSTokenizer._line.length) {
                 // Tokenizer is stateless, so restore its condition before tokenizing and save it after.
                 this._internalCSSTokenizer.condition = this._condition.internalCSSTokenizerCondition;
                 var result = this._internalCSSTokenizer.nextToken(cursor);
                 this.tokenType = this._internalCSSTokenizer.tokenType;
                 this._condition.internalCSSTokenizerCondition = this._internalCSSTokenizer.condition;
                 return result;
             } else if (cursor !== this._line.length)
                 delete this._condition.internalCSSTokenizerCondition;
         }

         var cursorOnEnter = cursor;
         var gotoCase = 1;
         var YYMARKER;
         while (1) {
             switch (gotoCase)
             // Following comment is replaced with generated state machine.
             /*!re2c
                 re2c:define:YYCTYPE  = "var";
                 re2c:define:YYCURSOR = cursor;
                 re2c:define:YYGETCONDITION = "this.getLexCondition";
                 re2c:define:YYSETCONDITION = "this.setLexCondition";
                 re2c:condprefix = "case this.case_";
                 re2c:condenumprefix = "this._lexConditions.";
                 re2c:yyfill:enable = 0;
                 re2c:labelprefix = "case ";
                 re2c:indent:top = 2;
                 re2c:indent:string = "    ";

                 CommentContent = ([^-\r\n] | ("--" [^>]))*;
                 Comment = "<!--" CommentContent "-->";
                 CommentStart = "<!--" CommentContent [\r\n];
                 CommentEnd = CommentContent "-->";

                 DocTypeStart = "<!" [Dd] [Oo] [Cc] [Tt] [Yy] [Pp] [Ee];
                 DocTypeContent = [^\r\n>]*;

                 ScriptStart = "<" [Ss] [Cc] [Rr] [Ii] [Pp] [Tt];
                 ScriptEnd = "</" [Ss] [Cc] [Rr] [Ii] [Pp] [Tt];

                 StyleStart = "<" [Ss] [Tt] [Yy] [Ll] [Ee];
                 StyleEnd = "</" [Ss] [Tt] [Yy] [Ll] [Ee];

                 LT = "<" | "</";
                 GT = ">";
                 EqualSign = "=";

                 DoubleStringContent = [^\r\n\"]*;
                 SingleStringContent = [^\r\n\']*;
                 StringLiteral = "\"" DoubleStringContent "\"" | "'" SingleStringContent "'";
                 DoubleStringStart = "\"" DoubleStringContent [\r\n];
                 DoubleStringEnd = DoubleStringContent "\"";
                 SingleStringStart = "'" SingleStringContent [\r\n];
                 SingleStringEnd = SingleStringContent "'";

                 Identifier = [^ \r\n"'<>\[\]=]+;

                 <INITIAL> Comment { this.tokenType = "html-comment"; return cursor; }
                 <INITIAL> CommentStart => COMMENT { this.tokenType = "html-comment"; return cursor; }
                 <COMMENT> CommentContent => COMMENT { this.tokenType = "html-comment"; return cursor; }
                 <COMMENT> CommentEnd => INITIAL { this.tokenType = "html-comment"; return cursor; }

                 <INITIAL> DocTypeStart => DOCTYPE { this.tokenType = "html-doctype"; return cursor; }
                 <DOCTYPE> DocTypeContent => DOCTYPE { this.tokenType = "html-doctype"; return cursor; }
                 <DOCTYPE> GT => INITIAL { this.tokenType = "html-doctype"; return cursor; }

                 <INITIAL> ScriptStart => TAG
                 {
                     if (this._condition.parseCondition & this._parseConditions.SCRIPT) {
                         // Do not tokenize script tag contents, keep lexer state, even though processing "<".
                         this.setLexCondition(this._lexConditions.INITIAL);
                         this.tokenType = null;
                         return cursor;
                     }
                     this.tokenType = "html-tag";
                     this._condition.parseCondition = this._parseConditions.SCRIPT;
                     this._setExpectingAttribute();
                     return cursor;
                 }

                 <INITIAL> ScriptEnd => TAG
                 {
                     this.tokenType = "html-tag";
                     this._condition.parseCondition = this._parseConditions.INITIAL;
                     this.scriptEnded(cursor - 8);
                     return cursor;
                 }

                 <INITIAL> StyleStart => TAG
                 {
                     if (this._condition.parseCondition & this._parseConditions.STYLE) {
                         // Do not tokenize style tag contents, keep lexer state, even though processing "<".
                         this.setLexCondition(this._lexConditions.INITIAL);
                         this.tokenType = null;
                         return cursor;
                     }
                     this.tokenType = "html-tag";
                     this._condition.parseCondition = this._parseConditions.STYLE;
                     this._setExpectingAttribute();
                     return cursor;
                 }

                 <INITIAL> StyleEnd => TAG
                 {
                     this.tokenType = "html-tag";
                     this._condition.parseCondition = this._parseConditions.INITIAL;
                     this.styleSheetEnded(cursor - 7);
                     return cursor;
                 }

                 <INITIAL> LT => TAG
                 {
                     if (this._condition.parseCondition & (this._parseConditions.SCRIPT | this._parseConditions.STYLE)) {
                         // Do not tokenize script and style tag contents, keep lexer state, even though processing "<".
                         this.setLexCondition(this._lexConditions.INITIAL);
                         this.tokenType = null;
                         return cursor;
                     }

                     this._condition.parseCondition = this._parseConditions.INITIAL;
                     this.tokenType = "html-tag";
                     return cursor;
                 }

                 <TAG> GT => INITIAL
                 {
                     this.tokenType = "html-tag";
                     if (this._condition.parseCondition & this._parseConditions.SCRIPT) {
                         this.scriptStarted(cursor);
                         // Do not tokenize script tag contents.
                         return cursor;
                     }

                     if (this._condition.parseCondition & this._parseConditions.STYLE) {
                         this.styleSheetStarted(cursor);
                         // Do not tokenize style tag contents.
                         return cursor;
                     }

                     this._condition.parseCondition = this._parseConditions.INITIAL;
                     return cursor;
                 }

                 <TAG> StringLiteral { return this._stringToken(cursor, true); }
                 <TAG> DoubleStringStart => DSTRING { return this._stringToken(cursor); }
                 <DSTRING> DoubleStringContent => DSTRING { return this._stringToken(cursor); }
                 <DSTRING> DoubleStringEnd => TAG { return this._stringToken(cursor, true); }
                 <TAG> SingleStringStart => SSTRING { return this._stringToken(cursor); }
                 <SSTRING> SingleStringContent => SSTRING { return this._stringToken(cursor); }
                 <SSTRING> SingleStringEnd => TAG { return this._stringToken(cursor, true); }

                 <TAG> EqualSign => TAG
                 {
                     if (this._isExpectingAttribute())
                         this._setExpectingAttributeValue();
                     this.tokenType = null;
                     return cursor;
                 }

                 <TAG> Identifier
                 {
                     if (this._condition.parseCondition === this._parseConditions.SCRIPT || this._condition.parseCondition === this._parseConditions.STYLE) {
                         // Fall through if expecting attributes.
                         this.tokenType = null;
                         return cursor;
                     }

                     if (this._condition.parseCondition === this._parseConditions.INITIAL) {
                         this.tokenType = "html-tag";
                         this._setExpectingAttribute();
                         var token = this._line.substring(cursorOnEnter, cursor);
                         if (token === "a")
                             this._condition.parseCondition |= this._parseConditions.A_NODE;
                         else if (this._condition.parseCondition & this._parseConditions.A_NODE)
                             this._condition.parseCondition ^= this._parseConditions.A_NODE;
                     } else if (this._isExpectingAttribute()) {
                         var token = this._line.substring(cursorOnEnter, cursor);
                         if (token === "href" || token === "src")
                             this._condition.parseCondition |= this._parseConditions.LINKIFY;
                         else if (this._condition.parseCondition |= this._parseConditions.LINKIFY)
                             this._condition.parseCondition ^= this._parseConditions.LINKIFY;
                         this.tokenType = "html-attribute-name";
                     } else if (this._isExpectingAttributeValue())
                         this.tokenType = this._attrValueTokenType();
                     else
                         this.tokenType = null;
                     return cursor;
                 }
                 <*> [^] { this.tokenType = null; return cursor; }
             */
         }
     },

     __proto__: WebInspector.SourceTokenizer.prototype
 }
	/*
	* Copyright (C) 2009 Google Inc. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions are
	* met:
	*
	* * Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* * Redistributions in binary form must reproduce the above
	* copyright notice, this list of conditions and the following disclaimer
	* in the documentation and/or other materials provided with the
	* distribution.
	* * Neither the name of Google Inc. nor the names of its
	* contributors may be used to endorse or promote products derived from
	* this software without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/

	// Generate js file as follows:
	//
	// re2c -isc devtools/front_end/SourceHTMLTokenizer.re2js \
	// \| sed 's\|^yy\([^:]\)\:\|case \1:\|' \
	// \| sed 's\|[*]cursor[+][+]\|this._charAt(cursor++)\|' \
	// \| sed 's\|[[*][+][+]cursor\|this._charAt(++cursor)\|' \
	// \| sed 's\|[*]cursor\|this._charAt(cursor)\|' \
	// \| sed 's\|yych = \\([^;]\)\|yych = this._charAt\1\|' \
	// \| sed 's\|goto case \([^;]*\)\|{ gotoCase = \1; continue; }\|' \
	// \| sed 's\|unsigned\ int\|var\|' \
	// \| sed 's\|var\ yych\|case 1: var yych\|' > devtools/front_end/SourceHTMLTokenizer.js

	/**
	* @constructor
	* @extends {WebInspector.SourceTokenizer}
	*/
	WebInspector.SourceHTMLTokenizer = function()
	{
	WebInspector.SourceTokenizer.call(this);

	// The order is determined by the generated code.
	this._lexConditions = {
	INITIAL: 0,
	COMMENT: 1,
	DOCTYPE: 2,
	TAG: 3,
	DSTRING: 4,
	SSTRING: 5
	};
	this.case_INITIAL = 1000;
	this.case_COMMENT = 1001;
	this.case_DOCTYPE = 1002;
	this.case_TAG = 1003;
	this.case_DSTRING = 1004;
	this.case_SSTRING = 1005;

	this._parseConditions = {
	INITIAL: 0,
	ATTRIBUTE: 1,
	ATTRIBUTE_VALUE: 2,
	LINKIFY: 4,
	A_NODE: 8,
	SCRIPT: 16,
	STYLE: 32
	};

	this.condition = this.createInitialCondition();
	}

	WebInspector.SourceHTMLTokenizer.prototype = {
	createInitialCondition: function()
	{
	return { lexCondition: this._lexConditions.INITIAL, parseCondition: this._parseConditions.INITIAL };
	},

	set line(line) {
	if (this._condition.internalJavaScriptTokenizerCondition) {
	var match = /<\/script/i.exec(line);
	if (match) {
	this._internalJavaScriptTokenizer.line = line.substring(0, match.index);
	} else
	this._internalJavaScriptTokenizer.line = line;
	} else if (this._condition.internalCSSTokenizerCondition) {
	var match = /<\/style/i.exec(line);
	if (match) {
	this._internalCSSTokenizer.line = line.substring(0, match.index);
	} else
	this._internalCSSTokenizer.line = line;
	}
	this._line = line;
	},

	_isExpectingAttribute: function()
	{
	return this._condition.parseCondition & this._parseConditions.ATTRIBUTE;
	},

	_isExpectingAttributeValue: function()
	{
	return this._condition.parseCondition & this._parseConditions.ATTRIBUTE_VALUE;
	},

	_setExpectingAttribute: function()
	{
	if (this._isExpectingAttributeValue())
	this._condition.parseCondition ^= this._parseConditions.ATTRIBUTE_VALUE;
	this._condition.parseCondition \|= this._parseConditions.ATTRIBUTE;
	},

	_setExpectingAttributeValue: function()
	{
	if (this._isExpectingAttribute())
	this._condition.parseCondition ^= this._parseConditions.ATTRIBUTE;
	this._condition.parseCondition \|= this._parseConditions.ATTRIBUTE_VALUE;
	},

	/**
	* @param {boolean=} stringEnds
	*/
	_stringToken: function(cursor, stringEnds)
	{
	if (!this._isExpectingAttributeValue()) {
	this.tokenType = null;
	return cursor;
	}
	this.tokenType = this._attrValueTokenType();
	if (stringEnds)
	this._setExpectingAttribute();
	return cursor;
	},

	_attrValueTokenType: function()
	{
	if (this._condition.parseCondition & this._parseConditions.LINKIFY) {
	if (this._condition.parseCondition & this._parseConditions.A_NODE)
	return "html-external-link";
	return "html-resource-link";
	}
	return "html-attribute-value";
	},

	get _internalJavaScriptTokenizer()
	{
	return WebInspector.SourceTokenizer.Registry.getInstance().getTokenizer("text/javascript");
	},

	get _internalCSSTokenizer()
	{
	return WebInspector.SourceTokenizer.Registry.getInstance().getTokenizer("text/css");
	},

	scriptStarted: function(cursor)
	{
	this._condition.internalJavaScriptTokenizerCondition = this._internalJavaScriptTokenizer.createInitialCondition();
	},

	scriptEnded: function(cursor)
	{
	},

	styleSheetStarted: function(cursor)
	{
	this._condition.internalCSSTokenizerCondition = this._internalCSSTokenizer.createInitialCondition();
	},

	styleSheetEnded: function(cursor)
	{
	},

	nextToken: function(cursor)
	{
	if (this._condition.internalJavaScriptTokenizerCondition) {
	// Re-set line to force </script> detection first.
	this.line = this._line;
	if (cursor !== this._internalJavaScriptTokenizer._line.length) {
	// Tokenizer is stateless, so restore its condition before tokenizing and save it after.
	this._internalJavaScriptTokenizer.condition = this._condition.internalJavaScriptTokenizerCondition;
	var result = this._internalJavaScriptTokenizer.nextToken(cursor);
	this.tokenType = this._internalJavaScriptTokenizer.tokenType;
	this._condition.internalJavaScriptTokenizerCondition = this._internalJavaScriptTokenizer.condition;
	return result;
	} else if (cursor !== this._line.length)
	delete this._condition.internalJavaScriptTokenizerCondition;
	} else if (this._condition.internalCSSTokenizerCondition) {
	// Re-set line to force </style> detection first.
	this.line = this._line;
	if (cursor !== this._internalCSSTokenizer._line.length) {
	// Tokenizer is stateless, so restore its condition before tokenizing and save it after.
	this._internalCSSTokenizer.condition = this._condition.internalCSSTokenizerCondition;
	var result = this._internalCSSTokenizer.nextToken(cursor);
	this.tokenType = this._internalCSSTokenizer.tokenType;
	this._condition.internalCSSTokenizerCondition = this._internalCSSTokenizer.condition;
	return result;
	} else if (cursor !== this._line.length)
	delete this._condition.internalCSSTokenizerCondition;
	}

	var cursorOnEnter = cursor;
	var gotoCase = 1;
	var YYMARKER;
	while (1) {
	switch (gotoCase)
	// Following comment is replaced with generated state machine.
	/*!re2c
	re2c:define:YYCTYPE = "var";
	re2c:define:YYCURSOR = cursor;
	re2c:define:YYGETCONDITION = "this.getLexCondition";
	re2c:define:YYSETCONDITION = "this.setLexCondition";
	re2c:condprefix = "case this.case_";
	re2c:condenumprefix = "this._lexConditions.";
	re2c:yyfill:enable = 0;
	re2c:labelprefix = "case ";
	re2c:indent:top = 2;
	re2c:indent:string = " ";

	CommentContent = ([^-\r\n] \| ("--" [^>]))*;
	Comment = "<!--" CommentContent "-->";
	CommentStart = "<!--" CommentContent [\r\n];
	CommentEnd = CommentContent "-->";

	DocTypeStart = "<!" [Dd] [Oo] [Cc] [Tt] [Yy] [Pp] [Ee];
	DocTypeContent = [^\r\n>]*;

	ScriptStart = "<" [Ss] [Cc] [Rr] [Ii] [Pp] [Tt];
	ScriptEnd = "</" [Ss] [Cc] [Rr] [Ii] [Pp] [Tt];

	StyleStart = "<" [Ss] [Tt] [Yy] [Ll] [Ee];
	StyleEnd = "</" [Ss] [Tt] [Yy] [Ll] [Ee];

	LT = "<" \| "</";
	GT = ">";
	EqualSign = "=";

	DoubleStringContent = [^\r\n\"]*;
	SingleStringContent = [^\r\n\']*;
	StringLiteral = "\"" DoubleStringContent "\"" \| "'" SingleStringContent "'";
	DoubleStringStart = "\"" DoubleStringContent [\r\n];
	DoubleStringEnd = DoubleStringContent "\"";
	SingleStringStart = "'" SingleStringContent [\r\n];
	SingleStringEnd = SingleStringContent "'";

	Identifier = [^ \r\n"'<>\[\]=]+;

	<INITIAL> Comment { this.tokenType = "html-comment"; return cursor; }
	<INITIAL> CommentStart => COMMENT { this.tokenType = "html-comment"; return cursor; }
	<COMMENT> CommentContent => COMMENT { this.tokenType = "html-comment"; return cursor; }
	<COMMENT> CommentEnd => INITIAL { this.tokenType = "html-comment"; return cursor; }

	<INITIAL> DocTypeStart => DOCTYPE { this.tokenType = "html-doctype"; return cursor; }
	<DOCTYPE> DocTypeContent => DOCTYPE { this.tokenType = "html-doctype"; return cursor; }
	<DOCTYPE> GT => INITIAL { this.tokenType = "html-doctype"; return cursor; }

	<INITIAL> ScriptStart => TAG
	{
	if (this._condition.parseCondition & this._parseConditions.SCRIPT) {
	// Do not tokenize script tag contents, keep lexer state, even though processing "<".
	this.setLexCondition(this._lexConditions.INITIAL);
	this.tokenType = null;
	return cursor;
	}
	this.tokenType = "html-tag";
	this._condition.parseCondition = this._parseConditions.SCRIPT;
	this._setExpectingAttribute();
	return cursor;
	}

	<INITIAL> ScriptEnd => TAG
	{
	this.tokenType = "html-tag";
	this._condition.parseCondition = this._parseConditions.INITIAL;
	this.scriptEnded(cursor - 8);
	return cursor;
	}

	<INITIAL> StyleStart => TAG
	{
	if (this._condition.parseCondition & this._parseConditions.STYLE) {
	// Do not tokenize style tag contents, keep lexer state, even though processing "<".
	this.setLexCondition(this._lexConditions.INITIAL);
	this.tokenType = null;
	return cursor;
	}
	this.tokenType = "html-tag";
	this._condition.parseCondition = this._parseConditions.STYLE;
	this._setExpectingAttribute();
	return cursor;
	}

	<INITIAL> StyleEnd => TAG
	{
	this.tokenType = "html-tag";
	this._condition.parseCondition = this._parseConditions.INITIAL;
	this.styleSheetEnded(cursor - 7);
	return cursor;
	}

	<INITIAL> LT => TAG
	{
	if (this._condition.parseCondition & (this._parseConditions.SCRIPT \| this._parseConditions.STYLE)) {
	// Do not tokenize script and style tag contents, keep lexer state, even though processing "<".
	this.setLexCondition(this._lexConditions.INITIAL);
	this.tokenType = null;
	return cursor;
	}

	this._condition.parseCondition = this._parseConditions.INITIAL;
	this.tokenType = "html-tag";
	return cursor;
	}

	<TAG> GT => INITIAL
	{
	this.tokenType = "html-tag";
	if (this._condition.parseCondition & this._parseConditions.SCRIPT) {
	this.scriptStarted(cursor);
	// Do not tokenize script tag contents.
	return cursor;
	}

	if (this._condition.parseCondition & this._parseConditions.STYLE) {
	this.styleSheetStarted(cursor);
	// Do not tokenize style tag contents.
	return cursor;
	}

	this._condition.parseCondition = this._parseConditions.INITIAL;
	return cursor;
	}

	<TAG> StringLiteral { return this._stringToken(cursor, true); }
	<TAG> DoubleStringStart => DSTRING { return this._stringToken(cursor); }
	<DSTRING> DoubleStringContent => DSTRING { return this._stringToken(cursor); }
	<DSTRING> DoubleStringEnd => TAG { return this._stringToken(cursor, true); }
	<TAG> SingleStringStart => SSTRING { return this._stringToken(cursor); }
	<SSTRING> SingleStringContent => SSTRING { return this._stringToken(cursor); }
	<SSTRING> SingleStringEnd => TAG { return this._stringToken(cursor, true); }

	<TAG> EqualSign => TAG
	{
	if (this._isExpectingAttribute())
	this._setExpectingAttributeValue();
	this.tokenType = null;
	return cursor;
	}

	<TAG> Identifier
	{
	if (this._condition.parseCondition === this._parseConditions.SCRIPT \|\| this._condition.parseCondition === this._parseConditions.STYLE) {
	// Fall through if expecting attributes.
	this.tokenType = null;
	return cursor;
	}

	if (this._condition.parseCondition === this._parseConditions.INITIAL) {
	this.tokenType = "html-tag";
	this._setExpectingAttribute();
	var token = this._line.substring(cursorOnEnter, cursor);
	if (token === "a")
	this._condition.parseCondition \|= this._parseConditions.A_NODE;
	else if (this._condition.parseCondition & this._parseConditions.A_NODE)
	this._condition.parseCondition ^= this._parseConditions.A_NODE;
	} else if (this._isExpectingAttribute()) {
	var token = this._line.substring(cursorOnEnter, cursor);
	if (token === "href" \|\| token === "src")
	this._condition.parseCondition \|= this._parseConditions.LINKIFY;
	else if (this._condition.parseCondition \|= this._parseConditions.LINKIFY)
	this._condition.parseCondition ^= this._parseConditions.LINKIFY;
	this.tokenType = "html-attribute-name";
	} else if (this._isExpectingAttributeValue())
	this.tokenType = this._attrValueTokenType();
	else
	this.tokenType = null;
	return cursor;
	}
	<*> [^] { this.tokenType = null; return cursor; }
	*/
	}
	},

	__proto__: WebInspector.SourceTokenizer.prototype
	}