python/testSrc/com/jetbrains/python/PythonHighlightingLexerTest.java

/*
 * Copyright 2000-2013 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.jetbrains.python;

import com.jetbrains.python.fixtures.PyLexerTestCase;
import com.jetbrains.python.lexer.PythonHighlightingLexer;
import com.jetbrains.python.psi.LanguageLevel;

/**
 * user : catherine
 */
public class PythonHighlightingLexerTest extends PyLexerTestCase {

  public void testFromFutureUnicode() {
    doTest(LanguageLevel.PYTHON26, "from __future__ import unicode_literals\n\n" +
                                   "s = \"some string\"",
           "Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE", "Py:IMPORT_KEYWORD", "Py:SPACE", "Py:IDENTIFIER",
           "Py:LINE_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_UNICODE");
  }

  public void testFromFutureUnicodeDocstring() {
    doTest(LanguageLevel.PYTHON26, "\"\"\"docstring\"\"\"\n" +
                                   "from __future__ import unicode_literals\n\n" +
                                   "s = \"some string\"",
           "Py:DOCSTRING", "Py:LINE_BREAK",
           "Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE", "Py:IMPORT_KEYWORD", "Py:SPACE", "Py:IDENTIFIER",
           "Py:LINE_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_UNICODE");
  }

  public void testFromFutureUnicodeWithBackslash() {
    doTest(LanguageLevel.PYTHON26, "from __future__ \\\nimport unicode_literals\n\n" +
                                   "s = \"some string\"",
           "Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE", "Py:BACKSLASH", "Py:LINE_BREAK",
           "Py:IMPORT_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:LINE_BREAK",
           "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_UNICODE");
  }

  public void testFromFutureUnicodeWithBrace() {
    doTest(LanguageLevel.PYTHON26, "from __future__ import (unicode_literals)\n\n" +
                                   "s = \"some string\"",
           "Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE",
           "Py:IMPORT_KEYWORD", "Py:SPACE", "Py:LPAR", "Py:IDENTIFIER", "Py:RPAR", "Py:LINE_BREAK",
           "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_UNICODE");
  }

  public void testFromFutureUnicodeComment() {
    doTest(LanguageLevel.PYTHON26, "#one comment\n" +
                                   "from __future__ import ((unicode_literals))\n" +
                                   "s = \"some string\"",
           "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK", "Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE",
           "Py:IMPORT_KEYWORD", "Py:SPACE", "Py:LPAR", "Py:LPAR", "Py:IDENTIFIER", "Py:RPAR", "Py:RPAR",
           "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING");
  }

  public void testFromFutureBytes() {
    doTest(LanguageLevel.PYTHON26, "from __future__ import unicode_literals\n" +
                                   "s = b\"some string\"",
           "Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE",
           "Py:IMPORT_KEYWORD", "Py:SPACE", "Py:IDENTIFIER",
           "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING");
  }

  public void testBytes30() {
    doTest(LanguageLevel.PYTHON30, "s = b\"some string\"",
                            "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING");
  }

  public void testBytes() {
    doTest(LanguageLevel.PYTHON26, "s = b\"some string\"",
                            "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING");
  }

  public void testFromFutureUnicodeWithBraceFail() {
    doTest(LanguageLevel.PYTHON26, "from __future__ import ((unicode_literals))\n" +
                                   "s = \"some string\"",
           "Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE", "Py:IMPORT_KEYWORD", "Py:SPACE",
           "Py:LPAR", "Py:LPAR", "Py:IDENTIFIER", "Py:RPAR", "Py:RPAR",
           "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING");
  }

  public void testDoubleFromFutureUnicode() {
    doTest(LanguageLevel.PYTHON26, "from __future__ import absolute_import\n" +
                                   "from __future__ import (unicode_literals)\n\n" +
                                   "s = \"some string\"",
           "Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE", "Py:IMPORT_KEYWORD", "Py:SPACE", "Py:IDENTIFIER",
           "Py:LINE_BREAK", "Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE",
           "Py:IMPORT_KEYWORD", "Py:SPACE", "Py:LPAR", "Py:IDENTIFIER", "Py:RPAR", "Py:LINE_BREAK",
           "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_UNICODE");
  }

  public void testDoubleFromFutureUnicodeWithComma() {
    doTest(LanguageLevel.PYTHON26, "from __future__ import absolute_import, unicode_literals\n" +
                                   "s = \"some string\"",
           "Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE", "Py:IMPORT_KEYWORD", "Py:SPACE", "Py:IDENTIFIER",
           "Py:COMMA", "Py:SPACE", "Py:IDENTIFIER",
           "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_UNICODE");
  }

  public void testDoubleFromFutureUnicodeWithCommaFail() {
    doTest(LanguageLevel.PYTHON26, "from __future__ import absolute_import, (unicode_literals)\n" +
                                   "s = \"some string\"",
           "Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE", "Py:IMPORT_KEYWORD", "Py:SPACE", "Py:IDENTIFIER",
           "Py:COMMA", "Py:SPACE", "Py:LPAR", "Py:IDENTIFIER", "Py:RPAR",
           "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING");
  }

  public void testFromFutureUnicodeFail() {
    doTest(LanguageLevel.PYTHON26, "a = 2\n" +
                                   "from __future__ import unicode_literals\n\n" +
                                   "s = \"some string\"",
           "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE","Py:INTEGER_LITERAL", "Py:LINE_BREAK",
           "Py:FROM_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE", "Py:IMPORT_KEYWORD", "Py:SPACE", "Py:IDENTIFIER",
           "Py:LINE_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING");
  }

  public void testUnicode30() {
    doTest(LanguageLevel.PYTHON30, "s = \"some string\"",
                      "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_UNICODE");
  }

  public void testString() {
    doTest(LanguageLevel.PYTHON27, "s = \"some string\"",
                      "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING");
  }

  public void testUnicode() {
    doTest(LanguageLevel.PYTHON27, "s = u\"some string\"",
                      "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_UNICODE");
  }

  public void testDocString() {
    doTest(LanguageLevel.PYTHON27, "\"\"\"one docstrings \"\"\"\n",
                      "Py:DOCSTRING", "Py:LINE_BREAK");
  }

  public void testSingleDocStringWithBackslash() {
    doTest(LanguageLevel.PYTHON27, "\"one docstring \" \\\n\"new line of docstring\"\n",
                      "Py:DOCSTRING", "Py:SPACE", "Py:BACKSLASH", "Py:LINE_BREAK", "Py:DOCSTRING", "Py:LINE_BREAK");
  }

  public void testSingleDocstringFunction() {
    doTest(LanguageLevel.PYTHON27, "def foo():\n" +
                                   "  \"\"\"function foo\"\"\"\n" +
                                   "  a = \"string\"",
                      "Py:DEF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:LPAR", "Py:RPAR", "Py:COLON", "Py:LINE_BREAK",
                      "Py:SPACE", "Py:SPACE", "Py:DOCSTRING", "Py:LINE_BREAK", "Py:SPACE", "Py:SPACE", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE",
                      "Py:SINGLE_QUOTED_STRING");
  }

  public void testNotDocstring() { // PY-4481
      doTest(LanguageLevel.PYTHON27, "d = {\n" +
                                     " 'abc': 'def',\n" +
                                     " 'ghi': 'jkl'\n" +
                                     " }",
             "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:LBRACE", "Py:LINE_BREAK", "Py:SPACE", "Py:SINGLE_QUOTED_STRING",
             "Py:COLON", "Py:SPACE", "Py:SINGLE_QUOTED_STRING", "Py:COMMA", "Py:LINE_BREAK", "Py:SPACE", "Py:SINGLE_QUOTED_STRING",
             "Py:COLON", "Py:SPACE", "Py:SINGLE_QUOTED_STRING", "Py:LINE_BREAK", "Py:SPACE", "Py:RBRACE");
    }

  private static void doTest(LanguageLevel languageLevel, String text, String... expectedTokens) {
    PyLexerTestCase.doLexerTest(text, new PythonHighlightingLexer(languageLevel), expectedTokens);
  }
}