Source/devtools/scripts/compile_frontend.py

#!/usr/bin/env python
# Copyright (c) 2012 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os
import os.path as path
import generate_protocol_externs
import re
import shutil
import subprocess
import sys
import tempfile
try:
    import json
except ImportError:
    import simplejson as json

scripts_path = path.dirname(path.abspath(__file__))
devtools_path = path.dirname(scripts_path)
inspector_path = path.join(path.dirname(devtools_path), "core", "inspector")
devtools_frontend_path = path.join(devtools_path, "front_end")
global_externs_file = path.join(devtools_frontend_path, "externs.js")
protocol_externs_file = path.join(devtools_frontend_path, "protocol_externs.js")
webgl_rendering_context_idl_path = path.join(path.dirname(devtools_path), "core", "html", "canvas", "WebGLRenderingContextBase.idl")
injected_script_source_name = path.join(inspector_path, "InjectedScriptSource.js")
canvas_injected_script_source_name = path.join(inspector_path, "InjectedScriptCanvasModuleSource.js")
closure_compiler_jar = path.join(scripts_path, "closure", "compiler.jar")
closure_runner_jar = path.join(scripts_path, "compiler-runner", "closure-runner.jar")
jsdoc_validator_jar = path.join(scripts_path, "jsdoc-validator", "jsdoc-validator.jar")
java_exec = "java -Xms1024m -server -XX:+TieredCompilation"

generate_protocol_externs.generate_protocol_externs(protocol_externs_file, path.join(devtools_path, "protocol.json"))

jsmodule_name_prefix = "jsmodule_"
js_modules_name = "frontend_modules.json"

try:
    with open(path.join(scripts_path, js_modules_name), "rt") as js_modules_file:
        modules = json.loads(js_modules_file.read())
except:
    print "ERROR: Failed to read %s" % js_modules_name
    raise

# `importScript` function must not be used in any files
# except module headers. Refer to devtools.gyp file for
# the module header list.
allowed_import_statements_files = [
    "search/AdvancedSearchView.js",
    "console/ConsolePanel.js",
    "elements/ElementsPanel.js",
    "resources/ResourcesPanel.js",
    "network/NetworkPanel.js",
    "settings/SettingsScreen.js",
    "sources/SourcesPanel.js",
    "timeline/TimelinePanel.js",
    "profiler/ProfilesPanel.js",
    "audits/AuditsPanel.js",
    "layers/LayersPanel.js",
    "extensions/ExtensionServer.js",
    "source_frame/SourceFrame.js",
]

type_checked_jsdoc_tags_list = ["param", "return", "type", "enum"]

type_checked_jsdoc_tags_or = "|".join(type_checked_jsdoc_tags_list)

# Basic regex for invalid JsDoc types: an object type name ([A-Z][A-Za-z0-9.]+[A-Za-z0-9]) not preceded by '!', '?', ':' (this, new), or '.' (object property).
invalid_type_regex = re.compile(r"@(?:" + type_checked_jsdoc_tags_or + r")\s*\{.*(?<![!?:.A-Za-z0-9])([A-Z][A-Za-z0-9.]+[A-Za-z0-9])[^/]*\}")

invalid_type_designator_regex = re.compile(r"@(?:" + type_checked_jsdoc_tags_or + r")\s*.*(?<![{: ])([?!])=?\}")

importscript_regex = re.compile(r"importScript\(\s*[\"']")
error_warning_regex = re.compile(r"(?:WARNING|ERROR)")

errors_found = False


def hasErrors(output):
    return re.search(error_warning_regex, output) != None


def verify_importScript_usage():
    errors_found = False
    for module in modules:
        for file_name in module['sources']:
            if file_name in allowed_import_statements_files:
                continue
            try:
                with open(path.join(devtools_frontend_path, file_name), "r") as sourceFile:
                    source = sourceFile.read()
                    if re.search(importscript_regex, source):
                        print "ERROR: importScript function call is allowed in module header files only (found in %s)" % file_name
                        errors_found = True
            except:
                print "ERROR: Failed to access %s" % file_name
                raise
    return errors_found


def dump_all_checked_files():
    files = {}
    for module in modules:
        for source in module["sources"]:
            files[path.join(devtools_frontend_path, source)] = True
    return " ".join(files.keys())


def verify_jsdoc_extra():
    return subprocess.Popen("%s -jar %s %s" % (java_exec, jsdoc_validator_jar, dump_all_checked_files()), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)


def verify_jsdoc():
    errors_found = False
    for module in modules:
        for file_name in module["sources"]:
            lineIndex = 0
            full_file_name = path.join(devtools_frontend_path, file_name)
            with open(full_file_name, "r") as sourceFile:
                for line in sourceFile:
                    line = line.rstrip()
                    lineIndex += 1
                    if not line:
                        continue
                    if verify_jsdoc_line(full_file_name, lineIndex, line):
                        errors_found = True
    return errors_found


def verify_jsdoc_line(fileName, lineIndex, line):
    def print_error(message, errorPosition):
        print "%s:%s: ERROR - %s\n%s\n%s\n" % (fileName, lineIndex, message, line, " " * errorPosition + "^")

    errors_found = False
    match = re.search(invalid_type_regex, line)
    if match:
        print_error("Type '%s' nullability not marked explicitly with '?' (nullable) or '!' (non-nullable)" % match.group(1), match.start(1))
        errors_found = True

    match = re.search(invalid_type_designator_regex, line)
    if (match):
        print_error("Type nullability indicator misplaced, should precede type", match.start(1))
        errors_found = True
    return errors_found


def check_java_path():
    proc = subprocess.Popen("which java", stdout=subprocess.PIPE, shell=True)
    (javaPath, _) = proc.communicate()

    if proc.returncode != 0:
        print "Cannot find java ('which java' return code = %d, should be 0)" % proc.returncode
        sys.exit(1)
    print "Java executable: " + re.sub(r"\n$", "", javaPath)

check_java_path()

print "Verifying 'importScript' function usage..."
errors_found |= verify_importScript_usage()

print "Verifying JSDoc comments..."
errors_found |= verify_jsdoc()
jsdocValidatorProc = verify_jsdoc_extra()

modules_dir = tempfile.mkdtemp()
common_closure_args = " --summary_detail_level 3 --compilation_level SIMPLE_OPTIMIZATIONS --warning_level VERBOSE --language_in ECMASCRIPT5 --accept_const_keyword --module_output_path_prefix %s/" % modules_dir

spawned_compiler_command = "%s -jar %s %s \\\n" % (java_exec, closure_compiler_jar, common_closure_args)

modules_by_name = {}
standalone_modules_by_name = {}
dependents_by_module_name = {}

for module in modules:
    name = module["name"]
    modules_by_name[name] = module
    if "standalone" in module:
        standalone_modules_by_name[name] = module
    for dep in module["dependencies"]:
        list = dependents_by_module_name.get(dep)
        if not list:
            list = []
            dependents_by_module_name[dep] = list
        list.append(name)


def verify_standalone_modules():
    for module in modules:
        for dependency in module["dependencies"]:
            if dependency in standalone_modules_by_name:
                print "ERROR: Standalone module %s may not be present among the dependencies of %s" % (dependency, module["name"])
                errors_found = True

verify_standalone_modules()


def check_duplicate_files():

    def check_module(module, seen_files, seen_modules):
        name = module["name"]
        seen_modules[name] = True
        for dep_name in module["dependencies"]:
            if not dep_name in seen_modules:
                check_module(modules_by_name[dep_name], seen_files, seen_modules)
        for source in module["sources"]:
            referencing_module = seen_files.get(source)
            if referencing_module:
                print "ERROR: Duplicate use of %s in '%s' (previously seen in '%s')" % (source, name, referencing_module)
            seen_files[source] = name

    for module_name in standalone_modules_by_name:
        check_module(standalone_modules_by_name[module_name], {}, {})

print "Checking duplicate files across modules..."
check_duplicate_files()


def dump_module(name, recursively, processed_modules):
    if name in processed_modules:
        return ""
    processed_modules[name] = True
    module = modules_by_name[name]
    command = ""
    if recursively:
        for dependency in module["dependencies"]:
            command += dump_module(dependency, recursively, processed_modules)
    command += " --module " + jsmodule_name_prefix + module["name"] + ":"
    command += str(len(module["sources"]))
    firstDependency = True
    for dependency in module["dependencies"]:
        if firstDependency:
            command += ":"
        else:
            command += ","
        firstDependency = False
        command += jsmodule_name_prefix + dependency
    for script in module["sources"]:
        command += " --js " + path.join(devtools_frontend_path, script)
    return command

print "Compiling frontend..."

compiler_args_file = tempfile.NamedTemporaryFile(mode='wt', delete=False)
closure_runner_command = "%s -jar %s --compiler-args-file %s" % (java_exec, closure_runner_jar, compiler_args_file.name)

for module in modules:
    closure_args = common_closure_args
    closure_args += " --externs " + global_externs_file
    closure_args += " --externs " + protocol_externs_file
    closure_args += dump_module(module["name"], True, {})
    compiler_args_file.write("%s %s\n" % (module["name"], closure_args))

compiler_args_file.close()
modular_compiler_proc = subprocess.Popen(closure_runner_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)


def unclosure_injected_script(sourceFileName, outFileName):
    with open(sourceFileName, "r") as sourceFile:
        source = sourceFile.read()

    def replace_function(matchobj):
        return re.sub(r"@param", "param", matchobj.group(1) or "") + "\n//" + matchobj.group(2)

    # Comment out the closure function and its jsdocs
    source = re.sub(r"(/\*\*(?:[\s\n]*\*\s*@param[^\n]+\n)+\s*\*/\s*)?\n(\(function)", replace_function, source, count=1)

    # Comment out its return statement
    source = re.sub(r"\n(\s*return\s+[^;]+;\s*\n\}\)\s*)$", "\n/*\\1*/", source)

    # Replace the "var Object" override with a "self.Object" one
    source = re.sub(r"\nvar Object =", "\nself.Object =", source, count=1)

    with open(outFileName, "w") as outFileName:
        outFileName.write(source)

injectedScriptSourceTmpFile = path.join(inspector_path, "InjectedScriptSourceTmp.js")
injectedScriptCanvasModuleSourceTmpFile = path.join(inspector_path, "InjectedScriptCanvasModuleSourceTmp.js")

unclosure_injected_script(injected_script_source_name, injectedScriptSourceTmpFile)
unclosure_injected_script(canvas_injected_script_source_name, injectedScriptCanvasModuleSourceTmpFile)

print "Compiling InjectedScriptSource.js and InjectedScriptCanvasModuleSource.js..."
command = spawned_compiler_command
command += "    --externs " + path.join(inspector_path, "InjectedScriptExterns.js") + " \\\n"
command += "    --externs " + protocol_externs_file + " \\\n"
command += "    --module " + jsmodule_name_prefix + "injected_script" + ":1" + " \\\n"
command += "        --js " + injectedScriptSourceTmpFile + " \\\n"
command += "    --module " + jsmodule_name_prefix + "injected_canvas_script" + ":1:" + jsmodule_name_prefix + "injected_script" + " \\\n"
command += "        --js " + injectedScriptCanvasModuleSourceTmpFile + " \\\n"
command += "\n"

injectedScriptCompileProc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)

print "Checking generated code in InjectedScriptCanvasModuleSource.js..."
check_injected_webgl_calls_command = "%s/check_injected_webgl_calls_info.py %s %s" % (scripts_path, webgl_rendering_context_idl_path, canvas_injected_script_source_name)
canvasModuleCompileProc = subprocess.Popen(check_injected_webgl_calls_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)

print "Validating InjectedScriptSource.js..."
check_injected_script_command = "%s/check_injected_script_source.py %s" % (scripts_path, injected_script_source_name)
validateInjectedScriptProc = subprocess.Popen(check_injected_script_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)

print

(jsdocValidatorOut, _) = jsdocValidatorProc.communicate()
if jsdocValidatorOut:
    print ("JSDoc validator output:\n%s" % jsdocValidatorOut)
    errors_found = True

(moduleCompileOut, _) = modular_compiler_proc.communicate()
print "Modular compilation output:"

start_module_regex = re.compile(r"^@@ START_MODULE:(.+) @@$")
end_module_regex = re.compile(r"^@@ END_MODULE @@$")

in_module = False
skipped_modules = {}
error_count = 0

def skip_dependents(module_name):
    for skipped_module in dependents_by_module_name.get(module_name, []):
        skipped_modules[skipped_module] = True

# pylint: disable=E1103
for line in moduleCompileOut.splitlines():
    if not in_module:
        match = re.search(start_module_regex, line)
        if not match:
            continue
        in_module = True
        module_error_count = 0
        module_output = []
        module_name = match.group(1)
        skip_module = skipped_modules.get(module_name)
        if skip_module:
            skip_dependents(module_name)
    else:
        match = re.search(end_module_regex, line)
        if not match:
            if not skip_module:
                module_output.append(line)
                if hasErrors(line):
                    error_count += 1
                    module_error_count += 1
                    skip_dependents(module_name)
            continue

        in_module = False
        if skip_module:
            print "Skipping module %s..." % module_name
        elif not module_error_count:
            print "Module %s compiled successfully: %s" % (module_name, module_output[0])
        else:
            print "Module %s compile failed: %s errors\n" % (module_name, module_error_count)
            print os.linesep.join(module_output)

if error_count:
    print "Total Closure errors: %d\n" % error_count
    errors_found = True

(injectedScriptCompileOut, _) = injectedScriptCompileProc.communicate()
print "InjectedScriptSource.js and InjectedScriptCanvasModuleSource.js compilation output:\n", injectedScriptCompileOut
errors_found |= hasErrors(injectedScriptCompileOut)

(canvasModuleCompileOut, _) = canvasModuleCompileProc.communicate()
print "InjectedScriptCanvasModuleSource.js generated code check output:\n", canvasModuleCompileOut
errors_found |= hasErrors(canvasModuleCompileOut)

(validateInjectedScriptOut, _) = validateInjectedScriptProc.communicate()
print "Validate InjectedScriptSource.js output:\n", (validateInjectedScriptOut if validateInjectedScriptOut else "<empty>")
errors_found |= hasErrors(validateInjectedScriptOut)

if errors_found:
    print "ERRORS DETECTED"

os.remove(injectedScriptSourceTmpFile)
os.remove(injectedScriptCanvasModuleSourceTmpFile)
os.remove(compiler_args_file.name)
os.remove(protocol_externs_file)
shutil.rmtree(modules_dir, True)