[ci] use lintrunner in CI
This changes our lint workflows to use lintrunner for the linters that
are currently supported
+ some random fixes to make things lint clean on master
+ changes to Makefile to use lintrunner
Pull Request resolved: https://github.com/pytorch/pytorch/pull/68460
Approved by: https://github.com/t10-13rocket, https://github.com/seemethere, https://github.com/janeyx99
diff --git a/.github/scripts/convert_lintrunner_annotations_to_github.py b/.github/scripts/convert_lintrunner_annotations_to_github.py
new file mode 100644
index 0000000..11901bc
--- /dev/null
+++ b/.github/scripts/convert_lintrunner_annotations_to_github.py
@@ -0,0 +1,63 @@
+import json
+import subprocess
+import sys
+
+from enum import Enum
+from pathlib import Path
+from typing import NamedTuple, Optional
+
+# From: https://docs.github.com/en/rest/reference/checks
+class GitHubAnnotationLevel(str, Enum):
+ NOTICE = "notice"
+ WARNING = "warning"
+ FAILURE = "failure"
+
+
+class GitHubAnnotation(NamedTuple):
+ path: str
+ start_line: int
+ end_line: int
+ start_column: Optional[int]
+ end_column: Optional[int]
+ annotation_level: GitHubAnnotationLevel
+ message: str
+ title: Optional[str]
+ raw_details: Optional[str]
+
+PYTORCH_ROOT = Path(subprocess.check_output(['git', 'rev-parse', '--show-toplevel']).decode('ascii').strip())
+
+annotations = []
+for line in sys.stdin:
+ lint_message = json.loads(line)
+
+ path = lint_message.get("path")
+ line = lint_message.get("line")
+
+
+ code = lint_message["code"]
+ severity = lint_message["severity"]
+ name = lint_message["name"]
+ description = lint_message.get("description")
+
+ # These fields are required by the GitHub API, but optional in lintrunner.
+ # If they don't exist, just skip.
+ if path is None or line is None:
+ print(f"No path/line for lint: ({code}) {name}", file=sys.stderr)
+ continue
+
+ # normalize path relative to git root
+ path = Path(path).relative_to(PYTORCH_ROOT)
+
+ annotations.append(GitHubAnnotation(
+ path=str(path),
+ start_line=int(line),
+ end_line=int(line),
+ start_column=None,
+ end_column=None,
+ annotation_level=GitHubAnnotationLevel.FAILURE,
+ message=description,
+ title=f"({code}) {name}",
+ raw_details=None,
+ )._asdict())
+
+print(json.dumps(annotations), flush=True)
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index 228cd84..6a4dbbf 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -7,6 +7,72 @@
pull_request:
jobs:
+ lintrunner:
+ runs-on: ubuntu-18.04
+ steps:
+ - name: Setup Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: 3.8
+ architecture: x64
+
+ - name: Checkout PyTorch
+ uses: pytorch/pytorch/.github/actions/checkout-pytorch@master
+ with:
+ submodules: false
+
+ - name: Install lintrunner
+ run: pip install lintrunner==0.5.*
+
+ - name: Initialize lint dependencies
+ run: lintrunner init
+
+ - name: Do build steps necessary for linters
+ run: |
+ python3 -m tools.linter.clang_tidy.generate_build_files
+ python3 -m tools.generate_torch_version --is_debug=false
+ python3 -m tools.pyi.gen_pyi \
+ --native-functions-path aten/src/ATen/native/native_functions.yaml \
+ --deprecated-functions-path "tools/autograd/deprecated.yaml"
+
+ - name: Run lintrunner on all files
+ if: github.event_name == 'push'
+ run: lintrunner -vv --paths-cmd='git grep -Il .' --force-color
+
+ - name: Run lintrunner on PR files
+ if: github.event_name == 'pull_request'
+ env:
+ PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+ PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ run: |
+ MERGE_BASE=$(git merge-base "$PR_BASE_SHA" "$PR_HEAD_SHA")
+ lintrunner -vv --force-color --revision "${MERGE_BASE}"
+ echo ""
+ echo -e "\e[1m\e[36mYou can reproduce these results locally by using \`lintrunner\`.\e[0m"
+ echo -e "\e[1m\e[36mSee https://github.com/pytorch/pytorch/wiki/lintrunner for setup instructions.\e[0m"
+
+ - name: Store annotations
+ # Don't run on forked pull requests
+ if: failure() && github.event.pull_request.head.repo.full_name == github.repository
+ run: |
+ lintrunner --json \
+ | python .github/scripts/convert_lintrunner_annotations_to_github.py \
+ > annotations.json
+
+ cat annotations.json
+
+ - name: Add annotations
+ # Don't run on forked pull requests
+ if: failure() && github.event.pull_request.head.repo.full_name == github.repository
+ uses: pytorch/add-annotations-github-action@master
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ with:
+ check_name: 'lintrunner'
+ linter_output_path: annotations.json
+ commit_sha: ${{ github.event.pull_request.head.sha }}
+ mode: json
+
quick-checks:
name: quick-checks
runs-on: ubuntu-18.04
@@ -28,70 +94,16 @@
- name: Install requirements
id: requirements
run: pip3 install -r requirements.txt --user
- - name: Ensure consistent CircleCI YAML config
- if: ${{ always() && steps.requirements.outcome == 'success' }}
- run: cd .circleci && ./ensure-consistency.py
- - name: Lint native_functions.yaml
- if: ${{ always() && steps.requirements.outcome == 'success' }}
- run: |
- pip3 install ruamel.yaml==0.17.4 --user
- .github/scripts/lint_native_functions.py
- - name: Ensure correct trailing newlines
- if: ${{ always() && steps.requirements.outcome == 'success' }}
- run: |
- (! git --no-pager grep -Il '' -- . ':(exclude)**/contrib/**' ':(exclude)third_party' ':(exclude)**.expect' ':(exclude)**.ipynb' ':(exclude)tools/clang_format_hash' | tools/linter/trailing_newlines.py || (echo "The above files do not have correct trailing newlines; please normalize them"; false))
- - name: Ensure no trailing spaces
- if: always()
- run: |
- (! git --no-pager grep -In '[[:blank:]]$' -- . ':(exclude)**/contrib/**' ':(exclude)**.diff' ':(exclude)third_party' || (echo "The above lines have trailing spaces; please remove them"; false))
- - name: Ensure no tabs
- if: always()
- run: |
- (! git --no-pager grep -In $'\t' -- . ':(exclude)*.svg' ':(exclude)**Makefile' ':(exclude)**/contrib/**' ':(exclude)third_party' ':(exclude).gitattributes' ':(exclude).gitmodules' || (echo "The above lines have tabs; please convert them to spaces"; false))
- name: Ensure no non-breaking spaces
if: always()
run: |
# NB: We use 'printf' below rather than '\u000a' since bash pre-4.2
# does not support the '\u000a' syntax (which is relevant for local linters)
(! git --no-pager grep -In "$(printf '\xC2\xA0')" -- . || (echo "The above lines have non-breaking spaces (U+00A0); please convert them to spaces (U+0020)"; false))
- - name: Ensure canonical include
- if: always()
- run: |
- (! git --no-pager grep -In $'#include "' -- ./c10 ./aten ./torch/csrc ':(exclude)aten/src/ATen/native/quantized/cpu/qnnpack/**' ':(exclude)torch/csrc/jit/serialization/mobile_bytecode_generated.h'|| (echo "The above lines have include with quotes; please convert them to #include <xxxx>"; false))
- name: Ensure no versionless Python shebangs
if: always()
run: |
(! git --no-pager grep -In '#!.*python$' -- . || (echo "The above lines have versionless Python shebangs; please specify either python2 or python3"; false))
- - name: Ensure no unqualified noqa
- if: always()
- run: |
- # shellcheck disable=SC2016
- (! git --no-pager grep -InP '# noqa(?!: [A-Z]+\d{3})' -- '**.py' '**.pyi' ':(exclude)caffe2' || (echo 'The above lines have unqualified `noqa`; please convert them to `noqa: XXXX`'; false))
- - name: Ensure no unqualified type ignore
- if: always()
- run: |
- # shellcheck disable=SC2016
- (! git --no-pager grep -InP '# type:\s*ignore(?!\[)' -- '**.py' '**.pyi' ':(exclude)test/test_jit.py' || (echo 'The above lines have unqualified `type: ignore`; please convert them to `type: ignore[xxxx]`'; false))
- - name: Ensure GitHub PyPi dependencies are pinned
- if: always()
- run: |
- (! git --no-pager grep --color=always -InP \
- '(pip|pip3|python -m pip|python3 -m pip|python3 -mpip|python -mpip) install ([a-z][\.a-z-0-9]*+(?!(=|.*\.whl))([[:blank:]]|))+' \
- -- .github \
- ':(exclude)**.rst' \
- ':(exclude)**.py' \
- ':(exclude)**.md' \
- ':(exclude)**.diff' \
- ':(exclude)third_party' ||
- (echo "The above lines have unpinned PyPi installs; please pin them to a specific version: e.g. 'thepackage==1.2'"; false))
- # note that this next step depends on a clean checkout;
- # if you run it locally then it will likely to complain
- # about all the generated files in torch/test
- - name: Ensure C++ source files are not executable
- if: always()
- run: |
- # shellcheck disable=SC2016
- (! find . \( -path ./third_party -o -path ./.git -o -path ./torch/bin -o -path ./build \) -prune -o -type f -executable -regextype posix-egrep -not -regex '.+(\.(bash|sh|py|so)|git-pre-commit|git-clang-format|gradlew)$' -print | grep . || (echo 'The above files have executable permission; please remove their executable permission by using `chmod -x`'; false))
- name: C++ docs check
if: ${{ always() && steps.requirements.outcome == 'success' }}
run: |
@@ -102,86 +114,12 @@
run: |
set -eux
python torch/testing/_check_kernel_launches.py |& tee "${GITHUB_WORKSPACE}"/cuda_kernel_launch_checks.txt
- - name: Ensure no direct cub include
- if: always()
- run: |
- (! git --no-pager grep -I -no $'#include <cub/' -- ./aten ':(exclude)aten/src/ATen/cuda/cub*.cuh' || (echo "The above files have direct cub include; please include ATen/cuda/cub.cuh instead and wrap your cub calls in at::native namespace if necessary"; false))
- - name: Ensure no raw cuda api calls
- if: always()
- run: |
- (! git --no-pager grep -I -no $'cudaStreamSynchronize' -- ./aten ./c10 ':(exclude)aten/src/ATen/test' ':(exclude)c10/cuda/CUDAFunctions.h' || (echo "The above files call raw cuda APIs directly; please use at::cuda wrappers instead"; false))
- name: Ensure all test files have header containing ownership information
if: always()
run: |
python3 -m pip install boto3==1.19.12
.github/scripts/lint_test_ownership.py
- clang-format:
- name: clang-format
- runs-on: ubuntu-18.04
- if: ${{ github.event_name == 'pull_request' }}
- steps:
- - name: Setup Python
- uses: actions/setup-python@v2
- with:
- python-version: 3.x
- architecture: x64
- # [see note: pytorch repo ref]
- # deep clone (fetch-depth 0 required to use git merge-base)
- - name: Checkout PyTorch
- uses: pytorch/pytorch/.github/actions/checkout-pytorch@master
- with:
- submodules: false
- - name: Run clang-format
- env:
- BASE_SHA: ${{ github.event.pull_request.base.sha }}
- run: |
- set -eu
- # This is necessary to get the same results regardless of whether the
- # PR was opened directly or from a forked repo. See: `9f890a92` for more info.
- git remote add upstream https://github.com/pytorch/pytorch
- git fetch upstream "$GITHUB_BASE_REF"
-
- # only run clang-format on allowlisted files
- echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
- echo "| clang-format failures found! Run: "
- echo "| tools/linter/clang_format_ci.sh ${BASE_SHA} "
- echo "| to fix this error. "
- echo "| For more info, see: https://github.com/pytorch/pytorch/wiki/clang-format "
- echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-
- tools/linter/clang_format_ci.sh "${BASE_SHA}"
-
- GIT_DIFF=$(git diff)
- if [[ -z $GIT_DIFF ]]; then
- exit 0
- fi
- echo "$GIT_DIFF"
- exit 1
-
- py2-setup-validate-errormsg:
- name: py2-setup-validate-errormsg
- runs-on: ubuntu-18.04
- steps:
- - name: Setup Python
- uses: actions/setup-python@v2
- with:
- python-version: 2.x
- architecture: x64
- # [see note: pytorch repo ref]
- - name: Checkout PyTorch
- uses: pytorch/pytorch/.github/actions/checkout-pytorch@master
- with:
- submodules: false
- - name: Attempt to run setup.py
- run: |
- if ! python2 setup.py | grep -q "Python 2 has reached end-of-life and is no longer supported by PyTorch."; then
- echo 'Running setup.py with Python 2 did not give the expected error message.'
- false
- fi
- - name: Keep torch.utils.collect_env python2 compliant
- run: python2 -m py_compile torch/utils/collect_env.py
-
shellcheck:
name: shellcheck
runs-on: ubuntu-18.04
@@ -266,7 +204,6 @@
./actionlint --color
rm actionlint
-
toc:
name: toc
runs-on: ubuntu-18.04
@@ -307,239 +244,6 @@
false
fi
- flake8-py3:
- name: flake8-py3
- runs-on: ubuntu-18.04
- steps:
- - name: Setup Python
- uses: actions/setup-python@v2
- with:
- python-version: 3.x
- architecture: x64
- # [see note: pytorch repo ref]
- # fetch-depth 2 required to allow us to use github.event.pull_request.head.sha
- - name: Checkout PyTorch
- uses: pytorch/pytorch/.github/actions/checkout-pytorch@master
- with:
- submodules: false
- - name: Prepare output dir with HEAD commit SHA
- env:
- HEAD_SHA: ${{ github.event.pull_request.head.sha }}
- run: |
- mkdir flake8-output
- cd flake8-output
- echo "$HEAD_SHA" > commit-sha.txt
- - name: Install dependencies
- run: |
- set -eux
- pip3 install typing-extensions==3.10 --user # for tools/linter/translate_annotations.py
- pip3 install -r requirements-flake8.txt --user
- flake8 --version
- - name: Run flake8
- run: |
- set -eux
- flake8 | tee "${GITHUB_WORKSPACE}"/flake8-output.txt
- - name: Translate annotations
- if: ${{ github.event_name == 'pull_request' }}
- env:
- HEAD_SHA: ${{ github.event.pull_request.head.sha }}
- run: |
- tools/linter/translate_annotations.py \
- --file="${GITHUB_WORKSPACE}"/flake8-output.txt \
- --regex='^(?P<filename>.*?):(?P<lineNumber>\d+):(?P<columnNumber>\d+): (?P<errorCode>\w+\d+) (?P<errorDesc>.*)' \
- --commit="$HEAD_SHA" \
- > flake8-output/annotations.json
- - name: Fail if there were any warnings
- run: |
- set -eu
- # Re-output flake8 status so GitHub logs show it on the step that actually failed
- cat "${GITHUB_WORKSPACE}"/flake8-output.txt
- if [ -s "${GITHUB_WORKSPACE}"/flake8-output.txt ]; then
- echo 'Please fix the above Flake8 warnings.'
- false
- fi
- - name: Add annotations
- # Don't run on forked pull requests
- if: ${{ failure() && github.event.pull_request.head.repo.full_name == github.repository }}
- uses: pytorch/add-annotations-github-action@master
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- with:
- check_name: 'flake8-py3'
- linter_output_path: flake8-output/annotations.json
- commit_sha: ${{ github.event.pull_request.head.sha }}
- mode: json
-
- clang-tidy:
- name: clang-tidy
- runs-on: [self-hosted, linux.2xlarge]
- container:
- # ubuntu20.04-cuda11.2-py3.8-tidy11
- image: ghcr.io/pytorch/cilint-clang-tidy:d8f0c777964d0dd8a147360de80aed1a13eb613a
- steps:
- - name: Clean workspace
- run: |
- rm -rf "${GITHUB_WORKSPACE}"
- mkdir "${GITHUB_WORKSPACE}"
- # [see note: pytorch repo ref]
- # deep clone (fetch-depth 0) to allow tools/linter/clang_tidy.py to do its thing
- - name: Checkout PyTorch
- uses: pytorch/pytorch/.github/actions/checkout-pytorch@master
- with:
- no-sudo: true
- submodules: false
- - name: Prepare output dir with HEAD commit SHA
- env:
- HEAD_SHA: ${{ github.event.pull_request.head.sha }}
- run: |
- cd "${GITHUB_WORKSPACE}"
- mkdir clang-tidy-output
- cd clang-tidy-output
- echo "$HEAD_SHA" > commit-sha.txt
- - name: Fetch PR diff
- if: ${{ github.event_name == 'pull_request' }}
- env:
- PR_NUMBER: ${{ github.event.pull_request.number }}
- run: |
- cd "${GITHUB_WORKSPACE}"
- wget -O pr.diff "https://patch-diff.githubusercontent.com/raw/pytorch/pytorch/pull/$PR_NUMBER.diff"
- - name: Generate build files
- run: |
- cd "${GITHUB_WORKSPACE}"
- python3 -m tools.linter.clang_tidy.generate_build_files
- - name: Run PR clang-tidy
- if: ${{ github.event_name == 'pull_request' }}
- run: |
- cd "${GITHUB_WORKSPACE}"
-
- # The Docker image has our custom build, so we don't need to install it
- python3 -m tools.linter.clang_tidy \
- --clang-tidy-exe "$(which clang-tidy)" \
- --diff-file pr.diff \
- --disable-progress-bar 2>&1 | tee "${GITHUB_WORKSPACE}"/clang-tidy-output.txt
-
- # Run clang-tidy on a smaller subset of the codebase on master until we
- # make the repository clang-tidy clean
- - name: Run master clang-tidy
- run: |
- cd "${GITHUB_WORKSPACE}"
-
- python3 -m tools.linter.clang_tidy \
- --paths \
- torch/csrc/cuda \
- torch/csrc/fx \
- torch/csrc/utils \
- torch/csrc/generic \
- torch/csrc/deploy \
- torch/csrc/onnx \
- torch/csrc/tensor \
- --clang-tidy-exe "$(which clang-tidy)" \
- --disable-progress-bar 2>&1 | tee -a "${GITHUB_WORKSPACE}"/clang-tidy-output.txt
-
- - name: Annotate output
- if: ${{ github.event_name == 'pull_request' }}
- env:
- HEAD_SHA: ${{ github.event.pull_request.head.sha }}
- run: |
- cd "${GITHUB_WORKSPACE}"
- sed --in-place 's/^\.\.\///g' clang-tidy-output.txt
- tools/linter/translate_annotations.py \
- --file=clang-tidy-output.txt \
- --regex='^(?P<filename>.*?):(?P<lineNumber>\d+):(?P<columnNumber>\d+): (?P<errorDesc>.*?) \[(?P<errorCode>.*)\]' \
- --commit="$HEAD_SHA" \
- > clang-tidy-output/annotations.json
- - name: Check for warnings
- run: |
- cd "${GITHUB_WORKSPACE}"
- set -eu
- cat "${GITHUB_WORKSPACE}"/clang-tidy-output.txt
- if grep -Fq "Warnings detected!" "${GITHUB_WORKSPACE}"/clang-tidy-output.txt; then
- echo 'Please fix the above clang-tidy warnings.'
- false
- fi
- - name: Add annotations
- # Don't run on forked pull requests
- if: ${{ failure() && github.event.pull_request.head.repo.full_name == github.repository }}
- uses: pytorch/add-annotations-github-action@master
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- with:
- check_name: 'clang-tidy'
- linter_output_path: clang-tidy/annotations.json
- commit_sha: ${{ github.event.pull_request.head.sha }}
- mode: json
-
- cmakelint:
- name: cmakelint
- runs-on: ubuntu-18.04
- steps:
- - name: Setup Python
- uses: actions/setup-python@v2
- with:
- python-version: 3.x
- architecture: x64
- # [see note: pytorch repo ref]
- - name: Checkout PyTorch
- uses: pytorch/pytorch/.github/actions/checkout-pytorch@master
- with:
- submodules: false
- - name: Install dependencies
- run: |
- set -eux
- pip3 install cmakelint==1.4.1 --user
- cmakelint --version
- - name: Run cmakelint
- run: |
- set -eux
- git ls-files -z -- bootstrap '*.cmake' '*.cmake.in' '*CMakeLists.txt' | \
- grep -E -z -v '^(cmake/Modules/|cmake/Modules_CUDA_fix/|cmake/Caffe2Config.cmake.in|aten/src/ATen/ATenConfig.cmake.in|cmake/Caffe2ConfigVersion.cmake.in|cmake/TorchConfig.cmake.in|cmake/TorchConfigVersion.cmake.in|cmake/cmake_uninstall.cmake.in)' | \
- xargs -0 cmakelint --config=.cmakelintrc --spaces=2 --quiet
-
- mypy:
- name: mypy
- runs-on: ubuntu-18.04
- steps:
- - name: Setup Python
- uses: actions/setup-python@v2
- with:
- python-version: 3.8
- architecture: x64
- # [see note: pytorch repo ref]
- - name: Checkout PyTorch
- uses: pytorch/pytorch/.github/actions/checkout-pytorch@master
- with:
- submodules: false
- - name: Install dependencies
- run: |
- set -eux
- python3 -mpip install -r requirements.txt --user
- python3 -mpip install numpy==1.20 --user # https://github.com/pytorch/pytorch/pull/60472
- python3 -mpip install expecttest==0.1.3 mypy==0.812 --user
- # Needed to check tools/render_junit.py
- python3 -mpip install junitparser==2.1.1 rich==10.9.0 --user
- - name: Run autogen
- run: |
- set -eux
- time python3 -mtools.generate_torch_version --is_debug=false
- time python3 -mtools.codegen.gen -s aten/src/ATen -d build/aten/src/ATen
- time python3 -mtools.pyi.gen_pyi --native-functions-path aten/src/ATen/native/native_functions.yaml --deprecated-functions-path "tools/autograd/deprecated.yaml"
- - name: Run mypy
- env:
- MYPY_FORCE_COLOR: 1
- TERM: xterm-color
- run: |
- set -eux
- STATUS=
- for CONFIG in mypy*.ini; do
- if ! python3 -mmypy --config="$CONFIG"; then
- STATUS=fail
- fi
- done
- if [ -n "$STATUS" ]; then
- echo 'Please fix the above mypy warnings.'
- false
- fi
-
test-tools:
name: Test tools
if: ${{ github.repository == 'pytorch/pytorch' }}
@@ -563,6 +267,10 @@
set -eux
python3 -mpip install -r requirements.txt
python3 -mpip install boto3==1.16.34
+ pip3 install typing-extensions==3.10 --user
+ pip3 install -r requirements-flake8.txt --user
+ python3 -mpip install -r requirements.txt --user
+ python3 -mpip install mypy==0.812 --user
make setup_lint
- name: Test tools
run: |
diff --git a/Makefile b/Makefile
index e74f571..e35a5dd 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,6 @@
# This makefile does nothing but delegating the actual building to cmake.
PYTHON = python3
+PIP = pip3
all:
@mkdir -p build && cd build && cmake .. $(shell $(PYTHON) ./scripts/get_python_cmake_flags.py) && $(MAKE)
@@ -51,12 +52,7 @@
--job 'shellcheck'
setup_lint:
- $(PYTHON) tools/actions_local_runner.py --file .github/workflows/lint.yml \
- --job 'flake8-py3' --step 'Install dependencies' --no-quiet
- $(PYTHON) tools/actions_local_runner.py --file .github/workflows/lint.yml \
- --job 'cmakelint' --step 'Install dependencies' --no-quiet
- $(PYTHON) tools/actions_local_runner.py --file .github/workflows/lint.yml \
- --job 'mypy' --step 'Install dependencies' --no-quiet
+ $(PIP) install lintrunner
$(PYTHON) tools/actions_local_runner.py --file .github/workflows/lint.yml \
--job 'shellcheck' --step 'Install Jinja2' --no-quiet
@@ -71,8 +67,6 @@
--job 'shellcheck' --step 'Install ShellCheck' --no-quiet; \
fi
$(PYTHON) -mpip install jinja2 --user
- $(PYTHON) -mpip install -r tools/linter/clang_tidy/requirements.txt --user
- $(PYTHON) -m tools.linter.install.clang_tidy
quick_checks:
# TODO: This is broken when 'git config submodule.recurse' is 'true' since the
@@ -80,41 +74,7 @@
@$(PYTHON) tools/actions_local_runner.py \
--file .github/workflows/lint.yml \
--job 'quick-checks' \
- --step 'Ensure no trailing spaces' \
- --step 'Ensure no tabs' \
- --step 'Ensure no non-breaking spaces' \
- --step 'Ensure canonical include' \
- --step 'Ensure no versionless Python shebangs' \
- --step 'Ensure no unqualified noqa' \
- --step 'Ensure GitHub PyPi dependencies are pinned' \
- --step 'Ensure no unqualified type ignore' \
- --step 'Ensure no direct cub include' \
- --step 'Ensure correct trailing newlines' \
- --step 'Ensure no raw cuda api calls'
-
-flake8:
- @$(PYTHON) tools/actions_local_runner.py \
- $(CHANGED_ONLY) \
- $(REF_BRANCH) \
- --job 'flake8-py3'
-
-mypy:
- @$(PYTHON) tools/actions_local_runner.py \
- $(CHANGED_ONLY) \
- $(REF_BRANCH) \
- --job 'mypy'
-
-cmakelint:
- @$(PYTHON) tools/actions_local_runner.py \
- --file .github/workflows/lint.yml \
- --job 'cmakelint' \
- --step 'Run cmakelint'
-
-clang-tidy:
- @$(PYTHON) tools/actions_local_runner.py \
- $(CHANGED_ONLY) \
- $(REF_BRANCH) \
- --job 'clang-tidy'
+ --step 'Ensure no versionless Python shebangs'
toc:
@$(PYTHON) tools/actions_local_runner.py \
@@ -122,7 +82,9 @@
--job 'toc' \
--step "Regenerate ToCs and check that they didn't change"
-lint: flake8 mypy quick_checks cmakelint shellcheck
+lint: quick_checks shellcheck
+ lintrunner
quicklint: CHANGED_ONLY=--changed-only
-quicklint: mypy flake8 quick_checks cmakelint shellcheck clang-tidy
+quicklint: quick_checks shellcheck
+ lintrunner
diff --git a/tools/extract_scripts.py b/tools/extract_scripts.py
index 5312ed0..1090886 100755
--- a/tools/extract_scripts.py
+++ b/tools/extract_scripts.py
@@ -58,7 +58,7 @@
gha_expressions_found = False
for p in Path('.github/workflows').iterdir():
- with open(p) as f:
+ with open(p, "rb") as f:
workflow = yaml.safe_load(f)
for job_name, job in workflow['jobs'].items():
diff --git a/tools/linter/adapters/pip_init.py b/tools/linter/adapters/pip_init.py
index b4451be..10fdcea 100644
--- a/tools/linter/adapters/pip_init.py
+++ b/tools/linter/adapters/pip_init.py
@@ -45,7 +45,7 @@
"Package {package_name} did not have a version specified. "
"Please specify a version to product a consistent linting experience."
)
- pip_args = ["pip3", "install", "--user"]
+ pip_args = ["pip3", "install"]
pip_args.extend(args.packages)
dry_run = args.dry_run == "1"
diff --git a/tools/test/test_actions_local_runner.py b/tools/test/test_actions_local_runner.py
index ba4e6fd..3ef7c6a 100644
--- a/tools/test/test_actions_local_runner.py
+++ b/tools/test/test_actions_local_runner.py
@@ -47,16 +47,6 @@
class TestEndToEnd(unittest.TestCase):
expected = [
- "cmakelint: Run cmakelint",
- "quick-checks: Ensure no direct cub include",
- "quick-checks: Ensure no unqualified type ignore",
- "quick-checks: Ensure no unqualified noqa",
- "quick-checks: Ensure canonical include",
- "quick-checks: Ensure no non-breaking spaces",
- "quick-checks: Ensure no tabs",
- "flake8",
- "quick-checks: Ensure correct trailing newlines",
- "quick-checks: Ensure no trailing spaces",
"shellcheck: Regenerate workflows",
"shellcheck: Assert that regenerating the workflows didn't change them",
"shellcheck: Extract scripts from GitHub Actions workflows",
@@ -73,8 +63,6 @@
for line in self.expected:
self.assertIn(line, stdout)
- self.assertIn("mypy", stdout)
-
def test_quicklint(self):
cmd = ["make", "quicklint", "-j", str(multiprocessing.cpu_count())]
proc = subprocess.run(
@@ -85,9 +73,6 @@
for line in self.expected:
self.assertIn(line, stdout)
- # TODO: See https://github.com/pytorch/pytorch/issues/57967
- self.assertIn("mypy (skipped typestub generation)", stdout)
-
class TestQuicklint(unittest.IsolatedAsyncioTestCase):
test_files = [
os.path.join("caffe2", "some_cool_file.py"),
@@ -147,45 +132,6 @@
self.assertIn("SC2148: Tips depend on target shell", f.getvalue())
self.assertIn("SC2283: Remove spaces around = to assign", f.getvalue())
- async def test_mypy(self):
- self.maxDiff = None
- f = io.StringIO()
- with contextlib.redirect_stdout(f):
- # Quicklint assumes this has been run already and doesn't work
- # without it
- _, _, _ = await actions_local_runner.shell_cmd(
- [
- f"{sys.executable}",
- "tools/actions_local_runner.py",
- "--job",
- "mypy",
- "--file",
- ".github/workflows/lint.yml",
- "--step",
- "Run autogen",
- ],
- redirect=True,
- )
-
- await actions_local_runner.Mypy(self.test_py_files, True).run()
-
- # Should exclude the aten/ file; also, apparently mypy
- # typechecks files in reverse order
- expected = textwrap.dedent(
- """
- x mypy (skipped typestub generation)
- torch/some_stubs.pyi:3:17: error: Incompatible types in assignment (expression has type "None", variable has type "str") [assignment]
- torch/some_stubs.pyi:4:17: error: Incompatible types in assignment (expression has type "float", variable has type "str") [assignment]
- torch/some_cool_file.py:3:17: error: Incompatible types in assignment (expression has type "None", variable has type "str") [assignment]
- torch/some_cool_file.py:4:17: error: Incompatible types in assignment (expression has type "float", variable has type "str") [assignment]
- caffe2/some_cool_file.py:3:17: error: Incompatible types in assignment (expression has type "None", variable has type "str") [assignment]
- caffe2/some_cool_file.py:4:17: error: Incompatible types in assignment (expression has type "float", variable has type "str") [assignment]
- """ # noqa: B950
- ).lstrip(
- "\n"
- )
- self.assertEqual(expected, f.getvalue())
-
if __name__ == "__main__":
unittest.main()
