diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000000..dd8140504f93 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,21 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Dependabot version updates. See +# https://docs.github.com/en/code-security/reference/supply-chain-security/dependabot-options-reference + +version: 2 +updates: + - package-ecosystem: "github-actions" + # "/" covers the workflows in .github/workflows. + directories: + - "/" + schedule: + interval: "daily" + commit-message: + prefix: "chore" + include: "scope" + groups: + github-actions: + patterns: + - "*" diff --git a/.github/scan_tools_configs/bandit.yml b/.github/scan_tools_configs/bandit.yml new file mode 100644 index 000000000000..999f59c87e86 --- /dev/null +++ b/.github/scan_tools_configs/bandit.yml @@ -0,0 +1,19 @@ +# bandit configuration for rocMLIR +# Reference: https://bandit.readthedocs.io/en/latest/config.html +# +# pr-security-scan.yml and weekly-security-scan.yml pass this file to the +# shared ROCm/rocm-security-gh scan, where it replaces the shared default +# config wholesale rather than layering onto it. Exclusions and suppressions +# for this repository (e.g. `skips: [...]`) belong here. + +exclude_dirs: + - .git + - .venv + - venv + - build + - dist + - node_modules + - __pycache__ + # Vendored upstream trees. Bandit treats each entry as a substring of the + # scanned path, so the slashes keep this exclusion to the directory. + - /external/ diff --git a/.github/scan_tools_configs/gitleaks.toml b/.github/scan_tools_configs/gitleaks.toml new file mode 100644 index 000000000000..85dafbd30c30 --- /dev/null +++ b/.github/scan_tools_configs/gitleaks.toml @@ -0,0 +1,84 @@ +# gitleaks configuration for rocMLIR +# Reference: https://github.com/gitleaks/gitleaks/blob/master/config/gitleaks.toml +# +# pr-security-scan.yml and weekly-security-scan.yml pass this file to the +# shared ROCm/rocm-security-gh scan, where it replaces the shared default +# config wholesale rather than layering onto it. +# +# Within a scan, the layering is: +# * the `[[allowlists]]` below, applying to every rule. +# * `.gitleaksignore` at the scanned repo's root, carrying one-off +# fingerprint suppressions for historical findings that don't fit +# a structural rule. Keep that file shrinking; move recurring +# patterns up into a config when you see them. + +title = "gitleaks config" + +[extend] +# Inherit gitleaks' built-in ruleset (rotations, AWS, GCP, Slack, ...). +# Setting this to false would mean rolling our own detections, which +# is almost never what you want. +useDefault = true + + + +# Allowlist field-vs-target reference: +# stopwords -> always matched against the SECRET value. +# regexes -> matched against `regexTarget` (default: "secret"). +# regexTarget -> "secret" | "match" | "line". +# Use "match" or "line" when the "tell" is the variable name or +# surrounding syntax, not the value itself. + +[[allowlists]] +# Allowlist by location only where a real first-party secret structurally +# can't live. Dependency lock files qualify: their contents are generated +# from a manifest, and the high-entropy strings they carry are artifact +# digests rather than credentials. +description = "Generated dependency lock files" +paths = [ + '''.*\.lock$''', +] + +[[allowlists]] +description = "Vendored upstream trees" +paths = [ + '''^external/''', +] + +[[allowlists]] +description = "Test fixtures using well-known dummy variable names" +# The "tell" here is the variable name on the same line, not the value +# (the value is a random-looking string by design). Match against the +# full rule match (variable + value) so we anchor on the identifier. +regexTarget = "match" +regexes = [ + '''(?i)\bINVALID_TOKEN\s*=''', + '''(?i)\bwrong_secret\s*=''', + '''(?i)\bsecret\s*=''' +] + +[[allowlists]] +description = "Dummy gateway key in the Claude sanitizer tests" +# The sanitizer tests feed this made-up key to the sanitizer to verify that it +# is rejected. Requiring both the exact path and value means a real secret in +# the test file is still reported. +condition = "AND" +paths = [ + '''^\.github/scripts/tests/test_sanitize\.sh$''', +] +regexes = [ + '''^sk-secret-xyzzy-12345$''', +] + +[[allowlists]] +description = "Public artifact digests (variable names ending in SHA256/512/MD5)" +# SHA-256/512/MD5 hashes of public artifacts (tarballs, GPG keys, +# mirrored binaries) are not secrets - they're one-way digests used +# for integrity verification. Match by variable name (against the +# whole line) so we don't broadly allowlist every 64-char hex string. +regexTarget = "line" +regexes = [ + '''[A-Z][A-Z0-9_]*SHA256\s*[=:]''', + '''[A-Z][A-Z0-9_]*SHA512\s*[=:]''', + '''[A-Z][A-Z0-9_]*MD5\s*[=:]''', +] diff --git a/.github/scan_tools_configs/trivy.yml b/.github/scan_tools_configs/trivy.yml new file mode 100644 index 000000000000..990f8f548494 --- /dev/null +++ b/.github/scan_tools_configs/trivy.yml @@ -0,0 +1,19 @@ +# trivy configuration for rocMLIR +# Reference: https://trivy.dev/latest/docs/references/configuration/config-file/ +# +# pr-security-scan.yml and weekly-security-scan.yml pass this file to the +# shared ROCm/rocm-security-gh scan, where it replaces the shared default +# config wholesale rather than layering onto it. Tuning for this repository +# belongs here; per-finding suppressions go in a .trivyignore at the root. + +scan: + skip-dirs: + - .git + - .venv + - venv + - build + - dist + - node_modules + - __pycache__ + # Vendored upstream trees. + - external diff --git a/.github/scan_tools_configs/zizmor.yml b/.github/scan_tools_configs/zizmor.yml new file mode 100644 index 000000000000..1eada006f679 --- /dev/null +++ b/.github/scan_tools_configs/zizmor.yml @@ -0,0 +1,21 @@ +# zizmor configuration for rocMLIR +# Reference: https://docs.zizmor.sh/configuration/ +# +# pr-security-scan.yml and weekly-security-scan.yml pass this file to the +# shared ROCm/rocm-security-gh scan, where it replaces the shared default +# config wholesale rather than layering onto it. Zizmor has no path +# exclusions; vendored workflows are kept out by /external/*/.github in the +# root .gitignore. +# +# Per-rule ignore lists and tuning live under `rules:`, as +# `: { ignore: [file.yml:line:col] }`, which keeps a suppression +# reviewable in one place rather than scattered as inline comments +# across workflows. +rules: + dangerous-triggers: + ignore: + # These companions need pull_request_target to label and comment on fork + # PRs. Neither checks out nor runs PR code; both default to no permissions + # and use explicitly scoped App tokens. See CLAUDE_AUTO_REVIEW.md. + - claude_auto_review_fork_notify.yml + - claude_auto_review_perimeter_banner.yml diff --git a/.github/workflows/CLAUDE_AUTO_REVIEW.md b/.github/workflows/CLAUDE_AUTO_REVIEW.md index 71d0338e8345..7f8dfa0663bb 100644 --- a/.github/workflows/CLAUDE_AUTO_REVIEW.md +++ b/.github/workflows/CLAUDE_AUTO_REVIEW.md @@ -1120,6 +1120,8 @@ env vars can't reference a single source. When you change one, update all: | PR-review-checklist content | **No content sync to maintain** -- `docs/PR_REVIEW_CHECKLIST.md` is the only copy. The `snapshot_review_checklist` step reads the (overlaid, trusted) file, drops its first 2 lines (H1 + blank), and substitutes the remainder into the prompt heredoc between `` markers; the same file is also overlaid into the workspace so `Read('docs/PR_REVIEW_CHECKLIST.md')` returns matching bytes. Drift is impossible by construction. **Contract when editing:** line 1 must be H1, line 2 must be blank (the step skips exactly 2 lines and asserts this with `::error file=...::`; same contract documented as a trailing HTML comment in the file). Local sanity check: `tail -n +3 docs/PR_REVIEW_CHECKLIST.md | head`. | | `verdict` enum (`APPROVE` / `REQUEST_CHANGES` / `COMMENT`) | `--json-schema` enum in `claude_auto_review.yml`; sanitizer's verdict check; verdict→annotation case in `post_claude_review.sh`'s `post_review` (the submitted `gh pr review` event is hardcoded to `--comment`; the case selects only the body-header annotation); both skills' output-schema sections. The **COMMENT-only submission policy** is intentionally NOT a sync point — it lives only in `post_claude_review.sh`'s `post_review` (no env var, no workflow input, no repo variable) so a misconfiguration cannot flip it. | | Pinned action SHAs | `claude-code-action`, `create-github-app-token`, `checkout`, `upload/download-artifact` — re-verify internals on bump (esp. the credential-helper branch behind `allowed_non_write_users`). | +| Companion workflow filenames | The two `pull_request_target` companions (`claude_auto_review_perimeter_banner.yml`, `claude_auto_review_fork_notify.yml`) ↔ the `dangerous-triggers` ignore list in `.github/scan_tools_configs/zizmor.yml`, which matches by bare filename. Renaming a companion, or adding another `pull_request_target` workflow, needs a matching security review and configuration update. | +| Sanitizer-test dummy gateway key | The dummy gateway-key value in `.github/scripts/tests/test_sanitize.sh` ↔ the allowlist in `.github/scan_tools_configs/gitleaks.toml`, which matches only that path and exact value. Changing the value or moving the test file needs a matching configuration update. | ### 15.1 Why `--no-renames` and `-c core.quotePath=false`? diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a57cd16cb5e..2af0c682f725 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,7 +157,7 @@ jobs: image: rocm/mlir:rocm7.2-latest options: --user root steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 if: needs.relevant-changes.outputs.relevant != 'false' with: # Full history is required for git-restore-mtime below (mtimes @@ -174,8 +174,8 @@ jobs: if: needs.relevant-changes.outputs.relevant != 'false' && github.event_name == 'pull_request' shell: bash run: | - BASE_REF="origin/${{ github.base_ref }}" - BASE_BRANCH="${{ github.base_ref }}" + BASE_REF="origin/$GITHUB_BASE_REF" + BASE_BRANCH="$GITHUB_BASE_REF" git fetch origin "$BASE_BRANCH:refs/remotes/origin/$BASE_BRANCH" || \ git fetch origin "$BASE_BRANCH" || true @@ -224,7 +224,7 @@ jobs: shell: bash run: | python3 -m pip install --upgrade pip - python3 -m pip install pathspec==0.12.1 unidiff==0.7.5 GitPython==3.1.44 + python3 -m pip install -r mlir/utils/jenkins/static-checks/requirements.txt - name: Compute external/ tree SHA (for cache key) id: ext @@ -235,9 +235,10 @@ jobs: # any LLVM subtree pull, [EXTERNAL] patch, or mlir-hal edit. echo "hash=$(git rev-parse HEAD:external | cut -c1-12)" >> "$GITHUB_OUTPUT" - - name: Cache CMake build directory (compile_commands.json generation) + - name: Restore CMake build directory cache + id: cpp-cache if: needs.relevant-changes.outputs.relevant != 'false' - uses: actions/cache@v4 + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: build # ext SHA prefix locks the cache to the current external/ state. @@ -340,6 +341,16 @@ jobs: python3 ./mlir/utils/jenkins/static-checks/premerge-checks.py \ --base-commit="${{ steps.base.outputs.ref }}" "${ignore_external[@]}" + - name: Save CMake build directory cache + if: >- + needs.relevant-changes.outputs.relevant != 'false' + && github.event_name != 'pull_request' + && steps.cpp-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: build + key: ${{ steps.cpp-cache.outputs.cache-primary-key }} + - name: Nothing to check if: needs.relevant-changes.outputs.relevant == 'false' run: echo "Only external/llvm-project/** changed - no C/C++ sources in scope." @@ -353,7 +364,7 @@ jobs: image: python:3.10 options: --user root steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 if: needs.relevant-changes.outputs.relevant != 'false' with: fetch-depth: 0 @@ -376,12 +387,12 @@ jobs: run: | echo "Determining merge base..." - if [ "${{ github.event_name }}" = "pull_request" ]; then - BASE_REF="origin/${{ github.base_ref }}" - BASE_BRANCH="${{ github.base_ref }}" + if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + BASE_REF="origin/$GITHUB_BASE_REF" + BASE_BRANCH="$GITHUB_BASE_REF" else - BASE_REF="origin/${{ github.ref_name }}" - BASE_BRANCH="${{ github.ref_name }}" + BASE_REF="origin/$GITHUB_REF_NAME" + BASE_BRANCH="$GITHUB_REF_NAME" fi echo "BASE_REF=$BASE_REF" @@ -442,7 +453,7 @@ jobs: image: python:3.10 options: --user root steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 if: needs.relevant-changes.outputs.relevant != 'false' with: fetch-depth: 0 diff --git a/.github/workflows/claude_auto_review.yml b/.github/workflows/claude_auto_review.yml index 3256b15f6654..0976f8c07d83 100644 --- a/.github/workflows/claude_auto_review.yml +++ b/.github/workflows/claude_auto_review.yml @@ -272,6 +272,9 @@ jobs: # private key as a secret. The variable NAME is historical. client-id: ${{ vars.ROCMLIR_PR_REVIEWER_APP_ID }} private-key: ${{ secrets.ROCMLIR_PR_REVIEWER_PRIVATE_KEY }} + permission-checks: read + permission-pull-requests: read + permission-statuses: read # Pre-fetch PR context in a plain shell step (no model). Runs before Claude # and consumes no user-controlled data, so it can't be prompt-injected; the @@ -1005,6 +1008,7 @@ jobs: with: client-id: ${{ vars.ROCMLIR_PR_REVIEWER_APP_ID }} private-key: ${{ secrets.ROCMLIR_PR_REVIEWER_PRIVATE_KEY }} + permission-pull-requests: write - name: Post inline comments, thread updates, and submit review env: @@ -1040,6 +1044,8 @@ jobs: with: client-id: ${{ vars.ROCMLIR_PR_REVIEWER_APP_ID }} private-key: ${{ secrets.ROCMLIR_PR_REVIEWER_PRIVATE_KEY }} + permission-issues: write + permission-pull-requests: write - name: Remove claude-review label id: remove-label diff --git a/.github/workflows/claude_auto_review_fork_notify.yml b/.github/workflows/claude_auto_review_fork_notify.yml index db16bf4bb792..032841058575 100644 --- a/.github/workflows/claude_auto_review_fork_notify.yml +++ b/.github/workflows/claude_auto_review_fork_notify.yml @@ -42,6 +42,8 @@ jobs: with: client-id: ${{ vars.ROCMLIR_PR_REVIEWER_APP_ID }} private-key: ${{ secrets.ROCMLIR_PR_REVIEWER_PRIVATE_KEY }} + permission-issues: write + permission-pull-requests: write - name: Post fork-PR explanation comment and remove the label env: diff --git a/.github/workflows/pr-security-scan.yml b/.github/workflows/pr-security-scan.yml new file mode 100644 index 000000000000..2ba2f5f53da8 --- /dev/null +++ b/.github/workflows/pr-security-scan.yml @@ -0,0 +1,38 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# PR security gate: fast, PR-side security scanners that run on every pull +# request. This complements the premerge checks in `ci.yml`. The set of +# scanners lives in `security-baseline.yml`, so scanners coming online are +# picked up here without touching this file. +# +# `report_formats: human` applies to Bandit, Gitleaks, Trivy, and Zizmor: +# their findings are uploaded as build artifacts and printed to job summaries. +# GitHub permits code-scanning uploads triggered by `pull_request` even without +# `security-events: write`, so CodeQL uploads its results independently. See +# weekly-security-scan.yml for non-PR SARIF uploads and explicit permission. + +name: Security scan (PR) + +on: + pull_request: + # Spelled out the default to avoid confusion. + types: [opened, synchronize, reopened] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + security: + name: Security scan + uses: ROCm/rocm-security-gh/.github/workflows/security-baseline.yml@62c2c4f3effcd47711da28276815c90cb4aa6444 # v1.0.2 + with: + report_formats: human + bandit_config_path: .github/scan_tools_configs/bandit.yml + gitleaks_config_path: .github/scan_tools_configs/gitleaks.toml + trivy_config_path: .github/scan_tools_configs/trivy.yml + zizmor_config_path: .github/scan_tools_configs/zizmor.yml diff --git a/.github/workflows/weekly-security-scan.yml b/.github/workflows/weekly-security-scan.yml new file mode 100644 index 000000000000..0ba132f60214 --- /dev/null +++ b/.github/workflows/weekly-security-scan.yml @@ -0,0 +1,39 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Weekly security scan: runs on a fixed cadence and pushes SARIF findings +# to the repository's code-scanning Security tab. Complements +# `pr-security-scan.yml`, which scans only what changed in a PR and +# uploads reviewer-readable artifacts for in-review browsing. +# +# `security-events: write` is granted on the `uses:` job itself, not just +# the top-level `permissions:` block above it, since a `permissions:` +# block (wherever declared) implicitly zeroes out anything it doesn't +# list. + +name: Security scan (Weekly) + +on: + # Run every Saturday at 10:00 UTC. + schedule: + - cron: "0 10 * * 6" + workflow_dispatch: + +permissions: + contents: read + +jobs: + security: + name: Security scan + # Only the job that uploads SARIF gets write access to code scanning. + permissions: + contents: read + security-events: write + uses: ROCm/rocm-security-gh/.github/workflows/security-baseline.yml@62c2c4f3effcd47711da28276815c90cb4aa6444 # v1.0.2 + with: + scan_mode: all + report_formats: sarif + bandit_config_path: .github/scan_tools_configs/bandit.yml + gitleaks_config_path: .github/scan_tools_configs/gitleaks.toml + trivy_config_path: .github/scan_tools_configs/trivy.yml + zizmor_config_path: .github/scan_tools_configs/zizmor.yml diff --git a/.gitignore b/.gitignore index e457e3ca6072..b4652c9394d7 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,6 @@ __pycache__/ .pytest_cache/ .venv venv/ +# CI definitions inside vendored upstream trees never run in this repository. +# Zizmor honors .gitignore when collecting workflows. +/external/*/.github diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 88f41470d966..2958139fbf8f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,6 +38,11 @@ Use [GitHub Issues](../../issues) to report bugs or request features. Include a ``` 5. Open a PR against `develop`. Describe *what* changed and *why*; link any related issue. 6. Ensure CI passes and request review from the relevant [CODEOWNERS](.github/CODEOWNERS). + CI includes Bandit, Gitleaks, Trivy, and Zizmor security scans of first-party + code. Their configurations live in + [`.github/scan_tools_configs/`](.github/scan_tools_configs/). If a scan + reports a false positive, add a narrowly scoped, documented suppression + instead of disabling the scan. By opening a PR, you agree your contribution is licensed under the terms in [LICENSE](LICENSE). diff --git a/mlir/utils/buildbot/mlir-rocm-mi200/Dockerfile b/mlir/utils/buildbot/mlir-rocm-mi200/Dockerfile index a42d3314b93c..e561db683881 100644 --- a/mlir/utils/buildbot/mlir-rocm-mi200/Dockerfile +++ b/mlir/utils/buildbot/mlir-rocm-mi200/Dockerfile @@ -15,8 +15,12 @@ # TO-DO: Update to rocm7.0 when we add newer machines on buildbot FROM rocm/mlir:rocm6.3-latest +# The base image defaults to an unprivileged user; package installation and +# buildbot account setup require root. The final USER below remains buildbot. +USER root + RUN apt-get update; \ - apt-get install -y dumb-init; + apt-get install -y --no-install-recommends dumb-init; # Accounts for PyYAML, numpy, pybind11, ml_dtypes ADD "https://raw.githubusercontent.com/llvm/llvm-project/main/mlir/python/requirements.txt" requirements.txt diff --git a/mlir/utils/jenkins/Dockerfile b/mlir/utils/jenkins/Dockerfile index a65f3ded7f9a..a4607f791e59 100644 --- a/mlir/utils/jenkins/Dockerfile +++ b/mlir/utils/jenkins/Dockerfile @@ -56,7 +56,7 @@ RUN echo 'install llvm ${LLVM_VERSION}' && \ chmod +x llvm.sh && \ ./llvm.sh ${LLVM_VERSION} && \ apt-get update && \ - apt-get install -y clang-${LLVM_VERSION} clang-format-${LLVM_VERSION} clang-tidy-${LLVM_VERSION} lld-${LLVM_VERSION} && \ + apt-get install -y --no-install-recommends clang-${LLVM_VERSION} clang-format-${LLVM_VERSION} clang-tidy-${LLVM_VERSION} lld-${LLVM_VERSION} && \ ln -s /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang && \ ln -s /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ && \ ln -s /usr/bin/clang-tidy-${LLVM_VERSION} /usr/bin/clang-tidy && \ @@ -140,3 +140,8 @@ RUN apt-get update && \ parallel && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* + +# Default to the base image's unprivileged user. Jenkins replaces this with the +# agent uid through `docker run -u`; derived images switch to root while +# installing their own dependencies. +USER ubuntu diff --git a/mlir/utils/jenkins/Dockerfile.migraphx-ci b/mlir/utils/jenkins/Dockerfile.migraphx-ci index 021eb9ae224b..8fe6da13b1e8 100644 --- a/mlir/utils/jenkins/Dockerfile.migraphx-ci +++ b/mlir/utils/jenkins/Dockerfile.migraphx-ci @@ -6,6 +6,10 @@ FROM rocm/mlir:${ROCM_BASE_TAG}-latest ARG ROCM_VERSION=7.2.4 ARG ROCM_PATH=/opt/rocm-${ROCM_VERSION} +# The base image defaults to an unprivileged user. Install this derived +# image's dependencies as root, then switch back at the end. +USER root + # --------------------- Section 4: MIGraphX dependencies --------------- WORKDIR /MIGraphXDeps RUN pip3 install setuptools wheel @@ -30,3 +34,4 @@ RUN wget https://github.com/onnx/models/raw/ba629906dd91872def671e70177c5544e0ea RUN wget https://github.com/stefankoncarevic/Onnx_models/raw/master/bert_base_cased_1.onnx RUN wget https://github.com/stefankoncarevic/Onnx_models/raw/master/distilgpt2_1.onnx WORKDIR / +USER ubuntu diff --git a/mlir/utils/jenkins/static-checks/premerge-checks.py b/mlir/utils/jenkins/static-checks/premerge-checks.py index ec970d8236ee..3e7a396343fc 100755 --- a/mlir/utils/jenkins/static-checks/premerge-checks.py +++ b/mlir/utils/jenkins/static-checks/premerge-checks.py @@ -28,10 +28,21 @@ def get_diff(base_commit, ignore_external_files: bool) -> Tuple[bool, str]: - command = f"git-clang-format --diff {base_commit}" + command = ['git-clang-format', '--diff', base_commit] if ignore_external_files: - command = f"git-clang-format --diff {base_commit} $(git diff --name-only {base_commit} | grep -v '^external/')" - diff_run = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + changed_run = subprocess.run(['git', 'diff', '--name-only', '--diff-filter=d', base_commit], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if changed_run.returncode != 0: + print('git diff failed while collecting files for clang-format:', file=sys.stderr) + print(changed_run.stderr.decode('utf-8', 'replace'), file=sys.stderr) + return False, '' + changed = changed_run.stdout.decode().splitlines() + changed = [path for path in changed if not path.startswith('external/')] + if not changed: + return True, '' + command += ['--'] + changed + diff_run = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) is_diff_run_succesful = diff_run.returncode <= 1 diff = diff_run.stdout.decode() print(diff) @@ -109,10 +120,14 @@ def run_clang_tidy(base_commit, ignore_config, ignore_external_files: bool = Fal """Apply clang-tidy and return if no issues were found. Extracted from https://github.com/google/llvm-premerge-checks/blob/master/scripts/clang_tidy_report.py""" - r = subprocess.run(f'git diff -U0 --no-prefix {base_commit}', - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + diff_command = ['git', 'diff', '-U0', '--no-prefix', base_commit] + if ignore_external_files: + diff_command += ['--', '.', ':!external'] + r = subprocess.run(diff_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if r.returncode != 0: + print('git diff failed while collecting changes for clang-tidy:', file=sys.stderr) + print(r.stderr.decode('utf-8', 'replace'), file=sys.stderr) + return False diff = r.stdout.decode("utf-8", "ignore") if ignore_config is not None and os.path.exists(ignore_config): ignore = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, diff --git a/mlir/utils/jenkins/static-checks/requirements.txt b/mlir/utils/jenkins/static-checks/requirements.txt index b029c092abf5..ee9dc10a4375 100644 --- a/mlir/utils/jenkins/static-checks/requirements.txt +++ b/mlir/utils/jenkins/static-checks/requirements.txt @@ -1,4 +1,4 @@ # Python dependencies for static-checks/premerge-checks.py. -GitPython==3.1.43 +GitPython==3.1.62 pathspec==0.12.1 unidiff==0.7.5