diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index adcc14c6a..4545c3111 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,7 +18,7 @@ jobs: # Useful info: https://help.github.com/en/articles/workflow-syntax-for-github-actions include: - name: ubuntu - os: ubuntu-16.04 + os: ubuntu-18.04 python-version: 3.6 - name: windows-msvc os: windows-2019 @@ -99,9 +99,4 @@ jobs: continue-on-error: true if: runner.os == 'Linux' shell: bash -e -o pipefail -l {0} - run: | - sudo apt-get install -qq -o=Dpkg::Use-Pty=0 iwyu - bazel build \ - --keep_going \ - --config=iwyu \ - "//:*" + run: ci/travis/iwyu.sh process "//:*" diff --git a/BUILD.bazel b/BUILD.bazel index dd4a323a7..7fc231602 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1309,7 +1309,10 @@ action_listener( extra_action( name = "iwyu_action", - cmd = "$(location :iwyu_sh) $(location @com_google_protobuf//:protoc) $(location :extra_actions_base_proto) --extra_action_file=$(EXTRA_ACTION_FILE)", + cmd = "$(location :iwyu_sh) postbuild $(location @com_google_protobuf//:protoc) $(location :extra_actions_base_proto) --extra_action_file=$(EXTRA_ACTION_FILE) > $(output $(ACTION_ID).iwyu.txt)", + out_templates = [ + "$(ACTION_ID).iwyu.txt", + ], tools = [ ":extra_actions_base_proto", ":iwyu_sh", diff --git a/ci/travis/bazel.py b/ci/travis/bazel.py index 3b01c1d7c..d731734b6 100755 --- a/ci/travis/bazel.py +++ b/ci/travis/bazel.py @@ -71,7 +71,7 @@ def textproto_split(input_lines, json_encoder): def textproto_parse(stream, encoding, json_encoder): for item in textproto_split(stream, json_encoder): - yield json.loads(item) + yield json.loads(item.decode(encoding)) class Bazel(object): diff --git a/ci/travis/iwyu.sh b/ci/travis/iwyu.sh index db8ac1383..38b9e7fbb 100755 --- a/ci/travis/iwyu.sh +++ b/ci/travis/iwyu.sh @@ -2,6 +2,9 @@ set -euo pipefail +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)" +WORKSPACE_DIR="${ROOT_DIR}/../.." + if [ "${OSTYPE-}" = msys ] && [ -z "${MINGW_DIR+x}" ]; then # On Windows MINGW_DIR (analogous to /usr) might not always be defined when we need it for some tools if [ "${HOSTTYPE-}" = x86_64 ]; then @@ -11,10 +14,10 @@ if [ "${OSTYPE-}" = msys ] && [ -z "${MINGW_DIR+x}" ]; then fi fi invoke_cc() { - local env_vars=() args=() env_parsed=0 + local env_vars=() args=() env_parsed=0 result=0 if [ "${OSTYPE}" = msys ]; then # On Windows we don't want automatic path conversion, since it can break non-path arguments - env_vars+=(MSYS2_ARG_CONV_EXCL="*") + env_vars+=("MSYS2_ARG_CONV_EXCL=*") fi local arg; for arg in "$@"; do if [ 0 -ne "${env_parsed}" ]; then @@ -58,22 +61,59 @@ invoke_cc() { case "${cc##*/}" in clang*) # Call iwyu with the modified arguments and environment variables (env -i starts with a blank slate) + local output # shellcheck disable=SC2016 - { PATH="${PATH}:/usr/bin" env -i "${env_vars[@]}" "${SHELL-/bin/bash}" -c 'iwyu -isystem "$("$1" -print-resource-dir "${@:2}")/include" "${@:2}"' exec "${args[@]}" 2>&1 || true; } | awk ' - # Awk script to remove noise in the iwyu output - { header = 0; } - /^(The full include-list for .*|.* should (add|remove) these lines:)$/ { keep = 1; header = 1; } - /^(The full include-list for ([^\/]*\/)*external\/.*|([^\/]*\/)*external\/.* should (add|remove) these lines:)$/ { keep = 0; header = 1; } - /^The full include-list for .*$/ { keep = 0; header = 1; } - /^---$/ { keep = 0; header = 1; } - keep { print; n += header ? 0 : 1; } - END { exit n; } - ' + output="$(PATH="${PATH}:/usr/bin" env -i "${env_vars[@]}" "${SHELL-/bin/bash}" -c 'iwyu -isystem "$("$1" -print-resource-dir "${@:2}")/include" "${@:2}"' exec "${args[@]}" 3>&1 1>&2 2>&3- || true)." || result=$? + output="${output%.}" + if [ 0 -eq "${result}" ]; then + printf "%s" "${output}" + else + printf "%s" "${output}" 1>&2 + fi ;; esac + return "${result}" } -main() { +_fix_include() { + "$(command -v python2 || echo python)" "$(command -v fix_include)" "$@" +} + +process() { + # TODO(mehrdadn): Install later versions of iwyu that avoid suggesting headers + case "${OSTYPE}" in + linux*) + sudo apt-get install -qq -o=Dpkg::Use-Pty=0 clang iwyu + ;; + esac + + local target + target="$1" + CC=clang bazel build -k --config=iwyu "${target}" + CC=clang bazel aquery -k --config=iwyu --output=textproto "${target}" | + "$(command -v python3 || echo python)" "${ROOT_DIR}"/bazel.py textproto2json | + jq -s -r '{ + "artifacts": map(select(.[0] == "artifacts") | (.[1] | map({(.[0]): .[1]}) | add) | {(.id): .exec_path}) | add, + "outputs": map(select(.[0] == "actions") | (.[1] | map({(.[0]): .[1]}) | add | select(.mnemonic == "iwyu_action") | .output_ids)) + } | (. as $parent | .outputs | map($parent.artifacts[.])) | .[]' | + sed "s|^|$(bazel info | sed -n "s/execution_root: //p")/|" | + xargs -r -I {} -- sed -e "/^clang: /d" -e "s|[^ ]*_virtual_includes/[^/]*/||g" {} | + ( + # Checkout & modify a clean copy of the repo to affecting the current one + local data new_worktree args=(--nocomments) + data="$(cat)" + new_worktree="$(TMPDIR="${WORKSPACE_DIR}/.." mktemp -d)" + git worktree add -q "${new_worktree}" + pushd "${new_worktree}" + echo "${data}" | _fix_include "${args[@]}" || true # HACK: For files are only accessible from the workspace root + echo "${data}" | { cd src && _fix_include "${args[@]}"; } || true # For files accessible from src/ + git diff + popd # this is required so we can remove the worktree when we're done + git worktree remove --force "${new_worktree}" + ) +} + +postbuild() { # Parsing might fail due to various things like aspects (e.g. BazelCcProtoAspect), but we don't want to check generated files anyway local data="" # initialize in case next line fails data="$(exec "$1" --decode=blaze.CppCompileInfo "$2" < "${3#*=}" 2>&-)" || true @@ -92,7 +132,7 @@ main() { + [.[1:][] | select (.[0] == 6) | "\(.[1][1])=\(.[2][1])" | gsub("'\''"; "'\''\\'\'''\''") | "'\''\(.)'\''"] + ["--"] + [.[1:][] | select (.[0] == 1) | .[1] | gsub("'\''"; "'\''\\'\'''\''") | "'\''\(.)'\''"] - + [.[1:][] | select (.[0] == 2) | .[1] ] + + [.[1:][] | select (.[0] == 2 and (.[1] | type) == "string") | .[1] ] ) | .[]' <<< "${data}")" # On Windows, jq can insert carriage returns; remove them data="$(exec tr -d "\r" <<< "${data}")" @@ -102,4 +142,4 @@ main() { fi } -main "$@" +"$@"