Improve Clang-IWYU to automatically make #include fixes (#9858)

Co-authored-by: Mehrdad <noreply@github.com>

.github/workflows/main.yml

@@ -18,7 +18,7 @@ jobs:
         # Useful info: https://help.github.com/en/articles/workflow-syntax-for-github-actions
         include:
           - name: ubuntu
-            os: ubuntu-16.04
+            os: ubuntu-18.04
             python-version: 3.6
           - name: windows-msvc
             os: windows-2019
@@ -99,9 +99,4 @@ jobs:
       continue-on-error: true
       if: runner.os == 'Linux'
       shell: bash -e -o pipefail -l {0}
-      run: |
-        sudo apt-get install -qq -o=Dpkg::Use-Pty=0 iwyu
-        bazel build \
-          --keep_going \
-          --config=iwyu \
-          "//:*"
+      run: ci/travis/iwyu.sh process "//:*"

BUILD.bazel

@@ -1309,7 +1309,10 @@ action_listener(
 
 extra_action(
     name = "iwyu_action",
-    cmd = "$(location :iwyu_sh) $(location @com_google_protobuf//:protoc) $(location :extra_actions_base_proto) --extra_action_file=$(EXTRA_ACTION_FILE)",
+    cmd = "$(location :iwyu_sh) postbuild $(location @com_google_protobuf//:protoc) $(location :extra_actions_base_proto) --extra_action_file=$(EXTRA_ACTION_FILE) > $(output $(ACTION_ID).iwyu.txt)",
+    out_templates = [
+        "$(ACTION_ID).iwyu.txt",
+    ],
     tools = [
         ":extra_actions_base_proto",
         ":iwyu_sh",

ci/travis/bazel.py

@@ -71,7 +71,7 @@ def textproto_split(input_lines, json_encoder):
 
 def textproto_parse(stream, encoding, json_encoder):
     for item in textproto_split(stream, json_encoder):
-        yield json.loads(item)
+        yield json.loads(item.decode(encoding))
 
 
 class Bazel(object):

ci/travis/iwyu.sh

@@ -2,6 +2,9 @@
 
 set -euo pipefail
 
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)"
+WORKSPACE_DIR="${ROOT_DIR}/../.."
+
 if [ "${OSTYPE-}" = msys ] && [ -z "${MINGW_DIR+x}" ]; then
   # On Windows MINGW_DIR (analogous to /usr) might not always be defined when we need it for some tools
   if [ "${HOSTTYPE-}" = x86_64 ]; then
@@ -11,10 +14,10 @@ if [ "${OSTYPE-}" = msys ] && [ -z "${MINGW_DIR+x}" ]; then
   fi
 fi
 invoke_cc() {
-  local env_vars=() args=() env_parsed=0
+  local env_vars=() args=() env_parsed=0 result=0
   if [ "${OSTYPE}" = msys ]; then
     # On Windows we don't want automatic path conversion, since it can break non-path arguments
-    env_vars+=(MSYS2_ARG_CONV_EXCL="*")
+    env_vars+=("MSYS2_ARG_CONV_EXCL=*")
   fi
   local arg; for arg in "$@"; do
     if [ 0 -ne "${env_parsed}" ]; then
@@ -58,22 +61,59 @@ invoke_cc() {
   case "${cc##*/}" in
     clang*)
       # Call iwyu with the modified arguments and environment variables (env -i starts with a blank slate)
+      local output
       # shellcheck disable=SC2016
-      { PATH="${PATH}:/usr/bin" env -i "${env_vars[@]}" "${SHELL-/bin/bash}" -c 'iwyu -isystem "$("$1" -print-resource-dir "${@:2}")/include" "${@:2}"' exec "${args[@]}" 2>&1 || true; } | awk '
-        # Awk script to remove noise in the iwyu output
-        { header = 0; }
-        /^(The full include-list for .*|.* should (add|remove) these lines:)$/ { keep = 1; header = 1; }
-        /^(The full include-list for ([^\/]*\/)*external\/.*|([^\/]*\/)*external\/.* should (add|remove) these lines:)$/ { keep = 0; header = 1; }
-        /^The full include-list for .*$/ { keep = 0; header = 1; }
-        /^---$/ { keep = 0; header = 1; }
-        keep { print; n += header ? 0 : 1; }
-        END { exit n; }
-      '
+      output="$(PATH="${PATH}:/usr/bin" env -i "${env_vars[@]}" "${SHELL-/bin/bash}" -c 'iwyu -isystem "$("$1" -print-resource-dir "${@:2}")/include" "${@:2}"' exec "${args[@]}" 3>&1 1>&2 2>&3- || true)." || result=$?
+      output="${output%.}"
+      if [ 0 -eq "${result}" ]; then
+        printf "%s" "${output}"
+      else
+        printf "%s" "${output}" 1>&2
+      fi
       ;;
   esac
+  return "${result}"
 }
 
-main() {
+_fix_include() {
+  "$(command -v python2 || echo python)" "$(command -v fix_include)" "$@"
+}
+
+process() {
+  # TODO(mehrdadn): Install later versions of iwyu that avoid suggesting <bits/...> headers
+  case "${OSTYPE}" in
+    linux*)
+      sudo apt-get install -qq -o=Dpkg::Use-Pty=0 clang iwyu
+      ;;
+  esac
+
+  local target
+  target="$1"
+  CC=clang bazel build -k --config=iwyu "${target}"
+  CC=clang bazel aquery -k --config=iwyu --output=textproto "${target}" |
+    "$(command -v python3 || echo python)" "${ROOT_DIR}"/bazel.py textproto2json |
+    jq -s -r '{
+      "artifacts": map(select(.[0] == "artifacts") | (.[1] | map({(.[0]): .[1]}) | add) | {(.id): .exec_path}) | add,
+      "outputs": map(select(.[0] == "actions") | (.[1] | map({(.[0]): .[1]}) | add | select(.mnemonic == "iwyu_action") | .output_ids))
+    } | (. as $parent | .outputs | map($parent.artifacts[.])) | .[]' |
+    sed "s|^|$(bazel info | sed -n "s/execution_root: //p")/|" |
+    xargs -r -I {} -- sed -e "/^clang: /d" -e "s|[^ ]*_virtual_includes/[^/]*/||g" {} |
+    (
+      # Checkout & modify a clean copy of the repo to affecting the current one
+      local data new_worktree args=(--nocomments)
+      data="$(cat)"
+      new_worktree="$(TMPDIR="${WORKSPACE_DIR}/.." mktemp -d)"
+      git worktree add -q "${new_worktree}"
+      pushd "${new_worktree}"
+        echo "${data}" |             _fix_include "${args[@]}"    || true  # HACK: For files are only accessible from the workspace root
+        echo "${data}" | { cd src && _fix_include "${args[@]}"; } || true  # For files accessible from src/
+        git diff
+      popd  # this is required so we can remove the worktree when we're done
+      git worktree remove --force "${new_worktree}"
+    )
+}
+
+postbuild() {
   # Parsing might fail due to various things like aspects (e.g. BazelCcProtoAspect), but we don't want to check generated files anyway
   local data=""  # initialize in case next line fails
   data="$(exec "$1" --decode=blaze.CppCompileInfo "$2" < "${3#*=}" 2>&-)" || true
@@ -92,7 +132,7 @@ main() {
         + [.[1:][] | select (.[0] == 6) | "\(.[1][1])=\(.[2][1])" | gsub("'\''"; "'\''\\'\'''\''") | "'\''\(.)'\''"]
         + ["--"]
         + [.[1:][] | select (.[0] == 1) | .[1]                    | gsub("'\''"; "'\''\\'\'''\''") | "'\''\(.)'\''"]
-        + [.[1:][] | select (.[0] == 2) | .[1]                                                                     ]
+        + [.[1:][] | select (.[0] == 2 and (.[1] | type) == "string") | .[1]                                       ]
       ) | .[]' <<< "${data}")"
     # On Windows, jq can insert carriage returns; remove them
     data="$(exec tr -d "\r" <<< "${data}")"
@@ -102,4 +142,4 @@ main() {
   fi
 }
 
-main "$@"
+"$@"