feature/revert-include-all-file-types-in-personal-data-scan #1108
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: "Organisation ruleset: Terraform CI" | |
| permissions: | |
| contents: read | |
| on: | |
| push: | |
| branches-ignore: [main] | |
| paths: [.github/workflows/org.terraform-ci.yml] | |
| workflow_call: | |
| inputs: | |
| run_tflint: | |
| description: "Run TFLint after fmt/validate." | |
| required: false | |
| type: boolean | |
| default: true | |
| extra_skip_globs: | |
| description: "Newline-separated patterns to skip entirely (e.g., 'sandbox/*')." | |
| required: false | |
| type: string | |
| default: "" | |
| pull_request: | |
| types: [opened, edited, reopened, synchronize] | |
| branches: [main, master, dev] | |
| jobs: | |
| terraform-ci: | |
| if: ${{ github.actor != 'dependabot[bot]' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| env: | |
| TF_IN_AUTOMATION: "true" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Determine Terraform Version | |
| id: tf-version | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| VERSION="" | |
| # Check for .terraform-version file | |
| if [[ -f ".terraform-version" ]]; then | |
| VERSION=$(sed -E 's/#.*//; s/[[:space:]]+//g' .terraform-version) | |
| if [[ -n "$VERSION" ]]; then | |
| echo "version=$VERSION" >> "$GITHUB_OUTPUT" | |
| echo "Terraform version resolved from .terraform-version: $VERSION" | |
| exit 0 | |
| fi | |
| fi | |
| RV=$(grep -Rho 'required_version *= *"= *[0-9]+\.[0-9]+\.[0-9]+"' . || true) | |
| if [[ -n "$RV" ]]; then | |
| VERSION=$(echo "$RV" | sed -E 's/.*"= *([0-9]+\.[0-9]+\.[0-9]+)".*/\1/' | head -n1) | |
| if [[ -n "$VERSION" ]]; then | |
| echo "version=$VERSION" >> "$GITHUB_OUTPUT" | |
| echo "Terraform version resolved from required_version: $VERSION" | |
| exit 0 | |
| fi | |
| fi | |
| VERSION="latest" | |
| echo "version=latest" >> "$GITHUB_OUTPUT" | |
| echo "Terraform version resolved to fallback: $VERSION" | |
| - name: Setup Terraform | |
| id: setup-tf | |
| uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # v4.0.0 | |
| with: | |
| terraform_version: ${{ steps.tf-version.outputs.version }} | |
| terraform_wrapper: false | |
| # fmt is local-only | |
| - name: FMT (repo-wide) | |
| env: | |
| TF_PLUGIN_CACHE_DIR: "" | |
| run: terraform fmt -recursive -check | |
| # GitHub App token for cloning private module repos | |
| - name: Verify GitHub App secrets present | |
| env: | |
| APP_ID: ${{ secrets.TERRAFORM_MODULE_ACCESS_APP_ID }} | |
| APP_PRIVATE_KEY: ${{ secrets.TERRAFORM_MODULE_ACCESS_PRIVATE_KEY }} | |
| run: | | |
| set -euo pipefail | |
| [ -n "${APP_ID:-}" ] || { echo "Missing secret APP_ID"; exit 1; } | |
| [ -n "${APP_PRIVATE_KEY:-}" ] || { echo "Missing secret APP_PRIVATE_KEY"; exit 1; } | |
| - name: Get GitHub App token for modules | |
| id: app-token | |
| uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 | |
| with: | |
| owner: uktrade | |
| client-id: ${{ secrets.TERRAFORM_MODULE_ACCESS_APP_ID }} | |
| private-key: ${{ secrets.TERRAFORM_MODULE_ACCESS_PRIVATE_KEY }} | |
| - name: Configure git auth for private module clones | |
| env: | |
| TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| git config --global url."https://x-access-token:${TOKEN}@github.com/".insteadOf "https://github.com/" | |
| git config --global --get-regexp '^url\..*\.insteadOf$' || true | |
| - name: Add SSH Key | |
| id: ssh-agent | |
| uses: webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555 # v0.10.0 | |
| with: | |
| ssh-private-key: | | |
| ${{ secrets.TERRAFORM_CI_DUMMY_SSH }} | |
| ${{ secrets.TERRAFORM_CLOUDFRONT_CI_SSH }} | |
| ${{ secrets.TERRAFORM_DATADOG_CI_SSH }} | |
| ${{ secrets.TERRAFORM_PLATFORM_LOGGING_CI_SSH }} | |
| # ---- Provider cache (used by init/validate) ---- | |
| - name: Prepare Terraform provider cache dir | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| mkdir -p "${{ github.workspace }}/.terraform.d/plugin-cache" | |
| echo "TF_PLUGIN_CACHE_DIR=${{ github.workspace }}/.terraform.d/plugin-cache" >> "$GITHUB_ENV" | |
| - name: Cache Terraform provider plugins | |
| uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 | |
| with: | |
| path: ${{ github.workspace }}/.terraform.d/plugin-cache | |
| key: | |
| tf-plugins-${{ runner.os }}-tf${{ steps.setup-tf.outputs.terraform_version | |
| }}-${{ hashFiles('**/.terraform.lock.hcl') }} | |
| restore-keys: | | |
| tf-plugins-${{ runner.os }}-tf${{ steps.setup-tf.outputs.terraform_version }}- | |
| tf-plugins-${{ runner.os }}- | |
| - name: Clear per-directory .terraform providers | |
| run: | | |
| find . -type d -name ".terraform" -exec rm -rf {} + | |
| # ---- Discover roots (skip examples entirely) | |
| - name: Discover Terraform dirs (skip examples) | |
| id: discover | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| mapfile -t CANDIDATES < <(git ls-files -- '*.tf' '*.tf.json' 2>/dev/null | xargs -n1 dirname | sort -u) | |
| # Skip noise + examples entirely | |
| skip_regex='(^|/)(\.terraform|\.github|\.git|node_modules|vendor|docs|examples)(/|$)' | |
| EXTRA_SKIP="${{ inputs.extra_skip_globs }}" | |
| if [[ -n "$EXTRA_SKIP" ]]; then | |
| extra_regex="$(printf '%s\n' "$EXTRA_SKIP" | sed -E 's/[.[\]{}()+*^$\\|]/\\&/g; s,/,\\/,g' | paste -sd'|' -)" | |
| skip_regex="(${skip_regex}|${extra_regex})" | |
| fi | |
| declare -a ROOTS=() | |
| for d in "${CANDIDATES[@]}"; do | |
| [[ "$d" =~ $skip_regex ]] && continue | |
| shopt -s nullglob | |
| tf=( "$d"/*.tf "$d"/*.tf.json ) | |
| shopt -u nullglob | |
| [[ ${#tf[@]} -eq 0 ]] && continue | |
| ROOTS+=("$d") | |
| done | |
| { printf 'roots<<EOF\n'; printf '%s\n' "${ROOTS[@]}"; printf 'EOF\n'; } >> "$GITHUB_OUTPUT" | |
| echo "Discovered roots (examples skipped): ${#ROOTS[@]}" | |
| # ---- Validate (soft-skip auth/path/module-only) | |
| - name: Validate roots (soft) | |
| if: steps.discover.outputs.roots != '' | |
| shell: bash | |
| continue-on-error: true | |
| run: | | |
| set -euo pipefail | |
| mapfile -t DIRS < <(printf "%s\n" "${{ steps.discover.outputs.roots }}") | |
| echo "Validating roots:" | |
| printf ' - %s\n' "${DIRS[@]}" | |
| hard_fail=() | |
| soft_skipped=() | |
| for d in "${DIRS[@]}"; do | |
| # Skip empty/template-only dirs | |
| if ! find "$d" -maxdepth 1 -type f \( -name "*.tf" -o -name "*.tf.json" \) | grep -q .; then | |
| echo "::notice file=$d::Skipping (no Terraform files)" | |
| soft_skipped+=("$d (no tf files)") | |
| continue | |
| fi | |
| # Skip module directories outright | |
| if [[ "$d" =~ (^|/)modules(/|$) ]]; then | |
| echo "::notice file=$d::Skipping module directory" | |
| soft_skipped+=("$d (module dir)") | |
| continue | |
| fi | |
| echo; echo "==============================" | |
| echo "==> terraform init: $d" | |
| echo "==============================" | |
| lockflag=() | |
| [[ -f "$d/.terraform.lock.hcl" ]] && lockflag=(-lockfile=update) | |
| # INIT — SOFT-SKIP on auth/path/download issues | |
| log="$(mktemp)" | |
| if ! terraform -chdir="$d" init -backend=false -input=false "${lockflag[@]}" -no-color 2>&1 | tee "$log"; then | |
| if grep -Eq 'Failed to download module|Permission denied \(publickey\)|Authentication failed|Repository not found|could not read from remote repository|The requested URL returned error: (403|404)|Unreadable module directory|Unable to evaluate directory symlink|no such file or directory' "$log"; then | |
| echo "::warning file=$d::terraform init failed (private module auth or bad local module path). Skipping validate for this dir." | |
| soft_skipped+=("$d (init auth/path)") | |
| rm -f "$log" || true | |
| continue | |
| else | |
| echo "::error file=$d::terraform init failed (see output above)" | |
| hard_fail+=("$d (init failed)") | |
| rm -f "$log" || true | |
| continue | |
| fi | |
| fi | |
| rm -f "$log" || true | |
| echo; echo "==============================" | |
| echo "==> terraform validate: $d" | |
| echo "==============================" | |
| vlog="$(mktemp)" | |
| set +o pipefail | |
| terraform -chdir="$d" validate -no-color 2>&1 | tee "$vlog" | |
| ec=${PIPESTATUS[0]} | |
| set -o pipefail | |
| if (( ec != 0 )); then | |
| if grep -Eq 'Provider configuration not present|its original provider configuration at provider\[.*\] is required|has been removed\. This occurs when a provider configuration is removed|No value for required variable|Missing required argument|Module source cannot be determined' "$vlog"; then | |
| echo "::notice file=$d::Module-only or caller-dependent; skipping validate for this dir." | |
| soft_skipped+=("$d (module-only)") | |
| else | |
| echo "::error file=$d::terraform validate failed (see output above)" | |
| hard_fail+=("$d (validate failed)") | |
| fi | |
| fi | |
| rm -f "$vlog" || true | |
| done | |
| # Summary (no exit 1 — this step never fails the job) | |
| if [[ ${#soft_skipped[@]} -gt 0 ]]; then | |
| echo; echo "Soft-skipped directories:"; printf ' - %s\n' "${soft_skipped[@]}" | |
| fi | |
| if [[ ${#hard_fail[@]} -gt 0 ]]; then | |
| echo; echo "Validation hard failures (non-blocking):"; printf ' - %s\n' "${hard_fail[@]}" | |
| # To make hard validation failures fail the job, add: exit 1 | |
| fi | |
| # ---- TFLint (ALWAYS run)- | |
| - name: Setup TFLint | |
| if: ${{ always() }} | |
| uses: terraform-linters/setup-tflint@b480b8fcdaa6f2c577f8e4fa799e89e756bb7c93 # v6.2.2 | |
| - name: TFLint (all discovered roots) | |
| if: ${{ always() }} | |
| shell: bash | |
| run: |- | |
| set -euo pipefail | |
| if [[ -z "${{ steps.discover.outputs.roots }}" ]]; then | |
| echo "::notice::No Terraform roots discovered; skipping tflint." | |
| exit 0 | |
| fi | |
| mapfile -t DIRS < <(printf "%s\n" "${{ steps.discover.outputs.roots }}") | |
| for d in "${DIRS[@]}"; do | |
| if [[ "$d" =~ (^|/)modules(/|$) ]]; then | |
| echo "::notice file=$d::Skipping module directory for tflint" | |
| continue | |
| fi | |
| echo | |
| echo "==============================" | |
| echo "==> tflint: $d" | |
| echo "==============================" | |
| (cd "$d" && { [[ -f .tflint.hcl ]] && tflint --init || true; } && tflint) | |
| done |