#! /usr/bin/env bash # This script is copied from # https://github.com/labs42io/circleci-monorepo/blob/v2/.circleci/monorepo.sh set -e readonly REPO_TYPE=$( echo "${CIRCLE_REPOSITORY_URL}" | awk '{ match($0,/@github/) ? r="github" : r="bitbucket"; print r }' ) readonly PROJECT_SLUG="${REPO_TYPE}/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}" readonly SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) readonly TMP_DIR=${SCRIPT_DIR}/temp readonly CONFIG_FILE=${SCRIPT_DIR}/groups.json readonly CONCURRENCY=8 readonly TRIGGER_PARAM_NAME="trigger" readonly BUILDS_FILE=${TMP_DIR}/builds.json readonly DATA_FILE=${TMP_DIR}/data.json # Get the list of configured groups or default ones. function read_config_groups { c=$(jq --raw-output '(.groups // {}) | length' "$1") if [[ "${c}" == "0" ]]; then root_dir=$(jq --raw-output '.root // "groups"' "$1") find "${root_dir}/" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | awk -v d="${root_dir}" '{print $1 " " d "/" $1 "/"}' else jq -r '.groups | to_entries | map(([.key] + .value) | join(" ")) | join ("\n")' "$1" fi } # Download workflows status from CircleCI API (as JSON files). function get_workflows { seq 0 100 $((($1 - 1) * 100)) | \ awk \ -v api="https://circleci.com/api/v1.1/project/${PROJECT_SLUG}" \ -v tree="/tree/${CIRCLE_BRANCH}" \ -v token="${CIRCLE_API_TOKEN}" \ -v dir="${TMP_DIR}/data." \ '{ print $1 " " token " " api tree "?shallow=true&limit=100&offset=" $1 " " dir sprintf("%04d", $1) ".json" }' |\ xargs -n4 -P${CONCURRENCY} bash -c 'curl -u "$1:" -L -Ss -o $3 -w "\tGET: [%{response_code}] %{url_effective}\n" $2' } # Creates a map of workflows and commit SHAs for which build passed. function map { # Group by (workflow, commit sha, job name) and select # those workflows for which each job group contains at least one passed job. jq '.? | group_by(.workflows.workflow_name) | map({ (.[0].workflows.workflow_name): group_by(.vcs_revision) | map({ commit: .[0].vcs_revision, queued_at: .[0].queued_at, jobs: group_by(.workflows.job_name) | map({ success: any(.status == "success") }) }) | map(select(.jobs | all(.success))) | sort_by(.queued_at) | reverse | map(.commit) }) | add | select (. != null)' } # Get the nearest commit from which the current branch was created. function get_parent_commit { git_file="${TMP_DIR}/branches.txt" commit_sha=$1 if [[ ! -f "${git_file}" ]]; then git show-branch --topo-order --sha1-name --current --remote > "${git_file}" fi remote_name=$(git remote show | head -n1) indents=$(\ sed 's/].*/]/' "${git_file}" | # remove commit message awk '/^\-/ {exit} {print}' | # get lines until commits are listed awk -F '' -v b="[${remote_name}/${CIRCLE_BRANCH}]" 'match($0,/^ *\*/) || index($0, b)' | # get only current branch and remote awk -F '' '{ t = length($0); sub("^ *",""); print t - length($0) + 1 }') # calculate indentation level head_indent=$(\ sed 's/].*/]/' "${git_file}" | # remove commit message awk '/^\-/ {exit} {print}' | # get lines until commits are listed awk -F '' -v b="[${remote_name}/HEAD]" 'index($0, b)' | # get origin/HEAD line awk -F '' '{ t = length($0); sub("^ *",""); print t - length($0) + 1 }') # calculate indentation level i1=$(echo "${indents}" | head -n1) i2=$(echo "${indents}" | tail -n1) i3=${head_indent:-$i1} sed 's/].*//' "${git_file}" | # remove commit message awk -F '[' -v c="${commit_sha}" \ 'c == "null" || f; c!="null" && length($2) > 0 && index(c, $2) == 1 { f = 1; print }' | # skip until first commit in current branch awk -F '' -v i="${i1}" 'match(substr($0, i, 1), /[\+\-\*]/)' | # filter only commits (including merges) related to current branch awk -F '' -v i="${i1}" '{ print substr($0, 1, i - 1) " " substr($0, i + 1) }' | # excludes current branch awk -F '' -v i="${i2}" '{ print substr($0, 1, i - 1) " " substr($0, i + 1) }' | # excludes current remote branch awk -F '' -v i="${i3}" '{ print substr($0, 1, i - 1) " " substr($0, i + 1) }' | # excludes origin/HEAD branch awk -F '' 'gsub(/ /, "", $0)' | # remove white-space awk -F '' '/[\+\-]+\[/' | # match only lines with commit or merge head -n1 | # get the top most found commit sed 's/^.*\[//' # leave only the commit sha text } # GIT diff each package to calculate the number of changed files. function diff { parent_sha=$1 builds_file=$2 while read -r package paths; do last_build_sha=$(jq --raw-output --arg p "${package}" '.[$p][0]' "${builds_file}") if [[ "${last_build_sha}" != "null" && "x${last_build_sha}" != "x" ]]; then # diff changes since most recent successfull build for current workflow echo "$(git diff "${last_build_sha}"..HEAD --name-only -- ${paths} | wc -l)" "${last_build_sha:0:9}" built "${package}" elif [[ "x${parent_sha}" != "x" ]]; then # diff changes since parent branch commit sha echo "$(git diff "${parent_sha}"..HEAD --name-only -- ${paths} | wc -l)" "${parent_sha:0:9}" new "${package}" else # no builds and missing parent branch (detached?) echo 99999 - new "${package}" fi done } function print_status { echo -e "\nTrigger\tExists\tChanges\tParent\t\tPackage\n$(printf '=%0.s' {1..60})" echo "$1" | jq --raw-output ' def colors: { "red": "\u001b[31m", "green": "\u001b[32m", "yellow": "\u001b[33m", "default": "\u001b[39m", "reset": "\u001b[0m", }; def choose_color(a): if .changes == 99999 then colors.red elif .changes > 0 then colors.yellow elif .branch == "built" then colors.green else colors.default end; .[] | choose_color(.) + (if .changes > 0 then "[x]" else "[ ]" end) + "\t" + (if .branch == "built" then "[x]" else "[ ]" end) + "\t" + (.changes | tostring) + "\t" + .parent + "\t" + .package + colors.reset ' } function create_request_body { echo "$1" | jq --raw-output --arg branch "${CIRCLE_BRANCH}" --arg trigger "${TRIGGER_PARAM_NAME}" --argjson params "${CI_PARAMETERS:-null}" '. | map(select(.changes > 0)) | reduce .[] as $i (($params // {}) * { ($trigger): false }; .[$i.package] = true) | { branch: $branch, parameters: . } | @json' } function create_pipeline { url="https://circleci.com/api/v2/project/${PROJECT_SLUG}/pipeline" echo -e "Trigger:\n\tUrl: ${url}\n\tData: $1" if [[ "${CI}" != "true" ]]; then echo "Not a CI environment. Skip pipeline trigger." exit 0 fi; status_code=$(curl -s -u "${CIRCLE_API_TOKEN}:" -o response.json -w "%{http_code}" -X POST --header "Content-Type: application/json" -d "$1" "${url}") if [ "${status_code}" -ge "200" ] && [ "${status_code}" -lt "300" ]; then echo "API call succeeded [${status_code}]. Response: " cat response.json else echo "API call failed [${status_code}]. Response: " cat response.json exit 1 fi } function init { if [[ "x${CIRCLE_API_TOKEN}" == "x" ]]; then echo "ENV variable CIRCLE_API_TOKEN is empty. Please provide a user token." exit 1 fi mkdir -p "${TMP_DIR}" if [[ ! -f ${CONFIG_FILE} ]]; then echo "No config file found at ${CONFIG_FILE}. Using defaults." echo "{}" > "${CONFIG_FILE}" fi } function get_builds { echo "Getting workflow status:" get_workflows "$(jq '.pages // 1' "${CONFIG_FILE}")" wait cat "${TMP_DIR}"/data.*.json | jq --slurp 'reduce inputs as $i (.; . += $i) | flatten' > "${DATA_FILE}" map < "${DATA_FILE}" > "${BUILDS_FILE}" echo "Created build-commit map ${BUILDS_FILE}" } function debug { echo -e "\n\nDEBUG INFORMATION" echo -e "\n\n=== Branches ===" cat "${TMP_DIR}/branches.txt" echo -e "\n\n=== Builds ===" cat "${BUILDS_FILE}" } function get_parent { first_commit_in_branch=$(jq --raw-output 'map(select(.vcs_revision)) | last | .vcs_revision' "${DATA_FILE}") echo "First built commit in branch: ${first_commit_in_branch}" >&2 parent_commit=$(get_parent_commit "${first_commit_in_branch}") if [[ "x${parent_commit}" == "x" ]]; then # This could happen when branch is force pushed # and the build commit is no longer part of the history echo -e "\tCould not find parent commit relative to first build commit." >&2 echo -e "\tEither branch was force pushed or build commit too old." >&2 parent_commit=$(get_parent_commit null) fi echo "Parent commit: ${parent_commit}" >&2 echo ${parent_commit} } function main { init get_builds git_parent_commit=$( get_parent ) statuses=$(\ read_config_groups "${CONFIG_FILE}" | diff "${git_parent_commit}" "${BUILDS_FILE}" | jq --raw-input --slurp \ 'split("\n") | map(select(. != "")) | map(split(" ")) | map({ package: .[3], parent: .[1], branch: .[2], changes: .[0] | tonumber })') print_status "${statuses}" changed_groups=$( echo "${statuses}" | jq '. | map(select(.changes > 0)) | length' ) total_groups=$( echo "${statuses}" | jq '. | length' ) echo "Number of groups changed: ${changed_groups} / ${total_groups}" if [[ "${changed_groups}" != "0" ]]; then create_pipeline "$( create_request_body "${statuses}" )" else echo "No changes in groups. Skip workflow trigger." fi if [[ "${CONDITIONAL_DEBUG}" == "true" ]]; then debug fi } main "${@}"