Skip to content

Commit 6608126

Browse files
craigcomstockcraigcomstock
authored
Merge pull request #1669 from craigcomstock/3.21.x
Refactored github status get/set functions/scripts to hide secrets (3.21)
2 parents 7853060 + 2626444 commit 6608126

5 files changed

Lines changed: 212 additions & 119 deletions

File tree

‎build-scripts/bin/get-github-pull-request-info‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#!/usr/bin/env bash
2+
# Args:
3+
# $1 - repo identifier ("project/repo")
4+
# $2 - PR identifier (the PR number)
5+
# Env:
6+
# two github fine-grained personal access tokens are needed with read/write access to commit statuses
7+
# $CFENGINE_PR_TOKEN_PATH - file path containing token associated with CFEngine github organization
8+
# $NTHQ_PR_TOKEN_PATH - file path containing token associated with NorthernTechHQ github organization
9+
# see get-pr-token script along-side this script for details
10+
# Prints:
11+
# $REPO_ID $PR_ID $PR_STATUSES_URL
12+
# Where:
13+
# $PR_STATUSES_URL - GH API URL to set PR's statuses
14+
# Returns:
15+
# 0 - success, 1 - error
16+
if [ -z "$1" ]; then echo "First argument, project, is required"; exit 1; fi
17+
if [ -z "$2" ]; then echo "Second argument, pull request number, is required"; exit 1; fi
18+
19+
json_out="$(mktemp)"
20+
21+
# curl 7.88 ish supports --header @file but apparently 7.52 (on bootstrap vm (deb-9)) does not, so compose a script
22+
curl_script_file="$(mktemp)"
23+
chmod 600 "$curl_script_file"
24+
echo -n "curl --insecure --fail --header \"Authorization: Bearer " > "$curl_script_file"
25+
26+
_dir=$(readlink -e "$(dirname "$0")")
27+
"$_dir"/get-pr-token "$1" >> "$curl_script_file"
28+
echo "\" https://api.github.com/repos/$1/pulls/$2" >> "$curl_script_file"
29+
30+
if ( # sub-shell to preserve original shell -/+x -/+e state
31+
set +x # hide curl command below as it contains a secret! don't remove me!
32+
# uncomment the below to debug, warning: will reveal secrets in logs
33+
# cat "$curl_script_file" >&2
34+
bash "$curl_script_file" >"$json_out"
35+
); then
36+
if command -v jq > /dev/null; then
37+
URL=$(jq ".statuses_url" < "$json_out" | tr -d '"')
38+
status=$?
39+
else
40+
URL=$(grep "statuses_url" "$json_out" | head -n1 | sed -r 's/\s+"statuses_url": "([^"]+)",/\1/')
41+
status=$?
42+
fi
43+
else
44+
echo "Request failed. Response was $(cat "$json_out")" >&2
45+
status=1
46+
fi
47+
48+
# uncomment the below deletion of files for debugging
49+
rm "$curl_script_file"
50+
rm "$json_out"
51+
52+
echo "$1 $2 $URL"
53+
exit $status

‎build-scripts/bin/get-pr-token‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/usr/bin/env bash
2+
# get a pr token from the path in an environment variable depending on github organization.
3+
# used by get-github-pull-request-info and set-github-status which in turn are used by bootstrap-tarballs and testing-pr jenkins job
4+
# Args:
5+
# $1 - repository identifier ("organization/project")
6+
# Env:
7+
# two github fine-grained personal access tokens are needed with read/write access to commit statuses
8+
# $CFENGINE_PR_TOKEN_PATH - file path containing token associated with CFEngine github organization
9+
# $NTHQ_PR_TOKEN_PATH - file path containing token associated with NorthernTechHQ github organization
10+
( # hide commands as they may contain secrets or paths to secrets
11+
set +x
12+
if [ -z "$1" ]; then echo "Need repository identifier as first argument"; exit 1; fi
13+
if [ -z "$CFENGINE_PR_TOKEN_PATH" ]; then echo "Env var CFENGINE_PR_TOKEN_PATH is required"; exit 1; fi
14+
if [ ! -f "$CFENGINE_PR_TOKEN_PATH" ]; then echo "CFENGINE_PR_TOKEN_PATH file must exist"; exit 1; fi
15+
if [ -z "$NTHQ_PR_TOKEN_PATH" ]; then echo "Env var NTHQ_PR_TOKEN_PATH is required"; exit 1; fi
16+
if [ ! -f "$NTHQ_PR_TOKEN_PATH" ]; then echo "NTHQ_PR_TOKEN_PATH file must exist"; exit 1; fi
17+
)
18+
# debug the following sha256sum commands to help determine if the tokens are correct in jenkins builds
19+
#echo "sha256sum of CFENGINE_PR_TOKEN_PATH..." >&2
20+
#sha256sum "$CFENGINE_PR_TOKEN_PATH" >&2
21+
#echo "sha256sum of NTHQ_PR_TOKEN_PATH..." >&2
22+
#sha256sum "$NTHQ_PR_TOKEN_PATH" >&2
23+
if [ "${1%/*}" = "cfengine" ]; then
24+
tr -d '\n' < "$CFENGINE_PR_TOKEN_PATH"
25+
elif [ "${1%/*}" = "NorthernTechHQ" ]; then
26+
tr -d '\n' < "$NTHQ_PR_TOKEN_PATH"
27+
else
28+
echo "$0 doesn't know about tokens for organization ${1%/*}"
29+
exit 1
30+
fi

‎build-scripts/bin/set-github-status‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
#!/usr/bin/env bash
2+
# GitHub reporting script
3+
# Args:
4+
# Either:
5+
# $1 - where to get repos and PRs info from
6+
# $2 - what state to report to GitHub
7+
# $3 - job spec (e.g. "ci/testing-pr/PACKAGES_HUB_x86_64_linux_redhat_7")
8+
# $4 - description of the status
9+
# $5 - URL to link from the status (e.g. $JOB_URL of the jenkins job)
10+
# Or:
11+
# $1 - where to get repos and PRs info from
12+
# $2 - path to a JSON file ready to POST to GH
13+
# Env:
14+
# see get-pr-token adjacent to this file
15+
16+
PRs_file="$1"
17+
if [ -z "$PRs_file" ]; then
18+
exit 1
19+
fi
20+
21+
if [ $# = "2" ]; then
22+
# just two args, check if it is a file we can read
23+
if [ -r "$2" ]; then
24+
JSON_file="$2"
25+
else
26+
"Path to a readable JSON file or status details required!"
27+
exit 1
28+
fi
29+
else
30+
state="$2"
31+
job_spec="$3"
32+
description="$4"
33+
job_url="$5"
34+
if [ -z "$job_url" ]; then
35+
job_url="https://ci.cfengine.com/"
36+
fi
37+
38+
if [ -z "$state" ] || [ -z "$job_spec" ]; then
39+
echo "Missing arguments"
40+
exit 1
41+
fi
42+
fi
43+
44+
function set_status() {
45+
set -ex
46+
# Actually set status at GitHub
47+
# Args:
48+
# $1 - repo identifier (organization/project)
49+
# $2 - statuses API URL of the PR
50+
# Env:
51+
52+
if [ -z "$1" ]; then
53+
echo "Missing repo identifier (organization/project) as first argument"
54+
exit 1
55+
fi
56+
if [ -z "$2" ]; then
57+
echo "Missing pull request API URL as second argument"
58+
exit 1
59+
fi
60+
61+
# curl 7.88 ish supports --header @file but apparently 7.52 (on bootstrap vm (deb-9)) does not, so compose a script
62+
curl_script_file="$(mktemp)"
63+
chmod 600 "$curl_script_file"
64+
echo -n "curl --fail --insecure -X POST --header \"Authorization: Bearer " > "$curl_script_file"
65+
_dir=$(readlink -e "$(dirname "$0")")
66+
"$_dir"/get-pr-token "$1" >> "$curl_script_file"
67+
echo -n "\" $2 --data " >> "$curl_script_file"
68+
69+
if [ -n "$JSON_file" ]; then
70+
(
71+
set +x # hide secrets
72+
echo "@$JSON_file" >> "$curl_script_file"
73+
)
74+
else
75+
(
76+
set +x # hide secrets
77+
echo -n "@- <<EOF" >> "$curl_script_file"
78+
echo -n "
79+
{
80+
\"state\" : \"$state\",
81+
\"target_url\" : \"$job_url\",
82+
\"description\" : \"$description\",
83+
\"context\" : \"$job_spec\"
84+
}
85+
EOF" >> "$curl_script_file"
86+
)
87+
fi
88+
89+
# uncomment the below cat to see the curl_script_file location
90+
# cat "$curl_script_file" >&2
91+
bash "$curl_script_file"
92+
93+
# uncomment the below file deletion to debug curl_script_file
94+
rm "$curl_script_file"
95+
return $?
96+
}
97+
98+
while read -r line; do
99+
# the PRs file has lines in the following format:
100+
# REPO_IDENTIFIER PR_ID PR_STATUS_API_URL
101+
REPO_IDENTIFIER=$(echo "$line" | awk '{ print $1 };')
102+
STATUS_URL=$(echo "$line" | awk '{ print $3 };')
103+
set_status "$REPO_IDENTIFIER" "$STATUS_URL"
104+
done < "$PRs_file"

‎build-scripts/bootstrap-tarballs‎

Lines changed: 25 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,37 @@
11
#!/bin/bash -x
22

3+
_dir=$(readlink -e "$(dirname "$0")")
4+
# refactored a few functions into single file scripts for easier development/debugging, see ENT-12741 and ENT-12595
5+
# Easier to add a path to a script than source a file of functions.
6+
export PATH="$_dir"/bin:$PATH
37
. `dirname "$0"`/functions
48
. detect-environment
59
. compile-options
610
. version
711

8-
get_GH_PR_info() {
9-
# Args:
10-
# $1 - repo identifier ("project/repo")
11-
# $2 - PR identifier (the PR number)
12-
# Env:
13-
# $GITHUB_STATUS_TOKEN - token for GitHub authentication
14-
# Prints:
15-
# $REPO_ID $PR_ID $PR_STATUSES_URL
16-
# Where:
17-
# $PR_STATUSES_URL - GH API URL to set PR's statuses
18-
# Returns:
19-
# 0 - success, 1 - error
20-
if [ -z "$1" ] || [ -z "$2" ] || [ -z "$GITHUB_STATUS_TOKEN" ]; then return 1; fi
21-
22-
if which jq > /dev/null; then
23-
URL=$(curl -k -H "Authorization: token $GITHUB_STATUS_TOKEN" https://api.github.com/repos/$1/pulls/$2 |
24-
jq ".statuses_url" | tr -d '"')
25-
status=$?
26-
else
27-
URL=$(curl -k -H "Authorization: token $GITHUB_STATUS_TOKEN" https://api.github.com/repos/$1/pulls/$2 |
28-
grep "statuses_url" | head -n1 | sed -r 's/\s+"statuses_url": "([^"]+)",/\1/')
29-
status=$?
30-
fi
31-
32-
echo "$1 $2 $URL"
33-
return $status
34-
}
12+
mkdir -p $BASEDIR/output/tarballs
3513

14+
# the first part of the script is not really critical
15+
set +e
3616

37-
mkdir -p $BASEDIR/output/tarballs
17+
# Get information about PRs among the used revisions.
18+
# These PRs will have to be notified of build progress.
19+
for repo_spec in cfengine/buildscripts cfengine/core cfengine/masterfiles cfengine/enterprise cfengine/nova cfengine/mission-portal NorthernTechHQ/libntech; do
20+
# remove organization/ from start of repo_spec
21+
repo="${repo_spec#*/}"
22+
rev_param_name="$(echo $repo | tr '[:lower:]-' '[:upper:]_')_REV"
23+
revision="$(echo ${!rev_param_name})" || continue # dereference
24+
25+
# remove "origin/" (if any)
26+
revision="${revision##origin/}"
27+
if expr "$revision" : "pull/" >/dev/null; then
28+
pr_nr="$(echo $revision | cut -d/ -f2)"
29+
get-github-pull-request-info "$repo_spec" "$pr_nr" >> $BASEDIR/output/PRs
30+
fi
31+
done
32+
33+
# now script failures should fail the script
34+
set -e
3835

3936
cd $BASEDIR/core
4037
rm cfengine-3.*.tar.gz || true
@@ -109,20 +106,3 @@ if test -f "$BASEDIR/mission-portal/ldap/composer.json"; then
109106
fi
110107
)
111108

112-
# the rest of the script is not really critical
113-
set +e
114-
115-
# Get information about PRs among the used revisions.
116-
# These PRs will have to be notified of build progress.
117-
for repo in buildscripts core masterfiles enterprise nova mission-portal; do
118-
rev_param_name="$(echo $repo | tr '[:lower:]-' '[:upper:]_')_REV"
119-
revision="$(echo ${!rev_param_name})" || continue # dereference
120-
121-
# remove "origin/" (if any)
122-
revision="${revision##origin/}"
123-
if expr "$revision" : "pull/" >/dev/null; then
124-
repo_spec="cfengine/$repo"
125-
pr_nr="$(echo $revision | cut -d/ -f2)"
126-
get_GH_PR_info "$repo_spec" "$pr_nr" >> $BASEDIR/output/PRs
127-
fi
128-
done

‎build-scripts/set_github_status.sh‎

Lines changed: 0 additions & 74 deletions
This file was deleted.

0 commit comments

Comments
 (0)