diff --git a/ci/configure.sh b/ci/configure.sh
index 8d128b220..ae1fd0c37 100755
--- a/ci/configure.sh
+++ b/ci/configure.sh
@@ -1,7 +1,29 @@
 #!/usr/bin/env bash
 set -eu -o pipefail
 
-cd "$( dirname "${BASH_SOURCE[0]}" )"
+if [[ -n "${DEBUG:-}" ]]; then
+  set -x
+fi
 
-fly -t "${CONCOURSE_TARGET:-bosh}" set-pipeline -p bosh-cli \
-    -c ./pipeline.yml
+REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
+
+pipeline_name="bosh-cli"
+pipeline_yaml="${REPO_ROOT}/ci/pipeline.yml"
+
+concourse_target="${CONCOURSE_TARGET:-bosh}"
+fly="${FLY_CLI:-fly}"
+
+until "${fly}" -t "${concourse_target}" status; do
+  "${fly}" -t "${concourse_target}" login
+  sleep 1
+done
+
+echo "Validating..."
+"${fly}" validate-pipeline --strict --config "${pipeline_yaml}"
+echo ""
+
+echo "Configuring..."
+"${fly}" -t "${concourse_target}" \
+  set-pipeline \
+    --pipeline "${pipeline_name}" \
+    --config "${pipeline_yaml}"