158 lines
6.1 KiB
Bash
158 lines
6.1 KiB
Bash
#!/bin/sh
|
|
set -eu
|
|
|
|
deployment_directory=${1:-/opt/shd-build-production}
|
|
source_directory=${2:-}
|
|
candidate_image=${3:-}
|
|
candidate_directory=${4:-}
|
|
if [ -z "$source_directory" ] || [ ! -f "$source_directory/Dockerfile" ] || [ -z "$candidate_image" ] || [ ! -f "$candidate_directory/compose.production.yml" ] || [ ! -f "$candidate_directory/.env.deploy" ]; then
|
|
echo "Usage: deploy-production.sh <deployment-directory> <source-directory> <candidate-image> <candidate-directory>" >&2
|
|
exit 2
|
|
fi
|
|
case "$candidate_directory" in
|
|
"$deployment_directory"/candidate-*) ;;
|
|
*) echo "Candidate directory must be a candidate-* child of the deployment directory." >&2; exit 2 ;;
|
|
esac
|
|
candidate_compose_file="$candidate_directory/compose.production.yml"
|
|
candidate_environment="$candidate_directory/.env.deploy"
|
|
canonical_compose_file="$deployment_directory/compose.production.yml"
|
|
canonical_environment="$deployment_directory/.env.deploy"
|
|
previous_directory="$deployment_directory/.previous-release"
|
|
healthy_marker="$deployment_directory/.deployment-healthy"
|
|
release_files="compose.production.yml .env.deploy .infisical.app.client-id .infisical.app.client-secret .infisical.scheduler.client-id .infisical.scheduler.client-secret deploy-production.sh bootstrap-production.sh sync-production-isac-to-staging.sh"
|
|
rollback_required=false
|
|
candidate_promoted=false
|
|
had_previous_release=false
|
|
|
|
candidate_compose() {
|
|
docker compose --env-file "$candidate_environment" -f "$candidate_compose_file" "$@"
|
|
}
|
|
|
|
canonical_compose() {
|
|
docker compose --env-file "$canonical_environment" -f "$canonical_compose_file" "$@"
|
|
}
|
|
|
|
release_is_healthy() {
|
|
health_environment=$1
|
|
health_compose_file=$2
|
|
health_attempt=0
|
|
while [ "$health_attempt" -lt 30 ]; do
|
|
health_app_container=$(docker compose --env-file "$health_environment" -f "$health_compose_file" ps -q app)
|
|
health_scheduler_container=$(docker compose --env-file "$health_environment" -f "$health_compose_file" ps -q scheduler)
|
|
health_app_status=missing
|
|
health_scheduler_status=missing
|
|
if [ -n "$health_app_container" ]; then
|
|
health_app_status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$health_app_container" 2>/dev/null || echo missing)
|
|
fi
|
|
if [ -n "$health_scheduler_container" ]; then
|
|
health_scheduler_status=$(docker inspect --format '{{.State.Status}}' "$health_scheduler_container" 2>/dev/null || echo missing)
|
|
fi
|
|
if [ "$health_app_status" = healthy ] && [ "$health_scheduler_status" = running ]; then return 0; fi
|
|
if [ "$health_app_status" = unhealthy ] || [ "$health_app_status" = exited ] || [ "$health_app_status" = dead ] || [ "$health_scheduler_status" = exited ] || [ "$health_scheduler_status" = dead ]; then return 1; fi
|
|
health_attempt=$((health_attempt + 1))
|
|
sleep 2
|
|
done
|
|
return 1
|
|
}
|
|
|
|
copy_release() {
|
|
copy_source=$1
|
|
copy_target=$2
|
|
copy_status=0
|
|
for release_file in $release_files; do
|
|
if [ -f "$copy_source/$release_file" ]; then
|
|
copy_temporary=$(mktemp "$copy_target/.release.XXXXXX") || return 1
|
|
if cp -p "$copy_source/$release_file" "$copy_temporary"; then
|
|
mv "$copy_temporary" "$copy_target/$release_file" || copy_status=1
|
|
else
|
|
rm -f "$copy_temporary"
|
|
copy_status=1
|
|
fi
|
|
else
|
|
rm -f "$copy_target/$release_file" || copy_status=1
|
|
fi
|
|
done
|
|
return "$copy_status"
|
|
}
|
|
|
|
cleanup_candidate() {
|
|
case "$candidate_directory" in
|
|
"$deployment_directory"/candidate-*) rm -rf "$candidate_directory" ;;
|
|
esac
|
|
}
|
|
|
|
rollback() {
|
|
rollback_exit_status=$?
|
|
trap - EXIT HUP INT TERM
|
|
set +e
|
|
if [ "$rollback_required" = true ]; then
|
|
echo "Restoring the previous healthy production release." >&2
|
|
restore_status=0
|
|
candidate_compose stop scheduler app >/dev/null 2>&1 || restore_status=1
|
|
if [ "$candidate_promoted" = true ]; then copy_release "$previous_directory" "$deployment_directory" || restore_status=1; fi
|
|
if [ "$had_previous_release" = true ]; then
|
|
canonical_compose up -d --no-build app scheduler || restore_status=1
|
|
if [ "$restore_status" -eq 0 ] && ! release_is_healthy "$canonical_environment" "$canonical_compose_file"; then restore_status=1; fi
|
|
else
|
|
echo "No previous production release is available." >&2
|
|
restore_status=1
|
|
fi
|
|
if [ "$restore_status" -ne 0 ]; then
|
|
echo "The previous production release could not be restored." >&2
|
|
if [ "$rollback_exit_status" -eq 0 ]; then rollback_exit_status=1; fi
|
|
fi
|
|
fi
|
|
cleanup_candidate
|
|
exit "$rollback_exit_status"
|
|
}
|
|
|
|
trap rollback EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
echo "Building the candidate image while production remains online."
|
|
docker build --label org.opencontainers.image.title=shd-build -t "$candidate_image" "$source_directory"
|
|
|
|
echo "Validating Infisical access and the candidate runtime configuration."
|
|
candidate_compose run --rm --no-deps app node scripts/validate-runtime-environment.mjs app
|
|
candidate_compose run --rm --no-deps scheduler node scripts/validate-runtime-environment.mjs scheduler
|
|
|
|
rm -rf "$previous_directory"
|
|
mkdir -p "$previous_directory"
|
|
if [ -f "$canonical_compose_file" ] && [ -f "$canonical_environment" ]; then
|
|
had_previous_release=true
|
|
copy_release "$deployment_directory" "$previous_directory"
|
|
fi
|
|
|
|
echo "Stopping only SHD production processes before accessing the embedded database."
|
|
rollback_required=true
|
|
candidate_compose stop scheduler app
|
|
|
|
echo "Creating a timestamped production backup when the database exists."
|
|
candidate_compose run --rm --no-deps app sh -c \
|
|
'if [ -f "$TDR_DATABASE_PATH" ]; then node scripts/db-backup.mjs; else echo "No production database exists yet; skipping backup."; fi'
|
|
|
|
echo "Applying additive schema migrations."
|
|
candidate_compose run --rm --no-deps app node scripts/db-migrate.mjs
|
|
|
|
candidate_promoted=true
|
|
copy_release "$candidate_directory" "$deployment_directory"
|
|
|
|
echo "Starting production."
|
|
canonical_compose up -d --no-build --remove-orphans app scheduler
|
|
|
|
if release_is_healthy "$canonical_environment" "$canonical_compose_file"; then
|
|
echo "SHD production deployment is healthy."
|
|
rollback_required=false
|
|
umask 077
|
|
: > "$healthy_marker"
|
|
cleanup_candidate
|
|
trap - EXIT HUP INT TERM
|
|
exit 0
|
|
fi
|
|
|
|
echo "SHD production failed its health check."
|
|
canonical_compose logs --tail=150 app scheduler
|
|
exit 1
|