From 25d9241b510e0096b2dd09d44299a086c005b28c Mon Sep 17 00:00:00 2001 From: JB Date: Sun, 2 Aug 2026 01:19:41 -0400 Subject: [PATCH] feat: migrate secrets to Infisical --- .../replace-production-database/SKILL.md | 52 ++- .env.development | 32 -- .env.example | 16 +- .env.production | 32 -- .env.staging | 35 -- .gitea/workflows/deploy-production.yml | 71 +++- .gitea/workflows/deploy-staging.yml | 66 ++- .gitea/workflows/export-isac-data.yml | 33 +- .../workflows/export-production-isac-data.yml | 33 +- .../sync-production-isac-to-staging.yml | 2 +- .gitignore | 5 +- .infisical.json | 6 + AGENTS.md | 241 ++++++++--- Dockerfile | 25 +- README.md | 106 ++--- deploy/bootstrap-production.sh | 34 +- deploy/bootstrap-staging.sh | 34 +- deploy/compose.production.yml | 60 ++- deploy/compose.staging.yml | 56 ++- deploy/container-entrypoint.sh | 117 +++++- deploy/deploy-production.sh | 156 +++++-- deploy/deploy-staging.sh | 156 +++++-- deploy/runtime-start.test.ts | 393 ++++++++++++++++++ deploy/sync-production-isac-to-staging.sh | 56 ++- docs/DEPLOYMENT.md | 222 ++++++---- docs/architecture.md | 22 +- package-lock.json | 156 ++++--- package.json | 74 ++-- scripts/env-files.mjs | 35 -- scripts/runtime-environment.js | 232 +++++++++++ scripts/runtime-environment.test.ts | 161 +++++++ scripts/runtime-launcher.mjs | 251 +++++++++++ scripts/sync-staging-credentials.mjs | 54 --- scripts/validate-runtime-environment.mjs | 20 + 34 files changed, 2353 insertions(+), 691 deletions(-) delete mode 100644 .env.development delete mode 100644 .env.production delete mode 100644 .env.staging create mode 100644 .infisical.json create mode 100644 deploy/runtime-start.test.ts delete mode 100644 scripts/env-files.mjs create mode 100644 scripts/runtime-environment.js create mode 100644 scripts/runtime-environment.test.ts create mode 100644 scripts/runtime-launcher.mjs delete mode 100644 scripts/sync-staging-credentials.mjs create mode 100644 scripts/validate-runtime-environment.mjs diff --git a/.agents/skills/replace-production-database/SKILL.md b/.agents/skills/replace-production-database/SKILL.md index b84c984..7579248 100644 --- a/.agents/skills/replace-production-database/SKILL.md +++ b/.agents/skills/replace-production-database/SKILL.md @@ -1,32 +1,53 @@ --- name: replace-production-database -description: Replace the populated SHD.build production database with a local full bootstrap bundle after verified backups. Use only when the user explicitly authorizes replacement of all production data. +description: + Replace the populated SHD.build production database with a local full + bootstrap bundle after verified backups. Use only when the user explicitly + authorizes replacement of all production data. --- # Replace Production Database -This procedure overwrites production users, sessions, builds, guides, audits, proposals, settings, and source data. Obtain explicit approval for a full replacement. Use the ISAC import flow when the request only concerns canonical game data. +This procedure overwrites production users, sessions, builds, guides, audits, +proposals, settings, and source data. Obtain explicit approval for a full +replacement. Use the ISAC import flow when the request only concerns canonical +game data. ## Export -1. Stop only this repository's local development server. The embedded database must be closed during export. -2. Run `npm run production:bootstrap:export`. Record the generated bundle beneath `data/backups/`; it includes private account data and uploaded assets. -3. Transfer the bundle to `f302` over SSH. Keep it outside Git and confirm the remote copy is nonempty. +1. Stop only this repository's local development server. The embedded database + must be closed during export. +2. Run `npm run production:bootstrap:export`. Record the generated bundle + beneath `data/backups/`; it includes private account data and uploaded + assets. +3. Transfer the bundle to `f302` over SSH. Keep it outside Git and confirm the + remote copy is nonempty. ## Replace -1. Confirm the current production deployment is healthy and no Gitea production deployment is running. -2. On `f302`, stop only the production `app` and `scheduler` services from `/opt/shd-build-production`. -3. Run `node scripts/db-backup.mjs` through the production Compose service. Record and retain the JSON backup path before any destructive work. -4. Reset the production database file and its embedded-database support files. Keep the persistent volume and existing upload directories; the import writes the local avatar and managed-image files without deleting retained files. -5. Run `node scripts/db-migrate.mjs` through the production Compose service to create the current empty schema. -6. Run `environment-bootstrap-import.mjs` with `ALLOW_PRODUCTION_BOOTSTRAP=true`, the copied bundle mounted read-only, `--target production`, and `--confirm-empty-target-replace`. -7. Start `app` and `scheduler`, wait for app health, then remove the remote transfer bundle. +1. Confirm the current production deployment is healthy and no Gitea production + deployment is running. +2. On `f302`, stop only the production `app` and `scheduler` services from + `/opt/shd-build-production`. +3. Run `node scripts/db-backup.mjs` through the production Compose service. + Record and retain the JSON backup path before any destructive work. +4. Reset the production database file and its embedded-database support files. + Keep the persistent volume and existing upload directories; the import writes + the local avatar and managed-image files without deleting retained files. +5. Run `node scripts/db-migrate.mjs` through the production Compose service to + create the current empty schema. +6. Run `/opt/shd-build-production/bootstrap-production.sh `. The + script stages the bundle in the private data volume, assigns it to the + runtime UID, runs `environment-bootstrap-import.mjs` with the required + confirmation, removes the staged copy, and restarts both services. +7. Wait for app health, then remove the remote transfer bundle. ## Verify -1. Confirm the importer reports the expected row, avatar, and managed-image counts. -2. Confirm `https://shd.build/api/health` returns `ready: true` with an `ok` database check. +1. Confirm the importer reports the expected row, avatar, and managed-image + counts. +2. Confirm `https://shd.build/api/health` returns `ready: true` with an `ok` + database check. 3. Confirm the production app container is healthy and its scheduler is running. 4. Restart the local development server and verify it accepts an HTTP request. @@ -35,4 +56,5 @@ This procedure overwrites production users, sessions, builds, guides, audits, pr - Never replace a populated production database without explicit user approval. - Never delete the production Compose volume. - Do not replace the database while the app or scheduler can write to it. -- Keep the verified pre-replacement backup. It is the recovery source if the import or restart fails. +- Keep the verified pre-replacement backup. It is the recovery source if the + import or restart fails. diff --git a/.env.development b/.env.development deleted file mode 100644 index ce2ee73..0000000 --- a/.env.development +++ /dev/null @@ -1,32 +0,0 @@ -#/-------------------[DOTENV_PUBLIC_KEY]--------------------/ -#/ public-key encryption for .env files / -#/ [how it works](https://dotenvx.com/encryption) / -#/----------------------------------------------------------/ -DOTENV_PUBLIC_KEY_DEVELOPMENT="038fa4f9f7abd799fabebc69cb4d6a42a2ea361c3ec799580ddd213bcafac5c167" - -# .env.development -# Server-only OpenAI access for the permission-gated ISAC mechanic authoring -# assistant. Use a project-scoped, Responses-only restricted key. -OPENAI_MECHANIC_API_KEY=encrypted:BGRKl6pMDudJdr4yvye51GWuIsyjPfgarjsyxFnDs/HdgcRPdttk7nNz/40TEdfHidyaWHvag6HkM46zG/37VEhFpZ5N3iPPJmKV1a43NaAZNUeOP0dy6ct8UmOd6gEjDK9U6wzrjczRijgAffZ3GDyKyxkeDrn+8PHjng/0sXL78tpPr4p040cTEfSyet/XRsIUqeiV/6euS60afbnyzfOMUJ5liPdXK4VoJh6iLCsVrpmoNZ+rB5FMd65vpdSXOy6MKoG25aGq/1gtC5JOBiknrSE4vYGnuMriczklJviEBma+4rKMTSLTBOnYnHzpsuDC7+Tcft9VcluvNgcy0McGDExm -OPENAI_MECHANIC_MODEL=encrypted:BCCIuv04UwV44nzKVT3X7PpzbUJofJXPj/X2EYqErcSxedHh7IDAszLKgrBL+uzAT3xLOYS8lhOYfaFl/GaI12oOWAiZV2pijxRjpOE5Zv6X9wFJ7c+su44IRa7GIXF8wwnfWnSwSAg1S6kdS98= -OPENAI_MECHANIC_REASONING_EFFORT=encrypted:BDOTlCFCvs30g+to1IJD9aFhfPqrVPTccpRJKiUyywAm0ojSihpUL12guVnvjLG9NR2MrzkV1icqviDSR4yqsPC6Zz+Llf3KId4IBMXCa+OIUd+KOCM1AN3adoU9ZcBOIxuKZf6XTg== - -TDR_DATABASE_PATH=encrypted:BHzbwoU8Nc6yeHEwxLKFt08YVpz0+IRDBaRRu8mZlKtoi3R7qDOriv1+mIwLxkuZg+n8k7hGf9XhEipvfhbhx/N2sMcvsnVcrZfLIWItcNfrq9O8XmxE5oQpIQiYz/Osm9vclOeu06C6MwykowCtksIQD2xtCLLHInFPa8pJ0M0= -DISCORD_CLIENT_ID=encrypted:BAUbUqppB9HUQocELaV74nB1Ot6blLFNZ+640X8QLX5VWDzhABsw8JluD34L0ooYZve794HZH8ZFSn/zcvOsgSbGOdte9RQAa0YZpSz3Ih+TAmCvNUWg59A7iE2O1jCB5IEsCrRGbg4UslnP+I9ZHgnjRKo= -DISCORD_CLIENT_SECRET=encrypted:BA6jiQ2ws/e+y08nRwx2hyH2IlKBOjYVi+kk4gcYH6YyEvGw7HwIMRJ/n5eQx/IQB3KNDnklwL7x9XNGaq2Ht2UOHfrV+HFjPrTy/Kwt6retmMkfrA3PYJ7PdI+SlxE+hzvhYLvBJz6q1IcpRH09gz/x4jYeJPqD/LvPSKU2RlQ9 -DISCORD_REDIRECT_URI=encrypted:BJl92lm9C5zv7guW8O5faHE5t53c5orIMLte3TOOaMFMGcWlybqNuQEf6SmLfgwQ/AoAviJZDxOQRhRaKI5OR4s1r54Fbe2oHQBSlgcCyIKmACukOY9dg9nlpx/EkkvIY77hekCJhh/RfD3P5Os2OfSok7uiZxEGcagkA8H3/2KbpRUZSjWVEqNZjyM= - -# Passwordless authentication -APP_ORIGIN=encrypted:BBjQBgg8fXVjAjSrfzhAmuDuVRpvOlrKZTGqInncSwhJENhUs1EN/mBOlD769tAWEiNQ19mdDhIN1Dsiuu9Ps2VksDNdXjVNGwMYj3TRlX6v9wmzgODoCm6fvlseNJUQqXWSqEGgnrBzg90eg8bRMOzQMBByrA== -PASSKEY_RP_ID=encrypted:BM8t3iboOCY37Mg3UfYG1b17shBtNvM1QhkPAQPEq6gHIZz5nWeLVIqkZiLdMP81fmyEjmpD4cpnHNffy81xMyqSvzYZfIQ1UhNAM5vJMc6aDB535fRNVRYyf4+zMbiRJtyc+010RN3kCw== - -# Shared SMTP account for magic-link and email-verification delivery. -# Implicit TLS/SSL normally uses port 465. For STARTTLS on port 587, set -# SMTP_SECURE=false. -SMTP_HOST=encrypted:BE8dBQOkKsQuDh9SDx3CTyUOzQ8uuoDMlw4bCxQBs7apyg6P5FqZJIvEgnzCRHCo3dC7skgi/KDUDZI1stLX5A9tvlLvasCf2vGiMnn7NgG0e1nAYD1PiAnbQDmqfUcYBMLIWJibqIF96Yv1icnEcfNwkw== -SMTP_PORT=encrypted:BKt0uerEl2/lqwev72twYoeI0lUZdXgONWB+Xm14ftBG/6vY8E2UiH+/+6Zrf2b8Y1D11PzsZqEGaCYCJ8v1PyDnAQrE0pC4WjiYCl9kCkqsFvtZQcWys3rm0Nu6zrV8s0pYPQ== -SMTP_SECURE=encrypted:BFj5YB/gxpOwKdCrgUsydheEDV5aYAvtNMtLFSslYgsEkPLWe6nA1deQDXjVEwZxLC6iAgOgzxErRbem8nK5BNFfwpMuwwily+NQWZgPYMu7gD+BPmT9BMNvJkJvCwHEPj6Q6AT1 -SMTP_USER=encrypted:BC7+7c4vy1aS9mkuOaYZbxOdgZunja8UCz3P+ap+ZMK6qqV5KXvp+DPVvtJvVyCZejisS15LFsL+1VCeiD8hYyFKBrE2YJ+kx1hDwNezVMr1Dza548b9dXeMLh7Is6MjN5XBLLUjk2HcqgeXZ8WOmx01Dg== -SMTP_PASSWORD=encrypted:BOgcw3hrxXlMuQ92RNr+3M+myxlpd8lAtieebwXMsRQKiqLAKSDz3x0VXetvC2foMJ/KIEzTqM5scqkUA7dnm1/tQe0N3dG3ZZFzYIMz90Q0I5/XPWzli6cGtexoujswpkfnAa4EeGlvNEAhOYQrciU= -SMTP_FROM=encrypted:BGrMEhbploGfX29mvT+8XjgxvXK/QNjzr4CbAywkDyJmZ2xlscDbLC3+UoRSanGMdbVHvbDbYx6wG8M3MQ2QeIknf2AB75LpeeIVquSWa+u6OQBHmiRgEXWRuQJ7leTahd+B+E87gCpVtC3yFcHUtW3STyV1ZyidQJVwsI4HmQ== -REMOVE_BG_API_KEY=encrypted:BFeraOe9tiw3wCooTZ2ZE0qG9dDwqlpzq3HpEA0J+2LGckZSUxYP4f1Po1LNqWWihX+TzACONzHDhgjSaNq9XTMdjitYw7bEFehQQxsVi5iiJ/C6ZsN9M6vNIRP0FH+SfcYvFI41KJbSNOBFIX4zm5Y8hxG0aFJKkA== diff --git a/.env.example b/.env.example index 1ca1668..c0582b6 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,6 @@ +# Infisical secret inventory. This file contains names and safe defaults only; +# the application does not load it. Store values under /app unless noted. + # Server-only OpenAI access for the permission-gated ISAC mechanic and # calculation authoring assistants. Use a project-scoped, Responses-only restricted key. OPENAI_MECHANIC_API_KEY= @@ -13,7 +16,8 @@ TDR_DATABASE_PATH=./data/shd-build-development.db # Persistent directory for normalized user-uploaded avatars. TDR_AVATAR_STORAGE_PATH=./data/avatars -# Persistent directory for uploaded talent, brand, protocol, and skill-chip icons. +# Persistent directory for game-data icons and managed library images, including +# weapons, gear, mods, talents, brands, protocols, Skill Chips, and Specializations. TDR_GAME_ICON_STORAGE_PATH=./data/game-icons # Optional fixed credentials used only when initializing or explicitly resetting @@ -25,6 +29,7 @@ DEFAULT_USER_PASSWORD= DISCORD_CLIENT_ID= DISCORD_CLIENT_SECRET= # Proxied development: https://dev.shd.build/auth/discord/callback +# Staging: https://stage.shd.build/auth/discord/callback # Production: https://shd.build/auth/discord/callback DISCORD_REDIRECT_URI=https://dev.shd.build/auth/discord/callback @@ -45,9 +50,14 @@ SMTP_USER= SMTP_PASSWORD= SMTP_FROM=SHD.build -# Long random bearer token used only by the internal container scheduler. -SCHEDULER_TOKEN= +# Cloudflare Turnstile challenge keys for anonymous Guide activity. +TURNSTILE_SITE_KEY= +TURNSTILE_SECRET_KEY= + REMOVE_BG_API_KEY= REMOVE_BG_LOW_CREDIT_THRESHOLD=5 REMOVE_BG_LOW_FREE_CALL_THRESHOLD=10 REMOVE_BG_CREDIT_CHECK_HOURS=6 + +# /shared/scheduler +SCHEDULER_TOKEN= diff --git a/.env.production b/.env.production deleted file mode 100644 index 5837d52..0000000 --- a/.env.production +++ /dev/null @@ -1,32 +0,0 @@ -#/-------------------[DOTENV_PUBLIC_KEY]--------------------/ -#/ public-key encryption for .env files / -#/ [how it works](https://dotenvx.com/encryption) / -#/----------------------------------------------------------/ -DOTENV_PUBLIC_KEY_PRODUCTION="034d799df86cb16852742f1b2a42c69dde86396ddce96448c429c6ba68034a331d" - -# .env.production -# Server-only OpenAI access for the permission-gated ISAC mechanic authoring -# assistant. Use a project-scoped, Responses-only restricted key. -OPENAI_MECHANIC_API_KEY=encrypted:BIQtxqHheOrBdpeabt1p2Y1+TCHQDIxp5DHpQMUsS5N8aqukt/IELJSKWo9LXaFXbExHFB/X7Z2WvuWSyLiSA1QyZeUnUwMxnaJiYCpPlM3gTGvvlw/6U9uPxPBO7dLwchTMtqX9Sfx+GSWFzojfPOsVNaUVBRjBy5VTSOLwxMw0LMI+oYGVDEbjR3k/74sHlGXQazN0mThk8NClPJg6BSkKuKKFQLBjc4RyWUm94SE9KZwqiZVkfQaMmn+a9zLvbfd4yZaG4Ae/sneIU1T+cYjqdYAT05kBf8DikJkaezC2dRvcZ8I7jsp08pt+xHj3UC4XgbDLK2IVyLO4C8ELP5g7vQVy -OPENAI_MECHANIC_MODEL=encrypted:BLZffo6IeKN95zCneOq+FrdTjcjPF+vRkagV7pWAVgdusAK0OWR6B2bFqbqBuRVmos32uEWS4aP2GLrGpOr7gPlGDQTPu2EBAv4eT1OD5ewMC3lJFuqrXnr6j12bJOG8BUM3l+pMxwFBAXrDUew= -OPENAI_MECHANIC_REASONING_EFFORT=encrypted:BOrPunzMR1atHilHR6MEXaUDEMyPI7gkhyF4bSO1byh2DqGGEyKcGxLf6A2vIOcFM38/k9g9QCxuth54B4Zs5mHE5MUmvj6g8nwDSfO/ae5lAmYLSCqj4MLaDQ2LcJ+vYDrJ8aLpfA== - -TDR_DATABASE_PATH=encrypted:BL1wSNLW9BwOyzX89x73UjkxbJIUsOgOGkvLUfBRCqAAL2T4OVccEzh9aTWmLNaEICj6L795Tyd3Up+OZQ34NxqASKKk3o7piwQj8hiRU6W89fLVlSSZxdgLGq2S5YUdm3a/h9Loa1H4iDkg0TxO+MEQ2Qu53GD8zxOYWSQuBQ== -DISCORD_CLIENT_ID=encrypted:BB7n8VWz0GAV16Fkm7eJIfxI0a/rgHP1YakJ9XeiAvflWdYmndF5dvboOli+yLtuGCe+VqKUnzyLeQ7tk2YPQmMlkDXe75bhRkrLyPPvU0eg1DzhZ9ptU/qmmJZfrCQtT5o+/R61umVmfGkDB8nhpPkzqE8= -DISCORD_CLIENT_SECRET=encrypted:BJbCD14+3ZJ9T9+y90Uh3ZgRn4zyOqA0ckN/H+A9JfyWGgM2R5PBP2bmOPPGwRZk1Lg2GOWsowox4nrZEmINuUYirLRaszm9sqVejYXWsJPsTiXbukW+4rgtDpW7ifVkajbu/QGb8NqQR0tPUzJMYM0U5at5NM7TrqKWuu+4cLHf -DISCORD_REDIRECT_URI=encrypted:BKRD6GzZg6Qkt/ydrqkta4pUNlXNQzI4Kdq1ymn6qJH4O7NYVGE4J7MNO5biVsWPQuNU2ZG5qFatxzbbZMWT50x0o+A9wqmQ5XXt1kln8puHZsVOvOqSAI1z7mvWA+nuewv3vRDu0fchSsgSgFRjL1xwpp5tNac8DnvNaw0K/NlWHDdgZFIajQ== - -# Passwordless authentication -APP_ORIGIN=encrypted:BCDOpIDmpg3cipvTGTD/2XFxazXZBJLLf0/Y/sUDdRoKTEaF3m1dh7H3Ep9gyvRSLcbXxVnu3vPBowE5L7/pB6S3KRkjfdwvEImgUmvpfFfkLXM1/AiWO3M1drkfB+0lx5x5ktbtuNpQ22bckMraFzyx -PASSKEY_RP_ID=encrypted:BFyFsPV8oZCPkKROL0Lo8aDmKK+ay8jy5Z4HeYdkYvfF8e6eaaS29doIyXqEV0eeHEF/iFf5UspARhlf8To/lB1TE8KHXBiHAgfTVhur+tCN/TUzocH1jr1jnw77qykey5FXcoTKqD0+VQ== - -# Shared SMTP account for magic-link and email-verification delivery. -# Implicit TLS/SSL normally uses port 465. For STARTTLS on port 587, set -# SMTP_SECURE=false. -SMTP_HOST=encrypted:BFenO3Hh63xJz6dcJrEmCM40VGjPV7Y+F0TVgRRWAGMeBbrYZddhoJi7hhHZXpIX6B3nrSvq7dWIL+0aPKPzyWMgBsj3AnlXx8qi19CejStRQBHTQPyGnlpAZJK4SyBv24RwpwUs19T/TcK8b1DCX3tHYg== -SMTP_PORT=encrypted:BBxjsH6/dJcBap/sCS97SUPjuhuYSrFR1mqDvzaifCP52n/8jZ0+EYdkK0FcsU8ZhgHAWZ6vRkBjxr3nM39YXLEyATzbNF/tFZg29WGOk3kOxRhgemry0+kK2/UmZ9KUBTn3dQ== -SMTP_SECURE=encrypted:BAnU6hfdVX2xiPtP183dGjofqmz50/lB4MBeuWoer4KjwOsQXh1b/jOrGACUY7IKD2ngi5IMNdAtcqIkxdQGEfXj9GvPnkOIMNSwFwKNafIsZpS137UGmZCW1XopXU5cID28iSY7 -SMTP_USER=encrypted:BMREkzkhV9Aa4E22SskfpcTt3zbGSPwhvRqTmfXlclta6FZtQD7Oy8rDifhNaMf78V3hJkoaXPk9bcZ8zZn2NnG72NPbRXxukn6LiLePhb78z24g4UiiHON2MP7jIz4pvhPCSET08N/Km5kfiMLvp8otQA== -SMTP_PASSWORD=encrypted:BG39vFySsPvtm4AsuZZk5UbNaxHOOjxd8yffgsZebGNjxLY8aluuLf3/XMFquOz3bb59ZcyYSPv0Knp/N5c0vSHkl/ByC8WuQFTeLYH6BzpUADiVlGPE9Wg5Oa6dCOmianfRtHBUVrG1DC/BztBVYRg= -SMTP_FROM=encrypted:BMGyxQY0VdtmVfyiRl7NoO6dz8Q2pqvx8CLXDX/ZS9FsE98SUynl2MeOrOU8Vjz55V+KcLfFUooZoYT5Ke46YfRGMmE0ke2OaDZxFCBnD0rv6Mm9sRqXdyTn0vUxhjXmqLFkef32hN6VvBNN+xlppwwwbgYTg6pwtx6LKBQ1eg== -REMOVE_BG_API_KEY=encrypted:BLrlul9iafui2MD8NcqLYgwc2crtcC2HbXIX1QNDHy5sX2PMitvRCYOefEKzT6JmNXT2xvD0/uOKB7WTdKG1+qv7jVWl3IBK9wc2ftAtAWgESsUPy2mLKomXe8N04wz682om7EgwJ942294NWMWPLrWGYZCH1vOtog== diff --git a/.env.staging b/.env.staging deleted file mode 100644 index 9e4a599..0000000 --- a/.env.staging +++ /dev/null @@ -1,35 +0,0 @@ -#/-------------------[DOTENV_PUBLIC_KEY]--------------------/ -#/ public-key encryption for .env files / -#/ [how it works](https://dotenvx.com/encryption) / -#/----------------------------------------------------------/ -DOTENV_PUBLIC_KEY_STAGING="020bab072314925eb9a9cbaf42282e3a0d5c4a8de3d88a51ed92b4b3c83d598f98" - -# .env.staging -# Server-only OpenAI access for the permission-gated ISAC mechanic authoring -# assistant. Use a project-scoped, Responses-only restricted key. -OPENAI_MECHANIC_API_KEY=encrypted:BL4P5f594pMu3zF6onW128E+17uT7Vh7CwnpcAndHTNLAW3Vx3w99MawmLYZ17iiWgu6GF6ZzqavHpGgtTdYa1mU2zSwwfm5f0/gQP9lNM7KETdG8EIQfa167d3XixJu2EeW7Z17dTUEbEOOJ8xiI2HwsIZ/X37KEqYHZ0GJIzU0rXo4i6G17rHRs2OiByZL80ujQNsjU5lPHGnDikmAaSSczqtR0Q4/QmSd9EFDrRLIIGtDwqpCG8C4lV6JxRaU6SrfH+4GtUxSHyvITnsAAAzVVQ9yUmddmPkkfAdmAjz6XlrX5mdNnrjAJI9Z95RGTeCrwXCMI9ffulH84xxpXyXG+qez -OPENAI_MECHANIC_MODEL=encrypted:BLyNUZ2WPB+jMJOqLoV/xAbd31RsrXbGkV2EChn0vy8rh5yVoqbUX2aoqsTU6mv7oP1NT3CAsC/YkfiJFJ2FEiH9GT8Ddrs43QFMBvU2c3AUlZGZJeQDEIoqjWqmt7I6EYeM5gFiaR4o/i2NEHk= -OPENAI_MECHANIC_REASONING_EFFORT=encrypted:BHU/0x0Ez9jvwYPhmTCPh4ciVPYyVQTfjfCSb8zV2iNwH6c6LNkEs3omWXvA4wpUhNwub8z79/w4N2QGEnxOy/n8jFnhXqf+mI++niqmRXp8nuFXY0NUGF+THiZlssiKzIAyFXXeCQ== - -# Staging is deployed at https://stage.shd.build. -TDR_DATABASE_PATH=encrypted:BAh41Twl5JvpTKOaxlfw2J8dMBqRy5ontNeIPV8XFqrzQBTdQ7EmLn4gv+OOal4pUGlRQ2onC8TFN8vD0Xr83n1AQRYe48ovlDIRGeWFJoCimNSXqymrT+avXhNwRrMEjIu2E8YW0W8JWDXGWB1c4G8epRLy49Plq6Gzd7P+/RM= -TDR_AVATAR_STORAGE_PATH=encrypted:BH3DJiNNFUPnK83cby/HVCk0DHoBh2sNrnV1Txoz2Bh5EMPFSVpXdZaAz6C/LkdD5a2mKcDHFZb21Wy/GVJc9N3XNCJQz7GtvWrPNMv4oDFDJjDyeWHFHhPZmee6h3izWVO9mQDQTioamt7r50swMtP2 -TDR_GAME_ICON_STORAGE_PATH=encrypted:BKNWYaPZsXRaUr9gXAFfS5ULOqQPCBaO8fzsfpPMdh9L7dhMPgiIsqNzZlLUr5Ywl7JzKc9EFbdeuVYP2MFpeta0x/lwGnJFy+hjcBHlanXPUyLIYFEs0h2MZdwB4b24un9kZo6igsF478qWRfP2PLPCZ4eq - -APP_ORIGIN=encrypted:BAN8bBXyHn0fLmn/HqlK3cXfess/XF0HGYRAKxh89hyV26VHETLKgARQogdzjaAU4IRoTB6Lxr6pXzhuVRtQhsn4SiloTQ+t9Ecg/B7IbJ60F2TlcM5/aMq9QodpDGV2jZKVT7MsDl7yQuOyMca8BDJE6YiYGA== -PASSKEY_RP_ID=encrypted:BMF+2RmswtWB5Z8eaebDFdK306JvBeu8fqPLK7DmfObq980Tc57a8t0fSOUt13k+f16CnvRc0EeHORq+R2vN0jQW3korDKptBa4LHRJr68u8ITXKqU+gkzIzGWLOdfmkMdXKPN4AGeEryw== - -DISCORD_CLIENT_ID="encrypted:BE1JPS3brhFpQ7qvCqmAi9PoQxFKnLZGBe/IrZQ4atx6s0byUQ6RrRW4cRK+XG08FCikCbzW0BaDSjy3z/6I+0dNvCZEMcsfU7v/ZFlmrOpoPF+sxrQNHfbDHaYT0MnOSlvtpnPGAm/FoT4SVuS0XdTyY2M=" -DISCORD_CLIENT_SECRET="encrypted:BDEjA2trC6RqGqDkwq2R8pGdD8xWqnhyFhkS9hxxUt4GJpTtTSUteU1zwHodGc+Xj7UUXAd5o3+MVc31N2yPyVLVHkdN3tkEb87nuUOdxHhtbBW7fPAszq/n5jMJfIfEI9/lvCB4pq/yVKxDkHJt+sWoZhdqUuLgtRFOE0IPZwyW" -DISCORD_REDIRECT_URI=encrypted:BNM4ROa+P+E06QttCNhxzIwzTKmZb7IJLaK8gIH1n07gadfrNfAzbvFWCOZHnj4mcTvRnWym6761KnkL4FIgGOcLTx8eoGKmFM6WRAgA1u5MAz2k0fhyuNxAU/h0J+VdGLdFa3WpkjEY1TjD1A+6dhDv62A7XF8wCJtZx5mTqzjPjRdalxA/3GwgKRQ= - -SMTP_HOST="encrypted:BMjOo3Z/HyXK4c2fiVIqyud3bT83eBuD3uCEs97j2ymE0OAvoEqYaux8fMnSFull6kqvBLnfsxRR7BJaH6B15/Y8bhWmMIzPx1nfmkQTgbHY/esF+LSRZ18WeAIk7NOuvPRWacpY2CtadMZZ9hIe7AIhYg==" -SMTP_PORT="encrypted:BCaQ3+9VxAash3YSjHRQ/pt1RLy6xjVpzMA4MSDuyFlfnMjE20ywt5ZtnqRrmKzXQa2KELF0mm92v1VehREwUZReD+W8J3p9sSxpG+D4SjDRkXfVEAfXuYml9E+vqwirianL/g==" -SMTP_SECURE="encrypted:BAyezwpS+qNxZB0JmXJcQyiAS5Lua5XrURUnW1JUobBzTraqafs+2MpNJt3/LzVPayVv9nfuuQlUYfLJFnF6yRLQehqZobkaC6Hh+Q6AiWD3ZgkG+Sn6ON9c/Db+fltzt9D5lqbS" -SMTP_USER="encrypted:BHnngZLsV6cbONJ10YvT/46wYXA8zZQB4W50VodF/6E3MyRcY9GiUmW+vaCFsrq71KA8HMKQaJ1P+3dB5u/7kaWPzMW8UDUWkBtaBnr0CJLCxN3bff2SA2+iCcHvjgx0bcLJS83JhnKsdo6Fb42IkuEGgw==" -SMTP_PASSWORD="encrypted:BJs+zmP95i0SCm5h228DpILK5Q8BLX3wpsUx+5qEQgFakGV2b5h1x59q4TJcvX1wkTcbEHG+8ZYI1B2NJCgNgKFxRV4+uBnybIwP9rpHUQGL+kycYaM+hatvLzt9rA0eY+Gbm4AwhbiQ14rWxGqHnD4=" -SMTP_FROM="encrypted:BCrtD05OXCA8xlrFHvRLN/w16m43/S4da2xYWfYsvPIWeDWN56KURl9J6jljUpi5KzPutOfut5azWyKMssNdxAZqlRYQefFBBbrLK+lZjqccGEw2mKwLAibllYqPhiL5hK3bW5fUjojYqAL6Kb1UvN5KtkGzjIe2YW1jsc9eCg==" - -ORIGIN="encrypted:BMh6IEGoG+8iLK4a8vDpqduz3bGJ7ZI+X3oBnmmM0sbff3MaU/u3bdPuBtKcxxzMI8+Auc9TVlHOyDwM5pq40ny96CT/ZiMBQH+qFOjN797Vi7GO27eK58Z7iQuN99eQep1Pwxo84JruTUjhEuyYpVyqoQHl/ibN" -SCHEDULER_TOKEN="encrypted:BAUdHe+C2fE9hU+uzbpQM36hNC7ZY01smAcCuE8uUA8n2QhhVAD6z3Meb6k+IokarpxUrua7W1S7knQx7hdDgZfDPqWsbJW0GYP9Ym6VNe1chJ/F2ceaNxPAPp0jsZ55Vj22rQ1ljni5vD1oSBE2gTkrsqt95KS/qiaRMTRxEkQ0lmXZscmkXhmeNy0=" -REMOVE_BG_API_KEY=encrypted:BFcytS+vbpSp2AhQ7nnRZvGbQCCuDCcp5D6irWwodBVnKB1zosiXapU3Nw6lTwHepl6t+j8hakxF3mKyJmB+6I5hZlzH/v6VUYUQnlYaPE43dYYYbGrseg4ZYK0W9O7SWlOyHHgIv7Ktkx3nqBIwa6PS50Bjoq+8SA== diff --git a/.gitea/workflows/deploy-production.yml b/.gitea/workflows/deploy-production.yml index 8d5a20e..3af7315 100644 --- a/.gitea/workflows/deploy-production.yml +++ b/.gitea/workflows/deploy-production.yml @@ -7,7 +7,7 @@ on: workflow_dispatch: concurrency: - group: shd-build-production + group: shd-build-host-maintenance cancel-in-progress: false jobs: @@ -21,7 +21,7 @@ jobs: run: npm ci - name: Validate application - run: npm run check + run: npm run check && npm test - name: Build Node application run: npm run build:app @@ -33,30 +33,63 @@ jobs: - name: Check out repository uses: actions/checkout@v4 - - name: Install environment tooling - run: npm ci - - name: Prepare production configuration env: - DOTENV_PRIVATE_KEY_PRODUCTION: ${{ secrets.DOTENV_PRIVATE_KEY_PRODUCTION }} - SCHEDULER_TOKEN_PRODUCTION: ${{ secrets.SCHEDULER_TOKEN_PRODUCTION }} + INFISICAL_APP_CLIENT_ID: + ${{ vars.INFISICAL_PRODUCTION_APP_CLIENT_ID }} + INFISICAL_APP_CLIENT_SECRET: + ${{ secrets.INFISICAL_PRODUCTION_APP_CLIENT_SECRET }} + INFISICAL_SCHEDULER_CLIENT_ID: + ${{ vars.INFISICAL_PRODUCTION_SCHEDULER_CLIENT_ID }} + INFISICAL_SCHEDULER_CLIENT_SECRET: + ${{ secrets.INFISICAL_PRODUCTION_SCHEDULER_CLIENT_SECRET }} CONFIGURED_DEPLOY_PATH: ${{ vars.PRODUCTION_DEPLOY_PATH }} CONFIGURED_PRODUCTION_PORT: ${{ vars.PRODUCTION_PORT }} COMMIT_SHA: ${{ gitea.sha }} BUILD_NUMBER: ${{ gitea.run_number }} run: | set -eu - : "${DOTENV_PRIVATE_KEY_PRODUCTION:?DOTENV_PRIVATE_KEY_PRODUCTION is required}" - : "${SCHEDULER_TOKEN_PRODUCTION:?SCHEDULER_TOKEN_PRODUCTION is required}" + umask 077 + : "${INFISICAL_APP_CLIENT_ID:?INFISICAL_PRODUCTION_APP_CLIENT_ID is required}" + : "${INFISICAL_APP_CLIENT_SECRET:?INFISICAL_PRODUCTION_APP_CLIENT_SECRET is required}" + : "${INFISICAL_SCHEDULER_CLIENT_ID:?INFISICAL_PRODUCTION_SCHEDULER_CLIENT_ID is required}" + : "${INFISICAL_SCHEDULER_CLIENT_SECRET:?INFISICAL_PRODUCTION_SCHEDULER_CLIENT_SECRET is required}" deployment_path="${CONFIGURED_DEPLOY_PATH:-/opt/shd-build-production}" production_port="${CONFIGURED_PRODUCTION_PORT:-3102}" + case "$deployment_path" in + /opt/shd-build-*) ;; + *) echo "PRODUCTION_DEPLOY_PATH must be beneath /opt/shd-build-." >&2; exit 1 ;; + esac + case "$production_port" in + ''|*[!0-9]*) echo "PRODUCTION_PORT must be an integer." >&2; exit 1 ;; + esac + if [ "$production_port" -lt 1 ] || [ "$production_port" -gt 65535 ]; then + echo "PRODUCTION_PORT must be from 1 through 65535." >&2 + exit 1 + fi mkdir -p "$deployment_path" - cp deploy/compose.production.yml deploy/deploy-production.sh deploy/bootstrap-production.sh deploy/sync-production-isac-to-staging.sh "$deployment_path/" - DOTENV_PRIVATE_KEY_PRODUCTION="$DOTENV_PRIVATE_KEY_PRODUCTION" npx dotenvx decrypt -f .env.production --stdout > "$deployment_path/.env.production.runtime" - printf '\nSCHEDULER_TOKEN=%s\n' "$SCHEDULER_TOKEN_PRODUCTION" >> "$deployment_path/.env.production.runtime" - printf 'SHD_IMAGE=shd-build-production:%s\nSHD_PRODUCTION_PORT=%s\nSHD_BUILD_NUMBER=%s\n' "$COMMIT_SHA" "$production_port" "$BUILD_NUMBER" > "$deployment_path/.env.deploy" - chmod 600 "$deployment_path/.env.production.runtime" "$deployment_path/.env.deploy" - chmod 755 "$deployment_path/deploy-production.sh" "$deployment_path/bootstrap-production.sh" "$deployment_path/sync-production-isac-to-staging.sh" + release_id="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT:-1}" + candidate_path="$deployment_path/candidate-$release_id" + rm -rf "$candidate_path" + mkdir -p "$candidate_path" + cp deploy/compose.production.yml deploy/deploy-production.sh deploy/bootstrap-production.sh deploy/sync-production-isac-to-staging.sh "$candidate_path/" + write_credential() { + target="$1" + value="$2" + temporary_file=$(mktemp "${target%/*}/.infisical.XXXXXX") + printf '%s' "$value" > "$temporary_file" + chmod 600 "$temporary_file" + mv "$temporary_file" "$target" + } + write_credential "$candidate_path/.infisical.app.client-id" "$INFISICAL_APP_CLIENT_ID" + write_credential "$candidate_path/.infisical.app.client-secret" "$INFISICAL_APP_CLIENT_SECRET" + write_credential "$candidate_path/.infisical.scheduler.client-id" "$INFISICAL_SCHEDULER_CLIENT_ID" + write_credential "$candidate_path/.infisical.scheduler.client-secret" "$INFISICAL_SCHEDULER_CLIENT_SECRET" + temporary_deploy_environment=$(mktemp "$candidate_path/.env.deploy.XXXXXX") + printf 'SHD_IMAGE=shd-build-production:%s-%s\nSHD_PRODUCTION_PORT=%s\nSHD_BUILD_NUMBER=%s\nSHD_COMMIT_SHA=%s\n' "$COMMIT_SHA" "$release_id" "$production_port" "$BUILD_NUMBER" "$COMMIT_SHA" > "$temporary_deploy_environment" + chmod 600 "$temporary_deploy_environment" + mv "$temporary_deploy_environment" "$candidate_path/.env.deploy" + chmod 755 "$candidate_path/deploy-production.sh" "$candidate_path/bootstrap-production.sh" "$candidate_path/sync-production-isac-to-staging.sh" - name: Deploy production env: @@ -64,8 +97,12 @@ jobs: COMMIT_SHA: ${{ gitea.sha }} run: | deployment_path="${CONFIGURED_DEPLOY_PATH:-/opt/shd-build-production}" - "$deployment_path/deploy-production.sh" "$deployment_path" "$GITHUB_WORKSPACE" "shd-build-production:$COMMIT_SHA" + release_id="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT:-1}" + candidate_path="$deployment_path/candidate-$release_id" + "$candidate_path/deploy-production.sh" "$deployment_path" "$GITHUB_WORKSPACE" "shd-build-production:$COMMIT_SHA-$release_id" "$candidate_path" - name: Remove unused application images if: success() - run: docker image prune --force --filter "label=org.opencontainers.image.title=shd-build" + run: + docker image prune --force --filter + "label=org.opencontainers.image.title=shd-build" diff --git a/.gitea/workflows/deploy-staging.yml b/.gitea/workflows/deploy-staging.yml index e2e4d8c..3258dfe 100644 --- a/.gitea/workflows/deploy-staging.yml +++ b/.gitea/workflows/deploy-staging.yml @@ -7,7 +7,7 @@ on: workflow_dispatch: concurrency: - group: shd-build-staging + group: shd-build-host-maintenance cancel-in-progress: false jobs: @@ -21,7 +21,7 @@ jobs: run: npm ci - name: Validate application - run: npm run check + run: npm run check && npm test - name: Build Node application run: npm run build:app @@ -33,26 +33,62 @@ jobs: - name: Check out repository uses: actions/checkout@v4 - - name: Install environment tooling - run: npm ci - - name: Prepare deployment configuration env: - DOTENV_PRIVATE_KEY_STAGING: ${{ secrets.DOTENV_PRIVATE_KEY_STAGING }} + INFISICAL_APP_CLIENT_ID: ${{ vars.INFISICAL_STAGING_APP_CLIENT_ID }} + INFISICAL_APP_CLIENT_SECRET: + ${{ secrets.INFISICAL_STAGING_APP_CLIENT_SECRET }} + INFISICAL_SCHEDULER_CLIENT_ID: + ${{ vars.INFISICAL_STAGING_SCHEDULER_CLIENT_ID }} + INFISICAL_SCHEDULER_CLIENT_SECRET: + ${{ secrets.INFISICAL_STAGING_SCHEDULER_CLIENT_SECRET }} CONFIGURED_DEPLOY_PATH: ${{ vars.STAGE_DEPLOY_PATH }} CONFIGURED_STAGE_PORT: ${{ vars.STAGE_PORT }} COMMIT_SHA: ${{ gitea.sha }} BUILD_NUMBER: ${{ gitea.run_number }} run: | set -eu + umask 077 + : "${INFISICAL_APP_CLIENT_ID:?INFISICAL_STAGING_APP_CLIENT_ID is required}" + : "${INFISICAL_APP_CLIENT_SECRET:?INFISICAL_STAGING_APP_CLIENT_SECRET is required}" + : "${INFISICAL_SCHEDULER_CLIENT_ID:?INFISICAL_STAGING_SCHEDULER_CLIENT_ID is required}" + : "${INFISICAL_SCHEDULER_CLIENT_SECRET:?INFISICAL_STAGING_SCHEDULER_CLIENT_SECRET is required}" deployment_path="${CONFIGURED_DEPLOY_PATH:-/opt/shd-build-stage}" stage_port="${CONFIGURED_STAGE_PORT:-5569}" + case "$deployment_path" in + /opt/shd-build-*) ;; + *) echo "STAGE_DEPLOY_PATH must be beneath /opt/shd-build-." >&2; exit 1 ;; + esac + case "$stage_port" in + ''|*[!0-9]*) echo "STAGE_PORT must be an integer." >&2; exit 1 ;; + esac + if [ "$stage_port" -lt 1 ] || [ "$stage_port" -gt 65535 ]; then + echo "STAGE_PORT must be from 1 through 65535." >&2 + exit 1 + fi mkdir -p "$deployment_path" - cp deploy/compose.staging.yml deploy/deploy-staging.sh deploy/bootstrap-staging.sh "$deployment_path/" - DOTENV_PRIVATE_KEY_STAGING="$DOTENV_PRIVATE_KEY_STAGING" npx dotenvx decrypt -f .env.staging --stdout > "$deployment_path/.env.staging.runtime" - printf 'SHD_IMAGE=shd-build-stage:%s\nSHD_STAGE_PORT=%s\nSHD_BUILD_NUMBER=%s\n' "$COMMIT_SHA" "$stage_port" "$BUILD_NUMBER" > "$deployment_path/.env.deploy" - chmod 600 "$deployment_path/.env.staging.runtime" "$deployment_path/.env.deploy" - chmod 755 "$deployment_path/deploy-staging.sh" "$deployment_path/bootstrap-staging.sh" + release_id="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT:-1}" + candidate_path="$deployment_path/candidate-$release_id" + rm -rf "$candidate_path" + mkdir -p "$candidate_path" + cp deploy/compose.staging.yml deploy/deploy-staging.sh deploy/bootstrap-staging.sh "$candidate_path/" + write_credential() { + target="$1" + value="$2" + temporary_file=$(mktemp "${target%/*}/.infisical.XXXXXX") + printf '%s' "$value" > "$temporary_file" + chmod 600 "$temporary_file" + mv "$temporary_file" "$target" + } + write_credential "$candidate_path/.infisical.app.client-id" "$INFISICAL_APP_CLIENT_ID" + write_credential "$candidate_path/.infisical.app.client-secret" "$INFISICAL_APP_CLIENT_SECRET" + write_credential "$candidate_path/.infisical.scheduler.client-id" "$INFISICAL_SCHEDULER_CLIENT_ID" + write_credential "$candidate_path/.infisical.scheduler.client-secret" "$INFISICAL_SCHEDULER_CLIENT_SECRET" + temporary_deploy_environment=$(mktemp "$candidate_path/.env.deploy.XXXXXX") + printf 'SHD_IMAGE=shd-build-stage:%s-%s\nSHD_STAGE_PORT=%s\nSHD_BUILD_NUMBER=%s\nSHD_COMMIT_SHA=%s\n' "$COMMIT_SHA" "$release_id" "$stage_port" "$BUILD_NUMBER" "$COMMIT_SHA" > "$temporary_deploy_environment" + chmod 600 "$temporary_deploy_environment" + mv "$temporary_deploy_environment" "$candidate_path/.env.deploy" + chmod 755 "$candidate_path/deploy-staging.sh" "$candidate_path/bootstrap-staging.sh" - name: Deploy staging env: @@ -60,8 +96,12 @@ jobs: COMMIT_SHA: ${{ gitea.sha }} run: | deployment_path="${CONFIGURED_DEPLOY_PATH:-/opt/shd-build-stage}" - "$deployment_path/deploy-staging.sh" "$deployment_path" "$GITHUB_WORKSPACE" "shd-build-stage:$COMMIT_SHA" + release_id="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT:-1}" + candidate_path="$deployment_path/candidate-$release_id" + "$candidate_path/deploy-staging.sh" "$deployment_path" "$GITHUB_WORKSPACE" "shd-build-stage:$COMMIT_SHA-$release_id" "$candidate_path" - name: Remove unused application images if: success() - run: docker image prune --force --filter "label=org.opencontainers.image.title=shd-build" + run: + docker image prune --force --filter + "label=org.opencontainers.image.title=shd-build" diff --git a/.gitea/workflows/export-isac-data.yml b/.gitea/workflows/export-isac-data.yml index 05c57f4..1670013 100644 --- a/.gitea/workflows/export-isac-data.yml +++ b/.gitea/workflows/export-isac-data.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: concurrency: - group: shd-build-staging + group: shd-build-host-maintenance cancel-in-progress: false jobs: @@ -25,15 +25,32 @@ jobs: compose() { docker compose --env-file "$deploy_environment" -f "$compose_file" "$@" } - restart_services() { - compose up -d --no-build app scheduler - } - trap restart_services EXIT snapshot_name="isac-stage-${GITHUB_RUN_ID}.json" + volume_snapshot="/app/data/exports/$snapshot_name" + remove_volume_snapshot=false + cleanup() { + exit_status=$? + trap - EXIT + if [ "$remove_volume_snapshot" = true ]; then + compose run --rm --no-deps app rm -f "$volume_snapshot" || true + fi + if ! compose up -d --no-build app scheduler; then + if [ "$exit_status" -eq 0 ]; then exit_status=1; fi + fi + exit "$exit_status" + } + trap cleanup EXIT compose stop scheduler app - compose run --rm --no-deps app node scripts/isac-data-export.mjs "/app/data/exports/$snapshot_name" - compose run --rm --no-deps -v "$GITHUB_WORKSPACE:/export" app \ - cp "/app/data/exports/$snapshot_name" /export/isac-stage.json + app_container="$(compose ps --all --quiet app)" + if [ -z "$app_container" ]; then + echo "The stopped staging app container could not be found." >&2 + exit 1 + fi + remove_volume_snapshot=true + compose run --rm --no-deps app node scripts/isac-data-export.mjs "$volume_snapshot" + docker cp "$app_container:$volume_snapshot" "$GITHUB_WORKSPACE/isac-stage.json" + compose run --rm --no-deps app rm -f "$volume_snapshot" + remove_volume_snapshot=false - name: Publish ISAC snapshot uses: actions/upload-artifact@v3 diff --git a/.gitea/workflows/export-production-isac-data.yml b/.gitea/workflows/export-production-isac-data.yml index 3557e5c..932ba47 100644 --- a/.gitea/workflows/export-production-isac-data.yml +++ b/.gitea/workflows/export-production-isac-data.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: concurrency: - group: shd-build-production + group: shd-build-host-maintenance cancel-in-progress: false jobs: @@ -25,15 +25,32 @@ jobs: compose() { docker compose --env-file "$deploy_environment" -f "$compose_file" "$@" } - restart_services() { - compose up -d --no-build app scheduler - } - trap restart_services EXIT snapshot_name="isac-production-${GITHUB_RUN_ID}.json" + volume_snapshot="/app/data/exports/$snapshot_name" + remove_volume_snapshot=false + cleanup() { + exit_status=$? + trap - EXIT + if [ "$remove_volume_snapshot" = true ]; then + compose run --rm --no-deps app rm -f "$volume_snapshot" || true + fi + if ! compose up -d --no-build app scheduler; then + if [ "$exit_status" -eq 0 ]; then exit_status=1; fi + fi + exit "$exit_status" + } + trap cleanup EXIT compose stop scheduler app - compose run --rm --no-deps app node scripts/isac-data-export.mjs "/app/data/exports/$snapshot_name" - compose run --rm --no-deps -v "$GITHUB_WORKSPACE:/export" app \ - cp "/app/data/exports/$snapshot_name" /export/isac-production.json + app_container="$(compose ps --all --quiet app)" + if [ -z "$app_container" ]; then + echo "The stopped production app container could not be found." >&2 + exit 1 + fi + remove_volume_snapshot=true + compose run --rm --no-deps app node scripts/isac-data-export.mjs "$volume_snapshot" + docker cp "$app_container:$volume_snapshot" "$GITHUB_WORKSPACE/isac-production.json" + compose run --rm --no-deps app rm -f "$volume_snapshot" + remove_volume_snapshot=false - name: Publish production ISAC snapshot uses: actions/upload-artifact@v3 diff --git a/.gitea/workflows/sync-production-isac-to-staging.yml b/.gitea/workflows/sync-production-isac-to-staging.yml index 7d5e2ad..68ca0d4 100644 --- a/.gitea/workflows/sync-production-isac-to-staging.yml +++ b/.gitea/workflows/sync-production-isac-to-staging.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: concurrency: - group: shd-build-production-to-staging + group: shd-build-host-maintenance cancel-in-progress: false jobs: diff --git a/.gitignore b/.gitignore index cd26108..3abf46b 100644 --- a/.gitignore +++ b/.gitignore @@ -17,11 +17,10 @@ Thumbs.db .env.* !.env.example !.env.test -!.env.development -!.env.staging -!.env.production .env.keys .env.keys.* +.infisical.*.client-id +.infisical.*.client-secret # Embedded Turso database files /data/*.db diff --git a/.infisical.json b/.infisical.json new file mode 100644 index 0000000..8934ffd --- /dev/null +++ b/.infisical.json @@ -0,0 +1,6 @@ +{ + "workspaceId": "2ee94b27-e599-4194-9797-3372d42e53f6", + "defaultEnvironment": "dev", + "defaultSecretPath": "/app", + "domain": "https://secrets.jb.codes" +} diff --git a/AGENTS.md b/AGENTS.md index c9fcb44..01cc9f2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,39 +2,70 @@ ## Parallel task execution -For substantial changes, use subagents as a standard practice when the work can be divided into concrete, independent tasks that provide real parallel progress. +For substantial changes, use subagents as a standard practice when the work can +be divided into concrete, independent tasks that provide real parallel progress. -1. Delegate bounded mechanical work to faster, lower-reasoning agents (focused code discovery, repetitive call-site updates, documentation, isolated test coverage). -2. Keep architecture, security-sensitive behavior, database migrations, shared data-model decisions, and complex debugging with a suitably capable agent. -3. Assign explicit file or subsystem ownership before parallel edits. Avoid having multiple agents edit the same files unless their work is deliberately sequenced. -4. Do not delegate tightly coupled work merely to increase agent count. Keep it with the primary agent when coordination would cost more than it saves. -5. The primary agent remains responsible for integrating results, reviewing diffs, resolving cross-cutting concerns, and running the final required validation. -6. Subagents must follow every repository rule that applies to their assigned work, including design-system maintenance, audit logging, and database preservation. +1. Delegate bounded mechanical work to faster, lower-reasoning agents (focused + code discovery, repetitive call-site updates, documentation, isolated test + coverage). +2. Keep architecture, security-sensitive behavior, database migrations, shared + data-model decisions, and complex debugging with a suitably capable agent. +3. Assign explicit file or subsystem ownership before parallel edits. Avoid + having multiple agents edit the same files unless their work is deliberately + sequenced. +4. Do not delegate tightly coupled work merely to increase agent count. Keep it + with the primary agent when coordination would cost more than it saves. +5. The primary agent remains responsible for integrating results, reviewing + diffs, resolving cross-cutting concerns, and running the final required + validation. +6. Subagents must follow every repository rule that applies to their assigned + work, including design-system maintenance, audit logging, and database + preservation. ## Documentation prose -Use the `stop-slop` skill whenever an agent creates or changes project documentation. Apply its review to the complete affected document, including sections that the change did not introduce. Preserve technical meaning, commands, paths, formulas, links, policy strength, and product terminology. Remove filler, vague claims, formulaic contrasts, manufactured emphasis, false agency, and repetition. Prefer active voice and specific subjects. Do not weaken terms such as `must`, `never`, or `always` when they express a real requirement. +Use the `stop-slop` skill whenever an agent creates or changes project +documentation. Apply its review to the complete affected document, including +sections that the change did not introduce. Preserve technical meaning, +commands, paths, formulas, links, policy strength, and product terminology. +Remove filler, vague claims, formulaic contrasts, manufactured emphasis, false +agency, and repetition. Prefer active voice and specific subjects. Do not weaken +terms such as `must`, `never`, or `always` when they express a real requirement. ## Dependency source research -Use `npx opensrc` before inspecting an npm dependency's implementation. It resolves the package version from this workspace's lockfile, shallow-clones the matching source revision, and caches it outside the repository at `~/.opensrc/`. +Use `npx opensrc` before inspecting an npm dependency's implementation. It +resolves the package version from this workspace's lockfile, shallow-clones the +matching source revision, and caches it outside the repository at `~/.opensrc/`. -- Do not guess at dependency behavior or implementation details. Identify the needed function, type, option, or control flow in the cached source and use that evidence when designing a change. -- Search the smallest relevant source area first. Use the package entry point, exports, types, and targeted code search to locate the behavior instead of reading unrelated files. -- Fetch source before a dependency investigation: `npx opensrc fetch --cwd . `. -- Print the cached source path for use with repository search and file tools: `npx opensrc path --cwd . `. -- Fetch several declared packages in one command: `npx opensrc fetch --cwd . `. -- After adding or upgrading a direct dependency, fetch its source at the locked version. Do not commit `~/.opensrc/` or copy dependency source into this repository. -- Use `npx opensrc list` to inspect the cache. Use `npx opensrc remove ` or `npx opensrc clean` only when cache removal is intended. +- Do not guess at dependency behavior or implementation details. Identify the + needed function, type, option, or control flow in the cached source and use + that evidence when designing a change. +- Search the smallest relevant source area first. Use the package entry point, + exports, types, and targeted code search to locate the behavior instead of + reading unrelated files. +- Fetch source before a dependency investigation: + `npx opensrc fetch --cwd . `. +- Print the cached source path for use with repository search and file tools: + `npx opensrc path --cwd . `. +- Fetch several declared packages in one command: + `npx opensrc fetch --cwd . `. +- After adding or upgrading a direct dependency, fetch its source at the locked + version. Do not commit `~/.opensrc/` or copy dependency source into this + repository. +- Use `npx opensrc list` to inspect the cache. Use + `npx opensrc remove ` or `npx opensrc clean` only when cache removal + is intended. ## Available skills The `.agents/skills/` directory contains repo-specific skill files. Key skills: -- `stop-slop` — apply to documentation prose changes -- `frontend-design` / `impeccable` — UI design, refinement, and polish -- `import-talent-screenshot` / `import-weapon-screenshot` — extract Division Resurgence game data from screenshots and enter through ISAC -- `webapp-testing` — Playwright-based local web app testing +- `stop-slop`: apply to documentation prose changes +- `frontend-design` / `impeccable`: UI design, refinement, and polish +- `import-talent-screenshot` / `import-weapon-screenshot`: extract Division + Resurgence game data from screenshots and enter through ISAC +- `webapp-testing`: Playwright-based local web app testing Use `skill` tool to load one when the task matches. @@ -44,103 +75,183 @@ Run in order after source changes: ``` npm run check # svelte-kit sync + svelte-check (typecheck) -npm run build # dotenvx production env + vite build +npm run build # secretless Vite production build ``` -`npm run check` runs `svelte-kit sync` (generates `.svelte-kit/tsconfig.json`) then `svelte-check --tsconfig ./tsconfig.json`. `npm test` runs all vitest tests; there is no separate vitest config (vitest inherits from `vite.config.ts`). +`npm run check` runs `svelte-kit sync` (generates `.svelte-kit/tsconfig.json`) +then `svelte-check --tsconfig ./tsconfig.json`. `npm test` runs all vitest +tests; there is no separate vitest config (vitest inherits from +`vite.config.ts`). -Test files (`*.test.ts`) live adjacent to the code they test, mostly in `src/lib/`. Most server-unit tests mock `$lib/server/database` via `vi.mock(...)` — unnecessary for unit tests against pure logic. Integration tests that need database state use the embedded Turso engine. +Test files (`*.test.ts`) live adjacent to the code they test, mostly in +`src/lib/`. Most server-unit tests mock `$lib/server/database` via +`vi.mock(...)`; pure logic tests do not need that mock. Integration tests that +need database state use the embedded Turso engine. ## Framework and toolchain quirks -- **SvelteKit 2 / Svelte 5** with forced runes mode (configured in `vite.config.ts` via plugin `compilerOptions.runes`). There is no `svelte.config.js` — SvelteKit config lives in `vite.config.ts`. -- **Node adapter** (`@sveltejs/adapter-node`). Production entry point is `node build`. The container runs migrations on startup, then serves. -- **Embedded Turso** (`@tursodatabase/database`) — local SQLite, no Turso Cloud, no libSQL HTTP, no replicas. The dev database path is `./data/shd-build-development.db` (set in `.env.development`). -- **Passkey auth** (`@simplewebauthn`) is preferred. `APP_ORIGIN` and `PASSKEY_RP_ID` must match the public origin. -- **Environment files** are encrypted with `dotenvx`. Committed `.env.development`, `.env.staging`, `.env.production` are encrypted; `.env.keys` is git-ignored. The pre-commit hook runs `npm run env:stage` (encrypts and git-adds). To edit: `npm run env:decrypt`, make changes, `npm run env:stage`. +- **SvelteKit 2 / Svelte 5** with forced runes mode (configured in + `vite.config.ts` via plugin `compilerOptions.runes`). There is no + `svelte.config.js`; SvelteKit config lives in `vite.config.ts`. +- **Node adapter** (`@sveltejs/adapter-node`). Production entry point is + `node build`. The container runs migrations on startup, then serves. +- **Embedded Turso** (`@tursodatabase/database`): local SQLite, no Turso Cloud, + no libSQL HTTP, no replicas. The Infisical `dev` environment sets the database + path to `./data/shd-build-development.db`. +- **Passkey auth** (`@simplewebauthn`) is preferred. `APP_ORIGIN` must match the + public origin. `PASSKEY_RP_ID` must be that hostname or a registrable parent + domain; hosted environments use `shd.build`. +- **Infisical secrets** live in project `2ee94b27-e599-4194-9797-3372d42e53f6` + at `https://secrets.jb.codes`. Application values use `/app`; + `SCHEDULER_TOKEN` uses `/shared/scheduler`. Local commands use the + authenticated Infisical CLI and the `dev` environment. Deployed app and + scheduler containers use separate Universal Auth bootstrap credentials mounted + through Compose secrets. The root entrypoint exchanges those credentials for + one-shot JSON exports, deletes the export files, and starts the workload as + the dedicated `shd-runtime` UID 10001 without Infisical credentials or tokens. + The self-hosted Free plan cannot scope Viewer identities by environment or + path. - **Argon2id** password hashing with server-side sessions and CSRF protection. -- **ISAC namespace**: `/isac` is the admin console; `/admin` redirects there via 308. +- **ISAC namespace**: `/isac` is the admin console; `/admin` redirects there + via 308. ## Development server lifecycle -After changing source code, styles, configuration, dependencies, or routes, restart the development server before handing work back. +After changing source code, styles, configuration, dependencies, or routes, +restart the development server before handing work back. -1. Inspect running processes and stop only this project's development-server process tree. -2. Start one hidden `npm run dev` process from the repository root. Do not leave duplicate project development servers running. +1. Inspect running processes and stop only this project's development-server + process tree. +2. Start one hidden `npm run dev` process from the repository root. Do not leave + duplicate project development servers running. 3. Confirm the server accepts an HTTP request before completing the task. -4. Leave the verified development server running so the project owner can review the changes. +4. Leave the verified development server running so the project owner can review + the changes. -Database maintenance may require the server to remain stopped while the embedded database is locked. Restart and verify the server after maintenance completes. +Database maintenance may require the server to remain stopped while the embedded +database is locked. Restart and verify the server after maintenance completes. ## SHD design-system maintenance -Treat `docs/DESIGN_SYSTEM.md` and the shared tokens in `src/app.css` as maintained parts of the product. When a change makes a significant visual or interaction-design decision, update the design system in the same change. +Treat `docs/DESIGN_SYSTEM.md` and the shared tokens in `src/app.css` as +maintained parts of the product. When a change makes a significant visual or +interaction-design decision, update the design system in the same change. -For significant design changes: reuse or extend CSS custom properties from `src/app.css`; update `docs/DESIGN_SYSTEM.md`; refactor repeated UI into a reusable component; verify hover, pressed, focus-visible, pending, disabled, error, success, and reduced-motion behavior. Run `npm run check && npm run build` before completing. +For significant design changes: reuse or extend CSS custom properties from +`src/app.css`; update `docs/DESIGN_SYSTEM.md`; refactor repeated UI into a +reusable component; verify hover, pressed, focus-visible, pending, disabled, +error, success, and reduced-motion behavior. Run +`npm run check && npm run build` before completing. -Design principles: tactical but restrained, data-first, square and engineered, orange reserved for action/emphasis, explicit textual state indicators. +Design principles: tactical but restrained, data-first, square and engineered, +orange reserved for action/emphasis, explicit textual state indicators. -Minor alignment fixes, isolated bug fixes, and corrections that do not introduce a reusable design decision do not require a documentation update. +Minor alignment fixes, isolated bug fixes, and corrections that do not introduce +a reusable design decision do not require a documentation update. ## Reusable interface patterns -Favor shared components from `src/lib/components`. Before adding page-local markup, check if a shared component covers the need. When introducing a reusable pattern, migrate the clearest existing duplicate in the same change. +Favor shared components from `src/lib/components`. Before adding page-local +markup, check if a shared component covers the need. When introducing a reusable +pattern, migrate the clearest existing duplicate in the same change. -For small, finite option sets, prefer the shared segmented-choice pattern over a dropdown. Use radio semantics for exactly one choice and checkbox semantics for additive independent choices. Reserve dropdowns for long, dynamic, or search-oriented option sets. +For small, finite option sets, prefer the shared segmented-choice pattern over a +dropdown. Use radio semantics for exactly one choice and checkbox semantics for +additive independent choices. Reserve dropdowns for long, dynamic, or +search-oriented option sets. ## ISAC editor lifecycle All ISAC data-management editors must follow the same successful-save behavior: -1. Saving an edit to an existing record closes the editor and returns the user to the registry. +1. Saving an edit to an existing record closes the editor and returns the user + to the registry. 2. Saving a newly added record keeps the editor open with a fresh blank record. 3. Validation errors keep the editor open and preserve the entered values. -4. Server actions must return the add/edit mode and a unique result token; client-side reactive handling must consume each token only once to avoid repeated state updates. -5. When a successful add keeps an icon-enabled editor open, increment the shared upload reset token so the file input, preview, filename, drag/paste state, validation message, and removal state are fresh for the next record. +4. Server actions must return the add/edit mode and a unique result token; + client-side reactive handling must consume each token only once to avoid + repeated state updates. +5. When a successful add keeps an icon-enabled editor open, increment the shared + upload reset token so the file input, preview, filename, drag/paste state, + validation message, and removal state are fresh for the next record. Apply this convention whenever an ISAC editor is added or materially changed. ## ISAC audit logging -Every successful server-side ISAC mutation must create an append-only audit event. Follow these rules: +Every successful server-side ISAC mutation must create an append-only audit +event. Follow these rules: -1. Use the shared audit utilities in `src/lib/server/audit.ts`; do not write directly to `audit_log`. -2. Record the authenticated operator ID, stable category, specific action (namespaced past tense, e.g. `weapon.updated`), target entity ID, timestamp, and useful before/after snapshots. Categories include `game_data`, `user_access`, `user_moderation`, `identity`, `role_management`. -3. Capture changes to the primary row and its relationships. Include meaningful previous and resulting sets for linked records. -4. Create audit records only after validation succeeds. Rejected or failed requests must not appear as successful mutation events. -5. Never include password hashes, session tokens, magic-link tokens, OAuth state, passkey public-key bytes, SMTP credentials, or environment secrets in audit snapshots. -6. Audit visibility is controlled by `audit.view`. The read-only `Auditor` role grants `isac.access` and `audit.view` without granting mutation permissions. -7. Audit reversion is a compensating action, never a history rewrite. Require `audit.revert`, confirmation, current-state comparison, relationship validation, and a new `audit.reverted` event. -8. Tests for an ISAC mutation must confirm its audit event contains the correct actor, category, action, target, and change evidence. +1. Use the shared audit utilities in `src/lib/server/audit.ts`; do not write + directly to `audit_log`. +2. Record the authenticated operator ID, stable category, specific action + (namespaced past tense, e.g. `weapon.updated`), target entity ID, timestamp, + and useful before/after snapshots. Categories include `game_data`, + `user_access`, `user_moderation`, `identity`, `role_management`. +3. Capture changes to the primary row and its relationships. Include meaningful + previous and resulting sets for linked records. +4. Create audit records only after validation succeeds. Rejected or failed + requests must not appear as successful mutation events. +5. Never include password hashes, session tokens, magic-link tokens, OAuth + state, passkey public-key bytes, SMTP credentials, or environment secrets in + audit snapshots. +6. Audit visibility is controlled by `audit.view`. The read-only `Auditor` role + grants `isac.access` and `audit.view` without granting mutation permissions. +7. Audit reversion is a compensating action, never a history rewrite. Require + `audit.revert`, confirmation, current-state comparison, relationship + validation, and a new `audit.reverted` event. +8. Tests for an ISAC mutation must confirm its audit event contains the correct + actor, category, action, target, and change evidence. ## Prototype compatibility -Treat application data and client payloads as prototype formats until the project owner declares a production-stable format. +Treat application data and client payloads as prototype formats until the +project owner declares a production-stable format. -1. Prefer one current validated schema over compatibility readers, repair branches, dual writes, or fallback parsing for obsolete formats. -2. Remove incompatible local drafts, saved builds, calibration snapshots, and other derived records through a backed-up migration when the current model cannot represent them safely. -3. Preserve authored game data whenever the new model can retain it without guessing. -4. Follow the database-preservation rules before removing or reshaping persistent data. -5. Add compatibility support only when the project owner requires it for a production-stable format. +1. Prefer one current validated schema over compatibility readers, repair + branches, dual writes, or fallback parsing for obsolete formats. +2. Remove incompatible local drafts, saved builds, calibration snapshots, and + other derived records through a backed-up migration when the current model + cannot represent them safely. +3. Preserve authored game data whenever the new model can retain it without + guessing. +4. Follow the database-preservation rules before removing or reshaping + persistent data. +5. Add compatibility support only when the project owner requires it for a + production-stable format. ## Talent mechanic conventions -Before importing a Talent or changing its mechanic program, read `docs/TALENT_MECHANIC_RULES.md`. Treat the confirmed lifecycle, trigger, cooldown, and targeting rules in that document as authoritative game behavior. Do not mark a mechanic `requires_assumptions` for behavior that the document resolves. When the project owner clarifies another Talent mechanic, record the answer in `docs/TALENT_MECHANIC_RULES.md`. +Before importing a Talent or changing its mechanic program, read +`docs/TALENT_MECHANIC_RULES.md`. Treat the confirmed lifecycle, trigger, +cooldown, and targeting rules in that document as authoritative game behavior. +Do not mark a mechanic `requires_assumptions` for behavior that the document +resolves. When the project owner clarifies another Talent mechanic, record the +answer in `docs/TALENT_MECHANIC_RULES.md`. ## Database preservation -Development data is valuable. Treat embedded database files as persistent user data, not disposable build artifacts. +Treat embedded database files as persistent user data, not disposable build +artifacts. -Before destructive work on the database, stop the dev server, then create a verified timestamped JSON backup with `npm run db:backup`. Keep backups out of Git (they can contain user records). Prefer additive, repeatable schema migrations over database recreation. Report the backup path when completing database-destructive work. +Before destructive work on the database, stop the dev server, then create a +verified timestamped JSON backup with `npm run db:backup`. Keep backups out of +Git (they can contain user records). Prefer additive, repeatable schema +migrations over database recreation. Report the backup path when completing +database-destructive work. -Use `npm run db:restore -- ` only with an empty, migrated target database — it refuses to merge into a populated database. +Use `npm run db:restore -- ` only with an empty, migrated target +database. The command refuses to merge into a populated database. ## Development database maintenance 1. Stop only this project's development-server process tree. 2. Run and verify `npm run db:backup`. Retain and report the backup path. -3. Apply the migration or maintenance operation. Verify schema state, record counts, and critical entities. -4. If an operation fails, preserve logs and the verified backup rather than improvising against the database. +3. Apply the migration or maintenance operation. Verify schema state, record + counts, and critical entities. +4. If an operation fails, preserve logs and the verified backup rather than + improvising against the database. 5. Restart the dev server, then confirm a basic health or page check. -Production and staging backups and deployment are defined in `docs/DEPLOYMENT.md`. +Production and staging backups and deployment are defined in +`docs/DEPLOYMENT.md`. diff --git a/Dockerfile b/Dockerfile index 03efdd2..a43a522 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM node:24-bookworm-slim AS dependencies WORKDIR /app COPY package.json package-lock.json .npmrc ./ COPY scripts/prepare.mjs ./scripts/prepare.mjs -RUN npm ci +RUN npm ci --ignore-scripts && npm rebuild argon2 FROM dependencies AS build COPY vite.config.ts tsconfig.json ./ @@ -15,24 +15,41 @@ FROM node:24-bookworm-slim AS runtime-dependencies WORKDIR /app COPY package.json package-lock.json .npmrc ./ COPY scripts/prepare.mjs ./scripts/prepare.mjs -RUN npm ci --omit=dev && npm cache clean --force +RUN npm ci --omit=dev --ignore-scripts && npm rebuild argon2 && npm cache clean --force FROM node:24-bookworm-slim AS runtime +ARG TARGETARCH=amd64 +ARG INFISICAL_CLI_VERSION=0.43.116 LABEL org.opencontainers.image.title="shd-build" ENV NODE_ENV=production \ HOST=0.0.0.0 \ PORT=3000 \ + INFISICAL_DOMAIN=https://secrets.jb.codes \ + INFISICAL_DISABLE_UPDATE_CHECK=true \ TDR_DATABASE_PATH=/app/data/shd-build.db \ TDR_AVATAR_STORAGE_PATH=/app/data/avatars \ TDR_GAME_ICON_STORAGE_PATH=/app/data/game-icons WORKDIR /app +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl gosu \ + && useradd --uid 10001 --gid node --no-create-home --shell /usr/sbin/nologin shd-runtime \ + && case "$TARGETARCH" in \ + amd64) infisical_sha256=53abc96a3bc872ade81efe156275bd30c53f9ee8b52965b0c9cb320a9c814fbc ;; \ + arm64) infisical_sha256=0111d8e7daa5f3c498a8cc6cb9852bde2c5c21e53c935d4738ddd80c764554cc ;; \ + *) echo "Unsupported Infisical CLI architecture: $TARGETARCH" >&2; exit 1 ;; \ + esac \ + && curl -fsSL "https://github.com/Infisical/cli/releases/download/v${INFISICAL_CLI_VERSION}/cli_${INFISICAL_CLI_VERSION}_linux_${TARGETARCH}.tar.gz" -o /tmp/infisical.tar.gz \ + && echo "$infisical_sha256 /tmp/infisical.tar.gz" | sha256sum -c - \ + && tar -xzf /tmp/infisical.tar.gz -C /usr/local/bin infisical \ + && chmod 755 /usr/local/bin/infisical \ + && rm -f /tmp/infisical.tar.gz \ + && rm -rf /var/lib/apt/lists/* COPY --from=runtime-dependencies --chown=node:node /app/node_modules ./node_modules COPY --from=build --chown=node:node /app/build ./build COPY --chown=node:node database ./database COPY --chown=node:node scripts ./scripts COPY --chown=node:node deploy/container-entrypoint.sh ./deploy/container-entrypoint.sh -RUN chmod 755 ./deploy/container-entrypoint.sh && mkdir -p /app/data && chown node:node /app/data -USER node +RUN chmod 755 ./deploy/container-entrypoint.sh ./scripts/runtime-launcher.mjs && mkdir -p /app/data && chown node:node /app/data && chmod 0770 /app/data EXPOSE 3000 VOLUME ["/app/data"] ENTRYPOINT ["./deploy/container-entrypoint.sh"] diff --git a/README.md b/README.md index d3e3cc3..a0de5fe 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ Create or update the local database: npm run db:migrate ``` -The default file is `data/shd-build.db`. Copy `.env.example` to `.env` and set a -different location if needed. +The Infisical `dev` environment sets the local file to +`./data/shd-build-development.db`. Store custom avatars as metadata-free 256 × 256 WebP files. Set `TDR_AVATAR_STORAGE_PATH` to a persistent, writable directory in production; it @@ -30,9 +30,10 @@ Seed the initial local accounts once: npm run db:seed-users ``` -The seed creates administrator `Aewyn` and standard user `Quartermaster`. It +The seed creates `Aewyn` and `Quartermaster` with the standard user role. It generates and displays strong temporary passwords once, then stores only salted -Argon2id hashes. +Argon2id hashes. The separate bootstrap policy promotes the first authenticated +account when no administrator exists. ## Develop @@ -40,30 +41,28 @@ Argon2id hashes. npm run dev ``` -## Encrypted environment files +## Infisical secrets -Keep development, staging, and production settings in the committed, encrypted -`.env.development`, `.env.staging`, and `.env.production` files. Keep private -decryption keys only in the ignored `.env.keys` file and back it up in a -password manager. +The committed `.infisical.json` connects the project to the self-hosted +Infisical instance at `https://secrets.jb.codes`. Authenticate the local CLI +once. The CLI uses its configured system vault and can fall back to an encrypted +file vault when the operating-system keyring is unavailable. ```sh -npm run env:decrypt # decrypt all environment files for editing -npm run env:encrypt # encrypt all environment files again -npm run env:stage # encrypt and git-add all environment files -npm run env:decrypt:development # operate on one environment -npm run env:decrypt:staging -npm run env:decrypt:production -npm run env:sync:staging-credentials # copy shared Discord/SMTP values from development +npx infisical login --domain=https://secrets.jb.codes +npm run dev ``` -The repository pre-commit hook runs `env:stage` and replaces staged plaintext -environment files with encrypted versions. Git has no pre-add hook. Use -`npm run env:stage` instead of directly adding these two files. +Local commands use the `dev` environment. Application configuration lives under +`/app`; the scheduler bearer token lives under `/shared/scheduler`. Staging and +production commands select the `staging` and `prod` environments explicitly. +`npm run check`, `npm test`, and `npm run build` do not contact Infisical. -Keep `.env.keys` out of Git and retain a backup. Production deployments receive -`DOTENV_PRIVATE_KEY_PRODUCTION` through their secret manager; do not copy the -keys file to the server. +Staging and production containers authenticate at startup with separate +Universal Auth identities. Gitea stores each client secret, while repository +variables hold the matching non-secret client IDs. The containers fetch current +values before migration or startup and stop if authentication or validation +fails. ## Containers and staging deployment @@ -72,23 +71,25 @@ The application uses the official SvelteKit Node adapter and a multi-stage game-data icons under `/app/data` on a persistent volume. Inject runtime secrets when the container starts; keep them out of the image. -[docs/DEPLOYMENT.md](docs/DEPLOYMENT.md) documents the staging Compose topology -and Gitea Actions deployment. A push to `main` validates the application, builds -the candidate image while staging stays online, stops the application and -scheduler for the database backup and migration, then starts the release. +[docs/DEPLOYMENT.md](docs/DEPLOYMENT.md) documents the staging and production +Compose topologies. A push to `main` triggers both deployment workflows. Each +workflow validates Infisical access and the candidate image while the current +service stays online, then stops its application and scheduler for backup and +migration. ## Passwordless authentication Use passkeys as the preferred sign-in method. Set `APP_ORIGIN` to the exact public origin for registration and authentication. Set `PASSKEY_RP_ID` to that hostname or a registrable parent domain. Hosted environments use `shd.build`, -which permits credentials on `shd.build` and `dev.shd.build`. Use `localhost` -for direct local development. +which permits credentials on `shd.build`, `stage.shd.build`, and +`dev.shd.build`. Use `localhost` for direct local development. -Magic links require `SMTP_HOST`, `SMTP_PORT`, `SMTP_SECURE`, `SMTP_USER`, -`SMTP_PASSWORD`, and `SMTP_FROM`. Use `SMTP_SECURE=true` on port 465. Use -`false` on port 587 and upgrade with STARTTLS. An account owner activates an -email identity by following a 15-minute, single-use verification link. +Magic links require `SMTP_HOST` and `SMTP_FROM`. `SMTP_PORT` defaults to `587`. +Set `SMTP_SECURE=true` for implicit TLS, normally on port 465; the default +`false` uses STARTTLS. If the SMTP server requires authentication, set both +`SMTP_USER` and `SMTP_PASSWORD`. An account owner activates an email identity by +following a 15-minute, single-use verification link. ## Validate @@ -97,47 +98,6 @@ npm run check npm run build ``` -[`sv`](https://github.com/sveltejs/cli) creates and configures Svelte projects. - -## Creating a project - -```sh -# create a new project -npx sv create my-app -``` - -To recreate this project with the same configuration: - -```sh -# recreate this project -npx sv@0.16.5 create --template minimal --types ts --install npm . -``` - -## Developing - -Install dependencies with `npm install` (or `pnpm install` or `yarn`), then -start a development server: - -```sh -npm run dev - -# or start the server and open the app in a new browser tab -npm run dev -- --open -``` - -## Building - -Build a production version: - -```sh -npm run build -``` - -Preview the production build with `npm run preview`. - -> Deployments may require an [adapter](https://svelte.dev/docs/kit/adapters) for -> the target environment. - The shared visual language and dashboard conventions are documented in [docs/DESIGN_SYSTEM.md](docs/DESIGN_SYSTEM.md). diff --git a/deploy/bootstrap-production.sh b/deploy/bootstrap-production.sh index f2f5caa..94ea75b 100644 --- a/deploy/bootstrap-production.sh +++ b/deploy/bootstrap-production.sh @@ -7,23 +7,45 @@ if [ -z "$bundle_path" ] || [ ! -f "$bundle_path" ]; then echo "Usage: bootstrap-production.sh [deployment-directory]" >&2 exit 2 fi +bundle_path=$(readlink -f "$bundle_path") compose_file="$deployment_directory/compose.production.yml" deploy_environment="$deployment_directory/.env.deploy" +volume_bundle_path=/app/data/.production-bootstrap-import.json +remove_volume_bundle=false compose() { docker compose --env-file "$deploy_environment" -f "$compose_file" "$@" } -restart_services() { - compose up -d --no-build app scheduler +cleanup() { + status=$? + trap - EXIT HUP INT TERM + set +e + if [ "$remove_volume_bundle" = true ]; then + compose run --rm --no-deps --entrypoint rm --user root app -f "$volume_bundle_path" + fi + if ! compose up -d --no-build app scheduler; then status=1; fi + exit "$status" } -trap restart_services EXIT +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM echo "Stopping production before the one-time database bootstrap." -compose stop scheduler app || true +compose stop scheduler app +compose create app >/dev/null +app_container=$(compose ps --all --quiet app) +if [ -z "$app_container" ]; then + echo "The stopped production app container could not be found." >&2 + exit 1 +fi +docker cp "$bundle_path" "$app_container:$volume_bundle_path" +remove_volume_bundle=true +compose run --rm --no-deps --entrypoint sh --user root app -c \ + 'chown shd-runtime:node "$1" && chmod 600 "$1"' sh "$volume_bundle_path" compose run --rm --no-deps \ -e ALLOW_PRODUCTION_BOOTSTRAP=true \ - -v "$bundle_path:/import/production-bootstrap.json:ro" \ app node scripts/environment-bootstrap-import.mjs \ - /import/production-bootstrap.json --target production --confirm-empty-target-replace + "$volume_bundle_path" --target production --confirm-empty-target-replace echo "Production bootstrap completed." diff --git a/deploy/bootstrap-staging.sh b/deploy/bootstrap-staging.sh index 88c2dd1..f7602ac 100644 --- a/deploy/bootstrap-staging.sh +++ b/deploy/bootstrap-staging.sh @@ -7,23 +7,45 @@ if [ -z "$bundle_path" ] || [ ! -f "$bundle_path" ]; then echo "Usage: bootstrap-staging.sh [deployment-directory]" >&2 exit 2 fi +bundle_path=$(readlink -f "$bundle_path") compose_file="$deployment_directory/compose.staging.yml" deploy_environment="$deployment_directory/.env.deploy" +volume_bundle_path=/app/data/.staging-bootstrap-import.json +remove_volume_bundle=false compose() { docker compose --env-file "$deploy_environment" -f "$compose_file" "$@" } -restart_services() { - compose up -d --no-build app scheduler +cleanup() { + status=$? + trap - EXIT HUP INT TERM + set +e + if [ "$remove_volume_bundle" = true ]; then + compose run --rm --no-deps --entrypoint rm --user root app -f "$volume_bundle_path" + fi + if ! compose up -d --no-build app scheduler; then status=1; fi + exit "$status" } -trap restart_services EXIT +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM echo "Stopping staging before the one-time database bootstrap." -compose stop scheduler app || true +compose stop scheduler app +compose create app >/dev/null +app_container=$(compose ps --all --quiet app) +if [ -z "$app_container" ]; then + echo "The stopped staging app container could not be found." >&2 + exit 1 +fi +docker cp "$bundle_path" "$app_container:$volume_bundle_path" +remove_volume_bundle=true +compose run --rm --no-deps --entrypoint sh --user root app -c \ + 'chown shd-runtime:node "$1" && chmod 600 "$1"' sh "$volume_bundle_path" compose run --rm --no-deps \ -e ALLOW_STAGING_BOOTSTRAP=true \ - -v "$bundle_path:/import/staging-bootstrap.json:ro" \ app node scripts/stage-bootstrap-import.mjs \ - /import/staging-bootstrap.json --confirm-empty-staging-replace + "$volume_bundle_path" --confirm-empty-staging-replace echo "Staging bootstrap completed." diff --git a/deploy/compose.production.yml b/deploy/compose.production.yml index 361b97e..563f150 100644 --- a/deploy/compose.production.yml +++ b/deploy/compose.production.yml @@ -5,26 +5,29 @@ services: image: ${SHD_IMAGE:-shd-build-production:local} restart: unless-stopped init: true - env_file: - - .env.production.runtime environment: - HOST: 0.0.0.0 - PORT: 3000 - BODY_SIZE_LIMIT: 6M - SKIP_DEFAULT_USERS: 'true' - TDR_DATABASE_PATH: /app/data/shd-build-production.db - TDR_AVATAR_STORAGE_PATH: /app/data/avatars - TDR_GAME_ICON_STORAGE_PATH: /app/data/game-icons + INFISICAL_PROJECT_ID: 2ee94b27-e599-4194-9797-3372d42e53f6 + INFISICAL_ENVIRONMENT: prod + INFISICAL_RUNTIME_SCOPE: app + INFISICAL_UNIVERSAL_AUTH_CLIENT_ID_FILE: /run/secrets/infisical_client_id + INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET_FILE: /run/secrets/infisical_client_secret SHD_BUILD_NUMBER: ${SHD_BUILD_NUMBER:-0} - APP_ORIGIN: https://shd.build - ORIGIN: https://shd.build - PASSKEY_RP_ID: shd.build - DISCORD_REDIRECT_URI: https://shd.build/auth/discord/callback - UBISOFT_NEWS_IMPORT_ENABLED: 'true' + SHD_COMMIT_SHA: ${SHD_COMMIT_SHA:-} ports: - '0.0.0.0:${SHD_PRODUCTION_PORT:-3102}:3000' volumes: - shd_production_data:/app/data + secrets: + - source: infisical_app_client_id + target: infisical_client_id + uid: '0' + gid: '0' + mode: 0400 + - source: infisical_app_client_secret + target: infisical_client_secret + uid: '0' + gid: '0' + mode: 0400 healthcheck: test: - CMD @@ -43,10 +46,23 @@ services: init: true command: - scheduler - env_file: - - .env.production.runtime environment: - SCHEDULER_URL: http://app:3000/api/internal/scheduler + INFISICAL_PROJECT_ID: 2ee94b27-e599-4194-9797-3372d42e53f6 + INFISICAL_ENVIRONMENT: prod + INFISICAL_RUNTIME_SCOPE: scheduler + INFISICAL_UNIVERSAL_AUTH_CLIENT_ID_FILE: /run/secrets/infisical_client_id + INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET_FILE: /run/secrets/infisical_client_secret + secrets: + - source: infisical_scheduler_client_id + target: infisical_client_id + uid: '0' + gid: '0' + mode: 0400 + - source: infisical_scheduler_client_secret + target: infisical_client_secret + uid: '0' + gid: '0' + mode: 0400 depends_on: app: condition: service_healthy @@ -55,3 +71,13 @@ services: volumes: shd_production_data: name: shd-build-production-data + +secrets: + infisical_app_client_id: + file: .infisical.app.client-id + infisical_app_client_secret: + file: .infisical.app.client-secret + infisical_scheduler_client_id: + file: .infisical.scheduler.client-id + infisical_scheduler_client_secret: + file: .infisical.scheduler.client-secret diff --git a/deploy/compose.staging.yml b/deploy/compose.staging.yml index 80e1bf3..cefbe10 100644 --- a/deploy/compose.staging.yml +++ b/deploy/compose.staging.yml @@ -5,22 +5,29 @@ services: image: ${SHD_IMAGE:-shd-build-stage:local} restart: unless-stopped init: true - env_file: - - .env.staging.runtime environment: - HOST: 0.0.0.0 - PORT: 3000 - BODY_SIZE_LIMIT: 6M - SKIP_DEFAULT_USERS: 'true' - TDR_DATABASE_PATH: /app/data/shd-build-staging.db - TDR_AVATAR_STORAGE_PATH: /app/data/avatars - TDR_GAME_ICON_STORAGE_PATH: /app/data/game-icons + INFISICAL_PROJECT_ID: 2ee94b27-e599-4194-9797-3372d42e53f6 + INFISICAL_ENVIRONMENT: staging + INFISICAL_RUNTIME_SCOPE: app + INFISICAL_UNIVERSAL_AUTH_CLIENT_ID_FILE: /run/secrets/infisical_client_id + INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET_FILE: /run/secrets/infisical_client_secret SHD_BUILD_NUMBER: ${SHD_BUILD_NUMBER:-0} - UBISOFT_NEWS_IMPORT_ENABLED: 'false' + SHD_COMMIT_SHA: ${SHD_COMMIT_SHA:-} ports: - '0.0.0.0:${SHD_STAGE_PORT:-5569}:3000' volumes: - shd_stage_data:/app/data + secrets: + - source: infisical_app_client_id + target: infisical_client_id + uid: '0' + gid: '0' + mode: 0400 + - source: infisical_app_client_secret + target: infisical_client_secret + uid: '0' + gid: '0' + mode: 0400 healthcheck: test: - CMD @@ -39,10 +46,23 @@ services: init: true command: - scheduler - env_file: - - .env.staging.runtime environment: - SCHEDULER_URL: http://app:3000/api/internal/scheduler + INFISICAL_PROJECT_ID: 2ee94b27-e599-4194-9797-3372d42e53f6 + INFISICAL_ENVIRONMENT: staging + INFISICAL_RUNTIME_SCOPE: scheduler + INFISICAL_UNIVERSAL_AUTH_CLIENT_ID_FILE: /run/secrets/infisical_client_id + INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET_FILE: /run/secrets/infisical_client_secret + secrets: + - source: infisical_scheduler_client_id + target: infisical_client_id + uid: '0' + gid: '0' + mode: 0400 + - source: infisical_scheduler_client_secret + target: infisical_client_secret + uid: '0' + gid: '0' + mode: 0400 depends_on: app: condition: service_healthy @@ -51,3 +71,13 @@ services: volumes: shd_stage_data: name: shd-build-stage-data + +secrets: + infisical_app_client_id: + file: .infisical.app.client-id + infisical_app_client_secret: + file: .infisical.app.client-secret + infisical_scheduler_client_id: + file: .infisical.scheduler.client-id + infisical_scheduler_client_secret: + file: .infisical.scheduler.client-secret diff --git a/deploy/container-entrypoint.sh b/deploy/container-entrypoint.sh index 15c5d7f..0c984a7 100644 --- a/deploy/container-entrypoint.sh +++ b/deploy/container-entrypoint.sh @@ -1,13 +1,118 @@ #!/bin/sh set -eu -if [ "${1:-}" = "serve" ]; then - node scripts/db-migrate.mjs - exec node build +project_id=${INFISICAL_PROJECT_ID:?INFISICAL_PROJECT_ID is required} +environment=${INFISICAL_ENVIRONMENT:?INFISICAL_ENVIRONMENT is required} +runtime_scope=${INFISICAL_RUNTIME_SCOPE:?INFISICAL_RUNTIME_SCOPE is required} +client_id_file=${INFISICAL_UNIVERSAL_AUTH_CLIENT_ID_FILE:?INFISICAL_UNIVERSAL_AUTH_CLIENT_ID_FILE is required} +client_secret_file=${INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET_FILE:?INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET_FILE is required} +build_number=${SHD_BUILD_NUMBER:-0} +commit_sha=${SHD_COMMIT_SHA:-} +secret_file= +shared_secret_file= + +cleanup() { + if [ -n "$secret_file" ]; then rm -f "$secret_file"; fi + if [ -n "$shared_secret_file" ]; then rm -f "$shared_secret_file"; fi +} + +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +if [ ! -r "$client_id_file" ] || [ ! -r "$client_secret_file" ]; then + echo "Infisical Universal Auth credential files are not readable." >&2 + exit 1 fi -if [ "${1:-}" = "scheduler" ]; then - exec node scripts/scheduler-client.mjs +if [ "$(id -u)" -eq 0 ]; then + runtime_uid=$(id -u shd-runtime) + for credential_file in "$client_id_file" "$client_secret_file"; do + credential_uid=$(stat -c '%u' "$credential_file") + credential_mode=$(stat -c '%a' "$credential_file") + if [ "$credential_uid" -eq "$runtime_uid" ] || [ $((credential_mode % 100)) -ne 0 ]; then + echo "Infisical credential files must not be readable by the application runtime user." >&2 + exit 1 + fi + done + if [ "$runtime_scope" = app ] && ! { [ "${1:-}" = node ] && [ "${2:-}" = scripts/validate-runtime-environment.mjs ]; }; then + chgrp -R node /app/data + chmod -R g+rwX /app/data + fi fi -exec "$@" +client_id=$(cat "$client_id_file") +client_secret=$(cat "$client_secret_file") +if [ -z "$client_id" ] || [ -z "$client_secret" ]; then + echo "Infisical Universal Auth credential files must not be empty." >&2 + exit 1 +fi + +unset INFISICAL_TOKEN INFISICAL_UNIVERSAL_AUTH_ACCESS_TOKEN TOKEN +access_token=$( + INFISICAL_UNIVERSAL_AUTH_CLIENT_ID="$client_id" \ + INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET="$client_secret" \ + infisical login --method=universal-auth --silent --plain +) +unset client_id client_secret + +if [ -z "$access_token" ]; then + echo "Infisical Universal Auth returned an empty access token." >&2 + exit 1 +fi + +umask 077 +secret_file=$(mktemp /tmp/shd-infisical-app.XXXXXX) + +case "$runtime_scope" in + app) + INFISICAL_UNIVERSAL_AUTH_ACCESS_TOKEN="$access_token" infisical export \ + --projectId="$project_id" \ + --env="$environment" \ + --path=/app \ + --format=json \ + --output-file="$secret_file" \ + --silent + shared_secret_file=$(mktemp /tmp/shd-infisical-shared.XXXXXX) + INFISICAL_UNIVERSAL_AUTH_ACCESS_TOKEN="$access_token" infisical export \ + --projectId="$project_id" \ + --env="$environment" \ + --path=/shared/scheduler \ + --format=json \ + --output-file="$shared_secret_file" \ + --silent + ;; + scheduler) + INFISICAL_UNIVERSAL_AUTH_ACCESS_TOKEN="$access_token" infisical export \ + --projectId="$project_id" \ + --env="$environment" \ + --path=/shared/scheduler \ + --format=json \ + --output-file="$secret_file" \ + --silent + ;; + *) + echo "Unsupported INFISICAL_RUNTIME_SCOPE: $runtime_scope" >&2 + exit 2 + ;; +esac + +unset access_token \ + INFISICAL_TOKEN \ + INFISICAL_UNIVERSAL_AUTH_ACCESS_TOKEN \ + INFISICAL_UNIVERSAL_AUTH_CLIENT_ID \ + INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET + +if [ "$runtime_scope" = app ]; then + if [ "$(id -u)" -eq 0 ]; then + chown shd-runtime:node "$secret_file" "$shared_secret_file" + exec gosu shd-runtime node scripts/runtime-launcher.mjs app "$environment" "$build_number" "$commit_sha" "$secret_file" "$shared_secret_file" -- "$@" + fi + exec node scripts/runtime-launcher.mjs app "$environment" "$build_number" "$commit_sha" "$secret_file" "$shared_secret_file" -- "$@" +fi +if [ "$(id -u)" -eq 0 ]; then + chown shd-runtime:node "$secret_file" + exec gosu shd-runtime node scripts/runtime-launcher.mjs scheduler "$environment" "$build_number" "$commit_sha" "$secret_file" -- "$@" +fi +exec node scripts/runtime-launcher.mjs scheduler "$environment" "$build_number" "$commit_sha" "$secret_file" -- "$@" diff --git a/deploy/deploy-production.sh b/deploy/deploy-production.sh index 3b54bfc..e32076e 100644 --- a/deploy/deploy-production.sh +++ b/deploy/deploy-production.sh @@ -4,48 +4,154 @@ set -eu deployment_directory=${1:-/opt/shd-build-production} source_directory=${2:-} candidate_image=${3:-} -if [ -z "$source_directory" ] || [ ! -f "$source_directory/Dockerfile" ] || [ -z "$candidate_image" ]; then - echo "Usage: deploy-production.sh " >&2 +candidate_directory=${4:-} +if [ -z "$source_directory" ] || [ ! -f "$source_directory/Dockerfile" ] || [ -z "$candidate_image" ] || [ ! -f "$candidate_directory/compose.production.yml" ] || [ ! -f "$candidate_directory/.env.deploy" ]; then + echo "Usage: deploy-production.sh " >&2 exit 2 fi -compose_file="$deployment_directory/compose.production.yml" -deploy_environment="$deployment_directory/.env.deploy" +case "$candidate_directory" in + "$deployment_directory"/candidate-*) ;; + *) echo "Candidate directory must be a candidate-* child of the deployment directory." >&2; exit 2 ;; +esac +candidate_compose_file="$candidate_directory/compose.production.yml" +candidate_environment="$candidate_directory/.env.deploy" +canonical_compose_file="$deployment_directory/compose.production.yml" +canonical_environment="$deployment_directory/.env.deploy" +previous_directory="$deployment_directory/.previous-release" +healthy_marker="$deployment_directory/.deployment-healthy" +release_files="compose.production.yml .env.deploy .infisical.app.client-id .infisical.app.client-secret .infisical.scheduler.client-id .infisical.scheduler.client-secret deploy-production.sh bootstrap-production.sh sync-production-isac-to-staging.sh" +rollback_required=false +candidate_promoted=false +had_previous_release=false -compose() { - docker compose --env-file "$deploy_environment" -f "$compose_file" "$@" +candidate_compose() { + docker compose --env-file "$candidate_environment" -f "$candidate_compose_file" "$@" } +canonical_compose() { + docker compose --env-file "$canonical_environment" -f "$canonical_compose_file" "$@" +} + +release_is_healthy() { + health_environment=$1 + health_compose_file=$2 + health_attempt=0 + while [ "$health_attempt" -lt 30 ]; do + health_app_container=$(docker compose --env-file "$health_environment" -f "$health_compose_file" ps -q app) + health_scheduler_container=$(docker compose --env-file "$health_environment" -f "$health_compose_file" ps -q scheduler) + health_app_status=missing + health_scheduler_status=missing + if [ -n "$health_app_container" ]; then + health_app_status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$health_app_container" 2>/dev/null || echo missing) + fi + if [ -n "$health_scheduler_container" ]; then + health_scheduler_status=$(docker inspect --format '{{.State.Status}}' "$health_scheduler_container" 2>/dev/null || echo missing) + fi + if [ "$health_app_status" = healthy ] && [ "$health_scheduler_status" = running ]; then return 0; fi + if [ "$health_app_status" = unhealthy ] || [ "$health_app_status" = exited ] || [ "$health_app_status" = dead ] || [ "$health_scheduler_status" = exited ] || [ "$health_scheduler_status" = dead ]; then return 1; fi + health_attempt=$((health_attempt + 1)) + sleep 2 + done + return 1 +} + +copy_release() { + copy_source=$1 + copy_target=$2 + copy_status=0 + for release_file in $release_files; do + if [ -f "$copy_source/$release_file" ]; then + copy_temporary=$(mktemp "$copy_target/.release.XXXXXX") || return 1 + if cp -p "$copy_source/$release_file" "$copy_temporary"; then + mv "$copy_temporary" "$copy_target/$release_file" || copy_status=1 + else + rm -f "$copy_temporary" + copy_status=1 + fi + else + rm -f "$copy_target/$release_file" || copy_status=1 + fi + done + return "$copy_status" +} + +cleanup_candidate() { + case "$candidate_directory" in + "$deployment_directory"/candidate-*) rm -rf "$candidate_directory" ;; + esac +} + +rollback() { + rollback_exit_status=$? + trap - EXIT HUP INT TERM + set +e + if [ "$rollback_required" = true ]; then + echo "Restoring the previous healthy production release." >&2 + restore_status=0 + candidate_compose stop scheduler app >/dev/null 2>&1 || restore_status=1 + if [ "$candidate_promoted" = true ]; then copy_release "$previous_directory" "$deployment_directory" || restore_status=1; fi + if [ "$had_previous_release" = true ]; then + canonical_compose up -d --no-build app scheduler || restore_status=1 + if [ "$restore_status" -eq 0 ] && ! release_is_healthy "$canonical_environment" "$canonical_compose_file"; then restore_status=1; fi + else + echo "No previous production release is available." >&2 + restore_status=1 + fi + if [ "$restore_status" -ne 0 ]; then + echo "The previous production release could not be restored." >&2 + if [ "$rollback_exit_status" -eq 0 ]; then rollback_exit_status=1; fi + fi + fi + cleanup_candidate + exit "$rollback_exit_status" +} + +trap rollback EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + echo "Building the candidate image while production remains online." docker build --label org.opencontainers.image.title=shd-build -t "$candidate_image" "$source_directory" +echo "Validating Infisical access and the candidate runtime configuration." +candidate_compose run --rm --no-deps app node scripts/validate-runtime-environment.mjs app +candidate_compose run --rm --no-deps scheduler node scripts/validate-runtime-environment.mjs scheduler + +rm -rf "$previous_directory" +mkdir -p "$previous_directory" +if [ -f "$canonical_compose_file" ] && [ -f "$canonical_environment" ]; then + had_previous_release=true + copy_release "$deployment_directory" "$previous_directory" +fi + echo "Stopping only SHD production processes before accessing the embedded database." -compose stop scheduler app || true +rollback_required=true +candidate_compose stop scheduler app echo "Creating a timestamped production backup when the database exists." -compose run --rm --no-deps app sh -c \ +candidate_compose run --rm --no-deps app sh -c \ 'if [ -f "$TDR_DATABASE_PATH" ]; then node scripts/db-backup.mjs; else echo "No production database exists yet; skipping backup."; fi' echo "Applying additive schema migrations." -compose run --rm --no-deps app node scripts/db-migrate.mjs +candidate_compose run --rm --no-deps app node scripts/db-migrate.mjs + +candidate_promoted=true +copy_release "$candidate_directory" "$deployment_directory" echo "Starting production." -compose up -d --no-build --remove-orphans app scheduler +canonical_compose up -d --no-build --remove-orphans app scheduler -container_id=$(compose ps -q app) -attempt=0 -while [ "$attempt" -lt 30 ]; do - status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_id") - if [ "$status" = "healthy" ]; then - echo "SHD production deployment is healthy." - exit 0 - fi - if [ "$status" = "unhealthy" ] || [ "$status" = "exited" ] || [ "$status" = "dead" ]; then - break - fi - attempt=$((attempt + 1)) - sleep 2 -done +if release_is_healthy "$canonical_environment" "$canonical_compose_file"; then + echo "SHD production deployment is healthy." + rollback_required=false + umask 077 + : > "$healthy_marker" + cleanup_candidate + trap - EXIT HUP INT TERM + exit 0 +fi echo "SHD production failed its health check." -compose logs --tail=150 app +canonical_compose logs --tail=150 app scheduler exit 1 diff --git a/deploy/deploy-staging.sh b/deploy/deploy-staging.sh index bbb972d..d576ec6 100644 --- a/deploy/deploy-staging.sh +++ b/deploy/deploy-staging.sh @@ -4,48 +4,154 @@ set -eu deployment_directory=${1:-/opt/shd-build-stage} source_directory=${2:-} candidate_image=${3:-} -if [ -z "$source_directory" ] || [ ! -f "$source_directory/Dockerfile" ] || [ -z "$candidate_image" ]; then - echo "Usage: deploy-staging.sh " >&2 +candidate_directory=${4:-} +if [ -z "$source_directory" ] || [ ! -f "$source_directory/Dockerfile" ] || [ -z "$candidate_image" ] || [ ! -f "$candidate_directory/compose.staging.yml" ] || [ ! -f "$candidate_directory/.env.deploy" ]; then + echo "Usage: deploy-staging.sh " >&2 exit 2 fi -compose_file="$deployment_directory/compose.staging.yml" -deploy_environment="$deployment_directory/.env.deploy" +case "$candidate_directory" in + "$deployment_directory"/candidate-*) ;; + *) echo "Candidate directory must be a candidate-* child of the deployment directory." >&2; exit 2 ;; +esac +candidate_compose_file="$candidate_directory/compose.staging.yml" +candidate_environment="$candidate_directory/.env.deploy" +canonical_compose_file="$deployment_directory/compose.staging.yml" +canonical_environment="$deployment_directory/.env.deploy" +previous_directory="$deployment_directory/.previous-release" +healthy_marker="$deployment_directory/.deployment-healthy" +release_files="compose.staging.yml .env.deploy .infisical.app.client-id .infisical.app.client-secret .infisical.scheduler.client-id .infisical.scheduler.client-secret deploy-staging.sh bootstrap-staging.sh" +rollback_required=false +candidate_promoted=false +had_previous_release=false -compose() { - docker compose --env-file "$deploy_environment" -f "$compose_file" "$@" +candidate_compose() { + docker compose --env-file "$candidate_environment" -f "$candidate_compose_file" "$@" } +canonical_compose() { + docker compose --env-file "$canonical_environment" -f "$canonical_compose_file" "$@" +} + +release_is_healthy() { + health_environment=$1 + health_compose_file=$2 + health_attempt=0 + while [ "$health_attempt" -lt 30 ]; do + health_app_container=$(docker compose --env-file "$health_environment" -f "$health_compose_file" ps -q app) + health_scheduler_container=$(docker compose --env-file "$health_environment" -f "$health_compose_file" ps -q scheduler) + health_app_status=missing + health_scheduler_status=missing + if [ -n "$health_app_container" ]; then + health_app_status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$health_app_container" 2>/dev/null || echo missing) + fi + if [ -n "$health_scheduler_container" ]; then + health_scheduler_status=$(docker inspect --format '{{.State.Status}}' "$health_scheduler_container" 2>/dev/null || echo missing) + fi + if [ "$health_app_status" = healthy ] && [ "$health_scheduler_status" = running ]; then return 0; fi + if [ "$health_app_status" = unhealthy ] || [ "$health_app_status" = exited ] || [ "$health_app_status" = dead ] || [ "$health_scheduler_status" = exited ] || [ "$health_scheduler_status" = dead ]; then return 1; fi + health_attempt=$((health_attempt + 1)) + sleep 2 + done + return 1 +} + +copy_release() { + copy_source=$1 + copy_target=$2 + copy_status=0 + for release_file in $release_files; do + if [ -f "$copy_source/$release_file" ]; then + copy_temporary=$(mktemp "$copy_target/.release.XXXXXX") || return 1 + if cp -p "$copy_source/$release_file" "$copy_temporary"; then + mv "$copy_temporary" "$copy_target/$release_file" || copy_status=1 + else + rm -f "$copy_temporary" + copy_status=1 + fi + else + rm -f "$copy_target/$release_file" || copy_status=1 + fi + done + return "$copy_status" +} + +cleanup_candidate() { + case "$candidate_directory" in + "$deployment_directory"/candidate-*) rm -rf "$candidate_directory" ;; + esac +} + +rollback() { + rollback_exit_status=$? + trap - EXIT HUP INT TERM + set +e + if [ "$rollback_required" = true ]; then + echo "Restoring the previous healthy staging release." >&2 + restore_status=0 + candidate_compose stop scheduler app >/dev/null 2>&1 || restore_status=1 + if [ "$candidate_promoted" = true ]; then copy_release "$previous_directory" "$deployment_directory" || restore_status=1; fi + if [ "$had_previous_release" = true ]; then + canonical_compose up -d --no-build app scheduler || restore_status=1 + if [ "$restore_status" -eq 0 ] && ! release_is_healthy "$canonical_environment" "$canonical_compose_file"; then restore_status=1; fi + else + echo "No previous staging release is available." >&2 + restore_status=1 + fi + if [ "$restore_status" -ne 0 ]; then + echo "The previous staging release could not be restored." >&2 + if [ "$rollback_exit_status" -eq 0 ]; then rollback_exit_status=1; fi + fi + fi + cleanup_candidate + exit "$rollback_exit_status" +} + +trap rollback EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + echo "Building the candidate image while the current staging service remains online." docker build --label org.opencontainers.image.title=shd-build -t "$candidate_image" "$source_directory" +echo "Validating Infisical access and the candidate runtime configuration." +candidate_compose run --rm --no-deps app node scripts/validate-runtime-environment.mjs app +candidate_compose run --rm --no-deps scheduler node scripts/validate-runtime-environment.mjs scheduler + +rm -rf "$previous_directory" +mkdir -p "$previous_directory" +if [ -f "$canonical_compose_file" ] && [ -f "$canonical_environment" ]; then + had_previous_release=true + copy_release "$deployment_directory" "$previous_directory" +fi + echo "Stopping SHD staging processes before accessing the embedded database." -compose stop scheduler app || true +rollback_required=true +candidate_compose stop scheduler app echo "Creating a timestamped database backup when a staging database exists." -compose run --rm --no-deps app sh -c \ +candidate_compose run --rm --no-deps app sh -c \ 'if [ -f "$TDR_DATABASE_PATH" ]; then node scripts/db-backup.mjs; else echo "No staging database exists yet; skipping backup."; fi' echo "Applying additive schema migrations." -compose run --rm --no-deps app node scripts/db-migrate.mjs +candidate_compose run --rm --no-deps app node scripts/db-migrate.mjs + +candidate_promoted=true +copy_release "$candidate_directory" "$deployment_directory" echo "Starting the application and notification scheduler." -compose up -d --no-build --remove-orphans app scheduler +canonical_compose up -d --no-build --remove-orphans app scheduler -container_id=$(compose ps -q app) -attempt=0 -while [ "$attempt" -lt 30 ]; do - status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_id") - if [ "$status" = "healthy" ]; then - echo "SHD staging deployment is healthy." - exit 0 - fi - if [ "$status" = "unhealthy" ] || [ "$status" = "exited" ] || [ "$status" = "dead" ]; then - break - fi - attempt=$((attempt + 1)) - sleep 2 -done +if release_is_healthy "$canonical_environment" "$canonical_compose_file"; then + echo "SHD staging deployment is healthy." + rollback_required=false + umask 077 + : > "$healthy_marker" + cleanup_candidate + trap - EXIT HUP INT TERM + exit 0 +fi echo "SHD staging failed its health check." -compose logs --tail=150 app +canonical_compose logs --tail=150 app scheduler exit 1 diff --git a/deploy/runtime-start.test.ts b/deploy/runtime-start.test.ts new file mode 100644 index 0000000..22e504d --- /dev/null +++ b/deploy/runtime-start.test.ts @@ -0,0 +1,393 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { + copyFile, + mkdir, + mkdtemp, + readFile, + rm, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = resolve(import.meta.dirname, '..'); +const schedulerToken = 'scheduler-token-with-at-least-32-characters'; + +const infisicalStub = `#!/bin/sh +set -eu + +case "\${1:-}" in + login) + printf '%s\n' "$@" > "$STUB_LOGIN_LOG" + if [ "\${INFISICAL_UNIVERSAL_AUTH_CLIENT_ID:-}" != "$STUB_EXPECTED_CLIENT_ID" ] || \ + [ "\${INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET:-}" != "$STUB_EXPECTED_CLIENT_SECRET" ]; then + printf '%s\n' 'credentials=incorrect' >> "$STUB_LOGIN_LOG" + exit 26 + fi + printf '%s\n' 'credentials=ok' >> "$STUB_LOGIN_LOG" + case "\${STUB_LOGIN_RESULT:-success}" in + fail) exit 23 ;; + empty) exit 0 ;; + *) printf '%s\n' stub-access-token ;; + esac + ;; + export) + [ "\${INFISICAL_UNIVERSAL_AUTH_ACCESS_TOKEN:-}" = stub-access-token ] || exit 27 + secret_path= + output_file= + for argument in "$@"; do + case "$argument" in + --path=*) secret_path=\${argument#--path=} ;; + --output-file=*) output_file=\${argument#--output-file=} ;; + esac + done + [ -n "$secret_path" ] && [ -n "$output_file" ] || exit 28 + printf '%s|%s\n' "$secret_path" "$output_file" >> "$STUB_EXPORT_LOG" + case "$secret_path" in + /app) cp "$STUB_APP_EXPORT" "$output_file" ;; + /shared/scheduler) cp "$STUB_SHARED_EXPORT" "$output_file" ;; + *) exit 29 ;; + esac + ;; + *) exit 25 ;; +esac +`; + +const environmentCapture = `#!/bin/sh +env > "$CAPTURE_ENV_PATH" +`; + +const gosuStub = `#!/bin/sh +set -eu +[ "\${1:-}" = shd-runtime ] || exit 30 +printf '%s\n' 'target=shd-runtime' > "$STUB_GOSU_LOG" +shift +exec "$@" +`; + +const idStub = `#!/bin/sh +if [ "\${1:-}" = -u ] && [ "\${2:-}" = shd-runtime ]; then + printf '%s\n' 10001 + exit 0 +fi +exec /usr/bin/id "$@" +`; + +const chownStub = `#!/bin/sh +exit 0 +`; + +describe('container runtime startup', () => { + let temporaryRoot: string; + let clientIdFile: string; + let clientSecretFile: string; + let loginLog: string; + let exportLog: string; + let gosuLog: string; + let capturedEnvironment: string; + let appExport: string; + let sharedExport: string; + + beforeEach(async () => { + temporaryRoot = await mkdtemp(join(tmpdir(), 'shd-runtime-start-')); + const deployDirectory = join(temporaryRoot, 'deploy'); + const scriptsDirectory = join(temporaryRoot, 'scripts'); + await mkdir(deployDirectory); + await mkdir(scriptsDirectory); + await Promise.all([ + copyFile( + join(repositoryRoot, 'deploy/container-entrypoint.sh'), + join(deployDirectory, 'container-entrypoint.sh'), + ), + copyFile( + join(repositoryRoot, 'scripts/runtime-launcher.mjs'), + join(scriptsDirectory, 'runtime-launcher.mjs'), + ), + copyFile( + join(repositoryRoot, 'scripts/runtime-environment.js'), + join(scriptsDirectory, 'runtime-environment.js'), + ), + ]); + + clientIdFile = join(temporaryRoot, 'client-id'); + clientSecretFile = join(temporaryRoot, 'client-secret'); + loginLog = join(temporaryRoot, 'login.log'); + exportLog = join(temporaryRoot, 'export.log'); + gosuLog = join(temporaryRoot, 'gosu.log'); + capturedEnvironment = join(temporaryRoot, 'child.env'); + appExport = join(temporaryRoot, 'app-export.json'); + sharedExport = join(temporaryRoot, 'shared-export.json'); + + await Promise.all([ + writeFile(clientIdFile, 'universal-client-id\n', { mode: 0o600 }), + writeFile(clientSecretFile, 'universal-client-secret\n', { mode: 0o600 }), + writeFile(join(temporaryRoot, 'infisical'), infisicalStub, { + mode: 0o755, + }), + writeFile( + join(temporaryRoot, 'capture-environment'), + environmentCapture, + { + mode: 0o755, + }, + ), + writeFile(join(temporaryRoot, 'gosu'), gosuStub, { mode: 0o755 }), + writeFile(join(temporaryRoot, 'id'), idStub, { mode: 0o755 }), + writeFile(join(temporaryRoot, 'chown'), chownStub, { mode: 0o755 }), + writeExport(appExport, [ + { key: 'APP_ORIGIN', value: 'https://exported.invalid' }, + { key: 'DISCORD_CLIENT_ID', value: 'exported-discord-client' }, + { key: 'DISCORD_CLIENT_SECRET', value: 'exported-discord-secret' }, + ]), + writeExport(sharedExport, [ + { key: 'SCHEDULER_TOKEN', value: schedulerToken }, + ]), + ]); + }); + + afterEach(async () => { + await rm(temporaryRoot, { recursive: true, force: true }); + }); + + function runtimeEnvironment( + scope: 'app' | 'scheduler', + ): Record { + return { + PATH: `${temporaryRoot}:${process.env.PATH ?? ''}`, + INFISICAL_PROJECT_ID: 'project-id', + INFISICAL_ENVIRONMENT: 'staging', + INFISICAL_RUNTIME_SCOPE: scope, + INFISICAL_UNIVERSAL_AUTH_CLIENT_ID_FILE: clientIdFile, + INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET_FILE: clientSecretFile, + INFISICAL_UNIVERSAL_AUTH_CLIENT_ID: 'bootstrap-client-id', + INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET: 'bootstrap-client-secret', + INFISICAL_UNIVERSAL_AUTH_ACCESS_TOKEN: 'bootstrap-access-token', + INFISICAL_TOKEN: 'bootstrap-token', + TOKEN: 'generic-bootstrap-token', + STUB_EXPECTED_CLIENT_ID: 'universal-client-id', + STUB_EXPECTED_CLIENT_SECRET: 'universal-client-secret', + STUB_LOGIN_LOG: loginLog, + STUB_EXPORT_LOG: exportLog, + STUB_GOSU_LOG: gosuLog, + STUB_APP_EXPORT: appExport, + STUB_SHARED_EXPORT: sharedExport, + CAPTURE_ENV_PATH: capturedEnvironment, + SHD_BUILD_NUMBER: '42', + SHD_COMMIT_SHA: 'abc123', + APP_ORIGIN: 'https://parent.invalid', + SCHEDULER_URL: 'https://parent.invalid/scheduler', + BASH_ENV: '', + CDPATH: temporaryRoot, + ENV: '', + IFS: 'hostile-ifs', + LD_LIBRARY_PATH: temporaryRoot, + LD_PRELOAD: '', + NODE_OPTIONS: '--no-warnings', + NODE_PATH: temporaryRoot, + PYTHONPATH: temporaryRoot, + }; + } + + function start(environment: Record) { + return spawnSync( + '/bin/sh', + ['./deploy/container-entrypoint.sh', 'capture-environment'], + { + cwd: temporaryRoot, + env: environment, + encoding: 'utf8', + }, + ); + } + + it('exports app and shared secrets into a scrubbed app child environment', async () => { + const result = start(runtimeEnvironment('app')); + + expect(result.status, result.stderr).toBe(0); + expect(await exportPaths()).toEqual(['/app', '/shared/scheduler']); + expect(await readFile(loginLog, 'utf8')).toBe( + 'login\n--method=universal-auth\n--silent\n--plain\ncredentials=ok\n', + ); + + const childEnvironment = parseEnvironment( + await readFile(capturedEnvironment, 'utf8'), + ); + expect(childEnvironment).toMatchObject({ + APP_ORIGIN: 'https://stage.shd.build', + DISCORD_CLIENT_ID: 'exported-discord-client', + DISCORD_CLIENT_SECRET: 'exported-discord-secret', + ORIGIN: 'https://stage.shd.build', + PASSKEY_RP_ID: 'shd.build', + SCHEDULER_TOKEN: schedulerToken, + SHD_BUILD_NUMBER: '42', + SHD_COMMIT_SHA: 'abc123', + TDR_AVATAR_STORAGE_PATH: '/app/data/avatars', + TDR_DATABASE_PATH: '/app/data/shd-build-staging.db', + TDR_GAME_ICON_STORAGE_PATH: '/app/data/game-icons', + }); + for (const name of removedEnvironmentNames) { + expect(childEnvironment).not.toHaveProperty(name); + } + await expectSecretFilesRemoved(); + await expectRootPrivilegeDrop(); + }); + + it('exports only shared secrets into the scheduler child environment', async () => { + const result = start(runtimeEnvironment('scheduler')); + + expect(result.status, result.stderr).toBe(0); + expect(await exportPaths()).toEqual(['/shared/scheduler']); + expect( + parseEnvironment(await readFile(capturedEnvironment, 'utf8')), + ).toMatchObject({ + SCHEDULER_TOKEN: schedulerToken, + SCHEDULER_URL: 'http://app:3000/api/internal/scheduler', + }); + await expectSecretFilesRemoved(); + await expectRootPrivilegeDrop(); + }); + + it('fails closed on an unknown secret before launching the child', async () => { + await writeExport(appExport, [ + { key: 'DISCORD_CLIENT_ID', value: 'exported-discord-client' }, + { key: 'UNKNOWN_SECRET', value: 'must-not-reach-child' }, + ]); + + const result = start(runtimeEnvironment('app')); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain( + 'Infisical secret UNKNOWN_SECRET is not allowed in this runtime scope.', + ); + expect(existsSync(capturedEnvironment)).toBe(false); + await expectSecretFilesRemoved(); + }); + + it('fails closed when a credential file setting is missing', () => { + const environment = runtimeEnvironment('app'); + delete environment.INFISICAL_UNIVERSAL_AUTH_CLIENT_ID_FILE; + + const result = start(environment); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain( + 'INFISICAL_UNIVERSAL_AUTH_CLIENT_ID_FILE is required', + ); + expect(existsSync(exportLog)).toBe(false); + expect(existsSync(capturedEnvironment)).toBe(false); + }); + + it('fails closed when either credential is empty', async () => { + for (const credentialFile of [clientIdFile, clientSecretFile]) { + await writeFile(credentialFile, ''); + + const result = start(runtimeEnvironment('app')); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain('credential files must not be empty'); + expect(existsSync(exportLog)).toBe(false); + expect(existsSync(capturedEnvironment)).toBe(false); + await writeFile( + credentialFile, + credentialFile === clientIdFile + ? 'universal-client-id\n' + : 'universal-client-secret\n', + ); + } + }); + + it.each(['fail', 'empty'])( + 'fails closed when login returns %s', + (loginResult) => { + const result = start({ + ...runtimeEnvironment('app'), + STUB_LOGIN_RESULT: loginResult, + }); + + expect(result.status).not.toBe(0); + expect(existsSync(exportLog)).toBe(false); + expect(existsSync(capturedEnvironment)).toBe(false); + }, + ); + + it('fails closed on a malformed export and removes every secret file', async () => { + await writeFile(appExport, '{not-json'); + + const result = start(runtimeEnvironment('app')); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain( + 'Infisical returned an unreadable secret export.', + ); + expect(existsSync(capturedEnvironment)).toBe(false); + await expectSecretFilesRemoved(); + }); + + async function exportPaths() { + return (await exportRecords()).map(([path]) => path); + } + + async function expectSecretFilesRemoved() { + for (const [, file] of await exportRecords()) { + expect(existsSync(file)).toBe(false); + } + } + + async function exportRecords() { + return (await readFile(exportLog, 'utf8')) + .trim() + .split('\n') + .map((line) => line.split('|')); + } + + async function expectRootPrivilegeDrop() { + if (typeof process.getuid === 'function' && process.getuid() === 0) { + expect(await readFile(gosuLog, 'utf8')).toBe('target=shd-runtime\n'); + } else { + expect(existsSync(gosuLog)).toBe(false); + } + } +}); + +const removedEnvironmentNames = [ + 'BASH_ENV', + 'CDPATH', + 'ENV', + 'IFS', + 'INFISICAL_ENVIRONMENT', + 'INFISICAL_PROJECT_ID', + 'INFISICAL_RUNTIME_SCOPE', + 'INFISICAL_TOKEN', + 'INFISICAL_UNIVERSAL_AUTH_ACCESS_TOKEN', + 'INFISICAL_UNIVERSAL_AUTH_CLIENT_ID', + 'INFISICAL_UNIVERSAL_AUTH_CLIENT_ID_FILE', + 'INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET', + 'INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET_FILE', + 'LD_LIBRARY_PATH', + 'LD_PRELOAD', + 'NODE_OPTIONS', + 'NODE_PATH', + 'PYTHONPATH', + 'TOKEN', +]; + +function writeExport( + file: string, + entries: Array<{ key: string; value: string }>, +) { + return writeFile(file, `${JSON.stringify(entries)}\n`); +} + +function parseEnvironment(value: string) { + return Object.fromEntries( + value + .trim() + .split('\n') + .map((line) => { + const separator = line.indexOf('='); + return [line.slice(0, separator), line.slice(separator + 1)]; + }), + ); +} diff --git a/deploy/sync-production-isac-to-staging.sh b/deploy/sync-production-isac-to-staging.sh index 3ea5272..40710b0 100644 --- a/deploy/sync-production-isac-to-staging.sh +++ b/deploy/sync-production-isac-to-staging.sh @@ -14,7 +14,11 @@ production_environment="$production_directory/.env.deploy" staging_compose_file="$staging_directory/compose.staging.yml" staging_environment="$staging_directory/.env.deploy" snapshot_name="isac-production-$(date -u +%Y-%m-%dT%H-%M-%SZ).json" -snapshot_path="/tmp/$snapshot_name" +snapshot_path=$(mktemp "/tmp/$snapshot_name.XXXXXX") +production_snapshot_path="/app/data/exports/$snapshot_name" +staging_snapshot_path="/app/data/$snapshot_name" +remove_production_snapshot=false +remove_staging_snapshot=false production_compose() { docker compose --env-file "$production_environment" -f "$production_compose_file" "$@" @@ -22,22 +26,54 @@ production_compose() { staging_compose() { docker compose --env-file "$staging_environment" -f "$staging_compose_file" "$@" } -restart_services() { - production_compose up -d --no-build app scheduler || true - staging_compose up -d --no-build app scheduler || true - rm -f "$snapshot_path" +cleanup() { + exit_status=$? + trap - EXIT + if [ "$remove_production_snapshot" = true ]; then + production_compose run --rm --no-deps app rm -f "$production_snapshot_path" || true + fi + if [ "$remove_staging_snapshot" = true ]; then + staging_compose run --rm --no-deps app rm -f "$staging_snapshot_path" || true + fi + rm -f "$snapshot_path" || true + if ! production_compose up -d --no-build app scheduler; then + if [ "$exit_status" -eq 0 ]; then exit_status=1; fi + fi + if ! staging_compose up -d --no-build app scheduler; then + if [ "$exit_status" -eq 0 ]; then exit_status=1; fi + fi + exit "$exit_status" } -trap restart_services EXIT +trap cleanup EXIT echo "Stopping production writers for a consistent canonical export." production_compose stop scheduler app -production_compose run --rm --no-deps -v "/tmp:/export" app \ - node scripts/isac-data-export.mjs "/export/$snapshot_name" +production_app_container=$(production_compose ps --all --quiet app) +if [ -z "$production_app_container" ]; then + echo "The stopped production app container could not be found." >&2 + exit 1 +fi +remove_production_snapshot=true +production_compose run --rm --no-deps app \ + node scripts/isac-data-export.mjs "$production_snapshot_path" +docker cp "$production_app_container:$production_snapshot_path" "$snapshot_path" +production_compose run --rm --no-deps app rm -f "$production_snapshot_path" +remove_production_snapshot=false production_compose up -d --no-build app scheduler echo "Stopping staging writers before its automatic full and ISAC safety backups." staging_compose stop scheduler app -staging_compose run --rm --no-deps -v "/tmp:/import:ro" app \ - node scripts/isac-data-import.mjs "/import/$snapshot_name" --confirm-replace-isac-data +staging_app_container=$(staging_compose ps --all --quiet app) +if [ -z "$staging_app_container" ]; then + echo "The stopped staging app container could not be found." >&2 + exit 1 +fi +remove_staging_snapshot=true +docker cp "$snapshot_path" "$staging_app_container:$staging_snapshot_path" +staging_compose run --rm --no-deps app \ + node scripts/isac-data-import.mjs "$staging_snapshot_path" --confirm-replace-isac-data +staging_compose run --rm --no-deps app rm -f "$staging_snapshot_path" +remove_staging_snapshot=false +rm -f "$snapshot_path" echo "Production ISAC data was safely synchronized into staging." diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 14c3b0c..edf21b1 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -2,8 +2,8 @@ SHD.build runs in a Node 24 container through the official SvelteKit Node adapter. The image includes application code, migration and notification -scripts, and native runtime dependencies. Environment files, databases, backups, -and user uploads remain outside the image. +scripts, the Infisical CLI, and native runtime dependencies. Databases, backups, +user uploads, and Universal Auth bootstrap credentials remain outside the image. ## Persistent data @@ -30,7 +30,7 @@ Staging defaults to: - Adapter canonical origin: `https://stage.shd.build` Guide ratings and reports use Cloudflare Turnstile after repeated anonymous -activity. Configure both values in each encrypted environment: +activity. Configure both values under `/app` in each Infisical environment: ```text TURNSTILE_SITE_KEY= @@ -44,12 +44,13 @@ and the request address to Cloudflare for verification; it never exposes valid secret, a visitor who reaches the challenge threshold cannot submit more anonymous ratings or Guide reports. -Edit its encrypted configuration locally: +Edit staging configuration in Infisical project +`2ee94b27-e599-4194-9797-3372d42e53f6`: -```sh -npm run env:decrypt:staging -# Fill in Discord and SMTP values. -npm run env:encrypt:staging +```text +Environment: staging +Application path: /app +Scheduler path: /shared/scheduler ``` Add `https://stage.shd.build/auth/discord/callback` to the Discord application's @@ -59,9 +60,43 @@ and configure the host's TLS reverse proxy to forward `stage.shd.build` to port a containerized reverse proxy or tunnel can reach it; firewall the port to trusted LAN/tunnel sources when the host is reachable from other networks. -The trusted staging runner decrypts `.env.staging` into a mode-`0600` runtime -file. Gitea Actions holds the decryption key; Git and the container image do -not. +The trusted staging runner writes separate mode-`0600` Universal Auth client-ID +and client-secret files for the app and scheduler. Compose mounts each pair only +into its assigned service. Each container exchanges those credentials for a +short-lived access token and writes the assigned paths to mode-`0600` temporary +JSON files. The entrypoint rejects unknown secret names, deletes the token, +transfers the exports to the dedicated `shd-runtime` UID 10001, and drops root +before parsing them. The launcher deletes the files before validation, +migration, or startup. Startup fails if the mounted credential files are +readable by UID 10001 or by their group or other users. Runtime-created data +retains group access for rollback to the previous container UID. + +| Machine identity | Environment | Injected paths | Gitea variable | Gitea secret | +| -------------------------------- | ----------- | --------------------------- | ------------------------------------------ | ---------------------------------------------- | +| `shd-build-staging-app` | `staging` | `/app`, `/shared/scheduler` | `INFISICAL_STAGING_APP_CLIENT_ID` | `INFISICAL_STAGING_APP_CLIENT_SECRET` | +| `shd-build-staging-scheduler` | `staging` | `/shared/scheduler` | `INFISICAL_STAGING_SCHEDULER_CLIENT_ID` | `INFISICAL_STAGING_SCHEDULER_CLIENT_SECRET` | +| `shd-build-production-app` | `prod` | `/app`, `/shared/scheduler` | `INFISICAL_PRODUCTION_APP_CLIENT_ID` | `INFISICAL_PRODUCTION_APP_CLIENT_SECRET` | +| `shd-build-production-scheduler` | `prod` | `/shared/scheduler` | `INFISICAL_PRODUCTION_SCHEDULER_CLIENT_ID` | `INFISICAL_PRODUCTION_SCHEDULER_CLIENT_SECRET` | + +The self-hosted Free plan allows five machine identities and does not include +Custom Project Roles. The four runtime identities use the built-in Viewer role, +which can read the complete project. Service-specific paths limit injected +values, but they do not restrict what a stolen root credential could request. +Use an Enterprise license with Custom Project Roles, or split runtime scopes +into separate projects, before treating identity authorization as least +privilege. + +An Infisical outage does not stop a running container. A new container or direct +restart fails closed until `https://secrets.jb.codes` becomes available and the +identity can authenticate. + +Keep the old `.env.*.runtime` files and `DOTENV_PRIVATE_KEY_*` Gitea secrets +only through the first successful staging and production cutovers. The rollback +path may need them to start the previous image. After both environments pass +direct restart checks, rotate every migrated provider credential in Infisical, +remove the old dotenv private keys and `SCHEDULER_TOKEN_PRODUCTION` from Gitea, +and delete local and hosted dotenv key material. Removing encrypted files from +the current Git tree does not revoke values stored in Git history. The staging container disables default-user seeding. On a brand-new staging database, the existing bootstrap policy grants Admin to the first real account @@ -99,31 +134,51 @@ Restart the runner session after adding it to the Docker group. In `https://git.jb.codes/jb/shd.build/settings/actions`: -1. Add repository secret `DOTENV_PRIVATE_KEY_STAGING`, using the matching value - from the ignored local `.env.keys`. -2. Optionally add repository variable `STAGE_DEPLOY_PATH`. It defaults to +1. Add repository secrets `INFISICAL_STAGING_APP_CLIENT_SECRET` and + `INFISICAL_STAGING_SCHEDULER_CLIENT_SECRET`. +2. Add repository variables `INFISICAL_STAGING_APP_CLIENT_ID` and + `INFISICAL_STAGING_SCHEDULER_CLIENT_ID`. +3. Optionally add repository variable `STAGE_DEPLOY_PATH`. It defaults to `/opt/shd-build-stage`. -3. Optionally add repository variable `STAGE_PORT`. It defaults to `5569`. -4. Ensure Actions are enabled and the `stage-deploy` runner is online. +4. Optionally add repository variable `STAGE_PORT`. It defaults to `5569`. +5. Ensure Actions are enabled and the `stage-deploy` runner is online. `.gitea/workflows/deploy-staging.yml` runs for pushes to `main` and manual -dispatches. The trusted staging runner builds each commit-tagged image locally, -so the workflow needs no registry credentials. Gitea assigns `gitea.run_number` -to `SHD_BUILD_NUMBER`; each new workflow run receives the next integer, while -retries of the same run retain their number. +dispatches. The trusted staging runner builds each image locally with a tag that +contains the commit, workflow run, and attempt, so the workflow needs no +registry credentials and a retry cannot overwrite the active image. Gitea +assigns `gitea.run_number` to `SHD_BUILD_NUMBER`; each new workflow run receives +the next integer, while retries of the same run retain their number. + +Staging and production deployments, ISAC exports, and production-to-staging +synchronization share the `shd-build-host-maintenance` concurrency group. Gitea +runs only one of these database maintenance workflows on the deployment host at +a time and never cancels one in progress. ## Deployment lifecycle -The workflow: +The workflow stages Compose, credentials, and release metadata in an isolated +candidate directory. It leaves the canonical deployment files unchanged until +the candidate passes preflight, backup, and migration. The workflow then: -1. installs dependencies, runs `npm run check`, and produces the Node build; +1. installs dependencies, runs `npm run check` and `npm test`, and produces the + Node build; 2. builds the candidate container while the current site remains online; -3. stops only the staging application and notification scheduler; -4. creates a timestamped JSON backup if the staging database exists; -5. applies additive, repeatable migrations; -6. starts the application and scheduler with the new image; -7. waits for `/api/health` to report healthy; -8. retains the logs and exits unsuccessfully if health verification fails. +3. starts one-off app and scheduler containers to verify Infisical access and + validate the candidate environment; +4. stops only the staging application and notification scheduler; +5. creates a timestamped JSON backup if the staging database exists; +6. applies additive, repeatable migrations in a one-off container; +7. promotes the candidate files to the canonical deployment paths; +8. starts the application and scheduler with the new image; +9. validates the app environment and reruns the repeatable migration in the app + entrypoint before serving; +10. verifies `/api/health` and scheduler liveness; +11. restores the previous canonical files and verifies the previous app and + scheduler after a failed stop, backup, migration, startup, or health check. + +On a failed health check, the deploy script prints the final 150 application log +lines and exits unsuccessfully. It does not create a separate log artifact. The outage covers backup, migration, and container startup. Migrations may also remove records that use retired data formats when a release explicitly ends @@ -145,11 +200,10 @@ feed remain published; the importer performs no automatic removal. The scheduler runs notification delivery before it starts the bounded Ubisoft request, so a slow provider does not postpone notifications for the current tick. -`deploy/compose.production.yml` declares `UBISOFT_NEWS_IMPORT_ENABLED=true`, -while the staging Compose file declares `false`. Development does not set the -flag and remains disabled because the importer requires the exact value `true`. -The application process evaluates the flag when it handles -`/api/internal/scheduler`. +The runtime launcher sets `UBISOFT_NEWS_IMPORT_ENABLED=true` for production and +`false` for staging. Development does not set the flag, so the importer remains +disabled because it requires the exact value `true`. The application process +evaluates the flag when it handles `/api/internal/scheduler`. ## Prelaunch operation @@ -216,10 +270,11 @@ channel, keep it out of Git, then copy it to the staging host and run: /secure/path/shd-stage-bootstrap-YYYY-MM-DDTHH-MM-SSZ.json ``` -The bootstrap flow stops the app and scheduler, creates a safety backup of the -target, verifies that staging is unused, imports the entire development database -and uploads, and restarts both services. Delete the transfer copy from the -staging host after verifying the imported record counts and critical entities. +The bootstrap flow requires both services to stop, copies the bundle into the +private data volume, assigns it to the runtime UID, creates a safety backup, +verifies that staging is unused, imports the entire development database and +uploads, deletes the volume copy, and restarts both services. Delete the host +transfer copy after verifying the imported record counts and critical entities. The copy under the local ignored backup directory remains the recovery source. Because this is a full initial clone, development session tokens and account @@ -279,16 +334,21 @@ environment, port, and deployment workflow: - Database: `/app/data/shd-build-production.db` - Persistent volume: `shd-build-production-data` -Routine pushes leave production data unchanged. The production deploy stops its -application and scheduler, creates a timestamped JSON backup, applies repeatable -migrations, restarts, and verifies database health. Keep -`docker compose down --volumes` away from production. +Routine pushes preserve production data unless an included migration removes a +retired prototype format. The production deploy stops its application and +scheduler, creates a timestamped JSON backup, applies repeatable migrations, +restarts, and verifies database health. Keep `docker compose down --volumes` +away from production. Add these repository Action secrets: -- `DOTENV_PRIVATE_KEY_PRODUCTION`, matching the ignored local `.env.keys` -- `SCHEDULER_TOKEN_PRODUCTION`, a long random value used only by the internal - scheduler endpoint +- `INFISICAL_PRODUCTION_APP_CLIENT_SECRET` +- `INFISICAL_PRODUCTION_SCHEDULER_CLIENT_SECRET` + +Add the matching repository variables: + +- `INFISICAL_PRODUCTION_APP_CLIENT_ID` +- `INFISICAL_PRODUCTION_SCHEDULER_CLIENT_ID` Optional repository variables are `PRODUCTION_DEPLOY_PATH` (default `/opt/shd-build-production`) and `PRODUCTION_PORT` (default `3102`). The @@ -299,12 +359,12 @@ workflow run number supplies the production `SHD_BUILD_NUMBER`. Prepare the host directory once from an interactive SSH session: ```sh -sudo install -d -o jb -g jb -m 0750 /opt/shd-build-production +sudo install -d -o act_runner -g act_runner -m 0750 /opt/shd-build-production ``` -Generate the scheduler secret locally without printing it into a shared log, -then enter it directly in Gitea's repository-secret form. Do not commit it or -reuse the staging scheduler token. +Store a separate production `SCHEDULER_TOKEN` under Infisical path +`/shared/scheduler`. The application and scheduler fetch the same value from +that path. Do not reuse the staging token. ### Initial production bootstrap @@ -322,9 +382,11 @@ Copy the resulting ignored JSON bundle to the deployment host over SSH, then: /secure/path/shd-production-bootstrap-YYYY-MM-DDTHH-MM-SSZ.json ``` -The importer verifies the bundle target, creates a production safety backup, and -refuses to replace a production database containing users, builds, audit events, -or proposals. Remove the remote transfer copy after verification. +The bootstrap script requires both services to stop, stages the bundle in the +private data volume with runtime-only access, and removes that copy after the +import. The importer verifies the bundle target, creates a production safety +backup, and refuses to replace a production database containing users, builds, +audit events, or proposals. Remove the remote host copy after verification. ### Promote one News article @@ -340,13 +402,14 @@ managed-image metadata, and image bytes. It excludes comments, reports, user records, Builds, and ISAC data. After deploying the matching application schema, stop the production app and -scheduler and mount the transferred bundle into a one-off application container. -Run `scripts/article-transfer.mjs import` with `ARTICLE_IMPORT_ENABLED=true` and +scheduler. Copy the transferred bundle into `/app/data` with `docker cp`, assign +it to `shd-runtime:node` with mode `0600` through a root one-off container, then +run `scripts/article-transfer.mjs import` with `ARTICLE_IMPORT_ENABLED=true` and `--confirm-article-import`. The importer creates a production database backup before opening the database, resolves the article owner by stable ID or username, rejects ID and slug conflicts, verifies image checksums, and appends -`editorial_content / news.promoted`. Restart and health-check both services, -then remove the transferred bundle from the host. +`editorial_content / news.promoted`. Remove both bundle copies, restart the app +and scheduler, and health-check both services. ### Pull production ISAC data into development @@ -373,11 +436,11 @@ safety backups before replacement. Deployments do not run this flow. Image Library assets use stable IDs and can be promoted additively from local development. Export a bundle after finishing local uploads and metadata edits: -Set the server-only `REMOVE_BG_API_KEY` in each encrypted environment that -accepts ISAC uploads. It is sent only from the server in the `X-Api-Key` header. -Transparent icon uploads skip remove.bg. Opaque sources use its preview output, -then pass through local trimming, white normalization, WebP conversion, and -storage. +Set the server-only `REMOVE_BG_API_KEY` under `/app` in each Infisical +environment that accepts ISAC uploads. The server sends it only in the +`X-Api-Key` header. Transparent icon uploads skip remove.bg. Opaque sources use +its preview output, then pass through local trimming, white normalization, WebP +conversion, and storage. The application stores a remove.bg capacity snapshot and refreshes it from the internal scheduler. `REMOVE_BG_CREDIT_CHECK_HOURS` controls the account lookup @@ -399,19 +462,33 @@ embedded database is unlocked, and run the merge using the target image: ```sh # Staging cd /opt/shd-build-stage -docker compose -f compose.staging.yml stop scheduler app -docker compose -f compose.staging.yml run --rm \ - -v /secure/path:/import:ro app \ - node scripts/managed-images-import.mjs /import/managed-images.json --confirm-merge -docker compose -f compose.staging.yml up -d app scheduler +docker compose --env-file .env.deploy -f compose.staging.yml stop scheduler app +docker compose --env-file .env.deploy -f compose.staging.yml create app +app_container=$(docker compose --env-file .env.deploy -f compose.staging.yml ps --all --quiet app) +docker cp /secure/path/managed-images.json "$app_container:/app/data/.managed-images-import.json" +docker compose --env-file .env.deploy -f compose.staging.yml run --rm --no-deps \ + --entrypoint sh --user root app -c \ + 'chown shd-runtime:node /app/data/.managed-images-import.json && chmod 600 /app/data/.managed-images-import.json' +docker compose --env-file .env.deploy -f compose.staging.yml run --rm --no-deps app \ + node scripts/managed-images-import.mjs /app/data/.managed-images-import.json --confirm-merge +docker compose --env-file .env.deploy -f compose.staging.yml run --rm --no-deps \ + --entrypoint rm --user root app -f /app/data/.managed-images-import.json +docker compose --env-file .env.deploy -f compose.staging.yml up -d app scheduler # Production cd /opt/shd-build-production -docker compose -f compose.production.yml stop scheduler app -docker compose -f compose.production.yml run --rm \ - -v /secure/path:/import:ro app \ - node scripts/managed-images-import.mjs /import/managed-images.json --confirm-merge -docker compose -f compose.production.yml up -d app scheduler +docker compose --env-file .env.deploy -f compose.production.yml stop scheduler app +docker compose --env-file .env.deploy -f compose.production.yml create app +app_container=$(docker compose --env-file .env.deploy -f compose.production.yml ps --all --quiet app) +docker cp /secure/path/managed-images.json "$app_container:/app/data/.managed-images-import.json" +docker compose --env-file .env.deploy -f compose.production.yml run --rm --no-deps \ + --entrypoint sh --user root app -c \ + 'chown shd-runtime:node /app/data/.managed-images-import.json && chmod 600 /app/data/.managed-images-import.json' +docker compose --env-file .env.deploy -f compose.production.yml run --rm --no-deps app \ + node scripts/managed-images-import.mjs /app/data/.managed-images-import.json --confirm-merge +docker compose --env-file .env.deploy -f compose.production.yml run --rm --no-deps \ + --entrypoint rm --user root app -f /app/data/.managed-images-import.json +docker compose --env-file .env.deploy -f compose.production.yml up -d app scheduler ``` Each import creates a full target database backup before changes. The importer @@ -422,9 +499,8 @@ After reviewing an intentional metadata or file replacement, add verifying both environments because it may contain unpublished visual assets and creator identifiers. -The `.env.staging` and `.env.production` npm import commands are intended for -maintenance performed directly in an environment where those database and -storage paths are mounted: +The staging and production npm import commands fetch `/app` from Infisical. Run +them only where the target database and storage paths are mounted: ```sh npm run images:import:staging -- /path/to/bundle.json --confirm-merge diff --git a/docs/architecture.md b/docs/architecture.md index dbe0edc..85a86b8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -911,12 +911,9 @@ The notification router sends Data Change review items to agents who hold delivery for each authorized queue. Submission creates Immediate alerts; `oversight_notification_queue` stores consolidated alerts. -Production must run the idempotent dispatcher once per minute from one scheduler -instance: - -```text -* * * * * cd /path/to/shd.build && npm run notifications:oversight:production -``` +The production Compose scheduler calls the internal scheduler endpoint once per +minute. That endpoint runs the idempotent dispatcher in the application process, +which owns the embedded-database connection. Do not run a second host cron job. The dispatcher groups due rows by reviewer, queue, cadence, and delivery window. Daily rows use the next 10:00 AM wall-clock boundary in `America/New_York`, @@ -941,12 +938,7 @@ default welcome introduces News, Database, Builds, Guides, Builder, ISAC Contributor Mode, and the SHD Discord community. The registration request delivers Immediate definitions. -`automated_system_message_deliveries` stores definitions with a delay. -Production must run the delayed dispatcher once per minute: - -```text -* * * * * cd /path/to/shd.build && npm run notifications:system:production -``` - -Only one dispatcher instance should run for the embedded database. Delivered -rows retain both their delivery timestamp and generated notification ID. +`automated_system_message_deliveries` stores definitions with a delay. The same +Compose scheduler tick runs the delayed dispatcher. Only one scheduler instance +should call the embedded-database application. Delivered rows retain both their +delivery timestamp and generated notification ID. diff --git a/package-lock.json b/package-lock.json index 36fc581..47a105d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,7 @@ "sharp": "^0.35.3" }, "devDependencies": { - "@dotenvx/dotenvx": "^2.17.4", + "@infisical/cli": "0.43.116", "@sveltejs/adapter-node": "^5.5.7", "@sveltejs/kit": "^2.63.0", "@sveltejs/vite-plugin-svelte": "^7.1.2", @@ -41,38 +41,6 @@ "vitest": "^4.1.10" } }, - "node_modules/@dotenvx/dotenvx": { - "version": "2.17.4", - "resolved": "https://registry.npmjs.org/@dotenvx/dotenvx/-/dotenvx-2.17.4.tgz", - "integrity": "sha512-hzW9xsf9bOUoY1V7xXnl+vZfjqdsdUs0zSOWzm6Zqfcm55nq3YaV+lXmeSzT3ju7xQ6fdSlLjwDQFqrIsMV3wA==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@dotenvx/primitives": "^2.1.0", - "@dotenvx/tooling": "^1.0.3", - "yocto-spinner": "^1.2.1" - }, - "bin": { - "dotenvx": "src/cli/dotenvx.js" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/@dotenvx/primitives": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@dotenvx/primitives/-/primitives-2.1.0.tgz", - "integrity": "sha512-GIpMSSgjZk4URRCtAw12vBmobnw3VeSKL+p7/y8ypWvTHFW+StAMZvq4wCjdX+QKuw4alXu0IMGW6QPhKo6ufg==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/@dotenvx/tooling": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@dotenvx/tooling/-/tooling-1.0.3.tgz", - "integrity": "sha512-Gd6qQol/ICb4MPRWpiIH56y+pR9AVB7kPBkbhrLSGLtAiF4Qgy+k651/f2pze0Xbm3H5XewbPfIm4y9K+ic30Q==", - "dev": true, - "license": "BSD-3-Clause" - }, "node_modules/@emnapi/core": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", @@ -636,6 +604,33 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@infisical/cli": { + "version": "0.43.116", + "resolved": "https://registry.npmjs.org/@infisical/cli/-/cli-0.43.116.tgz", + "integrity": "sha512-RV2vOPrCfML7c2Qnw84N/nxn31f7gMEjw32Bv7ZrNfuXDOB1H3DtW3PAC6APXcjEEDRgSczr4MxH1k4HkuZt6w==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "tar": "^7.5.13", + "yauzl": "^3.2.0" + }, + "bin": { + "infisical": "bin/infisical" + } + }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "dev": true, + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -2561,6 +2556,16 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/cli-cursor": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-5.0.0.tgz", @@ -3314,6 +3319,29 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/mri": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", @@ -3481,6 +3509,13 @@ "dev": true, "license": "MIT" }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "dev": true, + "license": "MIT" + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -4141,6 +4176,23 @@ "typescript": ">=5.0.0" } }, + "node_modules/tar": { + "version": "7.5.22", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.22.tgz", + "integrity": "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", @@ -4502,6 +4554,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/yaml": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", @@ -4518,33 +4580,17 @@ "url": "https://github.com/sponsors/eemeli" } }, - "node_modules/yocto-spinner": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/yocto-spinner/-/yocto-spinner-1.2.2.tgz", - "integrity": "sha512-DODGl1wJjA/s5pnJFKau9lIYHT81lnhob1i3e1TjxZRxEhWRKl74nTbWE6H5KlkViQQTo/Z29YFdxzTZAMY3ng==", + "node_modules/yauzl": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-3.4.0.tgz", + "integrity": "sha512-jIH9yLR9wqr0wOS0TpBvo/g/2UgZH5qePVbjgRliiF0BYvOZyaBknKsF+x9Iht0O6sqgnB93rCICdOZFecJuDw==", "dev": true, "license": "MIT", "dependencies": { - "yoctocolors": "^2.1.1" + "pend": "~1.2.0" }, "engines": { - "node": ">=18.19" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/yoctocolors": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/yoctocolors/-/yoctocolors-2.1.2.tgz", - "integrity": "sha512-CzhO+pFNo8ajLM2d2IW/R93ipy99LWjtwblvC1RsoSUMZgyLbYFr221TnSNT7GjGdYui6P459mw9JH/g/zW2ug==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node": ">=12" } }, "node_modules/zimmerframe": { diff --git a/package.json b/package.json index 096f6ce..c01af3b 100644 --- a/package.json +++ b/package.json @@ -4,52 +4,42 @@ "version": "0.0.1", "type": "module", "scripts": { - "dev": "dotenvx run -f .env.development -- vite dev --host 0.0.0.0", + "dev": "infisical run --env=dev --path=/app --path=/shared/scheduler -- vite dev --host 0.0.0.0", "build:app": "vite build", - "build:staging": "dotenvx run -f .env.staging -- vite build", - "build": "dotenvx run -f .env.production -- vite build", + "build:staging": "vite build", + "build": "vite build", "brand:render": "node scripts/render-brand-assets.mjs", "start": "node build", "preview": "node build", - "db:migrate": "dotenvx run -f .env.development -- node scripts/db-migrate.mjs", - "db:migrate:staging": "dotenvx run -f .env.staging -- node scripts/db-migrate.mjs", - "db:migrate:production": "dotenvx run -f .env.production -- node scripts/db-migrate.mjs", - "db:seed-users": "dotenvx run -f .env.development -- node scripts/seed-initial-users.mjs", - "db:seed-users:reset": "dotenvx run -f .env.development -- node scripts/seed-initial-users.mjs --reset-passwords", - "db:backup": "dotenvx run -f .env.development -- node scripts/db-backup.mjs", - "db:backup:staging": "dotenvx run -f .env.staging -- node scripts/db-backup.mjs", - "db:backup:production": "dotenvx run -f .env.production -- node scripts/db-backup.mjs", - "db:restore": "dotenvx run -f .env.development -- node scripts/db-restore.mjs", - "db:restore:production": "dotenvx run -f .env.production -- node scripts/db-restore.mjs", - "article:export": "dotenvx run -f .env.development -- node scripts/article-transfer.mjs export", - "isac:export": "dotenvx run -f .env.development -- node scripts/isac-data-export.mjs", - "isac:import": "dotenvx run -f .env.development -- node scripts/isac-data-import.mjs", - "isac:attributes:import": "dotenvx run -f .env.development -- node scripts/import-resurgence-attributes.mjs", - "isac:export:staging": "dotenvx run -f .env.staging -- node scripts/isac-data-export.mjs", - "images:export": "dotenvx run -f .env.development -- node scripts/managed-images-export.mjs", - "images:import": "dotenvx run -f .env.development -- node scripts/managed-images-import.mjs", - "images:import:staging": "dotenvx run -f .env.staging -- node scripts/managed-images-import.mjs", - "images:import:production": "dotenvx run -f .env.production -- node scripts/managed-images-import.mjs", - "stage:bootstrap:export": "dotenvx run -f .env.development -- node scripts/stage-bootstrap-export.mjs", - "production:bootstrap:export": "dotenvx run -f .env.development -- node scripts/environment-bootstrap-export.mjs production", - "notifications:digest": "dotenvx run -f .env.development -- node scripts/notification-digest.mjs", - "notifications:digest:production": "dotenvx run -f .env.production -- node scripts/notification-digest.mjs", - "notifications:oversight": "dotenvx run -f .env.development -- node scripts/oversight-notification-dispatch.mjs", - "notifications:oversight:production": "dotenvx run -f .env.production -- node scripts/oversight-notification-dispatch.mjs", - "notifications:system": "dotenvx run -f .env.development -- node scripts/system-notification-dispatch.mjs", - "notifications:system:production": "dotenvx run -f .env.production -- node scripts/system-notification-dispatch.mjs", - "notifications:seed-test": "dotenvx run -f .env.development -- node scripts/seed-test-notifications.mjs", + "db:migrate": "infisical run --env=dev --path=/app --path=/shared/scheduler -- node scripts/db-migrate.mjs", + "db:migrate:staging": "infisical run --env=staging --path=/app --path=/shared/scheduler -- node scripts/db-migrate.mjs", + "db:migrate:production": "infisical run --env=prod --path=/app --path=/shared/scheduler -- node scripts/db-migrate.mjs", + "db:seed-users": "infisical run --env=dev --path=/app -- node scripts/seed-initial-users.mjs", + "db:seed-users:reset": "infisical run --env=dev --path=/app -- node scripts/seed-initial-users.mjs --reset-passwords", + "db:backup": "infisical run --env=dev --path=/app -- node scripts/db-backup.mjs", + "db:backup:staging": "infisical run --env=staging --path=/app -- node scripts/db-backup.mjs", + "db:backup:production": "infisical run --env=prod --path=/app -- node scripts/db-backup.mjs", + "db:restore": "infisical run --env=dev --path=/app -- node scripts/db-restore.mjs", + "db:restore:production": "infisical run --env=prod --path=/app -- node scripts/db-restore.mjs", + "article:export": "infisical run --env=dev --path=/app -- node scripts/article-transfer.mjs export", + "isac:export": "infisical run --env=dev --path=/app -- node scripts/isac-data-export.mjs", + "isac:import": "infisical run --env=dev --path=/app -- node scripts/isac-data-import.mjs", + "isac:attributes:import": "infisical run --env=dev --path=/app -- node scripts/import-resurgence-attributes.mjs", + "isac:export:staging": "infisical run --env=staging --path=/app -- node scripts/isac-data-export.mjs", + "images:export": "infisical run --env=dev --path=/app -- node scripts/managed-images-export.mjs", + "images:import": "infisical run --env=dev --path=/app -- node scripts/managed-images-import.mjs", + "images:import:staging": "infisical run --env=staging --path=/app -- node scripts/managed-images-import.mjs", + "images:import:production": "infisical run --env=prod --path=/app -- node scripts/managed-images-import.mjs", + "stage:bootstrap:export": "infisical run --env=dev --path=/app -- node scripts/stage-bootstrap-export.mjs", + "production:bootstrap:export": "infisical run --env=dev --path=/app -- node scripts/environment-bootstrap-export.mjs production", + "notifications:digest": "infisical run --env=dev --path=/app -- node scripts/notification-digest.mjs", + "notifications:digest:production": "infisical run --env=prod --path=/app -- node scripts/notification-digest.mjs", + "notifications:oversight": "infisical run --env=dev --path=/app -- node scripts/oversight-notification-dispatch.mjs", + "notifications:oversight:production": "infisical run --env=prod --path=/app -- node scripts/oversight-notification-dispatch.mjs", + "notifications:system": "infisical run --env=dev --path=/app -- node scripts/system-notification-dispatch.mjs", + "notifications:system:production": "infisical run --env=prod --path=/app -- node scripts/system-notification-dispatch.mjs", + "notifications:seed-test": "infisical run --env=dev --path=/app -- node scripts/seed-test-notifications.mjs", "assets:trim-icons": "node scripts/trim-static-icons.mjs", - "env:encrypt": "node scripts/env-files.mjs encrypt", - "env:decrypt": "node scripts/env-files.mjs decrypt", - "env:encrypt:development": "dotenvx encrypt -f .env.development", - "env:decrypt:development": "dotenvx decrypt -f .env.development", - "env:encrypt:staging": "dotenvx encrypt -f .env.staging", - "env:decrypt:staging": "dotenvx decrypt -f .env.staging", - "env:sync:staging-credentials": "dotenvx run -f .env.development -- node scripts/sync-staging-credentials.mjs", - "env:encrypt:production": "dotenvx encrypt -f .env.production", - "env:decrypt:production": "dotenvx decrypt -f .env.production", - "env:stage": "node scripts/env-files.mjs stage", "prepare": "node scripts/prepare.mjs", "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", "test": "vitest run", @@ -59,7 +49,7 @@ "**/*": "prettier --write --ignore-unknown" }, "devDependencies": { - "@dotenvx/dotenvx": "^2.17.4", + "@infisical/cli": "0.43.116", "@sveltejs/adapter-node": "^5.5.7", "@sveltejs/kit": "^2.63.0", "@sveltejs/vite-plugin-svelte": "^7.1.2", diff --git a/scripts/env-files.mjs b/scripts/env-files.mjs deleted file mode 100644 index b506ef6..0000000 --- a/scripts/env-files.mjs +++ /dev/null @@ -1,35 +0,0 @@ -import { spawnSync } from 'node:child_process'; -import { resolve } from 'node:path'; -import process from 'node:process'; - -const files = ['.env.development', '.env.staging', '.env.production']; -const action = process.argv[2]; -const gitCommand = process.platform === 'win32' ? 'git.exe' : 'git'; -const dotenvxCli = resolve('node_modules/@dotenvx/dotenvx/src/cli/dotenvx.js'); - -function run(command, args) { - const result = spawnSync(command, args, { - cwd: process.cwd(), - stdio: 'inherit', - }); - if (result.error) throw result.error; - if (result.status !== 0) process.exit(result.status ?? 1); -} - -function dotenvx(command, file) { - run(process.execPath, [dotenvxCli, command, '-f', file]); -} - -if (!['encrypt', 'decrypt', 'stage'].includes(action)) { - console.error('Usage: node scripts/env-files.mjs '); - process.exit(2); -} - -for (const file of files) { - dotenvx(action === 'decrypt' ? 'decrypt' : 'encrypt', file); -} - -if (action === 'stage') { - run(gitCommand, ['add', '--', ...files]); - console.log('Encrypted and staged environment files.'); -} diff --git a/scripts/runtime-environment.js b/scripts/runtime-environment.js new file mode 100644 index 0000000..3cf1ff3 --- /dev/null +++ b/scripts/runtime-environment.js @@ -0,0 +1,232 @@ +/** @typedef {'app' | 'scheduler'} RuntimeMode */ +/** @typedef {Record} RuntimeEnvironment */ + +const APP_REQUIRED = [ + 'APP_ORIGIN', + 'ORIGIN', + 'PASSKEY_RP_ID', + 'SCHEDULER_TOKEN', + 'TDR_DATABASE_PATH', + 'TDR_AVATAR_STORAGE_PATH', + 'TDR_GAME_ICON_STORAGE_PATH', +]; + +/** + * Validate environment variables after the runtime secret provider has injected them. + * + * @param {RuntimeMode} mode + * @param {RuntimeEnvironment} environment + * @returns {string[]} + */ +export function validateRuntimeEnvironment(mode, environment) { + if (mode !== 'app' && mode !== 'scheduler') { + throw new TypeError(`Unsupported runtime mode: ${mode}`); + } + + const errors = []; + const get = (name) => environment[name]?.trim() ?? ''; + const requireValues = (names) => { + for (const name of names) { + if (!get(name)) errors.push(`${name} is required.`); + } + }; + + if (mode === 'scheduler') { + requireValues(['SCHEDULER_URL', 'SCHEDULER_TOKEN']); + validateHttpUrl('SCHEDULER_URL', get('SCHEDULER_URL'), errors); + if ( + get('SCHEDULER_URL') && + get('SCHEDULER_URL') !== 'http://app:3000/api/internal/scheduler' + ) { + errors.push( + 'SCHEDULER_URL must use the internal app scheduler endpoint.', + ); + } + validateSchedulerToken(get('SCHEDULER_TOKEN'), errors); + return errors; + } + + requireValues(APP_REQUIRED); + validateSchedulerToken(get('SCHEDULER_TOKEN'), errors); + + const appOrigin = validateOrigin('APP_ORIGIN', get('APP_ORIGIN'), errors); + const adapterOrigin = validateOrigin('ORIGIN', get('ORIGIN'), errors); + if ( + appOrigin && + adapterOrigin && + appOrigin.hostname !== adapterOrigin.hostname + ) { + errors.push('APP_ORIGIN and ORIGIN must use the same hostname.'); + } + + const rpId = get('PASSKEY_RP_ID').toLowerCase(); + if (rpId && !isHostname(rpId)) { + errors.push('PASSKEY_RP_ID must be a hostname.'); + } else if ( + appOrigin && + rpId && + appOrigin.hostname !== rpId && + !appOrigin.hostname.endsWith(`.${rpId}`) + ) { + errors.push( + 'PASSKEY_RP_ID must equal APP_ORIGIN hostname or be its registrable parent domain.', + ); + } + + requirePair( + 'DISCORD_CLIENT_ID', + get('DISCORD_CLIENT_ID'), + 'DISCORD_CLIENT_SECRET', + get('DISCORD_CLIENT_SECRET'), + errors, + ); + requirePair( + 'TURNSTILE_SITE_KEY', + get('TURNSTILE_SITE_KEY'), + 'TURNSTILE_SECRET_KEY', + get('TURNSTILE_SECRET_KEY'), + errors, + ); + + const smtpEnabled = [ + 'SMTP_HOST', + 'SMTP_FROM', + 'SMTP_USER', + 'SMTP_PASSWORD', + ].some((name) => Boolean(get(name))); + if (smtpEnabled) { + requirePair( + 'SMTP_HOST', + get('SMTP_HOST'), + 'SMTP_FROM', + get('SMTP_FROM'), + errors, + ); + requirePair( + 'SMTP_USER', + get('SMTP_USER'), + 'SMTP_PASSWORD', + get('SMTP_PASSWORD'), + errors, + ); + } + + const smtpPort = get('SMTP_PORT'); + if ( + smtpPort && + (!/^\d+$/.test(smtpPort) || + Number(smtpPort) < 1 || + Number(smtpPort) > 65535) + ) { + errors.push('SMTP_PORT must be an integer from 1 through 65535.'); + } + + const smtpSecure = get('SMTP_SECURE'); + if (smtpSecure && !['true', 'false'].includes(smtpSecure)) { + errors.push('SMTP_SECURE must be true or false.'); + } + + const discordRedirect = validateHttpUrl( + 'DISCORD_REDIRECT_URI', + get('DISCORD_REDIRECT_URI'), + errors, + ); + if ( + appOrigin && + discordRedirect && + (discordRedirect.hostname !== appOrigin.hostname || + discordRedirect.pathname !== '/auth/discord/callback') + ) { + errors.push( + 'DISCORD_REDIRECT_URI must use APP_ORIGIN hostname and /auth/discord/callback.', + ); + } + + for (const name of [ + 'TDR_DATABASE_PATH', + 'TDR_AVATAR_STORAGE_PATH', + 'TDR_GAME_ICON_STORAGE_PATH', + ]) { + if (get(name) && !get(name).startsWith('/')) { + errors.push(`${name} must be an absolute hosted path.`); + } + } + + return errors; +} + +/** + * @param {string} firstName + * @param {string} firstValue + * @param {string} secondName + * @param {string} secondValue + * @param {string[]} errors + */ +function requirePair(firstName, firstValue, secondName, secondValue, errors) { + if (Boolean(firstValue) !== Boolean(secondValue)) { + errors.push(`${firstName} and ${secondName} must be configured together.`); + } +} + +/** + * @param {string} name + * @param {string} value + * @param {string[]} errors + * @returns {URL | undefined} + */ +function validateHttpUrl(name, value, errors) { + if (!value) return undefined; + try { + const url = new URL(value); + if (!['http:', 'https:'].includes(url.protocol) || !url.hostname) { + throw new TypeError('not an HTTP URL'); + } + return url; + } catch { + errors.push(`${name} must be a valid HTTP or HTTPS URL.`); + return undefined; + } +} + +/** + * @param {string} name + * @param {string} value + * @param {string[]} errors + * @returns {URL | undefined} + */ +function validateOrigin(name, value, errors) { + const url = validateHttpUrl(name, value, errors); + if (!url) return undefined; + if ( + url.username || + url.password || + url.pathname !== '/' || + url.search || + url.hash + ) { + errors.push(`${name} must contain only a URL origin.`); + return undefined; + } + return url; +} + +/** @param {string} value */ +function isHostname(value) { + if (value === 'localhost') return true; + if (value.length > 253 || !value.includes('.')) return false; + return value + .split('.') + .every( + (label) => + label.length <= 63 && /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(label), + ); +} + +/** @param {string} value @param {string[]} errors */ +function validateSchedulerToken(value, errors) { + if (value && (value.length < 32 || /\s/.test(value))) { + errors.push( + 'SCHEDULER_TOKEN must contain at least 32 non-whitespace characters.', + ); + } +} diff --git a/scripts/runtime-environment.test.ts b/scripts/runtime-environment.test.ts new file mode 100644 index 0000000..6ea55f4 --- /dev/null +++ b/scripts/runtime-environment.test.ts @@ -0,0 +1,161 @@ +import { describe, expect, it } from 'vitest'; +import { validateRuntimeEnvironment } from './runtime-environment.js'; + +const validAppEnvironment = { + APP_ORIGIN: 'https://stage.shd.build', + ORIGIN: 'https://stage.shd.build', + PASSKEY_RP_ID: 'shd.build', + SCHEDULER_TOKEN: 'scheduler-secret-with-32-characters', + TDR_DATABASE_PATH: '/app/data/shd-build-staging.db', + TDR_AVATAR_STORAGE_PATH: '/app/data/avatars', + TDR_GAME_ICON_STORAGE_PATH: '/app/data/game-icons', +}; + +describe('validateRuntimeEnvironment', () => { + it('accepts the hosted app contract without optional providers', () => { + expect(validateRuntimeEnvironment('app', validAppEnvironment)).toEqual([]); + }); + + it('reports every missing required app value', () => { + expect(validateRuntimeEnvironment('app', {})).toEqual([ + 'APP_ORIGIN is required.', + 'ORIGIN is required.', + 'PASSKEY_RP_ID is required.', + 'SCHEDULER_TOKEN is required.', + 'TDR_DATABASE_PATH is required.', + 'TDR_AVATAR_STORAGE_PATH is required.', + 'TDR_GAME_ICON_STORAGE_PATH is required.', + ]); + }); + + it('validates canonical origins and passkey hostname scope', () => { + expect( + validateRuntimeEnvironment('app', { + ...validAppEnvironment, + APP_ORIGIN: 'https://stage.shd.build/sign-in', + }), + ).toContain('APP_ORIGIN must contain only a URL origin.'); + + expect( + validateRuntimeEnvironment('app', { + ...validAppEnvironment, + PASSKEY_RP_ID: 'example.com', + }), + ).toContain( + 'PASSKEY_RP_ID must equal APP_ORIGIN hostname or be its registrable parent domain.', + ); + }); + + it('rejects mismatched app and adapter hostnames', () => { + expect( + validateRuntimeEnvironment('app', { + ...validAppEnvironment, + ORIGIN: 'https://other.shd.build', + }), + ).toContain('APP_ORIGIN and ORIGIN must use the same hostname.'); + }); + + it('requires optional integration credentials only as pairs', () => { + expect( + validateRuntimeEnvironment('app', { + ...validAppEnvironment, + DISCORD_CLIENT_ID: 'client-id', + TURNSTILE_SECRET_KEY: 'secret', + }), + ).toEqual([ + 'DISCORD_CLIENT_ID and DISCORD_CLIENT_SECRET must be configured together.', + 'TURNSTILE_SITE_KEY and TURNSTILE_SECRET_KEY must be configured together.', + ]); + }); + + it('allows SMTP defaults without enabling email and validates enabled SMTP', () => { + expect( + validateRuntimeEnvironment('app', { + ...validAppEnvironment, + SMTP_PORT: '587', + SMTP_SECURE: 'false', + }), + ).toEqual([]); + + expect( + validateRuntimeEnvironment('app', { + ...validAppEnvironment, + SMTP_HOST: 'smtp.example.com', + SMTP_USER: 'agent', + SMTP_PORT: '70000', + }), + ).toEqual([ + 'SMTP_HOST and SMTP_FROM must be configured together.', + 'SMTP_USER and SMTP_PASSWORD must be configured together.', + 'SMTP_PORT must be an integer from 1 through 65535.', + ]); + }); + + it('validates optional redirect URLs when supplied', () => { + expect( + validateRuntimeEnvironment('app', { + ...validAppEnvironment, + DISCORD_REDIRECT_URI: 'ftp://stage.shd.build/callback', + }), + ).toContain('DISCORD_REDIRECT_URI must be a valid HTTP or HTTPS URL.'); + + expect( + validateRuntimeEnvironment('app', { + ...validAppEnvironment, + DISCORD_REDIRECT_URI: 'https://dev.shd.build/auth/discord/callback', + }), + ).toContain( + 'DISCORD_REDIRECT_URI must use APP_ORIGIN hostname and /auth/discord/callback.', + ); + }); + + it('validates hosted paths and SMTP TLS configuration', () => { + expect( + validateRuntimeEnvironment('app', { + ...validAppEnvironment, + SMTP_SECURE: 'yes', + TDR_DATABASE_PATH: './data/staging.db', + }), + ).toEqual([ + 'SMTP_SECURE must be true or false.', + 'TDR_DATABASE_PATH must be an absolute hosted path.', + ]); + }); + + it('accepts a configured scheduler and rejects missing or invalid values', () => { + expect( + validateRuntimeEnvironment('scheduler', { + SCHEDULER_URL: 'http://app:3000/api/internal/scheduler', + SCHEDULER_TOKEN: 'scheduler-secret-with-32-characters', + }), + ).toEqual([]); + + expect( + validateRuntimeEnvironment('scheduler', { + SCHEDULER_URL: 'not a URL', + }), + ).toEqual([ + 'SCHEDULER_TOKEN is required.', + 'SCHEDULER_URL must be a valid HTTP or HTTPS URL.', + 'SCHEDULER_URL must use the internal app scheduler endpoint.', + ]); + }); + + it('requires a strong scheduler token and fixed internal endpoint', () => { + expect( + validateRuntimeEnvironment('scheduler', { + SCHEDULER_URL: 'https://example.com/api/internal/scheduler', + SCHEDULER_TOKEN: 'too-short', + }), + ).toEqual([ + 'SCHEDULER_URL must use the internal app scheduler endpoint.', + 'SCHEDULER_TOKEN must contain at least 32 non-whitespace characters.', + ]); + }); + + it('rejects unsupported runtime modes', () => { + expect(() => + validateRuntimeEnvironment('worker' as 'app', validAppEnvironment), + ).toThrow('Unsupported runtime mode: worker'); + }); +}); diff --git a/scripts/runtime-launcher.mjs b/scripts/runtime-launcher.mjs new file mode 100644 index 0000000..e96c7dd --- /dev/null +++ b/scripts/runtime-launcher.mjs @@ -0,0 +1,251 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import { readFile, unlink } from 'node:fs/promises'; +import process from 'node:process'; +import { validateRuntimeEnvironment } from './runtime-environment.js'; + +const APP_SECRET_NAMES = new Set([ + 'APP_ORIGIN', + 'DISCORD_CLIENT_ID', + 'DISCORD_CLIENT_SECRET', + 'DISCORD_REDIRECT_URI', + 'OPENAI_MECHANIC_API_KEY', + 'OPENAI_MECHANIC_MODEL', + 'OPENAI_MECHANIC_REASONING_EFFORT', + 'OPENAI_MODERATION_API_KEY', + 'ORIGIN', + 'PASSKEY_RP_ID', + 'REMOVE_BG_API_KEY', + 'REMOVE_BG_CREDIT_CHECK_HOURS', + 'REMOVE_BG_LOW_CREDIT_THRESHOLD', + 'REMOVE_BG_LOW_FREE_CALL_THRESHOLD', + 'SMTP_FROM', + 'SMTP_HOST', + 'SMTP_PASSWORD', + 'SMTP_PORT', + 'SMTP_SECURE', + 'SMTP_USER', + 'TDR_AVATAR_STORAGE_PATH', + 'TDR_DATABASE_PATH', + 'TDR_GAME_ICON_STORAGE_PATH', + 'TURNSTILE_SECRET_KEY', + 'TURNSTILE_SITE_KEY', +]); +const SHARED_SECRET_NAMES = new Set(['SCHEDULER_TOKEN']); +const REMOVED_ENVIRONMENT_NAMES = [ + 'BASH_ENV', + 'CDPATH', + 'ENV', + 'IFS', + 'INFISICAL_ENVIRONMENT', + 'INFISICAL_PROJECT_ID', + 'INFISICAL_RUNTIME_SCOPE', + 'INFISICAL_TOKEN', + 'INFISICAL_UNIVERSAL_AUTH_ACCESS_TOKEN', + 'INFISICAL_UNIVERSAL_AUTH_CLIENT_ID', + 'INFISICAL_UNIVERSAL_AUTH_CLIENT_ID_FILE', + 'INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET', + 'INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET_FILE', + 'LD_LIBRARY_PATH', + 'LD_PRELOAD', + 'NODE_OPTIONS', + 'NODE_PATH', + 'PYTHONPATH', + 'TOKEN', +]; + +const [runtimeScope, environment, buildNumber = '0', commitSha = '', ...rest] = + process.argv.slice(2); +const separatorIndex = rest.indexOf('--'); +if ( + !['app', 'scheduler'].includes(runtimeScope) || + !['staging', 'prod'].includes(environment) || + separatorIndex < 1 +) { + console.error( + 'Usage: runtime-launcher.mjs -- ', + ); + process.exit(2); +} + +const secretFiles = rest.slice(0, separatorIndex); +const requestedCommand = rest.slice(separatorIndex + 1); +if (requestedCommand.length === 0) { + console.error('A runtime command is required.'); + process.exit(2); +} + +const expectedFileCount = runtimeScope === 'app' ? 2 : 1; +if (secretFiles.length !== expectedFileCount) { + console.error( + `${runtimeScope} requires ${expectedFileCount} secret export file(s).`, + ); + process.exit(2); +} + +const secrets = {}; +try { + await loadSecretFile( + secretFiles[0], + runtimeScope === 'app' ? APP_SECRET_NAMES : SHARED_SECRET_NAMES, + secrets, + ); + if (runtimeScope === 'app') { + await loadSecretFile(secretFiles[1], SHARED_SECRET_NAMES, secrets); + } +} finally { + const deletionResults = await Promise.allSettled( + secretFiles.map((file) => unlink(file)), + ); + const undeletedFiles = secretFiles.filter( + (_, index) => deletionResults[index].status === 'rejected', + ); + if (undeletedFiles.length > 0) { + throw new Error( + `Failed to delete Infisical secret export: ${undeletedFiles.join(', ')}`, + ); + } +} + +const childEnvironment = { ...process.env, ...secrets }; +for (const name of REMOVED_ENVIRONMENT_NAMES) delete childEnvironment[name]; +Object.assign( + childEnvironment, + trustedRuntimeEnvironment(runtimeScope, environment, buildNumber, commitSha), +); + +const errors = validateRuntimeEnvironment(runtimeScope, childEnvironment); +if (errors.length > 0) { + console.error( + `Invalid ${runtimeScope} runtime environment:\n${errors.map((error) => `- ${error}`).join('\n')}`, + ); + process.exit(1); +} + +process.umask(0o007); + +if (requestedCommand[0] === 'serve') { + await runAsNode(['node', 'scripts/db-migrate.mjs'], childEnvironment); + await runAsNode(['node', 'build'], childEnvironment); +} else if (requestedCommand[0] === 'scheduler') { + await runAsNode(['node', 'scripts/scheduler-client.mjs'], childEnvironment); +} else { + await runAsNode(requestedCommand, childEnvironment); +} + +async function loadSecretFile(file, allowedNames, target) { + let entries; + try { + entries = JSON.parse(await readFile(file, 'utf8')); + } catch { + throw new Error('Infisical returned an unreadable secret export.'); + } + if (!Array.isArray(entries)) { + throw new Error('Infisical secret export must be an array.'); + } + for (const entry of entries) { + const name = entry?.key; + const value = entry?.value; + if (typeof name !== 'string' || typeof value !== 'string') { + throw new Error('Infisical secret export contains an invalid entry.'); + } + if (!allowedNames.has(name)) { + throw new Error( + `Infisical secret ${name} is not allowed in this runtime scope.`, + ); + } + if (Object.hasOwn(target, name)) { + throw new Error(`Infisical secret ${name} was exported more than once.`); + } + target[name] = value; + } +} + +function trustedRuntimeEnvironment( + scope, + targetEnvironment, + release, + revision, +) { + const common = { + BODY_SIZE_LIMIT: '6M', + HOST: '0.0.0.0', + NODE_ENV: 'production', + PORT: '3000', + SHD_BUILD_NUMBER: release, + SHD_COMMIT_SHA: revision, + }; + if (scope === 'scheduler') { + return { + ...common, + SCHEDULER_URL: 'http://app:3000/api/internal/scheduler', + }; + } + if (targetEnvironment === 'staging') { + return { + ...common, + APP_ORIGIN: 'https://stage.shd.build', + DISCORD_REDIRECT_URI: 'https://stage.shd.build/auth/discord/callback', + ORIGIN: 'https://stage.shd.build', + PASSKEY_RP_ID: 'shd.build', + SKIP_DEFAULT_USERS: 'true', + TDR_AVATAR_STORAGE_PATH: '/app/data/avatars', + TDR_DATABASE_PATH: '/app/data/shd-build-staging.db', + TDR_GAME_ICON_STORAGE_PATH: '/app/data/game-icons', + UBISOFT_NEWS_IMPORT_ENABLED: 'false', + }; + } + return { + ...common, + APP_ORIGIN: 'https://shd.build', + DISCORD_REDIRECT_URI: 'https://shd.build/auth/discord/callback', + ORIGIN: 'https://shd.build', + PASSKEY_RP_ID: 'shd.build', + SKIP_DEFAULT_USERS: 'true', + TDR_AVATAR_STORAGE_PATH: '/app/data/avatars', + TDR_DATABASE_PATH: '/app/data/shd-build-production.db', + TDR_GAME_ICON_STORAGE_PATH: '/app/data/game-icons', + UBISOFT_NEWS_IMPORT_ENABLED: 'true', + }; +} + +function runAsNode(command, environmentVariables) { + const usesPrivilegeDrop = + typeof process.getuid === 'function' && process.getuid() === 0; + const executable = usesPrivilegeDrop ? 'gosu' : command[0]; + const args = usesPrivilegeDrop + ? ['shd-runtime', ...command] + : command.slice(1); + return new Promise((resolve, reject) => { + const child = spawn(executable, args, { + env: environmentVariables, + stdio: 'inherit', + }); + const signalHandlers = new Map(); + for (const signal of ['SIGHUP', 'SIGINT', 'SIGTERM']) { + const handler = () => child.kill(signal); + signalHandlers.set(signal, handler); + process.once(signal, handler); + } + const cleanup = () => { + for (const [signal, handler] of signalHandlers) { + process.off(signal, handler); + } + }; + child.once('error', (error) => { + cleanup(); + reject(error); + }); + child.once('exit', (code, signal) => { + cleanup(); + if (signal) { + process.kill(process.pid, signal); + } else if (code === 0) { + resolve(); + } else { + reject(new Error(`Runtime command exited with status ${code ?? 1}.`)); + } + }); + }); +} diff --git a/scripts/sync-staging-credentials.mjs b/scripts/sync-staging-credentials.mjs deleted file mode 100644 index 61638f1..0000000 --- a/scripts/sync-staging-credentials.mjs +++ /dev/null @@ -1,54 +0,0 @@ -import { spawnSync } from 'node:child_process'; -import { readFile, writeFile } from 'node:fs/promises'; -import { resolve } from 'node:path'; -import process from 'node:process'; - -const stagingFile = resolve('.env.staging'); -const dotenvxCli = resolve('node_modules/@dotenvx/dotenvx/src/cli/dotenvx.js'); -const sharedKeys = [ - 'DISCORD_CLIENT_ID', - 'DISCORD_CLIENT_SECRET', - 'SMTP_HOST', - 'SMTP_PORT', - 'SMTP_SECURE', - 'SMTP_USER', - 'SMTP_PASSWORD', - 'SMTP_FROM', -]; - -for (const key of sharedKeys) { - if (!process.env[key]) - throw new Error(`Source environment is missing ${key}.`); -} - -function dotenvx(action) { - const result = spawnSync( - process.execPath, - [dotenvxCli, action, '-f', stagingFile], - { - cwd: process.cwd(), - stdio: 'inherit', - }, - ); - if (result.error) throw result.error; - if (result.status !== 0) throw new Error(`dotenvx ${action} failed.`); -} - -dotenvx('decrypt'); -try { - let contents = await readFile(stagingFile, 'utf8'); - for (const key of sharedKeys) { - const replacement = `${key}=${JSON.stringify(process.env[key])}`; - const pattern = new RegExp(`^${key}=.*$`, 'm'); - if (!pattern.test(contents)) - throw new Error(`Staging environment is missing ${key}.`); - contents = contents.replace(pattern, replacement); - } - await writeFile(stagingFile, contents, 'utf8'); -} finally { - dotenvx('encrypt'); -} - -console.log( - 'Copied shared Discord and SMTP credentials into encrypted staging configuration.', -); diff --git a/scripts/validate-runtime-environment.mjs b/scripts/validate-runtime-environment.mjs new file mode 100644 index 0000000..0e9a288 --- /dev/null +++ b/scripts/validate-runtime-environment.mjs @@ -0,0 +1,20 @@ +#!/usr/bin/env node + +import process from 'node:process'; +import { validateRuntimeEnvironment } from './runtime-environment.js'; + +const mode = process.argv[2]; +if (mode !== 'app' && mode !== 'scheduler') { + console.error( + 'Usage: node scripts/validate-runtime-environment.mjs ', + ); + process.exit(2); +} + +const errors = validateRuntimeEnvironment(mode, process.env); +if (errors.length > 0) { + console.error( + `Invalid ${mode} runtime environment:\n${errors.map((error) => `- ${error}`).join('\n')}`, + ); + process.exit(1); +}