feat(deploy): support systemd --user process manager for nextjs-ssr

.agents/skills/smbcloud-deploy-nextjs/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: smbcloud-deploy-nextjs
-description: Use when deploying or debugging Next.js apps on smbCloud, especially the dedicated `nextjs-ssr` flow that builds locally, uploads `.next/standalone` via rsync, restarts PM2 over SSH, and serves traffic behind Nginx.
+description: Use when deploying or debugging Next.js apps on smbCloud, especially the dedicated `nextjs-ssr` flow that builds locally, uploads `.next/standalone` via rsync, restarts the app over SSH (PM2 by default, or a git-owned `systemctl --user` service when `process_manager = "systemd"`), and serves traffic behind Nginx.
 ---
 
 # smbCloud Deploy Next.js
@@ -83,6 +83,7 @@ package_manager = "pnpm"
 pm2_app = "my-app"
 path = "apps/web/my-app"
 port = 3028
+# process_manager = "systemd"  # optional; default is pm2 (see "systemd --user" below)
 ```
 
 Important rules:
@@ -93,8 +94,13 @@ Important rules:
 - `deploy_repo_id` identifies the DeployRepo record backing this app — include it when available
 - `source` is the local Next.js app directory
 - `path` is the remote destination directory on the server
-- `pm2_app` is mandatory because the CLI restarts PM2 by name
+- `pm2_app` is mandatory because the CLI restarts PM2 by name (with
+  `process_manager = "systemd"` this same value is the systemd `--user` unit
+  name, `<pm2_app>.service`)
 - `port` should be set explicitly and must match nginx
+- `process_manager` is optional. Omit it (or set `"pm2"`) for the default PM2
+  path; set `"systemd"` to restart via a git-owned `systemctl --user` service
+  instead (see "Process manager: systemd --user" below)
 - `deployment_method = 1` may still be present, but the `kind` routing is what matters most
 
 ## Expected remote layout
@@ -147,6 +153,45 @@ This has two consequences:
 - the configured `port` in `.smb/config.toml` is used for the fallback start path
 - the live server `ecosystem.config.cjs` (or `.js`) is the runtime source of truth when present
 
+## Process manager: systemd --user (alternative to PM2)
+
+Set `process_manager = "systemd"` in `[project]` to have the deploy restart the
+app through a **git-owned `systemctl --user` service** instead of PM2. The unit
+is named after `pm2_app` (`<pm2_app>.service`). This is the recommended target
+for the nbg box — a cgroup-supervised unit kills the whole process tree on
+`restart` (no orphaned `next-server` holding the port, the failure mode that
+caused the karokowe.com gateway-timeout outage) and survives reboot via linger,
+with no `pm2 save` to forget.
+
+The deploy restart step becomes (no sudo — the CLI SSHes as the git app-owner
+and drives that user's own manager):
+
+```sh
+export XDG_RUNTIME_DIR="/run/user/$(id -u)"
+UNIT="$PM2_APP.service"
+# fails loudly if the unit was never created
+systemctl --user cat "$UNIT" >/dev/null || exit 1
+systemctl --user restart "$UNIT"
+systemctl --user is-active "$UNIT"    # verified after a short sleep
+```
+
+Prerequisites (one-time per app, done out-of-band before the first
+`process_manager = "systemd"` deploy):
+
+1. Enable linger for the app-owner so its units run without an active login and
+   start at boot: `sudo loginctl enable-linger git`.
+2. Create the `~/.config/systemd/user/<pm2_app>.service` (+ a mode-600
+   `<pm2_app>.env`) and cut over off PM2. Use the migration generator
+   `pm2-to-systemd.sh <app> --apply`, which reads the live `pm2 jlist`,
+   reconstructs `ExecStart`/env faithfully, enables the unit, and `pm2 delete`s
+   the app.
+3. Set `process_manager = "systemd"` in the app's `.smb/config.toml`.
+
+After that, every `smb deploy` restarts via `systemctl --user` and the deploy
+aborts if the unit is missing (so a misconfigured `process_manager` can't
+silently leave the app down).
+
+
 ### Environment variables — ecosystem file (standard pattern)
 
 The server runs in a **multi-tenant setup**: multiple Next.js apps share one server, each managed as a separate PM2 process. The standard way to manage per-app environment variables is an `ecosystem.config.cjs` file kept **on the server only** (never committed to the app repo).

crates/cli/src/cloud-deploy/config.rs

@@ -124,6 +124,9 @@ pub(crate) async fn overlay_server_config(
             if let Some(remote_pm2_env) = deploy_config.pm2_env {
                 config.project.pm2_env = Some(remote_pm2_env);
             }
+            if let Some(remote_process_manager) = deploy_config.process_manager {
+                config.project.process_manager = Some(remote_process_manager);
+            }
             if let Some(remote_port) = deploy_config.port {
                 config.project.port = Some(remote_port);
             }

crates/cli/src/cloud-deploy/process_deploy_nextjs_ssr.rs

@@ -85,6 +85,67 @@ fn prune_dangling_symlinks(root: &std::path::Path) -> std::io::Result<usize> {
     Ok(removed)
 }
 
+/// Build the shell snippet that (re)starts the app after the standalone bundle
+/// is on the server.
+///
+/// Two supervisors are supported, selected by `process_manager`:
+/// - `None` / anything but `"systemd"` → **pm2** (default): delete + start
+///   fresh, preferring an operator-managed `ecosystem.config.cjs`/`.js`, else
+///   `pm2 start node -- server.js` with `PORT`/`HOSTNAME` inline.
+/// - `"systemd"` → a **git-owned `systemctl --user` service** named
+///   `<pm2_app>.service` (created out-of-band by the pm2-to-systemd migration).
+///   Restarted with no sudo — the deploy SSHes as the git app-owner, and
+///   `systemctl --user` (with `XDG_RUNTIME_DIR` set) drives that user's own
+///   manager. Fails loudly if the unit does not exist.
+///
+/// The returned snippet references the shell var `$PM2_APP` (set earlier in the
+/// deploy script). `port` is only used by the pm2 fallback path.
+fn restart_stanza(process_manager: Option<&str>, port: u16) -> String {
+    let use_systemd = process_manager
+        .map(|m| m.eq_ignore_ascii_case("systemd"))
+        .unwrap_or(false);
+
+    if use_systemd {
+        r#"    echo "Restarting $PM2_APP via systemd --user…"
+    export XDG_RUNTIME_DIR="/run/user/$(id -u)"
+    UNIT="$PM2_APP.service"
+    if ! systemctl --user cat "$UNIT" >/dev/null 2>&1; then
+        echo "Error: systemd --user unit $UNIT not found on the server."
+        echo "Create it first with the pm2-to-systemd migration, then re-deploy."
+        exit 1
+    fi
+    systemctl --user restart "$UNIT"
+    sleep 3
+    if ! systemctl --user is-active "$UNIT" >/dev/null 2>&1; then
+        echo "Error: $UNIT failed to become active after restart."
+        systemctl --user status "$UNIT" --no-pager 2>&1 | head -20 || true
+        exit 1
+    fi
+    echo "Done."
+"#
+        .to_string()
+    } else {
+        format!(
+            r#"    echo "Starting $PM2_APP with pm2..."
+    if pm2 describe "$PM2_APP" > /dev/null 2>&1; then
+        pm2 delete "$PM2_APP"
+    fi
+
+    if [ -f ecosystem.config.cjs ]; then
+        pm2 start ecosystem.config.cjs --only "$PM2_APP" --env production
+    elif [ -f ecosystem.config.js ]; then
+        pm2 start ecosystem.config.js --only "$PM2_APP" --env production
+    else
+        NODE_ENV=production PORT={port} HOSTNAME=127.0.0.1 pm2 start node --name "$PM2_APP" -- server.js
+    fi
+
+    pm2 save
+    echo "Done."
+"#
+        )
+    }
+}
+
 /// Deploys a Next.js SSR app using standalone output mode.
 ///
 /// Requires `output: 'standalone'` in next.config.js. This produces a
@@ -99,7 +160,8 @@ fn prune_dangling_symlinks(root: &std::path::Path) -> std::io::Result<usize> {
 ///   4. rsync .next/standalone/  → server:path/          (server + bundled deps)
 ///   5. rsync .next/static/      → server:path/.next/static/  (static chunks)
 ///   6. rsync public/            → server:path/public/   (public assets)
-///   7. SSH: pm2 delete + start fresh (prefer server `ecosystem.config.cjs` or `.js` if present)
+///   7. SSH: restart the app — pm2 (default) or a git-owned systemd --user
+///      service when `process_manager = "systemd"` (see `restart_stanza`)
 ///   8. PATCH deployment record as Done
 pub async fn process_deploy_nextjs_ssr(env: Environment, config: Config) -> Result<CommandResult> {
     let source = config.project.source.as_deref().unwrap_or(".");
@@ -530,6 +592,9 @@ pub async fn process_deploy_nextjs_ssr(env: Environment, config: Config) -> Resu
         )
     };
 
+    // Runtime supervisor: pm2 (default) or a git-owned systemd --user service.
+    let restart_stanza = restart_stanza(config.project.process_manager.as_deref(), port);
+
     let deploy_script = format!(
         r#"set -e
     APP_PATH="{remote_path}"
@@ -662,26 +727,11 @@ EOF
         ln -sfn ../../server.js .next/standalone/server.js
     fi
 
-    echo "Starting $PM2_APP with pm2..."
-    if pm2 describe "$PM2_APP" > /dev/null 2>&1; then
-        pm2 delete "$PM2_APP"
-    fi
-
-    if [ -f ecosystem.config.cjs ]; then
-        pm2 start ecosystem.config.cjs --only "$PM2_APP" --env production
-    elif [ -f ecosystem.config.js ]; then
-        pm2 start ecosystem.config.js --only "$PM2_APP" --env production
-    else
-        NODE_ENV=production PORT={port} HOSTNAME=127.0.0.1 pm2 start node --name "$PM2_APP" -- server.js
-    fi
-
-    pm2 save
-    echo "Done."
-    "#,
+{restart}    "#,
         remote_path = remote_path,
         pm2_app = pm2_app,
         runtime_subdir = runtime_subdir_for_shell,
-        port = port,
+        restart = restart_stanza,
         ecosystem_config = ecosystem_config_content,
     );
 
@@ -828,7 +878,11 @@ async fn mark_failed(
 
 #[cfg(all(test, unix))]
 mod tests {
-    use {super::prune_dangling_symlinks, std::os::unix::fs::symlink, tempfile::tempdir};
+    use {
+        super::{prune_dangling_symlinks, restart_stanza},
+        std::os::unix::fs::symlink,
+        tempfile::tempdir,
+    };
 
     /// Mirrors the real failure: a pnpm compat link points at a version dir that
     /// was never traced into the bundle, so it dangles and must be pruned, while
@@ -915,4 +969,27 @@ mod tests {
             "the dangling root symlink is gone"
         );
     }
+
+    #[test]
+    fn restart_stanza_defaults_to_pm2() {
+        for pm in [None, Some("pm2"), Some("PM2"), Some("other")] {
+            let s = restart_stanza(pm, 3025);
+            assert!(s.contains("pm2 start"), "pm={pm:?} should use pm2");
+            assert!(s.contains("PORT=3025"), "pm={pm:?} injects the port");
+            assert!(!s.contains("systemctl"), "pm={pm:?} must not touch systemd");
+        }
+    }
+
+    #[test]
+    fn restart_stanza_systemd_uses_user_unit_no_sudo() {
+        let s = restart_stanza(Some("systemd"), 3025);
+        assert!(s.contains("systemctl --user restart \"$UNIT\""));
+        assert!(s.contains("UNIT=\"$PM2_APP.service\""));
+        assert!(s.contains("XDG_RUNTIME_DIR=\"/run/user/$(id -u)\""));
+        // No sudo (git drives its own user manager) and no pm2 fallback.
+        assert!(!s.contains("sudo"), "must not require sudo");
+        assert!(!s.contains("pm2 "), "systemd path must not call pm2");
+        // Case-insensitive selector.
+        assert_eq!(s, restart_stanza(Some("SystemD"), 3025));
+    }
 }

crates/cli/src/cloud-deploy/process_migrate.rs

@@ -197,6 +197,7 @@ fn strip_project(project: &Project) -> Project {
         package_manager: None,
         pm2_app: None,
         pm2_env: None,
+        process_manager: None,
         port: None,
         shared_lib: None,
         compile_cmd: None,

crates/smbcloud-model/src/deploy_config.rs

@@ -35,6 +35,11 @@ pub struct DeployConfig {
     pub pm2_app: Option<String>,
     #[serde(default)]
     pub pm2_env: Option<std::collections::HashMap<String, serde_json::Value>>,
+    /// Runtime supervisor for `nextjs-ssr` apps: `"pm2"` (default) or
+    /// `"systemd"`. When `systemd`, deploys restart via a git-owned
+    /// `systemctl --user restart <pm2_app>.service` instead of `pm2`.
+    #[serde(default)]
+    pub process_manager: Option<String>,
     #[serde(default)]
     pub port: Option<u16>,
     #[serde(default)]

crates/smbcloud-model/src/project.rs

@@ -99,6 +99,13 @@ pub struct Project {
     /// Populated from the server-side App record; not written to `.smb/config.toml`.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub pm2_env: Option<std::collections::HashMap<String, serde_json::Value>>,
+    /// Runtime supervisor for `nextjs-ssr` apps: `"pm2"` (default) or `"systemd"`.
+    /// When `"systemd"`, the deploy restart step drives a git-owned
+    /// `systemctl --user restart <pm2_app>.service` (created by the
+    /// pm2-to-systemd migration) instead of `pm2 delete`/`pm2 start`.
+    /// `pm2_app` is still used as the unit name.
+    #[serde(default)]
+    pub process_manager: Option<String>,
     /// Port the standalone server binds to (default: 3000). Must match nginx upstream configuration.
     #[serde(default)]
     pub port: Option<u16>,