#!/usr/bin/env python3 """123cluster installer — install, update and uninstall the 123cluster application. Two installation modes are supported: • Typical — full install with built-in defaults; no questions asked. • Custom — prompts for PostgreSQL parameters (ports, credentials, backup info) and then lets you run individual steps from the menu. The four main steps are: 1. System checks — hardware requirements, OS detection, fresh-system check 2. Install PostgreSQL — install PG, configure, tune, restore DB from dump, PgBouncer 3. Install backend — base packages, Python 3.12, Node.js, app dirs, services 4. Install frontend — React build, Nginx Usage: sudo python3 123cluster_install.py # interactive menu sudo python3 123cluster_install.py --dry-run # preview changes without executing sudo python3 123cluster_install.py --debug # enable verbose debug logging Environment variables (all optional; defaults shown): LOGFILE /var/log/123cluster_install.log DISKSPACE 100 Minimum required free disk space in GB PGVERSION 17 PostgreSQL version to install DBPORT 5432 PostgreSQL port BOUNCER_PORT 6432 PgBouncer listen port BACKPORT 8000 Backend (uvicorn) listen port FRONTPORT 80 Frontend (Nginx) listen port APP_DIR /123cluster Application root directory BACKUPS_DIR /backups Backups root directory DB_NAME DB_123cluster DB_USER postgres DB_PASSWORD root123cluster DB_TIMEZONE Asia/Jerusalem DUMP_BASE_URL (auto) Base URL or local dir for dump files Defaults to {ARTIFACTS_URL}/dbbackup/ when set Defaults to {OFFLINE_DIR}/dbbackup/ when OFFLINE_DIR is set REPO_URL (empty) Internal package mirror base URL (e.g. http://172.16.0.10) When set, PostgreSQL, Node.js 20 and Python 3.12 are installed from the local mirror instead of the internet. ARTIFACTS_URL (empty) Base URL for application tarballs and pip wheels. Defaults to REPO_URL when not set (backward compatible). Set this independently to use online packages + a local artifact server (e.g. ARTIFACTS_URL=http://172.16.2.38). OFFLINE_DIR (empty) Path to offline bundle directory. When set: • ONLINE is forced to false • PGVERSION defaults to 15 • DNF repos point to ISO + RPM bundle (Rocky Linux only) • pip uses local wheel cache (no PyPI) • tarballs and dumps are read from local dirs Bundle layout: iso/ rpms/ wheels/ backend/ clustertasks/ frontend/ dbbackup/ ISO_MOUNT (auto) ISO mount path; defaults to {OFFLINE_DIR}/iso RELEASES_URL https://releases.123cluster.com Update manifest endpoint ONLINE true LOG_LEVEL INFO Python logging level (DEBUG / INFO / WARNING / ERROR) """ import os import shutil import subprocess import sys # ── Python version guard ────────────────────────────────────────────────────── # This block uses only syntax valid on Python 3.x so that older interpreters # can parse and execute it. If the version is too old, the script re-execs # itself via python3.10 (installing it first if necessary) before any # Python-3.10-specific code is ever reached. _REQUIRED_PY = (3, 10) if sys.version_info < _REQUIRED_PY: _cur = "{}.{}".format(sys.version_info.major, sys.version_info.minor) _req = "{}.{}".format(_REQUIRED_PY[0], _REQUIRED_PY[1]) print("Python {} detected — this installer requires Python {}+.".format(_cur, _req)) # Re-exec immediately if python3.12 or python3.10 is already on PATH for _pybin in ("python3.12", "python3.10"): _found = shutil.which(_pybin) if _found: print("{} found — restarting ...".format(_pybin)) os.execv(_found, [_pybin] + sys.argv) # Offer automatic installation print("Python 3.12 is not installed on this system.") try: _answer = input("Install Python 3.12 automatically? [Y/n]: ").strip().lower() except (EOFError, KeyboardInterrupt): _answer = "n" if _answer in ("n", "no"): sys.exit("Aborted — install Python 3.12 manually and re-run.") _pkg = None if shutil.which("apt-get"): _pkg = "apt" elif shutil.which("dnf"): _pkg = "dnf" elif shutil.which("yum"): _pkg = "yum" if _pkg is None: sys.exit("No supported package manager found. Install Python 3.12 manually.") _repo_url = os.environ.get("REPO_URL", "").rstrip("/") print("Installing Python 3.12 ...") if _pkg == "apt": if _repo_url: subprocess.run(["mkdir", "-p", "/etc/apt/keyrings"], check=False, capture_output=True) subprocess.run( "curl -fsSL {}/gpg/deadsnakes.asc" " | gpg --yes --dearmor -o /etc/apt/keyrings/local-deadsnakes.gpg".format( _repo_url), shell=True, check=False, capture_output=True, ) _raw_cn = subprocess.run( ["lsb_release", "-cs"], capture_output=True, text=True, ).stdout.strip() or "noble" _SUPPORTED = ("noble", "resolute", "trixie") _codename = _raw_cn if _raw_cn in _SUPPORTED else _SUPPORTED[-1] with open("/etc/apt/sources.list.d/local-python312.list", "w") as _f: _f.write( "deb [signed-by=/etc/apt/keyrings/local-deadsnakes.gpg]" " {}/deb/{}/python312/ {} main\n".format( _repo_url, _codename, _codename) ) subprocess.run(["apt-get", "update", "-y"], check=False, capture_output=True) else: # Detect OS family: deadsnakes PPA is Ubuntu-only. # For Debian use packages.sury.org instead. _os_id_b = "" try: with open("/etc/os-release") as _f: for _line in _f: if _line.startswith("ID="): _os_id_b = _line.split("=", 1)[1].strip().strip('"').lower() break except Exception: pass if _os_id_b == "ubuntu": subprocess.run(["apt-get", "install", "-y", "software-properties-common"], check=False, capture_output=True) subprocess.run(["add-apt-repository", "-y", "ppa:deadsnakes/ppa"], check=False, capture_output=True) elif _os_id_b == "debian": subprocess.run(["mkdir", "-p", "/etc/apt/keyrings"], check=False, capture_output=True) subprocess.run( "curl -fsSL https://packages.sury.org/python/apt.gpg" " | gpg --yes --dearmor -o /etc/apt/keyrings/sury-python.gpg", shell=True, check=False, capture_output=True, ) _deb_cn = subprocess.run( ["lsb_release", "-cs"], capture_output=True, text=True, ).stdout.strip() or "bookworm" with open("/etc/apt/sources.list.d/sury-python.list", "w") as _f: _f.write( "deb [signed-by=/etc/apt/keyrings/sury-python.gpg]" " https://packages.sury.org/python/ {} main\n".format(_deb_cn) ) subprocess.run(["apt-get", "update", "-y"], check=False, capture_output=True) subprocess.run( ["apt-get", "install", "-y", "python3.12", "python3.12-venv", "python3.12-dev"], check=False, capture_output=True, ) else: if _repo_url: _el_ver = "9" try: with open("/etc/os-release") as _f: for _line in _f: if _line.startswith("VERSION_ID="): _el_ver = _line.split("=", 1)[1].strip().strip('"').split(".")[0] break except Exception: pass _py312_gpg = "epel{}".format(_el_ver) with open("/etc/yum.repos.d/local-python312-bootstrap.repo", "w") as _f: _f.write( "[local-python312-bootstrap]\n" "name=Local Python 3.12 bootstrap\n" "baseurl={}/rpm/el{}/python312/\n" "enabled=1\ngpgcheck=1\nrepo_gpgcheck=0\n" "module_hotfixes=1\n" "gpgkey={}/gpg/{}.asc\n".format( _repo_url, _el_ver, _repo_url, _py312_gpg) ) else: subprocess.run([_pkg, "install", "-y", "epel-release"], check=False, capture_output=True) subprocess.run([_pkg, "install", "-y", "python3.12"], check=False, capture_output=True) _py312 = shutil.which("python3.12") if not _py312: sys.exit( "Python 3.12 installation failed. " "Install it manually and re-run this script." ) print("Python 3.12 installed — restarting script ...") os.execv(_py312, ["python3.12"] + sys.argv) sys.exit(1) # unreachable; silences linters # ── Remaining imports (Python 3.10 guaranteed beyond this point) ────────────── import getpass import glob import json import re import socket import secrets import logging import shlex import textwrap import threading import time from pathlib import Path from typing import Callable, List, Optional, Tuple # ── Config ──────────────────────────────────────────────────────────────────── LOGFILE = os.environ.get("LOGFILE", "/var/log/123cluster_install.log") APP_NAME = "123cluster" DISKSPACE = int(os.environ.get("DISKSPACE", 100)) _offline_dir_env = os.environ.get("OFFLINE_DIR", "") PGVERSION = os.environ.get("PGVERSION", "17") DBPORT = int(os.environ.get("DBPORT", 5432)) BOUNCER_PORT = int(os.environ.get("BOUNCER_PORT", 6432)) BACKPORT = int(os.environ.get("BACKPORT", 8000)) FRONTPORT = int(os.environ.get("FRONTPORT", 80)) APP_DIR = Path(os.environ.get("APP_DIR", "/123cluster")) BACKUPS_DIR = Path(os.environ.get("BACKUPS_DIR", "/backups")) DB_NAME = os.environ.get("DB_NAME", "DB_123cluster") DB_USER = os.environ.get("DB_USER", "postgres") DB_PASSWORD = os.environ.get("DB_PASSWORD", "root123cluster") DB_TIMEZONE = os.environ.get("DB_TIMEZONE", "Asia/Jerusalem") # Base URL or local directory where the dump files are stored. # Defaults to {ARTIFACTS_URL}/dbbackup/ when ARTIFACTS_URL (or REPO_URL) is set. # Override explicitly with DUMP_BASE_URL env var if needed. DUMP_BASE_URL = os.environ.get("DUMP_BASE_URL", "") DUMP_FILES: tuple = ( "schema_public.sql", "schema_app_123monitor.sql", "backup_schema.sql", ) # Internal package mirror base URL (Proxmox repo CT or NFS/local path). # When set, PGDG, Node.js 20 and Python 3.12 are installed from the local # mirror instead of the internet. # Accepted forms: http://172.16.2.38 | /mnt/repo | file:///mnt/repo REPO_URL = os.environ.get("REPO_URL", "").rstrip("/") if REPO_URL and not REPO_URL.startswith(("http://", "https://", "/", "file://")): REPO_URL = "http://" + REPO_URL ARTIFACTS_URL = os.environ.get("ARTIFACTS_URL", "").rstrip("/") if ARTIFACTS_URL and not ARTIFACTS_URL.startswith(("http://", "https://", "/", "file://")): ARTIFACTS_URL = "http://" + ARTIFACTS_URL ARTIFACTS_URL = ARTIFACTS_URL or REPO_URL # fall back to package mirror when not set explicitly def _is_local_path(url: str) -> bool: """True when url is a local filesystem path (absolute path or file:// scheme).""" return bool(url) and (url.startswith("/") or url.startswith("file://")) def _to_file_baseurl(url: str) -> str: """Convert a local path or file:// URL to file:// form for dnf/apt repo config.""" if url.startswith("file://"): return url.rstrip("/") return "file://" + url.rstrip("/") def _local_path_of(url: str) -> str: """Strip file:// scheme to return a plain filesystem path.""" if url.startswith("file://"): return url[7:].rstrip("/") return url.rstrip("/") def _fetch_artifact(base_url: str, rel_path: str, dest: "Path", spin: str = ""): """Copy or download an artifact. base_url may be http(s):// or a local/NFS path.""" if _is_local_path(base_url): src = Path(_local_path_of(base_url)) / rel_path return run(f"cp {src} {dest}", check=False, _spin=spin) return run(f"curl -fsSL --retry 3 {base_url}/{rel_path} -o {dest}", check=False, _spin=spin) def _artifact_exists(base_url: str, rel_path: str) -> bool: """Return True if the artifact is reachable on the mirror without downloading it.""" if _is_local_path(base_url): return (Path(_local_path_of(base_url)) / rel_path).exists() r = subprocess.run( ["curl", "-fsSL", "--head", "--max-time", "5", f"{base_url}/{rel_path}"], capture_output=True, ) return r.returncode == 0 ADMIN_USER = os.environ.get("ADMIN_USER", "admin") ADMIN_PASSWORD = os.environ.get("ADMIN_PASSWORD", "admin123cluster") ORG_NAME = os.environ.get("ORG_NAME", "initial") # SQL schema files to apply in order during database restore. # When ARTIFACTS_URL is set, files are fetched from {ARTIFACTS_URL}/dbbackup/. # DUMP_BASE_URL can override the source explicitly. RELEASES_URL = os.environ.get("RELEASES_URL", "https://releases.123cluster.com") ONLINE = os.environ.get("ONLINE", "true").lower() == "true" LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO").upper() # Offline install mode — set OFFLINE_DIR to a directory containing: # iso/ Rocky Linux 9.8 Everything ISO mounted here (BaseOS + AppStream) # rpms/ PGDG PG15 + PgBouncer RPMs (with createrepo metadata) # wheels/ pip wheels for requirements.txt + tasks_requirements.txt # backend/backend.tar.gz, clustertasks/clustertasks.tar.gz, frontend/build.tar.gz # dbbackup/ SQL dump files OFFLINE_DIR: Optional[Path] = Path(_offline_dir_env) if _offline_dir_env else None ISO_MOUNT: Optional[Path] = ( Path(os.environ["ISO_MOUNT"]) if "ISO_MOUNT" in os.environ else (OFFLINE_DIR / "iso" if OFFLINE_DIR else None) ) if OFFLINE_DIR: ONLINE = False # no internet in offline mode if OFFLINE_DIR and not DUMP_BASE_URL: DUMP_BASE_URL = str(OFFLINE_DIR / "dbbackup") BINARY_PKGS: bool = True # always use Nuitka binaries; source tarballs are not distributed INSTALL_PGBOUNCER = True # PgBouncer is mandatory; not configurable DRY_RUN = "--dry-run" in sys.argv DEBUG = "--debug" in sys.argv or LOG_LEVEL == "DEBUG" # Hard minimums — the script refuses to proceed below these thresholds _MIN_RAM_MB = 3584 # 3.5 GB — fits a 4 GB VPS after OS overhead _MIN_CPUS = 2 _MIN_DISK_GB = 100 # ── Logging ─────────────────────────────────────────────────────────────────── Path(LOGFILE).parent.mkdir(parents=True, exist_ok=True) _level = logging.DEBUG if DEBUG else getattr(logging, LOG_LEVEL, logging.INFO) _fmt = logging.Formatter("%(asctime)s %(message)s", datefmt="%H:%M:%S") # File handler — receives everything: status messages AND raw command output. _file_handler = logging.FileHandler(LOGFILE) _file_handler.setLevel(logging.DEBUG) _file_handler.setFormatter(_fmt) # Stream handler — terminal only shows status lines (INFO+). # Raw command stdout/stderr is suppressed from the terminal. # When a spinner is active it prepends \n so the log line doesn't overwrite # the spinner text mid-line. class _SpinnerAwareStreamHandler(logging.StreamHandler): def emit(self, record: logging.LogRecord) -> None: if _Spinner.active: self.stream.write("\n") self.stream.flush() super().emit(record) _stream_handler = _SpinnerAwareStreamHandler(sys.stdout) _stream_handler.setLevel(_level) _stream_handler.setFormatter(_fmt) log = logging.getLogger() log.setLevel(logging.DEBUG) log.addHandler(_file_handler) log.addHandler(_stream_handler) # Separate logger for raw subprocess output. It writes to the same file # but has propagate=False so its records never reach the stream handler. cmd_log = logging.getLogger("cmd_output") cmd_log.setLevel(logging.DEBUG) cmd_log.addHandler(_file_handler) cmd_log.propagate = False class _Spinner: """Display an animated spinner with elapsed time for long-running operations. Usage:: with _Spinner("Installing packages"): run(...) Or via run()'s _spin= keyword:: run("apt-get install ...", _spin="Installing packages") """ _CHARS = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏" active: bool = False # class-level flag read by _SpinnerAwareStreamHandler def __init__(self, msg: str) -> None: self._msg = msg self._stop = threading.Event() self._t = threading.Thread(target=self._loop, daemon=True) def _loop(self) -> None: t0, i = time.monotonic(), 0 while not self._stop.is_set(): c = self._CHARS[i % len(self._CHARS)] elapsed = int(time.monotonic() - t0) sys.stdout.write(f"\r {c} {self._msg} ({elapsed}s)") sys.stdout.flush() i += 1 self._stop.wait(0.1) sys.stdout.write("\r" + " " * 72 + "\r") sys.stdout.flush() def __enter__(self) -> "_Spinner": _Spinner.active = True self._t.start() return self def __exit__(self, *_) -> None: _Spinner.active = False self._stop.set() self._t.join() # ── Output helpers ──────────────────────────────────────────────────────────── def ok(msg: str) -> None: """Print a green success line.""" log.info(f"\033[32m✔\033[0m {msg}") def warn(msg: str) -> None: """Print a yellow warning line.""" log.warning(f"\033[33m⚠\033[0m {msg}") def err(msg: str) -> None: """Print a red error line.""" log.error(f"\033[31m✖\033[0m {msg}") def info(msg: str) -> None: """Print a plain informational line.""" log.info(f" {msg}") # ── System helpers ───────────────────────────────────────────────────────────── def require_root() -> None: """Exit with an error message if the script is not running as root.""" if os.geteuid() != 0: err("Run as root") sys.exit(1) def have_cmd(cmd: str) -> bool: """Return True if `cmd` is found on PATH.""" return shutil.which(cmd) is not None PKG_MGR: str = "" def pkg_detect() -> None: """Detect the system package manager and store it in the global PKG_MGR. Supports apt (Debian/Ubuntu), dnf (Fedora/RHEL 8+/Rocky/Alma), and yum (RHEL 7). Exits if none of these are found. """ global PKG_MGR if have_cmd("apt-get"): PKG_MGR = "apt" elif have_cmd("dnf"): PKG_MGR = "dnf" elif have_cmd("yum"): PKG_MGR = "yum" else: err("No supported package manager found (apt-get / dnf / yum)") sys.exit(1) def run(cmd: str, check: bool = True, _spin: str = "") -> subprocess.CompletedProcess: """Run a shell command. stdout and stderr are captured and written to the log file only — the terminal stays clean and shows only ok/warn/err/info status lines. Pass _spin="message" to display an animated spinner while the command runs. When check=True and the command exits non-zero the last 20 lines of combined output are echoed to the terminal as errors before the exception is raised, so the operator can see what went wrong without opening the log. In DRY_RUN mode the command is printed but not executed. """ if DRY_RUN: log.info(f"[DRY RUN] {cmd}") return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="") cmd_log.debug(f"$ {cmd}") if _spin: spinner = _Spinner(_spin) spinner.__enter__() result = subprocess.run( cmd, shell=True, capture_output=True, text=True, errors="replace", ) if _spin: spinner.__exit__(None, None, None) combined = (result.stdout or "") + (result.stderr or "") for line in combined.splitlines(): if line.strip(): cmd_log.debug(line) if result.returncode != 0 and check: tail = [l for l in combined.splitlines() if l.strip()][-20:] for line in tail: err(f" {line}") raise subprocess.CalledProcessError( result.returncode, cmd, result.stdout, result.stderr, ) return result def run_output(cmd: str) -> str: """Run a command and return its stdout as a stripped string. Always executes even in DRY_RUN mode — read-only queries must run so detection logic (IP, package versions, service state) works in previews. Returns an empty string on error. """ cmd_log.debug(f"$ {cmd}") result = subprocess.run( cmd, shell=True, capture_output=True, text=True, errors="replace", ) if result.stdout: cmd_log.debug(result.stdout.strip()) return result.stdout.strip() def write_file(path: Path, content: str, mode: int = 0o644) -> None: """Write `content` to `path`, creating parent directories if needed. In DRY_RUN mode logs the operation without touching the filesystem. """ if DRY_RUN: log.info(f"[DRY RUN] write {path} ({len(content)} chars)") return path.parent.mkdir(parents=True, exist_ok=True) path.write_text(content) os.chmod(path, mode) def append_file(path: Path, content: str) -> None: """Append `content` to `path` only if it is not already present (idempotent).""" if DRY_RUN: log.info(f"[DRY RUN] append {path}") return existing = path.read_text() if path.exists() else "" if content.strip() not in existing: with open(path, "a") as fh: fh.write(content) def copy_file(src: str, dst: str) -> None: """Copy `src` to `dst`, preserving metadata. No-op in DRY_RUN mode.""" if DRY_RUN: log.info(f"[DRY RUN] copy {src} → {dst}") return shutil.copy2(src, dst) def _wait_apt_lock(timeout: int = 120) -> None: import time deadline = time.time() + timeout while time.time() < deadline: r = subprocess.run( "flock -n /var/lib/dpkg/lock-frontend true", shell=True, capture_output=True, ) if r.returncode == 0: return time.sleep(3) warn("apt lock still held after timeout — proceeding anyway") def pkg_install(*packages: str) -> None: """Install one or more packages using the detected package manager.""" pkgs = " ".join(packages) label = f"Installing {pkgs[:50]}{'…' if len(pkgs) > 50 else ''}" if PKG_MGR == "apt": _wait_apt_lock() run(f"DEBIAN_FRONTEND=noninteractive apt-get install -y {pkgs}", _spin=label) elif PKG_MGR in ("dnf", "yum"): run(f"{PKG_MGR} install -y {pkgs}", _spin=label) def _pip_flags() -> str: """Return extra pip flags for offline mode (no-index + local wheel dir).""" if OFFLINE_DIR: wheels = OFFLINE_DIR / "wheels" if wheels.exists(): return f"--no-index --find-links={wheels}" return "" def service_active(name: str) -> bool: """Return True if the named systemd unit is currently running.""" r = subprocess.run( f"systemctl is-active {name}", shell=True, capture_output=True, text=True, ) return r.stdout.strip() == "active" def get_server_ip() -> str: """Return the primary outbound IPv4 address of this host. Uses the routing table to find the source address used when reaching the internet. Falls back to the hostname resolution, then to 127.0.0.1. """ result = run_output( "ip route get 8.8.8.8 2>/dev/null" " | awk '{for(i=1;i<=NF;i++) if($i==\"src\") print $(i+1); exit}'" ) if result: return result try: import socket return socket.gethostbyname(socket.gethostname()) except Exception: return "127.0.0.1" def _os_version_id() -> str: """Return the major OS version number from /etc/os-release (e.g. '9').""" try: for line in Path("/etc/os-release").read_text().splitlines(): if line.startswith("VERSION_ID="): return line.split("=", 1)[1].strip('"').split(".")[0] except FileNotFoundError: pass return "8" def _os_id() -> str: """Return the OS ID from /etc/os-release (e.g. 'ubuntu', 'rocky', 'debian').""" try: for line in Path("/etc/os-release").read_text().splitlines(): if line.startswith("ID="): return line.split("=", 1)[1].strip('"').lower() except FileNotFoundError: pass return "unknown" def _os_codename() -> str: """Return the OS version codename (e.g. 'jammy', 'bookworm'). Falls back to lsb_release when the value is absent from /etc/os-release. """ try: for line in Path("/etc/os-release").read_text().splitlines(): if line.startswith("VERSION_CODENAME="): return line.split("=", 1)[1].strip('"').lower() except FileNotFoundError: pass return run_output("lsb_release -cs 2>/dev/null") or "noble" # Codenames carried by the local CT 500 mirror, oldest→newest. # When the host OS has a newer codename not yet mirrored, fall back to the # most recent one here rather than pointing at a non-existent repo path. _MIRROR_DEB_CODENAMES: tuple[str, ...] = ("noble", "resolute", "trixie") _MIRROR_DEB_DEFAULT = _MIRROR_DEB_CODENAMES[-1] # newest supported def _mirror_codename() -> str: """Return the codename to use for local-mirror APT source entries. If the host OS codename is not yet mirrored (e.g. a distro newer than noble) this returns the latest supported codename and emits a warning so operators know to add it to the CT 500 sync scripts. """ cn = _os_codename() if cn in _MIRROR_DEB_CODENAMES: return cn warn( f"OS codename '{cn}' is not in the local mirror " f"(supported: {', '.join(_MIRROR_DEB_CODENAMES)}) — " f"falling back to '{_MIRROR_DEB_DEFAULT}' packages" ) return _MIRROR_DEB_DEFAULT def setup_offline_repos() -> None: """Configure DNF to use the offline bundle (ISO + RPM bundle) instead of internet repos. Creates two local repo files: offline-iso.repo — Rocky Linux 9.8 Everything ISO (BaseOS + AppStream) offline-rpms.repo — PGDG PG15 + PgBouncer RPM bundle All pre-existing internet repos are disabled so DNF never reaches the network. Called automatically from setup_local_repos() when OFFLINE_DIR is set. """ if not OFFLINE_DIR: return pkg_detect() if PKG_MGR not in ("dnf", "yum"): warn("Offline DNF repo setup skipped — not a dnf/yum system") return iso_path = ISO_MOUNT or (OFFLINE_DIR / "iso") rpms_path = OFFLINE_DIR / "rpms" if not iso_path.exists(): warn(f"ISO mount not found at {iso_path} — DNF may fail to locate packages") if not rpms_path.exists(): warn(f"RPM bundle not found at {rpms_path} — PGDG packages will be unavailable") run("dnf config-manager --set-disabled '*'", check=False) write_file( Path("/etc/yum.repos.d/offline-iso.repo"), f"""\ [offline-baseos] name=Rocky Linux 9.8 - BaseOS (offline) baseurl=file://{iso_path}/BaseOS enabled=1 gpgcheck=0 [offline-appstream] name=Rocky Linux 9.8 - AppStream (offline) baseurl=file://{iso_path}/AppStream enabled=1 gpgcheck=0 """, ) if rpms_path.exists(): write_file( Path("/etc/yum.repos.d/offline-rpms.repo"), f"""\ [offline-rpms] name=Offline RPM Bundle (PGDG PG15 + PgBouncer) baseurl=file://{rpms_path} enabled=1 gpgcheck=0 """, ) run("dnf -qy module disable postgresql 2>/dev/null || true", check=False) run("dnf makecache", check=False, _spin="Building offline DNF cache") ok(f"Offline DNF repos configured (ISO: {iso_path})") def setup_local_repos() -> None: """Configure package-manager repos to use the internal mirror (REPO_URL). No-op when REPO_URL is empty. On RPM systems writes /etc/yum.repos.d/local-mirror.repo; on APT systems writes source list files under /etc/apt/sources.list.d/ and imports GPG keys from the mirror. Idempotent: safe to call multiple times. """ if OFFLINE_DIR: setup_offline_repos() return if not REPO_URL: return pkg_detect() ver = _os_version_id() codename = _mirror_codename() # normalized to nearest mirrored release if PKG_MGR in ("dnf", "yum"): # For local/NFS paths use file:// scheme; HTTP(S) URLs are used as-is. _repo_base = _to_file_baseurl(REPO_URL) if _is_local_path(REPO_URL) else REPO_URL write_file( Path("/etc/yum.repos.d/local-mirror.repo"), f"""\ [local-baseos] name=Local BaseOS baseurl={_repo_base}/iso/BaseOS/ enabled=1 gpgcheck=0 skip_if_unavailable=1 [local-appstream] name=Local AppStream baseurl={_repo_base}/iso/AppStream/ enabled=1 gpgcheck=0 skip_if_unavailable=1 [local-pgdg{PGVERSION}] name=Local PGDG {PGVERSION} baseurl={_repo_base}/rpm/el{ver}/pgdg{PGVERSION}/ enabled=1 gpgcheck=0 repo_gpgcheck=0 module_hotfixes=1 [local-nodejs20] name=Local Node.js 20 baseurl={_repo_base}/rpm/nodejs20/ enabled=1 gpgcheck=0 repo_gpgcheck=0 module_hotfixes=1 [local-python312] name=Local Python 3.12 baseurl={_repo_base}/rpm/el{ver}/python312/ enabled=1 gpgcheck=0 repo_gpgcheck=0 module_hotfixes=1 [local-mariadb-connector] name=Local MariaDB Connector/C baseurl={_repo_base}/rpm/el{ver}/mariadb-connector-c/ enabled=1 gpgcheck=0 skip_if_unavailable=1 """, ) ok(f"Local RPM repos configured ({REPO_URL})") # Disable any online PGDG repo files left by a previous run so DNF # cannot reach them and pull a different PG version from the internet. run( "find /etc/yum.repos.d/ -name 'pgdg*.repo' ! -name 'local-mirror.repo'" " -exec sed -i 's/^enabled *= *1/enabled=0/g' {} + 2>/dev/null || true", check=False, ) run("dnf clean metadata -q 2>/dev/null || true", check=False) elif PKG_MGR == "apt": # Remove any stale local-repo files from previous partial runs so they # don't cause GPG-signature errors that block the initial apt-get update # and prevent curl/gnupg from being installed. for _stale in [ "/etc/apt/sources.list.d/local-pgdg.list", "/etc/apt/sources.list.d/local-nodejs.list", "/etc/apt/sources.list.d/local-python312.list", "/etc/apt/keyrings/local-pgdg.gpg", "/etc/apt/keyrings/local-nodesource.gpg", "/etc/apt/keyrings/local-deadsnakes.gpg", ]: Path(_stale).unlink(missing_ok=True) # Refresh base Ubuntu package lists so curl/gnupg can be fetched from # the clean ubuntu archive before any local repo files are written. _wait_apt_lock() run("apt-get update -y", check=False, _spin="apt-get update") run("DEBIAN_FRONTEND=noninteractive apt-get install -y curl gnupg", check=False, _spin="Installing curl gnupg") run("mkdir -p /etc/apt/keyrings", check=False) if _is_local_path(REPO_URL): # Local path or NFS mount — skip GPG key fetching; use trusted=yes _repo_base = _to_file_baseurl(REPO_URL) write_file( Path("/etc/apt/sources.list.d/local-pgdg.list"), f"deb [trusted=yes] {_repo_base}/deb/{codename}/pgdg/ {codename}-pgdg main\n", ) write_file( Path("/etc/apt/sources.list.d/local-nodejs.list"), f"deb [trusted=yes] {_repo_base}/deb/nodejs20/ nodistro main\n", ) write_file( Path("/etc/apt/sources.list.d/local-python312.list"), f"deb [trusted=yes] {_repo_base}/deb/{codename}/python312/ {codename} main\n", ) else: def _import_key(name: str, url: str, out: str) -> bool: r = run(f"curl -fsSL {url} | gpg --dearmor > {out}", check=False) if r.returncode != 0 or not Path(out).exists() or not Path(out).stat().st_size: warn(f"GPG key import failed for {name} — repo will be skipped") Path(out).unlink(missing_ok=True) return False return True _pgdg_key_ok = _import_key("pgdg", f"{REPO_URL}/gpg/pgdg.asc", "/etc/apt/keyrings/local-pgdg.gpg") _nodesource_key_ok = _import_key("nodesource", f"{REPO_URL}/gpg/nodesource-deb.asc", "/etc/apt/keyrings/local-nodesource.gpg") _deadsnakes_key_ok = _import_key("deadsnakes", f"{REPO_URL}/gpg/deadsnakes.asc", "/etc/apt/keyrings/local-deadsnakes.gpg") if _pgdg_key_ok: write_file( Path("/etc/apt/sources.list.d/local-pgdg.list"), f"deb [signed-by=/etc/apt/keyrings/local-pgdg.gpg]" f" {REPO_URL}/deb/{codename}/pgdg/ {codename}-pgdg main\n", ) if _nodesource_key_ok: write_file( Path("/etc/apt/sources.list.d/local-nodejs.list"), f"deb [signed-by=/etc/apt/keyrings/local-nodesource.gpg]" f" {REPO_URL}/deb/nodejs20/ nodistro main\n", ) if _deadsnakes_key_ok: write_file( Path("/etc/apt/sources.list.d/local-python312.list"), f"deb [signed-by=/etc/apt/keyrings/local-deadsnakes.gpg]" f" {REPO_URL}/deb/{codename}/python312/ {codename} main\n", ) run("apt-get update -y", check=False, _spin="apt-get update") ok(f"Local APT repos configured ({REPO_URL})") # ── Modules ─────────────────────────────────────────────────────────────────── def sys_checks(check_freshness: bool = True) -> bool: """Verify minimum system requirements and (optionally) freshness. Hard requirements (returns False if not met): • Running as root • At least 2 CPU cores • At least 3.5 GB RAM • At least 100 GB free disk space (or DISKSPACE env var, whichever is larger) Freshness requirements (only checked when check_freshness=True): • No PostgreSQL instance already installed or running • No Nginx service already running • No PgBouncer service already running • No existing 123cluster backend virtualenv found Pass check_freshness=False for Force-mode installs that intentionally run over an existing installation. Soft warnings are emitted for below-recommended values (8 CPU / 16 GB RAM / 500 GB disk) without terminating the check. """ require_root() pkg_detect() ok(f"Package manager: {PKG_MGR}") # ── Hardware checks ─────────────────────────────────────────────────────── meminfo = Path("/proc/meminfo").read_text() ram_kb = int(next(l for l in meminfo.splitlines() if "MemTotal" in l).split()[1]) ram_mb = ram_kb // 1024 cpus = os.cpu_count() or 1 st = os.statvfs("/") free_gb = (st.f_bavail * st.f_frsize) // (1024 ** 3) failed = False if ram_mb < _MIN_RAM_MB: err(f"RAM {ram_mb} MB — minimum required is {_MIN_RAM_MB} MB (3.5 GB)") failed = True elif ram_mb < 16_000: warn(f"RAM {ram_mb} MB — recommended minimum is 16 384 MB") if cpus < _MIN_CPUS: err(f"CPU count {cpus} — minimum required is {_MIN_CPUS}") failed = True elif cpus < 8: warn(f"CPU count {cpus} — recommended minimum is 8 cores") min_disk = max(_MIN_DISK_GB, DISKSPACE) if free_gb < min_disk: err(f"Free disk {free_gb} GB < {min_disk} GB required") failed = True elif free_gb < 500: warn(f"Disk {free_gb} GB free — 500 GB recommended for production use") if failed: return False # ── Freshness checks ────────────────────────────────────────────────────── if not check_freshness: info("Freshness check skipped (Force mode)") ok("Hardware checks passed") # Still print the summary os_info: dict = {} try: for line in Path("/etc/os-release").read_text().splitlines(): if "=" in line: k, _, v = line.partition("=") os_info[k] = v.strip('"') except FileNotFoundError: pass info(f"OS: {os_info.get('PRETTY_NAME', 'unknown')}") info(f"RAM: {ram_mb} MB") info(f"CPU: {cpus} cores") info(f"Disk: {free_gb} GB free") info(f"IP: {get_server_ip()}") return True # The system must be clean before a fresh install can proceed. conflicts: List[str] = [] if detect_existing_postgres(): conflicts.append("PostgreSQL is already installed or running") if service_active("nginx"): conflicts.append("Nginx is already running") if service_active("pgbouncer"): conflicts.append("PgBouncer is already running") if (APP_DIR / "backend" / "venv" / "bin" / "activate").exists(): conflicts.append(f"123cluster backend already installed at {APP_DIR}") elif APP_DIR.exists() and any(APP_DIR.iterdir()): conflicts.append( f"Partial 123cluster install found at {APP_DIR} (no venv — previous install failed). " "Run W to wipe first, or use Force install (option 2)." ) # Check that required ports are not already bound by foreign processes. _busy: list[str] = [] for _port, _label in ( (FRONTPORT, f"port {FRONTPORT} (FRONTPORT/nginx)"), (BACKPORT, f"port {BACKPORT} (BACKPORT/backend)"), (BOUNCER_PORT, f"port {BOUNCER_PORT} (BOUNCER_PORT/pgbouncer)"), ): try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as _s: _s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) _s.bind(("", _port)) except OSError: _busy.append(_label) if _busy: conflicts.append("Port(s) already in use by another process: " + ", ".join(_busy)) if conflicts: err("System is not clean — cannot proceed with a fresh install:") for c in conflicts: err(f" • {c}") err("Remove existing installations before running this installer.") return False # ── OS / hardware summary ───────────────────────────────────────────────── _os_info: dict = {} try: for line in Path("/etc/os-release").read_text().splitlines(): if "=" in line: k, _, v = line.partition("=") _os_info[k] = v.strip('"') except FileNotFoundError: pass info(f"OS: {_os_info.get('PRETTY_NAME', 'unknown')}") info(f"RAM: {ram_mb} MB") info(f"CPU: {cpus} cores") info(f"Disk: {free_gb} GB free") info(f"IP: {get_server_ip()}") ok("System checks passed — system is clean and meets all requirements") return True def prompt_mirror_urls() -> None: """Interactively set REPO_URL and ARTIFACTS_URL. Shows current values; press Enter to keep them. Both accept a bare hostname/IP — 'http://' is prepended automatically. """ global REPO_URL, ARTIFACTS_URL _arts_explicit = os.environ.get("ARTIFACTS_URL", "").strip() # was it set by the user? print("\n" + "─" * 58) print(" Mirror URLs (Enter = keep current value, '-' = clear)") print("─" * 58) _repo_hint = REPO_URL or "not set — use official online repos" raw = input(f" Package mirror REPO_URL [{_repo_hint}]: ").strip() if raw == "-": REPO_URL = "" elif raw: REPO_URL = raw if raw.startswith(("http://", "https://", "/", "file://")) else "http://" + raw _arts_hint = ( _arts_explicit or (f"{REPO_URL} (same as REPO_URL)" if REPO_URL else "not set") ) raw = input(f" App artifacts ARTIFACTS_URL [{_arts_hint}]: ").strip() if raw == "-": ARTIFACTS_URL = REPO_URL # fall back to package mirror elif raw: ARTIFACTS_URL = raw if raw.startswith(("http://", "https://", "/", "file://")) else "http://" + raw else: ARTIFACTS_URL = _arts_explicit if _arts_explicit else REPO_URL print("─" * 58) info(f" REPO_URL : {REPO_URL or '(not set)'}") info(f" ARTIFACTS_URL : {ARTIFACTS_URL or '(not set)'}") def ask_postgres_params() -> None: """Interactively collect PostgreSQL parameters and print a disk-space estimate. Updates the globals DBPORT, BOUNCER_PORT, DB_NAME, DB_USER, DB_PASSWORD, BACKUPS_DIR, and DUMP_BASE_URL in place. Also asks the customer about the number of databases, estimated DB size, and backup retention to derive a recommended total disk requirement. """ global DBPORT, BOUNCER_PORT, DB_NAME, DB_USER, DB_PASSWORD global BACKUPS_DIR, DUMP_BASE_URL print("\n" + "─" * 58) print(" PostgreSQL configuration") print("─" * 58) # ── Ports ───────────────────────────────────────────────────────────────── raw = input(f" PostgreSQL port [{DBPORT}]: ").strip() if raw.isdigit(): DBPORT = int(raw) raw = input(f" PgBouncer port [{BOUNCER_PORT}]: ").strip() if raw.isdigit(): BOUNCER_PORT = int(raw) # ── Credentials ─────────────────────────────────────────────────────────── raw = input(f" Database name [{DB_NAME}]: ").strip() if raw: DB_NAME = raw raw = input(f" Database user (low pri.) [{DB_USER}]: ").strip() if raw: DB_USER = raw raw = getpass.getpass(" Database password [keep default, hidden]: ").strip() if raw: DB_PASSWORD = raw # ── Capacity planning ───────────────────────────────────────────────────── print() raw = input(" Number of databases [1]: ").strip() num_dbs = int(raw) if raw.isdigit() and int(raw) > 0 else 1 raw = input(" Estimated size per database (GB) [10]: ").strip() try: db_size_gb = float(raw) if raw else 10.0 except ValueError: db_size_gb = 10.0 raw = input(" Backup copies to retain [7]: ").strip() backup_copies = int(raw) if raw.isdigit() and int(raw) > 0 else 7 raw = input(f" Backup storage path [{BACKUPS_DIR}]: ").strip() if raw: BACKUPS_DIR = Path(raw) data_gb = num_dbs * db_size_gb backup_gb = data_gb * backup_copies * 1.1 # 10 % overhead/compression margin total_gb = data_gb + backup_gb + 20 # +20 GB for OS/app/logs print() info("── Disk-space estimate ──────────────────────────────") info(f" Databases : {num_dbs} × {db_size_gb:.0f} GB = {data_gb:.0f} GB") info(f" Backups : {backup_copies} copies × {data_gb:.0f} GB × 1.1 overhead = {backup_gb:.0f} GB") info(f" OS / app : ~20 GB") info(" ─────────────────────────────────────────────────") info(f" Recommended: ~{total_gb * 1.3:.0f} GB (estimated {total_gb:.0f} GB + 30 % headroom)") # ── Dump base URL / directory ───────────────────────────────────────────── print() _default_dump = f"{ARTIFACTS_URL}/dbbackup" if ARTIFACTS_URL else "(not set)" info(f" Schema dumps are fetched from: {DUMP_BASE_URL or _default_dump}") info(f" Files applied in order: {', '.join(DUMP_FILES)}") info(" Provide a base URL (http://…) or a local directory path.") raw = input(f" Dump base URL / dir [{DUMP_BASE_URL}]: ").strip() if raw: DUMP_BASE_URL = raw print("─" * 58) ok("PostgreSQL parameters saved") def base_packages() -> None: """Install base system packages required by the application and build tooling. Includes compiler tooling, Python development headers, PostgreSQL client headers, and ODBC libraries needed by some Django database drivers. """ pkg_detect() common = "curl ca-certificates tar unzip rsync net-tools jq git wget" if PKG_MGR == "apt": run("apt-get update -y", check=False, _spin="apt-get update") pkg_install( common, "build-essential", "libpq-dev", "python3-dev", "unixodbc", "unixodbc-dev", "libmariadb-dev", ) else: if BINARY_PKGS: pkg_install(common, "unixODBC") else: pkg_install( common, "gcc", "gcc-c++", "make", "python3-devel", "unixODBC", ) # postgresql-devel requires perl(IPC::Run) which may be absent from # minimal mirrors; non-fatal — psycopg2 can install from a binary wheel r = run("dnf install -y postgresql-devel", check=False) if r.returncode != 0: warn("postgresql-devel not available — psycopg2 will need a binary wheel") # unixODBC-devel was removed in EL10 — try but don't abort if missing r = run("dnf install -y unixODBC-devel", check=False) if r.returncode != 0: warn("unixODBC-devel not available on this platform — skipping") ok("Base packages installed") def install_python312() -> None: """Install Python 3.12 if not already present. Uses the internal mirror (REPO_URL) when configured; otherwise falls back to the deadsnakes PPA on Debian/Ubuntu and EPEL on RHEL/Rocky/Alma. Emits a warning rather than failing if 3.12 is not found after the install attempt so that callers can decide how to handle it. """ pkg_detect() os_id = _os_id() if PKG_MGR == "apt": if have_cmd("python3.12"): ok("Python 3.12 already installed — ensuring venv/dev packages") # python3.12-venv/dev may be bundled on newer distros or absent from # deadsnakes when the mirror is unreachable; non-fatal if not found. _venv_r = run( "DEBIAN_FRONTEND=noninteractive apt-get install -y python3.12-venv python3.12-dev", check=False, ) if _venv_r.returncode != 0: warn("python3.12-venv/dev not found — trying python3-venv python3-dev") run("DEBIAN_FRONTEND=noninteractive apt-get install -y python3-venv python3-dev", check=False) return if not REPO_URL: if os_id == "ubuntu": pkg_install("software-properties-common") run("add-apt-repository -y ppa:deadsnakes/ppa", check=False) elif os_id == "debian": # deadsnakes PPA is Ubuntu-only; use packages.sury.org for Debian run("mkdir -p /etc/apt/keyrings", check=False) run( "curl -fsSL https://packages.sury.org/python/apt.gpg" " | gpg --yes --dearmor -o /etc/apt/keyrings/sury-python.gpg", check=False, ) _deb_cn = _os_codename() write_file( Path("/etc/apt/sources.list.d/sury-python.list"), f"deb [signed-by=/etc/apt/keyrings/sury-python.gpg]" f" https://packages.sury.org/python/ {_deb_cn} main\n", ) run("apt-get update -y", check=False, _spin="apt-get update") _py_r = run( "DEBIAN_FRONTEND=noninteractive apt-get install -y python3.12 python3.12-venv python3.12-dev", check=False, ) if _py_r.returncode != 0: warn("Some python3.12 packages not found — retrying with --fix-missing") run( "DEBIAN_FRONTEND=noninteractive apt-get install -y --fix-missing" " python3.12 python3.12-venv python3.12-dev", check=False, ) else: if not REPO_URL: pkg_install("epel-release") run(f"{PKG_MGR} install -y python3.12 python3.12-devel", check=False) if have_cmd("python3.12"): ok("Python 3.12 installed") else: warn("python3.12 not found after install — check your distro repositories") def detect_existing_postgres() -> Optional[str]: """Return the major version of a running PostgreSQL instance, or None. Checks pg_lsclusters (Debian/Ubuntu), then systemctl unit names for each supported version, and finally the plain 'postgresql' service name. Does not modify any configuration. """ if have_cmd("pg_lsclusters"): output = run_output("pg_lsclusters --no-header 2>/dev/null") for line in output.splitlines(): parts = line.split() if parts and parts[0].isdigit(): return parts[0] for ver in ("17", "16", "15", "14", "13"): r = subprocess.run( f"systemctl is-active postgresql-{ver}", shell=True, capture_output=True, text=True, ) if r.stdout.strip() == "active": return ver r = subprocess.run( "systemctl is-active postgresql", shell=True, capture_output=True, text=True, ) if r.stdout.strip() == "active": v = run_output("psql --version 2>/dev/null | awk '{print $3}' | cut -d. -f1") return v or "unknown" # A stopped but installed PostgreSQL must also be detected — checking only # systemctl is-active misses it and allows a second PG to be installed. if have_cmd("rpm"): _pg_pkgs = subprocess.run( "rpm -qa --queryformat '%{NAME}\\n' 2>/dev/null | grep -E '^postgresql[0-9]+-server$'", shell=True, capture_output=True, text=True, ).stdout.strip() for _pkg in _pg_pkgs.splitlines(): _m = re.search(r"postgresql(\d+)-server", _pkg) if _m: return _m.group(1) elif have_cmd("dpkg"): _pg_pkgs = subprocess.run( "dpkg -l 'postgresql-[0-9]*' 2>/dev/null | awk '/^ii/{print $2}'", shell=True, capture_output=True, text=True, ).stdout.strip() for _pkg in _pg_pkgs.splitlines(): _m = re.search(r"postgresql-(\d+)$", _pkg) if _m: return _m.group(1) return None def install_postgres() -> None: """Install PostgreSQL if not already present; otherwise reuse the running instance. Installs from the official PGDG repository to ensure the requested version is available across all supported distributions. If an existing PostgreSQL instance is already running, updates the global PGVERSION so that all subsequent functions target the correct version. """ global PGVERSION existing = detect_existing_postgres() if existing: ok(f"PostgreSQL {existing} already running — reusing existing installation") if existing not in ("unknown",): PGVERSION = existing return pkg_detect() pgv = PGVERSION pgv_nodot = pgv.replace(".", "") if PKG_MGR == "apt": pkg_install("wget", "gnupg", "lsb-release") codename = _os_codename() if not REPO_URL: run( f'echo "deb http://apt.postgresql.org/pub/repos/apt {codename}-pgdg main"' " | tee /etc/apt/sources.list.d/pgdg.list" ) run( "wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc" " | gpg --yes --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg" ) run("apt-get update -y", check=False, _spin="apt-get update") _pg_apt_r = run( f"DEBIAN_FRONTEND=noninteractive apt-get install -y" f" postgresql-{pgv} postgresql-client-{pgv} postgresql-contrib-{pgv}", check=False, ) if _pg_apt_r.returncode != 0 and REPO_URL: warn("Local PGDG mirror failed for postgresql — retrying with --fix-missing") run( f"DEBIAN_FRONTEND=noninteractive apt-get install -y --fix-missing" f" postgresql-{pgv} postgresql-client-{pgv} postgresql-contrib-{pgv}", check=False, ) if not have_cmd("pg_lsclusters"): if ONLINE: warn("Falling back to official PGDG APT repo for postgresql") run( f'echo "deb http://apt.postgresql.org/pub/repos/apt {codename}-pgdg main"' " | tee /etc/apt/sources.list.d/pgdg.list", check=False, ) run( "wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc" " | gpg --yes --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg", check=False, ) run("apt-get update -y", check=False, _spin="apt-get update") run( f"DEBIAN_FRONTEND=noninteractive apt-get install -y" f" postgresql-{pgv} postgresql-client-{pgv} postgresql-contrib-{pgv}", check=False, ) else: warn("Local PGDG mirror failed and ONLINE=false — cannot fall back") run( f"systemctl enable --now postgresql@{pgv}-main 2>/dev/null" " || systemctl enable --now postgresql", check=False, ) else: ver = _os_version_id() if not REPO_URL and not OFFLINE_DIR: repo_rpm_url = ( f"https://download.postgresql.org/pub/repos/yum/reporpms/" f"EL-{ver}-x86_64/pgdg-redhat-repo-latest.noarch.rpm" ) # Remove any stale pgdg repo files left by a previous failed run so # dnf doesn't try to refresh broken repo metadata before the RPM installs. run("rm -f /etc/yum.repos.d/pgdg-redhat-all.repo", check=False) # Use rpm (not dnf) to install the PGDG repo RPM — dnf would try to # refresh all enabled repos first, including the just-removed (or # not-yet-fixed) pgdg-common repo, causing a 404 on el10. _pgdg_tmp = f"/tmp/pgdg-el{ver}.noarch.rpm" run(f"curl -fsSL {repo_rpm_url} -o {_pgdg_tmp}") run(f"rpm -Uvh --nosignature --replacepkgs {_pgdg_tmp}", check=False) run(f"rm -f {_pgdg_tmp}", check=False) # PGDG EL10 repo files use dnf5 variables ($releasever_major and # $releasever_minor) which produce broken URLs on CentOS Stream 10 # (e.g. rhel-10_major.10_minor-x86_64). Replace all three variable # forms with the literal version number so dnf never evaluates them. # Order matters: combined form first, then individual, then plain. if int(ver) >= 10: run( f"find /etc/yum.repos.d/ -name 'pgdg*.repo' -exec sed -i" f" -e 's|\\$releasever_major\\.\\$releasever_minor|{ver}|g'" f" -e 's|\\$releasever_major|{ver}|g'" f" -e 's|\\$releasever_minor||g'" f" -e 's|\\$releasever|{ver}|g'" f" {{}} +", check=False, ) # Force fresh metadata so dnf picks up the newly installed/patched repos. run("dnf clean metadata -q 2>/dev/null || true", check=False) run("dnf -qy module disable postgresql 2>/dev/null || true", check=False) _pg_r = run( f"dnf install -y postgresql{pgv_nodot}-server postgresql{pgv_nodot}", check=False, ) if _pg_r.returncode != 0 and not REPO_URL: # No local mirror configured — fall back to the official PGDG repo # if online access is available. if ONLINE: warn( f"PGDG install failed for EL{ver} — falling back to " "official PGDG repository" ) _official_rpm = ( f"https://download.postgresql.org/pub/repos/yum/reporpms/" f"EL-{ver}-x86_64/pgdg-redhat-repo-latest.noarch.rpm" ) run(f"{PKG_MGR} install -y {_official_rpm}", check=False) run("dnf -qy module disable postgresql 2>/dev/null || true", check=False) _pg_r = run( f"dnf install -y postgresql{pgv_nodot}-server postgresql{pgv_nodot}", check=False, ) else: warn("PGDG install failed and ONLINE=false — cannot fall back") if _pg_r.returncode != 0: _pg_out = ((_pg_r.stdout or "") + "\n" + (_pg_r.stderr or "")).strip() for _line in _pg_out.splitlines()[-20:]: if _line.strip(): warn(f" dnf: {_line}") _hint = ( f"Check that {REPO_URL}/rpm/el{ver}/pgdg{pgv_nodot}/ has packages" if REPO_URL else f"PGDG may not yet publish EL{ver} packages, or check network/DNF errors above" ) raise RuntimeError( f"Failed to install PostgreSQL {pgv} on EL{ver}. {_hint}." ) run(f"/usr/pgsql-{pgv}/bin/postgresql-{pgv}-setup initdb", check=False) run(f"systemctl enable --now postgresql-{pgv}") ok(f"PostgreSQL {pgv} installed") def configure_postgres() -> bool: """Configure PostgreSQL for localhost-only access using scram-sha-256 auth. Backs up the original pg_hba.conf and postgresql.conf before modifying them. Sets listen_addresses = 'localhost' so PostgreSQL is not reachable from the network, and ensures the 127.0.0.1/32 host rule uses scram-sha-256. Returns False if the config files cannot be located. """ pgv = PGVERSION pghba = "" pgconf = "" pg_etc = Path(f"/etc/postgresql/{pgv}") if pg_etc.exists(): pghba = str(pg_etc / "main/pg_hba.conf") pgconf = str(pg_etc / "main/postgresql.conf") else: pghba = run_output("find /var/lib/pgsql -name pg_hba.conf | head -n1") pgconf = run_output("find /var/lib/pgsql -name postgresql.conf | head -n1") if not pghba: err("pg_hba.conf not found — is PostgreSQL installed?") return False copy_file(pghba, pghba + ".bak") copy_file(pgconf, pgconf + ".bak") # Ensure localhost host rule uses scram-sha-256 run( f'sed -i "s|^host.*all.*all.*127.0.0.1/32.*|' f'host all all 127.0.0.1/32 scram-sha-256|" {pghba}', check=False, ) # Listen on localhost only — no network exposure run( f"sed -i \"s|^#*listen_addresses.*|listen_addresses = 'localhost'|\" {pgconf}", check=False, ) # Apply non-default port if configured if DBPORT != 5432: run( f"sed -i \"s|^#*port.*|port = {DBPORT}|\" {pgconf}", check=False, ) run("systemctl restart postgresql* 2>/dev/null || systemctl restart postgresql", check=False) ok(f"PostgreSQL configured (localhost only, port {DBPORT})") return True def tune_postgres() -> None: """Apply basic PostgreSQL performance tuning based on available CPU and RAM. Sets shared_buffers to 25 % of total RAM, derives work_mem and effective_cache_size from the remaining memory, and adjusts max_connections to be proportional to the CPU count. The tuning block is appended to postgresql.conf and tagged so it is only added once. """ pgv = PGVERSION pgconf = "" pg_etc = Path(f"/etc/postgresql/{pgv}") if pg_etc.exists(): pgconf = str(pg_etc / "main/postgresql.conf") else: pgconf = run_output("find /var/lib/pgsql -name postgresql.conf | head -n1") if not pgconf: warn("postgresql.conf not found — skipping tuning") return meminfo = Path("/proc/meminfo").read_text() ram_kb = int(next(l for l in meminfo.splitlines() if "MemTotal" in l).split()[1]) ram_mb = ram_kb // 1024 cpus = os.cpu_count() or 1 shared_buffers_mb = ram_mb // 4 effective_cache_mb = (ram_mb * 3) // 4 max_connections = max(100, cpus * 10) work_mem_mb = max(4, ram_mb // (max_connections * 2)) maint_work_mem_mb = min(256, ram_mb // 8) tuning_block = ( "\n# --- 123cluster auto-tuning ---" f" ({ram_mb} MB RAM, {cpus} CPU cores)\n" f"shared_buffers = {shared_buffers_mb}MB\n" f"work_mem = {work_mem_mb}MB\n" f"effective_cache_size = {effective_cache_mb}MB\n" f"maintenance_work_mem = {maint_work_mem_mb}MB\n" f"max_connections = {max_connections}\n" "checkpoint_completion_target = 0.9\n" "wal_buffers = 16MB\n" "random_page_cost = 1.1\n" "# --- end 123cluster auto-tuning ---\n" ) append_file(Path(pgconf), tuning_block) run("systemctl restart postgresql* 2>/dev/null || systemctl restart postgresql", check=False) ok( f"PostgreSQL tuned (shared_buffers={shared_buffers_mb}MB" f" work_mem={work_mem_mb}MB max_connections={max_connections})" ) def install_db_clients() -> None: """Install PostgreSQL and MongoDB client utilities for all supported versions. Installs only client tools (psql, pg_dump, pg_restore, mongodump, mongorestore, mongosh) without setting up any database server. Individual package failures are non-fatal — some versions may not be available on all distributions. """ pkg_detect() for ver in ("13", "14", "15", "16", "17"): if PKG_MGR == "apt": run(f"apt-get install -y postgresql-client-{ver}", check=False) else: nodot = ver.replace(".", "") run(f"{PKG_MGR} install -y postgresql{nodot}", check=False) if PKG_MGR == "apt": os_id = _os_id() codename = _os_codename() if os_id in ("ubuntu", "debian"): run( "curl -fsSL https://www.mongodb.org/static/pgp/server-7.0.asc" " | gpg --yes --dearmor -o /etc/apt/trusted.gpg.d/mongodb-server-7.0.gpg", check=False, ) run( f'echo "deb [ arch=amd64,arm64 ]' f" https://repo.mongodb.org/apt/{os_id}" f' {codename}/mongodb-org/7.0 multiverse"' " | tee /etc/apt/sources.list.d/mongodb-org-7.0.list", check=False, ) run("apt-get update -y", check=False, _spin="apt-get update") run("apt-get install -y mongodb-database-tools mongosh", check=False) else: ver_id = _os_version_id() mongo_repo = ( "[mongodb-org-7.0]\n" "name=MongoDB Repository\n" f"baseurl=https://repo.mongodb.org/yum/redhat/" f"{ver_id}/mongodb-org/7.0/x86_64/\n" "gpgcheck=1\n" "enabled=1\n" "gpgkey=https://www.mongodb.org/static/pgp/server-7.0.asc\n" ) write_file(Path("/etc/yum.repos.d/mongodb-org-7.0.repo"), mongo_repo) run(f"{PKG_MGR} install -y mongodb-database-tools mongosh", check=False) ok("Database client utilities installed") def install_nodejs() -> None: """Install the latest Node.js LTS release and NPM via the NodeSource repository. Idempotent: skips installation when Node.js >= 18 is already present. """ _node_path = shutil.which("node") or "" _npm_path = shutil.which("npm") or "" # Snap-installed node/npm is excluded from non-interactive shell PATH — # fall through to install the proper deb package from the NodeSource mirror. _snap_node = "/snap/" in _node_path or "/snap/" in _npm_path if not _snap_node and _node_path and _npm_path: ver = run_output("node --version 2>/dev/null").lstrip("v") major = int(ver.split(".")[0]) if ver and ver.split(".")[0].isdigit() else 0 if major >= 18: ok(f"Node.js {ver} already installed") return pkg_detect() # Remove snap-installed nodejs if present — snap binaries aren't in non-interactive # shell PATH, which breaks subprocess npm calls later in the install. if PKG_MGR == "apt" and "/snap/" in (shutil.which("node") or ""): run("snap remove node 2>/dev/null || snap remove nodejs 2>/dev/null || true", check=False) if not REPO_URL: if PKG_MGR == "apt": run("curl -fsSL https://deb.nodesource.com/setup_lts.x | bash -") else: run("curl -fsSL https://rpm.nodesource.com/setup_lts.x | bash -") if PKG_MGR == "apt": if REPO_URL: # Local mirror may ship separate npm package (no NodeSource bundling) pkg_install("nodejs", "npm") else: # NodeSource's nodejs bundles npm; installing the distro npm alongside # it causes "held broken packages" conflicts on Debian/Ubuntu. pkg_install("nodejs") # Mark as manually installed so apt-get autoremove doesn't remove it. run("apt-mark manual nodejs 2>/dev/null || true", check=False) else: pkg_install("nodejs") ok(f"Node.js LTS installed ({run_output('node --version 2>/dev/null')})") def create_app_dirs() -> None: """Create the required application and backup directory structure. Creates APP_DIR with subdirectories for backend, frontend, clustertasks, config and logs, and BACKUPS_DIR with subdirectories for postgres and mongodb backups. All directories are created with mkdir -p semantics. """ dirs = [ APP_DIR, APP_DIR / "backend", APP_DIR / "frontend", APP_DIR / "clustertasks", APP_DIR / "config", APP_DIR / "logs", BACKUPS_DIR, BACKUPS_DIR / "postgres", BACKUPS_DIR / "mongodb", ] for d in dirs: if DRY_RUN: log.info(f"[DRY RUN] mkdir -p {d}") else: d.mkdir(parents=True, exist_ok=True) ok(f"Directories created under {APP_DIR} and {BACKUPS_DIR}") def _find_latest_schema() -> Optional[Path]: """Return the most-recently modified db_schema*.sql file, or None. Searches APP_DIR/backend and the current working directory. Using modification time avoids depending on date formats embedded in filenames. """ candidates: List[Path] = [] for pat in ( str(APP_DIR / "backend" / "db_schema*.sql"), str(Path.cwd() / "db_schema*.sql"), ): candidates.extend(Path(p) for p in glob.glob(pat)) if not candidates: return None return max(candidates, key=lambda p: p.stat().st_mtime) def _create_database() -> bool: """Create the application PostgreSQL database and DB user if they do not exist. Idempotent: returns True immediately when the database already exists. Returns False if creation fails so callers can abort early. """ r = subprocess.run( f"sudo -u postgres psql -lqt 2>/dev/null" f" | cut -d'|' -f1 | grep -qw {DB_NAME}", shell=True, capture_output=True, ) if r.returncode == 0: ok(f"Database '{DB_NAME}' already exists") return True if DRY_RUN: log.info(f"[DRY RUN] createdb {DB_NAME}") return True run( f"sudo -u postgres psql -c" f" \"ALTER USER {DB_USER} WITH ENCRYPTED PASSWORD '{DB_PASSWORD}';\"", check=False, ) result = run(f"sudo -u postgres createdb {DB_NAME}", check=False) if result.returncode != 0: err(f"Failed to create database '{DB_NAME}'") return False ok(f"Database '{DB_NAME}' created") return True def restore_db_from_dump() -> bool: """Apply schema/data dump files into the application database. Mirrors the logic of ci/init-db-dumps.sh: 1. Ensure public.db_log and public.node_json support tables exist. 2. Extract and apply only CREATE FUNCTION public.* from schema_public. 3. Apply schema_app_123monitor in full (skipped if schema already exists). 4. Apply backup_schema in full (skipped if backup schema already exists). 5. Copy config rows from backup.* into live public.* / app_123monitor.* tables. Dump source: DUMP_BASE_URL env var, or {ARTIFACTS_URL}/dbbackup/ when ARTIFACTS_URL is set. For local directories the latest date-stamped file (e.g. schema_public_20260101.sql) is used as a fallback when the plain name is not found. Returns False only if no dump source is configured. """ _dump_url = DUMP_BASE_URL or (f"{ARTIFACTS_URL}/dbbackup" if ARTIFACTS_URL else "") if not _dump_url: warn("No dump source configured (DUMP_BASE_URL / ARTIFACTS_URL) — skipping dump restore") return False if DRY_RUN: log.info("[DRY RUN] restore_db_from_dump()") return True base = _dump_url.rstrip("/").rstrip(os.sep) is_url = base.startswith("http://") or base.startswith("https://") _psql_bin = ( f"/usr/pgsql-{PGVERSION}/bin/psql" if PKG_MGR in ("dnf", "yum") and Path(f"/usr/pgsql-{PGVERSION}/bin/psql").exists() else "psql" ) def _psql_run(sql: str, label: str) -> bool: cmd_log.debug(f"--- psql: {label} ---") r = subprocess.run( f"LANG=C LC_ALL=C sudo -u postgres {_psql_bin} -d {DB_NAME}", shell=True, input=sql, capture_output=True, text=True, errors="replace", ) for line in (r.stdout + r.stderr).splitlines(): if line.strip(): cmd_log.debug(line) return r.returncode == 0 def _schema_exists(name: str) -> bool: r = subprocess.run( f"LANG=C LC_ALL=C sudo -u postgres {_psql_bin} -d {DB_NAME} -Atc " f"\"SELECT EXISTS (SELECT 1 FROM information_schema.schemata " f"WHERE schema_name = '{name}')\"", shell=True, capture_output=True, text=True, ) return "t" in r.stdout def _fetch(filename: str) -> Optional[Path]: if is_url: tmp = Path(f"/tmp/123cluster_dump_{os.getpid()}_{filename}") info(f"Downloading {base}/{filename} ...") r = run(f"curl -fL --retry 3 {base}/{filename} -o {tmp}", check=False) if r.returncode != 0 or not tmp.exists(): err(f"Failed to download {base}/{filename} — skipping") return None return tmp plain = Path(base) / filename if plain.exists(): return plain # Fall back to latest date-stamped variant: schema_public_20260101.sql stem = filename[:-4] # strip .sql matches = sorted(Path(base).glob(f"{stem}_[0-9]*.sql")) if matches: return matches[-1] warn(f"Dump file not found: {plain} — skipping") return None def _filter_dump(content: str) -> str: """Strip commands that fail on plain psql.""" out = [] for line in content.splitlines(keepends=True): s = line.lstrip() if (s.startswith("\\restrict ") or s.startswith("\\unrestrict ") or s.startswith("SET transaction_timeout = 0;")): continue out.append(line) return "".join(out) def _extract_public_functions(content: str) -> str: """Return only CREATE FUNCTION public.* blocks, with CREATE OR REPLACE.""" result = ["SET check_function_bodies = false;\n\n"] in_fn = False for line in content.splitlines(keepends=True): if line.startswith("CREATE FUNCTION public."): in_fn = True if in_fn: line = line.replace("CREATE FUNCTION ", "CREATE OR REPLACE FUNCTION ", 1) result.append(line) if line.rstrip() == "$$;": in_fn = False result.append("\n") return "".join(result) def _apply_file(path: Path, label: str) -> bool: return _psql_run(_filter_dump(path.read_text(errors="replace")), label) def _cleanup(path: Optional[Path]) -> None: if path and str(path).startswith("/tmp/"): path.unlink(missing_ok=True) # ── Step 1: support tables ───────────────────────────────────────────────── _support_sql = textwrap.dedent("""\ CREATE UNLOGGED TABLE IF NOT EXISTS public.db_log ( node_id integer, monitor_name character varying(4000), ecode character varying(4000), errmsg text, module character varying(4000), step character varying(4000), severity_id integer, date_created timestamp with time zone ); CREATE TABLE IF NOT EXISTS public.node_json ( row_to_json jsonb ); """) if _psql_run(_support_sql, "ensure support tables"): ok("Support tables ready") else: warn("Could not create support tables — check log") # ── Step 2: public functions from schema_public ──────────────────────────── pub_path = _fetch("schema_public.sql") if pub_path: fn_sql = _extract_public_functions(_filter_dump(pub_path.read_text(errors="replace"))) if _psql_run(fn_sql, "schema_public functions"): ok("Public functions restored") else: warn("Errors restoring public functions — check log") _cleanup(pub_path) # ── Step 3: app_123monitor schema ───────────────────────────────────────── if not _schema_exists("app_123monitor"): mon_path = _fetch("schema_app_123monitor.sql") if mon_path: if _apply_file(mon_path, "schema_app_123monitor"): ok("app_123monitor schema applied") else: warn("Errors applying app_123monitor schema — check log") _cleanup(mon_path) else: ok("app_123monitor schema already exists — skipping") # ── Step 4: backup schema ────────────────────────────────────────────────── if not _schema_exists("backup"): bak_path = _fetch("backup_schema.sql") if bak_path: if _apply_file(bak_path, "backup_schema"): ok("backup schema applied") else: warn("Errors applying backup schema — check log") _cleanup(bak_path) else: ok("backup schema already exists — skipping") # ── Step 5: migrate config rows from backup.* into live tables ───────────── _migrate_sql = textwrap.dedent("""\ DO $$ DECLARE has_admin boolean; has_org boolean; BEGIN IF to_regclass('public."clusterCore_systemconfig"') IS NOT NULL AND to_regclass('backup."clusterCore_systemconfig"') IS NOT NULL THEN INSERT INTO public."clusterCore_systemconfig" (id, db_type, backups_path, exports_path) SELECT id, db_type, backups_path, exports_path FROM backup."clusterCore_systemconfig" ON CONFLICT (id) DO NOTHING; PERFORM setval( pg_get_serial_sequence('public."clusterCore_systemconfig"', 'id'), COALESCE((SELECT MAX(id) FROM public."clusterCore_systemconfig"), 1), true); END IF; IF to_regclass('public."clusterCore_systemsettings"') IS NOT NULL AND to_regclass('backup."clusterCore_systemsettings"') IS NOT NULL THEN INSERT INTO public."clusterCore_systemsettings" (id, name, value) SELECT id, name, value FROM backup."clusterCore_systemsettings" ON CONFLICT (id) DO NOTHING; PERFORM setval( pg_get_serial_sequence('public."clusterCore_systemsettings"', 'id'), COALESCE((SELECT MAX(id) FROM public."clusterCore_systemsettings"), 1), true); END IF; IF to_regclass('app_123monitor.rp_dashboards') IS NOT NULL AND to_regclass('backup.rp_dashboards') IS NOT NULL THEN INSERT INTO app_123monitor.rp_dashboards (id, node_id, node_type, name, ind_template, owner) SELECT id, node_id, node_type, name, ind_template, owner FROM backup.rp_dashboards ON CONFLICT (id) DO NOTHING; PERFORM setval( 'app_123monitor.rp_dashboards_id_seq', COALESCE((SELECT MAX(id) FROM app_123monitor.rp_dashboards), 1), true); END IF; IF to_regclass('app_123monitor.rp_metrics') IS NOT NULL AND to_regclass('backup.rp_metrics') IS NOT NULL THEN INSERT INTO app_123monitor.rp_metrics (id, dashboard_id, node_id, database_type, name, metric_type, sql, "order") SELECT id, dashboard_id, node_id, database_type, name, metric_type, sql, "order" FROM backup.rp_metrics ON CONFLICT (id) DO NOTHING; PERFORM setval( 'app_123monitor.rp_metrics_id_seq', COALESCE((SELECT MAX(id) FROM app_123monitor.rp_metrics), 1), true); END IF; IF to_regclass('app_123monitor.rp_monitor_columns') IS NOT NULL AND to_regclass('backup.rp_monitor_columns') IS NOT NULL THEN INSERT INTO app_123monitor.rp_monitor_columns (org_id, node_id, monitor_name, column_name, display_name, column_type, node_type) SELECT b.org_id, b.node_id, b.monitor_name, b.column_name, b.display_name, b.column_type, b.node_type FROM backup.rp_monitor_columns b WHERE NOT EXISTS ( SELECT 1 FROM app_123monitor.rp_monitor_columns c WHERE c.org_id = b.org_id AND c.node_id IS NOT DISTINCT FROM b.node_id AND c.monitor_name = b.monitor_name AND c.column_name = b.column_name AND c.node_type = b.node_type ); END IF; IF to_regclass('app_123monitor.rp_monitors') IS NOT NULL AND to_regclass('backup.rp_monitors') IS NOT NULL THEN INSERT INTO app_123monitor.rp_monitors ( org_id, node_id, db_type, monitor_name, monitor_description, pre_script, monitor_script, ind_enabled, red_rule, yellow_rule, red_rule_message, yellow_rule_message, ind_send_email_notifications, monitor_interval, runtime_red_threshold, runtime_yellow_threshold, return_code_threshold, ind_status_graph, ind_create_timeline, ind_timeline_full_impact, cmd_type, ind_first_column_unique, ind_alert_on_added_missing, alert_on_added_missing_severity, ind_alert_compare_to_previous_check, alert_compare_to_previous_check_severity, alert_compare_to_previous_check_higher_lower, alert_compare_to_previous_check_percent, ind_break_long_running, ind_compute_baseline, data_reduction_function, renew_alert_every, ind_red_by_row_count, red_by_row_count_option, red_by_row_count_items, ind_yellow_by_row_count, yellow_by_row_count_option, yellow_by_row_count_items, graph_aggregation, graph_aggregation_column, snoozed_until, count_long_running, ind_broken, num_weeks_in_baseline, task_id, module, ind_display_alerts, ind_disable_edit ) SELECT b.org_id, b.node_id, b.db_type, b.monitor_name, b.monitor_description, b.pre_script, b.monitor_script, b.ind_enabled, b.red_rule, b.yellow_rule, b.red_rule_message, b.yellow_rule_message, b.ind_send_email_notifications, b.monitor_interval, b.runtime_red_threshold, b.runtime_yellow_threshold, b.return_code_threshold, b.ind_status_graph, b.ind_create_timeline, b.ind_timeline_full_impact, b.cmd_type, b.ind_first_column_unique, b.ind_alert_on_added_missing, b.alert_on_added_missing_severity, b.ind_alert_compare_to_previous_check, b.alert_compare_to_previous_check_severity, b.alert_compare_to_previous_check_higher_lower, b.alert_compare_to_previous_check_percent, b.ind_break_long_running, b.ind_compute_baseline, b.data_reduction_function, b.renew_alert_every, b.ind_red_by_row_count, b.red_by_row_count_option, b.red_by_row_count_items, b.ind_yellow_by_row_count, b.yellow_by_row_count_option, b.yellow_by_row_count_items, b.graph_aggregation, b.graph_aggregation_column, b.snoozed_until, b.count_long_running, b.ind_broken, b.num_weeks_in_baseline, b.task_id, b.module, b.ind_display_alerts, b.ind_disable_edit FROM backup.rp_monitors b WHERE NOT EXISTS ( SELECT 1 FROM app_123monitor.rp_monitors m WHERE m.org_id = b.org_id AND m.node_id IS NOT DISTINCT FROM b.node_id AND m.db_type = b.db_type AND m.monitor_name = b.monitor_name ); END IF; IF to_regclass('public."clusterCore_task"') IS NOT NULL AND to_regclass('backup."clusterCore_task"') IS NOT NULL THEN SELECT EXISTS (SELECT 1 FROM public.auth_user WHERE id = 1) INTO has_admin; SELECT EXISTS (SELECT 1 FROM public.organization WHERE org_id = 1) INTO has_org; IF has_admin AND has_org THEN INSERT INTO public."clusterCore_task" ( task_id, created_date, status, task, monitor_name, displayment, target_type, target_id, agent_id, schedule, parameters, created_by_id, org_id, next_due_date, estimated_time, started_time ) SELECT task_id, created_date, status, task, monitor_name, displayment, target_type, target_id, agent_id, schedule, parameters, created_by_id, org_id, next_due_date, estimated_time, started_time FROM backup."clusterCore_task" ON CONFLICT (task_id) DO NOTHING; PERFORM setval( pg_get_serial_sequence('public."clusterCore_task"', 'task_id'), COALESCE((SELECT MAX(task_id) FROM public."clusterCore_task"), 1), true); ELSE RAISE NOTICE 'Skipping clusterCore_task: auth_user id=1 or organization org_id=1 not found.'; END IF; END IF; END $$; """) if _psql_run(_migrate_sql, "apply_backup_config_data"): ok("Config data migrated from backup schema") else: warn("Errors migrating config data — check log") return True def _seed_initial_data() -> None: """Insert the initial organization and profile rows required by the app. Idempotent: uses WHERE NOT EXISTS guards so re-runs are safe. Must be called after manage.py migrate and after restore_db_from_dump(). """ if DRY_RUN: log.info("[DRY RUN] seed organization + clusterCore_profile") return _psql = ( f"/usr/pgsql-{PGVERSION}/bin/psql" if PKG_MGR in ("dnf", "yum") and Path(f"/usr/pgsql-{PGVERSION}/bin/psql").exists() else "psql" ) org_sql = ( "INSERT INTO public.organization " "(date_created, ind_disable_all_nodes, organization_name, " "ind_enable_notifications, ind_housekeeper, " "housekeeper_short_retention, housekeeper_long_retention, " "max_notifications_per_hour) " f"SELECT now(),'N','{ORG_NAME}','Y','Y',10,11,10 " "WHERE NOT EXISTS " f"(SELECT 1 FROM public.organization WHERE organization_name='{ORG_NAME}');" ) profile_sql = ( 'INSERT INTO public."clusterCore_profile" (org_id, user_id, is_demo) ' "SELECT 1, u.id, false " "FROM auth_user u " "WHERE u.is_superuser = true " 'AND NOT EXISTS (SELECT 1 FROM public."clusterCore_profile" WHERE user_id = u.id) ' "LIMIT 1;" ) for label, sql in (("organization", org_sql), ("clusterCore_profile", profile_sql)): r = run( f"LANG=C LC_ALL=C sudo -u postgres {_psql} -d {DB_NAME} -c {shlex.quote(sql)}", check=False, ) if r.returncode == 0: ok(f"Seeded {label}") else: _lines = [ l for l in ((r.stdout or "") + (r.stderr or "")).splitlines() if l.strip() and not l.startswith("perl:") and not l.strip().startswith(("LC_", "LANGUAGE =", "are inadequate", "Falling back")) and "locale" not in l.lower() ] warn(f"Could not seed {label} — add manually via admin panel") for _l in _lines: warn(f" psql: {_l}") def setup_pgbouncer() -> None: """Install and configure PgBouncer as a connection pooler for the application. Writes /etc/pgbouncer/pgbouncer.ini with transaction-mode pooling tuned for the application. Generates /etc/pgbouncer/userlist.txt from the password hash stored in pg_authid for the database user. """ pkg_detect() _pgb_r = run( f"DEBIAN_FRONTEND=noninteractive apt-get install -y pgbouncer" if PKG_MGR == "apt" else f"{PKG_MGR} install -y pgbouncer", check=False, ) if _pgb_r.returncode != 0: if PKG_MGR == "apt": # Local PGDG mirror unreachable — install the Ubuntu system pgbouncer. warn("PGDG mirror failed for pgbouncer — retrying with --fix-missing") run( "DEBIAN_FRONTEND=noninteractive apt-get install -y --fix-missing pgbouncer", check=False, ) elif ONLINE: # Local PGDG mirror unreachable — fall back to official PGDG RPM repo. warn("Local PGDG mirror failed for pgbouncer — falling back to official PGDG") _ver = _os_version_id() _rpm = ( f"https://download.postgresql.org/pub/repos/yum/reporpms/" f"EL-{_ver}-x86_64/pgdg-redhat-repo-latest.noarch.rpm" ) run(f"{PKG_MGR} install -y {_rpm}", check=False) run(f"{PKG_MGR} install -y pgbouncer", check=False) pgbouncer_ini = f"""\ [databases] {DB_NAME} = host=127.0.0.1 port={DBPORT} dbname={DB_NAME} [pgbouncer] listen_addr = * listen_port = {BOUNCER_PORT} auth_type = __AUTH_TYPE__ auth_file = /etc/pgbouncer/userlist.txt ; Pooling pool_mode = transaction server_reset_query = DISCARD ALL max_client_conn = 2147483647 default_pool_size = 30 min_pool_size = 5 reserve_pool_size = 10 reserve_pool_timeout = 5 ; Timeouts client_idle_timeout = 60 server_lifetime = 7200 server_idle_timeout = 600 server_login_retry = 10 query_timeout = 3600 tcp_keepalive = 1 ; Logging log_connections = 0 log_disconnections = 0 log_stats = 0 logfile = /var/log/pgbouncer/pgbouncer.log ; Admin admin_users = {DB_USER} """ # Store plaintext password: pgbouncer uses it to verify MD5 from clients # AND to authenticate to the postgres backend via SCRAM. auth_type = "md5" write_file(Path("/etc/pgbouncer/pgbouncer.ini"), pgbouncer_ini.replace("__AUTH_TYPE__", auth_type)) # Resolve which OS user pgbouncer.service actually runs as. # RHEL/Rocky: pgbouncer user; Ubuntu/Debian: postgres user. pgb_user = run_output( "systemctl show pgbouncer.service -p User --value 2>/dev/null" ).strip() or "pgbouncer" if not DRY_RUN: write_file( Path("/etc/pgbouncer/userlist.txt"), f'"{DB_USER}" "{DB_PASSWORD}"\n', mode=0o640, ) run(f"chown {pgb_user}:{pgb_user} /etc/pgbouncer/userlist.txt", check=False) Path("/var/log/pgbouncer").mkdir(parents=True, exist_ok=True) run(f"chown {pgb_user}:{pgb_user} /var/log/pgbouncer", check=False) run("systemctl enable --now pgbouncer", check=False) run("systemctl restart pgbouncer", check=False) ok(f"PgBouncer installed and configured (port {BOUNCER_PORT})") def setup_backend() -> None: """Set up the Django backend service. When ARTIFACTS_URL is set and the backend directory has no code yet, downloads backend.tar.gz from {ARTIFACTS_URL}/backend/ and extracts it. Creates a Python 3.12 virtual environment at APP_DIR/backend/venv, installs pip dependencies, writes the .env file (never overwrites an existing one), runs makemigrations / migrate / collectstatic, and creates a systemd service unit. Idempotent: each step checks whether it has already been done. """ backend_dir = APP_DIR / "backend" venv_dir = backend_dir / "venv" env_file = backend_dir / ".env" # ── Deploy backend code from offline bundle or repo server ──────────────── if BINARY_PKGS: _offline_backend = OFFLINE_DIR / "backend" / "backend-build-bin.tar.gz" if OFFLINE_DIR else None _deployed = (backend_dir / "bin" / "run_asgi").exists() if not _deployed and (OFFLINE_DIR or ARTIFACTS_URL): backend_dir.mkdir(parents=True, exist_ok=True) for _attempt in range(1, 3): tmp_tar = Path(f"/tmp/123cluster_backend_{os.getpid()}.tar.gz") tmp_tar.unlink(missing_ok=True) if _offline_backend and _offline_backend.exists(): result = run( f"cp {_offline_backend} {tmp_tar}", check=False, _spin=f"Copying backend binary from offline bundle (attempt {_attempt}/2)", ) elif ARTIFACTS_URL: result = _fetch_artifact( ARTIFACTS_URL, "backend/backend-build-bin.tar.gz", tmp_tar, f"Fetching backend binary (attempt {_attempt}/2)", ) else: break if result.returncode == 0 and tmp_tar.exists(): run(f"tar -xzf {tmp_tar} -C {backend_dir}", _spin="Extracting backend binary", check=False) tmp_tar.unlink(missing_ok=True) if (backend_dir / "bin" / "run_asgi").exists(): run(f"chmod +x {backend_dir}/bin/run_asgi {backend_dir}/bin/manage", check=False) ok("Backend binary extracted") break warn(f"bin/run_asgi missing after extraction (attempt {_attempt}/2)") else: src = str(_offline_backend) if _offline_backend else f"{ARTIFACTS_URL}/backend/backend-build-bin.tar.gz" warn(f"Could not deploy backend binary from {src} — deploy manually") pkg_detect() if PKG_MGR in ("dnf", "yum"): run(f"{PKG_MGR} install -y mariadb-connector-c postgresql{PGVERSION}-libs openssl libffi", check=False) ensure_mariadb_connector_c() elif PKG_MGR == "apt": _wait_apt_lock() run("DEBIAN_FRONTEND=noninteractive apt-get install -y libmariadb3 libssl3 libffi8", check=False) else: _offline_backend = OFFLINE_DIR / "backend" / "backend.tar.gz" if OFFLINE_DIR else None if not (backend_dir / "requirements.txt").exists() and (OFFLINE_DIR or ARTIFACTS_URL): backend_dir.mkdir(parents=True, exist_ok=True) for _attempt in range(1, 3): tmp_tar = Path(f"/tmp/123cluster_backend_{os.getpid()}.tar.gz") tmp_tar.unlink(missing_ok=True) if _offline_backend and _offline_backend.exists(): result = run( f"cp {_offline_backend} {tmp_tar}", check=False, _spin=f"Copying backend from offline bundle (attempt {_attempt}/2)", ) elif ARTIFACTS_URL: result = _fetch_artifact( ARTIFACTS_URL, "backend/backend.tar.gz", tmp_tar, f"Fetching backend (attempt {_attempt}/2)", ) else: break if result.returncode == 0 and tmp_tar.exists(): run(f"tar -xzf {tmp_tar} -C {backend_dir}", _spin="Extracting backend", check=False) tmp_tar.unlink(missing_ok=True) if (backend_dir / "requirements.txt").exists(): ok("Backend code extracted") break warn(f"backend requirements.txt missing after extraction (attempt {_attempt}/2)") else: src = str(_offline_backend) if _offline_backend else f"{ARTIFACTS_URL}/backend/backend.tar.gz" warn(f"Could not deploy backend from {src} — deploy manually") if not (venv_dir / "bin" / "activate").exists(): python_bin = "python3.12" if have_cmd("python3.12") else "python3" run(f"{python_bin} -m venv {venv_dir}") pip = venv_dir / "bin" / "pip" python = venv_dir / "bin" / "python" # Install C-extension build dependencies before pip pkg_detect() if PKG_MGR in ("dnf", "yum"): # Try PGDG postgresql-devel first (needs perl-IPC-Run which may be absent) _pgdevel_r = run( f"{PKG_MGR} install -y mariadb-connector-c-devel python3.12-devel " f"postgresql{PGVERSION}-devel gcc openssl-devel libffi-devel", check=False, ) if _pgdevel_r.returncode != 0: # postgresql-devel unavailable (missing perl-IPC-Run in EL9/EL10) — fall back # to system libpq-devel which provides libpq-fe.h without that dep. # Disable the local PGDG repo so DNF picks AppStream's libpq-devel # (pg13) rather than upgrading to postgresql17-devel (which also lacks # perl-IPC-Run and would fail with the same error). warn( f"postgresql{PGVERSION}-devel not installable — " "falling back to system libpq-devel for psycopg2 build headers" ) _pgdg_repo_id = f"local-pgdg{PGVERSION}" run( f"{PKG_MGR} install -y --disablerepo={_pgdg_repo_id} " "mariadb-connector-c-devel python3.12-devel " "libpq-devel gcc openssl-devel libffi-devel", check=False, ) # Last resort: if neither system pg_config nor PGDG headers appeared # (e.g. libpq-devel absent from EL10 AppStream), install PGDG devel # headers alone by skipping the unsatisfiable perl-IPC-Run dependency. _pgdg_inc = Path(f"/usr/pgsql-{PGVERSION}/include/libpq-fe.h") if not _pgdg_inc.exists() and not Path("/usr/bin/pg_config").exists(): warn( f"libpq-devel unavailable — installing postgresql{PGVERSION}-devel " "without dependencies (skipping perl-IPC-Run)" ) run( f"{PKG_MGR} install -y --nodeps postgresql{PGVERSION}-devel", check=False, ) # EL10 may need the mariadb module enabled first if not have_cmd("mariadb_config"): run("dnf module enable mariadb -y 2>/dev/null || true", check=False) run("dnf install -y mariadb-connector-c-devel", check=False) elif PKG_MGR == "apt": _wait_apt_lock() run("DEBIAN_FRONTEND=noninteractive apt-get install -y " "libmariadb-dev python3.12-dev libpq-dev gcc libssl-dev libffi-dev", check=False) run(f"{pip} install --upgrade pip wheel") # Pre-install license wheels (data_verifier + data_signer) from the artifact server. if ARTIFACTS_URL: pkg_detect() _manylinux = "manylinux_2_34" for _whl_name in ( f"data_verifier-0.1.0-cp312-cp312-{_manylinux}_x86_64.whl", f"data_signer-0.1.0-cp312-cp312-{_manylinux}_x86_64.whl", ): _tmp_whl = Path(f"/tmp/{_whl_name}") _r = _fetch_artifact(ARTIFACTS_URL, f"backend/wheels/{_whl_name}", _tmp_whl) if _r.returncode == 0 and _tmp_whl.exists(): run(f"{pip} install --no-deps {_tmp_whl}") _tmp_whl.unlink() else: warn(f"Could not download {_whl_name} — backend may fail to start") if not BINARY_PKGS: python = venv_dir / "bin" / "python" req_file = backend_dir / "requirements.txt" if not BINARY_PKGS and req_file.exists(): # Drop local .whl references whose file is missing or incompatible (e.g. wrong # Python ABI tag). pip would abort on the first bad line; we skip and warn instead. # Also skip the mariadb package when mariadb_config isn't available (e.g. EL10 # where mariadb-connector-c-devel may not be in the default repos). _has_mariadb_cfg = have_cmd("mariadb_config") lines = req_file.read_text().splitlines(keepends=True) filtered = [] for line in lines: stripped = line.strip() if stripped.endswith(".whl") and not Path(stripped).exists(): warn(f"Skipping missing/incompatible wheel in requirements.txt: {stripped}") continue pkg_name = stripped.split("=")[0].split(">")[0].split("<")[0].split("[")[0].lower() if pkg_name == "mariadb": if not _has_mariadb_cfg: warn( "Skipping mariadb in requirements.txt — mariadb_config not found. " "Install mariadb-connector-c-devel and re-run to enable MariaDB support." ) continue # mariadb Python package >= 1.1.8 requires Connector/C >= 3.3.1 # mariadb_config --version returns the server version, not Connector/C; # query the RPM or dpkg for the actual connector library version. # rpm -q prints "package X is not installed" to STDOUT when not found, # so filter that out before checking the fallback Provides approach. # MariaDB 10.11+ ships as "MariaDB-shared" which obsoletes the old # "mariadb-connector-c" package — check its Provides for the connector ver. _cc_ver_str = ( run_output( "rpm -q mariadb-connector-c --queryformat '%{VERSION}' 2>/dev/null" " | grep -v 'not installed'" ) or run_output( "rpm -q --provides MariaDB-shared 2>/dev/null" " | grep 'mariadb-connector-c =' | head -1 | awk '{print $3}'" ) or run_output("dpkg-query -f '${Version}' -W libmariadb3 2>/dev/null") or "0.0.0" ).strip().split("-")[0] # strip distro suffix e.g. "3.2.6-1.el9" # strip Debian epoch prefix e.g. "1:11.8.6" → "11.8.6" if ":" in _cc_ver_str: _cc_ver_str = _cc_ver_str.split(":", 1)[1] try: _cc_tuple = tuple(int(x) for x in _cc_ver_str.split(".")[:3]) except ValueError: _cc_tuple = (0, 0, 0) if _cc_tuple < (3, 3, 1): warn( f"Skipping mariadb in requirements.txt — " f"Connector/C {_cc_ver_str} < 3.3.1 required. " f"Upgrade mariadb-connector-c to >= 3.3.1 to enable MariaDB support." ) continue filtered.append(line) if len(filtered) != len(lines) and not DRY_RUN: tmp_req = backend_dir / "requirements_filtered.txt" tmp_req.write_text("".join(filtered)) req_file = tmp_req # pg_config lives in /usr/pgsql-N/bin on PGDG installs; psycopg2 needs it in PATH. # Add PGDG bin to PATH whenever pg_config is there (from server or devel install). # System pg_config (/usr/bin/pg_config from libpq-devel) is already on default PATH. _pgdg_pgconfig = Path(f"/usr/pgsql-{PGVERSION}/bin/pg_config") if PKG_MGR in ("dnf", "yum") and _pgdg_pgconfig.exists(): pg_path = f"PATH=/usr/pgsql-{PGVERSION}/bin:$PATH " else: pg_path = "" # Pre-patch psycopg2 → psycopg2-binary when either pg_config or libpq # headers are absent. Both are required to compile from source; having # pg_config alone (e.g. from the server package) is not sufficient — # psycopg2 will find the include path via pg_config but fail if the # header files are not actually installed (EL10: postgresql17-devel # --nodeps installs pg_config but not the headers). _have_pgconfig = ( _pgdg_pgconfig.exists() or Path("/usr/bin/pg_config").exists() or have_cmd("pg_config") ) _have_pg_headers = ( Path(f"/usr/pgsql-{PGVERSION}/include/libpq-fe.h").exists() or Path("/usr/include/libpq-fe.h").exists() or Path("/usr/include/postgresql/libpq-fe.h").exists() ) if not (_have_pgconfig and _have_pg_headers): warn("pg_config not found — using psycopg2-binary (pre-compiled wheel)") _patched_lines = [] for _l in req_file.read_text().splitlines(keepends=True): if _l.lower().startswith("psycopg2") and "binary" not in _l.lower(): _patched_lines.append( _l.replace("psycopg2==", "psycopg2-binary==") .replace("psycopg2 ", "psycopg2-binary ") .replace("psycopg2\n", "psycopg2-binary\n") .replace("psycopg2>=", "psycopg2-binary>=") ) else: _patched_lines.append(_l) _patched_req = req_file.parent / (req_file.stem + "_patched.txt") _patched_req.write_text("".join(_patched_lines)) req_file = _patched_req run(f"{pg_path}{pip} install {_pip_flags()} -r {req_file}", _spin="Installing Python dependencies") else: warn(f"requirements.txt not found at {req_file} — skipping pip install") if not env_file.exists(): secret_key = f"django-insecure-{secrets.token_urlsafe(40)}" write_file( env_file, f"DB_NAME={DB_NAME}\n" f"DB_USER={DB_USER}\n" f"DB_PASSWORD={DB_PASSWORD}\n" f"DB_HOST=127.0.0.1\n" f"DB_PORT={BOUNCER_PORT}\n" f"DB_TIMEZONE={DB_TIMEZONE}\n" f"SECRET_KEY='{secret_key}'\n", mode=0o600, ) ok(f".env written to {env_file}") else: info(f".env already exists at {env_file} — not overwritten") # Export .env into the installer process so manage.py subprocesses inherit # SECRET_KEY and DB_* without needing systemd's EnvironmentFile. if env_file.exists() and not DRY_RUN: for _line in env_file.read_text().splitlines(): _line = _line.strip() if not _line or _line.startswith("#") or "=" not in _line: continue _k, _, _v = _line.partition("=") _k = _k.strip() _v = _v.strip().strip("'\"") if _k and _k not in os.environ: os.environ[_k] = _v if BINARY_PKGS: _manage_bin = backend_dir / "bin" / "manage" _manage_cmd = str(_manage_bin) if _manage_bin.exists() else None _exec_start = f"{backend_dir}/bin/run_asgi --host 0.0.0.0 --port {BACKPORT}" else: _manage_cmd = f"{python} manage.py" if (backend_dir / "manage.py").exists() else None _exec_start = f"{venv_dir}/bin/uvicorn core.asgi:application --host 0.0.0.0 --port {BACKPORT}" if _manage_cmd: run(f"cd {backend_dir} && {_manage_cmd} makemigrations --noinput", check=False) # Run full migrations directly against PostgreSQL (port DBPORT) to # create the complete schema before the dump is imported. Using the # direct port rather than PgBouncer avoids DDL transaction restrictions. _migrate_r = run( f"cd {backend_dir} && DB_PORT={DBPORT} {_manage_cmd} migrate --noinput", check=False, _spin="Running Django migrations", ) if _migrate_r.returncode == 0: ok("Django migrations applied") else: warn( f"Django migrate failed (rc={_migrate_r.returncode}) — " "check /var/log/123cluster_install.log for details" ) run(f"cd {backend_dir} && {_manage_cmd} collectstatic --noinput", check=False, _spin="Collecting static files") # Import DB dumps AFTER migrations so every table exists first. restore_db_from_dump() # Create Django superuser before seeding so user_id=1 exists for the # FK constraint on clusterCore_profile. _su_exists = run( f"cd {backend_dir} && " f"{_manage_cmd} shell -c " f"\"from django.contrib.auth import get_user_model; U=get_user_model();" f" exit(0 if U.objects.filter(username='{ADMIN_USER}').exists() else 1)\"", check=False, ) if _su_exists.returncode == 0: ok(f"Django superuser '{ADMIN_USER}' already exists") else: _su_result = run( f"cd {backend_dir} && " f"DJANGO_SUPERUSER_USERNAME={ADMIN_USER} " f"DJANGO_SUPERUSER_PASSWORD={ADMIN_PASSWORD} " f"DJANGO_SUPERUSER_EMAIL=admin@localhost " f"{_manage_cmd} createsuperuser --noinput", check=False, ) if _su_result.returncode == 0: ok(f"Django superuser '{ADMIN_USER}' created") else: warn(f"Django superuser creation failed (rc={_su_result.returncode}) — create manually") # Seed the mandatory initial rows (organization + profile). _seed_initial_data() else: warn("manage entry point not found — skipping Django management commands") write_file( Path(f"/etc/systemd/system/{APP_NAME}-backend.service"), f"""\ [Unit] Description=123cluster backend (uvicorn / Django) After=network.target postgresql.service pgbouncer.service PartOf={APP_NAME}.target [Service] Type=simple WorkingDirectory={backend_dir} EnvironmentFile={env_file} ExecStart={_exec_start} Restart=always RestartSec=5 StandardOutput=append:{APP_DIR}/logs/backend.log StandardError=append:{APP_DIR}/logs/backend.log [Install] WantedBy={APP_NAME}.target """, ) run("systemctl daemon-reload") run(f"systemctl enable {APP_NAME}-backend", check=False) ok("Backend service configured") def setup_clustertasks() -> None: """Set up the clustertasks service (separate Python process alongside Django). Creates a dedicated virtual environment at APP_DIR/clustertasks/venv, installs tasks_requirements.txt, writes the .env file, and registers a systemd service unit that starts after the backend. """ tasks_dir = APP_DIR / "clustertasks" venv_dir = tasks_dir / "venv" env_file = tasks_dir / ".env" if BINARY_PKGS: _offline_tasks = OFFLINE_DIR / "clustertasks" / "task-service-build-bin.tar.gz" if OFFLINE_DIR else None _ready = (tasks_dir / "bin" / "taskservice").exists() if not _ready and (OFFLINE_DIR or ARTIFACTS_URL): tasks_dir.mkdir(parents=True, exist_ok=True) for _attempt in range(1, 3): tmp_tar = Path("/tmp/task-service-build-bin.tar.gz") tmp_tar.unlink(missing_ok=True) if _offline_tasks and _offline_tasks.exists(): r = run( f"cp {_offline_tasks} {tmp_tar}", check=False, _spin=f"Copying task service binary from offline bundle (attempt {_attempt}/2)", ) elif ARTIFACTS_URL: r = _fetch_artifact( ARTIFACTS_URL, "clustertasks/task-service-build-bin.tar.gz", tmp_tar, f"Fetching task service binary (attempt {_attempt}/2)", ) else: break if r.returncode == 0 and tmp_tar.exists(): run(f"tar -xzf {tmp_tar} -C {tasks_dir}", _spin="Extracting task service binary", check=False) tmp_tar.unlink(missing_ok=True) if (tasks_dir / "bin" / "taskservice").exists(): run(f"chmod +x {tasks_dir}/bin/taskservice", check=False) ok("Task service binary extracted") break warn(f"bin/taskservice missing after extraction (attempt {_attempt}/2)") else: src = str(_offline_tasks) if _offline_tasks else f"{ARTIFACTS_URL}/clustertasks/task-service-build-bin.tar.gz" warn(f"Could not deploy task service binary from {src} — deploy manually") _exec_start = f"{tasks_dir}/bin/taskservice runserver" _ready = (tasks_dir / "bin" / "taskservice").exists() else: _offline_tasks = OFFLINE_DIR / "clustertasks" / "clustertasks.tar.gz" if OFFLINE_DIR else None if OFFLINE_DIR or ARTIFACTS_URL: for _attempt in range(1, 3): tmp_tar = Path("/tmp/clustertasks.tar.gz") tmp_tar.unlink(missing_ok=True) if _offline_tasks and _offline_tasks.exists(): r = run( f"cp {_offline_tasks} {tmp_tar}", check=False, _spin=f"Copying clustertasks from offline bundle (attempt {_attempt}/2)", ) elif ARTIFACTS_URL: r = _fetch_artifact( ARTIFACTS_URL, "clustertasks/clustertasks.tar.gz", tmp_tar, f"Fetching clustertasks (attempt {_attempt}/2)", ) else: break if r.returncode == 0 and tmp_tar.exists(): run(f"tar -xzf {tmp_tar} -C {tasks_dir}", _spin="Extracting clustertasks", check=False) tmp_tar.unlink(missing_ok=True) if (tasks_dir / "task_service_ps.py").exists(): ok("Clustertasks deployed") break warn(f"task_service_ps.py missing after extraction (attempt {_attempt}/2)") else: src = str(_offline_tasks) if _offline_tasks else f"{ARTIFACTS_URL}/clustertasks/clustertasks.tar.gz" warn(f"Could not deploy clustertasks from {src} — deploy manually") if not (venv_dir / "bin" / "activate").exists(): python_bin = "python3.12" if have_cmd("python3.12") else "python3" run(f"{python_bin} -m venv {venv_dir}") pip = venv_dir / "bin" / "pip" python = venv_dir / "bin" / "python" if not OFFLINE_DIR: run(f"{pip} install --upgrade pip wheel", _spin="Upgrading pip/wheel") req_file = tasks_dir / "tasks_requirements.txt" if req_file.exists(): _pgdg_pgconfig_ct = Path(f"/usr/pgsql-{PGVERSION}/bin/pg_config") _ct_pg_path = ( f"PATH=/usr/pgsql-{PGVERSION}/bin:$PATH " if PKG_MGR in ("dnf", "yum") and _pgdg_pgconfig_ct.exists() else "" ) _ct_have_pgconfig = ( _pgdg_pgconfig_ct.exists() or Path("/usr/bin/pg_config").exists() or have_cmd("pg_config") ) _ct_have_pg_headers = ( Path(f"/usr/pgsql-{PGVERSION}/include/libpq-fe.h").exists() or Path("/usr/include/libpq-fe.h").exists() or Path("/usr/include/postgresql/libpq-fe.h").exists() ) if not (_ct_have_pgconfig and _ct_have_pg_headers): warn("pg_config or libpq headers not found — using psycopg2-binary for clustertasks (pre-compiled wheel)") _ct_lines = [] for _l in req_file.read_text().splitlines(keepends=True): if _l.lower().startswith("psycopg2") and "binary" not in _l.lower(): _ct_lines.append( _l.replace("psycopg2==", "psycopg2-binary==") .replace("psycopg2 ", "psycopg2-binary ") .replace("psycopg2\n", "psycopg2-binary\n") .replace("psycopg2>=", "psycopg2-binary>=") ) else: _ct_lines.append(_l) _ct_patched = req_file.parent / "tasks_requirements_patched.txt" _ct_patched.write_text("".join(_ct_lines)) req_file = _ct_patched run( f"{_ct_pg_path}{pip} install {_pip_flags()} -r {req_file}", _spin="Installing clustertasks dependencies", ) else: warn(f"tasks_requirements.txt not found at {req_file} — skipping pip install") _exec_start = f"{python} task_service_ps.py runserver" _ready = (tasks_dir / "task_service_ps.py").exists() if not env_file.exists(): write_file( env_file, f"DB_NAME={DB_NAME}\n" f"DB_USER={DB_USER}\n" f"DB_PASSWORD={DB_PASSWORD}\n" f"DB_HOST=127.0.0.1\n" f"DB_PORT={BOUNCER_PORT}\n" f"DB_TIMEZONE={DB_TIMEZONE}\n", mode=0o600, ) if not _ready: warn( "Task service entry point not found — " "clustertasks code must be deployed manually before the service can start" ) write_file( Path(f"/etc/systemd/system/{APP_NAME}-tasks.service"), f"""\ [Unit] Description=123cluster task service After=network.target postgresql.service pgbouncer.service {APP_NAME}-backend.service PartOf={APP_NAME}.target [Service] Type=simple WorkingDirectory={tasks_dir} EnvironmentFile={env_file} ExecStart={_exec_start} Restart=always RestartSec=5 StandardOutput=append:{APP_DIR}/logs/clustertasks.log StandardError=append:{APP_DIR}/logs/clustertasks.log [Install] WantedBy={APP_NAME}.target """, ) run("systemctl daemon-reload") run(f"systemctl enable {APP_NAME}-tasks", check=False) if _ready: ok("Clustertasks service configured") else: warn( "Clustertasks service unit written and enabled but will not start until " f"the task service binary is deployed to {tasks_dir}" ) def _disable_nginx_default_server() -> None: """Comment out the inline default server block in /etc/nginx/nginx.conf. RHEL/CentOS ship nginx.conf with a built-in server block (listen 80; server_name _) that conflicts with our conf.d config. nginx picks whichever comes first — nginx.conf wins, so our config is silently ignored. Commenting it out lets conf.d take over. """ nginx_conf = Path("/etc/nginx/nginx.conf") if DRY_RUN or not nginx_conf.exists(): return lines = nginx_conf.read_text().splitlines(keepends=True) out: list[str] = [] in_server = False depth = 0 done = False # only comment the first server block (the RHEL/CentOS default) for line in lines: if not done and not in_server and line.strip() == "server {": in_server, depth = True, 1 out.append("#" + line) continue if in_server: out.append("#" + line) depth += line.count("{") - line.count("}") if depth == 0: in_server = False done = True else: out.append(line) nginx_conf.write_text("".join(out)) def _restore_nginx_default_server() -> None: """Undo _disable_nginx_default_server(): uncomment the first server block. Called by purge_all() so that after a Wipe the stock nginx.conf is restored and any remaining custom sites still work when nginx restarts. """ nginx_conf = Path("/etc/nginx/nginx.conf") if DRY_RUN or not nginx_conf.exists(): return lines = nginx_conf.read_text().splitlines(keepends=True) out: list[str] = [] in_server = False depth = 0 done = False for line in lines: stripped_of_hash = line[1:] if line.startswith("#") else line if not done and not in_server and stripped_of_hash.strip() == "server {" and line.startswith("#"): in_server, depth = True, 1 out.append(stripped_of_hash) continue if in_server: out.append(stripped_of_hash if line.startswith("#") else line) content = stripped_of_hash if line.startswith("#") else line depth += content.count("{") - content.count("}") if depth == 0: in_server, done = False, True else: out.append(line) nginx_conf.write_text("".join(out)) def setup_frontend() -> None: """Build the React frontend and configure Nginx to serve the static files. Runs npm install and npm run build in APP_DIR/frontend, then adds a new Nginx site that serves the build directory and proxies /api/ and /admin/ to the Django backend. Existing Nginx virtual hosts are not modified. """ frontend_dir = APP_DIR / "frontend" if BINARY_PKGS: # Frontend is bundled inside the backend binary archive at frontend-build/ build_dir = APP_DIR / "backend" / "frontend-build" if (build_dir / "index.html").exists(): ok("Frontend already present inside backend binary archive") else: warn(f"frontend-build/index.html not found at {build_dir} — backend binary may not have been extracted yet") else: build_dir = frontend_dir / "build" _offline_frontend = OFFLINE_DIR / "frontend" / "build.tar.gz" if OFFLINE_DIR else None if OFFLINE_DIR or ARTIFACTS_URL: for _attempt in range(1, 3): tmp_tar = Path("/tmp/frontend-build.tar.gz") tmp_tar.unlink(missing_ok=True) if _offline_frontend and _offline_frontend.exists(): r = run( f"cp {_offline_frontend} {tmp_tar}", check=False, _spin=f"Copying frontend from offline bundle (attempt {_attempt}/2)", ) elif ARTIFACTS_URL: r = _fetch_artifact( ARTIFACTS_URL, "frontend/build.tar.gz", tmp_tar, f"Fetching frontend build (attempt {_attempt}/2)", ) else: break if r.returncode == 0 and tmp_tar.exists(): build_dir.mkdir(parents=True, exist_ok=True) run(f"tar -xzf {tmp_tar} -C {build_dir}", _spin="Extracting frontend build", check=False) tmp_tar.unlink(missing_ok=True) if (build_dir / "index.html").exists(): ok("Frontend deployed") break warn(f"index.html missing after frontend extraction (attempt {_attempt}/2)") else: src = str(_offline_frontend) if _offline_frontend else f"{ARTIFACTS_URL}/frontend/build.tar.gz" warn(f"Could not deploy frontend build from {src}") elif (frontend_dir / "package.json").exists(): run(f"cd {frontend_dir} && npm install", _spin="npm install") run(f"cd {frontend_dir} && npm run build", _spin="npm run build") ok("Frontend built") else: warn(f"No ARTIFACTS_URL/OFFLINE_DIR and no package.json at {frontend_dir} — skipping frontend build") pkg_detect() if not have_cmd("nginx"): pkg_install("nginx") server_ip = get_server_ip() nginx_site = f"""\ # 123cluster — managed by installer, do not edit manually server {{ listen {FRONTPORT}; server_name _; root {build_dir}; index index.html; # React client-side routing: fall back to index.html for unknown paths location / {{ try_files $uri $uri/ /index.html; }} # Django backend — all paths routed through the app location /api/ {{ proxy_pass http://127.0.0.1:{BACKPORT}; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_read_timeout 300; }} location /cluster-core/ {{ proxy_pass http://127.0.0.1:{BACKPORT}; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_read_timeout 300; }} # Django admin panel location /admin/ {{ proxy_pass http://127.0.0.1:{BACKPORT}; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; }} access_log /var/log/nginx/123cluster_access.log; error_log /var/log/nginx/123cluster_error.log; }} """ sites_available = Path("/etc/nginx/sites-available") sites_enabled = Path("/etc/nginx/sites-enabled") conf_d = Path("/etc/nginx/conf.d") if sites_available.exists(): site_conf = sites_available / "123cluster" write_file(site_conf, nginx_site) link = sites_enabled / "123cluster" if not link.exists() and not DRY_RUN: link.symlink_to(site_conf) # Remove the Ubuntu default site which conflicts (same server_name _). default_link = sites_enabled / "default" if default_link.exists() and not DRY_RUN: default_link.unlink() else: write_file(conf_d / "123cluster.conf", nginx_site) # On RHEL/CentOS, nginx.conf ships with an inline default server block # that conflicts with our conf.d config (both use server_name _). # Comment it out so our config wins. _disable_nginx_default_server() run("nginx -t", check=False) run("systemctl enable --now nginx", check=False) run("systemctl reload nginx", check=False) ok(f"Frontend available at http://{server_ip}:{FRONTPORT}/") # TODO: confirm with CEO whether Azurite is needed before re-enabling. # def setup_azurite() -> None: # """Install Azurite (Azure Blob Storage emulator) as a systemd service. # # Azurite is installed globally via npm. Persistent data is stored in # APP_DIR/azurite. Idempotent: skips npm install if azurite is already on PATH. # """ # npm_bin = shutil.which("npm") # if not npm_bin: # warn("npm not found — skipping Azurite install") # return # if not have_cmd("azurite"): # run(f"{npm_bin} install -g azurite") # # azurite_dir = APP_DIR / "azurite" # if DRY_RUN: # log.info(f"[DRY RUN] mkdir -p {azurite_dir}") # else: # azurite_dir.mkdir(parents=True, exist_ok=True) # # azurite_bin = run_output("which azurite 2>/dev/null") or "/usr/bin/azurite" # write_file( # Path(f"/etc/systemd/system/{APP_NAME}-azurite.service"), # f"""\ # [Unit] # Description=Azurite Azure Blob Storage emulator # After=network.target # PartOf={APP_NAME}.target # # [Service] # Type=simple # ExecStart={azurite_bin} --location {azurite_dir} \ # --debug {APP_DIR}/logs/azurite.log # Restart=always # RestartSec=5 # # [Install] # WantedBy={APP_NAME}.target # """, # ) # run("systemctl daemon-reload") # run(f"systemctl enable {APP_NAME}-azurite", check=False) # ok("Azurite service configured") def deploy_systemd_target() -> None: """Create the 123cluster.target unit that groups all application services. After this step, all components can be started or stopped as a group: systemctl start 123cluster.target systemctl stop 123cluster.target """ write_file( Path(f"/etc/systemd/system/{APP_NAME}.target"), f"""\ [Unit] Description=123cluster application Documentation=https://123cluster.com After=network.target [Install] WantedBy=multi-user.target """, ) run("systemctl daemon-reload") run(f"systemctl enable {APP_NAME}.target", check=False) ok("systemd target created") def setup_ssh_keys() -> None: """Generate a dedicated SSH key pair and wire up localhost access. Creates /root/.ssh/123cluster_id_ed25519 (private) and the matching .pub file. Adds the public key to /root/.ssh/authorized_keys so the backend service can SSH to the local task service without a password. Also enables key-based root login in sshd if not already set. Idempotent: skips generation if the key already exists. """ ssh_dir = Path("/root/.ssh") key_path = ssh_dir / "123cluster_id_ed25519" ssh_dir.mkdir(parents=True, exist_ok=True) if key_path.exists(): ok(f"SSH key already exists: {key_path}") else: run(f"ssh-keygen -t ed25519 -N '' -C '123cluster-app' -f {key_path}") ok(f"SSH key pair generated: {key_path}") pub_key = ( (key_path.with_suffix(".pub").read_text().strip()) if key_path.with_suffix(".pub").exists() and not DRY_RUN else "" ) info(f"Public key: {pub_key}") # Add the public key to local authorized_keys so the backend can SSH # to the task service running on the same host. if not DRY_RUN and pub_key != "": auth_keys = ssh_dir / "authorized_keys" auth_keys.touch(mode=0o600, exist_ok=True) auth_keys.chmod(0o600) existing = auth_keys.read_text() if pub_key not in existing: with auth_keys.open("a") as _f: _f.write(pub_key + "\n") ok("Public key added to /root/.ssh/authorized_keys (localhost SSH)") else: ok("Public key already in /root/.ssh/authorized_keys") # Ensure sshd allows key-based root login (prohibit-password keeps it safe) sshd_cfg = Path("/etc/ssh/sshd_config") if not DRY_RUN and sshd_cfg.exists(): cfg_text = sshd_cfg.read_text() if "PermitRootLogin prohibit-password" not in cfg_text: run( "sed -i 's|^#*PermitRootLogin.*|PermitRootLogin prohibit-password|' " "/etc/ssh/sshd_config", check=False, ) run("systemctl reload sshd 2>/dev/null || systemctl reload ssh 2>/dev/null || true", check=False) ok("sshd: PermitRootLogin set to prohibit-password") # Create (or verify) the c123_host record for this machine with # ex_params={"myself": true} so the backend can look up its own SSH # credentials and connect to the local task service. backend_dir = APP_DIR / "backend" if BINARY_PKGS: _manage_cmd = str(backend_dir / "bin" / "manage") _can_manage = (backend_dir / "bin" / "manage").exists() else: _python = backend_dir / "venv" / "bin" / "python" _manage_cmd = f"{_python} manage.py" _can_manage = (backend_dir / "manage.py").exists() and _python.exists() if not DRY_RUN and key_path.exists() and _can_manage: _myself_py = textwrap.dedent("""\ import os, socket, sys key_path = '/root/.ssh/123cluster_id_ed25519' try: key_content = open(key_path).read().strip() except FileNotFoundError: print('KEY_MISSING'); sys.exit(0) try: from clusterCore.models import c123_host from django.contrib.auth.models import User admin = User.objects.filter(is_superuser=True).first() if not admin: print('NO_ADMIN'); sys.exit(0) existing = c123_host.objects.filter(ex_params__myself=True, ind_deleted=False).first() if existing: if not existing.private_key: existing.private_key = key_content existing.save() print('UPDATED') else: print('EXISTS') else: local_ip = socket.gethostbyname(socket.gethostname()) c123_host.objects.create( name=local_ip, ssh_port='22', username='root', password='', private_key=key_content, status='skip', ex_params={'myself': True}, created_by=admin, last_update_by=admin, ind_deleted=False, ) print(f'CREATED {local_ip}') except Exception as e: print(f'ERROR: {e}'); sys.exit(1) """) _r = run( f"cd {backend_dir} && {_manage_cmd} shell -c {shlex.quote(_myself_py)}", check=False, ) if _r.returncode == 0: ok("Localhost host record (myself=true) created/verified in database") else: warn("Could not create localhost host record — add it manually via admin panel: " "C123_hosts → Add → ex_params={\"myself\": true}, private_key=<123cluster_id_ed25519>") _SMB_SHARE_CONF = """\ [123Cluster] path = /srv/123Cluster force user = nobody public = yes browsable = yes guest ok = yes guest only = yes writable = yes create mask = 0444 directory mask = 0555 writeable = yes delete readonly = no veto files = /*.bak/*.tmp/ delete veto files = no vfs objects = worm worm:grace_period = 60 """ _SMB_CONF = """\ [global] workgroup = WORKGROUP security = user server role = standalone server map to guest = Bad User guest account = nobody usershare allow guests = yes server min protocol = NT1 client min protocol = NT1 """ + _SMB_SHARE_CONF def install_samba() -> None: """Install Samba and configure a read-only 123Cluster share at /srv/123Cluster.""" pkg_detect() pkg_install("samba", "samba-common") share = Path("/srv/123Cluster") if DRY_RUN: log.info("[DRY RUN] mkdir /srv/123Cluster && chmod 0555") else: share.mkdir(parents=True, exist_ok=True) share.chmod(0o555) run("chown nobody:nogroup /srv/123Cluster || chown nobody:nobody /srv/123Cluster", check=False) _smb_conf_path = Path("/etc/samba/smb.conf") if _smb_conf_path.exists(): _existing_smb = _smb_conf_path.read_text() if "[123Cluster]" in _existing_smb: ok("123Cluster Samba share already present in smb.conf") else: # Preserve existing config; back it up and append only the share section. if not DRY_RUN: shutil.copy(str(_smb_conf_path), str(_smb_conf_path) + ".pre-123cluster.bak") with _smb_conf_path.open("a") as _f: _f.write("\n" + _SMB_SHARE_CONF) ok("123Cluster share appended to existing smb.conf (backup: smb.conf.pre-123cluster.bak)") else: write_file(_smb_conf_path, _SMB_CONF) ok("Samba smb.conf written") run("systemctl enable smb || systemctl enable smbd", check=False) run("systemctl restart smb || systemctl restart smbd", check=False) ok("Samba and 123Cluster share configured") # The prebuilt backend binary embeds mariadb Python bindings compiled against this # Connector/C; the bindings refuse to load against an older system libmariadb # (ImportError "was build with ... while the loaded ... library has version ..."). # CentOS Stream / RHEL 10 ship 3.4.4 in AppStream, so MariaDB-shared from MariaDB.org is needed. MIN_CONNECTOR_C = "3.4.9" MARIADB_REPO_SERIES = "11.8" def _connector_c_version() -> str: """Version of the Connector/C that provides libmariadb.so.3 (RPM systems), or ''.""" return run_output( "rpm -q --provides $(rpm -q --whatprovides 'libmariadb.so.3()(64bit)' 2>/dev/null | head -1)" " 2>/dev/null | grep -E '^mariadb-connector-c = ' | head -1 | awk '{print $3}'" ).strip().split("-")[0] def _ver_tuple(v: str) -> tuple: return tuple(int(x) for x in v.split(".") if x.isdigit()) def ensure_mariadb_connector_c() -> None: """Make sure libmariadb is at least MIN_CONNECTOR_C (RPM systems only).""" have = _connector_c_version() if have and _ver_tuple(have) >= _ver_tuple(MIN_CONNECTOR_C): ok(f"MariaDB Connector/C {have} (>= {MIN_CONNECTOR_C})") return ver = _os_version_id() if not REPO_URL: # Online install: add the MariaDB.org repo. With a local mirror the # [local-mariadb-connector] repo from local-mirror.repo is already configured. write_file( Path("/etc/yum.repos.d/mariadb-connector.repo"), f"""\ [mariadb-connector] name=MariaDB {MARIADB_REPO_SERIES} (Connector/C) baseurl=https://rpm.mariadb.org/{MARIADB_REPO_SERIES}/rhel/{ver}/x86_64 gpgkey=https://supplychain.mariadb.com/MariaDB-Server-GPG-KEY gpgcheck=1 enabled=1 """, ) run(f"{PKG_MGR} install -y --allowerasing MariaDB-shared", check=False, _spin=f"Installing newer MariaDB Connector/C (have {have or 'none'}, need >= {MIN_CONNECTOR_C})") have = _connector_c_version() if have and _ver_tuple(have) >= _ver_tuple(MIN_CONNECTOR_C): ok(f"MariaDB Connector/C {have} installed") else: warn(f"MariaDB Connector/C is {have or 'missing'} (< {MIN_CONNECTOR_C}) — " "the backend will fail with a MariaDB ImportError; install MariaDB-shared manually") def fix_selinux_labels() -> None: """Apply correct SELinux file contexts on RHEL/Rocky (SELinux enforcing). systemd reads EnvironmentFile as init_t → file must be etc_t. systemd opens StandardOutput log files as init_t → must be var_log_t. systemd exec's venv binaries as init_t → must be bin_t. venv shared libraries referenced by bin_t executables → lib_t. """ if PKG_MGR not in ("dnf", "yum"): return _se = run_output("getenforce 2>/dev/null").strip() if _se != "Enforcing": return for _env in (APP_DIR / "backend" / ".env", APP_DIR / "clustertasks" / ".env"): if _env.exists(): run(f"chcon -t etc_t {_env}", check=False) _logs = APP_DIR / "logs" if _logs.exists(): run(f"chcon -R -t var_log_t {_logs}", check=False) for _base in (APP_DIR / "backend", APP_DIR / "clustertasks"): _vbin = _base / "venv" / "bin" _vlib = _base / "venv" / "lib" if _vbin.exists(): run(f"chcon -R -t bin_t {_vbin}", check=False) if _vlib.exists(): run(f"chcon -R -t lib_t {_vlib}", check=False) # Prebuilt binary layout: bin/* are symlinks into .dist/ (executables # plus bundled .so files); systemd (init_t) can only exec/map bin_t, not default_t. _bbin = _base / "bin" if _bbin.exists(): run(f"chcon -R -h -t bin_t {_bbin}", check=False) for _dist in _base.glob("*.dist"): run(f"chcon -R -t bin_t {_dist}", check=False) # Nginx (httpd_t) cannot read files with default_t — label frontend with # httpd_sys_content_t so nginx can serve the React build. _frontend = APP_DIR / "frontend" / "build" if _frontend.exists(): run(f"chcon -R -t httpd_sys_content_t {_frontend}", check=False) # Allow nginx to proxy to the uvicorn backend on port 8000. run("setsebool -P httpd_can_network_connect 1", check=False) ok("SELinux file contexts updated") def fix_apparmor_nginx() -> None: """Allow nginx to serve files from APP_DIR on Ubuntu (AppArmor-enforced). Ubuntu's default nginx AppArmor profile only allows reading from /var/www/ and /usr/share/nginx/. Files extracted to APP_DIR (/123cluster/) are blocked with EACCES. This adds a local profile override so nginx can read APP_DIR. """ if PKG_MGR != "apt": return if not have_cmd("aa-status"): return # Check if AppArmor is running and nginx is under a profile _aa_out = run_output("aa-status 2>/dev/null") if "nginx" not in _aa_out: return _local_dir = Path("/etc/apparmor.d/local") _local_dir.mkdir(parents=True, exist_ok=True) _override = _local_dir / "usr.sbin.nginx" _rule = f" {APP_DIR}/ r,\n {APP_DIR}/** r,\n" _existing = _override.read_text() if _override.exists() else "" if _rule not in _existing: write_file(_override, _existing + _rule) run("apparmor_parser -r /etc/apparmor.d/usr.sbin.nginx 2>/dev/null || true", check=False) ok(f"AppArmor: nginx profile updated to allow {APP_DIR}") def start_all_services() -> None: """Start all 123cluster systemd services via the application target.""" pkg_detect() fix_selinux_labels() fix_apparmor_nginx() run(f"systemctl start {APP_NAME}.target", check=False) ok("All services started") def restart_all_services() -> None: """Restart each 123cluster service individually, then reload Nginx.""" pkg_detect() fix_selinux_labels() fix_apparmor_nginx() for svc in ( f"{APP_NAME}-backend", f"{APP_NAME}-tasks", # f"{APP_NAME}-azurite", # TODO: confirm with CEO whether Azurite is needed "pgbouncer", ): run(f"systemctl restart {svc}", check=False) run("systemctl reload nginx", check=False) ok("All services restarted") # ── Grouped install steps ───────────────────────────────────────────────────── def install_postgresql_all() -> bool: """Install and fully configure the PostgreSQL stack. Runs in sequence: install PG server → configure → tune → DB clients → create (empty) database → PgBouncer. The dump restore and initial data seeding happen later in the backend phase (setup_backend) so that Django migrations can create the full schema first — the dumps only carry data for tables that existed at dump time. In custom mode this function expects ask_postgres_params() to have been called beforehand so that DBPORT, BOUNCER_PORT, DB_USER, DB_PASSWORD, DUMP_BASE_URL is already set. Returns False if any critical step fails. """ setup_local_repos() install_postgres() if not configure_postgres(): return False tune_postgres() install_db_clients() if not _create_database(): return False setup_pgbouncer() ok("PostgreSQL stack ready") return True def install_backend_all() -> None: """Install all backend components. Runs in sequence: base packages → Python 3.10 → Node.js → application directories → systemd target → backend service → clustertasks service → Azurite → Samba → SSH keys. """ setup_local_repos() base_packages() install_python312() install_nodejs() create_app_dirs() deploy_systemd_target() setup_backend() setup_clustertasks() # setup_azurite() # TODO: confirm with CEO whether Azurite is needed install_samba() setup_ssh_keys() ok("Backend stack ready") def update_app() -> None: """Check for a newer application version and apply available updates. Reads the installed version from APP_DIR/VERSION and compares it with the latest release published at RELEASES_URL/latest.json. Downloads and unpacks the new release archive, re-runs pip install, npm build, and Django migrations, then restarts all services. """ version_file = APP_DIR / "VERSION" current = version_file.read_text().strip() if version_file.exists() else "0.0.0" info(f"Installed version: {current}") manifest_raw = run_output(f"curl -sf {RELEASES_URL}/latest.json") if not manifest_raw: warn(f"Could not reach {RELEASES_URL}/latest.json — is RELEASES_URL configured?") return try: manifest = json.loads(manifest_raw) except json.JSONDecodeError: err("Invalid JSON in release manifest") return latest = manifest.get("version", current) dl_url = manifest.get("download_url", "") info(f"Latest version: {latest}") if latest == current: ok("Application is already up to date") return if not dl_url: err("No download_url in manifest") return archive = Path(f"/tmp/123cluster-{latest}.tar.gz") run(f"curl -fL {dl_url} -o {archive}") run(f"tar -xzf {archive} -C {APP_DIR} --strip-components=1") backend_dir = APP_DIR / "backend" if BINARY_PKGS: _upd_manage = str(backend_dir / "bin" / "manage") if (backend_dir / "bin" / "manage").exists() else None else: venv_pip = APP_DIR / "backend" / "venv" / "bin" / "pip" req_file = APP_DIR / "backend" / "requirements.txt" if req_file.exists(): run(f"{venv_pip} install -r {req_file}") python = APP_DIR / "backend" / "venv" / "bin" / "python" _upd_manage = f"{python} manage.py" if (backend_dir / "manage.py").exists() else None if _upd_manage: run(f"cd {backend_dir} && {_upd_manage} migrate --noinput", check=False) run(f"cd {backend_dir} && {_upd_manage} collectstatic --noinput", check=False) if not BINARY_PKGS: frontend_dir = APP_DIR / "frontend" if (frontend_dir / "package.json").exists(): run(f"cd {frontend_dir} && npm install && npm run build") write_file(version_file, latest + "\n") restart_all_services() ok(f"Updated to version {latest}") def purge_all() -> None: """Full system reset: remove the app, PostgreSQL, PgBouncer, and Nginx. Equivalent to the manual cleanup sequence used between test installs. Prompts for explicit confirmation — this is not reversible. """ print(f"\n This will COMPLETELY remove 123cluster, PostgreSQL, PgBouncer, and Nginx.") print(f" ALL database data under /var/lib/pgsql will be deleted.") print(f" APP_DIR ({APP_DIR}) and /backups/123cluster will be deleted.") answer = input(" Type 'yes' to confirm full purge: ").strip().lower() if answer != "yes": info("Purge cancelled") return # Stop and disable services — stop all postgresql-* variants in case # the installed PG version differs from PGVERSION (e.g. after a failed run). run("systemctl stop 'postgresql-*' 2>/dev/null || true", check=False) run("systemctl disable 'postgresql-*' 2>/dev/null || true", check=False) for svc in [ f"{APP_NAME}-backend", f"{APP_NAME}-tasks", f"{APP_NAME}.target", "pgbouncer", "nginx", ]: run(f"systemctl stop {svc}", check=False) run(f"systemctl disable {svc}", check=False) # Remove service unit files for unit in [ f"{APP_NAME}-backend.service", f"{APP_NAME}-tasks.service", f"{APP_NAME}.target", ]: p = Path(f"/etc/systemd/system/{unit}") if p.exists() and not DRY_RUN: p.unlink() run("systemctl daemon-reload", check=False) # A1: warn before deleting foreign databases — if postgres has databases we # didn't create, require a second explicit confirmation before wiping. _pg_ver = detect_existing_postgres() if _pg_ver: _foreign_dbs = run_output( "sudo -u postgres psql -tAc " f"\"SELECT datname FROM pg_database WHERE datname NOT IN " f"('postgres','template0','template1','{DB_NAME}')\" 2>/dev/null" ).strip() if _foreign_dbs: print(f"\n ⚠ Databases NOT created by this installer:") for _db in _foreign_dbs.splitlines(): print(f" • {_db}") print(" These will be PERMANENTLY DELETED by the Wipe.") _c2 = input(" Type 'delete' to confirm deleting foreign databases, or anything else to cancel: ").strip().lower() if _c2 != "delete": info("Wipe cancelled to protect foreign databases") return # Remove OS packages — use rpm -qa to catch whatever PG version is actually # installed, not just PGVERSION (they can differ after a version mismatch). pkg_detect() if PKG_MGR in ("dnf", "yum"): run( "rpm -qa --queryformat '%{NAME}\\n' | grep -E '^postgresql[0-9]'" " | xargs -r dnf remove -y 2>/dev/null || true", check=False, ) run("dnf remove -y pgbouncer nginx", check=False) elif PKG_MGR == "apt": run(f"DEBIAN_FRONTEND=noninteractive apt-get purge -y " f"postgresql-{PGVERSION} pgbouncer nginx nginx-common nginx-full", check=False) run("apt-get autoremove -y --purge", check=False) # Remove data directories and config — wipe all of /var/lib/pgsql so no # version-specific subdirectory is left behind regardless of which PG version ran. for path in [ APP_DIR, Path("/backups/123cluster"), Path("/var/lib/pgsql"), Path("/etc/pgbouncer"), Path("/var/lib/pgbouncer"), Path("/var/log/nginx"), ]: if not DRY_RUN and path.exists(): shutil.rmtree(path, ignore_errors=True) # A6: Remove only the installer's own nginx files — do NOT delete /etc/nginx # wholesale, which would destroy any custom site configs the user has there. for _nf in ( Path("/etc/nginx/sites-enabled/123cluster"), Path("/etc/nginx/sites-available/123cluster"), Path("/etc/nginx/conf.d/123cluster.conf"), ): if not DRY_RUN and _nf.exists(): _nf.unlink(missing_ok=True) # Restore the default server block in nginx.conf that the installer commented out. if not DRY_RUN: _restore_nginx_default_server() # Remove installer-written repo files so the next install starts from a clean # package-manager state (no stale mirror URLs, no disabled online PGDG repos). for _rf in ( # DNF / yum Path("/etc/yum.repos.d/local-mirror.repo"), Path("/etc/yum.repos.d/local-python312-bootstrap.repo"), Path("/etc/yum.repos.d/offline-iso.repo"), Path("/etc/yum.repos.d/offline-rpms.repo"), # APT source lists Path("/etc/apt/sources.list.d/local-pgdg.list"), Path("/etc/apt/sources.list.d/local-nodejs.list"), Path("/etc/apt/sources.list.d/local-python312.list"), Path("/etc/apt/sources.list.d/sury-python.list"), # APT GPG keys Path("/etc/apt/keyrings/local-pgdg.gpg"), Path("/etc/apt/keyrings/local-nodesource.gpg"), Path("/etc/apt/keyrings/local-deadsnakes.gpg"), Path("/etc/apt/keyrings/sury-python.gpg"), ): if not DRY_RUN and _rf.exists(): _rf.unlink(missing_ok=True) # Re-enable any online PGDG repos that configure_local_repos() disabled. if not DRY_RUN and have_cmd("rpm"): run( "find /etc/yum.repos.d/ -name 'pgdg*.repo' ! -name 'local-mirror.repo'" " -exec sed -i 's/^enabled *= *0/enabled=1/g' {} + 2>/dev/null || true", check=False, ) ok("Full purge complete — system is ready for a fresh install") def uninstall_app() -> None: """Remove the 123cluster application while preserving PostgreSQL and Nginx. Stops and disables all application services, removes the service unit files, removes APP_DIR, and deletes the Nginx site config. PostgreSQL, its data, and Nginx itself are kept intact. Prompts for explicit confirmation before removing any data. """ print(f"\n This will remove {APP_DIR} and all application services.") print(" PostgreSQL databases and Nginx will NOT be removed.") answer = input(" Type 'yes' to confirm: ").strip().lower() if answer != "yes": info("Uninstall cancelled") return services = [ f"{APP_NAME}-backend", f"{APP_NAME}-tasks", # f"{APP_NAME}-azurite", # TODO: confirm with CEO whether Azurite is needed f"{APP_NAME}.target", ] for svc in services: run(f"systemctl stop {svc}", check=False) run(f"systemctl disable {svc}", check=False) for unit in [ f"{APP_NAME}-backend.service", f"{APP_NAME}-tasks.service", # f"{APP_NAME}-azurite.service", # TODO: confirm with CEO whether Azurite is needed f"{APP_NAME}.target", ]: p = Path(f"/etc/systemd/system/{unit}") if p.exists() and not DRY_RUN: p.unlink() run("systemctl daemon-reload") for nginx_conf in ( Path("/etc/nginx/sites-enabled/123cluster"), Path("/etc/nginx/sites-available/123cluster"), Path("/etc/nginx/conf.d/123cluster.conf"), ): if nginx_conf.exists() and not DRY_RUN: nginx_conf.unlink() run("systemctl reload nginx", check=False) if APP_DIR.exists() and not DRY_RUN: shutil.rmtree(APP_DIR) ok("123cluster uninstalled") info("PostgreSQL databases and Nginx configuration are preserved.") def verify_installation() -> None: """Check every installed component and print a short status report. Checks PostgreSQL, PgBouncer, the backend service, clustertasks, Azurite, Nginx, the application database, and the .env file. Prints a compact table with PASS / WARN / FAIL for each item. """ WIDTH = 56 PASS = "\033[32mPASS\033[0m" WARN = "\033[33mWARN\033[0m" FAIL = "\033[31mFAIL\033[0m" # (label, status_tag, note, service_name) rows: List[Tuple[str, str, str, str]] = [] def _journal_hint(svc: str) -> str: """Return the last non-empty journal line for a service (1 line max).""" out = run_output( f"journalctl -u {svc} -n 5 --no-pager -o short-monotonic 2>/dev/null" " | grep -v '^--' | tail -1" ).strip() # Strip the monotonic timestamp prefix (e.g. " 123.456789 hostname svc: msg") if out and " " in out: out = out.split(" ", 2)[-1] if out.count(" ") >= 2 else out return out[:80] if out else "" def _row(label: str, status: str, note: str = "", svc: str = "") -> None: rows.append((label, status, note, svc)) # PostgreSQL pg_ver = detect_existing_postgres() if pg_ver: _row("PostgreSQL", PASS, f"version {pg_ver}, port {DBPORT}") else: _row("PostgreSQL", FAIL, "not running", "postgresql") # PgBouncer if service_active("pgbouncer"): _row("PgBouncer", PASS, f"port {BOUNCER_PORT}") else: _row("PgBouncer", FAIL, "not running", "pgbouncer") # Backend service svc_backend = f"{APP_NAME}-backend" if service_active(svc_backend): _row("Backend service", PASS, f"port {BACKPORT}", svc_backend) else: _row("Backend service", FAIL, "not running", svc_backend) # Clustertasks svc_tasks = f"{APP_NAME}-tasks" if service_active(svc_tasks): _row("Clustertasks", PASS, "", svc_tasks) else: _row("Clustertasks", WARN, "not running", svc_tasks) # Azurite — commented out pending CEO decision on whether it's needed # svc_azurite = f"{APP_NAME}-azurite" # if service_active(svc_azurite): # _row("Azurite", PASS, "", svc_azurite) # else: # _row("Azurite", WARN, "not running", svc_azurite) # Nginx / frontend if service_active("nginx"): _row("Nginx / frontend", PASS, f"port {FRONTPORT}") else: _row("Nginx / frontend", FAIL, "not running", "nginx") # Application database # On PGDG RPM installs psql lives in /usr/pgsql-N/bin, not on the default PATH. pkg_detect() _psql = ( f"/usr/pgsql-{PGVERSION}/bin/psql" if PKG_MGR in ("dnf", "yum") and Path(f"/usr/pgsql-{PGVERSION}/bin/psql").exists() else "psql" ) r = subprocess.run( f"sudo -u postgres {_psql} -lqt 2>/dev/null" f" | cut -d'|' -f1 | grep -qw {DB_NAME}", shell=True, capture_output=True, ) if r.returncode == 0: _row(f"Database '{DB_NAME}'", PASS, "") else: _row(f"Database '{DB_NAME}'", FAIL, "not found") # .env file env_path = APP_DIR / "backend" / ".env" if env_path.exists(): _row(".env file", PASS, str(env_path)) else: _row(".env file", WARN, f"not found at {env_path}") # Dump base URL / directory _base = DUMP_BASE_URL.rstrip("/") _is_url = _base.startswith("http://") or _base.startswith("https://") if _is_url: _row("Dump base URL", PASS, _base) elif Path(_base).is_dir(): _row("Dump base dir", PASS, _base) else: _row("Dump base dir", WARN, f"not found: {_base}") # Print report print(f"\n{'═' * WIDTH}") print(" Installation status report") print(f"{'─' * WIDTH}") details: List[str] = [] # WARN/FAIL detail blocks printed after the table for label, status, note, svc in rows: is_problem = "FAIL" in status or "WARN" in status line = f" {label:<24} {status}" if note: line += f" ({note})" if svc and is_problem: line += f" → journalctl -u {svc}" print(line) if svc and is_problem: hint = _journal_hint(svc) if hint: details.append(f" [{svc}] {hint}") print(f"{'═' * WIDTH}") failures = sum(1 for _, s, _, _ in rows if "FAIL" in s) warnings = sum(1 for _, s, _, _ in rows if "WARN" in s) if failures: print(f" {failures} component(s) FAILED — review errors above\n") elif warnings: print(f" All critical components OK — {warnings} warning(s)\n") else: print(" All components healthy\n") if details: print(" Last log line per failed/warned service:") for d in details: print(d) print() def install_all(force: bool = False) -> None: """Run a complete installation of all 123cluster components in order. When force=True the freshness check is skipped so the installer can run over an existing installation (Force mode). The caller must have already called ask_postgres_params() when running in Custom mode; Typical and Force modes use the built-in defaults. """ if not sys_checks(check_freshness=not force): err("System checks failed — installation aborted") return if not install_postgresql_all(): err("PostgreSQL installation failed — aborting") return install_backend_all() setup_frontend() start_all_services() _failed_svcs = [ s for s in (f"{APP_NAME}-backend", f"{APP_NAME}-tasks") if not service_active(s) ] if _failed_svcs: err(f"Services not running after start: {', '.join(_failed_svcs)}") err("Check: journalctl -u 123cluster-backend -n 50") err("If ARTIFACTS_URL was not set, app code was not downloaded — set it and re-run option B then F.") return server_ip = get_server_ip() ok("Installation complete") info(f"Frontend: http://{server_ip}:{FRONTPORT}/") info(f"Admin panel: http://{server_ip}:{BACKPORT}/admin/") info(f"DB admin: {ADMIN_USER} / {ADMIN_PASSWORD}") # ── Menu ────────────────────────────────────────────────────────────────────── def main_menu() -> None: """Display the interactive main menu and dispatch the user's selection. Top-level install modes: 1 — Typical install: full install with built-in defaults, no prompts. 2 — Force install: same as Typical but skips freshness checks — safe to run over an existing installation. 3 — Custom install: collects PostgreSQL parameters first, then lets you run individual steps or all at once. 4 — Verify installation: checks every component and prints a report. Individual steps (run with current parameter values): S — System checks P — Install PostgreSQL B — Install backend F — Install frontend Maintenance: A — Start all services R — Restart all services U — Update application X — Uninstall application W — Wipe everything (full reset, removes PostgreSQL/Nginx/PgBouncer) Q — Quit """ while True: print(f"\n{'═' * 58}") print(" 123cluster Installer") print(f"{'═' * 58}") print(f" 1. Typical install — full install with defaults") print(f" 2. Force install — reinstall over existing setup") print(f" 3. Custom install — configure parameters first") print(f" 4. Verify install — check components, print report") print(f"{'─' * 58}") print(f" S. System checks only") print(f" P. Install PostgreSQL (step 2)") print(f" B. Install backend (step 3)") print(f" F. Install frontend (step 4)") print(f"{'─' * 58}") print(f" A. Start all services") print(f" R. Restart all services") print(f" U. Update application") print(f" X. Uninstall application") print(f" W. Wipe everything — full reset (removes PostgreSQL / Nginx)") print(f" Q. Quit") print(f"{'═' * 58}") choice = input("Select: ").strip().upper() if choice == "1": prompt_mirror_urls() install_all(force=False) elif choice == "2": warn("Force mode: freshness checks will be skipped.") answer = input(" Continue? [y/N]: ").strip().lower() if answer in ("y", "yes"): prompt_mirror_urls() install_all(force=True) else: info("Force install cancelled") elif choice == "3": prompt_mirror_urls() ask_postgres_params() print(f"\n{'─' * 58}") print(" Parameters saved. Use S/P/B/F to run individual steps") print(" or press 1 / 2 to run the full install now.") print(f"{'─' * 58}") elif choice == "4": verify_installation() elif choice == "S": sys_checks() elif choice == "P": install_postgresql_all() elif choice == "B": install_backend_all() elif choice == "F": setup_frontend() elif choice == "A": start_all_services() elif choice == "R": restart_all_services() elif choice == "U": update_app() elif choice == "X": uninstall_app() elif choice == "W": purge_all() elif choice == "Q": print("Bye.") break else: warn(f"Unknown option: {choice}") input("\nPress Enter to continue...") # ── Entry ───────────────────────────────────────────────────────────────────── if __name__ == "__main__": if DRY_RUN: log.info("=" * 58) log.info(" DRY RUN mode — no changes will be made") log.info("=" * 58) require_root() pkg_detect() main_menu()