Add installation and update scripts for Apache Airflow

2026-05-10 22:47:20 +02:00
parent 22976049b6
commit e6cabc342c
3 changed files with 267 additions and 0 deletions
--- a/ct/airflow.sh
+++ b/ct/airflow.sh
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+source <(curl -fsSL https://raw.githubusercontent.com/community-scripts/ProxmoxVED/main/misc/build.func)
+# Copyright (c) 2021-2026 community-scripts ORG
+# Author: MickLesk (CanbiZ)
+# License: MIT | https://github.com/community-scripts/ProxmoxVED/raw/main/LICENSE
+# Source: https://github.com/apache/airflow
+
+APP="Apache Airflow"
+var_tags="${var_tags:-workflow;scheduler;automation}"
+var_cpu="${var_cpu:-2}"
+var_ram="${var_ram:-4096}"
+var_disk="${var_disk:-16}"
+var_os="${var_os:-debian}"
+var_version="${var_version:-13}"
+var_unprivileged="${var_unprivileged:-1}"
+
+header_info "$APP"
+variables
+color
+catch_errors
+
+function update_script() {
+  header_info
+  check_container_storage
+  check_container_resources
+
+  if [[ ! -d /opt/airflow ]]; then
+    msg_error "No ${APP} Installation Found!"
+    exit
+  fi
+
+  INSTALLED=$(cat ~/.airflow 2>/dev/null || echo "0")
+  LATEST=$(curl -fsSL "https://pypi.org/pypi/apache-airflow/json" | jq -r '.info.version')
+
+  if [[ "$INSTALLED" == "$LATEST" ]]; then
+    msg_ok "Already on the latest version (${LATEST})"
+    exit
+  fi
+
+  msg_info "Stopping Services"
+  systemctl stop airflow-api-server airflow-scheduler airflow-dag-processor airflow-triggerer
+  msg_ok "Stopped Services"
+
+  msg_info "Backing up Configuration"
+  cp /opt/airflow/.env /opt/airflow.env.bak
+  msg_ok "Backed up Configuration"
+
+  msg_info "Updating Apache Airflow to ${LATEST}"
+  $STD uv pip install --python /opt/airflow/.venv/bin/python \
+    "apache-airflow[postgres,fab]==${LATEST}" \
+    --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-${LATEST}/constraints-3.14.txt"
+  echo "${LATEST}" >~/.airflow
+  msg_ok "Updated Apache Airflow to ${LATEST}"
+
+  msg_info "Restoring Configuration"
+  cp /opt/airflow.env.bak /opt/airflow/.env
+  rm -f /opt/airflow.env.bak
+  msg_ok "Restored Configuration"
+
+  msg_info "Running Database Migrations"
+  set -a && source /opt/airflow/.env && set +a
+  $STD /opt/airflow/.venv/bin/airflow db migrate
+  msg_ok "Ran Database Migrations"
+
+  msg_info "Starting Services"
+  systemctl start airflow-api-server airflow-scheduler airflow-dag-processor airflow-triggerer
+  msg_ok "Started Services"
+  msg_ok "Updated successfully!"
+  exit
+}
+
+start
+build_container
+description
+
+msg_ok "Completed Successfully!\n"
+echo -e "${CREATING}${GN}${APP} setup has been successfully initialized!${CL}"
+echo -e "${INFO}${YW} Access it using the following URL:${CL}"
+echo -e "${TAB}${GATEWAY}${BGN}http://${IP}:8080${CL}"
--- a/install/airflow-install.sh
+++ b/install/airflow-install.sh
@@ -0,0 +1,140 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2021-2026 community-scripts ORG
+# Author: MickLesk (CanbiZ)
+# License: MIT | https://github.com/community-scripts/ProxmoxVED/raw/main/LICENSE
+# Source: https://github.com/apache/airflow
+
+source /dev/stdin <<<"$FUNCTIONS_FILE_PATH"
+color
+verb_ip6
+catch_errors
+setting_up_container
+network_check
+update_os
+
+msg_info "Installing Dependencies"
+$STD apt install -y \
+  build-essential \
+  libpq-dev \
+  libssl-dev \
+  libffi-dev \
+  python3-dev
+msg_ok "Installed Dependencies"
+
+UV_PYTHON="3.14" setup_uv
+PG_VERSION="16" setup_postgresql
+PG_DB_NAME="airflow" PG_DB_USER="airflow" setup_postgresql_db
+
+msg_info "Installing Apache Airflow"
+AIRFLOW_VERSION="3.2.1"
+mkdir -p /opt/airflow/{dags,logs,plugins}
+cd /opt/airflow
+$STD uv venv --python 3.14
+$STD uv pip install --python /opt/airflow/.venv/bin/python \
+  "apache-airflow[postgres,fab]==${AIRFLOW_VERSION}" \
+  --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-3.14.txt"
+echo "${AIRFLOW_VERSION}" >~/.airflow
+msg_ok "Installed Apache Airflow"
+
+msg_info "Configuring Application"
+FERNET_KEY=$(/opt/airflow/.venv/bin/python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())")
+SECRET_KEY=$(openssl rand -hex 32)
+ADMIN_PASS=$(openssl rand -base64 12 | tr -dc 'a-zA-Z0-9' | cut -c1-12)
+cat <<EOF >/opt/airflow/.env
+AIRFLOW_HOME=/opt/airflow
+AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://${PG_DB_USER}:${PG_DB_PASS}@localhost:5432/${PG_DB_NAME}
+AIRFLOW__CORE__EXECUTOR=LocalExecutor
+AIRFLOW__CORE__FERNET_KEY=${FERNET_KEY}
+AIRFLOW__CORE__DAGS_FOLDER=/opt/airflow/dags
+AIRFLOW__CORE__LOAD_EXAMPLES=false
+AIRFLOW__API__AUTH_BACKENDS=airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session
+AIRFLOW__WEBSERVER__SECRET_KEY=${SECRET_KEY}
+AIRFLOW__WEBSERVER__BASE_URL=http://${LOCAL_IP}:8080
+AIRFLOW_ADMIN_PASSWORD=${ADMIN_PASS}
+EOF
+set -a && source /opt/airflow/.env && set +a
+$STD /opt/airflow/.venv/bin/airflow db migrate
+$STD /opt/airflow/.venv/bin/airflow users create \
+  --username admin \
+  --firstname Admin \
+  --lastname User \
+  --role Admin \
+  --email admin@example.com \
+  --password "${ADMIN_PASS}"
+msg_ok "Configured Application"
+
+msg_info "Creating Services"
+cat <<EOF >/etc/systemd/system/airflow-api-server.service
+[Unit]
+Description=Apache Airflow API Server
+After=network.target postgresql.service
+
+[Service]
+Type=simple
+User=root
+EnvironmentFile=/opt/airflow/.env
+ExecStart=/opt/airflow/.venv/bin/airflow api-server --port 8080
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+cat <<EOF >/etc/systemd/system/airflow-scheduler.service
+[Unit]
+Description=Apache Airflow Scheduler
+After=network.target postgresql.service
+
+[Service]
+Type=simple
+User=root
+EnvironmentFile=/opt/airflow/.env
+ExecStart=/opt/airflow/.venv/bin/airflow scheduler
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+cat <<EOF >/etc/systemd/system/airflow-dag-processor.service
+[Unit]
+Description=Apache Airflow DAG Processor
+After=network.target postgresql.service
+
+[Service]
+Type=simple
+User=root
+EnvironmentFile=/opt/airflow/.env
+ExecStart=/opt/airflow/.venv/bin/airflow dag-processor
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+cat <<EOF >/etc/systemd/system/airflow-triggerer.service
+[Unit]
+Description=Apache Airflow Triggerer
+After=network.target postgresql.service
+
+[Service]
+Type=simple
+User=root
+EnvironmentFile=/opt/airflow/.env
+ExecStart=/opt/airflow/.venv/bin/airflow triggerer
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+EOF
+systemctl enable -q --now airflow-api-server airflow-scheduler airflow-dag-processor airflow-triggerer
+msg_ok "Created Services"
+
+motd_ssh
+customize
+cleanup_lxc
--- a/json/airflow.json
+++ b/json/airflow.json
@@ -0,0 +1,48 @@
+{
+  "name": "Apache Airflow",
+  "slug": "airflow",
+  "categories": [
+    19
+  ],
+  "date_created": "2026-05-10",
+  "type": "ct",
+  "updateable": true,
+  "privileged": false,
+  "interface_port": 8080,
+  "documentation": "https://airflow.apache.org/docs/",
+  "website": "https://airflow.apache.org/",
+  "logo": "https://cdn.jsdelivr.net/gh/selfhst/icons@main/webp/apache-airflow.webp",
+  "description": "Apache Airflow is an open-source platform for developing, scheduling, and monitoring batch-oriented workflows. It uses Python to author DAGs (Directed Acyclic Graphs) that represent workflows, and provides a rich web UI for managing and observing pipelines.",
+  "install_methods": [
+    {
+      "type": "default",
+      "script": "ct/airflow.sh",
+      "config_path": "/opt/airflow/.env",
+      "resources": {
+        "cpu": 2,
+        "ram": 4096,
+        "hdd": 16,
+        "os": "Debian",
+        "version": "13"
+      }
+    }
+  ],
+  "default_credentials": {
+    "username": "admin",
+    "password": null
+  },
+  "notes": [
+    {
+      "text": "The initial admin password is randomly generated and stored in /opt/airflow/.env (AIRFLOW_ADMIN_PASSWORD).",
+      "type": "info"
+    },
+    {
+      "text": "Place your DAG files in /opt/airflow/dags/. The scheduler will pick them up automatically.",
+      "type": "info"
+    },
+    {
+      "text": "This installs Airflow with LocalExecutor. For distributed task execution, configure CeleryExecutor manually.",
+      "type": "info"
+    }
+  ]
+}