diff --git a/ct/airflow.sh b/ct/airflow.sh new file mode 100644 index 00000000..35e9b8bf --- /dev/null +++ b/ct/airflow.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +source <(curl -fsSL https://raw.githubusercontent.com/community-scripts/ProxmoxVED/main/misc/build.func) +# Copyright (c) 2021-2026 community-scripts ORG +# Author: MickLesk (CanbiZ) +# License: MIT | https://github.com/community-scripts/ProxmoxVED/raw/main/LICENSE +# Source: https://github.com/apache/airflow + +APP="Apache Airflow" +var_tags="${var_tags:-workflow;scheduler;automation}" +var_cpu="${var_cpu:-2}" +var_ram="${var_ram:-4096}" +var_disk="${var_disk:-16}" +var_os="${var_os:-debian}" +var_version="${var_version:-13}" +var_unprivileged="${var_unprivileged:-1}" + +header_info "$APP" +variables +color +catch_errors + +function update_script() { + header_info + check_container_storage + check_container_resources + + if [[ ! -d /opt/airflow ]]; then + msg_error "No ${APP} Installation Found!" + exit + fi + + INSTALLED=$(cat ~/.airflow 2>/dev/null || echo "0") + LATEST=$(curl -fsSL "https://pypi.org/pypi/apache-airflow/json" | jq -r '.info.version') + + if [[ "$INSTALLED" == "$LATEST" ]]; then + msg_ok "Already on the latest version (${LATEST})" + exit + fi + + msg_info "Stopping Services" + systemctl stop airflow-api-server airflow-scheduler airflow-dag-processor airflow-triggerer + msg_ok "Stopped Services" + + msg_info "Backing up Configuration" + cp /opt/airflow/.env /opt/airflow.env.bak + msg_ok "Backed up Configuration" + + msg_info "Updating Apache Airflow to ${LATEST}" + $STD uv pip install --python /opt/airflow/.venv/bin/python \ + "apache-airflow[postgres,fab]==${LATEST}" \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-${LATEST}/constraints-3.14.txt" + echo "${LATEST}" >~/.airflow + msg_ok "Updated Apache Airflow to ${LATEST}" + + msg_info "Restoring Configuration" + cp /opt/airflow.env.bak /opt/airflow/.env + rm -f /opt/airflow.env.bak + msg_ok "Restored Configuration" + + msg_info "Running Database Migrations" + set -a && source /opt/airflow/.env && set +a + $STD /opt/airflow/.venv/bin/airflow db migrate + msg_ok "Ran Database Migrations" + + msg_info "Starting Services" + systemctl start airflow-api-server airflow-scheduler airflow-dag-processor airflow-triggerer + msg_ok "Started Services" + msg_ok "Updated successfully!" + exit +} + +start +build_container +description + +msg_ok "Completed Successfully!\n" +echo -e "${CREATING}${GN}${APP} setup has been successfully initialized!${CL}" +echo -e "${INFO}${YW} Access it using the following URL:${CL}" +echo -e "${TAB}${GATEWAY}${BGN}http://${IP}:8080${CL}" diff --git a/install/airflow-install.sh b/install/airflow-install.sh new file mode 100644 index 00000000..334e8229 --- /dev/null +++ b/install/airflow-install.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash + +# Copyright (c) 2021-2026 community-scripts ORG +# Author: MickLesk (CanbiZ) +# License: MIT | https://github.com/community-scripts/ProxmoxVED/raw/main/LICENSE +# Source: https://github.com/apache/airflow + +source /dev/stdin <<<"$FUNCTIONS_FILE_PATH" +color +verb_ip6 +catch_errors +setting_up_container +network_check +update_os + +msg_info "Installing Dependencies" +$STD apt install -y \ + build-essential \ + libpq-dev \ + libssl-dev \ + libffi-dev \ + python3-dev +msg_ok "Installed Dependencies" + +UV_PYTHON="3.14" setup_uv +PG_VERSION="16" setup_postgresql +PG_DB_NAME="airflow" PG_DB_USER="airflow" setup_postgresql_db + +msg_info "Installing Apache Airflow" +AIRFLOW_VERSION="3.2.1" +mkdir -p /opt/airflow/{dags,logs,plugins} +cd /opt/airflow +$STD uv venv --python 3.14 +$STD uv pip install --python /opt/airflow/.venv/bin/python \ + "apache-airflow[postgres,fab]==${AIRFLOW_VERSION}" \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-3.14.txt" +echo "${AIRFLOW_VERSION}" >~/.airflow +msg_ok "Installed Apache Airflow" + +msg_info "Configuring Application" +FERNET_KEY=$(/opt/airflow/.venv/bin/python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())") +SECRET_KEY=$(openssl rand -hex 32) +ADMIN_PASS=$(openssl rand -base64 12 | tr -dc 'a-zA-Z0-9' | cut -c1-12) +cat </opt/airflow/.env +AIRFLOW_HOME=/opt/airflow +AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://${PG_DB_USER}:${PG_DB_PASS}@localhost:5432/${PG_DB_NAME} +AIRFLOW__CORE__EXECUTOR=LocalExecutor +AIRFLOW__CORE__FERNET_KEY=${FERNET_KEY} +AIRFLOW__CORE__DAGS_FOLDER=/opt/airflow/dags +AIRFLOW__CORE__LOAD_EXAMPLES=false +AIRFLOW__API__AUTH_BACKENDS=airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session +AIRFLOW__WEBSERVER__SECRET_KEY=${SECRET_KEY} +AIRFLOW__WEBSERVER__BASE_URL=http://${LOCAL_IP}:8080 +AIRFLOW_ADMIN_PASSWORD=${ADMIN_PASS} +EOF +set -a && source /opt/airflow/.env && set +a +$STD /opt/airflow/.venv/bin/airflow db migrate +$STD /opt/airflow/.venv/bin/airflow users create \ + --username admin \ + --firstname Admin \ + --lastname User \ + --role Admin \ + --email admin@example.com \ + --password "${ADMIN_PASS}" +msg_ok "Configured Application" + +msg_info "Creating Services" +cat </etc/systemd/system/airflow-api-server.service +[Unit] +Description=Apache Airflow API Server +After=network.target postgresql.service + +[Service] +Type=simple +User=root +EnvironmentFile=/opt/airflow/.env +ExecStart=/opt/airflow/.venv/bin/airflow api-server --port 8080 +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOF + +cat </etc/systemd/system/airflow-scheduler.service +[Unit] +Description=Apache Airflow Scheduler +After=network.target postgresql.service + +[Service] +Type=simple +User=root +EnvironmentFile=/opt/airflow/.env +ExecStart=/opt/airflow/.venv/bin/airflow scheduler +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOF + +cat </etc/systemd/system/airflow-dag-processor.service +[Unit] +Description=Apache Airflow DAG Processor +After=network.target postgresql.service + +[Service] +Type=simple +User=root +EnvironmentFile=/opt/airflow/.env +ExecStart=/opt/airflow/.venv/bin/airflow dag-processor +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOF + +cat </etc/systemd/system/airflow-triggerer.service +[Unit] +Description=Apache Airflow Triggerer +After=network.target postgresql.service + +[Service] +Type=simple +User=root +EnvironmentFile=/opt/airflow/.env +ExecStart=/opt/airflow/.venv/bin/airflow triggerer +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOF +systemctl enable -q --now airflow-api-server airflow-scheduler airflow-dag-processor airflow-triggerer +msg_ok "Created Services" + +motd_ssh +customize +cleanup_lxc diff --git a/json/airflow.json b/json/airflow.json new file mode 100644 index 00000000..5f4920e3 --- /dev/null +++ b/json/airflow.json @@ -0,0 +1,48 @@ +{ + "name": "Apache Airflow", + "slug": "airflow", + "categories": [ + 19 + ], + "date_created": "2026-05-10", + "type": "ct", + "updateable": true, + "privileged": false, + "interface_port": 8080, + "documentation": "https://airflow.apache.org/docs/", + "website": "https://airflow.apache.org/", + "logo": "https://cdn.jsdelivr.net/gh/selfhst/icons@main/webp/apache-airflow.webp", + "description": "Apache Airflow is an open-source platform for developing, scheduling, and monitoring batch-oriented workflows. It uses Python to author DAGs (Directed Acyclic Graphs) that represent workflows, and provides a rich web UI for managing and observing pipelines.", + "install_methods": [ + { + "type": "default", + "script": "ct/airflow.sh", + "config_path": "/opt/airflow/.env", + "resources": { + "cpu": 2, + "ram": 4096, + "hdd": 16, + "os": "Debian", + "version": "13" + } + } + ], + "default_credentials": { + "username": "admin", + "password": null + }, + "notes": [ + { + "text": "The initial admin password is randomly generated and stored in /opt/airflow/.env (AIRFLOW_ADMIN_PASSWORD).", + "type": "info" + }, + { + "text": "Place your DAG files in /opt/airflow/dags/. The scheduler will pick them up automatically.", + "type": "info" + }, + { + "text": "This installs Airflow with LocalExecutor. For distributed task execution, configure CeleryExecutor manually.", + "type": "info" + } + ] +}