T-Gander/ProxmoxVEDHelperScripts
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

refactor: streamline ClickHouse and OxiCloud installation scripts; update RAM requirements

Browse Source
This commit is contained in:
MickLesk
2026-06-02 11:32:19 +02:00
parent a942dd7282
commit 25514eda40
7 changed files with 21 additions and 364 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
ct/clickhouse.sh
View File

@@ -31,87 +31,15 @@ function update_script() {
fi
setup_clickhouse
if [[ -f /opt/clickstack/.env ]]; then
if check_for_gh_release "clickstack" "hyperdxio/hyperdx"; then
msg_info "Stopping ClickStack Services"
systemctl stop clickstack-app clickstack-api
msg_ok "Stopped ClickStack Services"
msg_info "Backing up Data"
cp /opt/clickstack/.env /opt/clickstack.env.bak
msg_ok "Backed up Data"
CLEAN_INSTALL=1 fetch_and_deploy_gh_release "clickstack" "hyperdxio/hyperdx" "tarball" "latest" "/opt/clickstack"
cd /opt/clickstack
$STD corepack enable
YARN_SPEC=$(node -e "const p=require('./package.json');process.stdout.write(p.packageManager||'yarn@stable')" 2>/dev/null || echo "yarn@stable")
$STD corepack prepare "${YARN_SPEC}" --activate
msg_info "Building HyperDX"
$STD yarn install
$STD yarn workspace @hyperdx/common-utils run build
rm -rf /opt/clickstack/packages/api/build
yarn workspace @hyperdx/api exec tsc >>"$(get_active_logfile)" 2>&1 || true
$STD yarn workspace @hyperdx/api exec tsc-alias
cp -r /opt/clickstack/packages/api/src/opamp/proto /opt/clickstack/packages/api/build/opamp/ 2>/dev/null || true
[[ -f /opt/clickstack/packages/api/build/index.js ]] || {
msg_error "HyperDX API build failed: build/index.js not found"
exit 1
}
$STD yarn workspace @hyperdx/app run build
msg_ok "Built HyperDX"
msg_info "Restoring Data"
cp /opt/clickstack.env.bak /opt/clickstack/.env
rm -f /opt/clickstack.env.bak
msg_ok "Restored Data"
msg_info "Starting ClickStack Services"
systemctl start clickstack-api clickstack-app
msg_ok "Started ClickStack Services"
msg_ok "Updated successfully!"
fi
if check_for_gh_release "otelcol" "open-telemetry/opentelemetry-collector-releases"; then
msg_info "Stopping OTel Collector"
systemctl stop clickstack-otel
msg_ok "Stopped OTel Collector"
CLEAN_INSTALL=1 fetch_and_deploy_gh_release "otelcol" "open-telemetry/opentelemetry-collector-releases" "prebuild" "latest" "/opt/otelcol" "otelcol-contrib_*_linux_amd64.tar.gz"
msg_info "Starting OTel Collector"
systemctl start clickstack-otel
msg_ok "Started OTel Collector"
msg_ok "Updated OTel Collector!"
fi
fi
msg_ok "Updated successfully!"
exit
}
export CLICKSTACK="no"
if (whiptail --backtitle "Proxmox VE Helper Scripts" --title "CLICKSTACK" --yesno "Install ClickStack observability stack?\n\n(HyperDX UI + OTel Collector + MongoDB)\nRequires: 4 CPU, 8GB RAM, 30GB Disk" 12 58); then
export CLICKSTACK="yes"
var_cpu="4"
var_ram="8192"
var_disk="30"
fi
start
build_container
description
msg_ok "Completed Successfully!\n"
echo -e "${CREATING}${GN}${APP} setup has been successfully initialized!${CL}"
if [[ "${CLICKSTACK}" == "yes" ]]; then
echo -e "${INFO}${YW} Access HyperDX UI using the following URL:${CL}"
echo -e "${TAB}${GATEWAY}${BGN}http://${IP}:8080${CL}"
echo -e "${INFO}${YW} ClickHouse Play UI / HTTP API:${CL}"
echo -e "${TAB}${GATEWAY}${BGN}http://${IP}:8123${CL}"
echo -e "${INFO}${YW} OTel Collector (gRPC: 4317, HTTP: 4318)${CL}"
else
echo -e "${INFO}${YW} ClickHouse Play UI / HTTP API:${CL}"
echo -e "${TAB}${GATEWAY}${BGN}http://${IP}:8123${CL}"
fi
echo -e "${INFO}${YW} Access it using the following URL:${CL}"
echo -e "${TAB}${GATEWAY}${BGN}http://${IP}:8123${CL}"

2
ct/oxicloud.sh
View File

@@ -9,7 +9,7 @@ source <(curl -fsSL https://raw.githubusercontent.com/community-scripts/ProxmoxV
APP="OxiCloud"
var_tags="${var_tags:-files;documents}"
var_cpu="${var_cpu:-2}"
var_ram="${var_ram:-3072}"
var_ram="${var_ram:-4096}"
var_disk="${var_disk:-20}"
var_os="${var_os:-debian}"
var_version="${var_version:-13}"

280
install/clickhouse-install.sh
View File

@@ -24,286 +24,6 @@ EOF
systemctl restart clickhouse-server
msg_ok "Configured ClickHouse"
if [[ "${CLICKSTACK:-}" == "yes" ]]; then
msg_info "Installing Dependencies"
$STD apt install -y \
build-essential \
python3
msg_ok "Installed Dependencies"
setup_mongodb
NODE_VERSION="22" setup_nodejs
msg_info "Initializing ClickHouse Schema"
clickhouse client -n <<'EOSQL'
CREATE DATABASE IF NOT EXISTS default;
CREATE TABLE IF NOT EXISTS default.otel_logs
(
`Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
`TimestampTime` DateTime DEFAULT toDateTime(Timestamp),
`TraceId` String CODEC(ZSTD(1)),
`SpanId` String CODEC(ZSTD(1)),
`TraceFlags` UInt8,
`SeverityText` LowCardinality(String) CODEC(ZSTD(1)),
`SeverityNumber` UInt8,
`ServiceName` LowCardinality(String) CODEC(ZSTD(1)),
`Body` String CODEC(ZSTD(1)),
`ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)),
`ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
`ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)),
`ScopeName` String CODEC(ZSTD(1)),
`ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)),
`ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
`LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
)
ENGINE = MergeTree
PARTITION BY toDate(TimestampTime)
PRIMARY KEY (ServiceName, TimestampTime)
ORDER BY (ServiceName, TimestampTime, Timestamp)
TTL TimestampTime + toIntervalDay(30)
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
CREATE TABLE IF NOT EXISTS default.otel_traces
(
`Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
`TraceId` String CODEC(ZSTD(1)),
`SpanId` String CODEC(ZSTD(1)),
`ParentSpanId` String CODEC(ZSTD(1)),
`TraceState` String CODEC(ZSTD(1)),
`SpanName` LowCardinality(String) CODEC(ZSTD(1)),
`SpanKind` LowCardinality(String) CODEC(ZSTD(1)),
`ServiceName` LowCardinality(String) CODEC(ZSTD(1)),
`ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
`ScopeName` String CODEC(ZSTD(1)),
`ScopeVersion` String CODEC(ZSTD(1)),
`SpanAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
`Duration` UInt64 CODEC(ZSTD(1)),
`StatusCode` LowCardinality(String) CODEC(ZSTD(1)),
`StatusMessage` String CODEC(ZSTD(1)),
`Events.Timestamp` Array(DateTime64(9)) CODEC(ZSTD(1)),
`Events.Name` Array(LowCardinality(String)) CODEC(ZSTD(1)),
`Events.Attributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
`Links.TraceId` Array(String) CODEC(ZSTD(1)),
`Links.SpanId` Array(String) CODEC(ZSTD(1)),
`Links.TraceState` Array(String) CODEC(ZSTD(1)),
`Links.Attributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_duration Duration TYPE minmax GRANULARITY 1,
INDEX idx_lower_span_name lower(SpanName) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
)
ENGINE = MergeTree
PARTITION BY toDate(Timestamp)
ORDER BY (ServiceName, SpanName, toDateTime(Timestamp))
TTL toDate(Timestamp) + toIntervalDay(30)
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
CREATE TABLE IF NOT EXISTS default.hyperdx_sessions
(
`Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
`TimestampTime` DateTime DEFAULT toDateTime(Timestamp),
`TraceId` String CODEC(ZSTD(1)),
`SpanId` String CODEC(ZSTD(1)),
`TraceFlags` UInt8,
`SeverityText` LowCardinality(String) CODEC(ZSTD(1)),
`SeverityNumber` UInt8,
`ServiceName` LowCardinality(String) CODEC(ZSTD(1)),
`Body` String CODEC(ZSTD(1)),
`ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)),
`ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
`ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)),
`ScopeName` String CODEC(ZSTD(1)),
`ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)),
`ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
`LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_body Body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
)
ENGINE = MergeTree
PARTITION BY toDate(TimestampTime)
PRIMARY KEY (ServiceName, TimestampTime)
ORDER BY (ServiceName, TimestampTime, Timestamp)
TTL TimestampTime + toIntervalDay(30)
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
EOSQL
msg_ok "Initialized ClickHouse Schema"
fetch_and_deploy_gh_release "otelcol" "open-telemetry/opentelemetry-collector-releases" "prebuild" "latest" "/opt/otelcol" "otelcol-contrib_*_linux_amd64.tar.gz"
msg_info "Configuring OTel Collector"
cat <<'EOF' >/opt/otelcol/config.yaml
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
fluentforward:
endpoint: 0.0.0.0:24225
processors:
batch:
timeout: 5s
send_batch_size: 10000
exporters:
clickhouse:
endpoint: tcp://127.0.0.1:9000?dial_timeout=10s
database: default
create_schema: false
logs_table_name: otel_logs
traces_table_name: otel_traces
extensions:
health_check:
endpoint: 0.0.0.0:13133
service:
extensions: [health_check]
pipelines:
logs:
receivers: [otlp, fluentforward]
processors: [batch]
exporters: [clickhouse]
traces:
receivers: [otlp]
processors: [batch]
exporters: [clickhouse]
metrics:
receivers: [otlp]
processors: [batch]
exporters: [clickhouse]
EOF
msg_ok "Configured OTel Collector"
fetch_and_deploy_gh_release "clickstack" "hyperdxio/hyperdx" "tarball" "latest" "/opt/clickstack"
msg_info "Enabling Corepack"
cd /opt/clickstack
$STD corepack enable
YARN_SPEC=$(node -e "const p=require('./package.json');process.stdout.write(p.packageManager||'yarn@stable')" 2>/dev/null || echo "yarn@stable")
$STD corepack prepare "${YARN_SPEC}" --activate
msg_ok "Enabled Corepack"
msg_info "Building HyperDX"
$STD yarn install
$STD yarn workspace @hyperdx/common-utils run build
rm -rf /opt/clickstack/packages/api/build
yarn workspace @hyperdx/api exec tsc >>"$(get_active_logfile)" 2>&1 || true
$STD yarn workspace @hyperdx/api exec tsc-alias
cp -r /opt/clickstack/packages/api/src/opamp/proto /opt/clickstack/packages/api/build/opamp/ 2>/dev/null || true
[[ -f /opt/clickstack/packages/api/build/index.js ]] || {
msg_error "HyperDX API build failed: build/index.js not found"
exit 1
}
$STD yarn workspace @hyperdx/app run build
msg_ok "Built HyperDX"
msg_info "Configuring ClickStack"
HYPERDX_API_KEY=$(openssl rand -hex 16)
cat <<EOF >/opt/clickstack/.env
FRONTEND_URL=http://${LOCAL_IP}:8080
HYPERDX_API_KEY=${HYPERDX_API_KEY}
HYPERDX_API_PORT=8000
HYPERDX_APP_PORT=8080
HYPERDX_APP_URL=http://${LOCAL_IP}
HYPERDX_LOG_LEVEL=info
MONGO_URI=mongodb://127.0.0.1:27017/hyperdx
SERVER_URL=http://127.0.0.1:8000
PORT=8000
OPAMP_PORT=4320
OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318
OTEL_SERVICE_NAME=hdx-oss-app
NODE_ENV=production
IS_LOCAL_APP_MODE=DANGEROUSLY_is_local_app_mode💀
NEXT_PUBLIC_IS_LOCAL_MODE=true
DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://127.0.0.1:8123","username":"default","password":""}]
DEFAULT_SOURCES=[{"name":"Logs","kind":"log","connection":"Local ClickHouse","from":"otel_logs","timestampValueExpression":"Timestamp","defaultTableSelectExpression":"*","serviceNameExpression":"ServiceName","bodyExpression":"Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"name":"Traces","kind":"trace","connection":"Local ClickHouse","from":"otel_traces","timestampValueExpression":"Timestamp","defaultTableSelectExpression":"*","serviceNameExpression":"ServiceName","bodyExpression":"SpanName","durationExpression":"Duration / 1000000","traceIdExpression":"TraceId","spanIdExpression":"SpanId","parentSpanIdExpression":"ParentSpanId","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","logSourceId":"Logs","sessionSourceId":"Sessions"},{"name":"Sessions","kind":"session","connection":"Local ClickHouse","from":"hyperdx_sessions","timestampValueExpression":"Timestamp","defaultTableSelectExpression":"*","serviceNameExpression":"ServiceName","bodyExpression":"Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","logSourceId":"Logs","traceSourceId":"Traces"}]
EOF
msg_ok "Configured ClickStack"
msg_info "Creating Services"
cat <<EOF >/etc/systemd/system/clickstack-otel.service
[Unit]
Description=ClickStack OTel Collector
After=network.target clickhouse-server.service
[Service]
Type=simple
User=root
WorkingDirectory=/opt/otelcol
ExecStart=/opt/otelcol/otelcol-contrib --config /opt/otelcol/config.yaml
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
cat <<EOF >/etc/systemd/system/clickstack-api.service
[Unit]
Description=ClickStack HyperDX API
After=network.target mongod.service clickhouse-server.service
[Service]
Type=simple
User=root
WorkingDirectory=/opt/clickstack/packages/api
EnvironmentFile=/opt/clickstack/.env
ExecStart=/usr/bin/node /opt/clickstack/packages/api/build/index.js
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
cat <<EOF >/etc/systemd/system/clickstack-app.service
[Unit]
Description=ClickStack HyperDX Frontend
After=network.target clickstack-api.service
[Service]
Type=simple
User=root
WorkingDirectory=/opt/clickstack/packages/app
EnvironmentFile=/opt/clickstack/.env
Environment=PORT=8080
Environment=HOSTNAME=0.0.0.0
ExecStart=/usr/bin/node /opt/clickstack/node_modules/next/dist/bin/next start
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
systemctl enable -q --now clickstack-otel
systemctl enable -q --now clickstack-api
systemctl enable -q --now clickstack-app
msg_ok "Created Services"
fi
motd_ssh
customize
cleanup_lxc

2
install/oxicloud-install.sh
View File

@@ -18,7 +18,7 @@ $STD apt install -y \
build-essential
msg_ok "Installed Dependencies"
PG_VERSION="17" PG_MODULES="pg_trgm,ltree" setup_postgresql
PG_VERSION="17" setup_postgresql
PG_DB_NAME="oxicloud" PG_DB_USER="oxicloud" setup_postgresql_db
fetch_and_deploy_gh_release "OxiCloud" "DioCrafts/OxiCloud" "tarball" "latest" "/opt/oxicloud"
TOOLCHAIN="$(sed -n '2s/[^:]*://p' /opt/oxicloud/Dockerfile | awk -F- '{print $1}')"

4
json/excalidash.json
View File

@@ -40,6 +40,10 @@
{
"text": "Create your admin account on first visit at the web UI.",
"type": "info"
},
{
"text": "The one-time admin setup code can be retrieved with: journalctl -u excalidash | grep -i 'setup\\|bootstrap\\|code'",
"type": "info"
}
]
}

11
json/oxicloud.json
View File

@@ -22,7 +22,7 @@
"config_path": "/etc/oxicloud/.env",
"resources": {
"cpu": 2,
"ram": 3072,
"ram": 4096,
"hdd": 20,
"os": "Debian",
"version": "13"
@@ -33,5 +33,10 @@
"username": null,
"password": null
},
"notes": []
}
"notes": [
{
"text": "The initial installation compiles OxiCloud from source and requires at least 4GB RAM. Insufficient memory will cause the build to be killed (OOM).",
"type": "info"
}
]
}

8
misc/alpine-install.func
View File

@@ -39,9 +39,9 @@ trap on_interrupt INT
trap on_terminate TERM
error_handler() {
local exit_code="$1"
local line_number="$2"
local command="$3"
local exit_code="${1:-$?}"
local line_number="${2:-unknown}"
local command="${3:-unknown}"
# Exitcode 0 = kein Fehler → ignorieren
if [[ "$exit_code" -eq 0 ]]; then
@@ -113,7 +113,7 @@ network_check() {
msg_ok "Internet: ${ipv4_status} DNS: ${BL}${RESOLVEDIP}${CL}"
fi
set -e
trap 'error_handler $LINENO "$BASH_COMMAND"' ERR
trap 'error_handler $? $LINENO "$BASH_COMMAND"' ERR
}
# This function updates the Container OS by running apt-get update and upgrade