Add optional ClickStack observability stack
Introduce an optional ClickStack observability install (HyperDX UI + OTel Collector + MongoDB). Adds a Whiptail prompt to enable ClickStack and adjusts default resource suggestions when selected. Updates ct/clickhouse.sh to include observability tag, author, update logic to auto-update HyperDX and the OTel Collector, and prints appropriate access URLs. Extends install/clickhouse-install.sh to install dependencies, initialize ClickHouse schemas for logs/traces/sessions, deploy the OTel Collector with a ClickHouse exporter, clone/build HyperDX, create systemd services (clickstack-otel, clickstack-api, clickstack-app), and enable them. Updates json/clickhouse.json to include an informational note about the optional ClickStack requirements and usage.
This commit is contained in:
CanbiZ (MickLesk)
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright (c) 2021-2026 community-scripts ORG
|
||||
# Author: community-scripts ORG
|
||||
# Author: MickLesk (CanbiZ)
|
||||
# License: MIT | https://github.com/community-scripts/ProxmoxVED/raw/main/LICENSE
|
||||
# Source: https://clickhouse.com
|
||||
|
||||
@@ -24,6 +24,279 @@ EOF
|
||||
systemctl restart clickhouse-server
|
||||
msg_ok "Configured ClickHouse"
|
||||
|
||||
if [[ "$CLICKSTACK" == "yes" ]]; then
|
||||
msg_info "Installing Dependencies"
|
||||
$STD apt install -y \
|
||||
build-essential \
|
||||
git \
|
||||
python3
|
||||
msg_ok "Installed Dependencies"
|
||||
|
||||
setup_mongodb
|
||||
NODE_VERSION="22" setup_nodejs
|
||||
|
||||
msg_info "Initializing ClickHouse Schema"
|
||||
clickhouse client -n <<'EOSQL'
|
||||
CREATE DATABASE IF NOT EXISTS default;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS default.otel_logs
|
||||
(
|
||||
`Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
`TimestampTime` DateTime DEFAULT toDateTime(Timestamp),
|
||||
`TraceId` String CODEC(ZSTD(1)),
|
||||
`SpanId` String CODEC(ZSTD(1)),
|
||||
`TraceFlags` UInt8,
|
||||
`SeverityText` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
`SeverityNumber` UInt8,
|
||||
`ServiceName` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
`Body` String CODEC(ZSTD(1)),
|
||||
`ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
`ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
`ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
`ScopeName` String CODEC(ZSTD(1)),
|
||||
`ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
`ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
`LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
|
||||
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(TimestampTime)
|
||||
PRIMARY KEY (ServiceName, TimestampTime)
|
||||
ORDER BY (ServiceName, TimestampTime, Timestamp)
|
||||
TTL TimestampTime + toIntervalDay(30)
|
||||
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS default.otel_traces
|
||||
(
|
||||
`Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
`TraceId` String CODEC(ZSTD(1)),
|
||||
`SpanId` String CODEC(ZSTD(1)),
|
||||
`ParentSpanId` String CODEC(ZSTD(1)),
|
||||
`TraceState` String CODEC(ZSTD(1)),
|
||||
`SpanName` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
`SpanKind` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
`ServiceName` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
`ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
`ScopeName` String CODEC(ZSTD(1)),
|
||||
`ScopeVersion` String CODEC(ZSTD(1)),
|
||||
`SpanAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
`Duration` UInt64 CODEC(ZSTD(1)),
|
||||
`StatusCode` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
`StatusMessage` String CODEC(ZSTD(1)),
|
||||
`Events.Timestamp` Array(DateTime64(9)) CODEC(ZSTD(1)),
|
||||
`Events.Name` Array(LowCardinality(String)) CODEC(ZSTD(1)),
|
||||
`Events.Attributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
|
||||
`Links.TraceId` Array(String) CODEC(ZSTD(1)),
|
||||
`Links.SpanId` Array(String) CODEC(ZSTD(1)),
|
||||
`Links.TraceState` Array(String) CODEC(ZSTD(1)),
|
||||
`Links.Attributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
|
||||
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
|
||||
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_duration Duration TYPE minmax GRANULARITY 1,
|
||||
INDEX idx_lower_span_name lower(SpanName) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(Timestamp)
|
||||
ORDER BY (ServiceName, SpanName, toDateTime(Timestamp))
|
||||
TTL toDate(Timestamp) + toIntervalDay(30)
|
||||
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS default.hyperdx_sessions
|
||||
(
|
||||
`Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
`TimestampTime` DateTime DEFAULT toDateTime(Timestamp),
|
||||
`TraceId` String CODEC(ZSTD(1)),
|
||||
`SpanId` String CODEC(ZSTD(1)),
|
||||
`TraceFlags` UInt8,
|
||||
`SeverityText` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
`SeverityNumber` UInt8,
|
||||
`ServiceName` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
`Body` String CODEC(ZSTD(1)),
|
||||
`ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
`ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
`ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
`ScopeName` String CODEC(ZSTD(1)),
|
||||
`ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
`ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
`LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
|
||||
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_body Body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(TimestampTime)
|
||||
PRIMARY KEY (ServiceName, TimestampTime)
|
||||
ORDER BY (ServiceName, TimestampTime, Timestamp)
|
||||
TTL TimestampTime + toIntervalDay(30)
|
||||
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
|
||||
EOSQL
|
||||
msg_ok "Initialized ClickHouse Schema"
|
||||
|
||||
fetch_and_deploy_gh_release "otelcol" "open-telemetry/opentelemetry-collector-releases" "prebuild" "latest" "/opt/otelcol" "otelcol-contrib_*_linux_amd64.tar.gz"
|
||||
|
||||
msg_info "Configuring OTel Collector"
|
||||
cat <<'EOF' >/opt/otelcol/config.yaml
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
fluentforward:
|
||||
endpoint: 0.0.0.0:24225
|
||||
|
||||
processors:
|
||||
batch:
|
||||
timeout: 5s
|
||||
send_batch_size: 10000
|
||||
|
||||
exporters:
|
||||
clickhouse:
|
||||
endpoint: tcp://127.0.0.1:9000?dial_timeout=10s
|
||||
database: default
|
||||
create_schema: false
|
||||
logs_table_name: otel_logs
|
||||
traces_table_name: otel_traces
|
||||
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
|
||||
service:
|
||||
extensions: [health_check]
|
||||
pipelines:
|
||||
logs:
|
||||
receivers: [otlp, fluentforward]
|
||||
processors: [batch]
|
||||
exporters: [clickhouse]
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [batch]
|
||||
exporters: [clickhouse]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [batch]
|
||||
exporters: [clickhouse]
|
||||
EOF
|
||||
msg_ok "Configured OTel Collector"
|
||||
|
||||
msg_info "Cloning HyperDX"
|
||||
HDX_VERSION=$(curl -fsSL "https://api.github.com/repos/hyperdxio/hyperdx/tags?per_page=1" | grep -oP '"name": "hyperdx@\K[^"]+' | head -1)
|
||||
$STD git clone --depth 1 --branch "hyperdx@${HDX_VERSION}" https://github.com/hyperdxio/hyperdx.git /opt/clickstack
|
||||
echo "${HDX_VERSION}" >~/.clickstack
|
||||
msg_ok "Cloned HyperDX v${HDX_VERSION}"
|
||||
|
||||
msg_info "Enabling Corepack"
|
||||
$STD corepack enable
|
||||
cd /opt/clickstack
|
||||
$STD corepack prepare --activate
|
||||
msg_ok "Enabled Corepack"
|
||||
|
||||
msg_info "Building HyperDX"
|
||||
$STD yarn install --immutable
|
||||
$STD yarn workspace @hyperdx/common-utils run build
|
||||
$STD yarn workspace @hyperdx/api run build
|
||||
NEXT_OUTPUT_STANDALONE=true $STD yarn workspace @hyperdx/app run build
|
||||
msg_ok "Built HyperDX"
|
||||
|
||||
msg_info "Configuring ClickStack"
|
||||
HYPERDX_API_KEY=$(openssl rand -hex 16)
|
||||
|
||||
cat <<EOF >/opt/clickstack/.env
|
||||
FRONTEND_URL=http://${LOCAL_IP}:8080
|
||||
HYPERDX_API_KEY=${HYPERDX_API_KEY}
|
||||
HYPERDX_API_PORT=8000
|
||||
HYPERDX_APP_PORT=8080
|
||||
HYPERDX_APP_URL=http://${LOCAL_IP}
|
||||
HYPERDX_LOG_LEVEL=info
|
||||
MONGO_URI=mongodb://127.0.0.1:27017/hyperdx
|
||||
SERVER_URL=http://127.0.0.1:8000
|
||||
OPAMP_PORT=4320
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318
|
||||
OTEL_SERVICE_NAME=hdx-oss-app
|
||||
NODE_ENV=production
|
||||
DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://127.0.0.1:8123","username":"default","password":""}]
|
||||
EOF
|
||||
msg_ok "Configured ClickStack"
|
||||
|
||||
msg_info "Creating Services"
|
||||
cat <<EOF >/etc/systemd/system/clickstack-otel.service
|
||||
[Unit]
|
||||
Description=ClickStack OTel Collector
|
||||
After=network.target clickhouse-server.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
WorkingDirectory=/opt/otelcol
|
||||
ExecStart=/opt/otelcol/otelcol-contrib --config /opt/otelcol/config.yaml
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
cat <<EOF >/etc/systemd/system/clickstack-api.service
|
||||
[Unit]
|
||||
Description=ClickStack HyperDX API
|
||||
After=network.target mongod.service clickhouse-server.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
WorkingDirectory=/opt/clickstack/packages/api
|
||||
EnvironmentFile=/opt/clickstack/.env
|
||||
ExecStart=/usr/bin/node /opt/clickstack/packages/api/dist/index.js
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
cat <<EOF >/etc/systemd/system/clickstack-app.service
|
||||
[Unit]
|
||||
Description=ClickStack HyperDX Frontend
|
||||
After=network.target clickstack-api.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
WorkingDirectory=/opt/clickstack/packages/app
|
||||
EnvironmentFile=/opt/clickstack/.env
|
||||
Environment=PORT=8080
|
||||
Environment=HOSTNAME=0.0.0.0
|
||||
ExecStart=/usr/bin/node /opt/clickstack/packages/app/.next/standalone/server.js
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
systemctl enable -q --now clickstack-otel
|
||||
systemctl enable -q --now clickstack-api
|
||||
systemctl enable -q --now clickstack-app
|
||||
msg_ok "Created Services"
|
||||
fi
|
||||
|
||||
motd_ssh
|
||||
customize
|
||||
cleanup_lxc
|
||||
|
||||
Reference in New Issue
Block a user