diff --git a/ct/clickhouse.sh b/ct/clickhouse.sh index 1adadf5b..42013668 100644 --- a/ct/clickhouse.sh +++ b/ct/clickhouse.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash source <(curl -fsSL https://raw.githubusercontent.com/community-scripts/ProxmoxVED/main/misc/build.func) # Copyright (c) 2021-2026 community-scripts ORG -# Author: community-scripts ORG +# Author: MickLesk (CanbiZ) # License: MIT | https://github.com/community-scripts/ProxmoxVED/raw/main/LICENSE # Source: https://clickhouse.com APP="ClickHouse" -var_tags="${var_tags:-database;analytics}" +var_tags="${var_tags:-database;analytics;observability}" var_cpu="${var_cpu:-2}" var_ram="${var_ram:-4096}" var_disk="${var_disk:-10}" @@ -30,14 +30,83 @@ function update_script() { fi setup_clickhouse + + if [[ -f /opt/clickstack/.env ]]; then + CURRENT_HDX_VERSION=$(cat ~/.clickstack 2>/dev/null || echo "none") + LATEST_HDX_VERSION=$(curl -fsSL "https://api.github.com/repos/hyperdxio/hyperdx/tags?per_page=1" | grep -oP '"name": "hyperdx@\K[^"]+' | head -1) + + if [[ "$CURRENT_HDX_VERSION" != "$LATEST_HDX_VERSION" ]]; then + msg_info "Stopping ClickStack Services" + systemctl stop clickstack-app clickstack-api + msg_ok "Stopped ClickStack Services" + + msg_info "Backing up Data" + cp /opt/clickstack/.env /opt/clickstack.env.bak + msg_ok "Backed up Data" + + cd /opt/clickstack + $STD git fetch --all --tags + $STD git checkout "hyperdx@${LATEST_HDX_VERSION}" + + msg_info "Building HyperDX" + $STD yarn install --immutable + $STD yarn workspace @hyperdx/common-utils run build + $STD yarn workspace @hyperdx/api run build + NEXT_OUTPUT_STANDALONE=true $STD yarn workspace @hyperdx/app run build + msg_ok "Built HyperDX" + + msg_info "Restoring Data" + cp /opt/clickstack.env.bak /opt/clickstack/.env + rm -f /opt/clickstack.env.bak + msg_ok "Restored Data" + + msg_info "Starting ClickStack Services" + systemctl start clickstack-api clickstack-app + msg_ok "Started ClickStack Services" + + echo "${LATEST_HDX_VERSION}" >~/.clickstack + msg_ok "Updated HyperDX to v${LATEST_HDX_VERSION}" + else + msg_ok "HyperDX is already up to date (v${CURRENT_HDX_VERSION})" + fi + + if check_for_gh_release "otelcol" "open-telemetry/opentelemetry-collector-releases"; then + msg_info "Stopping OTel Collector" + systemctl stop clickstack-otel + msg_ok "Stopped OTel Collector" + + CLEAN_INSTALL=1 fetch_and_deploy_gh_release "otelcol" "open-telemetry/opentelemetry-collector-releases" "prebuild" "latest" "/opt/otelcol" "otelcol-contrib_*_linux_amd64.tar.gz" + + msg_info "Starting OTel Collector" + systemctl start clickstack-otel + msg_ok "Started OTel Collector" + msg_ok "Updated OTel Collector!" + fi + fi + exit } +if (whiptail --backtitle "Proxmox VE Helper Scripts" --title "CLICKSTACK" --yesno "Install ClickStack observability stack?\n\n(HyperDX UI + OTel Collector + MongoDB)\nRequires: 4 CPU, 8GB RAM, 30GB Disk" 12 58); then + export CLICKSTACK="yes" + var_cpu="4" + var_ram="8192" + var_disk="30" +fi + start build_container description msg_ok "Completed Successfully!\n" echo -e "${CREATING}${GN}${APP} setup has been successfully initialized!${CL}" -echo -e "${INFO}${YW} Access it using the following URL:${CL}" -echo -e "${TAB}${GATEWAY}${BGN}http://${IP}:8123${CL}" +if [[ "${CLICKSTACK}" == "yes" ]]; then + echo -e "${INFO}${YW} Access HyperDX UI using the following URL:${CL}" + echo -e "${TAB}${GATEWAY}${BGN}http://${IP}:8080${CL}" + echo -e "${INFO}${YW} ClickHouse HTTP API:${CL}" + echo -e "${TAB}${GATEWAY}${BGN}http://${IP}:8123${CL}" + echo -e "${INFO}${YW} OTel Collector (gRPC: 4317, HTTP: 4318)${CL}" +else + echo -e "${INFO}${YW} Access it using the following URL:${CL}" + echo -e "${TAB}${GATEWAY}${BGN}http://${IP}:8123${CL}" +fi diff --git a/install/clickhouse-install.sh b/install/clickhouse-install.sh index b63a8c77..b24151be 100644 --- a/install/clickhouse-install.sh +++ b/install/clickhouse-install.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Copyright (c) 2021-2026 community-scripts ORG -# Author: community-scripts ORG +# Author: MickLesk (CanbiZ) # License: MIT | https://github.com/community-scripts/ProxmoxVED/raw/main/LICENSE # Source: https://clickhouse.com @@ -24,6 +24,279 @@ EOF systemctl restart clickhouse-server msg_ok "Configured ClickHouse" +if [[ "$CLICKSTACK" == "yes" ]]; then + msg_info "Installing Dependencies" + $STD apt install -y \ + build-essential \ + git \ + python3 + msg_ok "Installed Dependencies" + + setup_mongodb + NODE_VERSION="22" setup_nodejs + + msg_info "Initializing ClickHouse Schema" + clickhouse client -n <<'EOSQL' +CREATE DATABASE IF NOT EXISTS default; + +CREATE TABLE IF NOT EXISTS default.otel_logs +( + `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `TimestampTime` DateTime DEFAULT toDateTime(Timestamp), + `TraceId` String CODEC(ZSTD(1)), + `SpanId` String CODEC(ZSTD(1)), + `TraceFlags` UInt8, + `SeverityText` LowCardinality(String) CODEC(ZSTD(1)), + `SeverityNumber` UInt8, + `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), + `Body` String CODEC(ZSTD(1)), + `ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), + `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), + `ScopeName` String CODEC(ZSTD(1)), + `ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)), + `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +PRIMARY KEY (ServiceName, TimestampTime) +ORDER BY (ServiceName, TimestampTime, Timestamp) +TTL TimestampTime + toIntervalDay(30) +SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; + +CREATE TABLE IF NOT EXISTS default.otel_traces +( + `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `TraceId` String CODEC(ZSTD(1)), + `SpanId` String CODEC(ZSTD(1)), + `ParentSpanId` String CODEC(ZSTD(1)), + `TraceState` String CODEC(ZSTD(1)), + `SpanName` LowCardinality(String) CODEC(ZSTD(1)), + `SpanKind` LowCardinality(String) CODEC(ZSTD(1)), + `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), + `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `ScopeName` String CODEC(ZSTD(1)), + `ScopeVersion` String CODEC(ZSTD(1)), + `SpanAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `Duration` UInt64 CODEC(ZSTD(1)), + `StatusCode` LowCardinality(String) CODEC(ZSTD(1)), + `StatusMessage` String CODEC(ZSTD(1)), + `Events.Timestamp` Array(DateTime64(9)) CODEC(ZSTD(1)), + `Events.Name` Array(LowCardinality(String)) CODEC(ZSTD(1)), + `Events.Attributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), + `Links.TraceId` Array(String) CODEC(ZSTD(1)), + `Links.SpanId` Array(String) CODEC(ZSTD(1)), + `Links.TraceState` Array(String) CODEC(ZSTD(1)), + `Links.Attributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_duration Duration TYPE minmax GRANULARITY 1, + INDEX idx_lower_span_name lower(SpanName) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + toIntervalDay(30) +SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; + +CREATE TABLE IF NOT EXISTS default.hyperdx_sessions +( + `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `TimestampTime` DateTime DEFAULT toDateTime(Timestamp), + `TraceId` String CODEC(ZSTD(1)), + `SpanId` String CODEC(ZSTD(1)), + `TraceFlags` UInt8, + `SeverityText` LowCardinality(String) CODEC(ZSTD(1)), + `SeverityNumber` UInt8, + `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), + `Body` String CODEC(ZSTD(1)), + `ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), + `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), + `ScopeName` String CODEC(ZSTD(1)), + `ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)), + `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_body Body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +PRIMARY KEY (ServiceName, TimestampTime) +ORDER BY (ServiceName, TimestampTime, Timestamp) +TTL TimestampTime + toIntervalDay(30) +SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; +EOSQL + msg_ok "Initialized ClickHouse Schema" + + fetch_and_deploy_gh_release "otelcol" "open-telemetry/opentelemetry-collector-releases" "prebuild" "latest" "/opt/otelcol" "otelcol-contrib_*_linux_amd64.tar.gz" + + msg_info "Configuring OTel Collector" + cat <<'EOF' >/opt/otelcol/config.yaml +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + fluentforward: + endpoint: 0.0.0.0:24225 + +processors: + batch: + timeout: 5s + send_batch_size: 10000 + +exporters: + clickhouse: + endpoint: tcp://127.0.0.1:9000?dial_timeout=10s + database: default + create_schema: false + logs_table_name: otel_logs + traces_table_name: otel_traces + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +service: + extensions: [health_check] + pipelines: + logs: + receivers: [otlp, fluentforward] + processors: [batch] + exporters: [clickhouse] + traces: + receivers: [otlp] + processors: [batch] + exporters: [clickhouse] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [clickhouse] +EOF + msg_ok "Configured OTel Collector" + + msg_info "Cloning HyperDX" + HDX_VERSION=$(curl -fsSL "https://api.github.com/repos/hyperdxio/hyperdx/tags?per_page=1" | grep -oP '"name": "hyperdx@\K[^"]+' | head -1) + $STD git clone --depth 1 --branch "hyperdx@${HDX_VERSION}" https://github.com/hyperdxio/hyperdx.git /opt/clickstack + echo "${HDX_VERSION}" >~/.clickstack + msg_ok "Cloned HyperDX v${HDX_VERSION}" + + msg_info "Enabling Corepack" + $STD corepack enable + cd /opt/clickstack + $STD corepack prepare --activate + msg_ok "Enabled Corepack" + + msg_info "Building HyperDX" + $STD yarn install --immutable + $STD yarn workspace @hyperdx/common-utils run build + $STD yarn workspace @hyperdx/api run build + NEXT_OUTPUT_STANDALONE=true $STD yarn workspace @hyperdx/app run build + msg_ok "Built HyperDX" + + msg_info "Configuring ClickStack" + HYPERDX_API_KEY=$(openssl rand -hex 16) + + cat </opt/clickstack/.env +FRONTEND_URL=http://${LOCAL_IP}:8080 +HYPERDX_API_KEY=${HYPERDX_API_KEY} +HYPERDX_API_PORT=8000 +HYPERDX_APP_PORT=8080 +HYPERDX_APP_URL=http://${LOCAL_IP} +HYPERDX_LOG_LEVEL=info +MONGO_URI=mongodb://127.0.0.1:27017/hyperdx +SERVER_URL=http://127.0.0.1:8000 +OPAMP_PORT=4320 +OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318 +OTEL_SERVICE_NAME=hdx-oss-app +NODE_ENV=production +DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://127.0.0.1:8123","username":"default","password":""}] +EOF + msg_ok "Configured ClickStack" + + msg_info "Creating Services" + cat </etc/systemd/system/clickstack-otel.service +[Unit] +Description=ClickStack OTel Collector +After=network.target clickhouse-server.service + +[Service] +Type=simple +User=root +WorkingDirectory=/opt/otelcol +ExecStart=/opt/otelcol/otelcol-contrib --config /opt/otelcol/config.yaml +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOF + + cat </etc/systemd/system/clickstack-api.service +[Unit] +Description=ClickStack HyperDX API +After=network.target mongod.service clickhouse-server.service + +[Service] +Type=simple +User=root +WorkingDirectory=/opt/clickstack/packages/api +EnvironmentFile=/opt/clickstack/.env +ExecStart=/usr/bin/node /opt/clickstack/packages/api/dist/index.js +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOF + + cat </etc/systemd/system/clickstack-app.service +[Unit] +Description=ClickStack HyperDX Frontend +After=network.target clickstack-api.service + +[Service] +Type=simple +User=root +WorkingDirectory=/opt/clickstack/packages/app +EnvironmentFile=/opt/clickstack/.env +Environment=PORT=8080 +Environment=HOSTNAME=0.0.0.0 +ExecStart=/usr/bin/node /opt/clickstack/packages/app/.next/standalone/server.js +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOF + + systemctl enable -q --now clickstack-otel + systemctl enable -q --now clickstack-api + systemctl enable -q --now clickstack-app + msg_ok "Created Services" +fi + motd_ssh customize cleanup_lxc diff --git a/json/clickhouse.json b/json/clickhouse.json index 13374201..99a4f82e 100644 --- a/json/clickhouse.json +++ b/json/clickhouse.json @@ -1,38 +1,44 @@ { - "name": "ClickHouse", - "slug": "clickhouse", - "categories": [8], - "date_created": "2026-04-14", - "type": "ct", - "updateable": true, - "privileged": false, - "interface_port": 8123, - "documentation": "https://clickhouse.com/docs/", - "website": "https://clickhouse.com/", - "logo": "https://cdn.jsdelivr.net/gh/selfhst/icons@main/webp/clickhouse.webp", - "description": "ClickHouse is an open-source, high-performance columnar database management system designed for real-time analytics and data processing using SQL queries.", - "install_methods": [ - { - "type": "default", - "script": "ct/clickhouse.sh", - "config_path": "/etc/clickhouse-server/config.xml", - "resources": { - "cpu": 2, - "ram": 4096, - "hdd": 10, - "os": "Debian", - "version": "13" - } - } - ], - "default_credentials": { - "username": "default", - "password": null - }, - "notes": [ - { - "text": "The default user 'default' has no password. Set a password for production use.", - "type": "warning" - } - ] -} + "name": "ClickHouse", + "slug": "clickhouse", + "categories": [ + 8 + ], + "date_created": "2026-04-14", + "type": "ct", + "updateable": true, + "privileged": false, + "interface_port": 8123, + "documentation": "https://clickhouse.com/docs/", + "website": "https://clickhouse.com/", + "logo": "https://cdn.jsdelivr.net/gh/selfhst/icons@main/webp/clickhouse.webp", + "description": "ClickHouse is an open-source, high-performance columnar database management system designed for real-time analytics and data processing using SQL queries.", + "install_methods": [ + { + "type": "default", + "script": "ct/clickhouse.sh", + "config_path": "/etc/clickhouse-server/config.xml", + "resources": { + "cpu": 2, + "ram": 4096, + "hdd": 10, + "os": "Debian", + "version": "13" + } + } + ], + "default_credentials": { + "username": "default", + "password": null + }, + "notes": [ + { + "text": "The default user 'default' has no password. Set a password for production use.", + "type": "warning" + }, + { + "text": "During setup you can optionally install ClickStack (HyperDX UI + OTel Collector + MongoDB) for full observability. This requires 4 CPU, 8GB RAM, and 30GB disk.", + "type": "info" + } + ] +} \ No newline at end of file