diff --git a/ct/clickhouse.sh b/ct/clickhouse.sh index e6f9e4e0..0fe8a296 100644 --- a/ct/clickhouse.sh +++ b/ct/clickhouse.sh @@ -31,87 +31,15 @@ function update_script() { fi setup_clickhouse - - if [[ -f /opt/clickstack/.env ]]; then - if check_for_gh_release "clickstack" "hyperdxio/hyperdx"; then - msg_info "Stopping ClickStack Services" - systemctl stop clickstack-app clickstack-api - msg_ok "Stopped ClickStack Services" - - msg_info "Backing up Data" - cp /opt/clickstack/.env /opt/clickstack.env.bak - msg_ok "Backed up Data" - - CLEAN_INSTALL=1 fetch_and_deploy_gh_release "clickstack" "hyperdxio/hyperdx" "tarball" "latest" "/opt/clickstack" - - cd /opt/clickstack - $STD corepack enable - YARN_SPEC=$(node -e "const p=require('./package.json');process.stdout.write(p.packageManager||'yarn@stable')" 2>/dev/null || echo "yarn@stable") - $STD corepack prepare "${YARN_SPEC}" --activate - - msg_info "Building HyperDX" - $STD yarn install - $STD yarn workspace @hyperdx/common-utils run build - rm -rf /opt/clickstack/packages/api/build - yarn workspace @hyperdx/api exec tsc >>"$(get_active_logfile)" 2>&1 || true - $STD yarn workspace @hyperdx/api exec tsc-alias - cp -r /opt/clickstack/packages/api/src/opamp/proto /opt/clickstack/packages/api/build/opamp/ 2>/dev/null || true - [[ -f /opt/clickstack/packages/api/build/index.js ]] || { - msg_error "HyperDX API build failed: build/index.js not found" - exit 1 - } - $STD yarn workspace @hyperdx/app run build - msg_ok "Built HyperDX" - - msg_info "Restoring Data" - cp /opt/clickstack.env.bak /opt/clickstack/.env - rm -f /opt/clickstack.env.bak - msg_ok "Restored Data" - - msg_info "Starting ClickStack Services" - systemctl start clickstack-api clickstack-app - msg_ok "Started ClickStack Services" - msg_ok "Updated successfully!" - fi - - if check_for_gh_release "otelcol" "open-telemetry/opentelemetry-collector-releases"; then - msg_info "Stopping OTel Collector" - systemctl stop clickstack-otel - msg_ok "Stopped OTel Collector" - - CLEAN_INSTALL=1 fetch_and_deploy_gh_release "otelcol" "open-telemetry/opentelemetry-collector-releases" "prebuild" "latest" "/opt/otelcol" "otelcol-contrib_*_linux_amd64.tar.gz" - - msg_info "Starting OTel Collector" - systemctl start clickstack-otel - msg_ok "Started OTel Collector" - msg_ok "Updated OTel Collector!" - fi - fi - + msg_ok "Updated successfully!" exit } -export CLICKSTACK="no" -if (whiptail --backtitle "Proxmox VE Helper Scripts" --title "CLICKSTACK" --yesno "Install ClickStack observability stack?\n\n(HyperDX UI + OTel Collector + MongoDB)\nRequires: 4 CPU, 8GB RAM, 30GB Disk" 12 58); then - export CLICKSTACK="yes" - var_cpu="4" - var_ram="8192" - var_disk="30" -fi - start build_container description msg_ok "Completed Successfully!\n" echo -e "${CREATING}${GN}${APP} setup has been successfully initialized!${CL}" -if [[ "${CLICKSTACK}" == "yes" ]]; then - echo -e "${INFO}${YW} Access HyperDX UI using the following URL:${CL}" - echo -e "${TAB}${GATEWAY}${BGN}http://${IP}:8080${CL}" - echo -e "${INFO}${YW} ClickHouse Play UI / HTTP API:${CL}" - echo -e "${TAB}${GATEWAY}${BGN}http://${IP}:8123${CL}" - echo -e "${INFO}${YW} OTel Collector (gRPC: 4317, HTTP: 4318)${CL}" -else - echo -e "${INFO}${YW} ClickHouse Play UI / HTTP API:${CL}" - echo -e "${TAB}${GATEWAY}${BGN}http://${IP}:8123${CL}" -fi +echo -e "${INFO}${YW} Access it using the following URL:${CL}" +echo -e "${TAB}${GATEWAY}${BGN}http://${IP}:8123${CL}" diff --git a/ct/oxicloud.sh b/ct/oxicloud.sh index 56cd4a03..5ffcc16b 100644 --- a/ct/oxicloud.sh +++ b/ct/oxicloud.sh @@ -9,7 +9,7 @@ source <(curl -fsSL https://raw.githubusercontent.com/community-scripts/ProxmoxV APP="OxiCloud" var_tags="${var_tags:-files;documents}" var_cpu="${var_cpu:-2}" -var_ram="${var_ram:-3072}" +var_ram="${var_ram:-4096}" var_disk="${var_disk:-20}" var_os="${var_os:-debian}" var_version="${var_version:-13}" diff --git a/install/clickhouse-install.sh b/install/clickhouse-install.sh index 7098b2dc..acef52c5 100644 --- a/install/clickhouse-install.sh +++ b/install/clickhouse-install.sh @@ -24,286 +24,6 @@ EOF systemctl restart clickhouse-server msg_ok "Configured ClickHouse" -if [[ "${CLICKSTACK:-}" == "yes" ]]; then - msg_info "Installing Dependencies" - $STD apt install -y \ - build-essential \ - python3 - msg_ok "Installed Dependencies" - - setup_mongodb - NODE_VERSION="22" setup_nodejs - - msg_info "Initializing ClickHouse Schema" - clickhouse client -n <<'EOSQL' -CREATE DATABASE IF NOT EXISTS default; - -CREATE TABLE IF NOT EXISTS default.otel_logs -( - `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TimestampTime` DateTime DEFAULT toDateTime(Timestamp), - `TraceId` String CODEC(ZSTD(1)), - `SpanId` String CODEC(ZSTD(1)), - `TraceFlags` UInt8, - `SeverityText` LowCardinality(String) CODEC(ZSTD(1)), - `SeverityNumber` UInt8, - `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `Body` String CODEC(ZSTD(1)), - `ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), - `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), - `ScopeName` String CODEC(ZSTD(1)), - `ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)), - `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 -) -ENGINE = MergeTree -PARTITION BY toDate(TimestampTime) -PRIMARY KEY (ServiceName, TimestampTime) -ORDER BY (ServiceName, TimestampTime, Timestamp) -TTL TimestampTime + toIntervalDay(30) -SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; - -CREATE TABLE IF NOT EXISTS default.otel_traces -( - `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TraceId` String CODEC(ZSTD(1)), - `SpanId` String CODEC(ZSTD(1)), - `ParentSpanId` String CODEC(ZSTD(1)), - `TraceState` String CODEC(ZSTD(1)), - `SpanName` LowCardinality(String) CODEC(ZSTD(1)), - `SpanKind` LowCardinality(String) CODEC(ZSTD(1)), - `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `ScopeName` String CODEC(ZSTD(1)), - `ScopeVersion` String CODEC(ZSTD(1)), - `SpanAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `Duration` UInt64 CODEC(ZSTD(1)), - `StatusCode` LowCardinality(String) CODEC(ZSTD(1)), - `StatusMessage` String CODEC(ZSTD(1)), - `Events.Timestamp` Array(DateTime64(9)) CODEC(ZSTD(1)), - `Events.Name` Array(LowCardinality(String)) CODEC(ZSTD(1)), - `Events.Attributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), - `Links.TraceId` Array(String) CODEC(ZSTD(1)), - `Links.SpanId` Array(String) CODEC(ZSTD(1)), - `Links.TraceState` Array(String) CODEC(ZSTD(1)), - `Links.Attributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), - INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_duration Duration TYPE minmax GRANULARITY 1, - INDEX idx_lower_span_name lower(SpanName) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 -) -ENGINE = MergeTree -PARTITION BY toDate(Timestamp) -ORDER BY (ServiceName, SpanName, toDateTime(Timestamp)) -TTL toDate(Timestamp) + toIntervalDay(30) -SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; - -CREATE TABLE IF NOT EXISTS default.hyperdx_sessions -( - `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TimestampTime` DateTime DEFAULT toDateTime(Timestamp), - `TraceId` String CODEC(ZSTD(1)), - `SpanId` String CODEC(ZSTD(1)), - `TraceFlags` UInt8, - `SeverityText` LowCardinality(String) CODEC(ZSTD(1)), - `SeverityNumber` UInt8, - `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `Body` String CODEC(ZSTD(1)), - `ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), - `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), - `ScopeName` String CODEC(ZSTD(1)), - `ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)), - `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_body Body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 -) -ENGINE = MergeTree -PARTITION BY toDate(TimestampTime) -PRIMARY KEY (ServiceName, TimestampTime) -ORDER BY (ServiceName, TimestampTime, Timestamp) -TTL TimestampTime + toIntervalDay(30) -SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; -EOSQL - msg_ok "Initialized ClickHouse Schema" - - fetch_and_deploy_gh_release "otelcol" "open-telemetry/opentelemetry-collector-releases" "prebuild" "latest" "/opt/otelcol" "otelcol-contrib_*_linux_amd64.tar.gz" - - msg_info "Configuring OTel Collector" - cat <<'EOF' >/opt/otelcol/config.yaml -receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 - fluentforward: - endpoint: 0.0.0.0:24225 - -processors: - batch: - timeout: 5s - send_batch_size: 10000 - -exporters: - clickhouse: - endpoint: tcp://127.0.0.1:9000?dial_timeout=10s - database: default - create_schema: false - logs_table_name: otel_logs - traces_table_name: otel_traces - -extensions: - health_check: - endpoint: 0.0.0.0:13133 - -service: - extensions: [health_check] - pipelines: - logs: - receivers: [otlp, fluentforward] - processors: [batch] - exporters: [clickhouse] - traces: - receivers: [otlp] - processors: [batch] - exporters: [clickhouse] - metrics: - receivers: [otlp] - processors: [batch] - exporters: [clickhouse] -EOF - msg_ok "Configured OTel Collector" - - fetch_and_deploy_gh_release "clickstack" "hyperdxio/hyperdx" "tarball" "latest" "/opt/clickstack" - - msg_info "Enabling Corepack" - cd /opt/clickstack - $STD corepack enable - YARN_SPEC=$(node -e "const p=require('./package.json');process.stdout.write(p.packageManager||'yarn@stable')" 2>/dev/null || echo "yarn@stable") - $STD corepack prepare "${YARN_SPEC}" --activate - msg_ok "Enabled Corepack" - - msg_info "Building HyperDX" - $STD yarn install - $STD yarn workspace @hyperdx/common-utils run build - rm -rf /opt/clickstack/packages/api/build - yarn workspace @hyperdx/api exec tsc >>"$(get_active_logfile)" 2>&1 || true - $STD yarn workspace @hyperdx/api exec tsc-alias - cp -r /opt/clickstack/packages/api/src/opamp/proto /opt/clickstack/packages/api/build/opamp/ 2>/dev/null || true - [[ -f /opt/clickstack/packages/api/build/index.js ]] || { - msg_error "HyperDX API build failed: build/index.js not found" - exit 1 - } - $STD yarn workspace @hyperdx/app run build - msg_ok "Built HyperDX" - - msg_info "Configuring ClickStack" - HYPERDX_API_KEY=$(openssl rand -hex 16) - - cat </opt/clickstack/.env -FRONTEND_URL=http://${LOCAL_IP}:8080 -HYPERDX_API_KEY=${HYPERDX_API_KEY} -HYPERDX_API_PORT=8000 -HYPERDX_APP_PORT=8080 -HYPERDX_APP_URL=http://${LOCAL_IP} -HYPERDX_LOG_LEVEL=info -MONGO_URI=mongodb://127.0.0.1:27017/hyperdx -SERVER_URL=http://127.0.0.1:8000 -PORT=8000 -OPAMP_PORT=4320 -OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318 -OTEL_SERVICE_NAME=hdx-oss-app -NODE_ENV=production -IS_LOCAL_APP_MODE=DANGEROUSLY_is_local_app_mode💀 -NEXT_PUBLIC_IS_LOCAL_MODE=true -DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://127.0.0.1:8123","username":"default","password":""}] -DEFAULT_SOURCES=[{"name":"Logs","kind":"log","connection":"Local ClickHouse","from":"otel_logs","timestampValueExpression":"Timestamp","defaultTableSelectExpression":"*","serviceNameExpression":"ServiceName","bodyExpression":"Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"name":"Traces","kind":"trace","connection":"Local ClickHouse","from":"otel_traces","timestampValueExpression":"Timestamp","defaultTableSelectExpression":"*","serviceNameExpression":"ServiceName","bodyExpression":"SpanName","durationExpression":"Duration / 1000000","traceIdExpression":"TraceId","spanIdExpression":"SpanId","parentSpanIdExpression":"ParentSpanId","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","logSourceId":"Logs","sessionSourceId":"Sessions"},{"name":"Sessions","kind":"session","connection":"Local ClickHouse","from":"hyperdx_sessions","timestampValueExpression":"Timestamp","defaultTableSelectExpression":"*","serviceNameExpression":"ServiceName","bodyExpression":"Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","logSourceId":"Logs","traceSourceId":"Traces"}] -EOF - msg_ok "Configured ClickStack" - - msg_info "Creating Services" - cat </etc/systemd/system/clickstack-otel.service -[Unit] -Description=ClickStack OTel Collector -After=network.target clickhouse-server.service - -[Service] -Type=simple -User=root -WorkingDirectory=/opt/otelcol -ExecStart=/opt/otelcol/otelcol-contrib --config /opt/otelcol/config.yaml -Restart=on-failure -RestartSec=5 - -[Install] -WantedBy=multi-user.target -EOF - - cat </etc/systemd/system/clickstack-api.service -[Unit] -Description=ClickStack HyperDX API -After=network.target mongod.service clickhouse-server.service - -[Service] -Type=simple -User=root -WorkingDirectory=/opt/clickstack/packages/api -EnvironmentFile=/opt/clickstack/.env -ExecStart=/usr/bin/node /opt/clickstack/packages/api/build/index.js -Restart=on-failure -RestartSec=5 - -[Install] -WantedBy=multi-user.target -EOF - - cat </etc/systemd/system/clickstack-app.service -[Unit] -Description=ClickStack HyperDX Frontend -After=network.target clickstack-api.service - -[Service] -Type=simple -User=root -WorkingDirectory=/opt/clickstack/packages/app -EnvironmentFile=/opt/clickstack/.env -Environment=PORT=8080 -Environment=HOSTNAME=0.0.0.0 -ExecStart=/usr/bin/node /opt/clickstack/node_modules/next/dist/bin/next start -Restart=on-failure -RestartSec=5 - -[Install] -WantedBy=multi-user.target -EOF - - systemctl enable -q --now clickstack-otel - systemctl enable -q --now clickstack-api - systemctl enable -q --now clickstack-app - msg_ok "Created Services" -fi - motd_ssh customize cleanup_lxc diff --git a/install/oxicloud-install.sh b/install/oxicloud-install.sh index 9a8cb616..00fc0005 100644 --- a/install/oxicloud-install.sh +++ b/install/oxicloud-install.sh @@ -18,7 +18,7 @@ $STD apt install -y \ build-essential msg_ok "Installed Dependencies" -PG_VERSION="17" PG_MODULES="pg_trgm,ltree" setup_postgresql +PG_VERSION="17" setup_postgresql PG_DB_NAME="oxicloud" PG_DB_USER="oxicloud" setup_postgresql_db fetch_and_deploy_gh_release "OxiCloud" "DioCrafts/OxiCloud" "tarball" "latest" "/opt/oxicloud" TOOLCHAIN="$(sed -n '2s/[^:]*://p' /opt/oxicloud/Dockerfile | awk -F- '{print $1}')" diff --git a/json/excalidash.json b/json/excalidash.json index f0681e28..651af494 100644 --- a/json/excalidash.json +++ b/json/excalidash.json @@ -40,6 +40,10 @@ { "text": "Create your admin account on first visit at the web UI.", "type": "info" + }, + { + "text": "The one-time admin setup code can be retrieved with: journalctl -u excalidash | grep -i 'setup\\|bootstrap\\|code'", + "type": "info" } ] } \ No newline at end of file diff --git a/json/oxicloud.json b/json/oxicloud.json index 75815dd8..db2d1692 100644 --- a/json/oxicloud.json +++ b/json/oxicloud.json @@ -22,7 +22,7 @@ "config_path": "/etc/oxicloud/.env", "resources": { "cpu": 2, - "ram": 3072, + "ram": 4096, "hdd": 20, "os": "Debian", "version": "13" @@ -33,5 +33,10 @@ "username": null, "password": null }, - "notes": [] -} + "notes": [ + { + "text": "The initial installation compiles OxiCloud from source and requires at least 4GB RAM. Insufficient memory will cause the build to be killed (OOM).", + "type": "info" + } + ] +} \ No newline at end of file diff --git a/misc/alpine-install.func b/misc/alpine-install.func index 31b34bae..09098435 100644 --- a/misc/alpine-install.func +++ b/misc/alpine-install.func @@ -39,9 +39,9 @@ trap on_interrupt INT trap on_terminate TERM error_handler() { - local exit_code="$1" - local line_number="$2" - local command="$3" + local exit_code="${1:-$?}" + local line_number="${2:-unknown}" + local command="${3:-unknown}" # Exitcode 0 = kein Fehler → ignorieren if [[ "$exit_code" -eq 0 ]]; then @@ -113,7 +113,7 @@ network_check() { msg_ok "Internet: ${ipv4_status} DNS: ${BL}${RESOLVEDIP}${CL}" fi set -e - trap 'error_handler $LINENO "$BASH_COMMAND"' ERR + trap 'error_handler $? $LINENO "$BASH_COMMAND"' ERR } # This function updates the Container OS by running apt-get update and upgrade