Add telemetry data service and dashboard revamp
Introduce a telemetry data microservice under misc/data: add Dockerfile, entrypoint, migration tools, README, LICENSE and a .gitignore. Increase Docker CACHE_TTL_SECONDS to 300s. Implement extensive dashboard and analytics updates in dashboard.go: add total_all_time and sample_size, return total item counts from fetchRecords (with page/limit handling and a maxRecords guard), raise top-N limits, add a minimum-installs threshold for failed-apps, and numerous UI/style/layout improvements in the embedded DashboardHTML. Minor formatting tweak to misc/api.func.
This commit is contained in:
CanbiZ (MickLesk)
106
misc/data/migration/fix-timestamps.sh
Normal file
106
misc/data/migration/fix-timestamps.sh
Normal file
@@ -0,0 +1,106 @@
|
||||
#!/bin/bash
|
||||
# Post-migration script to fix timestamps in PocketBase
|
||||
# Run this INSIDE the PocketBase container after migration completes
|
||||
#
|
||||
# Usage: ./fix-timestamps.sh
|
||||
|
||||
set -e
|
||||
|
||||
DB_PATH="/app/pb_data/data.db"
|
||||
|
||||
echo "==========================================================="
|
||||
echo " Fix Timestamps in PocketBase"
|
||||
echo "==========================================================="
|
||||
echo ""
|
||||
|
||||
# Check if sqlite3 is available
|
||||
if ! command -v sqlite3 &> /dev/null; then
|
||||
echo "sqlite3 not found. Installing..."
|
||||
apk add sqlite 2>/dev/null || apt-get update && apt-get install -y sqlite3
|
||||
fi
|
||||
|
||||
# Check if database exists
|
||||
if [ ! -f "$DB_PATH" ]; then
|
||||
echo "Database not found at $DB_PATH"
|
||||
echo "Trying alternative paths..."
|
||||
|
||||
if [ -f "/pb_data/data.db" ]; then
|
||||
DB_PATH="/pb_data/data.db"
|
||||
elif [ -f "/pb/pb_data/data.db" ]; then
|
||||
DB_PATH="/pb/pb_data/data.db"
|
||||
else
|
||||
DB_PATH=$(find / -name "data.db" 2>/dev/null | head -1)
|
||||
fi
|
||||
|
||||
if [ -z "$DB_PATH" ] || [ ! -f "$DB_PATH" ]; then
|
||||
echo "Could not find PocketBase database!"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Database: $DB_PATH"
|
||||
echo ""
|
||||
|
||||
# List tables
|
||||
echo "Tables in database:"
|
||||
sqlite3 "$DB_PATH" ".tables"
|
||||
echo ""
|
||||
|
||||
# Find the telemetry table (usually matches collection name)
|
||||
echo "Looking for telemetry/installations table..."
|
||||
TABLE_NAME=$(sqlite3 "$DB_PATH" ".tables" | tr ' ' '\n' | grep -E "telemetry|installations" | head -1)
|
||||
|
||||
if [ -z "$TABLE_NAME" ]; then
|
||||
echo "Could not auto-detect table. Available tables:"
|
||||
sqlite3 "$DB_PATH" ".tables"
|
||||
echo ""
|
||||
read -p "Enter table name: " TABLE_NAME
|
||||
fi
|
||||
|
||||
echo "Using table: $TABLE_NAME"
|
||||
echo ""
|
||||
|
||||
# Check if old_created column exists
|
||||
HAS_OLD_CREATED=$(sqlite3 "$DB_PATH" "PRAGMA table_info($TABLE_NAME);" | grep -c "old_created" || echo "0")
|
||||
|
||||
if [ "$HAS_OLD_CREATED" -eq "0" ]; then
|
||||
echo "Column 'old_created' not found in table $TABLE_NAME"
|
||||
echo "Migration may not have been run with timestamp preservation."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Show sample data before update
|
||||
echo "Sample data BEFORE update:"
|
||||
sqlite3 "$DB_PATH" "SELECT id, created, old_created FROM $TABLE_NAME WHERE old_created IS NOT NULL AND old_created != '' LIMIT 3;"
|
||||
echo ""
|
||||
|
||||
# Count records to update
|
||||
COUNT=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM $TABLE_NAME WHERE old_created IS NOT NULL AND old_created != '';")
|
||||
echo "Records to update: $COUNT"
|
||||
echo ""
|
||||
|
||||
read -p "Proceed with timestamp update? [y/N] " -n 1 -r
|
||||
echo ""
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
echo "Aborted."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Perform the update
|
||||
echo "Updating timestamps..."
|
||||
sqlite3 "$DB_PATH" "UPDATE $TABLE_NAME SET created = old_created, updated = old_created WHERE old_created IS NOT NULL AND old_created != '';"
|
||||
|
||||
# Show sample data after update
|
||||
echo ""
|
||||
echo "Sample data AFTER update:"
|
||||
sqlite3 "$DB_PATH" "SELECT id, created, old_created FROM $TABLE_NAME LIMIT 3;"
|
||||
echo ""
|
||||
|
||||
echo "==========================================================="
|
||||
echo " Timestamp Update Complete!"
|
||||
echo "==========================================================="
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo "1. Verify data in PocketBase Admin UI"
|
||||
echo "2. Remove the 'old_created' field from the collection schema"
|
||||
echo ""
|
||||
77
misc/data/migration/import-direct.sh
Normal file
77
misc/data/migration/import-direct.sh
Normal file
@@ -0,0 +1,77 @@
|
||||
#!/bin/sh
|
||||
# Direct SQLite Import - Pure Shell, FAST batch mode!
|
||||
# Imports MongoDB Extended JSON directly into PocketBase SQLite
|
||||
#
|
||||
# Usage:
|
||||
# docker cp import-direct.sh pocketbase:/tmp/
|
||||
# docker cp data.json pocketbase:/tmp/
|
||||
# docker exec -it pocketbase sh -c "cd /tmp && chmod +x import-direct.sh && ./import-direct.sh"
|
||||
|
||||
set -e
|
||||
|
||||
JSON_FILE="${1:-/tmp/data.json}"
|
||||
TABLE="${2:-telemetry}"
|
||||
REPO="${3:-Proxmox VE}"
|
||||
DB="${4:-/app/pb_data/data.db}"
|
||||
BATCH=5000
|
||||
|
||||
echo "========================================================="
|
||||
echo " Direct SQLite Import (Batch Mode)"
|
||||
echo "========================================================="
|
||||
echo "JSON: $JSON_FILE"
|
||||
echo "Table: $TABLE"
|
||||
echo "Repo: $REPO"
|
||||
echo "Batch: $BATCH"
|
||||
echo "---------------------------------------------------------"
|
||||
|
||||
# Install jq if missing
|
||||
command -v jq >/dev/null || apk add --no-cache jq
|
||||
|
||||
# Optimize SQLite for bulk
|
||||
sqlite3 "$DB" "PRAGMA journal_mode=WAL; PRAGMA synchronous=OFF; PRAGMA cache_size=100000;"
|
||||
|
||||
SQL_FILE="/tmp/batch.sql"
|
||||
echo "[INFO] Converting JSON to SQL..."
|
||||
START=$(date +%s)
|
||||
|
||||
# Convert entire JSON to SQL file (much faster than line-by-line sqlite3 calls)
|
||||
{
|
||||
echo "BEGIN TRANSACTION;"
|
||||
jq -r '.[] | @json' "$JSON_FILE" | while read -r r; do
|
||||
CT=$(echo "$r" | jq -r 'if .ct_type|type=="object" then .ct_type["$numberLong"] else .ct_type end // 0')
|
||||
DISK=$(echo "$r" | jq -r 'if .disk_size|type=="object" then .disk_size["$numberLong"] else .disk_size end // 0')
|
||||
CORE=$(echo "$r" | jq -r 'if .core_count|type=="object" then .core_count["$numberLong"] else .core_count end // 0')
|
||||
RAM=$(echo "$r" | jq -r 'if .ram_size|type=="object" then .ram_size["$numberLong"] else .ram_size end // 0')
|
||||
OS=$(echo "$r" | jq -r '.os_type // ""' | sed "s/'/''/g")
|
||||
OSVER=$(echo "$r" | jq -r '.os_version // ""' | sed "s/'/''/g")
|
||||
DIS6=$(echo "$r" | jq -r '.disable_ip6 // "no"' | sed "s/'/''/g")
|
||||
APP=$(echo "$r" | jq -r '.nsapp // "unknown"' | sed "s/'/''/g")
|
||||
METH=$(echo "$r" | jq -r '.method // ""' | sed "s/'/''/g")
|
||||
PVE=$(echo "$r" | jq -r '.pveversion // ""' | sed "s/'/''/g")
|
||||
STAT=$(echo "$r" | jq -r '.status // "unknown"')
|
||||
[ "$STAT" = "done" ] && STAT="success"
|
||||
RID=$(echo "$r" | jq -r '.random_id // ""' | sed "s/'/''/g")
|
||||
TYPE=$(echo "$r" | jq -r '.type // "lxc"' | sed "s/'/''/g")
|
||||
ERR=$(echo "$r" | jq -r '.error // ""' | sed "s/'/''/g")
|
||||
DATE=$(echo "$r" | jq -r 'if .created_at|type=="object" then .created_at["$date"] else .created_at end // ""')
|
||||
ID=$(head -c 100 /dev/urandom | tr -dc 'a-z0-9' | head -c 15)
|
||||
REPO_ESC=$(echo "$REPO" | sed "s/'/''/g")
|
||||
|
||||
echo "INSERT OR IGNORE INTO $TABLE (id,created,updated,ct_type,disk_size,core_count,ram_size,os_type,os_version,disableip6,nsapp,method,pve_version,status,random_id,type,error,repo_source) VALUES ('$ID','$DATE','$DATE',$CT,$DISK,$CORE,$RAM,'$OS','$OSVER','$DIS6','$APP','$METH','$PVE','$STAT','$RID','$TYPE','$ERR','$REPO_ESC');"
|
||||
done
|
||||
echo "COMMIT;"
|
||||
} > "$SQL_FILE"
|
||||
|
||||
MID=$(date +%s)
|
||||
echo "[INFO] SQL generated in $((MID - START))s"
|
||||
echo "[INFO] Importing into SQLite..."
|
||||
|
||||
sqlite3 "$DB" < "$SQL_FILE"
|
||||
|
||||
END=$(date +%s)
|
||||
COUNT=$(wc -l < "$SQL_FILE")
|
||||
rm -f "$SQL_FILE"
|
||||
|
||||
echo "========================================================="
|
||||
echo "Done! ~$((COUNT - 2)) records in $((END - START)) seconds"
|
||||
echo "========================================================="
|
||||
89
misc/data/migration/migrate-linux.sh
Normal file
89
misc/data/migration/migrate-linux.sh
Normal file
@@ -0,0 +1,89 @@
|
||||
#!/bin/bash
|
||||
# Migration script for Proxmox VE data
|
||||
# Run directly on the server machine
|
||||
#
|
||||
# Usage: ./migrate-linux.sh
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Go installed (apt install golang-go)
|
||||
# - Network access to source API and PocketBase
|
||||
|
||||
set -e
|
||||
|
||||
echo "==========================================================="
|
||||
echo " Proxmox VE Data Migration to PocketBase"
|
||||
echo "==========================================================="
|
||||
|
||||
# Configuration - EDIT THESE VALUES
|
||||
export MIGRATION_SOURCE_URL="https://api.htl-braunau.at/data"
|
||||
export POCKETBASE_URL="http://db.community-scripts.org"
|
||||
export POCKETBASE_COLLECTION="telemetry"
|
||||
export PB_AUTH_COLLECTION="_superusers"
|
||||
export PB_IDENTITY="db_admin@community-scripts.org"
|
||||
export PB_PASSWORD="YOUR_PASSWORD_HERE" # <-- CHANGE THIS!
|
||||
export REPO_SOURCE="Proxmox VE"
|
||||
export DATE_UNTIL="2026-02-10"
|
||||
export BATCH_SIZE="500"
|
||||
|
||||
# Optional: Resume from specific page
|
||||
# export START_PAGE="100"
|
||||
|
||||
# Optional: Only import records after this date
|
||||
# export DATE_FROM="2020-01-01"
|
||||
|
||||
echo ""
|
||||
echo "Configuration:"
|
||||
echo " Source: $MIGRATION_SOURCE_URL"
|
||||
echo " Target: $POCKETBASE_URL"
|
||||
echo " Collection: $POCKETBASE_COLLECTION"
|
||||
echo " Repo: $REPO_SOURCE"
|
||||
echo " Until: $DATE_UNTIL"
|
||||
echo " Batch: $BATCH_SIZE"
|
||||
echo ""
|
||||
|
||||
# Check if Go is installed
|
||||
if ! command -v go &> /dev/null; then
|
||||
echo "Go is not installed. Installing..."
|
||||
apt-get update && apt-get install -y golang-go
|
||||
fi
|
||||
|
||||
# Download migrate.go if not present
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
MIGRATE_GO="$SCRIPT_DIR/migrate.go"
|
||||
|
||||
if [ ! -f "$MIGRATE_GO" ]; then
|
||||
echo "migrate.go not found in $SCRIPT_DIR"
|
||||
echo "Please copy migrate.go to this directory first."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Building migration tool..."
|
||||
cd "$SCRIPT_DIR"
|
||||
go build -o migrate migrate.go
|
||||
|
||||
echo ""
|
||||
echo "Starting migration..."
|
||||
echo "Press Ctrl+C to stop (you can resume later with START_PAGE)"
|
||||
echo ""
|
||||
|
||||
./migrate
|
||||
|
||||
echo ""
|
||||
echo "==========================================================="
|
||||
echo " Post-Migration Steps"
|
||||
echo "==========================================================="
|
||||
echo ""
|
||||
echo "1. Connect to PocketBase container:"
|
||||
echo " docker exec -it <pocketbase-container> sh"
|
||||
echo ""
|
||||
echo "2. Find the table name:"
|
||||
echo " sqlite3 /app/pb_data/data.db '.tables'"
|
||||
echo ""
|
||||
echo "3. Update timestamps (replace <table> with actual name):"
|
||||
echo " sqlite3 /app/pb_data/data.db \"UPDATE <table> SET created = old_created, updated = old_created WHERE old_created IS NOT NULL AND old_created != ''\""
|
||||
echo ""
|
||||
echo "4. Verify timestamps:"
|
||||
echo " sqlite3 /app/pb_data/data.db \"SELECT created, old_created FROM <table> LIMIT 5\""
|
||||
echo ""
|
||||
echo "5. Remove old_created field in PocketBase Admin UI"
|
||||
echo ""
|
||||
1295
misc/data/migration/migrate.go
Normal file
1295
misc/data/migration/migrate.go
Normal file
File diff suppressed because it is too large
Load Diff
@@ -13,7 +13,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# Default values
|
||||
POCKETBASE_URL="${1:-http://localhost:8090}"
|
||||
POCKETBASE_COLLECTION="${2:-_telemetry_data}"
|
||||
POCKETBASE_COLLECTION="${2:-telemetry}"
|
||||
|
||||
echo "============================================="
|
||||
echo " ProxmoxVED Data Migration Tool"
|
||||
|
||||
@@ -95,13 +95,13 @@ func main() {
|
||||
pbCollection = os.Getenv("PB_TARGET_COLLECTION")
|
||||
}
|
||||
if pbCollection == "" {
|
||||
pbCollection = "_telemetry_data"
|
||||
pbCollection = "telemetry"
|
||||
}
|
||||
|
||||
// Auth collection
|
||||
authCollection := os.Getenv("PB_AUTH_COLLECTION")
|
||||
if authCollection == "" {
|
||||
authCollection = "_telemetry_service"
|
||||
authCollection = "telemetry_service_user"
|
||||
}
|
||||
|
||||
// Credentials - prefer admin auth for timestamp preservation
|
||||
|
||||
Reference in New Issue
Block a user