diff --git a/scripts/ingest.ps1 b/scripts/ingest.ps1 index 7e153d54a..b4ef2f796 100644 --- a/scripts/ingest.ps1 +++ b/scripts/ingest.ps1 @@ -3,7 +3,7 @@ param( [Parameter(Position=0)] [string]$Param1, - + [Parameter(Position=1)] [string]$Param2, @@ -16,13 +16,14 @@ param( $ProgressPreference = 'SilentlyContinue' -$INSTALL_DIR = "$env:LOCALAPPDATA\fluent-bit" -$BIN_DIR = "$INSTALL_DIR\bin" -$FLUENT_BIT_EXE = "$BIN_DIR\fluent-bit.exe" -$CONFIG_FILE = "$PSScriptRoot\fluent-bit.conf" -$PID_FILE = "$PSScriptRoot\fluent-bit.pid" -$LOG_FILE = "$PSScriptRoot\fluent-bit.log" -$ERROR_LOG_FILE = "$PSScriptRoot\fluent-bit.err.log" +$COLLECTOR_VERSION = "0.157.0" +$INSTALL_DIR = "$env:LOCALAPPDATA\parseable-otelcol" +$COLLECTOR_EXE = "$INSTALL_DIR\otelcol.exe" +$CONFIG_FILE = "$PSScriptRoot\otelcol.yaml" +$PID_FILE = "$PSScriptRoot\otelcol.pid" +$LOG_FILE = "$PSScriptRoot\otelcol.log" +$ERROR_LOG_FILE = "$PSScriptRoot\otelcol.err.log" + $SCRIPT_PATH = $PSCommandPath if ([string]::IsNullOrWhiteSpace($SCRIPT_PATH)) { $SCRIPT_PATH = $MyInvocation.MyCommand.Path @@ -32,8 +33,6 @@ if ([string]::IsNullOrWhiteSpace($SCRIPT_PATH)) { } $SCRIPT_CMD = "powershell -NoProfile -ExecutionPolicy Bypass -File '$SCRIPT_PATH'" -$SUPPORTED_ARCH = @("AMD64", "ARM64") - function Write-Info { param([string]$Message) Write-Host "[INFO] $Message" -ForegroundColor Green @@ -76,13 +75,29 @@ function Write-SetupComplete { $reset = "$([char]27)[0m" Write-Host "" Write-Host "${ok}[OK] You're all set!${reset}" - Write-Host "Host metrics are now being sent to Parseable." + Write-Host "Host metrics are now being sent to Parseable as OTLP JSON." Write-Host "Dataset: " -NoNewline Write-Host "${accent}${StreamName}${reset}" Write-Host "Return to Parseable and click Continue to verify your data." } -function Test-FluentBitRunning { +function Get-Architecture { + $arch = $env:PROCESSOR_ARCHITEW6432 + if ([string]::IsNullOrWhiteSpace($arch)) { + $arch = $env:PROCESSOR_ARCHITECTURE + } + if ($arch -eq "AMD64") { + return "AMD64" + } + if ($arch -eq "ARM64") { + return "ARM64" + } + + Write-ErrorMsg "Unsupported CPU architecture: $arch" + exit 1 +} + +function Test-CollectorRunning { if (Test-Path $PID_FILE) { $processId = Get-Content $PID_FILE @@ -103,7 +118,7 @@ function Test-FluentBitRunning { } $actualPath = [System.IO.Path]::GetFullPath($process.ExecutablePath) - $expectedPath = [System.IO.Path]::GetFullPath($FLUENT_BIT_EXE) + $expectedPath = [System.IO.Path]::GetFullPath($COLLECTOR_EXE) if ([System.StringComparer]::OrdinalIgnoreCase.Equals($actualPath, $expectedPath)) { return $true } @@ -113,44 +128,41 @@ function Test-FluentBitRunning { return $false } -function Stop-FluentBit { - if (Test-FluentBitRunning) { +function Stop-Collector { + if (Test-CollectorRunning) { $processId = Get-Content $PID_FILE - Write-Info "Stopping Fluent Bit (PID: $processId)..." - + Write-Info "Stopping OpenTelemetry Collector (PID: $processId)..." + try { Stop-Process -Id $processId -Force -ErrorAction Stop Start-Sleep -Seconds 2 - Write-Info "Fluent Bit stopped successfully" + Write-Info "OpenTelemetry Collector stopped successfully" Remove-Item $PID_FILE -ErrorAction SilentlyContinue } catch { - Write-ErrorMsg "Failed to stop Fluent Bit: $_" + Write-ErrorMsg "Failed to stop OpenTelemetry Collector: $_" exit 1 } } else { - Write-Warning "Fluent Bit is not running" + Write-Warning "OpenTelemetry Collector is not running" } } function Show-Status { - if (Test-FluentBitRunning) { + if (Test-CollectorRunning) { $processId = Get-Content $PID_FILE - Write-Info "Fluent Bit is running (PID: $processId)" + Write-Info "OpenTelemetry Collector is running (PID: $processId)" Write-Host "" - Write-Info "Process details:" Get-Process -Id $processId | Format-Table Id, ProcessName, CPU, WS, StartTime -AutoSize - Write-Host "" Write-Info "Config file: $CONFIG_FILE" Write-Info "Log file: $LOG_FILE" Write-Info "Error log file: $ERROR_LOG_FILE" - Write-Host "" Write-Info "To see logs: $SCRIPT_CMD logs" Write-Info "To stop: $SCRIPT_CMD stop" } else { - Write-Warning "Fluent Bit is not running" + Write-Warning "OpenTelemetry Collector is not running" if (Test-Path $PID_FILE) { Write-Info "Cleaning up stale PID file..." Remove-Item $PID_FILE -ErrorAction SilentlyContinue @@ -177,155 +189,140 @@ function Show-Logs { } } -function Get-Architecture { - $arch = $env:PROCESSOR_ARCHITECTURE - if ($arch -eq "AMD64") { - return "AMD64" - } - elseif ($arch -eq "ARM64") { - return "ARM64" +function Install-Collector { + $arch = Get-Architecture + $archSuffix = if ($arch -eq "ARM64") { "arm64" } else { "amd64" } + $expectedHash = if ($arch -eq "ARM64") { + "5dbbd3dd0344f759f41ab3557604d73ab720a17827375c346af6d4c2234ce776" } else { - Write-ErrorMsg "Unsupported architecture: $arch" - exit 1 + "f1468356aee226c4bf8bb846d260e3bada0121ef7da31db2e2e023f8207b7b9e" } -} -function Install-FluentBit { - - $arch = Get-Architecture - - if ($SUPPORTED_ARCH -notcontains $arch) { - Write-ErrorMsg "Unsupported CPU architecture: $arch" - exit 1 - } - - if (Test-Path $FLUENT_BIT_EXE) { - $version = & $FLUENT_BIT_EXE --version 2>$null | Select-Object -First 1 - return - } - - if (-not (Test-Path $INSTALL_DIR)) { - New-Item -ItemType Directory -Path $INSTALL_DIR -Force | Out-Null - } - if (-not (Test-Path $BIN_DIR)) { - New-Item -ItemType Directory -Path $BIN_DIR -Force | Out-Null + if (Test-Path $COLLECTOR_EXE) { + $installedVersion = & $COLLECTOR_EXE --version 2>$null | Select-Object -First 1 + if ($installedVersion -match [regex]::Escape($COLLECTOR_VERSION)) { + return + } } - + + $archiveName = "otelcol_${COLLECTOR_VERSION}_windows_${archSuffix}.tar.gz" + $downloadUrl = "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v${COLLECTOR_VERSION}/${archiveName}" + $tempDir = Join-Path $env:TEMP ("parseable-otelcol-" + [guid]::NewGuid().ToString("N")) + $archivePath = Join-Path $tempDir $archiveName + try { - $version = "3.2.2" - $archSuffix = if ($arch -eq "ARM64") { "winarm64" } else { "win64" } - $downloadUrl = "https://packages.fluentbit.io/windows/fluent-bit-$version-$archSuffix.zip" - $zipFile = "$env:TEMP\fluent-bit-$version-$archSuffix.zip" - Invoke-WebRequest -Uri $downloadUrl -OutFile $zipFile - Expand-Archive -Path $zipFile -DestinationPath $INSTALL_DIR -Force - - $extractedExe = Get-ChildItem -Path $INSTALL_DIR -Filter "fluent-bit.exe" -Recurse | Select-Object -First 1 - - if ($extractedExe) { - Copy-Item -Path $extractedExe.FullName -Destination $FLUENT_BIT_EXE -Force - - $dllPath = Split-Path $extractedExe.FullName - Get-ChildItem -Path $dllPath -Filter "*.dll" -ErrorAction SilentlyContinue | ForEach-Object { - Copy-Item -Path $_.FullName -Destination $BIN_DIR -Force - } - - $pluginsDir = Join-Path $dllPath "plugins" - if (Test-Path $pluginsDir) { - $targetPluginsDir = Join-Path $BIN_DIR "plugins" - if (-not (Test-Path $targetPluginsDir)) { - New-Item -ItemType Directory -Path $targetPluginsDir -Force | Out-Null - } - Copy-Item -Path "$pluginsDir\*" -Destination $targetPluginsDir -Recurse -Force - } + Write-Info "Installing OpenTelemetry Collector v$COLLECTOR_VERSION..." + New-Item -ItemType Directory -Path $tempDir -Force | Out-Null + Invoke-WebRequest -Uri $downloadUrl -OutFile $archivePath + + $actualHash = (Get-FileHash -Path $archivePath -Algorithm SHA256).Hash + if (-not [System.StringComparer]::OrdinalIgnoreCase.Equals($actualHash, $expectedHash)) { + throw "OpenTelemetry Collector checksum verification failed" } - else { - Write-ErrorMsg "Could not find fluent-bit.exe in the downloaded package" - exit 1 + + $tarCommand = Get-Command tar.exe -ErrorAction SilentlyContinue + if ($null -eq $tarCommand) { + throw "tar.exe is required to extract OpenTelemetry Collector" + } + + & $tarCommand.Source -xzf $archivePath -C $tempDir + if ($LASTEXITCODE -ne 0) { + throw "Failed to extract OpenTelemetry Collector archive" + } + + $extractedExe = Get-ChildItem -Path $tempDir -Filter "otelcol.exe" -Recurse | Select-Object -First 1 + if ($null -eq $extractedExe) { + throw "OpenTelemetry Collector executable not found in downloaded archive" } - - Remove-Item $zipFile -Force -ErrorAction SilentlyContinue - - $installedVersion = & $FLUENT_BIT_EXE --version 2>$null | Select-Object -First 1 + + if (Test-CollectorRunning) { + Stop-Collector + } + + if (-not (Test-Path $INSTALL_DIR)) { + New-Item -ItemType Directory -Path $INSTALL_DIR -Force | Out-Null + } + Copy-Item -Path $extractedExe.FullName -Destination $COLLECTOR_EXE -Force } catch { - Write-ErrorMsg "Failed to install Fluent Bit: $_" + Write-ErrorMsg "Failed to install OpenTelemetry Collector: $_" exit 1 } + finally { + Remove-Item $tempDir -Recurse -Force -ErrorAction SilentlyContinue + } } -function Start-FluentBit { - if (Test-FluentBitRunning) { +function Start-Collector { + if (Test-CollectorRunning) { $processId = Get-Content $PID_FILE - Write-Warning "Fluent Bit is already running (PID: $processId)" - Write-Info "Use '$SCRIPT_CMD stop' to stop it first" + Write-Warning "OpenTelemetry Collector is already running (PID: $processId)" return $false } - + if (-not (Test-Path $CONFIG_FILE)) { Write-ErrorMsg "Configuration file not found: $CONFIG_FILE" Write-ErrorMsg "Please run setup first" exit 1 } - - if (-not (Test-Path $FLUENT_BIT_EXE)) { - Write-ErrorMsg "Fluent Bit not installed. Installing..." - Install-FluentBit + + Install-Collector + + & $COLLECTOR_EXE validate --config $CONFIG_FILE *> $null + if ($LASTEXITCODE -ne 0) { + Write-ErrorMsg "OpenTelemetry Collector configuration validation failed" + exit 1 } - - $version = & $FLUENT_BIT_EXE --version 2>$null | Select-Object -First 1 - + Remove-Item $LOG_FILE, $ERROR_LOG_FILE -ErrorAction SilentlyContinue - # Start Fluent Bit process in background and capture logs - $process = Start-Process -FilePath $FLUENT_BIT_EXE ` - -ArgumentList "-c", "`"$CONFIG_FILE`"" ` - -WorkingDirectory $BIN_DIR ` + $process = Start-Process -FilePath $COLLECTOR_EXE ` + -ArgumentList "--config", "`"$CONFIG_FILE`"" ` + -WorkingDirectory $INSTALL_DIR ` -WindowStyle Hidden ` -RedirectStandardOutput $LOG_FILE ` -RedirectStandardError $ERROR_LOG_FILE ` -PassThru - - # Save PID + $process.Id | Out-File -FilePath $PID_FILE -Force - - Write-Info "Process started with PID: $($process.Id)" + Write-Info "OpenTelemetry Collector started with PID: $($process.Id)" Start-Sleep -Seconds 3 - - # Check if process is still running + $stillRunning = Get-Process -Id $process.Id -ErrorAction SilentlyContinue - if (-not $stillRunning) { - Write-ErrorMsg "Fluent Bit exited immediately" - Write-ErrorMsg "Run '$SCRIPT_CMD debug' to see error details" + Write-ErrorMsg "OpenTelemetry Collector exited immediately" + Write-ErrorMsg "Run '$SCRIPT_CMD logs' to see error details" Remove-Item $PID_FILE -ErrorAction SilentlyContinue exit 1 } - else { - Write-Info "Fluent Bit started successfully (PID: $($process.Id))" - Write-Host "" - Write-Info "To debug: $SCRIPT_CMD debug" - Write-Info "To check status: $SCRIPT_CMD status" - Write-Info "To see logs: $SCRIPT_CMD logs" - Write-Info "To stop: $SCRIPT_CMD stop" - return $true - } + + Write-Info "OpenTelemetry Collector started successfully (PID: $($process.Id))" + Write-Info "To check status: $SCRIPT_CMD status" + Write-Info "To see logs: $SCRIPT_CMD logs" + Write-Info "To stop: $SCRIPT_CMD stop" + return $true } -function Restart-FluentBit { - Stop-FluentBit +function Restart-Collector { + Stop-Collector Start-Sleep -Seconds 2 - [void](Start-FluentBit) + [void](Start-Collector) +} + +function ConvertTo-YamlSingleQuoted { + param([string]$Value) + return "'" + $Value.Replace("'", "''") + "'" } -function Setup-FluentBit { +function Setup-Collector { param( [string]$IngestorHost, [string]$StreamName, [string]$ApiKey, [string]$TenantId ) - + if ([string]::IsNullOrWhiteSpace($IngestorHost) -or [string]::IsNullOrWhiteSpace($StreamName) -or [string]::IsNullOrWhiteSpace($ApiKey)) { Write-ErrorMsg "Invalid setup parameters" exit 1 @@ -333,21 +330,29 @@ function Setup-FluentBit { Write-ParseableBanner - $tlsSetting = "On" + $ingestorScheme = "https" $defaultPort = "443" if ($IngestorHost -like "https://*") { $IngestorHost = $IngestorHost.Substring("https://".Length) - $tlsSetting = "On" + $ingestorScheme = "https" $defaultPort = "443" } elseif ($IngestorHost -like "http://*") { $IngestorHost = $IngestorHost.Substring("http://".Length) - $tlsSetting = "Off" + $ingestorScheme = "http" $defaultPort = "80" } $IngestorHost = ($IngestorHost -split '/', 2)[0] - if ($IngestorHost -match '^(.*):([0-9]+)$') { + if ($IngestorHost -match '^(\[[^]]+\])(?::([0-9]+))?$') { + $IngestorHost = $Matches[1] + $Port = if ([string]::IsNullOrWhiteSpace($Matches[2])) { $defaultPort } else { $Matches[2] } + } + elseif (($IngestorHost -split ':').Count -gt 2) { + Write-ErrorMsg "IPv6 hosts must be enclosed in brackets" + exit 1 + } + elseif ($IngestorHost -match '^(.*):([0-9]+)$') { $Port = $Matches[2] $IngestorHost = $Matches[1] } @@ -367,63 +372,122 @@ function Setup-FluentBit { exit 1 } - Install-FluentBit + Install-Collector + $endpoint = ConvertTo-YamlSingleQuoted "${ingestorScheme}://${IngestorHost}:${Port}" + $apiKeyValue = ConvertTo-YamlSingleQuoted $ApiKey + $streamNameValue = ConvertTo-YamlSingleQuoted $StreamName + $hostNameValue = ConvertTo-YamlSingleQuoted $env:COMPUTERNAME $configLines = @( - "[SERVICE]", - " flush 1", - " log_level info", + "receivers:", + " host_metrics:", + " collection_interval: 2s", + " scrapers:", + " cpu:", + " disk:", + " filesystem:", + " load:", + " memory:", + " network:", + " paging:", + " system:", "", - "[INPUT]", - " Name windows_exporter_metrics", - " Tag node_metrics", - " Scrape_interval 1", - " # Collect only essential metrics", - " metrics cpu", + "processors:", + " resource:", + " attributes:", + " - key: host.name", + " value: $hostNameValue", + " action: upsert", + " batch:", + " timeout: 1s", "", - "[OUTPUT]", - " Name opentelemetry", - " Match node_metrics", - " Host $IngestorHost", - " Port $Port", - " Metrics_uri /v1/metrics", - " Log_response_payload True", - " TLS $tlsSetting", - " Grpc Off", - " Http2 Off", - " Header X-API-Key $ApiKey" + "exporters:", + " otlp_http/parseable:", + " endpoint: $endpoint", + " encoding: json", + " compression: none", + " headers:", + " X-API-Key: $apiKeyValue", + " X-P-Stream: $streamNameValue", + " X-P-Log-Source: otel-metrics" ) if (-not [string]::IsNullOrWhiteSpace($TenantId)) { - $configLines += " Header X-P-Tenant $TenantId" + $tenantIdValue = ConvertTo-YamlSingleQuoted $TenantId + $configLines += " X-P-Tenant: $tenantIdValue" } $configLines += @( - " Header X-P-Stream $StreamName", - " Header X-P-Log-Source otel-metrics" + "", + "service:", + " telemetry:", + " metrics:", + " level: none", + " pipelines:", + " metrics:", + " receivers: [host_metrics]", + " processors: [resource, batch]", + " exporters: [otlp_http/parseable]" ) $configContent = ($configLines -join [Environment]::NewLine) + [Environment]::NewLine - - # Use UTF8 without BOM (important for Fluent Bit) $utf8NoBom = New-Object System.Text.UTF8Encoding $false - [System.IO.File]::WriteAllText($CONFIG_FILE, $configContent, $utf8NoBom) + $tempConfigFile = "$CONFIG_FILE.$([guid]::NewGuid().ToString('N')).tmp" + + try { + $tempConfigStream = [System.IO.File]::Create($tempConfigFile) + $tempConfigStream.Dispose() + + $currentIdentity = [System.Security.Principal.WindowsIdentity]::GetCurrent() + $fileRights = [System.Security.AccessControl.FileSystemRights]::Read -bor ` + [System.Security.AccessControl.FileSystemRights]::Write + $accessRule = [System.Security.AccessControl.FileSystemAccessRule]::new( + $currentIdentity.User, + $fileRights, + [System.Security.AccessControl.AccessControlType]::Allow + ) + $acl = [System.Security.AccessControl.FileSecurity]::new() + $acl.SetOwner($currentIdentity.User) + $acl.SetAccessRuleProtection($true, $false) + $acl.AddAccessRule($accessRule) + Set-Acl -Path $tempConfigFile -AclObject $acl -ErrorAction Stop + + [System.IO.File]::WriteAllText($tempConfigFile, $configContent, $utf8NoBom) + + & $COLLECTOR_EXE validate --config $tempConfigFile *> $null + if ($LASTEXITCODE -ne 0) { + Write-ErrorMsg "OpenTelemetry Collector configuration validation failed" + exit 1 + } - if (Test-FluentBitRunning) { - Write-Info "Restarting Fluent Bit to apply the updated configuration..." - Stop-FluentBit + if (Test-Path $CONFIG_FILE) { + Set-Acl -Path $CONFIG_FILE -AclObject $acl -ErrorAction Stop + [System.IO.File]::Replace($tempConfigFile, $CONFIG_FILE, $null) + } + else { + [System.IO.File]::Move($tempConfigFile, $CONFIG_FILE) + } + Set-Acl -Path $CONFIG_FILE -AclObject $acl -ErrorAction Stop + } + finally { + Remove-Item $tempConfigFile -Force -ErrorAction SilentlyContinue + } + + if (Test-CollectorRunning) { + Write-Info "Restarting OpenTelemetry Collector to apply updated configuration..." + Stop-Collector Start-Sleep -Seconds 2 } - + Write-Host "" - if (Start-FluentBit) { + if (Start-Collector) { Write-SetupComplete -StreamName $StreamName } } function Show-Help { Write-Host @" -Fluent Bit Setup and Management Script for Windows +OpenTelemetry Collector Host Metrics Setup and Management Script for Windows Usage: Setup: $SCRIPT_CMD [host[:port]] [stream] [api_key] [tenant_id] @@ -441,20 +505,16 @@ Example: "@ } -function Debug-FluentBit { +function Debug-Collector { if (-not (Test-Path $CONFIG_FILE)) { Write-ErrorMsg "Configuration file not found: $CONFIG_FILE" exit 1 } - - if (-not (Test-Path $FLUENT_BIT_EXE)) { - Write-ErrorMsg "Fluent Bit not installed" - exit 1 - } + + Install-Collector Write-Info "Config: $CONFIG_FILE" Write-Host "" - - & $FLUENT_BIT_EXE -c "$CONFIG_FILE" + & $COLLECTOR_EXE --config $CONFIG_FILE } if ([string]::IsNullOrWhiteSpace($Param1)) { @@ -464,13 +524,13 @@ if ([string]::IsNullOrWhiteSpace($Param1)) { switch ($Param1.ToLower()) { "stop" { - Stop-FluentBit + Stop-Collector } "restart" { - Restart-FluentBit + Restart-Collector } "start" { - [void](Start-FluentBit) + [void](Start-Collector) } "status" { Show-Status @@ -479,7 +539,7 @@ switch ($Param1.ToLower()) { Show-Logs } "debug" { - Debug-FluentBit + Debug-Collector } "help" { Show-Help @@ -496,6 +556,6 @@ switch ($Param1.ToLower()) { Write-ErrorMsg " Or: $SCRIPT_CMD [start|stop|restart|status|logs|debug|help]" exit 1 } - Setup-FluentBit -IngestorHost $Param1 -StreamName $Param2 -ApiKey $Param3 -TenantId $Param4 + Setup-Collector -IngestorHost $Param1 -StreamName $Param2 -ApiKey $Param3 -TenantId $Param4 } } diff --git a/scripts/ingest.sh b/scripts/ingest.sh index c47fc04ed..6ba35dc58 100755 --- a/scripts/ingest.sh +++ b/scripts/ingest.sh @@ -1,7 +1,7 @@ #!/bin/bash -# Fluent Bit Setup and Management Script -# Usage: +# OpenTelemetry Collector host-metrics setup and management script +# Usage: # Setup: ./ingest.sh [tenant_id] # Stop: ./ingest.sh stop # Restart: ./ingest.sh restart @@ -10,21 +10,21 @@ set -e -# Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' -ACCENT='\033[38;2;158;158;240m' -OK='\033[38;2;52;211;153m' +ACCENT='\033[38;2;158;158;240m' +OK='\033[38;2;52;211;153m' BOLD='\033[1m' -NC='\033[0m' # No Color +NC='\033[0m' -# File locations -PID_FILE="./fluent-bit.pid" -LOG_FILE="./fluent-bit.log" -CONFIG_FILE="./fluent-bit.conf" +COLLECTOR_VERSION="0.157.0" +COLLECTOR_DIR="./otelcol" +COLLECTOR_BIN="$COLLECTOR_DIR/otelcol" +PID_FILE="./otelcol.pid" +LOG_FILE="./otelcol.log" +CONFIG_FILE="./otelcol.yaml" -# Function to print colored output print_info() { echo -e "${GREEN}[INFO]${NC} $1" } @@ -55,55 +55,61 @@ print_setup_complete() { echo "" echo -e "${OK}${BOLD}✓ You're all set!${NC}" - echo "Host metrics are now being sent to Parseable." + echo "Host metrics are now being sent to Parseable as OTLP JSON." echo -e "Dataset: ${BOLD}${stream_name}${NC}" echo "Return to Parseable and click Continue to verify your data." } -# Function to check if Fluent Bit is running is_running() { + local process_command + local config_base + + config_base=$(basename "$CONFIG_FILE") + if [ -f "$PID_FILE" ]; then PID=$(cat "$PID_FILE") - if ps -p "$PID" > /dev/null 2>&1; then - return 0 + if [[ "$PID" =~ ^[0-9]+$ ]] && ps -p "$PID" > /dev/null 2>&1; then + process_command=$(ps -p "$PID" -o command= 2>/dev/null || true) + case "$process_command" in + *otelcol*"$config_base"*) return 0 ;; + esac fi fi return 1 } -# Function to stop Fluent Bit -stop_fluent_bit() { +stop_collector() { if is_running; then PID=$(cat "$PID_FILE") - print_info "Stopping Fluent Bit (PID: $PID)..." + print_info "Stopping OpenTelemetry Collector (PID: $PID)..." kill "$PID" - - # Wait for process to stop (max 10 seconds) - for i in {1..10}; do + + for _ in {1..10}; do if ! ps -p "$PID" > /dev/null 2>&1; then - print_info "✓ Fluent Bit stopped successfully" + print_info "✓ OpenTelemetry Collector stopped successfully" rm -f "$PID_FILE" return 0 fi sleep 1 done - - # Force kill if still running + if ps -p "$PID" > /dev/null 2>&1; then - print_warning "Force killing Fluent Bit..." + print_warning "Force killing OpenTelemetry Collector..." kill -9 "$PID" rm -f "$PID_FILE" fi else - print_warning "Fluent Bit is not running" + print_warning "OpenTelemetry Collector is not running" + if [ -f "$PID_FILE" ]; then + rm -f "$PID_FILE" + fi fi } -# Function to show status show_status() { if is_running; then PID=$(cat "$PID_FILE") - print_info "✓ Fluent Bit is running (PID: $PID)" + print_info "✓ OpenTelemetry Collector is running (PID: $PID)" print_info "" print_info "Process details:" ps -p "$PID" -o pid,ppid,user,%cpu,%mem,etime,command @@ -111,7 +117,7 @@ show_status() { print_info "Log file: $LOG_FILE" print_info "Config file: $CONFIG_FILE" else - print_warning "✗ Fluent Bit is not running" + print_warning "✗ OpenTelemetry Collector is not running" if [ -f "$PID_FILE" ]; then print_info "Cleaning up stale PID file..." rm -f "$PID_FILE" @@ -119,12 +125,11 @@ show_status() { fi } -# Function to show logs show_logs() { if [ -f "$LOG_FILE" ]; then - print_info "Showing last 50 lines of logs (Ctrl+C to exit)..." + print_info "Showing last 80 OpenTelemetry Collector log lines..." echo "" - tail -50 "$LOG_FILE" + tail -80 "$LOG_FILE" echo "" print_info "To follow logs in real-time, run:" print_info " tail -f $LOG_FILE" @@ -133,150 +138,162 @@ show_logs() { fi } -# Function to get Fluent Bit binary path -get_fluent_bit_bin() { - if [ -f /opt/fluent-bit/bin/fluent-bit ]; then - echo "/opt/fluent-bit/bin/fluent-bit" - elif [ -f /opt/homebrew/bin/fluent-bit ]; then - echo "/opt/homebrew/bin/fluent-bit" - elif [ -f /usr/local/bin/fluent-bit ]; then - echo "/usr/local/bin/fluent-bit" +install_collector() { + local collector_os + local collector_arch + local expected_hash + local archive_name + local download_url + local temp_dir + local archive_path + local actual_hash + local extracted_bin + + if [ -x "$COLLECTOR_BIN" ] && "$COLLECTOR_BIN" --version 2>/dev/null | grep -q "$COLLECTOR_VERSION"; then + return 0 + fi + + case "$(uname -s)" in + Linux) collector_os="linux" ;; + Darwin) collector_os="darwin" ;; + *) + print_error "Unsupported OS: $(uname -s)" + exit 1 + ;; + esac + + case "$(uname -m)" in + x86_64|amd64) collector_arch="amd64" ;; + arm64|aarch64) collector_arch="arm64" ;; + *) + print_error "Unsupported CPU architecture: $(uname -m)" + exit 1 + ;; + esac + + case "$collector_os/$collector_arch" in + linux/amd64) expected_hash="2937cf24892af55b143c072fddece17862239cf78280620029276493eb81beae" ;; + linux/arm64) expected_hash="59b63b99bab315509375fee76e22a9065eb9d9ba0a8995f8e985d60ca50d34ea" ;; + darwin/amd64) expected_hash="974420dce3aa9ba22b9e4e26cd68761f91e439bea441482166f19ece3fc186c3" ;; + darwin/arm64) expected_hash="1ea74db004f247948db7f5f99bc88a38a3c017cd5fb9b3a1fb62a98af0caa8c8" ;; + esac + + if ! command -v curl > /dev/null 2>&1; then + print_error "curl is required to install OpenTelemetry Collector" + exit 1 + fi + + archive_name="otelcol_${COLLECTOR_VERSION}_${collector_os}_${collector_arch}.tar.gz" + download_url="https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v${COLLECTOR_VERSION}/${archive_name}" + temp_dir=$(mktemp -d) + archive_path="$temp_dir/$archive_name" + INGEST_INSTALL_TEMP_DIR="$temp_dir" + INGEST_INSTALL_NEW_BIN="$COLLECTOR_BIN.new" + trap 'rm -rf "$INGEST_INSTALL_TEMP_DIR"; rm -f "$INGEST_INSTALL_NEW_BIN"' EXIT + + print_info "Installing OpenTelemetry Collector v$COLLECTOR_VERSION..." + if ! curl -fsSL --retry 3 --retry-delay 2 --connect-timeout 10 --max-time 300 \ + "$download_url" -o "$archive_path"; then + print_error "Failed to download OpenTelemetry Collector from $download_url" + exit 1 + fi + + if command -v sha256sum > /dev/null 2>&1; then + actual_hash=$(sha256sum "$archive_path" | awk '{print $1}') + elif command -v shasum > /dev/null 2>&1; then + actual_hash=$(shasum -a 256 "$archive_path" | awk '{print $1}') else - which fluent-bit 2>/dev/null || echo "fluent-bit" + print_error "Cannot verify download: sha256sum or shasum is required" + exit 1 fi + + if [ "$actual_hash" != "$expected_hash" ]; then + print_error "OpenTelemetry Collector checksum verification failed" + exit 1 + fi + + tar -xzf "$archive_path" -C "$temp_dir" + extracted_bin=$(find "$temp_dir" -type f -name otelcol -print -quit) + if [ -z "$extracted_bin" ]; then + print_error "OpenTelemetry Collector executable not found in downloaded archive" + exit 1 + fi + + mkdir -p "$COLLECTOR_DIR" + cp "$extracted_bin" "$COLLECTOR_BIN.new" + chmod 755 "$COLLECTOR_BIN.new" + mv "$COLLECTOR_BIN.new" "$COLLECTOR_BIN" + rm -rf "$temp_dir" + trap - EXIT + unset INGEST_INSTALL_TEMP_DIR INGEST_INSTALL_NEW_BIN } -# Function to start Fluent Bit -start_fluent_bit() { +start_collector() { if is_running; then PID=$(cat "$PID_FILE") - print_warning "Fluent Bit is already running (PID: $PID)" - print_info "Use '$0 stop' to stop it first, or '$0 restart' to restart" - exit 0 + print_warning "OpenTelemetry Collector is already running (PID: $PID)" + return 0 fi - + if [ ! -f "$CONFIG_FILE" ]; then print_error "Configuration file not found: $CONFIG_FILE" - print_error "Please run setup first with: $0 [tenant_id]" + print_error "Please run setup first" exit 1 fi - - FLUENT_BIT_BIN=$(get_fluent_bit_bin) - FINAL_VERSION=$("$FLUENT_BIT_BIN" --version 2>/dev/null | head -n1 | sed -n 's/.*v\([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' || echo "unknown") - - nohup "$FLUENT_BIT_BIN" -c "$CONFIG_FILE" > "$LOG_FILE" 2>&1 & - FLUENT_PID=$! - echo "$FLUENT_PID" > "$PID_FILE" - + + install_collector + + if ! "$COLLECTOR_BIN" validate --config "$CONFIG_FILE" > /dev/null; then + print_error "OpenTelemetry Collector configuration validation failed" + exit 1 + fi + + nohup "$COLLECTOR_BIN" --config "$CONFIG_FILE" > "$LOG_FILE" 2>&1 & + PID=$! + echo "$PID" > "$PID_FILE" + sleep 2 - if ps -p "$FLUENT_PID" > /dev/null 2>&1; then - print_info "✓ Fluent Bit started successfully (PID: $FLUENT_PID)" + if ps -p "$PID" > /dev/null 2>&1; then + print_info "✓ OpenTelemetry Collector started successfully (PID: $PID)" print_info "View logs: tail -f $LOG_FILE" - print_info "Check status: ps -p \$(cat $PID_FILE)" - print_info "Stop: kill \$(cat $PID_FILE)" + print_info "Check status: $0 status" + print_info "Stop: $0 stop" else - print_error "✗ Fluent Bit failed to start. Check logs: cat $LOG_FILE" + print_error "✗ OpenTelemetry Collector failed to start. Check logs: cat $LOG_FILE" rm -f "$PID_FILE" exit 1 fi } -# Function to restart Fluent Bit -restart_fluent_bit() { - stop_fluent_bit +restart_collector() { + stop_collector sleep 2 - start_fluent_bit + start_collector } -# Function to compare versions -version_gt() { - test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1" +yaml_escape() { + printf '%s' "$1" | sed "s/'/''/g" } -# Function to get Fluent Bit version -get_fluent_bit_version() { - local version_output - local fluent_bit_cmd - - # Try different installation locations - if [ -f /opt/fluent-bit/bin/fluent-bit ]; then - fluent_bit_cmd="/opt/fluent-bit/bin/fluent-bit" - elif [ -f /opt/homebrew/bin/fluent-bit ]; then - fluent_bit_cmd="/opt/homebrew/bin/fluent-bit" - elif [ -f /usr/local/bin/fluent-bit ]; then - fluent_bit_cmd="/usr/local/bin/fluent-bit" - else - fluent_bit_cmd="fluent-bit" - fi - - version_output=$($fluent_bit_cmd --version 2>/dev/null | head -n1) - - # Extract version using sed (portable across macOS and Linux) - echo "$version_output" | sed -n 's/.*v\([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' || echo "0.0.0" -} - -# Detect OS -detect_os() { - if [[ "$OSTYPE" == "darwin"* ]]; then - echo "macos" - elif [[ "$OSTYPE" == "linux-gnu"* ]]; then - if [ -f /etc/os-release ]; then - . /etc/os-release - echo "$ID" - else - echo "linux" - fi - else - echo "unknown" - fi -} - -# Install Fluent Bit based on OS -install_fluent_bit() { - case "$OS" in - macos) - print_info "Installing Fluent Bit on macOS using Homebrew..." - if ! command -v brew &> /dev/null; then - print_error "Homebrew is not installed. Please install Homebrew first:" - print_error '/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"' - exit 1 - fi - brew install fluent-bit - ;; - ubuntu|debian) - print_info "Installing Fluent Bit on Ubuntu/Debian..." - curl -fsSL https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh -o /tmp/install-fluentbit.sh - chmod +x /tmp/install-fluentbit.sh - /tmp/install-fluentbit.sh - ;; - - centos|rhel|fedora) - print_info "Installing Fluent Bit on CentOS/RHEL/Fedora..." - curl -fsSL https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh -o /tmp/install-fluentbit.sh - chmod +x /tmp/install-fluentbit.sh - /tmp/install-fluentbit.sh - ;; - *) - print_error "Unsupported OS: $OS" - print_info "Please install Fluent Bit manually from: https://docs.fluentbit.io/manual/installation/getting-started-with-fluent-bit" - exit 1 - ;; - esac -} - -# Setup function -setup_fluent_bit() { - local INGESTOR_HOST="$1" - local STREAM_NAME="$2" - local API_KEY="$3" - local TENANT_ID="${4:-}" - local TENANT_HEADER="" - local TLS_SETTING="On" - local DEFAULT_PORT="443" - local PORT="" - - # Validate all fields are present - if [ -z "$INGESTOR_HOST" ] || [ -z "$STREAM_NAME" ] || [ -z "$API_KEY" ]; then +setup_collector() { + local ingestor_host="$1" + local stream_name="$2" + local api_key="$3" + local tenant_id="${4:-}" + local ingestor_scheme="https" + local default_port="443" + local port + local endpoint_yaml + local api_key_yaml + local stream_name_yaml + local tenant_id_yaml + local host_name_yaml + local tenant_header="" + local scrapers + local bracketed_host_pattern='^(\[[^]]+\])(:([0-9]+))?$' + local temp_config + + if [ -z "$ingestor_host" ] || [ -z "$stream_name" ] || [ -z "$api_key" ]; then print_error "Invalid setup parameters" print_error "Expected format: $0 [tenant_id]" exit 1 @@ -284,109 +301,137 @@ setup_fluent_bit() { print_parseable_banner - if [[ "$INGESTOR_HOST" =~ ^[Hh][Tt][Tt][Pp][Ss]:// ]]; then - INGESTOR_HOST="${INGESTOR_HOST#*://}" - TLS_SETTING="On" - DEFAULT_PORT="443" - elif [[ "$INGESTOR_HOST" =~ ^[Hh][Tt][Tt][Pp]:// ]]; then - INGESTOR_HOST="${INGESTOR_HOST#*://}" - TLS_SETTING="Off" - DEFAULT_PORT="80" + if [[ "$ingestor_host" =~ ^[Hh][Tt][Tt][Pp][Ss]:// ]]; then + ingestor_host="${ingestor_host#*://}" + ingestor_scheme="https" + default_port="443" + elif [[ "$ingestor_host" =~ ^[Hh][Tt][Tt][Pp]:// ]]; then + ingestor_host="${ingestor_host#*://}" + ingestor_scheme="http" + default_port="80" fi - INGESTOR_HOST="${INGESTOR_HOST%%/*}" + ingestor_host="${ingestor_host%%/*}" - if [[ "$INGESTOR_HOST" == *:* ]]; then - PORT="${INGESTOR_HOST##*:}" - INGESTOR_HOST="${INGESTOR_HOST%:*}" + if [[ "$ingestor_host" =~ $bracketed_host_pattern ]]; then + port="${BASH_REMATCH[3]:-$default_port}" + ingestor_host="${BASH_REMATCH[1]}" + elif [[ "$ingestor_host" == *:*:* ]]; then + print_error "IPv6 hosts must be enclosed in brackets" + exit 1 + elif [[ "$ingestor_host" == *:* ]]; then + port="${ingestor_host##*:}" + ingestor_host="${ingestor_host%:*}" else - PORT="$DEFAULT_PORT" + port="$default_port" fi - if [ -z "$INGESTOR_HOST" ]; then + if [ -z "$ingestor_host" ]; then print_error "Invalid host" exit 1 fi - if ! [[ "$PORT" =~ ^[0-9]+$ ]] || [ "$PORT" -lt 1 ] || [ "$PORT" -gt 65535 ]; then - print_error "Invalid port: $PORT" + if ! [[ "$port" =~ ^[0-9]+$ ]] || [ "$port" -lt 1 ] || [ "$port" -gt 65535 ]; then + print_error "Invalid port: $port" print_error "Port must be a number between 1 and 65535" exit 1 fi - if [ -n "$TENANT_ID" ]; then - TENANT_HEADER=" Header X-P-Tenant $TENANT_ID" - fi - - OS=$(detect_os) - - # Minimum version required for node_exporter_metrics plugin - MIN_VERSION="1.9.0" - - # Check if Fluent Bit is already installed - if command -v fluent-bit &> /dev/null || [ -f /opt/homebrew/bin/fluent-bit ] || [ -f /usr/local/bin/fluent-bit ]; then - CURRENT_VERSION=$(get_fluent_bit_version) - - if version_gt "$MIN_VERSION" "$CURRENT_VERSION"; then - install_fluent_bit - # Clear command hash to get updated binary - hash -r 2>/dev/null || true - NEW_VERSION=$(get_fluent_bit_version) - fi - else - install_fluent_bit - # Clear command hash to get updated binary - hash -r 2>/dev/null || true - NEW_VERSION=$(get_fluent_bit_version) + install_collector + + endpoint_yaml=$(yaml_escape "${ingestor_scheme}://${ingestor_host}:${port}") + api_key_yaml=$(yaml_escape "$api_key") + stream_name_yaml=$(yaml_escape "$stream_name") + tenant_id_yaml=$(yaml_escape "$tenant_id") + host_name_yaml=$(yaml_escape "$(hostname)") + + if [ -n "$tenant_id" ]; then + tenant_header=" X-P-Tenant: '$tenant_id_yaml'" fi - - cat > "$CONFIG_FILE" << EOF -[SERVICE] - flush 1 - log_level info - -[INPUT] - Name node_exporter_metrics - Tag node_metrics - Scrape_interval 2 - -[OUTPUT] - Name opentelemetry - Match node_metrics - Host $INGESTOR_HOST - Port $PORT - Metrics_uri /v1/metrics - Log_response_payload True - TLS $TLS_SETTING - Header X-API-Key $API_KEY -${TENANT_HEADER} - Header X-P-Stream $STREAM_NAME - Header X-P-Log-Source otel-metrics + + scrapers=$(cat <<'EOF' + cpu: + disk: + filesystem: + load: + memory: + network: + paging: + processes: + system: +EOF +) + + temp_config=$(mktemp "${CONFIG_FILE}.tmp.XXXXXX") + INGEST_TEMP_CONFIG="$temp_config" + trap 'rm -f "$INGEST_TEMP_CONFIG"' EXIT + chmod 600 "$temp_config" + cat > "$temp_config" << EOF +receivers: + host_metrics: + collection_interval: 2s + scrapers: +$scrapers + +processors: + resource: + attributes: + - key: host.name + value: '$host_name_yaml' + action: upsert + batch: + timeout: 1s + +exporters: + otlp_http/parseable: + endpoint: '$endpoint_yaml' + encoding: json + compression: none + headers: + X-API-Key: '$api_key_yaml' + X-P-Stream: '$stream_name_yaml' + X-P-Log-Source: otel-metrics +${tenant_header} + +service: + telemetry: + metrics: + level: none + pipelines: + metrics: + receivers: [host_metrics] + processors: [resource, batch] + exporters: [otlp_http/parseable] EOF - chmod 600 "$CONFIG_FILE" - sed "s/Header X-API-Key.*/Header X-API-Key [REDACTED]/" "$CONFIG_FILE" + + if ! "$COLLECTOR_BIN" validate --config "$temp_config" > /dev/null; then + print_error "OpenTelemetry Collector configuration validation failed" + exit 1 + fi + + mv -f "$temp_config" "$CONFIG_FILE" + trap - EXIT + unset INGEST_TEMP_CONFIG if is_running; then - print_info "Restarting Fluent Bit to apply the updated configuration..." - stop_fluent_bit + print_info "Restarting OpenTelemetry Collector to apply updated configuration..." + stop_collector sleep 2 fi - - # Start Fluent Bit + echo "" - start_fluent_bit - print_setup_complete "$STREAM_NAME" + start_collector + print_setup_complete "$stream_name" } -# Main script logic case "${1:-}" in stop) - stop_fluent_bit + stop_collector ;; restart) - restart_fluent_bit + restart_collector ;; start) - start_fluent_bit + start_collector ;; status) show_status @@ -395,40 +440,29 @@ case "${1:-}" in show_logs ;; -h|--help|help) - echo "Fluent Bit Setup and Management Script" + echo "OpenTelemetry Collector Host Metrics Setup and Management Script" echo "" echo "Usage:" echo " Setup and start:" echo " $0 [tenant_id]" echo "" echo " Management commands:" - echo " $0 start - Start Fluent Bit (if config exists)" - echo " $0 stop - Stop Fluent Bit" - echo " $0 restart - Restart Fluent Bit" - echo " $0 status - Show Fluent Bit status" - echo " $0 logs - Show Fluent Bit logs" + echo " $0 start - Start OpenTelemetry Collector" + echo " $0 stop - Stop OpenTelemetry Collector" + echo " $0 restart - Restart OpenTelemetry Collector" + echo " $0 status - Show OpenTelemetry Collector status" + echo " $0 logs - Show OpenTelemetry Collector logs" echo "" echo "Example:" echo " $0 https://example.parseable.com:443 node-metrics px_api_key" echo " $0 http://localhost:8000 node-metrics px_api_key tenant-id" ;; *) - # If not a command, treat as setup parameters if [ $# -lt 3 ] || [ $# -gt 4 ]; then print_error "Usage: $0 [tenant_id]" print_error " Or: $0 [start|stop|restart|status|logs|help]" - print_error "" - print_error "Example:" - print_error " $0 https://ec9cfee0-2fd4-45eb-8209-d7cd992c4bcc-ingestor.workspace-staging.parseable.com:443 node-metrics px_api_key" - print_error " $0 http://localhost:8000 node-metrics px_api_key tenant-id" - print_error "" - print_error "Management commands:" - print_error " $0 status - Check if running" - print_error " $0 stop - Stop Fluent Bit" - print_error " $0 restart - Restart Fluent Bit" - print_error " $0 logs - View logs" exit 1 fi - setup_fluent_bit "$1" "$2" "$3" "${4:-}" + setup_collector "$1" "$2" "$3" "${4:-}" ;; esac