From d1475ef7e01aa8ca275ca8cbfc3dba0dc446a2f0 Mon Sep 17 00:00:00 2001 From: John Essien Date: Tue, 22 Sep 2026 08:10:38 +0100 Subject: [PATCH] feat: add idle Azure VPN Gateway detection --- cloudcost.idle-vpn-gateway-real.yml | 35 ++++ policies/idle_vpn_gateway.sql | 19 ++ src/cloudcost/engine/runner.py | 1 + .../sources/azure/idle_vpn_gateway.py | 187 ++++++++++++++++++ 4 files changed, 242 insertions(+) create mode 100644 cloudcost.idle-vpn-gateway-real.yml create mode 100644 policies/idle_vpn_gateway.sql create mode 100644 src/cloudcost/sources/azure/idle_vpn_gateway.py diff --git a/cloudcost.idle-vpn-gateway-real.yml b/cloudcost.idle-vpn-gateway-real.yml new file mode 100644 index 0000000..ccabebf --- /dev/null +++ b/cloudcost.idle-vpn-gateway-real.yml @@ -0,0 +1,35 @@ +version: "1" + +pipeline: + name: idle-vpn-gateway-real-test + mode: incremental + timezone: UTC + +sources: + - name: vpn_gateway_metrics + type: azure.idle_vpn_gateway + config: + resource_group: cost-compare-3day-rg + lookback_days: 1 + +destinations: + - name: local_duckdb + type: duckdb + config: + path: ./data/idle-vpn-gateway-real.duckdb + +loads: + - input: vpn_gateway_metrics + destination: local_duckdb + table: fact_idle_vpn_gateway + mode: overwrite + +observability: + log_level: info + emit_run_summary: true + +policies: + - name: idle-vpn-gateway + type: sql + query_file: policies/idle_vpn_gateway.sql + severity: medium diff --git a/policies/idle_vpn_gateway.sql b/policies/idle_vpn_gateway.sql new file mode 100644 index 0000000..5c30987 --- /dev/null +++ b/policies/idle_vpn_gateway.sql @@ -0,0 +1,19 @@ +-- Real check: Azure VPN Gateways with zero average bandwidth over +-- the lookback window. billed_cost uses the live hourly VPN Gateway +-- price fetched by the source plugin from the Azure Retail Prices API +-- (serviceName = 'VPN Gateway', priceType = 'Consumption', exact +-- gateway SKU and region), multiplied by 730 hours/month. + +SELECT + 'azure' AS provider, + resource_id, + 'Azure VPN Gateway' AS service_name, + ROUND(hourly_price * 730, 2) AS billed_cost, + resource_name, + sku, + avg_bandwidth_bps, + lookback_days, + 'zero average bandwidth over ' || lookback_days || ' days' + AS evidence_reason +FROM fact_idle_vpn_gateway +WHERE avg_bandwidth_bps = 0; diff --git a/src/cloudcost/engine/runner.py b/src/cloudcost/engine/runner.py index 91f7b3a..d1dd9c9 100644 --- a/src/cloudcost/engine/runner.py +++ b/src/cloudcost/engine/runner.py @@ -30,6 +30,7 @@ import cloudcost.sources.azure.idle_private_endpoints import cloudcost.sources.azure.cosmosdb_idle_ru import cloudcost.sources.azure.idle_firewall + import cloudcost.sources.azure.idle_vpn_gateway import cloudcost.sources.azure.log_analytics_idle_commitment import cloudcost.sources.azure.idle_container_instances import cloudcost.sources.azure.idle_eventhub diff --git a/src/cloudcost/sources/azure/idle_vpn_gateway.py b/src/cloudcost/sources/azure/idle_vpn_gateway.py new file mode 100644 index 0000000..bcc8a99 --- /dev/null +++ b/src/cloudcost/sources/azure/idle_vpn_gateway.py @@ -0,0 +1,187 @@ +import json +import shutil +import subprocess +from datetime import datetime, timedelta, timezone +from typing import Any + +import pyarrow as pa + +from cloudcost.core.registry import registry + +AZ_CLI = shutil.which("az.cmd") or shutil.which("az") or "az" + +def _normalize_region(location: str) -> str: + """Normalize Azure CLI display region for the Retail Prices API.""" + return location.lower().replace(" ", "") + + +def _fetch_hourly_price(sku_name: str, region: str) -> float: + """Fetch the live hourly VPN Gateway deployment price.""" + region = _normalize_region(region) + + filter_str = ( + f"armRegionName eq '{region}' " + f"and serviceName eq 'VPN Gateway' " + f"and skuName eq '{sku_name}' " + f"and meterName eq '{sku_name}' " + f"and priceType eq 'Consumption'" + ) + + try: + raw = subprocess.run( + [ + "curl", + "-s", + "-G", + "https://prices.azure.com/api/retail/prices", + "--data-urlencode", + f"$filter={filter_str}", + ], + capture_output=True, + text=True, + check=True, + timeout=15, + ).stdout + + data = json.loads(raw) + + items = [ + item + for item in data.get("Items", []) + if ( + item.get("unitOfMeasure") == "1 Hour" + and item.get("meterName") == sku_name + ) + ] + + return float(items[0]["retailPrice"]) if items else 0.0 + + except Exception: + return 0.0 + + +# Real Azure VPN Gateway metric: +# "AverageBandwidth" with Average aggregation and PT1H interval. +# Verified against a live Azure VPN Gateway during real-world testing. +@registry.register_source("azure.idle_vpn_gateway") +class AzureIdleVpnGatewaySource: + def __init__(self, config: dict): + self.resource_group = config.get("resource_group") + self.lookback_days = config.get("lookback_days", 7) + + if not self.resource_group: + raise ValueError( + "azure.idle_vpn_gateway requires 'resource_group' in config" + ) + + def extract(self, context: Any = None) -> pa.Table: + gateways_raw = subprocess.run( + [ + AZ_CLI, + "network", + "vnet-gateway", + "list", + "--resource-group", + self.resource_group, + "--query", + "[?gatewayType=='Vpn'].{id:id,name:name,sku:sku.name,location:location}", + "-o", + "json", + ], + capture_output=True, + text=True, + check=True, + ).stdout + + gateways = json.loads(gateways_raw) + + end_time = datetime.now(timezone.utc) + start_time = end_time - timedelta(days=self.lookback_days) + + rows = [] + + for gateway in gateways: + raw = subprocess.run( + [ + AZ_CLI, + "monitor", + "metrics", + "list", + "--resource", + gateway["id"], + "--metric", + "AverageBandwidth", + "--aggregation", + "Average", + "--interval", + "PT1H", + "--start-time", + start_time.strftime("%Y-%m-%dT%H:%M:%SZ"), + "--end-time", + end_time.strftime("%Y-%m-%dT%H:%M:%SZ"), + ], + capture_output=True, + text=True, + check=True, + ).stdout + + parsed = json.loads(raw) + + bandwidth_values = [] + + for timeseries in parsed.get("value", []): + for series in timeseries.get("timeseries", []): + for point in series.get("data", []): + average = point.get("average") + + if average is not None: + bandwidth_values.append(float(average)) + + avg_bandwidth = ( + sum(bandwidth_values) / len(bandwidth_values) + if bandwidth_values + else 0.0 + ) + + sku_name = gateway.get("sku", "unknown") + location = gateway.get("location", "eastus") + + rows.append( + { + "resource_id": gateway["id"].lower(), + "resource_name": gateway["name"], + "sku": sku_name, + "location": _normalize_region(location), + "avg_bandwidth_bps": avg_bandwidth, + "hourly_price": _fetch_hourly_price( + sku_name, + location, + ), + "lookback_days": self.lookback_days, + } + ) + + if not rows: + return pa.table( + { + "resource_id": pa.array([], type=pa.string()), + "resource_name": pa.array([], type=pa.string()), + "sku": pa.array([], type=pa.string()), + "location": pa.array([], type=pa.string()), + "avg_bandwidth_bps": pa.array([], type=pa.float64()), + "hourly_price": pa.array([], type=pa.float64()), + "lookback_days": pa.array([], type=pa.int64()), + } + ) + + return pa.table( + { + "resource_id": [r["resource_id"] for r in rows], + "resource_name": [r["resource_name"] for r in rows], + "sku": [r["sku"] for r in rows], + "location": [r["location"] for r in rows], + "avg_bandwidth_bps": [r["avg_bandwidth_bps"] for r in rows], + "hourly_price": [r["hourly_price"] for r in rows], + "lookback_days": [r["lookback_days"] for r in rows], + } + )