From cddd562eb64a849e6f63f023770794829562b2d2 Mon Sep 17 00:00:00 2001 From: DYNOSuprovo Date: Tue, 22 Sep 2026 01:26:50 +0530 Subject: [PATCH] feat(azure): add idle Cosmos DB for Gremlin API check (#20) --- README.md | 2 +- cloudcost.cosmosdb-gremlin-idle-real.yml | 35 ++++ policies/cosmosdb_gremlin_idle_ru.sql | 22 +++ src/cloudcost/engine/runner.py | 1 + .../sources/azure/cosmosdb_gremlin_idle_ru.py | 151 ++++++++++++++++++ .../policies/test_cosmosdb_gremlin_idle_ru.py | 123 ++++++++++++++ 6 files changed, 333 insertions(+), 1 deletion(-) create mode 100644 cloudcost.cosmosdb-gremlin-idle-real.yml create mode 100644 policies/cosmosdb_gremlin_idle_ru.sql create mode 100644 src/cloudcost/sources/azure/cosmosdb_gremlin_idle_ru.py create mode 100644 tests/policies/test_cosmosdb_gremlin_idle_ru.py diff --git a/README.md b/README.md index 5f0a821..bf74c18 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Every check below was built the same way: create the actual cloud resource, run **Databases & messaging** - Azure SQL DTU underutilization — `sql_dtu_underutilized` -- Idle Cosmos DB provisioned throughput (SQL, MongoDB, and Cassandra APIs) — `cosmosdb_idle_ru`, `cosmosdb_mongo_idle_ru`, `cosmosdb_cassandra_idle_ru` +- Idle Cosmos DB provisioned throughput (SQL, MongoDB, Cassandra, and Gremlin APIs) — `cosmosdb_idle_ru`, `cosmosdb_mongo_idle_ru`, `cosmosdb_cassandra_idle_ru`, `cosmosdb_gremlin_idle_ru` - Idle PostgreSQL / MySQL Flexible Server — `postgres_idle_flexible`, `mysql_idle_flexible` - Idle Redis Cache (live per-tier/SKU pricing) — `redis_idle` - Idle Event Hubs Namespace (Standard) — `idle_eventhub` diff --git a/cloudcost.cosmosdb-gremlin-idle-real.yml b/cloudcost.cosmosdb-gremlin-idle-real.yml new file mode 100644 index 0000000..288bbc8 --- /dev/null +++ b/cloudcost.cosmosdb-gremlin-idle-real.yml @@ -0,0 +1,35 @@ +version: "1" + +pipeline: + name: cosmosdb-gremlin-idle-real-test + mode: incremental + timezone: UTC + +sources: + - name: cosmos_gremlin_ru + type: azure.cosmosdb_gremlin_idle_ru + config: + resource_group: cost-compare-3day-rg + lookback_days: 1 + +destinations: + - name: local_duckdb + type: duckdb + config: + path: ./data/cosmosdb-gremlin-idle-real.duckdb + +loads: + - input: cosmos_gremlin_ru + destination: local_duckdb + table: fact_cosmosdb_gremlin_idle_ru + mode: overwrite + +observability: + log_level: info + emit_run_summary: true + +policies: + - name: cosmosdb-gremlin-idle-ru + type: sql + query_file: policies/cosmosdb_gremlin_idle_ru.sql + severity: medium diff --git a/policies/cosmosdb_gremlin_idle_ru.sql b/policies/cosmosdb_gremlin_idle_ru.sql new file mode 100644 index 0000000..13d119f --- /dev/null +++ b/policies/cosmosdb_gremlin_idle_ru.sql @@ -0,0 +1,22 @@ +-- Real check: Cosmos DB for Apache Gremlin (Graph) API accounts with fixed +-- provisioned throughput (RU/s reserved, billed hourly regardless of use) +-- and near-zero actual consumption over the lookback window. Same real +-- price as the SQL, MongoDB, and Cassandra Cosmos checks: $0.008/hour per +-- 100 RU/s (Retail Prices API, 'Azure Cosmos DB 100 RU/s' meter) -- RU +-- billing is identical across Cosmos APIs, but Gremlin uses its own CLI +-- surface (database/graph) for managing throughput. Serverless accounts +-- are excluded upstream. +SELECT + 'azure' AS provider, + resource_id, + 'Azure Cosmos DB for Gremlin' AS service_name, + ROUND((provisioned_ru / 100.0) * 0.008 * 730, 2) AS billed_cost, + resource_name, + provisioned_ru, + total_ru_consumed, + lookback_days, + provisioned_ru || ' RU/s provisioned, only ' || ROUND(total_ru_consumed, 0) || + ' RU consumed over ' || lookback_days || ' days' AS evidence_reason +FROM fact_cosmosdb_gremlin_idle_ru +WHERE provisioned_ru > 0 + AND total_ru_consumed < (provisioned_ru * 0.01); diff --git a/src/cloudcost/engine/runner.py b/src/cloudcost/engine/runner.py index 260f0c7..91f7b3a 100644 --- a/src/cloudcost/engine/runner.py +++ b/src/cloudcost/engine/runner.py @@ -49,6 +49,7 @@ import cloudcost.sources.azure.idle_premiumv2_disk_overage import cloudcost.sources.azure.cosmosdb_mongo_idle_ru import cloudcost.sources.azure.cosmosdb_cassandra_idle_ru + import cloudcost.sources.azure.cosmosdb_gremlin_idle_ru import cloudcost.sources.azure.idle_container_apps_dedicated import cloudcost.sources.azure.idle_acr_geo_replication import cloudcost.sources.oci.billing diff --git a/src/cloudcost/sources/azure/cosmosdb_gremlin_idle_ru.py b/src/cloudcost/sources/azure/cosmosdb_gremlin_idle_ru.py new file mode 100644 index 0000000..451ed8a --- /dev/null +++ b/src/cloudcost/sources/azure/cosmosdb_gremlin_idle_ru.py @@ -0,0 +1,151 @@ +import json +import subprocess +from datetime import datetime, timedelta, timezone +from typing import Any + +import pyarrow as pa + +from cloudcost.core.registry import registry + + +# Real check: Cosmos DB for Apache Gremlin (Graph) API databases and +# graphs with fixed provisioned throughput bill for it whether consumed +# or not -- same real RU price as azure.cosmosdb_idle_ru, +# azure.cosmosdb_mongo_idle_ru, and azure.cosmosdb_cassandra_idle_ru, +# but Gremlin API needs its own CLI surface (`az cosmosdb gremlin ...`) +# and its own resource scope (database and graph). +@registry.register_source("azure.cosmosdb_gremlin_idle_ru") +class AzureCosmosDbGremlinIdleRuSource: + def __init__(self, config: dict): + self.resource_group = config.get("resource_group") + self.lookback_days = config.get("lookback_days", 7) + if not self.resource_group: + raise ValueError("azure.cosmosdb_gremlin_idle_ru requires 'resource_group' in config") + + def extract(self, context: Any = None) -> pa.Table: + accounts_raw = subprocess.run( + [ + "az", "cosmosdb", "list", "--resource-group", self.resource_group, + "--query", "[?capabilities[?name=='EnableGremlin']].{id:id,name:name,capabilities:capabilities}", + "-o", "json", + ], + capture_output=True, text=True, check=True, + ).stdout + accounts = json.loads(accounts_raw) + + end_time = datetime.now(timezone.utc) + start_time = end_time - timedelta(days=self.lookback_days) + + rows = [] + for acct in accounts: + capability_names = [c.get("name") for c in (acct.get("capabilities") or [])] + if "EnableServerless" in capability_names: + continue + + dbs_raw = subprocess.run( + [ + "az", "cosmosdb", "gremlin", "database", "list", + "--account-name", acct["name"], + "--resource-group", self.resource_group, + "--query", "[].name", "-o", "json", + ], + capture_output=True, text=True, check=True, + ).stdout + db_names = json.loads(dbs_raw) + + provisioned_ru = 0 + for db_name in db_names: + # Check database-level throughput + try: + throughput_raw = subprocess.run( + [ + "az", "cosmosdb", "gremlin", "database", "throughput", "show", + "--account-name", acct["name"], + "--resource-group", self.resource_group, + "--name", db_name, + "--query", "resource.throughput", "-o", "tsv", + ], + capture_output=True, text=True, check=True, + ).stdout.strip() + if throughput_raw and throughput_raw != "None": + provisioned_ru += int(throughput_raw) + except subprocess.CalledProcessError: + pass + + # Check graph-level throughput + try: + graphs_raw = subprocess.run( + [ + "az", "cosmosdb", "gremlin", "graph", "list", + "--account-name", acct["name"], + "--resource-group", self.resource_group, + "--database-name", db_name, + "--query", "[].name", "-o", "json", + ], + capture_output=True, text=True, check=True, + ).stdout + graph_names = json.loads(graphs_raw) + for graph_name in graph_names: + try: + g_throughput_raw = subprocess.run( + [ + "az", "cosmosdb", "gremlin", "graph", "throughput", "show", + "--account-name", acct["name"], + "--resource-group", self.resource_group, + "--database-name", db_name, + "--name", graph_name, + "--query", "resource.throughput", "-o", "tsv", + ], + capture_output=True, text=True, check=True, + ).stdout.strip() + if g_throughput_raw and g_throughput_raw != "None": + provisioned_ru += int(g_throughput_raw) + except subprocess.CalledProcessError: + continue + except subprocess.CalledProcessError: + continue + + raw = subprocess.run( + [ + "az", "monitor", "metrics", "list", + "--resource", acct["id"], + "--metric", "TotalRequestUnits", + "--aggregation", "Total", + "--interval", "PT1H", + "--start-time", start_time.strftime("%Y-%m-%dT%H:%M:%SZ"), + "--end-time", end_time.strftime("%Y-%m-%dT%H:%M:%SZ"), + ], + capture_output=True, text=True, check=True, + ).stdout + parsed = json.loads(raw) + + total_ru_consumed = 0.0 + for timeseries in parsed.get("value", []): + for series in timeseries.get("timeseries", []): + for point in series.get("data", []): + total_ru_consumed += point.get("total") or 0.0 + + rows.append({ + "resource_id": acct["id"].lower(), + "resource_name": acct["name"], + "provisioned_ru": provisioned_ru, + "total_ru_consumed": total_ru_consumed, + "lookback_days": self.lookback_days, + }) + + if not rows: + return pa.table({ + "resource_id": pa.array([], type=pa.string()), + "resource_name": pa.array([], type=pa.string()), + "provisioned_ru": pa.array([], type=pa.int64()), + "total_ru_consumed": pa.array([], type=pa.float64()), + "lookback_days": pa.array([], type=pa.int64()), + }) + + return pa.table({ + "resource_id": [r["resource_id"] for r in rows], + "resource_name": [r["resource_name"] for r in rows], + "provisioned_ru": [r["provisioned_ru"] for r in rows], + "total_ru_consumed": [r["total_ru_consumed"] for r in rows], + "lookback_days": [r["lookback_days"] for r in rows], + }) diff --git a/tests/policies/test_cosmosdb_gremlin_idle_ru.py b/tests/policies/test_cosmosdb_gremlin_idle_ru.py new file mode 100644 index 0000000..4eaf0ed --- /dev/null +++ b/tests/policies/test_cosmosdb_gremlin_idle_ru.py @@ -0,0 +1,123 @@ +import pyarrow as pa + + +def gremlin_accounts(*rows: tuple[str, str, int, float, int]) -> pa.Table: + """Each row is (resource_id, resource_name, provisioned_ru, total_ru_consumed, lookback_days).""" + return pa.table( + { + "resource_id": pa.array([r[0] for r in rows], type=pa.string()), + "resource_name": pa.array([r[1] for r in rows], type=pa.string()), + "provisioned_ru": pa.array([r[2] for r in rows], type=pa.int64()), + "total_ru_consumed": pa.array([r[3] for r in rows], type=pa.float64()), + "lookback_days": pa.array([r[4] for r in rows], type=pa.int64()), + } + ) + + +def test_flags_idle_gremlin_account_below_consumption_threshold(run_policy) -> None: + table = gremlin_accounts( + ( + "/subscriptions/sub/resourcegroups/rg/providers/microsoft.documentdb/databaseaccounts/gremlindb1", + "gremlindb1", + 1000, + 5.0, + 7, + ) + ) + + findings = run_policy( + "cosmosdb_gremlin_idle_ru", + {"fact_cosmosdb_gremlin_idle_ru": table}, + ) + + assert len(findings) == 1 + finding = findings[0] + assert finding.provider == "azure" + assert finding.service_name == "Azure Cosmos DB for Gremlin" + assert finding.estimated_impact == 58.40 # (1000 / 100) * 0.008 * 730 = 58.4 + assert finding.evidence["provisioned_ru"] == 1000 + assert finding.evidence["total_ru_consumed"] == 5.0 + assert finding.evidence["lookback_days"] == 7 + assert ( + finding.evidence["evidence_reason"] + == "1000 RU/s provisioned, only 5.0 RU consumed over 7 days" + ) + + +def test_active_gremlin_account_is_not_flagged(run_policy) -> None: + # 100 RU consumed >= 1% of 1000 provisioned RU + table = gremlin_accounts( + ( + "/subscriptions/sub/resourcegroups/rg/providers/microsoft.documentdb/databaseaccounts/activegremlin", + "activegremlin", + 1000, + 100.0, + 7, + ) + ) + + findings = run_policy( + "cosmosdb_gremlin_idle_ru", + {"fact_cosmosdb_gremlin_idle_ru": table}, + ) + + assert findings == [] + + +def test_account_with_zero_provisioned_ru_is_not_flagged(run_policy) -> None: + table = gremlin_accounts( + ( + "/subscriptions/sub/resourcegroups/rg/providers/microsoft.documentdb/databaseaccounts/zero-ru", + "zero-ru", + 0, + 0.0, + 7, + ) + ) + + findings = run_policy( + "cosmosdb_gremlin_idle_ru", + {"fact_cosmosdb_gremlin_idle_ru": table}, + ) + + assert findings == [] + + +def test_mixed_accounts_returns_only_idle(run_policy) -> None: + table = gremlin_accounts( + ( + "/subscriptions/sub/resourcegroups/rg/providers/microsoft.documentdb/databaseaccounts/idle-account", + "idle-account", + 400, + 1.0, + 7, + ), + ( + "/subscriptions/sub/resourcegroups/rg/providers/microsoft.documentdb/databaseaccounts/busy-account", + "busy-account", + 400, + 5000.0, + 7, + ), + ) + + findings = run_policy( + "cosmosdb_gremlin_idle_ru", + {"fact_cosmosdb_gremlin_idle_ru": table}, + ) + + assert len(findings) == 1 + assert "idle-account" in findings[0].resource_id + assert findings[0].estimated_impact == 23.36 # (400 / 100) * 0.008 * 730 = 23.36 + + +def test_empty_table_returns_no_findings(run_policy) -> None: + table = gremlin_accounts() + + assert ( + run_policy( + "cosmosdb_gremlin_idle_ru", + {"fact_cosmosdb_gremlin_idle_ru": table}, + ) + == [] + )