feat(azure): add idle Cosmos DB for Gremlin API check (#20)

README.md

@@ -47,7 +47,7 @@ Every check below was built the same way: create the actual cloud resource, run
 
 **Databases & messaging**
 - Azure SQL DTU underutilization — `sql_dtu_underutilized`
-- Idle Cosmos DB provisioned throughput (SQL, MongoDB, and Cassandra APIs) — `cosmosdb_idle_ru`, `cosmosdb_mongo_idle_ru`, `cosmosdb_cassandra_idle_ru`
+- Idle Cosmos DB provisioned throughput (SQL, MongoDB, Cassandra, and Gremlin APIs) — `cosmosdb_idle_ru`, `cosmosdb_mongo_idle_ru`, `cosmosdb_cassandra_idle_ru`, `cosmosdb_gremlin_idle_ru`
 - Idle PostgreSQL / MySQL Flexible Server — `postgres_idle_flexible`, `mysql_idle_flexible`
 - Idle Redis Cache (live per-tier/SKU pricing) — `redis_idle`
 - Idle Event Hubs Namespace (Standard) — `idle_eventhub`

cloudcost.cosmosdb-gremlin-idle-real.yml

@@ -0,0 +1,35 @@
+version: "1"
+
+pipeline:
+  name: cosmosdb-gremlin-idle-real-test
+  mode: incremental
+  timezone: UTC
+
+sources:
+  - name: cosmos_gremlin_ru
+    type: azure.cosmosdb_gremlin_idle_ru
+    config:
+      resource_group: cost-compare-3day-rg
+      lookback_days: 1
+
+destinations:
+  - name: local_duckdb
+    type: duckdb
+    config:
+      path: ./data/cosmosdb-gremlin-idle-real.duckdb
+
+loads:
+  - input: cosmos_gremlin_ru
+    destination: local_duckdb
+    table: fact_cosmosdb_gremlin_idle_ru
+    mode: overwrite
+
+observability:
+  log_level: info
+  emit_run_summary: true
+
+policies:
+  - name: cosmosdb-gremlin-idle-ru
+    type: sql
+    query_file: policies/cosmosdb_gremlin_idle_ru.sql
+    severity: medium

policies/cosmosdb_gremlin_idle_ru.sql

@@ -0,0 +1,22 @@
+-- Real check: Cosmos DB for Apache Gremlin (Graph) API accounts with fixed
+-- provisioned throughput (RU/s reserved, billed hourly regardless of use)
+-- and near-zero actual consumption over the lookback window. Same real
+-- price as the SQL, MongoDB, and Cassandra Cosmos checks: $0.008/hour per
+-- 100 RU/s (Retail Prices API, 'Azure Cosmos DB 100 RU/s' meter) -- RU
+-- billing is identical across Cosmos APIs, but Gremlin uses its own CLI
+-- surface (database/graph) for managing throughput. Serverless accounts
+-- are excluded upstream.
+SELECT
+    'azure' AS provider,
+    resource_id,
+    'Azure Cosmos DB for Gremlin' AS service_name,
+    ROUND((provisioned_ru / 100.0) * 0.008 * 730, 2) AS billed_cost,
+    resource_name,
+    provisioned_ru,
+    total_ru_consumed,
+    lookback_days,
+    provisioned_ru || ' RU/s provisioned, only ' || ROUND(total_ru_consumed, 0) ||
+        ' RU consumed over ' || lookback_days || ' days' AS evidence_reason
+FROM fact_cosmosdb_gremlin_idle_ru
+WHERE provisioned_ru > 0
+  AND total_ru_consumed < (provisioned_ru * 0.01);

src/cloudcost/engine/runner.py

@@ -49,6 +49,7 @@
     import cloudcost.sources.azure.idle_premiumv2_disk_overage
     import cloudcost.sources.azure.cosmosdb_mongo_idle_ru
     import cloudcost.sources.azure.cosmosdb_cassandra_idle_ru
+    import cloudcost.sources.azure.cosmosdb_gremlin_idle_ru
     import cloudcost.sources.azure.idle_container_apps_dedicated
     import cloudcost.sources.azure.idle_acr_geo_replication
     import cloudcost.sources.oci.billing

src/cloudcost/sources/azure/cosmosdb_gremlin_idle_ru.py

@@ -0,0 +1,151 @@
+import json
+import subprocess
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+import pyarrow as pa
+
+from cloudcost.core.registry import registry
+
+
+# Real check: Cosmos DB for Apache Gremlin (Graph) API databases and
+# graphs with fixed provisioned throughput bill for it whether consumed
+# or not -- same real RU price as azure.cosmosdb_idle_ru,
+# azure.cosmosdb_mongo_idle_ru, and azure.cosmosdb_cassandra_idle_ru,
+# but Gremlin API needs its own CLI surface (`az cosmosdb gremlin ...`)
+# and its own resource scope (database and graph).
+@registry.register_source("azure.cosmosdb_gremlin_idle_ru")
+class AzureCosmosDbGremlinIdleRuSource:
+    def __init__(self, config: dict):
+        self.resource_group = config.get("resource_group")
+        self.lookback_days = config.get("lookback_days", 7)
+        if not self.resource_group:
+            raise ValueError("azure.cosmosdb_gremlin_idle_ru requires 'resource_group' in config")
+
+    def extract(self, context: Any = None) -> pa.Table:
+        accounts_raw = subprocess.run(
+            [
+                "az", "cosmosdb", "list", "--resource-group", self.resource_group,
+                "--query", "[?capabilities[?name=='EnableGremlin']].{id:id,name:name,capabilities:capabilities}",
+                "-o", "json",
+            ],
+            capture_output=True, text=True, check=True,
+        ).stdout
+        accounts = json.loads(accounts_raw)
+
+        end_time = datetime.now(timezone.utc)
+        start_time = end_time - timedelta(days=self.lookback_days)
+
+        rows = []
+        for acct in accounts:
+            capability_names = [c.get("name") for c in (acct.get("capabilities") or [])]
+            if "EnableServerless" in capability_names:
+                continue
+
+            dbs_raw = subprocess.run(
+                [
+                    "az", "cosmosdb", "gremlin", "database", "list",
+                    "--account-name", acct["name"],
+                    "--resource-group", self.resource_group,
+                    "--query", "[].name", "-o", "json",
+                ],
+                capture_output=True, text=True, check=True,
+            ).stdout
+            db_names = json.loads(dbs_raw)
+
+            provisioned_ru = 0
+            for db_name in db_names:
+                # Check database-level throughput
+                try:
+                    throughput_raw = subprocess.run(
+                        [
+                            "az", "cosmosdb", "gremlin", "database", "throughput", "show",
+                            "--account-name", acct["name"],
+                            "--resource-group", self.resource_group,
+                            "--name", db_name,
+                            "--query", "resource.throughput", "-o", "tsv",
+                        ],
+                        capture_output=True, text=True, check=True,
+                    ).stdout.strip()
+                    if throughput_raw and throughput_raw != "None":
+                        provisioned_ru += int(throughput_raw)
+                except subprocess.CalledProcessError:
+                    pass
+
+                # Check graph-level throughput
+                try:
+                    graphs_raw = subprocess.run(
+                        [
+                            "az", "cosmosdb", "gremlin", "graph", "list",
+                            "--account-name", acct["name"],
+                            "--resource-group", self.resource_group,
+                            "--database-name", db_name,
+                            "--query", "[].name", "-o", "json",
+                        ],
+                        capture_output=True, text=True, check=True,
+                    ).stdout
+                    graph_names = json.loads(graphs_raw)
+                    for graph_name in graph_names:
+                        try:
+                            g_throughput_raw = subprocess.run(
+                                [
+                                    "az", "cosmosdb", "gremlin", "graph", "throughput", "show",
+                                    "--account-name", acct["name"],
+                                    "--resource-group", self.resource_group,
+                                    "--database-name", db_name,
+                                    "--name", graph_name,
+                                    "--query", "resource.throughput", "-o", "tsv",
+                                ],
+                                capture_output=True, text=True, check=True,
+                            ).stdout.strip()
+                            if g_throughput_raw and g_throughput_raw != "None":
+                                provisioned_ru += int(g_throughput_raw)
+                        except subprocess.CalledProcessError:
+                            continue
+                except subprocess.CalledProcessError:
+                    continue
+
+            raw = subprocess.run(
+                [
+                    "az", "monitor", "metrics", "list",
+                    "--resource", acct["id"],
+                    "--metric", "TotalRequestUnits",
+                    "--aggregation", "Total",
+                    "--interval", "PT1H",
+                    "--start-time", start_time.strftime("%Y-%m-%dT%H:%M:%SZ"),
+                    "--end-time", end_time.strftime("%Y-%m-%dT%H:%M:%SZ"),
+                ],
+                capture_output=True, text=True, check=True,
+            ).stdout
+            parsed = json.loads(raw)
+
+            total_ru_consumed = 0.0
+            for timeseries in parsed.get("value", []):
+                for series in timeseries.get("timeseries", []):
+                    for point in series.get("data", []):
+                        total_ru_consumed += point.get("total") or 0.0
+
+            rows.append({
+                "resource_id": acct["id"].lower(),
+                "resource_name": acct["name"],
+                "provisioned_ru": provisioned_ru,
+                "total_ru_consumed": total_ru_consumed,
+                "lookback_days": self.lookback_days,
+            })
+
+        if not rows:
+            return pa.table({
+                "resource_id": pa.array([], type=pa.string()),
+                "resource_name": pa.array([], type=pa.string()),
+                "provisioned_ru": pa.array([], type=pa.int64()),
+                "total_ru_consumed": pa.array([], type=pa.float64()),
+                "lookback_days": pa.array([], type=pa.int64()),
+            })
+
+        return pa.table({
+            "resource_id": [r["resource_id"] for r in rows],
+            "resource_name": [r["resource_name"] for r in rows],
+            "provisioned_ru": [r["provisioned_ru"] for r in rows],
+            "total_ru_consumed": [r["total_ru_consumed"] for r in rows],
+            "lookback_days": [r["lookback_days"] for r in rows],
+        })

tests/policies/test_cosmosdb_gremlin_idle_ru.py

@@ -0,0 +1,123 @@
+import pyarrow as pa
+
+
+def gremlin_accounts(*rows: tuple[str, str, int, float, int]) -> pa.Table:
+    """Each row is (resource_id, resource_name, provisioned_ru, total_ru_consumed, lookback_days)."""
+    return pa.table(
+        {
+            "resource_id": pa.array([r[0] for r in rows], type=pa.string()),
+            "resource_name": pa.array([r[1] for r in rows], type=pa.string()),
+            "provisioned_ru": pa.array([r[2] for r in rows], type=pa.int64()),
+            "total_ru_consumed": pa.array([r[3] for r in rows], type=pa.float64()),
+            "lookback_days": pa.array([r[4] for r in rows], type=pa.int64()),
+        }
+    )
+
+
+def test_flags_idle_gremlin_account_below_consumption_threshold(run_policy) -> None:
+    table = gremlin_accounts(
+        (
+            "/subscriptions/sub/resourcegroups/rg/providers/microsoft.documentdb/databaseaccounts/gremlindb1",
+            "gremlindb1",
+            1000,
+            5.0,
+            7,
+        )
+    )
+
+    findings = run_policy(
+        "cosmosdb_gremlin_idle_ru",
+        {"fact_cosmosdb_gremlin_idle_ru": table},
+    )
+
+    assert len(findings) == 1
+    finding = findings[0]
+    assert finding.provider == "azure"
+    assert finding.service_name == "Azure Cosmos DB for Gremlin"
+    assert finding.estimated_impact == 58.40  # (1000 / 100) * 0.008 * 730 = 58.4
+    assert finding.evidence["provisioned_ru"] == 1000
+    assert finding.evidence["total_ru_consumed"] == 5.0
+    assert finding.evidence["lookback_days"] == 7
+    assert (
+        finding.evidence["evidence_reason"]
+        == "1000 RU/s provisioned, only 5.0 RU consumed over 7 days"
+    )
+
+
+def test_active_gremlin_account_is_not_flagged(run_policy) -> None:
+    # 100 RU consumed >= 1% of 1000 provisioned RU
+    table = gremlin_accounts(
+        (
+            "/subscriptions/sub/resourcegroups/rg/providers/microsoft.documentdb/databaseaccounts/activegremlin",
+            "activegremlin",
+            1000,
+            100.0,
+            7,
+        )
+    )
+
+    findings = run_policy(
+        "cosmosdb_gremlin_idle_ru",
+        {"fact_cosmosdb_gremlin_idle_ru": table},
+    )
+
+    assert findings == []
+
+
+def test_account_with_zero_provisioned_ru_is_not_flagged(run_policy) -> None:
+    table = gremlin_accounts(
+        (
+            "/subscriptions/sub/resourcegroups/rg/providers/microsoft.documentdb/databaseaccounts/zero-ru",
+            "zero-ru",
+            0,
+            0.0,
+            7,
+        )
+    )
+
+    findings = run_policy(
+        "cosmosdb_gremlin_idle_ru",
+        {"fact_cosmosdb_gremlin_idle_ru": table},
+    )
+
+    assert findings == []
+
+
+def test_mixed_accounts_returns_only_idle(run_policy) -> None:
+    table = gremlin_accounts(
+        (
+            "/subscriptions/sub/resourcegroups/rg/providers/microsoft.documentdb/databaseaccounts/idle-account",
+            "idle-account",
+            400,
+            1.0,
+            7,
+        ),
+        (
+            "/subscriptions/sub/resourcegroups/rg/providers/microsoft.documentdb/databaseaccounts/busy-account",
+            "busy-account",
+            400,
+            5000.0,
+            7,
+        ),
+    )
+
+    findings = run_policy(
+        "cosmosdb_gremlin_idle_ru",
+        {"fact_cosmosdb_gremlin_idle_ru": table},
+    )
+
+    assert len(findings) == 1
+    assert "idle-account" in findings[0].resource_id
+    assert findings[0].estimated_impact == 23.36  # (400 / 100) * 0.008 * 730 = 23.36
+
+
+def test_empty_table_returns_no_findings(run_policy) -> None:
+    table = gremlin_accounts()
+
+    assert (
+        run_policy(
+            "cosmosdb_gremlin_idle_ru",
+            {"fact_cosmosdb_gremlin_idle_ru": table},
+        )
+        == []
+    )