Record API requests in Matomo

.env.example

@@ -7,3 +7,7 @@ CLEAN_DB_PASSWORD=mmli
 CLEAN_DB_HOST=host.docker.internal
 CLEAN_DB_PORT=5432
 CLEAN_DB_NAME=mmlidb
+
+# Matomo usage tracking (optional; disabled unless MATOMO_SITE_ID is set)
+# MATOMO_SITE_ID=
+# MATOMO_TOKEN=

app/core/config.py

@@ -1,6 +1,6 @@
-from typing import List
+from typing import List, Optional
 
-from pydantic import ConfigDict
+from pydantic import ConfigDict, field_validator
 from pydantic_settings import BaseSettings
 
 
@@ -16,6 +16,18 @@ class Settings(BaseSettings):
     # CORS configuration
     CORS_ORIGINS: List[str] = ["*"]
 
+    # Matomo usage tracking; disabled unless MATOMO_SITE_ID is set
+    MATOMO_URL: str = "https://matomo.mmli2.ncsa.illinois.edu"
+    MATOMO_SITE_ID: Optional[int] = None
+    # Token of a user with write access to the site; lets Matomo record the caller's IP
+    MATOMO_TOKEN: Optional[str] = None
+
+    @field_validator("MATOMO_SITE_ID", "MATOMO_TOKEN", mode="before")
+    @classmethod
+    def empty_is_unset(cls, value):
+        """Treat MATOMO_SITE_ID= (as in .env.example) as unset instead of failing to start."""
+        return None if value == "" else value
+
     # Database configuration
     CLEAN_DB_USER: str
     CLEAN_DB_PASSWORD: str

app/core/matomo.py

@@ -0,0 +1,94 @@
+import asyncio
+import hashlib
+import hmac
+
+import httpx
+from loguru import logger
+from starlette.requests import Request
+
+from app.core.config import settings
+
+# Shared client, created in start() only when tracking is configured; None means tracking is off.
+_client: httpx.AsyncClient | None = None
+# Hits still being sent; holding a reference keeps the tasks from being garbage collected.
+_pending: set[asyncio.Task] = set()
+
+
+def start() -> None:
+    global _client
+    if settings.MATOMO_SITE_ID is not None:
+        # The connection limit caps concurrent hits; hits waiting for a connection give up
+        # after the pool timeout, so a slow Matomo can't pile up open connections.
+        _client = httpx.AsyncClient(timeout=5, limits=httpx.Limits(max_connections=10))
+
+
+async def stop() -> None:
+    """Give in-flight hits a few seconds to finish, then close the client."""
+    global _client
+    if _client is None:
+        return
+    if _pending:
+        await asyncio.wait(_pending, timeout=5)
+    await _client.aclose()
+    _client = None
+
+
+def build_hit(request: Request) -> dict:
+    """Build the Matomo tracking parameters for one API request."""
+    ua = request.headers.get("user-agent", "")
+    params = {
+        "idsite": settings.MATOMO_SITE_ID,
+        "rec": 1,
+        "apiv": 1,
+        "send_image": 0,
+        # Path only: query strings would make every search and typeahead keystroke its own page.
+        "url": str(request.url.replace(query="")),
+        "action_name": request.url.path,
+        "ua": ua,
+        "urlref": request.headers.get("referer", ""),
+        "lang": request.headers.get("accept-language", ""),
+    }
+    ip = request.client.host if request.client else None
+    # Matomo ignores cip (the caller's IP) unless the request carries a token with write access.
+    if settings.MATOMO_TOKEN and ip:
+        params.update(
+            token_auth=settings.MATOMO_TOKEN,
+            cip=ip,
+            # API clients send no Matomo cookie, so derive a stable visitor id from IP + user
+            # agent. Keyed with the token so the id can't be reversed into the caller's IP;
+            # rotating the token resets visitor ids.
+            cid=hmac.new(settings.MATOMO_TOKEN.encode(), f"{ip}|{ua}".encode(),
+                         hashlib.sha256).hexdigest()[:16],
+        )
+    return params
+
+
+async def send_hit(params: dict) -> None:
+    try:
+        response = await _client.post(f"{settings.MATOMO_URL}/matomo.php", data=params)
+        response.raise_for_status()
+    except Exception as e:
+        # Tracking must never affect the API, whatever goes wrong.
+        logger.warning(f"Matomo tracking failed: {e!r}")
+
+
+class MatomoTrackingMiddleware:
+    """Record each API request in Matomo, after the response, without delaying it.
+
+    Disabled unless MATOMO_SITE_ID is set (see start()). CORS preflight (OPTIONS) requests are
+    skipped so a browser call is counted once.
+    """
+
+    def __init__(self, app):
+        self.app = app
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] != "http" or _client is None or scope["method"] == "OPTIONS":
+            await self.app(scope, receive, send)
+            return
+        try:
+            await self.app(scope, receive, send)
+        finally:
+            task = asyncio.create_task(send_hit(build_hit(Request(scope))))
+            _pending.add(task)
+            task.add_done_callback(_pending.discard)

app/main.py

@@ -4,6 +4,7 @@
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.middleware.gzip import GZipMiddleware
 
+from app.core import matomo
 from app.core.config import settings
 from app.db.database import _db
 from app.routers import search
@@ -14,9 +15,11 @@ async def lifespan(app: FastAPI):
     """Lifespan context manager for database connection handling."""
     # Connect to database on startup
     await _db.connect()
+    matomo.start()
     yield
     # Disconnect from database on shutdown
     await _db.disconnect()
+    await matomo.stop()
 
 
 # Initialize FastAPI application
@@ -100,5 +103,8 @@ async def lifespan(app: FastAPI):
 # Add GZip compression middleware to compress large responses
 app.add_middleware(GZipMiddleware, minimum_size=1000)
 
+# Record API usage in Matomo
+app.add_middleware(matomo.MatomoTrackingMiddleware)
+
 # Include API routers
 app.include_router(search.router, prefix="/api/v1")

tests/conftest.py

@@ -6,6 +6,8 @@
 # Settings require DB credentials at import time; tests mock the DB calls.
 for key in ("CLEAN_DB_HOST", "CLEAN_DB_USER", "CLEAN_DB_PASSWORD", "CLEAN_DB_NAME"):
     os.environ.setdefault(key, "test")
+# Keep a developer's .env from sending test requests to the real Matomo site.
+os.environ["MATOMO_SITE_ID"] = ""
 
 
 @pytest.fixture

tests/test_matomo.py

@@ -0,0 +1,131 @@
+import asyncio
+import hashlib
+from unittest.mock import AsyncMock, patch
+
+import httpx
+import pytest
+from starlette.requests import Request
+
+import app.core.matomo as matomo
+from app.core.config import Settings, settings
+from app.main import app
+
+
+@pytest.fixture
+async def tracking(monkeypatch):
+    monkeypatch.setattr(settings, "MATOMO_SITE_ID", 7)
+    monkeypatch.setattr(settings, "MATOMO_TOKEN", None)
+    matomo.start()
+    with patch.object(matomo, "send_hit", AsyncMock()) as send_hit:
+        yield send_hit
+    await matomo.stop()
+
+
+async def request(method, path, **kwargs):
+    transport = httpx.ASGITransport(app=app, client=("203.0.113.5", 4321))
+    async with httpx.AsyncClient(transport=transport, base_url="https://cleandb-api.test") as client:
+        response = await client.request(method, path, **kwargs)
+    await asyncio.gather(*matomo._pending)
+    return response
+
+
+async def test_records_request_path_without_query(tracking):
+    response = await request("GET", "/api/v1/openapi.json?accession=P51451",
+                             headers={"user-agent": "curl/8", "referer": "https://example.org/"})
+    assert response.status_code == 200
+    params = tracking.await_args.args[0]
+    assert params["idsite"] == 7
+    assert params["rec"] == 1
+    assert params["url"] == "https://cleandb-api.test/api/v1/openapi.json"
+    assert params["action_name"] == "/api/v1/openapi.json"
+    assert params["ua"] == "curl/8"
+    assert params["urlref"] == "https://example.org/"
+    assert not {"cip", "cid", "token_auth"} & params.keys()
+
+
+async def test_token_sends_caller_ip_and_keyed_visitor_id(tracking, monkeypatch):
+    monkeypatch.setattr(settings, "MATOMO_TOKEN", "secret")
+    await request("GET", "/api/v1/openapi.json", headers={"user-agent": "a"})
+    await request("GET", "/api/v1/openapi.json", headers={"user-agent": "a"})
+    await request("GET", "/api/v1/openapi.json", headers={"user-agent": "b"})
+    first, again, other_ua = (c.args[0] for c in tracking.await_args_list)
+    assert first["token_auth"] == "secret"
+    assert first["cip"] == "203.0.113.5"
+    assert len(first["cid"]) == 16
+    assert first["cid"] == again["cid"] != other_ua["cid"]
+    # Not recoverable by hashing candidate IPs and user agents without the token.
+    assert first["cid"] != hashlib.sha256(b"203.0.113.5|a").hexdigest()[:16]
+
+    monkeypatch.setattr(settings, "MATOMO_TOKEN", "other")
+    await request("GET", "/api/v1/openapi.json", headers={"user-agent": "a"})
+    assert tracking.await_args.args[0]["cid"] != first["cid"]
+
+
+def test_no_caller_address_sends_no_ip_or_visitor_id(monkeypatch):
+    monkeypatch.setattr(settings, "MATOMO_TOKEN", "secret")
+    scope = {"type": "http", "method": "GET", "scheme": "https", "server": ("api.test", 443),
+             "path": "/api/v1/search", "query_string": b"", "headers": [], "client": None}
+    params = matomo.build_hit(Request(scope))
+    assert not {"cip", "cid", "token_auth"} & params.keys()
+
+
+async def test_records_failed_requests(tracking):
+    response = await request("GET", "/api/v1/no-such-endpoint")
+    assert response.status_code == 404
+    tracking.assert_awaited_once()
+
+
+async def test_skips_cors_preflight(tracking):
+    await request("OPTIONS", "/api/v1/search",
+                  headers={"origin": "https://example.org", "access-control-request-method": "GET"})
+    tracking.assert_not_awaited()
+
+
+async def test_disabled_without_site_id(monkeypatch):
+    monkeypatch.setattr(settings, "MATOMO_SITE_ID", None)
+    matomo.start()
+    with patch.object(matomo, "send_hit", AsyncMock()) as send_hit:
+        response = await request("GET", "/api/v1/openapi.json")
+    assert response.status_code == 200
+    send_hit.assert_not_awaited()
+
+
+def test_empty_settings_mean_unset(monkeypatch):
+    monkeypatch.setenv("MATOMO_SITE_ID", "")
+    monkeypatch.setenv("MATOMO_TOKEN", "")
+    loaded = Settings(_env_file=None)
+    assert loaded.MATOMO_SITE_ID is None
+    assert loaded.MATOMO_TOKEN is None
+
+
+@pytest.mark.parametrize("url", [
+    "http://127.0.0.1:9",  # nothing listens on port 9, so the POST fails to connect
+    "http://matomo:not-a-port",  # httpx.InvalidURL, which is not an httpx.HTTPError
+])
+async def test_send_failure_is_logged_not_raised(monkeypatch, url):
+    monkeypatch.setattr(settings, "MATOMO_SITE_ID", 7)
+    monkeypatch.setattr(settings, "MATOMO_URL", url)
+    matomo.start()
+    with patch.object(matomo.logger, "warning") as warning:
+        await matomo.send_hit({"idsite": 7, "rec": 1})
+    await matomo.stop()
+    warning.assert_called_once()
+
+
+async def test_stop_waits_for_in_flight_hits(monkeypatch):
+    monkeypatch.setattr(settings, "MATOMO_SITE_ID", 7)
+    matomo.start()
+
+    async def slow_post(*args, **kwargs):
+        await asyncio.sleep(0.1)
+        return httpx.Response(204, request=httpx.Request("POST", "https://matomo.test"))
+
+    with patch.object(matomo._client, "post", side_effect=slow_post) as post:
+        transport = httpx.ASGITransport(app=app)
+        async with httpx.AsyncClient(transport=transport, base_url="https://cleandb-api.test") as client:
+            await client.get("/api/v1/openapi.json")
+        assert matomo._pending
+        await matomo.stop()
+    post.assert_awaited_once()
+    assert not matomo._pending
+    assert matomo._client is None