Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

## System Profile
- **Project Name:** GHAADD (GitHub Asset/Asset Downloader Daemon)
- **Tech Stack:** Python 3.11+ (Typing annotations required; `typing.NotRequired` in `daemon_control.py` and `daemon_lock.py` needs 3.11, found by the CI on 2026-10-13, and 3.10 reaches EOL in October 2026, so 3.11 is the declared minimum; `vermin` does not catch names imported from `typing`; developed and tested on 3.14; the CI runs 3.11 and 3.14 on Windows, macOS and Linux), SQLite (WAL mode enabled), JSON Configs, Cross-platform Filelocks.
- **Tech Stack:** Python 3.11+ (Typing annotations required, enforced since 2026-10-25: every function, nested ones and tests included, has a type on each parameter (Pyright `reportMissingParameterType` is an error in `pyrightconfig.json`) and a return type (`tests/test_annotations.py` reads the source, because Pyright has no rule for that); `Any` is the honest answer for test fakes and sqlite rows, not for new production code; the rule exists so Pyright has something to check; `typing.NotRequired` in `daemon_control.py` and `daemon_lock.py` needs 3.11, found by the CI on 2026-10-13, and 3.10 reaches EOL in October 2026, so 3.11 is the declared minimum; `vermin` does not catch names imported from `typing`; developed and tested on 3.14; the CI runs 3.11 and 3.14 on Windows, macOS and Linux), SQLite (WAL mode enabled), JSON Configs, Cross-platform Filelocks.
- **Dependencies:** `requests`, `python-dotenv`, `imapclient`, `filelock` (pinned lower bounds in `requirements.txt`).
- **Version:** see `modules/app_info.py` (the single source, used by `main.py` and the GUI window title; do not copy the number here)
- **Architecture:** Monolithic entry points (`main.py`, `main_gui.py`) orchestrating isolated, highly specific functional files within the `/modules` directory.
Expand Down
63 changes: 35 additions & 28 deletions main.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@
import sys
import time
from contextlib import contextmanager, nullcontext
from typing import Any, Dict, Iterator, Optional, TextIO, Tuple, Union
from modules.config_manager import (
get_config_fingerprint,
get_destination_check_every_n_polls,
get_polling_settings,
TerminalLogSettings,
get_terminal_log_settings,
get_viewer_settings,
load_config,
Expand Down Expand Up @@ -42,7 +44,7 @@
from modules.queue_worker import process_queue_once, run_ingest_and_queue_cycle, run_single_cycle


def _ensure_standard_streams():
def _ensure_standard_streams() -> None:
"""Give a console-less run (pythonw.exe, e.g. from Task Scheduler) real stdout/stderr objects.

Without a console both are None, which breaks the TeeStream wrapping and any write to them. Output is
Expand All @@ -65,7 +67,7 @@ def _ensure_standard_streams():
_ORIGINAL_STDERR = sys.stderr


def _configure_output_encoding():
def _configure_output_encoding() -> None:
"""Make console output UTF-8 and unable to crash on a character the stream cannot encode.

A redirected stream (GUI Start button, service, "> file") falls back to cp1252 on Windows, where the
Expand All @@ -84,12 +86,12 @@ def _configure_output_encoding():
class TeeStream:
"""Mirror writes to terminal and a log file stream."""

def __init__(self, primary_stream, secondary_stream):
def __init__(self, primary_stream: TextIO, secondary_stream: Optional[Union[TextIO, RollingLogFile]]) -> None:
"""Remember the real stream (primary) and the optional log file stream (secondary)."""
self.primary_stream = primary_stream
self.secondary_stream = secondary_stream

def _disable_secondary_stream(self, exc):
def _disable_secondary_stream(self, exc: Exception) -> None:
"""Drop the log file after a write error so logging trouble never stops the daemon.

The failure is reported once on the original stderr; the console keeps working.
Expand All @@ -109,7 +111,7 @@ def _disable_secondary_stream(self, exc):
file=_ORIGINAL_STDERR,
)

def write(self, data):
def write(self, data: str) -> int:
"""Write to the console, and to the log file while one is attached."""
self.primary_stream.write(data)
if self.secondary_stream is not None:
Expand All @@ -120,7 +122,7 @@ def write(self, data):
self._disable_secondary_stream(exc)
return len(data)

def flush(self):
def flush(self) -> None:
"""Flush the console and the log file (if any)."""
self.primary_stream.flush()
if self.secondary_stream is not None:
Expand All @@ -129,12 +131,12 @@ def flush(self):
except (OSError, ValueError) as exc:
self._disable_secondary_stream(exc)

def isatty(self):
def isatty(self) -> bool:
"""Report whether the console is a terminal (the log file does not count)."""
return self.primary_stream.isatty()

@property
def encoding(self):
def encoding(self) -> str:
"""Report the console's encoding, UTF-8 when the stream has none."""
return getattr(self.primary_stream, "encoding", "utf-8")

Expand All @@ -147,18 +149,18 @@ class TerminalLog:
daemon runs (each start() opens a new file).
"""

def __init__(self, settings, tees):
def __init__(self, settings: TerminalLogSettings, tees: Tuple[TeeStream, TeeStream]) -> None:
"""Keep the log settings and the two TeeStreams (stdout, stderr) whose file side this class switches."""
self.settings = settings
self._tees = tees
self._log_file = None

@property
def active(self):
def active(self) -> bool:
"""Return True while a log file is open and attached to both streams."""
return self._log_file is not None and all(tee.secondary_stream is not None for tee in self._tees)

def start(self):
def start(self) -> bool:
"""Open a new log file and mirror output into it; return True if logging is on."""
if self.active:
return True
Expand All @@ -183,7 +185,7 @@ def start(self):
print(f"Logging enabled. Writing terminal output to: {self._log_file.path}")
return True

def stop(self):
def stop(self) -> None:
"""Stop mirroring and close the log file (no-op when logging is off)."""
if self._log_file is None:
return
Expand All @@ -196,11 +198,11 @@ def stop(self):
except (OSError, ValueError):
pass

def prune(self):
def prune(self) -> int:
"""Apply the retention limit; returns how many old files were removed."""
return self._log_file.prune() if self._log_file is not None else 0

def apply_override(self, override):
def apply_override(self, override: Optional[bool]) -> None:
"""Follow a live switch: True/False force logging on/off, None returns to config.json."""
want = self.settings["enabled"] if override is None else override
if want and not self.active:
Expand All @@ -212,7 +214,7 @@ def apply_override(self, override):
self.stop()


def setup_terminal_logging(config):
def setup_terminal_logging(config: Optional[Dict[str, Any]]) -> TerminalLog:
"""Route console output through TeeStream and return the (still closed) TerminalLog.

The log file itself starts later, once main() knows it is entering a run mode, so --help and the
Expand All @@ -223,7 +225,7 @@ def setup_terminal_logging(config):
return TerminalLog(get_terminal_log_settings(config), tees)


def run_internal_smoke_tests():
def run_internal_smoke_tests() -> None:
"""Run internal smoke tests for baseline release-download behavior."""
print("🚀 Starting Internal Smoke Tests...")

Expand All @@ -248,7 +250,7 @@ def run_internal_smoke_tests():


@contextmanager
def _viewer_push_scope(pusher):
def _viewer_push_scope(pusher: Optional[ViewerPusher]) -> Iterator[None]:
"""Stop pushing to the viewer (telling it goodbye) however the polling loop ends."""
try:
yield
Expand All @@ -258,8 +260,13 @@ def _viewer_push_scope(pusher):


def run_polling_loop(
interval_seconds, jitter_min_seconds, jitter_max_seconds, terminal_log=None, config_fingerprint=None, idle=False
):
interval_seconds: float,
jitter_min_seconds: int,
jitter_max_seconds: int,
terminal_log: Optional[TerminalLog] = None,
config_fingerprint: Optional[str] = None,
idle: bool = False,
) -> None:
"""Run processing continuously with randomized jitter between cycles.

With `idle` (polling.enabled is false, started with --daemon) nothing is polled on a schedule: the daemon
Expand Down Expand Up @@ -292,24 +299,24 @@ def run_polling_loop(
else None
)

def apply_push_override(override):
def apply_push_override(override: Optional[bool]) -> None:
"""Follow a live push switch (None = the viewer.enabled setting of config.json)."""
if pusher is not None:
pusher.apply_override(override, viewer_settings["enabled"])

def log_is_active():
def log_is_active() -> bool:
"""Return whether the terminal log file is open right now (published for the GUI)."""
return bool(terminal_log is not None and terminal_log.active)

def apply_log_override(override):
def apply_log_override(override: Optional[bool]) -> None:
"""Follow a live log switch, then publish whether the file really is open (the GUI shows it)."""
if terminal_log is None: # nothing to switch (never registered then, but keeps the types honest)
return
terminal_log.apply_override(override)
if control_enabled:
update_daemon_status(log_active=log_is_active())

def publish_wait_state(paused, next_poll_at):
def publish_wait_state(paused: bool, next_poll_at: Optional[float]) -> None:
"""Publish pause state and next poll time to the status file, and announce pause/resume once."""
nonlocal announced_paused
if control_enabled:
Expand Down Expand Up @@ -358,14 +365,14 @@ def run_folder_checks(reason: str) -> None:
run_folder_checks("at start") # counts and warnings are there right away, not after N polls
run_backup_check() # a daemon that was off for a while catches up at once

def announce_requested_poll(wait_result):
def announce_requested_poll(wait_result: Optional[str]) -> None:
"""Say which poll was asked for; and note that a pause goes on afterwards."""
what = "Single poll requested (one notification, one queue item)" if wait_result == "single" else "Poll now requested"
print(f"⏩ {what}." + (" Polling stays paused afterwards." if watcher.forced_while_paused else ""))
if control_enabled:
update_daemon_status(last_forced_poll_handled=watcher.last_handled_request)

def wait_for_poll_now():
def wait_for_poll_now() -> Optional[str]:
"""Idle mode: wait (no countdown) until Poll now / Single poll or Stop; the request, or None on Stop."""
wait_result = watcher.wait(0, on_change=publish_wait_state, idle=True)
if wait_result == "stop":
Expand Down Expand Up @@ -434,7 +441,7 @@ def wait_for_poll_now():
cycle += 1


def run_backup_check():
def run_backup_check() -> None:
"""Make a backup when the schedule in config.json says one is due (never stops the daemon).

The settings are read afresh each time, so a change in the Backup window applies without a restart.
Expand All @@ -455,7 +462,7 @@ def run_backup_check():
log_warning("BACKUP", result.message)


def run_drain_queue_loop(github_token):
def run_drain_queue_loop(github_token: Optional[str]) -> None:
"""Process only due queue jobs (no email ingestion) until the queue is fully empty."""
print("Drain-queue mode enabled. Email ingestion is skipped.\n")

Expand Down Expand Up @@ -484,7 +491,7 @@ def run_drain_queue_loop(github_token):
cycle += 1


def main():
def main() -> None:
"""Entry point: parse the CLI, then run one of the modes (one-shot command, single, drain, once, polling).

Only mutating run modes take the daemon lock and start the terminal log; dry-run and the read-only
Expand Down
24 changes: 12 additions & 12 deletions modules/asset_downloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
)
from modules.payload_types import DownloadReleaseResult, DownloadResultPayload, ReleaseAssetQueueItem, SkippedItemPayload
from requests.exceptions import ChunkedEncodingError, ConnectionError, Timeout
from typing import Literal, Optional, cast
from typing import Any, cast, Dict, Literal, Optional
from urllib.parse import quote, urljoin

# Load environment variables from .env.
Expand Down Expand Up @@ -806,7 +806,7 @@ def _github_archive_name(repo_name: str, tag: str, extension: str) -> str:
return f"{repo_name}-{version}.{extension}"


def _probe_archive_name(session, repo: str, tag: str, extension: str, fallback: str) -> str:
def _probe_archive_name(session: requests.Session, repo: str, tag: str, extension: str, fallback: str) -> str:
"""Return the file name GitHub sends for the release page's own archive link, else the fallback."""
url = f"https://github.com/{repo}/archive/refs/tags/{quote(tag, safe='/')}.{extension}"
try:
Expand All @@ -820,7 +820,7 @@ def _probe_archive_name(session, repo: str, tag: str, extension: str, fallback:
return name if name and name != "unknown" else fallback


def sanitize_folder_name(text):
def sanitize_folder_name(text: Optional[str]) -> str:
"""Return a Windows-safe folder name using JS-compatible rules."""
if not text:
return "unknown"
Expand All @@ -829,7 +829,7 @@ def sanitize_folder_name(text):
return text.strip()


def get_short_commit_hash(repo, tag, headers):
def get_short_commit_hash(repo: str, tag: str, headers: Dict[str, str]) -> Any | Literal['unknown-commit']:
"""Return the 7-character commit hash for a specific tag."""
url = f"https://api.github.com/repos/{repo}/commits/{tag}"
response = requests.get(url, headers=headers)
Expand All @@ -838,7 +838,7 @@ def get_short_commit_hash(repo, tag, headers):
return "unknown-commit"


def build_folder_name(repo, release_data, headers, raw_name):
def build_folder_name(repo: str, release_data: Dict[str, Any], headers: Dict[str, str], raw_name: str) -> str:
"""Build a release folder name from a specific display name."""
pub_time_str = release_data.get("published_at", "")
if pub_time_str:
Expand All @@ -858,13 +858,13 @@ def build_folder_name(repo, release_data, headers, raw_name):
return f"{formatted_date}, {safe_name}, {safe_tag}, {safe_commit}"


def generate_folder_name(repo, release_data, headers):
def generate_folder_name(repo: str, release_data: Dict[str, Any], headers: Dict[str, str]) -> str:
"""Generate the default folder name based on the release title."""
release_name = release_data.get("name") or release_data.get("tag_name", "unknown-name")
return build_folder_name(repo, release_data, headers, release_name)


def is_state_persistence_enabled():
def is_state_persistence_enabled() -> bool:
"""Return whether persistent duplicate-detection state is enabled."""
return not is_state_persistence_disabled()

Expand All @@ -883,7 +883,7 @@ def build_expected_signature(item: ReleaseAssetQueueItem) -> Optional[str]:
return "|".join(signature_parts)


def get_release_data(repo, tag):
def get_release_data(repo: str, tag: str) -> Any | None:
"""Return release JSON for a repository tag, or None when unavailable."""
url = f"https://api.github.com/repos/{repo}/releases/tags/{tag}"

Expand All @@ -907,7 +907,7 @@ def get_release_data(repo, tag):
return None


def get_release_attestation_url(release_data, headers):
def get_release_attestation_url(release_data: Dict[str, Any], headers: Dict[str, str]) -> str | None:
"""Return the release attestation download URL from the expanded assets page.

GitHub exposes this link in expanded-assets HTML, not in release JSON.
Expand Down Expand Up @@ -1117,15 +1117,15 @@ def download_all_assets(
per_file_retry_delay = 2
normal_downloaded_count = 0

def refresh_release_state():
def refresh_release_state() -> None:
"""Reload the stored state of this release from state.db into the enclosing function's variable."""
nonlocal release_state
if state_db is not None:
release_state = load_release_state(state_db, release_key)

def should_skip_existing_file(file_path, item_key, expected_signature, remote_size, remote_last_modified, remote_etag):
def should_skip_existing_file(file_path: Optional[str], item_key: str, expected_signature: Optional[str], remote_size: Any, remote_last_modified: Any, remote_etag: Any) -> bool:
"""Return True when the local file can be treated as up to date."""
if not os.path.exists(file_path):
if file_path is None or not os.path.exists(file_path):
return False

state_entry = release_state.get(item_key, {})
Expand Down
Loading
Loading