diff --git a/common/lib/archive_metadata.py b/common/lib/archive_metadata.py new file mode 100644 index 000000000..c4b8886a6 --- /dev/null +++ b/common/lib/archive_metadata.py @@ -0,0 +1,648 @@ +""" +Read and write metadata files packaged inside dataset archives. + +4CAT processors that produce ZIP archives can include a small JSON file +describing the other files in the archive — what they are and where they came +from. This module provides `ArchiveMetadataFile`, a base class handling the +shared plumbing (locating the file inside a zip or results folder, schema +versioning, atomic writes), and `MediaArchiveMetadata`, the schema for media +download archives. + +`MediaArchiveMetadata` record per media file the source post IDs, +originating URL (if any), and any platform-specific data. Download attempts +that produced no file are recorded separately as failures. + +TODO: +There are other metadata files (e.g. token sets, topic models) with their own +metadata files with quite different schemas. They seem like they need their +own classes as well (or would be super redundant in `ArchiveMetadataFile` +format). +""" +import json +import os +import zipfile +from pathlib import Path +from typing import Iterator, Optional + +from common.lib.exceptions import MetadataException + + +class ArchiveMetadataFile: + """ + Base class for JSON metadata files inside dataset archives. + + Handles only the shared plumbing: locate the file (zip member or + results folder), read it, check the schema version, and writing it + back. Subclasses define the actual schema by implementing + `_populate_from_raw` and `to_dict`, and optionally `validate`. + """ + + # Subclasses must set these. + SCHEMA_VERSION: Optional[int] = None + DEFAULT_FILENAME: Optional[str] = None + + def __init__(self, dataset=None, processor_type: Optional[str] = None, filename: Optional[str] = None): + self.dataset = dataset + # store the processor that created the archive and metadata + # leaving optional in cases we create metadata... some other way + self.processor_type = processor_type + # the 4CAT version that created the metadata: resolved by `new()` + self.software_version = None + self.software_source = None + + # filename and schema + self._filename = filename or self.DEFAULT_FILENAME + self.schema_version = self.SCHEMA_VERSION + + def _resolve_software_version(self) -> None: + """ + Record the 4CAT commit / repository creating this metadata. + + Called by `new()` on the producer path. The dataset already records + the version its processor runs with (stored when the processor + starts), so it is copied from there. A no-op without a dataset (e.g. + metadata constructed outside a processor run). + """ + if self.dataset is None: + return + self.software_version = self.dataset.data.get("software_version") + self.software_source = self.dataset.data.get("software_source") + + @classmethod + def read(cls, dataset, *, filename: Optional[str] = None): + """ + Load metadata from a finished dataset's archive or results folder. + + :param dataset: the DataSet to read from. + :param filename: metadata filename; defaults to the class's + `DEFAULT_FILENAME`. + + :raises FileNotFoundError: the metadata file is not present. + :raises MetadataException: the dataset is unfinished, or the file is + malformed or claims an unsupported schema version. + """ + filename = filename or cls.DEFAULT_FILENAME + + # an unfinished dataset may still be writing its metadata. A finished + # dataset with no items is still read: an archive in which every + # download failed, or whose files were all copied from an earlier + # download, still has metadata worth reading. If there is no archive + # at all, reading it raises FileNotFoundError below. + if not dataset.is_finished(): + raise MetadataException("Dataset is not finished; metadata may be incomplete.") + + # load it + raw = cls._load_raw(dataset, Path(dataset.get_results_path()), filename) + # create the class + instance = cls(dataset, filename=filename) + # and populate + instance._populate_from_raw(raw) + return instance + + @classmethod + def _load_raw(cls, dataset, results_file: Path, filename: str) -> dict: + """ + Locate and JSON-decode the metadata file (zip member or folder). + + :raises FileNotFoundError: the file is not present. + :raises MetadataException: the archive is damaged, or the file is + present but not valid JSON. + """ + # if the dataset has a zip archive, look inside it + if results_file.suffix == ".zip": + try: + with zipfile.ZipFile(results_file, "r") as archive: + if filename not in archive.namelist(): + raise FileNotFoundError(f"No metadata file {filename} in archive {results_file.name}.") + with archive.open(filename) as f: + return cls._decode_json(f, filename) + except zipfile.BadZipFile as e: + raise MetadataException(f"Archive {results_file.name} cannot be opened: {e}") from e + + # it is possible we have a results folder w/ metadata instead; look there + results_folder = dataset.get_results_folder_path() + if not results_folder.is_dir(): + raise FileNotFoundError(f"No metadata file {filename} found (no results folder).") + + # check that the file exists + metadata_path = results_folder / filename + if not metadata_path.is_file(): + raise FileNotFoundError(f"No metadata file {filename} in {results_folder}.") + + # load the metadata! (`write` always writes UTF-8) + with open(metadata_path, encoding="utf-8") as f: + return cls._decode_json(f, filename) + + @staticmethod + def _decode_json(fp, filename: str) -> dict: + """ + JSON-decode an open file object, surfacing decode errors as + `MetadataException`. Without this a corrupt metadata file raises + `json.JSONDecodeError` (or `UnicodeDecodeError` for text that is not + UTF-8), which escapes the `(FileNotFoundError, MetadataException)` + handler every consumer uses. + """ + try: + return json.load(fp) + except ValueError as e: + # JSONDecodeError and UnicodeDecodeError are both ValueErrors + raise MetadataException(f"Metadata file {filename} is not valid JSON: {e}") from e + + @classmethod + def _check_schema_version(cls, raw: dict) -> Optional[int]: + """ + Read the `schema_version` of a raw metadata dict. + + Returns the declared version, or None if the file predates schema + versioning (legacy). Raises MetadataException for a version this + class does not know how to read. + + This is meant to allow us to change the schema later if needed + """ + version = raw.get("schema_version") + if version is not None and version != cls.SCHEMA_VERSION: + raise MetadataException( + f"Unsupported metadata schema_version {version}; expected {cls.SCHEMA_VERSION}." + ) + return version + + # -- subclass hooks -- + + def _populate_from_raw(self, raw: dict) -> None: + """Populate instance state from a raw decoded dict.""" + raise NotImplementedError + + def to_dict(self) -> dict: + """Serialise instance state to a JSON-encodable dict.""" + raise NotImplementedError + + def validate(self) -> None: + """Strict schema check run before writing. Override as needed.""" + pass + + # -- output -- + + def write(self, staging_area, *, filename: Optional[str] = None) -> Path: + """ + Validate and atomically write the metadata file to + `/`. Returns the path written. + """ + if filename is None: + filename = self._filename + + # validate before writing; this will be a sanity check for fields (or whatever) + self.validate() + + # write it first to a staging area, then move it to the target location + # should avoid leaving a half-written file if something goes wrong + staging_area = Path(staging_area) + target = staging_area / filename + tmp = target.parent / (target.name + ".tmp") + try: + with open(tmp, "w", encoding="utf-8") as f: + json.dump(self.to_dict(), f) + os.replace(tmp, target) + except Exception: + # don't leave a partial .tmp file behind to be zipped into the archive + tmp.unlink(missing_ok=True) + raise + return target + + +class MediaArchiveMetadata(ArchiveMetadataFile): + """ + Metadata for media download archives (`.metadata.json`). + + Keyed by output filename; each item records the source post IDs, the + originating URL (if any), and an `extra` blob of platform-specific data. + Download attempts that produced no file are recorded in `failures`. + + An item or failure can be reached through more than one URL, e.g. a short + and a long link to the same video, or a link in a post that redirected to + the file. The entry's `url` is one of them; the others are listed in + `extra["other_urls"]`, so that the entry can be found by any of them. + """ + + SCHEMA_VERSION = 1 + DEFAULT_FILENAME = ".metadata.json" + + def __init__(self, dataset=None, *, processor_type: Optional[str] = None, + from_dataset: Optional[str] = None, + filename: Optional[str] = None): + # `dataset` is the target dataset (the one the archive belongs to). + # `from_dataset` is the source dataset key whose posts are being + # downloaded from; these are not generally the same and there is no + # safe default — producers must set it explicitly. + super().__init__(dataset, processor_type=processor_type, filename=filename) + # store the source dataset key (e.g. top dataset or some subdataset) + self.from_dataset = from_dataset + + # the actual metadata content + self.items: dict = {} + self.failures: list = [] + + # -- constructors -- + + @classmethod + def new(cls, dataset=None, *, processor_type: str, + from_dataset: Optional[str] = None, + filename: Optional[str] = None) -> "MediaArchiveMetadata": + """ + Empty container for a producer that is building an archive. + + :param dataset: the DataSet the archive will belong to. + :param processor_type: producer identifier, typically `processor.type`. + :param from_dataset: key of the source dataset whose posts are being + downloaded from. + :param filename: metadata filename inside the archive. + """ + instance = cls(dataset, processor_type=processor_type, + from_dataset=from_dataset, filename=filename) + instance._resolve_software_version() + return instance + + # -- schema population / legacy normalization -- + + def _populate_from_raw(self, raw: dict) -> None: + """ + Populate fields from a raw dict. v1 is read directly; older shapes + are translated through `_normalize_legacy`. + """ + if not isinstance(raw, dict): + raise MetadataException("Metadata file is not a JSON object.") + + version = self._check_schema_version(raw) + if version == self.SCHEMA_VERSION: + # current schema. `_check_schema_version` has already rejected any + # mismatching version, so this is the only non-legacy branch. + self.schema_version = version + self.from_dataset = raw.get("from_dataset", self.from_dataset) + # pretty sure we did not store the processor or version info previously + self.processor_type = raw.get("processor_type") + self.software_version = raw.get("software_version") + self.software_source = raw.get("software_source") + + # archives can be imported from elsewhere, so the file may have + # been edited by hand: refuse a file whose overall layout is wrong, + # and skip single entries that are not what consumers expect + items = raw.get("items") or {} + failures = raw.get("failures") or [] + if not isinstance(items, dict): + raise MetadataException("Metadata 'items' must be a JSON object.") + if not isinstance(failures, list): + raise MetadataException("Metadata 'failures' must be a list.") + + # post_ids are string-coerced on write, but normalize on read too + # so consumers never have to: some source datasets use integer ids, + # and hand-edited or externally-produced files may carry them. + self.items = { + filename: self._normalize_entry_post_ids(entry) + for filename, entry in items.items() + if isinstance(entry, dict) + } + self.failures = [ + self._normalize_entry_post_ids(failure) + for failure in failures + if isinstance(failure, dict) + ] + return + + # version is None here (a mismatching version would have raised + # above): this is a pre-schema-versioned file. + self._normalize_legacy(raw) + + def _normalize_legacy(self, raw: dict) -> None: + """ + Translate pre-v1 metadata to v1. + + Old format is a flat dict keyed by URL (image/video) or filename + (Telegram). Each entry has a `success` flag; failed entries become + `failures[]`, successful entries become `items[filename]`. The video + downloader's nested `files[]` list explodes to one item per file, and + a file listed under several URLs gets the post IDs of all of them. + All producer-specific data is carried across into `extra` (items) or + the failure dict (failures) so the migration loses nothing — see + `_legacy_extra`. + """ + for outer_key, entry in raw.items(): + if not isinstance(entry, dict): + # malformed; should not exist + continue + + post_ids = entry.get("post_ids") + if post_ids is None and "post_id" in entry: + post_ids = [entry["post_id"]] + post_ids = self._normalize_post_ids(post_ids) + + url = entry.get("url") + # check if the key was a URL + if url is None and isinstance(outer_key, str) and ( + outer_key.startswith("http://") or outer_key.startswith("https://")): + url = outer_key + + # existing `from_dataset` is correct if exists; video downloads + # from early 2023 called it `source_dataset` + recorded_source = entry.get("from_dataset") or entry.get("source_dataset") + if recorded_source: + self.from_dataset = recorded_source + + success = entry.get("success", True) + + # `files` key from video downloader due to channels and playlists. + # A URL can list files even when the URL as a whole was not marked + # successful: e.g. yt-dlp recognised a video it had already + # downloaded for an earlier URL, or stopped part-way through a + # playlist. Those files are in the archive, so each one is recorded; + # the URL itself only counts as a failure if it has no file at all. + files = entry.get("files") + if isinstance(files, list) and files: + files_recorded = False + for file in files: + if not isinstance(file, dict): + continue + file_filename = file.get("filename") + if not file.get("success", True): + # the shared outer entry data is preserved alongside the + # per-file data so e.g. the video `downloader` survives + self.failures.append(self._legacy_failure( + post_ids, url, + file.get("reason"), + file.get("reason_description") or file.get("error"), + entry, file, + )) + files_recorded = True + continue + if not file_filename or not isinstance(file_filename, str): + continue + # the same file can be listed under more than one URL; + # merge so the posts of every URL stay connected to it + self.add_item(file_filename, post_ids=post_ids, url=url, + extra=self._legacy_extra(entry, file), merge=True) + files_recorded = True + if files_recorded or success: + continue + + if not success: + reason = entry.get("reason") + if reason is None and entry.get("retry") is False: + # the old video downloader marked URLs that turned out not + # to be videos with `retry: False` rather than a reason + reason = "not_a_video" + self.failures.append(self._legacy_failure( + post_ids, url, + reason, + entry.get("reason_description") or entry.get("error"), + entry, + )) + elif entry.get("filename") and isinstance(entry["filename"], str): + self.add_item(entry["filename"], post_ids=post_ids, url=url, + extra=self._legacy_extra(entry), merge=True) + # else: malformed-but-tolerated; drop + + # Legacy entry keys promoted to first-class v1 fields, or pure structure; + # every other key is producer data and is preserved under `extra`. + _LEGACY_NON_EXTRA_KEYS = frozenset({ + "filename", "post_ids", "post_id", "url", "from_dataset", "source_dataset", + "success", "files", "metadata", "extra", + }) + # Additionally promoted on the failure path: failures have no `extra` of + # their own, but carry these as first-class fields. + _LEGACY_FAILURE_KEYS = frozenset({"reason", "reason_description", "error"}) + + @classmethod + def _legacy_extra(cls, *sources, exclude=frozenset()) -> dict: + """ + Flatten producer data from one or more legacy dicts into a single flat + `extra` dict; later sources win on a key clash. + + Pre-v1 files had no `extra` key — per-file data (yt-dlp dumps, the + Stable Diffusion `prompt`, the video `downloader`, ...) sat at the top + level or inside a `metadata` sub-dict. Everything not promoted to a + first-class v1 field is kept so the migration loses nothing. `extra` + is kept flat because v1 consumers read keys off it directly. + """ + skip = cls._LEGACY_NON_EXTRA_KEYS | exclude + extra = {} + for source in sources: + if not isinstance(source, dict): + continue + extra.update({k: v for k, v in source.items() if k not in skip}) + # a `metadata` (yt-dlp dump) or explicit `extra` is flattened in + for nested_key in ("metadata", "extra"): + nested = source.get(nested_key) + if isinstance(nested, dict): + extra.update(nested) + return extra + + @classmethod + def _legacy_failure(cls, post_ids, url, reason, reason_description, + *extra_sources) -> dict: + """ + Build a v1 failure dict from legacy data. Leftover producer data is + preserved under the failure's own `extra` key. + """ + failure = { + "post_ids": post_ids, + "reason": reason or "error", + "reason_description": reason_description or "", + } + if url is not None: + failure["url"] = url + extra = cls._legacy_extra(*extra_sources, exclude=cls._LEGACY_FAILURE_KEYS) + if extra: + failure["extra"] = extra + return failure + + @staticmethod + def _build_item(filename: str, post_ids: list, url, extra: dict) -> dict: + item = {"filename": filename, "post_ids": list(post_ids)} + if url is not None: + item["url"] = url + if extra: + item["extra"] = extra + return item + + # -- mutation -- + + def add_item(self, filename: str, *, post_ids, + url: Optional[str] = None, + extra: Optional[dict] = None, + replace: bool = False, + merge: bool = False) -> None: + """ + Record a successfully produced file. + + :param filename: the file's name inside the archive. Must be unique + within this metadata instance unless `replace` or `merge` is set. + :param post_ids: list of source post IDs (string-coerced). + :param url: optional originating URL. + :param extra: optional free-form per-file data (e.g. yt-dlp dump). + :param replace: overwrite an existing entry with the same filename. + :param merge: if an entry with the same filename exists, add these + post IDs to it instead of overwriting it. Use this when one file + can be reached through several URLs (e.g. the same video linked in + different ways), so that every post linking to it stays connected + to the file. The existing URL is kept, and any other URL that led + to the same file is listed in `extra["other_urls"]`. The existing + extra data is kept; extra keys the entry does not have yet are + added. + """ + if not filename or not isinstance(filename, str): + raise MetadataException("add_item: 'filename' must be a non-empty string.") + if replace and merge: + raise MetadataException("add_item: use either 'replace' or 'merge', not both.") + + existing = self.items.get(filename) + if existing is not None and merge: + known_ids = existing.setdefault("post_ids", []) + for post_id in self._normalize_post_ids(post_ids): + if post_id not in known_ids: + known_ids.append(post_id) + if extra: + existing["extra"] = {**extra, **existing.get("extra", {})} + if url is not None: + if existing.get("url") is None: + existing["url"] = url + elif url != existing["url"]: + entry_extra = existing.setdefault("extra", {}) + other_urls = list(entry_extra.get("other_urls", [])) + if url not in other_urls: + other_urls.append(url) + entry_extra["other_urls"] = other_urls + return + + if existing is not None and not replace: + raise MetadataException(f"add_item: filename {filename!r} already present.") + self.items[filename] = self._build_item( + filename, self._normalize_post_ids(post_ids), url, dict(extra) if extra else {} + ) + + def add_failure(self, *, post_ids, reason: str, + reason_description: Optional[str] = None, + url: Optional[str] = None, + extra: Optional[dict] = None) -> None: + """ + Record a download attempt that did not produce a file. + + :param post_ids: list of source post IDs (string-coerced; may be empty). + :param reason: structured failure code (e.g. "error", "no_media"). + :param reason_description: optional human-readable explanation. + :param url: optional URL that was attempted. + :param extra: optional free-form data about the attempt. + """ + if not reason or not isinstance(reason, str): + raise MetadataException("add_failure: 'reason' must be a non-empty string.") + failure = { + "post_ids": self._normalize_post_ids(post_ids), + "reason": reason, + } + if url is not None: + failure["url"] = url + if reason_description is not None: + failure["reason_description"] = reason_description + if extra: + failure["extra"] = dict(extra) + self.failures.append(failure) + + @staticmethod + def _normalize_post_ids(post_ids) -> list: + if post_ids is None: + return [] + if isinstance(post_ids, (str, int)): + post_ids = [post_ids] + # coerce to stripped strings, dropping null and blank ids rather than + # keeping them as the text "None" or "": some media (e.g. Instagram ads) + # records no usable id, and a fabricated id would only mislead anything + # that later matches on it. + return [str(p).strip() for p in post_ids if p is not None and str(p).strip()] + + @classmethod + def _normalize_entry_post_ids(cls, entry): + """ + Return a copy of an item/failure dict with `post_ids` coerced to a + list of strings. Non-dict entries are returned unchanged. + """ + if not isinstance(entry, dict) or "post_ids" not in entry: + return entry + entry = dict(entry) + entry["post_ids"] = cls._normalize_post_ids(entry["post_ids"]) + return entry + + # -- access -- + + def get_entry(self, filename: str) -> Optional[dict]: + return self.items.get(filename) + + def iter_entries(self) -> Iterator[tuple]: + return iter(self.items.items()) + + def iter_failures(self) -> Iterator[dict]: + return iter(self.failures) + + def filename_to_post_ids(self) -> dict: + """ + Return a `{filename: [post_id, ...]}` mapping for successful entries. + + Replaces the per-consumer map-building that strips extensions + and walks nested `files[]` lists. + """ + return {fn: list(entry.get("post_ids", [])) for fn, entry in self.items.items()} + + def post_ids_for(self, filename: str) -> list: + """ + Return the list of source post IDs associated with a given filename, or + empty list if the filename is not present or has no post IDs. + """ + entry = self.items.get(filename) + return list(entry.get("post_ids", [])) if entry else [] + + # -- some helpers for common access patterns -- + + def __len__(self) -> int: + return len(self.items) + + def __contains__(self, filename) -> bool: + return filename in self.items + + # -- output -- + + def to_dict(self) -> dict: + return { + "schema_version": self.schema_version, + "from_dataset": self.from_dataset, + "processor_type": self.processor_type, + "software_version": self.software_version, + "software_source": self.software_source, + "items": self.items, + "failures": self.failures, + } + + def validate(self) -> None: + """ + Strict schema check used before writing. Raises MetadataException on + any violation. + """ + if self.schema_version != self.SCHEMA_VERSION: + raise MetadataException(f"Unsupported schema_version {self.schema_version}.") + if not isinstance(self.items, dict): + raise MetadataException("items must be a dict.") + if not isinstance(self.failures, list): + raise MetadataException("failures must be a list.") + for filename, entry in self.items.items(): + if not isinstance(entry, dict): + raise MetadataException(f"items[{filename!r}] must be a dict.") + if entry.get("filename") != filename: + raise MetadataException( + f"items[{filename!r}].filename must match its key " + f"(got {entry.get('filename')!r})." + ) + if not isinstance(entry.get("post_ids"), list): + raise MetadataException(f"items[{filename!r}].post_ids must be a list.") + for i, failure in enumerate(self.failures): + if not isinstance(failure, dict): + raise MetadataException(f"failures[{i}] must be a dict.") + if not isinstance(failure.get("post_ids"), list): + raise MetadataException(f"failures[{i}].post_ids must be a list.") + reason = failure.get("reason") + if not isinstance(reason, str) or not reason: + raise MetadataException(f"failures[{i}].reason must be a non-empty string.") diff --git a/common/lib/dataset.py b/common/lib/dataset.py index 471fd77d7..0dcd50102 100644 --- a/common/lib/dataset.py +++ b/common/lib/dataset.py @@ -1861,15 +1861,19 @@ def update_version(self, version): except AttributeError: processor_path = "" + version_data = { + "software_version": version[0], + "software_source": version[1], + "software_file": processor_path, + } updated = self.db.update( "datasets", where={"key": self.data["key"]}, - data={ - "software_version": version[0], - "software_source": version[1], - "software_file": processor_path, - }, + data=version_data, ) + # keep this object in step with the database, so code that runs later + # in the same processor (e.g. writing archive metadata) sees the version + self.data.update(version_data) return updated > 0 @@ -2409,80 +2413,44 @@ def get_media_from_children(self, item_ids=None) -> dict: :param list item_ids: A list of item IDs to limit the filename retrieval to. returns dict: item_id as key and a list of tuples (child dataset key -> filename) as items """ - children = self.get_children() + from common.lib.exceptions import MetadataException + children = self.get_children() if not children: return {} if item_ids is None: item_ids = [] - media_map = {} - - # Get children that are image/video downloaders media_datasets = [ p for p in children if ("video-downloader" in p.type or "image-downloader" in p.type) and p.data.get("num_rows", 0) > 0 and p.is_finished() ] - # Loop through media datasets and create a map of dataset key -> filenames - # Skip files that were downloaded multiple times + media_map = {} seen_files = set() + item_ids = [str(i_id) for i_id in item_ids] for media_dataset in media_datasets: - for item in media_dataset.iterate_items(): - - if item.file.name == ".metadata.json": - with item.file.open() as infile: - metadata = json.load(infile) - - for url_key, item_metadata in metadata.items(): - media_items = set() - post_ids = item_metadata.get("post_ids", []) # Required - - if not post_ids: - continue - - # Make sure we're matching and passing strings - post_ids = [str(p_id) for p_id in post_ids] - item_ids = [str(i_id) for i_id in item_ids] - - # Skip items that are not in the requested item_ids - if item_ids and not any(p_id in item_ids for p_id in post_ids): - continue - - # Single file (images usually format like this) - is_success = item_metadata.get("success", True) - - if is_success and "filename" in item_metadata: - media_info = (media_dataset.key, item_metadata["filename"]) - media_items.add(media_info) - - # Multiple files (videos with the 'files' array) - if item_metadata.get("files"): - for file in item_metadata["files"]: - if file.get("success") and "filename" in file: - media_info = (media_dataset.key, file["filename"]) - media_items.add(media_info) - - if not media_items: - continue - - # Append to post_id list - for post_id in post_ids: - if post_id not in media_map: - media_map[post_id] = [] - - for media_item in media_items: - if media_item not in media_map[post_id]: - # Don't add post_id -> filename couplings that we've already seen - media_ref = (post_id, media_item[1]) - if media_ref not in seen_files: - media_map[post_id].append(media_item) - seen_files.add(media_ref) - - # break after .metadata.json - break + try: + metadata = media_dataset.read_media_metadata() + except (FileNotFoundError, MetadataException): + continue + + for filename, entry in metadata.iter_entries(): + post_ids = [str(p_id) for p_id in entry.get("post_ids", [])] + if not post_ids: + continue + if item_ids and not any(p_id in item_ids for p_id in post_ids): + continue + + media_info = (media_dataset.key, filename) + for post_id in post_ids: + media_ref = (post_id, filename) + if media_ref in seen_files: + continue + seen_files.add(media_ref) + media_map.setdefault(post_id, []).append(media_info) return media_map @@ -2502,6 +2470,33 @@ def get_metadata(self): metadata["current_4CAT_version"] = get_software_version() return metadata + def read_media_metadata(self, filename=".metadata.json"): + """ + Load this dataset's media-archive `.metadata.json`. + + Returns a `MediaArchiveMetadata` instance. Legacy formats are + normalized on load so callers always see the current schema. + + Raises `FileNotFoundError` if no metadata file is present and + `MetadataException` if the dataset is unfinished or the file is + malformed. + """ + from common.lib.archive_metadata import MediaArchiveMetadata + return MediaArchiveMetadata.read(self, filename=filename) + + def new_media_metadata(self, from_dataset, processor_type, filename=".metadata.json"): + """ + Empty `MediaArchiveMetadata` container for a processor producing a + media archive for this dataset. + + :param str from_dataset: key of the *source* dataset whose posts are + being downloaded from (this is generally not the same as `self`). + :param str processor_type: type of the processor, used for metadata. + """ + from common.lib.archive_metadata import MediaArchiveMetadata + return MediaArchiveMetadata.new(self, processor_type=processor_type, + from_dataset=from_dataset, filename=filename) + def get_result_url(self): """ Gets the 4CAT frontend URL of a dataset file. diff --git a/common/lib/exceptions.py b/common/lib/exceptions.py index b0b798a4c..209d76db6 100644 --- a/common/lib/exceptions.py +++ b/common/lib/exceptions.py @@ -162,3 +162,9 @@ class MediaSignatureException(FourcatException): Raise in media processors when the media cannot be read """ pass + +class MetadataException(FourcatException): + """ + Raise when there is an issue with metadata + """ + pass diff --git a/common/lib/media_archive_library.py b/common/lib/media_archive_library.py new file mode 100644 index 000000000..3145deeb1 --- /dev/null +++ b/common/lib/media_archive_library.py @@ -0,0 +1,199 @@ +""" +Look up media that previous downloader runs already fetched. + +When a downloader (currently the video downloader) runs, other downloaders of +the same kind may have already fetched some of the same URLs from the same +source data. `MediaArchiveLibrary` aggregates the `MediaArchiveMetadata` of +those previous runs +""" +from common.lib.exceptions import MetadataException, DataSetException + + +class MediaLibraryHit: + """ + Result of a `MediaArchiveLibrary.find()` lookup. + + A success hit carries the archive the files live in (`metadata`, whose + `.dataset` locates the zip) and the matching `(filename, item)` entries. + A failure hit carries every failure recorded for the URL, as + `(metadata, failure)` pairs, so that a consumer can see not only why each + attempt failed but which dataset made it - and therefore what settings it + ran with, since a dataset stores the parameters it was created with. + """ + + def __init__(self, is_success, metadata=None, entries=None, failures=None): + self.is_success = is_success + self.metadata = metadata + self.entries = entries or [] + self.failures = failures or [] + + @property + def reasons(self) -> set: + """ + The failure reasons seen for this URL across all previous archives. + + :return set: reason codes, e.g. `{"not_a_video"}` + """ + return {failure.get("reason") for _, failure in self.failures if failure.get("reason")} + + def datasets_for_reason(self, reason) -> list: + """ + The datasets whose archives recorded a given failure reason for this URL. + + Use this to compare a previous attempt's settings against the current + run's: each dataset stores the parameters it ran with, so nothing about + those settings has to be repeated in the metadata file. + + :param str reason: reason code to look for, e.g. `"not_a_video"` + :return list: the datasets that recorded it, skipping any metadata that + was not read from a dataset + """ + return [metadata.dataset for metadata, failure in self.failures + if failure.get("reason") == reason and metadata.dataset is not None] + + +class MediaArchiveLibrary: + """ + Aggregate of `MediaArchiveMetadata` from previous downloader datasets. + + Construct via `collect()` inside a processor; the bare constructor takes + metadata objects directly and is intended for testing. + """ + + def __init__(self, metadata_objects, current_dataset=None): + self.metadata_objects = list(metadata_objects) + self.current_dataset = current_dataset + self._url_index = None + + @classmethod + def collect(cls, current_dataset, modules, compatible_types): + """ + Build a library from finished downloader datasets that share + `current_dataset`'s source data. + + :param current_dataset: the dataset being produced now (excluded + from the result). + :param modules: module registry, used to resolve the original + dataset of a filtered set. + :param list compatible_types: processor types whose archives count, + e.g. `["video-downloader"]`. + """ + datasets = cls._collect_previous_downloaders(current_dataset, modules, compatible_types) + metadata_objects = [] + for dataset in datasets: + try: + metadata_objects.append(dataset.read_media_metadata()) + except (FileNotFoundError, MetadataException): + # no metadata, or unfinished/malformed — nothing to reuse + continue + + if current_dataset is not None: + current_dataset.log( + f"Media library: {len(metadata_objects)} previous " + f"{'/'.join(compatible_types)} archive(s) available for reuse" + ) + return cls(metadata_objects, current_dataset=current_dataset) + + @staticmethod + def _collect_previous_downloaders(current_dataset, modules, compatible_types): + """ + Sibling datasets of a compatible processor type, plus — if the + current dataset's parent is a filtered copy — the downloaders of the + dataset it was copied from. Excludes the current dataset itself. + """ + from common.lib.dataset import DataSet + + # kids from the parent + parent_dataset = current_dataset.get_parent() + downloaders = [ + child for child in parent_dataset.get_children() + if child.type in compatible_types and child.key != current_dataset.key + ] + + # kids from the original (if filtered dataset) + if "copied_from" in parent_dataset.parameters and parent_dataset.is_top_dataset(): + try: + original = DataSet(key=parent_dataset.parameters["copied_from"], + db=current_dataset.db, modules=modules) + downloaders += [ + child for child in original.top_parent().get_children() + if child.type in compatible_types and child.key != current_dataset.key + ] + except DataSetException: + # the original dataset no longer exists + pass + + return downloaders + + def _build_index(self) -> dict: + """ + `{url: {"items": [(metadata, filename, item), ...], + "failures": [(metadata, failure), ...]}}` + + A URL can map to entries from several archives (downloaded more than + once) and to several files within one archive (e.g. a playlist). A + file or failed download can also be reached through more than one URL + (e.g. a short and a long link to the same video, or a link that + redirected): see `_urls_of`. + """ + index = {} + for metadata in self.metadata_objects: + for filename, item in metadata.iter_entries(): + for url in self._urls_of(item): + index.setdefault(url, {"items": [], "failures": []}) + index[url]["items"].append((metadata, filename, item)) + for failure in metadata.iter_failures(): + for url in self._urls_of(failure): + index.setdefault(url, {"items": [], "failures": []}) + index[url]["failures"].append((metadata, failure)) + return index + + @staticmethod + def _urls_of(entry) -> list: + """ + All URLs that lead to an item or failed download: its `url`, and every + URL in its `extra["other_urls"]`. + + :param dict entry: item or failure from the metadata + :return list: the URLs, leaving out any that are empty or not text + """ + other_urls = (entry.get("extra") or {}).get("other_urls") + if not isinstance(other_urls, list): + other_urls = [] + return [url for url in [entry.get("url"), *other_urls] if url and isinstance(url, str)] + + @property + def url_index(self) -> dict: + if self._url_index is None: + self._url_index = self._build_index() + return self._url_index + + def find(self, url: str): + """ + Look up a URL across all previous downloader archives. + + Returns a `MediaLibraryHit` — a success hit if any archive + downloaded it (including as one of a file's other URLs), otherwise a + failure hit carrying every failure `reason` seen — or `None` if the + URL was never seen. + """ + bucket = self.url_index.get(url) + if not bucket: + return None + + if bucket["items"]: + # success beats failure; take the first archive that has the URL + # and copy all of its files for that URL (one URL may yield many, + # e.g. a playlist) + first_metadata = bucket["items"][0][0] + entries = [(filename, item) for metadata, filename, item in bucket["items"] + if metadata is first_metadata] + return MediaLibraryHit(is_success=True, metadata=first_metadata, entries=entries) + + # pass all the failures (e.g. in case one is "error", but later is "not_a_video", or + # something else the consumer wants to interpret), each with the archive + # that recorded it, so the consumer can also tell which run it came from + return MediaLibraryHit(is_success=False, failures=list(bucket["failures"])) + + def __len__(self) -> int: + return len(self.metadata_objects) diff --git a/processors/audio/audio_extractor.py b/processors/audio/audio_extractor.py index 637798836..ba80e94a2 100644 --- a/processors/audio/audio_extractor.py +++ b/processors/audio/audio_extractor.py @@ -11,7 +11,7 @@ from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility, is_executable -from common.lib.exceptions import ProcessorInterruptedException +from common.lib.exceptions import ProcessorInterruptedException, MetadataException __author__ = "Dale Wahl" __credits__ = ["Dale Wahl"] @@ -74,12 +74,27 @@ def process(self): total_possible_videos = sum( 1 for archived_file in archive_file.infolist() - if not archived_file.is_dir() and Path(archived_file.filename).name != ".metadata.json" + if not archived_file.is_dir() + and Path(archived_file.filename).name not in (".metadata.json", "video_archive") ) if max_files != 0: total_possible_videos = min(total_possible_videos, max_files) + # Read the source video archive's metadata so each extracted audio + # file can carry its video's provenance (source posts, URL). + try: + source_metadata = self.source_dataset.read_media_metadata() + except (FileNotFoundError, MetadataException): + source_metadata = None + + # Build our own metadata describing the audio files we produce, + # rather than passing the (video-keyed) source metadata through. + metadata = self.dataset.new_media_metadata( + processor_type=self.type, + from_dataset=(source_metadata.from_dataset if source_metadata else self.source_dataset.key), + ) + processed_videos = 0 written = 0 @@ -88,9 +103,12 @@ def process(self): if self.interrupted: raise ProcessorInterruptedException("Interrupted while determining image wall order") - # Check for 4CAT's metadata JSON and copy it + # the source archive's metadata describes videos; we write our + # own (audio-keyed) metadata below, so skip the original if item.file.name == '.metadata.json': - shutil.copy(item.file, output_dir.joinpath(".metadata.json")) + continue + elif item.file.name == "video_archive": + # yt-dlp's list of what it downloaded; not a video continue if max_files != 0 and processed_videos >= max_files: @@ -114,9 +132,23 @@ def process(self): ffmpeg_output = result.stdout.decode("utf-8") ffmpeg_error = result.stderr.decode("utf-8") - audio_file = output_dir.joinpath(f"{vid_name}.wav") + audio_filename = f"{vid_name}.wav" + audio_file = output_dir.joinpath(audio_filename) + + # carry the source video's provenance onto the extracted audio + video_item = source_metadata.get_entry(item.file.name) if source_metadata else None + post_ids = video_item.get("post_ids", []) if video_item else [] + source_url = video_item.get("url") if video_item else None + extra = dict(video_item.get("extra") or {}) if video_item else {} + if audio_file.exists(): written += 1 + metadata.add_item(audio_filename, post_ids=post_ids, url=source_url, + extra=extra, replace=True) + else: + metadata.add_failure(post_ids=post_ids, reason="extraction_failed", + reason_description=f"ffmpeg exited with code {result.returncode}", + url=source_url) if ffmpeg_output: with open(str(output_dir.joinpath(f"{vid_name}_stdout.log")), 'w', encoding="utf-8") as outfile: @@ -134,6 +166,10 @@ def process(self): self.dataset.update_status(f"Extracted audio from {written} of {processed_videos} attempted videos") self.dataset.update_progress(min(1, processed_videos / max(total_possible_videos, 1))) + # Write our own metadata describing the extracted audio files (and + # any extraction failures), keyed by audio filename. + metadata.write(output_dir) + # Finish up warning = f"Extracted {written}/{processed_videos} audio files, check the logs for errors." \ if written < processed_videos else None diff --git a/processors/conversion/hash_images.py b/processors/conversion/hash_images.py index 13b0414ca..291b409fc 100644 --- a/processors/conversion/hash_images.py +++ b/processors/conversion/hash_images.py @@ -2,12 +2,11 @@ Hash images """ import csv -import json from PIL import UnidentifiedImageError from backend.lib.processor import BasicProcessor -from common.lib.exceptions import ProcessorInterruptedException +from common.lib.exceptions import ProcessorInterruptedException, MetadataException from common.lib.helpers import UserInput, hash_image, stringify_hash from common.lib.compatibility import Compatibility from processors.metrics.group_hashes import HashGrouper @@ -111,50 +110,25 @@ def process(self): group_by = self.parameters.get("group-by", False) similarity_pct = float(self.parameters.get("similarity-threshold", 4)) - # Get staging area - staging_area = self.dataset.get_staging_area() - - # Extract metadata if present try: - metadata_file = self.extract_archived_file_by_name(".metadata.json", self.source_file, staging_area) - except FileNotFoundError: - metadata_file = None + source_metadata = self.source_dataset.read_media_metadata() + except (FileNotFoundError, MetadataException): + source_metadata = None - metadata_extra_fields = ["post_ids", "post_id", "url", "from_dataset"] image_data_by_filename = {} - if metadata_file: - with open(metadata_file) as file: - image_data = json.load(file) - for url, item in image_data.items(): - if "filename" in item: - image_data_by_filename[item["filename"]] = item - elif "files" in item: - files = item.get('files') - if not isinstance(files, list): - self.log.warning(f"Invalid 'files' entry in metadata (expected list, got {type(files)}); cannot use file metadata") - image_data_by_filename = {} - break - for file in files: - if "filename" in file: - # add extra fields from parent item if not present in file entry - for key in metadata_extra_fields: - if key in file: - continue - elif key in item: - file[key] = item[key] - image_data_by_filename[file["filename"]] = file - - self.dataset.log("Found and loaded image metadata") + if source_metadata is not None: + for filename, item in source_metadata.iter_entries(): + image_data_by_filename[filename] = item + self.dataset.log("Found and loaded image metadata") else: self.dataset.log("No image metadata found") - image_data_by_filename = {} # Set up CSV fieldnames (always include hash_size even if crhash to indicate 'None') base_fields = ["filename", "image_hash", "hash_type", "hash_size"] + metadata_extra_fields = ["post_ids", "url", "from_dataset"] fieldnames = (["group"] if group_by else []) + base_fields if image_data_by_filename: - example = next(iter(image_data_by_filename.values())) - fieldnames.extend([key for key in metadata_extra_fields if key in example.keys()]) + fieldnames.extend(metadata_extra_fields) processed = 0 skipped = 0 @@ -203,9 +177,10 @@ def process(self): with self.dataset.get_results_path().open("w", newline="", encoding="utf-8") as output: writer = csv.DictWriter(output, fieldnames=fieldnames) writer.writeheader() + from_dataset = source_metadata.from_dataset if source_metadata else "" for it in items: image_metadata = image_data_by_filename.get(it["filename"], {}) - + # Convert hash object to string for CSV storage hash_str = stringify_hash(it["hash_obj"], it["hash_type"]) row = { @@ -215,10 +190,10 @@ def process(self): "image_hash": hash_str, "hash_type": it["hash_type"], } - # Add optional metadata fields if present - for key in fieldnames: - if key not in row: - row[key] = image_metadata.get(key, "") + if image_data_by_filename: + row["post_ids"] = ", ".join(image_metadata.get("post_ids", [])) + row["url"] = image_metadata.get("url", "") + row["from_dataset"] = from_dataset writer.writerow(row) final_msg = f"Processed {processed:,} images" diff --git a/processors/conversion/view_metadata.py b/processors/conversion/view_metadata.py index f9e3f9cd4..0aa34ba99 100644 --- a/processors/conversion/view_metadata.py +++ b/processors/conversion/view_metadata.py @@ -4,13 +4,11 @@ Designed to work with any processor that has a 'map_metadata' method """ import csv -import json -import zipfile from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility from common.lib.dataset import DataSet -from common.lib.exceptions import DataSetException, ProcessorInterruptedException +from common.lib.exceptions import DataSetException, MetadataException, ProcessorInterruptedException from common.lib.helpers import remove_nuls from common.lib.user_input import UserInput @@ -79,11 +77,19 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: def process(self): """ - Grabs .metadata.json, reformats it, and combines it with the dataset the - media was downloaded from where that dataset can still be found. + Read .metadata.json from the parent archive and reformat as CSV using the + parent producer's `map_metadata` / `map_failure_metadata` hooks, adding + the data of the dataset the media was downloaded from where that dataset + can still be found. """ - metadata = self.read_metadata_file() - if metadata is None: + self.dataset.update_status("Collecting .metadata.json file") + try: + metadata = self.source_dataset.read_media_metadata() + except FileNotFoundError: + self.dataset.finish_with_error("Unable to identify metadata file") + return + except MetadataException as e: + self.dataset.finish_with_error(f"Unable to read metadata: {e}") return producer = self.source_dataset.get_own_processor() @@ -100,161 +106,233 @@ def process(self): return self.dataset.finish_with_error("No valid metadata could be read from the dataset.") # find the dataset the media was downloaded from, if there is one left - source_dataset = None - source_columns = [] + source_columns, source_items = [], {} if self.parameters.get("join_source", True) and wanted_ids: - source_dataset, found_via = self.find_origin_dataset(metadata) - if source_dataset: - source_columns = source_dataset.get_columns() - if source_columns: - self.dataset.update_status(f"Combining metadata with dataset '{source_dataset.get_label()}'") - self.dataset.log(f"Adding data from dataset {source_dataset.key} ({found_via})") - else: - # e.g. an archive, or an NDJSON file whose items cannot be - # read as columns: there is nothing to add - self.dataset.log(f"Dataset {source_dataset.key} has no readable columns; writing metadata only") - source_dataset = None - - fieldnames, source_column_map = self.build_fieldnames(media_rows, source_columns) - matched_rows = set() - num_rows = 0 - - with self.dataset.get_results_path().open("w", encoding="utf-8", newline="") as outfile: - writer = csv.DictWriter(outfile, fieldnames=fieldnames, restval="", extrasaction="ignore") - writer.writeheader() - - if source_dataset: - # One pass over the source dataset. Only items that a media file - # refers to are used. - item_index = {} - for row_index, (_, post_ids) in enumerate(media_rows): - for post_id in post_ids: - item_index.setdefault(post_id, []).append(row_index) - - seen_ids = set() - items_read = 0 - for item in source_dataset.iterate_items(self): - if self.interrupted: - raise ProcessorInterruptedException("Interrupted while reading source dataset") - - for post_id in self.item_ids(item): - if post_id not in item_index or post_id in seen_ids: - # not referred to by any media file, or the item ID - # occurs more than once in the dataset - continue - seen_ids.add(post_id) - - item_columns = {written_as: item.get(column, "") - for column, written_as in source_column_map.items()} - # a media file used by several items becomes one row per - # item, so that nothing has to be merged into one cell - for row_index in item_index[post_id]: - writer.writerow(remove_nuls({ - **media_rows[row_index][0], - self.matched_column: True, - **item_columns - })) - matched_rows.add(row_index) - num_rows += 1 - - if len(seen_ids) == len(item_index): - # every item referred to has been found - break - - items_read += 1 - if items_read % 500 == 0: - self.dataset.update_status(f"Looked for media in {items_read:,} of " - f"{source_dataset.num_rows:,} item(s)") - - # media files that could not be traced back to an item still get a - # row, with the columns of the source dataset left empty - for row_index, (row, _) in enumerate(media_rows): - if row_index in matched_rows: - continue - if source_dataset: - row = {**row, self.matched_column: False} - writer.writerow(remove_nuls(row)) - num_rows += 1 + source_columns, source_items = self.collect_source_data(metadata, wanted_ids) - if source_dataset: - self.dataset.log(f"Traced {len(matched_rows):,} of {len(media_rows):,} metadata row(s) back to an item " - f"in dataset {source_dataset.key}") - - self.dataset.update_status(f"Read metadata for {num_rows:,} item(s).") - - if source_dataset and not matched_rows: - self.dataset.finish_with_warning( - num_rows, - f"Wrote {num_rows:,} metadata row(s), but none of them could be traced back to an item in the " - f"dataset the media was downloaded from; its columns are empty." - ) - else: - self.dataset.finish(num_rows) - - def read_metadata_file(self): - """ - Read the .metadata.json file from the archive this processor runs on - - :return dict: The metadata file's contents, or `None` if it could not - be read, in which case the dataset is finished with an error. - """ - self.dataset.update_status("Collecting .metadata.json file") - with zipfile.ZipFile(self.source_file, "r") as archive_file: - if ".metadata.json" not in archive_file.namelist(): - self.dataset.finish_with_error("Unable to identify metadata file") - return None - - staging_area = self.dataset.get_staging_area() - archive_file.extract(".metadata.json", staging_area) - - with staging_area.joinpath(".metadata.json").open() as file: - return json.load(file) + self.write_rows(media_rows, source_columns, source_items) def collect_media_rows(self, metadata, producer): """ Turn the metadata file into rows The processor that made the archive decides what the rows look like, via - its `map_metadata` method. Their columns are prefixed here so that they - cannot collide with those of the dataset the media came from. + its `map_metadata` and `map_failure_metadata` methods. Their columns are + prefixed here so that they cannot collide with those of the dataset the + media came from. - :param dict metadata: Contents of the metadata file + :param MediaArchiveMetadata metadata: The archive's metadata :param producer: Processor class that made the archive :return tuple: A list of (row, item IDs) tuples, and the set of all item IDs those rows refer to """ - include_failed = self.parameters.get("include_failed", False) media_rows = [] wanted_ids = set() - for key, value in metadata.items(): + def add(rows, entry): + # a download that produced no file still records the items it was + # for, so failures can be traced back to an item as well. Post IDs + # are already normalised to non-blank strings when the metadata is + # read, so media with no traceable item (e.g. Instagram ads) simply + # has an empty list here rather than a placeholder id. + post_ids = list(dict.fromkeys(entry.get("post_ids", []))) + for row in rows: + media_rows.append(({self.media_prefix + column: cell for column, cell in row.items()}, post_ids)) + wanted_ids.update(post_ids) + + for filename, item in metadata.iter_entries(): if self.interrupted: raise ProcessorInterruptedException("Interrupted while reading metadata file") + add(producer.map_metadata(filename, item), item) - if not isinstance(value, dict): - # metadata files can be uploaded by users, so may be malformed - continue + map_failure = getattr(producer, "map_failure_metadata", None) + if self.parameters.get("include_failed", False) and map_failure is not None: + for failure in metadata.iter_failures(): + if self.interrupted: + raise ProcessorInterruptedException("Interrupted while reading metadata file") + add(map_failure(failure), failure) + + return media_rows, wanted_ids - if not include_failed and not value.get("success", True): + def collect_source_data(self, metadata, wanted_ids): + """ + Read the items the media was downloaded from + + More than one dataset can look like the right one to use, so each is + tried in turn and the first that actually accounts for some of the media + is used. If none of them do, the best guess is still used, so that the + result has its columns and the mismatch is reported rather than passing + silently. + + :param MediaArchiveMetadata metadata: The archive's metadata + :param set wanted_ids: IDs of the items the media refers to + :return tuple: The columns to add, and a map of item ID to the values + of those columns; both empty if there is no dataset to use + """ + fallback = None + for candidate, found_via in self.find_origin_datasets(metadata): + columns = candidate.get_columns() + if not columns: + # e.g. an archive, or an NDJSON file whose items cannot be read + # as columns: there is nothing to add from this one + self.dataset.log(f"Dataset {candidate.key} ({found_via}) has no readable columns; skipping it") continue - # some processors write item IDs as text and others as numbers - post_ids = value.get("post_ids") or [] - if isinstance(post_ids, (str, int)): - post_ids = [post_ids] - post_ids = [str(post_id).strip() for post_id in post_ids] - entry = {**value, "post_ids": post_ids} + self.dataset.update_status(f"Combining metadata with dataset '{candidate.get_label()}'") + items = self.collect_source_items(candidate, columns, wanted_ids) + if items: + self.dataset.log(f"Adding data from dataset {candidate.key} ({found_via}); " + f"accounts for {len(items):,} of {len(wanted_ids):,} item(s) the media refers to") + return columns, items - # an item referred to twice would otherwise produce the same row twice - unique_ids = list(dict.fromkeys(post_ids)) + self.dataset.log(f"Dataset {candidate.key} ({found_via}) has none of the items the media refers to") + if fallback is None: + fallback = (columns, {}) - # Metadata may contain more than one row/item per key, value pair - for row in producer.map_metadata(key, entry): - prefixed = {self.media_prefix + column: cell for column, cell in row.items()} - media_rows.append((prefixed, unique_ids)) - wanted_ids.update(unique_ids) + if fallback is not None: + return fallback - return media_rows, wanted_ids + self.dataset.log("Could not find the dataset the media was downloaded from; writing metadata only") + return [], {} + + def find_origin_datasets(self, metadata): + """ + Find the datasets the media in this archive may have been downloaded from + + The metadata file records the key of that dataset, which is normally the + right one, but it can name a helper dataset that a preset built for the + downloader rather than the dataset a researcher would recognise. As a + second option, walk up the chain of parent datasets, skipping any that + are media archives themselves - an archive can be made from another + archive, for example by filtering out duplicate images. + + :param MediaArchiveMetadata metadata: The archive's metadata + :return list: (dataset, how it was found) tuples, best guess first + """ + candidates = [] + + if metadata.from_dataset: + try: + recorded = DataSet(key=metadata.from_dataset, db=self.db, modules=self.modules) + if recorded.is_finished() and recorded.num_rows: + candidates.append((recorded, "recorded in the metadata file")) + except DataSetException: + # deleted since the media was downloaded + pass + + parent = self.source_dataset.get_parent() + while parent is not None and self.is_media_archive(parent): + parent = parent.get_parent() + if parent is not None and parent.num_rows and parent.key not in [c.key for c, _ in candidates]: + candidates.append((parent, "found by walking up the chain of parent datasets")) + + return candidates + + def collect_source_items(self, dataset, columns, wanted_ids): + """ + Read the wanted items from a dataset in one pass + + Only items the media refers to are kept, so the amount of data held here + depends on the size of the archive and not on that of the dataset. + + :param DataSet dataset: Dataset to read + :param list columns: Columns to keep + :param set wanted_ids: IDs of the items to keep + :return dict: Item ID to the values of those columns + """ + found = {} + items_read = 0 + for item in dataset.iterate_items(self): + if self.interrupted: + raise ProcessorInterruptedException("Interrupted while reading source dataset") + items_read += 1 + + for post_id in self.item_ids(item): + if post_id in wanted_ids and post_id not in found: + found[post_id] = {column: item.get(column, "") for column in columns} + + if len(found) == len(wanted_ids): + # every item the media refers to has been found + break + + if items_read % 500 == 0: + self.dataset.update_status(f"Looked for media in {items_read:,} of {dataset.num_rows:,} item(s)") + + # When some media could not be traced and the whole dataset was read, + # items the datasource could not read back (e.g. Instagram ads, which its + # map_item rejects) are the likely reason: they are skipped on read, so + # the media pointing at them has nothing to match. Say so, so a partial + # join is not mistaken for missing or corrupt metadata. + if len(found) < len(wanted_ids): + unread = dataset.num_rows - items_read + if unread > 0: + self.dataset.log( + f"{unread:,} of {dataset.num_rows:,} item(s) in '{dataset.get_label()}' could not be read " + f"and were skipped (e.g. items a datasource cannot map, such as ads); media " + f"downloaded from them cannot be traced back and is left unmatched." + ) + + return found + + def write_rows(self, media_rows, source_columns, source_items): + """ + Write the result file and finish the dataset + + A media file used by several items becomes one row per item, so that + nothing has to be merged into one cell. One that cannot be traced back to + an item still gets a row, with the source columns left empty. + + :param list media_rows: Rows as returned by `collect_media_rows` + :param list source_columns: Columns to add from the source dataset + :param dict source_items: Item ID to source column values + """ + fieldnames, source_column_map = self.build_fieldnames(media_rows, source_columns) + num_rows = 0 + matched = 0 + + with self.dataset.get_results_path().open("w", encoding="utf-8", newline="") as outfile: + writer = csv.DictWriter(outfile, fieldnames=fieldnames, restval="", extrasaction="ignore") + writer.writeheader() + + for row, post_ids in media_rows: + items = [source_items[post_id] for post_id in post_ids if post_id in source_items] + + if not items: + if source_columns: + row = {**row, self.matched_column: False} + writer.writerow(remove_nuls(row)) + num_rows += 1 + continue + + matched += 1 + for item in items: + writer.writerow(remove_nuls({ + **row, + self.matched_column: True, + **{source_column_map[column]: value for column, value in item.items()} + })) + num_rows += 1 + + total = len(media_rows) + if source_columns: + self.dataset.log(f"Traced {matched:,} of {total:,} media file(s) back to an item") + + self.dataset.update_status(f"Read metadata for {num_rows:,} item(s).") + + # When combining with a source dataset, any media that could not be traced + # back to an item is worth flagging: the row is still written, but its + # source columns are empty. This is a warning rather than an error - the + # result is usable - and the front end shows it to the user as such. + if source_columns and matched < total: + unmatched = total - matched + if not matched: + warning = f"None of the {total:,} media file(s) could be traced back to an item in the dataset, " + else: + warning = f"{unmatched:,} of {total:,} media file(s) could not be traced back to an item in the dataset, " + warning += "so their added columns are empty (e.g. ads or other unsupported items)." + self.dataset.finish_with_warning(num_rows, warning) + else: + self.dataset.finish(num_rows) def build_fieldnames(self, media_rows, source_columns): """ @@ -273,7 +351,6 @@ def build_fieldnames(self, media_rows, source_columns): """ fieldnames = [] for row, _ in media_rows: - # TODO: look into MediaArchive class PR to see if this can be removed fieldnames.extend(column for column in row if column not in fieldnames) if not source_columns: @@ -294,54 +371,15 @@ def build_fieldnames(self, media_rows, source_columns): return fieldnames, source_column_map - def find_origin_dataset(self, metadata): - """ - Find the dataset the media in this archive was downloaded from - - The metadata file records the key of that dataset, which is the most - reliable answer, but it may have been deleted since (e.g. a filter - was removed). Walk up the chain of parent datasets. - - :param dict metadata: Contents of the metadata file - :return tuple: The dataset and a short description of how it was found, - or `(None, None)` if there is none that can be used - """ - recorded_keys = dict.fromkeys(value["from_dataset"] for value in metadata.values() - if isinstance(value, dict) and value.get("from_dataset")) - if len(recorded_keys) > 1: - # I do not think this is possible and can be removed with MediaArchive class PR - self.dataset.log(f"Metadata refers to more than one source dataset ({', '.join(recorded_keys)}); " - f"using the first one that can be read") - - for key in recorded_keys: - try: - candidate = DataSet(key=key, db=self.db, modules=self.modules) - except DataSetException: - # deleted since the media was downloaded - continue - if candidate.is_finished() and candidate.num_rows: - return candidate, "recorded in the metadata file" - - candidate = self.source_dataset.get_parent() - while candidate is not None and self.is_media_archive(candidate): - candidate = candidate.get_parent() - if candidate is not None and candidate.num_rows: - return candidate, "found by walking up the chain of parent datasets" - - self.dataset.log("Could not find the dataset the media was downloaded from; writing metadata only") - return None, None - @classmethod - def is_media_archive(cls, dataset): + def is_media_archive(cls, module): """ Is this dataset an archive of downloaded media files? - # TODO: Remove with MediaArchive class PR - - :param DataSet dataset: Dataset to check + :param module: Dataset or processor to check :return bool: """ - return any(dataset.type.startswith(prefix) for prefix in cls.media_archive_prefixes) + return any(module.type.startswith(prefix) for prefix in cls.media_archive_prefixes) @staticmethod def item_ids(item): diff --git a/processors/filtering/unique_images.py b/processors/filtering/unique_images.py index 8f5b448eb..ea7b1ee10 100644 --- a/processors/filtering/unique_images.py +++ b/processors/filtering/unique_images.py @@ -2,10 +2,9 @@ Filter by unique images """ import shutil -import json from backend.lib.processor import BasicProcessor -from common.lib.exceptions import ProcessorInterruptedException +from common.lib.exceptions import ProcessorInterruptedException, MetadataException from common.lib.helpers import UserInput, hash_file from common.lib.compatibility import Compatibility @@ -70,7 +69,7 @@ def process(self): """ seen_hashes = set() hash_map = {} - metadata = None + hash_type = self.parameters.get("hash-type") dupes = 0 processed = 0 staging_area = self.dataset.get_staging_area() @@ -87,11 +86,9 @@ def process(self): processed += 1 if image.file.name == ".metadata.json": - with image.file.open() as infile: - metadata = json.load(infile) continue - image_hash = hash_file(image.file, self.parameters.get("hash-type")) + image_hash = hash_file(image.file, hash_type) if image_hash not in seen_hashes: seen_hashes.add(image_hash) @@ -101,21 +98,37 @@ def process(self): self.dataset.log(f"{image.file.name} is a duplicate of {hash_map[image_hash]} - skipping") dupes += 1 - new_metadata = {} + try: + source_metadata = self.source_dataset.read_media_metadata() + except (FileNotFoundError, MetadataException): + source_metadata = None + + new_metadata = self.dataset.new_media_metadata( + processor_type=self.type, + from_dataset=(source_metadata.from_dataset if source_metadata else self.source_dataset.key), + ) inverse_hashmap = {v: k for k, v in hash_map.items()} - if metadata: - for url, item in metadata.items(): - if item["filename"] in inverse_hashmap: - new_metadata[inverse_hashmap[item["filename"]]] = { - **item, - "hash": inverse_hashmap[item["filename"]], - "hash_type": self.parameters.get("hash-type") - } + if source_metadata is not None: + for filename, item in source_metadata.iter_entries(): + if filename not in inverse_hashmap: + continue + extra = dict(item.get("extra") or {}) + extra["hash"] = inverse_hashmap[filename] + extra["hash_type"] = hash_type + new_metadata.add_item( + filename, + post_ids=item.get("post_ids", []), + url=item.get("url"), + extra=extra, + ) else: - new_metadata = {hash_map[k]: {"filename": hash_map[k], "hash": k, "hash_type": self.parameters.get("hash-type")} for k in hash_map} + for h, filename in hash_map.items(): + new_metadata.add_item( + filename, post_ids=[], + extra={"hash": h, "hash_type": hash_type}, + ) - with staging_area.joinpath(".metadata.json").open("w") as outfile: - json.dump(new_metadata, outfile) + new_metadata.write(staging_area) self.dataset.update_status(f"Image archive filtered, found {dupes:,} duplicate(s)", is_final=True) self.write_archive_and_finish(staging_area, len(hash_map), finish=True) diff --git a/processors/machine_learning/audio_to_text.py b/processors/machine_learning/audio_to_text.py index ed4078ba0..d631beb1a 100644 --- a/processors/machine_learning/audio_to_text.py +++ b/processors/machine_learning/audio_to_text.py @@ -9,7 +9,7 @@ from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility from common.lib.dmi_service_manager import DmiServiceManager, DmiServiceManagerException, DsmOutOfMemory -from common.lib.exceptions import ProcessorInterruptedException +from common.lib.exceptions import ProcessorInterruptedException, MetadataException from common.lib.user_input import UserInput from common.lib.item_mapping import MappedItem @@ -506,18 +506,20 @@ def process(self): self.dataset.update_status(f"Got {i + 1} transcription{s} from OpenAI") self.dataset.update_progress((i + 1) / max_files) - # Load the video metadata if available - video_metadata = None + # Load the source archive's metadata if available. Map by + # extension-less filename: the transcription result files share their + # stem with the input audio (e.g. `foo.wav` -> `foo.json`). file_metadata_map = {} - if staging_area.joinpath(".metadata.json").is_file(): - with open(staging_area.joinpath(".metadata.json")) as file: - video_metadata = json.load(file) - self.dataset.log("Found and loaded video metadata") - - for video_id, video_data in video_metadata.items(): - for file in video_data.get("files", []): - file = ".".join(file["filename"].split(".")[:-1]) - file_metadata_map[file] = video_data + try: + source_metadata = self.source_dataset.read_media_metadata() + except (FileNotFoundError, MetadataException): + source_metadata = None + if source_metadata is not None: + self.dataset.log("Found and loaded audio metadata") + from_dataset = source_metadata.from_dataset + for filename, item in source_metadata.iter_entries(): + basename = filename.rsplit(".", 1)[0] + file_metadata_map[basename] = {**item, "from_dataset": from_dataset} self.dataset.update_status("Processing results...") diff --git a/processors/machine_learning/blip2_image_caption.py b/processors/machine_learning/blip2_image_caption.py index 109d811d2..c4d95855e 100644 --- a/processors/machine_learning/blip2_image_caption.py +++ b/processors/machine_learning/blip2_image_caption.py @@ -6,7 +6,7 @@ from backend.lib.processor import BasicProcessor from common.lib.dmi_service_manager import DmiServiceManager, DmiServiceManagerException, DsmOutOfMemory, DsmConnectionError -from common.lib.exceptions import ProcessorInterruptedException +from common.lib.exceptions import ProcessorInterruptedException, MetadataException from common.lib.user_input import UserInput from common.lib.item_mapping import MappedItem from common.lib.compatibility import Compatibility @@ -198,21 +198,19 @@ def process(self): self.dataset.finish_with_error("Error with BLIP2 model; please contact 4CAT admins.") return - # Load the video metadata if available + # Load the image metadata if available. Key by basename because BLIP2 + # results share the input image's stem but use a different extension. image_metadata = {} try: - metadata_file = self.extract_archived_file_by_name(".metadata.json", self.source_file, staging_area) - except FileNotFoundError: - metadata_file = None - if metadata_file: - with open(metadata_file) as file: - image_data = json.load(file) - self.dataset.log("Found and loaded image metadata") - for url, data in image_data.items(): - if data.get('success'): - data.update({"url": url}) - # using the filename without extension as the key; since that is how the results form their filename - image_metadata[".".join(data['filename'].split(".")[:-1])] = data + source_metadata = self.source_dataset.read_media_metadata() + except (FileNotFoundError, MetadataException): + source_metadata = None + if source_metadata is not None: + self.dataset.log("Found and loaded image metadata") + from_dataset = source_metadata.from_dataset + for filename, item in source_metadata.iter_entries(): + basename = filename.rsplit(".", 1)[0] + image_metadata[basename] = {**item, "from_dataset": from_dataset} else: self.dataset.log("No image metadata found") diff --git a/processors/machine_learning/clarifai_api.py b/processors/machine_learning/clarifai_api.py index 63b69f312..e45873a9f 100644 --- a/processors/machine_learning/clarifai_api.py +++ b/processors/machine_learning/clarifai_api.py @@ -9,6 +9,7 @@ from clarifai_grpc.channel.clarifai_channel import ClarifaiChannel from common.lib.helpers import UserInput, convert_to_int +from common.lib.exceptions import MetadataException from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility @@ -140,6 +141,12 @@ def process(self): processed = 0 annotated = 0 + try: + source_metadata = self.source_dataset.read_media_metadata() + filename_to_post_ids = source_metadata.filename_to_post_ids() + except (FileNotFoundError, MetadataException): + filename_to_post_ids = {} + # send batched requests per model for model_id in models: iterator = self.source_dataset.iterate_items() @@ -160,10 +167,6 @@ def process(self): send_batch = True if image: - if image.file.name == ".metadata.json": - # Metadata, we keep this for writing annotations - image_metadata = json.load(image.file.open("r")) - if image.file.suffix in (".json", ".log"): continue @@ -225,8 +228,6 @@ def process(self): # Get original post IDs for annotations fourcat_annotations = [] annotated = 0 - if save_annotations: - filename_and_post_ids = {v["filename"]: v["post_ids"] for v in image_metadata.values()} # save the buffered results (we do this only now so we can also store # combined annotations) @@ -247,7 +248,7 @@ def process(self): if save_annotations: for label, confidence in all_annotations.items(): if confidence > annotation_threshold: - for item_id in filename_and_post_ids[image]: + for item_id in filename_to_post_ids.get(image, []): fourcat_annotations.append({ "label": "clarifai_" + label, "value": confidence, diff --git a/processors/machine_learning/clip_categorize_images.py b/processors/machine_learning/clip_categorize_images.py index 99aa084d1..ada4e1c48 100644 --- a/processors/machine_learning/clip_categorize_images.py +++ b/processors/machine_learning/clip_categorize_images.py @@ -7,7 +7,7 @@ from backend.lib.processor import BasicProcessor from common.lib.dmi_service_manager import DmiServiceManager, DmiServiceManagerException, DsmOutOfMemory, DsmConnectionError -from common.lib.exceptions import ProcessorInterruptedException +from common.lib.exceptions import ProcessorInterruptedException, MetadataException from common.lib.user_input import UserInput from common.lib.item_mapping import MappedItem from common.lib.compatibility import Compatibility @@ -221,17 +221,19 @@ def process(self): self.dataset.finish_with_error("Error with CLIP model; please contact 4CAT admins.") return - # Load the video metadata if available + # Load the image metadata if available. Key by basename because CLIP + # result files share the input image's stem but use `.json` extension. image_metadata = {} - if staging_area.joinpath(".metadata.json").is_file(): - with open(staging_area.joinpath(".metadata.json")) as file: - image_data = json.load(file) - self.dataset.log("Found and loaded image metadata") - for url, data in image_data.items(): - if data.get('success'): - data.update({"url": url}) - # using the filename without extension as the key; since that is how the results form their filename - image_metadata[".".join(data['filename'].split(".")[:-1])] = data + try: + source_metadata = self.source_dataset.read_media_metadata() + except (FileNotFoundError, MetadataException): + source_metadata = None + if source_metadata is not None: + self.dataset.log("Found and loaded image metadata") + from_dataset = source_metadata.from_dataset + for filename, item in source_metadata.iter_entries(): + basename = filename.rsplit(".", 1)[0] + image_metadata[basename] = {**item, "from_dataset": from_dataset} self.dataset.update_status("Processing CLIP results...") # Download the result files diff --git a/processors/machine_learning/generate_images.py b/processors/machine_learning/generate_images.py index 273d0835c..91658935e 100644 --- a/processors/machine_learning/generate_images.py +++ b/processors/machine_learning/generate_images.py @@ -276,22 +276,23 @@ def make_filename(id, prompt): return f"{id}-{safe_prompt}.jpeg" self.dataset.update_status("Verifying results") - with output_dir.joinpath(".metadata.json").open("w") as outfile: - metadata = { - prompt_id: { - "from_dataset": self.source_dataset.key, - "filename": make_filename(prompt_id, data["prompt"]), - "success": output_dir.joinpath(make_filename(prompt_id, data["prompt"])).exists(), - "post_ids": [prompt_id], - "prompt": data["prompt"], - "negative-prompt": data["negative"], - } for prompt_id, data in prompts.items() - } - json.dump(metadata, outfile) + metadata = self.dataset.new_media_metadata( + processor_type=self.type, from_dataset=self.source_dataset.key + ) + for prompt_id, data in prompts.items(): + filename = make_filename(prompt_id, data["prompt"]) + extra = {"prompt": data["prompt"], "negative-prompt": data["negative"]} + if output_dir.joinpath(filename).exists(): + metadata.add_item(filename, post_ids=[prompt_id], extra=extra, replace=True) + else: + metadata.add_failure(post_ids=[prompt_id], reason="error", + reason_description="Generated image not found on disk") + metadata.write(output_dir) shutil.rmtree(staging_area) + successful = len(metadata) self.dataset.update_status( - f"Generated {len([r for r in metadata.values() if r['success']]):,} image(s) for {len(prompts):,} prompt(s)", + f"Generated {successful:,} image(s) for {len(prompts):,} prompt(s)", is_final=True) - self.write_archive_and_finish(output_dir, num_items=len([r for r in metadata.values() if r['success']])) + self.write_archive_and_finish(output_dir, num_items=successful) diff --git a/processors/machine_learning/google_vision_api.py b/processors/machine_learning/google_vision_api.py index 73cae0d2a..c55f6a431 100644 --- a/processors/machine_learning/google_vision_api.py +++ b/processors/machine_learning/google_vision_api.py @@ -11,7 +11,7 @@ from common.lib.helpers import UserInput, convert_to_int from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility -from common.lib.exceptions import ProcessorInterruptedException +from common.lib.exceptions import ProcessorInterruptedException, MetadataException __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters"] @@ -118,8 +118,12 @@ def process(self): processed = 0 done = 0 - # Get the .metadata.json file if we're writing annotations so we can add them to specific rows. - img_metadata = [] + # Load the metadata file so we can attach source post IDs to annotations. + try: + source_metadata = self.source_dataset.read_media_metadata() + img_metadata = source_metadata.filename_to_post_ids() + except (FileNotFoundError, MetadataException): + img_metadata = {} # Loop through images for image in self.source_dataset.iterate_items(): @@ -130,13 +134,7 @@ def process(self): self.dataset.update_progress(done / total) if image.file.name.startswith(".") or image.file.suffix in (".json", ".log"): - - # Get the .metadata.json file so we can also save item IDs. - if image.file.name == ".metadata.json": - img_metadata = json.load(image.file.open()) - if img_metadata: - img_metadata = {v["filename"]: v.get("post_ids", []) for v in img_metadata.values()} - else: + if image.file.name != ".metadata.json": self.dataset.log(f"Skipping file {image.file.name}, probably not an image.") continue @@ -151,7 +149,9 @@ def process(self): annotations = { "file_name": image.file.name, - "post_ids": img_metadata[image.file.name], + # images from an archive without metadata (e.g. an upload or + # video frames) have no posts to connect to + "post_ids": img_metadata.get(image.file.name, []), **annotations} with self.dataset.get_results_path().open("a", encoding="utf-8") as outfile: diff --git a/processors/machine_learning/llm_prompter.py b/processors/machine_learning/llm_prompter.py index 7848a860a..e697cbfc9 100644 --- a/processors/machine_learning/llm_prompter.py +++ b/processors/machine_learning/llm_prompter.py @@ -16,7 +16,7 @@ from datetime import datetime, timedelta from common.lib.item_mapping import MappedItem -from common.lib.exceptions import ProcessorInterruptedException, QueryParametersException, QueryNeedsExplicitConfirmationException +from common.lib.exceptions import ProcessorInterruptedException, QueryParametersException, QueryNeedsExplicitConfirmationException, MetadataException from common.lib.helpers import UserInput, nthify, andify, remove_nuls, flatten_dict, urls_in_text from common.lib.llm.adapter import LLMAdapter from backend.lib.processor import BasicProcessor @@ -519,28 +519,8 @@ def process(self): filename_to_post_ids = {} if save_annotations: try: - self.extract_archived_file_by_name(".metadata.json", self.source_file, staging_area) - with open(staging_area.joinpath(".metadata.json")) as meta_file: - archive_metadata = json.load(meta_file) - for url, data in archive_metadata.items(): - if data.get("success") and data.get("post_ids"): - post_ids = [str(pid) for pid in data["post_ids"]] - # A single URL may map to one filename or multiple files (e.g. video + thumbnail) - filenames_for_url = [] - if data.get("filename"): - filenames_for_url.append(data["filename"]) - for file_entry in data.get("files", []): - if file_entry.get("success") and file_entry.get("filename"): - filenames_for_url.append(file_entry["filename"]) - # Merge post_ids per filename; extend rather than overwrite so that - # multiple URLs pointing to the same file don't lose earlier post_ids. - for filename in filenames_for_url: - existing = filename_to_post_ids.setdefault(filename, []) - for post_id in post_ids: - if post_id not in existing: - existing.append(post_id) - - except (FileNotFoundError, json.JSONDecodeError, KeyError) as e: + filename_to_post_ids = self.source_dataset.read_media_metadata().filename_to_post_ids() + except (FileNotFoundError, MetadataException) as e: self.dataset.log(f"Could not load .metadata.json for annotation mapping: {e}. " f"Annotations will use filenames as item IDs.") diff --git a/processors/machine_learning/pix-plot.py b/processors/machine_learning/pix-plot.py index 397fa554d..9386629b6 100644 --- a/processors/machine_learning/pix-plot.py +++ b/processors/machine_learning/pix-plot.py @@ -2,7 +2,6 @@ Create an PixPlot of downloaded images """ import shutil -import json from datetime import datetime import csv import os @@ -10,6 +9,7 @@ from werkzeug.utils import secure_filename from common.lib.dmi_service_manager import DmiServiceManager, DsmOutOfMemory, DmiServiceManagerException +from common.lib.exceptions import MetadataException from common.lib.helpers import UserInput, ellipsiate, urls_in_text from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility @@ -263,11 +263,11 @@ def format_metadata(self, temp_path): """ # Get image data - if not os.path.isfile(os.path.join(temp_path, '.metadata.json')): - # No metadata + try: + source_metadata = self.source_dataset.read_media_metadata() + except (FileNotFoundError, MetadataException): return False - top_dataset = self.dataset.top_parent() # Check that this is not already a top dataset if top_dataset.key == self.source_dataset.key: @@ -280,38 +280,29 @@ def format_metadata(self, temp_path): # e.g., image dataset uploaded as zip and later filtered via unique_images return False - with open(os.path.join(temp_path, '.metadata.json')) as file: - image_data = json.load(file) - - # Images can belong to multiple posts, so we must build this file as we go + # Images can belong to multiple posts, so we must build this file as we go. + # Keyed by archive filename so the post-loop below can look up images by + # post id without going through the originating URL. images = {} - - # Reformat image_data to access by filename and begin metadata post_id_image_dictionary = {} successful_image_count = 0 - for url, data in image_data.items(): - # Check if image successfully downloaded for image - if data.get('success') and data.get('filename') is not None and data.get('post_ids'): - successful_image_count += 1 - # if no filename, bad metadata; file was not actually downloaded, fixed in 9b603cd1ecdf97fd92c3e1c6200e4b6700dc1e37 - - # dmi_pix_plot API uses secure_filename while pixplot.py (in PixPlot library) uses clean_filename - filename = self.clean_filename(secure_filename(data.get('filename'))) - - for post_id in data.get('post_ids'): - # Add key to post ID dictionary - if post_id in post_id_image_dictionary.keys(): - post_id_image_dictionary[post_id].append(url) - else: - post_id_image_dictionary[post_id] = [url] - - # Add to metadata - images[url] = {'filename': filename, - 'permalink': url, - 'description': 'Number of posts with this image: ' + str(len(data.get('post_ids'))), - 'tags': '', - 'number_of_posts': 0, - } + for archive_filename, item in source_metadata.iter_entries(): + post_ids = item.get("post_ids", []) + if not post_ids: + continue + successful_image_count += 1 + # dmi_pix_plot API uses secure_filename while pixplot.py (in PixPlot library) uses clean_filename + filename = self.clean_filename(secure_filename(archive_filename)) + url = item.get("url", "") + for post_id in post_ids: + post_id_image_dictionary.setdefault(post_id, []).append(archive_filename) + images[archive_filename] = { + 'filename': filename, + 'permalink': url, + 'description': 'Number of posts with this image: ' + str(len(post_ids)), + 'tags': '', + 'number_of_posts': 0, + } self.dataset.log(f"Metadata for {successful_image_count} images collected from {len(post_id_image_dictionary)} posts") @@ -319,9 +310,10 @@ def format_metadata(self, temp_path): posts_with_images = 0 for post in self.dataset.top_parent().iterate_items(self): # Check if post contains one of the downloaded images - if post['id'] in post_id_image_dictionary.keys(): + post_id = str(post['id']) + if post_id in post_id_image_dictionary: posts_with_images += 1 - for img_name in post_id_image_dictionary[post['id']]: + for img_name in post_id_image_dictionary[post_id]: image = images[img_name] # Update description diff --git a/processors/machine_learning/text_from_image.py b/processors/machine_learning/text_from_image.py index 0edce68a9..b8cf96bcc 100644 --- a/processors/machine_learning/text_from_image.py +++ b/processors/machine_learning/text_from_image.py @@ -11,7 +11,7 @@ from common.lib.dmi_service_manager import DmiServiceManager, DsmOutOfMemory, DmiServiceManagerException, DsmConnectionError from common.lib.helpers import UserInput, hash_to_md5 from backend.lib.processor import BasicProcessor -from common.lib.exceptions import ProcessorInterruptedException +from common.lib.exceptions import ProcessorInterruptedException, MetadataException from common.lib.item_mapping import MappedItem from common.lib.compatibility import Compatibility @@ -125,15 +125,11 @@ def process(self): # Collect filenames and metadata image_filenames = [] skipped_images = 0 - metadata_file = None staging_area = self.dataset.get_staging_area() for image in self.source_dataset.iterate_items(staging_area=staging_area, immediately_delete=False): if self.interrupted: raise ProcessorInterruptedException("Interrupted while unzipping images") - if image.file.name == ".metadata.json": - metadata_file = image.file.name - continue elif image.file.name.split('.')[-1] in ["json", "log"]: continue elif image.file.name.split('.')[-1] == "svg": @@ -200,38 +196,20 @@ def process(self): dmi_service_manager.process_results(output_dir) # Load the metadata from the archive - image_metadata = {} - if metadata_file is None: - try: - self.extract_archived_file_by_name(".metadata.json", self.source_file, staging_area) - metadata_exists = True - except FileNotFoundError: - self.dataset.update_status("No metadata file found") - metadata_exists = False - else: - # Previously extracted - metadata_exists = True - - if metadata_exists: - with open(os.path.join(staging_area, '.metadata.json')) as file: - image_data = json.load(file) - for url, data in image_data.items(): - if data.get('success'): - data.update({"url": url}) - image_metadata[data['filename']] = data + try: + source_metadata = self.source_dataset.read_media_metadata() + except (FileNotFoundError, MetadataException): + source_metadata = None + self.dataset.update_status("No metadata file found") # Check if we need to collect data for updating the original dataset save_annotations = self.parameters.get("save_annotations", False) if save_annotations: - if not metadata_exists: + if source_metadata is None: self.dataset.update_status("No metadata file found, cannot write to original dataset") save_annotations = False else: - # Create filename to post id mapping - filename_to_post_id = {} - for url, data in image_data.items(): - if data.get("success"): - filename_to_post_id[data.get("filename")] = data.get("post_ids") + filename_to_post_id = source_metadata.filename_to_post_ids() post_id_to_results = {} # Save files as NDJSON, then use map_item for 4CAT to interact @@ -251,7 +229,7 @@ def process(self): # Need to include filename as there may be many images to a single post detected_text = '%s:"""%s"""' % (image_name, result_data.get('simplified_text', {}).get('raw_text', '')) - post_ids = filename_to_post_id[image_name] + post_ids = filename_to_post_id.get(image_name, []) for post_id in post_ids: # Posts can have multiple images if post_id in post_id_to_results.keys(): @@ -262,7 +240,7 @@ def process(self): data = { "id": image_name, **result_data, - "image_metadata": image_metadata.get(image_name, {}) if image_metadata else {}, + "image_metadata": (source_metadata.get_entry(image_name) or {}) if source_metadata else {}, } outfile.write(json.dumps(data) + "\n") diff --git a/processors/networks/image-network.py b/processors/networks/image-network.py index ee573604b..d1cec0d9b 100644 --- a/processors/networks/image-network.py +++ b/processors/networks/image-network.py @@ -1,8 +1,6 @@ """ Make a bipartite Image-Item network """ -import json - from backend.lib.processor import BasicProcessor from common.lib.helpers import hash_file @@ -13,7 +11,7 @@ __maintainer__ = "Stijn Peeters" __email__ = "4cat@oilab.eu" -from common.lib.exceptions import ProcessorInterruptedException +from common.lib.exceptions import ProcessorInterruptedException, MetadataException from common.lib.user_input import UserInput from common.lib.compatibility import Compatibility @@ -114,49 +112,47 @@ def process(self): column = self.parameters.get("column") hash_type = self.parameters.get("deduplicate") filename_filter = [".metadata.json"] if hash_type == "none" else [] - metadata = None hashed = 0 - # some maps to make sure we use the right value in the right place - # url or filename, original image or duplicate, etc - file_hash_map = {} + try: + metadata = self.source_dataset.read_media_metadata() + except (FileNotFoundError, MetadataException): + return self.dataset.finish_with_error("No valid metadata found in image archive - this processor can only " + "be run on sets of images sourced from another 4CAT dataset.") + + # File maps used during graph construction. With v1 metadata each + # entry is already keyed by filename, so the file_hash_map starts as + # the identity mapping; if image-value is "url" we'll resolve via + # file_url_map at edge-write time. + file_hash_map = {filename: filename for filename, _ in metadata.iter_entries()} hash_file_map = {} seen_hashes = set() id_file_map = {} for image in self.source_dataset.iterate_items(filename_filter=filename_filter): if image.file.name == ".metadata.json": - with image.file.open() as infile: - try: - metadata = json.load(infile) - file_hash_map = {i: v["filename"] for i, v in metadata.items()} if self.parameters.get("image-value") == "url" else {i["filename"]: i["filename"] for i in metadata.values()} - except json.JSONDecodeError: - pass - else: - try: - hashed += 1 - if hashed % 100 == 0: - self.dataset.update_status(f"Generated identity hashes for {hashed:,} of {self.source_dataset.num_rows-1:,} item(s)") - self.dataset.update_progress(hashed / (self.source_dataset.num_rows-1) * 0.5) - file_hash = hash_file(image.file, hash_type) - file_hash_map[image.file.name] = file_hash - if file_hash not in hash_file_map: - hash_file_map[file_hash] = image.file.name - - except (FileNotFoundError, ValueError): - continue - - if not metadata: - return self.dataset.finish_with_error("No valid metadata found in image archive - this processor can only " - "be run on sets of images sourced from another 4CAT dataset.") + continue + try: + hashed += 1 + if hashed % 100 == 0: + self.dataset.update_status(f"Generated identity hashes for {hashed:,} of {self.source_dataset.num_rows-1:,} item(s)") + self.dataset.update_progress(hashed / (self.source_dataset.num_rows-1) * 0.5) + file_hash = hash_file(image.file, hash_type) + file_hash_map[image.file.name] = file_hash + if file_hash not in hash_file_map: + hash_file_map[file_hash] = image.file.name + except (FileNotFoundError, ValueError): + continue - file_url_map = {v["filename"]: u for u, v in metadata.items()} - for url, details in metadata.items(): - for item_id in details.get("post_ids", []): + # not every downloader records a URL (e.g. Telegram, Stable Diffusion); + # use the filename for those, so each image still gets its own node + file_url_map = {filename: item.get("url") or filename for filename, item in metadata.iter_entries()} + for filename, item in metadata.iter_entries(): + for item_id in item.get("post_ids", []): if self.source_dataset.type.endswith("-telegram"): # telegram has weird IDs - item_id = "-".join(details["filename"].split("-")[:-1]) + "-" + str(item_id) - id_file_map[item_id] = details["filename"] + item_id = "-".join(filename.split("-")[:-1]) + "-" + str(item_id) + id_file_map[item_id] = filename root_dataset = self.get_root_dataset(self.dataset) self.for_cleanup.append(root_dataset) @@ -207,7 +203,7 @@ def process(self): continue if self.parameters.get("image-value") == "url": - to_node = file_url_map[to_node] + to_node = file_url_map.get(to_node, to_node) to_node_id = f"image-{to_node}" if from_node_id not in network.nodes: diff --git a/processors/visualisation/download_images.py b/processors/visualisation/download_images.py index f4572106d..6db591a0b 100644 --- a/processors/visualisation/download_images.py +++ b/processors/visualisation/download_images.py @@ -224,7 +224,9 @@ def process(self): ua = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:123.0) Gecko/20100101 Firefox/123.0" downloaded_files = set() failures = [] - metadata = {} + metadata = self.dataset.new_media_metadata( + processor_type=self.type, from_dataset=self.source_dataset.key + ) self.dataset.log(f"Filename prep for {len(urls)} URLs") # prepare filenames for each url @@ -384,25 +386,29 @@ def process(self): # before ending the loop and wrapping up self.complete = True + resolved_url, url_extra = self.urls_for_metadata(url) if failure: failures.append(url) downloaded_file.unlink(missing_ok=True) - - metadata[url] = { - "filename": self.filenames[url], - "url": self.resolve_url(url), - "success": not failure, - "from_dataset": self.source_dataset.key, - "post_ids": item_map[url], - } + metadata.add_failure( + post_ids=item_map[url], + reason="error", + url=resolved_url, + extra=url_extra, + ) + else: + metadata.add_item( + filename=self.filenames[url], + post_ids=item_map[url], + url=resolved_url, + extra=url_extra, + replace=True, + ) if self.complete: break - with self.staging_area.joinpath(".metadata.json").open( - "w", encoding="utf-8" - ) as outfile: - json.dump(metadata, outfile) + metadata.write(self.staging_area) # delete supernumerary partially downloaded files self.flush_proxied_requests() # get rid of remaining queue @@ -415,7 +421,7 @@ def process(self): # finish up self.dataset.update_progress(1.0) self.write_archive_and_finish( - self.staging_area, len([x for x in metadata.values() if x.get("success")]), + self.staging_area, len(metadata), finish=False ) if self.warning_message: @@ -563,6 +569,22 @@ def resolve_url(self, url): return url + def urls_for_metadata(self, url): + """ + Get the URLs to record in the archive metadata for a download + + The metadata's `url` is where the file was downloaded from, after + following any redirects. If the URL as it appeared in the dataset is + different, it is listed in `other_urls`, so the file can be found by + either. + + :param str url: URL as it appeared in the dataset + :return tuple: The URL to record, and extra data for the metadata + """ + resolved_url = self.resolve_url(url) + extra = {"other_urls": [url]} if resolved_url != url else {} + return resolved_url, extra + @staticmethod def find_image_urls(value): """ @@ -610,20 +632,29 @@ def get_valid_file_extension(self, file): raise InvalidDownloadedFileException(getattr(e, "message", str(e))) from e @staticmethod - def map_metadata(url, data): + def map_metadata(filename, item): """ - Iterator to yield modified metadata for CSV - - :param str url: string that may contain URLs - :param dict data: dictionary with metadata collected previously - :yield dict: iterator containing reformated metadata + Yield CSV row(s) for a successful `items[filename]` entry. """ - row = { - "url": url, - "number_of_posts_with_url": len(data.get("post_ids", [])), - "post_ids": ", ".join(data.get("post_ids", [])), - "filename": data.get("filename"), - "download_successful": data.get("success", ""), + yield { + "url": item.get("url", ""), + "number_of_posts_with_url": len(item.get("post_ids", [])), + "post_ids": ", ".join(item.get("post_ids", [])), + "filename": filename, + "download_successful": True, } - yield row + @staticmethod + def map_failure_metadata(failure): + """ + Yield CSV row(s) for a `failures[]` entry. + """ + yield { + "url": failure.get("url", ""), + "number_of_posts_with_url": len(failure.get("post_ids", [])), + "post_ids": ", ".join(failure.get("post_ids", [])), + "filename": "", + "download_successful": False, + "reason": failure.get("reason", ""), + "reason_description": failure.get("reason_description", ""), + } diff --git a/processors/visualisation/download_telegram_videos.py b/processors/visualisation/download_telegram_videos.py index a463d0126..76252ebce 100644 --- a/processors/visualisation/download_telegram_videos.py +++ b/processors/visualisation/download_telegram_videos.py @@ -9,7 +9,6 @@ class attributes to switch behavior for a different media type. """ import asyncio import hashlib -import json import re from collections import Counter @@ -156,14 +155,15 @@ def process(self): """ self.staging_area = self.dataset.get_staging_area() self.eventloop = None - self.metadata = {} + self.metadata = self.dataset.new_media_metadata( + processor_type=self.type, from_dataset=self.source_dataset.key + ) self.reason_counts = Counter() asyncio.run(self.get_media()) # finish up - with self.staging_area.joinpath(".metadata.json").open("w", encoding="utf-8") as outfile: - json.dump(self.metadata, outfile) + self.metadata.write(self.staging_area) self.dataset.update_status(f"Compressing {self._media_label}s") successful = self.reason_counts.get("ok", 0) @@ -321,21 +321,20 @@ async def get_media(self): reason_code, reason_text = self.categorize_download_error(e) self.reason_counts[reason_code] += 1 - if not success: + post_ids = [msg_id] if msg_id else [] + if success: + self.metadata.add_item(filename, post_ids=post_ids, replace=True) + else: msg_id_log = msg_id if msg_id else f"index {media_done:,}" self.dataset.log( f"Skipped {self._media_label} for message {msg_id_log} " f"[{reason_code}]: {reason_text}") self.flawless = False - - self.metadata[filename] = { - "filename": filename, - "success": success, - "reason": reason_code, - "reason_description": reason_text, - "from_dataset": self.source_dataset.key, - "post_ids": [msg_id] if msg_id else [] - } + self.metadata.add_failure( + post_ids=post_ids, + reason=reason_code, + reason_description=reason_text, + ) media_done += 1 except FloodError as e: @@ -364,14 +363,11 @@ async def get_media(self): for mid in message_ids: msg_id_full = f"{entity}-{mid}" self.reason_counts[reason_code] += 1 - self.metadata[msg_id_full] = { - "filename": "", - "success": False, - "reason": reason_code, - "reason_description": reason_text, - "from_dataset": self.source_dataset.key, - "post_ids": [msg_id_full] - } + self.metadata.add_failure( + post_ids=[msg_id_full], + reason=reason_code, + reason_description=reason_text, + ) # end-of-run outcome breakdown into the dataset log so researchers can # see counts per category without parsing per-message lines @@ -384,17 +380,25 @@ async def get_media(self): await client.disconnect() @classmethod - def map_metadata(cls, filename, data): - """ - Iterator to yield modified metadata for CSV - """ - row = { - cls._metadata_count_label: len(data.get("post_ids", [])), - "post_ids": ", ".join(map(str, data.get("post_ids", []))), + def map_metadata(cls, filename, item): + """Yield CSV row(s) for a successful items[filename] entry.""" + yield { + cls._metadata_count_label: len(item.get("post_ids", [])), + "post_ids": ", ".join(map(str, item.get("post_ids", []))), "filename": filename, - "download_successful": data.get('success', ""), - "reason": data.get("reason", ""), - "reason_description": data.get("reason_description", "") + "download_successful": True, + "reason": "ok", + "reason_description": "", } - yield row + @classmethod + def map_failure_metadata(cls, failure): + """Yield CSV row(s) for a failures[] entry.""" + yield { + cls._metadata_count_label: len(failure.get("post_ids", [])), + "post_ids": ", ".join(map(str, failure.get("post_ids", []))), + "filename": "", + "download_successful": False, + "reason": failure.get("reason", ""), + "reason_description": failure.get("reason_description", ""), + } diff --git a/processors/visualisation/download_tiktok.py b/processors/visualisation/download_tiktok.py index 04e0a1f92..6dc7b978a 100644 --- a/processors/visualisation/download_tiktok.py +++ b/processors/visualisation/download_tiktok.py @@ -3,7 +3,6 @@ """ import asyncio import datetime -import json from io import BytesIO from pathlib import Path from PIL import Image, UnidentifiedImageError @@ -114,7 +113,9 @@ def process(self): urls_to_refresh = [] url_to_item_id = {} max_fails_exceeded = 0 - metadata = {} + metadata = self.dataset.new_media_metadata( + processor_type=self.type, from_dataset=self.source_dataset.key + ) # Loop through items and collect URLs for mapped_item in self.source_dataset.iterate_items(self): @@ -156,12 +157,12 @@ def process(self): downloaded_media += 1 self.dataset.update_status(f"Downloaded image {downloaded_media}/{max_amount}") - metadata[url] = { - "filename": filename, - "success": success, - "from_dataset": self.source_dataset.key, - "post_ids": [post_id] - } + metadata.add_item( + filename, + post_ids=[post_id], + url=url, + replace=True, + ) if refresh_tiktok_urls: @@ -217,29 +218,33 @@ def process(self): filename = '' # Record metadata - metadata[url] = { - "filename": filename, - "success": success, - "from_dataset": self.source_dataset.key, - "post_ids": [post_id] - } - if success: + metadata.add_item( + filename, + post_ids=[post_id], + url=url, + replace=True, + ) self.dataset.update_status(f"Downloaded image for {url}") downloaded_media += 1 - elif not url: - self.dataset.log( - f"No {url_column} identified for {refreshed_mapped_item.get('tiktok_url')}, skipping") else: - self.dataset.log(f"Unable to save image for {url}, skipping") + metadata.add_failure( + post_ids=[post_id], + reason="error", + url=url, + ) + if not url: + self.dataset.log( + f"No {url_column} identified for {refreshed_mapped_item.get('tiktok_url')}, skipping") + else: + self.dataset.log(f"Unable to save image for {url}, skipping") # In case some images failed to download, we update our starting points last_url_index += need_more need_more = max_amount - downloaded_media # Write metadata file - with results_path.joinpath(".metadata.json").open("w", encoding="utf-8") as outfile: - json.dump(metadata, outfile) + metadata.write(results_path) warning = None if downloaded_media < max_amount: @@ -299,20 +304,17 @@ def collect_image(url, user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_ return picture, extension @staticmethod - def map_metadata(url, data): + def map_metadata(filename, item): """ - Iterator to yield modified metadata for CSV - - :param str url: string that may contain URLs - :param dict data: dictionary with metadata collected previously - :yield dict: iterator containing reformated metadata + Yield CSV row(s) for a successful metadata entry, in the same format + as the general image downloader """ - row = { - "url": url, - "number_of_posts_with_url": len(data.get("post_ids", [])), - "post_ids": ", ".join(data.get("post_ids", [])), - "filename": data.get("filename"), - "download_successful": data.get('success', "") - } + yield from ImageDownloader.map_metadata(filename, item) - yield row + @staticmethod + def map_failure_metadata(failure): + """ + Yield CSV row(s) for a failed download, in the same format as the + general image downloader + """ + yield from ImageDownloader.map_failure_metadata(failure) diff --git a/processors/visualisation/download_tiktok_video.py b/processors/visualisation/download_tiktok_video.py index c6e0b6c3b..a193dfb79 100644 --- a/processors/visualisation/download_tiktok_video.py +++ b/processors/visualisation/download_tiktok_video.py @@ -106,27 +106,37 @@ def get_processor_pipeline(self): "amount": amount, "columns": ["video_url"], "also_indirect": "all", # enabled YT-DLP - "_ytdlp_fallback_column": "tiktok_url" # YT-DLP uses TikTok post URL + "_ytdlp_fallback_column": "tiktok_url", # YT-DLP uses TikTok post URL + # The helper dataset above copies each item's own id through, + # so the archive's metadata can credit the dataset this preset + # was run on rather than the helper. Only when the id column is + # the items' own `id`: with any other column the recorded post + # IDs would not match that dataset's items, and the helper - + # whose `id` column does hold them - stays the right answer. + **({"_from_dataset": self.source_dataset.key} if column == "id" else {}), } }, ] return pipeline - @staticmethod - def map_metadata(url, data): + @classmethod + def map_metadata(cls, filename, item): """ - Iterator to yield modified metadata for CSV + Yield CSV row(s) for a successful metadata entry The archive this preset produces is written by the video downloader in its pipeline and copied to this dataset, so the metadata file inside it has the video downloader's format and is read the same way. + """ + yield from VideoDownloaderPlus.map_metadata(filename, item) - :param str url: string that may contain URLs - :param dict data: dictionary with metadata collected previously - :yield dict: iterator containing reformated metadata + @classmethod + def map_failure_metadata(cls, failure): + """ + Yield CSV row(s) for a failed download, in the video downloader's format """ - yield from VideoDownloaderPlus.map_metadata(url, data) + yield from VideoDownloaderPlus.map_failure_metadata(failure) class TikTokVideoMetadata(BasicProcessor): diff --git a/processors/visualisation/download_videos.py b/processors/visualisation/download_videos.py index 66dfc9198..bdb255bf1 100644 --- a/processors/visualisation/download_videos.py +++ b/processors/visualisation/download_videos.py @@ -4,9 +4,7 @@ First attempt to download via request, but if that fails use yt-dlp. """ import copy -import json import re -import shutil import time import zipfile from pathlib import Path @@ -20,9 +18,9 @@ from backend.lib.processor import BasicProcessor from backend.lib.proxied_requests import FailedProxiedRequest from common.lib.compatibility import Compatibility -from common.lib.dataset import DataSet -from common.lib.exceptions import ProcessorInterruptedException, ProcessorException, DataSetException +from common.lib.exceptions import ProcessorInterruptedException, ProcessorException from common.lib.helpers import UserInput, sets_to_lists, url_to_filename, split_urls, urls_in_text +from common.lib.media_archive_library import MediaArchiveLibrary __author__ = "Dale Wahl" __credits__ = ["Dale Wahl"] @@ -324,7 +322,7 @@ def process(self): else: self.dataset.update_status('Collected %i urls.' % len(urls)) - vid_lib = DatasetVideoLibrary(self.dataset, modules=self.modules) + vid_lib = MediaArchiveLibrary.collect(self.dataset, self.modules, ["video-downloader"]) # Prepare staging area for videos and video tracking results_path = self.dataset.get_staging_area() @@ -353,6 +351,9 @@ def process(self): self.downloaded_videos = 0 self.failed_downloads = 0 self.copied_videos = 0 + # filename -> key of the earlier dataset it was copied from, so a video + # that several URLs lead to is copied (and counted) only once + self.files_copied_from_library = {} consecutive_errors = 0 # not_a_video tracked within direct_results self.total_not_a_video = 0 @@ -518,7 +519,7 @@ def add_to_queue(self, url_list, queue, url_dict, skip_channels=False, copy_if_e :param dict url_dict: Dictionary to update with URL metadata :param bool skip_channels: Whether to skip known channels :param bool copy_if_existing: Whether to attempt to copy video from previous dataset if URL exists in library - :param DatasetVideoLibrary existing_video_library: Video library to check for existing videos if `copy_if_existing` is `True` + :param MediaArchiveLibrary existing_video_library: Video library to check for existing videos if `copy_if_existing` is `True` :param Path copy_output_path: Path to copy videos to if copied from library :return: None """ @@ -540,7 +541,6 @@ def add_to_queue(self, url_list, queue, url_dict, skip_channels=False, copy_if_e # Initialize URL metadata url_dict[url]["success"] = False - url_dict[url]["retry"] = True # Skip known channels if not downloading channels if skip_channels and any([sub_url in url for sub_url in self.known_channels]): @@ -556,50 +556,95 @@ def add_to_queue(self, url_list, queue, url_dict, skip_channels=False, copy_if_e def _try_copy_from_library(self, url, urls_dict, vid_lib, results_path): """ - Try to copy video from previously downloaded library - + Try to copy a video already fetched by a previous downloader run. + :param str url: URL to check :param dict urls_dict: URLs dictionary to update - :param DatasetVideoLibrary vid_lib: Video library instance + :param MediaArchiveLibrary vid_lib: Video library instance :param Path results_path: Path to staging area :return dict: Result with 'copied', 'count', and 'skip' keys """ result = {"copied": False, "count": 0, "skip": False} - - if url not in vid_lib.library: + + hit = vid_lib.find(url) + if hit is None: return result - - previous_vid_metadata = vid_lib.library[url] - - if previous_vid_metadata.get('success', False): - # Use previous downloaded video + + if hit.is_success: + # copy the previously downloaded file(s) instead of re-fetching try: self.dataset.log(f"Copying previously downloaded video for url: {url}") - num_copied = self.copy_previous_video(previous_vid_metadata, results_path, vid_lib.previous_downloaders) - urls_dict[url] = previous_vid_metadata + num_copied = self.copy_previous_video(hit, results_path) + # keep this run's post_ids; fill in the download outcome + urls_dict[url].update(self._copied_url_fields(hit)) self.dataset.update_status("Copied previously downloaded video to current dataset.") result["copied"] = True result["count"] = num_copied except FailedToCopy as e: self.dataset.log(f"{str(e)}; attempting to download again") - elif previous_vid_metadata.get("retry", True) is False: - # Check if the current run has a more inclusive yt-dlp setting than the previous attempt - # Inclusivity order: "none" < "yt_only" < "all" - indirect_levels = {"none": 0, "yt_only": 1, "all": 2} - previous_also_indirect = previous_vid_metadata.get("also_indirect", "none") - current_also_indirect = self.parameters.get("also_indirect", "none") - previous_level = indirect_levels.get(previous_also_indirect, 0) - current_level = indirect_levels.get(current_also_indirect, 0) - if current_level > previous_level: - # Current run has more inclusive yt-dlp setting; retry - self.dataset.log(f"Previously identified as not a video with also_indirect={previous_also_indirect}; retrying with also_indirect={current_also_indirect}: {url}") + elif "not_a_video" in hit.reasons: + # a previous run established this URL is not a video, so normally + # don't retry. But that conclusion only holds for the kinds of URL + # that run was allowed to hand to yt-dlp: if this run is allowed + # more, the URL deserves another try. + if self._widens_indirect_downloads(hit): + self.dataset.log(f"Previously identified as not a video, but this run allows yt-dlp more " + f"URLs than the run that decided that; trying again: {url}") else: - urls_dict[url] = previous_vid_metadata + urls_dict[url]["success"] = False + urls_dict[url]["reason"] = "not_a_video" self.dataset.log(f"Skipping; previously identified url as not a video: {url}") result["skip"] = True - + return result + # How much of the web yt-dlp is allowed to be pointed at, least to most. + # A run can only reach URLs a lower setting would have skipped. + INDIRECT_DOWNLOAD_LEVELS = {"none": 0, "yt_only": 1, "all": 2} + + def _widens_indirect_downloads(self, hit): + """ + Does this run allow yt-dlp more URLs than every run that decided this URL + was not a video? + + The datasets those runs produced store the parameters they ran with, so + their settings are read back from there rather than from the archive's + metadata. A dataset created before this setting existed reads as the + lowest level, so it is retried rather than wrongly skipped. + + :param MediaLibraryHit hit: failure hit for the URL + :return bool: True if this run is more permissive than all of them + """ + levels = self.INDIRECT_DOWNLOAD_LEVELS + current = levels.get(self.parameters.get("also_indirect", "none"), 0) + previous = max( + (levels.get(dataset.parameters.get("also_indirect", "none"), 0) + for dataset in hit.datasets_for_reason("not_a_video")), + default=0 + ) + return current > previous + + @staticmethod + def _copied_url_fields(hit): + """ + Build the `urls_dict` fields for a URL whose file(s) were copied from + a previous archive, from a successful `MediaLibraryHit`. + """ + files = [] + downloader = None + for filename, item in hit.entries: + extra = dict(item.get("extra") or {}) + # `downloader` lived at URL level in the working dict, not per file + downloader = downloader or extra.pop("downloader", None) + # the earlier archive's other URLs came from the posts of its own + # dataset; this archive records the URLs of this dataset's posts + extra.pop("other_urls", None) + files.append({"filename": filename, "success": True, "metadata": extra}) + fields = {"success": True, "files": files} + if downloader: + fields["downloader"] = downloader + return fields + def _process_direct_downloads(self, url_list, urls_dict, results_path, max_video_size, also_indirect, amount, last_domains, ignore_not_video): """ @@ -775,8 +820,7 @@ def _handle_direct_download_response(self, url, response, urls_dict, results_pat else: self.dataset.log(f"NotVideoLinkError: {str(e)}") urls_dict[url]["error"] = str(e) - urls_dict[url]["retry"] = False - urls_dict[url]["also_indirect"] = also_indirect + urls_dict[url]["reason"] = "not_a_video" result["not_a_video"] = True if last_domains.count(domain) >= 2: @@ -790,8 +834,7 @@ def _handle_direct_download_response(self, url, response, urls_dict, results_pat except NotAVideo as e: self.dataset.log(f"Request Error: {str(e)}") urls_dict[url]["error"] = str(e) - urls_dict[url]["retry"] = False - urls_dict[url]["also_indirect"] = also_indirect + urls_dict[url]["reason"] = "not_a_video" result["not_a_video"] = True if last_domains.count(domain) >= 2: @@ -955,16 +998,48 @@ def _download_single_ytdlp(self, url, ydl_opts, urls_dict): return result def _save_metadata(self, urls_dict, results_path): - """Save metadata to JSON file""" + """Save metadata to .metadata.json in the staging area.""" self.dataset.update_status("Updating and saving metadata") - metadata = { - url: { - "from_dataset": self.source_dataset.key, - **sets_to_lists(data) - } for url, data in urls_dict.items() - } - with results_path.joinpath(".metadata.json").open("w", encoding="utf-8") as outfile: - json.dump(metadata, outfile) + # A preset may run this downloader on a stand-in dataset it built itself. + # Where that stand-in carries the original items' IDs, the preset names + # the original dataset here, so the metadata credits the dataset a + # researcher would recognise instead of the stand-in. + from_dataset = self.parameters.get("_from_dataset") or self.source_dataset.key + metadata = self.dataset.new_media_metadata( + processor_type=self.type, from_dataset=from_dataset + ) + for url, data in urls_dict.items(): + data = sets_to_lists(data) + post_ids = list(data.get("post_ids", [])) + files = data.get("files") or [] + had_successful_file = False + for file in files: + if not isinstance(file, dict): + continue + filename = file.get("filename") + if not filename or not file.get("success", True): + continue + extra = dict(file.get("metadata") or {}) + if data.get("downloader"): + extra["downloader"] = data["downloader"] + # merge=True: one file can belong to several URLs, e.g. when + # yt-dlp recognises a video it already downloaded for an earlier + # URL, or the same video appears twice in a playlist. Every URL's + # posts must stay connected to the file; the URLs after the first + # are listed in the file's extra["other_urls"]. + metadata.add_item(filename, post_ids=post_ids, url=url, + extra=extra, merge=True) + had_successful_file = True + + if not had_successful_file: + metadata.add_failure( + post_ids=post_ids, + reason=data.get("reason") or "error", + reason_description=data.get("error", "") or "", + url=url, + ) + + metadata.write(results_path) def _log_statistics(self, total_urls): """Log comprehensive download statistics""" @@ -986,8 +1061,10 @@ def _log_statistics(self, total_urls): def _finish_processing(self, results_path, total_urls): """Finish processing and create result archive""" self.dataset.update_status("Writing downloaded videos to zip archive") - self.write_archive_and_finish(results_path, self.downloaded_videos + self.copied_videos, finish=False) - + # videos copied from earlier downloads are in the archive too + num_videos = self.downloaded_videos + self.copied_videos + self.write_archive_and_finish(results_path, num_videos, finish=False) + status_msg = f"Downloaded {self.downloaded_videos} videos" if self.copied_videos > 0: status_msg += f"; {self.copied_videos} videos copied from previous downloads" @@ -1000,10 +1077,10 @@ def _finish_processing(self, results_path, total_urls): if self.warning_message: self.dataset.update_status(status_msg) - self.dataset.finish_with_warning(self.downloaded_videos, f"Incomplete: {self.warning_message}") + self.dataset.finish_with_warning(num_videos, f"Incomplete: {self.warning_message}") else: self.dataset.update_status(status_msg, is_final=True) - self.dataset.finish(self.downloaded_videos) + self.dataset.finish(num_videos) def yt_dlp_monitor(self, d): """Can be used to gather information from yt-dlp while downloading""" @@ -1241,143 +1318,92 @@ def identify_video_urls_in_string(self, text): """Search string of text for URLs that may contain video links""" return urls_in_text(text) - def copy_previous_video(self, previous_vid_metadata, staging_area, previous_downloaders): - """Copy existing video to new staging area""" - num_copied = 0 - dataset_key = previous_vid_metadata.get("file_dataset_key") - dataset = [dataset for dataset in previous_downloaders if dataset.key == dataset_key] - - if "files" in previous_vid_metadata: - files = previous_vid_metadata.get('files') - elif "filename" in previous_vid_metadata: - files = [{"filename": previous_vid_metadata.get("filename"), "success": True}] - else: - raise FailedToCopy("Unable to read video metadata") + def copy_previous_video(self, hit, staging_area): + """ + Copy the file(s) of a previously downloaded video into the current + staging area. + + Several URLs can lead to the same earlier file (e.g. a short and a + long link to one video). A file that was already copied for another + URL in this run is not copied or counted again. + + :param MediaLibraryHit hit: a successful library lookup; its + `metadata.dataset` locates the source archive and `entries` + lists the files to copy. + :param Path staging_area: where to extract the files to. + :return int: number of files newly copied. + """ + source_dataset = hit.metadata.dataset + if source_dataset is None: + raise FailedToCopy("Previous metadata is not bound to a dataset") - if not files: + filenames = [filename for filename, _ in hit.entries] + if not filenames: raise FailedToCopy("No file found in metadata") - if not dataset: - raise FailedToCopy(f"Dataset with key {dataset_key} not found") - else: - dataset = dataset[0] - - with zipfile.ZipFile(dataset.get_results_path(), "r") as archive_file: - archive_contents = sorted(archive_file.namelist()) - - for file in files: - if file.get("filename") not in archive_contents: - raise FailedToCopy(f"Previously downloaded video {file.get('filename')} not found") - - self.dataset.log(f"Copying previously downloaded video {file.get('filename')} to new staging area") - archive_file.extract(file.get("filename"), staging_area) + num_copied = 0 + with zipfile.ZipFile(source_dataset.get_results_path(), "r") as archive_file: + archive_contents = set(archive_file.namelist()) + + # check every file before copying any, so that a URL is never left + # with only some of its files copied + to_copy = [] + for filename in filenames: + if filename not in archive_contents: + raise FailedToCopy(f"Previously downloaded video {filename} not found") + copied_from = self.files_copied_from_library.get(filename) + if copied_from == source_dataset.key: + self.dataset.log(f"Previously downloaded video {filename} was already copied for another URL") + continue + if copied_from is not None: + # a file of the same name from a different earlier dataset + # may be a different video; copying it would overwrite the + # one already copied + raise FailedToCopy(f"A video named {filename} was already copied from dataset {copied_from}") + to_copy.append(filename) + + for filename in to_copy: + self.dataset.log(f"Copying previously downloaded video {filename} to new staging area") + archive_file.extract(filename, staging_area) + self.files_copied_from_library[filename] = source_dataset.key num_copied += 1 return num_copied - @staticmethod - def map_metadata(url, data): - """Iterator to yield modified metadata for CSV""" + YT_DLP_EXTRACTED_FIELDS = ( + "title", "artist", "description", "view_count", "like_count", + "repost_count", "comment_count", "uploader", "creator", "uploader_id", + ) + + @classmethod + def map_metadata(cls, filename, item): + """Yield CSV row(s) for a successful items[filename] entry.""" + extra = item.get("extra") or {} row = { - "url": url, - "number_of_posts_with_url": len(data.get("post_ids", [])), - "post_ids": ", ".join(data.get("post_ids", [])), - "downloader": data.get("downloader", ""), - "download_successful": data.get('success', "") + "url": item.get("url", ""), + "number_of_posts_with_url": len(item.get("post_ids", [])), + "post_ids": ", ".join(item.get("post_ids", [])), + "downloader": extra.get("downloader", ""), + "download_successful": True, + "filename": filename, } + for field in cls.YT_DLP_EXTRACTED_FIELDS: + row[f"extracted_{field}"] = extra.get(field, "N/A") + row["error"] = "N/A" + yield row - for file in data.get("files", [{}]): - row["filename"] = file.get("filename", "N/A") - yt_dlp_data = file.get("metadata", {}) - for common_column in ["title", "artist", "description", "view_count", "like_count", "repost_count", "comment_count", "uploader", "creator", "uploader_id"]: - if yt_dlp_data: - row[f"extracted_{common_column}"] = yt_dlp_data.get(common_column) - else: - row[f"extracted_{common_column}"] = "N/A" - row["error"] = data.get("error", "N/A") - yield row - - -class DatasetVideoLibrary: - """ - Library for managing video downloads across multiple processors - """ - def __init__(self, current_dataset, modules): - self.modules = modules - self.current_dataset = current_dataset - self.previous_downloaders = self.collect_previous_downloaders() - self.current_dataset.log(f"Previously video downloaders: {[downloader.key for downloader in self.previous_downloaders]}") - - metadata_files = self.collect_all_metadata_files() - - # Build library - library = {} - for metadata_file in metadata_files: - for url, data in metadata_file[1].items(): - if data.get("success", False): - # Always overwrite for success - library[url] = { - **data, - "file_dataset_key": metadata_file[0] - } - elif url not in library: - # Do not overwrite failures, but do add if missing - library[url] = { - **data, - "file_dataset_key": metadata_file[0] - } - - self.current_dataset.log(f"Total URLs previously seen: {len(library)}") - self.library = library - - def collect_previous_downloaders(self): - """ - Check for other video-downloader processors run on the dataset and create library for reference - """ - # NOTE: this only checks parent dataset, not full ancestry (e.g. other filters with video downloaders) - parent_dataset = self.current_dataset.get_parent() - # Note: exclude current dataset - previous_downloaders = [child for child in parent_dataset.get_children() if - (child.type in ["video-downloader"] and child.key != self.current_dataset.key)] - - # Check to see if filtered dataset - if "copied_from" in parent_dataset.parameters and parent_dataset.is_top_dataset(): - try: - original_dataset = DataSet(key=parent_dataset.parameters["copied_from"], db=self.current_dataset.db, modules=self.modules) - previous_downloaders += [child for child in original_dataset.top_parent().get_children() if - (child.type in ["video-downloader"] and child.key != self.current_dataset.key)] - except DataSetException: - # parent dataset no longer exists! - pass - - return previous_downloaders - - def collect_metadata_file(self, dataset, staging_area): - """Collect metadata from a dataset's video archive""" - source_file = dataset.get_results_path() - if not source_file.exists(): - return None - - with zipfile.ZipFile(dataset.get_results_path(), "r") as archive_file: - archive_contents = sorted(archive_file.namelist()) - if '.metadata.json' not in archive_contents: - return None - - archive_file.extract(".metadata.json", staging_area) - - with open(staging_area.joinpath(".metadata.json")) as file: - return json.load(file) - - def collect_all_metadata_files(self): - """Collect all metadata files from previous downloaders""" - metadata_staging_area = self.current_dataset.get_staging_area() - - metadata_files = [(downloader.key, self.collect_metadata_file(downloader, metadata_staging_area)) - for downloader in self.previous_downloaders] - metadata_files = [file for file in metadata_files if file[1] is not None] - self.current_dataset.log(f"Metadata files collected: {len(metadata_files)}; with {[len(urls[1]) for urls in metadata_files]}") - - # Delete staging area - shutil.rmtree(metadata_staging_area) - - return metadata_files \ No newline at end of file + @classmethod + def map_failure_metadata(cls, failure): + """Yield CSV row(s) for a failures[] entry.""" + row = { + "url": failure.get("url", ""), + "number_of_posts_with_url": len(failure.get("post_ids", [])), + "post_ids": ", ".join(failure.get("post_ids", [])), + "downloader": "", + "download_successful": False, + "filename": "", + } + for field in cls.YT_DLP_EXTRACTED_FIELDS: + row[f"extracted_{field}"] = "N/A" + row["error"] = failure.get("reason_description") or failure.get("reason") or "N/A" + yield row diff --git a/processors/visualisation/image_category_wall.py b/processors/visualisation/image_category_wall.py index 7dd9df5c2..29a4276af 100644 --- a/processors/visualisation/image_category_wall.py +++ b/processors/visualisation/image_category_wall.py @@ -3,7 +3,6 @@ """ import io import base64 -import json import math from svgwrite.image import Image as ImageElement @@ -16,7 +15,7 @@ from common.lib.helpers import UserInput, convert_to_int, get_4cat_canvas from backend.lib.processor import BasicProcessor -from common.lib.exceptions import ProcessorInterruptedException +from common.lib.exceptions import ProcessorInterruptedException, MetadataException from common.lib.compatibility import Compatibility __author__ = "Dale Wahl" @@ -195,17 +194,16 @@ def process(self): filename_map = {filename.stem: [filename] for filename in staging_area.iterdir()} else: # Use image metadata to map post IDs to filenames - with open(staging_area.joinpath('.metadata.json')) as file: - image_data = json.load(file) + try: + metadata = image_dataset.read_media_metadata() + except (FileNotFoundError, MetadataException): + self.dataset.finish_with_error("No metadata file found") + return filename_map = {} # Images can belong to multiple posts; posts can have multiple images - for image in image_data.values(): - if image.get("success"): - for post_id in image.get("post_ids"): - if post_id not in filename_map: - filename_map[post_id] = [staging_area.joinpath(image.get("filename"))] - else: - filename_map[post_id].append(staging_area.joinpath(image.get("filename"))) + for filename, item in metadata.iter_entries(): + for post_id in item.get("post_ids", []): + filename_map.setdefault(post_id, []).append(staging_area.joinpath(filename)) # Organize posts into categories category_type = None diff --git a/processors/visualisation/image_wall_w_text.py b/processors/visualisation/image_wall_w_text.py index e0da2638c..633a392fc 100644 --- a/processors/visualisation/image_wall_w_text.py +++ b/processors/visualisation/image_wall_w_text.py @@ -3,7 +3,6 @@ """ import io import base64 -import json import math import textwrap @@ -17,7 +16,7 @@ from common.lib.helpers import UserInput, convert_to_int, get_4cat_canvas from backend.lib.processor import BasicProcessor -from common.lib.exceptions import ProcessorInterruptedException +from common.lib.exceptions import ProcessorInterruptedException, MetadataException from common.lib.compatibility import Compatibility __author__ = "Dale Wahl" @@ -159,21 +158,19 @@ def process(self): max_text_len = 0 filename_to_text_mapping = {} if text_dataset.type in ImageTextWallGenerator.combined_dataset: - # For datasets with both images and text, use .metadata.json - metadata_file = self.extract_archived_file_by_name(".metadata.json", self.source_file) - if metadata_file is None: + # For datasets with both images and text (Stable Diffusion output), + # the prompt is stored as item.extra.prompt in the archive metadata. + try: + metadata = text_dataset.read_media_metadata() + except (FileNotFoundError, MetadataException): self.dataset.finish_with_error("No metadata file found") return - with metadata_file.open() as f: - metadata = json.load(f) - for item in metadata.values(): + for filename, item in metadata.iter_entries(): if self.interrupted: raise ProcessorInterruptedException("Interrupted while collecting text") - if "filename" in item: - # "image-downloader-stable-diffusion" datasets - image_text = item.get("prompt", "") - max_text_len = max(max_text_len, len(image_text)) - filename_to_text_mapping[item["filename"]] = image_text + image_text = (item.get("extra") or {}).get("prompt", "") + max_text_len = max(max_text_len, len(image_text)) + filename_to_text_mapping[filename] = image_text elif text_dataset.type == "llm-prompter": # llm-prompter store filename as "id" and text as "output" for item in text_dataset.iterate_items(self): diff --git a/processors/visualisation/video_hasher.py b/processors/visualisation/video_hasher.py index 8271e7b69..858987a3f 100644 --- a/processors/visualisation/video_hasher.py +++ b/processors/visualisation/video_hasher.py @@ -5,7 +5,6 @@ https://ffmpeg.org/ """ import csv -import json import shutil import zipfile @@ -17,7 +16,7 @@ from backend.lib.processor import BasicProcessor from backend.lib.preset import ProcessorAdvancedPreset from common.lib.compatibility import Compatibility, is_executable -from common.lib.exceptions import ProcessorInterruptedException, ProcessorException +from common.lib.exceptions import ProcessorInterruptedException, ProcessorException, MetadataException from common.lib.user_input import UserInput __author__ = "Dale Wahl" @@ -186,7 +185,10 @@ def process(self): output_dir = self.dataset.get_staging_area() video_hashes = {} - video_metadata = None + try: + video_metadata = self.source_dataset.read_media_metadata() + except (FileNotFoundError, MetadataException): + video_metadata = None total_possible_videos = max((min(self.source_dataset.num_rows - 1, max_videos) if max_videos != 0 else self.source_dataset.num_rows), 1) processed_videos = 0 @@ -199,9 +201,6 @@ def process(self): break if video.file.name == '.metadata.json': - # Keep it and move on - with video.file.open() as file: - video_metadata = json.load(file) continue elif video.file.name == "video_archive": # yt-dlp file @@ -244,16 +243,6 @@ def process(self): num_posts = 0 rows = [] annotations = [] - if video_metadata is None: - # Grab the metadata directly, if it exists but was skipped (e.g., not found prior to max_videos) - try: - metadata_path = self.extract_archived_file_by_name(".metadata.json", self.source_file, output_dir) - except FileNotFoundError: - metadata_path = None - if metadata_path: - with open(metadata_path) as file: - video_metadata = json.load(file) - if video_metadata is None: self.dataset.log( "No video metadata (i.e., from video downloader) found; unable to connect original posts. Saving video hashes only.") @@ -270,43 +259,31 @@ def process(self): num_posts += 1 else: self.dataset.update_status("Saving video hash results") - for url, data in video_metadata.items(): - if not data.get("success"): - continue - if "files" in data: - files = data.get('files') - elif "filename" in data: - files = [{"filename": data.get("filename"), "success": True}] - else: - self.dataset.log(f"Metadata Error: {url} with {data}") + from_dataset = video_metadata.from_dataset + for filename, item in video_metadata.iter_entries(): + if filename not in video_hashes: continue + video_hash = video_hashes[filename].get('videohash') + post_ids = item.get("post_ids", []) + rows.append({ + 'id': filename, + 'url': item.get("url", ""), + "from_dataset": from_dataset, + 'video_hash': video_hash.hash, + 'video_duration': video_hash.video_duration, + 'video_count': len(post_ids), + "post_ids": ','.join(str(p) for p in post_ids), + 'video_collage_filename': video_hashes[filename].get('video_collage_filename'), + }) + if save_annotations: + for item_id in post_ids: + annotations.append({ + "label": "video-hash", + "value": video_hash.hash, + "item_id": item_id + }) - for file in files: - if not file.get("success"): - continue - if file.get('filename') not in video_hashes: - self.dataset.log(f"Metadata Error: {file.get('filename')} with {url} - {data}") - continue - video_hash = video_hashes[file.get('filename')].get('videohash') - rows.append({ - 'id': file.get('filename'), # best if all datasets have unique identifier - 'url': url, - "from_dataset": data.get("from_dataset"), - 'video_hash': video_hash.hash, - 'video_duration': video_hash.video_duration, - 'video_count': len(data.get('post_ids', [])), - "post_ids": ','.join([str(post_id) for post_id in data.get("post_ids", [])]), - 'video_collage_filename': video_hashes[file.get('filename')].get('video_collage_filename'), - }) - if save_annotations: - for item_id in data.get("post_ids", []): - annotations.append({ - "label": "video-hash", - "value": video_hash.hash, - "item_id": item_id - }) - - num_posts += 1 + num_posts += 1 writer = None diff --git a/processors/visualisation/video_scene_identifier.py b/processors/visualisation/video_scene_identifier.py index 8a85dec55..de40a8efe 100644 --- a/processors/visualisation/video_scene_identifier.py +++ b/processors/visualisation/video_scene_identifier.py @@ -10,7 +10,7 @@ from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility, ExecutableSibling -from common.lib.exceptions import ProcessorInterruptedException, ProcessorException +from common.lib.exceptions import ProcessorInterruptedException, ProcessorException, MetadataException from common.lib.user_input import UserInput __author__ = "Dale Wahl" @@ -215,17 +215,16 @@ def process(self): self.dataset.update_status("Detecting video scenes") skipped = 0 processed_videos = 0 - video_metadata = None + try: + video_metadata = self.source_dataset.read_media_metadata() + except (FileNotFoundError, MetadataException): + video_metadata = None collected_scenes = {} for original_video in self.source_dataset.iterate_items(self, immediately_delete=False): if self.interrupted: raise ProcessorInterruptedException("Interrupted while detecting video scenes") - # Check for 4CAT's metadata JSON and copy it if original_video.file.name == ".metadata.json": - # Keep it and move on - with open(original_video.file) as file: - video_metadata = json.load(file) continue elif original_video.file.name == "video_archive": # yt-dlp file @@ -264,39 +263,29 @@ def process(self): num_posts += 1 else: self.dataset.update_status("Saving video scene results") - for url, video_data in video_metadata.items(): - if video_data.get('success'): - files = video_data.get('files') if 'files' in video_data else [{"filename": video_data.get("filename"), "success":True}] - for file in files: - if not file.get("success") or file.get("filename") not in collected_scenes: - continue - - # List types are not super fun for CSV - if 'post_ids' in video_data: - video_data['post_ids'] = ','.join(video_data['post_ids']) - - - for i, scene in enumerate(collected_scenes[file.get('filename')]): - rows.append({ - 'id': file.get('filename') + '_scene_' + str(i+1), # best if all datasets have unique identifier - 'url': url, - "from_dataset": video_data.get("from_dataset"), - **scene, - "post_ids": ','.join(video_data.get("post_ids", [])), + from_dataset = video_metadata.from_dataset + for filename, item in video_metadata.iter_entries(): + if filename not in collected_scenes: + continue + post_ids = item.get("post_ids", []) + post_ids_csv = ','.join(post_ids) + for i, scene in enumerate(collected_scenes[filename]): + rows.append({ + 'id': filename + '_scene_' + str(i + 1), + 'url': item.get("url", ""), + "from_dataset": from_dataset, + **scene, + "post_ids": post_ids_csv, + }) + num_posts += 1 + + if save_annotations and i == 0: + for item_id in post_ids: + annotations.append({ + "label": "scene_amount", + "value": scene.get("num_scenes_detected", ""), + "item_id": item_id, }) - num_posts += 1 - - # Write amount of scenes for first scene detected - if save_annotations and i == 0: - item_ids = video_data.get("post_ids", []) - item_ids = [item_ids] if isinstance(item_ids, str) else item_ids - for item_id in item_ids: - annotation = { - "label": "scene_amount", - "value": scene.get("num_scenes_detected", ""), - "item_id": item_id - } - annotations.append(annotation) if save_annotations and annotations: self.save_annotations(annotations) diff --git a/processors/visualisation/video_timelines.py b/processors/visualisation/video_timelines.py index df98f612a..d701f1828 100644 --- a/processors/visualisation/video_timelines.py +++ b/processors/visualisation/video_timelines.py @@ -2,7 +2,6 @@ Create timelines from collections of video frames """ import base64 -import json import io from PIL import Image @@ -15,7 +14,7 @@ from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility -from common.lib.exceptions import DataSetNotFoundException, ProcessorInterruptedException +from common.lib.exceptions import DataSetNotFoundException, ProcessorInterruptedException, MetadataException from common.lib.user_input import UserInput from common.lib.helpers import get_4cat_canvas from common.lib.dataset import DataSet @@ -68,7 +67,10 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: } def process(self): - metadata = {} + try: + metadata = self.source_dataset.read_media_metadata() + except (FileNotFoundError, MetadataException): + metadata = None base_height = self.parameters.get("height", 100) fontsize = 12 @@ -99,11 +101,9 @@ def process(self): # final iteration in which we finish things up (see below) looping = False - # there is a metadata file, always read first, which we can use + # skip the metadata file (already read via read_media_metadata) if item.file.name == ".metadata.json": - with item.file.open() as infile: - metadata = json.load(infile) - continue + continue # skip if file is a real file but not an image if looping and item.file.suffix not in (".jpeg", ".jpg", ".png", ".gif"): @@ -195,25 +195,23 @@ def get_video_labels(self, metadata): determine an appropriate label. There is a generalised heuristic and some data source-specific pathways. - :param metadata: Metadata as parsed from the 'Extract Frames' JSON + :param MediaArchiveMetadata metadata: Loaded archive metadata. :return dict: Filename -> label mapping """ + if not metadata: + return {} + mapping_dataset = {} mapping_ids = {} labels = {} + from_dataset = metadata.from_dataset - if not metadata: - return {} - - for url, data in metadata.items(): - if data.get('success'): - for filename in [f["filename"] for f in data.get("files", [])]: - filename = ".".join(filename.split(".")[:-1]) - mapping_ids[filename] = data["post_ids"] - if data.get("from_dataset", data.get("source_dataset")) not in mapping_dataset: - mapping_dataset[data.get("from_dataset", data.get("source_dataset"))] = [] - mapping_dataset[data.get("from_dataset", data.get("source_dataset"))].append(filename) - labels[filename] = filename + for filename, item in metadata.iter_entries(): + basename = filename.rsplit(".", 1)[0] + mapping_ids[basename] = item.get("post_ids", []) + if from_dataset: + mapping_dataset.setdefault(from_dataset, []).append(basename) + labels[basename] = basename for dataset, urls in mapping_dataset.items(): try: @@ -243,7 +241,7 @@ def get_video_labels(self, metadata): if source_dataset is not None: for item in source_dataset.iterate_items(self): for filename in urls: - if item["id"] in mapping_ids[filename]: + if str(item["id"]) in mapping_ids[filename]: labels[filename] = mapper(item) return labels diff --git a/tests/test_archive_metadata.py b/tests/test_archive_metadata.py new file mode 100644 index 000000000..de0f921e3 --- /dev/null +++ b/tests/test_archive_metadata.py @@ -0,0 +1,599 @@ +""" +Tests for `common.lib.archive_metadata.MediaArchiveMetadata`. + +Covers v1 round-trips, legacy format normalization (URL-keyed flat, +filename-keyed flat, video-style `files[]`, singular `post_id`), validation, +and the convenience helpers consumers rely on. +""" +import json +import zipfile + +import pytest + +from common.lib.archive_metadata import MediaArchiveMetadata +from common.lib.exceptions import MetadataException + +CURRENT_SCHEMA_VERSION = MediaArchiveMetadata.SCHEMA_VERSION + + +class FakeDataset: + """Just enough surface for MediaArchiveMetadata.read.""" + + def __init__(self, key, results_path, results_folder=None, finished=True): + self.key = key + self._results = results_path + self._results_folder = results_folder + self._finished = finished + + def is_finished(self): + return self._finished + + def get_results_path(self): + return self._results + + def get_results_folder_path(self): + return self._results_folder + + def get_own_processor(self): + return None + + +# -- v1 writer / reader round-trip -- + +def test_round_trip_minimal(tmp_path): + meta = MediaArchiveMetadata.new(processor_type="image-downloader", from_dataset="src123") + meta.add_item("a.jpg", post_ids=["p1", "p2"], url="https://example.com/a") + meta.add_item("b.jpg", post_ids=["p3"]) + meta.add_failure(post_ids=["p4"], reason="error", + reason_description="boom", url="https://example.com/c") + + target = meta.write(tmp_path) + assert target == tmp_path / ".metadata.json" + + raw = json.loads(target.read_text()) + assert raw["schema_version"] == 1 + assert raw["processor_type"] == "image-downloader" + assert raw["from_dataset"] == "src123" + assert set(raw["items"]) == {"a.jpg", "b.jpg"} + assert raw["items"]["a.jpg"] == { + "filename": "a.jpg", + "post_ids": ["p1", "p2"], + "url": "https://example.com/a", + } + assert raw["failures"] == [{ + "post_ids": ["p4"], + "reason": "error", + "url": "https://example.com/c", + "reason_description": "boom", + }] + + +def test_new_copies_software_version_from_dataset(): + """The version a processor runs with is already stored on its dataset.""" + dataset = FakeDataset("k", None) + dataset.data = {"software_version": "abc123", "software_source": "https://github.com/x/4cat"} + meta = MediaArchiveMetadata.new(dataset, processor_type="image-downloader", from_dataset="src") + assert meta.software_version == "abc123" + assert meta.software_source == "https://github.com/x/4cat" + + +def test_read_v1_from_zip(tmp_path): + meta = MediaArchiveMetadata.new(processor_type="image-downloader", from_dataset="src") + meta.add_item("a.jpg", post_ids=["1"], url="u") + meta.write(tmp_path) + + archive = tmp_path / "results.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.write(tmp_path / ".metadata.json", ".metadata.json") + + loaded = MediaArchiveMetadata.read(FakeDataset("dskey", archive)) + assert loaded.schema_version == CURRENT_SCHEMA_VERSION + assert loaded.processor_type == "image-downloader" + assert loaded.from_dataset == "src" + assert loaded.get_entry("a.jpg")["post_ids"] == ["1"] + + +def test_read_v1_from_folder(tmp_path): + meta = MediaArchiveMetadata.new(processor_type="image-downloader", from_dataset="src") + meta.add_item("a.jpg", post_ids=["1"]) + folder = tmp_path / "folder_dskey" + folder.mkdir() + meta.write(folder) + + ds = FakeDataset("dskey", tmp_path / "results.csv", results_folder=folder) + loaded = MediaArchiveMetadata.read(ds) + assert "a.jpg" in loaded + assert len(loaded) == 1 + + +# -- legacy normalization -- + +def test_legacy_url_keyed_flat(tmp_path): + """Image-downloader style: URL keys, flat filenames, success bool.""" + legacy = { + "https://example.com/a.jpg": { + "filename": "a.jpg", + "success": True, + "from_dataset": "src", + "post_ids": ["p1", "p2"], + "url": "https://example.com/a.jpg", + }, + "https://example.com/b.jpg": { + "filename": "b.jpg", + "success": False, + "from_dataset": "src", + "post_ids": ["p3"], + }, + } + path = tmp_path / ".metadata.json" + path.write_text(json.dumps(legacy)) + + # Easiest to invoke `_populate_from_raw` directly rather than building a + # fake on-disk archive. + m = MediaArchiveMetadata() + m._populate_from_raw(legacy) + + assert m.from_dataset == "src" + assert m.get_entry("a.jpg")["post_ids"] == ["p1", "p2"] + assert m.get_entry("a.jpg")["url"] == "https://example.com/a.jpg" + assert "b.jpg" not in m + assert len(m.failures) == 1 + assert m.failures[0]["post_ids"] == ["p3"] + assert m.failures[0]["url"] == "https://example.com/b.jpg" + assert m.failures[0]["reason"] == "error" + + +def test_legacy_video_files_array(tmp_path): + """Video-downloader style: URL keys, nested files[] with per-file metadata.""" + legacy = { + "https://example.com/playlist": { + "success": True, + "from_dataset": "src", + "post_ids": ["p1"], + "downloader": "yt_dlp", + "files": [ + {"filename": "vid1.mp4", "success": True, + "metadata": {"title": "One", "view_count": 100}}, + {"filename": "vid2.mp4", "success": True, + "metadata": {"title": "Two"}}, + ], + }, + "https://example.com/dead": { + "success": False, + "from_dataset": "src", + "post_ids": ["p2"], + "error": "404 Not Found", + }, + } + m = MediaArchiveMetadata() + m._populate_from_raw(legacy) + + assert set(m.items) == {"vid1.mp4", "vid2.mp4"} + assert m.get_entry("vid1.mp4")["url"] == "https://example.com/playlist" + assert m.get_entry("vid1.mp4")["post_ids"] == ["p1"] + # per-file yt-dlp metadata is flattened in, and the outer-level + # `downloader` is carried onto every file so nothing is lost + assert m.get_entry("vid1.mp4")["extra"] == { + "downloader": "yt_dlp", "title": "One", "view_count": 100, + } + assert m.get_entry("vid2.mp4")["extra"] == { + "downloader": "yt_dlp", "title": "Two", + } + assert m.get_entry("vid2.mp4")["post_ids"] == ["p1"] + + # url is shared across the playlist's outputs + assert m.get_entry("vid2.mp4")["url"] == "https://example.com/playlist" + + # failure preserved with description from `error` + assert len(m.failures) == 1 + assert m.failures[0]["url"] == "https://example.com/dead" + assert m.failures[0]["reason_description"] == "404 Not Found" + + +def test_legacy_telegram_filename_keyed(): + """Telegram style: filename keys, structured reason codes.""" + legacy = { + "chat-100.mp4": { + "filename": "chat-100.mp4", "success": True, + "from_dataset": "src", "post_ids": ["chat-100"], + "reason": "ok", "reason_description": "downloaded", + }, + "chat-101.mp4": { + "filename": "chat-101.mp4", "success": False, + "from_dataset": "src", "post_ids": ["chat-101"], + "reason": "restricted_channel", + "reason_description": "Telegram refused: channel restrictions.", + }, + } + m = MediaArchiveMetadata() + m._populate_from_raw(legacy) + + assert "chat-100.mp4" in m + assert "chat-101.mp4" not in m + assert m.failures[0]["reason"] == "restricted_channel" + assert m.failures[0]["post_ids"] == ["chat-101"] + # no URL stored for telegram entries + assert "url" not in m.failures[0] + + +def test_legacy_not_a_video_keeps_its_reason(): + """The old video downloader marked URLs that were not videos with `retry: False`.""" + legacy = { + "https://example.com/page": { + "success": False, "retry": False, "also_indirect": "none", + "post_ids": ["p1"], "error": "Not a video", + }, + } + m = MediaArchiveMetadata() + m._populate_from_raw(legacy) + assert m.failures[0]["reason"] == "not_a_video" + assert m.failures[0]["reason_description"] == "Not a video" + + +def test_legacy_singular_post_id(): + legacy = { + "https://example.com/a.jpg": { + "filename": "a.jpg", "success": True, "post_id": "p1", + }, + } + m = MediaArchiveMetadata() + m._populate_from_raw(legacy) + assert m.get_entry("a.jpg")["post_ids"] == ["p1"] + + +def test_legacy_unsupported_schema_version_raises(): + with pytest.raises(MetadataException, match="Unsupported"): + m = MediaArchiveMetadata() + m._populate_from_raw({"schema_version": 99, "items": {}}) + + +def test_legacy_single_file_preserves_top_level_extra(): + """Pre-v1 files kept producer data (e.g. the SD prompt) at the top level.""" + legacy = { + "p1": { + "filename": "p1-cat.jpeg", "success": True, "post_ids": ["p1"], + "from_dataset": "src", + "prompt": "a cat", "negative-prompt": "a dog", + }, + } + m = MediaArchiveMetadata() + m._populate_from_raw(legacy) + assert m.get_entry("p1-cat.jpeg")["extra"] == { + "prompt": "a cat", "negative-prompt": "a dog", + } + + +def test_legacy_failure_preserves_extra(): + """Producer data on a failed legacy entry survives under failure['extra'].""" + legacy = { + "https://example.com/playlist": { + "success": True, "from_dataset": "src", "post_ids": ["p1"], + "downloader": "yt_dlp", + "files": [ + {"filename": "good.mp4", "success": True, "metadata": {"title": "ok"}}, + {"filename": "", "success": False, "error": "geo-blocked", + "region": "NL"}, + ], + }, + } + m = MediaArchiveMetadata() + m._populate_from_raw(legacy) + + assert len(m.failures) == 1 + failure = m.failures[0] + assert failure["reason_description"] == "geo-blocked" + # the outer `downloader` and the per-file `region` are both kept; the + # promoted reason/error keys are not duplicated into extra + assert failure["extra"] == {"downloader": "yt_dlp", "region": "NL"} + + +# -- helper API -- + +def test_filename_to_post_ids(): + m = MediaArchiveMetadata.new(processor_type="x", from_dataset="y") + m.add_item("a.jpg", post_ids=["1", "2"]) + m.add_item("b.jpg", post_ids=["3"]) + assert m.filename_to_post_ids() == {"a.jpg": ["1", "2"], "b.jpg": ["3"]} + + +def test_post_ids_for(): + m = MediaArchiveMetadata.new(processor_type="x", from_dataset="y") + m.add_item("a.jpg", post_ids=["1"]) + assert m.post_ids_for("a.jpg") == ["1"] + assert m.post_ids_for("missing.jpg") == [] + + +def test_add_item_normalizes_post_ids(): + m = MediaArchiveMetadata.new(processor_type="x", from_dataset="y") + m.add_item("a.jpg", post_ids="p1") # bare string + m.add_item("b.jpg", post_ids=42) # bare int + m.add_item("c.jpg", post_ids=[1, 2, 3]) # ints in list + assert m.get_entry("a.jpg")["post_ids"] == ["p1"] + assert m.get_entry("b.jpg")["post_ids"] == ["42"] + assert m.get_entry("c.jpg")["post_ids"] == ["1", "2", "3"] + + +def test_add_item_drops_null_and_blank_post_ids(): + """ + Media with no traceable item (e.g. Instagram ads) records a null or blank + post id. These are dropped, not kept as the text "None" or "", so nothing + downstream tries to match on a fabricated id. + """ + m = MediaArchiveMetadata.new(processor_type="x", from_dataset="y") + m.add_item("a.jpg", post_ids=[None]) + m.add_item("b.jpg", post_ids=[None, "p1", " ", "p2"]) + m.add_item("c.jpg", post_ids=[]) + assert m.get_entry("a.jpg")["post_ids"] == [] + assert m.get_entry("b.jpg")["post_ids"] == ["p1", "p2"] + assert m.get_entry("c.jpg")["post_ids"] == [] + + +def test_legacy_null_post_ids_dropped(): + """The old image downloader wrote `post_ids: [null]` for ad media.""" + legacy = { + "https://example.com/ad.jpg": { + "filename": "ad.jpg", + "success": True, + "from_dataset": "src", + "post_ids": [None], + }, + } + m = MediaArchiveMetadata() + m._populate_from_raw(legacy) + # the file is still recorded, it just has no item to trace back to + assert m.get_entry("ad.jpg")["post_ids"] == [] + + +def test_add_item_duplicate_raises(): + m = MediaArchiveMetadata.new(processor_type="x", from_dataset="y") + m.add_item("a.jpg", post_ids=["1"]) + with pytest.raises(MetadataException, match="already present"): + m.add_item("a.jpg", post_ids=["2"]) + + +def test_add_item_replace_overwrites(): + m = MediaArchiveMetadata.new(processor_type="x", from_dataset="y") + m.add_item("a.jpg", post_ids=["1"]) + m.add_item("a.jpg", post_ids=["2"], replace=True) + assert m.post_ids_for("a.jpg") == ["2"] + + +def test_add_failure_keeps_extra(): + m = MediaArchiveMetadata.new(processor_type="x", from_dataset="y") + m.add_failure(post_ids=["1"], reason="error", url="https://cdn.example/a.jpg", + extra={"original_url": "https://t.example/a"}) + m.add_failure(post_ids=["2"], reason="error") + assert m.failures[0]["extra"] == {"original_url": "https://t.example/a"} + assert "extra" not in m.failures[1] + m.validate() + + +def test_add_item_merge_keeps_posts_of_every_url(): + """One file reached through two URLs stays connected to the posts of both.""" + m = MediaArchiveMetadata.new(processor_type="x", from_dataset="y") + m.add_item("v.mp4", post_ids=["1", "2"], url="https://a.example/v", + extra={"title": "first"}) + m.add_item("v.mp4", post_ids=["2", "3"], url="https://b.example/v", + extra={"title": "second", "downloader": "yt_dlp"}, merge=True) + + entry = m.get_entry("v.mp4") + assert entry["post_ids"] == ["1", "2", "3"] + # the first URL and the data recorded with it are kept; only new keys are + # added, and the second URL is listed as another way to reach the file + assert entry["url"] == "https://a.example/v" + assert entry["extra"] == { + "title": "first", "downloader": "yt_dlp", + "other_urls": ["https://b.example/v"], + } + + +def test_add_item_merge_lists_each_other_url_once(): + m = MediaArchiveMetadata.new(processor_type="x", from_dataset="y") + m.add_item("v.mp4", post_ids=["1"], url="https://a.example/v") + m.add_item("v.mp4", post_ids=["2"], url="https://b.example/v", merge=True) + m.add_item("v.mp4", post_ids=["3"], url="https://b.example/v", merge=True) + m.add_item("v.mp4", post_ids=["4"], url="https://c.example/v", merge=True) + # the same URL as the entry's own, or no URL at all, adds nothing + m.add_item("v.mp4", post_ids=["5"], url="https://a.example/v", merge=True) + m.add_item("v.mp4", post_ids=["6"], merge=True) + + entry = m.get_entry("v.mp4") + assert entry["post_ids"] == ["1", "2", "3", "4", "5", "6"] + assert entry["extra"]["other_urls"] == ["https://b.example/v", "https://c.example/v"] + + +def test_add_item_merge_fills_in_missing_url(): + """A file first recorded without a URL takes the next URL as its own.""" + m = MediaArchiveMetadata.new(processor_type="x", from_dataset="y") + m.add_item("v.mp4", post_ids=["1"]) + m.add_item("v.mp4", post_ids=["2"], url="https://a.example/v", merge=True) + entry = m.get_entry("v.mp4") + assert entry["url"] == "https://a.example/v" + assert "extra" not in entry + + +def test_add_item_merge_adds_new_file(): + m = MediaArchiveMetadata.new(processor_type="x", from_dataset="y") + m.add_item("v.mp4", post_ids=["1"], merge=True) + assert m.post_ids_for("v.mp4") == ["1"] + + +def test_add_item_merge_and_replace_together_raises(): + m = MediaArchiveMetadata.new(processor_type="x", from_dataset="y") + with pytest.raises(MetadataException, match="either"): + m.add_item("v.mp4", post_ids=["1"], replace=True, merge=True) + + +def test_legacy_file_shared_by_two_urls_keeps_all_post_ids(): + """ + The old video downloader could list one file under two URLs, when yt-dlp + recognised a video it had already downloaded for an earlier URL. The second + URL was then marked unsuccessful, but still listed the file. + """ + legacy = { + "https://youtube.com/watch?v=abc": { + "success": True, "post_ids": ["p1"], "downloader": "yt_dlp", + "files": [{"filename": "abc.mp4", "success": True, "metadata": {"id": "abc"}}], + }, + "https://youtu.be/abc": { + "success": False, "post_ids": ["p2"], "downloader": "yt_dlp", + "files": [{"filename": "abc.mp4", "success": True, "metadata": {"id": "abc"}}], + }, + } + m = MediaArchiveMetadata() + m._populate_from_raw(legacy) + + assert m.post_ids_for("abc.mp4") == ["p1", "p2"] + assert m.get_entry("abc.mp4")["url"] == "https://youtube.com/watch?v=abc" + assert m.get_entry("abc.mp4")["extra"]["other_urls"] == ["https://youtu.be/abc"] + # the second URL did produce a file, so it is not also a failure + assert m.failures == [] + + +def test_legacy_unsuccessful_url_without_files_is_a_failure(): + legacy = { + "https://example.com/gone": { + "success": False, "post_ids": ["p1"], "error": "404", "files": [], + }, + } + m = MediaArchiveMetadata() + m._populate_from_raw(legacy) + assert len(m) == 0 + assert m.failures[0]["post_ids"] == ["p1"] + assert m.failures[0]["reason_description"] == "404" + + +# -- validation -- + +def test_validate_rejects_filename_mismatch(tmp_path): + m = MediaArchiveMetadata.new(processor_type="x", from_dataset="y") + m.items["a.jpg"] = {"filename": "b.jpg", "post_ids": []} + with pytest.raises(MetadataException, match="must match its key"): + m.validate() + + +def test_validate_rejects_non_list_post_ids(): + m = MediaArchiveMetadata.new(processor_type="x", from_dataset="y") + m.items["a.jpg"] = {"filename": "a.jpg", "post_ids": "p1"} + with pytest.raises(MetadataException, match="post_ids must be a list"): + m.validate() + + +def test_validate_rejects_failure_without_reason(): + m = MediaArchiveMetadata.new(processor_type="x", from_dataset="y") + m.failures.append({"post_ids": [], "reason": ""}) + with pytest.raises(MetadataException, match="reason"): + m.validate() + + +def test_read_raises_when_dataset_unfinished(tmp_path): + ds = FakeDataset("k", tmp_path / "results.zip", finished=False) + with pytest.raises(MetadataException, match="not finished"): + MediaArchiveMetadata.read(ds) + + +def test_read_finished_dataset_without_items(tmp_path): + """ + An archive in which every download failed (or every file was copied from an + earlier download) finishes with no items, but its metadata is still read. + """ + meta = MediaArchiveMetadata.new(processor_type="video-downloader", from_dataset="src") + meta.add_failure(post_ids=["1"], reason="not_a_video", url="https://example.com/page") + meta.write(tmp_path) + archive = tmp_path / "results.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.write(tmp_path / ".metadata.json", ".metadata.json") + + loaded = MediaArchiveMetadata.read(FakeDataset("k", archive)) + assert loaded.failures[0]["reason"] == "not_a_video" + + +def test_read_raises_when_finished_dataset_has_no_archive(tmp_path): + ds = FakeDataset("k", tmp_path / "results.zip") + with pytest.raises(FileNotFoundError): + MediaArchiveMetadata.read(ds) + + +def test_read_raises_when_file_missing_from_zip(tmp_path): + archive = tmp_path / "empty.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("README", "no metadata here") + ds = FakeDataset("k", archive) + with pytest.raises(FileNotFoundError): + MediaArchiveMetadata.read(ds) + + +def test_read_raises_metadata_exception_on_malformed_json(tmp_path): + """A corrupt metadata file surfaces as MetadataException, not JSONDecodeError.""" + archive = tmp_path / "bad.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr(".metadata.json", "{ not valid json ]") + ds = FakeDataset("k", archive) + with pytest.raises(MetadataException, match="not valid JSON"): + MediaArchiveMetadata.read(ds) + + +def test_read_raises_metadata_exception_on_damaged_zip(tmp_path): + archive = tmp_path / "results.zip" + archive.write_bytes(b"this is not a zip file") + with pytest.raises(MetadataException, match="cannot be opened"): + MediaArchiveMetadata.read(FakeDataset("k", archive)) + + +def test_read_raises_metadata_exception_on_text_that_is_not_utf8(tmp_path): + folder = tmp_path / "folder_k" + folder.mkdir() + (folder / ".metadata.json").write_bytes(b'{"items": "\xff\xfe"}') + ds = FakeDataset("k", tmp_path / "results.csv", results_folder=folder) + with pytest.raises(MetadataException, match="not valid JSON"): + MediaArchiveMetadata.read(ds) + + +def test_v1_read_rejects_items_that_are_not_an_object(): + m = MediaArchiveMetadata() + with pytest.raises(MetadataException, match="items"): + m._populate_from_raw({"schema_version": 1, "items": ["a.jpg"]}) + + +def test_v1_read_tolerates_empty_and_malformed_entries(): + """Missing or null lists count as empty; entries that are not objects are skipped.""" + m = MediaArchiveMetadata() + m._populate_from_raw({"schema_version": 1, "items": None, "failures": None}) + assert len(m) == 0 + assert m.failures == [] + + m = MediaArchiveMetadata() + m._populate_from_raw({ + "schema_version": 1, + "items": {"a.jpg": {"filename": "a.jpg", "post_ids": ["1"]}, "b.jpg": "oops"}, + "failures": [{"post_ids": ["2"], "reason": "error"}, "oops"], + }) + assert list(m.items) == ["a.jpg"] + assert len(m.failures) == 1 + + +def test_legacy_source_dataset_key_is_read_as_from_dataset(): + """Video downloads from early 2023 recorded the source as `source_dataset`.""" + legacy = { + "https://example.com/v": { + "success": True, "source_dataset": "src", "post_ids": ["p1"], + "files": [{"filename": "v.mp4", "success": True}], + }, + } + m = MediaArchiveMetadata() + m._populate_from_raw(legacy) + assert m.from_dataset == "src" + assert "source_dataset" not in m.get_entry("v.mp4").get("extra", {}) + + +def test_v1_read_normalizes_integer_post_ids(): + """Integer post_ids in a v1 file are coerced to strings on read.""" + raw = { + "schema_version": 1, + "items": {"a.jpg": {"filename": "a.jpg", "post_ids": [1, 2]}}, + "failures": [{"post_ids": [3], "reason": "error"}], + } + m = MediaArchiveMetadata() + m._populate_from_raw(raw) + assert m.get_entry("a.jpg")["post_ids"] == ["1", "2"] + assert m.failures[0]["post_ids"] == ["3"] diff --git a/tests/test_image_downloader_metadata.py b/tests/test_image_downloader_metadata.py new file mode 100644 index 000000000..ae7c7d6fb --- /dev/null +++ b/tests/test_image_downloader_metadata.py @@ -0,0 +1,59 @@ +""" +Tests for which URLs the image downloader records in its archive metadata. + +The downloader is built with `__new__` and given only the redirects it has +seen, so none of the worker machinery has to be started up. +""" +from common.lib.archive_metadata import MediaArchiveMetadata +from common.lib.media_archive_library import MediaArchiveLibrary +from processors.visualisation.download_images import ImageDownloader + + +def make_downloader(redirects=None): + downloader = ImageDownloader.__new__(ImageDownloader) + downloader.url_redirects = redirects or {} + return downloader + + +def test_redirected_url_is_recorded_with_the_dataset_url_as_other_url(): + downloader = make_downloader({ + "http://example.com/a.jpg": "https://example.com/a.jpg", + "https://example.com/a.jpg": "https://cdn.example.com/a.jpg", + }) + + url, extra = downloader.urls_for_metadata("http://example.com/a.jpg") + + # where the file was downloaded from, after following every redirect + assert url == "https://cdn.example.com/a.jpg" + # the URL as it appeared in the dataset + assert extra == {"other_urls": ["http://example.com/a.jpg"]} + + +def test_url_without_redirect_has_no_other_urls(): + url, extra = make_downloader().urls_for_metadata("https://example.com/a.jpg") + assert url == "https://example.com/a.jpg" + assert extra == {} + + +def test_redirected_image_can_be_found_by_either_url(): + """Both downloads and failures can be looked up by the dataset's URL or the final one.""" + downloader = make_downloader({ + "http://example.com/a.jpg": "https://cdn.example.com/a.jpg", + "http://example.com/b.jpg": "https://cdn.example.com/b.jpg", + }) + metadata = MediaArchiveMetadata.new(processor_type="image-downloader", from_dataset="posts") + + url, extra = downloader.urls_for_metadata("http://example.com/a.jpg") + metadata.add_item("a.jpg", post_ids=["p1"], url=url, extra=extra, replace=True) + url, extra = downloader.urls_for_metadata("http://example.com/b.jpg") + metadata.add_failure(post_ids=["p2"], reason="error", url=url, extra=extra) + + library = MediaArchiveLibrary([metadata]) + for found_by in ("http://example.com/a.jpg", "https://cdn.example.com/a.jpg"): + hit = library.find(found_by) + assert hit.is_success + assert [filename for filename, _ in hit.entries] == ["a.jpg"] + for found_by in ("http://example.com/b.jpg", "https://cdn.example.com/b.jpg"): + hit = library.find(found_by) + assert not hit.is_success + assert hit.reasons == {"error"} diff --git a/tests/test_media_archive_library.py b/tests/test_media_archive_library.py new file mode 100644 index 000000000..0261f92d5 --- /dev/null +++ b/tests/test_media_archive_library.py @@ -0,0 +1,218 @@ +""" +Tests for `common.lib.media_archive_library.MediaArchiveLibrary`. + +Exercises `find()` resolution via the pure constructor (metadata objects +injected directly). The `collect()` path — walking parent/child datasets — +is left to integration/manual verification. +""" +from common.lib.archive_metadata import MediaArchiveMetadata +from common.lib.media_archive_library import MediaArchiveLibrary + + +def _meta(): + """Build an empty MediaArchiveMetadata for tests to populate.""" + return MediaArchiveMetadata.new(processor_type="video-downloader", from_dataset="src") + + +# -- success lookups -- + +def test_find_success_returns_entry(): + m = _meta() + m.add_item("v.mp4", post_ids=["p1"], url="https://example.com/v") + lib = MediaArchiveLibrary([m]) + + hit = lib.find("https://example.com/v") + assert hit is not None + assert hit.is_success + assert hit.metadata is m + assert hit.entries == [("v.mp4", m.get_entry("v.mp4"))] + + +def test_find_unknown_url_returns_none(): + m = _meta() + m.add_item("v.mp4", post_ids=["p1"], url="https://example.com/v") + lib = MediaArchiveLibrary([m]) + + assert lib.find("https://example.com/never-seen") is None + + +def test_playlist_one_url_many_files(): + """A single source URL that produced several files (yt-dlp playlist).""" + m = _meta() + m.add_item("v1.mp4", post_ids=["p1"], url="https://example.com/playlist") + m.add_item("v2.mp4", post_ids=["p1"], url="https://example.com/playlist") + lib = MediaArchiveLibrary([m]) + + hit = lib.find("https://example.com/playlist") + assert hit.is_success + assert {fn for fn, _ in hit.entries} == {"v1.mp4", "v2.mp4"} + + +def test_success_groups_by_single_archive(): + """URL downloaded by two archives: hit resolves to one archive's files.""" + m1 = _meta() + m1.add_item("a.mp4", post_ids=["p1"], url="https://example.com/v") + m2 = _meta() + m2.add_item("b.mp4", post_ids=["p1"], url="https://example.com/v") + lib = MediaArchiveLibrary([m1, m2]) + + hit = lib.find("https://example.com/v") + assert hit.is_success + # all returned entries belong to the same (first) archive + assert hit.metadata is m1 + assert {fn for fn, _ in hit.entries} == {"a.mp4"} + + +# -- failure lookups -- + +def test_find_failure_surfaces_reason(): + m = _meta() + m.add_failure(post_ids=["p1"], reason="not_a_video", url="https://example.com/x") + lib = MediaArchiveLibrary([m]) + + hit = lib.find("https://example.com/x") + assert hit is not None + assert not hit.is_success + assert hit.reasons == {"not_a_video"} + + +def test_failure_collects_all_reasons_across_archives(): + m1 = _meta() + m1.add_failure(post_ids=["p1"], reason="error", url="https://example.com/x") + m2 = _meta() + m2.add_failure(post_ids=["p2"], reason="not_a_video", url="https://example.com/x") + lib = MediaArchiveLibrary([m1, m2]) + + hit = lib.find("https://example.com/x") + assert not hit.is_success + assert hit.reasons == {"error", "not_a_video"} + + +def test_success_beats_failure(): + """URL downloaded in one archive, failed in another — success wins.""" + failed = _meta() + failed.add_failure(post_ids=["p1"], reason="error", url="https://example.com/v") + ok = _meta() + ok.add_item("v.mp4", post_ids=["p1"], url="https://example.com/v") + lib = MediaArchiveLibrary([failed, ok]) + + hit = lib.find("https://example.com/v") + assert hit.is_success + assert hit.metadata is ok + + +# -- tracing a failure back to the run that recorded it -- + +class FakeDataset: + """A dataset stores the parameters it was created with; that is all that is needed here.""" + + def __init__(self, key, parameters=None): + self.key = key + self.parameters = parameters or {} + + +def _meta_from(dataset): + """ + Metadata as read from a dataset. + + Uses the plain constructor rather than `new()`, which would try to resolve + the 4CAT version from the dataset's processor. + """ + return MediaArchiveMetadata(dataset, processor_type="video-downloader", from_dataset="src") + + +def test_datasets_for_reason_returns_the_recording_dataset(): + dataset = FakeDataset("previous", {"also_indirect": "none"}) + m = _meta_from(dataset) + m.add_failure(post_ids=["p1"], reason="not_a_video", url="https://example.com/x") + lib = MediaArchiveLibrary([m]) + + hit = lib.find("https://example.com/x") + assert hit.datasets_for_reason("not_a_video") == [dataset] + + +def test_datasets_for_reason_ignores_other_reasons(): + """ + Each reason has to be traced separately: a URL can fail one way in one run + and another way in another, and the settings that produced each differ. + """ + errored = FakeDataset("errored", {"also_indirect": "all"}) + not_video = FakeDataset("not-video", {"also_indirect": "none"}) + m1 = _meta_from(errored) + m1.add_failure(post_ids=["p1"], reason="error", url="https://example.com/x") + m2 = _meta_from(not_video) + m2.add_failure(post_ids=["p1"], reason="not_a_video", url="https://example.com/x") + lib = MediaArchiveLibrary([m1, m2]) + + hit = lib.find("https://example.com/x") + assert hit.reasons == {"error", "not_a_video"} + assert hit.datasets_for_reason("not_a_video") == [not_video] + assert hit.datasets_for_reason("error") == [errored] + + +def test_datasets_for_reason_skips_metadata_without_a_dataset(): + """Metadata built outside a dataset (e.g. in a test) has nothing to trace to.""" + m = _meta() + m.add_failure(post_ids=["p1"], reason="not_a_video", url="https://example.com/x") + lib = MediaArchiveLibrary([m]) + + hit = lib.find("https://example.com/x") + assert hit.reasons == {"not_a_video"} + assert hit.datasets_for_reason("not_a_video") == [] + + +# -- misc -- + +def test_items_without_url_are_not_indexed(): + """Telegram-style entries have no URL; they must not crash indexing.""" + m = _meta() + m.add_item("chat-1.mp4", post_ids=["chat-1"]) # no url + lib = MediaArchiveLibrary([m]) + + assert lib.find(None) is None + assert len(lib) == 1 + + +def test_find_through_other_url(): + """A file reached through a second URL can be found through that URL too.""" + m = _meta() + m.add_item("v.mp4", post_ids=["p1"], url="https://youtube.com/watch?v=abc") + m.add_item("v.mp4", post_ids=["p2"], url="https://youtu.be/abc", merge=True) + lib = MediaArchiveLibrary([m]) + + hit = lib.find("https://youtu.be/abc") + assert hit.is_success + assert hit.metadata is m + assert [fn for fn, _ in hit.entries] == ["v.mp4"] + # its own URL still finds it, listed once + assert [fn for fn, _ in lib.find("https://youtube.com/watch?v=abc").entries] == ["v.mp4"] + + +def test_find_failure_through_other_url(): + """A failed download can also be found through each of its URLs.""" + m = _meta() + m.add_failure(post_ids=["p1"], reason="not_a_video", url="https://example.com/page", + extra={"other_urls": ["http://example.com/page"]}) + lib = MediaArchiveLibrary([m]) + + for url in ("https://example.com/page", "http://example.com/page"): + hit = lib.find(url) + assert not hit.is_success + assert hit.reasons == {"not_a_video"} + assert len(hit.failures) == 1 + + +def test_other_urls_that_are_not_a_list_are_ignored(): + """A hand-edited file may hold something else under `other_urls`.""" + m = _meta() + m.add_item("v.mp4", post_ids=["p1"], url="https://example.com/v", + extra={"other_urls": "https://example.com/w"}) + lib = MediaArchiveLibrary([m]) + + assert lib.find("https://example.com/v").is_success + assert lib.find("https://example.com/w") is None + + +def test_len_reports_archive_count(): + lib = MediaArchiveLibrary([_meta(), _meta(), _meta()]) + assert len(lib) == 3 diff --git a/tests/test_video_downloader_metadata.py b/tests/test_video_downloader_metadata.py new file mode 100644 index 000000000..62a04c132 --- /dev/null +++ b/tests/test_video_downloader_metadata.py @@ -0,0 +1,198 @@ +""" +Tests for how the video downloader writes its archive metadata and reuses the +metadata of earlier downloads. + +The downloader is built with `__new__` and given stand-ins for the few +attributes it uses, so none of the worker machinery (database, job queue, +config) has to be started up. +""" +import zipfile +from types import SimpleNamespace + +from common.lib.archive_metadata import MediaArchiveMetadata +from common.lib.media_archive_library import MediaArchiveLibrary +from processors.visualisation.download_videos import VideoDownloaderPlus + + +class DatasetStub: + """Stands in for the dataset the downloader is producing.""" + + def __init__(self): + self.metadata = None + self.logs = [] + + def update_status(self, *args, **kwargs): + pass + + def log(self, message): + self.logs.append(message) + + def new_media_metadata(self, processor_type=None, from_dataset=None): + self.metadata = MediaArchiveMetadata(from_dataset=from_dataset) + return self.metadata + + +def make_downloader(parameters=None): + downloader = VideoDownloaderPlus.__new__(VideoDownloaderPlus) + downloader.dataset = DatasetStub() + downloader.source_dataset = SimpleNamespace(key="posts") + downloader.parameters = parameters or {} + downloader.files_copied_from_library = {} + return downloader + + +# -- writing metadata -- + +def test_file_reached_through_two_urls_keeps_posts_of_both(tmp_path): + """ + When yt-dlp recognises a video it already downloaded for an earlier URL, the + later URL lists the same file. Both URLs' posts must stay connected to it. + """ + video = {"filename": "abc.mp4", "success": True, "metadata": {"id": "abc"}} + urls = { + "https://youtube.com/watch?v=abc": { + "post_ids": {"p1"}, "success": True, "downloader": "yt_dlp", "files": [video], + }, + # not marked successful, because nothing new was downloaded for it + "https://youtu.be/abc": { + "post_ids": {"p2"}, "success": False, "downloader": "yt_dlp", "files": [video], + }, + } + downloader = make_downloader() + downloader._save_metadata(urls, tmp_path) + + metadata = downloader.dataset.metadata + assert metadata.post_ids_for("abc.mp4") == ["p1", "p2"] + assert metadata.get_entry("abc.mp4")["url"] == "https://youtube.com/watch?v=abc" + assert metadata.get_entry("abc.mp4")["extra"]["other_urls"] == ["https://youtu.be/abc"] + assert metadata.failures == [] + + +def test_copied_video_does_not_carry_the_earlier_archives_other_urls(): + """ + Other URLs recorded by an earlier download came from that dataset's posts, + so they are not copied into this archive's metadata along with the file. + """ + earlier = MediaArchiveMetadata(SimpleNamespace(parameters={})) + earlier.add_item("abc.mp4", post_ids=["p1"], url="https://youtube.com/watch?v=abc", + extra={"title": "a video", "downloader": "yt_dlp"}) + earlier.add_item("abc.mp4", post_ids=["p2"], url="https://youtu.be/abc", merge=True) + hit = MediaArchiveLibrary([earlier]).find("https://youtube.com/watch?v=abc") + + fields = VideoDownloaderPlus._copied_url_fields(hit) + + assert fields["files"][0]["metadata"] == {"title": "a video"} + assert fields["downloader"] == "yt_dlp" + # the earlier archive itself is left as it was + assert earlier.get_entry("abc.mp4")["extra"]["other_urls"] == ["https://youtu.be/abc"] + + +# -- reusing earlier downloads -- + +def old_not_a_video_archive(also_indirect): + """ + Metadata of a video download made before archive metadata had a schema + version, in which a URL was found not to be a video. + """ + previous_run = SimpleNamespace(parameters={"also_indirect": also_indirect}) + metadata = MediaArchiveMetadata(previous_run) + metadata._populate_from_raw({ + "https://example.com/page": { + "success": False, "retry": False, "also_indirect": also_indirect, + "post_ids": ["p1"], "error": "Not a video", + }, + }) + return metadata + + +def test_url_an_old_download_found_not_to_be_a_video_is_skipped(tmp_path): + library = MediaArchiveLibrary([old_not_a_video_archive("none")]) + urls = {"https://example.com/page": {"post_ids": {"p9"}}} + downloader = make_downloader({"also_indirect": "none"}) + + result = downloader._try_copy_from_library("https://example.com/page", urls, library, tmp_path) + + assert result["skip"] is True + assert urls["https://example.com/page"]["reason"] == "not_a_video" + + +def test_url_is_tried_again_when_this_run_allows_yt_dlp_more_urls(tmp_path): + library = MediaArchiveLibrary([old_not_a_video_archive("none")]) + urls = {"https://example.com/page": {"post_ids": {"p9"}}} + downloader = make_downloader({"also_indirect": "all"}) + + result = downloader._try_copy_from_library("https://example.com/page", urls, library, tmp_path) + + assert result["skip"] is False + + +def earlier_video_archive(tmp_path, key, urls): + """ + An earlier video download whose zip holds one video, `abc.mp4`, that the + given URLs all led to. + """ + folder = tmp_path / key + folder.mkdir() + zip_path = folder / "results.zip" + with zipfile.ZipFile(zip_path, "w") as archive: + archive.writestr("abc.mp4", b"video bytes") + + dataset = SimpleNamespace(key=key, parameters={}, get_results_path=lambda: zip_path) + metadata = MediaArchiveMetadata(dataset) + for i, url in enumerate(urls): + metadata.add_item("abc.mp4", post_ids=[f"{key}-{i}"], url=url, + extra={"downloader": "yt_dlp"}, merge=True) + return metadata + + +def test_video_reached_through_two_urls_is_copied_once(tmp_path): + """ + An earlier download reached one video through two URLs. When this dataset + links to both, the video is found through either URL but copied once. + """ + library = MediaArchiveLibrary([earlier_video_archive( + tmp_path, "earlier", ["https://youtube.com/watch?v=abc", "https://youtu.be/abc"])]) + staging = tmp_path / "staging" + staging.mkdir() + urls = { + "https://youtube.com/watch?v=abc": {"post_ids": {"p1"}}, + "https://youtu.be/abc": {"post_ids": {"p2"}}, + } + downloader = make_downloader() + + first = downloader._try_copy_from_library("https://youtube.com/watch?v=abc", urls, library, staging) + second = downloader._try_copy_from_library("https://youtu.be/abc", urls, library, staging) + + assert (first["copied"], first["count"]) == (True, 1) + # found through the file's other URLs, but the video is already there + assert (second["copied"], second["count"]) == (True, 0) + assert [file.name for file in staging.iterdir()] == ["abc.mp4"] + + # both URLs' posts end up connected to the one file + downloader._save_metadata(urls, staging) + metadata = downloader.dataset.metadata + assert metadata.post_ids_for("abc.mp4") == ["p1", "p2"] + assert metadata.get_entry("abc.mp4")["extra"]["other_urls"] == ["https://youtu.be/abc"] + + +def test_same_filename_from_a_different_earlier_download_is_not_copied_over(tmp_path): + """ + Two earlier downloads can each hold a file of the same name, which may be + different videos. Copying the second would overwrite the first, so that URL + is downloaded again instead. + """ + library = MediaArchiveLibrary([ + earlier_video_archive(tmp_path, "earlier1", ["https://example.com/one"]), + earlier_video_archive(tmp_path, "earlier2", ["https://example.com/two"]), + ]) + staging = tmp_path / "staging" + staging.mkdir() + urls = {"https://example.com/one": {"post_ids": {"p1"}}, "https://example.com/two": {"post_ids": {"p2"}}} + downloader = make_downloader() + + first = downloader._try_copy_from_library("https://example.com/one", urls, library, staging) + second = downloader._try_copy_from_library("https://example.com/two", urls, library, staging) + + assert first["copied"] is True + assert second["copied"] is False + assert "files" not in urls["https://example.com/two"] diff --git a/tests/test_view_metadata.py b/tests/test_view_metadata.py new file mode 100644 index 000000000..e1c259f1c --- /dev/null +++ b/tests/test_view_metadata.py @@ -0,0 +1,611 @@ +""" +Tests for the metadata viewer processor. + +The processor reads a media archive's metadata and, where it can find the dataset +the media was downloaded from, adds that dataset's columns to each row. These +tests cover that join: which dataset is picked, how a media file used by several +items is written out, and what happens to media that cannot be traced back. + +Also covers the `_from_dataset` override that lets a preset tell the video +downloader which dataset to credit, instead of the helper dataset it built. + +The processors are built with `__new__` and given stand-ins for the few +attributes they use, so none of the worker machinery (database, job queue, +config) has to be started up. +""" +import csv +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from common.lib.archive_metadata import MediaArchiveMetadata +# imported as a module as well so that `DataSet` and `DataSetException` can be +# reached exactly as the processor sees them; another test module replaces some +# `common.lib` entries in sys.modules, so importing them again here is not safe +from processors.conversion import view_metadata +from processors.conversion.view_metadata import ViewMetadata + + +class FakeProducer: + """Stands in for the downloader that made the archive.""" + type = "image-downloader" + + @classmethod + def map_metadata(cls, filename, item): + yield { + "url": item.get("url", ""), + "filename": filename, + "post_ids": ", ".join(item.get("post_ids", [])), + "download_successful": True, + } + + @classmethod + def map_failure_metadata(cls, failure): + yield { + "url": failure.get("url", ""), + "filename": "", + "post_ids": ", ".join(failure.get("post_ids", [])), + "download_successful": False, + } + + +class FakeDataset: + """Records what the processor reports, so tests can assert on it.""" + + def __init__(self, results_path=None, key="fake", num_rows=0, + dataset_type="image-downloader", columns=None, items=None, + metadata=None): + self.key = key + self.type = dataset_type + self.num_rows = num_rows + self._results_path = results_path + self._columns = columns or [] + self._items = items or [] + self._metadata = metadata + self.logs = [] + self.statuses = [] + self.finished = None + self.warning = None + self.error = None + self.parent = None + + # -- used when this dataset is the processor's own output -- + + def get_results_path(self): + return self._results_path + + def log(self, message): + self.logs.append(message) + + def update_status(self, message, is_final=False): + self.statuses.append(message) + + def finish(self, num_rows=0): + self.finished = num_rows + + def finish_with_warning(self, num_rows, warning): + self.finished = num_rows + self.warning = warning + + def finish_with_error(self, error): + self.error = error + + # -- used when this dataset is the archive being read -- + + def read_media_metadata(self, filename=".metadata.json"): + if self._metadata is None: + raise FileNotFoundError("no metadata") + return self._metadata + + def get_own_processor(self): + return FakeProducer + + # -- used when this dataset is a source of items -- + + def get_columns(self): + return list(self._columns) + + def get_label(self): + return f"dataset {self.key}" + + def get_parent(self): + return self.parent + + def is_finished(self): + return True + + def iterate_items(self, processor=None, **kwargs): + for item in self._items: + yield dict(item) + + +def make_metadata(from_dataset="posts", items=(), failures=()): + """ + Build real archive metadata, so the tests go through the same normalising + the processor relies on. + + :param str from_dataset: key the archive records as its source + :param items: (filename, post_ids, url) tuples + :param failures: (post_ids, reason) tuples + """ + metadata = MediaArchiveMetadata(from_dataset=from_dataset) + for filename, post_ids, url in items: + metadata.add_item(filename, post_ids=post_ids, url=url) + for post_ids, reason in failures: + metadata.add_failure(post_ids=post_ids, reason=reason) + return metadata + + +def make_processor(tmp_path, metadata=None, parameters=None, source_dataset=None): + """ + Build a ViewMetadata with stand-ins for everything it reads. + + :param tmp_path: pytest temporary directory + :param metadata: archive metadata, as built by `make_metadata` + :param dict parameters: processor parameters + :param source_dataset: dataset the media came from; when given, the search + for it is replaced by this, since that search is covered on its own + :return tuple: the processor and its output dataset + """ + output = FakeDataset(results_path=tmp_path.joinpath("result.csv")) + + processor = ViewMetadata.__new__(ViewMetadata) + processor.dataset = output + processor.source_dataset = FakeDataset(dataset_type="image-downloader", key="archive", + metadata=metadata) + processor.parameters = parameters if parameters is not None else {} + processor.interrupted = False + processor.log = MagicMock() + if source_dataset is not None: + processor.find_origin_datasets = lambda _: [(source_dataset, "test")] + + return processor, output + + +def read_result(output): + with output.get_results_path().open(encoding="utf-8", newline="") as infile: + reader = csv.DictReader(infile) + return reader.fieldnames, list(reader) + + +# -- item IDs -- + +@pytest.mark.parametrize("item,expected", [ + ({"id": "p1"}, ["p1"]), + ({"id": 12}, ["12"]), # some datasources number their items + ({"id": " p1 "}, ["p1"]), + ({}, [""]), + ({"ids": "1,2,3"}, ["1", "2", "3"]), # rows that combine several items + ({"ids": "1, 2", "id": "ignored"}, ["1", "2"]), +]) +def test_item_ids(item, expected): + assert ViewMetadata.item_ids(item) == expected + + +# -- recognising media archives -- + +@pytest.mark.parametrize("dataset_type,expected", [ + ("image-downloader", True), + ("image-downloader-unique", True), # a filtered image archive + ("video-downloader-telegram", True), + ("video-downloader-tiktok", True), # the preset, which carries its own type + ("tiktok-search", False), + ("metadata-viewer", False), +]) +def test_is_media_archive(dataset_type, expected): + module = SimpleNamespace(type=dataset_type) + assert ViewMetadata.is_media_archive(module) is expected + assert ViewMetadata.is_compatible_with(module) is expected + + +# -- reading the metadata -- + +def test_collect_media_rows_prefixes_and_dedupes(tmp_path): + metadata = make_metadata(items=[ + ("1.jpg", ["p1", "p2"], "http://example.com/1.jpg"), + ("2.jpg", [3, 3, " 4 "], "http://example.com/2.jpg"), + ]) + processor, _ = make_processor(tmp_path, metadata) + + rows, wanted_ids = processor.collect_media_rows(metadata, FakeProducer) + + assert len(rows) == 2 + # every column read from the metadata is prefixed, so it cannot collide with + # a column of the dataset the media came from + assert set(rows[0][0]) == {"media_url", "media_filename", "media_post_ids", + "media_download_successful"} + assert rows[0][1] == ["p1", "p2"] + # numbers become text, whitespace is trimmed and duplicates are dropped + assert rows[1][1] == ["3", "4"] + assert wanted_ids == {"p1", "p2", "3", "4"} + + +def test_collect_media_rows_skips_failures_unless_asked(tmp_path): + metadata = make_metadata(items=[("1.jpg", ["p1"], None)], + failures=[(["p2"], "error")]) + processor, _ = make_processor(tmp_path, metadata) + + rows, wanted_ids = processor.collect_media_rows(metadata, FakeProducer) + assert [row["media_filename"] for row, _ in rows] == ["1.jpg"] + assert wanted_ids == {"p1"} + + processor.parameters = {"include_failed": True} + rows, wanted_ids = processor.collect_media_rows(metadata, FakeProducer) + assert [row["media_filename"] for row, _ in rows] == ["1.jpg", ""] + # a download that produced no file still records what it was for, so the + # failure can be traced back to an item too + assert rows[1][1] == ["p2"] + assert wanted_ids == {"p1", "p2"} + + +# -- working out the columns -- + +def test_build_fieldnames_unions_rows_and_appends_source_columns(tmp_path): + processor, _ = make_processor(tmp_path) + media_rows = [ + ({"media_url": "a", "media_filename": "1.jpg"}, []), + ({"media_url": "b", "media_error": "timed out"}, []), # a column only some rows have + ] + + fieldnames, source_map = processor.build_fieldnames(media_rows, ["id", "body"]) + + assert fieldnames == ["media_url", "media_filename", "media_error", + "media_source_matched", "id", "body"] + assert source_map == {"id": "id", "body": "body"} + + +def test_build_fieldnames_without_source_dataset(tmp_path): + processor, _ = make_processor(tmp_path) + fieldnames, source_map = processor.build_fieldnames([({"media_url": "a"}, [])], []) + + # nothing to combine with, so no column saying whether a row was traced back + assert fieldnames == ["media_url"] + assert source_map == {} + + +def test_build_fieldnames_renames_clashing_source_column(tmp_path): + processor, _ = make_processor(tmp_path) + media_rows = [({"media_url": "a", "media_filename": "1.jpg"}, [])] + + fieldnames, source_map = processor.build_fieldnames(media_rows, ["id", "media_filename"]) + + assert fieldnames == ["media_url", "media_filename", "media_source_matched", "id", + "source_media_filename"] + # the renamed column must still be filled from the original column name + assert source_map == {"id": "id", "media_filename": "source_media_filename"} + + +# -- finding the dataset the media came from -- + +def test_find_origin_datasets_orders_recorded_first(tmp_path, monkeypatch): + metadata = make_metadata(from_dataset="recorded") + processor, _ = make_processor(tmp_path, metadata) + processor.db = MagicMock() + processor.modules = MagicMock() + + recorded = FakeDataset(key="recorded", num_rows=5, dataset_type="tiktok-search") + posts = FakeDataset(key="posts", num_rows=10, dataset_type="tiktok-search") + processor.source_dataset.parent = posts + + monkeypatch.setattr(view_metadata, "DataSet", lambda **kwargs: recorded) + candidates = processor.find_origin_datasets(metadata) + + assert [(dataset.key, how) for dataset, how in candidates] == [ + ("recorded", "recorded in the metadata file"), + ("posts", "found by walking up the chain of parent datasets"), + ] + + +def test_find_origin_datasets_walks_past_media_archives(tmp_path, monkeypatch): + """A deleted source dataset leaves only the chain of parent datasets.""" + metadata = make_metadata(from_dataset="deleted") + processor, _ = make_processor(tmp_path, metadata) + processor.db = MagicMock() + processor.modules = MagicMock() + + posts = FakeDataset(key="posts", num_rows=10, dataset_type="tiktok-search") + # the archive was made from another archive, e.g. by filtering duplicates + intermediate = FakeDataset(key="unique", num_rows=4, dataset_type="image-downloader-unique") + intermediate.parent = posts + processor.source_dataset.parent = intermediate + + def raise_missing(**kwargs): + raise view_metadata.DataSetException("gone") + + monkeypatch.setattr(view_metadata, "DataSet", raise_missing) + candidates = processor.find_origin_datasets(metadata) + + assert [dataset.key for dataset, _ in candidates] == ["posts"] + + +def test_find_origin_datasets_does_not_repeat_one_dataset(tmp_path, monkeypatch): + metadata = make_metadata(from_dataset="posts") + processor, _ = make_processor(tmp_path, metadata) + processor.db = MagicMock() + processor.modules = MagicMock() + + posts = FakeDataset(key="posts", num_rows=10, dataset_type="tiktok-search") + processor.source_dataset.parent = posts + + monkeypatch.setattr(view_metadata, "DataSet", lambda **kwargs: posts) + assert len(processor.find_origin_datasets(metadata)) == 1 + + +def test_collect_source_data_falls_back_when_recorded_dataset_has_no_items(tmp_path): + """ + The recorded dataset can be a helper a preset built, whose items are not the + ones the media refers to. When it accounts for nothing, the next candidate is + tried rather than reporting an empty join. + """ + processor, output = make_processor(tmp_path) + helper = FakeDataset(key="helper", num_rows=1, columns=["id", "video_url"], + items=[{"id": "unrelated", "video_url": "http://a"}]) + posts = FakeDataset(key="posts", num_rows=1, columns=["id", "body"], + items=[{"id": "p1", "body": "first"}]) + processor.find_origin_datasets = lambda _: [(helper, "recorded in the metadata file"), + (posts, "found by walking up")] + + columns, items = processor.collect_source_data(None, {"p1"}) + + assert columns == ["id", "body"] + assert items == {"p1": {"id": "p1", "body": "first"}} + assert any("has none of the items" in message for message in output.logs) + + +def test_collect_source_data_keeps_columns_when_nothing_matches(tmp_path): + """ + If no candidate accounts for any media, the best guess is still used so the + result carries its columns and the mismatch gets reported. + """ + processor, _ = make_processor(tmp_path) + helper = FakeDataset(key="helper", num_rows=1, columns=["id", "video_url"], + items=[{"id": "unrelated", "video_url": "http://a"}]) + processor.find_origin_datasets = lambda _: [(helper, "recorded in the metadata file")] + + columns, items = processor.collect_source_data(None, {"p1"}) + + assert columns == ["id", "video_url"] + assert items == {} + + +def test_collect_source_data_skips_unreadable_dataset(tmp_path): + processor, output = make_processor(tmp_path) + archive = FakeDataset(key="zip", num_rows=3, columns=[]) # no readable columns + processor.find_origin_datasets = lambda _: [(archive, "recorded in the metadata file")] + + assert processor.collect_source_data(None, {"p1"}) == ([], {}) + assert any("no readable columns" in message for message in output.logs) + + +def test_collect_source_items_reports_unmappable_source_items(tmp_path): + """ + A dataset can have more rows than it yields when some items cannot be mapped + (e.g. Instagram ads). If that leaves media untraced, the gap is reported so + a partial join is not mistaken for missing metadata. + """ + processor, output = make_processor(tmp_path) + # 5 rows, but only 3 items come back from iteration: 2 were unmappable + dataset = FakeDataset(key="posts", num_rows=5, columns=["id", "body"], + items=[{"id": "p1", "body": "a"}, {"id": "p2", "body": "b"}, + {"id": "p3", "body": "c"}]) + + found = processor.collect_source_items(dataset, ["id", "body"], {"p1", "p2", "gone"}) + + assert set(found) == {"p1", "p2"} + assert any("could not be read" in message and "2 of 5" in message for message in output.logs) + + +def test_collect_source_items_silent_when_all_media_traced(tmp_path): + """No report when every item the media refers to was found, even if the + dataset also holds unrelated rows that were never reached.""" + processor, output = make_processor(tmp_path) + dataset = FakeDataset(key="posts", num_rows=9, columns=["id", "body"], + items=[{"id": "p1", "body": "a"}, {"id": "p2", "body": "b"}]) + + found = processor.collect_source_items(dataset, ["id", "body"], {"p1", "p2"}) + + assert set(found) == {"p1", "p2"} + assert not any("could not be read" in message for message in output.logs) + + +# -- the whole thing -- + +def test_process_joins_source_dataset(tmp_path): + """ + A media file used by two items becomes two rows; one that cannot be traced + back to an item keeps its row with the source columns empty. + """ + metadata = make_metadata(items=[ + ("1.jpg", ["p1", "p2"], "http://example.com/1.jpg"), + ("2.jpg", [3], "http://example.com/2.jpg"), + ("4.jpg", ["gone"], "http://example.com/4.jpg"), + ]) + source = FakeDataset(key="posts", num_rows=3, dataset_type="tiktok-search", + columns=["id", "body"], + items=[{"id": "p1", "body": "first"}, + {"id": "p2", "body": "second"}, + {"id": 3, "body": "third"}]) # a numeric item ID + processor, output = make_processor(tmp_path, metadata, {"join_source": True}, source) + + processor.process() + + fieldnames, rows = read_result(output) + assert fieldnames == ["media_url", "media_filename", "media_post_ids", + "media_download_successful", "media_source_matched", "id", "body"] + + # 1.jpg twice (once per item), 2.jpg once, 4.jpg once + assert len(rows) == 4 + assert output.finished == 4 + # 4.jpg did not trace back to an item, so a partial-match warning is raised + assert output.warning is not None + assert "1 of 3 media file(s) could not be traced" in output.warning + + by_file = {} + for row in rows: + by_file.setdefault(row["media_filename"], []).append(row) + + assert sorted(row["body"] for row in by_file["1.jpg"]) == ["first", "second"] + assert all(row["media_source_matched"] == "True" for row in by_file["1.jpg"]) + # the metadata column keeps listing every item, prefixed and untouched + assert by_file["1.jpg"][0]["media_post_ids"] == "p1, p2" + + # a numeric item ID in the dataset still matches the text one in the metadata + assert by_file["2.jpg"][0]["body"] == "third" + + # nothing to trace 4.jpg back to, but it is not dropped + assert by_file["4.jpg"][0]["media_source_matched"] == "False" + assert by_file["4.jpg"][0]["body"] == "" + + +def test_process_media_without_post_id_renders_empty_not_none(tmp_path): + """ + Media with no traceable item (e.g. an Instagram ad) has a null post id in + the archive. It must show as an empty post-id cell and an unmatched row, not + the literal text "None". + """ + metadata = make_metadata(items=[ + ("ad.jpg", [None], "http://example.com/ad.jpg"), + ("ok.jpg", ["p1"], "http://example.com/ok.jpg"), + ]) + source = FakeDataset(key="posts", num_rows=1, columns=["id", "body"], + items=[{"id": "p1", "body": "first"}]) + processor, output = make_processor(tmp_path, metadata, {"join_source": True}, source) + + processor.process() + + _, rows = read_result(output) + by_file = {r["media_filename"]: r for r in rows} + assert by_file["ad.jpg"]["media_post_ids"] == "" + assert by_file["ad.jpg"]["media_source_matched"] == "False" + assert by_file["ad.jpg"]["body"] == "" + assert by_file["ok.jpg"]["media_source_matched"] == "True" + + +def test_process_no_warning_when_all_media_traced(tmp_path): + """When every media file traces back to an item, the dataset finishes cleanly.""" + metadata = make_metadata(items=[ + ("1.jpg", ["p1"], "http://example.com/1.jpg"), + ("2.jpg", ["p2"], "http://example.com/2.jpg"), + ]) + source = FakeDataset(key="posts", num_rows=2, columns=["id", "body"], + items=[{"id": "p1", "body": "first"}, {"id": "p2", "body": "second"}]) + processor, output = make_processor(tmp_path, metadata, {"join_source": True}, source) + + processor.process() + + assert output.warning is None + assert output.finished == 2 + + +def test_process_without_join(tmp_path): + """Turning the join off gives the metadata on its own.""" + metadata = make_metadata(items=[("1.jpg", ["p1"], "http://example.com/1.jpg")]) + source = FakeDataset(key="posts", num_rows=1, columns=["id", "body"], + items=[{"id": "p1", "body": "first"}]) + processor, output = make_processor(tmp_path, metadata, {"join_source": False}, source) + + processor.process() + + fieldnames, rows = read_result(output) + assert fieldnames == ["media_url", "media_filename", "media_post_ids", + "media_download_successful"] + assert len(rows) == 1 + assert output.finished == 1 + + +def test_process_warns_when_nothing_matches(tmp_path): + metadata = make_metadata(items=[("1.jpg", ["p1"], "http://example.com/1.jpg")]) + source = FakeDataset(key="posts", num_rows=1, columns=["id", "body"], + items=[{"id": "somethingelse", "body": "first"}]) + processor, output = make_processor(tmp_path, metadata, {"join_source": True}, source) + + processor.process() + + _, rows = read_result(output) + assert len(rows) == 1 + assert rows[0]["media_source_matched"] == "False" + assert output.warning is not None + assert "None of the 1 media file(s) could be traced back" in output.warning + + +def test_process_errors_without_metadata_file(tmp_path): + processor, output = make_processor(tmp_path, metadata=None) + + processor.process() + + assert output.error == "Unable to identify metadata file" + assert not output.get_results_path().exists() + + +# -- crediting the right dataset when a preset builds a helper -- + +def test_video_downloader_honours_from_dataset_override(tmp_path): + """ + The downloader normally credits the dataset it ran on, but a preset can name + a different one, because the dataset it ran on may be a helper. + """ + from processors.visualisation.download_videos import VideoDownloaderPlus + + credited = [] + + class DatasetStub: + def update_status(self, *args, **kwargs): + pass + + def new_media_metadata(self, processor_type=None, from_dataset=None): + credited.append(from_dataset) + return MediaArchiveMetadata(from_dataset=from_dataset) + + downloader = VideoDownloaderPlus.__new__(VideoDownloaderPlus) + downloader.dataset = DatasetStub() + downloader.source_dataset = SimpleNamespace(key="helper") + + downloader.parameters = {"_from_dataset": "original"} + downloader._save_metadata({}, tmp_path) + + downloader.parameters = {} + downloader._save_metadata({}, tmp_path) + + assert credited == ["original", "helper"] + + +@pytest.mark.parametrize("dataset_type,parameters,expected", [ + # the id column is the items' own `id`, so the original can be credited + ("tiktok-search", {}, "original"), + ("tiktok-urls-search", {}, "original"), + ("upload-search", {"column": "id"}, "original"), + # a different id column: the recorded post IDs would not match that + # dataset's items, so the helper stays the right answer + ("upload-search", {"column": "tiktok_post_id"}, None), +]) +def test_tiktok_preset_credits_original_only_when_ids_line_up(dataset_type, parameters, expected): + from processors.visualisation.download_tiktok_video import TikTokVideoDownloader + + preset = TikTokVideoDownloader.__new__(TikTokVideoDownloader) + preset.source_dataset = SimpleNamespace(type=dataset_type, key="original") + preset.parameters = {"amount": 10, **parameters} + + stages = preset.get_processor_pipeline() + video_stage = next(stage for stage in stages if stage["type"] == "video-downloader") + + assert video_stage["parameters"].get("_from_dataset") == expected + + +def test_tiktok_preset_reads_video_downloader_metadata(): + """ + The preset has the video downloader write its archive, so it reads that + archive the same way the video downloader would. + """ + from processors.visualisation.download_tiktok_video import TikTokVideoDownloader + from processors.visualisation.download_videos import VideoDownloaderPlus + + item = {"filename": "7241234567890.mp4", "post_ids": ["7241234567890"], + "url": "https://tiktok.com/x", "extra": {"title": "a video", "downloader": "yt_dlp"}} + failure = {"post_ids": ["7241234567891"], "reason": "error", + "reason_description": "no video found", "url": "https://tiktok.com/y"} + + assert (list(TikTokVideoDownloader.map_metadata("7241234567890.mp4", item)) + == list(VideoDownloaderPlus.map_metadata("7241234567890.mp4", item))) + assert (list(TikTokVideoDownloader.map_failure_metadata(failure)) + == list(VideoDownloaderPlus.map_failure_metadata(failure)))