From a9998948ca3b1b8d6baf59b739d525b434e295cb Mon Sep 17 00:00:00 2001 From: Dale Wahl Date: Tue, 13 Oct 2026 16:05:22 +0200 Subject: [PATCH 01/10] archive_metadata: create support file helper (based on import_media) --- common/lib/archive_metadata.py | 26 ++++++++++++++++++ datasources/media_import/import_media.py | 34 ++++++------------------ 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/common/lib/archive_metadata.py b/common/lib/archive_metadata.py index c6cd0ce38..39f84ef76 100644 --- a/common/lib/archive_metadata.py +++ b/common/lib/archive_metadata.py @@ -27,6 +27,32 @@ from common.lib.exceptions import MetadataException from common.lib.helpers import get_software_commit +# Files 4CAT adds to an archive next to its media or other content. They are +# not items of the dataset: metadata files, the list in which yt-dlp notes +# which videos it downloaded, and log files (e.g. from ffmpeg) +SUPPORT_FILENAMES = frozenset({ + ".metadata.json", + ".token_metadata.json", + ".model_metadata.json", + "video_archive", +}) + + +def is_support_file(filename: str) -> bool: + """ + Check if a file in an archive is a support file rather than an item + + Iterating through an archive dataset skips these files, unless asked not + to. Read a metadata file with its own class instead, e.g. via + `DataSet.read_media_metadata()`. + + :param str filename: File name, can include folders (e.g. a path inside + a zip archive) + :return bool: + """ + name = filename.split("/")[-1] + return name in SUPPORT_FILENAMES or name.endswith(".log") + class ArchiveMetadataFile: """ diff --git a/datasources/media_import/import_media.py b/datasources/media_import/import_media.py index 38003a228..57218b85e 100644 --- a/datasources/media_import/import_media.py +++ b/datasources/media_import/import_media.py @@ -6,6 +6,7 @@ from io import BytesIO from backend.lib.processor import BasicProcessor +from common.lib.archive_metadata import is_support_file from common.lib.exceptions import QueryParametersException, QueryNeedsExplicitConfirmationException from common.lib.user_input import UserInput from common.lib.helpers import andify @@ -86,7 +87,7 @@ def validate_query(query, request, config): # as they are, system files and folders are left out later files = [f for f in files if not ( f["filename"].split("/")[-1].startswith(".") - or SearchMedia.is_support_file(f["filename"]) + or is_support_file(f["filename"]) or f["filename"].split("/")[-1].startswith("__MACOSX") or f["filename"].endswith(".DS_Store") or f["filename"].endswith("/") # sub-directory @@ -119,7 +120,7 @@ def validate_query(query, request, config): # Check file types to ensure all are same type of media for file in uploaded_files: # Support files are kept as they are, without a type check - if SearchMedia.is_support_file(file.filename): + if is_support_file(file.filename): continue # Warn on SVG files @@ -200,12 +201,12 @@ def after_create(query, dataset, request): guessed_file_mime_type = mimetypes.guess_type(inner_file.filename) file_media_type = guessed_file_mime_type[0].split('/')[0] if guessed_file_mime_type[0] else None - is_support_file = SearchMedia.is_support_file(inner_file.filename) + support_file = is_support_file(inner_file.filename) # keep media files and support files; skip macOS # clutter and anything else clean_file_name = inner_file.filename.split("/")[-1] - if (not is_support_file and file_media_type not in SearchMedia.accepted_file_types) \ + if (not support_file and file_media_type not in SearchMedia.accepted_file_types) \ or clean_file_name.startswith("__MACOSX") \ or inner_file.filename.startswith("__MACOSX"): print(f"skipping {clean_file_name} ({guessed_file_mime_type})") @@ -214,14 +215,14 @@ def after_create(query, dataset, request): # support files are not media, so they neither set # the dataset's media type nor count as files - if not is_support_file: + if not support_file: mime_type = file_media_type # save inner file from the uploaded zip archive to the new zip with all files new_filename = SearchMedia.get_safe_filename(inner_file.filename, new_zip_archive) new_zip_archive.writestr(new_filename, inner_zip_archive.read(inner_file)) - if not is_support_file: + if not support_file: saved_files += 1 continue @@ -234,7 +235,7 @@ def after_create(query, dataset, request): break dest_file.write(chunk) - if not SearchMedia.is_support_file(new_filename): + if not is_support_file(new_filename): saved_files += 1 # update the number of files in the dataset @@ -263,25 +264,6 @@ def process(self): self.dataset.update_status(f"Uploaded {self.parameters.get('num_files')} files of type {self.parameters.get('media_type')}{'' if svg_warning == 0 else f' ({svg_warning} SVG files; see log)'}", is_final=True) self.dataset.finish(self.parameters.get("num_files")) - @staticmethod - def is_support_file(filename): - """ - Check if a file is a non-media file that belongs with the media - - Media archives made by 4CAT hold more than media: a .metadata.json - file that links each file to the item it was downloaded for, a - 'video_archive' file in which yt-dlp notes which videos it downloaded, - and log files (e.g. from extracting video frames). When such an - archive is uploaded again, these files are kept in the dataset, but - they are not media, so they are not type-checked or counted as files. - - :param str filename: File name, can include folders (e.g. a path - inside a zip archive) - :return bool: - """ - filename = filename.split("/")[-1] - return filename in (".metadata.json", "video_archive") or filename.endswith(".log") - @staticmethod def get_safe_filename(filename, zip_archive=None): new_filename = SearchMedia.disallowed_characters.sub("", filename) From 3b276316a460eac2460d062f09a6e5929dca8273 Mon Sep 17 00:00:00 2001 From: Dale Wahl Date: Tue, 13 Oct 2026 16:06:17 +0200 Subject: [PATCH 02/10] update dataset to allow skipping support files directly --- common/lib/dataset.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/common/lib/dataset.py b/common/lib/dataset.py index f04d8527e..81aeb5b63 100644 --- a/common/lib/dataset.py +++ b/common/lib/dataset.py @@ -16,6 +16,7 @@ from natsort import natsorted from common.lib.annotation import Annotation +from common.lib.archive_metadata import is_support_file from common.lib.job import Job, JobNotFoundException from common.lib.helpers import get_software_commit, NullAwareTextIOWrapper, convert_to_int, get_software_version, call_api, hash_to_md5, convert_to_float @@ -393,6 +394,7 @@ def _iterate_archive_contents( staging_area=None, immediately_delete=True, filename_filter=None, + include_support_files=False, processor=None, offset=0, *args, **kwargs @@ -406,6 +408,10 @@ def _iterate_archive_contents( Files are temporarily unzipped and deleted after use. + Support files (metadata, logs, and the like; see `is_support_file()`) + are not items of the dataset and are skipped, unless + `include_support_files` is set or they are named in `filename_filter`. + :param Path staging_area: Where to store the files while they're being worked with. If omitted, a temporary folder is created and marked for deletion after all files have been yielded @@ -413,10 +419,10 @@ def _iterate_archive_contents( yielding; False keeps files until the staging_area is removed :param list filename_filter: Whitelist of filenames to iterate. If empty, do not filter + :param bool include_support_files: Also yield support files :param BasicProcessor processor: A reference to the processor iterating the dataset. - :param int offset: Skip this many files before yielding (warning: may - skip a metadata file too!) + :param int offset: Skip this many files before yielding :return: An iterator with a dictionary for each file, containing an `id`, a `path`, and all attributes of the `ZipInfo` object as keys """ @@ -460,6 +466,10 @@ def metadata_priority_sort(file): if filename_filter and archived_file.filename not in filename_filter: continue + # a support file named in filename_filter is still yielded + if not include_support_files and not filename_filter and is_support_file(archived_file.filename): + continue + if archived_file.is_dir(): # do not yield folders - we'll get to the files in them continue @@ -556,6 +566,10 @@ def iterate_items( :param list filename_filter: Only used when iterating a file archive. Whitelist of filenames to iterate, others are skipped. If empty, do not filter. + :param bool include_support_files: Only used when iterating a file + archive. Defaults to `False`, which skips files that are not items + of the dataset, such as metadata and log files. Read metadata with + e.g. `read_media_metadata()` instead. :return generator: A generator that yields DatasetItems """ unmapped_items = 0 From af09c71335da6ed388c1ea3ae67992678bd3de94 Mon Sep 17 00:00:00 2001 From: Dale Wahl Date: Tue, 13 Oct 2026 16:08:25 +0200 Subject: [PATCH 03/10] include support files where needed. --- processors/text-analysis/collocations.py | 2 +- processors/text-analysis/topic_modeling.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/processors/text-analysis/collocations.py b/processors/text-analysis/collocations.py index 32f6c121f..98bf36298 100644 --- a/processors/text-analysis/collocations.py +++ b/processors/text-analysis/collocations.py @@ -157,7 +157,7 @@ def process(self): annotations = {} # Go through all archived token sets and generate collocations for each - for token_file in self.source_dataset.iterate_items(self): + for token_file in self.source_dataset.iterate_items(self, include_support_files=True): if token_file.file.name == '.token_metadata.json': # Get metadata if we write annotations diff --git a/processors/text-analysis/topic_modeling.py b/processors/text-analysis/topic_modeling.py index d9070ab67..87ee8a78d 100644 --- a/processors/text-analysis/topic_modeling.py +++ b/processors/text-analysis/topic_modeling.py @@ -105,7 +105,7 @@ def process(self): model_metadata = {'parameters': self.parameters} # go through all archived token sets and vectorise them index = 0 - for token in self.source_dataset.iterate_items(self): + for token in self.source_dataset.iterate_items(self, include_support_files=True): # Check for and open token metadata file if token.file.name == '.token_metadata.json': # Copy the token metadata into our staging area From a4ffc06c3b8d74fef80178a741c9812e5548f543 Mon Sep 17 00:00:00 2001 From: Dale Wahl Date: Tue, 13 Oct 2026 16:10:08 +0200 Subject: [PATCH 04/10] remove now unnecessary support file checks --- processors/conversion/hash_images.py | 2 -- processors/filtering/unique_images.py | 3 --- processors/machine_learning/google_vision_api.py | 3 +-- processors/networks/image-network.py | 8 ++++---- processors/text-analysis/generate_embeddings.py | 3 --- processors/text-analysis/tf_idf.py | 3 --- processors/text-analysis/vectorise.py | 3 --- processors/text-analysis/vectorise_by_cat.py | 3 --- processors/visualisation/image_wall_w_text.py | 5 ----- processors/visualisation/video_hasher.py | 6 ------ processors/visualisation/video_scene_identifier.py | 6 ------ processors/visualisation/video_stack.py | 4 ---- processors/visualisation/video_wall.py | 3 --- 13 files changed, 5 insertions(+), 47 deletions(-) diff --git a/processors/conversion/hash_images.py b/processors/conversion/hash_images.py index 291b409fc..ad74b91d7 100644 --- a/processors/conversion/hash_images.py +++ b/processors/conversion/hash_images.py @@ -144,8 +144,6 @@ def process(self): if processed % 100 == 0: self.dataset.update_status(f"Processed {processed:,} of {self.source_dataset.num_rows:,} images") - if item.file.name == ".metadata.json": - continue try: image_hash = hash_image(item.file, hash_type, hash_size, as_string=False) except FileNotFoundError as e: diff --git a/processors/filtering/unique_images.py b/processors/filtering/unique_images.py index ea7b1ee10..78bbdbb50 100644 --- a/processors/filtering/unique_images.py +++ b/processors/filtering/unique_images.py @@ -85,9 +85,6 @@ def process(self): f"found {dupes:,} duplicate(s)") processed += 1 - if image.file.name == ".metadata.json": - continue - image_hash = hash_file(image.file, hash_type) if image_hash not in seen_hashes: diff --git a/processors/machine_learning/google_vision_api.py b/processors/machine_learning/google_vision_api.py index ca00a6d97..c4ca591a8 100644 --- a/processors/machine_learning/google_vision_api.py +++ b/processors/machine_learning/google_vision_api.py @@ -134,8 +134,7 @@ def process(self): self.dataset.update_progress(done / total) if image.file.name.startswith(".") or image.file.suffix in (".json", ".log"): - if image.file.name != ".metadata.json": - self.dataset.log(f"Skipping file {image.file.name}, probably not an image.") + self.dataset.log(f"Skipping file {image.file.name}, probably not an image.") continue try: diff --git a/processors/networks/image-network.py b/processors/networks/image-network.py index afa49846a..5ebd1304d 100644 --- a/processors/networks/image-network.py +++ b/processors/networks/image-network.py @@ -111,7 +111,6 @@ def is_compatible_with(cls, module=None, config=None): def process(self): column = self.parameters.get("column") hash_type = self.parameters.get("deduplicate") - filename_filter = [".metadata.json"] if hash_type == "none" else [] hashed = 0 try: @@ -129,9 +128,10 @@ def process(self): seen_hashes = set() id_file_map = {} - for image in self.source_dataset.iterate_items(filename_filter=filename_filter): - if image.file.name == ".metadata.json": - continue + # without deduplication, images are told apart by file name, so the + # images themselves do not need to be read + images = self.source_dataset.iterate_items() if hash_type != "none" else [] + for image in images: try: hashed += 1 if hashed % 100 == 0: diff --git a/processors/text-analysis/generate_embeddings.py b/processors/text-analysis/generate_embeddings.py index ea352895b..dfc35e76a 100644 --- a/processors/text-analysis/generate_embeddings.py +++ b/processors/text-analysis/generate_embeddings.py @@ -145,9 +145,6 @@ def process(self): # go through all archived token sets and vectorise them models = 0 for tokens in self.source_dataset.iterate_items(self): - if tokens.file.name == '.token_metadata.json': - # Skip metadata - continue # use the "list of lists" as input for the word2vec model # by default the tokeniser generates one list of tokens per # post... which may actually be preferable for short diff --git a/processors/text-analysis/tf_idf.py b/processors/text-analysis/tf_idf.py index 4e7378c4f..97b13cbfb 100644 --- a/processors/text-analysis/tf_idf.py +++ b/processors/text-analysis/tf_idf.py @@ -141,9 +141,6 @@ def process(self): # Go through all archived token sets and generate collocations for each for token_file in self.source_dataset.iterate_items(self): - if token_file.file.name == '.token_metadata.json': - # Skip metadata - continue # Get the date date_string = token_file.file.stem dates.append(date_string) diff --git a/processors/text-analysis/vectorise.py b/processors/text-analysis/vectorise.py index 0b591afba..fbf74207b 100644 --- a/processors/text-analysis/vectorise.py +++ b/processors/text-analysis/vectorise.py @@ -40,9 +40,6 @@ def process(self): # go through all archived token sets and vectorise them index = 0 for packed_tokens in self.source_dataset.iterate_items(self): - if packed_tokens.file.name == '.token_metadata.json': - # Skip metadata - continue index += 1 vector_set_name = packed_tokens.file.stem # we don't need the full path self.dataset.update_status("Processing token set %i (%s)" % (index, vector_set_name)) diff --git a/processors/text-analysis/vectorise_by_cat.py b/processors/text-analysis/vectorise_by_cat.py index eacf9a2a1..a11b17353 100644 --- a/processors/text-analysis/vectorise_by_cat.py +++ b/processors/text-analysis/vectorise_by_cat.py @@ -212,9 +212,6 @@ def process(self): # Each file is a token set (Tokenize processor separates tokens by dates or all) and contains a list of tokens for each document # A single item/post may have multiple documents (e.g., if it was seperated by sentance) for packed_tokens in self.source_dataset.iterate_items(self): - if packed_tokens.file.name == '.token_metadata.json': - # Skip metadata - continue index += 1 vector_set_name = packed_tokens.file.stem # we don't need the full path diff --git a/processors/visualisation/image_wall_w_text.py b/processors/visualisation/image_wall_w_text.py index 633a392fc..2522acd2d 100644 --- a/processors/visualisation/image_wall_w_text.py +++ b/processors/visualisation/image_wall_w_text.py @@ -212,11 +212,6 @@ def process(self): self.dataset.update_status("Creating Image wall") self.dataset.log(f"Creating image wall with {max_images} images, size {base_height} and tile type {tile_type}") for image in image_dataset.iterate_items(self, immediately_delete=False): - if image.file.name in [".metadata.json"]: - if convert_to_int(self.parameters.get("amount"), 100) == 0: - max_images = max_images - 1 - continue - # Check image loads prior to any modifications to canvas space try: frame = Image.open(str(image.file)) diff --git a/processors/visualisation/video_hasher.py b/processors/visualisation/video_hasher.py index 858987a3f..0b3d2d7e2 100644 --- a/processors/visualisation/video_hasher.py +++ b/processors/visualisation/video_hasher.py @@ -200,12 +200,6 @@ def process(self): if max_videos != 0 and processed_videos >= max_videos: break - if video.file.name == '.metadata.json': - continue - elif video.file.name == "video_archive": - # yt-dlp file - continue - try: videohash = VideoHash(path=str(video.file), storage_path=str(staging_area), frame_interval=frame_interval, do_not_copy=True) except FFmpegNotFound: diff --git a/processors/visualisation/video_scene_identifier.py b/processors/visualisation/video_scene_identifier.py index a96bb1453..0b95f0425 100644 --- a/processors/visualisation/video_scene_identifier.py +++ b/processors/visualisation/video_scene_identifier.py @@ -225,12 +225,6 @@ def process(self): if self.interrupted: raise ProcessorInterruptedException("Interrupted while detecting video scenes") - if original_video.file.name == ".metadata.json": - continue - elif original_video.file.name == "video_archive": - # yt-dlp file - continue - detector = self.get_scenes_scenedetect if self.parameters.get("detector_type") != "ffmpeg_select" else self.get_scenes_ffmpeg self.dataset.update_progress(processed_videos / self.source_dataset.num_rows) diff --git a/processors/visualisation/video_stack.py b/processors/visualisation/video_stack.py index a2f2fcac5..6512e3d22 100644 --- a/processors/visualisation/video_stack.py +++ b/processors/visualisation/video_stack.py @@ -161,10 +161,6 @@ def process(self): if self.interrupted: return ProcessorInterruptedException("Interrupted while unpacking videos") - # skip JSON - if video.file.name == '.metadata.json': - continue - if len(videos) >= max_videos: break diff --git a/processors/visualisation/video_wall.py b/processors/visualisation/video_wall.py index ba00c6765..7c28608a4 100644 --- a/processors/visualisation/video_wall.py +++ b/processors/visualisation/video_wall.py @@ -193,9 +193,6 @@ def process(self): self.dataset.update_status(f"Determined dimensions of {len(media):,} of {self.source_dataset.num_rows:,} file(s)") if item.file.suffix.lower() in (".json", ".log"): continue - elif item.file.name == "video_archive": - # yt-dlp created file - continue try: dimensions[item.file.name], lengths[item.file.name], sort_values[item.file.name] = \ From 280c6cb3de49f17d89c912c5a485b17794c5d834 Mon Sep 17 00:00:00 2001 From: Dale Wahl Date: Tue, 13 Oct 2026 16:11:27 +0200 Subject: [PATCH 05/10] add some nuance: extract metadata files with new archive class if needed and update some old counting tricks (to avoid counting metadata files) --- processors/audio/audio_extractor.py | 11 +++-------- processors/machine_learning/clarifai_api.py | 2 +- processors/networks/image-network.py | 4 ++-- processors/visualisation/download_videos.py | 8 +++++--- processors/visualisation/video_frames.py | 12 +++++++----- processors/visualisation/video_hasher.py | 2 +- processors/visualisation/video_scene_frames.py | 12 ++++++++---- processors/visualisation/video_stack.py | 10 ++++++++-- processors/visualisation/video_timelines.py | 8 ++++---- 9 files changed, 39 insertions(+), 30 deletions(-) diff --git a/processors/audio/audio_extractor.py b/processors/audio/audio_extractor.py index 115351a24..d368f9dc6 100644 --- a/processors/audio/audio_extractor.py +++ b/processors/audio/audio_extractor.py @@ -6,10 +6,10 @@ """ import shutil import zipfile -from pathlib import Path import oslex from backend.lib.processor import BasicProcessor +from common.lib.archive_metadata import is_support_file from common.lib.compatibility import Compatibility, is_executable from common.lib.exceptions import ProcessorInterruptedException, MetadataException @@ -66,7 +66,7 @@ def process(self): # Prepare staging areas for videos and video tracking output_dir = self.dataset.get_staging_area() - # Estimate how many actual video files we will attempt, excluding archive metadata. + # Estimate how many actual video files we will attempt, excluding metadata and the like. total_possible_videos = self.source_dataset.num_rows source_archive = self.source_dataset.get_results_path() if source_archive.exists() and source_archive.suffix.lower() == ".zip": @@ -74,7 +74,7 @@ def process(self): total_possible_videos = sum( 1 for archived_file in archive_file.infolist() - if not archived_file.is_dir() and Path(archived_file.filename).name != ".metadata.json" + if not archived_file.is_dir() and not is_support_file(archived_file.filename) ) if max_files != 0: @@ -102,11 +102,6 @@ def process(self): if self.interrupted: raise ProcessorInterruptedException("Interrupted while determining image wall order") - # the source archive's metadata describes videos; we write our - # own (audio-keyed) metadata below, so skip the original - if item.file.name == '.metadata.json': - continue - if max_files != 0 and processed_videos >= max_files: break diff --git a/processors/machine_learning/clarifai_api.py b/processors/machine_learning/clarifai_api.py index e45873a9f..28024f17d 100644 --- a/processors/machine_learning/clarifai_api.py +++ b/processors/machine_learning/clarifai_api.py @@ -135,7 +135,7 @@ def process(self): buffer = {} batch_size = 16 - num_images = self.source_dataset.num_rows - 1 # .metadata.json + num_images = self.source_dataset.num_rows total_annotations = (num_images if limit == 0 else min(limit, num_images)) * len(models) errors = 0 processed = 0 diff --git a/processors/networks/image-network.py b/processors/networks/image-network.py index 5ebd1304d..fa2e478ff 100644 --- a/processors/networks/image-network.py +++ b/processors/networks/image-network.py @@ -135,8 +135,8 @@ def process(self): try: hashed += 1 if hashed % 100 == 0: - self.dataset.update_status(f"Generated identity hashes for {hashed:,} of {self.source_dataset.num_rows-1:,} item(s)") - self.dataset.update_progress(hashed / (self.source_dataset.num_rows-1) * 0.5) + self.dataset.update_status(f"Generated identity hashes for {hashed:,} of {self.source_dataset.num_rows:,} item(s)") + self.dataset.update_progress(hashed / self.source_dataset.num_rows * 0.5) file_hash = hash_file(image.file, hash_type) file_hash_map[image.file.name] = file_hash if file_hash not in hash_file_map: diff --git a/processors/visualisation/download_videos.py b/processors/visualisation/download_videos.py index 5a2793114..eff49afa3 100644 --- a/processors/visualisation/download_videos.py +++ b/processors/visualisation/download_videos.py @@ -1060,7 +1060,9 @@ def _log_statistics(self, total_urls): def _finish_processing(self, results_path, total_urls): """Finish processing and create result archive""" self.dataset.update_status("Writing downloaded videos to zip archive") - self.write_archive_and_finish(results_path, self.downloaded_videos + self.copied_videos, finish=False) + # videos copied from an earlier run are in the archive too + num_videos = self.downloaded_videos + self.copied_videos + self.write_archive_and_finish(results_path, num_videos, finish=False) status_msg = f"Downloaded {self.downloaded_videos} videos" if self.copied_videos > 0: @@ -1074,10 +1076,10 @@ def _finish_processing(self, results_path, total_urls): if self.warning_message: self.dataset.update_status(status_msg) - self.dataset.finish_with_warning(self.downloaded_videos, f"Incomplete: {self.warning_message}") + self.dataset.finish_with_warning(num_videos, f"Incomplete: {self.warning_message}") else: self.dataset.update_status(status_msg, is_final=True) - self.dataset.finish(self.downloaded_videos) + self.dataset.finish(num_videos) def yt_dlp_monitor(self, d): """Can be used to gather information from yt-dlp while downloading""" diff --git a/processors/visualisation/video_frames.py b/processors/visualisation/video_frames.py index 9a24621fa..de8510930 100644 --- a/processors/visualisation/video_frames.py +++ b/processors/visualisation/video_frames.py @@ -94,16 +94,18 @@ def process(self): total_possible_videos = self.source_dataset.num_rows processed_videos = 0 + # Copy the videos' metadata, so frames can be traced back to the items + # the videos were downloaded for + try: + self.extract_archived_file_by_name(".metadata.json", self.source_dataset.get_results_path(), output_directory) + except FileNotFoundError: + pass + self.dataset.update_status("Extracting video frames") for i, video in enumerate(self.source_dataset.iterate_items(self)): if self.interrupted: raise ProcessorInterruptedException("Interrupted while determining image wall order") - # Check for 4CAT's metadata JSON and copy it - if video.file.name == '.metadata.json': - shutil.copy(video.file, output_directory) - continue - vid_name = video.file.stem video_dir = output_directory.joinpath(vid_name) video_dir.mkdir(exist_ok=True) diff --git a/processors/visualisation/video_hasher.py b/processors/visualisation/video_hasher.py index 0b3d2d7e2..d4031ba7a 100644 --- a/processors/visualisation/video_hasher.py +++ b/processors/visualisation/video_hasher.py @@ -189,7 +189,7 @@ def process(self): video_metadata = self.source_dataset.read_media_metadata() except (FileNotFoundError, MetadataException): video_metadata = None - total_possible_videos = max((min(self.source_dataset.num_rows - 1, max_videos) if max_videos != 0 else self.source_dataset.num_rows), 1) + total_possible_videos = max((min(self.source_dataset.num_rows, max_videos) if max_videos != 0 else self.source_dataset.num_rows), 1) processed_videos = 0 self.dataset.update_status("Creating video hashes") diff --git a/processors/visualisation/video_scene_frames.py b/processors/visualisation/video_scene_frames.py index 65e136fee..ce3c4641b 100644 --- a/processors/visualisation/video_scene_frames.py +++ b/processors/visualisation/video_scene_frames.py @@ -116,11 +116,15 @@ def process(self): errors = 0 processed_frames = 0 num_scenes = self.source_dataset.num_rows - for video in video_dataset.iterate_items(self): - # Check for 4CAT's metadata JSON and copy it - if video.file.name == '.metadata.json': - shutil.copy(video.file, staging_area) + # Copy the videos' metadata, so frames can be traced back to the items + # the videos were downloaded for + try: + self.extract_archived_file_by_name(".metadata.json", video_dataset.get_results_path(), staging_area) + except FileNotFoundError: + pass + + for video in video_dataset.iterate_items(self): if video.file.name not in scenes: continue diff --git a/processors/visualisation/video_stack.py b/processors/visualisation/video_stack.py index 6512e3d22..6bfc2f0c3 100644 --- a/processors/visualisation/video_stack.py +++ b/processors/visualisation/video_stack.py @@ -108,7 +108,7 @@ def process(self): frames from the videos at intervals and create image collages to hashes for comparison of videos. """ # Check processor able to run - if self.source_dataset.num_rows < 3: + if self.source_dataset.num_rows < 2: self.dataset.finish_with_error("Not enough videos to stack (need at least 2)") return @@ -152,7 +152,7 @@ def process(self): except ValueError: transparency = 0.3 - max_videos = min(amount, self.source_dataset.num_rows - 1) # minus 1, because .metadata.json + max_videos = min(amount, self.source_dataset.num_rows) lengths = {} videos = [] @@ -186,6 +186,12 @@ def process(self): num_videos = len(videos) self.dataset.log(f"Collected {num_videos} videos to stack") + # the dataset's row count can be higher than the number of videos in + # it (e.g. archives from before July 2023 also counted their metadata + # file), so check the videos actually found + if num_videos < 2: + return self.dataset.finish_with_error("Not enough videos to stack (need at least 2)") + # longest video in set may be shorter than requested length if video_length: video_length = min(max(lengths.values()), video_length) diff --git a/processors/visualisation/video_timelines.py b/processors/visualisation/video_timelines.py index d701f1828..3df301292 100644 --- a/processors/visualisation/video_timelines.py +++ b/processors/visualisation/video_timelines.py @@ -79,6 +79,7 @@ def process(self): offset_y = -base_height timeline = None iterator = self.source_dataset.iterate_items(self) + item = None looping = True timelines = [] timeline_widths = {} @@ -100,10 +101,9 @@ def process(self): # this means that at the end of the iterator we complete one # final iteration in which we finish things up (see below) looping = False - - # skip the metadata file (already read via read_media_metadata) - if item.file.name == ".metadata.json": - continue + if item is None: + # the archive holds no files to make timelines from + break # skip if file is a real file but not an image if looping and item.file.suffix not in (".jpeg", ".jpg", ".png", ".gif"): From b8c354c31dc2ed0c3142f3fbca1f720d64f0106c Mon Sep 17 00:00:00 2001 From: Dale Wahl Date: Tue, 13 Oct 2026 16:11:52 +0200 Subject: [PATCH 06/10] test archive things --- tests/test_archive_metadata.py | 19 ++++++++++++++++- tests/test_dataset_items.py | 39 ++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/tests/test_archive_metadata.py b/tests/test_archive_metadata.py index d8793c8dd..c0eedcd3a 100644 --- a/tests/test_archive_metadata.py +++ b/tests/test_archive_metadata.py @@ -10,7 +10,7 @@ import pytest -from common.lib.archive_metadata import MediaArchiveMetadata +from common.lib.archive_metadata import MediaArchiveMetadata, is_support_file from common.lib.exceptions import MetadataException CURRENT_SCHEMA_VERSION = MediaArchiveMetadata.SCHEMA_VERSION @@ -396,3 +396,20 @@ def test_v1_read_normalizes_integer_post_ids(): m._populate_from_raw(raw) assert m.get_entry("a.jpg")["post_ids"] == ["1", "2"] assert m.failures[0]["post_ids"] == ["3"] + + +@pytest.mark.parametrize("filename, expected", [ + (".metadata.json", True), + (".token_metadata.json", True), + (".model_metadata.json", True), + ("video_archive", True), + ("video_1/ffmpeg_error.log", True), + ("frames/.metadata.json", True), + ("image.jpg", False), + ("video.mp4", False), + ("results.json", False), + (".DS_Store", False), +]) +def test_is_support_file(filename, expected): + """Metadata, yt-dlp's video list and logs are support files; media is not.""" + assert is_support_file(filename) is expected diff --git a/tests/test_dataset_items.py b/tests/test_dataset_items.py index 6924f5270..913e5f34c 100644 --- a/tests/test_dataset_items.py +++ b/tests/test_dataset_items.py @@ -1,3 +1,4 @@ +import zipfile from unittest.mock import MagicMock import pytest @@ -33,3 +34,41 @@ def test_row_with_more_values_than_columns_stops_reading(tmp_path): with pytest.raises(DataSetException, match=r"dataset abc123: line 3 of its CSV file has 2 more value"): next(items) + + +@pytest.fixture +def media_archive(tmp_path): + """A video archive as 4CAT makes it: videos plus support files""" + path = tmp_path / "results.zip" + with zipfile.ZipFile(path, "w") as archive: + for filename in (".metadata.json", "video_archive", "a.mp4", "b.mp4", "b/ffmpeg_error.log"): + archive.writestr(filename, "content") + + return path + + +def read_archive(path, staging_area, **kwargs): + """Iterate a zip archive the way a dataset iterates its own results file""" + staging_area.mkdir(exist_ok=True) + dataset = MagicMock(key="abc123") + dataset.get_results_path.return_value = path + return [item["id"] for item in DataSet._iterate_archive_contents(dataset, staging_area=staging_area, **kwargs)] + + +def test_archive_skips_support_files(media_archive, tmp_path): + assert read_archive(media_archive, tmp_path / "staging") == ["a.mp4", "b.mp4"] + + +def test_archive_includes_support_files_when_asked(media_archive, tmp_path): + files = read_archive(media_archive, tmp_path / "staging", include_support_files=True) + assert sorted(files) == sorted([".metadata.json", "video_archive", "a.mp4", "b.mp4", "b/ffmpeg_error.log"]) + # dotfiles come first, so a metadata file can be read before the files it describes + assert files[0] == ".metadata.json" + + +def test_archive_yields_support_file_named_in_filter(media_archive, tmp_path): + assert read_archive(media_archive, tmp_path / "staging", filename_filter=[".metadata.json"]) == [".metadata.json"] + + +def test_archive_offset_counts_items_only(media_archive, tmp_path): + assert read_archive(media_archive, tmp_path / "staging", offset=1) == ["b.mp4"] From ece562d2f7c18af4ea16218e8874156736422a9b Mon Sep 17 00:00:00 2001 From: Dale Wahl Date: Wed, 14 Oct 2026 11:13:43 +0200 Subject: [PATCH 07/10] protect base filter if metadata file does not exist --- processors/filtering/base_filter.py | 10 +++--- tests/test_base_filter.py | 55 +++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 tests/test_base_filter.py diff --git a/processors/filtering/base_filter.py b/processors/filtering/base_filter.py index 0e0e95cf6..2cf1b3f02 100644 --- a/processors/filtering/base_filter.py +++ b/processors/filtering/base_filter.py @@ -94,10 +94,12 @@ def process(self): # Write the archive if needed and finish the dataset if zip_file: - # check for metadata file and copy it over if it exists - metadata_file = self.extract_archived_file_by_name(".metadata.json", self.source_dataset.get_results_path()) - if metadata_file: - shutil.copy2(metadata_file, staging_area) + # copy the metadata file over, if the archive has one (e.g. an + # upload of loose media files does not) + try: + self.extract_archived_file_by_name(".metadata.json", self.source_dataset.get_results_path(), staging_area) + except FileNotFoundError: + pass self.write_archive_and_finish(staging_area, num_posts) else: self.dataset.finish(num_posts) diff --git a/tests/test_base_filter.py b/tests/test_base_filter.py new file mode 100644 index 000000000..6abeb45aa --- /dev/null +++ b/tests/test_base_filter.py @@ -0,0 +1,55 @@ +""" +Tests for filtering archive datasets with `BaseFilter` +""" +import zipfile +from types import SimpleNamespace +from unittest.mock import MagicMock + +from backend.lib.processor import BasicProcessor +from processors.filtering.base_filter import BaseFilter + + +def run_filter(tmp_path, archive_contents): + """ + Filter an archive dataset, keeping its one image + + :return tuple: The staging area the filter wrote to, and the mocked + `write_archive_and_finish` + """ + source_archive = tmp_path / "source.zip" + with zipfile.ZipFile(source_archive, "w") as archive: + for filename in archive_contents: + archive.writestr(filename, "content") + + image = tmp_path / "a.jpg" + image.write_text("content") + staging_area = tmp_path / "staging" + staging_area.mkdir() + + processor = MagicMock() + processor.source_dataset.get_extension.return_value = "zip" + processor.source_dataset.get_results_path.return_value = source_archive + processor.source_dataset.num_annotations.return_value = 0 + processor.dataset.get_results_path.return_value = tmp_path / "results.zip" + processor.dataset.get_staging_area.return_value = staging_area + processor.dataset.is_finished.return_value = False + processor.filter_items.return_value = [SimpleNamespace(original={"id": "a.jpg"}, file=image)] + processor.extract_archived_file_by_name = lambda *args: BasicProcessor.extract_archived_file_by_name(processor, *args) + + BaseFilter.process(processor) + return staging_area, processor.write_archive_and_finish + + +def test_filter_archive_without_metadata(tmp_path): + """An archive without .metadata.json (e.g. uploaded media) can be filtered""" + staging_area, write_archive_and_finish = run_filter(tmp_path, ["a.jpg"]) + + assert sorted(file.name for file in staging_area.iterdir()) == ["a.jpg"] + write_archive_and_finish.assert_called_once_with(staging_area, 1) + + +def test_filter_archive_keeps_metadata(tmp_path): + staging_area, write_archive_and_finish = run_filter(tmp_path, [".metadata.json", "a.jpg"]) + + assert sorted(file.name for file in staging_area.iterdir()) == [".metadata.json", "a.jpg"] + write_archive_and_finish.assert_called_once_with(staging_area, 1) From 3989bad51389dfb8c57b6b8dd40e81f8163cf080 Mon Sep 17 00:00:00 2001 From: Dale Wahl Date: Wed, 14 Oct 2026 11:18:00 +0200 Subject: [PATCH 08/10] refactor get_media_metadata to avoid missing file or metadata exception repetition --- backend/lib/processor.py | 27 +++++++++++++- processors/audio/audio_extractor.py | 7 ++-- processors/conversion/hash_images.py | 7 ++-- processors/filtering/unique_images.py | 7 ++-- processors/machine_learning/audio_to_text.py | 7 ++-- .../machine_learning/blip2_image_caption.py | 7 ++-- processors/machine_learning/clarifai_api.py | 8 ++--- .../clip_categorize_images.py | 7 ++-- .../machine_learning/google_vision_api.py | 9 ++--- processors/machine_learning/llm_prompter.py | 13 +++---- processors/machine_learning/pix-plot.py | 6 ++-- .../machine_learning/text_from_image.py | 8 ++--- processors/networks/image-network.py | 7 ++-- .../visualisation/image_category_wall.py | 7 ++-- processors/visualisation/image_wall_w_text.py | 7 ++-- processors/visualisation/video_hasher.py | 7 ++-- .../visualisation/video_scene_identifier.py | 7 ++-- processors/visualisation/video_timelines.py | 7 ++-- tests/test_get_media_metadata.py | 35 +++++++++++++++++++ 19 files changed, 105 insertions(+), 85 deletions(-) create mode 100644 tests/test_get_media_metadata.py diff --git a/backend/lib/processor.py b/backend/lib/processor.py index 72c735812..8fdfe50fd 100644 --- a/backend/lib/processor.py +++ b/backend/lib/processor.py @@ -20,7 +20,7 @@ from common.lib.fourcat_module import FourcatModule from common.lib.helpers import get_software_commit, remove_nuls, send_email, hash_to_md5, looks_like_email_address from common.lib.exceptions import (WorkerInterruptedException, ProcessorInterruptedException, ProcessorException, - DataSetException, MapItemException, AnnotationException) + DataSetException, MapItemException, AnnotationException, MetadataException) from common.config_manager import ConfigWrapper from common.lib.user import User @@ -587,6 +587,31 @@ def unpack_archive_contents(self, path, staging_area=None): return staging_area + def get_media_metadata(self, dataset=None): + """ + Read the media metadata of a dataset, if it can be used + + A media archive without a metadata file is normal (e.g. uploaded + media), so that is not logged. A metadata file that cannot be read + (e.g. it is damaged) is logged in this processor's dataset log, so + the researcher can see why the results are not linked to items. + + :param DataSet dataset: Dataset to read the metadata of. Defaults to + the source dataset. + :return MediaArchiveMetadata|None: The metadata, or `None` if there + is none that can be used + """ + if dataset is None: + dataset = self.source_dataset + + try: + return dataset.read_media_metadata() + except FileNotFoundError: + return None + except MetadataException as e: + self.dataset.log(f"Could not use the metadata file of dataset {dataset.key}: {e}") + return None + def extract_archived_file_by_name(self, filename, archive_path, staging_area=None): """ Extract a file from an archive by name diff --git a/processors/audio/audio_extractor.py b/processors/audio/audio_extractor.py index d368f9dc6..190b9c432 100644 --- a/processors/audio/audio_extractor.py +++ b/processors/audio/audio_extractor.py @@ -11,7 +11,7 @@ from backend.lib.processor import BasicProcessor from common.lib.archive_metadata import is_support_file from common.lib.compatibility import Compatibility, is_executable -from common.lib.exceptions import ProcessorInterruptedException, MetadataException +from common.lib.exceptions import ProcessorInterruptedException __author__ = "Dale Wahl" __credits__ = ["Dale Wahl"] @@ -82,10 +82,7 @@ def process(self): # Read the source video archive's metadata so each extracted audio # file can carry its video's provenance (source posts, URL). - try: - source_metadata = self.source_dataset.read_media_metadata() - except (FileNotFoundError, MetadataException): - source_metadata = None + source_metadata = self.get_media_metadata() # Build our own metadata describing the audio files we produce, # rather than passing the (video-keyed) source metadata through. diff --git a/processors/conversion/hash_images.py b/processors/conversion/hash_images.py index ad74b91d7..5e88550eb 100644 --- a/processors/conversion/hash_images.py +++ b/processors/conversion/hash_images.py @@ -6,7 +6,7 @@ from PIL import UnidentifiedImageError from backend.lib.processor import BasicProcessor -from common.lib.exceptions import ProcessorInterruptedException, MetadataException +from common.lib.exceptions import ProcessorInterruptedException from common.lib.helpers import UserInput, hash_image, stringify_hash from common.lib.compatibility import Compatibility from processors.metrics.group_hashes import HashGrouper @@ -110,10 +110,7 @@ def process(self): group_by = self.parameters.get("group-by", False) similarity_pct = float(self.parameters.get("similarity-threshold", 4)) - try: - source_metadata = self.source_dataset.read_media_metadata() - except (FileNotFoundError, MetadataException): - source_metadata = None + source_metadata = self.get_media_metadata() image_data_by_filename = {} if source_metadata is not None: diff --git a/processors/filtering/unique_images.py b/processors/filtering/unique_images.py index 78bbdbb50..ef28716c4 100644 --- a/processors/filtering/unique_images.py +++ b/processors/filtering/unique_images.py @@ -4,7 +4,7 @@ import shutil from backend.lib.processor import BasicProcessor -from common.lib.exceptions import ProcessorInterruptedException, MetadataException +from common.lib.exceptions import ProcessorInterruptedException from common.lib.helpers import UserInput, hash_file from common.lib.compatibility import Compatibility @@ -95,10 +95,7 @@ def process(self): self.dataset.log(f"{image.file.name} is a duplicate of {hash_map[image_hash]} - skipping") dupes += 1 - try: - source_metadata = self.source_dataset.read_media_metadata() - except (FileNotFoundError, MetadataException): - source_metadata = None + source_metadata = self.get_media_metadata() new_metadata = self.dataset.new_media_metadata( processor_type=self.type, diff --git a/processors/machine_learning/audio_to_text.py b/processors/machine_learning/audio_to_text.py index d631beb1a..4d9357730 100644 --- a/processors/machine_learning/audio_to_text.py +++ b/processors/machine_learning/audio_to_text.py @@ -9,7 +9,7 @@ from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility from common.lib.dmi_service_manager import DmiServiceManager, DmiServiceManagerException, DsmOutOfMemory -from common.lib.exceptions import ProcessorInterruptedException, MetadataException +from common.lib.exceptions import ProcessorInterruptedException from common.lib.user_input import UserInput from common.lib.item_mapping import MappedItem @@ -510,10 +510,7 @@ def process(self): # extension-less filename: the transcription result files share their # stem with the input audio (e.g. `foo.wav` -> `foo.json`). file_metadata_map = {} - try: - source_metadata = self.source_dataset.read_media_metadata() - except (FileNotFoundError, MetadataException): - source_metadata = None + source_metadata = self.get_media_metadata() if source_metadata is not None: self.dataset.log("Found and loaded audio metadata") from_dataset = source_metadata.from_dataset diff --git a/processors/machine_learning/blip2_image_caption.py b/processors/machine_learning/blip2_image_caption.py index c4d95855e..e94a1a108 100644 --- a/processors/machine_learning/blip2_image_caption.py +++ b/processors/machine_learning/blip2_image_caption.py @@ -6,7 +6,7 @@ from backend.lib.processor import BasicProcessor from common.lib.dmi_service_manager import DmiServiceManager, DmiServiceManagerException, DsmOutOfMemory, DsmConnectionError -from common.lib.exceptions import ProcessorInterruptedException, MetadataException +from common.lib.exceptions import ProcessorInterruptedException from common.lib.user_input import UserInput from common.lib.item_mapping import MappedItem from common.lib.compatibility import Compatibility @@ -201,10 +201,7 @@ def process(self): # Load the image metadata if available. Key by basename because BLIP2 # results share the input image's stem but use a different extension. image_metadata = {} - try: - source_metadata = self.source_dataset.read_media_metadata() - except (FileNotFoundError, MetadataException): - source_metadata = None + source_metadata = self.get_media_metadata() if source_metadata is not None: self.dataset.log("Found and loaded image metadata") from_dataset = source_metadata.from_dataset diff --git a/processors/machine_learning/clarifai_api.py b/processors/machine_learning/clarifai_api.py index 28024f17d..57325bea3 100644 --- a/processors/machine_learning/clarifai_api.py +++ b/processors/machine_learning/clarifai_api.py @@ -9,7 +9,6 @@ from clarifai_grpc.channel.clarifai_channel import ClarifaiChannel from common.lib.helpers import UserInput, convert_to_int -from common.lib.exceptions import MetadataException from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility @@ -141,11 +140,8 @@ def process(self): processed = 0 annotated = 0 - try: - source_metadata = self.source_dataset.read_media_metadata() - filename_to_post_ids = source_metadata.filename_to_post_ids() - except (FileNotFoundError, MetadataException): - filename_to_post_ids = {} + source_metadata = self.get_media_metadata() + filename_to_post_ids = source_metadata.filename_to_post_ids() if source_metadata is not None else {} # send batched requests per model for model_id in models: diff --git a/processors/machine_learning/clip_categorize_images.py b/processors/machine_learning/clip_categorize_images.py index ada4e1c48..5dc1867c3 100644 --- a/processors/machine_learning/clip_categorize_images.py +++ b/processors/machine_learning/clip_categorize_images.py @@ -7,7 +7,7 @@ from backend.lib.processor import BasicProcessor from common.lib.dmi_service_manager import DmiServiceManager, DmiServiceManagerException, DsmOutOfMemory, DsmConnectionError -from common.lib.exceptions import ProcessorInterruptedException, MetadataException +from common.lib.exceptions import ProcessorInterruptedException from common.lib.user_input import UserInput from common.lib.item_mapping import MappedItem from common.lib.compatibility import Compatibility @@ -224,10 +224,7 @@ def process(self): # Load the image metadata if available. Key by basename because CLIP # result files share the input image's stem but use `.json` extension. image_metadata = {} - try: - source_metadata = self.source_dataset.read_media_metadata() - except (FileNotFoundError, MetadataException): - source_metadata = None + source_metadata = self.get_media_metadata() if source_metadata is not None: self.dataset.log("Found and loaded image metadata") from_dataset = source_metadata.from_dataset diff --git a/processors/machine_learning/google_vision_api.py b/processors/machine_learning/google_vision_api.py index 0253ce326..7bcdaff59 100644 --- a/processors/machine_learning/google_vision_api.py +++ b/processors/machine_learning/google_vision_api.py @@ -11,7 +11,7 @@ from common.lib.helpers import UserInput, convert_to_int from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility -from common.lib.exceptions import ProcessorInterruptedException, MetadataException +from common.lib.exceptions import ProcessorInterruptedException __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters"] @@ -119,11 +119,8 @@ def process(self): done = 0 # Load the metadata file so we can attach source post IDs to annotations. - try: - source_metadata = self.source_dataset.read_media_metadata() - img_metadata = source_metadata.filename_to_post_ids() - except (FileNotFoundError, MetadataException): - img_metadata = {} + source_metadata = self.get_media_metadata() + img_metadata = source_metadata.filename_to_post_ids() if source_metadata is not None else {} # Loop through images for image in self.source_dataset.iterate_items(): diff --git a/processors/machine_learning/llm_prompter.py b/processors/machine_learning/llm_prompter.py index 6cb4187e0..ea31e99ce 100644 --- a/processors/machine_learning/llm_prompter.py +++ b/processors/machine_learning/llm_prompter.py @@ -16,7 +16,7 @@ from datetime import datetime, timedelta from common.lib.item_mapping import MappedItem -from common.lib.exceptions import ProcessorInterruptedException, QueryParametersException, QueryNeedsExplicitConfirmationException, MetadataException +from common.lib.exceptions import ProcessorInterruptedException, QueryParametersException, QueryNeedsExplicitConfirmationException from common.lib.helpers import UserInput, nthify, andify, remove_nuls, flatten_dict from common.lib.llm.adapter import LLMAdapter from backend.lib.processor import BasicProcessor @@ -518,11 +518,12 @@ def process(self): # Load metadata to map filenames back to original post IDs for annotations. filename_to_post_ids = {} if save_annotations: - try: - filename_to_post_ids = self.source_dataset.read_media_metadata().filename_to_post_ids() - except (FileNotFoundError, MetadataException) as e: - self.dataset.log(f"Could not load .metadata.json for annotation mapping: {e}. " - f"Annotations will use filenames as item IDs.") + source_metadata = self.get_media_metadata() + if source_metadata is not None: + filename_to_post_ids = source_metadata.filename_to_post_ids() + else: + self.dataset.log("No usable .metadata.json found for annotation mapping. " + "Annotations will use filenames as item IDs.") for item in self.source_dataset.iterate_items(staging_area=staging_area, immediately_delete=True, get_annotations=False): diff --git a/processors/machine_learning/pix-plot.py b/processors/machine_learning/pix-plot.py index 9386629b6..f5b7fcc6e 100644 --- a/processors/machine_learning/pix-plot.py +++ b/processors/machine_learning/pix-plot.py @@ -9,7 +9,6 @@ from werkzeug.utils import secure_filename from common.lib.dmi_service_manager import DmiServiceManager, DsmOutOfMemory, DmiServiceManagerException -from common.lib.exceptions import MetadataException from common.lib.helpers import UserInput, ellipsiate, urls_in_text from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility @@ -263,9 +262,8 @@ def format_metadata(self, temp_path): """ # Get image data - try: - source_metadata = self.source_dataset.read_media_metadata() - except (FileNotFoundError, MetadataException): + source_metadata = self.get_media_metadata() + if source_metadata is None: return False top_dataset = self.dataset.top_parent() diff --git a/processors/machine_learning/text_from_image.py b/processors/machine_learning/text_from_image.py index b8cf96bcc..373823415 100644 --- a/processors/machine_learning/text_from_image.py +++ b/processors/machine_learning/text_from_image.py @@ -11,7 +11,7 @@ from common.lib.dmi_service_manager import DmiServiceManager, DsmOutOfMemory, DmiServiceManagerException, DsmConnectionError from common.lib.helpers import UserInput, hash_to_md5 from backend.lib.processor import BasicProcessor -from common.lib.exceptions import ProcessorInterruptedException, MetadataException +from common.lib.exceptions import ProcessorInterruptedException from common.lib.item_mapping import MappedItem from common.lib.compatibility import Compatibility @@ -196,10 +196,8 @@ def process(self): dmi_service_manager.process_results(output_dir) # Load the metadata from the archive - try: - source_metadata = self.source_dataset.read_media_metadata() - except (FileNotFoundError, MetadataException): - source_metadata = None + source_metadata = self.get_media_metadata() + if source_metadata is None: self.dataset.update_status("No metadata file found") # Check if we need to collect data for updating the original dataset diff --git a/processors/networks/image-network.py b/processors/networks/image-network.py index 0fc852062..fbcad73e8 100644 --- a/processors/networks/image-network.py +++ b/processors/networks/image-network.py @@ -11,7 +11,7 @@ __maintainer__ = "Stijn Peeters" __email__ = "4cat@oilab.eu" -from common.lib.exceptions import ProcessorInterruptedException, MetadataException +from common.lib.exceptions import ProcessorInterruptedException from common.lib.user_input import UserInput from common.lib.compatibility import Compatibility @@ -113,9 +113,8 @@ def process(self): hash_type = self.parameters.get("deduplicate") hashed = 0 - try: - metadata = self.source_dataset.read_media_metadata() - except (FileNotFoundError, MetadataException): + metadata = self.get_media_metadata() + if metadata is None: return self.dataset.finish_with_error("No valid metadata found in image archive - this processor can only " "be run on sets of images sourced from another 4CAT dataset.") diff --git a/processors/visualisation/image_category_wall.py b/processors/visualisation/image_category_wall.py index 29a4276af..429e1404d 100644 --- a/processors/visualisation/image_category_wall.py +++ b/processors/visualisation/image_category_wall.py @@ -15,7 +15,7 @@ from common.lib.helpers import UserInput, convert_to_int, get_4cat_canvas from backend.lib.processor import BasicProcessor -from common.lib.exceptions import ProcessorInterruptedException, MetadataException +from common.lib.exceptions import ProcessorInterruptedException from common.lib.compatibility import Compatibility __author__ = "Dale Wahl" @@ -194,9 +194,8 @@ def process(self): filename_map = {filename.stem: [filename] for filename in staging_area.iterdir()} else: # Use image metadata to map post IDs to filenames - try: - metadata = image_dataset.read_media_metadata() - except (FileNotFoundError, MetadataException): + metadata = self.get_media_metadata(image_dataset) + if metadata is None: self.dataset.finish_with_error("No metadata file found") return filename_map = {} diff --git a/processors/visualisation/image_wall_w_text.py b/processors/visualisation/image_wall_w_text.py index 2522acd2d..9586b2303 100644 --- a/processors/visualisation/image_wall_w_text.py +++ b/processors/visualisation/image_wall_w_text.py @@ -16,7 +16,7 @@ from common.lib.helpers import UserInput, convert_to_int, get_4cat_canvas from backend.lib.processor import BasicProcessor -from common.lib.exceptions import ProcessorInterruptedException, MetadataException +from common.lib.exceptions import ProcessorInterruptedException from common.lib.compatibility import Compatibility __author__ = "Dale Wahl" @@ -160,9 +160,8 @@ def process(self): if text_dataset.type in ImageTextWallGenerator.combined_dataset: # For datasets with both images and text (Stable Diffusion output), # the prompt is stored as item.extra.prompt in the archive metadata. - try: - metadata = text_dataset.read_media_metadata() - except (FileNotFoundError, MetadataException): + metadata = self.get_media_metadata(text_dataset) + if metadata is None: self.dataset.finish_with_error("No metadata file found") return for filename, item in metadata.iter_entries(): diff --git a/processors/visualisation/video_hasher.py b/processors/visualisation/video_hasher.py index d4031ba7a..7221d6a29 100644 --- a/processors/visualisation/video_hasher.py +++ b/processors/visualisation/video_hasher.py @@ -16,7 +16,7 @@ from backend.lib.processor import BasicProcessor from backend.lib.preset import ProcessorAdvancedPreset from common.lib.compatibility import Compatibility, is_executable -from common.lib.exceptions import ProcessorInterruptedException, ProcessorException, MetadataException +from common.lib.exceptions import ProcessorInterruptedException, ProcessorException from common.lib.user_input import UserInput __author__ = "Dale Wahl" @@ -185,10 +185,7 @@ def process(self): output_dir = self.dataset.get_staging_area() video_hashes = {} - try: - video_metadata = self.source_dataset.read_media_metadata() - except (FileNotFoundError, MetadataException): - video_metadata = None + video_metadata = self.get_media_metadata() total_possible_videos = max((min(self.source_dataset.num_rows, max_videos) if max_videos != 0 else self.source_dataset.num_rows), 1) processed_videos = 0 diff --git a/processors/visualisation/video_scene_identifier.py b/processors/visualisation/video_scene_identifier.py index 0b95f0425..ba09e0620 100644 --- a/processors/visualisation/video_scene_identifier.py +++ b/processors/visualisation/video_scene_identifier.py @@ -10,7 +10,7 @@ from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility -from common.lib.exceptions import ProcessorInterruptedException, ProcessorException, MetadataException +from common.lib.exceptions import ProcessorInterruptedException, ProcessorException from common.lib.user_input import UserInput __author__ = "Dale Wahl" @@ -216,10 +216,7 @@ def process(self): self.dataset.update_status("Detecting video scenes") skipped = 0 processed_videos = 0 - try: - video_metadata = self.source_dataset.read_media_metadata() - except (FileNotFoundError, MetadataException): - video_metadata = None + video_metadata = self.get_media_metadata() collected_scenes = {} for original_video in self.source_dataset.iterate_items(self, immediately_delete=False): if self.interrupted: diff --git a/processors/visualisation/video_timelines.py b/processors/visualisation/video_timelines.py index 3df301292..f0c2d5fe8 100644 --- a/processors/visualisation/video_timelines.py +++ b/processors/visualisation/video_timelines.py @@ -14,7 +14,7 @@ from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility -from common.lib.exceptions import DataSetNotFoundException, ProcessorInterruptedException, MetadataException +from common.lib.exceptions import DataSetNotFoundException, ProcessorInterruptedException from common.lib.user_input import UserInput from common.lib.helpers import get_4cat_canvas from common.lib.dataset import DataSet @@ -67,10 +67,7 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: } def process(self): - try: - metadata = self.source_dataset.read_media_metadata() - except (FileNotFoundError, MetadataException): - metadata = None + metadata = self.get_media_metadata() base_height = self.parameters.get("height", 100) fontsize = 12 diff --git a/tests/test_get_media_metadata.py b/tests/test_get_media_metadata.py new file mode 100644 index 000000000..ab507c421 --- /dev/null +++ b/tests/test_get_media_metadata.py @@ -0,0 +1,35 @@ +""" +Tests for `BasicProcessor.get_media_metadata`, which reads the media metadata +of a dataset or carries on without it +""" +from unittest.mock import MagicMock + +from backend.lib.processor import BasicProcessor +from common.lib.exceptions import MetadataException + + +def test_metadata_is_read_from_the_source_dataset(): + processor = MagicMock() + metadata = BasicProcessor.get_media_metadata(processor) + + assert metadata is processor.source_dataset.read_media_metadata.return_value + + +def test_missing_metadata_file_is_not_logged(): + """Uploaded media has no metadata file; that is normal""" + processor = MagicMock() + processor.source_dataset.read_media_metadata.side_effect = FileNotFoundError("No metadata file") + + assert BasicProcessor.get_media_metadata(processor) is None + processor.dataset.log.assert_not_called() + + +def test_unusable_metadata_file_is_logged_with_reason(): + processor = MagicMock() + other_dataset = MagicMock(key="abc123") + other_dataset.read_media_metadata.side_effect = MetadataException("Unsupported metadata schema_version 2") + + assert BasicProcessor.get_media_metadata(processor, other_dataset) is None + message = processor.dataset.log.call_args.args[0] + assert "abc123" in message + assert "schema_version 2" in message From 0cfb2731d28e63f68d3208eff7aecbe30dc9330e Mon Sep 17 00:00:00 2001 From: Dale Wahl Date: Wed, 14 Oct 2026 11:34:39 +0200 Subject: [PATCH 09/10] unique_images: track duplicates updates to use other_urls from downloader and update metadata record --- processors/filtering/unique_images.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/processors/filtering/unique_images.py b/processors/filtering/unique_images.py index ef28716c4..dd7651b04 100644 --- a/processors/filtering/unique_images.py +++ b/processors/filtering/unique_images.py @@ -69,6 +69,7 @@ def process(self): """ seen_hashes = set() hash_map = {} + duplicate_of = {} hash_type = self.parameters.get("hash-type") dupes = 0 processed = 0 @@ -93,28 +94,44 @@ def process(self): hash_map[image_hash] = image.file.name else: self.dataset.log(f"{image.file.name} is a duplicate of {hash_map[image_hash]} - skipping") + duplicate_of[image.file.name] = hash_map[image_hash] dupes += 1 source_metadata = self.get_media_metadata() new_metadata = self.dataset.new_media_metadata( processor_type=self.type, - from_dataset=(source_metadata.from_dataset if source_metadata else self.source_dataset.key), + from_dataset=(source_metadata.from_dataset if source_metadata is not None else self.source_dataset.key), ) inverse_hashmap = {v: k for k, v in hash_map.items()} if source_metadata is not None: - for filename, item in source_metadata.iter_entries(): - if filename not in inverse_hashmap: + # each image that is kept stands in for its duplicates too, so the + # posts and URLs of a removed duplicate are added to the kept image + # and no post loses its image + kept_file = {filename: filename for filename in inverse_hashmap} + kept_file.update(duplicate_of) + + # kept images first, so each keeps its own URL; a duplicate's URL + # is then listed in the kept image's extra["other_urls"] + entries = sorted(source_metadata.iter_entries(), key=lambda entry: entry[0] not in inverse_hashmap) + for filename, item in entries: + if filename not in kept_file: continue extra = dict(item.get("extra") or {}) - extra["hash"] = inverse_hashmap[filename] + other_urls = extra.pop("other_urls", []) + extra["hash"] = inverse_hashmap[kept_file[filename]] extra["hash_type"] = hash_type new_metadata.add_item( - filename, + kept_file[filename], post_ids=item.get("post_ids", []), url=item.get("url"), extra=extra, + merge=True, ) + # adding these one by one keeps the other URLs of both the + # kept image and its duplicates + for other_url in other_urls: + new_metadata.add_item(kept_file[filename], post_ids=[], url=other_url, merge=True) else: for h, filename in hash_map.items(): new_metadata.add_item( From 5a30752cccd0563eed9a57e2db5dc21b6a9620a3 Mon Sep 17 00:00:00 2001 From: Dale Wahl Date: Wed, 14 Oct 2026 14:25:52 +0200 Subject: [PATCH 10/10] add default map_metadata functions for processors that do not declare one. --- common/lib/archive_metadata.py | 59 ++++++++++++++++++++++++++ processors/README.md | 2 +- processors/conversion/view_metadata.py | 35 ++++++++------- 3 files changed, 80 insertions(+), 16 deletions(-) diff --git a/common/lib/archive_metadata.py b/common/lib/archive_metadata.py index 9dca5929a..984091821 100644 --- a/common/lib/archive_metadata.py +++ b/common/lib/archive_metadata.py @@ -630,6 +630,65 @@ def __len__(self) -> int: def __contains__(self, filename) -> bool: return filename in self.items + # -- rows, e.g. for the 'View metadata' processor -- + + @classmethod + def map_metadata(cls, filename: str, item: dict) -> Iterator[dict]: + """ + Yield a row for an `items[filename]` entry. + + The processor that made an archive can define its own `map_metadata`, + with columns that suit its data; this is used for archives whose + processor does not have a custom implementation. Same columns as the + downloaders use, plus the simple values from `extra`. + """ + row = { + "url": item.get("url") or "", + "number_of_posts": len(item.get("post_ids", [])), + "post_ids": ", ".join(item.get("post_ids", [])), + "filename": filename, + "download_successful": True, + } + yield cls._add_simple_values(row, item.get("extra")) + + @classmethod + def map_failure_metadata(cls, failure: dict) -> Iterator[dict]: + """ + Yield a row for a `failures[]` entry; see `map_metadata()`. + """ + row = { + "url": failure.get("url") or "", + "number_of_posts": len(failure.get("post_ids", [])), + "post_ids": ", ".join(failure.get("post_ids", [])), + "filename": "", + "download_successful": False, + "reason": failure.get("reason", ""), + "reason_description": failure.get("reason_description", ""), + } + yield cls._add_simple_values(row, failure.get("extra")) + + @staticmethod + def _add_simple_values(row: dict, extra: Optional[dict]) -> dict: + """ + Add the values of `extra` that fit in one cell to a row. + + Text, numbers and true/false are added as they are, and lists of + those are joined with commas. Nested data, such as a full yt-dlp + dump, is left out. A key the row already has keeps its value. + """ + def is_simple(value): + return value is None or isinstance(value, (str, int, float, bool)) + + for key, value in (extra or {}).items(): + if key in row: + continue + if is_simple(value): + row[key] = value + elif isinstance(value, list) and all(is_simple(part) for part in value): + row[key] = ", ".join(str(part) for part in value if part is not None) + + return row + # -- output -- def to_dict(self) -> dict: diff --git a/processors/README.md b/processors/README.md index bdac27bd6..e13d35da0 100644 --- a/processors/README.md +++ b/processors/README.md @@ -4,7 +4,7 @@ Processors are modules for 4CAT that take a dataset as their input and produce another dataset as their output. They are self-contained Python files using a common API. Refer to the -source code (`backend/abstract/processor.py`) and the +source code (`backend/lib/processor.py`) and the [Wiki](https://github.com/digitalmethodsinitiative/4cat/wiki/How-to-make-a-processor) for more information on how to make a processor. Also see our [list of processors on the Wiki](https://github.com/digitalmethodsinitiative/4cat/wiki/Available-processors) for an overview. diff --git a/processors/conversion/view_metadata.py b/processors/conversion/view_metadata.py index 0aa34ba99..650039ff6 100644 --- a/processors/conversion/view_metadata.py +++ b/processors/conversion/view_metadata.py @@ -1,11 +1,13 @@ """ View .metadata.json -Designed to work with any processor that has a 'map_metadata' method +Uses the 'map_metadata' method of the processor that made the archive, if it +has one, and otherwise shows the metadata as it is stored """ import csv from backend.lib.processor import BasicProcessor +from common.lib.archive_metadata import MediaArchiveMetadata from common.lib.compatibility import Compatibility from common.lib.dataset import DataSet from common.lib.exceptions import DataSetException, MetadataException, ProcessorInterruptedException @@ -78,9 +80,9 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: def process(self): """ Read .metadata.json from the parent archive and reformat as CSV using the - parent producer's `map_metadata` / `map_failure_metadata` hooks, adding - the data of the dataset the media was downloaded from where that dataset - can still be found. + parent producer's `map_metadata` / `map_failure_metadata` hooks (or the + default ones of `MediaArchiveMetadata`), adding the data of the dataset + the media was downloaded from where that dataset can still be found. """ self.dataset.update_status("Collecting .metadata.json file") try: @@ -93,12 +95,12 @@ def process(self): return producer = self.source_dataset.get_own_processor() - if producer is None or not hasattr(producer, "map_metadata"): - if producer is not None: - self.log.warning(f"Metadata formatter processor cannot run on {producer.type}; map_metadata method not implemented") - self.dataset.finish_with_error("Cannot reformat metadata for this dataset") - return - self.dataset.log(f"Collecting metadata created by {producer.type}") + if producer is not None and hasattr(producer, "map_metadata"): + self.dataset.log(f"Collecting metadata created by {producer.type}") + else: + # e.g. an archive of unique images, whose processor does not say how + # to show its metadata + self.dataset.log("Collecting metadata as it is stored in the archive") # read the metadata file into rows, and note which items they refer to media_rows, wanted_ids = self.collect_media_rows(metadata, producer) @@ -117,17 +119,21 @@ def collect_media_rows(self, metadata, producer): Turn the metadata file into rows The processor that made the archive decides what the rows look like, via - its `map_metadata` and `map_failure_metadata` methods. Their columns are + its `map_metadata` and `map_failure_metadata` methods. Where it has none, + the default ones of `MediaArchiveMetadata` are used. Their columns are prefixed here so that they cannot collide with those of the dataset the media came from. :param MediaArchiveMetadata metadata: The archive's metadata - :param producer: Processor class that made the archive + :param producer: Processor class that made the archive, or `None` if + it is not known :return tuple: A list of (row, item IDs) tuples, and the set of all item IDs those rows refer to """ media_rows = [] wanted_ids = set() + map_item = getattr(producer, "map_metadata", MediaArchiveMetadata.map_metadata) + map_failure = getattr(producer, "map_failure_metadata", MediaArchiveMetadata.map_failure_metadata) def add(rows, entry): # a download that produced no file still records the items it was @@ -143,10 +149,9 @@ def add(rows, entry): for filename, item in metadata.iter_entries(): if self.interrupted: raise ProcessorInterruptedException("Interrupted while reading metadata file") - add(producer.map_metadata(filename, item), item) + add(map_item(filename, item), item) - map_failure = getattr(producer, "map_failure_metadata", None) - if self.parameters.get("include_failed", False) and map_failure is not None: + if self.parameters.get("include_failed", False): for failure in metadata.iter_failures(): if self.interrupted: raise ProcessorInterruptedException("Interrupted while reading metadata file")