Follow on to Metadata Class

backend/lib/processor.py

@@ -20,7 +20,7 @@
 from common.lib.fourcat_module import FourcatModule
 from common.lib.helpers import get_software_commit, remove_nuls, send_email, hash_to_md5, looks_like_email_address
 from common.lib.exceptions import (WorkerInterruptedException, ProcessorInterruptedException, ProcessorException,
-                                   DataSetException, MapItemException, AnnotationException)
+                                   DataSetException, MapItemException, AnnotationException, MetadataException)
 from common.config_manager import ConfigWrapper
 from common.lib.user import User
 
@@ -610,6 +610,31 @@ def unpack_archive_contents(self, path, staging_area=None):
 
         return staging_area
 
+    def get_media_metadata(self, dataset=None):
+        """
+        Read the media metadata of a dataset, if it can be used
+
+        A media archive without a metadata file is normal (e.g. uploaded
+        media), so that is not logged. A metadata file that cannot be read
+        (e.g. it is damaged) is logged in this processor's dataset log, so
+        the researcher can see why the results are not linked to items.
+
+        :param DataSet dataset:  Dataset to read the metadata of. Defaults to
+          the source dataset.
+        :return MediaArchiveMetadata|None:  The metadata, or `None` if there
+          is none that can be used
+        """
+        if dataset is None:
+            dataset = self.source_dataset
+
+        try:
+            return dataset.read_media_metadata()
+        except FileNotFoundError:
+            return None
+        except MetadataException as e:
+            self.dataset.log(f"Could not use the metadata file of dataset {dataset.key}: {e}")
+            return None
+
     def extract_archived_file_by_name(self, filename, archive_path, staging_area=None):
         """
         Extract a file from an archive by name

common/lib/archive_metadata.py

@@ -26,6 +26,32 @@
 
 from common.lib.exceptions import MetadataException
 
+# Files 4CAT adds to an archive next to its media or other content. They are
+# not items of the dataset: metadata files, the list in which yt-dlp notes
+# which videos it downloaded, and log files (e.g. from ffmpeg)
+SUPPORT_FILENAMES = frozenset({
+	".metadata.json",
+	".token_metadata.json",
+	".model_metadata.json",
+	"video_archive",
+})
+
+
+def is_support_file(filename: str) -> bool:
+	"""
+	Check if a file in an archive is a support file rather than an item
+
+	Iterating through an archive dataset skips these files, unless asked not
+	to. Read a metadata file with its own class instead, e.g. via
+	`DataSet.read_media_metadata()`.
+
+	:param str filename:  File name, can include folders (e.g. a path inside
+	  a zip archive)
+	:return bool:
+	"""
+	name = filename.split("/")[-1]
+	return name in SUPPORT_FILENAMES or name.endswith(".log")
+
 
 class ArchiveMetadataFile:
 	"""
@@ -604,6 +630,65 @@ def __len__(self) -> int:
 	def __contains__(self, filename) -> bool:
 		return filename in self.items
 
+	# -- rows, e.g. for the 'View metadata' processor --
+
+	@classmethod
+	def map_metadata(cls, filename: str, item: dict) -> Iterator[dict]:
+		"""
+		Yield a row for an `items[filename]` entry.
+
+		The processor that made an archive can define its own `map_metadata`,
+		with columns that suit its data; this is used for archives whose
+		processor does not have a custom implementation. Same columns as the 
+		downloaders use, plus the simple values from `extra`.
+		"""
+		row = {
+			"url": item.get("url") or "",
+			"number_of_posts": len(item.get("post_ids", [])),
+			"post_ids": ", ".join(item.get("post_ids", [])),
+			"filename": filename,
+			"download_successful": True,
+		}
+		yield cls._add_simple_values(row, item.get("extra"))
+
+	@classmethod
+	def map_failure_metadata(cls, failure: dict) -> Iterator[dict]:
+		"""
+		Yield a row for a `failures[]` entry; see `map_metadata()`.
+		"""
+		row = {
+			"url": failure.get("url") or "",
+			"number_of_posts": len(failure.get("post_ids", [])),
+			"post_ids": ", ".join(failure.get("post_ids", [])),
+			"filename": "",
+			"download_successful": False,
+			"reason": failure.get("reason", ""),
+			"reason_description": failure.get("reason_description", ""),
+		}
+		yield cls._add_simple_values(row, failure.get("extra"))
+
+	@staticmethod
+	def _add_simple_values(row: dict, extra: Optional[dict]) -> dict:
+		"""
+		Add the values of `extra` that fit in one cell to a row.
+
+		Text, numbers and true/false are added as they are, and lists of
+		those are joined with commas. Nested data, such as a full yt-dlp
+		dump, is left out. A key the row already has keeps its value.
+		"""
+		def is_simple(value):
+			return value is None or isinstance(value, (str, int, float, bool))
+
+		for key, value in (extra or {}).items():
+			if key in row:
+				continue
+			if is_simple(value):
+				row[key] = value
+			elif isinstance(value, list) and all(is_simple(part) for part in value):
+				row[key] = ", ".join(str(part) for part in value if part is not None)
+
+		return row
+
 	# -- output --
 
 	def to_dict(self) -> dict:

common/lib/dataset.py

@@ -16,6 +16,7 @@
 from natsort import natsorted
 
 from common.lib.annotation import Annotation
+from common.lib.archive_metadata import is_support_file
 from common.lib.job import Job, JobNotFoundException
 
 from common.lib.helpers import get_software_commit, NullAwareTextIOWrapper, convert_to_int, get_software_version, call_api, hash_to_md5, convert_to_float
@@ -393,6 +394,7 @@ def _iterate_archive_contents(
             staging_area=None,
             immediately_delete=True,
             filename_filter=None,
+            include_support_files=False,
             processor=None,
             offset=0,
             *args, **kwargs
@@ -406,17 +408,21 @@ def _iterate_archive_contents(
 
         Files are temporarily unzipped and deleted after use.
 
+        Support files (metadata, logs, and the like; see `is_support_file()`)
+        are not items of the dataset and are skipped, unless
+        `include_support_files` is set or they are named in `filename_filter`.
+
         :param Path staging_area:  Where to store the files while they're
           being worked with. If omitted, a temporary folder is created and
           marked for deletion after all files have been yielded
         :param bool immediately_delete:  Temporary files are removed after
           yielding; False keeps files until the staging_area is removed
         :param list filename_filter:  Whitelist of filenames to iterate. If
           empty, do not filter
+        :param bool include_support_files:  Also yield support files
         :param BasicProcessor processor:  A reference to the processor
           iterating the dataset.
-        :param int offset:  Skip this many files before yielding (warning: may
-          skip a metadata file too!)
+        :param int offset:  Skip this many files before yielding
         :return:  An iterator with a dictionary for each file, containing an
           `id`, a `path`, and all attributes of the `ZipInfo` object as keys
         """
@@ -460,6 +466,10 @@ def metadata_priority_sort(file):
                 if filename_filter and archived_file.filename not in filename_filter:
                     continue
 
+                # a support file named in filename_filter is still yielded
+                if not include_support_files and not filename_filter and is_support_file(archived_file.filename):
+                    continue
+
                 if archived_file.is_dir():
                     # do not yield folders - we'll get to the files in them
                     continue
@@ -556,6 +566,10 @@ def iterate_items(
         :param list filename_filter:  Only used when iterating a file archive.
           Whitelist of filenames to iterate, others are skipped. If empty, do
           not filter.
+        :param bool include_support_files:  Only used when iterating a file
+          archive. Defaults to `False`, which skips files that are not items
+          of the dataset, such as metadata and log files. Read metadata with
+          e.g. `read_media_metadata()` instead.
         :return generator:  A generator that yields DatasetItems
         """
         unmapped_items = 0

datasources/media_import/import_media.py

@@ -6,6 +6,7 @@
 from io import BytesIO
 
 from backend.lib.processor import BasicProcessor
+from common.lib.archive_metadata import is_support_file
 from common.lib.exceptions import QueryParametersException, QueryNeedsExplicitConfirmationException
 from common.lib.user_input import UserInput
 from common.lib.helpers import andify
@@ -86,7 +87,7 @@ def validate_query(query, request, config):
                 # as they are, system files and folders are left out later
                 files = [f for f in files if not (
                         f["filename"].split("/")[-1].startswith(".")
-                        or SearchMedia.is_support_file(f["filename"])
+                        or is_support_file(f["filename"])
                         or f["filename"].split("/")[-1].startswith("__MACOSX")
                         or f["filename"].endswith(".DS_Store")
                         or f["filename"].endswith("/")  # sub-directory
@@ -119,7 +120,7 @@ def validate_query(query, request, config):
             # Check file types to ensure all are same type of media
             for file in uploaded_files:
                 # Support files are kept as they are, without a type check
-                if SearchMedia.is_support_file(file.filename):
+                if is_support_file(file.filename):
                     continue
 
                 # Warn on SVG files
@@ -200,12 +201,12 @@ def after_create(query, dataset, request):
 
                             guessed_file_mime_type = mimetypes.guess_type(inner_file.filename)
                             file_media_type = guessed_file_mime_type[0].split('/')[0] if guessed_file_mime_type[0] else None
-                            is_support_file = SearchMedia.is_support_file(inner_file.filename)
+                            support_file = is_support_file(inner_file.filename)
 
                             # keep media files and support files; skip macOS
                             # clutter and anything else
                             clean_file_name = inner_file.filename.split("/")[-1]
-                            if (not is_support_file and file_media_type not in SearchMedia.accepted_file_types) \
+                            if (not support_file and file_media_type not in SearchMedia.accepted_file_types) \
                               or clean_file_name.startswith("__MACOSX") \
                               or inner_file.filename.startswith("__MACOSX"):
                                 print(f"skipping {clean_file_name} ({guessed_file_mime_type})")
@@ -214,14 +215,14 @@ def after_create(query, dataset, request):
 
                             # support files are not media, so they neither set
                             # the dataset's media type nor count as files
-                            if not is_support_file:
+                            if not support_file:
                                 mime_type = file_media_type
 
                             # save inner file from the uploaded zip archive to the new zip with all files
                             new_filename = SearchMedia.get_safe_filename(inner_file.filename, new_zip_archive)
                             new_zip_archive.writestr(new_filename, inner_zip_archive.read(inner_file))
 
-                            if not is_support_file:
+                            if not support_file:
                                 saved_files += 1
                     continue
 
@@ -234,7 +235,7 @@ def after_create(query, dataset, request):
                             break
                         dest_file.write(chunk)
 
-                if not SearchMedia.is_support_file(new_filename):
+                if not is_support_file(new_filename):
                     saved_files += 1
 
         # update the number of files in the dataset
@@ -263,25 +264,6 @@ def process(self):
         self.dataset.update_status(f"Uploaded {self.parameters.get('num_files')} files of type {self.parameters.get('media_type')}{'' if svg_warning == 0 else f' ({svg_warning} SVG files; see log)'}", is_final=True)
         self.dataset.finish(self.parameters.get("num_files"))
 
-    @staticmethod
-    def is_support_file(filename):
-        """
-        Check if a file is a non-media file that belongs with the media
-
-        Media archives made by 4CAT hold more than media: a .metadata.json
-        file that links each file to the item it was downloaded for, a
-        'video_archive' file in which yt-dlp notes which videos it downloaded,
-        and log files (e.g. from extracting video frames). When such an
-        archive is uploaded again, these files are kept in the dataset, but
-        they are not media, so they are not type-checked or counted as files.
-
-        :param str filename:  File name, can include folders (e.g. a path
-          inside a zip archive)
-        :return bool:
-        """
-        filename = filename.split("/")[-1]
-        return filename in (".metadata.json", "video_archive") or filename.endswith(".log")
-
     @staticmethod
     def get_safe_filename(filename, zip_archive=None):
         new_filename = SearchMedia.disallowed_characters.sub("", filename)

processors/README.md

@@ -4,7 +4,7 @@ Processors are modules for 4CAT that take a dataset as their input and
 produce another dataset as their output.
 
 They are self-contained Python files using a common API. Refer to the 
-source code (`backend/abstract/processor.py`) and the 
+source code (`backend/lib/processor.py`) and the 
 [Wiki](https://github.com/digitalmethodsinitiative/4cat/wiki/How-to-make-a-processor)
 for more information on how to make a processor.
 Also see our [list of processors on the Wiki](https://github.com/digitalmethodsinitiative/4cat/wiki/Available-processors) for an overview.

processors/audio/audio_extractor.py

@@ -6,12 +6,12 @@
 """
 import shutil
 import zipfile
-from pathlib import Path
 import oslex
 
 from backend.lib.processor import BasicProcessor
+from common.lib.archive_metadata import is_support_file
 from common.lib.compatibility import Compatibility, is_executable
-from common.lib.exceptions import ProcessorInterruptedException, MetadataException
+from common.lib.exceptions import ProcessorInterruptedException
 
 __author__ = "Dale Wahl"
 __credits__ = ["Dale Wahl"]
@@ -66,27 +66,23 @@ def process(self):
         # Prepare staging areas for videos and video tracking
         output_dir = self.dataset.get_staging_area()
 
-        # Estimate how many actual video files we will attempt, excluding archive metadata.
+        # Estimate how many actual video files we will attempt, excluding metadata and the like.
         total_possible_videos = self.source_dataset.num_rows
         source_archive = self.source_dataset.get_results_path()
         if source_archive.exists() and source_archive.suffix.lower() == ".zip":
             with zipfile.ZipFile(source_archive, "r") as archive_file:
                 total_possible_videos = sum(
                     1
                     for archived_file in archive_file.infolist()
-                    if not archived_file.is_dir()
-                    and Path(archived_file.filename).name not in (".metadata.json", "video_archive")
+                    if not archived_file.is_dir() and not is_support_file(archived_file.filename)
                 )
 
         if max_files != 0:
             total_possible_videos = min(total_possible_videos, max_files)
 
         # Read the source video archive's metadata so each extracted audio
         # file can carry its video's provenance (source posts, URL).
-        try:
-            source_metadata = self.source_dataset.read_media_metadata()
-        except (FileNotFoundError, MetadataException):
-            source_metadata = None
+        source_metadata = self.get_media_metadata()
 
         # Build our own metadata describing the audio files we produce,
         # rather than passing the (video-keyed) source metadata through.
@@ -103,14 +99,6 @@ def process(self):
             if self.interrupted:
                 raise ProcessorInterruptedException("Interrupted while determining image wall order")
 
-            # the source archive's metadata describes videos; we write our
-            # own (audio-keyed) metadata below, so skip the original
-            if item.file.name == '.metadata.json':
-                continue
-            elif item.file.name == "video_archive":
-                # yt-dlp's list of what it downloaded; not a video
-                continue
-
             if max_files != 0 and processed_videos >= max_files:
                 break
 

processors/conversion/hash_images.py

@@ -6,7 +6,7 @@
 from PIL import UnidentifiedImageError
 
 from backend.lib.processor import BasicProcessor
-from common.lib.exceptions import ProcessorInterruptedException, MetadataException
+from common.lib.exceptions import ProcessorInterruptedException
 from common.lib.helpers import UserInput, hash_image, stringify_hash
 from common.lib.compatibility import Compatibility
 from processors.metrics.group_hashes import HashGrouper
@@ -110,10 +110,7 @@ def process(self):
         group_by = self.parameters.get("group-by", False)
         similarity_pct = float(self.parameters.get("similarity-threshold", 4))
 
-        try:
-            source_metadata = self.source_dataset.read_media_metadata()
-        except (FileNotFoundError, MetadataException):
-            source_metadata = None
+        source_metadata = self.get_media_metadata()
 
         image_data_by_filename = {}
         if source_metadata is not None:
@@ -144,8 +141,6 @@ def process(self):
             if processed % 100 == 0:
                 self.dataset.update_status(f"Processed {processed:,} of {self.source_dataset.num_rows:,} images")
 
-            if item.file.name == ".metadata.json":
-                continue
             try:
                 image_hash = hash_image(item.file, hash_type, hash_size, as_string=False)
             except FileNotFoundError as e: