optimized metadata processing during collection upload (13ba107e) · Commits · trapper-project / trapper

trapper/trapper-project/trapper/apps/media_classification/forms.py

+27 −0

Original line number	Diff line number	Diff line
		@@ -85,6 +85,33 @@ class ProjectRoleForm(BaseCrispyModelForm):
		else:
		return bool(self.initial or changed_data)

		def clean_user(self):
		"""
		Validate that a user can't have multiple roles in the same project.
		Error is displayed on the user field in the formset row.
		"""
		user = self.cleaned_data.get("user")
		if (
		user
		and hasattr(self, "instance")
		and hasattr(self.instance, "classification_project")
		):
		existing_role = (
		ClassificationProjectRole.objects.filter(
		user=user,
		classification_project=self.instance.classification_project,
		)
		.exclude(pk=self.instance.pk)
		.exists()
		)
		if existing_role:
		raise forms.ValidationError(
		_("User %(user)s already has a role in this project."),
		params={"user": user},
		code="duplicate_role",
		)
		return user


		class ProjectRoleInline(InlineFormSetFactory):
		"""

trapper/trapper-project/trapper/apps/storage/collection_upload.py

+19 −19

Original line number	Diff line number	Diff line
		@@ -6,7 +6,6 @@ uploading collections of resources.

		from collections import defaultdict
		from datetime import datetime
		from dateutil import parser
		from io import BytesIO
		import os
		import zipfile
		@@ -49,7 +48,7 @@ class CollectionProcessor:
		owner=None,
		read_definition_file=True,
		celery_enabled=False,
		extract_metadata=True,
		extract_metadata=False,
		trigger_ai_pipeline=True,
		):
		"""
		@@ -64,7 +63,8 @@ class CollectionProcessor:
		celery_enabled (bool, optional): If True, enables Celery tasks for processing.
		Defaults to False.
		extract_metadata (bool, optional): If True, extracts metadata from resources.
		Defaults to True.
		Defaults to False as it is resource-intensive and we expect that all required
		metadata is already present in the YAML definition file.
		trigger_ai_pipeline (bool, optional): If True, triggers AI pipeline processing.
		Defaults to True.
		"""
		@@ -300,23 +300,32 @@ class CollectionProcessor:
		deployment=deployment,
		search_name_vector=SearchVector(Value(resource_def["name"])),
		file_checksum_hash=file_checksum_hash,
		file_size=resource_def.get("file_size", bin_data_file.getbuffer().nbytes),
		mime_type=resource_def.get("mime_type"),
		file_width=resource_def.get("file_width"),
		file_height=resource_def.get("file_height"),
		file_fps=resource_def.get("file_fps"),
		file_duration=resource_def.get("file_duration"),
		)

		suf_file = SimpleUploadedFile(
		resource_def.get("file"),
		bin_data_file.read(),
		content_type=file_ext,
		)
		resource.file.save(resource_def["file"], suf_file, save=False)
		resource.update_resource_type()

		# update basic metadata
		resource.update_metadata()

		# Define date_recorded fallback; must be a datetime object
		# Parse date_recorded from resource definition
		try:
		date_recorded = parser.parse(resource_def["date_recorded"])
		except (ValueError, TypeError):
		# If parsing fails, use current time as fallback
		date_recorded = datetime.strptime(
		resource_def["date_recorded"], "%Y-%m-%dT%H:%M:%S%z"
		)
		resource.date_recorded = date_recorded
		except (ValueError, TypeError, KeyError):
		# If parsing fails or key is missing, use current time as fallback
		date_recorded = datetime.now(utc)
		resource.date_recorded = date_recorded

		if self.metadata_extractor is not None:
		# Update Resource metadata using metadata extractor (based on exiftool)
		@@ -335,15 +344,6 @@ class CollectionProcessor:
		deployment, results.get("metadata", {})
		)

		else:
		# Ensure timezone awareness before converting to UTC
		if date_recorded.tzinfo is None:
		# Assume the date is in deployment's local timezone
		date_recorded = deployment.location.timezone.localize(date_recorded)
		date_recorded = date_recorded.astimezone(utc)
		# only name and date_recorded are supported in the current YAML schema
		resource.date_recorded = date_recorded

		return (resource, None)

		@transaction.atomic

trapper/trapper-project/trapper/apps/storage/models.py

+2 −1

Original line number	Diff line number	Diff line
		@@ -323,12 +323,13 @@ class Resource(AccessModelMixin, models.Model):
		self.date_recorded, self.timezone, self.ignore_DST
		)

		def save(self, **kwargs):
		def save(self, update_metadata=True, **kwargs):
		"""
		On save update Resource metadata and name search vector
		"""

		# first update mime_type and resource_type
		if update_metadata:
		self.update_metadata()

		if not self.search_name_vector:

trapper/trapper-project/trapper/apps/storage/schema/collection_schema.yaml

+23 −2

Original line number	Diff line number	Diff line
		@@ -50,12 +50,33 @@ mapping:
		"file":
		type: str
		required: true
		"date_recorded":
		type: str
		required: true
		"mime_type":
		type: str
		required: false
		"file_width":
		type: int
		required: false
		"file_height":
		type: int
		required: false
		"file_size":
		type: int
		required: false
		"file_fps":
		type: float
		required: false
		"file_duration":
		type: float
		required: false
		"extra_file":
		type: str
		required: false
		"date_recorded":
		"extra_mime_type":
		type: str
		required: true
		required: false

trapper/trapper-project/trapper/apps/storage/tasks.py

+7 −1

Original line number	Diff line number	Diff line
		@@ -57,7 +57,12 @@ def celery_update_thumbnails(resources):

		@shared_task(serializer="pickle")
		def celery_process_collection_upload(
		definition_file, archive_file, owner, remove_zip, trigger_ai_pipeline=False
		definition_file,
		archive_file,
		owner,
		remove_zip,
		extract_metadata=False,
		trigger_ai_pipeline=False,
		):
		"""
		Celery task that creates collections using provided (uploaded)
		@@ -88,6 +93,7 @@ def celery_process_collection_upload(
		archive_file=archive_file,
		owner=owner,
		celery_enabled=settings.CELERY_ENABLED,
		extract_metadata=extract_metadata,
		trigger_ai_pipeline=trigger_ai_pipeline,
		)
		processor.create()