hoelee/tubesync
1
0
Fork 0
mirror of https://github.com/meeb/tubesync.git synced 2025-06-23 13:36:35 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Reduce the amount of time the database is locked

Browse Source
This commit is contained in:
tcely 2025-03-17 09:50:15 -04:00 committed by GitHub
parent cda93f0df2
commit deed3a3393
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 45 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
tubesync/sync/tasks.py
View File

@ -181,7 +181,6 @@ def cleanup_removed_media(source, videos):
@background(schedule=300, remove_existing_tasks=True) @background(schedule=300, remove_existing_tasks=True)
@atomic(durable=True)
def index_source_task(source_id): def index_source_task(source_id):
''' '''
Indexes media available from a Source object. Indexes media available from a Source object.
@ -206,51 +205,54 @@ def index_source_task(source_id):
source.save() source.save()
log.info(f'Found {len(videos)} media items for source: {source}') log.info(f'Found {len(videos)} media items for source: {source}')
fields = lambda f, m: m.get_metadata_field(f) fields = lambda f, m: m.get_metadata_field(f)
for video in videos: with atomic(durable=True):
# Create or update each video as a Media object for video in videos:
key = video.get(source.key_field, None) # Create or update each video as a Media object
if not key: key = video.get(source.key_field, None)
# Video has no unique key (ID), it can't be indexed if not key:
continue # Video has no unique key (ID), it can't be indexed
try: continue
media = Media.objects.get(key=key, source=source) try:
except Media.DoesNotExist: media = Media.objects.get(key=key, source=source)
media = Media(key=key) except Media.DoesNotExist:
media.source = source media = Media(key=key)
media.duration = float(video.get(fields('duration', media), None) or 0) or None media.source = source
media.title = str(video.get(fields('title', media), ''))[:200] media.duration = float(video.get(fields('duration', media), None) or 0) or None
timestamp = video.get(fields('timestamp', media), None) media.title = str(video.get(fields('title', media), ''))[:200]
published_dt = media.metadata_published(timestamp) timestamp = video.get(fields('timestamp', media), None)
if published_dt is not None: published_dt = media.metadata_published(timestamp)
media.published = published_dt if published_dt is not None:
try: media.published = published_dt
with atomic(): try:
media.save() with atomic():
log.debug(f'Indexed media: {source} / {media}') media.save()
# log the new media instances except IntegrityError as e:
new_media_instance = ( log.error(f'Index media failed: {source} / {media} with "{e}"')
media.created and else:
source.last_crawl and log.debug(f'Indexed media: {source} / {media}')
media.created >= source.last_crawl # log the new media instances
) new_media_instance = (
if new_media_instance: media.created and
log.info(f'Indexed new media: {source} / {media}') source.last_crawl and
log.info(f'Scheduling task to download metadata for: {media.url}') media.created >= source.last_crawl
verbose_name = _('Downloading metadata for "{}"')
download_media_metadata(
str(media.pk),
priority=9,
verbose_name=verbose_name.format(media.pk),
) )
except IntegrityError as e: if new_media_instance:
log.error(f'Index media failed: {source} / {media} with "{e}"') log.info(f'Indexed new media: {source} / {media}')
log.info(f'Scheduling task to download metadata for: {media.url}')
verbose_name = _('Downloading metadata for "{}"')
download_media_metadata(
str(media.pk),
priority=20,
verbose_name=verbose_name.format(media.pk),
)
# Tack on a cleanup of old completed tasks # Tack on a cleanup of old completed tasks
cleanup_completed_tasks() cleanup_completed_tasks()
# Tack on a cleanup of old media with atomic(durable=True):
cleanup_old_media() # Tack on a cleanup of old media
if source.delete_removed_media: cleanup_old_media()
log.info(f'Cleaning up media no longer in source: {source}') if source.delete_removed_media:
cleanup_removed_media(source, videos) log.info(f'Cleaning up media no longer in source: {source}')
cleanup_removed_media(source, videos)
@background(schedule=0) @background(schedule=0)