diff --git a/tubesync/common/json.py b/tubesync/common/json.py index e8a22e1c..5a56a019 100644 --- a/tubesync/common/json.py +++ b/tubesync/common/json.py @@ -1,4 +1,6 @@ +from datetime import datetime from django.core.serializers.json import DjangoJSONEncoder +from yt_dlp.utils import LazyList class JSONEncoder(DjangoJSONEncoder): @@ -14,3 +16,11 @@ class JSONEncoder(DjangoJSONEncoder): return list(iterable) return super().default(obj) + +def json_serial(obj): + if isinstance(obj, datetime): + return obj.isoformat() + if isinstance(obj, LazyList): + return list(obj) + raise TypeError(f'Type {type(obj)} is not json_serial()-able') + diff --git a/tubesync/common/urls.py b/tubesync/common/urls.py index 1f29056d..a3e00a84 100644 --- a/tubesync/common/urls.py +++ b/tubesync/common/urls.py @@ -1,7 +1,6 @@ from django.conf import settings from django.urls import path from django.views.generic.base import RedirectView -from django.views.generic import TemplateView from django.http import HttpResponse from .views import error403, error404, error500, HealthCheckView diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index c4798943..8f7afc2c 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -6,10 +6,8 @@ import os import pstats import string import time -from datetime import datetime from django.core.paginator import Paginator from urllib.parse import urlunsplit, urlencode, urlparse -from yt_dlp.utils import LazyList from .errors import DatabaseConnectionError @@ -84,14 +82,11 @@ def parse_database_connection_string(database_connection_string): f'invalid driver, must be one of {valid_drivers}') django_driver = django_backends.get(driver) host_parts = user_pass_host_port.split('@') - if len(host_parts) != 2: - raise DatabaseConnectionError(f'Database connection string netloc must be in ' - f'the format of user:pass@host') + user_pass_parts = host_parts[0].split(':') + if len(host_parts) != 2 or len(user_pass_parts) != 2: + raise DatabaseConnectionError('Database connection string netloc must be in ' + 'the format of user:pass@host') user_pass, host_port = host_parts - user_pass_parts = user_pass.split(':') - if len(user_pass_parts) != 2: - raise DatabaseConnectionError(f'Database connection string netloc must be in ' - f'the format of user:pass@host') username, password = user_pass_parts host_port_parts = host_port.split(':') if len(host_port_parts) == 1: @@ -113,13 +108,13 @@ def parse_database_connection_string(database_connection_string): f'65535, got {port}') else: # Malformed - raise DatabaseConnectionError(f'Database connection host must be a hostname or ' - f'a hostname:port combination') + raise DatabaseConnectionError('Database connection host must be a hostname or ' + 'a hostname:port combination') if database.startswith('/'): database = database[1:] if not database: - raise DatabaseConnectionError(f'Database connection string path must be a ' - f'string in the format of /databasename') + raise DatabaseConnectionError('Database connection string path must be a ' + 'string in the format of /databasename') if '/' in database: raise DatabaseConnectionError(f'Database connection string path can only ' f'contain a single string name, got: {database}') @@ -172,14 +167,6 @@ def clean_emoji(s): return emoji.replace_emoji(s) -def json_serial(obj): - if isinstance(obj, datetime): - return obj.isoformat() - if isinstance(obj, LazyList): - return list(obj) - raise TypeError(f'Type {type(obj)} is not json_serial()-able') - - def time_func(func): def wrapper(*args, **kwargs): start = time.perf_counter() diff --git a/tubesync/sync/fields.py b/tubesync/sync/fields.py index 452fee77..e489ff08 100644 --- a/tubesync/sync/fields.py +++ b/tubesync/sync/fields.py @@ -63,7 +63,7 @@ class CommaSepChoiceField(models.CharField): def __init__(self, *args, separator=",", possible_choices=(("","")), all_choice="", all_label="All", allow_all=False, **kwargs): kwargs.setdefault('max_length', 128) self.separator = str(separator) - self.possible_choices = possible_choices or choices + self.possible_choices = possible_choices or kwargs.get('choices') self.selected_choices = list() self.allow_all = allow_all self.all_label = all_label diff --git a/tubesync/sync/filtering.py b/tubesync/sync/filtering.py index 3f7023c1..ef206696 100644 --- a/tubesync/sync/filtering.py +++ b/tubesync/sync/filtering.py @@ -5,7 +5,6 @@ from common.logger import log from .models import Media from datetime import datetime -from django.utils import timezone from .overrides.custom_filter import filter_custom diff --git a/tubesync/sync/hooks.py b/tubesync/sync/hooks.py index 3bb3ce0d..467e2df1 100644 --- a/tubesync/sync/hooks.py +++ b/tubesync/sync/hooks.py @@ -1,9 +1,7 @@ import os -import yt_dlp from common.logger import log from common.utils import remove_enclosed -from django.conf import settings progress_hook = { diff --git a/tubesync/sync/management/commands/delete-source.py b/tubesync/sync/management/commands/delete-source.py index 42f9d5ac..d9f8f204 100644 --- a/tubesync/sync/management/commands/delete-source.py +++ b/tubesync/sync/management/commands/delete-source.py @@ -1,16 +1,15 @@ -import os import uuid -from django.utils.translation import gettext_lazy as _ from django.core.management.base import BaseCommand, CommandError from django.db.transaction import atomic +from django.utils.translation import gettext_lazy as _ from common.logger import log -from sync.models import Source, Media, MediaServer +from sync.models import Source from sync.tasks import schedule_media_servers_update class Command(BaseCommand): - help = _('Deletes a source by UUID') + help = 'Deletes a source by UUID' def add_arguments(self, parser): parser.add_argument('--source', action='store', required=True, help=_('Source UUID')) diff --git a/tubesync/sync/management/commands/import-existing-media.py b/tubesync/sync/management/commands/import-existing-media.py index 3813b497..c05c630a 100644 --- a/tubesync/sync/management/commands/import-existing-media.py +++ b/tubesync/sync/management/commands/import-existing-media.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand, CommandError # noqa from common.logger import log from common.timestamp import timestamp_to_datetime from sync.choices import FileExtension @@ -18,7 +18,7 @@ class Command(BaseCommand): dirmap = {} for s in Source.objects.all(): dirmap[str(s.directory_path)] = s - log.info(f'Scanning sources...') + log.info('Scanning sources...') file_extensions = list(FileExtension.values) + self.extra_extensions for sourceroot, source in dirmap.items(): media = list(Media.objects.filter(source=source, downloaded=False, diff --git a/tubesync/sync/management/commands/list-sources.py b/tubesync/sync/management/commands/list-sources.py index 4ee177ae..25eae481 100644 --- a/tubesync/sync/management/commands/list-sources.py +++ b/tubesync/sync/management/commands/list-sources.py @@ -1,7 +1,6 @@ -import os -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand, CommandError # noqa from common.logger import log -from sync.models import Source, Media, MediaServer +from sync.models import Source class Command(BaseCommand): diff --git a/tubesync/sync/management/commands/reset-tasks.py b/tubesync/sync/management/commands/reset-tasks.py index 55436863..318a5b27 100644 --- a/tubesync/sync/management/commands/reset-tasks.py +++ b/tubesync/sync/management/commands/reset-tasks.py @@ -1,14 +1,12 @@ -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand, CommandError # noqa from django.db.transaction import atomic from django.utils.translation import gettext_lazy as _ from background_task.models import Task +from common.logger import log from sync.models import Source from sync.tasks import index_source_task, check_source_directory_exists -from common.logger import log - - class Command(BaseCommand): help = 'Resets all tasks' @@ -31,10 +29,10 @@ class Command(BaseCommand): index_source_task( str(source.pk), repeat=source.index_schedule, + schedule=source.index_schedule, verbose_name=verbose_name.format(source.name), ) - with atomic(durable=True): - for source in Source.objects.all(): # This also chains down to call each Media objects .save() as well source.save() + log.info('Done') diff --git a/tubesync/sync/management/commands/sync-missing-metadata.py b/tubesync/sync/management/commands/sync-missing-metadata.py index 21b25c52..863f83f9 100644 --- a/tubesync/sync/management/commands/sync-missing-metadata.py +++ b/tubesync/sync/management/commands/sync-missing-metadata.py @@ -1,6 +1,5 @@ -import os from shutil import copyfile -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand, CommandError # noqa from django.db.models import Q from common.logger import log from sync.models import Source, Media diff --git a/tubesync/sync/management/commands/youtube-dl-info.py b/tubesync/sync/management/commands/youtube-dl-info.py index 32a47402..c8099765 100644 --- a/tubesync/sync/management/commands/youtube-dl-info.py +++ b/tubesync/sync/management/commands/youtube-dl-info.py @@ -1,7 +1,7 @@ import json -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand, CommandError # noqa from sync.youtube import get_media_info -from common.utils import json_serial +from common.json import JSONEncoder class Command(BaseCommand): @@ -15,6 +15,6 @@ class Command(BaseCommand): url = options['url'] self.stdout.write(f'Showing information for URL: {url}') info = get_media_info(url) - d = json.dumps(info, indent=4, sort_keys=True, default=json_serial) + d = json.dumps(info, indent=4, sort_keys=True, cls=JSONEncoder) self.stdout.write(d) self.stdout.write('Done') diff --git a/tubesync/sync/mediaservers.py b/tubesync/sync/mediaservers.py index ceab239f..e141238d 100644 --- a/tubesync/sync/mediaservers.py +++ b/tubesync/sync/mediaservers.py @@ -117,9 +117,6 @@ class PlexMediaServer(MediaServer): raise ValidationError('Plex Media Server "port" must be between 1 ' 'and 65535') options = self.object.options - if 'token' not in options: - raise ValidationError('Plex Media Server requires a "token"') - token = options['token'].strip() if 'token' not in options: raise ValidationError('Plex Media Server requires a "token"') if 'libraries' not in options: diff --git a/tubesync/sync/migrations/0011_auto_20220201_1654.py b/tubesync/sync/migrations/0011_auto_20220201_1654.py index 96d9f4a7..51641ece 100644 --- a/tubesync/sync/migrations/0011_auto_20220201_1654.py +++ b/tubesync/sync/migrations/0011_auto_20220201_1654.py @@ -1,8 +1,6 @@ # Generated by Django 3.2.11 on 2022-02-01 16:54 -import django.core.files.storage from django.db import migrations, models -import sync.models class Migration(migrations.Migration): diff --git a/tubesync/sync/migrations/0013_fix_elative_media_file.py b/tubesync/sync/migrations/0013_fix_elative_media_file.py index c9eee22e..2f1ac385 100644 --- a/tubesync/sync/migrations/0013_fix_elative_media_file.py +++ b/tubesync/sync/migrations/0013_fix_elative_media_file.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.12 on 2022-04-06 06:19 from django.conf import settings -from django.db import migrations, models +from django.db import migrations def fix_media_file(apps, schema_editor): diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index d7ed077c..a850a4e0 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -17,3 +17,9 @@ from .media import Media from .metadata import Metadata from .metadata_format import MetadataFormat +__all__ = [ + 'get_media_file_path', 'get_media_thumb_path', + 'media_file_storage', 'MediaServer', 'Source', + 'Media', 'Metadata', 'MetadataFormat', +] + diff --git a/tubesync/sync/models/_private.py b/tubesync/sync/models/_private.py index 96539dbe..5ec14d7c 100644 --- a/tubesync/sync/models/_private.py +++ b/tubesync/sync/models/_private.py @@ -1,4 +1,4 @@ -from ..choices import Val, YouTube_SourceType +from ..choices import Val, YouTube_SourceType # noqa _srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 6eb0ed76..11b30f80 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -15,9 +15,9 @@ from django.utils import timezone from django.utils.translation import gettext_lazy as _ from common.logger import log from common.errors import NoFormatException +from common.json import JSONEncoder from common.utils import ( clean_filename, clean_emoji, - django_queryset_generator as qs_gen, ) from ..youtube import ( get_media_info as get_youtube_media_info, @@ -578,14 +578,13 @@ class Media(models.Model): def metadata_dumps(self, arg_dict=dict()): - from common.utils import json_serial fallback = dict() try: fallback.update(self.new_metadata.with_formats) except ObjectDoesNotExist: pass data = arg_dict or fallback - return json.dumps(data, separators=(',', ':'), default=json_serial) + return json.dumps(data, separators=(',', ':'), cls=JSONEncoder) def metadata_loads(self, arg_str='{}'): @@ -688,7 +687,7 @@ class Media(models.Model): pass setattr(self, '_cached_metadata_dict', data) return data - except Exception as e: + except Exception: return {} @@ -1219,7 +1218,7 @@ class Media(models.Model): parent_dir.rmdir() log.info(f'Removed empty directory: {parent_dir!s}') parent_dir = parent_dir.parent - except OSError as e: + except OSError: pass diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index 82486521..d85334f0 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -511,7 +511,7 @@ class Source(db.models.Model): def get_example_media_format(self): try: return self.media_format.format(**self.example_media_format_dict) - except Exception as e: + except Exception: return '' def is_regex_match(self, media_item_title): diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 790ce1c2..69254146 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -10,11 +10,11 @@ from django.utils.translation import gettext_lazy as _ from background_task.signals import task_failed from background_task.models import Task from common.logger import log -from .models import Source, Media, MediaServer, Metadata +from .models import Source, Media, Metadata from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task, download_media_thumbnail, download_media_metadata, map_task_to_instance, check_source_directory_exists, - download_media, rescan_media_server, download_source_images, + download_media, download_source_images, delete_all_media_for_source, save_all_media_for_source, rename_media, get_media_metadata_task, get_media_download_task) from .utils import delete_file, glob_quote, mkdir_p @@ -270,7 +270,7 @@ def media_post_save(sender, instance, created, **kwargs): if not (media_file_exists or existing_media_download_task): # The file was deleted after it was downloaded, skip this media. if instance.can_download and instance.downloaded: - skip_changed = True != instance.skip + skip_changed = True if not instance.skip else False instance.skip = True downloaded = False if (instance.source.download_media and instance.can_download) and not ( @@ -374,13 +374,13 @@ def media_post_delete(sender, instance, **kwargs): try: p.rmdir() log.info(f'Deleted directory for: {instance} path: {p!s}') - except OSError as e: + except OSError: pass # Delete the directory itself try: other_path.rmdir() log.info(f'Deleted directory for: {instance} path: {other_path!s}') - except OSError as e: + except OSError: pass # Get all files that start with the bare file path all_related_files = video_path.parent.glob(f'{glob_quote(video_path.with_suffix("").name)}*') diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 1cc3d8e0..61cb52a0 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -6,7 +6,6 @@ import os import json -import math import random import requests import time @@ -14,9 +13,8 @@ import uuid from io import BytesIO from hashlib import sha1 from pathlib import Path -from datetime import datetime, timedelta +from datetime import timedelta from shutil import copyfile, rmtree -from PIL import Image from django import db from django.conf import settings from django.core.files.base import ContentFile @@ -30,13 +28,13 @@ from background_task.exceptions import InvalidTaskError from background_task.models import Task, CompletedTask from common.logger import log from common.errors import ( NoFormatException, NoMediaException, - NoMetadataException, NoThumbnailException, + NoThumbnailException, DownloadFailedException, ) from common.utils import ( django_queryset_generator as qs_gen, remove_enclosed, ) from .choices import Val, TaskQueue from .models import Source, Media, MediaServer -from .utils import ( get_remote_image, resize_image_to_height, delete_file, +from .utils import ( get_remote_image, resize_image_to_height, write_text_file, filter_response, ) from .youtube import YouTubeError @@ -226,7 +224,7 @@ def save_model(instance): @atomic(durable=False) def schedule_media_servers_update(): # Schedule a task to update media servers - log.info(f'Scheduling media server updates') + log.info('Scheduling media server updates') verbose_name = _('Request media server rescan for "{}"') for mediaserver in MediaServer.objects.all(): rescan_media_server( @@ -435,7 +433,7 @@ def download_source_images(source_id): log.info(f'Thumbnail URL for source with ID: {source_id} / {source} ' f'Avatar: {avatar} ' f'Banner: {banner}') - if banner != None: + if banner is not None: url = banner i = get_remote_image(url) image_file = BytesIO() @@ -451,7 +449,7 @@ def download_source_images(source_id): f.write(django_file.read()) i = image_file = None - if avatar != None: + if avatar is not None: url = avatar i = get_remote_image(url) image_file = BytesIO() @@ -866,7 +864,7 @@ def delete_all_media_for_source(source_id, source_name, source_directory): assert source_directory try: source = Source.objects.get(pk=source_id) - except Source.DoesNotExist as e: + except Source.DoesNotExist: # Task triggered but the source no longer exists, do nothing log.warn(f'Task delete_all_media_for_source(pk={source_id}) called but no ' f'source exists with ID: {source_id}') diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index 24f0d092..47089673 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -6,7 +6,6 @@ import logging -import os from datetime import datetime, timedelta from pathlib import Path from urllib.parse import urlsplit @@ -1822,13 +1821,13 @@ class TasksTestCase(TestCase): now = timezone.now() - m11 = Media.objects.create(source=src1, downloaded=True, key='a11', download_date=now - timedelta(days=5)) - m12 = Media.objects.create(source=src1, downloaded=True, key='a12', download_date=now - timedelta(days=25)) - m13 = Media.objects.create(source=src1, downloaded=False, key='a13') + m11 = Media.objects.create(source=src1, downloaded=True, key='a11', download_date=now - timedelta(days=5)) # noqa + m12 = Media.objects.create(source=src1, downloaded=True, key='a12', download_date=now - timedelta(days=25)) # noqa + m13 = Media.objects.create(source=src1, downloaded=False, key='a13') # noqa - m21 = Media.objects.create(source=src2, downloaded=True, key='a21', download_date=now - timedelta(days=5)) + m21 = Media.objects.create(source=src2, downloaded=True, key='a21', download_date=now - timedelta(days=5)) # noqa m22 = Media.objects.create(source=src2, downloaded=True, key='a22', download_date=now - timedelta(days=25)) - m23 = Media.objects.create(source=src2, downloaded=False, key='a23') + m23 = Media.objects.create(source=src2, downloaded=False, key='a23') # noqa self.assertEqual(src1.media_source.all().count(), 3) self.assertEqual(src2.media_source.all().count(), 3) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 10cfc5db..4f683834 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -35,7 +35,7 @@ from .tasks import (map_task_to_instance, get_error_message, from .choices import (Val, MediaServerType, SourceResolution, IndexSchedule, YouTube_SourceType, youtube_long_source_types, youtube_help, youtube_validation_urls) -from . import signals +from . import signals # noqa from . import youtube @@ -258,7 +258,7 @@ class ValidateSourceView(FormView): self.key = youtube.get_channel_id( index_url.replace('/channel/', '/') ) - except youtube.YouTubeError as e: + except youtube.YouTubeError: # It did not work, revert to previous behavior self.key = old_key self.source_type = old_source_type @@ -296,10 +296,13 @@ class EditSourceMixin: def form_valid(self, form: Form): # Perform extra validation to make sure the media_format is valid obj = form.save(commit=False) - source_type = form.cleaned_data['media_format'] + # temporarily use media_format from the form + saved_media_format = obj.media_format + obj.media_format = form.cleaned_data['media_format'] example_media_file = obj.get_example_media_format() + obj.media_format = saved_media_format - if example_media_file == '': + if '' == example_media_file: form.add_error( 'media_format', ValidationError(self.errors['invalid_media_format']) @@ -307,12 +310,16 @@ class EditSourceMixin: # Check for suspicious file path(s) try: - targetCheck = form.cleaned_data['directory']+"/.virt" - newdir = safe_join(settings.DOWNLOAD_ROOT,targetCheck) + targetCheck = form.cleaned_data['directory'] + '/.virt' + safe_join(settings.DOWNLOAD_ROOT, targetCheck) except SuspiciousFileOperation: form.add_error( 'directory', - ValidationError(self.errors['dir_outside_dlroot'].replace("%BASEDIR%",str(settings.DOWNLOAD_ROOT))) + ValidationError( + self.errors['dir_outside_dlroot'].replace( + "%BASEDIR%", str(settings.DOWNLOAD_ROOT) + ) + ), ) if form.errors: diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index c795261a..7a2ce654 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -6,7 +6,6 @@ import os -from collections import namedtuple from common.logger import log from copy import deepcopy from pathlib import Path @@ -102,7 +101,7 @@ def get_channel_image_info(url): avatar_url = thumbnail['url'] if thumbnail['id'] == 'banner_uncropped': banner_url = thumbnail['url'] - if banner_url != None and avatar_url != None: + if banner_url is not None and avatar_url is not None: break return avatar_url, banner_url @@ -143,7 +142,7 @@ def get_media_info(url, /, *, days=None, info_json=None): if days is not None: try: days = int(str(days), 10) - except Exception as e: + except (TypeError, ValueError): days = None start = ( f'yesterday-{days!s}days' if days else None diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 798bd252..3ab7f9ff 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -198,7 +198,7 @@ RENAME_SOURCES = None # You have been warned! try: - from .local_settings import * + from .local_settings import * # noqa except ImportError as e: import sys sys.stderr.write(f'Unable to import local_settings: {e}\n') @@ -222,5 +222,5 @@ if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS: BACKGROUND_TASK_ASYNC_THREADS = MAX_BACKGROUND_TASK_ASYNC_THREADS -from .dbutils import patch_ensure_connection +from .dbutils import patch_ensure_connection # noqa patch_ensure_connection()