From 1aaed36985e2273b3d5a3a26875ee9965ea49557 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 21 Jun 2025 11:42:06 -0400 Subject: [PATCH 01/11] Add `XDG_CONFIG_HOME` directory --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a54c68e6..6924166d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -534,7 +534,9 @@ HEALTHCHECK --interval=1m --timeout=10s --start-period=3m CMD ["/app/healthcheck ENV PYTHONPATH="/app" \ PYTHONPYCACHEPREFIX="/config/cache/pycache" \ S6_CMD_WAIT_FOR_SERVICES_MAXTIME="0" \ - XDG_CACHE_HOME="/config/cache" + XDG_CACHE_HOME="/config/cache" \ + XDG_CONFIG_HOME="/config/tubesync" \ + XDG_STATE_HOME="/config/state" EXPOSE 4848 # Volumes From 7bcb06d271f715b385b4a4490ecd77d0899d3ce6 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 21 Jun 2025 11:45:23 -0400 Subject: [PATCH 02/11] Create the `XDG_STATE_HOME` directory --- config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run index df003540..4ab92e4f 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run @@ -5,7 +5,7 @@ groupmod -o -g "${PGID:=911}" app usermod -o -u "${PUID:=911}" app # Ensure /config directories exist -mkdir -v -p /config/{cache,media,tasks,tubesync} +mkdir -v -p /config/{cache,media,state,tasks,tubesync} # Copy local_settings.py for the user if [ -f /config/tubesync/local_settings.py ] From 9e98eedea7b33d328499a9c5cddb016274cb41d8 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 21 Jun 2025 12:59:14 -0400 Subject: [PATCH 03/11] Create cache-filesystem.py --- .../extractor/cache-filesystem.py | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 tubesync/yt_dlp_plugins/extractor/cache-filesystem.py diff --git a/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py b/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py new file mode 100644 index 00000000..f20d5e7b --- /dev/null +++ b/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py @@ -0,0 +1,98 @@ +from common.utils import getenv +from datetime import datetime, timezone +from hashlib import sha256 +from pathlib import Path +from yt_dlp.extractor.youtube.pot.cache import ( + PoTokenCacheProvider, + register_preference, + register_provider +) + +from yt_dlp.extractor.youtube.pot.provider import PoTokenRequest + + +@register_provider +class TubeSyncFileSystemPCP(PoTokenCacheProvider): # Provider class name must end with "PCP" + PROVIDER_VERSION = '0.0.1' + # Define a unique display name for the provider + PROVIDER_NAME = 'TubeSync-fs' + BUG_REPORT_LOCATION = 'https://github.com/meeb/tubesync/issues' + + def _make_filename(self, key: str, expires_at: int) -> str: + result = sha256(key.encode(), usedforsecurity=False) + return f'{expires_at or '*'}-{result.hexdigest()}' + + def is_available(self) -> bool: + """ + Check if the provider is available (e.g. all required dependencies are available) + This is used to determine if the provider should be used and to provide debug information. + + IMPORTANT: This method SHOULD NOT make any network requests or perform any expensive operations. + + Since this is called multiple times, we recommend caching the result. + """ + cache_home = getenv('XDG_CACHE_HOME') + if not cache_home: + return False + directory = Path(cache_home) / 'yt-dlp/youtube-pot' + if directory.exists() and directory.is_dir(): + self._storage_directory = directory + return True + return False + + def get(self, key: str): + # ℹ️ Similar to PO Token Providers, Cache Providers and Cache Spec Providers + # are passed down extractor args matching key youtubepot-. + # some_setting = self._configuration_arg('some_setting', default=['default_value'])[0] + found = None + now = datetime.utcnow().astimezone(timezone.utc) + files = Path(self._storage_directory).glob(self._make_filename(key, 0)) + for file in files: + if not file.is_file(): + continue + try: + expires_at = int(file.name.partition('-')[0]) + except ValueError: + continue + else: + expired = datetime.utcfromtimestamp(expires_at).astimezone(timezone.utc) + if expired < now: + file.unlink() + else: + found = file + + return None if found is None else found.read_bytes().decode() + + def store(self, key: str, value: str, expires_at: int): + # ⚠ expires_at MUST be respected. + # Cache entries should not be returned if they have expired. + dst = Path(self._storage_directory) / self._make_filename(key, expires_at) + now = datetime.utcnow().astimezone(timezone.utc) + expires = datetime.utcfromtimestamp(expires_at).astimezone(timezone.utc) + if expires > now: + dst.write_bytes(value.encode()) + + def delete(self, key: str): + files = Path(self._storage_directory).glob(self._make_filename(key, 0)) + for file in files: + if not file.is_file(): + continue + file.unlink() + + def close(self): + # Optional close hook, called when the YoutubeDL instance is closed. + pass + +# If there are multiple PO Token Cache Providers available, you can +# define a preference function to increase/decrease the priority of providers. + +# IMPORTANT: Providers should be in preference of cache lookup time. +# For example, a memory cache should have a higher preference than a disk cache. + +# VERY IMPORTANT: yt-dlp has a built-in memory cache with a priority of 10000. +# Your cache provider should be lower than this. + + +@register_preference(MyCacheProviderPCP) +def my_cache_preference(provider: PoTokenCacheProvider, request: PoTokenRequest) -> int: + return 10 From 13bd4c83e0fef2f4de30f230edfb32df54a1e888 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 21 Jun 2025 13:06:04 -0400 Subject: [PATCH 04/11] fixup: adjust the example preference --- tubesync/yt_dlp_plugins/extractor/cache-filesystem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py b/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py index f20d5e7b..631d5b0a 100644 --- a/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py +++ b/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py @@ -93,6 +93,6 @@ class TubeSyncFileSystemPCP(PoTokenCacheProvider): # Provider class name must e # Your cache provider should be lower than this. -@register_preference(MyCacheProviderPCP) -def my_cache_preference(provider: PoTokenCacheProvider, request: PoTokenRequest) -> int: +@register_preference(TubeSyncFileSystemPCP) +def filesystem_cache_preference(provider: PoTokenCacheProvider, request: PoTokenRequest) -> int: return 10 From b9270336fcf3b11b9bad17ab8f0ffbf1bcf45fd7 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 21 Jun 2025 13:08:04 -0400 Subject: [PATCH 05/11] fixup: quoting --- tubesync/yt_dlp_plugins/extractor/cache-filesystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py b/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py index 631d5b0a..0800fa75 100644 --- a/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py +++ b/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py @@ -20,7 +20,7 @@ class TubeSyncFileSystemPCP(PoTokenCacheProvider): # Provider class name must e def _make_filename(self, key: str, expires_at: int) -> str: result = sha256(key.encode(), usedforsecurity=False) - return f'{expires_at or '*'}-{result.hexdigest()}' + return f'{expires_at or "*"}-{result.hexdigest()}' def is_available(self) -> bool: """ From d66665ce69ba680d9c2f212e6f7a343ac6ff3aaf Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 21 Jun 2025 15:17:06 -0400 Subject: [PATCH 06/11] Adjust for key already being hashed and clean up --- .../extractor/cache-filesystem.py | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py b/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py index 0800fa75..dc1c952d 100644 --- a/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py +++ b/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py @@ -1,6 +1,5 @@ from common.utils import getenv from datetime import datetime, timezone -from hashlib import sha256 from pathlib import Path from yt_dlp.extractor.youtube.pot.cache import ( PoTokenCacheProvider, @@ -18,10 +17,18 @@ class TubeSyncFileSystemPCP(PoTokenCacheProvider): # Provider class name must e PROVIDER_NAME = 'TubeSync-fs' BUG_REPORT_LOCATION = 'https://github.com/meeb/tubesync/issues' + def _now(self) -> datetime: + return datetime.now(timezone.utc) + def _make_filename(self, key: str, expires_at: int) -> str: - result = sha256(key.encode(), usedforsecurity=False) - return f'{expires_at or "*"}-{result.hexdigest()}' + return f'{expires_at or "*"}-{key}' + def _expires(self, expires_at: int) -> datetime: + return datetime.utcfromtimestamp(expires_at).astimezone(timezone.utc) + + def _files(self, key: str) -> generator: + return Path(self._storage_directory).glob(self._make_filename(key, 0)) + def is_available(self) -> bool: """ Check if the provider is available (e.g. all required dependencies are available) @@ -45,9 +52,8 @@ class TubeSyncFileSystemPCP(PoTokenCacheProvider): # Provider class name must e # are passed down extractor args matching key youtubepot-. # some_setting = self._configuration_arg('some_setting', default=['default_value'])[0] found = None - now = datetime.utcnow().astimezone(timezone.utc) - files = Path(self._storage_directory).glob(self._make_filename(key, 0)) - for file in files: + now = self._now() + for file in self._files(key): if not file.is_file(): continue try: @@ -55,8 +61,7 @@ class TubeSyncFileSystemPCP(PoTokenCacheProvider): # Provider class name must e except ValueError: continue else: - expired = datetime.utcfromtimestamp(expires_at).astimezone(timezone.utc) - if expired < now: + if self._expires(expires_at) < now: file.unlink() else: found = file @@ -66,15 +71,12 @@ class TubeSyncFileSystemPCP(PoTokenCacheProvider): # Provider class name must e def store(self, key: str, value: str, expires_at: int): # ⚠ expires_at MUST be respected. # Cache entries should not be returned if they have expired. - dst = Path(self._storage_directory) / self._make_filename(key, expires_at) - now = datetime.utcnow().astimezone(timezone.utc) - expires = datetime.utcfromtimestamp(expires_at).astimezone(timezone.utc) - if expires > now: + if self._expires(expires_at) > self._now(): + dst = Path(self._storage_directory) / self._make_filename(key, expires_at) dst.write_bytes(value.encode()) def delete(self, key: str): - files = Path(self._storage_directory).glob(self._make_filename(key, 0)) - for file in files: + for file in self._files(key): if not file.is_file(): continue file.unlink() From b96348f3b0323272357167975acba1e002ac9dc9 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 21 Jun 2025 15:35:51 -0400 Subject: [PATCH 07/11] fixup: typing --- tubesync/yt_dlp_plugins/extractor/cache-filesystem.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py b/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py index dc1c952d..88daa051 100644 --- a/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py +++ b/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py @@ -1,3 +1,4 @@ +from collections.abc import Generator from common.utils import getenv from datetime import datetime, timezone from pathlib import Path @@ -26,7 +27,7 @@ class TubeSyncFileSystemPCP(PoTokenCacheProvider): # Provider class name must e def _expires(self, expires_at: int) -> datetime: return datetime.utcfromtimestamp(expires_at).astimezone(timezone.utc) - def _files(self, key: str) -> generator: + def _files(self, key: str) -> Generator[Path]: return Path(self._storage_directory).glob(self._make_filename(key, 0)) def is_available(self) -> bool: From 7c66b3b00574ed79477c6884ca9c18b34d4a097e Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 21 Jun 2025 18:56:22 -0400 Subject: [PATCH 08/11] Check that the cookies file exists --- .../yt_dlp_plugins/extractor/cache-filesystem.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py b/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py index 88daa051..619d56bd 100644 --- a/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py +++ b/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py @@ -44,11 +44,18 @@ class TubeSyncFileSystemPCP(PoTokenCacheProvider): # Provider class name must e return False directory = Path(cache_home) / 'yt-dlp/youtube-pot' if directory.exists() and directory.is_dir(): + cookiejar = self._configuration_arg( + 'cookies', + default=['cookies.txt'], + )[0] + if not Path(cookiejar).is_file(): + return False self._storage_directory = directory return True return False def get(self, key: str): + self.logger.trace(f'fs-get: {key=}') # ℹ️ Similar to PO Token Providers, Cache Providers and Cache Spec Providers # are passed down extractor args matching key youtubepot-. # some_setting = self._configuration_arg('some_setting', default=['default_value'])[0] @@ -63,23 +70,30 @@ class TubeSyncFileSystemPCP(PoTokenCacheProvider): # Provider class name must e continue else: if self._expires(expires_at) < now: + self.logger.trace(f'fs-get: unlinking: {file.name}') file.unlink() else: + self.logger.trace(f'fs-get: found: {file.name}') found = file + self.logger.trace(f'fs-get: {found=}') return None if found is None else found.read_bytes().decode() def store(self, key: str, value: str, expires_at: int): + self.logger.trace(f'fs-store: {expires_at=} {key=}') # ⚠ expires_at MUST be respected. # Cache entries should not be returned if they have expired. if self._expires(expires_at) > self._now(): dst = Path(self._storage_directory) / self._make_filename(key, expires_at) + self.logger.trace(f'fs-store: writing: {dst.name}') dst.write_bytes(value.encode()) def delete(self, key: str): + self.logger.trace(f'fs-delete: {key=}') for file in self._files(key): if not file.is_file(): continue + self.logger.trace(f'fs-delete: unlinking: {file.name}') file.unlink() def close(self): From 0ee9a2c0871122903d7d0615c36fbe10319303fa Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 21 Jun 2025 21:33:47 -0400 Subject: [PATCH 09/11] The settings are empty --- tubesync/yt_dlp_plugins/extractor/cache-filesystem.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py b/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py index 619d56bd..96855e74 100644 --- a/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py +++ b/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py @@ -42,14 +42,12 @@ class TubeSyncFileSystemPCP(PoTokenCacheProvider): # Provider class name must e cache_home = getenv('XDG_CACHE_HOME') if not cache_home: return False + # TODO: check the actual setting: cookiefile + cookie_file = cache_home / '../cookies.txt' + if not cookie_file.is_file(): + return False directory = Path(cache_home) / 'yt-dlp/youtube-pot' if directory.exists() and directory.is_dir(): - cookiejar = self._configuration_arg( - 'cookies', - default=['cookies.txt'], - )[0] - if not Path(cookiejar).is_file(): - return False self._storage_directory = directory return True return False From fff550c30195eb859669f9c7929b26859c055540 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 22 Jun 2025 12:39:49 -0400 Subject: [PATCH 10/11] fixup: need a `Path` not a `str` --- tubesync/yt_dlp_plugins/extractor/cache-filesystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py b/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py index 96855e74..883773f3 100644 --- a/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py +++ b/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py @@ -43,7 +43,7 @@ class TubeSyncFileSystemPCP(PoTokenCacheProvider): # Provider class name must e if not cache_home: return False # TODO: check the actual setting: cookiefile - cookie_file = cache_home / '../cookies.txt' + cookie_file = Path(cache_home) / '../cookies.txt' if not cookie_file.is_file(): return False directory = Path(cache_home) / 'yt-dlp/youtube-pot' From 9fe40cd81fe85ef3e858befecc77f519f80d665d Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 23 Jun 2025 00:28:52 -0400 Subject: [PATCH 11/11] Sort the files --- tubesync/yt_dlp_plugins/extractor/cache-filesystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py b/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py index 883773f3..6a938aa6 100644 --- a/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py +++ b/tubesync/yt_dlp_plugins/extractor/cache-filesystem.py @@ -59,7 +59,7 @@ class TubeSyncFileSystemPCP(PoTokenCacheProvider): # Provider class name must e # some_setting = self._configuration_arg('some_setting', default=['default_value'])[0] found = None now = self._now() - for file in self._files(key): + for file in sorted(self._files(key)): if not file.is_file(): continue try: