diff --git a/app/ytdl.py b/app/ytdl.py index 435e30c..2cc54b1 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -12,6 +12,7 @@ import logging import re import random import string +from concurrent.futures import ThreadPoolExecutor from urllib.parse import urlparse import yt_dlp.networking.impersonate @@ -20,6 +21,9 @@ from datetime import datetime log = logging.getLogger('ytdl') +_download_lifecycle_executor = ThreadPoolExecutor(max_workers=32, thread_name_prefix='metube-download') +_metadata_executor = ThreadPoolExecutor(max_workers=4, thread_name_prefix='metube-metadata') + def library_meta_path(video_path): """Hidden sidecar JSON carrying a library file's source metadata. @@ -322,7 +326,7 @@ class Download: self.info.status = 'preparing' await self.notifier.updated(self.info) asyncio.create_task(self.update_status()) - return await self.loop.run_in_executor(None, self.proc.join) + return await self.loop.run_in_executor(_download_lifecycle_executor, self.proc.join) def _resolve_filename_conflict(self, filepath): """ @@ -369,7 +373,7 @@ class Download: async def update_status(self): while True: - status = await self.loop.run_in_executor(None, self.status_queue.get) + status = await self.loop.run_in_executor(_download_lifecycle_executor, self.status_queue.get) if status is None: log.info(f"Status update finished for: {self.info.title}") return @@ -433,7 +437,7 @@ class PersistentQueue: return self.dict.items() def saved_items(self): - with shelve.open(self.path, 'r') as shelf: + with shelve.open(self.path, 'c') as shelf: return sorted(shelf.items(), key=lambda item: item[1].timestamp) def put(self, value): @@ -507,11 +511,11 @@ class DownloadQueue: async def __import_queue(self): for k, v in self.queue.saved_items(): - await self.__add_download(v, True) + await self.__add_download(v, True, restored=True) async def __import_pending(self): for k, v in self.pending.saved_items(): - await self.__add_download(v, False) + await self.__add_download(v, False, restored=True) async def initialize(self): log.info("Initializing DownloadQueue") @@ -548,6 +552,7 @@ class DownloadQueue: output_chapter = item['output_chapter'] ytdl_options = item['ytdl_options'] entry = item['entry'] + restored = item.get('restored', False) log.info(f"[PreCheck] Checking for filename conflicts before download") log.debug(f"[PreCheck] Original output template: {output}") @@ -564,13 +569,16 @@ class DownloadQueue: title == f"twitter video #{video_id}" or # Placeholder pattern video_id in title # ID is in title (likely placeholder) ) + if needs_full_extraction and restored: + log.debug(f"[PreCheck] Skipping repeated full extraction for restored item: {dl.url}") + needs_full_extraction = False if needs_full_extraction: log.debug(f"[PreCheck] Title appears to be placeholder: '{title}', doing full info extraction") try: # Do a full (non-flat) extraction to get real title full_entry = await asyncio.get_running_loop().run_in_executor( - None, + _metadata_executor, lambda: yt_dlp.YoutubeDL(params={ 'quiet': True, 'no_color': True, @@ -780,7 +788,7 @@ class DownloadQueue: dldirectory = base_directory return dldirectory, None - async def __add_download(self, dl, auto_start): + async def __add_download(self, dl, auto_start, restored=False): """Fast path: validate and queue for precheck processing. Returns immediately without blocking on slow operations.""" # Check if this exact URL is already being processed, in queue, or already downloaded @@ -872,6 +880,7 @@ class DownloadQueue: 'output_chapter': output_chapter, 'ytdl_options': ytdl_options, 'entry': entry, + 'restored': restored, }) log.debug(f"[PreCheck] Queued for processing: {dl.url}") @@ -968,7 +977,7 @@ class DownloadQueue: else: already.add(url) try: - entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url, playlist_strict_mode) + entry = await asyncio.get_running_loop().run_in_executor(_metadata_executor, self.__extract_info, url, playlist_strict_mode) except yt_dlp.utils.YoutubeDLError as exc: return {'status': 'error', 'msg': str(exc)} return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already)