Update app/ytdl.py

app/ytdl.py

@@ -12,6 +12,7 @@ import logging
 import re
 import random
 import string
+from concurrent.futures import ThreadPoolExecutor
 from urllib.parse import urlparse
 
 import yt_dlp.networking.impersonate
@@ -20,6 +21,9 @@ from datetime import datetime
 
 log = logging.getLogger('ytdl')
 
+_download_lifecycle_executor = ThreadPoolExecutor(max_workers=32, thread_name_prefix='metube-download')
+_metadata_executor = ThreadPoolExecutor(max_workers=4, thread_name_prefix='metube-metadata')
+
 
 def library_meta_path(video_path):
     """Hidden sidecar JSON carrying a library file's source metadata.
@@ -322,7 +326,7 @@ class Download:
         self.info.status = 'preparing'
         await self.notifier.updated(self.info)
         asyncio.create_task(self.update_status())
-        return await self.loop.run_in_executor(None, self.proc.join)
+        return await self.loop.run_in_executor(_download_lifecycle_executor, self.proc.join)
 
     def _resolve_filename_conflict(self, filepath):
         """
@@ -369,7 +373,7 @@ class Download:
 
     async def update_status(self):
         while True:
-            status = await self.loop.run_in_executor(None, self.status_queue.get)
+            status = await self.loop.run_in_executor(_download_lifecycle_executor, self.status_queue.get)
             if status is None:
                 log.info(f"Status update finished for: {self.info.title}")
                 return
@@ -433,7 +437,7 @@ class PersistentQueue:
         return self.dict.items()
 
     def saved_items(self):
-        with shelve.open(self.path, 'r') as shelf:
+        with shelve.open(self.path, 'c') as shelf:
             return sorted(shelf.items(), key=lambda item: item[1].timestamp)
 
     def put(self, value):
@@ -507,11 +511,11 @@ class DownloadQueue:
 
     async def __import_queue(self):
         for k, v in self.queue.saved_items():
-            await self.__add_download(v, True)
+            await self.__add_download(v, True, restored=True)
 
     async def __import_pending(self):
         for k, v in self.pending.saved_items():
-            await self.__add_download(v, False)
+            await self.__add_download(v, False, restored=True)
 
     async def initialize(self):
         log.info("Initializing DownloadQueue")
@@ -548,6 +552,7 @@ class DownloadQueue:
         output_chapter = item['output_chapter']
         ytdl_options = item['ytdl_options']
         entry = item['entry']
+        restored = item.get('restored', False)
         
         log.info(f"[PreCheck] Checking for filename conflicts before download")
         log.debug(f"[PreCheck] Original output template: {output}")
@@ -564,13 +569,16 @@ class DownloadQueue:
                 title == f"twitter video #{video_id}" or  # Placeholder pattern
                 video_id in title  # ID is in title (likely placeholder)
             )
+            if needs_full_extraction and restored:
+                log.debug(f"[PreCheck] Skipping repeated full extraction for restored item: {dl.url}")
+                needs_full_extraction = False
             
             if needs_full_extraction:
                 log.debug(f"[PreCheck] Title appears to be placeholder: '{title}', doing full info extraction")
                 try:
                     # Do a full (non-flat) extraction to get real title
                     full_entry = await asyncio.get_running_loop().run_in_executor(
-                        None,
+                        _metadata_executor,
                         lambda: yt_dlp.YoutubeDL(params={
                             'quiet': True,
                             'no_color': True,
@@ -780,7 +788,7 @@ class DownloadQueue:
             dldirectory = base_directory
         return dldirectory, None
 
-    async def __add_download(self, dl, auto_start):
+    async def __add_download(self, dl, auto_start, restored=False):
         """Fast path: validate and queue for precheck processing.
         Returns immediately without blocking on slow operations."""
         # Check if this exact URL is already being processed, in queue, or already downloaded
@@ -872,6 +880,7 @@ class DownloadQueue:
             'output_chapter': output_chapter,
             'ytdl_options': ytdl_options,
             'entry': entry,
+            'restored': restored,
         })
         log.debug(f"[PreCheck] Queued for processing: {dl.url}")
         
@@ -968,7 +977,7 @@ class DownloadQueue:
         else:
             already.add(url)
         try:
-            entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url, playlist_strict_mode)
+            entry = await asyncio.get_running_loop().run_in_executor(_metadata_executor, self.__extract_info, url, playlist_strict_mode)
         except yt_dlp.utils.YoutubeDLError as exc:
             return {'status': 'error', 'msg': str(exc)}
         return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already)