xorbitlab
/
metube
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update app/ytdl.py

This commit is contained in:
tigerenwork 2026-11-05 16:59:25 +00:00
parent a0f7ecec9a
commit 496978078b
1 changed files with 17 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
app/ytdl.py
View File

@ -12,6 +12,7 @@ import logging
import re import re
import random import random
import string import string
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import urlparse from urllib.parse import urlparse
import yt_dlp.networking.impersonate import yt_dlp.networking.impersonate
@ -20,6 +21,9 @@ from datetime import datetime
log = logging.getLogger('ytdl') log = logging.getLogger('ytdl')
_download_lifecycle_executor = ThreadPoolExecutor(max_workers=32, thread_name_prefix='metube-download')
_metadata_executor = ThreadPoolExecutor(max_workers=4, thread_name_prefix='metube-metadata')
def library_meta_path(video_path): def library_meta_path(video_path):
"""Hidden sidecar JSON carrying a library file's source metadata. """Hidden sidecar JSON carrying a library file's source metadata.
@ -322,7 +326,7 @@ class Download:
self.info.status = 'preparing' self.info.status = 'preparing'
await self.notifier.updated(self.info) await self.notifier.updated(self.info)
asyncio.create_task(self.update_status()) asyncio.create_task(self.update_status())
return await self.loop.run_in_executor(None, self.proc.join) return await self.loop.run_in_executor(_download_lifecycle_executor, self.proc.join)
def _resolve_filename_conflict(self, filepath): def _resolve_filename_conflict(self, filepath):
""" """
@ -369,7 +373,7 @@ class Download:
async def update_status(self): async def update_status(self):
while True: while True:
status = await self.loop.run_in_executor(None, self.status_queue.get) status = await self.loop.run_in_executor(_download_lifecycle_executor, self.status_queue.get)
if status is None: if status is None:
log.info(f"Status update finished for: {self.info.title}") log.info(f"Status update finished for: {self.info.title}")
return return
@ -433,7 +437,7 @@ class PersistentQueue:
return self.dict.items() return self.dict.items()
def saved_items(self): def saved_items(self):
with shelve.open(self.path, 'r') as shelf: with shelve.open(self.path, 'c') as shelf:
return sorted(shelf.items(), key=lambda item: item[1].timestamp) return sorted(shelf.items(), key=lambda item: item[1].timestamp)
def put(self, value): def put(self, value):
@ -507,11 +511,11 @@ class DownloadQueue:
async def __import_queue(self): async def __import_queue(self):
for k, v in self.queue.saved_items(): for k, v in self.queue.saved_items():
await self.__add_download(v, True) await self.__add_download(v, True, restored=True)
async def __import_pending(self): async def __import_pending(self):
for k, v in self.pending.saved_items(): for k, v in self.pending.saved_items():
await self.__add_download(v, False) await self.__add_download(v, False, restored=True)
async def initialize(self): async def initialize(self):
log.info("Initializing DownloadQueue") log.info("Initializing DownloadQueue")
@ -548,6 +552,7 @@ class DownloadQueue:
output_chapter = item['output_chapter'] output_chapter = item['output_chapter']
ytdl_options = item['ytdl_options'] ytdl_options = item['ytdl_options']
entry = item['entry'] entry = item['entry']
restored = item.get('restored', False)
log.info(f"[PreCheck] Checking for filename conflicts before download") log.info(f"[PreCheck] Checking for filename conflicts before download")
log.debug(f"[PreCheck] Original output template: {output}") log.debug(f"[PreCheck] Original output template: {output}")
@ -564,13 +569,16 @@ class DownloadQueue:
title == f"twitter video #{video_id}" or # Placeholder pattern title == f"twitter video #{video_id}" or # Placeholder pattern
video_id in title # ID is in title (likely placeholder) video_id in title # ID is in title (likely placeholder)
) )
if needs_full_extraction and restored:
log.debug(f"[PreCheck] Skipping repeated full extraction for restored item: {dl.url}")
needs_full_extraction = False
if needs_full_extraction: if needs_full_extraction:
log.debug(f"[PreCheck] Title appears to be placeholder: '{title}', doing full info extraction") log.debug(f"[PreCheck] Title appears to be placeholder: '{title}', doing full info extraction")
try: try:
# Do a full (non-flat) extraction to get real title # Do a full (non-flat) extraction to get real title
full_entry = await asyncio.get_running_loop().run_in_executor( full_entry = await asyncio.get_running_loop().run_in_executor(
None, _metadata_executor,
lambda: yt_dlp.YoutubeDL(params={ lambda: yt_dlp.YoutubeDL(params={
'quiet': True, 'quiet': True,
'no_color': True, 'no_color': True,
@ -780,7 +788,7 @@ class DownloadQueue:
dldirectory = base_directory dldirectory = base_directory
return dldirectory, None return dldirectory, None
async def __add_download(self, dl, auto_start): async def __add_download(self, dl, auto_start, restored=False):
"""Fast path: validate and queue for precheck processing. """Fast path: validate and queue for precheck processing.
Returns immediately without blocking on slow operations.""" Returns immediately without blocking on slow operations."""
# Check if this exact URL is already being processed, in queue, or already downloaded # Check if this exact URL is already being processed, in queue, or already downloaded
@ -872,6 +880,7 @@ class DownloadQueue:
'output_chapter': output_chapter, 'output_chapter': output_chapter,
'ytdl_options': ytdl_options, 'ytdl_options': ytdl_options,
'entry': entry, 'entry': entry,
'restored': restored,
}) })
log.debug(f"[PreCheck] Queued for processing: {dl.url}") log.debug(f"[PreCheck] Queued for processing: {dl.url}")
@ -968,7 +977,7 @@ class DownloadQueue:
else: else:
already.add(url) already.add(url)
try: try:
entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url, playlist_strict_mode) entry = await asyncio.get_running_loop().run_in_executor(_metadata_executor, self.__extract_info, url, playlist_strict_mode)
except yt_dlp.utils.YoutubeDLError as exc: except yt_dlp.utils.YoutubeDLError as exc:
return {'status': 'error', 'msg': str(exc)} return {'status': 'error', 'msg': str(exc)}
return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already) return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already)