929 lines
36 KiB
Python
929 lines
36 KiB
Python
#!/usr/bin/env python3
|
|
# pylint: disable=no-member,method-hidden
|
|
|
|
import os
|
|
import sys
|
|
import asyncio
|
|
import shutil
|
|
from pathlib import Path
|
|
from aiohttp import web
|
|
from aiohttp.log import access_logger
|
|
import ssl
|
|
import socket
|
|
import socketio
|
|
import logging
|
|
import json
|
|
import pathlib
|
|
import re
|
|
import time
|
|
import base64
|
|
import hashlib
|
|
import aiohttp
|
|
from urllib.parse import urlparse
|
|
from watchfiles import DefaultFilter, Change, awatch
|
|
|
|
from ytdl import (
|
|
DownloadQueueNotifier, DownloadQueue, library_meta_path,
|
|
build_library_roots, normalize_library_roots, resolve_library_path,
|
|
)
|
|
from yt_dlp.version import __version__ as yt_dlp_version
|
|
|
|
log = logging.getLogger('main')
|
|
|
|
class Config:
|
|
_DEFAULTS = {
|
|
'DOWNLOAD_DIR': '.',
|
|
'AUDIO_DOWNLOAD_DIR': '%%DOWNLOAD_DIR',
|
|
'TEMP_DIR': '%%DOWNLOAD_DIR',
|
|
'DOWNLOAD_DIRS_INDEXABLE': 'false',
|
|
'CUSTOM_DIRS': 'true',
|
|
'CREATE_CUSTOM_DIRS': 'true',
|
|
'CUSTOM_DIRS_EXCLUDE_REGEX': r'(^|/)[.@].*$',
|
|
'DELETE_FILE_ON_TRASHCAN': 'true',
|
|
'STATE_DIR': '.',
|
|
'LIBRARY_DIR': '',
|
|
'LIBRARY_DIRS': '%%LIBRARY_DIR',
|
|
'PUBLIC_HOST_LIBRARY_URL': 'library/',
|
|
'URL_PREFIX': '',
|
|
'PUBLIC_HOST_URL': 'download/',
|
|
'PUBLIC_HOST_AUDIO_URL': 'audio_download/',
|
|
'OUTPUT_TEMPLATE': '%(title)s.%(ext)s',
|
|
'OUTPUT_TEMPLATE_CHAPTER': '%(title)s - %(section_number)s %(section_title)s.%(ext)s',
|
|
'OUTPUT_TEMPLATE_PLAYLIST': '%(playlist_title)s/%(title)s.%(ext)s',
|
|
'DEFAULT_OPTION_PLAYLIST_STRICT_MODE' : 'false',
|
|
'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT' : '0',
|
|
'YTDL_OPTIONS': '{}',
|
|
'YTDL_OPTIONS_FILE': '',
|
|
'ROBOTS_TXT': '',
|
|
'HOST': '0.0.0.0',
|
|
'PORT': '8081',
|
|
'HTTPS': 'false',
|
|
'CERTFILE': '',
|
|
'KEYFILE': '',
|
|
'BASE_DIR': '',
|
|
'DEFAULT_THEME': 'auto',
|
|
'DOWNLOAD_MODE': 'limited',
|
|
'MAX_CONCURRENT_DOWNLOADS': 3,
|
|
'LOGLEVEL': 'INFO',
|
|
'ENABLE_ACCESSLOG': 'false',
|
|
'MARK_WATCHED_ON_COMPLETE': 'false',
|
|
}
|
|
|
|
_BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE', 'HTTPS', 'ENABLE_ACCESSLOG', 'MARK_WATCHED_ON_COMPLETE')
|
|
|
|
def __init__(self):
|
|
for k, v in self._DEFAULTS.items():
|
|
setattr(self, k, os.environ.get(k, v))
|
|
|
|
for k, v in self.__dict__.items():
|
|
if isinstance(v, str) and v.startswith('%%'):
|
|
setattr(self, k, getattr(self, v[2:]))
|
|
if k in self._BOOLEAN:
|
|
if v not in ('true', 'false', 'True', 'False', 'on', 'off', '1', '0'):
|
|
log.error(f'Environment variable "{k}" is set to a non-boolean value "{v}"')
|
|
sys.exit(1)
|
|
setattr(self, k, v in ('true', 'True', 'on', '1'))
|
|
|
|
# Media library: comma-separated container paths; empty = feature off.
|
|
# Kept as a list so the UI can check `LIBRARY_DIRS` directly.
|
|
self.LIBRARY_DIRS = [d.strip() for d in str(self.LIBRARY_DIRS).split(',') if d.strip()]
|
|
try:
|
|
roots = build_library_roots(self.LIBRARY_DIRS)
|
|
except ValueError as e:
|
|
log.error(str(e))
|
|
sys.exit(1)
|
|
self.LIBRARY_ROOTS = normalize_library_roots(roots)
|
|
if self.LIBRARY_DIRS and not self.LIBRARY_ROOTS:
|
|
log.warning('Library feature disabled: none of LIBRARY_DIRS exists')
|
|
if self.LIBRARY_ROOTS:
|
|
log.info('Library roots: ' + ', '.join(f'{label or "(root)"} -> {path}' for label, path in self.LIBRARY_ROOTS))
|
|
|
|
if not self.URL_PREFIX.endswith('/'):
|
|
self.URL_PREFIX += '/'
|
|
|
|
# Convert relative addresses to absolute addresses to prevent the failure of file address comparison
|
|
if self.YTDL_OPTIONS_FILE and self.YTDL_OPTIONS_FILE.startswith('.'):
|
|
self.YTDL_OPTIONS_FILE = str(Path(self.YTDL_OPTIONS_FILE).resolve())
|
|
|
|
success,_ = self.load_ytdl_options()
|
|
if not success:
|
|
sys.exit(1)
|
|
|
|
def load_ytdl_options(self) -> tuple[bool, str]:
|
|
try:
|
|
self.YTDL_OPTIONS = json.loads(os.environ.get('YTDL_OPTIONS', '{}'))
|
|
assert isinstance(self.YTDL_OPTIONS, dict)
|
|
except (json.decoder.JSONDecodeError, AssertionError):
|
|
msg = 'Environment variable YTDL_OPTIONS is invalid'
|
|
log.error(msg)
|
|
return (False, msg)
|
|
|
|
if not self.YTDL_OPTIONS_FILE:
|
|
return (True, '')
|
|
|
|
log.info(f'Loading yt-dlp custom options from "{self.YTDL_OPTIONS_FILE}"')
|
|
if not os.path.exists(self.YTDL_OPTIONS_FILE):
|
|
msg = f'File "{self.YTDL_OPTIONS_FILE}" not found'
|
|
log.error(msg)
|
|
return (False, msg)
|
|
try:
|
|
with open(self.YTDL_OPTIONS_FILE) as json_data:
|
|
opts = json.load(json_data)
|
|
assert isinstance(opts, dict)
|
|
except (json.decoder.JSONDecodeError, AssertionError):
|
|
msg = 'YTDL_OPTIONS_FILE contents is invalid'
|
|
log.error(msg)
|
|
return (False, msg)
|
|
|
|
self.YTDL_OPTIONS.update(opts)
|
|
return (True, '')
|
|
|
|
config = Config()
|
|
|
|
class ObjectSerializer(json.JSONEncoder):
|
|
def default(self, obj):
|
|
# First try to use __dict__ for custom objects
|
|
if hasattr(obj, '__dict__'):
|
|
return obj.__dict__
|
|
# Convert iterables (generators, dict_items, etc.) to lists
|
|
# Exclude strings and bytes which are also iterable
|
|
elif hasattr(obj, '__iter__') and not isinstance(obj, (str, bytes)):
|
|
try:
|
|
return list(obj)
|
|
except:
|
|
pass
|
|
# Fall back to default behavior
|
|
return json.JSONEncoder.default(self, obj)
|
|
|
|
serializer = ObjectSerializer()
|
|
app = web.Application()
|
|
sio = socketio.AsyncServer(cors_allowed_origins='*')
|
|
sio.attach(app, socketio_path=config.URL_PREFIX + 'socket.io')
|
|
routes = web.RouteTableDef()
|
|
|
|
class Notifier(DownloadQueueNotifier):
|
|
async def added(self, dl):
|
|
log.info(f"Notifier: Download added - {dl.title}")
|
|
await sio.emit('added', serializer.encode(dl))
|
|
|
|
async def updated(self, dl):
|
|
log.info(f"Notifier: Download updated - {dl.title}")
|
|
await sio.emit('updated', serializer.encode(dl))
|
|
|
|
async def completed(self, dl):
|
|
log.info(f"Notifier: Download completed - {dl.title}")
|
|
await sio.emit('completed', serializer.encode(dl))
|
|
|
|
async def canceled(self, id):
|
|
log.info(f"Notifier: Download canceled - {id}")
|
|
await sio.emit('canceled', serializer.encode(id))
|
|
|
|
async def cleared(self, id):
|
|
log.info(f"Notifier: Download cleared - {id}")
|
|
await sio.emit('cleared', serializer.encode(id))
|
|
|
|
async def event(self, event):
|
|
log.info(f"Notifier: Event - {event['type']}")
|
|
await sio.emit('event', serializer.encode(event))
|
|
|
|
dqueue = DownloadQueue(config, Notifier())
|
|
app.on_startup.append(lambda app: dqueue.initialize())
|
|
|
|
class FileOpsFilter(DefaultFilter):
|
|
def __call__(self, change_type: int, path: str) -> bool:
|
|
# Check if this path matches our YTDL_OPTIONS_FILE
|
|
if path != config.YTDL_OPTIONS_FILE:
|
|
return False
|
|
|
|
# For existing files, use samefile comparison to handle symlinks correctly
|
|
if os.path.exists(config.YTDL_OPTIONS_FILE):
|
|
try:
|
|
if not os.path.samefile(path, config.YTDL_OPTIONS_FILE):
|
|
return False
|
|
except (OSError, IOError):
|
|
# If samefile fails, fall back to string comparison
|
|
if path != config.YTDL_OPTIONS_FILE:
|
|
return False
|
|
|
|
# Accept all change types for our file: modified, added, deleted
|
|
return change_type in (Change.modified, Change.added, Change.deleted)
|
|
|
|
def get_options_update_time(success=True, msg=''):
|
|
result = {
|
|
'success': success,
|
|
'msg': msg,
|
|
'update_time': None
|
|
}
|
|
|
|
# Only try to get file modification time if YTDL_OPTIONS_FILE is set and file exists
|
|
if config.YTDL_OPTIONS_FILE and os.path.exists(config.YTDL_OPTIONS_FILE):
|
|
try:
|
|
result['update_time'] = os.path.getmtime(config.YTDL_OPTIONS_FILE)
|
|
except (OSError, IOError) as e:
|
|
log.warning(f"Could not get modification time for {config.YTDL_OPTIONS_FILE}: {e}")
|
|
result['update_time'] = None
|
|
|
|
return result
|
|
|
|
async def watch_files():
|
|
async def _watch_files():
|
|
async for changes in awatch(config.YTDL_OPTIONS_FILE, watch_filter=FileOpsFilter()):
|
|
success, msg = config.load_ytdl_options()
|
|
result = get_options_update_time(success, msg)
|
|
await sio.emit('ytdl_options_changed', serializer.encode(result))
|
|
|
|
log.info(f'Starting Watch File: {config.YTDL_OPTIONS_FILE}')
|
|
asyncio.create_task(_watch_files())
|
|
|
|
if config.YTDL_OPTIONS_FILE:
|
|
app.on_startup.append(lambda app: watch_files())
|
|
|
|
@routes.post(config.URL_PREFIX + 'add')
|
|
async def add(request):
|
|
log.info("Received request to add download")
|
|
post = await request.json()
|
|
log.info(f"Request data: {post}")
|
|
url = post.get('url')
|
|
quality = post.get('quality')
|
|
if not url or not quality:
|
|
log.error("Bad request: missing 'url' or 'quality'")
|
|
raise web.HTTPBadRequest()
|
|
format = post.get('format')
|
|
folder = post.get('folder')
|
|
custom_name_prefix = post.get('custom_name_prefix')
|
|
playlist_strict_mode = post.get('playlist_strict_mode')
|
|
playlist_item_limit = post.get('playlist_item_limit')
|
|
auto_start = post.get('auto_start')
|
|
|
|
if custom_name_prefix is None:
|
|
custom_name_prefix = ''
|
|
if auto_start is None:
|
|
auto_start = True
|
|
if playlist_strict_mode is None:
|
|
playlist_strict_mode = config.DEFAULT_OPTION_PLAYLIST_STRICT_MODE
|
|
if playlist_item_limit is None:
|
|
playlist_item_limit = config.DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT
|
|
|
|
playlist_item_limit = int(playlist_item_limit)
|
|
|
|
status = await dqueue.add(url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start)
|
|
return web.Response(text=serializer.encode(status))
|
|
|
|
@routes.post(config.URL_PREFIX + 'cookie')
|
|
async def set_cookie(request):
|
|
"""Accept cookie string and save as cookie file for domain"""
|
|
log.info("Received request to set cookie")
|
|
post = await request.json()
|
|
url = post.get('url')
|
|
cookie = post.get('cookie')
|
|
domain = post.get('domain')
|
|
|
|
if not cookie:
|
|
log.error("Bad request: missing 'cookie'")
|
|
raise web.HTTPBadRequest()
|
|
|
|
# Determine domain from either explicit domain field or URL
|
|
if not domain:
|
|
if url:
|
|
parsed_url = urlparse(url)
|
|
domain = parsed_url.netloc
|
|
else:
|
|
log.error("Bad request: missing both 'url' and 'domain'")
|
|
raise web.HTTPBadRequest()
|
|
|
|
log.info(f"Processing cookie for domain: {domain}")
|
|
|
|
try:
|
|
# Decode base64 cookie if it appears to be encoded
|
|
try:
|
|
# Check if cookie is base64 encoded
|
|
decoded_cookie = base64.b64decode(cookie).decode('utf-8')
|
|
log.info(f"Cookie was base64 encoded, decoded successfully")
|
|
cookie = decoded_cookie
|
|
except Exception as e:
|
|
# If decoding fails, assume it's already plain text
|
|
log.info(f"Cookie is not base64 encoded or decode failed ({e}), using as-is")
|
|
|
|
log.debug(f"Cookie content: {cookie[:100]}...") # Log first 100 chars
|
|
|
|
# Create cookies directory if it doesn't exist
|
|
cookies_dir = os.path.join(config.STATE_DIR, 'cookies')
|
|
os.makedirs(cookies_dir, exist_ok=True)
|
|
|
|
# Use domain as filename (sanitized)
|
|
safe_domain = domain.replace(':', '_').replace('/', '_')
|
|
cookie_file = os.path.join(cookies_dir, f'{safe_domain}.txt')
|
|
|
|
log.info(f"Writing cookie file to: {cookie_file}")
|
|
|
|
# Convert cookie string to Netscape cookie file format
|
|
with open(cookie_file, 'w') as f:
|
|
f.write('# Netscape HTTP Cookie File\n')
|
|
f.write(f'# This file was generated by MeTube for {domain}\n')
|
|
f.write('# Edit at your own risk.\n\n')
|
|
|
|
# Parse cookie string (format: "key1=value1; key2=value2; ...")
|
|
cookie_count = 0
|
|
for cookie_pair in cookie.split(';'):
|
|
cookie_pair = cookie_pair.strip()
|
|
if '=' in cookie_pair:
|
|
key, value = cookie_pair.split('=', 1)
|
|
key = key.strip()
|
|
value = value.strip()
|
|
# Netscape format: domain\tflag\tpath\tsecure\texpiration\tname\tvalue
|
|
# domain: .domain.com (with leading dot for all subdomains)
|
|
# flag: TRUE (include subdomains)
|
|
# path: / (all paths)
|
|
# secure: FALSE (http and https)
|
|
# expiration: 2147483647 (max 32-bit timestamp - Jan 2038)
|
|
# name: cookie name
|
|
# value: cookie value
|
|
f.write(f'.{domain}\tTRUE\t/\tFALSE\t2147483647\t{key}\t{value}\n')
|
|
cookie_count += 1
|
|
log.debug(f"Added cookie: {key}={value[:20]}...")
|
|
|
|
log.info(f"Cookie file created successfully with {cookie_count} cookies at {cookie_file}")
|
|
return web.Response(text=serializer.encode({
|
|
'status': 'ok',
|
|
'cookie_file': cookie_file,
|
|
'cookie_count': cookie_count,
|
|
'msg': f'Cookie saved successfully for {domain} ({cookie_count} cookies)'
|
|
}))
|
|
except Exception as e:
|
|
log.error(f"Error saving cookie: {str(e)}", exc_info=True)
|
|
return web.Response(text=serializer.encode({
|
|
'status': 'error',
|
|
'msg': f'Failed to save cookie: {str(e)}'
|
|
}))
|
|
|
|
PLACEHOLDER_THUMB = '''<svg xmlns="http://www.w3.org/2000/svg" width="320" height="180" viewBox="0 0 320 180">
|
|
<rect width="320" height="180" fill="#e8eaed"/>
|
|
<g fill="none" stroke="#9aa0a6" stroke-width="6" stroke-linecap="round" stroke-linejoin="round">
|
|
<rect x="125" y="60" width="70" height="60" rx="8"/>
|
|
<polyline points="150 75 165 90 150 105"/>
|
|
</g>
|
|
</svg>'''
|
|
|
|
|
|
def sniff_image_type(data: bytes) -> str:
|
|
if data[:3] == b'\xff\xd8\xff':
|
|
return 'image/jpeg'
|
|
if data[:8] == b'\x89PNG\r\n\x1a\n':
|
|
return 'image/png'
|
|
if data[:4] == b'GIF8':
|
|
return 'image/gif'
|
|
if data[:4] == b'RIFF' and data[8:12] == b'WEBP':
|
|
return 'image/webp'
|
|
if data[:12] == b'\x00\x00\x00\x1cftyp':
|
|
return 'image/avif'
|
|
return 'application/octet-stream'
|
|
|
|
|
|
# ---------- Library (large media collection, separate from DOWNLOAD_DIR) ----------
|
|
|
|
VIDEO_EXTENSIONS = ('.mp4', '.mkv', '.webm', '.avi', '.mov', '.m4v', '.ts', '.flv', '.mpg', '.mpeg', '.wmv', '.m2ts')
|
|
|
|
|
|
def _library_root():
|
|
"""Normalized [(label, realpath)] library roots; empty = feature off."""
|
|
return config.LIBRARY_ROOTS
|
|
|
|
|
|
def _resolve_library_path(rel):
|
|
"""Resolve a client-supplied library-relative path. Returns (root, path);
|
|
path is None when it would escape the library root(s)."""
|
|
_, root, path = resolve_library_path(config.LIBRARY_ROOTS, rel or '')
|
|
return root, path
|
|
|
|
|
|
_library_folders_cache = {'time': 0.0, 'data': None}
|
|
_LIBRARY_FOLDERS_TTL = 60
|
|
|
|
|
|
def _scan_library_folders(roots):
|
|
exclude = re.compile(config.CUSTOM_DIRS_EXCLUDE_REGEX) if config.CUSTOM_DIRS_EXCLUDE_REGEX else None
|
|
folders = []
|
|
for label, root in roots:
|
|
def _on_walk_error(exc):
|
|
log.warning(f'Library scan error under "{root}": {exc!r}')
|
|
|
|
for dirpath, dirnames, filenames in os.walk(root, onerror=_on_walk_error):
|
|
rel = os.path.relpath(dirpath, root)
|
|
dirnames[:] = [d for d in dirnames
|
|
if not (exclude and exclude.search(os.path.normpath(os.path.join(rel, d))))]
|
|
count = sum(1 for f in filenames if os.path.splitext(f)[1].lower() in VIDEO_EXTENSIONS)
|
|
name = '' if rel == '.' else rel
|
|
path = name if not label else (label + '/' + name if name else label)
|
|
folders.append({'label': label, 'name': name, 'path': path, 'count': count})
|
|
folders.sort(key=lambda f: (f['label'], f['name'].lower()))
|
|
return folders
|
|
|
|
|
|
def _invalidate_library_folders():
|
|
_library_folders_cache['time'] = 0.0
|
|
|
|
|
|
@routes.get(config.URL_PREFIX + 'library/folders')
|
|
async def library_folders(request):
|
|
roots = _library_root()
|
|
if not roots:
|
|
raise web.HTTPBadRequest(text='LIBRARY_DIRS is not configured')
|
|
now = time.time()
|
|
if _library_folders_cache['data'] is None or now - _library_folders_cache['time'] > _LIBRARY_FOLDERS_TTL:
|
|
_library_folders_cache['data'] = await asyncio.get_running_loop().run_in_executor(None, _scan_library_folders, roots)
|
|
_library_folders_cache['time'] = now
|
|
return web.Response(text=serializer.encode({'folders': _library_folders_cache['data']}))
|
|
|
|
|
|
@routes.get(config.URL_PREFIX + 'library/files')
|
|
async def library_files(request):
|
|
folder = request.query.get('folder', '')
|
|
root, path = _resolve_library_path(folder)
|
|
if root is None:
|
|
raise web.HTTPBadRequest(text='LIBRARY_DIRS is not configured')
|
|
if path is None or not os.path.isdir(path):
|
|
raise web.HTTPBadRequest(text='Invalid library folder')
|
|
|
|
def _list():
|
|
entries = []
|
|
with os.scandir(path) as it:
|
|
for e in it:
|
|
if not e.is_file(follow_symlinks=False):
|
|
continue
|
|
if os.path.splitext(e.name)[1].lower() not in VIDEO_EXTENSIONS:
|
|
continue
|
|
st = e.stat()
|
|
entry = {
|
|
'name': e.name,
|
|
'path': folder + '/' + e.name if folder else e.name,
|
|
'size': st.st_size,
|
|
'mtime': st.st_mtime,
|
|
}
|
|
meta_file = library_meta_path(e.path)
|
|
if os.path.isfile(meta_file):
|
|
try:
|
|
with open(meta_file, encoding='utf-8') as f:
|
|
entry['meta'] = json.load(f)
|
|
except (OSError, ValueError) as ex:
|
|
log.debug(f'Ignoring unreadable library metadata sidecar {meta_file}: {ex!r}')
|
|
entries.append(entry)
|
|
entries.sort(key=lambda f: f['name'].lower())
|
|
return entries
|
|
|
|
files = await asyncio.get_running_loop().run_in_executor(None, _list)
|
|
return web.Response(text=serializer.encode({'files': files}))
|
|
|
|
|
|
@routes.post(config.URL_PREFIX + 'library/organize')
|
|
async def library_organize(request):
|
|
post = await request.json()
|
|
ids = post.get('ids')
|
|
folder = post.get('folder', '')
|
|
if not ids:
|
|
log.error("Bad request: missing 'ids'")
|
|
raise web.HTTPBadRequest()
|
|
status = await dqueue.organize_to_library(ids, folder)
|
|
if status.get('moved'):
|
|
_invalidate_library_folders()
|
|
log.info(f"Organize request processed: moved={status.get('moved')} errors={len(status.get('errors', []))}")
|
|
return web.Response(text=serializer.encode(status))
|
|
|
|
|
|
@routes.post(config.URL_PREFIX + 'library/delete')
|
|
async def library_delete(request):
|
|
post = await request.json()
|
|
paths = post.get('paths')
|
|
if not paths:
|
|
raise web.HTTPBadRequest()
|
|
if not _library_root():
|
|
raise web.HTTPBadRequest(text='LIBRARY_DIRS is not configured')
|
|
deleted = 0
|
|
errors = []
|
|
for rel in paths:
|
|
root, path = _resolve_library_path(rel)
|
|
if path is None or not os.path.isfile(path):
|
|
errors.append(f'{rel}: invalid path')
|
|
continue
|
|
try:
|
|
os.remove(path)
|
|
deleted += 1
|
|
# Drop the source-metadata sidecar too, if one was written at organize time
|
|
try:
|
|
os.remove(library_meta_path(path))
|
|
except OSError:
|
|
pass
|
|
# Clean up emptied parent directories, never the library root itself
|
|
parent = os.path.dirname(path)
|
|
while parent != root and parent.startswith(root + os.sep):
|
|
try:
|
|
os.rmdir(parent)
|
|
except OSError:
|
|
break
|
|
parent = os.path.dirname(parent)
|
|
except OSError as e:
|
|
errors.append(f'{rel}: {e}')
|
|
if deleted:
|
|
_invalidate_library_folders()
|
|
log.info(f"Library delete request processed: deleted={deleted} errors={len(errors)}")
|
|
return web.Response(text=serializer.encode({'status': 'ok', 'deleted': deleted, 'errors': errors}))
|
|
|
|
|
|
@routes.post(config.URL_PREFIX + 'library/move')
|
|
async def library_move(request):
|
|
post = await request.json()
|
|
paths = post.get('paths')
|
|
folder = post.get('folder', '')
|
|
if not paths:
|
|
raise web.HTTPBadRequest()
|
|
roots = _library_root()
|
|
if not roots:
|
|
raise web.HTTPBadRequest(text='LIBRARY_DIRS is not configured')
|
|
_, _, target_dir = resolve_library_path(roots, folder or '')
|
|
if target_dir is None:
|
|
raise web.HTTPBadRequest(text='Invalid library folder')
|
|
|
|
def _move():
|
|
os.makedirs(target_dir, exist_ok=True)
|
|
moved = 0
|
|
errors = []
|
|
for rel in paths:
|
|
root, src = _resolve_library_path(rel)
|
|
if src is None or not os.path.isfile(src):
|
|
errors.append(f'{rel}: invalid path')
|
|
continue
|
|
if os.path.realpath(os.path.dirname(src)) == target_dir:
|
|
continue
|
|
dst = os.path.join(target_dir, os.path.basename(src))
|
|
if os.path.exists(dst):
|
|
errors.append(f'{rel}: target already exists')
|
|
continue
|
|
try:
|
|
shutil.move(src, dst)
|
|
except OSError as e:
|
|
errors.append(f'{rel}: {e}')
|
|
continue
|
|
meta_src = library_meta_path(src)
|
|
if os.path.isfile(meta_src):
|
|
try:
|
|
shutil.move(meta_src, library_meta_path(dst))
|
|
except OSError:
|
|
pass
|
|
moved += 1
|
|
# Clean up emptied parent directories, never the library root itself
|
|
parent = os.path.dirname(src)
|
|
while parent != root and parent.startswith(root + os.sep):
|
|
try:
|
|
os.rmdir(parent)
|
|
except OSError:
|
|
break
|
|
parent = os.path.dirname(parent)
|
|
return moved, errors
|
|
|
|
moved, errors = await asyncio.get_running_loop().run_in_executor(None, _move)
|
|
if moved:
|
|
_invalidate_library_folders()
|
|
log.info(f"Library move request processed: moved={moved} errors={len(errors)}")
|
|
return web.Response(text=serializer.encode({'status': 'ok', 'moved': moved, 'errors': errors}))
|
|
|
|
|
|
@routes.get(config.URL_PREFIX + 'library/thumbnail')
|
|
async def library_thumbnail(request):
|
|
"""Thumbnail for a library file: ffmpeg frame extraction, cached on disk."""
|
|
rel = request.query.get('path')
|
|
root, path = _resolve_library_path(rel)
|
|
if root is None or path is None or not os.path.isfile(path):
|
|
return _placeholder_thumb_response()
|
|
thumb_dir = os.path.join(config.STATE_DIR, 'thumbnails')
|
|
os.makedirs(thumb_dir, exist_ok=True)
|
|
thumb_path = os.path.join(thumb_dir, hashlib.sha1(('lib:' + rel).encode('utf-8')).hexdigest() + '.img')
|
|
if not os.path.exists(thumb_path):
|
|
await _extract_video_frame(path, thumb_path, None)
|
|
if not os.path.exists(thumb_path):
|
|
return _placeholder_thumb_response()
|
|
with open(thumb_path, 'rb') as f:
|
|
data = f.read()
|
|
return web.Response(
|
|
body=data,
|
|
content_type=sniff_image_type(data[:16]),
|
|
headers={'Cache-Control': 'public, max-age=31536000, immutable'}
|
|
)
|
|
|
|
|
|
def _placeholder_thumb_response():
|
|
return web.Response(text=PLACEHOLDER_THUMB, content_type='image/svg+xml', headers={'Cache-Control': 'public, max-age=3600'})
|
|
|
|
|
|
async def _probe_duration(video_path):
|
|
"""Duration of a local media file in seconds via ffprobe, or None."""
|
|
cmd = ['ffprobe', '-v', 'error', '-show_entries', 'format=duration',
|
|
'-of', 'csv=p=0', video_path]
|
|
try:
|
|
proc = await asyncio.create_subprocess_exec(
|
|
*cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL)
|
|
out, _ = await asyncio.wait_for(proc.communicate(), timeout=15)
|
|
return float(out.decode().strip())
|
|
except Exception: # noqa: BLE001 - probing is best-effort
|
|
return None
|
|
|
|
|
|
async def _extract_video_frame(video_path, thumb_path, duration):
|
|
"""Extract a frame from a local video file with ffmpeg. Returns True on success."""
|
|
if not duration:
|
|
duration = await _probe_duration(video_path)
|
|
seek = 1.0
|
|
if duration:
|
|
try:
|
|
duration = float(duration)
|
|
seek = max(0.0, min(duration * 0.25, duration - 1.0))
|
|
except (TypeError, ValueError):
|
|
pass
|
|
cmd = ['ffmpeg', '-hide_banner', '-loglevel', 'error',
|
|
'-ss', str(seek), '-i', video_path,
|
|
'-frames:v', '1', '-vf', 'scale=480:-2', '-q:v', '4',
|
|
# thumb_path has a generic .img extension: force the JPEG encoder and
|
|
# image2 muxer explicitly (ffmpeg would otherwise guess from the name)
|
|
'-c:v', 'mjpeg', '-f', 'image2',
|
|
'-y', thumb_path]
|
|
try:
|
|
proc = await asyncio.create_subprocess_exec(
|
|
*cmd, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE)
|
|
_, err = await asyncio.wait_for(proc.communicate(), timeout=30)
|
|
except Exception as exc: # noqa: BLE001 - includes TimeoutError; fall through to other sources
|
|
log.warning(f"ffmpeg frame extraction failed for {video_path}: {exc}")
|
|
try:
|
|
proc.kill()
|
|
except Exception: # noqa: BLE001 - process may not exist
|
|
pass
|
|
return False
|
|
if proc.returncode != 0:
|
|
log.warning(f"ffmpeg frame extraction failed for {video_path}: {err.decode(errors='replace')[-200:]}")
|
|
return False
|
|
return os.path.exists(thumb_path)
|
|
|
|
|
|
@routes.get(config.URL_PREFIX + 'thumbnail')
|
|
async def thumbnail(request):
|
|
"""Serve a cached thumbnail for a download.
|
|
|
|
Sources, in order: a frame extracted from the local file (ffmpeg), the
|
|
remote thumbnail URL from stored yt-dlp metadata — clients can never point
|
|
this endpoint at arbitrary URLs — and finally a placeholder SVG.
|
|
"""
|
|
key = request.query.get('id')
|
|
if not key:
|
|
raise web.HTTPBadRequest()
|
|
|
|
info = dqueue.find_info(key)
|
|
thumb_url = getattr(info, 'thumbnail', None) if info is not None else None
|
|
log.debug(f"[Thumb] key={key[:60]} found={info is not None} has_url={bool(thumb_url)}")
|
|
|
|
thumb_dir = os.path.join(config.STATE_DIR, 'thumbnails')
|
|
os.makedirs(thumb_dir, exist_ok=True)
|
|
thumb_path = os.path.join(thumb_dir, hashlib.sha1(key.encode('utf-8')).hexdigest() + '.img')
|
|
|
|
if not os.path.exists(thumb_path):
|
|
# Prefer a frame from the local file: works even for downloads whose
|
|
# metadata predates thumbnail storage, and survives expiring remote URLs
|
|
local_file = dqueue.find_download_file(key)
|
|
log.debug(f"[Thumb] local_file={local_file}")
|
|
if local_file:
|
|
await _extract_video_frame(local_file, thumb_path, getattr(info, 'duration', None))
|
|
|
|
if not os.path.exists(thumb_path) and thumb_url:
|
|
try:
|
|
timeout = aiohttp.ClientTimeout(total=15)
|
|
async with aiohttp.ClientSession(timeout=timeout) as session:
|
|
async with session.get(thumb_url, headers={'User-Agent': 'Mozilla/5.0 MeTube'}) as resp:
|
|
if resp.status == 200:
|
|
data = await resp.read()
|
|
with open(thumb_path, 'wb') as f:
|
|
f.write(data)
|
|
except Exception as exc: # noqa: BLE001 - fall back to placeholder on any fetch error
|
|
log.warning(f"Failed to fetch thumbnail for {key}: {exc}")
|
|
|
|
if not os.path.exists(thumb_path):
|
|
return _placeholder_thumb_response()
|
|
|
|
with open(thumb_path, 'rb') as f:
|
|
data = f.read()
|
|
return web.Response(
|
|
body=data,
|
|
content_type=sniff_image_type(data[:16]),
|
|
headers={'Cache-Control': 'public, max-age=31536000, immutable'}
|
|
)
|
|
|
|
@routes.post(config.URL_PREFIX + 'delete')
|
|
async def delete(request):
|
|
post = await request.json()
|
|
ids = post.get('ids')
|
|
where = post.get('where')
|
|
if not ids or where not in ['queue', 'done']:
|
|
log.error("Bad request: missing 'ids' or incorrect 'where' value")
|
|
raise web.HTTPBadRequest()
|
|
status = await (dqueue.cancel(ids) if where == 'queue' else dqueue.clear(ids))
|
|
log.info(f"Download delete request processed for ids: {ids}, where: {where}")
|
|
return web.Response(text=serializer.encode(status))
|
|
|
|
@routes.post(config.URL_PREFIX + 'start')
|
|
async def start(request):
|
|
post = await request.json()
|
|
ids = post.get('ids')
|
|
log.info(f"Received request to start pending downloads for ids: {ids}")
|
|
status = await dqueue.start_pending(ids)
|
|
return web.Response(text=serializer.encode(status))
|
|
|
|
@routes.post(config.URL_PREFIX + 'set_folder')
|
|
async def set_folder(request):
|
|
post = await request.json()
|
|
ids = post.get('ids')
|
|
folder = post.get('folder', '')
|
|
if not ids:
|
|
log.error("Bad request: missing 'ids'")
|
|
raise web.HTTPBadRequest()
|
|
status = await dqueue.set_folder(ids, folder)
|
|
log.info(f"Folder change request processed for ids: {ids}, folder: '{folder}'")
|
|
return web.Response(text=serializer.encode(status))
|
|
|
|
@routes.get(config.URL_PREFIX + 'history')
|
|
async def history(request):
|
|
history = { 'done': [], 'queue': [], 'pending': []}
|
|
|
|
for _, v in dqueue.queue.saved_items():
|
|
history['queue'].append(v)
|
|
for _, v in dqueue.done.saved_items():
|
|
history['done'].append(v)
|
|
for _, v in dqueue.pending.saved_items():
|
|
history['pending'].append(v)
|
|
|
|
log.info("Sending download history")
|
|
return web.Response(text=serializer.encode(history))
|
|
|
|
@sio.event
|
|
async def connect(sid, environ):
|
|
log.info(f"Client connected: {sid}")
|
|
await sio.emit('all', serializer.encode(dqueue.get()), to=sid)
|
|
config_payload = {k: v for k, v in config.__dict__.items() if k != 'LIBRARY_ROOTS'}
|
|
await sio.emit('configuration', serializer.encode(config_payload), to=sid)
|
|
if config.CUSTOM_DIRS:
|
|
await sio.emit('custom_dirs', serializer.encode(get_custom_dirs()), to=sid)
|
|
if config.YTDL_OPTIONS_FILE:
|
|
await sio.emit('ytdl_options_changed', serializer.encode(get_options_update_time()), to=sid)
|
|
|
|
def get_custom_dirs():
|
|
def recursive_dirs(base):
|
|
path = pathlib.Path(base)
|
|
|
|
# Converts PosixPath object to string, and remove base/ prefix
|
|
def convert(p):
|
|
s = str(p)
|
|
if s.startswith(base):
|
|
s = s[len(base):]
|
|
|
|
if s.startswith('/'):
|
|
s = s[1:]
|
|
|
|
return s
|
|
|
|
# Include only directories which do not match the exclude filter
|
|
def include_dir(d):
|
|
if len(config.CUSTOM_DIRS_EXCLUDE_REGEX) == 0:
|
|
return True
|
|
else:
|
|
return re.search(config.CUSTOM_DIRS_EXCLUDE_REGEX, d) is None
|
|
|
|
# Recursively lists all subdirectories of DOWNLOAD_DIR
|
|
dirs = list(filter(include_dir, map(convert, path.glob('**/'))))
|
|
|
|
return dirs
|
|
|
|
download_dir = recursive_dirs(config.DOWNLOAD_DIR)
|
|
|
|
audio_download_dir = download_dir
|
|
if config.DOWNLOAD_DIR != config.AUDIO_DOWNLOAD_DIR:
|
|
audio_download_dir = recursive_dirs(config.AUDIO_DOWNLOAD_DIR)
|
|
|
|
return {
|
|
"download_dir": download_dir,
|
|
"audio_download_dir": audio_download_dir
|
|
}
|
|
|
|
@routes.get(config.URL_PREFIX)
|
|
def index(request):
|
|
response = web.FileResponse(os.path.join(config.BASE_DIR, 'ui/dist/metube/browser/index.html'))
|
|
if 'metube_theme' not in request.cookies:
|
|
response.set_cookie('metube_theme', config.DEFAULT_THEME)
|
|
return response
|
|
|
|
@routes.get(config.URL_PREFIX + 'robots.txt')
|
|
def robots(request):
|
|
if config.ROBOTS_TXT:
|
|
response = web.FileResponse(os.path.join(config.BASE_DIR, config.ROBOTS_TXT))
|
|
else:
|
|
response = web.Response(
|
|
text="User-agent: *\nDisallow: /download/\nDisallow: /audio_download/\n"
|
|
)
|
|
return response
|
|
|
|
@routes.get(config.URL_PREFIX + 'version')
|
|
def version(request):
|
|
return web.json_response({
|
|
"yt-dlp": yt_dlp_version,
|
|
"version": os.getenv("METUBE_VERSION", "dev")
|
|
})
|
|
|
|
@routes.get(config.URL_PREFIX + 'events')
|
|
def get_events(request):
|
|
events = dqueue.get_events()
|
|
return web.Response(text=serializer.encode(events))
|
|
|
|
@routes.post(config.URL_PREFIX + 'events/clear')
|
|
async def clear_events(request):
|
|
dqueue.clear_events()
|
|
return web.Response(text=serializer.encode({'status': 'ok'}))
|
|
|
|
if config.URL_PREFIX != '/':
|
|
@routes.get('/')
|
|
def index_redirect_root(request):
|
|
return web.HTTPFound(config.URL_PREFIX)
|
|
|
|
@routes.get(config.URL_PREFIX[:-1])
|
|
def index_redirect_dir(request):
|
|
return web.HTTPFound(config.URL_PREFIX)
|
|
|
|
if config.LIBRARY_ROOTS:
|
|
if len(config.LIBRARY_ROOTS) == 1:
|
|
# Legacy single-root mode: library files are served from 'library/'
|
|
routes.static(config.URL_PREFIX + 'library/', config.LIBRARY_ROOTS[0][1], show_index=False)
|
|
else:
|
|
# Multi-root mode: each root gets its own mount at 'library/<label>/'
|
|
for label, root in config.LIBRARY_ROOTS:
|
|
routes.static(config.URL_PREFIX + 'library/' + label + '/', root, show_index=False)
|
|
routes.static(config.URL_PREFIX + 'download/', config.DOWNLOAD_DIR, show_index=config.DOWNLOAD_DIRS_INDEXABLE)
|
|
routes.static(config.URL_PREFIX + 'audio_download/', config.AUDIO_DOWNLOAD_DIR, show_index=config.DOWNLOAD_DIRS_INDEXABLE)
|
|
routes.static(config.URL_PREFIX, os.path.join(config.BASE_DIR, 'ui/dist/metube/browser'))
|
|
try:
|
|
app.add_routes(routes)
|
|
except ValueError as e:
|
|
if 'ui/dist/metube/browser' in str(e):
|
|
raise RuntimeError('Could not find the frontend UI static assets. Please run `node_modules/.bin/ng build` inside the ui folder') from e
|
|
raise e
|
|
|
|
# https://github.com/aio-libs/aiohttp/pull/4615 waiting for release
|
|
# @routes.options(config.URL_PREFIX + 'add')
|
|
async def add_cors(request):
|
|
return web.Response(text=serializer.encode({"status": "ok"}))
|
|
|
|
async def cookie_cors(request):
|
|
return web.Response(text=serializer.encode({"status": "ok"}))
|
|
|
|
app.router.add_route('OPTIONS', config.URL_PREFIX + 'add', add_cors)
|
|
app.router.add_route('OPTIONS', config.URL_PREFIX + 'cookie', cookie_cors)
|
|
|
|
async def on_prepare(request, response):
|
|
if 'Origin' in request.headers:
|
|
response.headers['Access-Control-Allow-Origin'] = request.headers['Origin']
|
|
response.headers['Access-Control-Allow-Headers'] = 'Content-Type'
|
|
|
|
app.on_response_prepare.append(on_prepare)
|
|
|
|
def supports_reuse_port():
|
|
try:
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
|
|
sock.close()
|
|
return True
|
|
except (AttributeError, OSError):
|
|
return False
|
|
|
|
def parseLogLevel(logLevel):
|
|
match logLevel:
|
|
case 'DEBUG':
|
|
return logging.DEBUG
|
|
case 'INFO':
|
|
return logging.INFO
|
|
case 'WARNING':
|
|
return logging.WARNING
|
|
case 'ERROR':
|
|
return logging.ERROR
|
|
case 'CRITICAL':
|
|
return logging.CRITICAL
|
|
case _:
|
|
return None
|
|
|
|
def isAccessLogEnabled():
|
|
if config.ENABLE_ACCESSLOG:
|
|
return access_logger
|
|
else:
|
|
return None
|
|
|
|
if __name__ == '__main__':
|
|
logging.basicConfig(level=parseLogLevel(config.LOGLEVEL))
|
|
log.info(f"Listening on {config.HOST}:{config.PORT}")
|
|
|
|
if config.HTTPS:
|
|
ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
|
|
ssl_context.load_cert_chain(certfile=config.CERTFILE, keyfile=config.KEYFILE)
|
|
web.run_app(app, host=config.HOST, port=int(config.PORT), reuse_port=supports_reuse_port(), ssl_context=ssl_context, access_log=isAccessLogEnabled())
|
|
else:
|
|
web.run_app(app, host=config.HOST, port=int(config.PORT), reuse_port=supports_reuse_port(), access_log=isAccessLogEnabled())
|