From 3aa106f003baf05c916fa5e060bee37c69e54c7a Mon Sep 17 00:00:00 2001 From: tigerenwork Date: Thu, 13 Aug 2026 00:28:35 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E5=AA=92=E4=BD=93?= =?UTF-8?q?=E5=BA=93=E6=95=B4=E7=90=86=E4=B8=8E=E6=B5=8F=E8=A7=88=E5=8A=9F?= =?UTF-8?q?=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AGENTS.md | 6 + LIBRARY_ORGANIZE_DESIGN.md | 114 ++++++++ README.md | 2 + app/main.py | 160 +++++++++++ app/ytdl.py | 49 ++++ docker-compose.yml | 4 + ui/src/app/app.component.html | 58 +++- ui/src/app/app.component.ts | 162 ++++++++++- ui/src/app/app.module.ts | 2 + ui/src/app/download-card.component.html | 3 + ui/src/app/download-card.component.ts | 5 +- ui/src/app/download-list.component.html | 12 +- ui/src/app/download-list.component.ts | 7 + ui/src/app/download-row.component.html | 3 + ui/src/app/download-row.component.ts | 5 +- ui/src/app/downloads.service.ts | 43 +++ ui/src/app/library-browser.component.html | 88 ++++++ ui/src/app/library-browser.component.sass | 325 ++++++++++++++++++++++ ui/src/app/library-browser.component.ts | 65 +++++ ui/src/app/sidebar.component.html | 27 +- ui/src/app/sidebar.component.ts | 28 +- 21 files changed, 1156 insertions(+), 12 deletions(-) create mode 100644 LIBRARY_ORGANIZE_DESIGN.md create mode 100644 ui/src/app/library-browser.component.html create mode 100644 ui/src/app/library-browser.component.sass create mode 100644 ui/src/app/library-browser.component.ts diff --git a/AGENTS.md b/AGENTS.md index bc7c53f..d67759b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,3 +11,9 @@ ## UI - Angular app in `ui/`. Build with `cd ui && npm run build`. The Docker image builds the UI itself (see `Dockerfile`), so no need to commit `ui/dist`. + +## Media library (organize/browse) feature + +- `LIBRARY_DIR` (empty = feature off) points to a separate media library directory, mounted as its own volume. See `LIBRARY_ORGANIZE_DESIGN.md` for the full design. +- Library state is read from disk on demand — no DB/index. `/library/folders` is cached ~60s in memory; organize/delete invalidate it. +- Organizing a completed download moves its file into the library and removes the done entry (a `cleared` socket event updates the UI). diff --git a/LIBRARY_ORGANIZE_DESIGN.md b/LIBRARY_ORGANIZE_DESIGN.md new file mode 100644 index 0000000..54f25c8 --- /dev/null +++ b/LIBRARY_ORGANIZE_DESIGN.md @@ -0,0 +1,114 @@ +# Library & Organize 功能设计文档 + +## 背景与目标 + +目前 MeTube 只有一个下载目录(`DOWNLOAD_DIR`),"文件夹" 只是其子目录。用户另有一个大型媒体库(数百个文件夹,按演员/主题组织),需要: + +1. **整理(Organize)**:把下载目录中的文件移动进媒体库的某个文件夹;目标文件夹选择要支持"快速匹配"(搜索 + 自动建议)。 +2. **浏览媒体库**:按文件夹导航、页内播放视频、可删除文件。媒体库在体验上类似"已完成",但数据源是磁盘上的大库,而非下载记录。 + +### 已确认的关键决策 + +- 文件整理进媒体库后,**从"已完成"列表移除**(媒体库是它的唯一家)。 +- 快速匹配 = **可搜索的文件夹选择器 + 基于标题/uploader 的自动建议**,两者都要。 +- 播放走 **aiohttp 静态路由**(与下载目录一致,天然支持 Range 拖拽)。 + +## 总体架构 + +``` +下载区 (DOWNLOAD_DIR) 媒体库 (LIBRARY_DIR, 新 volume) +┌────────────────────┐ organize ┌──────────────────────────┐ +│ 已完成条目 (shelve) │ ──────────→ │ 文件夹A/ 文件夹B/ ... │ +│ 文件 + 元数据 │ 移动文件 │ 纯磁盘状态,无 DB │ +└────────────────────┘ 删除 shelve └──────────────────────────┘ + 条目 ↑ 浏览/播放/删除 + (实时扫盘 + 短 TTL 缓存) +``` + +媒体库**不建索引、不入库**:状态实时从磁盘读取,文件夹列表做 ~60s 内存缓存。库可能很大,但 `os.scandir` 列目录是毫秒级;只有"文件夹列表 + 文件数"需要遍历一次顶层目录,缓存即可。 + +## 后端设计 (app/main.py 为主) + +### 配置 + +- 新增 `LIBRARY_DIR`(默认空 = 功能关闭)、`PUBLIC_HOST_LIBRARY_URL`(默认 `library/`)。 +- `docker-compose.yml`:挂载 `./library:/library`,设 `LIBRARY_DIR=/library`。 +- 功能开关:`LIBRARY_DIR` 为空时,所有 `/library/*` API 返回 400,前端隐藏媒体库入口和"整理"按钮。configuration socket 事件已会把配置发给前端,UI 据此显隐。 + +### API 一览 + +| 端点 | 方法 | 说明 | +|---|---|---| +| `/library/folders` | GET | 递归列出库内文件夹(相对路径)+ 每个文件夹的视频文件数。内存缓存 ~60s,organize/delete 后主动失效。排除 `CUSTOM_DIRS_EXCLUDE_REGEX` 匹配的隐藏目录 | +| `/library/files?folder=` | GET | 列出某文件夹内的视频文件:文件名、大小、mtime;按名称排序。`folder` 为空 = 库根目录 | +| `/library/organize` | POST | `{ids: [...], folder: ''}`。仅接受 **done** 条目:逐个解析文件路径 → 校验/创建目标文件夹(必须在 LIBRARY_DIR 内)→ `shutil.move`(executor 中执行,跨盘时为拷贝+删除,大文件可能较慢)→ 冲突时按现有约定追加 5 位随机后缀并改名 → 从 done 存储删除 → 发送 `cleared` socket 事件让前端移除条目。逐项收集成败,返回 `{status, moved: n, errors: [...]}` | +| `/library/delete` | POST | `{paths: [...]}`(库内相对路径)。仅删文件、拒绝目录;删除后尝试清理变空的父目录(不删库根) | +| `/library/thumbnail?path=` | GET | 复用现有 `_extract_video_frame`(ffprobe 定位 25% + ffmpeg 抽帧),缓存键用 `sha1('lib:'+relpath)`,与下载缩略图同目录但命名空间隔离。无本地文件概念之外的 fallback:失败返回占位 SVG | + +### 静态播放路由 + +- `routes.static(config.URL_PREFIX + 'library/', config.LIBRARY_DIR)` — 注册在 `/library/*` 动态 API **之后**(aiohttp 先注册先匹配,现有静态路由本就在文件末尾)。 +- 前端拼播放 URL:`library/` + `encodeURIComponent(folder + '/' + filename)`。 + +### 路径安全 + +所有接受相对路径的端点统一走一个校验函数:`realpath(join(LIBRARY_DIR, rel))` 必须以 `realpath(LIBRARY_DIR)` 为前缀,否则 400。与现有 `__calc_download_path` 同款逻辑。 + +### 自动建议匹配(前端实现) + +文件夹列表本来就要发给前端,因此匹配放客户端,零额外请求: + +- 输入:待整理条目的 `title` 与 `entry.uploader`(uploader 通常就是演员名,是最强信号)。 +- 归一化:小写、去非字母数字、按空格分词。 +- 打分:uploader 与文件夹名完全相等(归一化后)= 最高分;子串包含次之;token 交集比例再次。取 Top 3 作为建议。 + +## 前端设计 (ui/src/app) + +### 导航模式 + +- `AppComponent` 增加 `mode: 'downloads' | 'library'`(持久化到 preferences)。 +- 侧边栏"状态"区下方新增"媒体库"入口(带视频总数);点击切到 library 模式,此时"文件夹"区改列**媒体库文件夹**(数据来自 `/library/folders`,复用现有 nav-item 样式与计数)。 +- 主区域:library 模式渲染新的 `LibraryBrowserComponent`,否则渲染现有的 active-downloads + download-list。 + +### LibraryBrowserComponent(新) + +- 工具条:搜索框(按文件名过滤)、排序(名称/时间/大小)、视图切换(列表/网格)——均复用现有模式。 +- 条目卡片/行:缩略图(`/library/thumbnail?path=`)、文件名、大小、mtime;操作:**播放**(复用现有页内播放器 modal,URL 指向静态路由)、**删除**(confirm 后调 `/library/delete`)。 +- 数据量:单文件夹内文件数通常有限(几十~几百),无需虚拟滚动;若后续需要可加 cdk-virtual-scroll。 + +### 整理流程(已完成视图) + +- 批量操作条新增"**整理到媒体库…**"按钮(在"移动到…"旁),选中条目后点击打开整理面板: + 1. **建议文件夹**(Top 3,客户端匹配,一键选择) + 2. **搜索框**:输入即过滤全部库文件夹(ng-select 或自绘列表,复用样式) + 3. **新建文件夹**:输入名创建(多层如 `A/B` 允许) +- 确认 → `POST /library/organize` → 成功条目从"已完成"消失(`cleared` 事件驱动)+ toast 提示移动数量;部分失败时 toast 列出失败原因。 +- 单条整理:已完成行/卡片的操作区加"整理"图标,打开同一面板(预选该条目)。 + +## 边界与异常 + +- `LIBRARY_DIR` 未配置:API 400 + UI 隐藏入口。 +- 目标/待删路径逃逸库根(`../`):400。 +- 整理 pending/downloading 条目:跳过并计入 errors(只有已完成文件可整理)。 +- 目标已存在同名文件:追加 `_XXXXX` 后缀(与下载区 move 行为一致),条目不丢失。 +- 跨文件系统移动大文件:executor 中执行,前端给 organizing 状态;失败保留原文件与条目。 +- 库文件被外部改动:文件夹缓存 60s 过期后自动反映;播放/删除前都实时校验路径存在。 + +## 实施步骤 + +1. 后端:`Config` 增加 `LIBRARY_DIR` / `PUBLIC_HOST_LIBRARY_URL`;路径校验辅助函数。 +2. 后端:`/library/folders`、`/library/files`(含缓存)。 +3. 后端:`/library/organize`(复用 move/conflict 逻辑 + done 条目删除 + cleared 事件)。 +4. 后端:`/library/delete`、`/library/thumbnail`、静态路由。 +5. 前端:`downloads.service.ts` 增加 library 相关方法(或直接 HttpClient)。 +6. 前端:侧边栏模式切换 + 媒体库文件夹列表。 +7. 前端:`LibraryBrowserComponent`(列表/网格、播放、删除)。 +8. 前端:整理面板(建议 + 搜索 + 新建),接入批量条与行操作。 +9. 样式与中文文案,与现有设计语言对齐。 +10. 验证:curl 端到端(建库目录→整理→文件移动+条目消失→播放 URL 200 且支持 Range→删除生效);UI build;重启持久性检查。 +11. 文档:README 与 AGENTS.md 提及新环境变量;docker-compose.yml 增加挂载示例。 + +## 交付物 + +- 本文档保存为仓库根的 `LIBRARY_ORGANIZE_DESIGN.md`(批准后开始实现时一并提交)。 +- 全部代码改动 + 本地容器验证。 diff --git a/README.md b/README.md index 6bade27..2e1dd91 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,8 @@ Certain values can be set via environment variables, using the `-e` parameter on * __TEMP_DIR__: Path where intermediary download files will be saved. Defaults to `/downloads` in the Docker image, and `.` otherwise. * Set this to an SSD or RAM filesystem (e.g., `tmpfs`) for better performance. * __Note__: Using a RAM filesystem may prevent downloads from being resumed. +* __LIBRARY_DIR__: Path to a separate media library directory (mounted into the container, e.g. `/library`). When set, the UI gains a **媒体库 (Library)** view — browse library folders, play videos in-page, delete files — and completed downloads can be **organized** (moved) into library folders, with folder search and title/uploader-based suggestions. Organized items are removed from the Completed list. Leave empty to disable the feature. Defaults to empty. +* __PUBLIC_HOST_LIBRARY_URL__: URL prefix under which library files are served. Defaults to `library/`. ### 📝 File Naming & yt-dlp diff --git a/app/main.py b/app/main.py index ae33ea0..df8735b 100644 --- a/app/main.py +++ b/app/main.py @@ -14,6 +14,7 @@ import logging import json import pathlib import re +import time import base64 import hashlib import aiohttp @@ -36,6 +37,8 @@ class Config: 'CUSTOM_DIRS_EXCLUDE_REGEX': r'(^|/)[.@].*$', 'DELETE_FILE_ON_TRASHCAN': 'true', 'STATE_DIR': '.', + 'LIBRARY_DIR': '', + 'PUBLIC_HOST_LIBRARY_URL': 'library/', 'URL_PREFIX': '', 'PUBLIC_HOST_URL': 'download/', 'PUBLIC_HOST_AUDIO_URL': 'audio_download/', @@ -357,6 +360,161 @@ def sniff_image_type(data: bytes) -> str: return 'application/octet-stream' +# ---------- Library (large media collection, separate from DOWNLOAD_DIR) ---------- + +VIDEO_EXTENSIONS = ('.mp4', '.mkv', '.webm', '.avi', '.mov', '.m4v', '.ts', '.flv', '.mpg', '.mpeg', '.wmv', '.m2ts') + + +def _library_root(): + """Realpath of the configured library directory, or None when the feature is off.""" + if not config.LIBRARY_DIR: + return None + return os.path.realpath(config.LIBRARY_DIR) + + +def _resolve_library_path(rel): + """Resolve a client-supplied library-relative path. Returns (root, path); + path is None when it would escape the library root.""" + root = _library_root() + if root is None: + return None, None + path = os.path.realpath(os.path.join(root, rel or '')) + if path != root and not path.startswith(root + os.sep): + return root, None + return root, path + + +_library_folders_cache = {'time': 0.0, 'data': None} +_LIBRARY_FOLDERS_TTL = 60 + + +def _scan_library_folders(root): + exclude = re.compile(config.CUSTOM_DIRS_EXCLUDE_REGEX) if config.CUSTOM_DIRS_EXCLUDE_REGEX else None + folders = [] + for dirpath, dirnames, filenames in os.walk(root): + rel = os.path.relpath(dirpath, root) + dirnames[:] = [d for d in dirnames + if not (exclude and exclude.search(os.path.normpath(os.path.join(rel, d))))] + count = sum(1 for f in filenames if os.path.splitext(f)[1].lower() in VIDEO_EXTENSIONS) + folders.append({'name': '' if rel == '.' else rel, 'count': count}) + folders.sort(key=lambda f: f['name'].lower()) + return folders + + +def _invalidate_library_folders(): + _library_folders_cache['time'] = 0.0 + + +@routes.get(config.URL_PREFIX + 'library/folders') +async def library_folders(request): + root = _library_root() + if root is None: + raise web.HTTPBadRequest(text='LIBRARY_DIR is not configured') + now = time.time() + if _library_folders_cache['data'] is None or now - _library_folders_cache['time'] > _LIBRARY_FOLDERS_TTL: + _library_folders_cache['data'] = await asyncio.get_running_loop().run_in_executor(None, _scan_library_folders, root) + _library_folders_cache['time'] = now + return web.Response(text=serializer.encode({'folders': _library_folders_cache['data']})) + + +@routes.get(config.URL_PREFIX + 'library/files') +async def library_files(request): + root, path = _resolve_library_path(request.query.get('folder', '')) + if root is None: + raise web.HTTPBadRequest(text='LIBRARY_DIR is not configured') + if path is None or not os.path.isdir(path): + raise web.HTTPBadRequest(text='Invalid library folder') + + def _list(): + entries = [] + with os.scandir(path) as it: + for e in it: + if not e.is_file(follow_symlinks=False): + continue + if os.path.splitext(e.name)[1].lower() not in VIDEO_EXTENSIONS: + continue + st = e.stat() + entries.append({'name': e.name, 'size': st.st_size, 'mtime': st.st_mtime}) + entries.sort(key=lambda f: f['name'].lower()) + return entries + + files = await asyncio.get_running_loop().run_in_executor(None, _list) + return web.Response(text=serializer.encode({'files': files})) + + +@routes.post(config.URL_PREFIX + 'library/organize') +async def library_organize(request): + post = await request.json() + ids = post.get('ids') + folder = post.get('folder', '') + if not ids: + log.error("Bad request: missing 'ids'") + raise web.HTTPBadRequest() + status = await dqueue.organize_to_library(ids, folder) + if status.get('moved'): + _invalidate_library_folders() + log.info(f"Organize request processed: moved={status.get('moved')} errors={len(status.get('errors', []))}") + return web.Response(text=serializer.encode(status)) + + +@routes.post(config.URL_PREFIX + 'library/delete') +async def library_delete(request): + post = await request.json() + paths = post.get('paths') + if not paths: + raise web.HTTPBadRequest() + root = _library_root() + if root is None: + raise web.HTTPBadRequest(text='LIBRARY_DIR is not configured') + deleted = 0 + errors = [] + for rel in paths: + _, path = _resolve_library_path(rel) + if path is None or not os.path.isfile(path): + errors.append(f'{rel}: invalid path') + continue + try: + os.remove(path) + deleted += 1 + # Clean up emptied parent directories, never the library root itself + parent = os.path.dirname(path) + while parent != root and parent.startswith(root + os.sep): + try: + os.rmdir(parent) + except OSError: + break + parent = os.path.dirname(parent) + except OSError as e: + errors.append(f'{rel}: {e}') + if deleted: + _invalidate_library_folders() + log.info(f"Library delete request processed: deleted={deleted} errors={len(errors)}") + return web.Response(text=serializer.encode({'status': 'ok', 'deleted': deleted, 'errors': errors})) + + +@routes.get(config.URL_PREFIX + 'library/thumbnail') +async def library_thumbnail(request): + """Thumbnail for a library file: ffmpeg frame extraction, cached on disk.""" + rel = request.query.get('path') + root, path = _resolve_library_path(rel) + if root is None or path is None or not os.path.isfile(path): + return _placeholder_thumb_response() + thumb_dir = os.path.join(config.STATE_DIR, 'thumbnails') + os.makedirs(thumb_dir, exist_ok=True) + thumb_path = os.path.join(thumb_dir, hashlib.sha1(('lib:' + rel).encode('utf-8')).hexdigest() + '.img') + if not os.path.exists(thumb_path): + await _extract_video_frame(path, thumb_path, None) + if not os.path.exists(thumb_path): + return _placeholder_thumb_response() + with open(thumb_path, 'rb') as f: + data = f.read() + return web.Response( + body=data, + content_type=sniff_image_type(data[:16]), + headers={'Cache-Control': 'public, max-age=31536000, immutable'} + ) + + def _placeholder_thumb_response(): return web.Response(text=PLACEHOLDER_THUMB, content_type='image/svg+xml', headers={'Cache-Control': 'public, max-age=3600'}) @@ -597,6 +755,8 @@ if config.URL_PREFIX != '/': def index_redirect_dir(request): return web.HTTPFound(config.URL_PREFIX) +if config.LIBRARY_DIR: + routes.static(config.URL_PREFIX + 'library/', config.LIBRARY_DIR, show_index=False) routes.static(config.URL_PREFIX + 'download/', config.DOWNLOAD_DIR, show_index=config.DOWNLOAD_DIRS_INDEXABLE) routes.static(config.URL_PREFIX + 'audio_download/', config.AUDIO_DOWNLOAD_DIR, show_index=config.DOWNLOAD_DIRS_INDEXABLE) routes.static(config.URL_PREFIX, os.path.join(config.BASE_DIR, 'ui/dist/metube/browser')) diff --git a/app/ytdl.py b/app/ytdl.py index efdd5d6..f90ae3a 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -967,6 +967,55 @@ class DownloadQueue: log.info(f"Changed folder for completed download {id} to '{folder or 'Default'}'") return None + async def organize_to_library(self, ids, folder): + """Move completed downloads' files into the media library and remove + their done entries — the library becomes the file's only home.""" + if not self.config.LIBRARY_DIR: + return {'status': 'error', 'msg': 'LIBRARY_DIR is not configured', 'moved': 0, 'errors': []} + library_root = os.path.realpath(self.config.LIBRARY_DIR) + target_dir = os.path.realpath(os.path.join(library_root, folder or '')) + if target_dir != library_root and not target_dir.startswith(library_root + os.sep): + return {'status': 'error', 'msg': f'Folder "{folder}" must resolve inside the library directory', 'moved': 0, 'errors': []} + os.makedirs(target_dir, exist_ok=True) + moved = 0 + errors = [] + for id in ids: + if not self.done.exists(id): + errors.append(f'{id}: not a completed download') + continue + dl = self.done.get(id) + filename = getattr(dl.info, 'filename', None) + if not filename: + errors.append(f'{dl.info.title}: no file on record') + continue + src_dir, error = self.__calc_download_path(dl.info.quality, dl.info.format, getattr(dl.info, 'folder', None)) + src = os.path.join(src_dir, filename) if src_dir else None + if not src or not os.path.exists(src): + errors.append(f'{dl.info.title}: file not found on disk') + continue + dst = os.path.join(target_dir, os.path.basename(filename)) + if os.path.exists(dst): + base, ext = os.path.splitext(os.path.basename(filename)) + unique_id = ''.join(random.choices(string.ascii_lowercase + string.digits, k=5)) + dst = os.path.join(target_dir, f"{base}_{unique_id}{ext}") + try: + await asyncio.get_running_loop().run_in_executor(None, shutil.move, src, dst) + except OSError as e: + errors.append(f'{dl.info.title}: {e}') + continue + log.info(f"Organized download {id} into library: {src} -> {dst}") + self.done.delete(id) + await self.notifier.cleared(id) + moved += 1 + # Clean up emptied source subdirectories (e.g. playlist dirs) + src_subdir = os.path.dirname(src) + if src_subdir != src_dir: + try: + os.removedirs(src_subdir) + except OSError: + pass + return {'status': 'ok', 'moved': moved, 'errors': errors} + async def cancel(self, ids): for id in ids: if self.pending.exists(id): diff --git a/docker-compose.yml b/docker-compose.yml index 08e99ed..b1644c5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,8 @@ services: volumes: - ./downloads:/downloads - ./metube-config:/config + # Media library for the organize/browse feature (change the host path to your real library) + - ./library:/library # Optional: mount cookies file for authenticated downloads # - ./cookies:/cookies:ro environment: @@ -23,6 +25,8 @@ services: - DOWNLOAD_DIR=/downloads - STATE_DIR=/config - TEMP_DIR=/downloads + # Media library (organize/browse feature); leave empty to disable + - LIBRARY_DIR=/library # Download behavior - DOWNLOAD_MODE=limited diff --git a/ui/src/app/app.component.html b/ui/src/app/app.component.html index 764eca4..3136496 100644 --- a/ui/src/app/app.component.html +++ b/ui/src/app/app.component.html @@ -9,8 +9,15 @@ [themes]="themes" [activeTheme]="activeTheme" [open]="sidebarOpen" + [mode]="mode" + [libraryEnabled]="libraryEnabled" + [libraryFolders]="libraryFolders" + [libraryTotal]="libraryTotal" + [selectedLibraryFolder]="libraryFolder" (statusChange)="onStatusChange($event)" (folderChange)="onFolderChange($event)" + (modeChange)="setMode($event)" + (libraryFolderChange)="selectLibraryFolder($event)" (openAdvanced)="toggleAdvanced(true)" (openActivity)="activityOpen = !activityOpen" (themeChanged)="themeChanged($event)" @@ -44,7 +51,7 @@ [storage]="librarySize"> -
+
+ (play)="openPlayer($event)" + (organize)="openOrganize($event)">
+ +
+ + +
@@ -127,6 +147,40 @@ + + +