xorbitlab
/
metube
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add headless watcher and mark-watched features; update Dockerfile and docs 

- Add app/headless_watcher.py and app/mark_watched.py
- Update main.py, ytdl.py, Dockerfile, docker-compose.yml
- Update DEPLOY.md, README.md, pyproject.toml, uv.lock
- Ignore /downloads and /metube-config/cookies in .gitignore
This commit is contained in:
tigeren 2026-08-07 05:39:28 +00:00
parent 159bf3b6e6
commit f3be397a96
11 changed files with 2200 additions and 1780 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored
View File

@ -51,3 +51,9 @@ pending*
__pycache__
.venv
# Downloaded media
/downloads
# Cookies (sensitive)
/metube-config/cookies

4
DEPLOY.md
View File

@ -1,5 +1,5 @@
docker build -t 192.168.2.212:3000/tigeren/metube:1.8 .
docker build -t 192.168.2.212:3000/tigeren/metube:1.9 .
docker push 192.168.2.212:3000/tigeren/metube:1.8
docker push 192.168.2.212:3000/tigeren/metube:1.9
docker compose up -d --build --force-recreate

8
Dockerfile
View File

@ -16,9 +16,9 @@ COPY pyproject.toml uv.lock docker-entrypoint.sh ./
# Install dependencies
RUN sed -i 's/\r$//g' docker-entrypoint.sh && \
chmod +x docker-entrypoint.sh && \
apk add --update ffmpeg aria2 coreutils shadow su-exec curl tini deno && \
apk add --update ffmpeg aria2 coreutils shadow su-exec curl tini deno chromium nss freetype harfbuzz ca-certificates && \
apk add --update --virtual .build-deps gcc g++ musl-dev uv && \
UV_PROJECT_ENVIRONMENT=/usr/local uv sync --frozen --no-dev --compile-bytecode && \
UV_PROJECT_ENVIRONMENT=/usr/local uv sync --frozen --no-dev --compile-bytecode --extra headless && \
apk del .build-deps && \
rm -rf /var/cache/apk/* && \
mkdir /.cache && chmod 777 /.cache
@ -30,6 +30,10 @@ ENV UID=0
ENV GID=0
ENV UMASK=022
# Playwright settings for Alpine Chromium
ENV PLAYWRIGHT_BROWSERS_PATH=0
ENV PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/bin/chromium-browser
ENV DOWNLOAD_DIR /downloads
ENV STATE_DIR /downloads/.metube
ENV TEMP_DIR /downloads

1
README.md
View File

@ -42,6 +42,7 @@ Certain values can be set via environment variables, using the `-e` parameter on
* __DELETE_FILE_ON_TRASHCAN__: if `true`, downloaded files are deleted on the server, when they are trashed from the "Completed" section of the UI. Defaults to `false`.
* __DEFAULT_OPTION_PLAYLIST_STRICT_MODE__: if `true`, the "Strict Playlist mode" switch will be enabled by default. In this mode the playlists will be downloaded only if the URL strictly points to a playlist. URLs to videos inside a playlist will be treated same as direct video URL. Defaults to `false` .
* __DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT__: Maximum number of playlist items that can be downloaded. Defaults to `0` (no limit).
* __MARK_WATCHED_ON_COMPLETE__: if `true`, videos will be marked as "watched" on the source website after successful download. Requires cookies to be configured for the website. Currently supported: PornHub. Defaults to `false`. Uses headless browser technique for sites where API is not available.
### 📁 Storage & Directories

144
app/headless_watcher.py Normal file
View File

@ -0,0 +1,144 @@
"""
Mark videos as watched using a headless browser.
Visiting the page with authenticated cookies triggers the watch tracking JavaScript.
"""
import os
import re
import logging
import asyncio
from urllib.parse import urlparse, parse_qs
from typing import Optional, Dict
log = logging.getLogger("headless_watcher")
# Try to import playwright
try:
from playwright.async_api import async_playwright
HAS_PLAYWRIGHT = True
except ImportError:
HAS_PLAYWRIGHT = False
log.warning("Playwright not installed, headless watching will not work")
class HeadlessWatcher:
"""Uses headless browser to visit pages and trigger watch tracking."""
def __init__(self, cookie_file: str):
self.cookie_file = cookie_file
self.domain_cookies = self._parse_cookie_file()
def _parse_cookie_file(self) -> Dict[str, list]:
"""Parse Netscape cookie file and group by domain."""
cookies_by_domain = {}
if not os.path.exists(self.cookie_file):
log.warning(f"Cookie file not found: {self.cookie_file}")
return cookies_by_domain
try:
with open(self.cookie_file, "r") as f:
for line in f:
line = line.strip()
if not line or line.startswith("#"):
continue
parts = line.split("\t")
if len(parts) >= 7:
domain = parts[0].lstrip(".")
path = parts[2]
secure = parts[3] == "TRUE"
name = parts[5]
value = parts[6]
if domain not in cookies_by_domain:
cookies_by_domain[domain] = []
cookies_by_domain[domain].append({
"name": name,
"value": value,
"domain": parts[0],
"path": path,
"secure": secure,
"httpOnly": False,
})
log.debug(f"Parsed cookies for {len(cookies_by_domain)} domains")
except Exception as e:
log.error(f"Error parsing cookie file: {e}")
return cookies_by_domain
async def visit_page(self, url: str, wait_seconds: int = 5) -> bool:
"""Visit a page with cookies to trigger watch tracking."""
if not HAS_PLAYWRIGHT:
log.error("Playwright not installed")
return False
parsed_url = urlparse(url)
domain = parsed_url.netloc.lower()
cookies = []
for cookie_domain, domain_cookies in self.domain_cookies.items():
if domain in cookie_domain or cookie_domain in domain:
cookies.extend(domain_cookies)
if not cookies:
log.warning(f"No cookies found for domain: {domain}")
return False
log.info(f"Visiting {url} with {len(cookies)} cookies")
try:
async with async_playwright() as p:
# Use system Chromium if available (for Alpine/Docker)
chromium_path = os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH")
if chromium_path and os.path.exists(chromium_path):
log.debug(f"Using system Chromium: {chromium_path}")
browser = await p.chromium.launch(headless=True, executable_path=chromium_path)
else:
browser = await p.chromium.launch(headless=True)
try:
context = await browser.new_context(
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
)
await context.add_cookies(cookies)
page = await context.new_page()
response = await page.goto(url, wait_until="networkidle", timeout=30000)
if response and response.status == 200:
log.debug(f"Page loaded, waiting {wait_seconds}s")
await asyncio.sleep(wait_seconds)
log.info("Successfully triggered watch tracking")
return True
else:
status = response.status if response else "no response"
log.warning(f"Failed to load page, status: {status}")
return False
finally:
await browser.close()
except Exception as e:
log.error(f"Error in headless browser: {e}")
return False
class PHEadlessWatcher(HeadlessWatcher):
"""PornHub specific headless watcher."""
DOMAINS = ["pornhub.com", "www.pornhub.com", "de.pornhub.com", "fr.pornhub.com", "es.pornhub.com", "it.pornhub.com", "rt.pornhub.com"]
def can_handle(self, url: str) -> bool:
parsed = urlparse(url)
domain = parsed.netloc.lower()
return any(d in domain for d in self.DOMAINS)
async def mark_watched(self, url: str, wait_seconds: int = 5) -> bool:
return await self.visit_page(url, wait_seconds=wait_seconds)
async def headless_mark_watched(url: str, cookie_file: str, wait_seconds: int = 5) -> bool:
"""Mark a video as watched using headless browser."""
if not HAS_PLAYWRIGHT:
log.error("Playwright is not installed")
return False
# Try PH handler first
ph_handler = PHHeadlessWatcher(cookie_file)
if ph_handler.can_handle(url):
return await ph_handler.mark_watched(url, wait_seconds)
# Fallback to generic handler
generic_handler = HeadlessWatcher(cookie_file)
return await generic_handler.visit_page(url, wait_seconds)

3
app/main.py
View File

@ -56,9 +56,10 @@ class Config:
'MAX_CONCURRENT_DOWNLOADS': 3,
'LOGLEVEL': 'INFO',
'ENABLE_ACCESSLOG': 'false',
'MARK_WATCHED_ON_COMPLETE': 'false',
}
_BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE', 'HTTPS', 'ENABLE_ACCESSLOG')
_BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE', 'HTTPS', 'ENABLE_ACCESSLOG', 'MARK_WATCHED_ON_COMPLETE')
def __init__(self):
for k, v in self._DEFAULTS.items():

228
app/mark_watched.py Normal file
View File

@ -0,0 +1,228 @@
"""
Mark videos as watched on various websites after successful download.
Uses the same cookies that were used for downloading.
"""
import os
import re
import logging
from urllib.parse import urlparse, parse_qs
from typing import Optional, Dict, Callable
from headless_watcher import headless_mark_watched, HAS_PLAYWRIGHT
log = logging.getLogger("mark_watched")
# Try to import curl_cffi for better impersonation, fallback to requests
try:
from curl_cffi import requests as curl_requests
HAS_CURL_CFFI = True
except ImportError:
HAS_CURL_CFFI = False
import requests
class MarkWatchedHandler:
"""Base class for mark-as-watched handlers."""
def __init__(self, cookie_file: str):
self.cookie_file = cookie_file
def can_handle(self, url: str) -> bool:
"""Check if this handler can handle the given URL."""
raise NotImplementedError
async def mark_watched(self, url: str) -> bool:
"""Mark the video as watched. Returns True on success."""
raise NotImplementedError
def _load_cookies(self) -> Dict[str, str]:
"""Load cookies from the Netscape cookie file."""
cookies = {}
if not os.path.exists(self.cookie_file):
log.warning(f"Cookie file not found: {self.cookie_file}")
return cookies
try:
with open(self.cookie_file, "r") as f:
for line in f:
line = line.strip()
if not line or line.startswith("#"):
continue
# Netscape format: domain flag path secure expiration name value
parts = line.split("\t")
if len(parts) >= 7:
name = parts[5]
value = parts[6]
cookies[name] = value
log.debug(f"Loaded {len(cookies)} cookies from {self.cookie_file}")
except Exception as e:
log.error(f"Error loading cookies: {e}")
return cookies
def _make_request(self, method: str, url: str, **kwargs) -> Optional[object]:
"""Make HTTP request using curl_cffi if available, else requests."""
try:
if HAS_CURL_CFFI:
# Use curl_cffi for better impersonation
session = curl_requests.Session()
# Set browser impersonation
session.impersonate = "chrome124"
response = session.request(method, url, **kwargs)
return response
else:
return requests.request(method, url, **kwargs)
except Exception as e:
log.error(f"Request failed: {e}")
return None
class PHHandler(MarkWatchedHandler):
"""Handler for PornHub."""
DOMAINS = ["pornhub.com", "www.pornhub.com", "de.pornhub.com", "fr.pornhub.com", "es.pornhub.com", "it.pornhub.com", "rt.pornhub.com"]
def can_handle(self, url):
parsed = urlparse(url)
domain = parsed.netloc.lower()
return any(d in domain for d in self.DOMAINS)
async def mark_watched(self, url):
# Extract viewkey from URL
parsed = urlparse(url)
params = parse_qs(parsed.query)
viewkey = params.get("viewkey", [None])[0]
if not viewkey:
# Try to extract from path
match = re.search(r"viewkey=([^&]+)", url)
if match:
viewkey = match.group(1)
if not viewkey:
log.warning(f"Could not extract viewkey from URL: {url}")
return False
log.info(f"Marking video as watched, viewkey: {viewkey}")
# Load cookies
cookies = self._load_cookies()
if not cookies:
log.warning("No cookies available, cannot mark as watched")
return False
# Try multiple endpoints as PH may use different URLs
# First try the AJAX endpoint which is most commonly used
endpoints = [
# AJAX endpoint (most common)
("POST", f"https://www.pornhub.com/user/watched/add/video/{viewkey}"),
# Alternative format
("GET", f"https://www.pornhub.com/user/watched/add/video/{viewkey}"),
# Legacy endpoint
("POST", f"https://www.pornhub.com/user/watched/video/viewkey/{viewkey}"),
]
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Accept": "application/json, text/javascript, */*; q=0.01",
"Accept-Language": "en-US,en;q=0.9",
"X-Requested-With": "XMLHttpRequest",
"Referer": url,
}
for method, api_url in endpoints:
log.debug(f"Trying {method} {api_url}")
response = self._make_request(method, api_url, headers=headers, cookies=cookies, allow_redirects=True)
if response is None:
continue
log.debug(f"Response status: {response.status_code}")
if response.status_code == 200:
log.info(f"Successfully marked video as watched using {api_url}")
return True
elif response.status_code == 302:
# Redirect often means success on PH
log.info(f"Successfully marked video as watched (redirect from {api_url})")
return True
# API methods failed, try headless browser as fallback
log.info("API methods failed, trying headless browser approach")
if HAS_PLAYWRIGHT:
return await headless_mark_watched(url, self.cookie_file, wait_seconds=5)
else:
log.warning("Playwright not available for headless browser fallback")
return False
class YouTubeHandler(MarkWatchedHandler):
"""Handler for YouTube - uses YouTube API or internal endpoints."""
DOMAINS = ["youtube.com", "www.youtube.com", "youtu.be", "m.youtube.com"]
def can_handle(self, url):
parsed = urlparse(url)
domain = parsed.netloc.lower()
return any(d in domain for d in self.DOMAINS)
async def mark_watched(self, url):
# YouTube marking as watched requires more complex handling
# typically done through the browse endpoint with protobuf
# This is a simplified implementation
log.info("YouTube mark-as-watched not yet fully implemented")
return False
# Registry of all handlers
HANDLERS = [
PHHandler,
YouTubeHandler,
]
def get_handler(url: str, cookie_file: str) -> Optional[MarkWatchedHandler]:
"""
Get the appropriate handler for a URL.
Args:
url: The video URL
cookie_file: Path to the Netscape cookie file
Returns:
Handler instance if found, None otherwise
"""
if not cookie_file or not os.path.exists(cookie_file):
return None
for handler_class in HANDLERS:
handler = handler_class(cookie_file)
if handler.can_handle(url):
return handler
return None
async def mark_watched(url: str, cookie_file: str) -> bool:
"""
Mark a video as watched on its respective site.
Args:
url: The video URL
cookie_file: Path to the Netscape cookie file
Returns:
True if successfully marked as watched, False otherwise
"""
handler = get_handler(url, cookie_file)
if not handler:
log.debug(f"No mark-watched handler available for URL: {url}")
return False
try:
return await handler.mark_watched(url)
except Exception as e:
log.error(f"Error marking video as watched: {e}")
return False

28
app/ytdl.py
View File

@ -1,6 +1,7 @@
import os
import yt_dlp
from collections import OrderedDict
from mark_watched import mark_watched
import shelve
import time
import asyncio
@ -544,6 +545,7 @@ class DownloadQueue:
async with self.seq_lock:
log.info("Starting sequential download.")
await download.start(self.notifier)
await self._try_mark_watched(download)
self._post_download_cleanup(download)
elif self.config.DOWNLOAD_MODE == 'limited' and self.semaphore is not None:
await self.__limited_concurrent_download(download)
@ -564,8 +566,34 @@ class DownloadQueue:
log.info(f"Download {download.info.title} is canceled; skipping start.")
return
await download.start(self.notifier)
await self._try_mark_watched(download)
self._post_download_cleanup(download)
async def _try_mark_watched(self, download):
"""Try to mark the video as watched on the source website."""
if not self.config.MARK_WATCHED_ON_COMPLETE:
return
# Only mark as watched if download was successful
if download.info.status != 'finished':
return
# Get the cookie file path from ytdl_options
cookie_file = download.ytdl_opts.get('cookiefile')
if not cookie_file:
log.debug(f"No cookie file for download, skipping mark-watched: {download.info.url}")
return
log.info(f"Attempting to mark video as watched: {download.info.url}")
try:
success = await mark_watched(download.info.url, cookie_file)
if success:
log.info(f"Successfully marked video as watched: {download.info.title}")
else:
log.debug(f"Could not mark video as watched: {download.info.title}")
except Exception as e:
log.error(f"Error in mark-watched: {e}")
def _post_download_cleanup(self, download):
# Release filename reservation if it exists
if hasattr(download, 'reserved_filepath') and download.reserved_filepath:

3
docker-compose.yml
View File

@ -73,6 +73,9 @@ services:
# Optional: robots.txt
# - ROBOTS_TXT=/app/robots.txt
# MARK_WATCHED_ON_COMPLETE
- MARK_WATCHED_ON_COMPLETE=true
# Optional: health check
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8081/version"]

5
pyproject.toml
View File

@ -16,3 +16,8 @@ dependencies = [
dev = [
"pylint",
]
[project.optional-dependencies]
headless = [
"playwright",
]

12
uv.lock
View File

@ -744,11 +744,11 @@ wheels = [
[[package]]
name = "yt-dlp"
version = "2025.12.8"
version = "2026.2.21"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/14/77/db924ebbd99d0b2b571c184cb08ed232cf4906c6f9b76eed763cd2c84170/yt_dlp-2025.12.8.tar.gz", hash = "sha256:b773c81bb6b71cb2c111cfb859f453c7a71cf2ef44eff234ff155877184c3e4f", size = 3088947, upload-time = "2025-12-08T00:16:01.649Z" }
sdist = { url = "https://files.pythonhosted.org/packages/58/d9/55ffff25204733e94a507552ad984d5a8a8e4f9d1f0d91763e6b1a41c79b/yt_dlp-2026.2.21.tar.gz", hash = "sha256:4407dfc1a71fec0dee5ef916a8d4b66057812939b509ae45451fa8fb4376b539", size = 3116630, upload-time = "2026-02-21T20:40:53.522Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/6e/2f/98c3596ad923f8efd32c90dca62e241e8ad9efcebf20831173c357042ba0/yt_dlp-2025.12.8-py3-none-any.whl", hash = "sha256:36e2584342e409cfbfa0b5e61448a1c5189e345cf4564294456ee509e7d3e065", size = 3291464, upload-time = "2025-12-08T00:15:58.556Z" },
{ url = "https://files.pythonhosted.org/packages/5a/40/664c99ee36d80d84ce7a96cd98aebcb3d16c19e6c3ad3461d2cf5424040e/yt_dlp-2026.2.21-py3-none-any.whl", hash = "sha256:0d8408f5b6d20487f5caeb946dfd04f9bcd2f1a3a125b744a0a982b590e449f7", size = 3313392, upload-time = "2026-02-21T20:40:51.514Z" },
]
[package.optional-dependencies]
@ -769,9 +769,9 @@ default = [
[[package]]
name = "yt-dlp-ejs"
version = "0.3.2"
version = "0.5.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/de/72/57d02cf78eb45126bd171298d6a58a5bd48ce1a398b6b7ff00fc904f1f0c/yt_dlp_ejs-0.3.2.tar.gz", hash = "sha256:31a41292799992bdc913e03c9fac2a8c90c82a5cbbc792b2e3373b01da841e3e", size = 34678, upload-time = "2025-12-07T23:44:48.258Z" }
sdist = { url = "https://files.pythonhosted.org/packages/6b/0d/b9e4ab1b47cdeba0842df634b74b3c0144307640ad5b632a5e189c4ab7ce/yt_dlp_ejs-0.5.0.tar.gz", hash = "sha256:8dfae59e418232f485253dcf8e197fefa232423c3af7824fe19e4517b173293b", size = 98925, upload-time = "2026-02-21T19:29:16.844Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/9d/0d/1f0d7a735ca60b87953271b15d00eff5eef05f6118390ddf6f81982526ed/yt_dlp_ejs-0.3.2-py3-none-any.whl", hash = "sha256:f2dc6b3d1b909af1f13e021621b0af048056fca5fb07c4db6aa9bbb37a4f66a9", size = 53252, upload-time = "2025-12-07T23:44:46.605Z" },
{ url = "https://files.pythonhosted.org/packages/7e/5b/1283356b70d4893a8a050cee15092e1b08ea15310b94365f88067146721b/yt_dlp_ejs-0.5.0-py3-none-any.whl", hash = "sha256:674fc0efea741d3100cdf3f0f9e123150715ee41edf47ea7a62fbdeda204bdec", size = 54032, upload-time = "2026-02-21T19:29:15.408Z" },
]