Add headless watcher and mark-watched features; update Dockerfile and docs

- Add app/headless_watcher.py and app/mark_watched.py
- Update main.py, ytdl.py, Dockerfile, docker-compose.yml
- Update DEPLOY.md, README.md, pyproject.toml, uv.lock
- Ignore /downloads and /metube-config/cookies in .gitignore

.gitignore

@@ -51,3 +51,9 @@ pending*
 
 __pycache__
 .venv
+
+# Downloaded media
+/downloads
+
+# Cookies (sensitive)
+/metube-config/cookies

DEPLOY.md

@@ -1,5 +1,5 @@
-docker build -t 192.168.2.212:3000/tigeren/metube:1.8 .
+docker build -t 192.168.2.212:3000/tigeren/metube:1.9 .
 
-docker push 192.168.2.212:3000/tigeren/metube:1.8
+docker push 192.168.2.212:3000/tigeren/metube:1.9
 
 docker compose up -d --build --force-recreate

Dockerfile

@@ -16,9 +16,9 @@ COPY pyproject.toml uv.lock docker-entrypoint.sh ./
 # Install dependencies
 RUN sed -i 's/\r$//g' docker-entrypoint.sh && \
     chmod +x docker-entrypoint.sh && \
-    apk add --update ffmpeg aria2 coreutils shadow su-exec curl tini deno && \
+    apk add --update ffmpeg aria2 coreutils shadow su-exec curl tini deno chromium nss freetype harfbuzz ca-certificates && \
     apk add --update --virtual .build-deps gcc g++ musl-dev uv && \
-    UV_PROJECT_ENVIRONMENT=/usr/local uv sync --frozen --no-dev --compile-bytecode && \
+    UV_PROJECT_ENVIRONMENT=/usr/local uv sync --frozen --no-dev --compile-bytecode --extra headless && \
     apk del .build-deps && \
     rm -rf /var/cache/apk/* && \
     mkdir /.cache && chmod 777 /.cache
@@ -30,6 +30,10 @@ ENV UID=0
 ENV GID=0
 ENV UMASK=022
 
+# Playwright settings for Alpine Chromium
+ENV PLAYWRIGHT_BROWSERS_PATH=0
+ENV PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/bin/chromium-browser
+
 ENV DOWNLOAD_DIR /downloads
 ENV STATE_DIR /downloads/.metube
 ENV TEMP_DIR /downloads

README.md

@@ -42,6 +42,7 @@ Certain values can be set via environment variables, using the `-e` parameter on
 * __DELETE_FILE_ON_TRASHCAN__: if `true`, downloaded files are deleted on the server, when they are trashed from the "Completed" section of the UI. Defaults to `false`.
 * __DEFAULT_OPTION_PLAYLIST_STRICT_MODE__: if `true`, the "Strict Playlist mode" switch will be enabled by default. In this mode the playlists will be downloaded only if the URL strictly points to a playlist. URLs to videos inside a playlist will be treated same as direct video URL. Defaults to `false` .
 * __DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT__: Maximum number of playlist items that can be downloaded. Defaults to `0` (no limit).
+* __MARK_WATCHED_ON_COMPLETE__: if `true`, videos will be marked as "watched" on the source website after successful download. Requires cookies to be configured for the website. Currently supported: PornHub. Defaults to `false`. Uses headless browser technique for sites where API is not available.
 
 ### 📁 Storage & Directories
 

app/headless_watcher.py

@@ -0,0 +1,144 @@
+"""
+Mark videos as watched using a headless browser.
+Visiting the page with authenticated cookies triggers the watch tracking JavaScript.
+"""
+
+import os
+import re
+import logging
+import asyncio
+from urllib.parse import urlparse, parse_qs
+from typing import Optional, Dict
+
+log = logging.getLogger("headless_watcher")
+
+# Try to import playwright
+try:
+    from playwright.async_api import async_playwright
+    HAS_PLAYWRIGHT = True
+except ImportError:
+    HAS_PLAYWRIGHT = False
+    log.warning("Playwright not installed, headless watching will not work")
+
+
+class HeadlessWatcher:
+    """Uses headless browser to visit pages and trigger watch tracking."""
+    def __init__(self, cookie_file: str):
+        self.cookie_file = cookie_file
+        self.domain_cookies = self._parse_cookie_file()
+
+    def _parse_cookie_file(self) -> Dict[str, list]:
+        """Parse Netscape cookie file and group by domain."""
+        cookies_by_domain = {}
+
+        if not os.path.exists(self.cookie_file):
+            log.warning(f"Cookie file not found: {self.cookie_file}")
+            return cookies_by_domain
+
+        try:
+            with open(self.cookie_file, "r") as f:
+                for line in f:
+                    line = line.strip()
+                    if not line or line.startswith("#"):
+                        continue
+                    parts = line.split("\t")
+                    if len(parts) >= 7:
+                        domain = parts[0].lstrip(".")
+                        path = parts[2]
+                        secure = parts[3] == "TRUE"
+                        name = parts[5]
+                        value = parts[6]
+                        if domain not in cookies_by_domain:
+                            cookies_by_domain[domain] = []
+                        cookies_by_domain[domain].append({
+                            "name": name,
+                            "value": value,
+                            "domain": parts[0],
+                            "path": path,
+                            "secure": secure,
+                            "httpOnly": False,
+                        })
+            log.debug(f"Parsed cookies for {len(cookies_by_domain)} domains")
+        except Exception as e:
+            log.error(f"Error parsing cookie file: {e}")
+        return cookies_by_domain
+
+    async def visit_page(self, url: str, wait_seconds: int = 5) -> bool:
+        """Visit a page with cookies to trigger watch tracking."""
+        if not HAS_PLAYWRIGHT:
+            log.error("Playwright not installed")
+            return False
+
+        parsed_url = urlparse(url)
+        domain = parsed_url.netloc.lower()
+
+        cookies = []
+        for cookie_domain, domain_cookies in self.domain_cookies.items():
+            if domain in cookie_domain or cookie_domain in domain:
+                cookies.extend(domain_cookies)
+
+        if not cookies:
+            log.warning(f"No cookies found for domain: {domain}")
+            return False
+
+        log.info(f"Visiting {url} with {len(cookies)} cookies")
+
+        try:
+            async with async_playwright() as p:
+                # Use system Chromium if available (for Alpine/Docker)
+                chromium_path = os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH")
+                if chromium_path and os.path.exists(chromium_path):
+                    log.debug(f"Using system Chromium: {chromium_path}")
+                    browser = await p.chromium.launch(headless=True, executable_path=chromium_path)
+                else:
+                    browser = await p.chromium.launch(headless=True)
+                try:
+                    context = await browser.new_context(
+                        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+                    )
+                    await context.add_cookies(cookies)
+                    page = await context.new_page()
+                    response = await page.goto(url, wait_until="networkidle", timeout=30000)
+                    if response and response.status == 200:
+                        log.debug(f"Page loaded, waiting {wait_seconds}s")
+                        await asyncio.sleep(wait_seconds)
+                        log.info("Successfully triggered watch tracking")
+                        return True
+                    else:
+                        status = response.status if response else "no response"
+                        log.warning(f"Failed to load page, status: {status}")
+                        return False
+                finally:
+                    await browser.close()
+        except Exception as e:
+            log.error(f"Error in headless browser: {e}")
+            return False
+
+class PHEadlessWatcher(HeadlessWatcher):
+    """PornHub specific headless watcher."""
+    
+    DOMAINS = ["pornhub.com", "www.pornhub.com", "de.pornhub.com", "fr.pornhub.com", "es.pornhub.com", "it.pornhub.com", "rt.pornhub.com"]
+    
+    def can_handle(self, url: str) -> bool:
+        parsed = urlparse(url)
+        domain = parsed.netloc.lower()
+        return any(d in domain for d in self.DOMAINS)
+    
+    async def mark_watched(self, url: str, wait_seconds: int = 5) -> bool:
+        return await self.visit_page(url, wait_seconds=wait_seconds)
+
+
+async def headless_mark_watched(url: str, cookie_file: str, wait_seconds: int = 5) -> bool:
+    """Mark a video as watched using headless browser."""
+    if not HAS_PLAYWRIGHT:
+        log.error("Playwright is not installed")
+        return False
+    
+    # Try PH handler first
+    ph_handler = PHHeadlessWatcher(cookie_file)
+    if ph_handler.can_handle(url):
+        return await ph_handler.mark_watched(url, wait_seconds)
+    
+    # Fallback to generic handler
+    generic_handler = HeadlessWatcher(cookie_file)
+    return await generic_handler.visit_page(url, wait_seconds)

app/main.py

@@ -56,9 +56,10 @@ class Config:
         'MAX_CONCURRENT_DOWNLOADS': 3,
         'LOGLEVEL': 'INFO',
         'ENABLE_ACCESSLOG': 'false',
+        'MARK_WATCHED_ON_COMPLETE': 'false',
     }
 
-    _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE', 'HTTPS', 'ENABLE_ACCESSLOG')
+    _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE', 'HTTPS', 'ENABLE_ACCESSLOG', 'MARK_WATCHED_ON_COMPLETE')
 
     def __init__(self):
         for k, v in self._DEFAULTS.items():

app/mark_watched.py

@@ -0,0 +1,228 @@
+"""
+Mark videos as watched on various websites after successful download.
+Uses the same cookies that were used for downloading.
+"""
+
+import os
+import re
+import logging
+from urllib.parse import urlparse, parse_qs
+from typing import Optional, Dict, Callable
+
+from headless_watcher import headless_mark_watched, HAS_PLAYWRIGHT
+
+log = logging.getLogger("mark_watched")
+
+# Try to import curl_cffi for better impersonation, fallback to requests
+try:
+    from curl_cffi import requests as curl_requests
+    HAS_CURL_CFFI = True
+except ImportError:
+    HAS_CURL_CFFI = False
+    import requests
+
+
+class MarkWatchedHandler:
+    """Base class for mark-as-watched handlers."""
+    
+    def __init__(self, cookie_file: str):
+        self.cookie_file = cookie_file
+    
+    def can_handle(self, url: str) -> bool:
+        """Check if this handler can handle the given URL."""
+        raise NotImplementedError
+    
+    async def mark_watched(self, url: str) -> bool:
+        """Mark the video as watched. Returns True on success."""
+        raise NotImplementedError
+    
+    def _load_cookies(self) -> Dict[str, str]:
+        """Load cookies from the Netscape cookie file."""
+        cookies = {}
+        if not os.path.exists(self.cookie_file):
+            log.warning(f"Cookie file not found: {self.cookie_file}")
+            return cookies
+        
+        try:
+            with open(self.cookie_file, "r") as f:
+                for line in f:
+                    line = line.strip()
+                    if not line or line.startswith("#"):
+                        continue
+                    # Netscape format: domain flag path secure expiration name value
+                    parts = line.split("\t")
+                    if len(parts) >= 7:
+                        name = parts[5]
+                        value = parts[6]
+                        cookies[name] = value
+            log.debug(f"Loaded {len(cookies)} cookies from {self.cookie_file}")
+        except Exception as e:
+            log.error(f"Error loading cookies: {e}")
+        
+        return cookies
+    
+    def _make_request(self, method: str, url: str, **kwargs) -> Optional[object]:
+        """Make HTTP request using curl_cffi if available, else requests."""
+        try:
+            if HAS_CURL_CFFI:
+                # Use curl_cffi for better impersonation
+                session = curl_requests.Session()
+                # Set browser impersonation
+                session.impersonate = "chrome124"
+                response = session.request(method, url, **kwargs)
+                return response
+            else:
+                return requests.request(method, url, **kwargs)
+        except Exception as e:
+            log.error(f"Request failed: {e}")
+            return None
+
+
+class PHHandler(MarkWatchedHandler):
+    """Handler for PornHub."""
+    
+    DOMAINS = ["pornhub.com", "www.pornhub.com", "de.pornhub.com", "fr.pornhub.com", "es.pornhub.com", "it.pornhub.com", "rt.pornhub.com"]
+
+    def can_handle(self, url):
+        parsed = urlparse(url)
+        domain = parsed.netloc.lower()
+        return any(d in domain for d in self.DOMAINS)
+    
+    async def mark_watched(self, url):
+        # Extract viewkey from URL
+        parsed = urlparse(url)
+        params = parse_qs(parsed.query)
+        viewkey = params.get("viewkey", [None])[0]
+        
+        if not viewkey:
+            # Try to extract from path
+            match = re.search(r"viewkey=([^&]+)", url)
+            if match:
+                viewkey = match.group(1)
+        
+        if not viewkey:
+            log.warning(f"Could not extract viewkey from URL: {url}")
+            return False
+        
+        log.info(f"Marking video as watched, viewkey: {viewkey}")
+        
+        # Load cookies
+        cookies = self._load_cookies()
+        if not cookies:
+            log.warning("No cookies available, cannot mark as watched")
+            return False
+        
+        # Try multiple endpoints as PH may use different URLs
+        # First try the AJAX endpoint which is most commonly used
+        endpoints = [
+            # AJAX endpoint (most common)
+            ("POST", f"https://www.pornhub.com/user/watched/add/video/{viewkey}"),
+            # Alternative format
+            ("GET", f"https://www.pornhub.com/user/watched/add/video/{viewkey}"),
+            # Legacy endpoint
+            ("POST", f"https://www.pornhub.com/user/watched/video/viewkey/{viewkey}"),
+        ]
+        
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Accept-Language": "en-US,en;q=0.9",
+            "X-Requested-With": "XMLHttpRequest",
+            "Referer": url,
+        }
+        
+        for method, api_url in endpoints:
+            log.debug(f"Trying {method} {api_url}")
+            response = self._make_request(method, api_url, headers=headers, cookies=cookies, allow_redirects=True)
+            
+            if response is None:
+                continue
+                
+            log.debug(f"Response status: {response.status_code}")
+            
+            if response.status_code == 200:
+                log.info(f"Successfully marked video as watched using {api_url}")
+                return True
+            elif response.status_code == 302:
+                # Redirect often means success on PH
+                log.info(f"Successfully marked video as watched (redirect from {api_url})")
+                return True
+        
+        # API methods failed, try headless browser as fallback
+        log.info("API methods failed, trying headless browser approach")
+        if HAS_PLAYWRIGHT:
+            return await headless_mark_watched(url, self.cookie_file, wait_seconds=5)
+        else:
+            log.warning("Playwright not available for headless browser fallback")
+        
+        return False
+
+
+class YouTubeHandler(MarkWatchedHandler):
+    """Handler for YouTube - uses YouTube API or internal endpoints."""
+    
+    DOMAINS = ["youtube.com", "www.youtube.com", "youtu.be", "m.youtube.com"]
+    
+    def can_handle(self, url):
+        parsed = urlparse(url)
+        domain = parsed.netloc.lower()
+        return any(d in domain for d in self.DOMAINS)
+    
+    async def mark_watched(self, url):
+        # YouTube marking as watched requires more complex handling
+        # typically done through the browse endpoint with protobuf
+        # This is a simplified implementation
+        log.info("YouTube mark-as-watched not yet fully implemented")
+        return False
+
+
+# Registry of all handlers
+HANDLERS = [
+    PHHandler,
+    YouTubeHandler,
+]
+
+
+def get_handler(url: str, cookie_file: str) -> Optional[MarkWatchedHandler]:
+    """
+    Get the appropriate handler for a URL.
+    
+    Args:
+        url: The video URL
+        cookie_file: Path to the Netscape cookie file
+    
+    Returns:
+        Handler instance if found, None otherwise
+    """
+    if not cookie_file or not os.path.exists(cookie_file):
+        return None
+    
+    for handler_class in HANDLERS:
+        handler = handler_class(cookie_file)
+        if handler.can_handle(url):
+            return handler
+    
+    return None
+
+
+async def mark_watched(url: str, cookie_file: str) -> bool:
+    """
+    Mark a video as watched on its respective site.
+    
+    Args:
+        url: The video URL
+        cookie_file: Path to the Netscape cookie file
+    
+    Returns:
+        True if successfully marked as watched, False otherwise
+    """
+    handler = get_handler(url, cookie_file)
+    if not handler:
+        log.debug(f"No mark-watched handler available for URL: {url}")
+        return False
+    
+    try:
+        return await handler.mark_watched(url)
+    except Exception as e:
+        log.error(f"Error marking video as watched: {e}")
+        return False

app/ytdl.py

@@ -1,6 +1,7 @@
 import os
 import yt_dlp
 from collections import OrderedDict
+from mark_watched import mark_watched
 import shelve
 import time
 import asyncio
@@ -544,6 +545,7 @@ class DownloadQueue:
             async with self.seq_lock:
                 log.info("Starting sequential download.")
                 await download.start(self.notifier)
+                await self._try_mark_watched(download)
                 self._post_download_cleanup(download)
         elif self.config.DOWNLOAD_MODE == 'limited' and self.semaphore is not None:
             await self.__limited_concurrent_download(download)
@@ -564,8 +566,34 @@ class DownloadQueue:
             log.info(f"Download {download.info.title} is canceled; skipping start.")
             return
         await download.start(self.notifier)
+        await self._try_mark_watched(download)
         self._post_download_cleanup(download)
 
+    async def _try_mark_watched(self, download):
+        """Try to mark the video as watched on the source website."""
+        if not self.config.MARK_WATCHED_ON_COMPLETE:
+            return
+        
+        # Only mark as watched if download was successful
+        if download.info.status != 'finished':
+            return
+        
+        # Get the cookie file path from ytdl_options
+        cookie_file = download.ytdl_opts.get('cookiefile')
+        if not cookie_file:
+            log.debug(f"No cookie file for download, skipping mark-watched: {download.info.url}")
+            return
+        
+        log.info(f"Attempting to mark video as watched: {download.info.url}")
+        try:
+            success = await mark_watched(download.info.url, cookie_file)
+            if success:
+                log.info(f"Successfully marked video as watched: {download.info.title}")
+            else:
+                log.debug(f"Could not mark video as watched: {download.info.title}")
+        except Exception as e:
+            log.error(f"Error in mark-watched: {e}")
+
     def _post_download_cleanup(self, download):
         # Release filename reservation if it exists
         if hasattr(download, 'reserved_filepath') and download.reserved_filepath:

docker-compose.yml

@@ -73,6 +73,9 @@ services:
       # Optional: robots.txt
       # - ROBOTS_TXT=/app/robots.txt
 
+      # MARK_WATCHED_ON_COMPLETE
+      - MARK_WATCHED_ON_COMPLETE=true
+
     # Optional: health check
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8081/version"]

pyproject.toml

@@ -16,3 +16,8 @@ dependencies = [
 dev = [
     "pylint",
 ]
+
+[project.optional-dependencies]
+headless = [
+    "playwright",
+]

uv.lock

@@ -744,11 +744,11 @@ wheels = [
 
 [[package]]
 name = "yt-dlp"
-version = "2025.12.8"
+version = "2026.2.21"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/14/77/db924ebbd99d0b2b571c184cb08ed232cf4906c6f9b76eed763cd2c84170/yt_dlp-2025.12.8.tar.gz", hash = "sha256:b773c81bb6b71cb2c111cfb859f453c7a71cf2ef44eff234ff155877184c3e4f", size = 3088947, upload-time = "2025-12-08T00:16:01.649Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/58/d9/55ffff25204733e94a507552ad984d5a8a8e4f9d1f0d91763e6b1a41c79b/yt_dlp-2026.2.21.tar.gz", hash = "sha256:4407dfc1a71fec0dee5ef916a8d4b66057812939b509ae45451fa8fb4376b539", size = 3116630, upload-time = "2026-02-21T20:40:53.522Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/6e/2f/98c3596ad923f8efd32c90dca62e241e8ad9efcebf20831173c357042ba0/yt_dlp-2025.12.8-py3-none-any.whl", hash = "sha256:36e2584342e409cfbfa0b5e61448a1c5189e345cf4564294456ee509e7d3e065", size = 3291464, upload-time = "2025-12-08T00:15:58.556Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/40/664c99ee36d80d84ce7a96cd98aebcb3d16c19e6c3ad3461d2cf5424040e/yt_dlp-2026.2.21-py3-none-any.whl", hash = "sha256:0d8408f5b6d20487f5caeb946dfd04f9bcd2f1a3a125b744a0a982b590e449f7", size = 3313392, upload-time = "2026-02-21T20:40:51.514Z" },
 ]
 
 [package.optional-dependencies]
@@ -769,9 +769,9 @@ default = [
 
 [[package]]
 name = "yt-dlp-ejs"
-version = "0.3.2"
+version = "0.5.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/de/72/57d02cf78eb45126bd171298d6a58a5bd48ce1a398b6b7ff00fc904f1f0c/yt_dlp_ejs-0.3.2.tar.gz", hash = "sha256:31a41292799992bdc913e03c9fac2a8c90c82a5cbbc792b2e3373b01da841e3e", size = 34678, upload-time = "2025-12-07T23:44:48.258Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/6b/0d/b9e4ab1b47cdeba0842df634b74b3c0144307640ad5b632a5e189c4ab7ce/yt_dlp_ejs-0.5.0.tar.gz", hash = "sha256:8dfae59e418232f485253dcf8e197fefa232423c3af7824fe19e4517b173293b", size = 98925, upload-time = "2026-02-21T19:29:16.844Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/9d/0d/1f0d7a735ca60b87953271b15d00eff5eef05f6118390ddf6f81982526ed/yt_dlp_ejs-0.3.2-py3-none-any.whl", hash = "sha256:f2dc6b3d1b909af1f13e021621b0af048056fca5fb07c4db6aa9bbb37a4f66a9", size = 53252, upload-time = "2025-12-07T23:44:46.605Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/5b/1283356b70d4893a8a050cee15092e1b08ea15310b94365f88067146721b/yt_dlp_ejs-0.5.0-py3-none-any.whl", hash = "sha256:674fc0efea741d3100cdf3f0f9e123150715ee41edf47ea7a62fbdeda204bdec", size = 54032, upload-time = "2026-02-21T19:29:15.408Z" },
 ]