diff --git a/.gitignore b/.gitignore index 58ea7ad..2b606d7 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,9 @@ pending* __pycache__ .venv + +# Downloaded media +/downloads + +# Cookies (sensitive) +/metube-config/cookies diff --git a/DEPLOY.md b/DEPLOY.md index 98fd683..6dd6f53 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -1,5 +1,5 @@ -docker build -t 192.168.2.212:3000/tigeren/metube:1.8 . +docker build -t 192.168.2.212:3000/tigeren/metube:1.9 . -docker push 192.168.2.212:3000/tigeren/metube:1.8 +docker push 192.168.2.212:3000/tigeren/metube:1.9 docker compose up -d --build --force-recreate \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 11f8b48..6e3eed6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,43 +1,47 @@ -FROM node:lts-alpine AS builder - -WORKDIR /metube -COPY ui ./ -RUN npm ci && \ - node_modules/.bin/ng build --configuration production - - -FROM python:3.13-alpine - -WORKDIR /app - -COPY pyproject.toml uv.lock docker-entrypoint.sh ./ - -# Use sed to strip carriage-return characters from the entrypoint script (in case building on Windows) -# Install dependencies -RUN sed -i 's/\r$//g' docker-entrypoint.sh && \ - chmod +x docker-entrypoint.sh && \ - apk add --update ffmpeg aria2 coreutils shadow su-exec curl tini deno && \ - apk add --update --virtual .build-deps gcc g++ musl-dev uv && \ - UV_PROJECT_ENVIRONMENT=/usr/local uv sync --frozen --no-dev --compile-bytecode && \ - apk del .build-deps && \ - rm -rf /var/cache/apk/* && \ - mkdir /.cache && chmod 777 /.cache - -COPY app ./app -COPY --from=builder /metube/dist/metube ./ui/dist/metube - -ENV UID=0 -ENV GID=0 -ENV UMASK=022 - -ENV DOWNLOAD_DIR /downloads -ENV STATE_DIR /downloads/.metube -ENV TEMP_DIR /downloads -VOLUME /downloads -EXPOSE 8081 - -# Add build-time argument for version -ARG VERSION=dev -ENV METUBE_VERSION=$VERSION - -ENTRYPOINT ["/sbin/tini", "-g", "--", "./docker-entrypoint.sh"] +FROM node:lts-alpine AS builder + +WORKDIR /metube +COPY ui ./ +RUN npm ci && \ + node_modules/.bin/ng build --configuration production + + +FROM python:3.13-alpine + +WORKDIR /app + +COPY pyproject.toml uv.lock docker-entrypoint.sh ./ + +# Use sed to strip carriage-return characters from the entrypoint script (in case building on Windows) +# Install dependencies +RUN sed -i 's/\r$//g' docker-entrypoint.sh && \ + chmod +x docker-entrypoint.sh && \ + apk add --update ffmpeg aria2 coreutils shadow su-exec curl tini deno chromium nss freetype harfbuzz ca-certificates && \ + apk add --update --virtual .build-deps gcc g++ musl-dev uv && \ + UV_PROJECT_ENVIRONMENT=/usr/local uv sync --frozen --no-dev --compile-bytecode --extra headless && \ + apk del .build-deps && \ + rm -rf /var/cache/apk/* && \ + mkdir /.cache && chmod 777 /.cache + +COPY app ./app +COPY --from=builder /metube/dist/metube ./ui/dist/metube + +ENV UID=0 +ENV GID=0 +ENV UMASK=022 + +# Playwright settings for Alpine Chromium +ENV PLAYWRIGHT_BROWSERS_PATH=0 +ENV PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/bin/chromium-browser + +ENV DOWNLOAD_DIR /downloads +ENV STATE_DIR /downloads/.metube +ENV TEMP_DIR /downloads +VOLUME /downloads +EXPOSE 8081 + +# Add build-time argument for version +ARG VERSION=dev +ENV METUBE_VERSION=$VERSION + +ENTRYPOINT ["/sbin/tini", "-g", "--", "./docker-entrypoint.sh"] diff --git a/README.md b/README.md index 4d451ec..6bade27 100644 --- a/README.md +++ b/README.md @@ -1,291 +1,292 @@ -# MeTube - -![Build Status](https://github.com/alexta69/metube/actions/workflows/main.yml/badge.svg) -![Docker Pulls](https://img.shields.io/docker/pulls/alexta69/metube.svg) - -Web GUI for youtube-dl (using the [yt-dlp](https://github.com/yt-dlp/yt-dlp) fork) with playlist support. Allows you to download videos from YouTube and [dozens of other sites](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md). - -![screenshot1](https://github.com/alexta69/metube/raw/master/screenshot.gif) - -## 🐳 Run using Docker - -```bash -docker run -d -p 8081:8081 -v /path/to/downloads:/downloads ghcr.io/alexta69/metube -``` - -## 🐳 Run using docker-compose - -```yaml -services: - metube: - image: ghcr.io/alexta69/metube - container_name: metube - restart: unless-stopped - ports: - - "8081:8081" - volumes: - - /path/to/downloads:/downloads -``` - -## βš™οΈ Configuration via environment variables - -Certain values can be set via environment variables, using the `-e` parameter on the docker command line, or the `environment:` section in docker-compose. - -### ⬇️ Download Behavior - -* __DOWNLOAD_MODE__: This flag controls how downloads are scheduled and executed. Options are `sequential`, `concurrent`, and `limited`. Defaults to `limited`: - * `sequential`: Downloads are processed one at a time. A new download won't start until the previous one has finished. This mode is useful for conserving system resources or ensuring downloads occur in strict order. - * `concurrent`: Downloads are started immediately as they are added, with no built-in limit on how many run simultaneously. This mode may overwhelm your system if too many downloads start at once. - * `limited`: Downloads are started concurrently but are capped by a concurrency limit. In this mode, a semaphore is used so that at most a fixed number of downloads run at any given time. -* __MAX_CONCURRENT_DOWNLOADS__: This flag is used only when `DOWNLOAD_MODE` is set to `limited`. - It specifies the maximum number of simultaneous downloads allowed. For example, if set to `5`, then at most five downloads will run concurrently, and any additional downloads will wait until one of the active downloads completes. Defaults to `3`. -* __DELETE_FILE_ON_TRASHCAN__: if `true`, downloaded files are deleted on the server, when they are trashed from the "Completed" section of the UI. Defaults to `false`. -* __DEFAULT_OPTION_PLAYLIST_STRICT_MODE__: if `true`, the "Strict Playlist mode" switch will be enabled by default. In this mode the playlists will be downloaded only if the URL strictly points to a playlist. URLs to videos inside a playlist will be treated same as direct video URL. Defaults to `false` . -* __DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT__: Maximum number of playlist items that can be downloaded. Defaults to `0` (no limit). - -### πŸ“ Storage & Directories - -* __DOWNLOAD_DIR__: Path to where the downloads will be saved. Defaults to `/downloads` in the Docker image, and `.` otherwise. -* __AUDIO_DOWNLOAD_DIR__: Path to where audio-only downloads will be saved, if you wish to separate them from the video downloads. Defaults to the value of `DOWNLOAD_DIR`. -* __CUSTOM_DIRS__: Whether to enable downloading videos into custom directories within the __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__). When enabled, a dropdown appears next to the Add button to specify the download directory. Defaults to `true`. -* __CREATE_CUSTOM_DIRS__: Whether to support automatically creating directories within the __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__) if they do not exist. When enabled, the download directory selector supports free-text input, and the specified directory will be created recursively. Defaults to `true`. -* __CUSTOM_DIRS_EXCLUDE_REGEX__: Regular expression to exclude some custom directories from the dropdown. Empty regex disables exclusion. Defaults to `(^|/)[.@].*$`, which means directories starting with `.` or `@`. -* __DOWNLOAD_DIRS_INDEXABLE__: If `true`, the download directories (__DOWNLOAD_DIR__ and __AUDIO_DOWNLOAD_DIR__) are indexable on the web server. Defaults to `false`. -* __STATE_DIR__: Path to where the queue persistence files will be saved. Defaults to `/downloads/.metube` in the Docker image, and `.` otherwise. -* __TEMP_DIR__: Path where intermediary download files will be saved. Defaults to `/downloads` in the Docker image, and `.` otherwise. - * Set this to an SSD or RAM filesystem (e.g., `tmpfs`) for better performance. - * __Note__: Using a RAM filesystem may prevent downloads from being resumed. - -### πŸ“ File Naming & yt-dlp - -* __OUTPUT_TEMPLATE__: The template for the filenames of the downloaded videos, formatted according to [this spec](https://github.com/yt-dlp/yt-dlp/blob/master/README.md#output-template). Defaults to `%(title)s.%(ext)s`. -* __OUTPUT_TEMPLATE_CHAPTER__: The template for the filenames of the downloaded videos when split into chapters via postprocessors. Defaults to `%(title)s - %(section_number)s %(section_title)s.%(ext)s`. -* __OUTPUT_TEMPLATE_PLAYLIST__: The template for the filenames of the downloaded videos when downloaded as a playlist. Defaults to `%(playlist_title)s/%(title)s.%(ext)s`. When empty, then `OUTPUT_TEMPLATE` is used. -* __YTDL_OPTIONS__: Additional options to pass to yt-dlp in JSON format. [See available options here](https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L222). They roughly correspond to command-line options, though some do not have exact equivalents here. For example, `--recode-video` has to be specified via `postprocessors`. Also note that dashes are replaced with underscores. You may find [this script](https://github.com/yt-dlp/yt-dlp/blob/master/devscripts/cli_to_api.py) helpful for converting from command-line options to `YTDL_OPTIONS`. -* __YTDL_OPTIONS_FILE__: A path to a JSON file that will be loaded and used for populating `YTDL_OPTIONS` above. Please note that if both `YTDL_OPTIONS_FILE` and `YTDL_OPTIONS` are specified, the options in `YTDL_OPTIONS` take precedence. The file will be monitored for changes and reloaded automatically when changes are detected. - -### 🌐 Web Server & URLs - -* __URL_PREFIX__: Base path for the web server (for use when hosting behind a reverse proxy). Defaults to `/`. -* __PUBLIC_HOST_URL__: Base URL for the download links shown in the UI for completed files. By default, MeTube serves them under its own URL. If your download directory is accessible on another URL and you want the download links to be based there, use this variable to set it. -* __PUBLIC_HOST_AUDIO_URL__: Same as PUBLIC_HOST_URL but for audio downloads. -* __HTTPS__: Use `https` instead of `http` (__CERTFILE__ and __KEYFILE__ required). Defaults to `false`. -* __CERTFILE__: HTTPS certificate file path. -* __KEYFILE__: HTTPS key file path. -* __ROBOTS_TXT__: A path to a `robots.txt` file mounted in the container. - -### 🏠 Basic Setup - -* __UID__: User under which MeTube will run. Defaults to `1000`. -* __GID__: Group under which MeTube will run. Defaults to `1000`. -* __UMASK__: Umask value used by MeTube. Defaults to `022`. -* __DEFAULT_THEME__: Default theme to use for the UI, can be set to `light`, `dark`, or `auto`. Defaults to `auto`. -* __LOGLEVEL__: Log level, can be set to `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`, or `NONE`. Defaults to `INFO`. -* __ENABLE_ACCESSLOG__: Whether to enable access log. Defaults to `false`. - -The project's Wiki contains examples of useful configurations contributed by users of MeTube: -* [YTDL_OPTIONS Cookbook](https://github.com/alexta69/metube/wiki/YTDL_OPTIONS-Cookbook) -* [OUTPUT_TEMPLATE Cookbook](https://github.com/alexta69/metube/wiki/OUTPUT_TEMPLATE-Cookbook) - -## πŸͺ Using browser cookies - -In case you need to use your browser's cookies with MeTube, for example to download restricted or private videos: - -* Add the following to your docker-compose.yml: - -```yaml - volumes: - - /path/to/cookies:/cookies - environment: - - YTDL_OPTIONS={"cookiefile":"/cookies/cookies.txt"} -``` - -* Install in your browser an extension to extract cookies: - * [Firefox](https://addons.mozilla.org/en-US/firefox/addon/export-cookies-txt/) - * [Chrome](https://chrome.google.com/webstore/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc) -* Extract the cookies you need with the extension and rename the file `cookies.txt` -* Drop the file in the folder you configured in the docker-compose.yml above -* Restart the container - -## πŸ”Œ Browser extensions - -Browser extensions allow right-clicking videos and sending them directly to MeTube. Please note that if you're on an HTTPS page, your MeTube instance must be behind an HTTPS reverse proxy (see below) for the extensions to work. - -__Chrome:__ contributed by [Rpsl](https://github.com/rpsl). You can install it from [Google Chrome Webstore](https://chrome.google.com/webstore/detail/metube-downloader/fbmkmdnlhacefjljljlbhkodfmfkijdh) or use developer mode and install [from sources](https://github.com/Rpsl/metube-browser-extension). - -__Firefox:__ contributed by [nanocortex](https://github.com/nanocortex). You can install it from [Firefox Addons](https://addons.mozilla.org/en-US/firefox/addon/metube-downloader) or get sources from [here](https://github.com/nanocortex/metube-firefox-addon). - -## πŸ“± iOS Shortcut - -[rithask](https://github.com/rithask) created an iOS shortcut to send URLs to MeTube from Safari. Enter the MeTube instance address when prompted which will be saved for later use. You can run the shortcut from Safari’s share menu. The shortcut can be downloaded from [this iCloud link](https://www.icloud.com/shortcuts/66627a9f334c467baabdb2769763a1a6). - -## πŸ“± iOS Compatibility - -iOS has strict requirements for video files, requiring h264 or h265 video codec and aac audio codec in MP4 container. This can sometimes be a lower quality than the best quality available. To accommodate iOS requirements, when downloading a MP4 format you can choose "Best (iOS)" to get the best quality formats as compatible as possible with iOS requirements. - -To force all downloads to be converted to an iOS-compatible codec, insert this as an environment variable: - -```yaml - environment: - - 'YTDL_OPTIONS={"format": "best", "exec": "ffmpeg -i %(filepath)q -c:v libx264 -c:a aac %(filepath)q.h264.mp4"}' -``` - -## πŸ”– Bookmarklet - -[kushfest](https://github.com/kushfest) has created a Chrome bookmarklet for sending the currently open webpage to MeTube. Please note that if you're on an HTTPS page, your MeTube instance must be configured with `HTTPS` as `true` in the environment, or be behind an HTTPS reverse proxy (see below) for the bookmarklet to work. - -GitHub doesn't allow embedding JavaScript as a link, so the bookmarklet has to be created manually by copying the following code to a new bookmark you create on your bookmarks bar. Change the hostname in the URL below to point to your MeTube instance. - -```javascript -javascript:!function(){xhr=new XMLHttpRequest();xhr.open("POST","https://metube.domain.com/add");xhr.withCredentials=true;xhr.send(JSON.stringify({"url":document.location.href,"quality":"best"}));xhr.onload=function(){if(xhr.status==200){alert("Sent to metube!")}else{alert("Send to metube failed. Check the javascript console for clues.")}}}(); -``` - -[shoonya75](https://github.com/shoonya75) has contributed a Firefox version: - -```javascript -javascript:(function(){xhr=new XMLHttpRequest();xhr.open("POST","https://metube.domain.com/add");xhr.send(JSON.stringify({"url":document.location.href,"quality":"best"}));xhr.onload=function(){if(xhr.status==200){alert("Sent to metube!")}else{alert("Send to metube failed. Check the javascript console for clues.")}}})(); -``` - -The above bookmarklets use `alert()` as a success/failure notification. The following will show a toast message instead: - -Chrome: - -```javascript -javascript:!function(){function notify(msg) {var sc = document.scrollingElement.scrollTop; var text = document.createElement('span');text.innerHTML=msg;var ts = text.style;ts.all = 'revert';ts.color = '#000';ts.fontFamily = 'Verdana, sans-serif';ts.fontSize = '15px';ts.backgroundColor = 'white';ts.padding = '15px';ts.border = '1px solid gainsboro';ts.boxShadow = '3px 3px 10px';ts.zIndex = '100';document.body.appendChild(text);ts.position = 'absolute'; ts.top = 50 + sc + 'px'; ts.left = (window.innerWidth / 2)-(text.offsetWidth / 2) + 'px'; setTimeout(function () { text.style.visibility = "hidden"; }, 1500);}xhr=new XMLHttpRequest();xhr.open("POST","https://metube.domain.com/add");xhr.send(JSON.stringify({"url":document.location.href,"quality":"best"}));xhr.onload=function() { if(xhr.status==200){notify("Sent to metube!")}else {notify("Send to metube failed. Check the javascript console for clues.")}}}(); -``` - -Firefox: - -```javascript -javascript:(function(){function notify(msg) {var sc = document.scrollingElement.scrollTop; var text = document.createElement('span');text.innerHTML=msg;var ts = text.style;ts.all = 'revert';ts.color = '#000';ts.fontFamily = 'Verdana, sans-serif';ts.fontSize = '15px';ts.backgroundColor = 'white';ts.padding = '15px';ts.border = '1px solid gainsboro';ts.boxShadow = '3px 3px 10px';ts.zIndex = '100';document.body.appendChild(text);ts.position = 'absolute'; ts.top = 50 + sc + 'px'; ts.left = (window.innerWidth / 2)-(text.offsetWidth / 2) + 'px'; setTimeout(function () { text.style.visibility = "hidden"; }, 1500);}xhr=new XMLHttpRequest();xhr.open("POST","https://metube.domain.com/add");xhr.send(JSON.stringify({"url":document.location.href,"quality":"best"}));xhr.onload=function() { if(xhr.status==200){notify("Sent to metube!")}else {notify("Send to metube failed. Check the javascript console for clues.")}}})(); -``` - -## ⚑ Raycast extension - -[dotvhs](https://github.com/dotvhs) has created an [extension for Raycast](https://www.raycast.com/dot/metube) that allows adding videos to MeTube directly from Raycast. - -## πŸ”’ HTTPS support, and running behind a reverse proxy - -It's possible to configure MeTube to listen in HTTPS mode. `docker-compose` example: - -```yaml -services: - metube: - image: ghcr.io/alexta69/metube - container_name: metube - restart: unless-stopped - ports: - - "8081:8081" - volumes: - - /path/to/downloads:/downloads - - /path/to/ssl/crt:/ssl/crt.pem - - /path/to/ssl/key:/ssl/key.pem - environment: - - HTTPS=true - - CERTFILE=/ssl/crt.pem - - KEYFILE=/ssl/key.pem -``` - -It's also possible to run MeTube behind a reverse proxy, in order to support authentication. HTTPS support can also be added in this way. - -When running behind a reverse proxy which remaps the URL (i.e. serves MeTube under a subdirectory and not under root), don't forget to set the URL_PREFIX environment variable to the correct value. - -If you're using the [linuxserver/swag](https://docs.linuxserver.io/general/swag) image for your reverse proxying needs (which I can heartily recommend), it already includes ready snippets for proxying MeTube both in [subfolder](https://github.com/linuxserver/reverse-proxy-confs/blob/master/metube.subfolder.conf.sample) and [subdomain](https://github.com/linuxserver/reverse-proxy-confs/blob/master/metube.subdomain.conf.sample) modes under the `nginx/proxy-confs` directory in the configuration volume. It also includes Authelia which can be used for authentication. - -### 🌐 NGINX - -```nginx -location /metube/ { - proxy_pass http://metube:8081; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Host $host; -} -``` - -Note: the extra `proxy_set_header` directives are there to make WebSocket work. - -### 🌐 Apache - -Contributed by [PIE-yt](https://github.com/PIE-yt). Source [here](https://gist.github.com/PIE-yt/29e7116588379032427f5bd446b2cac4). - -```apache -# For putting in your Apache sites site.conf -# Serves MeTube under a /metube/ subdir (http://yourdomain.com/metube/) - - ProxyPass http://localhost:8081/ retry=0 timeout=30 - ProxyPassReverse http://localhost:8081/ - - - - RewriteEngine On - RewriteCond %{QUERY_STRING} transport=websocket [NC] - RewriteRule /(.*) ws://localhost:8081/socket.io/$1 [P,L] - ProxyPass http://localhost:8081/socket.io retry=0 timeout=30 - ProxyPassReverse http://localhost:8081/socket.io - -``` - -### 🌐 Caddy - -The following example Caddyfile gets a reverse proxy going behind [caddy](https://caddyserver.com). - -```caddyfile -example.com { - route /metube/* { - uri strip_prefix metube - reverse_proxy metube:8081 - } -} -``` - -## πŸ”„ Updating yt-dlp - -The engine which powers the actual video downloads in MeTube is [yt-dlp](https://github.com/yt-dlp/yt-dlp). Since video sites regularly change their layouts, frequent updates of yt-dlp are required to keep up. - -There's an automatic nightly build of MeTube which looks for a new version of yt-dlp, and if one exists, the build pulls it and publishes an updated docker image. Therefore, in order to keep up with the changes, it's recommended that you update your MeTube container regularly with the latest image. - -I recommend installing and setting up [watchtower](https://github.com/containrrr/watchtower) for this purpose. - -## πŸ”§ Troubleshooting and submitting issues - -Before asking a question or submitting an issue for MeTube, please remember that MeTube is only a UI for [yt-dlp](https://github.com/yt-dlp/yt-dlp). Any issues you might be experiencing with authentication to video websites, postprocessing, permissions, other `YTDL_OPTIONS` configurations which seem not to work, or anything else that concerns the workings of the underlying yt-dlp library, need not be opened on the MeTube project. In order to debug and troubleshoot them, it's advised to try using the yt-dlp binary directly first, bypassing the UI, and once that is working, importing the options that worked for you into `YTDL_OPTIONS`. - -In order to test with the yt-dlp command directly, you can either download it and run it locally, or for a better simulation of its actual conditions, you can run it within the MeTube container itself. Assuming your MeTube container is called `metube`, run the following on your Docker host to get a shell inside the container: - -```bash -docker exec -ti metube sh -cd /downloads -``` - -Once there, you can use the yt-dlp command freely. - -## πŸ’‘ Submitting feature requests - -MeTube development relies on code contributions by the community. The program as it currently stands fits my own use cases, and is therefore feature-complete as far as I'm concerned. If your use cases are different and require additional features, please feel free to submit PRs that implement those features. It's advisable to create an issue first to discuss the planned implementation, because in an effort to reduce bloat, some PRs may not be accepted. However, note that opening a feature request when you don't intend to implement the feature will rarely result in the request being fulfilled. - -## πŸ› οΈ Building and running locally - -Make sure you have Node.js and Python 3.13 installed. - -```bash -cd metube/ui -# install Angular and build the UI -npm install -node_modules/.bin/ng build -# install python dependencies -cd .. -curl -LsSf https://astral.sh/uv/install.sh | sh -uv sync -# run -uv run python3 app/main.py -``` - -A Docker image can be built locally (it will build the UI too): - -```bash -docker build -t metube . -``` - -Note that if you're running the server in VSCode, your downloads will go to your user's Downloads folder (this is configured via the environment in `.vscode/launch.json`). +# MeTube + +![Build Status](https://github.com/alexta69/metube/actions/workflows/main.yml/badge.svg) +![Docker Pulls](https://img.shields.io/docker/pulls/alexta69/metube.svg) + +Web GUI for youtube-dl (using the [yt-dlp](https://github.com/yt-dlp/yt-dlp) fork) with playlist support. Allows you to download videos from YouTube and [dozens of other sites](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md). + +![screenshot1](https://github.com/alexta69/metube/raw/master/screenshot.gif) + +## 🐳 Run using Docker + +```bash +docker run -d -p 8081:8081 -v /path/to/downloads:/downloads ghcr.io/alexta69/metube +``` + +## 🐳 Run using docker-compose + +```yaml +services: + metube: + image: ghcr.io/alexta69/metube + container_name: metube + restart: unless-stopped + ports: + - "8081:8081" + volumes: + - /path/to/downloads:/downloads +``` + +## βš™οΈ Configuration via environment variables + +Certain values can be set via environment variables, using the `-e` parameter on the docker command line, or the `environment:` section in docker-compose. + +### ⬇️ Download Behavior + +* __DOWNLOAD_MODE__: This flag controls how downloads are scheduled and executed. Options are `sequential`, `concurrent`, and `limited`. Defaults to `limited`: + * `sequential`: Downloads are processed one at a time. A new download won't start until the previous one has finished. This mode is useful for conserving system resources or ensuring downloads occur in strict order. + * `concurrent`: Downloads are started immediately as they are added, with no built-in limit on how many run simultaneously. This mode may overwhelm your system if too many downloads start at once. + * `limited`: Downloads are started concurrently but are capped by a concurrency limit. In this mode, a semaphore is used so that at most a fixed number of downloads run at any given time. +* __MAX_CONCURRENT_DOWNLOADS__: This flag is used only when `DOWNLOAD_MODE` is set to `limited`. + It specifies the maximum number of simultaneous downloads allowed. For example, if set to `5`, then at most five downloads will run concurrently, and any additional downloads will wait until one of the active downloads completes. Defaults to `3`. +* __DELETE_FILE_ON_TRASHCAN__: if `true`, downloaded files are deleted on the server, when they are trashed from the "Completed" section of the UI. Defaults to `false`. +* __DEFAULT_OPTION_PLAYLIST_STRICT_MODE__: if `true`, the "Strict Playlist mode" switch will be enabled by default. In this mode the playlists will be downloaded only if the URL strictly points to a playlist. URLs to videos inside a playlist will be treated same as direct video URL. Defaults to `false` . +* __DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT__: Maximum number of playlist items that can be downloaded. Defaults to `0` (no limit). +* __MARK_WATCHED_ON_COMPLETE__: if `true`, videos will be marked as "watched" on the source website after successful download. Requires cookies to be configured for the website. Currently supported: PornHub. Defaults to `false`. Uses headless browser technique for sites where API is not available. + +### πŸ“ Storage & Directories + +* __DOWNLOAD_DIR__: Path to where the downloads will be saved. Defaults to `/downloads` in the Docker image, and `.` otherwise. +* __AUDIO_DOWNLOAD_DIR__: Path to where audio-only downloads will be saved, if you wish to separate them from the video downloads. Defaults to the value of `DOWNLOAD_DIR`. +* __CUSTOM_DIRS__: Whether to enable downloading videos into custom directories within the __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__). When enabled, a dropdown appears next to the Add button to specify the download directory. Defaults to `true`. +* __CREATE_CUSTOM_DIRS__: Whether to support automatically creating directories within the __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__) if they do not exist. When enabled, the download directory selector supports free-text input, and the specified directory will be created recursively. Defaults to `true`. +* __CUSTOM_DIRS_EXCLUDE_REGEX__: Regular expression to exclude some custom directories from the dropdown. Empty regex disables exclusion. Defaults to `(^|/)[.@].*$`, which means directories starting with `.` or `@`. +* __DOWNLOAD_DIRS_INDEXABLE__: If `true`, the download directories (__DOWNLOAD_DIR__ and __AUDIO_DOWNLOAD_DIR__) are indexable on the web server. Defaults to `false`. +* __STATE_DIR__: Path to where the queue persistence files will be saved. Defaults to `/downloads/.metube` in the Docker image, and `.` otherwise. +* __TEMP_DIR__: Path where intermediary download files will be saved. Defaults to `/downloads` in the Docker image, and `.` otherwise. + * Set this to an SSD or RAM filesystem (e.g., `tmpfs`) for better performance. + * __Note__: Using a RAM filesystem may prevent downloads from being resumed. + +### πŸ“ File Naming & yt-dlp + +* __OUTPUT_TEMPLATE__: The template for the filenames of the downloaded videos, formatted according to [this spec](https://github.com/yt-dlp/yt-dlp/blob/master/README.md#output-template). Defaults to `%(title)s.%(ext)s`. +* __OUTPUT_TEMPLATE_CHAPTER__: The template for the filenames of the downloaded videos when split into chapters via postprocessors. Defaults to `%(title)s - %(section_number)s %(section_title)s.%(ext)s`. +* __OUTPUT_TEMPLATE_PLAYLIST__: The template for the filenames of the downloaded videos when downloaded as a playlist. Defaults to `%(playlist_title)s/%(title)s.%(ext)s`. When empty, then `OUTPUT_TEMPLATE` is used. +* __YTDL_OPTIONS__: Additional options to pass to yt-dlp in JSON format. [See available options here](https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L222). They roughly correspond to command-line options, though some do not have exact equivalents here. For example, `--recode-video` has to be specified via `postprocessors`. Also note that dashes are replaced with underscores. You may find [this script](https://github.com/yt-dlp/yt-dlp/blob/master/devscripts/cli_to_api.py) helpful for converting from command-line options to `YTDL_OPTIONS`. +* __YTDL_OPTIONS_FILE__: A path to a JSON file that will be loaded and used for populating `YTDL_OPTIONS` above. Please note that if both `YTDL_OPTIONS_FILE` and `YTDL_OPTIONS` are specified, the options in `YTDL_OPTIONS` take precedence. The file will be monitored for changes and reloaded automatically when changes are detected. + +### 🌐 Web Server & URLs + +* __URL_PREFIX__: Base path for the web server (for use when hosting behind a reverse proxy). Defaults to `/`. +* __PUBLIC_HOST_URL__: Base URL for the download links shown in the UI for completed files. By default, MeTube serves them under its own URL. If your download directory is accessible on another URL and you want the download links to be based there, use this variable to set it. +* __PUBLIC_HOST_AUDIO_URL__: Same as PUBLIC_HOST_URL but for audio downloads. +* __HTTPS__: Use `https` instead of `http` (__CERTFILE__ and __KEYFILE__ required). Defaults to `false`. +* __CERTFILE__: HTTPS certificate file path. +* __KEYFILE__: HTTPS key file path. +* __ROBOTS_TXT__: A path to a `robots.txt` file mounted in the container. + +### 🏠 Basic Setup + +* __UID__: User under which MeTube will run. Defaults to `1000`. +* __GID__: Group under which MeTube will run. Defaults to `1000`. +* __UMASK__: Umask value used by MeTube. Defaults to `022`. +* __DEFAULT_THEME__: Default theme to use for the UI, can be set to `light`, `dark`, or `auto`. Defaults to `auto`. +* __LOGLEVEL__: Log level, can be set to `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`, or `NONE`. Defaults to `INFO`. +* __ENABLE_ACCESSLOG__: Whether to enable access log. Defaults to `false`. + +The project's Wiki contains examples of useful configurations contributed by users of MeTube: +* [YTDL_OPTIONS Cookbook](https://github.com/alexta69/metube/wiki/YTDL_OPTIONS-Cookbook) +* [OUTPUT_TEMPLATE Cookbook](https://github.com/alexta69/metube/wiki/OUTPUT_TEMPLATE-Cookbook) + +## πŸͺ Using browser cookies + +In case you need to use your browser's cookies with MeTube, for example to download restricted or private videos: + +* Add the following to your docker-compose.yml: + +```yaml + volumes: + - /path/to/cookies:/cookies + environment: + - YTDL_OPTIONS={"cookiefile":"/cookies/cookies.txt"} +``` + +* Install in your browser an extension to extract cookies: + * [Firefox](https://addons.mozilla.org/en-US/firefox/addon/export-cookies-txt/) + * [Chrome](https://chrome.google.com/webstore/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc) +* Extract the cookies you need with the extension and rename the file `cookies.txt` +* Drop the file in the folder you configured in the docker-compose.yml above +* Restart the container + +## πŸ”Œ Browser extensions + +Browser extensions allow right-clicking videos and sending them directly to MeTube. Please note that if you're on an HTTPS page, your MeTube instance must be behind an HTTPS reverse proxy (see below) for the extensions to work. + +__Chrome:__ contributed by [Rpsl](https://github.com/rpsl). You can install it from [Google Chrome Webstore](https://chrome.google.com/webstore/detail/metube-downloader/fbmkmdnlhacefjljljlbhkodfmfkijdh) or use developer mode and install [from sources](https://github.com/Rpsl/metube-browser-extension). + +__Firefox:__ contributed by [nanocortex](https://github.com/nanocortex). You can install it from [Firefox Addons](https://addons.mozilla.org/en-US/firefox/addon/metube-downloader) or get sources from [here](https://github.com/nanocortex/metube-firefox-addon). + +## πŸ“± iOS Shortcut + +[rithask](https://github.com/rithask) created an iOS shortcut to send URLs to MeTube from Safari. Enter the MeTube instance address when prompted which will be saved for later use. You can run the shortcut from Safari’s share menu. The shortcut can be downloaded from [this iCloud link](https://www.icloud.com/shortcuts/66627a9f334c467baabdb2769763a1a6). + +## πŸ“± iOS Compatibility + +iOS has strict requirements for video files, requiring h264 or h265 video codec and aac audio codec in MP4 container. This can sometimes be a lower quality than the best quality available. To accommodate iOS requirements, when downloading a MP4 format you can choose "Best (iOS)" to get the best quality formats as compatible as possible with iOS requirements. + +To force all downloads to be converted to an iOS-compatible codec, insert this as an environment variable: + +```yaml + environment: + - 'YTDL_OPTIONS={"format": "best", "exec": "ffmpeg -i %(filepath)q -c:v libx264 -c:a aac %(filepath)q.h264.mp4"}' +``` + +## πŸ”– Bookmarklet + +[kushfest](https://github.com/kushfest) has created a Chrome bookmarklet for sending the currently open webpage to MeTube. Please note that if you're on an HTTPS page, your MeTube instance must be configured with `HTTPS` as `true` in the environment, or be behind an HTTPS reverse proxy (see below) for the bookmarklet to work. + +GitHub doesn't allow embedding JavaScript as a link, so the bookmarklet has to be created manually by copying the following code to a new bookmark you create on your bookmarks bar. Change the hostname in the URL below to point to your MeTube instance. + +```javascript +javascript:!function(){xhr=new XMLHttpRequest();xhr.open("POST","https://metube.domain.com/add");xhr.withCredentials=true;xhr.send(JSON.stringify({"url":document.location.href,"quality":"best"}));xhr.onload=function(){if(xhr.status==200){alert("Sent to metube!")}else{alert("Send to metube failed. Check the javascript console for clues.")}}}(); +``` + +[shoonya75](https://github.com/shoonya75) has contributed a Firefox version: + +```javascript +javascript:(function(){xhr=new XMLHttpRequest();xhr.open("POST","https://metube.domain.com/add");xhr.send(JSON.stringify({"url":document.location.href,"quality":"best"}));xhr.onload=function(){if(xhr.status==200){alert("Sent to metube!")}else{alert("Send to metube failed. Check the javascript console for clues.")}}})(); +``` + +The above bookmarklets use `alert()` as a success/failure notification. The following will show a toast message instead: + +Chrome: + +```javascript +javascript:!function(){function notify(msg) {var sc = document.scrollingElement.scrollTop; var text = document.createElement('span');text.innerHTML=msg;var ts = text.style;ts.all = 'revert';ts.color = '#000';ts.fontFamily = 'Verdana, sans-serif';ts.fontSize = '15px';ts.backgroundColor = 'white';ts.padding = '15px';ts.border = '1px solid gainsboro';ts.boxShadow = '3px 3px 10px';ts.zIndex = '100';document.body.appendChild(text);ts.position = 'absolute'; ts.top = 50 + sc + 'px'; ts.left = (window.innerWidth / 2)-(text.offsetWidth / 2) + 'px'; setTimeout(function () { text.style.visibility = "hidden"; }, 1500);}xhr=new XMLHttpRequest();xhr.open("POST","https://metube.domain.com/add");xhr.send(JSON.stringify({"url":document.location.href,"quality":"best"}));xhr.onload=function() { if(xhr.status==200){notify("Sent to metube!")}else {notify("Send to metube failed. Check the javascript console for clues.")}}}(); +``` + +Firefox: + +```javascript +javascript:(function(){function notify(msg) {var sc = document.scrollingElement.scrollTop; var text = document.createElement('span');text.innerHTML=msg;var ts = text.style;ts.all = 'revert';ts.color = '#000';ts.fontFamily = 'Verdana, sans-serif';ts.fontSize = '15px';ts.backgroundColor = 'white';ts.padding = '15px';ts.border = '1px solid gainsboro';ts.boxShadow = '3px 3px 10px';ts.zIndex = '100';document.body.appendChild(text);ts.position = 'absolute'; ts.top = 50 + sc + 'px'; ts.left = (window.innerWidth / 2)-(text.offsetWidth / 2) + 'px'; setTimeout(function () { text.style.visibility = "hidden"; }, 1500);}xhr=new XMLHttpRequest();xhr.open("POST","https://metube.domain.com/add");xhr.send(JSON.stringify({"url":document.location.href,"quality":"best"}));xhr.onload=function() { if(xhr.status==200){notify("Sent to metube!")}else {notify("Send to metube failed. Check the javascript console for clues.")}}})(); +``` + +## ⚑ Raycast extension + +[dotvhs](https://github.com/dotvhs) has created an [extension for Raycast](https://www.raycast.com/dot/metube) that allows adding videos to MeTube directly from Raycast. + +## πŸ”’ HTTPS support, and running behind a reverse proxy + +It's possible to configure MeTube to listen in HTTPS mode. `docker-compose` example: + +```yaml +services: + metube: + image: ghcr.io/alexta69/metube + container_name: metube + restart: unless-stopped + ports: + - "8081:8081" + volumes: + - /path/to/downloads:/downloads + - /path/to/ssl/crt:/ssl/crt.pem + - /path/to/ssl/key:/ssl/key.pem + environment: + - HTTPS=true + - CERTFILE=/ssl/crt.pem + - KEYFILE=/ssl/key.pem +``` + +It's also possible to run MeTube behind a reverse proxy, in order to support authentication. HTTPS support can also be added in this way. + +When running behind a reverse proxy which remaps the URL (i.e. serves MeTube under a subdirectory and not under root), don't forget to set the URL_PREFIX environment variable to the correct value. + +If you're using the [linuxserver/swag](https://docs.linuxserver.io/general/swag) image for your reverse proxying needs (which I can heartily recommend), it already includes ready snippets for proxying MeTube both in [subfolder](https://github.com/linuxserver/reverse-proxy-confs/blob/master/metube.subfolder.conf.sample) and [subdomain](https://github.com/linuxserver/reverse-proxy-confs/blob/master/metube.subdomain.conf.sample) modes under the `nginx/proxy-confs` directory in the configuration volume. It also includes Authelia which can be used for authentication. + +### 🌐 NGINX + +```nginx +location /metube/ { + proxy_pass http://metube:8081; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; +} +``` + +Note: the extra `proxy_set_header` directives are there to make WebSocket work. + +### 🌐 Apache + +Contributed by [PIE-yt](https://github.com/PIE-yt). Source [here](https://gist.github.com/PIE-yt/29e7116588379032427f5bd446b2cac4). + +```apache +# For putting in your Apache sites site.conf +# Serves MeTube under a /metube/ subdir (http://yourdomain.com/metube/) + + ProxyPass http://localhost:8081/ retry=0 timeout=30 + ProxyPassReverse http://localhost:8081/ + + + + RewriteEngine On + RewriteCond %{QUERY_STRING} transport=websocket [NC] + RewriteRule /(.*) ws://localhost:8081/socket.io/$1 [P,L] + ProxyPass http://localhost:8081/socket.io retry=0 timeout=30 + ProxyPassReverse http://localhost:8081/socket.io + +``` + +### 🌐 Caddy + +The following example Caddyfile gets a reverse proxy going behind [caddy](https://caddyserver.com). + +```caddyfile +example.com { + route /metube/* { + uri strip_prefix metube + reverse_proxy metube:8081 + } +} +``` + +## πŸ”„ Updating yt-dlp + +The engine which powers the actual video downloads in MeTube is [yt-dlp](https://github.com/yt-dlp/yt-dlp). Since video sites regularly change their layouts, frequent updates of yt-dlp are required to keep up. + +There's an automatic nightly build of MeTube which looks for a new version of yt-dlp, and if one exists, the build pulls it and publishes an updated docker image. Therefore, in order to keep up with the changes, it's recommended that you update your MeTube container regularly with the latest image. + +I recommend installing and setting up [watchtower](https://github.com/containrrr/watchtower) for this purpose. + +## πŸ”§ Troubleshooting and submitting issues + +Before asking a question or submitting an issue for MeTube, please remember that MeTube is only a UI for [yt-dlp](https://github.com/yt-dlp/yt-dlp). Any issues you might be experiencing with authentication to video websites, postprocessing, permissions, other `YTDL_OPTIONS` configurations which seem not to work, or anything else that concerns the workings of the underlying yt-dlp library, need not be opened on the MeTube project. In order to debug and troubleshoot them, it's advised to try using the yt-dlp binary directly first, bypassing the UI, and once that is working, importing the options that worked for you into `YTDL_OPTIONS`. + +In order to test with the yt-dlp command directly, you can either download it and run it locally, or for a better simulation of its actual conditions, you can run it within the MeTube container itself. Assuming your MeTube container is called `metube`, run the following on your Docker host to get a shell inside the container: + +```bash +docker exec -ti metube sh +cd /downloads +``` + +Once there, you can use the yt-dlp command freely. + +## πŸ’‘ Submitting feature requests + +MeTube development relies on code contributions by the community. The program as it currently stands fits my own use cases, and is therefore feature-complete as far as I'm concerned. If your use cases are different and require additional features, please feel free to submit PRs that implement those features. It's advisable to create an issue first to discuss the planned implementation, because in an effort to reduce bloat, some PRs may not be accepted. However, note that opening a feature request when you don't intend to implement the feature will rarely result in the request being fulfilled. + +## πŸ› οΈ Building and running locally + +Make sure you have Node.js and Python 3.13 installed. + +```bash +cd metube/ui +# install Angular and build the UI +npm install +node_modules/.bin/ng build +# install python dependencies +cd .. +curl -LsSf https://astral.sh/uv/install.sh | sh +uv sync +# run +uv run python3 app/main.py +``` + +A Docker image can be built locally (it will build the UI too): + +```bash +docker build -t metube . +``` + +Note that if you're running the server in VSCode, your downloads will go to your user's Downloads folder (this is configured via the environment in `.vscode/launch.json`). diff --git a/app/headless_watcher.py b/app/headless_watcher.py new file mode 100644 index 0000000..d6c9e36 --- /dev/null +++ b/app/headless_watcher.py @@ -0,0 +1,144 @@ +""" +Mark videos as watched using a headless browser. +Visiting the page with authenticated cookies triggers the watch tracking JavaScript. +""" + +import os +import re +import logging +import asyncio +from urllib.parse import urlparse, parse_qs +from typing import Optional, Dict + +log = logging.getLogger("headless_watcher") + +# Try to import playwright +try: + from playwright.async_api import async_playwright + HAS_PLAYWRIGHT = True +except ImportError: + HAS_PLAYWRIGHT = False + log.warning("Playwright not installed, headless watching will not work") + + +class HeadlessWatcher: + """Uses headless browser to visit pages and trigger watch tracking.""" + def __init__(self, cookie_file: str): + self.cookie_file = cookie_file + self.domain_cookies = self._parse_cookie_file() + + def _parse_cookie_file(self) -> Dict[str, list]: + """Parse Netscape cookie file and group by domain.""" + cookies_by_domain = {} + + if not os.path.exists(self.cookie_file): + log.warning(f"Cookie file not found: {self.cookie_file}") + return cookies_by_domain + + try: + with open(self.cookie_file, "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split("\t") + if len(parts) >= 7: + domain = parts[0].lstrip(".") + path = parts[2] + secure = parts[3] == "TRUE" + name = parts[5] + value = parts[6] + if domain not in cookies_by_domain: + cookies_by_domain[domain] = [] + cookies_by_domain[domain].append({ + "name": name, + "value": value, + "domain": parts[0], + "path": path, + "secure": secure, + "httpOnly": False, + }) + log.debug(f"Parsed cookies for {len(cookies_by_domain)} domains") + except Exception as e: + log.error(f"Error parsing cookie file: {e}") + return cookies_by_domain + + async def visit_page(self, url: str, wait_seconds: int = 5) -> bool: + """Visit a page with cookies to trigger watch tracking.""" + if not HAS_PLAYWRIGHT: + log.error("Playwright not installed") + return False + + parsed_url = urlparse(url) + domain = parsed_url.netloc.lower() + + cookies = [] + for cookie_domain, domain_cookies in self.domain_cookies.items(): + if domain in cookie_domain or cookie_domain in domain: + cookies.extend(domain_cookies) + + if not cookies: + log.warning(f"No cookies found for domain: {domain}") + return False + + log.info(f"Visiting {url} with {len(cookies)} cookies") + + try: + async with async_playwright() as p: + # Use system Chromium if available (for Alpine/Docker) + chromium_path = os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH") + if chromium_path and os.path.exists(chromium_path): + log.debug(f"Using system Chromium: {chromium_path}") + browser = await p.chromium.launch(headless=True, executable_path=chromium_path) + else: + browser = await p.chromium.launch(headless=True) + try: + context = await browser.new_context( + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + ) + await context.add_cookies(cookies) + page = await context.new_page() + response = await page.goto(url, wait_until="networkidle", timeout=30000) + if response and response.status == 200: + log.debug(f"Page loaded, waiting {wait_seconds}s") + await asyncio.sleep(wait_seconds) + log.info("Successfully triggered watch tracking") + return True + else: + status = response.status if response else "no response" + log.warning(f"Failed to load page, status: {status}") + return False + finally: + await browser.close() + except Exception as e: + log.error(f"Error in headless browser: {e}") + return False + +class PHEadlessWatcher(HeadlessWatcher): + """PornHub specific headless watcher.""" + + DOMAINS = ["pornhub.com", "www.pornhub.com", "de.pornhub.com", "fr.pornhub.com", "es.pornhub.com", "it.pornhub.com", "rt.pornhub.com"] + + def can_handle(self, url: str) -> bool: + parsed = urlparse(url) + domain = parsed.netloc.lower() + return any(d in domain for d in self.DOMAINS) + + async def mark_watched(self, url: str, wait_seconds: int = 5) -> bool: + return await self.visit_page(url, wait_seconds=wait_seconds) + + +async def headless_mark_watched(url: str, cookie_file: str, wait_seconds: int = 5) -> bool: + """Mark a video as watched using headless browser.""" + if not HAS_PLAYWRIGHT: + log.error("Playwright is not installed") + return False + + # Try PH handler first + ph_handler = PHHeadlessWatcher(cookie_file) + if ph_handler.can_handle(url): + return await ph_handler.mark_watched(url, wait_seconds) + + # Fallback to generic handler + generic_handler = HeadlessWatcher(cookie_file) + return await generic_handler.visit_page(url, wait_seconds) diff --git a/app/main.py b/app/main.py index f0c81d5..d46d6be 100644 --- a/app/main.py +++ b/app/main.py @@ -1,526 +1,527 @@ -#!/usr/bin/env python3 -# pylint: disable=no-member,method-hidden - -import os -import sys -import asyncio -from pathlib import Path -from aiohttp import web -from aiohttp.log import access_logger -import ssl -import socket -import socketio -import logging -import json -import pathlib -import re -import base64 -from urllib.parse import urlparse -from watchfiles import DefaultFilter, Change, awatch - -from ytdl import DownloadQueueNotifier, DownloadQueue -from yt_dlp.version import __version__ as yt_dlp_version - -log = logging.getLogger('main') - -class Config: - _DEFAULTS = { - 'DOWNLOAD_DIR': '.', - 'AUDIO_DOWNLOAD_DIR': '%%DOWNLOAD_DIR', - 'TEMP_DIR': '%%DOWNLOAD_DIR', - 'DOWNLOAD_DIRS_INDEXABLE': 'false', - 'CUSTOM_DIRS': 'true', - 'CREATE_CUSTOM_DIRS': 'true', - 'CUSTOM_DIRS_EXCLUDE_REGEX': r'(^|/)[.@].*$', - 'DELETE_FILE_ON_TRASHCAN': 'true', - 'STATE_DIR': '.', - 'URL_PREFIX': '', - 'PUBLIC_HOST_URL': 'download/', - 'PUBLIC_HOST_AUDIO_URL': 'audio_download/', - 'OUTPUT_TEMPLATE': '%(title)s.%(ext)s', - 'OUTPUT_TEMPLATE_CHAPTER': '%(title)s - %(section_number)s %(section_title)s.%(ext)s', - 'OUTPUT_TEMPLATE_PLAYLIST': '%(playlist_title)s/%(title)s.%(ext)s', - 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE' : 'false', - 'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT' : '0', - 'YTDL_OPTIONS': '{}', - 'YTDL_OPTIONS_FILE': '', - 'ROBOTS_TXT': '', - 'HOST': '0.0.0.0', - 'PORT': '8081', - 'HTTPS': 'false', - 'CERTFILE': '', - 'KEYFILE': '', - 'BASE_DIR': '', - 'DEFAULT_THEME': 'auto', - 'DOWNLOAD_MODE': 'limited', - 'MAX_CONCURRENT_DOWNLOADS': 3, - 'LOGLEVEL': 'INFO', - 'ENABLE_ACCESSLOG': 'false', - } - - _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE', 'HTTPS', 'ENABLE_ACCESSLOG') - - def __init__(self): - for k, v in self._DEFAULTS.items(): - setattr(self, k, os.environ.get(k, v)) - - for k, v in self.__dict__.items(): - if isinstance(v, str) and v.startswith('%%'): - setattr(self, k, getattr(self, v[2:])) - if k in self._BOOLEAN: - if v not in ('true', 'false', 'True', 'False', 'on', 'off', '1', '0'): - log.error(f'Environment variable "{k}" is set to a non-boolean value "{v}"') - sys.exit(1) - setattr(self, k, v in ('true', 'True', 'on', '1')) - - if not self.URL_PREFIX.endswith('/'): - self.URL_PREFIX += '/' - - # Convert relative addresses to absolute addresses to prevent the failure of file address comparison - if self.YTDL_OPTIONS_FILE and self.YTDL_OPTIONS_FILE.startswith('.'): - self.YTDL_OPTIONS_FILE = str(Path(self.YTDL_OPTIONS_FILE).resolve()) - - success,_ = self.load_ytdl_options() - if not success: - sys.exit(1) - - def load_ytdl_options(self) -> tuple[bool, str]: - try: - self.YTDL_OPTIONS = json.loads(os.environ.get('YTDL_OPTIONS', '{}')) - assert isinstance(self.YTDL_OPTIONS, dict) - except (json.decoder.JSONDecodeError, AssertionError): - msg = 'Environment variable YTDL_OPTIONS is invalid' - log.error(msg) - return (False, msg) - - if not self.YTDL_OPTIONS_FILE: - return (True, '') - - log.info(f'Loading yt-dlp custom options from "{self.YTDL_OPTIONS_FILE}"') - if not os.path.exists(self.YTDL_OPTIONS_FILE): - msg = f'File "{self.YTDL_OPTIONS_FILE}" not found' - log.error(msg) - return (False, msg) - try: - with open(self.YTDL_OPTIONS_FILE) as json_data: - opts = json.load(json_data) - assert isinstance(opts, dict) - except (json.decoder.JSONDecodeError, AssertionError): - msg = 'YTDL_OPTIONS_FILE contents is invalid' - log.error(msg) - return (False, msg) - - self.YTDL_OPTIONS.update(opts) - return (True, '') - -config = Config() - -class ObjectSerializer(json.JSONEncoder): - def default(self, obj): - # First try to use __dict__ for custom objects - if hasattr(obj, '__dict__'): - return obj.__dict__ - # Convert iterables (generators, dict_items, etc.) to lists - # Exclude strings and bytes which are also iterable - elif hasattr(obj, '__iter__') and not isinstance(obj, (str, bytes)): - try: - return list(obj) - except: - pass - # Fall back to default behavior - return json.JSONEncoder.default(self, obj) - -serializer = ObjectSerializer() -app = web.Application() -sio = socketio.AsyncServer(cors_allowed_origins='*') -sio.attach(app, socketio_path=config.URL_PREFIX + 'socket.io') -routes = web.RouteTableDef() - -class Notifier(DownloadQueueNotifier): - async def added(self, dl): - log.info(f"Notifier: Download added - {dl.title}") - await sio.emit('added', serializer.encode(dl)) - - async def updated(self, dl): - log.info(f"Notifier: Download updated - {dl.title}") - await sio.emit('updated', serializer.encode(dl)) - - async def completed(self, dl): - log.info(f"Notifier: Download completed - {dl.title}") - await sio.emit('completed', serializer.encode(dl)) - - async def canceled(self, id): - log.info(f"Notifier: Download canceled - {id}") - await sio.emit('canceled', serializer.encode(id)) - - async def cleared(self, id): - log.info(f"Notifier: Download cleared - {id}") - await sio.emit('cleared', serializer.encode(id)) - - async def event(self, event): - log.info(f"Notifier: Event - {event['type']}") - await sio.emit('event', serializer.encode(event)) - -dqueue = DownloadQueue(config, Notifier()) -app.on_startup.append(lambda app: dqueue.initialize()) - -class FileOpsFilter(DefaultFilter): - def __call__(self, change_type: int, path: str) -> bool: - # Check if this path matches our YTDL_OPTIONS_FILE - if path != config.YTDL_OPTIONS_FILE: - return False - - # For existing files, use samefile comparison to handle symlinks correctly - if os.path.exists(config.YTDL_OPTIONS_FILE): - try: - if not os.path.samefile(path, config.YTDL_OPTIONS_FILE): - return False - except (OSError, IOError): - # If samefile fails, fall back to string comparison - if path != config.YTDL_OPTIONS_FILE: - return False - - # Accept all change types for our file: modified, added, deleted - return change_type in (Change.modified, Change.added, Change.deleted) - -def get_options_update_time(success=True, msg=''): - result = { - 'success': success, - 'msg': msg, - 'update_time': None - } - - # Only try to get file modification time if YTDL_OPTIONS_FILE is set and file exists - if config.YTDL_OPTIONS_FILE and os.path.exists(config.YTDL_OPTIONS_FILE): - try: - result['update_time'] = os.path.getmtime(config.YTDL_OPTIONS_FILE) - except (OSError, IOError) as e: - log.warning(f"Could not get modification time for {config.YTDL_OPTIONS_FILE}: {e}") - result['update_time'] = None - - return result - -async def watch_files(): - async def _watch_files(): - async for changes in awatch(config.YTDL_OPTIONS_FILE, watch_filter=FileOpsFilter()): - success, msg = config.load_ytdl_options() - result = get_options_update_time(success, msg) - await sio.emit('ytdl_options_changed', serializer.encode(result)) - - log.info(f'Starting Watch File: {config.YTDL_OPTIONS_FILE}') - asyncio.create_task(_watch_files()) - -if config.YTDL_OPTIONS_FILE: - app.on_startup.append(lambda app: watch_files()) - -@routes.post(config.URL_PREFIX + 'add') -async def add(request): - log.info("Received request to add download") - post = await request.json() - log.info(f"Request data: {post}") - url = post.get('url') - quality = post.get('quality') - if not url or not quality: - log.error("Bad request: missing 'url' or 'quality'") - raise web.HTTPBadRequest() - format = post.get('format') - folder = post.get('folder') - custom_name_prefix = post.get('custom_name_prefix') - playlist_strict_mode = post.get('playlist_strict_mode') - playlist_item_limit = post.get('playlist_item_limit') - auto_start = post.get('auto_start') - - if custom_name_prefix is None: - custom_name_prefix = '' - if auto_start is None: - auto_start = True - if playlist_strict_mode is None: - playlist_strict_mode = config.DEFAULT_OPTION_PLAYLIST_STRICT_MODE - if playlist_item_limit is None: - playlist_item_limit = config.DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT - - playlist_item_limit = int(playlist_item_limit) - - status = await dqueue.add(url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start) - return web.Response(text=serializer.encode(status)) - -@routes.post(config.URL_PREFIX + 'cookie') -async def set_cookie(request): - """Accept cookie string and save as cookie file for domain""" - log.info("Received request to set cookie") - post = await request.json() - url = post.get('url') - cookie = post.get('cookie') - domain = post.get('domain') - - if not cookie: - log.error("Bad request: missing 'cookie'") - raise web.HTTPBadRequest() - - # Determine domain from either explicit domain field or URL - if not domain: - if url: - parsed_url = urlparse(url) - domain = parsed_url.netloc - else: - log.error("Bad request: missing both 'url' and 'domain'") - raise web.HTTPBadRequest() - - log.info(f"Processing cookie for domain: {domain}") - - try: - # Decode base64 cookie if it appears to be encoded - try: - # Check if cookie is base64 encoded - decoded_cookie = base64.b64decode(cookie).decode('utf-8') - log.info(f"Cookie was base64 encoded, decoded successfully") - cookie = decoded_cookie - except Exception as e: - # If decoding fails, assume it's already plain text - log.info(f"Cookie is not base64 encoded or decode failed ({e}), using as-is") - - log.debug(f"Cookie content: {cookie[:100]}...") # Log first 100 chars - - # Create cookies directory if it doesn't exist - cookies_dir = os.path.join(config.STATE_DIR, 'cookies') - os.makedirs(cookies_dir, exist_ok=True) - - # Use domain as filename (sanitized) - safe_domain = domain.replace(':', '_').replace('/', '_') - cookie_file = os.path.join(cookies_dir, f'{safe_domain}.txt') - - log.info(f"Writing cookie file to: {cookie_file}") - - # Convert cookie string to Netscape cookie file format - with open(cookie_file, 'w') as f: - f.write('# Netscape HTTP Cookie File\n') - f.write(f'# This file was generated by MeTube for {domain}\n') - f.write('# Edit at your own risk.\n\n') - - # Parse cookie string (format: "key1=value1; key2=value2; ...") - cookie_count = 0 - for cookie_pair in cookie.split(';'): - cookie_pair = cookie_pair.strip() - if '=' in cookie_pair: - key, value = cookie_pair.split('=', 1) - key = key.strip() - value = value.strip() - # Netscape format: domain\tflag\tpath\tsecure\texpiration\tname\tvalue - # domain: .domain.com (with leading dot for all subdomains) - # flag: TRUE (include subdomains) - # path: / (all paths) - # secure: FALSE (http and https) - # expiration: 2147483647 (max 32-bit timestamp - Jan 2038) - # name: cookie name - # value: cookie value - f.write(f'.{domain}\tTRUE\t/\tFALSE\t2147483647\t{key}\t{value}\n') - cookie_count += 1 - log.debug(f"Added cookie: {key}={value[:20]}...") - - log.info(f"Cookie file created successfully with {cookie_count} cookies at {cookie_file}") - return web.Response(text=serializer.encode({ - 'status': 'ok', - 'cookie_file': cookie_file, - 'cookie_count': cookie_count, - 'msg': f'Cookie saved successfully for {domain} ({cookie_count} cookies)' - })) - except Exception as e: - log.error(f"Error saving cookie: {str(e)}", exc_info=True) - return web.Response(text=serializer.encode({ - 'status': 'error', - 'msg': f'Failed to save cookie: {str(e)}' - })) - -@routes.post(config.URL_PREFIX + 'delete') -async def delete(request): - post = await request.json() - ids = post.get('ids') - where = post.get('where') - if not ids or where not in ['queue', 'done']: - log.error("Bad request: missing 'ids' or incorrect 'where' value") - raise web.HTTPBadRequest() - status = await (dqueue.cancel(ids) if where == 'queue' else dqueue.clear(ids)) - log.info(f"Download delete request processed for ids: {ids}, where: {where}") - return web.Response(text=serializer.encode(status)) - -@routes.post(config.URL_PREFIX + 'start') -async def start(request): - post = await request.json() - ids = post.get('ids') - log.info(f"Received request to start pending downloads for ids: {ids}") - status = await dqueue.start_pending(ids) - return web.Response(text=serializer.encode(status)) - -@routes.get(config.URL_PREFIX + 'history') -async def history(request): - history = { 'done': [], 'queue': [], 'pending': []} - - for _, v in dqueue.queue.saved_items(): - history['queue'].append(v) - for _, v in dqueue.done.saved_items(): - history['done'].append(v) - for _, v in dqueue.pending.saved_items(): - history['pending'].append(v) - - log.info("Sending download history") - return web.Response(text=serializer.encode(history)) - -@sio.event -async def connect(sid, environ): - log.info(f"Client connected: {sid}") - await sio.emit('all', serializer.encode(dqueue.get()), to=sid) - await sio.emit('configuration', serializer.encode(config), to=sid) - if config.CUSTOM_DIRS: - await sio.emit('custom_dirs', serializer.encode(get_custom_dirs()), to=sid) - if config.YTDL_OPTIONS_FILE: - await sio.emit('ytdl_options_changed', serializer.encode(get_options_update_time()), to=sid) - -def get_custom_dirs(): - def recursive_dirs(base): - path = pathlib.Path(base) - - # Converts PosixPath object to string, and remove base/ prefix - def convert(p): - s = str(p) - if s.startswith(base): - s = s[len(base):] - - if s.startswith('/'): - s = s[1:] - - return s - - # Include only directories which do not match the exclude filter - def include_dir(d): - if len(config.CUSTOM_DIRS_EXCLUDE_REGEX) == 0: - return True - else: - return re.search(config.CUSTOM_DIRS_EXCLUDE_REGEX, d) is None - - # Recursively lists all subdirectories of DOWNLOAD_DIR - dirs = list(filter(include_dir, map(convert, path.glob('**/')))) - - return dirs - - download_dir = recursive_dirs(config.DOWNLOAD_DIR) - - audio_download_dir = download_dir - if config.DOWNLOAD_DIR != config.AUDIO_DOWNLOAD_DIR: - audio_download_dir = recursive_dirs(config.AUDIO_DOWNLOAD_DIR) - - return { - "download_dir": download_dir, - "audio_download_dir": audio_download_dir - } - -@routes.get(config.URL_PREFIX) -def index(request): - response = web.FileResponse(os.path.join(config.BASE_DIR, 'ui/dist/metube/browser/index.html')) - if 'metube_theme' not in request.cookies: - response.set_cookie('metube_theme', config.DEFAULT_THEME) - return response - -@routes.get(config.URL_PREFIX + 'robots.txt') -def robots(request): - if config.ROBOTS_TXT: - response = web.FileResponse(os.path.join(config.BASE_DIR, config.ROBOTS_TXT)) - else: - response = web.Response( - text="User-agent: *\nDisallow: /download/\nDisallow: /audio_download/\n" - ) - return response - -@routes.get(config.URL_PREFIX + 'version') -def version(request): - return web.json_response({ - "yt-dlp": yt_dlp_version, - "version": os.getenv("METUBE_VERSION", "dev") - }) - -@routes.get(config.URL_PREFIX + 'events') -def get_events(request): - events = dqueue.get_events() - return web.Response(text=serializer.encode(events)) - -@routes.post(config.URL_PREFIX + 'events/clear') -async def clear_events(request): - dqueue.clear_events() - return web.Response(text=serializer.encode({'status': 'ok'})) - -if config.URL_PREFIX != '/': - @routes.get('/') - def index_redirect_root(request): - return web.HTTPFound(config.URL_PREFIX) - - @routes.get(config.URL_PREFIX[:-1]) - def index_redirect_dir(request): - return web.HTTPFound(config.URL_PREFIX) - -routes.static(config.URL_PREFIX + 'download/', config.DOWNLOAD_DIR, show_index=config.DOWNLOAD_DIRS_INDEXABLE) -routes.static(config.URL_PREFIX + 'audio_download/', config.AUDIO_DOWNLOAD_DIR, show_index=config.DOWNLOAD_DIRS_INDEXABLE) -routes.static(config.URL_PREFIX, os.path.join(config.BASE_DIR, 'ui/dist/metube/browser')) -try: - app.add_routes(routes) -except ValueError as e: - if 'ui/dist/metube/browser' in str(e): - raise RuntimeError('Could not find the frontend UI static assets. Please run `node_modules/.bin/ng build` inside the ui folder') from e - raise e - -# https://github.com/aio-libs/aiohttp/pull/4615 waiting for release -# @routes.options(config.URL_PREFIX + 'add') -async def add_cors(request): - return web.Response(text=serializer.encode({"status": "ok"})) - -async def cookie_cors(request): - return web.Response(text=serializer.encode({"status": "ok"})) - -app.router.add_route('OPTIONS', config.URL_PREFIX + 'add', add_cors) -app.router.add_route('OPTIONS', config.URL_PREFIX + 'cookie', cookie_cors) - -async def on_prepare(request, response): - if 'Origin' in request.headers: - response.headers['Access-Control-Allow-Origin'] = request.headers['Origin'] - response.headers['Access-Control-Allow-Headers'] = 'Content-Type' - -app.on_response_prepare.append(on_prepare) - -def supports_reuse_port(): - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - sock.close() - return True - except (AttributeError, OSError): - return False - -def parseLogLevel(logLevel): - match logLevel: - case 'DEBUG': - return logging.DEBUG - case 'INFO': - return logging.INFO - case 'WARNING': - return logging.WARNING - case 'ERROR': - return logging.ERROR - case 'CRITICAL': - return logging.CRITICAL - case _: - return None - -def isAccessLogEnabled(): - if config.ENABLE_ACCESSLOG: - return access_logger - else: - return None - -if __name__ == '__main__': - logging.basicConfig(level=parseLogLevel(config.LOGLEVEL)) - log.info(f"Listening on {config.HOST}:{config.PORT}") - - if config.HTTPS: - ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - ssl_context.load_cert_chain(certfile=config.CERTFILE, keyfile=config.KEYFILE) - web.run_app(app, host=config.HOST, port=int(config.PORT), reuse_port=supports_reuse_port(), ssl_context=ssl_context, access_log=isAccessLogEnabled()) - else: - web.run_app(app, host=config.HOST, port=int(config.PORT), reuse_port=supports_reuse_port(), access_log=isAccessLogEnabled()) +#!/usr/bin/env python3 +# pylint: disable=no-member,method-hidden + +import os +import sys +import asyncio +from pathlib import Path +from aiohttp import web +from aiohttp.log import access_logger +import ssl +import socket +import socketio +import logging +import json +import pathlib +import re +import base64 +from urllib.parse import urlparse +from watchfiles import DefaultFilter, Change, awatch + +from ytdl import DownloadQueueNotifier, DownloadQueue +from yt_dlp.version import __version__ as yt_dlp_version + +log = logging.getLogger('main') + +class Config: + _DEFAULTS = { + 'DOWNLOAD_DIR': '.', + 'AUDIO_DOWNLOAD_DIR': '%%DOWNLOAD_DIR', + 'TEMP_DIR': '%%DOWNLOAD_DIR', + 'DOWNLOAD_DIRS_INDEXABLE': 'false', + 'CUSTOM_DIRS': 'true', + 'CREATE_CUSTOM_DIRS': 'true', + 'CUSTOM_DIRS_EXCLUDE_REGEX': r'(^|/)[.@].*$', + 'DELETE_FILE_ON_TRASHCAN': 'true', + 'STATE_DIR': '.', + 'URL_PREFIX': '', + 'PUBLIC_HOST_URL': 'download/', + 'PUBLIC_HOST_AUDIO_URL': 'audio_download/', + 'OUTPUT_TEMPLATE': '%(title)s.%(ext)s', + 'OUTPUT_TEMPLATE_CHAPTER': '%(title)s - %(section_number)s %(section_title)s.%(ext)s', + 'OUTPUT_TEMPLATE_PLAYLIST': '%(playlist_title)s/%(title)s.%(ext)s', + 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE' : 'false', + 'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT' : '0', + 'YTDL_OPTIONS': '{}', + 'YTDL_OPTIONS_FILE': '', + 'ROBOTS_TXT': '', + 'HOST': '0.0.0.0', + 'PORT': '8081', + 'HTTPS': 'false', + 'CERTFILE': '', + 'KEYFILE': '', + 'BASE_DIR': '', + 'DEFAULT_THEME': 'auto', + 'DOWNLOAD_MODE': 'limited', + 'MAX_CONCURRENT_DOWNLOADS': 3, + 'LOGLEVEL': 'INFO', + 'ENABLE_ACCESSLOG': 'false', + 'MARK_WATCHED_ON_COMPLETE': 'false', + } + + _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE', 'HTTPS', 'ENABLE_ACCESSLOG', 'MARK_WATCHED_ON_COMPLETE') + + def __init__(self): + for k, v in self._DEFAULTS.items(): + setattr(self, k, os.environ.get(k, v)) + + for k, v in self.__dict__.items(): + if isinstance(v, str) and v.startswith('%%'): + setattr(self, k, getattr(self, v[2:])) + if k in self._BOOLEAN: + if v not in ('true', 'false', 'True', 'False', 'on', 'off', '1', '0'): + log.error(f'Environment variable "{k}" is set to a non-boolean value "{v}"') + sys.exit(1) + setattr(self, k, v in ('true', 'True', 'on', '1')) + + if not self.URL_PREFIX.endswith('/'): + self.URL_PREFIX += '/' + + # Convert relative addresses to absolute addresses to prevent the failure of file address comparison + if self.YTDL_OPTIONS_FILE and self.YTDL_OPTIONS_FILE.startswith('.'): + self.YTDL_OPTIONS_FILE = str(Path(self.YTDL_OPTIONS_FILE).resolve()) + + success,_ = self.load_ytdl_options() + if not success: + sys.exit(1) + + def load_ytdl_options(self) -> tuple[bool, str]: + try: + self.YTDL_OPTIONS = json.loads(os.environ.get('YTDL_OPTIONS', '{}')) + assert isinstance(self.YTDL_OPTIONS, dict) + except (json.decoder.JSONDecodeError, AssertionError): + msg = 'Environment variable YTDL_OPTIONS is invalid' + log.error(msg) + return (False, msg) + + if not self.YTDL_OPTIONS_FILE: + return (True, '') + + log.info(f'Loading yt-dlp custom options from "{self.YTDL_OPTIONS_FILE}"') + if not os.path.exists(self.YTDL_OPTIONS_FILE): + msg = f'File "{self.YTDL_OPTIONS_FILE}" not found' + log.error(msg) + return (False, msg) + try: + with open(self.YTDL_OPTIONS_FILE) as json_data: + opts = json.load(json_data) + assert isinstance(opts, dict) + except (json.decoder.JSONDecodeError, AssertionError): + msg = 'YTDL_OPTIONS_FILE contents is invalid' + log.error(msg) + return (False, msg) + + self.YTDL_OPTIONS.update(opts) + return (True, '') + +config = Config() + +class ObjectSerializer(json.JSONEncoder): + def default(self, obj): + # First try to use __dict__ for custom objects + if hasattr(obj, '__dict__'): + return obj.__dict__ + # Convert iterables (generators, dict_items, etc.) to lists + # Exclude strings and bytes which are also iterable + elif hasattr(obj, '__iter__') and not isinstance(obj, (str, bytes)): + try: + return list(obj) + except: + pass + # Fall back to default behavior + return json.JSONEncoder.default(self, obj) + +serializer = ObjectSerializer() +app = web.Application() +sio = socketio.AsyncServer(cors_allowed_origins='*') +sio.attach(app, socketio_path=config.URL_PREFIX + 'socket.io') +routes = web.RouteTableDef() + +class Notifier(DownloadQueueNotifier): + async def added(self, dl): + log.info(f"Notifier: Download added - {dl.title}") + await sio.emit('added', serializer.encode(dl)) + + async def updated(self, dl): + log.info(f"Notifier: Download updated - {dl.title}") + await sio.emit('updated', serializer.encode(dl)) + + async def completed(self, dl): + log.info(f"Notifier: Download completed - {dl.title}") + await sio.emit('completed', serializer.encode(dl)) + + async def canceled(self, id): + log.info(f"Notifier: Download canceled - {id}") + await sio.emit('canceled', serializer.encode(id)) + + async def cleared(self, id): + log.info(f"Notifier: Download cleared - {id}") + await sio.emit('cleared', serializer.encode(id)) + + async def event(self, event): + log.info(f"Notifier: Event - {event['type']}") + await sio.emit('event', serializer.encode(event)) + +dqueue = DownloadQueue(config, Notifier()) +app.on_startup.append(lambda app: dqueue.initialize()) + +class FileOpsFilter(DefaultFilter): + def __call__(self, change_type: int, path: str) -> bool: + # Check if this path matches our YTDL_OPTIONS_FILE + if path != config.YTDL_OPTIONS_FILE: + return False + + # For existing files, use samefile comparison to handle symlinks correctly + if os.path.exists(config.YTDL_OPTIONS_FILE): + try: + if not os.path.samefile(path, config.YTDL_OPTIONS_FILE): + return False + except (OSError, IOError): + # If samefile fails, fall back to string comparison + if path != config.YTDL_OPTIONS_FILE: + return False + + # Accept all change types for our file: modified, added, deleted + return change_type in (Change.modified, Change.added, Change.deleted) + +def get_options_update_time(success=True, msg=''): + result = { + 'success': success, + 'msg': msg, + 'update_time': None + } + + # Only try to get file modification time if YTDL_OPTIONS_FILE is set and file exists + if config.YTDL_OPTIONS_FILE and os.path.exists(config.YTDL_OPTIONS_FILE): + try: + result['update_time'] = os.path.getmtime(config.YTDL_OPTIONS_FILE) + except (OSError, IOError) as e: + log.warning(f"Could not get modification time for {config.YTDL_OPTIONS_FILE}: {e}") + result['update_time'] = None + + return result + +async def watch_files(): + async def _watch_files(): + async for changes in awatch(config.YTDL_OPTIONS_FILE, watch_filter=FileOpsFilter()): + success, msg = config.load_ytdl_options() + result = get_options_update_time(success, msg) + await sio.emit('ytdl_options_changed', serializer.encode(result)) + + log.info(f'Starting Watch File: {config.YTDL_OPTIONS_FILE}') + asyncio.create_task(_watch_files()) + +if config.YTDL_OPTIONS_FILE: + app.on_startup.append(lambda app: watch_files()) + +@routes.post(config.URL_PREFIX + 'add') +async def add(request): + log.info("Received request to add download") + post = await request.json() + log.info(f"Request data: {post}") + url = post.get('url') + quality = post.get('quality') + if not url or not quality: + log.error("Bad request: missing 'url' or 'quality'") + raise web.HTTPBadRequest() + format = post.get('format') + folder = post.get('folder') + custom_name_prefix = post.get('custom_name_prefix') + playlist_strict_mode = post.get('playlist_strict_mode') + playlist_item_limit = post.get('playlist_item_limit') + auto_start = post.get('auto_start') + + if custom_name_prefix is None: + custom_name_prefix = '' + if auto_start is None: + auto_start = True + if playlist_strict_mode is None: + playlist_strict_mode = config.DEFAULT_OPTION_PLAYLIST_STRICT_MODE + if playlist_item_limit is None: + playlist_item_limit = config.DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT + + playlist_item_limit = int(playlist_item_limit) + + status = await dqueue.add(url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start) + return web.Response(text=serializer.encode(status)) + +@routes.post(config.URL_PREFIX + 'cookie') +async def set_cookie(request): + """Accept cookie string and save as cookie file for domain""" + log.info("Received request to set cookie") + post = await request.json() + url = post.get('url') + cookie = post.get('cookie') + domain = post.get('domain') + + if not cookie: + log.error("Bad request: missing 'cookie'") + raise web.HTTPBadRequest() + + # Determine domain from either explicit domain field or URL + if not domain: + if url: + parsed_url = urlparse(url) + domain = parsed_url.netloc + else: + log.error("Bad request: missing both 'url' and 'domain'") + raise web.HTTPBadRequest() + + log.info(f"Processing cookie for domain: {domain}") + + try: + # Decode base64 cookie if it appears to be encoded + try: + # Check if cookie is base64 encoded + decoded_cookie = base64.b64decode(cookie).decode('utf-8') + log.info(f"Cookie was base64 encoded, decoded successfully") + cookie = decoded_cookie + except Exception as e: + # If decoding fails, assume it's already plain text + log.info(f"Cookie is not base64 encoded or decode failed ({e}), using as-is") + + log.debug(f"Cookie content: {cookie[:100]}...") # Log first 100 chars + + # Create cookies directory if it doesn't exist + cookies_dir = os.path.join(config.STATE_DIR, 'cookies') + os.makedirs(cookies_dir, exist_ok=True) + + # Use domain as filename (sanitized) + safe_domain = domain.replace(':', '_').replace('/', '_') + cookie_file = os.path.join(cookies_dir, f'{safe_domain}.txt') + + log.info(f"Writing cookie file to: {cookie_file}") + + # Convert cookie string to Netscape cookie file format + with open(cookie_file, 'w') as f: + f.write('# Netscape HTTP Cookie File\n') + f.write(f'# This file was generated by MeTube for {domain}\n') + f.write('# Edit at your own risk.\n\n') + + # Parse cookie string (format: "key1=value1; key2=value2; ...") + cookie_count = 0 + for cookie_pair in cookie.split(';'): + cookie_pair = cookie_pair.strip() + if '=' in cookie_pair: + key, value = cookie_pair.split('=', 1) + key = key.strip() + value = value.strip() + # Netscape format: domain\tflag\tpath\tsecure\texpiration\tname\tvalue + # domain: .domain.com (with leading dot for all subdomains) + # flag: TRUE (include subdomains) + # path: / (all paths) + # secure: FALSE (http and https) + # expiration: 2147483647 (max 32-bit timestamp - Jan 2038) + # name: cookie name + # value: cookie value + f.write(f'.{domain}\tTRUE\t/\tFALSE\t2147483647\t{key}\t{value}\n') + cookie_count += 1 + log.debug(f"Added cookie: {key}={value[:20]}...") + + log.info(f"Cookie file created successfully with {cookie_count} cookies at {cookie_file}") + return web.Response(text=serializer.encode({ + 'status': 'ok', + 'cookie_file': cookie_file, + 'cookie_count': cookie_count, + 'msg': f'Cookie saved successfully for {domain} ({cookie_count} cookies)' + })) + except Exception as e: + log.error(f"Error saving cookie: {str(e)}", exc_info=True) + return web.Response(text=serializer.encode({ + 'status': 'error', + 'msg': f'Failed to save cookie: {str(e)}' + })) + +@routes.post(config.URL_PREFIX + 'delete') +async def delete(request): + post = await request.json() + ids = post.get('ids') + where = post.get('where') + if not ids or where not in ['queue', 'done']: + log.error("Bad request: missing 'ids' or incorrect 'where' value") + raise web.HTTPBadRequest() + status = await (dqueue.cancel(ids) if where == 'queue' else dqueue.clear(ids)) + log.info(f"Download delete request processed for ids: {ids}, where: {where}") + return web.Response(text=serializer.encode(status)) + +@routes.post(config.URL_PREFIX + 'start') +async def start(request): + post = await request.json() + ids = post.get('ids') + log.info(f"Received request to start pending downloads for ids: {ids}") + status = await dqueue.start_pending(ids) + return web.Response(text=serializer.encode(status)) + +@routes.get(config.URL_PREFIX + 'history') +async def history(request): + history = { 'done': [], 'queue': [], 'pending': []} + + for _, v in dqueue.queue.saved_items(): + history['queue'].append(v) + for _, v in dqueue.done.saved_items(): + history['done'].append(v) + for _, v in dqueue.pending.saved_items(): + history['pending'].append(v) + + log.info("Sending download history") + return web.Response(text=serializer.encode(history)) + +@sio.event +async def connect(sid, environ): + log.info(f"Client connected: {sid}") + await sio.emit('all', serializer.encode(dqueue.get()), to=sid) + await sio.emit('configuration', serializer.encode(config), to=sid) + if config.CUSTOM_DIRS: + await sio.emit('custom_dirs', serializer.encode(get_custom_dirs()), to=sid) + if config.YTDL_OPTIONS_FILE: + await sio.emit('ytdl_options_changed', serializer.encode(get_options_update_time()), to=sid) + +def get_custom_dirs(): + def recursive_dirs(base): + path = pathlib.Path(base) + + # Converts PosixPath object to string, and remove base/ prefix + def convert(p): + s = str(p) + if s.startswith(base): + s = s[len(base):] + + if s.startswith('/'): + s = s[1:] + + return s + + # Include only directories which do not match the exclude filter + def include_dir(d): + if len(config.CUSTOM_DIRS_EXCLUDE_REGEX) == 0: + return True + else: + return re.search(config.CUSTOM_DIRS_EXCLUDE_REGEX, d) is None + + # Recursively lists all subdirectories of DOWNLOAD_DIR + dirs = list(filter(include_dir, map(convert, path.glob('**/')))) + + return dirs + + download_dir = recursive_dirs(config.DOWNLOAD_DIR) + + audio_download_dir = download_dir + if config.DOWNLOAD_DIR != config.AUDIO_DOWNLOAD_DIR: + audio_download_dir = recursive_dirs(config.AUDIO_DOWNLOAD_DIR) + + return { + "download_dir": download_dir, + "audio_download_dir": audio_download_dir + } + +@routes.get(config.URL_PREFIX) +def index(request): + response = web.FileResponse(os.path.join(config.BASE_DIR, 'ui/dist/metube/browser/index.html')) + if 'metube_theme' not in request.cookies: + response.set_cookie('metube_theme', config.DEFAULT_THEME) + return response + +@routes.get(config.URL_PREFIX + 'robots.txt') +def robots(request): + if config.ROBOTS_TXT: + response = web.FileResponse(os.path.join(config.BASE_DIR, config.ROBOTS_TXT)) + else: + response = web.Response( + text="User-agent: *\nDisallow: /download/\nDisallow: /audio_download/\n" + ) + return response + +@routes.get(config.URL_PREFIX + 'version') +def version(request): + return web.json_response({ + "yt-dlp": yt_dlp_version, + "version": os.getenv("METUBE_VERSION", "dev") + }) + +@routes.get(config.URL_PREFIX + 'events') +def get_events(request): + events = dqueue.get_events() + return web.Response(text=serializer.encode(events)) + +@routes.post(config.URL_PREFIX + 'events/clear') +async def clear_events(request): + dqueue.clear_events() + return web.Response(text=serializer.encode({'status': 'ok'})) + +if config.URL_PREFIX != '/': + @routes.get('/') + def index_redirect_root(request): + return web.HTTPFound(config.URL_PREFIX) + + @routes.get(config.URL_PREFIX[:-1]) + def index_redirect_dir(request): + return web.HTTPFound(config.URL_PREFIX) + +routes.static(config.URL_PREFIX + 'download/', config.DOWNLOAD_DIR, show_index=config.DOWNLOAD_DIRS_INDEXABLE) +routes.static(config.URL_PREFIX + 'audio_download/', config.AUDIO_DOWNLOAD_DIR, show_index=config.DOWNLOAD_DIRS_INDEXABLE) +routes.static(config.URL_PREFIX, os.path.join(config.BASE_DIR, 'ui/dist/metube/browser')) +try: + app.add_routes(routes) +except ValueError as e: + if 'ui/dist/metube/browser' in str(e): + raise RuntimeError('Could not find the frontend UI static assets. Please run `node_modules/.bin/ng build` inside the ui folder') from e + raise e + +# https://github.com/aio-libs/aiohttp/pull/4615 waiting for release +# @routes.options(config.URL_PREFIX + 'add') +async def add_cors(request): + return web.Response(text=serializer.encode({"status": "ok"})) + +async def cookie_cors(request): + return web.Response(text=serializer.encode({"status": "ok"})) + +app.router.add_route('OPTIONS', config.URL_PREFIX + 'add', add_cors) +app.router.add_route('OPTIONS', config.URL_PREFIX + 'cookie', cookie_cors) + +async def on_prepare(request, response): + if 'Origin' in request.headers: + response.headers['Access-Control-Allow-Origin'] = request.headers['Origin'] + response.headers['Access-Control-Allow-Headers'] = 'Content-Type' + +app.on_response_prepare.append(on_prepare) + +def supports_reuse_port(): + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + sock.close() + return True + except (AttributeError, OSError): + return False + +def parseLogLevel(logLevel): + match logLevel: + case 'DEBUG': + return logging.DEBUG + case 'INFO': + return logging.INFO + case 'WARNING': + return logging.WARNING + case 'ERROR': + return logging.ERROR + case 'CRITICAL': + return logging.CRITICAL + case _: + return None + +def isAccessLogEnabled(): + if config.ENABLE_ACCESSLOG: + return access_logger + else: + return None + +if __name__ == '__main__': + logging.basicConfig(level=parseLogLevel(config.LOGLEVEL)) + log.info(f"Listening on {config.HOST}:{config.PORT}") + + if config.HTTPS: + ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + ssl_context.load_cert_chain(certfile=config.CERTFILE, keyfile=config.KEYFILE) + web.run_app(app, host=config.HOST, port=int(config.PORT), reuse_port=supports_reuse_port(), ssl_context=ssl_context, access_log=isAccessLogEnabled()) + else: + web.run_app(app, host=config.HOST, port=int(config.PORT), reuse_port=supports_reuse_port(), access_log=isAccessLogEnabled()) diff --git a/app/mark_watched.py b/app/mark_watched.py new file mode 100644 index 0000000..a3c2f57 --- /dev/null +++ b/app/mark_watched.py @@ -0,0 +1,228 @@ +""" +Mark videos as watched on various websites after successful download. +Uses the same cookies that were used for downloading. +""" + +import os +import re +import logging +from urllib.parse import urlparse, parse_qs +from typing import Optional, Dict, Callable + +from headless_watcher import headless_mark_watched, HAS_PLAYWRIGHT + +log = logging.getLogger("mark_watched") + +# Try to import curl_cffi for better impersonation, fallback to requests +try: + from curl_cffi import requests as curl_requests + HAS_CURL_CFFI = True +except ImportError: + HAS_CURL_CFFI = False + import requests + + +class MarkWatchedHandler: + """Base class for mark-as-watched handlers.""" + + def __init__(self, cookie_file: str): + self.cookie_file = cookie_file + + def can_handle(self, url: str) -> bool: + """Check if this handler can handle the given URL.""" + raise NotImplementedError + + async def mark_watched(self, url: str) -> bool: + """Mark the video as watched. Returns True on success.""" + raise NotImplementedError + + def _load_cookies(self) -> Dict[str, str]: + """Load cookies from the Netscape cookie file.""" + cookies = {} + if not os.path.exists(self.cookie_file): + log.warning(f"Cookie file not found: {self.cookie_file}") + return cookies + + try: + with open(self.cookie_file, "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + # Netscape format: domain flag path secure expiration name value + parts = line.split("\t") + if len(parts) >= 7: + name = parts[5] + value = parts[6] + cookies[name] = value + log.debug(f"Loaded {len(cookies)} cookies from {self.cookie_file}") + except Exception as e: + log.error(f"Error loading cookies: {e}") + + return cookies + + def _make_request(self, method: str, url: str, **kwargs) -> Optional[object]: + """Make HTTP request using curl_cffi if available, else requests.""" + try: + if HAS_CURL_CFFI: + # Use curl_cffi for better impersonation + session = curl_requests.Session() + # Set browser impersonation + session.impersonate = "chrome124" + response = session.request(method, url, **kwargs) + return response + else: + return requests.request(method, url, **kwargs) + except Exception as e: + log.error(f"Request failed: {e}") + return None + + +class PHHandler(MarkWatchedHandler): + """Handler for PornHub.""" + + DOMAINS = ["pornhub.com", "www.pornhub.com", "de.pornhub.com", "fr.pornhub.com", "es.pornhub.com", "it.pornhub.com", "rt.pornhub.com"] + + def can_handle(self, url): + parsed = urlparse(url) + domain = parsed.netloc.lower() + return any(d in domain for d in self.DOMAINS) + + async def mark_watched(self, url): + # Extract viewkey from URL + parsed = urlparse(url) + params = parse_qs(parsed.query) + viewkey = params.get("viewkey", [None])[0] + + if not viewkey: + # Try to extract from path + match = re.search(r"viewkey=([^&]+)", url) + if match: + viewkey = match.group(1) + + if not viewkey: + log.warning(f"Could not extract viewkey from URL: {url}") + return False + + log.info(f"Marking video as watched, viewkey: {viewkey}") + + # Load cookies + cookies = self._load_cookies() + if not cookies: + log.warning("No cookies available, cannot mark as watched") + return False + + # Try multiple endpoints as PH may use different URLs + # First try the AJAX endpoint which is most commonly used + endpoints = [ + # AJAX endpoint (most common) + ("POST", f"https://www.pornhub.com/user/watched/add/video/{viewkey}"), + # Alternative format + ("GET", f"https://www.pornhub.com/user/watched/add/video/{viewkey}"), + # Legacy endpoint + ("POST", f"https://www.pornhub.com/user/watched/video/viewkey/{viewkey}"), + ] + + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "Accept": "application/json, text/javascript, */*; q=0.01", + "Accept-Language": "en-US,en;q=0.9", + "X-Requested-With": "XMLHttpRequest", + "Referer": url, + } + + for method, api_url in endpoints: + log.debug(f"Trying {method} {api_url}") + response = self._make_request(method, api_url, headers=headers, cookies=cookies, allow_redirects=True) + + if response is None: + continue + + log.debug(f"Response status: {response.status_code}") + + if response.status_code == 200: + log.info(f"Successfully marked video as watched using {api_url}") + return True + elif response.status_code == 302: + # Redirect often means success on PH + log.info(f"Successfully marked video as watched (redirect from {api_url})") + return True + + # API methods failed, try headless browser as fallback + log.info("API methods failed, trying headless browser approach") + if HAS_PLAYWRIGHT: + return await headless_mark_watched(url, self.cookie_file, wait_seconds=5) + else: + log.warning("Playwright not available for headless browser fallback") + + return False + + +class YouTubeHandler(MarkWatchedHandler): + """Handler for YouTube - uses YouTube API or internal endpoints.""" + + DOMAINS = ["youtube.com", "www.youtube.com", "youtu.be", "m.youtube.com"] + + def can_handle(self, url): + parsed = urlparse(url) + domain = parsed.netloc.lower() + return any(d in domain for d in self.DOMAINS) + + async def mark_watched(self, url): + # YouTube marking as watched requires more complex handling + # typically done through the browse endpoint with protobuf + # This is a simplified implementation + log.info("YouTube mark-as-watched not yet fully implemented") + return False + + +# Registry of all handlers +HANDLERS = [ + PHHandler, + YouTubeHandler, +] + + +def get_handler(url: str, cookie_file: str) -> Optional[MarkWatchedHandler]: + """ + Get the appropriate handler for a URL. + + Args: + url: The video URL + cookie_file: Path to the Netscape cookie file + + Returns: + Handler instance if found, None otherwise + """ + if not cookie_file or not os.path.exists(cookie_file): + return None + + for handler_class in HANDLERS: + handler = handler_class(cookie_file) + if handler.can_handle(url): + return handler + + return None + + +async def mark_watched(url: str, cookie_file: str) -> bool: + """ + Mark a video as watched on its respective site. + + Args: + url: The video URL + cookie_file: Path to the Netscape cookie file + + Returns: + True if successfully marked as watched, False otherwise + """ + handler = get_handler(url, cookie_file) + if not handler: + log.debug(f"No mark-watched handler available for URL: {url}") + return False + + try: + return await handler.mark_watched(url) + except Exception as e: + log.error(f"Error marking video as watched: {e}") + return False diff --git a/app/ytdl.py b/app/ytdl.py index 6ee44bf..fb6d2f4 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -1,912 +1,940 @@ -import os -import yt_dlp -from collections import OrderedDict -import shelve -import time -import asyncio -import multiprocessing -import logging -import re -import random -import string -from urllib.parse import urlparse - -import yt_dlp.networking.impersonate -from dl_formats import get_format, get_opts, AUDIO_FORMATS -from datetime import datetime - -log = logging.getLogger('ytdl') - -class DownloadQueueNotifier: - async def added(self, dl): - raise NotImplementedError - - async def updated(self, dl): - raise NotImplementedError - - async def completed(self, dl): - raise NotImplementedError - - async def canceled(self, id): - raise NotImplementedError - - async def cleared(self, id): - raise NotImplementedError - - async def event(self, event): - raise NotImplementedError - -class DownloadInfo: - def __init__(self, id, title, url, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit): - self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}' - self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}' - self.url = url - self.quality = quality - self.format = format - self.folder = folder - self.custom_name_prefix = custom_name_prefix - self.msg = self.percent = self.speed = self.eta = None - self.status = "pending" - self.size = None - self.timestamp = time.time_ns() - self.error = error - self.entry = entry - self.playlist_item_limit = playlist_item_limit - # Extract website domain from URL - parsed_url = urlparse(url) - self.website = parsed_url.netloc - self.file_exists = None - -class Download: - manager = None - - def __init__(self, download_dir, temp_dir, output_template, output_template_chapter, quality, format, ytdl_opts, info): - self.download_dir = download_dir - self.temp_dir = temp_dir - self.output_template = output_template - self.output_template_chapter = output_template_chapter - self.format = get_format(format, quality) - self.ytdl_opts = get_opts(format, quality, ytdl_opts) - if "impersonate" in self.ytdl_opts: - self.ytdl_opts["impersonate"] = yt_dlp.networking.impersonate.ImpersonateTarget.from_str(self.ytdl_opts["impersonate"]) - self.info = info - self.canceled = False - self.tmpfilename = None - self.status_queue = None - self.proc = None - self.loop = None - self.notifier = None - self.had_download = False # Track if actual download occurred - - def _download(self): - log.info(f"Starting download for: {self.info.title} ({self.info.url})") - log.info(f"[TRACE] Download config: download_dir={self.download_dir}, temp_dir={self.temp_dir}") - log.info(f"[TRACE] Output template: {self.output_template}") - try: - def put_status(st): - # Log every status update to trace the flow - status_type = st.get('status', 'unknown') - if status_type == 'downloading': - # Mark that we're actually downloading (not skipping) - self.had_download = True - if 'tmpfilename' in st: - log.debug(f"[TRACE] Downloading - tmpfile: {st.get('tmpfilename')}") - elif status_type == 'finished': - log.info(f"[TRACE] put_status FINISHED - filename: {st.get('filename')}, tmpfilename: {st.get('tmpfilename')}") - log.info(f"[TRACE] had_download flag: {self.had_download}") - if st.get('filename'): - exists = os.path.exists(st['filename']) - log.info(f"[TRACE] File exists at reported location? {exists}") - if exists: - log.info(f"[TRACE] File size: {os.path.getsize(st['filename'])} bytes") - elif status_type == 'error': - log.error(f"[TRACE] put_status ERROR - msg: {st.get('msg')}") - - self.status_queue.put({k: v for k, v in st.items() if k in ( - 'tmpfilename', - 'filename', - 'status', - 'msg', - 'total_bytes', - 'total_bytes_estimate', - 'downloaded_bytes', - 'speed', - 'eta', - )}) - - def put_status_postprocessor(d): - log.info(f"[TRACE] ===== POSTPROCESSOR CALLED =====") - log.info(f"[TRACE] Postprocessor: {d.get('postprocessor')}, Status: {d.get('status')}") - - if d['postprocessor'] == 'MoveFiles' and d['status'] == 'finished': - log.info(f"[TRACE] MoveFiles postprocessor triggered") - log.info(f"[TRACE] had_download flag in postprocessor: {self.had_download}") - log.info(f"[TRACE] info_dict keys: {list(d['info_dict'].keys())}") - log.info(f"[TRACE] info_dict filepath: {d['info_dict'].get('filepath')}") - log.info(f"[TRACE] info_dict __finaldir: {d['info_dict'].get('__finaldir')}") - - if '__finaldir' in d['info_dict']: - filename = os.path.join(d['info_dict']['__finaldir'], os.path.basename(d['info_dict']['filepath'])) - else: - filename = d['info_dict']['filepath'] - - log.info(f"[TRACE] Resolved filename: {filename}") - log.info(f"[TRACE] File exists? {os.path.exists(filename)}") - - # List files in directory - dir_name = os.path.dirname(filename) - if os.path.isdir(dir_name): - all_files = os.listdir(dir_name) - log.info(f"[TRACE] Files in {dir_name}: {all_files}") - - # Check if file exists at expected location - if os.path.exists(filename): - log.info(f"[TRACE] File FOUND at expected location") - - # If yt-dlp didn't actually download (skipped), just report the existing file - if not self.had_download: - log.info(f"[TRACE] No actual download occurred - yt-dlp reused existing file") - log.info(f"[TRACE] Sending status with existing filename: {filename}") - self.status_queue.put({'status': 'finished', 'filename': filename}) - else: - # Actual download happened - check for conflicts - log.info(f"[TRACE] Actual download occurred - checking for conflicts") - base_name = os.path.basename(filename) - name, ext = os.path.splitext(base_name) - - # Look for other files with same base name (excluding current file) - other_files = [] - if os.path.isdir(dir_name): - for existing_file in os.listdir(dir_name): - if existing_file == base_name: - log.debug(f"[TRACE] Skipping current file: {existing_file}") - continue # Skip the current file - existing_name, existing_ext = os.path.splitext(existing_file) - # Check for exact name match - if existing_ext == ext and existing_name == name: - log.info(f"[TRACE] Found matching file: {existing_file}") - other_files.append(existing_file) - - log.info(f"[TRACE] Found {len(other_files)} other files with same base name: {other_files}") - - # If other files exist with same name, we have a duplicate - rename the NEW file - if len(other_files) > 0: - log.info(f"[TRACE] CONFLICT DETECTED! Other files: {other_files}") - unique_id = ''.join(random.choices(string.ascii_lowercase + string.digits, k=5)) - new_filename = f"{name}_{unique_id}{ext}" - new_filepath = os.path.join(dir_name, new_filename) - - log.info(f"[TRACE] Attempting rename: {filename} -> {new_filepath}") - try: - os.rename(filename, new_filepath) - log.warning(f"Filename conflict detected. Renamed: {base_name} β†’ {new_filename}") - log.info(f"[TRACE] Rename successful") - filename = new_filepath - except Exception as e: - log.error(f"[TRACE] Rename FAILED: {e}") - log.error(f"Failed to rename file due to conflict: {e}") - else: - log.info(f"[TRACE] No conflict - this is the only file with this name") - - log.info(f"[TRACE] Sending status with filename: {filename}") - self.status_queue.put({'status': 'finished', 'filename': filename}) - else: - log.info(f"[TRACE] File NOT FOUND at expected location") - base_name = os.path.basename(filename) - self.status_queue.put({'status': 'error', 'msg': f'File not found: {base_name}'}) - else: - log.debug(f"[TRACE] Other postprocessor: {d.get('postprocessor')}") - - ret = yt_dlp.YoutubeDL(params={ - 'quiet': True, - 'no_color': True, - 'paths': {"home": self.download_dir, "temp": self.temp_dir}, - 'outtmpl': { "default": self.output_template, "chapter": self.output_template_chapter }, - 'format': self.format, - 'socket_timeout': 30, - 'ignore_no_formats_error': True, - 'progress_hooks': [put_status], - 'postprocessor_hooks': [put_status_postprocessor], - **self.ytdl_opts, - }).download([self.info.url]) - self.status_queue.put({'status': 'finished' if ret == 0 else 'error'}) - log.info(f"Finished download for: {self.info.title}") - except yt_dlp.utils.YoutubeDLError as exc: - log.error(f"Download error for {self.info.title}: {str(exc)}") - self.status_queue.put({'status': 'error', 'msg': str(exc)}) - - async def start(self, notifier): - log.info(f"Preparing download for: {self.info.title}") - if Download.manager is None: - Download.manager = multiprocessing.Manager() - self.status_queue = Download.manager.Queue() - self.proc = multiprocessing.Process(target=self._download) - self.proc.start() - self.loop = asyncio.get_running_loop() - self.notifier = notifier - self.info.status = 'preparing' - await self.notifier.updated(self.info) - asyncio.create_task(self.update_status()) - return await self.loop.run_in_executor(None, self.proc.join) - - def _resolve_filename_conflict(self, filepath): - """ - Resolve filename conflicts by appending a short unique ID. - Returns the final non-conflicting filepath. - """ - dir_name = os.path.dirname(filepath) - base_name = os.path.basename(filepath) - name, ext = os.path.splitext(base_name) - - # Generate a short unique ID (5 alphanumeric characters) - unique_id = ''.join(random.choices(string.ascii_lowercase + string.digits, k=5)) - new_filename = f"{name}_{unique_id}{ext}" - new_filepath = os.path.join(dir_name, new_filename) - - return new_filepath - - def cancel(self): - log.info(f"Cancelling download: {self.info.title}") - if self.running(): - try: - self.proc.kill() - except Exception as e: - log.error(f"Error killing process for {self.info.title}: {e}") - self.canceled = True - if self.status_queue is not None: - self.status_queue.put(None) - - def close(self): - log.info(f"Closing download process for: {self.info.title}") - if self.started(): - self.proc.close() - if self.status_queue is not None: - self.status_queue.put(None) - - def running(self): - try: - return self.proc is not None and self.proc.is_alive() - except ValueError: - return False - - def started(self): - return self.proc is not None - - async def update_status(self): - while True: - status = await self.loop.run_in_executor(None, self.status_queue.get) - if status is None: - log.info(f"Status update finished for: {self.info.title}") - return - if self.canceled: - log.info(f"Download {self.info.title} is canceled; stopping status updates.") - return - self.tmpfilename = status.get('tmpfilename') - if 'filename' in status: - fileName = status.get('filename') - self.info.filename = os.path.relpath(fileName, self.download_dir) - self.info.size = os.path.getsize(fileName) if os.path.exists(fileName) else None - if self.info.format == 'thumbnail': - self.info.filename = re.sub(r'\.webm$', '.jpg', self.info.filename) - self.info.status = status['status'] - self.info.msg = status.get('msg') - if 'downloaded_bytes' in status: - total = status.get('total_bytes') or status.get('total_bytes_estimate') - if total: - self.info.percent = status['downloaded_bytes'] / total * 100 - self.info.speed = status.get('speed') - self.info.eta = status.get('eta') - log.info(f"Updating status for {self.info.title}: {status}") - await self.notifier.updated(self.info) - -class PersistentQueue: - def __init__(self, path): - pdir = os.path.dirname(path) - if not os.path.isdir(pdir): - os.mkdir(pdir) - with shelve.open(path, 'c'): - pass - self.path = path - self.dict = OrderedDict() - - def load(self): - for k, v in self.saved_items(): - # Ensure website field is populated for older downloads - if not hasattr(v, 'website') or v.website is None: - parsed_url = urlparse(v.url) - v.website = parsed_url.netloc - # Ensure file_exists field exists - if not hasattr(v, 'file_exists'): - v.file_exists = None - self.dict[k] = Download(None, None, None, None, None, None, {}, v) - - def exists(self, key): - return key in self.dict - - def get(self, key): - return self.dict[key] - - def items(self): - return self.dict.items() - - def saved_items(self): - with shelve.open(self.path, 'r') as shelf: - return sorted(shelf.items(), key=lambda item: item[1].timestamp) - - def put(self, value): - key = value.info.url - self.dict[key] = value - with shelve.open(self.path, 'w') as shelf: - shelf[key] = value.info - - def delete(self, key): - if key in self.dict: - del self.dict[key] - with shelve.open(self.path, 'w') as shelf: - shelf.pop(key, None) - - def next(self): - k, v = next(iter(self.dict.items())) - return k, v - - def empty(self): - return not bool(self.dict) - -class DownloadQueue: - def __init__(self, config, notifier): - self.config = config - self.notifier = notifier - self.queue = PersistentQueue(self.config.STATE_DIR + '/queue') - self.done = PersistentQueue(self.config.STATE_DIR + '/completed') - self.pending = PersistentQueue(self.config.STATE_DIR + '/pending') - self.active_downloads = set() - self.semaphore = None - # For sequential mode, use an asyncio lock to ensure one-at-a-time execution. - if self.config.DOWNLOAD_MODE == 'sequential': - self.seq_lock = asyncio.Lock() - elif self.config.DOWNLOAD_MODE == 'limited': - self.semaphore = asyncio.Semaphore(int(self.config.MAX_CONCURRENT_DOWNLOADS)) - - # PreCheck queue for sequential conflict detection (no locks needed) - self.precheck_queue = asyncio.Queue() - self.reserved_filenames = set() # Track filenames being processed - self.precheck_in_progress = {} # Track URL -> DownloadInfo for items in precheck queue - - # Event notifications (keep last 5 in memory) - self.events = [] # List of {type, message, timestamp, url} - self.max_events = 5 - - self.done.load() - - async def __import_queue(self): - for k, v in self.queue.saved_items(): - await self.__add_download(v, True) - - async def __import_pending(self): - for k, v in self.pending.saved_items(): - await self.__add_download(v, False) - - async def initialize(self): - log.info("Initializing DownloadQueue") - # Start the precheck worker for sequential conflict detection - asyncio.create_task(self.__precheck_worker()) - asyncio.create_task(self.__import_queue()) - asyncio.create_task(self.__import_pending()) - - async def __precheck_worker(self): - """Background worker that processes precheck queue sequentially. - Sequential processing naturally prevents race conditions without locks.""" - log.info("[PreCheck] Worker started") - while True: - try: - # Get next item from queue (blocks if empty) - item = await self.precheck_queue.get() - log.debug(f"[PreCheck] Processing item: {item['dl'].url}") - - # Process the precheck and start download - await self.__process_precheck(item) - - # Mark task as done - self.precheck_queue.task_done() - except Exception as e: - log.error(f"[PreCheck] Worker error: {e}", exc_info=True) - - async def __process_precheck(self, item): - """Process a single download with conflict detection. - Called sequentially by worker - no race conditions possible.""" - dl = item['dl'] - auto_start = item['auto_start'] - dldirectory = item['dldirectory'] - output = item['output'] - output_chapter = item['output_chapter'] - ytdl_options = item['ytdl_options'] - entry = item['entry'] - - log.info(f"[PreCheck] Checking for filename conflicts before download") - log.debug(f"[PreCheck] Original output template: {output}") - - # Try to predict the filename that yt-dlp will generate - if entry and 'title' in entry: - # Check if we have the real title or just a placeholder - title = entry.get('title', '') - video_id = entry.get('id', '') - - # If title looks like a placeholder (contains the ID), we need full extraction - needs_full_extraction = ( - not title or # No title - title == f"twitter video #{video_id}" or # Placeholder pattern - video_id in title # ID is in title (likely placeholder) - ) - - if needs_full_extraction: - log.debug(f"[PreCheck] Title appears to be placeholder: '{title}', doing full info extraction") - try: - # Do a full (non-flat) extraction to get real title - full_entry = await asyncio.get_running_loop().run_in_executor( - None, - lambda: yt_dlp.YoutubeDL(params={ - 'quiet': True, - 'no_color': True, - 'extract_flat': False, # Full extraction - 'skip_download': True, # Don't download, just get info - 'paths': {"home": dldirectory, "temp": self.config.TEMP_DIR}, - **ytdl_options, - }).extract_info(dl.url, download=False) - ) - if full_entry and 'title' in full_entry: - title = full_entry['title'] - log.debug(f"[PreCheck] Got real title from full extraction: '{title}'") - except Exception as e: - log.warning(f"[PreCheck] Failed to get full info: {e}, using placeholder title") - - predicted_filename = output - # Replace title - if '%(title)s' in predicted_filename: - predicted_filename = predicted_filename.replace('%(title)s', title) - - # Replace id - if '%(id)s' in predicted_filename and video_id: - predicted_filename = predicted_filename.replace('%(id)s', video_id) - - # Handle ext specially - default to format's extension if not in entry - if '%(ext)s' in predicted_filename: - ext = entry.get('ext', dl.format if dl.format in ['mp4', 'mkv', 'webm', 'mp3', 'm4a'] else 'mp4') - predicted_filename = predicted_filename.replace('%(ext)s', ext) - - predicted_filepath = os.path.join(dldirectory, predicted_filename) - log.info(f"[PreCheck] Predicted filepath: {predicted_filepath}") - - # Check if file already exists OR is reserved by another download in queue - # Sequential processing means we check one at a time - no race condition - if os.path.exists(predicted_filepath) or predicted_filepath in self.reserved_filenames: - if predicted_filepath in self.reserved_filenames: - log.warning(f"[PreCheck] Filename is reserved by pending download! Will append unique ID") - else: - log.warning(f"[PreCheck] File already exists! Will append unique ID to avoid conflict") - - # Generate unique ID - unique_id = ''.join(random.choices(string.ascii_lowercase + string.digits, k=5)) - - # Modify output template to include unique ID before extension - # Change "%(title)s.%(ext)s" to "%(title)s_XXXXX.%(ext)s" - if '.%(ext)s' in output: - output = output.replace('.%(ext)s', f'_{unique_id}.%(ext)s') - else: - # Fallback: append to end - output = f"{output}_{unique_id}" - - # Re-predict the new filename - predicted_filename = output - if '%(title)s' in predicted_filename: - predicted_filename = predicted_filename.replace('%(title)s', title) - if '%(id)s' in predicted_filename and video_id: - predicted_filename = predicted_filename.replace('%(id)s', video_id) - if '%(ext)s' in predicted_filename: - ext = entry.get('ext', dl.format if dl.format in ['mp4', 'mkv', 'webm', 'mp3', 'm4a'] else 'mp4') - predicted_filename = predicted_filename.replace('%(ext)s', ext) - predicted_filepath = os.path.join(dldirectory, predicted_filename) - - log.info(f"[PreCheck] Modified output template: {output}") - log.info(f"[PreCheck] New predicted filepath: {predicted_filepath}") - else: - log.info(f"[PreCheck] No conflict detected, using original template") - - # Reserve this filename to prevent concurrent downloads from using it - self.reserved_filenames.add(predicted_filepath) - log.debug(f"[PreCheck] Reserved filename: {predicted_filepath}") - else: - predicted_filepath = None - log.debug(f"[PreCheck] No entry data available, skipping pre-check") - - log.debug(f"final resolved output template: {output}") - download = Download(dldirectory, self.config.TEMP_DIR, output, output_chapter, dl.quality, dl.format, ytdl_options, dl) - - # Store the reserved filepath for cleanup - download.reserved_filepath = predicted_filepath - - # Remove from in-progress set before adding to queue - # This allows checking queue.exists() to work properly - if dl.url in self.precheck_in_progress: - del self.precheck_in_progress[dl.url] - log.debug(f"[PreCheck] Removed from in-progress tracking: {dl.url}") - - if auto_start is True: - self.queue.put(download) - asyncio.create_task(self.__start_download(download)) - else: - self.pending.put(download) - - async def __start_download(self, download): - if download.canceled: - log.info(f"Download {download.info.title} was canceled, skipping start.") - return - if self.config.DOWNLOAD_MODE == 'sequential': - async with self.seq_lock: - log.info("Starting sequential download.") - await download.start(self.notifier) - self._post_download_cleanup(download) - elif self.config.DOWNLOAD_MODE == 'limited' and self.semaphore is not None: - await self.__limited_concurrent_download(download) - else: - await self.__concurrent_download(download) - - async def __concurrent_download(self, download): - log.info("Starting concurrent download without limits.") - asyncio.create_task(self._run_download(download)) - - async def __limited_concurrent_download(self, download): - log.info("Starting limited concurrent download.") - async with self.semaphore: - await self._run_download(download) - - async def _run_download(self, download): - if download.canceled: - log.info(f"Download {download.info.title} is canceled; skipping start.") - return - await download.start(self.notifier) - self._post_download_cleanup(download) - - def _post_download_cleanup(self, download): - # Release filename reservation if it exists - if hasattr(download, 'reserved_filepath') and download.reserved_filepath: - if download.reserved_filepath in self.reserved_filenames: - self.reserved_filenames.discard(download.reserved_filepath) - log.debug(f"[PreCheck] Released reservation for: {download.reserved_filepath}") - - if download.info.status != 'finished': - if download.tmpfilename and os.path.isfile(download.tmpfilename): - try: - os.remove(download.tmpfilename) - except: - pass - download.info.status = 'error' - download.close() - if self.queue.exists(download.info.url): - self.queue.delete(download.info.url) - if download.canceled: - asyncio.create_task(self.notifier.canceled(download.info.url)) - else: - self.done.put(download) - asyncio.create_task(self.notifier.completed(download.info)) - - def __extract_info(self, url, playlist_strict_mode): - return yt_dlp.YoutubeDL(params={ - 'quiet': True, - 'no_color': True, - 'extract_flat': True, - 'ignore_no_formats_error': True, - 'noplaylist': playlist_strict_mode, - 'paths': {"home": self.config.DOWNLOAD_DIR, "temp": self.config.TEMP_DIR}, - **self.config.YTDL_OPTIONS, - **({'impersonate': yt_dlp.networking.impersonate.ImpersonateTarget.from_str(self.config.YTDL_OPTIONS['impersonate'])} if 'impersonate' in self.config.YTDL_OPTIONS else {}), - }).extract_info(url, download=False) - - def __calc_download_path(self, quality, format, folder): - base_directory = self.config.DOWNLOAD_DIR if (quality != 'audio' and format not in AUDIO_FORMATS) else self.config.AUDIO_DOWNLOAD_DIR - if folder: - if not self.config.CUSTOM_DIRS: - return None, {'status': 'error', 'msg': f'A folder for the download was specified but CUSTOM_DIRS is not true in the configuration.'} - dldirectory = os.path.realpath(os.path.join(base_directory, folder)) - real_base_directory = os.path.realpath(base_directory) - if not dldirectory.startswith(real_base_directory): - return None, {'status': 'error', 'msg': f'Folder "{folder}" must resolve inside the base download directory "{real_base_directory}"'} - if not os.path.isdir(dldirectory): - if not self.config.CREATE_CUSTOM_DIRS: - return None, {'status': 'error', 'msg': f'Folder "{folder}" for download does not exist inside base directory "{real_base_directory}", and CREATE_CUSTOM_DIRS is not true in the configuration.'} - os.makedirs(dldirectory, exist_ok=True) - else: - dldirectory = base_directory - return dldirectory, None - - async def __add_download(self, dl, auto_start): - """Fast path: validate and queue for precheck processing. - Returns immediately without blocking on slow operations.""" - # Check if this exact URL is already being processed, in queue, or already downloaded - # This prevents duplicate downloads when same URL is submitted multiple times - if (dl.url in self.precheck_in_progress or - self.queue.exists(dl.url) or - self.pending.exists(dl.url) or - self.done.exists(dl.url)): - log.info(f"[PreCheck] URL already queued/processing/downloaded, skipping: {dl.url}") - # Add event notification - self._add_event('duplicate_skipped', 'URL already in queue or downloaded', dl.url) - return {'status': 'ok', 'msg': 'Download already exists'} - - dldirectory, error_message = self.__calc_download_path(dl.quality, dl.format, dl.folder) - if error_message is not None: - return error_message - - output = self.config.OUTPUT_TEMPLATE if len(dl.custom_name_prefix) == 0 else f'{dl.custom_name_prefix}.{self.config.OUTPUT_TEMPLATE}' - output_chapter = self.config.OUTPUT_TEMPLATE_CHAPTER - entry = getattr(dl, 'entry', None) - - if entry is not None and 'playlist' in entry and entry['playlist'] is not None: - if len(self.config.OUTPUT_TEMPLATE_PLAYLIST): - output = self.config.OUTPUT_TEMPLATE_PLAYLIST - for property, value in entry.items(): - if property.startswith("playlist"): - output = output.replace(f"%({property})s", str(value)) - - ytdl_options = dict(self.config.YTDL_OPTIONS) - playlist_item_limit = getattr(dl, 'playlist_item_limit', 0) - if playlist_item_limit > 0: - log.info(f'playlist limit is set. Processing only first {playlist_item_limit} entries') - ytdl_options['playlistend'] = playlist_item_limit - - # Check if cookie file exists for this domain - parsed_url = urlparse(dl.url) - domain = parsed_url.netloc - log.info(f"[Cookie] Checking for cookie file for domain: {domain}") - - cookies_dir = os.path.join(self.config.STATE_DIR, 'cookies') - log.debug(f"[Cookie] Cookies directory: {cookies_dir}") - - # Try domain-specific cookie file - safe_domain = domain.replace(':', '_').replace('/', '_') - cookie_file = os.path.join(cookies_dir, f'{safe_domain}.txt') - - log.debug(f"[Cookie] Looking for cookie file at: {cookie_file}") - - if os.path.exists(cookie_file): - log.info(f"[Cookie] Found cookie file: {cookie_file}") - # Verify file is readable and has content - try: - with open(cookie_file, 'r') as f: - lines = f.readlines() - cookie_lines = [l for l in lines if l.strip() and not l.startswith('#')] - log.info(f"[Cookie] Cookie file contains {len(cookie_lines)} cookie entries") - if len(cookie_lines) == 0: - log.warning(f"[Cookie] Cookie file exists but contains no cookies!") - else: - log.debug(f"[Cookie] First cookie entry: {cookie_lines[0][:50]}...") - except Exception as e: - log.error(f"[Cookie] Error reading cookie file: {e}", exc_info=True) - - ytdl_options['cookiefile'] = cookie_file - log.info(f"[Cookie] Configured yt-dlp to use cookiefile: {cookie_file}") - else: - log.info(f"[Cookie] No cookie file found for domain {domain}") - log.debug(f"[Cookie] Checked path: {cookie_file}") - # List available cookie files for debugging - if os.path.exists(cookies_dir): - available_cookies = os.listdir(cookies_dir) - if available_cookies: - log.debug(f"[Cookie] Available cookie files: {available_cookies}") - else: - log.debug(f"[Cookie] Cookies directory is empty") - else: - log.debug(f"[Cookie] Cookies directory does not exist") - - # Mark URL as being processed to prevent duplicates - # Store the DownloadInfo so we can display it in UI - self.precheck_in_progress[dl.url] = dl - - # Queue for sequential precheck processing (fast, non-blocking) - await self.precheck_queue.put({ - 'dl': dl, - 'auto_start': auto_start, - 'dldirectory': dldirectory, - 'output': output, - 'output_chapter': output_chapter, - 'ytdl_options': ytdl_options, - 'entry': entry, - }) - log.debug(f"[PreCheck] Queued for processing: {dl.url}") - - # Notify immediately (fast response to user) - await self.notifier.added(dl) - - async def __add_entry(self, entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already): - if not entry: - return {'status': 'error', 'msg': "Invalid/empty data was given."} - - error = None - if "live_status" in entry and "release_timestamp" in entry and entry.get("live_status") == "is_upcoming": - dt_ts = datetime.fromtimestamp(entry.get("release_timestamp")).strftime('%Y-%m-%d %H:%M:%S %z') - error = f"Live stream is scheduled to start at {dt_ts}" - else: - if "msg" in entry: - error = entry["msg"] - - etype = entry.get('_type') or 'video' - - if etype.startswith('url'): - log.debug('Processing as an url') - return await self.add(entry['url'], quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already) - elif etype == 'playlist': - log.debug('Processing as a playlist') - entries = entry['entries'] - log.info(f'playlist detected with {len(entries)} entries') - - playlist_index_digits = len(str(len(entries))) - results = [] - if playlist_item_limit > 0: - log.info(f'Playlist item limit is set. Processing only first {playlist_item_limit} entries') - entries = entries[:playlist_item_limit] - - # Verify playlist entry has 'id' before using it - playlist_id = entry.get("id", "unknown_playlist") - if "id" not in entry: - log.warning(f"Playlist entry missing 'id' field. Using fallback 'unknown_playlist'. Entry keys: {list(entry.keys())}") - - for index, etr in enumerate(entries, start=1): - etr["_type"] = "video" - etr["playlist"] = playlist_id - etr["playlist_index"] = '{{0:0{0:d}d}}'.format(playlist_index_digits).format(index) - for property in ("id", "title", "uploader", "uploader_id"): - if property in entry: - etr[f"playlist_{property}"] = entry[property] - results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already)) - if any(res['status'] == 'error' for res in results): - return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)} - return {'status': 'ok'} - elif etype == 'video' or (etype.startswith('url') and 'id' in entry and 'title' in entry): - log.debug('Processing as a video') - - # Extract ID from entry, or derive from URL if missing - video_id = entry.get('id') - if not video_id: - # Try to extract ID from URL (e.g., viewkey parameter or URL path) - video_url = entry.get('url', '') - if 'viewkey=' in video_url: - # Extract viewkey parameter (common in PornHub, etc.) - match = re.search(r'viewkey=([^&]+)', video_url) - if match: - video_id = match.group(1) - log.info(f"Extracted video ID from viewkey: {video_id}") - elif 'webpage_url' in entry: - # Use webpage_url as fallback - video_id = entry['webpage_url'] - else: - # Last resort: use the URL itself - video_id = video_url - - if not video_id: - log.error(f"Video entry missing 'id' field and could not extract from URL. Entry keys: {list(entry.keys())}") - return {'status': 'error', 'msg': "Video entry missing required 'id' field and URL extraction failed"} - - key = entry.get('webpage_url') or entry['url'] - if not self.queue.exists(key): - dl = DownloadInfo(video_id, entry.get('title') or video_id, key, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit) - await self.__add_download(dl, auto_start) - return {'status': 'ok'} - return {'status': 'error', 'msg': f'Unsupported resource "{etype}"'} - - async def add(self, url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start=True, already=None): - log.info(f'adding {url}: {quality=} {format=} {already=} {folder=} {custom_name_prefix=} {playlist_strict_mode=} {playlist_item_limit=} {auto_start=}') - already = set() if already is None else already - if url in already: - log.info('recursion detected, skipping') - return {'status': 'ok'} - else: - already.add(url) - try: - entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url, playlist_strict_mode) - except yt_dlp.utils.YoutubeDLError as exc: - return {'status': 'error', 'msg': str(exc)} - return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already) - - async def start_pending(self, ids): - for id in ids: - if not self.pending.exists(id): - log.warn(f'requested start for non-existent download {id}') - continue - dl = self.pending.get(id) - self.queue.put(dl) - self.pending.delete(id) - asyncio.create_task(self.__start_download(dl)) - return {'status': 'ok'} - - async def cancel(self, ids): - for id in ids: - if self.pending.exists(id): - self.pending.delete(id) - await self.notifier.canceled(id) - continue - if not self.queue.exists(id): - log.warn(f'requested cancel for non-existent download {id}') - continue - if self.queue.get(id).started(): - self.queue.get(id).cancel() - else: - self.queue.delete(id) - await self.notifier.canceled(id) - return {'status': 'ok'} - - async def clear(self, ids): - for id in ids: - if not self.done.exists(id): - log.warn(f'requested delete for non-existent download {id}') - continue - if self.config.DELETE_FILE_ON_TRASHCAN: - dl = self.done.get(id) - try: - dldirectory, _ = self.__calc_download_path(dl.info.quality, dl.info.format, dl.info.folder) - os.remove(os.path.join(dldirectory, dl.info.filename)) - except Exception as e: - log.warn(f'deleting file for download {id} failed with error message {e!r}') - self.done.delete(id) - await self.notifier.cleared(id) - return {'status': 'ok'} - - def get(self): - # Ensure website field is populated for all downloads - for k, v in self.queue.items(): - if not hasattr(v.info, 'website') or v.info.website is None: - parsed_url = urlparse(v.info.url) - v.info.website = parsed_url.netloc - - for k, v in self.pending.items(): - if not hasattr(v.info, 'website') or v.info.website is None: - parsed_url = urlparse(v.info.url) - v.info.website = parsed_url.netloc - - # Update file existence status for done downloads - for k, v in self.done.items(): - if not hasattr(v.info, 'website') or v.info.website is None: - parsed_url = urlparse(v.info.url) - v.info.website = parsed_url.netloc - - # Use getattr with default to safely check for filename attribute - filename = getattr(v.info, 'filename', None) - if filename: - dldirectory, _ = self.__calc_download_path(v.info.quality, v.info.format, v.info.folder) - if dldirectory: - filepath = os.path.join(dldirectory, filename) - v.info.file_exists = os.path.exists(filepath) - else: - v.info.file_exists = False - else: - v.info.file_exists = False - - # Create list from items in precheck queue - # These items have 'preparing' status to indicate they're being analyzed - precheck_list = [(dl.url, dl) for dl in self.precheck_in_progress.values()] - - return (precheck_list + - list((k, v.info) for k, v in self.queue.items()) + - list((k, v.info) for k, v in self.pending.items()), - list((k, v.info) for k, v in self.done.items())) - - def _add_event(self, event_type, message, url=None): - """Add an event to the events list (keep only last 5).""" - event = { - 'type': event_type, - 'message': message, - 'timestamp': int(time.time()), - 'url': url - } - self.events.append(event) - # Keep only last 5 events - if len(self.events) > self.max_events: - self.events = self.events[-self.max_events:] - # Notify frontend via WebSocket - asyncio.create_task(self.notifier.event(event)) - - def get_events(self): - """Get all events (last 5).""" - return self.events - - def clear_events(self): - """Clear all events.""" - self.events = [] +import os +import yt_dlp +from collections import OrderedDict +from mark_watched import mark_watched +import shelve +import time +import asyncio +import multiprocessing +import logging +import re +import random +import string +from urllib.parse import urlparse + +import yt_dlp.networking.impersonate +from dl_formats import get_format, get_opts, AUDIO_FORMATS +from datetime import datetime + +log = logging.getLogger('ytdl') + +class DownloadQueueNotifier: + async def added(self, dl): + raise NotImplementedError + + async def updated(self, dl): + raise NotImplementedError + + async def completed(self, dl): + raise NotImplementedError + + async def canceled(self, id): + raise NotImplementedError + + async def cleared(self, id): + raise NotImplementedError + + async def event(self, event): + raise NotImplementedError + +class DownloadInfo: + def __init__(self, id, title, url, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit): + self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}' + self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}' + self.url = url + self.quality = quality + self.format = format + self.folder = folder + self.custom_name_prefix = custom_name_prefix + self.msg = self.percent = self.speed = self.eta = None + self.status = "pending" + self.size = None + self.timestamp = time.time_ns() + self.error = error + self.entry = entry + self.playlist_item_limit = playlist_item_limit + # Extract website domain from URL + parsed_url = urlparse(url) + self.website = parsed_url.netloc + self.file_exists = None + +class Download: + manager = None + + def __init__(self, download_dir, temp_dir, output_template, output_template_chapter, quality, format, ytdl_opts, info): + self.download_dir = download_dir + self.temp_dir = temp_dir + self.output_template = output_template + self.output_template_chapter = output_template_chapter + self.format = get_format(format, quality) + self.ytdl_opts = get_opts(format, quality, ytdl_opts) + if "impersonate" in self.ytdl_opts: + self.ytdl_opts["impersonate"] = yt_dlp.networking.impersonate.ImpersonateTarget.from_str(self.ytdl_opts["impersonate"]) + self.info = info + self.canceled = False + self.tmpfilename = None + self.status_queue = None + self.proc = None + self.loop = None + self.notifier = None + self.had_download = False # Track if actual download occurred + + def _download(self): + log.info(f"Starting download for: {self.info.title} ({self.info.url})") + log.info(f"[TRACE] Download config: download_dir={self.download_dir}, temp_dir={self.temp_dir}") + log.info(f"[TRACE] Output template: {self.output_template}") + try: + def put_status(st): + # Log every status update to trace the flow + status_type = st.get('status', 'unknown') + if status_type == 'downloading': + # Mark that we're actually downloading (not skipping) + self.had_download = True + if 'tmpfilename' in st: + log.debug(f"[TRACE] Downloading - tmpfile: {st.get('tmpfilename')}") + elif status_type == 'finished': + log.info(f"[TRACE] put_status FINISHED - filename: {st.get('filename')}, tmpfilename: {st.get('tmpfilename')}") + log.info(f"[TRACE] had_download flag: {self.had_download}") + if st.get('filename'): + exists = os.path.exists(st['filename']) + log.info(f"[TRACE] File exists at reported location? {exists}") + if exists: + log.info(f"[TRACE] File size: {os.path.getsize(st['filename'])} bytes") + elif status_type == 'error': + log.error(f"[TRACE] put_status ERROR - msg: {st.get('msg')}") + + self.status_queue.put({k: v for k, v in st.items() if k in ( + 'tmpfilename', + 'filename', + 'status', + 'msg', + 'total_bytes', + 'total_bytes_estimate', + 'downloaded_bytes', + 'speed', + 'eta', + )}) + + def put_status_postprocessor(d): + log.info(f"[TRACE] ===== POSTPROCESSOR CALLED =====") + log.info(f"[TRACE] Postprocessor: {d.get('postprocessor')}, Status: {d.get('status')}") + + if d['postprocessor'] == 'MoveFiles' and d['status'] == 'finished': + log.info(f"[TRACE] MoveFiles postprocessor triggered") + log.info(f"[TRACE] had_download flag in postprocessor: {self.had_download}") + log.info(f"[TRACE] info_dict keys: {list(d['info_dict'].keys())}") + log.info(f"[TRACE] info_dict filepath: {d['info_dict'].get('filepath')}") + log.info(f"[TRACE] info_dict __finaldir: {d['info_dict'].get('__finaldir')}") + + if '__finaldir' in d['info_dict']: + filename = os.path.join(d['info_dict']['__finaldir'], os.path.basename(d['info_dict']['filepath'])) + else: + filename = d['info_dict']['filepath'] + + log.info(f"[TRACE] Resolved filename: {filename}") + log.info(f"[TRACE] File exists? {os.path.exists(filename)}") + + # List files in directory + dir_name = os.path.dirname(filename) + if os.path.isdir(dir_name): + all_files = os.listdir(dir_name) + log.info(f"[TRACE] Files in {dir_name}: {all_files}") + + # Check if file exists at expected location + if os.path.exists(filename): + log.info(f"[TRACE] File FOUND at expected location") + + # If yt-dlp didn't actually download (skipped), just report the existing file + if not self.had_download: + log.info(f"[TRACE] No actual download occurred - yt-dlp reused existing file") + log.info(f"[TRACE] Sending status with existing filename: {filename}") + self.status_queue.put({'status': 'finished', 'filename': filename}) + else: + # Actual download happened - check for conflicts + log.info(f"[TRACE] Actual download occurred - checking for conflicts") + base_name = os.path.basename(filename) + name, ext = os.path.splitext(base_name) + + # Look for other files with same base name (excluding current file) + other_files = [] + if os.path.isdir(dir_name): + for existing_file in os.listdir(dir_name): + if existing_file == base_name: + log.debug(f"[TRACE] Skipping current file: {existing_file}") + continue # Skip the current file + existing_name, existing_ext = os.path.splitext(existing_file) + # Check for exact name match + if existing_ext == ext and existing_name == name: + log.info(f"[TRACE] Found matching file: {existing_file}") + other_files.append(existing_file) + + log.info(f"[TRACE] Found {len(other_files)} other files with same base name: {other_files}") + + # If other files exist with same name, we have a duplicate - rename the NEW file + if len(other_files) > 0: + log.info(f"[TRACE] CONFLICT DETECTED! Other files: {other_files}") + unique_id = ''.join(random.choices(string.ascii_lowercase + string.digits, k=5)) + new_filename = f"{name}_{unique_id}{ext}" + new_filepath = os.path.join(dir_name, new_filename) + + log.info(f"[TRACE] Attempting rename: {filename} -> {new_filepath}") + try: + os.rename(filename, new_filepath) + log.warning(f"Filename conflict detected. Renamed: {base_name} β†’ {new_filename}") + log.info(f"[TRACE] Rename successful") + filename = new_filepath + except Exception as e: + log.error(f"[TRACE] Rename FAILED: {e}") + log.error(f"Failed to rename file due to conflict: {e}") + else: + log.info(f"[TRACE] No conflict - this is the only file with this name") + + log.info(f"[TRACE] Sending status with filename: {filename}") + self.status_queue.put({'status': 'finished', 'filename': filename}) + else: + log.info(f"[TRACE] File NOT FOUND at expected location") + base_name = os.path.basename(filename) + self.status_queue.put({'status': 'error', 'msg': f'File not found: {base_name}'}) + else: + log.debug(f"[TRACE] Other postprocessor: {d.get('postprocessor')}") + + ret = yt_dlp.YoutubeDL(params={ + 'quiet': True, + 'no_color': True, + 'paths': {"home": self.download_dir, "temp": self.temp_dir}, + 'outtmpl': { "default": self.output_template, "chapter": self.output_template_chapter }, + 'format': self.format, + 'socket_timeout': 30, + 'ignore_no_formats_error': True, + 'progress_hooks': [put_status], + 'postprocessor_hooks': [put_status_postprocessor], + **self.ytdl_opts, + }).download([self.info.url]) + self.status_queue.put({'status': 'finished' if ret == 0 else 'error'}) + log.info(f"Finished download for: {self.info.title}") + except yt_dlp.utils.YoutubeDLError as exc: + log.error(f"Download error for {self.info.title}: {str(exc)}") + self.status_queue.put({'status': 'error', 'msg': str(exc)}) + + async def start(self, notifier): + log.info(f"Preparing download for: {self.info.title}") + if Download.manager is None: + Download.manager = multiprocessing.Manager() + self.status_queue = Download.manager.Queue() + self.proc = multiprocessing.Process(target=self._download) + self.proc.start() + self.loop = asyncio.get_running_loop() + self.notifier = notifier + self.info.status = 'preparing' + await self.notifier.updated(self.info) + asyncio.create_task(self.update_status()) + return await self.loop.run_in_executor(None, self.proc.join) + + def _resolve_filename_conflict(self, filepath): + """ + Resolve filename conflicts by appending a short unique ID. + Returns the final non-conflicting filepath. + """ + dir_name = os.path.dirname(filepath) + base_name = os.path.basename(filepath) + name, ext = os.path.splitext(base_name) + + # Generate a short unique ID (5 alphanumeric characters) + unique_id = ''.join(random.choices(string.ascii_lowercase + string.digits, k=5)) + new_filename = f"{name}_{unique_id}{ext}" + new_filepath = os.path.join(dir_name, new_filename) + + return new_filepath + + def cancel(self): + log.info(f"Cancelling download: {self.info.title}") + if self.running(): + try: + self.proc.kill() + except Exception as e: + log.error(f"Error killing process for {self.info.title}: {e}") + self.canceled = True + if self.status_queue is not None: + self.status_queue.put(None) + + def close(self): + log.info(f"Closing download process for: {self.info.title}") + if self.started(): + self.proc.close() + if self.status_queue is not None: + self.status_queue.put(None) + + def running(self): + try: + return self.proc is not None and self.proc.is_alive() + except ValueError: + return False + + def started(self): + return self.proc is not None + + async def update_status(self): + while True: + status = await self.loop.run_in_executor(None, self.status_queue.get) + if status is None: + log.info(f"Status update finished for: {self.info.title}") + return + if self.canceled: + log.info(f"Download {self.info.title} is canceled; stopping status updates.") + return + self.tmpfilename = status.get('tmpfilename') + if 'filename' in status: + fileName = status.get('filename') + self.info.filename = os.path.relpath(fileName, self.download_dir) + self.info.size = os.path.getsize(fileName) if os.path.exists(fileName) else None + if self.info.format == 'thumbnail': + self.info.filename = re.sub(r'\.webm$', '.jpg', self.info.filename) + self.info.status = status['status'] + self.info.msg = status.get('msg') + if 'downloaded_bytes' in status: + total = status.get('total_bytes') or status.get('total_bytes_estimate') + if total: + self.info.percent = status['downloaded_bytes'] / total * 100 + self.info.speed = status.get('speed') + self.info.eta = status.get('eta') + log.info(f"Updating status for {self.info.title}: {status}") + await self.notifier.updated(self.info) + +class PersistentQueue: + def __init__(self, path): + pdir = os.path.dirname(path) + if not os.path.isdir(pdir): + os.mkdir(pdir) + with shelve.open(path, 'c'): + pass + self.path = path + self.dict = OrderedDict() + + def load(self): + for k, v in self.saved_items(): + # Ensure website field is populated for older downloads + if not hasattr(v, 'website') or v.website is None: + parsed_url = urlparse(v.url) + v.website = parsed_url.netloc + # Ensure file_exists field exists + if not hasattr(v, 'file_exists'): + v.file_exists = None + self.dict[k] = Download(None, None, None, None, None, None, {}, v) + + def exists(self, key): + return key in self.dict + + def get(self, key): + return self.dict[key] + + def items(self): + return self.dict.items() + + def saved_items(self): + with shelve.open(self.path, 'r') as shelf: + return sorted(shelf.items(), key=lambda item: item[1].timestamp) + + def put(self, value): + key = value.info.url + self.dict[key] = value + with shelve.open(self.path, 'w') as shelf: + shelf[key] = value.info + + def delete(self, key): + if key in self.dict: + del self.dict[key] + with shelve.open(self.path, 'w') as shelf: + shelf.pop(key, None) + + def next(self): + k, v = next(iter(self.dict.items())) + return k, v + + def empty(self): + return not bool(self.dict) + +class DownloadQueue: + def __init__(self, config, notifier): + self.config = config + self.notifier = notifier + self.queue = PersistentQueue(self.config.STATE_DIR + '/queue') + self.done = PersistentQueue(self.config.STATE_DIR + '/completed') + self.pending = PersistentQueue(self.config.STATE_DIR + '/pending') + self.active_downloads = set() + self.semaphore = None + # For sequential mode, use an asyncio lock to ensure one-at-a-time execution. + if self.config.DOWNLOAD_MODE == 'sequential': + self.seq_lock = asyncio.Lock() + elif self.config.DOWNLOAD_MODE == 'limited': + self.semaphore = asyncio.Semaphore(int(self.config.MAX_CONCURRENT_DOWNLOADS)) + + # PreCheck queue for sequential conflict detection (no locks needed) + self.precheck_queue = asyncio.Queue() + self.reserved_filenames = set() # Track filenames being processed + self.precheck_in_progress = {} # Track URL -> DownloadInfo for items in precheck queue + + # Event notifications (keep last 5 in memory) + self.events = [] # List of {type, message, timestamp, url} + self.max_events = 5 + + self.done.load() + + async def __import_queue(self): + for k, v in self.queue.saved_items(): + await self.__add_download(v, True) + + async def __import_pending(self): + for k, v in self.pending.saved_items(): + await self.__add_download(v, False) + + async def initialize(self): + log.info("Initializing DownloadQueue") + # Start the precheck worker for sequential conflict detection + asyncio.create_task(self.__precheck_worker()) + asyncio.create_task(self.__import_queue()) + asyncio.create_task(self.__import_pending()) + + async def __precheck_worker(self): + """Background worker that processes precheck queue sequentially. + Sequential processing naturally prevents race conditions without locks.""" + log.info("[PreCheck] Worker started") + while True: + try: + # Get next item from queue (blocks if empty) + item = await self.precheck_queue.get() + log.debug(f"[PreCheck] Processing item: {item['dl'].url}") + + # Process the precheck and start download + await self.__process_precheck(item) + + # Mark task as done + self.precheck_queue.task_done() + except Exception as e: + log.error(f"[PreCheck] Worker error: {e}", exc_info=True) + + async def __process_precheck(self, item): + """Process a single download with conflict detection. + Called sequentially by worker - no race conditions possible.""" + dl = item['dl'] + auto_start = item['auto_start'] + dldirectory = item['dldirectory'] + output = item['output'] + output_chapter = item['output_chapter'] + ytdl_options = item['ytdl_options'] + entry = item['entry'] + + log.info(f"[PreCheck] Checking for filename conflicts before download") + log.debug(f"[PreCheck] Original output template: {output}") + + # Try to predict the filename that yt-dlp will generate + if entry and 'title' in entry: + # Check if we have the real title or just a placeholder + title = entry.get('title', '') + video_id = entry.get('id', '') + + # If title looks like a placeholder (contains the ID), we need full extraction + needs_full_extraction = ( + not title or # No title + title == f"twitter video #{video_id}" or # Placeholder pattern + video_id in title # ID is in title (likely placeholder) + ) + + if needs_full_extraction: + log.debug(f"[PreCheck] Title appears to be placeholder: '{title}', doing full info extraction") + try: + # Do a full (non-flat) extraction to get real title + full_entry = await asyncio.get_running_loop().run_in_executor( + None, + lambda: yt_dlp.YoutubeDL(params={ + 'quiet': True, + 'no_color': True, + 'extract_flat': False, # Full extraction + 'skip_download': True, # Don't download, just get info + 'paths': {"home": dldirectory, "temp": self.config.TEMP_DIR}, + **ytdl_options, + }).extract_info(dl.url, download=False) + ) + if full_entry and 'title' in full_entry: + title = full_entry['title'] + log.debug(f"[PreCheck] Got real title from full extraction: '{title}'") + except Exception as e: + log.warning(f"[PreCheck] Failed to get full info: {e}, using placeholder title") + + predicted_filename = output + # Replace title + if '%(title)s' in predicted_filename: + predicted_filename = predicted_filename.replace('%(title)s', title) + + # Replace id + if '%(id)s' in predicted_filename and video_id: + predicted_filename = predicted_filename.replace('%(id)s', video_id) + + # Handle ext specially - default to format's extension if not in entry + if '%(ext)s' in predicted_filename: + ext = entry.get('ext', dl.format if dl.format in ['mp4', 'mkv', 'webm', 'mp3', 'm4a'] else 'mp4') + predicted_filename = predicted_filename.replace('%(ext)s', ext) + + predicted_filepath = os.path.join(dldirectory, predicted_filename) + log.info(f"[PreCheck] Predicted filepath: {predicted_filepath}") + + # Check if file already exists OR is reserved by another download in queue + # Sequential processing means we check one at a time - no race condition + if os.path.exists(predicted_filepath) or predicted_filepath in self.reserved_filenames: + if predicted_filepath in self.reserved_filenames: + log.warning(f"[PreCheck] Filename is reserved by pending download! Will append unique ID") + else: + log.warning(f"[PreCheck] File already exists! Will append unique ID to avoid conflict") + + # Generate unique ID + unique_id = ''.join(random.choices(string.ascii_lowercase + string.digits, k=5)) + + # Modify output template to include unique ID before extension + # Change "%(title)s.%(ext)s" to "%(title)s_XXXXX.%(ext)s" + if '.%(ext)s' in output: + output = output.replace('.%(ext)s', f'_{unique_id}.%(ext)s') + else: + # Fallback: append to end + output = f"{output}_{unique_id}" + + # Re-predict the new filename + predicted_filename = output + if '%(title)s' in predicted_filename: + predicted_filename = predicted_filename.replace('%(title)s', title) + if '%(id)s' in predicted_filename and video_id: + predicted_filename = predicted_filename.replace('%(id)s', video_id) + if '%(ext)s' in predicted_filename: + ext = entry.get('ext', dl.format if dl.format in ['mp4', 'mkv', 'webm', 'mp3', 'm4a'] else 'mp4') + predicted_filename = predicted_filename.replace('%(ext)s', ext) + predicted_filepath = os.path.join(dldirectory, predicted_filename) + + log.info(f"[PreCheck] Modified output template: {output}") + log.info(f"[PreCheck] New predicted filepath: {predicted_filepath}") + else: + log.info(f"[PreCheck] No conflict detected, using original template") + + # Reserve this filename to prevent concurrent downloads from using it + self.reserved_filenames.add(predicted_filepath) + log.debug(f"[PreCheck] Reserved filename: {predicted_filepath}") + else: + predicted_filepath = None + log.debug(f"[PreCheck] No entry data available, skipping pre-check") + + log.debug(f"final resolved output template: {output}") + download = Download(dldirectory, self.config.TEMP_DIR, output, output_chapter, dl.quality, dl.format, ytdl_options, dl) + + # Store the reserved filepath for cleanup + download.reserved_filepath = predicted_filepath + + # Remove from in-progress set before adding to queue + # This allows checking queue.exists() to work properly + if dl.url in self.precheck_in_progress: + del self.precheck_in_progress[dl.url] + log.debug(f"[PreCheck] Removed from in-progress tracking: {dl.url}") + + if auto_start is True: + self.queue.put(download) + asyncio.create_task(self.__start_download(download)) + else: + self.pending.put(download) + + async def __start_download(self, download): + if download.canceled: + log.info(f"Download {download.info.title} was canceled, skipping start.") + return + if self.config.DOWNLOAD_MODE == 'sequential': + async with self.seq_lock: + log.info("Starting sequential download.") + await download.start(self.notifier) + await self._try_mark_watched(download) + self._post_download_cleanup(download) + elif self.config.DOWNLOAD_MODE == 'limited' and self.semaphore is not None: + await self.__limited_concurrent_download(download) + else: + await self.__concurrent_download(download) + + async def __concurrent_download(self, download): + log.info("Starting concurrent download without limits.") + asyncio.create_task(self._run_download(download)) + + async def __limited_concurrent_download(self, download): + log.info("Starting limited concurrent download.") + async with self.semaphore: + await self._run_download(download) + + async def _run_download(self, download): + if download.canceled: + log.info(f"Download {download.info.title} is canceled; skipping start.") + return + await download.start(self.notifier) + await self._try_mark_watched(download) + self._post_download_cleanup(download) + + async def _try_mark_watched(self, download): + """Try to mark the video as watched on the source website.""" + if not self.config.MARK_WATCHED_ON_COMPLETE: + return + + # Only mark as watched if download was successful + if download.info.status != 'finished': + return + + # Get the cookie file path from ytdl_options + cookie_file = download.ytdl_opts.get('cookiefile') + if not cookie_file: + log.debug(f"No cookie file for download, skipping mark-watched: {download.info.url}") + return + + log.info(f"Attempting to mark video as watched: {download.info.url}") + try: + success = await mark_watched(download.info.url, cookie_file) + if success: + log.info(f"Successfully marked video as watched: {download.info.title}") + else: + log.debug(f"Could not mark video as watched: {download.info.title}") + except Exception as e: + log.error(f"Error in mark-watched: {e}") + + def _post_download_cleanup(self, download): + # Release filename reservation if it exists + if hasattr(download, 'reserved_filepath') and download.reserved_filepath: + if download.reserved_filepath in self.reserved_filenames: + self.reserved_filenames.discard(download.reserved_filepath) + log.debug(f"[PreCheck] Released reservation for: {download.reserved_filepath}") + + if download.info.status != 'finished': + if download.tmpfilename and os.path.isfile(download.tmpfilename): + try: + os.remove(download.tmpfilename) + except: + pass + download.info.status = 'error' + download.close() + if self.queue.exists(download.info.url): + self.queue.delete(download.info.url) + if download.canceled: + asyncio.create_task(self.notifier.canceled(download.info.url)) + else: + self.done.put(download) + asyncio.create_task(self.notifier.completed(download.info)) + + def __extract_info(self, url, playlist_strict_mode): + return yt_dlp.YoutubeDL(params={ + 'quiet': True, + 'no_color': True, + 'extract_flat': True, + 'ignore_no_formats_error': True, + 'noplaylist': playlist_strict_mode, + 'paths': {"home": self.config.DOWNLOAD_DIR, "temp": self.config.TEMP_DIR}, + **self.config.YTDL_OPTIONS, + **({'impersonate': yt_dlp.networking.impersonate.ImpersonateTarget.from_str(self.config.YTDL_OPTIONS['impersonate'])} if 'impersonate' in self.config.YTDL_OPTIONS else {}), + }).extract_info(url, download=False) + + def __calc_download_path(self, quality, format, folder): + base_directory = self.config.DOWNLOAD_DIR if (quality != 'audio' and format not in AUDIO_FORMATS) else self.config.AUDIO_DOWNLOAD_DIR + if folder: + if not self.config.CUSTOM_DIRS: + return None, {'status': 'error', 'msg': f'A folder for the download was specified but CUSTOM_DIRS is not true in the configuration.'} + dldirectory = os.path.realpath(os.path.join(base_directory, folder)) + real_base_directory = os.path.realpath(base_directory) + if not dldirectory.startswith(real_base_directory): + return None, {'status': 'error', 'msg': f'Folder "{folder}" must resolve inside the base download directory "{real_base_directory}"'} + if not os.path.isdir(dldirectory): + if not self.config.CREATE_CUSTOM_DIRS: + return None, {'status': 'error', 'msg': f'Folder "{folder}" for download does not exist inside base directory "{real_base_directory}", and CREATE_CUSTOM_DIRS is not true in the configuration.'} + os.makedirs(dldirectory, exist_ok=True) + else: + dldirectory = base_directory + return dldirectory, None + + async def __add_download(self, dl, auto_start): + """Fast path: validate and queue for precheck processing. + Returns immediately without blocking on slow operations.""" + # Check if this exact URL is already being processed, in queue, or already downloaded + # This prevents duplicate downloads when same URL is submitted multiple times + if (dl.url in self.precheck_in_progress or + self.queue.exists(dl.url) or + self.pending.exists(dl.url) or + self.done.exists(dl.url)): + log.info(f"[PreCheck] URL already queued/processing/downloaded, skipping: {dl.url}") + # Add event notification + self._add_event('duplicate_skipped', 'URL already in queue or downloaded', dl.url) + return {'status': 'ok', 'msg': 'Download already exists'} + + dldirectory, error_message = self.__calc_download_path(dl.quality, dl.format, dl.folder) + if error_message is not None: + return error_message + + output = self.config.OUTPUT_TEMPLATE if len(dl.custom_name_prefix) == 0 else f'{dl.custom_name_prefix}.{self.config.OUTPUT_TEMPLATE}' + output_chapter = self.config.OUTPUT_TEMPLATE_CHAPTER + entry = getattr(dl, 'entry', None) + + if entry is not None and 'playlist' in entry and entry['playlist'] is not None: + if len(self.config.OUTPUT_TEMPLATE_PLAYLIST): + output = self.config.OUTPUT_TEMPLATE_PLAYLIST + for property, value in entry.items(): + if property.startswith("playlist"): + output = output.replace(f"%({property})s", str(value)) + + ytdl_options = dict(self.config.YTDL_OPTIONS) + playlist_item_limit = getattr(dl, 'playlist_item_limit', 0) + if playlist_item_limit > 0: + log.info(f'playlist limit is set. Processing only first {playlist_item_limit} entries') + ytdl_options['playlistend'] = playlist_item_limit + + # Check if cookie file exists for this domain + parsed_url = urlparse(dl.url) + domain = parsed_url.netloc + log.info(f"[Cookie] Checking for cookie file for domain: {domain}") + + cookies_dir = os.path.join(self.config.STATE_DIR, 'cookies') + log.debug(f"[Cookie] Cookies directory: {cookies_dir}") + + # Try domain-specific cookie file + safe_domain = domain.replace(':', '_').replace('/', '_') + cookie_file = os.path.join(cookies_dir, f'{safe_domain}.txt') + + log.debug(f"[Cookie] Looking for cookie file at: {cookie_file}") + + if os.path.exists(cookie_file): + log.info(f"[Cookie] Found cookie file: {cookie_file}") + # Verify file is readable and has content + try: + with open(cookie_file, 'r') as f: + lines = f.readlines() + cookie_lines = [l for l in lines if l.strip() and not l.startswith('#')] + log.info(f"[Cookie] Cookie file contains {len(cookie_lines)} cookie entries") + if len(cookie_lines) == 0: + log.warning(f"[Cookie] Cookie file exists but contains no cookies!") + else: + log.debug(f"[Cookie] First cookie entry: {cookie_lines[0][:50]}...") + except Exception as e: + log.error(f"[Cookie] Error reading cookie file: {e}", exc_info=True) + + ytdl_options['cookiefile'] = cookie_file + log.info(f"[Cookie] Configured yt-dlp to use cookiefile: {cookie_file}") + else: + log.info(f"[Cookie] No cookie file found for domain {domain}") + log.debug(f"[Cookie] Checked path: {cookie_file}") + # List available cookie files for debugging + if os.path.exists(cookies_dir): + available_cookies = os.listdir(cookies_dir) + if available_cookies: + log.debug(f"[Cookie] Available cookie files: {available_cookies}") + else: + log.debug(f"[Cookie] Cookies directory is empty") + else: + log.debug(f"[Cookie] Cookies directory does not exist") + + # Mark URL as being processed to prevent duplicates + # Store the DownloadInfo so we can display it in UI + self.precheck_in_progress[dl.url] = dl + + # Queue for sequential precheck processing (fast, non-blocking) + await self.precheck_queue.put({ + 'dl': dl, + 'auto_start': auto_start, + 'dldirectory': dldirectory, + 'output': output, + 'output_chapter': output_chapter, + 'ytdl_options': ytdl_options, + 'entry': entry, + }) + log.debug(f"[PreCheck] Queued for processing: {dl.url}") + + # Notify immediately (fast response to user) + await self.notifier.added(dl) + + async def __add_entry(self, entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already): + if not entry: + return {'status': 'error', 'msg': "Invalid/empty data was given."} + + error = None + if "live_status" in entry and "release_timestamp" in entry and entry.get("live_status") == "is_upcoming": + dt_ts = datetime.fromtimestamp(entry.get("release_timestamp")).strftime('%Y-%m-%d %H:%M:%S %z') + error = f"Live stream is scheduled to start at {dt_ts}" + else: + if "msg" in entry: + error = entry["msg"] + + etype = entry.get('_type') or 'video' + + if etype.startswith('url'): + log.debug('Processing as an url') + return await self.add(entry['url'], quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already) + elif etype == 'playlist': + log.debug('Processing as a playlist') + entries = entry['entries'] + log.info(f'playlist detected with {len(entries)} entries') + + playlist_index_digits = len(str(len(entries))) + results = [] + if playlist_item_limit > 0: + log.info(f'Playlist item limit is set. Processing only first {playlist_item_limit} entries') + entries = entries[:playlist_item_limit] + + # Verify playlist entry has 'id' before using it + playlist_id = entry.get("id", "unknown_playlist") + if "id" not in entry: + log.warning(f"Playlist entry missing 'id' field. Using fallback 'unknown_playlist'. Entry keys: {list(entry.keys())}") + + for index, etr in enumerate(entries, start=1): + etr["_type"] = "video" + etr["playlist"] = playlist_id + etr["playlist_index"] = '{{0:0{0:d}d}}'.format(playlist_index_digits).format(index) + for property in ("id", "title", "uploader", "uploader_id"): + if property in entry: + etr[f"playlist_{property}"] = entry[property] + results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already)) + if any(res['status'] == 'error' for res in results): + return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)} + return {'status': 'ok'} + elif etype == 'video' or (etype.startswith('url') and 'id' in entry and 'title' in entry): + log.debug('Processing as a video') + + # Extract ID from entry, or derive from URL if missing + video_id = entry.get('id') + if not video_id: + # Try to extract ID from URL (e.g., viewkey parameter or URL path) + video_url = entry.get('url', '') + if 'viewkey=' in video_url: + # Extract viewkey parameter (common in PornHub, etc.) + match = re.search(r'viewkey=([^&]+)', video_url) + if match: + video_id = match.group(1) + log.info(f"Extracted video ID from viewkey: {video_id}") + elif 'webpage_url' in entry: + # Use webpage_url as fallback + video_id = entry['webpage_url'] + else: + # Last resort: use the URL itself + video_id = video_url + + if not video_id: + log.error(f"Video entry missing 'id' field and could not extract from URL. Entry keys: {list(entry.keys())}") + return {'status': 'error', 'msg': "Video entry missing required 'id' field and URL extraction failed"} + + key = entry.get('webpage_url') or entry['url'] + if not self.queue.exists(key): + dl = DownloadInfo(video_id, entry.get('title') or video_id, key, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit) + await self.__add_download(dl, auto_start) + return {'status': 'ok'} + return {'status': 'error', 'msg': f'Unsupported resource "{etype}"'} + + async def add(self, url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start=True, already=None): + log.info(f'adding {url}: {quality=} {format=} {already=} {folder=} {custom_name_prefix=} {playlist_strict_mode=} {playlist_item_limit=} {auto_start=}') + already = set() if already is None else already + if url in already: + log.info('recursion detected, skipping') + return {'status': 'ok'} + else: + already.add(url) + try: + entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url, playlist_strict_mode) + except yt_dlp.utils.YoutubeDLError as exc: + return {'status': 'error', 'msg': str(exc)} + return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already) + + async def start_pending(self, ids): + for id in ids: + if not self.pending.exists(id): + log.warn(f'requested start for non-existent download {id}') + continue + dl = self.pending.get(id) + self.queue.put(dl) + self.pending.delete(id) + asyncio.create_task(self.__start_download(dl)) + return {'status': 'ok'} + + async def cancel(self, ids): + for id in ids: + if self.pending.exists(id): + self.pending.delete(id) + await self.notifier.canceled(id) + continue + if not self.queue.exists(id): + log.warn(f'requested cancel for non-existent download {id}') + continue + if self.queue.get(id).started(): + self.queue.get(id).cancel() + else: + self.queue.delete(id) + await self.notifier.canceled(id) + return {'status': 'ok'} + + async def clear(self, ids): + for id in ids: + if not self.done.exists(id): + log.warn(f'requested delete for non-existent download {id}') + continue + if self.config.DELETE_FILE_ON_TRASHCAN: + dl = self.done.get(id) + try: + dldirectory, _ = self.__calc_download_path(dl.info.quality, dl.info.format, dl.info.folder) + os.remove(os.path.join(dldirectory, dl.info.filename)) + except Exception as e: + log.warn(f'deleting file for download {id} failed with error message {e!r}') + self.done.delete(id) + await self.notifier.cleared(id) + return {'status': 'ok'} + + def get(self): + # Ensure website field is populated for all downloads + for k, v in self.queue.items(): + if not hasattr(v.info, 'website') or v.info.website is None: + parsed_url = urlparse(v.info.url) + v.info.website = parsed_url.netloc + + for k, v in self.pending.items(): + if not hasattr(v.info, 'website') or v.info.website is None: + parsed_url = urlparse(v.info.url) + v.info.website = parsed_url.netloc + + # Update file existence status for done downloads + for k, v in self.done.items(): + if not hasattr(v.info, 'website') or v.info.website is None: + parsed_url = urlparse(v.info.url) + v.info.website = parsed_url.netloc + + # Use getattr with default to safely check for filename attribute + filename = getattr(v.info, 'filename', None) + if filename: + dldirectory, _ = self.__calc_download_path(v.info.quality, v.info.format, v.info.folder) + if dldirectory: + filepath = os.path.join(dldirectory, filename) + v.info.file_exists = os.path.exists(filepath) + else: + v.info.file_exists = False + else: + v.info.file_exists = False + + # Create list from items in precheck queue + # These items have 'preparing' status to indicate they're being analyzed + precheck_list = [(dl.url, dl) for dl in self.precheck_in_progress.values()] + + return (precheck_list + + list((k, v.info) for k, v in self.queue.items()) + + list((k, v.info) for k, v in self.pending.items()), + list((k, v.info) for k, v in self.done.items())) + + def _add_event(self, event_type, message, url=None): + """Add an event to the events list (keep only last 5).""" + event = { + 'type': event_type, + 'message': message, + 'timestamp': int(time.time()), + 'url': url + } + self.events.append(event) + # Keep only last 5 events + if len(self.events) > self.max_events: + self.events = self.events[-self.max_events:] + # Notify frontend via WebSocket + asyncio.create_task(self.notifier.event(event)) + + def get_events(self): + """Get all events (last 5).""" + return self.events + + def clear_events(self): + """Clear all events.""" + self.events = [] diff --git a/docker-compose.yml b/docker-compose.yml index fb4a1cd..d604be2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,6 +73,9 @@ services: # Optional: robots.txt # - ROBOTS_TXT=/app/robots.txt + # MARK_WATCHED_ON_COMPLETE + - MARK_WATCHED_ON_COMPLETE=true + # Optional: health check healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8081/version"] diff --git a/pyproject.toml b/pyproject.toml index fb76830..ef05d0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,3 +16,8 @@ dependencies = [ dev = [ "pylint", ] + +[project.optional-dependencies] +headless = [ + "playwright", +] diff --git a/uv.lock b/uv.lock index e5030cb..e21473d 100644 --- a/uv.lock +++ b/uv.lock @@ -744,11 +744,11 @@ wheels = [ [[package]] name = "yt-dlp" -version = "2025.12.8" +version = "2026.2.21" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/14/77/db924ebbd99d0b2b571c184cb08ed232cf4906c6f9b76eed763cd2c84170/yt_dlp-2025.12.8.tar.gz", hash = "sha256:b773c81bb6b71cb2c111cfb859f453c7a71cf2ef44eff234ff155877184c3e4f", size = 3088947, upload-time = "2025-12-08T00:16:01.649Z" } +sdist = { url = "https://files.pythonhosted.org/packages/58/d9/55ffff25204733e94a507552ad984d5a8a8e4f9d1f0d91763e6b1a41c79b/yt_dlp-2026.2.21.tar.gz", hash = "sha256:4407dfc1a71fec0dee5ef916a8d4b66057812939b509ae45451fa8fb4376b539", size = 3116630, upload-time = "2026-02-21T20:40:53.522Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/2f/98c3596ad923f8efd32c90dca62e241e8ad9efcebf20831173c357042ba0/yt_dlp-2025.12.8-py3-none-any.whl", hash = "sha256:36e2584342e409cfbfa0b5e61448a1c5189e345cf4564294456ee509e7d3e065", size = 3291464, upload-time = "2025-12-08T00:15:58.556Z" }, + { url = "https://files.pythonhosted.org/packages/5a/40/664c99ee36d80d84ce7a96cd98aebcb3d16c19e6c3ad3461d2cf5424040e/yt_dlp-2026.2.21-py3-none-any.whl", hash = "sha256:0d8408f5b6d20487f5caeb946dfd04f9bcd2f1a3a125b744a0a982b590e449f7", size = 3313392, upload-time = "2026-02-21T20:40:51.514Z" }, ] [package.optional-dependencies] @@ -769,9 +769,9 @@ default = [ [[package]] name = "yt-dlp-ejs" -version = "0.3.2" +version = "0.5.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/de/72/57d02cf78eb45126bd171298d6a58a5bd48ce1a398b6b7ff00fc904f1f0c/yt_dlp_ejs-0.3.2.tar.gz", hash = "sha256:31a41292799992bdc913e03c9fac2a8c90c82a5cbbc792b2e3373b01da841e3e", size = 34678, upload-time = "2025-12-07T23:44:48.258Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6b/0d/b9e4ab1b47cdeba0842df634b74b3c0144307640ad5b632a5e189c4ab7ce/yt_dlp_ejs-0.5.0.tar.gz", hash = "sha256:8dfae59e418232f485253dcf8e197fefa232423c3af7824fe19e4517b173293b", size = 98925, upload-time = "2026-02-21T19:29:16.844Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/0d/1f0d7a735ca60b87953271b15d00eff5eef05f6118390ddf6f81982526ed/yt_dlp_ejs-0.3.2-py3-none-any.whl", hash = "sha256:f2dc6b3d1b909af1f13e021621b0af048056fca5fb07c4db6aa9bbb37a4f66a9", size = 53252, upload-time = "2025-12-07T23:44:46.605Z" }, + { url = "https://files.pythonhosted.org/packages/7e/5b/1283356b70d4893a8a050cee15092e1b08ea15310b94365f88067146721b/yt_dlp_ejs-0.5.0-py3-none-any.whl", hash = "sha256:674fc0efea741d3100cdf3f0f9e123150715ee41edf47ea7a62fbdeda204bdec", size = 54032, upload-time = "2026-02-21T19:29:15.408Z" }, ]