Merge pull request #177 from jwoglom/custom-download-folder

Custom download folders (fixes #129)
This commit is contained in:
Alex 2022-09-30 09:10:57 +03:00 committed by GitHub
commit b7c8c80362
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 216 additions and 30 deletions

View File

@ -36,6 +36,8 @@ Certain values can be set via environment variables, using the `-e` parameter on
* __UMASK__: umask value used by MeTube. Defaults to `022`.
* __DOWNLOAD_DIR__: path to where the downloads will be saved. Defaults to `/downloads` in the docker image, and `.` otherwise.
* __AUDIO_DOWNLOAD_DIR__: path to where audio-only downloads will be saved, if you wish to separate them from the video downloads. Defaults to the value of `DOWNLOAD_DIR`.
* __CUSTOM_DIRS__: whether to enable downloading videos into custom directories within the __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__). When enabled, a drop-down appears next to the Add button to specify the download directory. Defaults to `true`.
* __CREATE_CUSTOM_DIRS__: whether to support automatically creating directories within the __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__) if they do not exist. When enabled, the download directory selector becomes supports free-text input, and the specified directory will be created recursively. Defaults to `false`.
* __STATE_DIR__: path to where the queue persistence files will be saved. Defaults to `/downloads/.metube` in the docker image, and `.` otherwise.
* __URL_PREFIX__: base path for the web server (for use when hosting behind a reverse proxy). Defaults to `/`.
* __OUTPUT_TEMPLATE__: the template for the filenames of the downloaded videos, formatted according to [this spec](https://github.com/yt-dlp/yt-dlp/blob/master/README.md#output-template). Defaults to `%(title)s.%(ext)s`.

View File

@ -7,6 +7,7 @@ from aiohttp import web
import socketio
import logging
import json
import pathlib
from ytdl import DownloadQueueNotifier, DownloadQueue
@ -16,6 +17,8 @@ class Config:
_DEFAULTS = {
'DOWNLOAD_DIR': '.',
'AUDIO_DOWNLOAD_DIR': '%%DOWNLOAD_DIR',
'CUSTOM_DIRS': 'true',
'CREATE_CUSTOM_DIRS': 'true',
'STATE_DIR': '.',
'URL_PREFIX': '',
'OUTPUT_TEMPLATE': '%(title)s.%(ext)s',
@ -80,7 +83,8 @@ async def add(request):
if not url or not quality:
raise web.HTTPBadRequest()
format = post.get('format')
status = await dqueue.add(url, quality, format)
folder = post.get('folder')
status = await dqueue.add(url, quality, format, folder)
return web.Response(text=serializer.encode(status))
@routes.post(config.URL_PREFIX + 'delete')
@ -96,6 +100,40 @@ async def delete(request):
@sio.event
async def connect(sid, environ):
await sio.emit('all', serializer.encode(dqueue.get()), to=sid)
await sio.emit('configuration', serializer.encode(config), to=sid)
if config.CUSTOM_DIRS:
await sio.emit('custom_dirs', serializer.encode(get_custom_dirs()), to=sid)
def get_custom_dirs():
def recursive_dirs(base):
path = pathlib.Path(base)
# Converts PosixPath object to string, and remove base/ prefix
def convert(p):
s = str(p)
if s.startswith(base):
s = s[len(base):]
if s.startswith('/'):
s = s[1:]
return s
# Recursively lists all subdirectories of DOWNLOAD_DIR
dirs = list(filter(None, map(convert, path.glob('**'))))
return dirs
download_dir = recursive_dirs(config.DOWNLOAD_DIR)
audio_download_dir = download_dir
if config.DOWNLOAD_DIR != config.AUDIO_DOWNLOAD_DIR:
audio_download_dir = recursive_dirs(config.AUDIO_DOWNLOAD_DIR)
return {
"download_dir": download_dir,
"audio_download_dir": audio_download_dir
}
@routes.get(config.URL_PREFIX)
def index(request):
@ -112,8 +150,15 @@ if config.URL_PREFIX != '/':
routes.static(config.URL_PREFIX + 'favicon/', 'favicon')
routes.static(config.URL_PREFIX + 'download/', config.DOWNLOAD_DIR)
routes.static(config.URL_PREFIX + 'audio_download/', config.AUDIO_DOWNLOAD_DIR)
routes.static(config.URL_PREFIX, 'ui/dist/metube')
app.add_routes(routes)
try:
app.add_routes(routes)
except ValueError as e:
if 'ui/dist/metube' in str(e):
raise RuntimeError('Could not find the frontend UI static assets. Please run `node_modules/.bin/ng build` inside the ui folder') from e
raise e
# https://github.com/aio-libs/aiohttp/pull/4615 waiting for release

View File

@ -28,10 +28,11 @@ class DownloadQueueNotifier:
raise NotImplementedError
class DownloadInfo:
def __init__(self, id, title, url, quality, format):
def __init__(self, id, title, url, quality, format, folder):
self.id, self.title, self.url = id, title, url
self.quality = quality
self.format = format
self.folder = folder
self.status = self.msg = self.percent = self.speed = self.eta = None
self.filename = None
self.timestamp = time.time_ns()
@ -197,7 +198,7 @@ class DownloadQueue:
async def __import_queue(self):
for k, v in self.queue.saved_items():
await self.add(v.url, v.quality, v.format)
await self.add(v.url, v.quality, v.format, folder=v.folder)
async def initialize(self):
self.event = asyncio.Event()
@ -212,7 +213,7 @@ class DownloadQueue:
**self.config.YTDL_OPTIONS,
}).extract_info(url, download=False)
async def __add_entry(self, entry, quality, format, already):
async def __add_entry(self, entry, quality, format, folder, already):
etype = entry.get('_type') or 'video'
if etype == 'playlist':
entries = entry['entries']
@ -225,14 +226,28 @@ class DownloadQueue:
for property in ("id", "title", "uploader", "uploader_id"):
if property in entry:
etr[f"playlist_{property}"] = entry[property]
results.append(await self.__add_entry(etr, quality, format, already))
results.append(await self.__add_entry(etr, quality, format, folder, already))
if any(res['status'] == 'error' for res in results):
return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)}
return {'status': 'ok'}
elif etype == 'video' or etype.startswith('url') and 'id' in entry and 'title' in entry:
if not self.queue.exists(entry['id']):
dl = DownloadInfo(entry['id'], entry['title'], entry.get('webpage_url') or entry['url'], quality, format)
dldirectory = self.config.DOWNLOAD_DIR if (quality != 'audio' and format != 'mp3') else self.config.AUDIO_DOWNLOAD_DIR
dl = DownloadInfo(entry['id'], entry['title'], entry.get('webpage_url') or entry['url'], quality, format, folder)
# Keep consistent with frontend
base_directory = self.config.DOWNLOAD_DIR if (quality != 'audio' and format != 'mp3') else self.config.AUDIO_DOWNLOAD_DIR
if folder:
if self.config.CUSTOM_DIRS != 'true':
return {'status': 'error', 'msg': f'A folder for the download was specified but CUSTOM_DIRS is not true in the configuration.'}
dldirectory = os.path.realpath(os.path.join(base_directory, folder))
real_base_directory = os.path.realpath(base_directory)
if not dldirectory.startswith(real_base_directory):
return {'status': 'error', 'msg': f'Folder "{folder}" must resolve inside the base download directory "{real_base_directory}"'}
if not os.path.isdir(dldirectory):
if self.config.CREATE_CUSTOM_DIRS != 'true':
return {'status': 'error', 'msg': f'Folder "{folder}" for download does not exist inside base directory "{real_base_directory}", and CREATE_CUSTOM_DIRS is not true in the configuration.'}
os.makedirs(dldirectory, exist_ok=True)
else:
dldirectory = base_directory
output = self.config.OUTPUT_TEMPLATE
output_chapter = self.config.OUTPUT_TEMPLATE_CHAPTER
for property, value in entry.items():
@ -243,11 +258,11 @@ class DownloadQueue:
await self.notifier.added(dl)
return {'status': 'ok'}
elif etype.startswith('url'):
return await self.add(entry['url'], quality, format, already)
return await self.add(entry['url'], quality, format, folder, already)
return {'status': 'error', 'msg': f'Unsupported resource "{etype}"'}
async def add(self, url, quality, format, already=None):
log.info(f'adding {url}')
async def add(self, url, quality, format, folder, already=None):
log.info(f'adding {url}: {quality=} {format=} {already=} {folder=}')
already = set() if already is None else already
if url in already:
log.info('recursion detected, skipping')
@ -258,7 +273,7 @@ class DownloadQueue:
entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url)
except yt_dlp.utils.YoutubeDLError as exc:
return {'status': 'error', 'msg': str(exc)}
return await self.__add_entry(entry, quality, format, already)
return await self.__add_entry(entry, quality, format, folder, already)
async def cancel(self, ids):
for id in ids:

View File

@ -30,7 +30,9 @@
"styles": [
"src/styles.sass"
],
"scripts": []
"scripts": [
"node_modules/bootstrap/dist/js/bootstrap.min.js",
]
},
"configurations": {
"production": {

26
ui/package-lock.json generated
View File

@ -22,6 +22,7 @@
"@fortawesome/free-regular-svg-icons": "^6.1.1",
"@fortawesome/free-solid-svg-icons": "^6.1.1",
"@ng-bootstrap/ng-bootstrap": "^12.0.0",
"@ng-select/ng-select": "^8.3.0",
"bootstrap": "^5.0.0",
"ngx-cookie-service": "^13.0.0",
"ngx-socket-io": "^4.2.0",
@ -2529,6 +2530,23 @@
"rxjs": "^6.5.3 || ^7.4.0"
}
},
"node_modules/@ng-select/ng-select": {
"version": "8.3.0",
"resolved": "https://registry.npmjs.org/@ng-select/ng-select/-/ng-select-8.3.0.tgz",
"integrity": "sha512-AwAuDs+86++D2kEsik2/ZiQuRk0khd1HESofOm1yMBwbzsw+xLSyVOMml/OehDFSOxli7fAkk07wYtzAhxSB3Q==",
"dependencies": {
"tslib": "^2.3.1"
},
"engines": {
"node": ">= 12.20.0",
"npm": ">= 6.0.0"
},
"peerDependencies": {
"@angular/common": ">=13.0.0 <14.0.0",
"@angular/core": ">=13.0.0 <14.0.0",
"@angular/forms": ">=13.0.0 <14.0.0"
}
},
"node_modules/@ngtools/webpack": {
"version": "13.3.8",
"resolved": "https://registry.npmjs.org/@ngtools/webpack/-/webpack-13.3.8.tgz",
@ -13397,6 +13415,14 @@
"tslib": "^2.3.0"
}
},
"@ng-select/ng-select": {
"version": "8.3.0",
"resolved": "https://registry.npmjs.org/@ng-select/ng-select/-/ng-select-8.3.0.tgz",
"integrity": "sha512-AwAuDs+86++D2kEsik2/ZiQuRk0khd1HESofOm1yMBwbzsw+xLSyVOMml/OehDFSOxli7fAkk07wYtzAhxSB3Q==",
"requires": {
"tslib": "^2.3.1"
}
},
"@ngtools/webpack": {
"version": "13.3.8",
"resolved": "https://registry.npmjs.org/@ngtools/webpack/-/webpack-13.3.8.tgz",

View File

@ -25,6 +25,7 @@
"@fortawesome/free-regular-svg-icons": "^6.1.1",
"@fortawesome/free-solid-svg-icons": "^6.1.1",
"@ng-bootstrap/ng-bootstrap": "^12.0.0",
"@ng-select/ng-select": "^8.3.0",
"bootstrap": "^5.0.0",
"ngx-cookie-service": "^13.0.0",
"ngx-socket-io": "^4.2.0",

View File

@ -47,10 +47,21 @@
</div>
</div>
<div class="col-md-3 add-url-component">
<button class="btn btn-primary add-url" type="submit" (click)="addDownload()" [disabled]="addInProgress || downloads.loading">
<span class="spinner-border spinner-border-sm" role="status" id="add-spinner" *ngIf="addInProgress"></span>
{{ addInProgress ? "Adding..." : "Add" }}
</button>
<div [attr.class]="showAdvanced() ? 'btn-group add-url-group' : 'add-url-group'" ngbDropdown #advancedDropdown="ngbDropdown" display="dynamic" placement="bottom-end">
<button class="btn btn-primary add-url" type="submit" (click)="addDownload()" [disabled]="addInProgress || downloads.loading">
<span class="spinner-border spinner-border-sm" role="status" id="add-spinner" *ngIf="addInProgress"></span>
{{ addInProgress ? "Adding..." : "Add" }}
</button>
<button class="btn btn-primary dropdown-toggle dropdown-toggle-split" id="advancedButton" type="button" title="Advanced options" [disabled]="addInProgress || downloads.loading" ngbDropdownAnchor (focus)="advancedDropdown.open()" *ngIf="showAdvanced()">
<span class="sr-only">Advanced options</span>
</button>
<div ngbDropdownMenu aria-labelledby="advancedButton" class="dropdown-menu dropdown-menu-end folder-dropdown-menu">
<div class="input-group">
<span class="input-group-text">Download Folder</span>
<ng-select [items]="customDirs$ | async" placeholder="Default" [addTag]="allowCustomDir.bind(this)" addTagText="Create directory" [ngStyle]="{'flex-grow':'1'}" bindLabel="folder" [(ngModel)]="folder" [disabled]="addInProgress || downloads.loading"></ng-select>
</div>
</div>
</div>
</div>
</div>
</div>
@ -118,11 +129,11 @@
<fa-icon *ngIf="download.value.status == 'finished'" [icon]="faCheckCircle" style="color: green;"></fa-icon>
<fa-icon *ngIf="download.value.status == 'error'" [icon]="faTimesCircle" style="color: red;"></fa-icon>
</div>
<span ngbTooltip="{{download.value.msg}}"><a *ngIf="!!download.value.filename; else noDownloadLink" href="download/{{download.value.filename | encodeURIComponent}}" target="_blank">{{ download.value.title }}</a></span>
<span ngbTooltip="{{download.value.msg}}"><a *ngIf="!!download.value.filename; else noDownloadLink" href="{{buildDownloadLink(download.value)}}" target="_blank">{{ download.value.title }}</a></span>
<ng-template #noDownloadLink>{{ download.value.title }}</ng-template>
</td>
<td>
<button *ngIf="download.value.status == 'error'" type="button" class="btn btn-link" (click)="retryDownload(download.key, download.value.url, download.value.quality)"><fa-icon [icon]="faRedoAlt"></fa-icon></button>
<button *ngIf="download.value.status == 'error'" type="button" class="btn btn-link" (click)="retryDownload(download.key, download.value.url, download.value.quality, download.value.folder)"><fa-icon [icon]="faRedoAlt"></fa-icon></button>
</td>
<td>
<a href="{{download.value.url}}" target="_blank"><fa-icon [icon]="faExternalLinkAlt"></fa-icon></a>

View File

@ -9,9 +9,20 @@
.add-url-component
margin: 0.5rem auto
.add-url-group
width: 100%
button.add-url
width: 100%
.folder-dropdown-menu
width: 500px
.folder-dropdown-menu .input-group
display: flex
padding-left: 5px
padding-right: 5px
$metube-section-color-bg: rgba(0,0,0,.07)
.metube-section-header

View File

@ -2,15 +2,16 @@ import { Component, ViewChild, ElementRef, AfterViewInit } from '@angular/core';
import { faTrashAlt, faCheckCircle, faTimesCircle } from '@fortawesome/free-regular-svg-icons';
import { faRedoAlt, faSun, faMoon, faExternalLinkAlt } from '@fortawesome/free-solid-svg-icons';
import { CookieService } from 'ngx-cookie-service';
import { map, Observable, of } from 'rxjs';
import { DownloadsService, Status } from './downloads.service';
import { Download, DownloadsService, Status } from './downloads.service';
import { MasterCheckboxComponent } from './master-checkbox.component';
import { Formats, Format, Quality } from './formats';
@Component({
selector: 'app-root',
templateUrl: './app.component.html',
styleUrls: ['./app.component.sass']
styleUrls: ['./app.component.sass'],
})
export class AppComponent implements AfterViewInit {
addUrl: string;
@ -18,8 +19,10 @@ export class AppComponent implements AfterViewInit {
qualities: Quality[];
quality: string;
format: string;
folder: string;
addInProgress = false;
darkMode: boolean;
customDirs$: Observable<string[]>;
@ViewChild('queueMasterCheckbox') queueMasterCheckbox: MasterCheckboxComponent;
@ViewChild('queueDelSelected') queueDelSelected: ElementRef;
@ -44,6 +47,10 @@ export class AppComponent implements AfterViewInit {
this.setupTheme(cookieService)
}
ngOnInit() {
this.customDirs$ = this.getMatchingCustomDir();
}
ngAfterViewInit() {
this.downloads.queueChanged.subscribe(() => {
this.queueMasterCheckbox.selectionChanged();
@ -70,6 +77,36 @@ export class AppComponent implements AfterViewInit {
qualityChanged() {
this.cookieService.set('metube_quality', this.quality, { expires: 3650 });
// Re-trigger custom directory change
this.downloads.customDirsChanged.next(this.downloads.customDirs);
}
showAdvanced() {
return this.downloads.configuration['CUSTOM_DIRS'] == 'true';
}
allowCustomDir(tag: string) {
if (this.downloads.configuration['CREATE_CUSTOM_DIRS'] == 'true') {
return tag;
}
return false;
}
isAudioType() {
return this.quality == 'audio' || this.format == 'mp3';
}
getMatchingCustomDir() : Observable<string[]> {
return this.downloads.customDirsChanged.asObservable().pipe(map((output) => {
// Keep logic consistent with app/ytdl.py
if (this.isAudioType()) {
console.debug("Showing audio-specific download directories");
return output["audio_download_dir"];
} else {
console.debug("Showing default download directories");
return output["download_dir"];
}
}));
}
setupTheme(cookieService) {
@ -97,6 +134,8 @@ export class AppComponent implements AfterViewInit {
this.cookieService.set('metube_format', this.format, { expires: 3650 });
// Updates to use qualities available
this.setQualities()
// Re-trigger custom directory change
this.downloads.customDirsChanged.next(this.downloads.customDirs);
}
queueSelectionChanged(checked: number) {
@ -114,13 +153,15 @@ export class AppComponent implements AfterViewInit {
this.quality = exists ? this.quality : 'best'
}
addDownload(url?: string, quality?: string, format?: string) {
addDownload(url?: string, quality?: string, format?: string, folder?: string) {
url = url ?? this.addUrl
quality = quality ?? this.quality
format = format ?? this.format
folder = folder ?? this.folder
console.debug('Downloading: url='+url+' quality='+quality+' format='+format+' folder='+folder);
this.addInProgress = true;
this.downloads.add(url, quality, format).subscribe((status: Status) => {
this.downloads.add(url, quality, format, folder).subscribe((status: Status) => {
if (status.status === 'error') {
alert(`Error adding URL: ${status.msg}`);
} else {
@ -130,8 +171,8 @@ export class AppComponent implements AfterViewInit {
});
}
retryDownload(key: string, url: string, quality: string, format: string) {
this.addDownload(url, quality, format);
retryDownload(key: string, url: string, quality: string, format: string, folder: string) {
this.addDownload(url, quality, format, folder);
this.downloads.delById('done', [key]).subscribe();
}
@ -150,4 +191,17 @@ export class AppComponent implements AfterViewInit {
clearFailedDownloads() {
this.downloads.delByFilter('done', dl => dl.status === 'error').subscribe();
}
buildDownloadLink(download: Download) {
let baseDir = 'download/';
if (download.quality == 'audio' || download.filename.endsWith('.mp3')) {
baseDir = 'audio_download/';
}
if (download.folder) {
baseDir += download.folder + '/';
}
return baseDir + encodeURIComponent(download.filename);
}
}

View File

@ -10,6 +10,7 @@ import { AppComponent } from './app.component';
import { EtaPipe, SpeedPipe, EncodeURIComponent } from './downloads.pipe';
import { MasterCheckboxComponent, SlaveCheckboxComponent } from './master-checkbox.component';
import { MeTubeSocket } from './metube-socket';
import { NgSelectModule } from '@ng-select/ng-select';
@NgModule({
declarations: [
@ -25,7 +26,8 @@ import { MeTubeSocket } from './metube-socket';
FormsModule,
NgbModule,
HttpClientModule,
FontAwesomeModule
FontAwesomeModule,
NgSelectModule
],
providers: [CookieService, MeTubeSocket],
bootstrap: [AppComponent]

View File

@ -1,6 +1,6 @@
import { Injectable } from '@angular/core';
import { HttpClient, HttpErrorResponse } from '@angular/common/http';
import { of, Subject } from 'rxjs';
import { Observable, of, Subject } from 'rxjs';
import { catchError } from 'rxjs/operators';
import { MeTubeSocket } from './metube-socket';
@ -9,13 +9,14 @@ export interface Status {
msg?: string;
}
interface Download {
export interface Download {
id: string;
title: string;
url: string,
status: string;
msg: string;
filename: string;
folder: string;
quality: string;
percent: number;
speed: number;
@ -33,6 +34,10 @@ export class DownloadsService {
done = new Map<string, Download>();
queueChanged = new Subject();
doneChanged = new Subject();
customDirsChanged = new Subject();
configuration = {};
customDirs = {};
constructor(private http: HttpClient, private socket: MeTubeSocket) {
socket.fromEvent('all').subscribe((strdata: string) => {
@ -74,6 +79,17 @@ export class DownloadsService {
this.done.delete(data);
this.doneChanged.next(null);
});
socket.fromEvent('configuration').subscribe((strdata: string) => {
let data = JSON.parse(strdata);
console.debug("got configuration:", data);
this.configuration = data;
});
socket.fromEvent('custom_dirs').subscribe((strdata: string) => {
let data = JSON.parse(strdata);
console.debug("got custom_dirs:", data);
this.customDirs = data;
this.customDirsChanged.next(data);
});
}
handleHTTPError(error: HttpErrorResponse) {
@ -81,8 +97,8 @@ export class DownloadsService {
return of({status: 'error', msg: msg})
}
public add(url: string, quality: string, format: string) {
return this.http.post<Status>('add', {url: url, quality: quality, format: format}).pipe(
public add(url: string, quality: string, format: string, folder: string) {
return this.http.post<Status>('add', {url: url, quality: quality, format: format, folder: folder}).pipe(
catchError(this.handleHTTPError)
);
}

View File

@ -2,3 +2,4 @@
/* Importing Bootstrap SCSS file. */
@import '~bootstrap/scss/bootstrap'
@import '~@ng-select/ng-select/themes/default.theme.css'