Merge pull request #177 from jwoglom/custom-download-folder

Custom download folders (fixes #129)

README.md

@@ -36,6 +36,8 @@ Certain values can be set via environment variables, using the `-e` parameter on
 * __UMASK__: umask value used by MeTube. Defaults to `022`.
 * __DOWNLOAD_DIR__: path to where the downloads will be saved. Defaults to `/downloads` in the docker image, and `.` otherwise.
 * __AUDIO_DOWNLOAD_DIR__: path to where audio-only downloads will be saved, if you wish to separate them from the video downloads. Defaults to the value of `DOWNLOAD_DIR`.
+* __CUSTOM_DIRS__: whether to enable downloading videos into custom directories within the __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__). When enabled, a drop-down appears next to the Add button to specify the download directory. Defaults to `true`.
+* __CREATE_CUSTOM_DIRS__: whether to support automatically creating directories within the __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__) if they do not exist. When enabled, the download directory selector becomes supports free-text input, and the specified directory will be created recursively. Defaults to `false`.
 * __STATE_DIR__: path to where the queue persistence files will be saved. Defaults to `/downloads/.metube` in the docker image, and `.` otherwise.
 * __URL_PREFIX__: base path for the web server (for use when hosting behind a reverse proxy). Defaults to `/`.
 * __OUTPUT_TEMPLATE__: the template for the filenames of the downloaded videos, formatted according to [this spec](https://github.com/yt-dlp/yt-dlp/blob/master/README.md#output-template). Defaults to `%(title)s.%(ext)s`.

app/main.py

@@ -7,6 +7,7 @@ from aiohttp import web
 import socketio
 import logging
 import json
+import pathlib
 
 from ytdl import DownloadQueueNotifier, DownloadQueue
 
@@ -16,6 +17,8 @@ class Config:
     _DEFAULTS = {
         'DOWNLOAD_DIR': '.',
         'AUDIO_DOWNLOAD_DIR': '%%DOWNLOAD_DIR',
+        'CUSTOM_DIRS': 'true',
+        'CREATE_CUSTOM_DIRS': 'true',
         'STATE_DIR': '.',
         'URL_PREFIX': '',
         'OUTPUT_TEMPLATE': '%(title)s.%(ext)s',
@@ -80,7 +83,8 @@ async def add(request):
     if not url or not quality:
         raise web.HTTPBadRequest()
     format = post.get('format')
-    status = await dqueue.add(url, quality, format)
+    folder = post.get('folder')
+    status = await dqueue.add(url, quality, format, folder)
     return web.Response(text=serializer.encode(status))
 
 @routes.post(config.URL_PREFIX + 'delete')
@@ -96,6 +100,40 @@ async def delete(request):
 @sio.event
 async def connect(sid, environ):
     await sio.emit('all', serializer.encode(dqueue.get()), to=sid)
+    await sio.emit('configuration', serializer.encode(config), to=sid)
+    if config.CUSTOM_DIRS:
+        await sio.emit('custom_dirs', serializer.encode(get_custom_dirs()), to=sid)
+
+def get_custom_dirs():
+    def recursive_dirs(base):
+        path = pathlib.Path(base)
+
+        # Converts PosixPath object to string, and remove base/ prefix
+        def convert(p):
+            s = str(p)
+            if s.startswith(base):
+                s = s[len(base):]
+
+            if s.startswith('/'):
+                s = s[1:]
+
+            return s
+
+        # Recursively lists all subdirectories of DOWNLOAD_DIR
+        dirs = list(filter(None, map(convert, path.glob('**'))))
+
+        return dirs
+    
+    download_dir = recursive_dirs(config.DOWNLOAD_DIR)
+
+    audio_download_dir = download_dir
+    if config.DOWNLOAD_DIR != config.AUDIO_DOWNLOAD_DIR:
+        audio_download_dir = recursive_dirs(config.AUDIO_DOWNLOAD_DIR)
+    
+    return {
+        "download_dir": download_dir,
+        "audio_download_dir": audio_download_dir
+    }
 
 @routes.get(config.URL_PREFIX)
 def index(request):
@@ -112,8 +150,15 @@ if config.URL_PREFIX != '/':
 
 routes.static(config.URL_PREFIX + 'favicon/', 'favicon')
 routes.static(config.URL_PREFIX + 'download/', config.DOWNLOAD_DIR)
+routes.static(config.URL_PREFIX + 'audio_download/', config.AUDIO_DOWNLOAD_DIR)
 routes.static(config.URL_PREFIX, 'ui/dist/metube')
+try:
     app.add_routes(routes)
+except ValueError as e:
+    if 'ui/dist/metube' in str(e):
+        raise RuntimeError('Could not find the frontend UI static assets. Please run `node_modules/.bin/ng build` inside the ui folder') from e
+    raise e
+
 
 
 # https://github.com/aio-libs/aiohttp/pull/4615 waiting for release

app/ytdl.py

@@ -28,10 +28,11 @@ class DownloadQueueNotifier:
         raise NotImplementedError
 
 class DownloadInfo:
-    def __init__(self, id, title, url, quality, format):
+    def __init__(self, id, title, url, quality, format, folder):
         self.id, self.title, self.url = id, title, url
         self.quality = quality
         self.format = format
+        self.folder = folder
         self.status = self.msg = self.percent = self.speed = self.eta = None
         self.filename = None
         self.timestamp = time.time_ns()
@@ -197,7 +198,7 @@ class DownloadQueue:
     
     async def __import_queue(self):
         for k, v in self.queue.saved_items():
-            await self.add(v.url, v.quality, v.format)
+            await self.add(v.url, v.quality, v.format, folder=v.folder)
 
     async def initialize(self):
         self.event = asyncio.Event()
@@ -212,7 +213,7 @@ class DownloadQueue:
             **self.config.YTDL_OPTIONS,
         }).extract_info(url, download=False)
 
-    async def __add_entry(self, entry, quality, format, already):
+    async def __add_entry(self, entry, quality, format, folder, already):
         etype = entry.get('_type') or 'video'
         if etype == 'playlist':
             entries = entry['entries']
@@ -225,14 +226,28 @@ class DownloadQueue:
                 for property in ("id", "title", "uploader", "uploader_id"):
                     if property in entry:
                         etr[f"playlist_{property}"] = entry[property]
-                results.append(await self.__add_entry(etr, quality, format, already))
+                results.append(await self.__add_entry(etr, quality, format, folder, already))
             if any(res['status'] == 'error' for res in results):
                 return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)}
             return {'status': 'ok'}
         elif etype == 'video' or etype.startswith('url') and 'id' in entry and 'title' in entry:
             if not self.queue.exists(entry['id']):
-                dl = DownloadInfo(entry['id'], entry['title'], entry.get('webpage_url') or entry['url'], quality, format)
+                dl = DownloadInfo(entry['id'], entry['title'], entry.get('webpage_url') or entry['url'], quality, format, folder)
-                dldirectory = self.config.DOWNLOAD_DIR if (quality != 'audio' and format != 'mp3') else self.config.AUDIO_DOWNLOAD_DIR
+                # Keep consistent with frontend
+                base_directory = self.config.DOWNLOAD_DIR if (quality != 'audio' and format != 'mp3') else self.config.AUDIO_DOWNLOAD_DIR
+                if folder:
+                    if self.config.CUSTOM_DIRS != 'true':
+                        return {'status': 'error', 'msg': f'A folder for the download was specified but CUSTOM_DIRS is not true in the configuration.'}
+                    dldirectory = os.path.realpath(os.path.join(base_directory, folder))
+                    real_base_directory = os.path.realpath(base_directory)
+                    if not dldirectory.startswith(real_base_directory):
+                        return {'status': 'error', 'msg': f'Folder "{folder}" must resolve inside the base download directory "{real_base_directory}"'}
+                    if not os.path.isdir(dldirectory):
+                        if self.config.CREATE_CUSTOM_DIRS != 'true':
+                            return {'status': 'error', 'msg': f'Folder "{folder}" for download does not exist inside base directory "{real_base_directory}", and CREATE_CUSTOM_DIRS is not true in the configuration.'}
+                        os.makedirs(dldirectory, exist_ok=True)
+                else:
+                    dldirectory = base_directory
                 output = self.config.OUTPUT_TEMPLATE
                 output_chapter = self.config.OUTPUT_TEMPLATE_CHAPTER
                 for property, value in entry.items():
@@ -243,11 +258,11 @@ class DownloadQueue:
                 await self.notifier.added(dl)
             return {'status': 'ok'}
         elif etype.startswith('url'):
-            return await self.add(entry['url'], quality, format, already)
+            return await self.add(entry['url'], quality, format, folder, already)
         return {'status': 'error', 'msg': f'Unsupported resource "{etype}"'}
 
-    async def add(self, url, quality, format, already=None):
+    async def add(self, url, quality, format, folder, already=None):
-        log.info(f'adding {url}')
+        log.info(f'adding {url}: {quality=} {format=} {already=} {folder=}')
         already = set() if already is None else already
         if url in already:
             log.info('recursion detected, skipping')
@@ -258,7 +273,7 @@ class DownloadQueue:
             entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url)
         except yt_dlp.utils.YoutubeDLError as exc:
             return {'status': 'error', 'msg': str(exc)}
-        return await self.__add_entry(entry, quality, format, already)
+        return await self.__add_entry(entry, quality, format, folder, already)
 
     async def cancel(self, ids):
         for id in ids:

ui/angular.json

@@ -30,7 +30,9 @@
             "styles": [
               "src/styles.sass"
             ],
-            "scripts": []
+            "scripts": [
+              "node_modules/bootstrap/dist/js/bootstrap.min.js",
+            ]
           },
           "configurations": {
             "production": {

ui/package-lock.json

@@ -22,6 +22,7 @@
         "@fortawesome/free-regular-svg-icons": "^6.1.1",
         "@fortawesome/free-solid-svg-icons": "^6.1.1",
         "@ng-bootstrap/ng-bootstrap": "^12.0.0",
+        "@ng-select/ng-select": "^8.3.0",
         "bootstrap": "^5.0.0",
         "ngx-cookie-service": "^13.0.0",
         "ngx-socket-io": "^4.2.0",
@@ -2529,6 +2530,23 @@
         "rxjs": "^6.5.3 || ^7.4.0"
       }
     },
+    "node_modules/@ng-select/ng-select": {
+      "version": "8.3.0",
+      "resolved": "https://registry.npmjs.org/@ng-select/ng-select/-/ng-select-8.3.0.tgz",
+      "integrity": "sha512-AwAuDs+86++D2kEsik2/ZiQuRk0khd1HESofOm1yMBwbzsw+xLSyVOMml/OehDFSOxli7fAkk07wYtzAhxSB3Q==",
+      "dependencies": {
+        "tslib": "^2.3.1"
+      },
+      "engines": {
+        "node": ">= 12.20.0",
+        "npm": ">= 6.0.0"
+      },
+      "peerDependencies": {
+        "@angular/common": ">=13.0.0 <14.0.0",
+        "@angular/core": ">=13.0.0 <14.0.0",
+        "@angular/forms": ">=13.0.0 <14.0.0"
+      }
+    },
     "node_modules/@ngtools/webpack": {
       "version": "13.3.8",
       "resolved": "https://registry.npmjs.org/@ngtools/webpack/-/webpack-13.3.8.tgz",
@@ -13397,6 +13415,14 @@
         "tslib": "^2.3.0"
       }
     },
+    "@ng-select/ng-select": {
+      "version": "8.3.0",
+      "resolved": "https://registry.npmjs.org/@ng-select/ng-select/-/ng-select-8.3.0.tgz",
+      "integrity": "sha512-AwAuDs+86++D2kEsik2/ZiQuRk0khd1HESofOm1yMBwbzsw+xLSyVOMml/OehDFSOxli7fAkk07wYtzAhxSB3Q==",
+      "requires": {
+        "tslib": "^2.3.1"
+      }
+    },
     "@ngtools/webpack": {
       "version": "13.3.8",
       "resolved": "https://registry.npmjs.org/@ngtools/webpack/-/webpack-13.3.8.tgz",

ui/package.json

@@ -25,6 +25,7 @@
     "@fortawesome/free-regular-svg-icons": "^6.1.1",
     "@fortawesome/free-solid-svg-icons": "^6.1.1",
     "@ng-bootstrap/ng-bootstrap": "^12.0.0",
+    "@ng-select/ng-select": "^8.3.0",
     "bootstrap": "^5.0.0",
     "ngx-cookie-service": "^13.0.0",
     "ngx-socket-io": "^4.2.0",

ui/src/app/app.component.html

@@ -47,10 +47,21 @@
           </div>
         </div>
         <div class="col-md-3 add-url-component">
+          <div [attr.class]="showAdvanced() ? 'btn-group add-url-group' : 'add-url-group'" ngbDropdown #advancedDropdown="ngbDropdown" display="dynamic" placement="bottom-end">
             <button class="btn btn-primary add-url" type="submit" (click)="addDownload()" [disabled]="addInProgress || downloads.loading">
               <span class="spinner-border spinner-border-sm" role="status" id="add-spinner" *ngIf="addInProgress"></span>
               {{ addInProgress ? "Adding..." : "Add" }}
             </button>
+            <button class="btn btn-primary dropdown-toggle dropdown-toggle-split" id="advancedButton" type="button" title="Advanced options" [disabled]="addInProgress || downloads.loading" ngbDropdownAnchor (focus)="advancedDropdown.open()" *ngIf="showAdvanced()">
+              <span class="sr-only">Advanced options</span>
+            </button>
+            <div ngbDropdownMenu aria-labelledby="advancedButton" class="dropdown-menu dropdown-menu-end folder-dropdown-menu">
+              <div class="input-group">
+                <span class="input-group-text">Download Folder</span>
+                <ng-select [items]="customDirs$ | async" placeholder="Default" [addTag]="allowCustomDir.bind(this)" addTagText="Create directory" [ngStyle]="{'flex-grow':'1'}" bindLabel="folder" [(ngModel)]="folder" [disabled]="addInProgress || downloads.loading"></ng-select>
+              </div>
+            </div>
+          </div>
         </div>
       </div>
     </div>
@@ -118,11 +129,11 @@
             <fa-icon *ngIf="download.value.status == 'finished'" [icon]="faCheckCircle" style="color: green;"></fa-icon>
             <fa-icon *ngIf="download.value.status == 'error'" [icon]="faTimesCircle" style="color: red;"></fa-icon>
           </div>
-          <span ngbTooltip="{{download.value.msg}}"><a *ngIf="!!download.value.filename; else noDownloadLink" href="download/{{download.value.filename | encodeURIComponent}}" target="_blank">{{ download.value.title }}</a></span>
+          <span ngbTooltip="{{download.value.msg}}"><a *ngIf="!!download.value.filename; else noDownloadLink" href="{{buildDownloadLink(download.value)}}" target="_blank">{{ download.value.title }}</a></span>
           <ng-template #noDownloadLink>{{ download.value.title }}</ng-template>
         </td>
         <td>
-          <button *ngIf="download.value.status == 'error'" type="button" class="btn btn-link" (click)="retryDownload(download.key, download.value.url, download.value.quality)"><fa-icon [icon]="faRedoAlt"></fa-icon></button>
+          <button *ngIf="download.value.status == 'error'" type="button" class="btn btn-link" (click)="retryDownload(download.key, download.value.url, download.value.quality, download.value.folder)"><fa-icon [icon]="faRedoAlt"></fa-icon></button>
         </td>
         <td>
           <a href="{{download.value.url}}" target="_blank"><fa-icon [icon]="faExternalLinkAlt"></fa-icon></a>

ui/src/app/app.component.sass

@@ -9,9 +9,20 @@
 .add-url-component
     margin: 0.5rem auto
 
+.add-url-group
+    width: 100%
+
 button.add-url
     width: 100%
 
+.folder-dropdown-menu
+    width: 500px
+
+.folder-dropdown-menu .input-group
+    display: flex
+    padding-left: 5px
+    padding-right: 5px
+
 $metube-section-color-bg: rgba(0,0,0,.07)
 
 .metube-section-header

ui/src/app/app.component.ts

@@ -2,15 +2,16 @@ import { Component, ViewChild, ElementRef, AfterViewInit } from '@angular/core';
 import { faTrashAlt, faCheckCircle, faTimesCircle } from '@fortawesome/free-regular-svg-icons';
 import { faRedoAlt, faSun, faMoon, faExternalLinkAlt } from '@fortawesome/free-solid-svg-icons';
 import { CookieService } from 'ngx-cookie-service';
+import { map, Observable, of } from 'rxjs';
 
-import { DownloadsService, Status } from './downloads.service';
+import { Download, DownloadsService, Status } from './downloads.service';
 import { MasterCheckboxComponent } from './master-checkbox.component';
 import { Formats, Format, Quality } from './formats';
 
 @Component({
   selector: 'app-root',
   templateUrl: './app.component.html',
-  styleUrls: ['./app.component.sass']
+  styleUrls: ['./app.component.sass'],
 })
 export class AppComponent implements AfterViewInit {
   addUrl: string;
@@ -18,8 +19,10 @@ export class AppComponent implements AfterViewInit {
   qualities: Quality[];
   quality: string;
   format: string;
+  folder: string;
   addInProgress = false;
   darkMode: boolean;
+  customDirs$: Observable<string[]>;
 
   @ViewChild('queueMasterCheckbox') queueMasterCheckbox: MasterCheckboxComponent;
   @ViewChild('queueDelSelected') queueDelSelected: ElementRef;
@@ -44,6 +47,10 @@ export class AppComponent implements AfterViewInit {
     this.setupTheme(cookieService)
   }
 
+  ngOnInit() {
+    this.customDirs$ = this.getMatchingCustomDir();
+  }
+
   ngAfterViewInit() {
     this.downloads.queueChanged.subscribe(() => {
       this.queueMasterCheckbox.selectionChanged();
@@ -70,6 +77,36 @@ export class AppComponent implements AfterViewInit {
 
   qualityChanged() {
     this.cookieService.set('metube_quality', this.quality, { expires: 3650 });
+    // Re-trigger custom directory change
+    this.downloads.customDirsChanged.next(this.downloads.customDirs);
+  }
+
+  showAdvanced() {
+    return this.downloads.configuration['CUSTOM_DIRS'] == 'true';
+  }
+
+  allowCustomDir(tag: string) {
+    if (this.downloads.configuration['CREATE_CUSTOM_DIRS'] == 'true') {
+      return tag;
+    }
+    return false;
+  }
+
+  isAudioType() {
+    return this.quality == 'audio' || this.format == 'mp3';
+  }
+
+  getMatchingCustomDir() : Observable<string[]> {
+    return this.downloads.customDirsChanged.asObservable().pipe(map((output) => {
+      // Keep logic consistent with app/ytdl.py
+      if (this.isAudioType()) {
+        console.debug("Showing audio-specific download directories");
+        return output["audio_download_dir"];
+      } else {
+        console.debug("Showing default download directories");
+        return output["download_dir"];
+      }
+    }));
   }
 
   setupTheme(cookieService) {
@@ -97,6 +134,8 @@ export class AppComponent implements AfterViewInit {
     this.cookieService.set('metube_format', this.format, { expires: 3650 });
     // Updates to use qualities available
     this.setQualities()
+    // Re-trigger custom directory change
+    this.downloads.customDirsChanged.next(this.downloads.customDirs);
   }
 
   queueSelectionChanged(checked: number) {
@@ -114,13 +153,15 @@ export class AppComponent implements AfterViewInit {
     this.quality = exists ? this.quality : 'best'
   }
 
-  addDownload(url?: string, quality?: string, format?: string) {
+  addDownload(url?: string, quality?: string, format?: string, folder?: string) {
     url = url ?? this.addUrl
     quality = quality ?? this.quality
     format = format ?? this.format
+    folder = folder ?? this.folder
 
+    console.debug('Downloading: url='+url+' quality='+quality+' format='+format+' folder='+folder);
     this.addInProgress = true;
-    this.downloads.add(url, quality, format).subscribe((status: Status) => {
+    this.downloads.add(url, quality, format, folder).subscribe((status: Status) => {
       if (status.status === 'error') {
         alert(`Error adding URL: ${status.msg}`);
       } else {
@@ -130,8 +171,8 @@ export class AppComponent implements AfterViewInit {
     });
   }
 
-  retryDownload(key: string, url: string, quality: string, format: string) {
+  retryDownload(key: string, url: string, quality: string, format: string, folder: string) {
-    this.addDownload(url, quality, format);
+    this.addDownload(url, quality, format, folder);
     this.downloads.delById('done', [key]).subscribe();
   }
 
@@ -150,4 +191,17 @@ export class AppComponent implements AfterViewInit {
   clearFailedDownloads() {
     this.downloads.delByFilter('done', dl => dl.status === 'error').subscribe();
   }
+
+  buildDownloadLink(download: Download) {
+    let baseDir = 'download/';
+    if (download.quality == 'audio' || download.filename.endsWith('.mp3')) {
+      baseDir = 'audio_download/';
+    }
+
+    if (download.folder) {
+      baseDir += download.folder + '/';
+    }
+
+    return baseDir + encodeURIComponent(download.filename);
+  }
 }

ui/src/app/app.module.ts

@@ -10,6 +10,7 @@ import { AppComponent } from './app.component';
 import { EtaPipe, SpeedPipe, EncodeURIComponent } from './downloads.pipe';
 import { MasterCheckboxComponent, SlaveCheckboxComponent } from './master-checkbox.component';
 import { MeTubeSocket } from './metube-socket';
+import { NgSelectModule } from '@ng-select/ng-select';
 
 @NgModule({
   declarations: [
@@ -25,7 +26,8 @@ import { MeTubeSocket } from './metube-socket';
     FormsModule,
     NgbModule,
     HttpClientModule,
-    FontAwesomeModule
+    FontAwesomeModule,
+    NgSelectModule
   ],
   providers: [CookieService, MeTubeSocket],
   bootstrap: [AppComponent]

ui/src/app/downloads.service.ts

@@ -1,6 +1,6 @@
 import { Injectable } from '@angular/core';
 import { HttpClient, HttpErrorResponse } from '@angular/common/http';
-import { of, Subject } from 'rxjs';
+import { Observable, of, Subject } from 'rxjs';
 import { catchError } from 'rxjs/operators';
 import { MeTubeSocket } from './metube-socket';
 
@@ -9,13 +9,14 @@ export interface Status {
   msg?: string;
 }
 
-interface Download {
+export interface Download {
   id: string;
   title: string;
   url: string,
   status: string;
   msg: string;
   filename: string;
+  folder: string;
   quality: string;
   percent: number;
   speed: number;
@@ -33,6 +34,10 @@ export class DownloadsService {
   done = new Map<string, Download>();
   queueChanged = new Subject();
   doneChanged = new Subject();
+  customDirsChanged = new Subject();
+
+  configuration = {};
+  customDirs = {};
 
   constructor(private http: HttpClient, private socket: MeTubeSocket) {
     socket.fromEvent('all').subscribe((strdata: string) => {
@@ -74,6 +79,17 @@ export class DownloadsService {
       this.done.delete(data);
       this.doneChanged.next(null);
     });
+    socket.fromEvent('configuration').subscribe((strdata: string) => {
+      let data = JSON.parse(strdata);
+      console.debug("got configuration:", data);
+      this.configuration = data;
+    });
+    socket.fromEvent('custom_dirs').subscribe((strdata: string) => {
+      let data = JSON.parse(strdata);
+      console.debug("got custom_dirs:", data);
+      this.customDirs = data;
+      this.customDirsChanged.next(data);
+    });
   }
 
   handleHTTPError(error: HttpErrorResponse) {
@@ -81,8 +97,8 @@ export class DownloadsService {
     return of({status: 'error', msg: msg})
   }
 
-  public add(url: string, quality: string, format: string) {
+  public add(url: string, quality: string, format: string, folder: string) {
-    return this.http.post<Status>('add', {url: url, quality: quality, format: format}).pipe(
+    return this.http.post<Status>('add', {url: url, quality: quality, format: format, folder: folder}).pipe(
       catchError(this.handleHTTPError)
     );
   }

ui/src/styles.sass

@@ -2,3 +2,4 @@
 
 /* Importing Bootstrap SCSS file. */
 @import '~bootstrap/scss/bootstrap'
+@import '~@ng-select/ng-select/themes/default.theme.css'