fix(dl): delete old file after repackage (#114)

* fix(dl): delete old file after repackage

* fix(dl): using original_path instead of self.path in repackage method

.gitignore

@@ -1,4 +1,6 @@
 # devine
+devine.yaml
+devine.yml
 *.mkv
 *.mp4
 *.exe
@@ -9,6 +11,8 @@
 *.pem
 *.bin
 *.db
+*.ttf
+*.otf
 device_cert
 device_client_id_blob
 device_private_key

.pre-commit-config.yaml

@@ -2,12 +2,17 @@
 # See https://pre-commit.com/hooks.html for more hooks
 
 repos:
-  - repo: https://github.com/mtkennerly/pre-commit-hooks
+  - repo: https://github.com/compilerla/conventional-pre-commit
-    rev: v0.3.0
+    rev: v3.2.0
     hooks:
-      - id: poetry-ruff
+      - id: conventional-pre-commit
+        stages: [commit-msg]
+  - repo: https://github.com/mtkennerly/pre-commit-hooks
+    rev: v0.4.0
+    hooks:
+      - id: poetry-ruff-check
   - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
       - id: isort
   - repo: https://github.com/pre-commit/pre-commit-hooks

CHANGELOG.md

@@ -2,8 +2,179 @@
 
 All notable changes to this project will be documented in this file.
 
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+Versions [3.0.0] and older use a format based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+but versions thereafter use a custom changelog format using [git-cliff](https://git-cliff.org).
+
+## [3.3.3] - 2024-05-07
+
+### Bug Fixes
+
+- *dl*: Automatically convert TTML Subs to WebVTT for MKV support
+- *Subtitle*: Correct timestamps when merging fragmented WebVTT
+
+### Changes
+
+- *env*: List all directories as table in info
+- *env*: List possible config path locations when not found
+- *binaries*: Move all binary definitions to core/binaries file
+- *curl-impersonate*: Remove manual fix for curl proxy SSL
+- *curl-impersonate*: Update the default browser to chrome124
+- *Config*: Move possible config paths out of func to constant
+- *utilities*: Remove get_binary_path, use binaries.find instead
+- *dl*: Improve readability of download worker errors
+- *env*: Shorten paths on Windows with env vars
+
+## [3.3.2] - 2024-04-16
+
+### Bug Fixes
+
+- *Video*: Ensure track is supported in change_color_range()
+- *Video*: Optionalise constructor args, add doc-string & checks
+- *Audio*: Optionalise constructor args, add doc-string & checks
+- *Subtitle*: Optionalise constructor args, add doc-string & checks
+- *HLS*: Ensure playlist.stream_info.codecs exists before use
+- *HLS*: Ensure playlist.stream_info.resolution exists before use
+- *env*: List used config path, otherwise the default path
+- *cfg*: Use loaded config path instead of hardcoded default
+- *Basic*: Return None not Exception if no proxy configured
+
+### Changes
+
+- *Video*: Do not print "?"/"Unknown" values in str()
+- *Audio*: Do not print "?"/"Unknown" values in str()
+- *Subtitle*: Do not print "?"/"Unknown" values in str()
+- *Audio*: List lang after codec for consistency with other Tracks
+- *Video*: Return None if no m3u RANGE, not SDR
+- *env*: Use -- to indicate no config found/loaded
+
+### New Contributors
+
+- [retouching](https://github.com/retouching)
+
+## [3.3.1] - 2024-04-05
+
+### Features
+
+- *dl*: Add *new* --workers to set download threads/workers
+
+### Bug Fixes
+
+- *Chapter*: Cast values to int prior to formatting
+- *requests*: Fix multithreaded downloads
+- *Events*: Dereference subscription store from ephemeral store
+
+### Changes
+
+- *dl*: Change --workers to --downloads
+
+### New Contributors
+
+- [knowhere01](https://github.com/knowhere01)
+
+## [3.3.0] - 2024-04-02
+
+### Features
+
+- Add support for MKV Attachments via Attachment class
+- *dl*: Automatically attach fonts used within SSAv4 subs
+- *dl*: Try find SSAv4 fonts in System OS fonts folder
+- *Basic*: Allow single string URIs for countries
+- *Basic*: Allow proxy selection by index (one-indexed)
+- *Events*: Add new global Event Observer API
+
+### Bug Fixes
+
+- *curl-impersonate*: Set Cert-Authority Bundle for HTTPS Proxies
+- *Basic*: Make query case-insensitive
+- *WVD*: Ensure WVDs dir exists before moving WVD file
+- *WVD*: Fix empty path to WVDs folder check
+- *WVD*: Move log out of loop to save performance
+- *WVD*: Move log with path before Device load
+- *WVD*: Add exists/empty checks to WVD folder dumps
+- *Basic*: Fix variable typo regression
+
+### Changes
+
+- *Basic*: Improve proxy format checks
+- *WVD*: Print error if path to parse doesn't exist
+- *WVD*: Seperate logs in loop for visual clarity
+- *Track*: Move from OnXyz callables to Event observer
+
+## [3.2.0] - 2024-03-25
+
+### Features
+
+- *ClearKey*: Pass session not proxy str in from_m3u_key method
+- *Track*: Allow Track to choose downloader to use
+- *search*: New Search command, Service method, SearchResult Class
+
+### Bug Fixes
+
+- *dl*: Include chapters when muxing
+- *aria2c*: Support aria2(c) 1.37.0 by handling upstream regression
+- *MultipleChoice*: Simplify super() call and value types
+- *dl*: Add single mux job if there's no video tracks
+- *Track*: Compute Track ID from the `this` variable, not `self`
+- *DASH/HLS*: Don't merge folders, skip final merge if only 1 segment
+- *dl*: Use click.command() instead of click.group()
+- *HLS*: Remove save dir even if final merge wasn't needed
+- *Track*: Fix order of operation mistake in get_track_name
+- *requests*: Set HTTP pool connections/maxsize to max workers
+- *Video*: Delete original file after using change_color_range()
+- *Video*: Delete original file after using remove_eia_cc()
+- *requests*: Manually compute default max_workers or pool size is None
+- *requests*: Block until connection freed if too many connections
+- *HLS*: Delete subtitle segments as they are merged
+- *HLS*: Delete video/audio segments after FFmpeg merge
+
+### Changes
+
+- *ClearKey*: Only use User-Agent if none set in from_m3u_key
+- *Track*: Remove TERRITORY_MAP constant, trim SAR China manually
+- *Track*: Default the track name to it's lang's script/territory
+- *Service*: Go back to the default pool_maxsize in Session
+
+## [3.1.0] - 2024-03-05
+
+### Features
+
+- *cli*: Implement MultipleChoice click param based on Choice param
+- *dl*: Skip video lang filter if --v-lang unused & only 1 video lang
+- *dl*: Change --vcodec default to None, use any codec
+- *dl*: Support multiple -r/--range and mux ranges separately
+- *Subtitle*: Convert from fTTML->TTML & fVTT->WebVTT post-download
+- *Track*: Make ID optional, Automatically compute one if not provided
+- *Track*: Add a name property to use for the Track Name
+
+### Bug Fixes
+
+- *dl*: Have --sub-format default to None to keep original sub format
+- *HLS*: Use filtered out segment key info
+- *Track*: Don't modify lang when getting name
+- *Track*: Don't use fallback values "Zzzz"/"ZZ" for track name
+- *version*: The `__version__` variable forgot to be updated
+
+### Changes
+
+- Move dl command's download_track() to Track.download()
+- *dl*: Remove unused `get_profiles()` method
+- *DASH*: Move data values from track url to track data property
+- *DASH*: Change how Video FPS is gotten to remove FutureWarning log
+- *Track*: Add type checks, improve typing
+- *Track*: Remove swap() method and it's uses
+- *Track*: Remove unused DRM enum
+- *Track*: Rename Descriptor's M3U & MPD to HLS & DASH
+- *Track*: Remove unnecessary bool casting
+- *Track*: Move the path class instance variable with the rest
+- *Track*: Return new path on move(), raise exceptions on errors
+- *Track*: Move delete and move methods near start of Class
+- *Track*: Rename extra to data, enforce type as dict
+
+### Builds
+
+- Explicitly use marisa-trie==1.1.0 for Python 3.12 wheels
 
 ## [3.0.0] - 2024-03-01
 
@@ -643,6 +814,12 @@ This release brings a huge change to the fundamentals of Devine's logging, UI, a
 
 Initial public release under the name Devine.
 
+[3.3.3]: https://github.com/devine-dl/devine/releases/tag/v3.3.3
+[3.3.2]: https://github.com/devine-dl/devine/releases/tag/v3.3.2
+[3.3.1]: https://github.com/devine-dl/devine/releases/tag/v3.3.1
+[3.3.0]: https://github.com/devine-dl/devine/releases/tag/v3.3.0
+[3.2.0]: https://github.com/devine-dl/devine/releases/tag/v3.2.0
+[3.1.0]: https://github.com/devine-dl/devine/releases/tag/v3.1.0
 [3.0.0]: https://github.com/devine-dl/devine/releases/tag/v3.0.0
 [2.2.0]: https://github.com/devine-dl/devine/releases/tag/v2.2.0
 [2.1.0]: https://github.com/devine-dl/devine/releases/tag/v2.1.0

CONFIG.md

@@ -122,6 +122,7 @@ The following directories are available and may be overridden,
 - `commands` - CLI Command Classes.
 - `services` - Service Classes.
 - `vaults` - Vault Classes.
+- `fonts` - Font files (ttf or otf).
 - `downloads` - Downloads.
 - `temp` - Temporary files or conversions during download.
 - `cache` - Expiring data like Authorization tokens, or other misc data.
@@ -152,6 +153,13 @@ For example to set the default primary language to download to German,
 lang: de
 ```
 
+to set how many tracks to download concurrently to 4 and download threads to 16,
+
+```yaml
+downloads: 4
+workers: 16
+```
+
 to set `--bitrate=CVBR` for the AMZN service,
 
 ```yaml
@@ -160,7 +168,7 @@ AMZN:
   bitrate: CVBR
 ```
 
-or to change the output subtitle format from the default (SubRip SRT) to WebVTT,
+or to change the output subtitle format from the default (original format) to WebVTT,
 
 ```yaml
 sub_format: vtt
@@ -281,27 +289,30 @@ together.
 
 ## proxy_providers (dict)
 
-Enable external proxy provider services.
+Enable external proxy provider services. These proxies will be used automatically where needed as defined by the
+Service's GEOFENCE class property, but can also be explicitly used with `--proxy`. You can specify which provider
+to use by prefixing it with the provider key name, e.g., `--proxy basic:de` or `--proxy nordvpn:de`. Some providers
+support specific query formats for selecting a country/server.
 
-### basic (list\[dict])
+### basic (dict[str, str|list])
 
 Define a mapping of country to proxy to use where required.  
 The keys are region Alpha 2 Country Codes. Alpha 2 Country Codes are `[a-z]{2}` codes, e.g., `us`, `gb`, and `jp`.  
 Don't get this mixed up with language codes like `en` vs. `gb`, or `ja` vs. `jp`.
 
-Do note that each key's value is not a string but a list or sequence.
+Do note that each key's value can be a list of strings, or a string. For example,
-It will randomly choose which entry to use.
-
-For example,
 
 ```yaml
 us:
   - "http://john%40email.tld:password123@proxy-us.domain.tld:8080"
   - "http://jane%40email.tld:password456@proxy-us.domain2.tld:8080"
-de:
+de: "https://127.0.0.1:8080"
-  - "http://127.0.0.1:8888"
 ```
 
+Note that if multiple proxies are defined for a region, then by default one will be randomly chosen.
+You can choose a specific one by specifying it's number, e.g., `--proxy basic:us2` will choose the
+second proxy of the US list.
+
 ### nordvpn (dict)
 
 Set your NordVPN Service credentials with `username` and `password` keys to automate the use of NordVPN as a Proxy

README.md

@@ -341,6 +341,10 @@ Please refrain from spam or asking for questions that infringe upon a Service's
 <a href="https://github.com/varyg1001"><img src="https://images.weserv.nl/?url=avatars.githubusercontent.com/u/88599103?v=4&h=25&w=25&fit=cover&mask=circle&maxage=7d" alt="varyg1001"/></a>
 <a href="https://github.com/Hollander-1908"><img src="https://images.weserv.nl/?url=avatars.githubusercontent.com/u/93162595?v=4&h=25&w=25&fit=cover&mask=circle&maxage=7d" alt="Hollander-1908"/></a>
 <a href="https://github.com/Shivelight"><img src="https://images.weserv.nl/?url=avatars.githubusercontent.com/u/20620780?v=4&h=25&w=25&fit=cover&mask=circle&maxage=7d" alt="Shivelight"/></a>
+<a href="https://github.com/knowhere01"><img src="https://images.weserv.nl/?url=avatars.githubusercontent.com/u/113712042?v=4&h=25&w=25&fit=cover&mask=circle&maxage=7d" alt="knowhere01"/></a>
+<a href="https://github.com/retouching"><img src="https://images.weserv.nl/?url=avatars.githubusercontent.com/u/33735357?v=4&h=25&w=25&fit=cover&mask=circle&maxage=7d" alt="retouching"/></a>
+<a href="https://github.com/pandamoon21"><img src="https://images.weserv.nl/?url=avatars.githubusercontent.com/u/33972938?v=4&h=25&w=25&fit=cover&mask=circle&maxage=7d" alt="pandamoon21"/></a>
+<a href="https://github.com/adbbbb"><img src="https://images.weserv.nl/?url=avatars.githubusercontent.com/u/56319336?v=4&h=25&w=25&fit=cover&mask=circle&maxage=7d" alt="adbbbb"/></a>
 
 ## Licensing
 

cliff.toml

@@ -0,0 +1,71 @@
+# git-cliff ~ default configuration file
+# https://git-cliff.org/docs/configuration
+
+[changelog]
+header = """
+# Changelog\n
+All notable changes to this project will be documented in this file.
+
+This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+Versions [3.0.0] and older use a format based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+but versions thereafter use a custom changelog format using [git-cliff](https://git-cliff.org).\n
+"""
+body = """
+{% if version -%}
+    ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }}
+{% else -%}
+    ## [Unreleased]
+{% endif -%}
+{% for group, commits in commits | group_by(attribute="group") %}
+    ### {{ group | striptags | trim | upper_first }}
+    {% for commit in commits %}
+        - {% if commit.scope %}*{{ commit.scope }}*: {% endif %}\
+            {% if commit.breaking %}[**breaking**] {% endif %}\
+            {{ commit.message | upper_first }}\
+    {% endfor %}
+{% endfor %}\n
+"""
+footer = """
+{% for release in releases -%}
+    {% if release.version -%}
+        {% if release.previous.version -%}
+            [{{ release.version | trim_start_matches(pat="v") }}]: \
+                https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}\
+                    /compare/{{ release.previous.version }}..{{ release.version }}
+        {% endif -%}
+    {% else -%}
+        [unreleased]: https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}\
+            /compare/{{ release.previous.version }}..HEAD
+    {% endif -%}
+{% endfor %}
+"""
+trim = true
+postprocessors = [
+  # { pattern = '<REPO>', replace = "https://github.com/orhun/git-cliff" }, # replace repository URL
+]
+
+[git]
+conventional_commits = true
+filter_unconventional = true
+split_commits = false
+commit_preprocessors = []
+commit_parsers = [
+  { message = "^feat", group = "<!-- 0 -->Features" },
+  { message = "^fix|revert", group = "<!-- 1 -->Bug Fixes" },
+  { message = "^docs", group = "<!-- 2 -->Documentation" },
+  { message = "^style", skip = true },
+  { message = "^refactor", group = "<!-- 3 -->Changes" },
+  { message = "^perf", group = "<!-- 4 -->Performance Improvements" },
+  { message = "^test", skip = true },
+  { message = "^build", group = "<!-- 5 -->Builds" },
+  { message = "^ci", skip = true },
+  { message = "^chore", skip = true },
+]
+protect_breaking_commits = false
+filter_commits = false
+# tag_pattern = "v[0-9].*"
+# skip_tags = ""
+# ignore_tags = ""
+topo_order = false
+sort_commits = "oldest"

devine/commands/cfg.py

@@ -5,7 +5,7 @@ import sys
 import click
 from ruamel.yaml import YAML
 
-from devine.core.config import config
+from devine.core.config import config, get_config_path
 from devine.core.constants import context_settings
 
 
@@ -36,15 +36,15 @@ def cfg(ctx: click.Context, key: str, value: str, unset: bool, list_: bool) -> N
 
     log = logging.getLogger("cfg")
 
-    config_path = config.directories.user_configs / config.filenames.root_config
-
     yaml, data = YAML(), None
     yaml.default_flow_style = False
-    if config_path.is_file():
+
+    config_path = get_config_path() or config.directories.user_configs / config.filenames.root_config
+    if config_path.exists():
         data = yaml.load(config_path)
 
     if not data:
-        log.warning(f"{config_path} has no configuration data, yet")
+        log.warning("No config file was found or it has no data, yet")
         # yaml.load() returns `None` if the input data is blank instead of a usable object
         # force a usable object by making one and removing the only item within it
         data = yaml.load("""__TEMP__: null""")

devine/commands/dl.py

@@ -14,7 +14,7 @@ from concurrent.futures import ThreadPoolExecutor
 from copy import deepcopy
 from functools import partial
 from http.cookiejar import CookieJar, MozillaCookieJar
-from itertools import zip_longest
+from itertools import product
 from pathlib import Path
 from threading import Lock
 from typing import Any, Callable, Optional
@@ -32,34 +32,35 @@ from rich.console import Group
 from rich.live import Live
 from rich.padding import Padding
 from rich.panel import Panel
-from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeRemainingColumn
+from rich.progress import BarColumn, Progress, SpinnerColumn, TaskID, TextColumn, TimeRemainingColumn
 from rich.rule import Rule
 from rich.table import Table
 from rich.text import Text
 from rich.tree import Tree
 
+from devine.core import binaries
 from devine.core.config import config
 from devine.core.console import console
-from devine.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack, context_settings
+from devine.core.constants import DOWNLOAD_LICENCE_ONLY, AnyTrack, context_settings
 from devine.core.credential import Credential
-from devine.core.downloaders import downloader
 from devine.core.drm import DRM_T, Widevine
-from devine.core.manifests import DASH, HLS
+from devine.core.events import events
 from devine.core.proxies import Basic, Hola, NordVPN
 from devine.core.service import Service
 from devine.core.services import Services
 from devine.core.titles import Movie, Song, Title_T
 from devine.core.titles.episode import Episode
 from devine.core.tracks import Audio, Subtitle, Tracks, Video
-from devine.core.utilities import get_binary_path, is_close_match, time_elapsed_since, try_ensure_utf8
+from devine.core.tracks.attachment import Attachment
-from devine.core.utils.click_types import LANGUAGE_RANGE, QUALITY_LIST, SEASON_RANGE, ContextData
+from devine.core.utilities import get_system_fonts, is_close_match, time_elapsed_since
+from devine.core.utils.click_types import LANGUAGE_RANGE, QUALITY_LIST, SEASON_RANGE, ContextData, MultipleChoice
 from devine.core.utils.collections import merge_dict
 from devine.core.utils.subprocess import ffprobe
 from devine.core.vaults import Vaults
 
 
 class dl:
-    @click.group(
+    @click.command(
         short_help="Download, Decrypt, and Mux tracks for titles from a Service.",
         cls=Services,
         context_settings=dict(
@@ -72,8 +73,8 @@ class dl:
     @click.option("-q", "--quality", type=QUALITY_LIST, default=[],
                   help="Download Resolution(s), defaults to the best available resolution.")
     @click.option("-v", "--vcodec", type=click.Choice(Video.Codec, case_sensitive=False),
-                  default=Video.Codec.AVC,
+                  default=None,
-                  help="Video Codec to download, defaults to H.264.")
+                  help="Video Codec to download, defaults to any codec.")
     @click.option("-a", "--acodec", type=click.Choice(Audio.Codec, case_sensitive=False),
                   default=None,
                   help="Audio Codec to download, defaults to any codec.")
@@ -83,9 +84,9 @@ class dl:
     @click.option("-ab", "--abitrate", type=int,
                   default=None,
                   help="Audio Bitrate to download (in kbps), defaults to highest available.")
-    @click.option("-r", "--range", "range_", type=click.Choice(Video.Range, case_sensitive=False),
+    @click.option("-r", "--range", "range_", type=MultipleChoice(Video.Range, case_sensitive=False),
-                  default=Video.Range.SDR,
+                  default=[Video.Range.SDR],
-                  help="Video Color Range, defaults to SDR.")
+                  help="Video Color Range(s) to download, defaults to SDR.")
     @click.option("-c", "--channels", type=float,
                   default=None,
                   help="Audio Channel(s) to download. Matches sub-channel layouts like 5.1 with 6.0 implicitly.")
@@ -102,7 +103,7 @@ class dl:
     @click.option("--tag", type=str, default=None,
                   help="Set the Group Tag to be used, overriding the one in config if any.")
     @click.option("--sub-format", type=click.Choice(Subtitle.Codec, case_sensitive=False),
-                  default=Subtitle.Codec.SubRip,
+                  default=None,
                   help="Set Output Subtitle Format, only converting if necessary.")
     @click.option("-V", "--video-only", is_flag=True, default=False,
                   help="Only download video tracks.")
@@ -131,8 +132,10 @@ class dl:
                   help="Disable folder creation for TV Shows.")
     @click.option("--no-source", is_flag=True, default=False,
                   help="Disable the source tag from the output file name and path.")
-    @click.option("--workers", type=int, default=1,
+    @click.option("--workers", type=int, default=None,
-                  help="Max concurrent workers to use throughout the code, particularly downloads.")
+                  help="Max workers/threads to download with per-track. Default depends on the downloader.")
+    @click.option("--downloads", type=int, default=1,
+                  help="Amount of tracks to download concurrently.")
     @click.pass_context
     def cli(ctx: click.Context, **kwargs: Any) -> dl:
         return dl(ctx, **kwargs)
@@ -175,9 +178,10 @@ class dl:
             except ValueError as e:
                 self.log.error(f"Failed to load Widevine CDM, {e}")
                 sys.exit(1)
-            self.log.info(
+            if self.cdm:
-                f"Loaded {self.cdm.__class__.__name__} Widevine CDM: {self.cdm.system_id} (L{self.cdm.security_level})"
+                self.log.info(
-            )
+                    f"Loaded {self.cdm.__class__.__name__} Widevine CDM: {self.cdm.system_id} (L{self.cdm.security_level})"
+                )
 
         with console.status("Loading Key Vaults...", spinner="dots"):
             self.vaults = Vaults(self.service)
@@ -196,7 +200,7 @@ class dl:
                     self.proxy_providers.append(Basic(**config.proxy_providers["basic"]))
                 if config.proxy_providers.get("nordvpn"):
                     self.proxy_providers.append(NordVPN(**config.proxy_providers["nordvpn"]))
-                if get_binary_path("hola-proxy"):
+                if binaries.HolaProxy:
                     self.proxy_providers.append(Hola())
                 for proxy_provider in self.proxy_providers:
                     self.log.info(f"Loaded {proxy_provider.__class__.__name__}: {proxy_provider}")
@@ -252,17 +256,17 @@ class dl:
         self,
         service: Service,
         quality: list[int],
-        vcodec: Video.Codec,
+        vcodec: Optional[Video.Codec],
         acodec: Optional[Audio.Codec],
         vbitrate: int,
         abitrate: int,
-        range_: Video.Range,
+        range_: list[Video.Range],
         channels: float,
         wanted: list[str],
         lang: list[str],
         v_lang: list[str],
         s_lang: list[str],
-        sub_format: Subtitle.Codec,
+        sub_format: Optional[Subtitle.Codec],
         video_only: bool,
         audio_only: bool,
         subs_only: bool,
@@ -275,7 +279,8 @@ class dl:
         no_proxy: bool,
         no_folder: bool,
         no_source: bool,
-        workers: int,
+        workers: Optional[int],
+        downloads: int,
         *_: Any,
         **__: Any
     ) -> None:
@@ -325,6 +330,14 @@ class dl:
                 with console.status(f"Delaying by {delay} seconds..."):
                     time.sleep(delay)
 
+            with console.status("Subscribing to events...", spinner="dots"):
+                events.reset()
+                events.subscribe(events.Types.SEGMENT_DOWNLOADED, service.on_segment_downloaded)
+                events.subscribe(events.Types.TRACK_DOWNLOADED, service.on_track_downloaded)
+                events.subscribe(events.Types.TRACK_DECRYPTED, service.on_track_decrypted)
+                events.subscribe(events.Types.TRACK_REPACKED, service.on_track_repacked)
+                events.subscribe(events.Types.TRACK_MULTIPLEX, service.on_track_multiplex)
+
             with console.status("Getting tracks...", spinner="dots"):
                 title.tracks.add(service.get_tracks(title), warn_only=True)
                 title.tracks.chapters = service.get_chapters(title)
@@ -340,8 +353,13 @@ class dl:
                     non_sdh_sub = deepcopy(subtitle)
                     non_sdh_sub.id += "_stripped"
                     non_sdh_sub.sdh = False
-                    non_sdh_sub.OnMultiplex = lambda: non_sdh_sub.strip_hearing_impaired()
                     title.tracks.add(non_sdh_sub)
+                    events.subscribe(
+                        events.Types.TRACK_MULTIPLEX,
+                        lambda track: (
+                            track.strip_hearing_impaired()
+                        ) if track.id == non_sdh_sub.id else None
+                    )
 
             with console.status("Sorting tracks by language and bitrate...", spinner="dots"):
                 title.tracks.sort_videos(by_language=v_lang or lang)
@@ -359,15 +377,18 @@ class dl:
             with console.status("Selecting tracks...", spinner="dots"):
                 if isinstance(title, (Movie, Episode)):
                     # filter video tracks
-                    title.tracks.select_video(lambda x: x.codec == vcodec)
+                    if vcodec:
-                    if not title.tracks.videos:
+                        title.tracks.select_video(lambda x: x.codec == vcodec)
-                        self.log.error(f"There's no {vcodec.name} Video Track...")
+                        if not title.tracks.videos:
-                        sys.exit(1)
+                            self.log.error(f"There's no {vcodec.name} Video Track...")
+                            sys.exit(1)
 
-                    title.tracks.select_video(lambda x: x.range == range_)
+                    if range_:
-                    if not title.tracks.videos:
+                        title.tracks.select_video(lambda x: x.range in range_)
-                        self.log.error(f"There's no {range_.name} Video Track...")
+                        for color_range in range_:
-                        sys.exit(1)
+                            if not any(x.range == color_range for x in title.tracks.videos):
+                                self.log.error(f"There's no {color_range.name} Video Tracks...")
+                                sys.exit(1)
 
                     if vbitrate:
                         title.tracks.select_video(lambda x: x.bitrate and x.bitrate // 1000 == vbitrate)
@@ -383,7 +404,7 @@ class dl:
                             sys.exit(1)
 
                     if quality:
-                        title.tracks.by_resolutions(quality, per_resolution=1)
+                        title.tracks.by_resolutions(quality)
                         missing_resolutions = []
                         for resolution in quality:
                             if any(video.height == resolution for video in title.tracks.videos):
@@ -399,8 +420,27 @@ class dl:
                             plural = "s" if len(missing_resolutions) > 1 else ""
                             self.log.error(f"There's no {res_list} Video Track{plural}...")
                             sys.exit(1)
-                    else:
+
-                        title.tracks.videos = [title.tracks.videos[0]]
+                    # choose best track by range and quality
+                    title.tracks.videos = [
+                        track
+                        for resolution, color_range in product(
+                            quality or [None],
+                            range_ or [None]
+                        )
+                        for track in [next(
+                            t
+                            for t in title.tracks.videos
+                            if (not resolution and not color_range) or
+                            (
+                                (not resolution or (
+                                   (t.height == resolution) or
+                                   (int(t.width * (9 / 16)) == resolution)
+                                ))
+                                and (not color_range or t.range == color_range)
+                            )
+                        )]
+                    ]
 
                     # filter subtitle tracks
                     if s_lang and "all" not in s_lang:
@@ -467,12 +507,11 @@ class dl:
                     console=console,
                     refresh_per_second=5
                 ):
-                    with ThreadPoolExecutor(workers) as pool:
+                    with ThreadPoolExecutor(downloads) as pool:
                         for download in futures.as_completed((
                             pool.submit(
-                                self.download_track,
+                                track.download,
-                                service=service,
+                                session=service.session,
-                                track=track,
                                 prepare_drm=partial(
                                     partial(
                                         self.prepare_drm,
@@ -494,6 +533,7 @@ class dl:
                                     vaults_only=vaults_only,
                                     export=export
                                 ),
+                                max_workers=workers,
                                 progress=tracks_progress_callables[i]
                             )
                             for i, track in enumerate(title.tracks)
@@ -508,14 +548,17 @@ class dl:
             except Exception as e:  # noqa
                 error_messages = [
                     ":x: Download Failed...",
-                    "   One of the download workers had an error!",
-                    "   See the error trace above for more information."
                 ]
-                if isinstance(e, subprocess.CalledProcessError):
+                if isinstance(e, EnvironmentError):
-                    # ignore process exceptions as proper error logs are already shown
+                    error_messages.append(f"   {e}")
-                    error_messages.append(f"   Process exit code: {e.returncode}")
                 else:
-                    console.print_exception()
+                    error_messages.append("   An unexpected error occurred in one of the download workers.",)
+                    if hasattr(e, "returncode"):
+                        error_messages.append(f"   Binary call failed, Process exit code: {e.returncode}")
+                    error_messages.append("   See the error trace above for more information.")
+                    if isinstance(e, subprocess.CalledProcessError):
+                        # CalledProcessError already lists the exception trace
+                        console.print_exception()
                 console.print(Padding(
                     Group(*error_messages),
                     (1, 5)
@@ -572,10 +615,46 @@ class dl:
                             break
                     video_track_n += 1
 
-                with console.status(f"Converting Subtitles to {sub_format.name}..."):
+                with console.status("Converting Subtitles..."):
                     for subtitle in title.tracks.subtitles:
-                        if subtitle.codec != sub_format:
+                        if sub_format:
-                            subtitle.convert(sub_format)
+                            if subtitle.codec != sub_format:
+                                subtitle.convert(sub_format)
+                        elif subtitle.codec == Subtitle.Codec.TimedTextMarkupLang:
+                            # MKV does not support TTML, VTT is the next best option
+                            subtitle.convert(Subtitle.Codec.WebVTT)
+
+                with console.status("Checking Subtitles for Fonts..."):
+                    font_names = []
+                    for subtitle in title.tracks.subtitles:
+                        if subtitle.codec == Subtitle.Codec.SubStationAlphav4:
+                            for line in subtitle.path.read_text("utf8").splitlines():
+                                if line.startswith("Style: "):
+                                    font_names.append(line.removesuffix("Style: ").split(",")[1])
+
+                    font_count = 0
+                    system_fonts = get_system_fonts()
+                    for font_name in set(font_names):
+                        family_dir = Path(config.directories.fonts, font_name)
+                        fonts_from_system = [
+                            file
+                            for name, file in system_fonts.items()
+                            if name.startswith(font_name)
+                        ]
+                        if family_dir.exists():
+                            fonts = family_dir.glob("*.*tf")
+                            for font in fonts:
+                                title.tracks.add(Attachment(font, f"{font_name} ({font.stem})"))
+                                font_count += 1
+                        elif fonts_from_system:
+                            for font in fonts_from_system:
+                                title.tracks.add(Attachment(font, f"{font_name} ({font.stem})"))
+                                font_count += 1
+                        else:
+                            self.log.warning(f"Subtitle uses font [text2]{font_name}[/] but it could not be found...")
+
+                    if font_count:
+                        self.log.info(f"Attached {font_count} fonts for the Subtitles")
 
                 with console.status("Repackaging tracks with FFMPEG..."):
                     has_repacked = False
@@ -583,8 +662,7 @@ class dl:
                         if track.needs_repack:
                             track.repackage()
                             has_repacked = True
-                            if callable(track.OnRepacked):
+                            events.emit(events.Types.TRACK_REPACKED, track=track)
-                                track.OnRepacked()
                     if has_repacked:
                         # we don't want to fill up the log with "Repacked x track"
                         self.log.info("Repacked one or more tracks with FFMPEG")
@@ -600,35 +678,48 @@ class dl:
                         TimeRemainingColumn(compact=True, elapsed_when_finished=True),
                         console=console
                     )
-                    multi_jobs = len(title.tracks.videos) > 1
+
-                    tasks = [
+                    multiplex_tasks: list[tuple[TaskID, Tracks]] = []
-                        progress.add_task(
+                    for video_track in title.tracks.videos or [None]:
-                            f"Multiplexing{f' {x.height}p' if multi_jobs else ''}...",
+                        task_description = "Multiplexing"
-                            total=None,
+                        if video_track:
-                            start=False
+                            if len(quality) > 1:
-                        )
+                                task_description += f" {video_track.height}p"
-                        for x in title.tracks.videos or [None]
+                            if len(range_) > 1:
-                    ]
+                                task_description += f" {video_track.range.name}"
+
+                        task_id = progress.add_task(f"{task_description}...", total=None, start=False)
+
+                        task_tracks = Tracks(title.tracks) + title.tracks.chapters + title.tracks.attachments
+                        if video_track:
+                            task_tracks.videos = [video_track]
+
+                        multiplex_tasks.append((task_id, task_tracks))
+
                     with Live(
                         Padding(progress, (0, 5, 1, 5)),
                         console=console
                     ):
-                        for task, video_track in zip_longest(tasks, title.tracks.videos, fillvalue=None):
+                        for task_id, task_tracks in multiplex_tasks:
-                            if video_track:
+                            progress.start_task(task_id)  # TODO: Needed?
-                                title.tracks.videos = [video_track]
+                            muxed_path, return_code, errors = task_tracks.mux(
-                            progress.start_task(task)  # TODO: Needed?
-                            muxed_path, return_code = title.tracks.mux(
                                 str(title),
-                                progress=partial(progress.update, task_id=task),
+                                progress=partial(progress.update, task_id=task_id),
                                 delete=False
                             )
                             muxed_paths.append(muxed_path)
-                            if return_code == 1:
+                            if return_code >= 2:
-                                self.log.warning("mkvmerge had at least one warning, will continue anyway...")
+                                self.log.error(f"Failed to Mux video to Matroska file ({return_code}):")
-                            elif return_code >= 2:
+                            elif return_code == 1 or errors:
-                                self.log.error(f"Failed to Mux video to Matroska file ({return_code})")
+                                self.log.warning("mkvmerge had at least one warning or error, continuing anyway...")
+                            for line in errors:
+                                if line.startswith("#GUI#error"):
+                                    self.log.error(line)
+                                else:
+                                    self.log.warning(line)
+                            if return_code >= 2:
                                 sys.exit(1)
-                            if video_track:
+                            for video_track in task_tracks.videos:
                                 video_track.delete()
                         for track in title.tracks:
                             track.delete()
@@ -788,168 +879,6 @@ class dl:
                     keys[str(title)][str(track)].update(drm.content_keys)
                     export.write_text(jsonpickle.dumps(keys, indent=4), encoding="utf8")
 
-    def download_track(
-        self,
-        service: Service,
-        track: AnyTrack,
-        prepare_drm: Callable,
-        progress: partial
-    ):
-        if DOWNLOAD_LICENCE_ONLY.is_set():
-            progress(downloaded="[yellow]SKIPPING")
-
-        if DOWNLOAD_CANCELLED.is_set():
-            progress(downloaded="[yellow]CANCELLED")
-            return
-
-        proxy = next(iter(service.session.proxies.values()), None)
-
-        save_path = config.directories.temp / f"{track.__class__.__name__}_{track.id}.mp4"
-        if isinstance(track, Subtitle):
-            save_path = save_path.with_suffix(f".{track.codec.extension}")
-
-        if track.descriptor != track.Descriptor.URL:
-            save_dir = save_path.with_name(save_path.name + "_segments")
-        else:
-            save_dir = save_path.parent
-
-        def cleanup():
-            # track file (e.g., "foo.mp4")
-            save_path.unlink(missing_ok=True)
-            # aria2c control file (e.g., "foo.mp4.aria2")
-            save_path.with_suffix(f"{save_path.suffix}.aria2").unlink(missing_ok=True)
-            if save_dir.exists() and save_dir.name.endswith("_segments"):
-                shutil.rmtree(save_dir)
-
-        if not DOWNLOAD_LICENCE_ONLY.is_set():
-            if config.directories.temp.is_file():
-                self.log.error(f"Temp Directory '{config.directories.temp}' must be a Directory, not a file")
-                sys.exit(1)
-
-            config.directories.temp.mkdir(parents=True, exist_ok=True)
-
-            # Delete any pre-existing temp files matching this track.
-            # We can't re-use or continue downloading these tracks as they do not use a
-            # lock file. Or at least the majority don't. Even if they did I've encountered
-            # corruptions caused by sudden interruptions to the lock file.
-            cleanup()
-
-        try:
-            if track.descriptor == track.Descriptor.M3U:
-                HLS.download_track(
-                    track=track,
-                    save_path=save_path,
-                    save_dir=save_dir,
-                    progress=progress,
-                    session=service.session,
-                    proxy=proxy,
-                    license_widevine=prepare_drm
-                )
-            elif track.descriptor == track.Descriptor.MPD:
-                DASH.download_track(
-                    track=track,
-                    save_path=save_path,
-                    save_dir=save_dir,
-                    progress=progress,
-                    session=service.session,
-                    proxy=proxy,
-                    license_widevine=prepare_drm
-                )
-            elif track.descriptor == track.Descriptor.URL:
-                try:
-                    if not track.drm and isinstance(track, (Video, Audio)):
-                        # the service might not have explicitly defined the `drm` property
-                        # try find widevine DRM information from the init data of URL
-                        try:
-                            track.drm = [Widevine.from_track(track, service.session)]
-                        except Widevine.Exceptions.PSSHNotFound:
-                            # it might not have Widevine DRM, or might not have found the PSSH
-                            self.log.warning("No Widevine PSSH was found for this track, is it DRM free?")
-
-                    if track.drm:
-                        track_kid = track.get_key_id(session=service.session)
-                        drm = track.drm[0]  # just use the first supported DRM system for now
-                        if isinstance(drm, Widevine):
-                            # license and grab content keys
-                            if not prepare_drm:
-                                raise ValueError("prepare_drm func must be supplied to use Widevine DRM")
-                            progress(downloaded="LICENSING")
-                            prepare_drm(drm, track_kid=track_kid)
-                            progress(downloaded="[yellow]LICENSED")
-                    else:
-                        drm = None
-
-                    if DOWNLOAD_LICENCE_ONLY.is_set():
-                        progress(downloaded="[yellow]SKIPPED")
-                    else:
-                        for status_update in downloader(
-                            urls=track.url,
-                            output_dir=save_path.parent,
-                            filename=save_path.name,
-                            headers=service.session.headers,
-                            cookies=service.session.cookies,
-                            proxy=proxy
-                        ):
-                            file_downloaded = status_update.get("file_downloaded")
-                            if not file_downloaded:
-                                progress(**status_update)
-
-                        track.path = save_path
-                        if callable(track.OnDownloaded):
-                            track.OnDownloaded()
-
-                        if drm:
-                            progress(downloaded="Decrypting", completed=0, total=100)
-                            drm.decrypt(save_path)
-                            track.drm = None
-                            if callable(track.OnDecrypted):
-                                track.OnDecrypted(drm)
-                            progress(downloaded="Decrypted", completed=100)
-
-                        if isinstance(track, Subtitle) and \
-                           track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML):
-                            track_data = track.path.read_bytes()
-                            track_data = try_ensure_utf8(track_data)
-                            track_data = track_data.decode("utf8"). \
-                                replace("‎", html.unescape("‎")). \
-                                replace("‏", html.unescape("‏")). \
-                                encode("utf8")
-                            track.path.write_bytes(track_data)
-
-                        progress(downloaded="Downloaded")
-                except KeyboardInterrupt:
-                    DOWNLOAD_CANCELLED.set()
-                    progress(downloaded="[yellow]CANCELLED")
-                    raise
-                except Exception:
-                    DOWNLOAD_CANCELLED.set()
-                    progress(downloaded="[red]FAILED")
-                    raise
-        except (Exception, KeyboardInterrupt):
-            if not DOWNLOAD_LICENCE_ONLY.is_set():
-                cleanup()
-            raise
-
-        if DOWNLOAD_CANCELLED.is_set():
-            # we stopped during the download, let's exit
-            return
-
-        if not DOWNLOAD_LICENCE_ONLY.is_set():
-            if track.path.stat().st_size <= 3:  # Empty UTF-8 BOM == 3 bytes
-                raise IOError("Download failed, the downloaded file is empty.")
-
-    @staticmethod
-    def get_profile(service: str) -> Optional[str]:
-        """Get profile for Service from config."""
-        profile = config.profiles.get(service)
-        if profile is False:
-            return None  # auth-less service if `false` in config
-        if not profile:
-            profile = config.profiles.get("default")
-        if not profile:
-            raise ValueError(f"No profile has been defined for '{service}' in the config.")
-        return profile
-
     @staticmethod
     def get_cookie_path(service: str, profile: Optional[str]) -> Optional[Path]:
         """Get Service Cookie File Path for Profile."""
@@ -1008,21 +937,21 @@ class dl:
                 return Credential.loads(credentials)  # type: ignore
 
     @staticmethod
-    def get_cdm(service: str, profile: Optional[str] = None) -> WidevineCdm:
+    def get_cdm(service: str, profile: Optional[str] = None) -> Optional[WidevineCdm]:
         """
         Get CDM for a specified service (either Local or Remote CDM).
         Raises a ValueError if there's a problem getting a CDM.
         """
         cdm_name = config.cdm.get(service) or config.cdm.get("default")
         if not cdm_name:
-            raise ValueError("A CDM to use wasn't listed in the config")
+            return None
 
         if isinstance(cdm_name, dict):
             if not profile:
-                raise ValueError("CDM config is mapped for profiles, but no profile was chosen")
+                return None
             cdm_name = cdm_name.get(profile) or config.cdm.get("default")
             if not cdm_name:
-                raise ValueError(f"A CDM to use was not mapped for the profile {profile}")
+                return None
 
         cdm_api = next(iter(x for x in config.remote_cdm if x["name"] == cdm_name), None)
         if cdm_api:

devine/commands/env.py

@@ -1,10 +1,17 @@
 import logging
+import os
 import shutil
+import sys
+from pathlib import Path
 from typing import Optional
 
 import click
+from rich.padding import Padding
+from rich.table import Table
+from rich.tree import Tree
 
-from devine.core.config import config
+from devine.core.config import POSSIBLE_CONFIG_PATHS, config, config_path
+from devine.core.console import console
 from devine.core.constants import context_settings
 from devine.core.services import Services
 
@@ -18,13 +25,42 @@ def env() -> None:
 def info() -> None:
     """Displays information about the current environment."""
     log = logging.getLogger("env")
-    log.info(f"[Root Config]     : {config.directories.user_configs / config.filenames.root_config}")
+
-    log.info(f"[Cookies]         : {config.directories.cookies}")
+    if config_path:
-    log.info(f"[WVDs]            : {config.directories.wvds}")
+        log.info(f"Config loaded from {config_path}")
-    log.info(f"[Cache]           : {config.directories.cache}")
+    else:
-    log.info(f"[Logs]            : {config.directories.logs}")
+        tree = Tree("No config file found, you can use any of the following locations:")
-    log.info(f"[Temp Files]      : {config.directories.temp}")
+        for i, path in enumerate(POSSIBLE_CONFIG_PATHS, start=1):
-    log.info(f"[Downloads]       : {config.directories.downloads}")
+            tree.add(f"[repr.number]{i}.[/] [text2]{path.resolve()}[/]")
+        console.print(Padding(
+            tree,
+            (0, 5)
+        ))
+
+    table = Table(title="Directories", expand=True)
+    table.add_column("Name", no_wrap=True)
+    table.add_column("Path")
+
+    path_vars = {
+        x: Path(os.getenv(x))
+        for x in ("TEMP", "APPDATA", "LOCALAPPDATA", "USERPROFILE")
+        if sys.platform == "win32" and os.getenv(x)
+    }
+
+    for name in sorted(dir(config.directories)):
+        if name.startswith("__") or name == "app_dirs":
+            continue
+        path = getattr(config.directories, name).resolve()
+        for var, var_path in path_vars.items():
+            if path.is_relative_to(var_path):
+                path = rf"%{var}%\{path.relative_to(var_path)}"
+                break
+        table.add_row(name.title(), str(path))
+
+    console.print(Padding(
+        table,
+        (1, 5)
+    ))
 
 
 @env.group(name="clear", short_help="Clear an environment directory.", context_settings=context_settings)

devine/commands/search.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import logging
+import re
+import sys
+from typing import Any, Optional
+
+import click
+import yaml
+from rich.padding import Padding
+from rich.rule import Rule
+from rich.tree import Tree
+
+from devine.commands.dl import dl
+from devine.core import binaries
+from devine.core.config import config
+from devine.core.console import console
+from devine.core.constants import context_settings
+from devine.core.proxies import Basic, Hola, NordVPN
+from devine.core.service import Service
+from devine.core.services import Services
+from devine.core.utils.click_types import ContextData
+from devine.core.utils.collections import merge_dict
+
+
+@click.command(
+    short_help="Search for titles from a Service.",
+    cls=Services,
+    context_settings=dict(
+        **context_settings,
+        token_normalize_func=Services.get_tag
+    ))
+@click.option("-p", "--profile", type=str, default=None,
+              help="Profile to use for Credentials and Cookies (if available).")
+@click.option("--proxy", type=str, default=None,
+              help="Proxy URI to use. If a 2-letter country is provided, it will try get a proxy from the config.")
+@click.option("--no-proxy", is_flag=True, default=False,
+              help="Force disable all proxy use.")
+@click.pass_context
+def search(
+    ctx: click.Context,
+    no_proxy: bool,
+    profile: Optional[str] = None,
+    proxy: Optional[str] = None
+):
+    if not ctx.invoked_subcommand:
+        raise ValueError("A subcommand to invoke was not specified, the main code cannot continue.")
+
+    log = logging.getLogger("search")
+
+    service = Services.get_tag(ctx.invoked_subcommand)
+    profile = profile
+
+    if profile:
+        log.info(f"Using profile: '{profile}'")
+
+    with console.status("Loading Service Config...", spinner="dots"):
+        service_config_path = Services.get_path(service) / config.filenames.config
+        if service_config_path.exists():
+            service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8"))
+            log.info("Service Config loaded")
+        else:
+            service_config = {}
+        merge_dict(config.services.get(service), service_config)
+
+    proxy_providers = []
+    if no_proxy:
+        ctx.params["proxy"] = None
+    else:
+        with console.status("Loading Proxy Providers...", spinner="dots"):
+            if config.proxy_providers.get("basic"):
+                proxy_providers.append(Basic(**config.proxy_providers["basic"]))
+            if config.proxy_providers.get("nordvpn"):
+                proxy_providers.append(NordVPN(**config.proxy_providers["nordvpn"]))
+            if binaries.HolaProxy:
+                proxy_providers.append(Hola())
+            for proxy_provider in proxy_providers:
+                log.info(f"Loaded {proxy_provider.__class__.__name__}: {proxy_provider}")
+
+        if proxy:
+            requested_provider = None
+            if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE):
+                # requesting proxy from a specific proxy provider
+                requested_provider, proxy = proxy.split(":", maxsplit=1)
+            if re.match(r"^[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE):
+                proxy = proxy.lower()
+                with console.status(f"Getting a Proxy to {proxy}...", spinner="dots"):
+                    if requested_provider:
+                        proxy_provider = next((
+                            x
+                            for x in proxy_providers
+                            if x.__class__.__name__.lower() == requested_provider
+                        ), None)
+                        if not proxy_provider:
+                            log.error(f"The proxy provider '{requested_provider}' was not recognised.")
+                            sys.exit(1)
+                        proxy_uri = proxy_provider.get_proxy(proxy)
+                        if not proxy_uri:
+                            log.error(f"The proxy provider {requested_provider} had no proxy for {proxy}")
+                            sys.exit(1)
+                        proxy = ctx.params["proxy"] = proxy_uri
+                        log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}")
+                    else:
+                        for proxy_provider in proxy_providers:
+                            proxy_uri = proxy_provider.get_proxy(proxy)
+                            if proxy_uri:
+                                proxy = ctx.params["proxy"] = proxy_uri
+                                log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}")
+                                break
+            else:
+                log.info(f"Using explicit Proxy: {proxy}")
+
+    ctx.obj = ContextData(
+        config=service_config,
+        cdm=None,
+        proxy_providers=proxy_providers,
+        profile=profile
+    )
+
+
+@search.result_callback()
+def result(service: Service, profile: Optional[str] = None, **_: Any) -> None:
+    log = logging.getLogger("search")
+
+    service_tag = service.__class__.__name__
+
+    with console.status("Authenticating with Service...", spinner="dots"):
+        cookies = dl.get_cookie_jar(service_tag, profile)
+        credential = dl.get_credentials(service_tag, profile)
+        service.authenticate(cookies, credential)
+        if cookies or credential:
+            log.info("Authenticated with Service")
+
+    search_results = Tree("Search Results", hide_root=True)
+    with console.status("Searching...", spinner="dots"):
+        for result in service.search():
+            result_text = f"[bold text]{result.title}[/]"
+            if result.url:
+                result_text = f"[link={result.url}]{result_text}[/link]"
+            if result.label:
+                result_text += f"  [pink]{result.label}[/]"
+            if result.description:
+                result_text += f"\n[text2]{result.description}[/]"
+            result_text += f"\n[bright_black]id: {result.id}[/]"
+            search_results.add(result_text + "\n")
+
+    # update cookies
+    cookie_file = dl.get_cookie_path(service_tag, profile)
+    if cookie_file:
+        dl.save_cookies(cookie_file, service.session.cookies)
+
+    console.print(Padding(
+        Rule(f"[rule.text]{len(search_results.children)} Search Results"),
+        (1, 2)
+    ))
+
+    if search_results.children:
+        console.print(Padding(
+            search_results,
+            (0, 5)
+        ))
+    else:
+        console.print(Padding(
+            "[bold text]No matches[/]\n[bright_black]Please check spelling and search again....[/]",
+            (0, 5)
+        ))

devine/commands/serve.py

@@ -2,9 +2,9 @@ import subprocess
 
 import click
 
+from devine.core import binaries
 from devine.core.config import config
 from devine.core.constants import context_settings
-from devine.core.utilities import get_binary_path
 
 
 @click.command(
@@ -29,11 +29,10 @@ def serve(host: str, port: int, caddy: bool) -> None:
     from pywidevine import serve
 
     if caddy:
-        executable = get_binary_path("caddy")
+        if not binaries.Caddy:
-        if not executable:
             raise click.ClickException("Caddy executable \"caddy\" not found but is required for --caddy.")
         caddy_p = subprocess.Popen([
-            executable,
+            binaries.Caddy,
             "run",
             "--config", str(config.directories.user_configs / "Caddyfile")
         ])

devine/commands/util.py

@@ -4,8 +4,8 @@ from pathlib import Path
 import click
 from pymediainfo import MediaInfo
 
+from devine.core import binaries
 from devine.core.constants import context_settings
-from devine.core.utilities import get_binary_path
 
 
 @click.group(short_help="Various helper scripts and programs.", context_settings=context_settings)
@@ -38,8 +38,7 @@ def crop(path: Path, aspect: str, letter: bool, offset: int, preview: bool) -> N
     as it may go from being 2px away from a perfect crop, to 20px over-cropping
     again due to sub-sampled chroma.
     """
-    executable = get_binary_path("ffmpeg")
+    if not binaries.FFMPEG:
-    if not executable:
         raise click.ClickException("FFmpeg executable \"ffmpeg\" not found but is required.")
 
     if path.is_dir():
@@ -87,7 +86,7 @@ def crop(path: Path, aspect: str, letter: bool, offset: int, preview: bool) -> N
             ]))))]
 
         ffmpeg_call = subprocess.Popen([
-            executable, "-y",
+            binaries.FFMPEG, "-y",
             "-i", str(video_path),
             "-map", "0:v:0",
             "-c", "copy",
@@ -95,7 +94,7 @@ def crop(path: Path, aspect: str, letter: bool, offset: int, preview: bool) -> N
         ] + out_path, stdout=subprocess.PIPE)
         try:
             if preview:
-                previewer = get_binary_path("mpv", "ffplay")
+                previewer = binaries.MPV or binaries.FFPlay
                 if not previewer:
                     raise click.ClickException("MPV/FFplay executables weren't found but are required for previewing.")
                 subprocess.Popen((previewer, "-"), stdin=ffmpeg_call.stdout)
@@ -120,8 +119,7 @@ def range_(path: Path, full: bool, preview: bool) -> None:
     then you're video may have the range set to the wrong value. Flip its range to the
     opposite value and see if that fixes it.
     """
-    executable = get_binary_path("ffmpeg")
+    if not binaries.FFMPEG:
-    if not executable:
         raise click.ClickException("FFmpeg executable \"ffmpeg\" not found but is required.")
 
     if path.is_dir():
@@ -157,7 +155,7 @@ def range_(path: Path, full: bool, preview: bool) -> None:
             ]))))]
 
         ffmpeg_call = subprocess.Popen([
-            executable, "-y",
+            binaries.FFMPEG, "-y",
             "-i", str(video_path),
             "-map", "0:v:0",
             "-c", "copy",
@@ -165,7 +163,7 @@ def range_(path: Path, full: bool, preview: bool) -> None:
         ] + out_path, stdout=subprocess.PIPE)
         try:
             if preview:
-                previewer = get_binary_path("mpv", "ffplay")
+                previewer = binaries.MPV or binaries.FFPlay
                 if not previewer:
                     raise click.ClickException("MPV/FFplay executables weren't found but are required for previewing.")
                 subprocess.Popen((previewer, "-"), stdin=ffmpeg_call.stdout)
@@ -188,8 +186,7 @@ def test(path: Path, map_: str) -> None:
     You may choose specific streams using the -m/--map parameter. E.g.,
     '0:v:0' to test the first video stream, or '0:a' to test all audio streams.
     """
-    executable = get_binary_path("ffmpeg")
+    if not binaries.FFMPEG:
-    if not executable:
         raise click.ClickException("FFmpeg executable \"ffmpeg\" not found but is required.")
 
     if path.is_dir():
@@ -199,7 +196,7 @@ def test(path: Path, map_: str) -> None:
     for video_path in paths:
         print("Starting...")
         p = subprocess.Popen([
-            executable, "-hide_banner",
+            binaries.FFMPEG, "-hide_banner",
             "-benchmark",
             "-i", str(video_path),
             "-map", map_,

devine/commands/wvd.py

@@ -38,6 +38,7 @@ def add(paths: list[Path]) -> None:
         else:
             # TODO: Check for and log errors
             _ = Device.load(path)  # test if WVD is valid
+            dst_path.parent.mkdir(parents=True, exist_ok=True)
             shutil.move(path, dst_path)
             log.info(f"Added {path.stem}")
 
@@ -83,6 +84,10 @@ def parse(path: Path) -> None:
 
     log = logging.getLogger("wvd")
 
+    if not path.exists():
+        console.log(f"[bright_blue]{path.absolute()}[/] does not exist...")
+        return
+
     device = Device.load(path)
 
     log.info(f"System ID: {device.system_id}")
@@ -115,9 +120,23 @@ def dump(wvd_paths: list[Path], out_dir: Path) -> None:
     If the path is relative, with no file extension, it will dump the WVD in the WVDs
     directory.
     """
-    if wvd_paths == (Path(""),):
+    log = logging.getLogger("wvd")
-        wvd_paths = list(config.directories.wvds.iterdir())
+
-    for wvd_path, out_path in zip(wvd_paths, (out_dir / x.stem for x in wvd_paths)):
+    if wvd_paths == ():
+        if not config.directories.wvds.exists():
+            console.log(f"[bright_blue]{config.directories.wvds.absolute()}[/] does not exist...")
+        wvd_paths = list(
+            x
+            for x in config.directories.wvds.iterdir()
+            if x.is_file() and x.suffix.lower() == ".wvd"
+        )
+        if not wvd_paths:
+            console.log(f"[bright_blue]{config.directories.wvds.absolute()}[/] is empty...")
+
+    for i, (wvd_path, out_path) in enumerate(zip(wvd_paths, (out_dir / x.stem for x in wvd_paths))):
+        if i > 0:
+            log.info("")
+
         try:
             named = not wvd_path.suffix and wvd_path.relative_to(Path(""))
         except ValueError:
@@ -126,10 +145,9 @@ def dump(wvd_paths: list[Path], out_dir: Path) -> None:
             wvd_path = config.directories.wvds / f"{wvd_path.stem}.wvd"
         out_path.mkdir(parents=True, exist_ok=True)
 
+        log.info(f"Dumping: {wvd_path}")
         device = Device.load(wvd_path)
 
-        log = logging.getLogger("wvd")
-        log.info(f"Dumping: {wvd_path}")
         log.info(f"L{device.security_level} {device.system_id} {device.type.name}")
         log.info(f"Saving to: {out_path}")
 

devine/core/__init__.py

@@ -1 +1 @@
-__version__ = "2.1.0"
+__version__ = "3.3.3"

devine/core/binaries.py

@@ -0,0 +1,46 @@
+import shutil
+import sys
+from pathlib import Path
+from typing import Optional
+
+__shaka_platform = {
+    "win32": "win",
+    "darwin": "osx"
+}.get(sys.platform, sys.platform)
+
+
+def find(*names: str) -> Optional[Path]:
+    """Find the path of the first found binary name."""
+    for name in names:
+        path = shutil.which(name)
+        if path:
+            return Path(path)
+    return None
+
+
+FFMPEG = find("ffmpeg")
+FFProbe = find("ffprobe")
+FFPlay = find("ffplay")
+SubtitleEdit = find("SubtitleEdit")
+ShakaPackager = find(
+    "shaka-packager",
+    "packager",
+    f"packager-{__shaka_platform}",
+    f"packager-{__shaka_platform}-arm64",
+    f"packager-{__shaka_platform}-x64"
+)
+Aria2 = find("aria2c", "aria2")
+CCExtractor = find(
+    "ccextractor",
+    "ccextractorwin",
+    "ccextractorwinfull"
+)
+HolaProxy = find("hola-proxy")
+MPV = find("mpv")
+Caddy = find("caddy")
+
+
+__all__ = (
+    "FFMPEG", "FFProbe", "FFPlay", "SubtitleEdit", "ShakaPackager",
+    "Aria2", "CCExtractor", "HolaProxy", "MPV", "Caddy", "find"
+)

devine/core/config.py

@@ -17,6 +17,7 @@ class Config:
         commands = namespace_dir / "commands"
         services = namespace_dir / "services"
         vaults = namespace_dir / "vaults"
+        fonts = namespace_dir / "fonts"
         user_configs = Path(app_dirs.user_config_dir)
         data = Path(app_dirs.user_data_dir)
         downloads = Path.home() / "Downloads" / "devine"
@@ -76,29 +77,27 @@ class Config:
         return cls(**yaml.safe_load(path.read_text(encoding="utf8")) or {})
 
 
+# noinspection PyProtectedMember
+POSSIBLE_CONFIG_PATHS = (
+    # The Devine Namespace Folder (e.g., %appdata%/Python/Python311/site-packages/devine)
+    Config._Directories.namespace_dir / Config._Filenames.root_config,
+    # The Parent Folder to the Devine Namespace Folder (e.g., %appdata%/Python/Python311/site-packages)
+    Config._Directories.namespace_dir.parent / Config._Filenames.root_config,
+    # The AppDirs User Config Folder (e.g., %localappdata%/devine)
+    Config._Directories.user_configs / Config._Filenames.root_config
+)
+
+
 def get_config_path() -> Optional[Path]:
     """
-    Get Path to Config from various locations.
+    Get Path to Config from any one of the possible locations.
-
-    Looks for a config file in the following folders in order:
-
-    1. The Devine Namespace Folder (e.g., %appdata%/Python/Python311/site-packages/devine)
-    2. The Parent Folder to the Devine Namespace Folder (e.g., %appdata%/Python/Python311/site-packages)
-    3. The AppDirs User Config Folder (e.g., %localappdata%/devine)
 
     Returns None if no config file could be found.
     """
-    # noinspection PyProtectedMember
+    for path in POSSIBLE_CONFIG_PATHS:
-    path = Config._Directories.namespace_dir / Config._Filenames.root_config
+        if path.exists():
-    if not path.exists():
+            return path
-        # noinspection PyProtectedMember
+    return None
-        path = Config._Directories.namespace_dir.parent / Config._Filenames.root_config
-    if not path.exists():
-        # noinspection PyProtectedMember
-        path = Config._Directories.user_configs / Config._Filenames.root_config
-    if not path.exists():
-        path = None
-    return path
 
 
 config_path = get_config_path()

devine/core/constants.py

@@ -5,9 +5,6 @@ DOWNLOAD_CANCELLED = Event()
 DOWNLOAD_LICENCE_ONLY = Event()
 
 DRM_SORT_MAP = ["ClearKey", "Widevine"]
-TERRITORY_MAP = {
-    "Hong Kong SAR China": "Hong Kong"
-}
 LANGUAGE_MAX_DISTANCE = 5  # this is max to be considered "same", e.g., en, en-US, en-AU
 VIDEO_CODEC_MAP = {
     "AVC": "H.264",

devine/core/downloaders/__init__.py

@@ -1,13 +1,5 @@
-from ..config import config
 from .aria2c import aria2c
 from .curl_impersonate import curl_impersonate
 from .requests import requests
 
-downloader = {
+__all__ = ("aria2c", "curl_impersonate", "requests")
-    "aria2c": aria2c,
-    "curl_impersonate": curl_impersonate,
-    "requests": requests
-}[config.downloader]
-
-
-__all__ = ("downloader", "aria2c", "curl_impersonate", "requests")

devine/core/downloaders/aria2c.py

@@ -15,10 +15,11 @@ from requests.cookies import cookiejar_from_dict, get_cookie_header
 from rich import filesize
 from rich.text import Text
 
+from devine.core import binaries
 from devine.core.config import config
 from devine.core.console import console
 from devine.core.constants import DOWNLOAD_CANCELLED
-from devine.core.utilities import get_binary_path, get_extension, get_free_port
+from devine.core.utilities import get_extension, get_free_port
 
 
 def rpc(caller: Callable, secret: str, method: str, params: Optional[list[Any]] = None) -> Any:
@@ -87,8 +88,7 @@ def download(
     if not isinstance(urls, list):
         urls = [urls]
 
-    executable = get_binary_path("aria2c", "aria2")
+    if not binaries.Aria2:
-    if not executable:
         raise EnvironmentError("Aria2c executable not found...")
 
     if proxy and not proxy.lower().startswith("http://"):
@@ -186,7 +186,7 @@ def download(
     try:
         p = subprocess.Popen(
             [
-                executable,
+                binaries.Aria2,
                 *arguments
             ],
             stdin=subprocess.PIPE,

devine/core/downloaders/curl_impersonate.py

@@ -17,13 +17,13 @@ MAX_ATTEMPTS = 5
 RETRY_WAIT = 2
 CHUNK_SIZE = 1024
 PROGRESS_WINDOW = 5
-BROWSER = config.curl_impersonate.get("browser", "chrome120")
+BROWSER = config.curl_impersonate.get("browser", "chrome124")
 
 
 def download(
     url: str,
     save_path: Path,
-    session: Optional[Session] = None,
+    session: Session,
     **kwargs: Any
 ) -> Generator[dict[str, Any], None, None]:
     """
@@ -52,9 +52,6 @@ def download(
             for one-time request changes like a header, cookie, or proxy. For example,
             to request Byte-ranges use e.g., `headers={"Range": "bytes=0-128"}`.
     """
-    if not session:
-        session = Session(impersonate=BROWSER)
-
     save_dir = save_path.parent
     control_file = save_path.with_name(f"{save_path.name}.!dev")
 
@@ -224,10 +221,7 @@ def curl_impersonate(
     if cookies:
         session.cookies.update(cookies)
     if proxy:
-        session.proxies.update({
+        session.proxies.update({"all": proxy})
-            "http": proxy.replace("https://", "http://"),
-            "https": proxy.replace("https://", "http://")
-        })
 
     yield dict(total=len(urls))
 

devine/core/downloaders/requests.py

@@ -1,12 +1,14 @@
 import math
+import os
 import time
-from concurrent import futures
+from concurrent.futures import as_completed
 from concurrent.futures.thread import ThreadPoolExecutor
 from http.cookiejar import CookieJar
 from pathlib import Path
 from typing import Any, Generator, MutableMapping, Optional, Union
 
 from requests import Session
+from requests.adapters import HTTPAdapter
 from rich import filesize
 
 from devine.core.constants import DOWNLOAD_CANCELLED
@@ -17,11 +19,14 @@ RETRY_WAIT = 2
 CHUNK_SIZE = 1024
 PROGRESS_WINDOW = 5
 
+DOWNLOAD_SIZES = []
+LAST_SPEED_REFRESH = time.time()
 
 def download(
     url: str,
     save_path: Path,
     session: Optional[Session] = None,
+    segmented: bool = False,
     **kwargs: Any
 ) -> Generator[dict[str, Any], None, None]:
     """
@@ -46,10 +51,13 @@ def download(
         session: The Requests Session to make HTTP requests with. Useful to set Header,
             Cookie, and Proxy data. Connections are saved and re-used with the session
             so long as the server keeps the connection alive.
+        segmented: If downloads are segments or parts of one bigger file.
         kwargs: Any extra keyword arguments to pass to the session.get() call. Use this
             for one-time request changes like a header, cookie, or proxy. For example,
             to request Byte-ranges use e.g., `headers={"Range": "bytes=0-128"}`.
     """
+    global LAST_SPEED_REFRESH
+
     session = session or Session()
 
     save_dir = save_path.parent
@@ -67,6 +75,7 @@ def download(
             file_downloaded=save_path,
             written=save_path.stat().st_size
         )
+        # TODO: This should return, potential recovery bug
 
     # TODO: Design a control file format so we know how much of the file is missing
     control_file.write_bytes(b"")
@@ -75,6 +84,8 @@ def download(
     try:
         while True:
             written = 0
+
+            # these are for single-url speed calcs only
             download_sizes = []
             last_speed_refresh = time.time()
 
@@ -82,16 +93,17 @@ def download(
                 stream = session.get(url, stream=True, **kwargs)
                 stream.raise_for_status()
 
-                try:
+                if not segmented:
-                    content_length = int(stream.headers.get("Content-Length", "0"))
+                    try:
-                except ValueError:
+                        content_length = int(stream.headers.get("Content-Length", "0"))
-                    content_length = 0
+                    except ValueError:
+                        content_length = 0
 
-                if content_length > 0:
+                    if content_length > 0:
-                    yield dict(total=math.ceil(content_length / CHUNK_SIZE))
+                        yield dict(total=math.ceil(content_length / CHUNK_SIZE))
-                else:
+                    else:
-                    # we have no data to calculate total chunks
+                        # we have no data to calculate total chunks
-                    yield dict(total=None)  # indeterminate mode
+                        yield dict(total=None)  # indeterminate mode
 
                 with open(save_path, "wb") as f:
                     for chunk in stream.iter_content(chunk_size=CHUNK_SIZE):
@@ -99,23 +111,32 @@ def download(
                         f.write(chunk)
                         written += download_size
 
-                        yield dict(advance=1)
+                        if not segmented:
+                            yield dict(advance=1)
+                            now = time.time()
+                            time_since = now - last_speed_refresh
+                            download_sizes.append(download_size)
+                            if time_since > PROGRESS_WINDOW or download_size < CHUNK_SIZE:
+                                data_size = sum(download_sizes)
+                                download_speed = math.ceil(data_size / (time_since or 1))
+                                yield dict(downloaded=f"{filesize.decimal(download_speed)}/s")
+                                last_speed_refresh = now
+                                download_sizes.clear()
 
-                        now = time.time()
+                yield dict(file_downloaded=save_path, written=written)
-                        time_since = now - last_speed_refresh
 
-                        download_sizes.append(download_size)
+                if segmented:
-                        if time_since > PROGRESS_WINDOW or download_size < CHUNK_SIZE:
+                    yield dict(advance=1)
-                            data_size = sum(download_sizes)
+                    now = time.time()
-                            download_speed = math.ceil(data_size / (time_since or 1))
+                    time_since = now - LAST_SPEED_REFRESH
-                            yield dict(downloaded=f"{filesize.decimal(download_speed)}/s")
+                    if written:  # no size == skipped dl
-                            last_speed_refresh = now
+                        DOWNLOAD_SIZES.append(written)
-                            download_sizes.clear()
+                    if DOWNLOAD_SIZES and time_since > PROGRESS_WINDOW:
-
+                        data_size = sum(DOWNLOAD_SIZES)
-                yield dict(
+                        download_speed = math.ceil(data_size / (time_since or 1))
-                    file_downloaded=save_path,
+                        yield dict(downloaded=f"{filesize.decimal(download_speed)}/s")
-                    written=written
+                        LAST_SPEED_REFRESH = now
-                )
+                        DOWNLOAD_SIZES.clear()
                 break
             except Exception as e:
                 save_path.unlink(missing_ok=True)
@@ -195,6 +216,9 @@ def requests(
     if not isinstance(urls, list):
         urls = [urls]
 
+    if not max_workers:
+        max_workers = min(32, (os.cpu_count() or 1) + 4)
+
     urls = [
         dict(
             save_path=save_path,
@@ -211,6 +235,13 @@ def requests(
     ]
 
     session = Session()
+    session.mount("https://", HTTPAdapter(
+        pool_connections=max_workers,
+        pool_maxsize=max_workers,
+        pool_block=True
+    ))
+    session.mount("http://", session.adapters["https://"])
+
     if headers:
         headers = {
             k: v
@@ -225,59 +256,37 @@ def requests(
 
     yield dict(total=len(urls))
 
-    download_sizes = []
+    try:
-    last_speed_refresh = time.time()
+        with ThreadPoolExecutor(max_workers=max_workers) as pool:
-
+            for future in as_completed(
-    with ThreadPoolExecutor(max_workers=max_workers) as pool:
+                pool.submit(
-        for i, future in enumerate(futures.as_completed((
+                    download,
-            pool.submit(
+                    session=session,
-                download,
+                    segmented=True,
-                session=session,
+                    **url
-                **url
+                )
-            )
+                for url in urls
-            for url in urls
+            ):
-        ))):
+                try:
-            file_path, download_size = None, None
+                    yield from future.result()
-            try:
+                except KeyboardInterrupt:
-                for status_update in future.result():
+                    DOWNLOAD_CANCELLED.set()  # skip pending track downloads
-                    if status_update.get("file_downloaded") and status_update.get("written"):
+                    yield dict(downloaded="[yellow]CANCELLING")
-                        file_path = status_update["file_downloaded"]
+                    pool.shutdown(wait=True, cancel_futures=True)
-                        download_size = status_update["written"]
+                    yield dict(downloaded="[yellow]CANCELLED")
-                    elif len(urls) == 1:
+                    # tell dl that it was cancelled
-                        # these are per-chunk updates, only useful if it's one big file
+                    # the pool is already shut down, so exiting loop is fine
-                        yield status_update
+                    raise
-            except KeyboardInterrupt:
+                except Exception:
-                DOWNLOAD_CANCELLED.set()  # skip pending track downloads
+                    DOWNLOAD_CANCELLED.set()  # skip pending track downloads
-                yield dict(downloaded="[yellow]CANCELLING")
+                    yield dict(downloaded="[red]FAILING")
-                pool.shutdown(wait=True, cancel_futures=True)
+                    pool.shutdown(wait=True, cancel_futures=True)
-                yield dict(downloaded="[yellow]CANCELLED")
+                    yield dict(downloaded="[red]FAILED")
-                # tell dl that it was cancelled
+                    # tell dl that it failed
-                # the pool is already shut down, so exiting loop is fine
+                    # the pool is already shut down, so exiting loop is fine
-                raise
+                    raise
-            except Exception:
+    finally:
-                DOWNLOAD_CANCELLED.set()  # skip pending track downloads
+        DOWNLOAD_SIZES.clear()
-                yield dict(downloaded="[red]FAILING")
-                pool.shutdown(wait=True, cancel_futures=True)
-                yield dict(downloaded="[red]FAILED")
-                # tell dl that it failed
-                # the pool is already shut down, so exiting loop is fine
-                raise
-            else:
-                yield dict(file_downloaded=file_path, written=download_size)
-                yield dict(advance=1)
-
-                now = time.time()
-                time_since = now - last_speed_refresh
-
-                if download_size:  # no size == skipped dl
-                    download_sizes.append(download_size)
-
-                if download_sizes and (time_since > PROGRESS_WINDOW or i == len(urls)):
-                    data_size = sum(download_sizes)
-                    download_speed = math.ceil(data_size / (time_since or 1))
-                    yield dict(downloaded=f"{filesize.decimal(download_speed)}/s")
-                    last_speed_refresh = now
-                    download_sizes.clear()
 
 
 __all__ = ("requests",)

devine/core/drm/clearkey.py

@@ -6,10 +6,10 @@ from pathlib import Path
 from typing import Optional, Union
 from urllib.parse import urljoin
 
-import requests
 from Cryptodome.Cipher import AES
-from Cryptodome.Util.Padding import pad, unpad
+from Cryptodome.Util.Padding import unpad
 from m3u8.model import Key
+from requests import Session
 
 
 class ClearKey:
@@ -43,7 +43,7 @@ class ClearKey:
 
         decrypted = AES. \
             new(self.key, AES.MODE_CBC, self.iv). \
-            decrypt(pad(path.read_bytes(), AES.block_size))
+            decrypt(path.read_bytes())
 
         try:
             decrypted = unpad(decrypted, AES.block_size)
@@ -58,14 +58,33 @@ class ClearKey:
         shutil.move(decrypted_path, path)
 
     @classmethod
-    def from_m3u_key(cls, m3u_key: Key, proxy: Optional[str] = None) -> ClearKey:
+    def from_m3u_key(cls, m3u_key: Key, session: Optional[Session] = None) -> ClearKey:
+        """
+        Load a ClearKey from an M3U(8) Playlist's EXT-X-KEY.
+
+        Parameters:
+            m3u_key: A Key object parsed from a m3u(8) playlist using
+                the `m3u8` library.
+            session: Optional session used to request external URIs with.
+                Useful to set headers, proxies, cookies, and so forth.
+        """
         if not isinstance(m3u_key, Key):
             raise ValueError(f"Provided M3U Key is in an unexpected type {m3u_key!r}")
+        if not isinstance(session, (Session, type(None))):
+            raise TypeError(f"Expected session to be a {Session}, not a {type(session)}")
+
         if not m3u_key.method.startswith("AES"):
             raise ValueError(f"Provided M3U Key is not an AES Clear Key, {m3u_key.method}")
         if not m3u_key.uri:
             raise ValueError("No URI in M3U Key, unable to get Key.")
 
+        if not session:
+            session = Session()
+
+        if not session.headers.get("User-Agent"):
+            # commonly needed default for HLS playlists
+            session.headers["User-Agent"] = "smartexoplayer/1.1.0 (Linux;Android 8.0.0) ExoPlayerLib/2.13.3"
+
         if m3u_key.uri.startswith("data:"):
             media_types, data = m3u_key.uri[5:].split(",")
             media_types = media_types.split(";")
@@ -74,13 +93,7 @@ class ClearKey:
             key = data
         else:
             url = urljoin(m3u_key.base_uri, m3u_key.uri)
-            res = requests.get(
+            res = session.get(url)
-                url=url,
-                headers={
-                    "User-Agent": "smartexoplayer/1.1.0 (Linux;Android 8.0.0) ExoPlayerLib/2.13.3"
-                },
-                proxies={"all": proxy} if proxy else None
-            )
             res.raise_for_status()
             if not res.content:
                 raise EOFError("Unexpected Empty Response by M3U Key URI.")

devine/core/drm/widevine.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import base64
 import shutil
 import subprocess
-import sys
 import textwrap
 from pathlib import Path
 from typing import Any, Callable, Optional, Union
@@ -17,10 +16,11 @@ from pywidevine.pssh import PSSH
 from requests import Session
 from rich.text import Text
 
+from devine.core import binaries
 from devine.core.config import config
 from devine.core.console import console
 from devine.core.constants import AnyTrack
-from devine.core.utilities import get_binary_path, get_boxes
+from devine.core.utilities import get_boxes
 from devine.core.utils.subprocess import ffprobe
 
 
@@ -78,7 +78,7 @@ class Widevine:
         pssh_boxes: list[Container] = []
         tenc_boxes: list[Container] = []
 
-        if track.descriptor == track.Descriptor.M3U:
+        if track.descriptor == track.Descriptor.HLS:
             m3u_url = track.url
             master = m3u8.loads(session.get(m3u_url).text, uri=m3u_url)
             pssh_boxes.extend(
@@ -223,9 +223,7 @@ class Widevine:
         if not self.content_keys:
             raise ValueError("Cannot decrypt a Track without any Content Keys...")
 
-        platform = {"win32": "win", "darwin": "osx"}.get(sys.platform, sys.platform)
+        if not binaries.ShakaPackager:
-        executable = get_binary_path("shaka-packager", "packager", f"packager-{platform}", f"packager-{platform}-x64")
-        if not executable:
             raise EnvironmentError("Shaka Packager executable not found but is required.")
         if not path or not path.exists():
             raise ValueError("Tried to decrypt a file that does not exist.")
@@ -252,7 +250,7 @@ class Widevine:
             ]
 
             p = subprocess.Popen(
-                [executable, *arguments],
+                [binaries.ShakaPackager, *arguments],
                 stdout=subprocess.DEVNULL,
                 stderr=subprocess.PIPE,
                 universal_newlines=True

devine/core/events.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from copy import deepcopy
+from enum import Enum
+from typing import Any, Callable
+
+
+class Events:
+    class Types(Enum):
+        _reserved = 0
+        # A Track's segment has finished downloading
+        SEGMENT_DOWNLOADED = 1
+        # Track has finished downloading
+        TRACK_DOWNLOADED = 2
+        # Track has finished decrypting
+        TRACK_DECRYPTED = 3
+        # Track has finished repacking
+        TRACK_REPACKED = 4
+        # Track is about to be Multiplexed into a Container
+        TRACK_MULTIPLEX = 5
+
+    def __init__(self):
+        self.__subscriptions: dict[Events.Types, list[Callable]] = {}
+        self.__ephemeral: dict[Events.Types, list[Callable]] = {}
+        self.reset()
+
+    def reset(self):
+        """Reset Event Observer clearing all Subscriptions."""
+        self.__subscriptions = {
+            k: []
+            for k in Events.Types.__members__.values()
+        }
+        self.__ephemeral = deepcopy(self.__subscriptions)
+
+    def subscribe(self, event_type: Events.Types, callback: Callable, ephemeral: bool = False) -> None:
+        """
+        Subscribe to an Event with a Callback.
+
+        Parameters:
+            event_type: The Events.Type to subscribe to.
+            callback: The function or lambda to call on event emit.
+            ephemeral: Unsubscribe the callback from the event on first emit.
+                Note that this is not thread-safe and may be called multiple
+                times at roughly the same time.
+        """
+        [self.__subscriptions, self.__ephemeral][ephemeral][event_type].append(callback)
+
+    def unsubscribe(self, event_type: Events.Types, callback: Callable) -> None:
+        """
+        Unsubscribe a Callback from an Event.
+
+        Parameters:
+            event_type: The Events.Type to unsubscribe from.
+            callback: The function or lambda to remove from event emit.
+        """
+        if callback in self.__subscriptions[event_type]:
+            self.__subscriptions[event_type].remove(callback)
+        if callback in self.__ephemeral[event_type]:
+            self.__ephemeral[event_type].remove(callback)
+
+    def emit(self, event_type: Events.Types, *args: Any, **kwargs: Any) -> None:
+        """
+        Emit an Event, executing all subscribed Callbacks.
+
+        Parameters:
+            event_type: The Events.Type to emit.
+            args: Positional arguments to pass to callbacks.
+            kwargs: Keyword arguments to pass to callbacks.
+        """
+        if event_type not in self.__subscriptions:
+            raise ValueError(f"Event type \"{event_type}\" is invalid")
+
+        for callback in self.__subscriptions[event_type] + self.__ephemeral[event_type]:
+            callback(*args, **kwargs)
+
+        self.__ephemeral[event_type].clear()
+
+
+events = Events()

devine/core/manifests/dash.py

@@ -16,15 +16,15 @@ from zlib import crc32
 
 import requests
 from langcodes import Language, tag_is_valid
-from lxml.etree import Element
+from lxml.etree import Element, ElementTree
 from pywidevine.cdm import Cdm as WidevineCdm
 from pywidevine.pssh import PSSH
 from requests import Session
 
 from devine.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack
-from devine.core.downloaders import downloader
 from devine.core.downloaders import requests as requests_downloader
 from devine.core.drm import Widevine
+from devine.core.events import events
 from devine.core.tracks import Audio, Subtitle, Tracks, Video
 from devine.core.utilities import is_close_match, try_ensure_utf8
 from devine.core.utils.xml import load_xml
@@ -114,6 +114,7 @@ class DASH:
                 for rep in adaptation_set.findall("Representation"):
                     get = partial(self._get, adaptation_set=adaptation_set, representation=rep)
                     findall = partial(self._findall, adaptation_set=adaptation_set, representation=rep, both=True)
+                    segment_base = rep.find("SegmentBase")
 
                     codecs = get("codecs")
                     content_type = get("contentType")
@@ -141,6 +142,10 @@ class DASH:
                     if content_type == "video":
                         track_type = Video
                         track_codec = Video.Codec.from_codecs(codecs)
+                        track_fps = get("frameRate")
+                        if not track_fps and segment_base is not None:
+                            track_fps = segment_base.get("timescale")
+
                         track_args = dict(
                             range_=self.get_video_range(
                                 codecs,
@@ -150,7 +155,7 @@ class DASH:
                             bitrate=get("bandwidth") or None,
                             width=get("width") or 0,
                             height=get("height") or 0,
-                            fps=get("frameRate") or (rep.find("SegmentBase") or {}).get("timescale") or None
+                            fps=track_fps or None
                         )
                     elif content_type == "audio":
                         track_type = Audio
@@ -202,12 +207,19 @@ class DASH:
 
                     tracks.add(track_type(
                         id_=track_id,
-                        url=(self.url, self.manifest, rep, adaptation_set, period),
+                        url=self.url,
                         codec=track_codec,
                         language=track_lang,
                         is_original_lang=language and is_close_match(track_lang, [language]),
-                        descriptor=Video.Descriptor.MPD,
+                        descriptor=Video.Descriptor.DASH,
-                        extra=(rep, adaptation_set),
+                        data={
+                            "dash": {
+                                "manifest": self.manifest,
+                                "period": period,
+                                "adaptation_set": adaptation_set,
+                                "representation": rep
+                            }
+                        },
                         **track_args
                     ))
 
@@ -224,6 +236,7 @@ class DASH:
         progress: partial,
         session: Optional[Session] = None,
         proxy: Optional[str] = None,
+        max_workers: Optional[int] = None,
         license_widevine: Optional[Callable] = None
     ):
         if not session:
@@ -238,20 +251,21 @@ class DASH:
 
         log = logging.getLogger("DASH")
 
-        manifest_url, manifest, representation, adaptation_set, period = track.url
+        manifest: ElementTree = track.data["dash"]["manifest"]
+        period: Element = track.data["dash"]["period"]
+        adaptation_set: Element = track.data["dash"]["adaptation_set"]
+        representation: Element = track.data["dash"]["representation"]
 
         track.drm = DASH.get_drm(
             representation.findall("ContentProtection") +
             adaptation_set.findall("ContentProtection")
         )
 
-        manifest_url_query = urlparse(manifest_url).query
-
         manifest_base_url = manifest.findtext("BaseURL")
         if not manifest_base_url:
-            manifest_base_url = manifest_url
+            manifest_base_url = track.url
         elif not re.match("^https?://", manifest_base_url, re.IGNORECASE):
-            manifest_base_url = urljoin(manifest_url, f"./{manifest_base_url}")
+            manifest_base_url = urljoin(track.url, f"./{manifest_base_url}")
         period_base_url = urljoin(manifest_base_url, period.findtext("BaseURL"))
         rep_base_url = urljoin(period_base_url, representation.findtext("BaseURL"))
 
@@ -271,12 +285,16 @@ class DASH:
             segment_base = adaptation_set.find("SegmentBase")
 
         segments: list[tuple[str, Optional[str]]] = []
+        segment_timescale: float = 0
+        segment_durations: list[int] = []
         track_kid: Optional[UUID] = None
 
         if segment_template is not None:
             segment_template = copy(segment_template)
             start_number = int(segment_template.get("startNumber") or 1)
+            end_number = int(segment_template.get("endNumber") or 0) or None
             segment_timeline = segment_template.find("SegmentTimeline")
+            segment_timescale = float(segment_template.get("timescale") or 1)
 
             for item in ("initialization", "media"):
                 value = segment_template.get(item)
@@ -286,8 +304,10 @@ class DASH:
                     if not rep_base_url:
                         raise ValueError("Resolved Segment URL is not absolute, and no Base URL is available.")
                     value = urljoin(rep_base_url, value)
-                if not urlparse(value).query and manifest_url_query:
+                if not urlparse(value).query:
-                    value += f"?{manifest_url_query}"
+                    manifest_url_query = urlparse(track.url).query
+                    if manifest_url_query:
+                        value += f"?{manifest_url_query}"
                 segment_template.set(item, value)
 
             init_url = segment_template.get("initialization")
@@ -302,17 +322,18 @@ class DASH:
                 track_kid = track.get_key_id(init_data)
 
             if segment_timeline is not None:
-                seg_time_list = []
                 current_time = 0
                 for s in segment_timeline.findall("S"):
                     if s.get("t"):
                         current_time = int(s.get("t"))
                     for _ in range(1 + (int(s.get("r") or 0))):
-                        seg_time_list.append(current_time)
+                        segment_durations.append(current_time)
                         current_time += int(s.get("d"))
-                seg_num_list = list(range(start_number, len(seg_time_list) + start_number))
 
-                for t, n in zip(seg_time_list, seg_num_list):
+                if not end_number:
+                    end_number = len(segment_durations)
+
+                for t, n in zip(segment_durations, range(start_number, end_number + 1)):
                     segments.append((
                         DASH.replace_fields(
                             segment_template.get("media"),
@@ -326,11 +347,12 @@ class DASH:
                 if not period_duration:
                     raise ValueError("Duration of the Period was unable to be determined.")
                 period_duration = DASH.pt_to_sec(period_duration)
-                segment_duration = float(segment_template.get("duration"))
+                segment_duration = float(segment_template.get("duration")) or 1
-                segment_timescale = float(segment_template.get("timescale") or 1)
-                total_segments = math.ceil(period_duration / (segment_duration / segment_timescale))
 
-                for s in range(start_number, start_number + total_segments):
+                if not end_number:
+                    end_number = math.ceil(period_duration / (segment_duration / segment_timescale))
+
+                for s in range(start_number, end_number + 1):
                     segments.append((
                         DASH.replace_fields(
                             segment_template.get("media"),
@@ -340,7 +362,11 @@ class DASH:
                             Time=s
                         ), None
                     ))
+                    # TODO: Should we floor/ceil/round, or is int() ok?
+                    segment_durations.append(int(segment_duration))
         elif segment_list is not None:
+            segment_timescale = float(segment_list.get("timescale") or 1)
+
             init_data = None
             initialization = segment_list.find("Initialization")
             if initialization is not None:
@@ -372,6 +398,7 @@ class DASH:
                     media_url,
                     segment_url.get("mediaRange")
                 ))
+                segment_durations.append(int(segment_url.get("duration") or 1))
         elif segment_base is not None:
             media_range = None
             init_data = None
@@ -401,9 +428,13 @@ class DASH:
             ))
         else:
             log.error("Could not find a way to get segments from this MPD manifest.")
-            log.debug(manifest_url)
+            log.debug(track.url)
             sys.exit(1)
 
+        # TODO: Should we floor/ceil/round, or is int() ok?
+        track.data["dash"]["timescale"] = int(segment_timescale)
+        track.data["dash"]["segment_durations"] = segment_durations
+
         if not track.drm and isinstance(track, (Video, Audio)):
             try:
                 track.drm = [Widevine.from_init_data(init_data)]
@@ -437,12 +468,13 @@ class DASH:
 
         progress(total=len(segments))
 
-        downloader_ = downloader
+        downloader = track.downloader
         if downloader.__name__ == "aria2c" and any(bytes_range is not None for url, bytes_range in segments):
             # aria2(c) is shit and doesn't support the Range header, fallback to the requests downloader
-            downloader_ = requests_downloader
+            downloader = requests_downloader
+            log.warning("Falling back to the requests downloader as aria2(c) doesn't support the Range header")
 
-        for status_update in downloader_(
+        for status_update in downloader(
             urls=[
                 {
                     "url": url,
@@ -457,23 +489,31 @@ class DASH:
             headers=session.headers,
             cookies=session.cookies,
             proxy=proxy,
-            max_workers=16
+            max_workers=max_workers
         ):
             file_downloaded = status_update.get("file_downloaded")
-            if file_downloaded and callable(track.OnSegmentDownloaded):
+            if file_downloaded:
-                track.OnSegmentDownloaded(file_downloaded)
+                events.emit(events.Types.SEGMENT_DOWNLOADED, track=track, segment=file_downloaded)
             else:
                 downloaded = status_update.get("downloaded")
                 if downloaded and downloaded.endswith("/s"):
                     status_update["downloaded"] = f"DASH {downloaded}"
                 progress(**status_update)
 
-        segments_to_merge = sorted(save_dir.iterdir())
+        # see https://github.com/devine-dl/devine/issues/71
-        progress(downloaded="Merging", completed=0, total=len(segments_to_merge))
+        for control_file in save_dir.glob("*.aria2__temp"):
+            control_file.unlink()
 
+        segments_to_merge = [
+            x
+            for x in sorted(save_dir.iterdir())
+            if x.is_file()
+        ]
         with open(save_path, "wb") as f:
             if init_data:
                 f.write(init_data)
+            if len(segments_to_merge) > 1:
+                progress(downloaded="Merging", completed=0, total=len(segments_to_merge))
             for segment_file in segments_to_merge:
                 segment_data = segment_file.read_bytes()
                 # TODO: fix encoding after decryption?
@@ -492,15 +532,18 @@ class DASH:
                 progress(advance=1)
 
         track.path = save_path
-        if callable(track.OnDownloaded):
+        events.emit(events.Types.TRACK_DOWNLOADED, track=track)
-            track.OnDownloaded()
 
         if drm:
             progress(downloaded="Decrypting", completed=0, total=100)
             drm.decrypt(save_path)
             track.drm = None
-            if callable(track.OnDecrypted):
+            events.emit(
-                track.OnDecrypted(drm)
+                events.Types.TRACK_DECRYPTED,
+                track=track,
+                drm=drm,
+                segment=None
+            )
             progress(downloaded="Decrypting", advance=100)
 
         save_dir.rmdir()

devine/core/manifests/hls.py

@@ -19,12 +19,13 @@ from pywidevine.cdm import Cdm as WidevineCdm
 from pywidevine.pssh import PSSH
 from requests import Session
 
+from devine.core import binaries
 from devine.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack
-from devine.core.downloaders import downloader
 from devine.core.downloaders import requests as requests_downloader
 from devine.core.drm import DRM_T, ClearKey, Widevine
+from devine.core.events import events
 from devine.core.tracks import Audio, Subtitle, Tracks, Video
-from devine.core.utilities import get_binary_path, get_extension, is_close_match, try_ensure_utf8
+from devine.core.utilities import get_extension, is_close_match, try_ensure_utf8
 
 
 class HLS:
@@ -101,7 +102,8 @@ class HLS:
 
             try:
                 # TODO: Any better way to figure out the primary track type?
-                Video.Codec.from_codecs(playlist.stream_info.codecs)
+                if playlist.stream_info.codecs:
+                    Video.Codec.from_codecs(playlist.stream_info.codecs)
             except ValueError:
                 primary_track_type = Audio
             else:
@@ -110,21 +112,28 @@ class HLS:
             tracks.add(primary_track_type(
                 id_=hex(crc32(str(playlist).encode()))[2:],
                 url=urljoin(playlist.base_uri, playlist.uri),
-                codec=primary_track_type.Codec.from_codecs(playlist.stream_info.codecs),
+                codec=(
+                    primary_track_type.Codec.from_codecs(playlist.stream_info.codecs)
+                    if playlist.stream_info.codecs else None
+                ),
                 language=language,  # HLS manifests do not seem to have language info
                 is_original_lang=True,  # TODO: All we can do is assume Yes
                 bitrate=playlist.stream_info.average_bandwidth or playlist.stream_info.bandwidth,
-                descriptor=Video.Descriptor.M3U,
+                descriptor=Video.Descriptor.HLS,
                 drm=session_drm,
-                extra=playlist,
+                data={
+                    "hls": {
+                        "playlist": playlist
+                    }
+                },
                 # video track args
                 **(dict(
                     range_=Video.Range.DV if any(
                         codec.split(".")[0] in ("dva1", "dvav", "dvhe", "dvh1")
-                        for codec in playlist.stream_info.codecs.lower().split(",")
+                        for codec in (playlist.stream_info.codecs or "").lower().split(",")
                     ) else Video.Range.from_m3u_range_tag(playlist.stream_info.video_range),
-                    width=playlist.stream_info.resolution[0],
+                    width=playlist.stream_info.resolution[0] if playlist.stream_info.resolution else None,
-                    height=playlist.stream_info.resolution[1],
+                    height=playlist.stream_info.resolution[1] if playlist.stream_info.resolution else None,
                     fps=playlist.stream_info.frame_rate
                 ) if primary_track_type is Video else {})
             ))
@@ -164,9 +173,13 @@ class HLS:
                 codec=codec,
                 language=track_lang,  # HLS media may not have language info, fallback if needed
                 is_original_lang=language and is_close_match(track_lang, [language]),
-                descriptor=Audio.Descriptor.M3U,
+                descriptor=Audio.Descriptor.HLS,
                 drm=session_drm if media.type == "AUDIO" else None,
-                extra=media,
+                data={
+                    "hls": {
+                        "media": media
+                    }
+                },
                 # audio track args
                 **(dict(
                     bitrate=0,  # TODO: M3U doesn't seem to state bitrate?
@@ -189,6 +202,7 @@ class HLS:
         progress: partial,
         session: Optional[Session] = None,
         proxy: Optional[str] = None,
+        max_workers: Optional[int] = None,
         license_widevine: Optional[Callable] = None
     ) -> None:
         if not session:
@@ -231,27 +245,38 @@ class HLS:
         else:
             session_drm = None
 
-        segments = [
+        unwanted_segments = [
             segment for segment in master.segments
-            if not callable(track.OnSegmentFilter) or not track.OnSegmentFilter(segment)
+            if callable(track.OnSegmentFilter) and track.OnSegmentFilter(segment)
         ]
 
-        total_segments = len(segments)
+        total_segments = len(master.segments) - len(unwanted_segments)
         progress(total=total_segments)
 
-        downloader_ = downloader
+        downloader = track.downloader
+        if (
+            downloader.__name__ == "aria2c" and
+            any(x.byterange for x in master.segments if x not in unwanted_segments)
+        ):
+            downloader = requests_downloader
+            log.warning("Falling back to the requests downloader as aria2(c) doesn't support the Range header")
 
         urls: list[dict[str, Any]] = []
+        segment_durations: list[int] = []
+
         range_offset = 0
-        for segment in segments:
+        for segment in master.segments:
+            if segment in unwanted_segments:
+                continue
+
+            segment_durations.append(int(segment.duration))
+
             if segment.byterange:
-                if downloader_.__name__ == "aria2c":
-                    # aria2(c) is shit and doesn't support the Range header, fallback to the requests downloader
-                    downloader_ = requests_downloader
                 byte_range = HLS.calculate_byte_range(segment.byterange, range_offset)
                 range_offset = byte_range.split("-")[0]
             else:
                 byte_range = None
+
             urls.append({
                 "url": urljoin(segment.base_uri, segment.uri),
                 "headers": {
@@ -259,41 +284,49 @@ class HLS:
                 } if byte_range else {}
             })
 
+        track.data["hls"]["segment_durations"] = segment_durations
+
         segment_save_dir = save_dir / "segments"
 
-        for status_update in downloader_(
+        for status_update in downloader(
             urls=urls,
             output_dir=segment_save_dir,
-            filename="{i:0%d}{ext}" % len(str(len(segments))),
+            filename="{i:0%d}{ext}" % len(str(len(urls))),
             headers=session.headers,
             cookies=session.cookies,
             proxy=proxy,
-            max_workers=16
+            max_workers=max_workers
         ):
             file_downloaded = status_update.get("file_downloaded")
-            if file_downloaded and callable(track.OnSegmentDownloaded):
+            if file_downloaded:
-                track.OnSegmentDownloaded(file_downloaded)
+                events.emit(events.Types.SEGMENT_DOWNLOADED, track=track, segment=file_downloaded)
             else:
                 downloaded = status_update.get("downloaded")
                 if downloaded and downloaded.endswith("/s"):
                     status_update["downloaded"] = f"HLS {downloaded}"
                 progress(**status_update)
 
+        # see https://github.com/devine-dl/devine/issues/71
+        for control_file in segment_save_dir.glob("*.aria2__temp"):
+            control_file.unlink()
+
         progress(total=total_segments, completed=0, downloaded="Merging")
 
+        name_len = len(str(total_segments))
         discon_i = 0
         range_offset = 0
         map_data: Optional[tuple[m3u8.model.InitializationSection, bytes]] = None
         if session_drm:
-            encryption_data: Optional[tuple[int, Optional[m3u8.Key], DRM_T]] = (0, None, session_drm)
+            encryption_data: Optional[tuple[Optional[m3u8.Key], DRM_T]] = (None, session_drm)
         else:
-            encryption_data: Optional[tuple[int, Optional[m3u8.Key], DRM_T]] = None
+            encryption_data: Optional[tuple[Optional[m3u8.Key], DRM_T]] = None
 
-        for i, segment in enumerate(segments):
+        i = -1
-            is_last_segment = (i + 1) == total_segments
+        for real_i, segment in enumerate(master.segments):
-            name_len = len(str(total_segments))
+            if segment not in unwanted_segments:
-            segment_file_ext = get_extension(segment.uri)
+                i += 1
-            segment_file_path = segment_save_dir / f"{str(i).zfill(name_len)}{segment_file_ext}"
+
+            is_last_segment = (real_i + 1) == len(master.segments)
 
             def merge(to: Path, via: list[Path], delete: bool = False, include_map_data: bool = False):
                 """
@@ -331,13 +364,17 @@ class HLS:
 
                 Returns the decrypted path.
                 """
-                drm = encryption_data[2]
+                drm = encryption_data[1]
-                first_segment_i = encryption_data[0]
+                first_segment_i = next(
+                    int(file.stem)
+                    for file in sorted(segment_save_dir.iterdir())
+                    if file.stem.isdigit()
+                )
                 last_segment_i = max(0, i - int(not include_this_segment))
                 range_len = (last_segment_i - first_segment_i) + 1
 
                 segment_range = f"{str(first_segment_i).zfill(name_len)}-{str(last_segment_i).zfill(name_len)}"
-                merged_path = segment_save_dir / f"{segment_range}{get_extension(segments[last_segment_i].uri)}"
+                merged_path = segment_save_dir / f"{segment_range}{get_extension(master.segments[last_segment_i].uri)}"
                 decrypted_path = segment_save_dir / f"{merged_path.stem}_decrypted{merged_path.suffix}"
 
                 files = [
@@ -350,19 +387,35 @@ class HLS:
                 elif len(files) != range_len:
                     raise ValueError(f"Missing {range_len - len(files)} segment files for {segment_range}...")
 
-                merge(
+                if isinstance(drm, Widevine):
-                    to=merged_path,
+                    # with widevine we can merge all segments and decrypt once
-                    via=files,
+                    merge(
-                    delete=True,
+                        to=merged_path,
-                    include_map_data=True
+                        via=files,
+                        delete=True,
+                        include_map_data=True
+                    )
+                    drm.decrypt(merged_path)
+                    merged_path.rename(decrypted_path)
+                else:
+                    # with other drm we must decrypt separately and then merge them
+                    # for aes this is because each segment likely has 16-byte padding
+                    for file in files:
+                        drm.decrypt(file)
+                    merge(
+                        to=merged_path,
+                        via=files,
+                        delete=True,
+                        include_map_data=True
+                    )
+
+                events.emit(
+                    events.Types.TRACK_DECRYPTED,
+                    track=track,
+                    drm=drm,
+                    segment=decrypted_path
                 )
 
-                drm.decrypt(merged_path)
-                merged_path.rename(decrypted_path)
-
-                if callable(track.OnDecrypted):
-                    track.OnDecrypted(drm, decrypted_path)
-
                 return decrypted_path
 
             def merge_discontinuity(include_this_segment: bool, include_map_data: bool = True):
@@ -397,58 +450,61 @@ class HLS:
                         include_map_data=include_map_data
                     )
 
-            if isinstance(track, Subtitle):
+            if segment not in unwanted_segments:
-                segment_data = try_ensure_utf8(segment_file_path.read_bytes())
+                if isinstance(track, Subtitle):
-                if track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML):
+                    segment_file_ext = get_extension(segment.uri)
-                    segment_data = segment_data.decode("utf8"). \
+                    segment_file_path = segment_save_dir / f"{str(i).zfill(name_len)}{segment_file_ext}"
-                        replace("‎", html.unescape("‎")). \
+                    segment_data = try_ensure_utf8(segment_file_path.read_bytes())
-                        replace("‏", html.unescape("‏")). \
+                    if track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML):
-                        encode("utf8")
+                        segment_data = segment_data.decode("utf8"). \
-                segment_file_path.write_bytes(segment_data)
+                            replace("‎", html.unescape("‎")). \
+                            replace("‏", html.unescape("‏")). \
+                            encode("utf8")
+                    segment_file_path.write_bytes(segment_data)
 
-            if segment.discontinuity and i != 0:
+                if segment.discontinuity and i != 0:
-                if encryption_data:
+                    if encryption_data:
-                    decrypt(include_this_segment=False)
+                        decrypt(include_this_segment=False)
-                merge_discontinuity(
+                    merge_discontinuity(
-                    include_this_segment=False,
+                        include_this_segment=False,
-                    include_map_data=not encryption_data or not encryption_data[2]
+                        include_map_data=not encryption_data or not encryption_data[1]
-                )
-
-                discon_i += 1
-                range_offset = 0  # TODO: Should this be reset or not?
-                map_data = None
-                if encryption_data:
-                    encryption_data = (i, encryption_data[1], encryption_data[2])
-
-            if segment.init_section and (not map_data or segment.init_section != map_data[0]):
-                if segment.init_section.byterange:
-                    init_byte_range = HLS.calculate_byte_range(
-                        segment.init_section.byterange,
-                        range_offset
                     )
-                    range_offset = init_byte_range.split("-")[0]
-                    init_range_header = {
-                        "Range": f"bytes={init_byte_range}"
-                    }
-                else:
-                    init_range_header = {}
 
-                res = session.get(
+                    discon_i += 1
-                    url=urljoin(segment.init_section.base_uri, segment.init_section.uri),
+                    range_offset = 0  # TODO: Should this be reset or not?
-                    headers=init_range_header
+                    map_data = None
-                )
+                    if encryption_data:
-                res.raise_for_status()
+                        encryption_data = (encryption_data[0], encryption_data[1])
-                map_data = (segment.init_section, res.content)
+
+                if segment.init_section and (not map_data or segment.init_section != map_data[0]):
+                    if segment.init_section.byterange:
+                        init_byte_range = HLS.calculate_byte_range(
+                            segment.init_section.byterange,
+                            range_offset
+                        )
+                        range_offset = init_byte_range.split("-")[0]
+                        init_range_header = {
+                            "Range": f"bytes={init_byte_range}"
+                        }
+                    else:
+                        init_range_header = {}
+
+                    res = session.get(
+                        url=urljoin(segment.init_section.base_uri, segment.init_section.uri),
+                        headers=init_range_header
+                    )
+                    res.raise_for_status()
+                    map_data = (segment.init_section, res.content)
 
             if segment.keys:
                 key = HLS.get_supported_key(segment.keys)
-                if encryption_data and encryption_data[1] != key and i != 0:
+                if encryption_data and encryption_data[0] != key and i != 0 and segment not in unwanted_segments:
                     decrypt(include_this_segment=False)
 
                 if key is None:
                     encryption_data = None
-                elif not encryption_data or encryption_data[1] != key:
+                elif not encryption_data or encryption_data[0] != key:
-                    drm = HLS.get_drm(key, proxy)
+                    drm = HLS.get_drm(key, session)
                     if isinstance(drm, Widevine):
                         try:
                             if map_data:
@@ -462,7 +518,7 @@ class HLS:
                             DOWNLOAD_CANCELLED.set()  # skip pending track downloads
                             progress(downloaded="[red]FAILED")
                             raise
-                    encryption_data = (i, key, drm)
+                    encryption_data = (key, drm)
 
             # TODO: This wont work as we already downloaded
             if DOWNLOAD_LICENCE_ONLY.is_set():
@@ -474,7 +530,7 @@ class HLS:
                     decrypt(include_this_segment=True)
                 merge_discontinuity(
                     include_this_segment=True,
-                    include_map_data=not encryption_data or not encryption_data[2]
+                    include_map_data=not encryption_data or not encryption_data[1]
                 )
 
             progress(advance=1)
@@ -483,29 +539,37 @@ class HLS:
         if DOWNLOAD_LICENCE_ONLY.is_set():
             return
 
+        segment_save_dir.rmdir()
+
         # finally merge all the discontinuity save files together to the final path
-        progress(downloaded="Merging")
+        segments_to_merge = [
-        if isinstance(track, (Video, Audio)):
+            x
-            HLS.merge_segments(
+            for x in sorted(save_dir.iterdir())
-                segments=sorted(list(save_dir.iterdir())),
+            if x.is_file()
-                save_path=save_path
+        ]
-            )
+        if len(segments_to_merge) == 1:
-            shutil.rmtree(save_dir)
+            shutil.move(segments_to_merge[0], save_path)
         else:
-            with open(save_path, "wb") as f:
+            progress(downloaded="Merging")
-                for discontinuity_file in sorted(save_dir.iterdir()):
+            if isinstance(track, (Video, Audio)):
-                    if discontinuity_file.is_dir():
+                HLS.merge_segments(
-                        continue
+                    segments=segments_to_merge,
-                    discontinuity_data = discontinuity_file.read_bytes()
+                    save_path=save_path
-                    f.write(discontinuity_data)
+                )
-                    f.flush()
+            else:
-            shutil.rmtree(save_dir)
+                with open(save_path, "wb") as f:
+                    for discontinuity_file in segments_to_merge:
+                        discontinuity_data = discontinuity_file.read_bytes()
+                        f.write(discontinuity_data)
+                        f.flush()
+                        discontinuity_file.unlink()
+
+        save_dir.rmdir()
 
         progress(downloaded="Downloaded")
 
         track.path = save_path
-        if callable(track.OnDownloaded):
+        events.emit(events.Types.TRACK_DOWNLOADED, track=track)
-            track.OnDownloaded()
 
     @staticmethod
     def merge_segments(segments: list[Path], save_path: Path) -> int:
@@ -514,8 +578,7 @@ class HLS:
 
         Returns the file size of the merged file.
         """
-        ffmpeg = get_binary_path("ffmpeg")
+        if not binaries.FFMPEG:
-        if not ffmpeg:
             raise EnvironmentError("FFmpeg executable was not found but is required to merge HLS segments.")
 
         demuxer_file = segments[0].parent / "ffmpeg_concat_demuxer.txt"
@@ -525,7 +588,7 @@ class HLS:
         ]))
 
         subprocess.check_call([
-            ffmpeg, "-hide_banner",
+            binaries.FFMPEG, "-hide_banner",
             "-loglevel", "panic",
             "-f", "concat",
             "-safe", "0",
@@ -536,6 +599,9 @@ class HLS:
         ])
         demuxer_file.unlink()
 
+        for segment in segments:
+            segment.unlink()
+
         return save_path.stat().st_size
 
     @staticmethod
@@ -560,30 +626,12 @@ class HLS:
                 continue
             # TODO: Add a way to specify which supported key system to use
             # TODO: Add support for 'SAMPLE-AES', 'AES-CTR', 'AES-CBC', 'ClearKey'
-            # if encryption_data and encryption_data[0] == key:
-            #     # no need to re-obtain the exact same encryption data
-            #     break
             elif key.method == "AES-128":
                 return key
-                # # TODO: Use a session instead of creating a new connection within
-                # encryption_data = (key, ClearKey.from_m3u_key(key, proxy))
-                # break
             elif key.method == "ISO-23001-7":
                 return key
-                # encryption_data = (key, Widevine(
-                #     pssh=PSSH.new(
-                #         key_ids=[key.uri.split(",")[-1]],
-                #         system_id=PSSH.SystemId.Widevine
-                #     )
-                # ))
-                # break
             elif key.keyformat and key.keyformat.lower() == WidevineCdm.urn:
                 return key
-                # encryption_data = (key, Widevine(
-                #     pssh=PSSH(key.uri.split(",")[-1]),
-                #     **key._extra_params  # noqa
-                # ))
-                # break
             else:
                 unsupported_systems.append(key.method + (f" ({key.keyformat})" if key.keyformat else ""))
         else:
@@ -592,21 +640,26 @@ class HLS:
     @staticmethod
     def get_drm(
         key: Union[m3u8.model.SessionKey, m3u8.model.Key],
-        proxy: Optional[str] = None
+        session: Optional[requests.Session] = None
     ) -> DRM_T:
         """
         Convert HLS EXT-X-KEY data to an initialized DRM object.
 
         Parameters:
             key: m3u8 key system (EXT-X-KEY) object.
-            proxy: Optional proxy string used for requesting AES-128 URIs.
+            session: Optional session used to request AES-128 URIs.
+                Useful to set headers, proxies, cookies, and so forth.
 
         Raises a NotImplementedError if the key system is not supported.
         """
+        if not isinstance(session, (Session, type(None))):
+            raise TypeError(f"Expected session to be a {Session}, not {type(session)}")
+        if not session:
+            session = Session()
+
         # TODO: Add support for 'SAMPLE-AES', 'AES-CTR', 'AES-CBC', 'ClearKey'
         if key.method == "AES-128":
-            # TODO: Use a session instead of creating a new connection within
+            drm = ClearKey.from_m3u_key(key, session)
-            drm = ClearKey.from_m3u_key(key, proxy)
         elif key.method == "ISO-23001-7":
             drm = Widevine(
                 pssh=PSSH.new(

devine/core/proxies/basic.py

@@ -1,13 +1,20 @@
 import random
-from typing import Optional
+import re
+from typing import Optional, Union
+
+from requests.utils import prepend_scheme_if_needed
+from urllib3.util import parse_url
 
 from devine.core.proxies.proxy import Proxy
 
 
 class Basic(Proxy):
-    def __init__(self, **countries):
+    def __init__(self, **countries: dict[str, Union[str, list[str]]]):
         """Basic Proxy Service using Proxies specified in the config."""
-        self.countries = countries
+        self.countries = {
+            k.lower(): v
+            for k, v in countries.items()
+        }
 
     def __repr__(self) -> str:
         countries = len(self.countries)
@@ -17,14 +24,35 @@ class Basic(Proxy):
 
     def get_proxy(self, query: str) -> Optional[str]:
         """Get a proxy URI from the config."""
-        servers = self.countries.get(query)
+        query = query.lower()
+
+        match = re.match(r"^([a-z]{2})(\d+)?$", query, re.IGNORECASE)
+        if not match:
+            raise ValueError(f"The query \"{query}\" was not recognized...")
+
+        country_code = match.group(1)
+        entry = match.group(2)
+
+        servers: Optional[Union[str, list[str]]] = self.countries.get(country_code)
         if not servers:
-            return
+            return None
 
-        proxy = random.choice(servers)
+        if isinstance(servers, str):
+            proxy = servers
+        elif entry:
+            try:
+                proxy = servers[int(entry) - 1]
+            except IndexError:
+                raise ValueError(
+                    f"There's only {len(servers)} prox{'y' if len(servers) == 1 else 'ies'} "
+                    f"for \"{country_code}\"..."
+                )
+        else:
+            proxy = random.choice(servers)
 
-        if "://" not in proxy:
+        proxy = prepend_scheme_if_needed(proxy, "http")
-            # TODO: Improve the test for a valid URI
+        parsed_proxy = parse_url(proxy)
+        if not parsed_proxy.host:
             raise ValueError(f"The proxy '{proxy}' is not a valid proxy URI supported by Python-Requests.")
 
         return proxy

devine/core/proxies/hola.py

@@ -3,8 +3,8 @@ import re
 import subprocess
 from typing import Optional
 
+from devine.core import binaries
 from devine.core.proxies.proxy import Proxy
-from devine.core.utilities import get_binary_path
 
 
 class Hola(Proxy):
@@ -13,7 +13,7 @@ class Hola(Proxy):
         Proxy Service using Hola's direct connections via the hola-proxy project.
         https://github.com/Snawoot/hola-proxy
         """
-        self.binary = get_binary_path("hola-proxy")
+        self.binary = binaries.HolaProxy
         if not self.binary:
             raise EnvironmentError("hola-proxy executable not found but is required for the Hola proxy provider.")
 

devine/core/search_result.py

@@ -0,0 +1,44 @@
+from typing import Optional, Union
+
+
+class SearchResult:
+    def __init__(
+        self,
+        id_: Union[str, int],
+        title: str,
+        description: Optional[str] = None,
+        label: Optional[str] = None,
+        url: Optional[str] = None
+    ):
+        """
+        A Search Result for any support Title Type.
+
+        Parameters:
+            id_: The search result's Title ID.
+            title: The primary display text, e.g., the Title's Name.
+            description: The secondary display text, e.g., the Title's Description or
+                further title information.
+            label: The tertiary display text. This will typically be used to display
+                an informative label or tag to the result. E.g., "unavailable", the
+                title's price tag, region, etc.
+            url: A hyperlink to the search result or title's page.
+        """
+        if not isinstance(id_, (str, int)):
+            raise TypeError(f"Expected id_ to be a {str} or {int}, not {type(id_)}")
+        if not isinstance(title, str):
+            raise TypeError(f"Expected title to be a {str}, not {type(title)}")
+        if not isinstance(description, (str, type(None))):
+            raise TypeError(f"Expected description to be a {str}, not {type(description)}")
+        if not isinstance(label, (str, type(None))):
+            raise TypeError(f"Expected label to be a {str}, not {type(label)}")
+        if not isinstance(url, (str, type(None))):
+            raise TypeError(f"Expected url to be a {str}, not {type(url)}")
+
+        self.id = id_
+        self.title = title
+        self.description = description
+        self.label = label
+        self.url = url
+
+
+__all__ = ("SearchResult",)

devine/core/service.py

@@ -1,11 +1,14 @@
 import base64
 import logging
 from abc import ABCMeta, abstractmethod
+from collections.abc import Generator
 from http.cookiejar import CookieJar
+from pathlib import Path
 from typing import Optional, Union
 from urllib.parse import urlparse
 
 import click
+import m3u8
 import requests
 from requests.adapters import HTTPAdapter, Retry
 from rich.padding import Padding
@@ -16,6 +19,8 @@ from devine.core.config import config
 from devine.core.console import console
 from devine.core.constants import AnyTrack
 from devine.core.credential import Credential
+from devine.core.drm import DRM_T
+from devine.core.search_result import SearchResult
 from devine.core.titles import Title_T, Titles_T
 from devine.core.tracks import Chapters, Tracks
 from devine.core.utilities import get_ip_info
@@ -96,9 +101,6 @@ class Service(metaclass=ABCMeta):
                 backoff_factor=0.2,
                 status_forcelist=[429, 500, 502, 503, 504]
             ),
-            # 16 connections is used for byte-ranged downloads
-            # double it to allow for 16 non-related connections
-            pool_maxsize=16 * 2,
             pool_block=True
         ))
         session.mount("http://", session.adapters["https://"])
@@ -123,6 +125,17 @@ class Service(metaclass=ABCMeta):
                 raise TypeError(f"Expected cookies to be a {CookieJar}, not {cookies!r}.")
             self.session.cookies.update(cookies)
 
+    def search(self) -> Generator[SearchResult, None, None]:
+        """
+        Search by query for titles from the Service.
+
+        The query must be taken as a CLI argument by the Service class.
+        Ideally just re-use the title ID argument (i.e. self.title).
+
+        Search results will be displayed in the order yielded.
+        """
+        raise NotImplementedError(f"Search functionality has not been implemented by {self.__class__.__name__}")
+
     def get_widevine_service_certificate(self, *, challenge: bytes, title: Title_T, track: AnyTrack) \
             -> Union[bytes, str]:
         """
@@ -225,5 +238,53 @@ class Service(metaclass=ABCMeta):
         option `chapter_fallback_name`. For example, `"Chapter {i:02}"` for "Chapter 01".
         """
 
+    # Optional Event methods
+
+    def on_segment_downloaded(self, track: AnyTrack, segment: Path) -> None:
+        """
+        Called when one of a Track's Segments has finished downloading.
+
+        Parameters:
+            track: The Track object that had a Segment downloaded.
+            segment: The Path to the Segment that was downloaded.
+        """
+
+    def on_track_downloaded(self, track: AnyTrack) -> None:
+        """
+        Called when a Track has finished downloading.
+
+        Parameters:
+            track: The Track object that was downloaded.
+        """
+
+    def on_track_decrypted(self, track: AnyTrack, drm: DRM_T, segment: Optional[m3u8.Segment] = None) -> None:
+        """
+        Called when a Track has finished decrypting.
+
+        Parameters:
+            track: The Track object that was decrypted.
+            drm: The DRM object it decrypted with.
+            segment: The HLS segment information that was decrypted.
+        """
+
+    def on_track_repacked(self, track: AnyTrack) -> None:
+        """
+        Called when a Track has finished repacking.
+
+        Parameters:
+            track: The Track object that was repacked.
+        """
+
+    def on_track_multiplex(self, track: AnyTrack) -> None:
+        """
+        Called when a Track is about to be Multiplexed into a Container.
+
+        Note: Right now only MKV containers are multiplexed but in the future
+        this may also be called when multiplexing to other containers like
+        MP4 via ffmpeg/mp4box.
+
+        Parameters:
+            track: The Track object that was repacked.
+        """
 
 __all__ = ("Service",)

devine/core/tracks/attachment.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import mimetypes
+from pathlib import Path
+from typing import Optional, Union
+from zlib import crc32
+
+
+class Attachment:
+    def __init__(
+        self,
+        path: Union[Path, str],
+        name: Optional[str] = None,
+        mime_type: Optional[str] = None,
+        description: Optional[str] = None
+    ):
+        """
+        Create a new Attachment.
+
+        If name is not provided it will use the file name (without extension).
+        If mime_type is not provided, it will try to guess it.
+        """
+        if not isinstance(path, (str, Path)):
+            raise ValueError("The attachment path must be provided.")
+        if not isinstance(name, (str, type(None))):
+            raise ValueError("The attachment name must be provided.")
+
+        path = Path(path)
+        if not path.exists():
+            raise ValueError("The attachment file does not exist.")
+
+        name = (name or path.stem).strip()
+        mime_type = (mime_type or "").strip() or None
+        description = (description or "").strip() or None
+
+        if not mime_type:
+            mime_type = {
+                ".ttf": "application/x-truetype-font",
+                ".otf": "application/vnd.ms-opentype"
+            }.get(path.suffix.lower(), mimetypes.guess_type(path)[0])
+            if not mime_type:
+                raise ValueError("The attachment mime-type could not be automatically detected.")
+
+        self.path = path
+        self.name = name
+        self.mime_type = mime_type
+        self.description = description
+
+    def __repr__(self) -> str:
+        return "{name}({items})".format(
+            name=self.__class__.__name__,
+            items=", ".join([f"{k}={repr(v)}" for k, v in self.__dict__.items()])
+        )
+
+    def __str__(self) -> str:
+        return " | ".join(filter(bool, [
+            "ATT",
+            self.name,
+            self.mime_type,
+            self.description
+        ]))
+
+    @property
+    def id(self) -> str:
+        """Compute an ID from the attachment data."""
+        checksum = crc32(self.path.read_bytes())
+        return hex(checksum)
+
+
+__all__ = ("Attachment",)

devine/core/tracks/audio.py

@@ -64,18 +64,80 @@ class Audio(Track):
                 return Audio.Codec.OGG
             raise ValueError(f"The Content Profile '{profile}' is not a supported Audio Codec")
 
-    def __init__(self, *args: Any, codec: Audio.Codec, bitrate: Union[str, int, float],
+    def __init__(
-                 channels: Optional[Union[str, int, float]] = None, joc: int = 0, descriptive: bool = False,
+        self,
-                 **kwargs: Any):
+        *args: Any,
+        codec: Optional[Audio.Codec] = None,
+        bitrate: Optional[Union[str, int, float]] = None,
+        channels: Optional[Union[str, int, float]] = None,
+        joc: Optional[int] = None,
+        descriptive: Union[bool, int] = False,
+        **kwargs: Any
+    ):
+        """
+        Create a new Audio track object.
+
+        Parameters:
+            codec: An Audio.Codec enum representing the audio codec.
+                If not specified, MediaInfo will be used to retrieve the codec
+                once the track has been downloaded.
+            bitrate: A number or float representing the average bandwidth in bytes/s.
+                Float values are rounded up to the nearest integer.
+            channels: A number, float, or string representing the number of audio channels.
+                Strings may represent numbers or floats. Expanded layouts like 7.1.1 is
+                not supported. All numbers and strings will be cast to float.
+            joc: The number of Joint-Object-Coding Channels/Objects in the audio stream.
+            descriptive: Mark this audio as being descriptive audio for the blind.
+
+        Note: If codec, bitrate, channels, or joc is not specified some checks may be
+        skipped or assume a value. Specifying as much information as possible is highly
+        recommended.
+        """
         super().__init__(*args, **kwargs)
-        # required
+
+        if not isinstance(codec, (Audio.Codec, type(None))):
+            raise TypeError(f"Expected codec to be a {Audio.Codec}, not {codec!r}")
+        if not isinstance(bitrate, (str, int, float, type(None))):
+            raise TypeError(f"Expected bitrate to be a {str}, {int}, or {float}, not {bitrate!r}")
+        if not isinstance(channels, (str, int, float, type(None))):
+            raise TypeError(f"Expected channels to be a {str}, {int}, or {float}, not {channels!r}")
+        if not isinstance(joc, (int, type(None))):
+            raise TypeError(f"Expected joc to be a {int}, not {joc!r}")
+        if (
+            not isinstance(descriptive, (bool, int)) or
+            (isinstance(descriptive, int) and descriptive not in (0, 1))
+        ):
+            raise TypeError(f"Expected descriptive to be a {bool} or bool-like {int}, not {descriptive!r}")
+
         self.codec = codec
-        self.bitrate = int(math.ceil(float(bitrate))) if bitrate else None
+
-        self.channels = self.parse_channels(channels) if channels else None
+        try:
-        # optional
+            self.bitrate = int(math.ceil(float(bitrate))) if bitrate else None
+        except (ValueError, TypeError) as e:
+            raise ValueError(f"Expected bitrate to be a number or float, {e}")
+
+        try:
+            self.channels = self.parse_channels(channels) if channels else None
+        except (ValueError, NotImplementedError) as e:
+            raise ValueError(f"Expected channels to be a number, float, or a string, {e}")
+
         self.joc = joc
         self.descriptive = bool(descriptive)
 
+    def __str__(self) -> str:
+        return " | ".join(filter(bool, [
+            "AUD",
+            f"[{self.codec.value}]" if self.codec else None,
+            str(self.language),
+            ", ".join(filter(bool, [
+                str(self.channels) if self.channels else None,
+                f"JOC {self.joc}" if self.joc else None,
+            ])),
+            f"{self.bitrate // 1000} kb/s" if self.bitrate else None,
+            self.get_track_name(),
+            self.edition
+        ]))
+
     @staticmethod
     def parse_channels(channels: Union[str, int, float]) -> float:
         """
@@ -109,16 +171,5 @@ class Audio(Track):
             track_name += flag
         return track_name or None
 
-    def __str__(self) -> str:
-        return " | ".join(filter(bool, [
-            "AUD",
-            f"[{self.codec.value}]",
-            str(self.channels or "?") + (f" (JOC {self.joc})" if self.joc else ""),
-            f"{self.bitrate // 1000 if self.bitrate else '?'} kb/s",
-            str(self.language),
-            self.get_track_name(),
-            self.edition
-        ]))
-
 
 __all__ = ("Audio",)

devine/core/tracks/chapter.py

@@ -41,7 +41,7 @@ class Chapter:
                 seconds, ms = divmod(int(remainder * 1000), 1000)
             else:
                 raise TypeError
-            timestamp = f"{hours:02}:{minutes:02}:{seconds:02}.{str(ms).zfill(3)[:3]}"
+            timestamp = f"{int(hours):02}:{int(minutes):02}:{int(seconds):02}.{str(ms).zfill(3)[:3]}"
 
         timestamp_m = TIMESTAMP_FORMAT.match(timestamp)
         if not timestamp_m:

devine/core/tracks/subtitle.py

@@ -4,19 +4,23 @@ import re
 import subprocess
 from collections import defaultdict
 from enum import Enum
+from functools import partial
 from io import BytesIO
 from pathlib import Path
-from typing import Any, Callable, Iterable, Optional
+from typing import Any, Callable, Iterable, Optional, Union
 
 import pycaption
+import requests
 from construct import Container
 from pycaption import Caption, CaptionList, CaptionNode, WebVTTReader
 from pycaption.geometry import Layout
 from pymp4.parser import MP4
 from subtitle_filter import Subtitles
 
+from devine.core import binaries
 from devine.core.tracks.track import Track
-from devine.core.utilities import get_binary_path, try_ensure_utf8
+from devine.core.utilities import try_ensure_utf8
+from devine.core.utils.webvtt import merge_segmented_webvtt
 
 
 class Subtitle(Track):
@@ -72,22 +76,22 @@ class Subtitle(Track):
                 return Subtitle.Codec.TimedTextMarkupLang
             raise ValueError(f"The Content Profile '{profile}' is not a supported Subtitle Codec")
 
-    def __init__(self, *args: Any, codec: Subtitle.Codec, cc: bool = False, sdh: bool = False, forced: bool = False,
+    def __init__(
-                 **kwargs: Any):
+        self,
+        *args: Any,
+        codec: Optional[Subtitle.Codec] = None,
+        cc: bool = False,
+        sdh: bool = False,
+        forced: bool = False,
+        **kwargs: Any
+    ):
         """
-        Information on Subtitle Types:
+        Create a new Subtitle track object.
-            https://bit.ly/2Oe4fLC (3PlayMedia Blog on SUB vs CC vs SDH).
-            However, I wouldn't pay much attention to the claims about SDH needing to
-            be in the original source language. It's logically not true.
-
-            CC == Closed Captions. Source: Basically every site.
-            SDH = Subtitles for the Deaf or Hard-of-Hearing. Source: Basically every site.
-            HOH = Exact same as SDH. Is a term used in the UK. Source: https://bit.ly/2PGJatz (ICO UK)
-
-            More in-depth information, examples, and stuff to look for can be found in the Parameter
-            explanation list below.
 
         Parameters:
+            codec: A Subtitle.Codec enum representing the subtitle format.
+                If not specified, MediaInfo will be used to retrieve the format
+                once the track has been downloaded.
             cc: Closed Caption.
                 - Intended as if you couldn't hear the audio at all.
                 - Can have Sound as well as Dialogue, but doesn't have to.
@@ -123,20 +127,57 @@ class Subtitle(Track):
                      no other way to reliably work with Forced subtitles where multiple
                      forced subtitles may be in the output file. Just know what to expect
                      with "forced" subtitles.
+
+        Note: If codec is not specified some checks may be skipped or assume a value.
+        Specifying as much information as possible is highly recommended.
+
+        Information on Subtitle Types:
+            https://bit.ly/2Oe4fLC (3PlayMedia Blog on SUB vs CC vs SDH).
+            However, I wouldn't pay much attention to the claims about SDH needing to
+            be in the original source language. It's logically not true.
+
+            CC == Closed Captions. Source: Basically every site.
+            SDH = Subtitles for the Deaf or Hard-of-Hearing. Source: Basically every site.
+            HOH = Exact same as SDH. Is a term used in the UK. Source: https://bit.ly/2PGJatz (ICO UK)
+
+            More in-depth information, examples, and stuff to look for can be found in the Parameter
+            explanation list above.
         """
         super().__init__(*args, **kwargs)
+
+        if not isinstance(codec, (Subtitle.Codec, type(None))):
+            raise TypeError(f"Expected codec to be a {Subtitle.Codec}, not {codec!r}")
+        if not isinstance(cc, (bool, int)) or (isinstance(cc, int) and cc not in (0, 1)):
+            raise TypeError(f"Expected cc to be a {bool} or bool-like {int}, not {cc!r}")
+        if not isinstance(sdh, (bool, int)) or (isinstance(sdh, int) and sdh not in (0, 1)):
+            raise TypeError(f"Expected sdh to be a {bool} or bool-like {int}, not {sdh!r}")
+        if not isinstance(forced, (bool, int)) or (isinstance(forced, int) and forced not in (0, 1)):
+            raise TypeError(f"Expected forced to be a {bool} or bool-like {int}, not {forced!r}")
+
         self.codec = codec
+
         self.cc = bool(cc)
         self.sdh = bool(sdh)
+        self.forced = bool(forced)
+
         if self.cc and self.sdh:
             raise ValueError("A text track cannot be both CC and SDH.")
-        self.forced = bool(forced)
+
-        if (self.cc or self.sdh) and self.forced:
+        if self.forced and (self.cc or self.sdh):
             raise ValueError("A text track cannot be CC/SDH as well as Forced.")
 
+        # TODO: Migrate to new event observer system
         # Called after Track has been converted to another format
         self.OnConverted: Optional[Callable[[Subtitle.Codec], None]] = None
 
+    def __str__(self) -> str:
+        return " | ".join(filter(bool, [
+            "SUB",
+            f"[{self.codec.value}]" if self.codec else None,
+            str(self.language),
+            self.get_track_name()
+        ]))
+
     def get_track_name(self) -> Optional[str]:
         """Return the base Track Name."""
         track_name = super().get_track_name() or ""
@@ -147,6 +188,42 @@ class Subtitle(Track):
             track_name += flag
         return track_name or None
 
+    def download(
+        self,
+        session: requests.Session,
+        prepare_drm: partial,
+        max_workers: Optional[int] = None,
+        progress: Optional[partial] = None
+    ):
+        super().download(session, prepare_drm, max_workers, progress)
+        if not self.path:
+            return
+
+        if self.codec == Subtitle.Codec.fTTML:
+            self.convert(Subtitle.Codec.TimedTextMarkupLang)
+        elif self.codec == Subtitle.Codec.fVTT:
+            self.convert(Subtitle.Codec.WebVTT)
+        elif self.codec == Subtitle.Codec.WebVTT:
+            text = self.path.read_text("utf8")
+            if self.descriptor == Track.Descriptor.DASH:
+                if len(self.data["dash"]["segment_durations"]) > 1:
+                    text = merge_segmented_webvtt(
+                        text,
+                        segment_durations=self.data["dash"]["segment_durations"],
+                        timescale=self.data["dash"]["timescale"]
+                    )
+            elif self.descriptor == Track.Descriptor.HLS:
+                if len(self.data["hls"]["segment_durations"]) > 1:
+                    text = merge_segmented_webvtt(
+                        text,
+                        segment_durations=self.data["hls"]["segment_durations"],
+                        timescale=1  # ?
+                    )
+            caption_set = pycaption.WebVTTReader().read(text)
+            Subtitle.merge_same_cues(caption_set)
+            subtitle_text = pycaption.WebVTTWriter().write(caption_set)
+            self.path.write_text(subtitle_text, encoding="utf8")
+
     def convert(self, codec: Subtitle.Codec) -> Path:
         """
         Convert this Subtitle to another Format.
@@ -178,14 +255,13 @@ class Subtitle(Track):
 
         output_path = self.path.with_suffix(f".{codec.value.lower()}")
 
-        sub_edit_executable = get_binary_path("SubtitleEdit")
+        if binaries.SubtitleEdit and self.codec not in (Subtitle.Codec.fTTML, Subtitle.Codec.fVTT):
-        if sub_edit_executable and self.codec not in (Subtitle.Codec.fTTML, Subtitle.Codec.fVTT):
             sub_edit_format = {
                 Subtitle.Codec.SubStationAlphav4: "AdvancedSubStationAlpha",
                 Subtitle.Codec.TimedTextMarkupLang: "TimedText1.0"
             }.get(codec, codec.name)
             sub_edit_args = [
-                sub_edit_executable,
+                binaries.SubtitleEdit,
                 "/Convert", self.path, sub_edit_format,
                 f"/outputfilename:{output_path.name}",
                 "/encoding:utf8"
@@ -214,7 +290,7 @@ class Subtitle(Track):
 
             output_path.write_text(subtitle_text, encoding="utf8")
 
-        self.swap(output_path)
+        self.path = output_path
         self.codec = codec
 
         if callable(self.OnConverted):
@@ -253,14 +329,7 @@ class Subtitle(Track):
                 caption_lists[language] = caption_list
                 caption_set: pycaption.CaptionSet = pycaption.CaptionSet(caption_lists)
             elif codec == Subtitle.Codec.WebVTT:
-                text = try_ensure_utf8(data).decode("utf8")
+                text = Subtitle.space_webvtt_headers(data)
-                # Segmented VTT when merged may have the WEBVTT headers part of the next caption
-                # if they are not separated far enough from the previous caption, hence the \n\n
-                text = text. \
-                    replace("WEBVTT", "\n\nWEBVTT"). \
-                    replace("\r", ""). \
-                    replace("\n\n\n", "\n \n\n"). \
-                    replace("\n\n<", "\n<")
                 caption_set = pycaption.WebVTTReader().read(text)
             else:
                 raise ValueError(f"Unknown Subtitle format \"{codec}\"...")
@@ -277,6 +346,27 @@ class Subtitle(Track):
 
         return caption_set
 
+    @staticmethod
+    def space_webvtt_headers(data: Union[str, bytes]):
+        """
+        Space out the WEBVTT Headers from Captions.
+
+        Segmented VTT when merged may have the WEBVTT headers part of the next caption
+        as they were not separated far enough from the previous caption and ended up
+        being considered as caption text rather than the header for the next segment.
+        """
+        if isinstance(data, bytes):
+            data = try_ensure_utf8(data).decode("utf8")
+        elif not isinstance(data, str):
+            raise ValueError(f"Expecting data to be a str, not {data!r}")
+
+        text = data.replace("WEBVTT", "\n\nWEBVTT").\
+            replace("\r", "").\
+            replace("\n\n\n", "\n \n\n").\
+            replace("\n\n<", "\n<")
+
+        return text
+
     @staticmethod
     def merge_same_cues(caption_set: pycaption.CaptionSet):
         """Merge captions with the same timecodes and text as one in-place."""
@@ -445,8 +535,7 @@ class Subtitle(Track):
         if not self.path or not self.path.exists():
             raise ValueError("You must download the subtitle track first.")
 
-        executable = get_binary_path("SubtitleEdit")
+        if binaries.SubtitleEdit:
-        if executable:
             if self.codec == Subtitle.Codec.SubStationAlphav4:
                 output_format = "AdvancedSubStationAlpha"
             elif self.codec == Subtitle.Codec.TimedTextMarkupLang:
@@ -455,7 +544,7 @@ class Subtitle(Track):
                 output_format = self.codec.name
             subprocess.run(
                 [
-                    executable,
+                    binaries.SubtitleEdit,
                     "/Convert", self.path, output_format,
                     "/encoding:utf8",
                     "/overwrite",
@@ -484,8 +573,7 @@ class Subtitle(Track):
         if not self.path or not self.path.exists():
             raise ValueError("You must download the subtitle track first.")
 
-        executable = get_binary_path("SubtitleEdit")
+        if not binaries.SubtitleEdit:
-        if not executable:
             raise EnvironmentError("SubtitleEdit executable not found...")
 
         if self.codec == Subtitle.Codec.SubStationAlphav4:
@@ -497,7 +585,7 @@ class Subtitle(Track):
 
         subprocess.run(
             [
-                executable,
+                binaries.SubtitleEdit,
                 "/Convert", self.path, output_format,
                 "/ReverseRtlStartEnd",
                 "/encoding:utf8",
@@ -507,13 +595,5 @@ class Subtitle(Track):
             stdout=subprocess.DEVNULL
         )
 
-    def __str__(self) -> str:
-        return " | ".join(filter(bool, [
-            "SUB",
-            f"[{self.codec.value}]",
-            str(self.language),
-            self.get_track_name()
-        ]))
-
 
 __all__ = ("Subtitle",)

devine/core/tracks/track.py

@@ -1,97 +1,380 @@
 import base64
+import html
+import logging
 import re
 import shutil
 import subprocess
+from collections import defaultdict
+from copy import copy
 from enum import Enum
+from functools import partial
 from pathlib import Path
 from typing import Any, Callable, Iterable, Optional, Union
 from uuid import UUID
+from zlib import crc32
 
-import m3u8
-import requests
 from langcodes import Language
+from requests import Session
 
-from devine.core.constants import TERRITORY_MAP
+from devine.core import binaries
-from devine.core.drm import DRM_T
+from devine.core.config import config
-from devine.core.utilities import get_binary_path, get_boxes
+from devine.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY
+from devine.core.downloaders import aria2c, curl_impersonate, requests
+from devine.core.drm import DRM_T, Widevine
+from devine.core.events import events
+from devine.core.utilities import get_boxes, try_ensure_utf8
 from devine.core.utils.subprocess import ffprobe
 
 
 class Track:
-    class DRM(Enum):
-        pass
-
     class Descriptor(Enum):
         URL = 1  # Direct URL, nothing fancy
-        M3U = 2  # https://en.wikipedia.org/wiki/M3U (and M3U8)
+        HLS = 2  # https://en.wikipedia.org/wiki/HTTP_Live_Streaming
-        MPD = 3  # https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP
+        DASH = 3  # https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP
 
     def __init__(
         self,
-        id_: str,
         url: Union[str, list[str]],
         language: Union[Language, str],
         is_original_lang: bool = False,
         descriptor: Descriptor = Descriptor.URL,
         needs_repack: bool = False,
+        name: Optional[str] = None,
         drm: Optional[Iterable[DRM_T]] = None,
         edition: Optional[str] = None,
-        extra: Optional[Any] = None
+        downloader: Optional[Callable] = None,
+        data: Optional[Union[dict, defaultdict]] = None,
+        id_: Optional[str] = None,
     ) -> None:
-        self.id = id_
+        if not isinstance(url, (str, list)):
+            raise TypeError(f"Expected url to be a {str}, or list of {str}, not {type(url)}")
+        if not isinstance(language, (Language, str)):
+            raise TypeError(f"Expected language to be a {Language} or {str}, not {type(language)}")
+        if not isinstance(is_original_lang, bool):
+            raise TypeError(f"Expected is_original_lang to be a {bool}, not {type(is_original_lang)}")
+        if not isinstance(descriptor, Track.Descriptor):
+            raise TypeError(f"Expected descriptor to be a {Track.Descriptor}, not {type(descriptor)}")
+        if not isinstance(needs_repack, bool):
+            raise TypeError(f"Expected needs_repack to be a {bool}, not {type(needs_repack)}")
+        if not isinstance(name, (str, type(None))):
+            raise TypeError(f"Expected name to be a {str}, not {type(name)}")
+        if not isinstance(id_, (str, type(None))):
+            raise TypeError(f"Expected id_ to be a {str}, not {type(id_)}")
+        if not isinstance(edition, (str, type(None))):
+            raise TypeError(f"Expected edition to be a {str}, not {type(edition)}")
+        if not isinstance(downloader, (Callable, type(None))):
+            raise TypeError(f"Expected downloader to be a {Callable}, not {type(downloader)}")
+        if not isinstance(data, (dict, defaultdict, type(None))):
+            raise TypeError(f"Expected data to be a {dict} or {defaultdict}, not {type(data)}")
+
+        invalid_urls = ", ".join(set(type(x) for x in url if not isinstance(x, str)))
+        if invalid_urls:
+            raise TypeError(f"Expected all items in url to be a {str}, but found {invalid_urls}")
+
+        if drm is not None:
+            try:
+                iter(drm)
+            except TypeError:
+                raise TypeError(f"Expected drm to be an iterable, not {type(drm)}")
+
+        if downloader is None:
+            downloader = {
+                "aria2c": aria2c,
+                "curl_impersonate": curl_impersonate,
+                "requests": requests
+            }[config.downloader]
+
+        self.path: Optional[Path] = None
         self.url = url
-        # required basic metadata
         self.language = Language.get(language)
-        self.is_original_lang = bool(is_original_lang)
+        self.is_original_lang = is_original_lang
-        # optional io metadata
         self.descriptor = descriptor
-        self.needs_repack = bool(needs_repack)
+        self.needs_repack = needs_repack
-        # drm
+        self.name = name
         self.drm = drm
-        # extra data
         self.edition: str = edition
-        self.extra: Any = extra or {}  # allow anything for extra, but default to a dict
+        self.downloader = downloader
+        self._data: defaultdict[Any, Any] = defaultdict(dict)
+        self.data = data or {}
+
+        if self.name is None:
+            lang = Language.get(self.language)
+            if (lang.language or "").lower() == (lang.territory or "").lower():
+                lang.territory = None  # e.g. en-en, de-DE
+            reduced = lang.simplify_script()
+            extra_parts = []
+            if reduced.script is not None:
+                script = reduced.script_name(max_distance=25)
+                if script and script != "Zzzz":
+                    extra_parts.append(script)
+            if reduced.territory is not None:
+                territory = reduced.territory_name(max_distance=25)
+                if territory and territory != "ZZ":
+                    territory = territory.removesuffix(" SAR China")
+                    extra_parts.append(territory)
+            self.name = ", ".join(extra_parts) or None
+
+        if not id_:
+            this = copy(self)
+            this.url = self.url.rsplit("?", maxsplit=1)[0]
+            checksum = crc32(repr(this).encode("utf8"))
+            id_ = hex(checksum)[2:]
+
+        self.id = id_
 
         # TODO: Currently using OnFoo event naming, change to just segment_filter
         self.OnSegmentFilter: Optional[Callable] = None
 
-        # Called after one of the Track's segments have downloaded
-        self.OnSegmentDownloaded: Optional[Callable[[Path], None]] = None
-        # Called after the Track has downloaded
-        self.OnDownloaded: Optional[Callable] = None
-        # Called after the Track or one of its segments have been decrypted
-        self.OnDecrypted: Optional[Callable[[DRM_T, Optional[m3u8.Segment]], None]] = None
-        # Called after the Track has been repackaged
-        self.OnRepacked: Optional[Callable] = None
-        # Called before the Track is multiplexed
-        self.OnMultiplex: Optional[Callable] = None
-
-        # should only be set internally
-        self.path: Optional[Path] = None
-
     def __repr__(self) -> str:
         return "{name}({items})".format(
             name=self.__class__.__name__,
             items=", ".join([f"{k}={repr(v)}" for k, v in self.__dict__.items()])
         )
 
-    def __eq__(self, other: object) -> bool:
+    def __eq__(self, other: Any) -> bool:
         return isinstance(other, Track) and self.id == other.id
 
+    @property
+    def data(self) -> defaultdict[Any, Any]:
+        """
+        Arbitrary track data dictionary.
+
+        A defaultdict is used with a dict as the factory for easier
+        nested saving and safer exists-checks.
+
+        Reserved keys:
+
+        - "hls" used by the HLS class.
+          - playlist: m3u8.model.Playlist - The primary track information.
+          - media: m3u8.model.Media - The audio/subtitle track information.
+          - segment_durations: list[int] - A list of each segment's duration.
+        - "dash" used by the DASH class.
+          - manifest: lxml.ElementTree - DASH MPD manifest.
+          - period: lxml.Element - The period of this track.
+          - adaptation_set: lxml.Element - The adaptation set of this track.
+          - representation: lxml.Element - The representation of this track.
+          - timescale: int - The timescale of the track's segments.
+          - segment_durations: list[int] - A list of each segment's duration.
+
+        You should not add, change, or remove any data within reserved keys.
+        You may use their data but do note that the values of them may change
+        or be removed at any point.
+        """
+        return self._data
+
+    @data.setter
+    def data(self, value: Union[dict, defaultdict]) -> None:
+        if not isinstance(value, (dict, defaultdict)):
+            raise TypeError(f"Expected data to be a {dict} or {defaultdict}, not {type(value)}")
+        if isinstance(value, dict):
+            value = defaultdict(dict, **value)
+        self._data = value
+
+    def download(
+        self,
+        session: Session,
+        prepare_drm: partial,
+        max_workers: Optional[int] = None,
+        progress: Optional[partial] = None
+    ):
+        """Download and optionally Decrypt this Track."""
+        from devine.core.manifests import DASH, HLS
+
+        if DOWNLOAD_LICENCE_ONLY.is_set():
+            progress(downloaded="[yellow]SKIPPING")
+
+        if DOWNLOAD_CANCELLED.is_set():
+            progress(downloaded="[yellow]SKIPPED")
+            return
+
+        log = logging.getLogger("track")
+
+        proxy = next(iter(session.proxies.values()), None)
+
+        track_type = self.__class__.__name__
+        save_path = config.directories.temp / f"{track_type}_{self.id}.mp4"
+        if track_type == "Subtitle":
+            save_path = save_path.with_suffix(f".{self.codec.extension}")
+
+        if self.descriptor != self.Descriptor.URL:
+            save_dir = save_path.with_name(save_path.name + "_segments")
+        else:
+            save_dir = save_path.parent
+
+        def cleanup():
+            # track file (e.g., "foo.mp4")
+            save_path.unlink(missing_ok=True)
+            # aria2c control file (e.g., "foo.mp4.aria2" or "foo.mp4.aria2__temp")
+            save_path.with_suffix(f"{save_path.suffix}.aria2").unlink(missing_ok=True)
+            save_path.with_suffix(f"{save_path.suffix}.aria2__temp").unlink(missing_ok=True)
+            if save_dir.exists() and save_dir.name.endswith("_segments"):
+                shutil.rmtree(save_dir)
+
+        if not DOWNLOAD_LICENCE_ONLY.is_set():
+            if config.directories.temp.is_file():
+                raise ValueError(f"Temp Directory '{config.directories.temp}' must be a Directory, not a file")
+
+            config.directories.temp.mkdir(parents=True, exist_ok=True)
+
+            # Delete any pre-existing temp files matching this track.
+            # We can't re-use or continue downloading these tracks as they do not use a
+            # lock file. Or at least the majority don't. Even if they did I've encountered
+            # corruptions caused by sudden interruptions to the lock file.
+            cleanup()
+
+        try:
+            if self.descriptor == self.Descriptor.HLS:
+                HLS.download_track(
+                    track=self,
+                    save_path=save_path,
+                    save_dir=save_dir,
+                    progress=progress,
+                    session=session,
+                    proxy=proxy,
+                    max_workers=max_workers,
+                    license_widevine=prepare_drm
+                )
+            elif self.descriptor == self.Descriptor.DASH:
+                DASH.download_track(
+                    track=self,
+                    save_path=save_path,
+                    save_dir=save_dir,
+                    progress=progress,
+                    session=session,
+                    proxy=proxy,
+                    max_workers=max_workers,
+                    license_widevine=prepare_drm
+                )
+            elif self.descriptor == self.Descriptor.URL:
+                try:
+                    if not self.drm and track_type in ("Video", "Audio"):
+                        # the service might not have explicitly defined the `drm` property
+                        # try find widevine DRM information from the init data of URL
+                        try:
+                            self.drm = [Widevine.from_track(self, session)]
+                        except Widevine.Exceptions.PSSHNotFound:
+                            # it might not have Widevine DRM, or might not have found the PSSH
+                            log.warning("No Widevine PSSH was found for this track, is it DRM free?")
+
+                    if self.drm:
+                        track_kid = self.get_key_id(session=session)
+                        drm = self.drm[0]  # just use the first supported DRM system for now
+                        if isinstance(drm, Widevine):
+                            # license and grab content keys
+                            if not prepare_drm:
+                                raise ValueError("prepare_drm func must be supplied to use Widevine DRM")
+                            progress(downloaded="LICENSING")
+                            prepare_drm(drm, track_kid=track_kid)
+                            progress(downloaded="[yellow]LICENSED")
+                    else:
+                        drm = None
+
+                    if DOWNLOAD_LICENCE_ONLY.is_set():
+                        progress(downloaded="[yellow]SKIPPED")
+                    else:
+                        for status_update in self.downloader(
+                            urls=self.url,
+                            output_dir=save_path.parent,
+                            filename=save_path.name,
+                            headers=session.headers,
+                            cookies=session.cookies,
+                            proxy=proxy,
+                            max_workers=max_workers
+                        ):
+                            file_downloaded = status_update.get("file_downloaded")
+                            if not file_downloaded:
+                                progress(**status_update)
+
+                        # see https://github.com/devine-dl/devine/issues/71
+                        save_path.with_suffix(f"{save_path.suffix}.aria2__temp").unlink(missing_ok=True)
+
+                        self.path = save_path
+                        events.emit(events.Types.TRACK_DOWNLOADED, track=self)
+
+                        if drm:
+                            progress(downloaded="Decrypting", completed=0, total=100)
+                            drm.decrypt(save_path)
+                            self.drm = None
+                            events.emit(
+                                events.Types.TRACK_DECRYPTED,
+                                track=self,
+                                drm=drm,
+                                segment=None
+                            )
+                            progress(downloaded="Decrypted", completed=100)
+
+                        if track_type == "Subtitle" and self.codec.name not in ("fVTT", "fTTML"):
+                            track_data = self.path.read_bytes()
+                            track_data = try_ensure_utf8(track_data)
+                            track_data = track_data.decode("utf8"). \
+                                replace("‎", html.unescape("‎")). \
+                                replace("‏", html.unescape("‏")). \
+                                encode("utf8")
+                            self.path.write_bytes(track_data)
+
+                        progress(downloaded="Downloaded")
+                except KeyboardInterrupt:
+                    DOWNLOAD_CANCELLED.set()
+                    progress(downloaded="[yellow]CANCELLED")
+                    raise
+                except Exception:
+                    DOWNLOAD_CANCELLED.set()
+                    progress(downloaded="[red]FAILED")
+                    raise
+        except (Exception, KeyboardInterrupt):
+            if not DOWNLOAD_LICENCE_ONLY.is_set():
+                cleanup()
+            raise
+
+        if DOWNLOAD_CANCELLED.is_set():
+            # we stopped during the download, let's exit
+            return
+
+        if not DOWNLOAD_LICENCE_ONLY.is_set():
+            if self.path.stat().st_size <= 3:  # Empty UTF-8 BOM == 3 bytes
+                raise IOError("Download failed, the downloaded file is empty.")
+
+        events.emit(events.Types.TRACK_DOWNLOADED, track=self)
+
+    def delete(self) -> None:
+        if self.path:
+            self.path.unlink()
+            self.path = None
+
+    def move(self, target: Union[Path, str]) -> Path:
+        """
+        Move the Track's file from current location, to target location.
+        This will overwrite anything at the target path.
+
+        Raises:
+            TypeError: If the target argument is not the expected type.
+            ValueError: If track has no file to move, or the target does not exist.
+            OSError: If the file somehow failed to move.
+
+        Returns the new location of the track.
+        """
+        if not isinstance(target, (str, Path)):
+            raise TypeError(f"Expected {target} to be a {Path} or {str}, not {type(target)}")
+
+        if not self.path:
+            raise ValueError("Track has no file to move")
+
+        if not isinstance(target, Path):
+            target = Path(target)
+
+        if not target.exists():
+            raise ValueError(f"Target file {repr(target)} does not exist")
+
+        moved_to = Path(shutil.move(self.path, target))
+        if moved_to.resolve() != target.resolve():
+            raise OSError(f"Failed to move {self.path} to {target}")
+
+        self.path = target
+        return target
+
     def get_track_name(self) -> Optional[str]:
-        """Return the base Track Name. This may be enhanced in sub-classes."""
+        """Get the Track Name."""
-        if (self.language.language or "").lower() == (self.language.territory or "").lower():
+        return self.name
-            self.language.territory = None  # e.g. en-en, de-DE
-        if self.language.territory == "US":
-            self.language.territory = None
-        reduced = self.language.simplify_script()
-        extra_parts = []
-        if reduced.script is not None:
-            extra_parts.append(reduced.script_name(max_distance=25))
-        if reduced.territory is not None:
-            territory = reduced.territory_name(max_distance=25)
-            extra_parts.append(TERRITORY_MAP.get(territory, territory))
-        return ", ".join(extra_parts) or None
 
     def get_key_id(self, init_data: Optional[bytes] = None, *args, **kwargs) -> Optional[UUID]:
         """
@@ -117,7 +400,6 @@ class Track:
         if not isinstance(init_data, bytes):
             raise TypeError(f"Expected init_data to be bytes, not {init_data!r}")
 
-        # try get via ffprobe, needed for non mp4 data e.g. WEBM from Google Play
         probe = ffprobe(init_data)
         if probe:
             for stream in probe.get("streams") or []:
@@ -125,14 +407,12 @@ class Track:
                 if enc_key_id:
                     return UUID(bytes=base64.b64decode(enc_key_id))
 
-        # look for track encryption mp4 boxes
         for tenc in get_boxes(init_data, b"tenc"):
             if tenc.key_ID.int != 0:
                 return tenc.key_ID
 
-        # look for UUID mp4 boxes holding track encryption mp4 boxes
         for uuid_box in get_boxes(init_data, b"uuid"):
-            if uuid_box.extended_type == UUID("8974dbce-7be7-4c51-84f9-7148f9882554"):
+            if uuid_box.extended_type == UUID("8974dbce-7be7-4c51-84f9-7148f9882554"):  # tenc
                 tenc = uuid_box.data
                 if tenc.key_ID.int != 0:
                     return tenc.key_ID
@@ -142,7 +422,7 @@ class Track:
         maximum_size: int = 20000,
         url: Optional[str] = None,
         byte_range: Optional[str] = None,
-        session: Optional[requests.Session] = None
+        session: Optional[Session] = None
     ) -> bytes:
         """
         Get the Track's Initial Segment Data Stream.
@@ -166,20 +446,24 @@ class Track:
             byte_range: Range of bytes to download from the explicit or implicit URL.
             session: Session context, e.g., authorization and headers.
         """
-        if not session:
+        if not isinstance(maximum_size, int):
-            session = requests.Session()
+            raise TypeError(f"Expected maximum_size to be an {int}, not {type(maximum_size)}")
+        if not isinstance(url, (str, type(None))):
+            raise TypeError(f"Expected url to be a {str}, not {type(url)}")
+        if not isinstance(byte_range, (str, type(None))):
+            raise TypeError(f"Expected byte_range to be a {str}, not {type(byte_range)}")
+        if not isinstance(session, (Session, type(None))):
+            raise TypeError(f"Expected session to be a {Session}, not {type(session)}")
 
-        if self.descriptor != self.Descriptor.URL and not url:
-            # We cannot know which init map from the HLS or DASH playlist is actually used.
-            # For DASH this could be from any adaptation set, any period, e.t.c.
-            # For HLS we could make some assumptions, but it's best that it is explicitly provided.
-            raise ValueError(
-                f"An explicit URL to an init map or file must be provided for {self.descriptor.name} tracks."
-            )
-
-        url = url or self.url
         if not url:
-            raise ValueError("The track must have an URL to point towards it's data.")
+            if self.descriptor != self.Descriptor.URL:
+                raise ValueError(f"An explicit URL must be provided for {self.descriptor.name} tracks")
+            if not self.url:
+                raise ValueError("An explicit URL must be provided as the track has no URL")
+            url = self.url
+
+        if not session:
+            session = Session()
 
         content_length = maximum_size
 
@@ -196,7 +480,6 @@ class Track:
             if "Content-Length" in size_test.headers:
                 content_length_header = int(size_test.headers["Content-Length"])
                 if content_length_header > 0:
-                    # use whichever is smaller in case this is a large file
                     content_length = min(content_length_header, maximum_size)
             range_test = session.head(url, headers={"Range": "bytes=0-1"})
             if range_test.status_code == 206:
@@ -212,8 +495,6 @@ class Track:
             res.raise_for_status()
             init_data = res.content
         else:
-            # Take advantage of streaming support to take just the first n bytes
-            # This is a hacky alternative to HTTP's Range on unsupported servers
             init_data = None
             with session.get(url, stream=True) as s:
                 for chunk in s.iter_content(content_length):
@@ -224,17 +505,11 @@ class Track:
 
         return init_data
 
-    def delete(self) -> None:
-        if self.path:
-            self.path.unlink()
-            self.path = None
-
     def repackage(self) -> None:
         if not self.path or not self.path.exists():
             raise ValueError("Cannot repackage a Track that has not been downloaded.")
 
-        executable = get_binary_path("ffmpeg")
+        if not binaries.FFMPEG:
-        if not executable:
             raise EnvironmentError("FFmpeg executable \"ffmpeg\" was not found but is required for this call.")
 
         original_path = self.path
@@ -243,7 +518,7 @@ class Track:
         def _ffmpeg(extra_args: list[str] = None):
             subprocess.run(
                 [
-                    executable, "-hide_banner",
+                    binaries.FFMPEG, "-hide_banner",
                     "-loglevel", "error",
                     "-i", original_path,
                     *(extra_args or []),
@@ -267,36 +542,8 @@ class Track:
             else:
                 raise
 
-        self.swap(output_path)
+        original_path.unlink()
-        self.move(original_path)
+        self.path = output_path
-
-    def move(self, target: Union[str, Path]) -> bool:
-        """
-        Move the Track's file from current location, to target location.
-        This will overwrite anything at the target path.
-        """
-        if not self.path:
-            return False
-        target = Path(target)
-
-        ok = Path(shutil.move(self.path, target)).resolve() == target.resolve()
-        if ok:
-            self.path = target
-        return ok
-
-    def swap(self, target: Union[str, Path]) -> bool:
-        """
-        Swaps the Track's file with the Target file. The current Track's file is deleted.
-        Returns False if the Track is not yet downloaded, or the target path does not exist.
-        """
-        target = Path(target)
-        if not target.exists() or not self.path:
-            return False
-        self.path.unlink()
-        ok = Path(shutil.move(target, self.path)).resolve() == self.path.resolve()
-        if not ok:
-            return False
-        return self.move(target)
 
 
 __all__ = ("Track",)

devine/core/tracks/tracks.py

@@ -14,6 +14,8 @@ from rich.tree import Tree
 from devine.core.config import config
 from devine.core.console import console
 from devine.core.constants import LANGUAGE_MAX_DISTANCE, AnyTrack, TrackT
+from devine.core.events import events
+from devine.core.tracks.attachment import Attachment
 from devine.core.tracks.audio import Audio
 from devine.core.tracks.chapters import Chapter, Chapters
 from devine.core.tracks.subtitle import Subtitle
@@ -25,7 +27,7 @@ from devine.core.utils.collections import as_list, flatten
 
 class Tracks:
     """
-    Video, Audio, Subtitle, and Chapter Track Store.
+    Video, Audio, Subtitle, Chapter, and Attachment Track Store.
     It provides convenience functions for listing, sorting, and selecting tracks.
     """
 
@@ -33,14 +35,23 @@ class Tracks:
         Video: 0,
         Audio: 1,
         Subtitle: 2,
-        Chapter: 3
+        Chapter: 3,
+        Attachment: 4
     }
 
-    def __init__(self, *args: Union[Tracks, list[Track], Track]):
+    def __init__(self, *args: Union[
+        Tracks,
+        Sequence[Union[AnyTrack, Chapter, Chapters, Attachment]],
+        Track,
+        Chapter,
+        Chapters,
+        Attachment
+    ]):
         self.videos: list[Video] = []
         self.audio: list[Audio] = []
         self.subtitles: list[Subtitle] = []
         self.chapters = Chapters()
+        self.attachments: list[Attachment] = []
 
         if args:
             self.add(args)
@@ -53,7 +64,14 @@ class Tracks:
 
     def __add__(
         self,
-        other: Union[Tracks, Sequence[Union[AnyTrack, Chapter, Chapters]], Track, Chapter, Chapters]
+        other: Union[
+            Tracks,
+            Sequence[Union[AnyTrack, Chapter, Chapters, Attachment]],
+            Track,
+            Chapter,
+            Chapters,
+            Attachment
+        ]
     ) -> Tracks:
         self.add(other)
         return self
@@ -69,7 +87,8 @@ class Tracks:
             Video: [],
             Audio: [],
             Subtitle: [],
-            Chapter: []
+            Chapter: [],
+            Attachment: []
         }
         tracks = [*list(self), *self.chapters]
 
@@ -98,7 +117,7 @@ class Tracks:
         return rep
 
     def tree(self, add_progress: bool = False) -> tuple[Tree, list[partial]]:
-        all_tracks = [*list(self), *self.chapters]
+        all_tracks = [*list(self), *self.chapters, *self.attachments]
 
         progress_callables = []
 
@@ -111,7 +130,7 @@ class Tracks:
             track_type_plural = track_type.__name__ + ("s" if track_type != Audio and num_tracks != 1 else "")
             tracks_tree = tree.add(f"[repr.number]{num_tracks}[/] {track_type_plural}")
             for track in tracks:
-                if add_progress and track_type != Chapter:
+                if add_progress and track_type not in (Chapter, Attachment):
                     progress = Progress(
                         SpinnerColumn(finished_text=""),
                         BarColumn(),
@@ -143,12 +162,19 @@ class Tracks:
 
     def add(
         self,
-        tracks: Union[Tracks, Sequence[Union[AnyTrack, Chapter, Chapters]], Track, Chapter, Chapters],
+        tracks: Union[
+            Tracks,
+            Sequence[Union[AnyTrack, Chapter, Chapters, Attachment]],
+            Track,
+            Chapter,
+            Chapters,
+            Attachment
+        ],
         warn_only: bool = False
     ) -> None:
         """Add a provided track to its appropriate array and ensuring it's not a duplicate."""
         if isinstance(tracks, Tracks):
-            tracks = [*list(tracks), *tracks.chapters]
+            tracks = [*list(tracks), *tracks.chapters, *tracks.attachments]
 
         duplicates = 0
         for track in flatten(tracks):
@@ -173,6 +199,8 @@ class Tracks:
                 self.subtitles.append(track)
             elif isinstance(track, Chapter):
                 self.chapters.add(track)
+            elif isinstance(track, Attachment):
+                self.attachments.append(track)
             else:
                 raise ValueError("Track type was not set or is invalid.")
 
@@ -288,7 +316,7 @@ class Tracks:
             ][:per_language or None])
         return selected
 
-    def mux(self, title: str, delete: bool = True, progress: Optional[partial] = None) -> tuple[Path, int]:
+    def mux(self, title: str, delete: bool = True, progress: Optional[partial] = None) -> tuple[Path, int, list[str]]:
         """
         Multiplex all the Tracks into a Matroska Container file.
 
@@ -310,8 +338,7 @@ class Tracks:
         for i, vt in enumerate(self.videos):
             if not vt.path or not vt.path.exists():
                 raise ValueError("Video Track must be downloaded before muxing...")
-            if callable(vt.OnMultiplex):
+            events.emit(events.Types.TRACK_MULTIPLEX, track=vt)
-                vt.OnMultiplex()
             cl.extend([
                 "--language", f"0:{vt.language}",
                 "--default-track", f"0:{i == 0}",
@@ -323,8 +350,7 @@ class Tracks:
         for i, at in enumerate(self.audio):
             if not at.path or not at.path.exists():
                 raise ValueError("Audio Track must be downloaded before muxing...")
-            if callable(at.OnMultiplex):
+            events.emit(events.Types.TRACK_MULTIPLEX, track=at)
-                at.OnMultiplex()
             cl.extend([
                 "--track-name", f"0:{at.get_track_name() or ''}",
                 "--language", f"0:{at.language}",
@@ -338,8 +364,7 @@ class Tracks:
         for st in self.subtitles:
             if not st.path or not st.path.exists():
                 raise ValueError("Text Track must be downloaded before muxing...")
-            if callable(st.OnMultiplex):
+            events.emit(events.Types.TRACK_MULTIPLEX, track=st)
-                st.OnMultiplex()
             default = bool(self.audio and is_close_match(st.language, [self.audio[0].language]) and st.forced)
             cl.extend([
                 "--track-name", f"0:{st.get_track_name() or ''}",
@@ -363,6 +388,16 @@ class Tracks:
         else:
             chapters_path = None
 
+        for attachment in self.attachments:
+            if not attachment.path or not attachment.path.exists():
+                raise ValueError("Attachment File was not found...")
+            cl.extend([
+                "--attachment-description", attachment.description or "",
+                "--attachment-mime-type", attachment.mime_type,
+                "--attachment-name", attachment.name,
+                "--attach-file", str(attachment.path.resolve())
+            ])
+
         output_path = (
             self.videos[0].path.with_suffix(".muxed.mkv") if self.videos else
             self.audio[0].path.with_suffix(".muxed.mka") if self.audio else
@@ -375,15 +410,18 @@ class Tracks:
 
         # let potential failures go to caller, caller should handle
         try:
+            errors = []
             p = subprocess.Popen([
                 *cl,
                 "--output", str(output_path),
                 "--gui-mode"
             ], text=True, stdout=subprocess.PIPE)
             for line in iter(p.stdout.readline, ""):
+                if line.startswith("#GUI#error") or line.startswith("#GUI#warning"):
+                    errors.append(line)
                 if "progress" in line:
                     progress(total=100, completed=int(line.strip()[14:-1]))
-            return output_path, p.wait()
+            return output_path, p.wait(), errors
         finally:
             if chapters_path:
                 # regardless of delete param, we delete as it's a file we made during muxing

devine/core/tracks/video.py

@@ -10,10 +10,11 @@ from typing import Any, Optional, Union
 
 from langcodes import Language
 
+from devine.core import binaries
 from devine.core.config import config
 from devine.core.tracks.subtitle import Subtitle
 from devine.core.tracks.track import Track
-from devine.core.utilities import FPS, get_binary_path, get_boxes
+from devine.core.utilities import FPS, get_boxes
 
 
 class Video(Track):
@@ -141,9 +142,11 @@ class Video(Track):
                 return Video.Range.SDR
 
         @staticmethod
-        def from_m3u_range_tag(tag: str) -> Video.Range:
+        def from_m3u_range_tag(tag: str) -> Optional[Video.Range]:
             tag = (tag or "").upper().replace('"', '').strip()
-            if not tag or tag == "SDR":
+            if not tag:
+                return None
+            if tag == "SDR":
                 return Video.Range.SDR
             elif tag == "PQ":
                 return Video.Range.HDR10  # technically could be any PQ-transfer range
@@ -152,35 +155,110 @@ class Video(Track):
             # for some reason there's no Dolby Vision info tag
             raise ValueError(f"The M3U Range Tag '{tag}' is not a supported Video Range")
 
-    def __init__(self, *args: Any, codec: Video.Codec, range_: Video.Range, bitrate: Union[str, int, float],
+    def __init__(
-                 width: int, height: int, fps: Optional[Union[str, int, float]] = None, **kwargs: Any) -> None:
+        self,
+        *args: Any,
+        codec: Optional[Video.Codec] = None,
+        range_: Optional[Video.Range] = None,
+        bitrate: Optional[Union[str, int, float]] = None,
+        width: Optional[int] = None,
+        height: Optional[int] = None,
+        fps: Optional[Union[str, int, float]] = None,
+        **kwargs: Any
+    ) -> None:
+        """
+        Create a new Video track object.
+
+        Parameters:
+            codec: A Video.Codec enum representing the video codec.
+                If not specified, MediaInfo will be used to retrieve the codec
+                once the track has been downloaded.
+            range_: A Video.Range enum representing the video color range.
+                Defaults to SDR if not specified.
+            bitrate: A number or float representing the average bandwidth in bytes/s.
+                Float values are rounded up to the nearest integer.
+            width: The horizontal resolution of the video.
+            height: The vertical resolution of the video.
+            fps: A number, float, or string representing the frames/s of the video.
+                Strings may represent numbers, floats, or a fraction (num/den).
+                All strings will be cast to either a number or float.
+
+        Note: If codec, bitrate, width, height, or fps is not specified some checks
+        may be skipped or assume a value. Specifying as much information as possible
+        is highly recommended.
+        """
         super().__init__(*args, **kwargs)
-        # required
+
+        if not isinstance(codec, (Video.Codec, type(None))):
+            raise TypeError(f"Expected codec to be a {Video.Codec}, not {codec!r}")
+        if not isinstance(range_, (Video.Range, type(None))):
+            raise TypeError(f"Expected range_ to be a {Video.Range}, not {range_!r}")
+        if not isinstance(bitrate, (str, int, float, type(None))):
+            raise TypeError(f"Expected bitrate to be a {str}, {int}, or {float}, not {bitrate!r}")
+        if not isinstance(width, (int, str, type(None))):
+            raise TypeError(f"Expected width to be a {int}, not {width!r}")
+        if not isinstance(height, (int, str, type(None))):
+            raise TypeError(f"Expected height to be a {int}, not {height!r}")
+        if not isinstance(fps, (str, int, float, type(None))):
+            raise TypeError(f"Expected fps to be a {str}, {int}, or {float}, not {fps!r}")
+
         self.codec = codec
         self.range = range_ or Video.Range.SDR
-        self.bitrate = int(math.ceil(float(bitrate))) if bitrate else None
+
-        self.width = int(width)
+        try:
-        self.height = int(height)
+            self.bitrate = int(math.ceil(float(bitrate))) if bitrate else None
-        # optional
+        except (ValueError, TypeError) as e:
-        self.fps = FPS.parse(str(fps)) if fps else None
+            raise ValueError(f"Expected bitrate to be a number or float, {e}")
+
+        try:
+            self.width = int(width or 0) or None
+        except ValueError as e:
+            raise ValueError(f"Expected width to be a number, not {width!r}, {e}")
+
+        try:
+            self.height = int(height or 0) or None
+        except ValueError as e:
+            raise ValueError(f"Expected height to be a number, not {height!r}, {e}")
+
+        try:
+            self.fps = (FPS.parse(str(fps)) or None) if fps else None
+        except Exception as e:
+            raise ValueError(
+                "Expected fps to be a number, float, or a string as numerator/denominator form, " +
+                str(e)
+            )
 
     def __str__(self) -> str:
-        fps = f"{self.fps:.3f}" if self.fps else "Unknown"
         return " | ".join(filter(bool, [
             "VID",
-            f"[{self.codec.value}, {self.range.name}]",
+            "[" + (", ".join(filter(bool, [
+                self.codec.value if self.codec else None,
+                self.range.name
+            ]))) + "]",
             str(self.language),
-            f"{self.width}x{self.height} @ {self.bitrate // 1000 if self.bitrate else '?'} kb/s, {fps} FPS",
+            ", ".join(filter(bool, [
+                " @ ".join(filter(bool, [
+                    f"{self.width}x{self.height}" if self.width and self.height else None,
+                    f"{self.bitrate // 1000} kb/s" if self.bitrate else None
+                ])),
+                f"{self.fps:.3f} FPS" if self.fps else None
+            ])),
             self.edition
         ]))
 
     def change_color_range(self, range_: int) -> None:
         """Change the Video's Color Range to Limited (0) or Full (1)."""
         if not self.path or not self.path.exists():
-            raise ValueError("Cannot repackage a Track that has not been downloaded.")
+            raise ValueError("Cannot change the color range flag on a Video that has not been downloaded.")
+        if not self.codec:
+            raise ValueError("Cannot change the color range flag on a Video that has no codec specified.")
+        if self.codec not in (Video.Codec.AVC, Video.Codec.HEVC):
+            raise NotImplementedError(
+                "Cannot change the color range flag on this Video as "
+                f"it's codec, {self.codec.value}, is not yet supported."
+            )
 
-        executable = get_binary_path("ffmpeg")
+        if not binaries.FFMPEG:
-        if not executable:
             raise EnvironmentError("FFmpeg executable \"ffmpeg\" was not found but is required for this call.")
 
         filter_key = {
@@ -192,7 +270,7 @@ class Video(Track):
         output_path = original_path.with_stem(f"{original_path.stem}_{['limited', 'full'][range_]}_range")
 
         subprocess.run([
-            executable, "-hide_banner",
+            binaries.FFMPEG, "-hide_banner",
             "-loglevel", "panic",
             "-i", original_path,
             "-codec", "copy",
@@ -200,8 +278,8 @@ class Video(Track):
             str(output_path)
         ], check=True)
 
-        self.swap(output_path)
+        self.path = output_path
-        self.move(original_path)
+        original_path.unlink()
 
     def ccextractor(
         self, track_id: Any, out_path: Union[Path, str], language: Language, original: bool = False
@@ -210,8 +288,7 @@ class Video(Track):
         if not self.path:
             raise ValueError("You must download the track first.")
 
-        executable = get_binary_path("ccextractor", "ccextractorwin", "ccextractorwinfull")
+        if not binaries.CCExtractor:
-        if not executable:
             raise EnvironmentError("ccextractor executable was not found.")
 
         # ccextractor often fails in weird ways unless we repack
@@ -221,7 +298,7 @@ class Video(Track):
 
         try:
             subprocess.run([
-                executable,
+                binaries.CCExtractor,
                 "-trim",
                 "-nobom",
                 "-noru", "-ru1",
@@ -302,8 +379,7 @@ class Video(Track):
         if not self.path or not self.path.exists():
             raise ValueError("Cannot clean a Track that has not been downloaded.")
 
-        executable = get_binary_path("ffmpeg")
+        if not binaries.FFMPEG:
-        if not executable:
             raise EnvironmentError("FFmpeg executable \"ffmpeg\" was not found but is required for this call.")
 
         log = logging.getLogger("x264-clean")
@@ -321,11 +397,12 @@ class Video(Track):
         i = file.index(b"x264")
         encoding_settings = file[i: i + file[i:].index(b"\x00")].replace(b":", br"\\:").replace(b",", br"\,").decode()
 
-        cleaned_path = self.path.with_suffix(f".cleaned{self.path.suffix}")
+        original_path = self.path
+        cleaned_path = original_path.with_suffix(f".cleaned{original_path.suffix}")
         subprocess.run([
-            executable, "-hide_banner",
+            binaries.FFMPEG, "-hide_banner",
             "-loglevel", "panic",
-            "-i", self.path,
+            "-i", original_path,
             "-map_metadata", "-1",
             "-fflags", "bitexact",
             "-bsf:v", f"filter_units=remove_types=6,h264_metadata=sei_user_data={uuid}+{encoding_settings}",
@@ -335,7 +412,8 @@ class Video(Track):
 
         log.info(" + Removed")
 
-        self.swap(cleaned_path)
+        self.path = cleaned_path
+        original_path.unlink()
 
         return True
 

devine/core/utilities.py

@@ -3,7 +3,6 @@ import contextlib
 import importlib.util
 import os
 import re
-import shutil
 import socket
 import sys
 import time
@@ -87,15 +86,6 @@ def import_module_by_path(path: Path) -> ModuleType:
     return module
 
 
-def get_binary_path(*names: str) -> Optional[Path]:
-    """Find the path of the first found binary name."""
-    for name in names:
-        path = shutil.which(name)
-        if path:
-            return Path(path)
-    return None
-
-
 def sanitize_filename(filename: str, spacer: str = ".") -> str:
     """
     Sanitize a string to be filename safe.
@@ -133,18 +123,18 @@ def get_boxes(data: bytes, box_type: bytes, as_bytes: bool = False) -> Box:
     # since it doesn't care what child box the wanted box is from, this works fine.
     if not isinstance(data, (bytes, bytearray)):
         raise ValueError("data must be bytes")
+
+    offset = 0
     while True:
         try:
-            index = data.index(box_type)
+            index = data[offset:].index(box_type)
         except ValueError:
             break
         if index < 0:
             break
-        if index > 4:
+        index -= 4  # size is before box type and is 4 bytes long
-            index -= 4  # size is before box type and is 4 bytes long
-        data = data[index:]
         try:
-            box = Box.parse(data)
+            box = Box.parse(data[offset:][index:])
         except IOError:
             # since get_init_segment might cut off unexpectedly, pymp4 may be unable to read
             # the expected amounts of data and complain, so let's just end the function here
@@ -157,6 +147,7 @@ def get_boxes(data: bytes, box_type: bytes, as_bytes: bool = False) -> Box:
             raise e
         if as_bytes:
             box = Box.build(box)
+        offset += index + len(Box.build(box))
         yield box
 
 
@@ -277,6 +268,27 @@ def get_extension(value: Union[str, Path, ParseResult]) -> Optional[str]:
             return ext
 
 
+def get_system_fonts() -> dict[str, Path]:
+    if sys.platform == "win32":
+        import winreg
+        with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as reg:
+            key = winreg.OpenKey(
+                reg,
+                r"SOFTWARE\Microsoft\Windows NT\CurrentVersion\Fonts",
+                0,
+                winreg.KEY_READ
+            )
+            total_fonts = winreg.QueryInfoKey(key)[1]
+            return {
+                name.replace(" (TrueType)", ""): Path(r"C:\Windows\Fonts", filename)
+                for n in range(0, total_fonts)
+                for name, filename, _ in [winreg.EnumValue(key, n)]
+            }
+    else:
+        # TODO: Get System Fonts for Linux and mac OS
+        return {}
+
+
 class FPS(ast.NodeVisitor):
     def visit_BinOp(self, node: ast.BinOp) -> float:
         if isinstance(node.op, ast.Div):

devine/core/utils/click_types.py

@@ -1,7 +1,8 @@
 import re
-from typing import Optional, Union
+from typing import Any, Optional, Union
 
 import click
+from click.shell_completion import CompletionItem
 from pywidevine.cdm import Cdm as WidevineCdm
 
 
@@ -122,6 +123,62 @@ class QualityList(click.ParamType):
         return sorted(resolutions, reverse=True)
 
 
+class MultipleChoice(click.Choice):
+    """
+    The multiple choice type allows multiple values to be checked against
+    a fixed set of supported values.
+
+    It internally uses and is based off of click.Choice.
+    """
+
+    name = "multiple_choice"
+
+    def __repr__(self) -> str:
+        return f"MultipleChoice({list(self.choices)})"
+
+    def convert(
+        self,
+        value: Any,
+        param: Optional[click.Parameter] = None,
+        ctx: Optional[click.Context] = None
+    ) -> list[Any]:
+        if not value:
+            return []
+        if isinstance(value, str):
+            values = value.split(",")
+        elif isinstance(value, list):
+            values = value
+        else:
+            self.fail(
+                f"{value!r} is not a supported value.",
+                param,
+                ctx
+            )
+
+        chosen_values: list[Any] = []
+        for value in values:
+            chosen_values.append(super().convert(value, param, ctx))
+
+        return chosen_values
+
+    def shell_complete(
+        self,
+        ctx: click.Context,
+        param: click.Parameter,
+        incomplete: str
+    ) -> list[CompletionItem]:
+        """
+        Complete choices that start with the incomplete value.
+
+        Parameters:
+            ctx: Invocation context for this command.
+            param: The parameter that is requesting completion.
+            incomplete: Value being completed. May be empty.
+        """
+        incomplete = incomplete.rsplit(",")[-1]
+        return super(self).shell_complete(ctx, param, incomplete)
+
+
 SEASON_RANGE = SeasonRange()
 LANGUAGE_RANGE = LanguageRange()
 QUALITY_LIST = QualityList()

devine/core/utils/subprocess.py

@@ -3,11 +3,16 @@ import subprocess
 from pathlib import Path
 from typing import Union
 
+from devine.core import binaries
+
 
 def ffprobe(uri: Union[bytes, Path]) -> dict:
     """Use ffprobe on the provided data to get stream information."""
+    if not binaries.FFProbe:
+        raise EnvironmentError("FFProbe executable \"ffprobe\" not found but is required.")
+
     args = [
-        "ffprobe",
+        binaries.FFProbe,
         "-v", "quiet",
         "-of", "json",
         "-show_streams"

devine/core/utils/webvtt.py

@@ -0,0 +1,191 @@
+import re
+import sys
+import typing
+from typing import Optional
+
+from pycaption import Caption, CaptionList, CaptionNode, CaptionReadError, WebVTTReader, WebVTTWriter
+
+
+class CaptionListExt(CaptionList):
+    @typing.no_type_check
+    def __init__(self, iterable=None, layout_info=None):
+        self.first_segment_mpegts = 0
+        super().__init__(iterable, layout_info)
+
+
+class CaptionExt(Caption):
+    @typing.no_type_check
+    def __init__(self, start, end, nodes, style=None, layout_info=None, segment_index=0, mpegts=0, cue_time=0.0):
+        style = style or {}
+        self.segment_index: int = segment_index
+        self.mpegts: float = mpegts
+        self.cue_time: float = cue_time
+        super().__init__(start, end, nodes, style, layout_info)
+
+
+class WebVTTReaderExt(WebVTTReader):
+    # HLS extension support <https://datatracker.ietf.org/doc/html/rfc8216#section-3.5>
+    RE_TIMESTAMP_MAP = re.compile(r"X-TIMESTAMP-MAP.*")
+    RE_MPEGTS = re.compile(r"MPEGTS:(\d+)")
+    RE_LOCAL = re.compile(r"LOCAL:((?:(\d{1,}):)?(\d{2}):(\d{2})\.(\d{3}))")
+
+    def _parse(self, lines: list[str]) -> CaptionList:
+        captions = CaptionListExt()
+        start = None
+        end = None
+        nodes: list[CaptionNode] = []
+        layout_info = None
+        found_timing = False
+        segment_index = -1
+        mpegts = 0
+        cue_time = 0.0
+
+        # The first segment MPEGTS is needed to calculate the rest. It is possible that
+        # the first segment contains no cue and is ignored by pycaption, this acts as a fallback.
+        captions.first_segment_mpegts = 0
+
+        for i, line in enumerate(lines):
+            if "-->" in line:
+                found_timing = True
+                timing_line = i
+                last_start_time = captions[-1].start if captions else 0
+                try:
+                    start, end, layout_info = self._parse_timing_line(line, last_start_time)
+                except CaptionReadError as e:
+                    new_msg = f"{e.args[0]} (line {timing_line})"
+                    tb = sys.exc_info()[2]
+                    raise type(e)(new_msg).with_traceback(tb) from None
+
+            elif "" == line:
+                if found_timing and nodes:
+                    found_timing = False
+                    caption = CaptionExt(
+                        start,
+                        end,
+                        nodes,
+                        layout_info=layout_info,
+                        segment_index=segment_index,
+                        mpegts=mpegts,
+                        cue_time=cue_time,
+                    )
+                    captions.append(caption)
+                    nodes = []
+
+            elif "WEBVTT" in line:
+                # Merged segmented VTT doesn't have index information, track manually.
+                segment_index += 1
+                mpegts = 0
+                cue_time = 0.0
+            elif m := self.RE_TIMESTAMP_MAP.match(line):
+                if r := self.RE_MPEGTS.search(m.group()):
+                    mpegts = int(r.group(1))
+
+                cue_time = self._parse_local(m.group())
+
+                # Early assignment in case the first segment contains no cue.
+                if segment_index == 0:
+                    captions.first_segment_mpegts = mpegts
+
+            else:
+                if found_timing:
+                    if nodes:
+                        nodes.append(CaptionNode.create_break())
+                    nodes.append(CaptionNode.create_text(self._decode(line)))
+                else:
+                    # it's a comment or some metadata; ignore it
+                    pass
+
+        # Add a last caption if there are remaining nodes
+        if nodes:
+            caption = CaptionExt(start, end, nodes, layout_info=layout_info, segment_index=segment_index, mpegts=mpegts)
+            captions.append(caption)
+
+        return captions
+
+    @staticmethod
+    def _parse_local(string: str) -> float:
+        """
+        Parse WebVTT LOCAL time and convert it to seconds.
+        """
+        m = WebVTTReaderExt.RE_LOCAL.search(string)
+        if not m:
+            return 0
+
+        parsed = m.groups()
+        if not parsed:
+            return 0
+        hours = int(parsed[1])
+        minutes = int(parsed[2])
+        seconds = int(parsed[3])
+        milliseconds = int(parsed[4])
+        return (milliseconds / 1000) + seconds + (minutes * 60) + (hours * 3600)
+
+
+def merge_segmented_webvtt(vtt_raw: str, segment_durations: Optional[list[int]] = None, timescale: int = 1) -> str:
+    """
+    Merge Segmented WebVTT data.
+
+    Parameters:
+        vtt_raw: The concatenated WebVTT files to merge. All WebVTT headers must be
+            appropriately spaced apart, or it may produce unwanted effects like
+            considering headers as captions, timestamp lines, etc.
+        segment_durations: A list of each segment's duration. If not provided it will try
+            to get it from the X-TIMESTAMP-MAP headers, specifically the MPEGTS number.
+        timescale: The number of time units per second.
+
+    This parses the X-TIMESTAMP-MAP data to compute new absolute timestamps, replacing
+    the old start and end timestamp values. All X-TIMESTAMP-MAP header information will
+    be removed from the output as they are no longer of concern. Consider this function
+    the opposite of a WebVTT Segmenter, a WebVTT Joiner of sorts.
+
+    Algorithm borrowed from N_m3u8DL-RE and shaka-player.
+    """
+    MPEG_TIMESCALE = 90_000
+
+    vtt = WebVTTReaderExt().read(vtt_raw)
+    for lang in vtt.get_languages():
+        prev_caption = None
+        duplicate_index: list[int] = []
+        captions = vtt.get_captions(lang)
+
+        if captions[0].segment_index == 0:
+            first_segment_mpegts = captions[0].mpegts
+        else:
+            first_segment_mpegts = segment_durations[0] if segment_durations else captions.first_segment_mpegts
+
+        caption: CaptionExt
+        for i, caption in enumerate(captions):
+            # DASH WebVTT doesn't have MPEGTS timestamp like HLS. Instead,
+            # calculate the timestamp from SegmentTemplate/SegmentList duration.
+            likely_dash = first_segment_mpegts == 0 and caption.mpegts == 0
+            if likely_dash and segment_durations:
+                duration = segment_durations[caption.segment_index]
+                caption.mpegts = MPEG_TIMESCALE * (duration / timescale)
+
+            if caption.mpegts == 0:
+                continue
+
+            seconds = (caption.mpegts - first_segment_mpegts) / MPEG_TIMESCALE - caption.cue_time
+            offset = seconds * 1_000_000  # pycaption use microseconds
+
+            if caption.start < offset:
+                caption.start += offset
+                caption.end += offset
+
+            # If the difference between current and previous captions is <=1ms
+            # and the payload is equal then splice.
+            if (
+                prev_caption
+                and not caption.is_empty()
+                and (caption.start - prev_caption.end) <= 1000  # 1ms in microseconds
+                and caption.get_text() == prev_caption.get_text()
+            ):
+                prev_caption.end = caption.end
+                duplicate_index.append(i)
+
+            prev_caption = caption
+
+        # Remove duplicate
+        captions[:] = [c for c_index, c in enumerate(captions) if c_index not in set(duplicate_index)]
+
+    return WebVTTWriter().write(vtt)

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "devine"
-version = "3.0.0"
+version = "3.3.3"
 description = "Modular Movie, TV, and Music Archival Software."
 license = "GPL-3.0-only"
 authors = ["rlaphoenix <rlaphoenix@pm.me>"]
@@ -39,12 +39,12 @@ Brotli = "^1.1.0"
 click = "^8.1.7"
 construct = "^2.8.8"
 crccheck = "^1.3.0"
-jsonpickle = "^3.0.3"
+jsonpickle = "^3.0.4"
-langcodes = { extras = ["data"], version = "^3.3.0" }
+langcodes = { extras = ["data"], version = "^3.4.0" }
-lxml = "^5.1.0"
+lxml = "^5.2.1"
 pproxy = "^2.7.9"
 protobuf = "^4.25.3"
-pycaption = "^2.2.4"
+pycaption = "^2.2.6"
 pycryptodomex = "^3.20.0"
 pyjwt = "^2.8.0"
 pymediainfo = "^6.1.0"
@@ -57,21 +57,21 @@ rich = "^13.7.1"
 "rlaphoenix.m3u8" = "^3.4.0"
 "ruamel.yaml" = "^0.18.6"
 sortedcontainers = "^2.4.0"
-subtitle-filter = "^1.4.8"
+subtitle-filter = "^1.4.9"
 Unidecode = "^1.3.8"
 urllib3 = "^2.2.1"
 chardet = "^5.2.0"
-curl-cffi = "^0.6.1"
+curl-cffi = "^0.7.0b4"
 
 [tool.poetry.dev-dependencies]
-pre-commit = "^3.6.2"
+pre-commit = "^3.7.0"
-mypy = "^1.8.0"
+mypy = "^1.9.0"
-mypy-protobuf = "^3.5.0"
+mypy-protobuf = "^3.6.0"
-types-protobuf = "^4.24.0.20240129"
+types-protobuf = "^4.24.0.20240408"
 types-PyMySQL = "^1.1.0.1"
-types-requests = "^2.31.0.20240218"
+types-requests = "^2.31.0.20240406"
 isort = "^5.13.2"
-ruff = "~0.3.0"
+ruff = "~0.3.7"
 
 [tool.poetry.scripts]
 devine = "devine.core.__main__:main"