Initial commit

devine/__main__.py

@@ -0,0 +1,3 @@
+if __name__ == "__main__":
+    from devine.core.__main__ import main
+    main()

devine/commands/auth.py

@@ -0,0 +1,252 @@
+import logging
+import tkinter.filedialog
+from pathlib import Path
+from typing import Optional
+
+import click
+from ruamel.yaml import YAML
+
+from devine.core.config import Config, config
+from devine.core.constants import context_settings
+from devine.core.credential import Credential
+
+
+@click.group(
+    short_help="Manage cookies and credentials for profiles of services.",
+    context_settings=context_settings)
+@click.pass_context
+def auth(ctx: click.Context) -> None:
+    """Manage cookies and credentials for profiles of services."""
+    ctx.obj = logging.getLogger("auth")
+
+
+@auth.command(
+    name="list",
+    short_help="List profiles and their state for a service or all services.",
+    context_settings=context_settings)
+@click.argument("service", type=str, required=False)
+@click.pass_context
+def list_(ctx: click.Context, service: Optional[str] = None) -> None:
+    """
+    List profiles and their state for a service or all services.
+
+    \b
+    Profile and Service names are case-insensitive.
+    """
+    log = ctx.obj
+    service_f = service
+
+    profiles: dict[str, dict[str, list]] = {}
+    for cookie_dir in config.directories.cookies.iterdir():
+        service = cookie_dir.name
+        profiles[service] = {}
+        for cookie in cookie_dir.glob("*.txt"):
+            if cookie.stem not in profiles[service]:
+                profiles[service][cookie.stem] = ["Cookie"]
+
+    for service, credentials in config.credentials.items():
+        if service not in profiles:
+            profiles[service] = {}
+        for profile, credential in credentials.items():
+            if profile not in profiles[service]:
+                profiles[service][profile] = []
+            profiles[service][profile].append("Credential")
+
+    for service, profiles in profiles.items():
+        if service_f and service != service_f.upper():
+            continue
+        log.info(service)
+        for profile, authorizations in profiles.items():
+            log.info(f'  "{profile}": {", ".join(authorizations)}')
+
+
+@auth.command(
+    short_help="View profile cookies and credentials for a service.",
+    context_settings=context_settings)
+@click.argument("profile", type=str)
+@click.argument("service", type=str)
+@click.pass_context
+def view(ctx: click.Context, profile: str, service: str) -> None:
+    """
+    View profile cookies and credentials for a service.
+
+    \b
+    Profile and Service names are case-sensitive.
+    """
+    log = ctx.obj
+    service_f = service
+    profile_f = profile
+    found = False
+
+    for cookie_dir in config.directories.cookies.iterdir():
+        if cookie_dir.name == service_f:
+            for cookie in cookie_dir.glob("*.txt"):
+                if cookie.stem == profile_f:
+                    log.info(f"Cookie: {cookie}")
+                    log.debug(cookie.read_text(encoding="utf8").strip())
+                    found = True
+                    break
+
+    for service, credentials in config.credentials.items():
+        if service == service_f:
+            for profile, credential in credentials.items():
+                if profile == profile_f:
+                    log.info(f"Credential: {':'.join(list(credential))}")
+                    found = True
+                    break
+
+    if not found:
+        raise click.ClickException(
+            f"Could not find Profile '{profile_f}' for Service '{service_f}'."
+            f"\nThe profile and service values are case-sensitive."
+        )
+
+
+@auth.command(
+    short_help="Check what profile is used by services.",
+    context_settings=context_settings)
+@click.argument("service", type=str, required=False)
+@click.pass_context
+def status(ctx: click.Context, service: Optional[str] = None) -> None:
+    """
+    Check what profile is used by services.
+
+    \b
+    Service names are case-sensitive.
+    """
+    log = ctx.obj
+    found_profile = False
+    for service_, profile in config.profiles.items():
+        if not service or service_.upper() == service.upper():
+            log.info(f"{service_}: {profile or '--'}")
+            found_profile = True
+
+    if not found_profile:
+        log.info(f"No profile has been explicitly set for {service}")
+
+    default = config.profiles.get("default", "not set")
+    log.info(f"The default profile is {default}")
+
+
+@auth.command(
+    short_help="Delete a profile and all of its authorization from a service.",
+    context_settings=context_settings)
+@click.argument("profile", type=str)
+@click.argument("service", type=str)
+@click.option("--cookie", is_flag=True, default=False, help="Only delete the cookie.")
+@click.option("--credential", is_flag=True, default=False, help="Only delete the credential.")
+@click.pass_context
+def delete(ctx: click.Context, profile: str, service: str, cookie: bool, credential: bool):
+    """
+    Delete a profile and all of its authorization from a service.
+
+    \b
+    By default this does remove both Cookies and Credentials.
+    You may remove only one of them with --cookie or --credential.
+
+    \b
+    Profile and Service names are case-sensitive.
+    Comments may be removed from config!
+    """
+    log = ctx.obj
+    service_f = service
+    profile_f = profile
+    found = False
+
+    if not credential:
+        for cookie_dir in config.directories.cookies.iterdir():
+            if cookie_dir.name == service_f:
+                for cookie_ in cookie_dir.glob("*.txt"):
+                    if cookie_.stem == profile_f:
+                        cookie_.unlink()
+                        log.info(f"Deleted Cookie: {cookie_}")
+                        found = True
+                        break
+
+    if not cookie:
+        for key, credentials in config.credentials.items():
+            if key == service_f:
+                for profile, credential_ in credentials.items():
+                    if profile == profile_f:
+                        config_path = Config._Directories.user_configs / Config._Filenames.root_config
+                        yaml, data = YAML(), None
+                        yaml.default_flow_style = False
+                        data = yaml.load(config_path)
+                        del data["credentials"][key][profile_f]
+                        yaml.dump(data, config_path)
+                        log.info(f"Deleted Credential: {credential_}")
+                        found = True
+                        break
+
+    if not found:
+        raise click.ClickException(
+            f"Could not find Profile '{profile_f}' for Service '{service_f}'."
+            f"\nThe profile and service values are case-sensitive."
+        )
+
+
+@auth.command(
+    short_help="Add a Credential and/or Cookies to an existing or new profile for a service.",
+    context_settings=context_settings)
+@click.argument("profile", type=str)
+@click.argument("service", type=str)
+@click.option("--cookie", type=str, default=None, help="Direct path to Cookies to add.")
+@click.option("--credential", type=str, default=None, help="Direct Credential string to add.")
+@click.pass_context
+def add(ctx: click.Context, profile: str, service: str, cookie: Optional[str] = None, credential: Optional[str] = None):
+    """
+    Add a Credential and/or Cookies to an existing or new profile for a service.
+
+    \b
+    Cancel the Open File dialogue when presented if you do not wish to provide
+    cookies. The Credential should be in `Username:Password` form. The username
+    may be an email. If you do not wish to add a Credential, just hit enter.
+
+    \b
+    Profile and Service names are case-sensitive!
+    Comments may be removed from config!
+    """
+    log = ctx.obj
+    service = service.upper()
+    profile = profile.lower()
+
+    if cookie:
+        cookie = Path(cookie)
+    else:
+        print("Opening File Dialogue, select a Cookie file to import.")
+        cookie = tkinter.filedialog.askopenfilename(
+            title="Select a Cookie file (Cancel to skip)",
+            filetypes=[("Cookies", "*.txt"), ("All files", "*.*")]
+        )
+        if cookie:
+            cookie = Path(cookie)
+        else:
+            log.info("Skipped adding a Cookie...")
+
+    if credential:
+        try:
+            credential = Credential.loads(credential)
+        except ValueError as e:
+            raise click.ClickException(str(e))
+    else:
+        credential = input("Credential: ")
+        if credential:
+            try:
+                credential = Credential.loads(credential)
+            except ValueError as e:
+                raise click.ClickException(str(e))
+        else:
+            log.info("Skipped adding a Credential...")
+
+    if cookie:
+        cookie = cookie.rename((config.directories.cookies / service / profile).with_suffix(".txt"))
+        log.info(f"Moved Cookie file to: {cookie}")
+
+    if credential:
+        config_path = Config._Directories.user_configs / Config._Filenames.root_config
+        yaml, data = YAML(), None
+        yaml.default_flow_style = False
+        data = yaml.load(config_path)
+        data["credentials"][service][profile] = credential.dumps()
+        yaml.dump(data, config_path)
+        log.info(f"Added Credential: {credential}")

devine/commands/cfg.py

@@ -0,0 +1,86 @@
+import ast
+import logging
+import sys
+
+import click
+from ruamel.yaml import YAML
+
+from devine.core.config import config
+from devine.core.constants import context_settings
+
+
+@click.command(
+    short_help="Manage configuration values for the program and its services.",
+    context_settings=context_settings)
+@click.argument("key", type=str, required=False)
+@click.argument("value", type=str, required=False)
+@click.option("--unset", is_flag=True, default=False, help="Unset/remove the configuration value.")
+@click.option("--list", "list_", is_flag=True, default=False, help="List all set configuration values.")
+@click.pass_context
+def cfg(ctx: click.Context, key: str, value: str, unset: bool, list_: bool) -> None:
+    """
+    Manage configuration values for the program and its services.
+
+    \b
+    Known Issues:
+    - Config changes remove all comments of the changed files, which may hold critical data. (#14)
+    """
+    if not key and not value and not list_:
+        raise click.UsageError("Nothing to do.", ctx)
+
+    if value:
+        try:
+            value = ast.literal_eval(value)
+        except (ValueError, SyntaxError):
+            pass  # probably a str without quotes or similar, assume it's a string value
+
+    log = logging.getLogger("cfg")
+
+    config_path = config.directories.user_configs / config.filenames.root_config
+
+    yaml, data = YAML(), None
+    yaml.default_flow_style = False
+    if config_path.is_file():
+        data = yaml.load(config_path)
+
+    if not data:
+        log.warning(f"{config_path} has no configuration data, yet")
+
+    if list_:
+        yaml.dump(data, sys.stdout)
+        return
+
+    key_items = key.split(".")
+    parent_key = key_items[:-1]
+    trailing_key = key_items[-1]
+
+    is_write = value is not None
+    is_delete = unset
+    if is_write and is_delete:
+        raise click.ClickException("You cannot set a value and use --unset at the same time.")
+
+    if not is_write and not is_delete:
+        data = data.mlget(key_items, default=KeyError)
+        if data == KeyError:
+            raise click.ClickException(f"Key '{key}' does not exist in the config.")
+        yaml.dump(data, sys.stdout)
+    else:
+        try:
+            parent_data = data
+            if parent_key:
+                parent_data = data.mlget(parent_key, default=data)
+                if parent_data == data:
+                    for key in parent_key:
+                        if not hasattr(parent_data, key):
+                            parent_data[key] = {}
+                        parent_data = parent_data[key]
+            if is_write:
+                parent_data[trailing_key] = value
+                log.info(f"Set {key} to {repr(value)}")
+            elif is_delete:
+                del parent_data[trailing_key]
+                log.info(f"Unset {key}")
+        except KeyError:
+            raise click.ClickException(f"Key '{key}' does not exist in the config.")
+        config_path.parent.mkdir(parents=True, exist_ok=True)
+        yaml.dump(data, config_path)

devine/commands/dl.py

@@ -0,0 +1,732 @@
+from __future__ import annotations
+
+import html
+import logging
+import random
+import re
+import sys
+import time
+from collections import defaultdict
+from concurrent import futures
+from concurrent.futures import ThreadPoolExecutor
+from copy import deepcopy
+from datetime import datetime
+from functools import partial
+from http.cookiejar import MozillaCookieJar
+from pathlib import Path
+from threading import Event
+from typing import Any, Optional, Callable
+
+import click
+import jsonpickle
+import yaml
+from pymediainfo import MediaInfo
+from pywidevine.cdm import Cdm as WidevineCdm
+from pywidevine.device import Device
+from pywidevine.remotecdm import RemoteCdm
+from tqdm import tqdm
+
+from devine.core.config import config
+from devine.core.constants import AnyTrack, context_settings, LOG_FORMATTER, DRM_SORT_MAP
+from devine.core.drm import Widevine, DRM_T
+from devine.core.proxies import Basic, NordVPN, Hola
+from devine.core.service import Service
+from devine.core.services import Services
+from devine.core.titles import Title_T, Movie, Song
+from devine.core.titles.episode import Episode
+from devine.core.tracks import Audio, Video
+from devine.core.utilities import is_close_match, get_binary_path
+from devine.core.utils.click_types import LANGUAGE_RANGE, QUALITY, SEASON_RANGE, ContextData
+from devine.core.utils.collections import merge_dict
+from devine.core.credential import Credential
+from devine.core.utils.subprocess import ffprobe
+from devine.core.vaults import Vaults
+
+
+class dl:
+    @click.group(
+        short_help="Download, Decrypt, and Mux tracks for titles from a Service.",
+        cls=Services,
+        context_settings=dict(
+            **context_settings,
+            default_map=config.dl,
+            token_normalize_func=Services.get_tag
+        ))
+    @click.option("-p", "--profile", type=str, default=None,
+                  help="Profile to use for Credentials and Cookies (if available). Overrides profile set by config.")
+    @click.option("-q", "--quality", type=QUALITY, default=None,
+                  help="Download Resolution, defaults to best available.")
+    @click.option("-v", "--vcodec", type=click.Choice(Video.Codec, case_sensitive=False),
+                  default=Video.Codec.AVC,
+                  help="Video Codec to download, defaults to H.264.")
+    @click.option("-a", "--acodec", type=click.Choice(Audio.Codec, case_sensitive=False),
+                  default=None,
+                  help="Audio Codec to download, defaults to any codec.")
+    @click.option("-r", "--range", "range_", type=click.Choice(Video.Range, case_sensitive=False),
+                  default=Video.Range.SDR,
+                  help="Video Color Range, defaults to SDR.")
+    @click.option("-w", "--wanted", type=SEASON_RANGE, default=None,
+                  help="Wanted episodes, e.g. `S01-S05,S07`, `S01E01-S02E03`, `S02-S02E03`, e.t.c, defaults to all.")
+    @click.option("-l", "--lang", type=LANGUAGE_RANGE, default="en",
+                  help="Language wanted for Video and Audio.")
+    @click.option("-vl", "--v-lang", type=LANGUAGE_RANGE, default=[],
+                  help="Language wanted for Video, you would use this if the video language doesn't match the audio.")
+    @click.option("-sl", "--s-lang", type=LANGUAGE_RANGE, default=["all"],
+                  help="Language wanted for Subtitles.")
+    @click.option("--proxy", type=str, default=None,
+                  help="Proxy URI to use. If a 2-letter country is provided, it will try get a proxy from the config.")
+    @click.option("--group", type=str, default=None,
+                  help="Set the Group Tag to be used, overriding the one in config if any.")
+    @click.option("-A", "--audio-only", is_flag=True, default=False,
+                  help="Only download audio tracks.")
+    @click.option("-S", "--subs-only", is_flag=True, default=False,
+                  help="Only download subtitle tracks.")
+    @click.option("-C", "--chapters-only", is_flag=True, default=False,
+                  help="Only download chapters.")
+    @click.option("--slow", is_flag=True, default=False,
+                  help="Add a 60-120 second delay between each Title download to act more like a real device. "
+                       "This is recommended if you are downloading high-risk titles or streams.")
+    @click.option("--list", "list_", is_flag=True, default=False,
+                  help="Skip downloading and list available tracks and what tracks would have been downloaded.")
+    @click.option("--list-titles", is_flag=True, default=False,
+                  help="Skip downloading, only list available titles that would have been downloaded.")
+    @click.option("--skip-dl", is_flag=True, default=False,
+                  help="Skip downloading while still retrieving the decryption keys.")
+    @click.option("--export", type=Path,
+                  help="Export Decryption Keys as you obtain them to a JSON file.")
+    @click.option("--cdm-only/--vaults-only", is_flag=True, default=None,
+                  help="Only use CDM, or only use Key Vaults for retrieval of Decryption Keys.")
+    @click.option("--no-proxy", is_flag=True, default=False,
+                  help="Force disable all proxy use.")
+    @click.option("--no-folder", is_flag=True, default=False,
+                  help="Disable folder creation for TV Shows.")
+    @click.option("--no-source", is_flag=True, default=False,
+                  help="Disable the source tag from the output file name and path.")
+    @click.option("--workers", type=int, default=1,
+                  help="Max concurrent workers to use throughout the code, particularly downloads.")
+    @click.option("--log", "log_path", type=Path, default=config.directories.logs / config.filenames.log,
+                  help="Log path (or filename). Path can contain the following f-string args: {name} {time}.")
+    @click.pass_context
+    def cli(ctx: click.Context, **kwargs: Any) -> dl:
+        return dl(ctx, **kwargs)
+
+    DL_POOL_STOP = Event()
+
+    def __init__(
+        self,
+        ctx: click.Context,
+        log_path: Path,
+        profile: Optional[str] = None,
+        proxy: Optional[str] = None,
+        group: Optional[str] = None,
+        *_: Any,
+        **__: Any
+    ):
+        if not ctx.invoked_subcommand:
+            raise ValueError("A subcommand to invoke was not specified, the main code cannot continue.")
+
+        self.log = logging.getLogger("download")
+        if log_path:
+            new_log_path = self.rotate_log_file(log_path)
+            fh = logging.FileHandler(new_log_path, encoding="utf8")
+            fh.setFormatter(LOG_FORMATTER)
+            self.log.addHandler(fh)
+
+        self.service = Services.get_tag(ctx.invoked_subcommand)
+
+        self.log.info(f"Loading Profile Data for {self.service}")
+        if profile:
+            self.profile = profile
+            self.log.info(f" + Profile: {self.profile} (explicit)")
+        else:
+            self.profile = self.get_profile(self.service)
+            self.log.info(f" + Profile: {self.profile} (from config)")
+
+        self.log.info("Initializing Widevine CDM")
+        try:
+            self.cdm = self.get_cdm(self.service, self.profile)
+        except ValueError as e:
+            self.log.error(f" - {e}")
+            sys.exit(1)
+        self.log.info(
+            f" + {self.cdm.__class__.__name__}: {self.cdm.system_id} (L{self.cdm.security_level})"
+        )
+
+        self.log.info("Loading Vaults")
+        self.vaults = Vaults(self.service)
+        for vault in config.key_vaults:
+            vault_type = vault["type"]
+            del vault["type"]
+            self.vaults.load(vault_type, **vault)
+        self.log.info(f" + {len(self.vaults)} Vaults")
+
+        self.log.info("Getting Service Config")
+        service_config_path = Services.get_path(self.service) / config.filenames.config
+        if service_config_path.is_file():
+            self.service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8"))
+            self.log.info(" + Got Service Config")
+        else:
+            self.service_config = {}
+            self.log.info(" - No Service Config")
+        merge_dict(config.services.get(self.service), self.service_config)
+
+        self.log.info("Loading Proxy Providers")
+        self.proxy_providers = []
+        if config.proxy_providers.get("basic"):
+            self.proxy_providers.append(Basic(**config.proxy_providers["basic"]))
+        if config.proxy_providers.get("nordvpn"):
+            self.proxy_providers.append(NordVPN(**config.proxy_providers["nordvpn"]))
+        if get_binary_path("hola-proxy"):
+            self.proxy_providers.append(Hola())
+        for proxy_provider in self.proxy_providers:
+            self.log.info(f" + {proxy_provider.__class__.__name__}: {repr(proxy_provider)}")
+
+        if proxy:
+            requested_provider = None
+            if re.match(rf"^[a-z]+:.+$", proxy, re.IGNORECASE):
+                # requesting proxy from a specific proxy provider
+                requested_provider, proxy = proxy.split(":", maxsplit=1)
+            if re.match(r"^[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE):
+                proxy = proxy.lower()
+                self.log.info(f"Getting a Proxy to '{proxy}'")
+                if requested_provider:
+                    proxy_provider = next((
+                        x
+                        for x in self.proxy_providers
+                        if x.__class__.__name__.lower() == requested_provider
+                    ), None)
+                    if not proxy_provider:
+                        self.log.error(f"The proxy provider '{requested_provider}' was not recognised.")
+                        sys.exit(1)
+                    proxy_uri = proxy_provider.get_proxy(proxy)
+                    if not proxy_uri:
+                        self.log.error(f"The proxy provider {requested_provider} had no proxy for {proxy}")
+                        sys.exit(1)
+                    proxy = ctx.params["proxy"] = proxy_uri
+                    self.log.info(f" + {proxy} (from {proxy_provider.__class__.__name__})")
+                else:
+                    for proxy_provider in self.proxy_providers:
+                        proxy_uri = proxy_provider.get_proxy(proxy)
+                        if proxy_uri:
+                            proxy = ctx.params["proxy"] = proxy_uri
+                            self.log.info(f" + {proxy} (from {proxy_provider.__class__.__name__})")
+                            break
+            else:
+                self.log.info(f"Proxy: {proxy} (from args)")
+
+        ctx.obj = ContextData(
+            config=self.service_config,
+            cdm=self.cdm,
+            proxy_providers=self.proxy_providers,
+            profile=self.profile
+        )
+
+        if group:
+            config.tag = group
+
+        # needs to be added this way instead of @cli.result_callback to be
+        # able to keep `self` as the first positional
+        self.cli._result_callback = self.result
+
+    def result(
+        self, service: Service, quality: Optional[int], vcodec: Video.Codec,
+        acodec: Optional[Audio.Codec], range_: Video.Range, wanted: list[str], lang: list[str], v_lang: list[str],
+        s_lang: list[str], audio_only: bool, subs_only: bool, chapters_only: bool, slow: bool, list_: bool,
+        list_titles: bool, skip_dl: bool, export: Optional[Path], cdm_only: Optional[bool], no_folder: bool,
+        no_source: bool, workers: int, *_: Any, **__: Any
+    ) -> None:
+        if cdm_only is None:
+            vaults_only = None
+        else:
+            vaults_only = not cdm_only
+
+        if self.profile:
+            cookies = self.get_cookie_jar(self.service, self.profile)
+            credential = self.get_credentials(self.service, self.profile)
+            if not cookies and not credential:
+                self.log.error(f"The Profile '{self.profile}' has no Cookies or Credentials. Check for typos.")
+                sys.exit(1)
+
+            self.log.info(f"Authenticating with Profile '{self.profile}'")
+            service.authenticate(cookies, credential)
+            self.log.info(" + Authenticated")
+
+        self.log.info("Retrieving Titles")
+        titles = service.get_titles()
+        if not titles:
+            self.log.error(" - No titles returned!")
+            sys.exit(1)
+
+        for line in str(titles).splitlines(keepends=False):
+            self.log.info(line)
+
+        if list_titles:
+            for title in titles:
+                self.log.info(title)
+            return
+
+        for i, title in enumerate(titles):
+            if isinstance(title, Episode) and wanted and f"{title.season}x{title.number}" not in wanted:
+                continue
+
+            self.log.info(f"Getting tracks for {title}")
+            if slow and i != 0:
+                delay = random.randint(60, 120)
+                self.log.info(f" - Delaying by {delay} seconds due to --slow ...")
+                time.sleep(delay)
+
+            title.tracks.add(service.get_tracks(title), warn_only=True)
+            title.tracks.add(service.get_chapters(title))
+
+            # strip SDH subs to non-SDH if no equivalent same-lang non-SDH is available
+            # uses a loose check, e.g, wont strip en-US SDH sub if a non-SDH en-GB is available
+            for subtitle in title.tracks.subtitles:
+                if subtitle.sdh and not any(
+                    is_close_match(subtitle.language, [x.language])
+                    for x in title.tracks.subtitles
+                    if not x.sdh and not x.forced
+                ):
+                    non_sdh_sub = deepcopy(subtitle)
+                    non_sdh_sub.id += "_stripped"
+                    non_sdh_sub.sdh = False
+                    non_sdh_sub.OnDownloaded = lambda x: x.strip_hearing_impaired()
+                    title.tracks.add(non_sdh_sub)
+
+            title.tracks.sort_videos(by_language=v_lang or lang)
+            title.tracks.sort_audio(by_language=lang)
+            title.tracks.sort_subtitles(by_language=s_lang)
+            title.tracks.sort_chapters()
+
+            self.log.info("> All Tracks:")
+            title.tracks.print()
+
+            self.log.info("> Selected Tracks:")  # log early so errors logs make sense
+
+            if isinstance(title, (Movie, Episode)):
+                # filter video tracks
+                title.tracks.select_video(lambda x: x.codec == vcodec)
+                title.tracks.select_video(lambda x: x.range == range_)
+                if quality:
+                    title.tracks.with_resolution(quality)
+                if not title.tracks.videos:
+                    self.log.error(f"There's no {quality}p {vcodec.name} ({range_.name}) Video Track...")
+                    sys.exit(1)
+
+                video_language = v_lang or lang
+                if video_language and "all" not in video_language:
+                    title.tracks.videos = title.tracks.select_per_language(title.tracks.videos, video_language)
+                    if not title.tracks.videos:
+                        self.log.error(f"There's no {video_language} Video Track...")
+                        sys.exit(1)
+
+                # filter subtitle tracks
+                if s_lang and "all" not in s_lang:
+                    title.tracks.select_subtitles(lambda x: is_close_match(x.language, s_lang))
+                    if not title.tracks.subtitles:
+                        self.log.error(f"There's no {s_lang} Subtitle Track...")
+                        sys.exit(1)
+
+                title.tracks.select_subtitles(lambda x: not x.forced or is_close_match(x.language, lang))
+
+            # filter audio tracks
+            title.tracks.select_audio(lambda x: not x.descriptive)  # exclude descriptive audio
+            if acodec:
+                title.tracks.select_audio(lambda x: x.codec == acodec)
+                if not title.tracks.audio:
+                    self.log.error(f"There's no {acodec.name} Audio Tracks...")
+                    sys.exit(1)
+
+            if lang and "all" not in lang:
+                title.tracks.audio = title.tracks.select_per_language(title.tracks.audio, lang)
+                if not title.tracks.audio:
+                    if all(x.descriptor == Video.Descriptor.M3U for x in title.tracks.videos):
+                        self.log.warning(f"There's no {lang} Audio Tracks, "
+                                         f"likely part of an invariant playlist, continuing...")
+                    else:
+                        self.log.error(f"There's no {lang} Audio Track, cannot continue...")
+                        sys.exit(1)
+
+            if audio_only or subs_only or chapters_only:
+                title.tracks.videos.clear()
+                if audio_only:
+                    if not subs_only:
+                        title.tracks.subtitles.clear()
+                    if not chapters_only:
+                        title.tracks.chapters.clear()
+                elif subs_only:
+                    if not audio_only:
+                        title.tracks.audio.clear()
+                    if not chapters_only:
+                        title.tracks.chapters.clear()
+                elif chapters_only:
+                    if not audio_only:
+                        title.tracks.audio.clear()
+                    if not subs_only:
+                        title.tracks.subtitles.clear()
+
+            title.tracks.print()
+
+            if list_:
+                continue  # only wanted to see what tracks were available and chosen
+
+            # Prepare Track DRM (if any)
+            for track in title.tracks:
+                if not track.drm and isinstance(track, (Video, Audio)):
+                    # service might not list DRM in manifest, get from stream data
+                    try:
+                        track.drm = [Widevine.from_track(track, service.session)]
+                    except Widevine.Exceptions.PSSHNotFound:
+                        # it might not have Widevine DRM, or might not have found the PSSH
+                        self.log.warning("No Widevine PSSH was found for this track, is it DRM free?")
+                if track.drm:
+                    # choose first-available DRM in order of Enum value
+                    track.drm = next(iter(sorted(track.drm, key=lambda x: DRM_SORT_MAP.index(x.__class__.__name__))))
+                    if isinstance(track.drm, Widevine):
+                        # Get Widevine Content Keys now, this must be done in main thread due to SQLite objects
+                        self.log.info(f"Getting {track.drm.__class__.__name__} Keys for: {track}")
+                        self.prepare_drm(
+                            drm=track.drm,
+                            licence=partial(
+                                service.get_widevine_license,
+                                title=title,
+                                track=track
+                            ),
+                            certificate=partial(
+                                service.get_widevine_service_certificate,
+                                title=title,
+                                track=track
+                            ),
+                            cdm_only=cdm_only,
+                            vaults_only=vaults_only
+                        )
+
+                        if export:
+                            keys = {}
+                            if export.is_file():
+                                keys = jsonpickle.loads(export.read_text(encoding="utf8"))
+                            if str(title) not in keys:
+                                keys[str(title)] = {}
+                            keys[str(title)][str(track)] = {
+                                kid: key
+                                for kid, key in track.drm.content_keys.items()
+                                if kid in track.drm.kids
+                            }
+                            export.write_text(jsonpickle.dumps(keys, indent=4), encoding="utf8")
+
+            if skip_dl:
+                self.log.info("Skipping Download...")
+            else:
+                with tqdm(total=len(title.tracks)) as pbar:
+                    with ThreadPoolExecutor(workers) as pool:
+                        try:
+                            for download in futures.as_completed((
+                                pool.submit(
+                                    self.download_track,
+                                    service=service,
+                                    track=track,
+                                    title=title
+                                )
+                                for track in title.tracks
+                            )):
+                                if download.cancelled():
+                                    continue
+                                e = download.exception()
+                                if e:
+                                    self.DL_POOL_STOP.set()
+                                    pool.shutdown(wait=False, cancel_futures=True)
+                                    self.log.error(f"Download worker threw an unhandled exception: {e!r}")
+                                    return
+                                else:
+                                    pbar.update(1)
+                        except KeyboardInterrupt:
+                            self.DL_POOL_STOP.set()
+                            pool.shutdown(wait=False, cancel_futures=True)
+                            self.log.info("Received Keyboard Interrupt, stopping...")
+                            return
+
+            if not skip_dl:
+                self.mux_tracks(title, not no_folder, not no_source)
+
+            # update cookies
+            cookie_file = config.directories.cookies / service.__class__.__name__ / f"{self.profile}.txt"
+            if cookie_file.exists():
+                cookie_jar = MozillaCookieJar(cookie_file)
+                cookie_jar.load()
+                for cookie in service.session.cookies:
+                    cookie_jar.set_cookie(cookie)
+                cookie_jar.save(ignore_discard=True)
+
+        self.log.info("Processed all titles!")
+
+    def download_track(
+        self,
+        service: Service,
+        track: AnyTrack,
+        title: Title_T
+    ):
+        time.sleep(1)
+        if self.DL_POOL_STOP.is_set():
+            return
+
+        if track.needs_proxy:
+            proxy = next(iter(service.session.proxies.values()), None)
+        else:
+            proxy = None
+
+        self.log.info(f"Downloading: {track}")
+        track.download(config.directories.temp, headers=service.session.headers, proxy=proxy)
+        if callable(track.OnDownloaded):
+            track.OnDownloaded(track)
+
+        if track.drm:
+            self.log.info(f"Decrypting file with {track.drm.__class__.__name__} DRM...")
+            track.drm.decrypt(track)
+            self.log.info(" + Decrypted")
+            if callable(track.OnDecrypted):
+                track.OnDecrypted(track)
+
+        if track.needs_repack:
+            self.log.info("Repackaging stream with FFMPEG (fix malformed streams)")
+            track.repackage()
+            self.log.info(" + Repackaged")
+            if callable(track.OnRepacked):
+                track.OnRepacked(track)
+
+        if (
+            isinstance(track, Video) and
+            not title.tracks.subtitles and
+            any(
+                x.get("codec_name", "").startswith("eia_")
+                for x in ffprobe(track.path).get("streams", [])
+            )
+        ):
+            self.log.info("Checking for EIA-CC Captions")
+            try:
+                # TODO: Figure out the real language, it might be different
+                #       EIA-CC tracks sadly don't carry language information :(
+                # TODO: Figure out if the CC language is original lang or not.
+                #       Will need to figure out above first to do so.
+                track_id = f"ccextractor-{track.id}"
+                cc_lang = track.language
+                cc = track.ccextractor(
+                    track_id=track_id,
+                    out_path=config.directories.temp / config.filenames.subtitle.format(
+                        id=track_id,
+                        language=cc_lang
+                    ),
+                    language=cc_lang,
+                    original=False
+                )
+                if cc:
+                    title.tracks.add(cc)
+                    self.log.info(" + Found & Extracted an EIA-CC Caption")
+            except EnvironmentError:
+                self.log.error(" - Track needs to have CC extracted, but ccextractor wasn't found")
+                sys.exit(1)
+            self.log.info(" + No EIA-CC Captions...")
+
+    def prepare_drm(
+        self,
+        drm: DRM_T,
+        certificate: Callable,
+        licence: Callable,
+        cdm_only: bool = False,
+        vaults_only: bool = False
+    ) -> None:
+        """
+        Prepare the DRM by getting decryption data like KIDs, Keys, and such.
+        The DRM object should be ready for decryption once this function ends.
+        """
+        if not drm:
+            return
+
+        if isinstance(drm, Widevine):
+            self.log.info(f"PSSH: {drm.pssh.dumps()}")
+            self.log.info("KIDs:")
+            for kid in drm.kids:
+                self.log.info(f" + {kid.hex}")
+
+            for kid in drm.kids:
+                if kid in drm.content_keys:
+                    continue
+
+                if not cdm_only:
+                    content_key, vault_used = self.vaults.get_key(kid)
+                    if content_key:
+                        drm.content_keys[kid] = content_key
+                        self.log.info(f"Content Key: {kid.hex}:{content_key} ({vault_used})")
+                        add_count = self.vaults.add_key(kid, content_key, excluding=vault_used)
+                        self.log.info(f" + Cached to {add_count}/{len(self.vaults) - 1} Vaults")
+                    elif vaults_only:
+                        self.log.error(f" - No Content Key found in vaults for {kid.hex}")
+                        sys.exit(1)
+
+                if kid not in drm.content_keys and not vaults_only:
+                    from_vaults = drm.content_keys.copy()
+
+                    try:
+                        drm.get_content_keys(
+                            cdm=self.cdm,
+                            licence=licence,
+                            certificate=certificate
+                        )
+                    except ValueError as e:
+                        self.log.error(str(e))
+                        sys.exit(1)
+
+                    self.log.info("Content Keys:")
+                    for kid_, key in drm.content_keys.items():
+                        msg = f" + {kid_.hex}:{key}"
+                        if kid_ == kid:
+                            msg += " *"
+                        if key == "0" * 32:
+                            msg += " [Unusable!]"
+                        self.log.info(msg)
+
+                    drm.content_keys = {
+                        kid_: key
+                        for kid_, key in drm.content_keys.items()
+                        if key and key.count("0") != len(key)
+                    }
+
+                    # The CDM keys may have returned blank content keys for KIDs we got from vaults.
+                    # So we re-add the keys from vaults earlier overwriting blanks or removed KIDs data.
+                    drm.content_keys.update(from_vaults)
+
+                    cached_keys = self.vaults.add_keys(drm.content_keys)
+                    self.log.info(f" + Newly added to {cached_keys}/{len(drm.content_keys)} Vaults")
+
+                    if kid not in drm.content_keys:
+                        self.log.error(f" - No Content Key with the KID ({kid.hex}) was returned...")
+                        sys.exit(1)
+
+    def mux_tracks(self, title: Title_T, season_folder: bool = True, add_source: bool = True) -> None:
+        """Mux Tracks, Delete Pre-Mux files, and move to the final location."""
+        self.log.info("Muxing Tracks into a Matroska Container")
+
+        if isinstance(title, (Movie, Episode)):
+            muxed_path, return_code = title.tracks.mux(str(title))
+            if return_code == 1:
+                self.log.warning("mkvmerge had at least one warning, will continue anyway...")
+            elif return_code >= 2:
+                self.log.error(" - Failed to Mux video to Matroska file")
+                sys.exit(1)
+            self.log.info(f" + Muxed to {muxed_path}")
+        else:
+            # dont mux
+            muxed_path = title.tracks.audio[0].path
+
+        media_info = MediaInfo.parse(muxed_path)
+        final_dir = config.directories.downloads
+        final_filename = title.get_filename(media_info, show_service=add_source)
+
+        if season_folder and isinstance(title, (Episode, Song)):
+            final_dir /= title.get_filename(media_info, show_service=add_source, folder=True)
+
+        final_dir.mkdir(parents=True, exist_ok=True)
+        final_path = final_dir / f"{final_filename}{muxed_path.suffix}"
+
+        muxed_path.rename(final_path)
+        self.log.info(f" + Moved to {final_path}")
+
+    @staticmethod
+    def rotate_log_file(log_path: Path, keep: int = 20) -> Path:
+        """
+        Update Log Filename and delete old log files.
+        It keeps only the 20 newest logs by default.
+        """
+        if not log_path:
+            raise ValueError("A log path must be provided")
+
+        try:
+            log_path.relative_to(Path(""))  # file name only
+        except ValueError:
+            pass
+        else:
+            log_path = config.directories.logs / log_path
+
+        log_path = log_path.parent / log_path.name.format_map(defaultdict(
+            str,
+            name="root",
+            time=datetime.now().strftime("%Y%m%d-%H%M%S")
+        ))
+
+        if log_path.parent.exists():
+            log_files = [x for x in log_path.parent.iterdir() if x.suffix == log_path.suffix]
+            for log_file in log_files[::-1][keep-1:]:
+                # keep n newest files and delete the rest
+                log_file.unlink()
+
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        return log_path
+
+    @staticmethod
+    def get_profile(service: str) -> Optional[str]:
+        """Get profile for Service from config."""
+        profile = config.profiles.get(service)
+        if profile is False:
+            return None  # auth-less service if `false` in config
+        if not profile:
+            profile = config.profiles.get("default")
+        if not profile:
+            raise ValueError(f"No profile has been defined for '{service}' in the config.")
+        return profile
+
+    @staticmethod
+    def get_cookie_jar(service: str, profile: str) -> Optional[MozillaCookieJar]:
+        """Get Profile's Cookies as Mozilla Cookie Jar if available."""
+        cookie_file = config.directories.cookies / service / f"{profile}.txt"
+        if cookie_file.is_file():
+            cookie_jar = MozillaCookieJar(cookie_file)
+            cookie_data = html.unescape(cookie_file.read_text("utf8")).splitlines(keepends=False)
+            for i, line in enumerate(cookie_data):
+                if line and not line.startswith("#"):
+                    line_data = line.lstrip().split("\t")
+                    # Disable client-side expiry checks completely across everywhere
+                    # Even though the cookies are loaded under ignore_expires=True, stuff
+                    # like python-requests may not use them if they are expired
+                    line_data[4] = ""
+                    cookie_data[i] = "\t".join(line_data)
+            cookie_data = "\n".join(cookie_data)
+            cookie_file.write_text(cookie_data, "utf8")
+            cookie_jar.load(ignore_discard=True, ignore_expires=True)
+            return cookie_jar
+        return None
+
+    @staticmethod
+    def get_credentials(service: str, profile: str) -> Optional[Credential]:
+        """Get Profile's Credential if available."""
+        cred = config.credentials.get(service, {}).get(profile)
+        if cred:
+            if isinstance(cred, list):
+                return Credential(*cred)
+            return Credential.loads(cred)
+        return None
+
+    @staticmethod
+    def get_cdm(service: str, profile: Optional[str] = None) -> WidevineCdm:
+        """
+        Get CDM for a specified service (either Local or Remote CDM).
+        Raises a ValueError if there's a problem getting a CDM.
+        """
+        cdm_name = config.cdm.get(service) or config.cdm.get("default")
+        if not cdm_name:
+            raise ValueError("A CDM to use wasn't listed in the config")
+
+        if isinstance(cdm_name, dict):
+            if not profile:
+                raise ValueError("CDM config is mapped for profiles, but no profile was chosen")
+            cdm_name = cdm_name.get(profile) or config.cdm.get("default")
+            if not cdm_name:
+                raise ValueError(f"A CDM to use was not mapped for the profile {profile}")
+
+        cdm_api = next(iter(x for x in config.remote_cdm if x["name"] == cdm_name), None)
+        if cdm_api:
+            del cdm_api["name"]
+            return RemoteCdm(**cdm_api)
+
+        cdm_path = config.directories.wvds / f"{cdm_name}.wvd"
+        if not cdm_path.is_file():
+            raise ValueError(f"{cdm_name} does not exist or is not a file")
+        device = Device.load(cdm_path)
+        return WidevineCdm.from_device(device)

devine/commands/env.py

@@ -0,0 +1,64 @@
+import logging
+import shutil
+from typing import Optional
+
+import click
+
+from devine.core.config import config
+from devine.core.constants import context_settings
+from devine.core.services import Services
+
+
+@click.group(short_help="Manage and configure the project environment.", context_settings=context_settings)
+def env() -> None:
+    """Manage and configure the project environment."""
+
+
+@env.command()
+def info() -> None:
+    """Displays information about the current environment."""
+    log = logging.getLogger("env")
+    log.info(f"[Root Config]     : {config.directories.user_configs / config.filenames.root_config}")
+    log.info(f"[Cookies]         : {config.directories.cookies}")
+    log.info(f"[WVDs]            : {config.directories.wvds}")
+    log.info(f"[Cache]           : {config.directories.cache}")
+    log.info(f"[Logs]            : {config.directories.logs}")
+    log.info(f"[Temp Files]      : {config.directories.temp}")
+    log.info(f"[Downloads]       : {config.directories.downloads}")
+
+
+@env.group(name="clear", short_help="Clear an environment directory.", context_settings=context_settings)
+def clear() -> None:
+    """Clear an environment directory."""
+
+
+@clear.command()
+@click.argument("service", type=str, required=False)
+def cache(service: Optional[str]) -> None:
+    """Clear the environment cache directory."""
+    log = logging.getLogger("env")
+    cache_dir = config.directories.cache
+    if service:
+        cache_dir = cache_dir / Services.get_tag(service)
+    log.info(f"Clearing cache directory: {cache_dir}")
+    files_count = len(list(cache_dir.glob("**/*")))
+    if not files_count:
+        log.info("No files to delete")
+    else:
+        log.info(f"Deleting {files_count} files...")
+        shutil.rmtree(cache_dir)
+        log.info("Cleared")
+
+
+@clear.command()
+def temp() -> None:
+    """Clear the environment temp directory."""
+    log = logging.getLogger("env")
+    log.info(f"Clearing temp directory: {config.directories.temp}")
+    files_count = len(list(config.directories.temp.glob("**/*")))
+    if not files_count:
+        log.info("No files to delete")
+    else:
+        log.info(f"Deleting {files_count} files...")
+        shutil.rmtree(config.directories.temp)
+        log.info("Cleared")

devine/commands/kv.py

@@ -0,0 +1,212 @@
+from __future__ import annotations
+
+import logging
+import re
+from pathlib import Path
+from typing import Optional
+
+import click
+
+from devine.core.vault import Vault
+from devine.core.config import config
+from devine.core.constants import context_settings
+from devine.core.services import Services
+from devine.core.vaults import Vaults
+
+
+@click.group(short_help="Manage and configure Key Vaults.", context_settings=context_settings)
+def kv() -> None:
+    """Manage and configure Key Vaults."""
+
+
+@kv.command()
+@click.argument("to_vault", type=str)
+@click.argument("from_vaults", nargs=-1, type=click.UNPROCESSED)
+@click.option("-s", "--service", type=str, default=None,
+              help="Only copy data to and from a specific service.")
+def copy(to_vault: str, from_vaults: list[str], service: Optional[str] = None) -> None:
+    """
+    Copy data from multiple Key Vaults into a single Key Vault.
+    Rows with matching KIDs are skipped unless there's no KEY set.
+    Existing data is not deleted or altered.
+
+    The `to_vault` argument is the key vault you wish to copy data to.
+    It should be the name of a Key Vault defined in the config.
+
+    The `from_vaults` argument is the key vault(s) you wish to take
+    data from. You may supply multiple key vaults.
+    """
+    if not from_vaults:
+        raise click.ClickException("No Vaults were specified to copy data from.")
+
+    log = logging.getLogger("kv")
+
+    vaults = Vaults()
+    for vault_name in [to_vault] + list(from_vaults):
+        vault = next((x for x in config.key_vaults if x["name"] == vault_name), None)
+        if not vault:
+            raise click.ClickException(f"Vault ({vault_name}) is not defined in the config.")
+        vault_type = vault["type"]
+        vault_args = vault.copy()
+        del vault_args["type"]
+        vaults.load(vault_type, **vault_args)
+
+    to_vault: Vault = vaults.vaults[0]
+    from_vaults: list[Vault] = vaults.vaults[1:]
+
+    log.info(f"Copying data from {', '.join([x.name for x in from_vaults])}, into {to_vault.name}")
+    if service:
+        service = Services.get_tag(service)
+        log.info(f"Only copying data for service {service}")
+
+    total_added = 0
+    for from_vault in from_vaults:
+        if service:
+            services = [service]
+        else:
+            services = from_vault.get_services()
+
+        for service_ in services:
+            log.info(f"Getting data from {from_vault} for {service_}")
+            content_keys = list(from_vault.get_keys(service_))  # important as it's a generator we iterate twice
+
+            bad_keys = {
+                kid: key
+                for kid, key in content_keys
+                if not key or key.count("0") == len(key)
+            }
+
+            for kid, key in bad_keys.items():
+                log.warning(f"Cannot add a NULL Content Key to a Vault, skipping: {kid}:{key}")
+
+            content_keys = {
+                kid: key
+                for kid, key in content_keys
+                if kid not in bad_keys
+            }
+
+            total_count = len(content_keys)
+            log.info(f"Adding {total_count} Content Keys to {to_vault} for {service_}")
+
+            try:
+                added = to_vault.add_keys(service_, content_keys, commit=True)
+            except PermissionError:
+                log.warning(f" - No permission to create table ({service_}) in {to_vault}, skipping...")
+                continue
+
+            total_added += added
+            existed = total_count - added
+
+            log.info(f"{to_vault} ({service_}): {added} newly added, {existed} already existed (skipped)")
+
+    log.info(f"{to_vault}: {total_added} total newly added")
+
+
+@kv.command()
+@click.argument("vaults", nargs=-1, type=click.UNPROCESSED)
+@click.option("-s", "--service", type=str, default=None,
+              help="Only sync data to and from a specific service.")
+@click.pass_context
+def sync(ctx: click.Context, vaults: list[str], service: Optional[str] = None) -> None:
+    """
+    Ensure multiple Key Vaults copies of all keys as each other.
+    It's essentially just a bi-way copy between each vault.
+    To see the precise details of what it's doing between each
+    provided vault, see the documentation for the `copy` command.
+    """
+    if not len(vaults) > 1:
+        raise click.ClickException("You must provide more than one Vault to sync.")
+
+    ctx.invoke(copy, to_vault=vaults[0], from_vaults=vaults[1:], service=service)
+    for i in range(1, len(vaults)):
+        ctx.invoke(copy, to_vault=vaults[i], from_vaults=[vaults[i-1]], service=service)
+
+
+@kv.command()
+@click.argument("file", type=Path)
+@click.argument("service", type=str)
+@click.argument("vaults", nargs=-1, type=click.UNPROCESSED)
+def add(file: Path, service: str, vaults: list[str]) -> None:
+    """
+    Add new Content Keys to Key Vault(s) by service.
+
+    File should contain one key per line in the format KID:KEY (HEX:HEX).
+    Each line should have nothing else within it except for the KID:KEY.
+    Encoding is presumed to be UTF8.
+    """
+    if not file.exists():
+        raise click.ClickException(f"File provided ({file}) does not exist.")
+    if not file.is_file():
+        raise click.ClickException(f"File provided ({file}) is not a file.")
+    if not service or not isinstance(service, str):
+        raise click.ClickException(f"Service provided ({service}) is invalid.")
+    if len(vaults) < 1:
+        raise click.ClickException("You must provide at least one Vault.")
+
+    log = logging.getLogger("kv")
+    service = Services.get_tag(service)
+
+    vaults_ = Vaults()
+    for vault_name in vaults:
+        vault = next((x for x in config.key_vaults if x["name"] == vault_name), None)
+        if not vault:
+            raise click.ClickException(f"Vault ({vault_name}) is not defined in the config.")
+        vault_type = vault["type"]
+        vault_args = vault.copy()
+        del vault_args["type"]
+        vaults_.load(vault_type, **vault_args)
+
+    data = file.read_text(encoding="utf8")
+    kid_keys: dict[str, str] = {}
+    for line in data.splitlines(keepends=False):
+        line = line.strip()
+        match = re.search(r"^(?P<kid>[0-9a-fA-F]{32}):(?P<key>[0-9a-fA-F]{32})$", line)
+        if not match:
+            continue
+        kid = match.group("kid").lower()
+        key = match.group("key").lower()
+        kid_keys[kid] = key
+
+    total_count = len(kid_keys)
+
+    for vault in vaults_:
+        log.info(f"Adding {total_count} Content Keys to {vault}")
+        added_count = vault.add_keys(service, kid_keys, commit=True)
+        existed_count = total_count - added_count
+        log.info(f"{vault}: {added_count} newly added, {existed_count} already existed (skipped)")
+
+    log.info("Done!")
+
+
+@kv.command()
+@click.argument("vaults", nargs=-1, type=click.UNPROCESSED)
+def prepare(vaults: list[str]) -> None:
+    """Create Service Tables on Vaults if not yet created."""
+    log = logging.getLogger("kv")
+
+    vaults_ = Vaults()
+    for vault_name in vaults:
+        vault = next((x for x in config.key_vaults if x["name"] == vault_name), None)
+        if not vault:
+            raise click.ClickException(f"Vault ({vault_name}) is not defined in the config.")
+        vault_type = vault["type"]
+        vault_args = vault.copy()
+        del vault_args["type"]
+        vaults_.load(vault_type, **vault_args)
+
+    for vault in vaults_:
+        if hasattr(vault, "has_table") and hasattr(vault, "create_table"):
+            for service_tag in Services.get_tags():
+                if vault.has_table(service_tag):
+                    log.info(f"{vault} already has a {service_tag} Table")
+                else:
+                    try:
+                        vault.create_table(service_tag, commit=True)
+                        log.info(f"{vault}: Created {service_tag} Table")
+                    except PermissionError:
+                        log.error(f"{vault} user has no create table permission, skipping...")
+                        continue
+        else:
+            log.info(f"{vault} does not use tables, skipping...")
+
+    log.info("Done!")

devine/commands/serve.py

@@ -0,0 +1,50 @@
+import subprocess
+
+import click
+
+from devine.core.config import config
+from devine.core.constants import context_settings
+from devine.core.utilities import get_binary_path
+
+
+@click.command(
+    short_help="Serve your Local Widevine Devices for Remote Access.",
+    context_settings=context_settings)
+@click.option("-h", "--host", type=str, default="0.0.0.0", help="Host to serve from.")
+@click.option("-p", "--port", type=int, default=8786, help="Port to serve from.")
+@click.option("--caddy", is_flag=True, default=False, help="Also serve with Caddy.")
+def serve(host: str, port: int, caddy: bool) -> None:
+    """
+    Serve your Local Widevine Devices for Remote Access.
+
+    \b
+    Host as 127.0.0.1 may block remote access even if port-forwarded.
+    Instead, use 0.0.0.0 and ensure the TCP port you choose is forwarded.
+
+    \b
+    You may serve with Caddy at the same time with --caddy. You can use Caddy
+    as a reverse-proxy to serve with HTTPS. The config used will be the Caddyfile
+    next to the devine config.
+    """
+    from pywidevine import serve
+
+    if caddy:
+        executable = get_binary_path("caddy")
+        if not executable:
+            raise click.ClickException("Caddy executable \"caddy\" not found but is required for --caddy.")
+        caddy_p = subprocess.Popen([
+            executable,
+            "run",
+            "--config", str(config.directories.user_configs / "Caddyfile")
+        ])
+    else:
+        caddy_p = None
+
+    try:
+        if not config.serve.get("devices"):
+            config.serve["devices"] = []
+        config.serve["devices"].extend(list(config.directories.wvds.glob("*.wvd")))
+        serve.run(config.serve, host, port)
+    finally:
+        if caddy_p:
+            caddy_p.kill()

devine/commands/util.py

@@ -0,0 +1,104 @@
+import subprocess
+from pathlib import Path
+
+import click
+from pymediainfo import MediaInfo
+
+from devine.core.constants import context_settings
+from devine.core.utilities import get_binary_path
+
+
+@click.group(short_help="Various helper scripts and programs.", context_settings=context_settings)
+def util() -> None:
+    """Various helper scripts and programs."""
+
+
+@util.command()
+@click.argument("path", type=Path)
+@click.argument("aspect", type=str)
+@click.option("--letter/--pillar", default=True,
+              help="Specify which direction to crop. Top and Bottom would be --letter, Sides would be --pillar.")
+@click.option("-o", "--offset", type=int, default=0,
+              help="Fine tune the computed crop area if not perfectly centered.")
+@click.option("-p", "--preview", is_flag=True, default=False,
+              help="Instantly preview the newly-set aspect crop in MPV (or ffplay if mpv is unavailable).")
+def crop(path: Path, aspect: str, letter: bool, offset: int, preview: bool) -> None:
+    """
+    Losslessly crop H.264 and H.265 video files at the bit-stream level.
+    You may provide a path to a file, or a folder of mkv and/or mp4 files.
+
+    Note: If you notice that the values you put in are not quite working, try
+    tune -o/--offset. This may be necessary on videos with sub-sampled chroma.
+
+    Do note that you may not get an ideal lossless cropping result on some
+    cases, again due to sub-sampled chroma.
+
+    It's recommended that you try -o about 10 or so pixels and lower it until
+    you get as close in as possible. Do make sure it's not over-cropping either
+    as it may go from being 2px away from a perfect crop, to 20px over-cropping
+    again due to sub-sampled chroma.
+    """
+    executable = get_binary_path("ffmpeg")
+    if not executable:
+        raise click.ClickException("FFmpeg executable \"ffmpeg\" not found but is required.")
+
+    if path.is_dir():
+        paths = list(path.glob("*.mkv")) + list(path.glob("*.mp4"))
+    else:
+        paths = [path]
+    for video_path in paths:
+        try:
+            video_track = next(iter(MediaInfo.parse(video_path).video_tracks or []))
+        except StopIteration:
+            raise click.ClickException("There's no video tracks in the provided file.")
+
+        crop_filter = {
+            "HEVC": "hevc_metadata",
+            "AVC": "h264_metadata"
+        }.get(video_track.commercial_name)
+        if not crop_filter:
+            raise click.ClickException(f"{video_track.commercial_name} Codec not supported.")
+
+        aspect_w, aspect_h = list(map(float, aspect.split(":")))
+        if letter:
+            crop_value = (video_track.height - (video_track.width / (aspect_w * aspect_h))) / 2
+            left, top, right, bottom = map(int, [0, crop_value + offset, 0, crop_value - offset])
+        else:
+            crop_value = (video_track.width - (video_track.height * (aspect_w / aspect_h))) / 2
+            left, top, right, bottom = map(int, [crop_value + offset, 0, crop_value - offset, 0])
+        crop_filter += f"=crop_left={left}:crop_top={top}:crop_right={right}:crop_bottom={bottom}"
+
+        if min(left, top, right, bottom) < 0:
+            raise click.ClickException("Cannot crop less than 0, are you cropping in the right direction?")
+
+        if preview:
+            out_path = ["-f", "mpegts", "-"]  # pipe
+        else:
+            out_path = [str(video_path.with_stem(".".join(filter(bool, [
+                video_path.stem,
+                video_track.language,
+                "crop",
+                str(offset or "")
+            ]))).with_suffix({
+                # ffmpeg's MKV muxer does not yet support HDR
+                "HEVC": ".h265",
+                "AVC": ".h264"
+            }.get(video_track.commercial_name, ".mp4")))]
+
+        ffmpeg_call = subprocess.Popen([
+            executable, "-y",
+            "-i", str(video_path),
+            "-map", "0:v:0",
+            "-c", "copy",
+            "-bsf:v", crop_filter
+        ] + out_path, stdout=subprocess.PIPE)
+        try:
+            if preview:
+                previewer = get_binary_path("mpv", "ffplay")
+                if not previewer:
+                    raise click.ClickException("MPV/FFplay executables weren't found but are required for previewing.")
+                subprocess.Popen((previewer, "-"), stdin=ffmpeg_call.stdout)
+        finally:
+            if ffmpeg_call.stdout:
+                ffmpeg_call.stdout.close()
+            ffmpeg_call.wait()

devine/commands/wvd.py

@@ -0,0 +1,215 @@
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Optional
+
+import click
+import yaml
+from google.protobuf.json_format import MessageToDict
+from pywidevine.device import Device
+from pywidevine.license_protocol_pb2 import FileHashes
+from unidecode import UnidecodeError, unidecode
+
+from devine.core.config import config
+from devine.core.constants import context_settings
+
+
+@click.group(
+    short_help="Manage configuration and creation of WVD (Widevine Device) files.",
+    context_settings=context_settings)
+def wvd() -> None:
+    """Manage configuration and creation of WVD (Widevine Device) files."""
+
+
+@wvd.command()
+@click.argument("path", type=Path)
+def parse(path: Path) -> None:
+    """
+    Parse a .WVD Widevine Device file to check information.
+    Relative paths are relative to the WVDs directory.
+    """
+    try:
+        named = not path.suffix and path.relative_to(Path(""))
+    except ValueError:
+        named = False
+    if named:
+        path = config.directories.wvds / f"{path.name}.wvd"
+
+    log = logging.getLogger("wvd")
+
+    device = Device.load(path)
+
+    log.info(f"System ID: {device.system_id}")
+    log.info(f"Security Level: {device.security_level}")
+    log.info(f"Type: {device.type}")
+    log.info(f"Flags: {device.flags}")
+    log.info(f"Private Key: {bool(device.private_key)}")
+    log.info(f"Client ID: {bool(device.client_id)}")
+    log.info(f"VMP: {bool(device.client_id.vmp_data)}")
+
+    log.info("Client ID:")
+    log.info(device.client_id)
+
+    log.info("VMP:")
+    if device.client_id.vmp_data:
+        file_hashes = FileHashes()
+        file_hashes.ParseFromString(device.client_id.vmp_data)
+        log.info(str(file_hashes))
+    else:
+        log.info("None")
+
+
+@wvd.command()
+@click.argument("wvd_paths", type=Path, nargs=-1)
+@click.argument("out_dir", type=Path, nargs=1)
+def dump(wvd_paths: list[Path], out_dir: Path) -> None:
+    """
+    Extract data from a .WVD Widevine Device file to a folder structure.
+
+    If the path is relative, with no file extension, it will dump the WVD in the WVDs
+    directory.
+    """
+    if wvd_paths == (Path(""),):
+        wvd_paths = list(config.directories.wvds.iterdir())
+    for wvd_path, out_path in zip(wvd_paths, (out_dir / x.stem for x in wvd_paths)):
+        try:
+            named = not wvd_path.suffix and wvd_path.relative_to(Path(""))
+        except ValueError:
+            named = False
+        if named:
+            wvd_path = config.directories.wvds / f"{wvd_path.stem}.wvd"
+        out_path.mkdir(parents=True, exist_ok=True)
+
+        device = Device.load(wvd_path)
+
+        log = logging.getLogger("wvd")
+        log.info(f"Dumping: {wvd_path}")
+        log.info(f"L{device.security_level} {device.system_id} {device.type.name}")
+        log.info(f"Saving to: {out_path}")
+
+        device_meta = {
+            "wvd": {
+                "device_type": device.type.name,
+                "security_level": device.security_level,
+                **device.flags
+            },
+            "client_info": {},
+            "capabilities": MessageToDict(device.client_id, preserving_proto_field_name=True)["client_capabilities"]
+        }
+        for client_info in device.client_id.client_info:
+            device_meta["client_info"][client_info.name] = client_info.value
+
+        device_meta_path = out_path / "metadata.yml"
+        device_meta_path.write_text(yaml.dump(device_meta), encoding="utf8")
+        log.info(" + Device Metadata")
+
+        if device.private_key:
+            private_key_path = out_path / "private_key.pem"
+            private_key_path.write_text(
+                data=device.private_key.export_key().decode(),
+                encoding="utf8"
+            )
+            private_key_path.with_suffix(".der").write_bytes(
+                device.private_key.export_key(format="DER")
+            )
+            log.info(" + Private Key")
+        else:
+            log.warning(" - No Private Key available")
+
+        if device.client_id:
+            client_id_path = out_path / "client_id.bin"
+            client_id_path.write_bytes(device.client_id.SerializeToString())
+            log.info(" + Client ID")
+        else:
+            log.warning(" - No Client ID available")
+
+        if device.client_id.vmp_data:
+            vmp_path = out_path / "vmp.bin"
+            vmp_path.write_bytes(device.client_id.vmp_data)
+            log.info(" + VMP (File Hashes)")
+        else:
+            log.info(" - No VMP (File Hashes) available")
+
+
+@wvd.command()
+@click.argument("name", type=str)
+@click.argument("private_key", type=Path)
+@click.argument("client_id", type=Path)
+@click.argument("file_hashes", type=Path, required=False)
+@click.option("-t", "--type", "type_", type=click.Choice([x.name for x in Device.Types], case_sensitive=False),
+              default="Android", help="Device Type")
+@click.option("-l", "--level", type=click.IntRange(1, 3), default=1, help="Device Security Level")
+@click.option("-o", "--output", type=Path, default=None, help="Output Directory")
+@click.pass_context
+def new(
+    ctx: click.Context,
+    name: str,
+    private_key: Path,
+    client_id: Path,
+    file_hashes: Optional[Path],
+    type_: str,
+    level: int,
+    output: Optional[Path]
+) -> None:
+    """
+    Create a new .WVD Widevine provision file.
+
+    name: The origin device name of the provided data. e.g. `Nexus 6P`. You do not need to
+        specify the security level, that will be done automatically.
+    private_key: A PEM file of a Device's private key.
+    client_id: A binary blob file which follows the Widevine ClientIdentification protobuf
+        schema.
+    file_hashes: A binary blob file with follows the Widevine FileHashes protobuf schema.
+        Also known as VMP as it's used for VMP (Verified Media Path) assurance.
+    """
+    try:
+        # TODO: Remove need for name, create name based on Client IDs ClientInfo values
+        name = unidecode(name.strip().lower().replace(" ", "_"))
+    except UnidecodeError as e:
+        raise click.UsageError(f"name: Failed to sanitize name, {e}", ctx)
+    if not name:
+        raise click.UsageError("name: Empty after sanitizing, please make sure the name is valid.", ctx)
+    if not private_key.is_file():
+        raise click.UsageError("private_key: Not a path to a file, or it doesn't exist.", ctx)
+    if not client_id.is_file():
+        raise click.UsageError("client_id: Not a path to a file, or it doesn't exist.", ctx)
+    if file_hashes and not file_hashes.is_file():
+        raise click.UsageError("file_hashes: Not a path to a file, or it doesn't exist.", ctx)
+
+    device = Device(
+        type_=Device.Types[type_.upper()],
+        security_level=level,
+        flags=None,
+        private_key=private_key.read_bytes(),
+        client_id=client_id.read_bytes()
+    )
+
+    if file_hashes:
+        device.client_id.vmp_data = file_hashes.read_bytes()
+
+    out_path = (output or config.directories.wvds) / f"{name}_{device.system_id}_l{device.security_level}.wvd"
+    device.dump(out_path)
+
+    log = logging.getLogger("wvd")
+
+    log.info(f"Created binary WVD file, {out_path.name}")
+    log.info(f" + Saved to: {out_path.absolute()}")
+    log.info(f" + System ID: {device.system_id}")
+    log.info(f" + Security Level: {device.security_level}")
+    log.info(f" + Type: {device.type}")
+    log.info(f" + Flags: {device.flags}")
+    log.info(f" + Private Key: {bool(device.private_key)}")
+    log.info(f" + Client ID: {bool(device.client_id)}")
+    log.info(f" + VMP: {bool(device.client_id.vmp_data)}")
+
+    log.debug("Client ID:")
+    log.debug(device.client_id)
+
+    log.debug("VMP:")
+    if device.client_id.vmp_data:
+        file_hashes = FileHashes()
+        file_hashes.ParseFromString(device.client_id.vmp_data)
+        log.info(str(file_hashes))
+    else:
+        log.info("None")

devine/core/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.0.0"

devine/core/__main__.py

@@ -0,0 +1,29 @@
+import logging
+from datetime import datetime
+
+import click
+import coloredlogs
+
+from devine.core import __version__
+from devine.core.commands import Commands
+from devine.core.constants import context_settings, LOG_FORMAT
+
+
+@click.command(cls=Commands, invoke_without_command=True, context_settings=context_settings)
+@click.option("-v", "--version", is_flag=True, default=False, help="Print version information.")
+@click.option("-d", "--debug", is_flag=True, default=False, help="Enable DEBUG level logs.")
+def main(version: bool, debug: bool) -> None:
+    """Devine—Open-Source Movie, TV, and Music Downloading Solution."""
+    logging.basicConfig(level=logging.DEBUG if debug else logging.INFO)
+    log = logging.getLogger()
+    coloredlogs.install(level=log.level, fmt=LOG_FORMAT, style="{")
+
+    log.info(f"Devine version {__version__} Copyright (c) 2019-{datetime.now().year} rlaphoenix")
+    log.info("Convenient Widevine-DRM Downloader and Decrypter.")
+    log.info("https://github.com/devine/devine")
+    if version:
+        return
+
+
+if __name__ == "__main__":
+    main()

devine/core/cacher.py

@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+import zlib
+from datetime import datetime, timedelta
+from os import stat_result
+from pathlib import Path
+from typing import Optional, Any, Union
+
+import jsonpickle
+import jwt
+
+from devine.core.config import config
+
+
+EXP_T = Union[datetime, str, int, float]
+
+
+class Cacher:
+    """Cacher for Services to get and set arbitrary data with expiration dates."""
+
+    def __init__(
+        self,
+        service_tag: str,
+        key: Optional[str] = None,
+        version: Optional[int] = 1,
+        data: Optional[Any] = None,
+        expiration: Optional[datetime] = None
+    ) -> None:
+        self.service_tag = service_tag
+        self.key = key
+        self.version = version
+        self.data = data or {}
+        self.expiration = expiration
+
+        if self.expiration and self.expired:
+            # if its expired, remove the data for safety and delete cache file
+            self.data = None
+            self.path.unlink()
+
+    def __bool__(self) -> bool:
+        return bool(self.data)
+
+    @property
+    def path(self) -> Path:
+        """Get the path at which the cache will be read and written."""
+        return (config.directories.cache / self.service_tag / self.key).with_suffix(".json")
+
+    @property
+    def expired(self) -> bool:
+        return self.expiration and self.expiration < datetime.utcnow()
+
+    def get(self, key: str, version: int = 1) -> Cacher:
+        """
+        Get Cached data for the Service by Key.
+        :param key: the filename to save the data to, should be url-safe.
+        :param version: the config data version you expect to use.
+        :returns: Cache object containing the cached data or None if the file does not exist.
+        """
+        cache = Cacher(self.service_tag, key, version)
+        if cache.path.is_file():
+            data = jsonpickle.loads(cache.path.read_text(encoding="utf8"))
+            payload = data.copy()
+            del payload["crc32"]
+            checksum = data["crc32"]
+            calculated = zlib.crc32(jsonpickle.dumps(payload).encode("utf8"))
+            if calculated != checksum:
+                raise ValueError(
+                    f"The checksum of the Cache payload mismatched. "
+                    f"Checksum: {checksum} !== Calculated: {calculated}"
+                )
+            cache.data = data["data"]
+            cache.expiration = data["expiration"]
+            cache.version = data["version"]
+            if cache.version != version:
+                raise ValueError(
+                    f"The version of your {self.service_tag} {key} cache is outdated. "
+                    f"Please delete: {cache.path}"
+                )
+        return cache
+
+    def set(self, data: Any, expiration: Optional[EXP_T] = None) -> Any:
+        """
+        Set Cached data for the Service by Key.
+        :param data: absolutely anything including None.
+        :param expiration: when the data expires, optional. Can be ISO 8601, seconds
+            til expiration, unix timestamp, or a datetime object.
+        :returns: the data provided for quick wrapping of functions or vars.
+        """
+        self.data = data
+
+        if not expiration:
+            try:
+                expiration = jwt.decode(self.data, options={"verify_signature": False})["exp"]
+            except jwt.DecodeError:
+                pass
+
+        self.expiration = self._resolve_datetime(expiration) if expiration else None
+
+        payload = {
+            "data": self.data,
+            "expiration": self.expiration,
+            "version": self.version
+        }
+        payload["crc32"] = zlib.crc32(jsonpickle.dumps(payload).encode("utf8"))
+
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.path.write_text(jsonpickle.dumps(payload))
+
+        return self.data
+
+    def stat(self) -> stat_result:
+        """
+        Get Cache file OS Stat data like Creation Time, Modified Time, and such.
+        :returns: an os.stat_result tuple
+        """
+        return self.path.stat()
+
+    @staticmethod
+    def _resolve_datetime(timestamp: EXP_T) -> datetime:
+        """
+        Resolve multiple formats of a Datetime or Timestamp to an absolute Datetime.
+
+        Examples:
+            >>> now = datetime.now()
+            datetime.datetime(2022, 6, 27, 9, 49, 13, 657208)
+            >>> iso8601 = now.isoformat()
+            '2022-06-27T09:49:13.657208'
+            >>> Cacher._resolve_datetime(iso8601)
+            datetime.datetime(2022, 6, 27, 9, 49, 13, 657208)
+            >>> Cacher._resolve_datetime(iso8601 + "Z")
+            datetime.datetime(2022, 6, 27, 9, 49, 13, 657208)
+            >>> Cacher._resolve_datetime(3600)
+            datetime.datetime(2022, 6, 27, 10, 52, 50, 657208)
+            >>> Cacher._resolve_datetime('3600')
+            datetime.datetime(2022, 6, 27, 10, 52, 51, 657208)
+            >>> Cacher._resolve_datetime(7800.113)
+            datetime.datetime(2022, 6, 27, 11, 59, 13, 770208)
+
+        In the int/float examples you may notice that it did not return now + 3600 seconds
+        but rather something a bit more than that. This is because it did not resolve 3600
+        seconds from the `now` variable but from right now as the function was called.
+        """
+        if isinstance(timestamp, datetime):
+            return timestamp
+        if isinstance(timestamp, str):
+            if timestamp.endswith("Z"):
+                # fromisoformat doesn't accept the final Z
+                timestamp = timestamp.split("Z")[0]
+            try:
+                return datetime.fromisoformat(timestamp)
+            except ValueError:
+                timestamp = float(timestamp)
+        try:
+            timestamp = datetime.fromtimestamp(timestamp)
+        except ValueError:
+            raise ValueError(f"Unrecognized Timestamp value {timestamp!r}")
+        if timestamp < datetime.now():
+            # timestamp is likely an amount of seconds til expiration
+            # or, it's an already expired timestamp which is unlikely
+            timestamp = timestamp + timedelta(seconds=datetime.now().timestamp())
+        return timestamp

devine/core/commands.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from typing import Optional
+
+import click
+
+from devine.core.config import config
+from devine.core.utilities import import_module_by_path
+
+_COMMANDS = sorted(
+    (
+        path
+        for path in config.directories.commands.glob("*.py")
+        if path.stem.lower() != "__init__"
+    ),
+    key=lambda x: x.stem
+)
+
+_MODULES = {
+    path.stem: getattr(import_module_by_path(path), path.stem)
+    for path in _COMMANDS
+}
+
+
+class Commands(click.MultiCommand):
+    """Lazy-loaded command group of project commands."""
+
+    def list_commands(self, ctx: click.Context) -> list[str]:
+        """Returns a list of command names from the command filenames."""
+        return [x.stem for x in _COMMANDS]
+
+    def get_command(self, ctx: click.Context, name: str) -> Optional[click.Command]:
+        """Load the command code and return the main click command function."""
+        module = _MODULES.get(name)
+        if not module:
+            raise click.ClickException(f"Unable to find command by the name '{name}'")
+
+        if hasattr(module, "cli"):
+            return module.cli
+
+        return module
+
+
+# Hide direct access to commands from quick import form, they shouldn't be accessed directly
+__ALL__ = (Commands,)

devine/core/config.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import tempfile
+from pathlib import Path
+from typing import Any
+
+import yaml
+from appdirs import AppDirs
+
+
+class Config:
+    class _Directories:
+        # default directories, do not modify here, set via config
+        app_dirs = AppDirs("devine", False)
+        core_dir = Path(__file__).resolve().parent
+        namespace_dir = core_dir.parent
+        commands = namespace_dir / "commands"
+        services = namespace_dir / "services"
+        vaults = namespace_dir / "vaults"
+        user_configs = Path(app_dirs.user_config_dir)
+        data = Path(app_dirs.user_data_dir)
+        downloads = Path.home() / "Downloads" / "devine"
+        temp = Path(tempfile.gettempdir()) / "devine"
+        cache = Path(app_dirs.user_cache_dir)
+        cookies = data / "Cookies"
+        logs = Path(app_dirs.user_log_dir)
+        wvds = data / "WVDs"
+        dcsl = data / "DCSL"
+
+    class _Filenames:
+        # default filenames, do not modify here, set via config
+        log = "devine_{name}_{time}.log"  # Directories.logs
+        config = "config.yaml"  # Directories.services / tag
+        root_config = "devine.yaml"  # Directories.user_configs
+        chapters = "Chapters_{title}_{random}.txt"  # Directories.temp
+        subtitle = "Subtitle_{id}_{language}.srt"  # Directories.temp
+
+    def __init__(self, **kwargs: Any):
+        self.dl: dict = kwargs.get("dl") or {}
+        self.aria2c: dict = kwargs.get("aria2c") or {}
+        self.cdm: dict = kwargs.get("cdm") or {}
+        self.remote_cdm: list[dict] = kwargs.get("remote_cdm") or []
+        self.credentials: dict = kwargs.get("credentials") or {}
+
+        self.directories = self._Directories()
+        for name, path in (kwargs.get("directories") or {}).items():
+            if name.lower() in ("app_dirs", "core_dir", "namespace_dir", "user_configs", "data"):
+                # these must not be modified by the user
+                continue
+            setattr(self.directories, name, Path(path).expanduser())
+
+        self.filenames = self._Filenames()
+        for name, filename in (kwargs.get("filenames") or {}).items():
+            setattr(self.filenames, name, filename)
+
+        self.headers: dict = kwargs.get("headers") or {}
+        self.key_vaults: list[dict[str, Any]] = kwargs.get("key_vaults")
+        self.muxing: dict = kwargs.get("muxing") or {}
+        self.nordvpn: dict = kwargs.get("nordvpn") or {}
+        self.profiles: dict = kwargs.get("profiles") or {}
+        self.proxies: dict = kwargs.get("proxies") or {}
+        self.proxy_providers: dict = kwargs.get("proxy_providers") or {}
+        self.serve: dict = kwargs.get("serve") or {}
+        self.services: dict = kwargs.get("services") or {}
+        self.tag: str = kwargs.get("tag") or ""
+
+    @classmethod
+    def from_yaml(cls, path: Path) -> Config:
+        if not path.exists():
+            raise FileNotFoundError(f"Config file path ({path}) was not found")
+        if not path.is_file():
+            raise FileNotFoundError(f"Config file path ({path}) is not to a file.")
+        return cls(**yaml.safe_load(path.read_text(encoding="utf8")))
+
+
+# noinspection PyProtectedMember
+config = Config.from_yaml(Config._Directories.user_configs / Config._Filenames.root_config)
+
+__ALL__ = (config,)

devine/core/constants.py

@@ -0,0 +1,51 @@
+import logging
+from typing import TypeVar, Union
+
+
+LOG_FORMAT = "{asctime} [{levelname[0]}] {name} : {message}"  # must be '{}' style
+LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+LOG_FORMATTER = logging.Formatter(LOG_FORMAT, LOG_DATE_FORMAT, "{")
+DRM_SORT_MAP = ["ClearKey", "Widevine"]
+LANGUAGE_MUX_MAP = {
+    # List of language tags that cannot be used by mkvmerge and need replacements.
+    # Try get the replacement to be as specific locale-wise as possible.
+    # A bcp47 as the replacement is recommended.
+    "cmn": "zh",
+    "cmn-Hant": "zh-Hant",
+    "cmn-Hans": "zh-Hans",
+    "none": "und",
+    "yue": "zh-yue",
+    "yue-Hant": "zh-yue-Hant",
+    "yue-Hans": "zh-yue-Hans"
+}
+TERRITORY_MAP = {
+    "Hong Kong SAR China": "Hong Kong"
+}
+LANGUAGE_MAX_DISTANCE = 5  # this is max to be considered "same", e.g., en, en-US, en-AU
+VIDEO_CODEC_MAP = {
+    "AVC": "H.264",
+    "HEVC": "H.265"
+}
+DYNAMIC_RANGE_MAP = {
+    "HDR10": "HDR",
+    "HDR10+": "HDR",
+    "Dolby Vision": "DV"
+}
+AUDIO_CODEC_MAP = {
+    "E-AC-3": "DDP",
+    "AC-3": "DD"
+}
+
+context_settings = dict(
+    help_option_names=["-?", "-h", "--help"],  # default only has --help
+    max_content_width=116,  # max PEP8 line-width, -4 to adjust for initial indent
+)
+
+# For use in signatures of functions which take one specific type of track at a time
+# (it can't be a list that contains e.g. both Video and Audio objects)
+TrackT = TypeVar("TrackT", bound="Track")  # noqa: F821
+
+# For general use in lists that can contain mixed types of tracks.
+# list[Track] won't work because list is invariant.
+# TODO: Add Chapter?
+AnyTrack = Union["Video", "Audio", "Subtitle"]  # noqa: F821

devine/core/credential.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+import base64
+import hashlib
+import re
+from pathlib import Path
+from typing import Optional, Union
+
+
+class Credential:
+    """Username (or Email) and Password Credential."""
+
+    def __init__(self, username: str, password: str, extra: Optional[str] = None):
+        self.username = username
+        self.password = password
+        self.extra = extra
+        self.sha1 = hashlib.sha1(self.dumps().encode()).hexdigest()
+
+    def __bool__(self) -> bool:
+        return bool(self.username) and bool(self.password)
+
+    def __str__(self) -> str:
+        return self.dumps()
+
+    def __repr__(self) -> str:
+        return "{name}({items})".format(
+            name=self.__class__.__name__,
+            items=", ".join([f"{k}={repr(v)}" for k, v in self.__dict__.items()])
+        )
+
+    def dumps(self) -> str:
+        """Return credential data as a string."""
+        return f"{self.username}:{self.password}" + (f":{self.extra}" if self.extra else "")
+
+    def dump(self, path: Union[Path, str]) -> int:
+        """Write credential data to a file."""
+        if isinstance(path, str):
+            path = Path(path)
+        return path.write_text(self.dumps(), encoding="utf8")
+
+    def as_base64(self, with_extra: bool = False, encode_password: bool = False, encode_extra: bool = False) -> str:
+        """
+        Dump Credential as a Base64-encoded string in Basic Authorization style.
+        encode_password and encode_extra will also Base64-encode the password and extra respectively.
+        """
+        value = f"{self.username}:"
+        if encode_password:
+            value += base64.b64encode(self.password.encode()).decode()
+        else:
+            value += self.password
+        if with_extra and self.extra:
+            if encode_extra:
+                value += f":{base64.b64encode(self.extra.encode()).decode()}"
+            else:
+                value += f":{self.extra}"
+        return base64.b64encode(value.encode()).decode()
+
+    @classmethod
+    def loads(cls, text: str) -> Credential:
+        """
+        Load credential from a text string.
+
+        Format: {username}:{password}
+        Rules:
+            Only one Credential must be in this text contents.
+            All whitespace before and after all text will be removed.
+            Any whitespace between text will be kept and used.
+            The credential can be spanned across one or multiple lines as long as it
+                abides with all the above rules and the format.
+
+        Example that follows the format and rules:
+            `\tJohnd\noe@gm\n\rail.com\n:Pass1\n23\n\r  \t  \t`
+            >>>Credential(username='Johndoe@gmail.com', password='Pass123')
+        """
+        text = "".join([
+            x.strip() for x in text.splitlines(keepends=False)
+        ]).strip()
+        credential = re.fullmatch(r"^([^:]+?):([^:]+?)(?::(.+))?$", text)
+        if credential:
+            return cls(*credential.groups())
+        raise ValueError("No credentials found in text string. Expecting the format `username:password`")
+
+    @classmethod
+    def load(cls, path: Path) -> Credential:
+        """
+        Load Credential from a file path.
+        Use Credential.loads() for loading from text content and seeing the rules and
+        format expected to be found in the URIs contents.
+        """
+        return cls.loads(path.read_text("utf8"))

devine/core/downloaders/__init__.py

@@ -0,0 +1,2 @@
+from .aria2c import aria2c
+from .saldl import saldl

devine/core/downloaders/aria2c.py

@@ -0,0 +1,88 @@
+import asyncio
+import subprocess
+from pathlib import Path
+from typing import Union, Optional
+
+from devine.core.config import config
+from devine.core.utilities import get_binary_path, start_pproxy
+
+
+async def aria2c(
+    uri: Union[str, list[str]],
+    out: Path,
+    headers: Optional[dict] = None,
+    proxy: Optional[str] = None
+) -> int:
+    """
+    Download files using Aria2(c).
+    https://aria2.github.io
+
+    If multiple URLs are provided they will be downloaded in the provided order
+    to the output directory. They will not be merged together.
+    """
+    segmented = False
+    if isinstance(uri, list) and len(uri) == 1:
+        uri = uri[0]
+    if isinstance(uri, list):
+        segmented = True
+        uri = "\n".join([
+            f"{url}\n"
+            f"\tdir={out}\n"
+            f"\tout={i:08}.mp4"
+            for i, url in enumerate(uri)
+        ])
+        if out.is_file():
+            raise ValueError("Provided multiple segments to download, expecting directory path")
+    elif "\t" not in uri:
+        uri = f"{uri}\n" \
+              f"\tdir={out.parent}\n" \
+              f"\tout={out.name}"
+
+    executable = get_binary_path("aria2c", "aria2")
+    if not executable:
+        raise EnvironmentError("Aria2c executable not found...")
+
+    arguments = [
+        "-c",  # Continue downloading a partially downloaded file
+        "--remote-time",  # Retrieve timestamp of the remote file from the and apply if available
+        "-x", "16",  # The maximum number of connections to one server for each download
+        "-j", "16",  # The maximum number of parallel downloads for every static (HTTP/FTP) URL
+        "-s", ("1" if segmented else "16"),  # Download a file using N connections
+        "--min-split-size", ("1024M" if segmented else "20M"),  # effectively disable split if segmented
+        "--allow-overwrite=true",
+        "--auto-file-renaming=false",
+        "--retry-wait", "2",  # Set the seconds to wait between retries.
+        "--max-tries", "5",
+        "--max-file-not-found", "5",
+        "--summary-interval", "0",
+        "--file-allocation", config.aria2c.get("file_allocation", "falloc"),
+        "--console-log-level", "warn",
+        "--download-result", "hide",
+        "-i", "-"
+    ]
+
+    for header, value in (headers or {}).items():
+        if header.lower() == "accept-encoding":
+            # we cannot set an allowed encoding, or it will return compressed
+            # and the code is not set up to uncompress the data
+            continue
+        arguments.extend(["--header", f"{header}: {value}"])
+
+    if proxy and proxy.lower().split(":")[0] != "http":
+        # HTTPS proxies not supported by Aria2c.
+        # Proxy the proxy via pproxy to access it as a HTTP proxy.
+        async with start_pproxy(proxy) as pproxy_:
+            return await aria2c(uri, out, headers, pproxy_)
+
+    if proxy:
+        arguments += ["--all-proxy", proxy]
+
+    p = await asyncio.create_subprocess_exec(executable, *arguments, stdin=subprocess.PIPE)
+    await p.communicate(uri.encode())
+    if p.returncode != 0:
+        raise subprocess.CalledProcessError(p.returncode, arguments)
+
+    return p.returncode
+
+
+__ALL__ = (aria2c,)

devine/core/downloaders/saldl.py

@@ -0,0 +1,51 @@
+import subprocess
+from pathlib import Path
+from typing import Union, Optional
+
+from devine.core.utilities import get_binary_path
+
+
+async def saldl(
+    uri: Union[str, list[str]],
+    out: Union[Path, str],
+    headers: Optional[dict] = None,
+    proxy: Optional[str] = None
+) -> int:
+    out = Path(out)
+
+    if headers:
+        headers.update({k: v for k, v in headers.items() if k.lower() != "accept-encoding"})
+
+    executable = get_binary_path("saldl", "saldl-win64", "saldl-win32")
+    if not executable:
+        raise EnvironmentError("Saldl executable not found...")
+
+    arguments = [
+        executable,
+        # "--no-status",
+        "--skip-TLS-verification",
+        "--resume",
+        "--merge-in-order",
+        "-c8",
+        "--auto-size", "1",
+        "-D", str(out.parent),
+        "-o", out.name
+    ]
+
+    if headers:
+        arguments.extend([
+            "--custom-headers",
+            "\r\n".join([f"{k}: {v}" for k, v in headers.items()])
+        ])
+
+    if proxy:
+        arguments.extend(["--proxy", proxy])
+
+    if isinstance(uri, list):
+        raise ValueError("Saldl code does not yet support multiple uri (e.g. segmented) downloads.")
+    arguments.append(uri)
+
+    return subprocess.check_call(arguments)
+
+
+__ALL__ = (saldl,)

devine/core/drm/__init__.py

@@ -0,0 +1,6 @@
+from typing import Union
+
+from devine.core.drm.clearkey import ClearKey
+from devine.core.drm.widevine import Widevine
+
+DRM_T = Union[ClearKey, Widevine]

devine/core/drm/clearkey.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+from typing import Optional, Union
+from urllib.parse import urljoin
+
+import requests
+from Cryptodome.Cipher import AES
+from m3u8.model import Key
+
+from devine.core.constants import TrackT
+
+
+class ClearKey:
+    """AES Clear Key DRM System."""
+    def __init__(self, key: Union[bytes, str], iv: Optional[Union[bytes, str]] = None):
+        """
+        Generally IV should be provided where possible. If not provided, it will be
+        set to \x00 of the same bit-size of the key.
+        """
+        if isinstance(key, str):
+            key = bytes.fromhex(key.replace("0x", ""))
+        if not isinstance(key, bytes):
+            raise ValueError(f"Expected AES Key to be bytes, not {key!r}")
+        if not iv:
+            iv = b"\x00"
+        if isinstance(iv, str):
+            iv = bytes.fromhex(iv.replace("0x", ""))
+        if not isinstance(iv, bytes):
+            raise ValueError(f"Expected IV to be bytes, not {iv!r}")
+
+        if len(iv) < len(key):
+            iv = iv * (len(key) - len(iv) + 1)
+
+        self.key: bytes = key
+        self.iv: bytes = iv
+
+    def decrypt(self, track: TrackT) -> None:
+        """Decrypt a Track with AES Clear Key DRM."""
+        if not track.path or not track.path.exists():
+            raise ValueError("Tried to decrypt a track that has not yet been downloaded.")
+
+        decrypted = AES. \
+            new(self.key, AES.MODE_CBC, self.iv). \
+            decrypt(track.path.read_bytes())
+
+        decrypted_path = track.path.with_suffix(f".decrypted{track.path.suffix}")
+        decrypted_path.write_bytes(decrypted)
+
+        track.swap(decrypted_path)
+        track.drm = None
+
+    @classmethod
+    def from_m3u_key(cls, m3u_key: Key, proxy: Optional[str] = None) -> ClearKey:
+        if not isinstance(m3u_key, Key):
+            raise ValueError(f"Provided M3U Key is in an unexpected type {m3u_key!r}")
+        if not m3u_key.method.startswith("AES"):
+            raise ValueError(f"Provided M3U Key is not an AES Clear Key, {m3u_key.method}")
+        if not m3u_key.uri:
+            raise ValueError("No URI in M3U Key, unable to get Key.")
+
+        res = requests.get(
+            url=urljoin(m3u_key.base_uri, m3u_key.uri),
+            headers={
+                "User-Agent": "smartexoplayer/1.1.0 (Linux;Android 8.0.0) ExoPlayerLib/2.13.3"
+            },
+            proxies={"all": proxy} if proxy else None
+        )
+        res.raise_for_status()
+        if not res.content:
+            raise EOFError("Unexpected Empty Response by M3U Key URI.")
+        if len(res.content) < 16:
+            raise EOFError(f"Unexpected Length of Key ({len(res.content)} bytes) in M3U Key.")
+
+        key = res.content
+        iv = None
+        if m3u_key.iv:
+            iv = bytes.fromhex(m3u_key.iv.replace("0x", ""))
+
+        return cls(key=key, iv=iv)
+
+
+__ALL__ = (ClearKey,)

devine/core/drm/widevine.py

@@ -0,0 +1,222 @@
+from __future__ import annotations
+
+import base64
+import subprocess
+import sys
+from typing import Any, Optional, Union, Callable
+from uuid import UUID
+
+import m3u8
+from construct import Container
+from pymp4.parser import Box
+from pywidevine.cdm import Cdm as WidevineCdm
+from pywidevine.pssh import PSSH
+from requests import Session
+
+from devine.core.config import config
+from devine.core.constants import AnyTrack, TrackT
+from devine.core.utilities import get_binary_path, get_boxes
+from devine.core.utils.subprocess import ffprobe
+
+
+class Widevine:
+    """Widevine DRM System."""
+    def __init__(self, pssh: PSSH, kid: Union[UUID, str, bytes, None] = None, **kwargs: Any):
+        if not pssh:
+            raise ValueError("Provided PSSH is empty.")
+        if not isinstance(pssh, PSSH):
+            raise TypeError(f"Expected pssh to be a {PSSH}, not {pssh!r}")
+
+        if pssh.system_id == PSSH.SystemId.PlayReady:
+            pssh.to_widevine()
+
+        if kid:
+            if isinstance(kid, str):
+                kid = UUID(hex=kid)
+            elif isinstance(kid, bytes):
+                kid = UUID(bytes=kid)
+            if not isinstance(kid, UUID):
+                raise ValueError(f"Expected kid to be a {UUID}, str, or bytes, not {kid!r}")
+            pssh.set_key_ids([kid])
+
+        self._pssh = pssh
+
+        if not self.kids:
+            raise Widevine.Exceptions.KIDNotFound("No Key ID was found within PSSH and none were provided.")
+
+        self.content_keys: dict[UUID, str] = {}
+        self.data: dict = kwargs or {}
+
+    @classmethod
+    def from_track(cls, track: AnyTrack, session: Optional[Session] = None) -> Widevine:
+        """
+        Get PSSH and KID from within the Initiation Segment of the Track Data.
+        It also tries to get PSSH and KID from other track data like M3U8 data
+        as well as through ffprobe.
+
+        Create a Widevine DRM System object from a track's information.
+        This should only be used if a PSSH could not be provided directly.
+        It is *rare* to need to use this.
+
+        You may provide your own requests session to be able to use custom
+        headers and more.
+
+        Raises:
+            PSSHNotFound - If the PSSH was not found within the data.
+            KIDNotFound - If the KID was not found within the data or PSSH.
+        """
+        if not session:
+            session = Session()
+            session.headers.update(config.headers)
+
+        kid: Optional[UUID] = None
+        pssh_boxes: list[Container] = []
+        tenc_boxes: list[Container] = []
+
+        if track.descriptor == track.Descriptor.M3U:
+            m3u_url = track.url
+            if isinstance(m3u_url, list):
+                # TODO: Find out why exactly the track url could be a list in this
+                #       scenario, as if its a list of segments, they would be files
+                #       not m3u documents
+                m3u_url = m3u_url[0]
+            master = m3u8.loads(session.get(m3u_url).text, uri=m3u_url)
+            pssh_boxes.extend(
+                Box.parse(base64.b64decode(x.uri.split(",")[-1]))
+                for x in (master.session_keys or master.keys)
+                if x and x.keyformat and x.keyformat.lower() == WidevineCdm.urn
+            )
+
+        init_data = track.get_init_segment(session)
+        if init_data:
+            # try get via ffprobe, needed for non mp4 data e.g. WEBM from Google Play
+            probe = ffprobe(init_data)
+            if probe:
+                for stream in probe.get("streams") or []:
+                    enc_key_id = stream.get("tags", {}).get("enc_key_id")
+                    if enc_key_id:
+                        kid = UUID(bytes=base64.b64decode(enc_key_id))
+            pssh_boxes.extend(list(get_boxes(init_data, b"pssh")))
+            tenc_boxes.extend(list(get_boxes(init_data, b"tenc")))
+
+        pssh_boxes.sort(key=lambda b: {
+            PSSH.SystemId.Widevine: 0,
+            PSSH.SystemId.PlayReady: 1
+        }[b.system_ID])
+
+        pssh = next(iter(pssh_boxes), None)
+        if not pssh:
+            raise Widevine.Exceptions.PSSHNotFound("PSSH was not found in track data.")
+
+        tenc = next(iter(tenc_boxes), None)
+        if not kid and tenc and tenc.key_ID.int != 0:
+            kid = tenc.key_ID
+
+        return cls(pssh=PSSH(pssh), kid=kid)
+
+    @property
+    def pssh(self) -> PSSH:
+        """Get Protection System Specific Header Box."""
+        return self._pssh
+
+    @property
+    def kid(self) -> Optional[UUID]:
+        """Get first Key ID, if any."""
+        return next(iter(self.kids), None)
+
+    @property
+    def kids(self) -> list[UUID]:
+        """Get all Key IDs."""
+        return self._pssh.key_ids
+
+    def get_content_keys(self, cdm: WidevineCdm, certificate: Callable, licence: Callable) -> None:
+        """
+        Create a CDM Session and obtain Content Keys for this DRM Instance.
+        The certificate and license params are expected to be a function and will
+        be provided with the challenge and session ID.
+        """
+        for kid in self.kids:
+            if kid in self.content_keys:
+                continue
+
+            session_id = cdm.open()
+
+            try:
+                cdm.set_service_certificate(
+                    session_id,
+                    certificate(
+                        challenge=cdm.service_certificate_challenge
+                    )
+                )
+
+                cdm.parse_license(
+                    session_id,
+                    licence(
+                        challenge=cdm.get_license_challenge(session_id, self.pssh)
+                    )
+                )
+
+                self.content_keys = {
+                    key.kid: key.key.hex()
+                    for key in cdm.get_keys(session_id, "CONTENT")
+                }
+                if not self.content_keys:
+                    raise ValueError("No Content Keys were returned by the License")
+
+                if kid not in self.content_keys:
+                    raise ValueError(f"No Content Key with the KID ({kid.hex}) was returned")
+            finally:
+                cdm.close(session_id)
+
+    def decrypt(self, track: TrackT) -> None:
+        """
+        Decrypt a Track with Widevine DRM.
+        Raises:
+            EnvironmentError if the Shaka Packager executable could not be found.
+            ValueError if the track has not yet been downloaded.
+            SubprocessError if Shaka Packager returned a non-zero exit code.
+        """
+        if not self.content_keys:
+            raise ValueError("Cannot decrypt a Track without any Content Keys...")
+
+        platform = {"win32": "win", "darwin": "osx"}.get(sys.platform, sys.platform)
+        executable = get_binary_path("shaka-packager", f"packager-{platform}", f"packager-{platform}-x64")
+        if not executable:
+            raise EnvironmentError("Shaka Packager executable not found but is required.")
+        if not track.path or not track.path.exists():
+            raise ValueError("Tried to decrypt a track that has not yet been downloaded.")
+
+        decrypted_path = track.path.with_suffix(f".decrypted{track.path.suffix}")
+        config.directories.temp.mkdir(parents=True, exist_ok=True)
+        try:
+            subprocess.check_call([
+                executable,
+                f"input={track.path},stream=0,output={decrypted_path}",
+                "--enable_raw_key_decryption", "--keys",
+                ",".join([
+                    *[
+                        "label={}:key_id={}:key={}".format(i, kid.hex, key.lower())
+                        for i, (kid, key) in enumerate(self.content_keys.items())
+                    ],
+                    *[
+                        # Apple TV+ needs this as their files do not use the KID supplied in it's manifest
+                        "label={}:key_id={}:key={}".format(i, "00" * 16, key.lower())
+                        for i, (kid, key) in enumerate(self.content_keys.items(), len(self.content_keys))
+                    ]
+                ]),
+                "--temp_dir", config.directories.temp
+            ])
+        except subprocess.CalledProcessError as e:
+            raise subprocess.SubprocessError(f"Failed to Decrypt! Shaka Packager Error: {e}")
+        track.swap(decrypted_path)
+        track.drm = None
+
+    class Exceptions:
+        class PSSHNotFound(Exception):
+            """PSSH (Protection System Specific Header) was not found."""
+
+        class KIDNotFound(Exception):
+            """KID (Encryption Key ID) was not found."""
+
+
+__ALL__ = (Widevine,)

devine/core/manifests/__init__.py

@@ -0,0 +1,2 @@
+from .dash import DASH
+from .hls import HLS

devine/core/manifests/dash.py

@@ -0,0 +1,432 @@
+from __future__ import annotations
+
+import base64
+from hashlib import md5
+
+import math
+import re
+from copy import copy
+from typing import Any, Optional, Union, Callable
+from urllib.parse import urljoin, urlparse
+from uuid import UUID
+
+import requests
+from langcodes import Language, tag_is_valid
+from pywidevine.cdm import Cdm as WidevineCdm
+from pywidevine.pssh import PSSH
+from requests import Session
+
+from devine.core.drm import Widevine
+from devine.core.tracks import Tracks, Video, Audio, Subtitle
+from devine.core.utilities import is_close_match, FPS
+from devine.core.utils.xml import load_xml
+
+
+class DASH:
+    def __init__(self, manifest, url: str):
+        if manifest is None:
+            raise ValueError("DASH manifest must be provided.")
+        if manifest.tag != "MPD":
+            raise TypeError(f"Expected 'MPD' document, but received a '{manifest.tag}' document instead.")
+
+        if not url:
+            raise requests.URLRequired("DASH manifest URL must be provided for relative path computations.")
+        if not isinstance(url, str):
+            raise TypeError(f"Expected url to be a {str}, not {url!r}")
+
+        self.manifest = manifest
+        self.url = url
+
+    @classmethod
+    def from_url(cls, url: str, session: Optional[Session] = None, **args: Any) -> DASH:
+        if not url:
+            raise requests.URLRequired("DASH manifest URL must be provided for relative path computations.")
+        if not isinstance(url, str):
+            raise TypeError(f"Expected url to be a {str}, not {url!r}")
+
+        if not session:
+            session = Session()
+        elif not isinstance(session, Session):
+            raise TypeError(f"Expected session to be a {Session}, not {session!r}")
+
+        res = session.get(url, **args)
+        if not res.ok:
+            raise requests.ConnectionError(
+                "Failed to request the MPD document.",
+                response=res
+            )
+
+        return DASH.from_text(res.text, url)
+
+    @classmethod
+    def from_text(cls, text: str, url: str) -> DASH:
+        if not text:
+            raise ValueError("DASH manifest Text must be provided.")
+        if not isinstance(text, str):
+            raise TypeError(f"Expected text to be a {str}, not {text!r}")
+
+        if not url:
+            raise requests.URLRequired("DASH manifest URL must be provided for relative path computations.")
+        if not isinstance(url, str):
+            raise TypeError(f"Expected url to be a {str}, not {url!r}")
+
+        manifest = load_xml(text)
+
+        return cls(manifest, url)
+
+    def to_tracks(self, language: Union[str, Language], period_filter: Optional[Callable] = None) -> Tracks:
+        """
+        Convert an MPEG-DASH MPD (Media Presentation Description) document to Video, Audio and Subtitle Track objects.
+
+        Parameters:
+            language: Language you expect the Primary Track to be in.
+            period_filter: Filter out period's within the manifest.
+
+        All Track URLs will be a list of segment URLs.
+        """
+        tracks = Tracks()
+
+        for period in self.manifest.findall("Period"):
+            if callable(period_filter) and period_filter(period):
+                continue
+
+            period_base_url = period.findtext("BaseURL") or self.manifest.findtext("BaseURL")
+            if not period_base_url or not re.match("^https?://", period_base_url, re.IGNORECASE):
+                period_base_url = urljoin(self.url, period_base_url)
+
+            for adaptation_set in period.findall("AdaptationSet"):
+                # flags
+                trick_mode = any(
+                    x.get("schemeIdUri") == "http://dashif.org/guidelines/trickmode"
+                    for x in (
+                            adaptation_set.findall("EssentialProperty") +
+                            adaptation_set.findall("SupplementalProperty")
+                    )
+                )
+                descriptive = any(
+                    (x.get("schemeIdUri"), x.get("value")) == ("urn:mpeg:dash:role:2011", "descriptive")
+                    for x in adaptation_set.findall("Accessibility")
+                ) or any(
+                    (x.get("schemeIdUri"), x.get("value")) == ("urn:tva:metadata:cs:AudioPurposeCS:2007", "1")
+                    for x in adaptation_set.findall("Accessibility")
+                )
+                forced = any(
+                    (x.get("schemeIdUri"), x.get("value")) == ("urn:mpeg:dash:role:2011", "forced-subtitle")
+                    for x in adaptation_set.findall("Role")
+                )
+                cc = any(
+                    (x.get("schemeIdUri"), x.get("value")) == ("urn:mpeg:dash:role:2011", "caption")
+                    for x in adaptation_set.findall("Role")
+                )
+
+                if trick_mode:
+                    # we don't want trick mode streams (they are only used for fast-forward/rewind)
+                    continue
+
+                for rep in adaptation_set.findall("Representation"):
+                    supplements = rep.findall("SupplementalProperty") + adaptation_set.findall("SupplementalProperty")
+
+                    content_type = adaptation_set.get("contentType") or \
+                        adaptation_set.get("mimeType") or \
+                        rep.get("contentType") or \
+                        rep.get("mimeType")
+                    if not content_type:
+                        raise ValueError("No content type value could be found")
+                    content_type = content_type.split("/")[0]
+
+                    codecs = rep.get("codecs") or adaptation_set.get("codecs")
+
+                    if content_type.startswith("image"):
+                        # we don't want what's likely thumbnails for the seekbar
+                        continue
+                    if content_type == "application":
+                        # possibly application/mp4 which could be mp4-boxed subtitles
+                        try:
+                            Subtitle.Codec.from_mime(codecs)
+                            content_type = "text"
+                        except ValueError:
+                            raise ValueError(f"Unsupported content type '{content_type}' with codecs of '{codecs}'")
+
+                    if content_type == "text":
+                        mime = adaptation_set.get("mimeType")
+                        if mime and not mime.endswith("/mp4"):
+                            codecs = mime.split("/")[1]
+
+                    joc = next((
+                        x.get("value")
+                        for x in supplements
+                        if x.get("schemeIdUri") == "tag:dolby.com,2018:dash:EC3_ExtensionComplexityIndex:2018"
+                    ), None)
+
+                    track_lang = DASH.get_language(rep.get("lang"), adaptation_set.get("lang"), language)
+                    if not track_lang:
+                        raise ValueError(
+                            "One or more Tracks had no Language information. "
+                            "The provided fallback language is not valid or is `None` or `und`."
+                        )
+
+                    drm = DASH.get_drm(rep.findall("ContentProtection") + adaptation_set.findall("ContentProtection"))
+
+                    # from here we need to calculate the Segment Template and compute a final list of URLs
+
+                    segment_urls = DASH.get_segment_urls(
+                        representation=rep,
+                        period_duration=period.get("duration") or self.manifest.get("mediaPresentationDuration"),
+                        fallback_segment_template=adaptation_set.find("SegmentTemplate"),
+                        fallback_base_url=period_base_url,
+                        fallback_query=urlparse(self.url).query
+                    )
+
+                    # for some reason it's incredibly common for services to not provide
+                    # a good and actually unique track ID, sometimes because of the lang
+                    # dialect not being represented in the id, or the bitrate, or such.
+                    # this combines all of them as one and hashes it to keep it small(ish).
+                    track_id = md5("{codec}-{lang}-{bitrate}-{base_url}-{extra}".format(
+                        codec=codecs,
+                        lang=track_lang,
+                        bitrate=rep.get("bandwidth") or 0,  # subs may not state bandwidth
+                        base_url=(rep.findtext("BaseURL") or "").split("?")[0],
+                        extra=(adaptation_set.get("audioTrackId") or "") + (rep.get("id") or "") +
+                              (period.get("id") or "")
+                    ).encode()).hexdigest()
+
+                    if content_type == "video":
+                        track_type = Video
+                        track_codec = Video.Codec.from_codecs(codecs)
+                    elif content_type == "audio":
+                        track_type = Audio
+                        track_codec = Audio.Codec.from_codecs(codecs)
+                    elif content_type == "text":
+                        track_type = Subtitle
+                        track_codec = Subtitle.Codec.from_codecs(codecs or "vtt")
+                    else:
+                        raise ValueError(f"Unknown Track Type '{content_type}'")
+
+                    tracks.add(track_type(
+                        id_=track_id,
+                        url=segment_urls,
+                        codec=track_codec,
+                        language=track_lang,
+                        is_original_lang=not track_lang or not language or is_close_match(track_lang, [language]),
+                        descriptor=Video.Descriptor.MPD,
+                        extra=(rep, adaptation_set),
+                        # video track args
+                        **(dict(
+                            range_=(
+                                Video.Range.DV
+                                if codecs.startswith(("dva1", "dvav", "dvhe", "dvh1")) else
+                                Video.Range.from_cicp(
+                                    primaries=next((
+                                        int(x.get("value"))
+                                        for x in (
+                                            adaptation_set.findall("SupplementalProperty")
+                                            + adaptation_set.findall("EssentialProperty")
+                                        )
+                                        if x.get("schemeIdUri") == "urn:mpeg:mpegB:cicp:ColourPrimaries"
+                                    ), 0),
+                                    transfer=next((
+                                        int(x.get("value"))
+                                        for x in (
+                                            adaptation_set.findall("SupplementalProperty")
+                                            + adaptation_set.findall("EssentialProperty")
+                                        )
+                                        if x.get("schemeIdUri") == "urn:mpeg:mpegB:cicp:TransferCharacteristics"
+                                    ), 0),
+                                    matrix=next((
+                                        int(x.get("value"))
+                                        for x in (
+                                            adaptation_set.findall("SupplementalProperty")
+                                            + adaptation_set.findall("EssentialProperty")
+                                        )
+                                        if x.get("schemeIdUri") == "urn:mpeg:mpegB:cicp:MatrixCoefficients"
+                                    ), 0)
+                                )
+                            ),
+                            bitrate=rep.get("bandwidth"),
+                            width=int(rep.get("width") or 0) or adaptation_set.get("width"),
+                            height=int(rep.get("height") or 0) or adaptation_set.get("height"),
+                            fps=(
+                                rep.get("frameRate") or
+                                adaptation_set.get("frameRate") or
+                                FPS.parse(rep.find("SegmentBase").get("timescale"))
+                            ),
+                            drm=drm
+                        ) if track_type is Video else dict(
+                            bitrate=rep.get("bandwidth"),
+                            channels=next(iter(
+                                rep.xpath("AudioChannelConfiguration/@value")
+                                or adaptation_set.xpath("AudioChannelConfiguration/@value")
+                            ), None),
+                            joc=joc,
+                            descriptive=descriptive,
+                            drm=drm
+                        ) if track_type is Audio else dict(
+                            forced=forced,
+                            cc=cc
+                        ) if track_type is Subtitle else {})
+                    ))
+
+            # only get tracks from the first main-content period
+            break
+
+        return tracks
+
+    @staticmethod
+    def get_language(*options: Any) -> Optional[Language]:
+        for option in options:
+            option = (str(option) or "").strip()
+            if not tag_is_valid(option) or option.startswith("und"):
+                continue
+            return Language.get(option)
+
+    @staticmethod
+    def get_drm(protections) -> Optional[list[Widevine]]:
+        drm = []
+        for protection in protections:
+            # TODO: Add checks for PlayReady, FairPlay, maybe more
+            urn = (protection.get("schemeIdUri") or "").lower()
+            if urn != WidevineCdm.urn:
+                continue
+
+            pssh = protection.findtext("pssh")
+            if not pssh:
+                continue
+            pssh = PSSH(pssh)
+
+            kid = protection.get("kid")
+            if kid:
+                kid = UUID(bytes=base64.b64decode(kid))
+
+            default_kid = protection.get("default_KID")
+            if default_kid:
+                kid = UUID(default_kid)
+
+            if not pssh.key_ids and not kid:
+                # weird manifest, look across all protections for a default_KID
+                kid = next((
+                    UUID(protection.get("default_KID"))
+                    for protection in protections
+                    if protection.get("default_KID")
+                ), None)
+
+            drm.append(Widevine(
+                pssh=pssh,
+                kid=kid
+            ))
+
+        if not drm:
+            drm = None
+
+        return drm
+
+    @staticmethod
+    def pt_to_sec(d: Union[str, float]) -> float:
+        if isinstance(d, float):
+            return d
+        has_ymd = d[0:8] == "P0Y0M0DT"
+        if d[0:2] != "PT" and not has_ymd:
+            raise ValueError("Input data is not a valid time string.")
+        if has_ymd:
+            d = d[6:].upper()  # skip `P0Y0M0DT`
+        else:
+            d = d[2:].upper()  # skip `PT`
+        m = re.findall(r"([\d.]+.)", d)
+        return sum(
+            float(x[0:-1]) * {"H": 60 * 60, "M": 60, "S": 1}[x[-1].upper()]
+            for x in m
+        )
+
+    @staticmethod
+    def replace_fields(url: str, **kwargs: Any) -> str:
+        for field, value in kwargs.items():
+            url = url.replace(f"${field}$", str(value))
+            m = re.search(fr"\${re.escape(field)}%([a-z0-9]+)\$", url, flags=re.I)
+            if m:
+                url = url.replace(m.group(), f"{value:{m.group(1)}}")
+        return url
+
+    @staticmethod
+    def get_segment_urls(
+        representation,
+        period_duration: str,
+        fallback_segment_template,
+        fallback_base_url: Optional[str] = None,
+        fallback_query: Optional[str] = None
+    ) -> list[str]:
+        segment_urls: list[str] = []
+        segment_template = representation.find("SegmentTemplate") or fallback_segment_template
+        base_url = representation.findtext("BaseURL") or fallback_base_url
+
+        if segment_template is None:
+            # We could implement SegmentBase, but it's basically a list of Byte Range's to download
+            # So just return the Base URL as a segment, why give the downloader extra effort
+            return [urljoin(fallback_base_url, base_url)]
+
+        segment_template = copy(segment_template)
+        start_number = int(segment_template.get("startNumber") or 1)
+        segment_timeline = segment_template.find("SegmentTimeline")
+
+        for item in ("initialization", "media"):
+            value = segment_template.get(item)
+            if not value:
+                continue
+            if not re.match("^https?://", value, re.IGNORECASE):
+                if not base_url:
+                    raise ValueError("Resolved Segment URL is not absolute, and no Base URL is available.")
+                value = urljoin(base_url, value)
+            if not urlparse(value).query and fallback_query:
+                value += f"?{fallback_query}"
+            segment_template.set(item, value)
+
+        initialization = segment_template.get("initialization")
+        if initialization:
+            segment_urls.append(DASH.replace_fields(
+                initialization,
+                Bandwidth=representation.get("bandwidth"),
+                RepresentationID=representation.get("id")
+            ))
+
+        if segment_timeline is not None:
+            seg_time_list = []
+            current_time = 0
+            for s in segment_timeline.findall("S"):
+                if s.get("t"):
+                    current_time = int(s.get("t"))
+                for _ in range(1 + (int(s.get("r") or 0))):
+                    seg_time_list.append(current_time)
+                    current_time += int(s.get("d"))
+            seg_num_list = list(range(start_number, len(seg_time_list) + start_number))
+            segment_urls += [
+                DASH.replace_fields(
+                    segment_template.get("media"),
+                    Bandwidth=representation.get("bandwidth"),
+                    Number=n,
+                    RepresentationID=representation.get("id"),
+                    Time=t
+                )
+                for t, n in zip(seg_time_list, seg_num_list)
+            ]
+        else:
+            if not period_duration:
+                raise ValueError("Duration of the Period was unable to be determined.")
+            period_duration = DASH.pt_to_sec(period_duration)
+
+            segment_duration = (
+                float(segment_template.get("duration")) / float(segment_template.get("timescale") or 1)
+            )
+            total_segments = math.ceil(period_duration / segment_duration)
+            segment_urls += [
+                DASH.replace_fields(
+                    segment_template.get("media"),
+                    Bandwidth=representation.get("bandwidth"),
+                    Number=s,
+                    RepresentationID=representation.get("id"),
+                    Time=s
+                )
+                for s in range(start_number, start_number + total_segments)
+            ]
+
+        return segment_urls
+
+
+__ALL__ = (DASH,)

devine/core/manifests/hls.py

@@ -0,0 +1,217 @@
+from __future__ import annotations
+
+import re
+from hashlib import md5
+from typing import Union, Any, Optional
+
+import m3u8
+import requests
+from langcodes import Language
+from m3u8 import M3U8
+from pywidevine.cdm import Cdm as WidevineCdm
+from pywidevine.pssh import PSSH
+from requests import Session
+
+from devine.core.drm import ClearKey, Widevine, DRM_T
+from devine.core.tracks import Tracks, Video, Audio, Subtitle
+from devine.core.utilities import is_close_match
+
+
+class HLS:
+    def __init__(self, manifest: M3U8, session: Optional[Session] = None):
+        if not manifest:
+            raise ValueError("HLS manifest must be provided.")
+        if not isinstance(manifest, M3U8):
+            raise TypeError(f"Expected manifest to be a {M3U8}, not {manifest!r}")
+        if not manifest.is_variant:
+            raise ValueError("Expected the M3U(8) manifest to be a Variant Playlist.")
+
+        self.manifest = manifest
+        self.session = session or Session()
+
+    @classmethod
+    def from_url(cls, url: str, session: Optional[Session] = None, **args: Any) -> HLS:
+        if not url:
+            raise requests.URLRequired("HLS manifest URL must be provided.")
+        if not isinstance(url, str):
+            raise TypeError(f"Expected url to be a {str}, not {url!r}")
+
+        if not session:
+            session = Session()
+        elif not isinstance(session, Session):
+            raise TypeError(f"Expected session to be a {Session}, not {session!r}")
+
+        res = session.get(url, **args)
+        if not res.ok:
+            raise requests.ConnectionError(
+                "Failed to request the M3U(8) document.",
+                response=res
+            )
+
+        master = m3u8.loads(res.text, uri=url)
+
+        return cls(master, session)
+
+    @classmethod
+    def from_text(cls, text: str, url: str) -> HLS:
+        if not text:
+            raise ValueError("HLS manifest Text must be provided.")
+        if not isinstance(text, str):
+            raise TypeError(f"Expected text to be a {str}, not {text!r}")
+
+        if not url:
+            raise requests.URLRequired("HLS manifest URL must be provided for relative path computations.")
+        if not isinstance(url, str):
+            raise TypeError(f"Expected url to be a {str}, not {url!r}")
+
+        master = m3u8.loads(text, uri=url)
+
+        return cls(master)
+
+    def to_tracks(self, language: Union[str, Language], **args: Any) -> Tracks:
+        """
+        Convert a Variant Playlist M3U(8) document to Video, Audio and Subtitle Track objects.
+
+        Parameters:
+            language: Language you expect the Primary Track to be in.
+            args: You may pass any arbitrary named header to be passed to all requests made within
+                this method.
+
+        All Track objects' URL will be to another M3U(8) document. However, these documents
+        will be Invariant Playlists and contain the list of segments URIs among other metadata.
+        """
+        session_drm = HLS.get_drm(self.manifest.session_keys)
+
+        audio_codecs_by_group_id: dict[str, Audio.Codec] = {}
+        tracks = Tracks()
+
+        for playlist in self.manifest.playlists:
+            url = playlist.uri
+            if not re.match("^https?://", url):
+                url = playlist.base_uri + url
+
+            audio_group = playlist.stream_info.audio
+            if audio_group:
+                audio_codec = Audio.Codec.from_codecs(playlist.stream_info.codecs)
+                audio_codecs_by_group_id[audio_group] = audio_codec
+
+            if session_drm:
+                drm = session_drm
+            else:
+                # keys may be in the invariant playlist instead, annoying...
+                res = self.session.get(url, **args)
+                if not res.ok:
+                    raise requests.ConnectionError(
+                        "Failed to request an invariant M3U(8) document.",
+                        response=res
+                    )
+
+                invariant_playlist = m3u8.loads(res.text, url)
+                drm = HLS.get_drm(invariant_playlist.keys)
+
+            try:
+                # TODO: Any better way to figure out the primary track type?
+                Video.Codec.from_codecs(playlist.stream_info.codecs)
+            except ValueError:
+                primary_track_type = Audio
+            else:
+                primary_track_type = Video
+
+            tracks.add(primary_track_type(
+                id_=md5(str(playlist).encode()).hexdigest()[0:7],  # 7 chars only for filename length
+                url=url,
+                codec=primary_track_type.Codec.from_codecs(playlist.stream_info.codecs),
+                language=language,  # HLS manifests do not seem to have language info
+                is_original_lang=True,  # TODO: All we can do is assume Yes
+                bitrate=playlist.stream_info.average_bandwidth or playlist.stream_info.bandwidth,
+                descriptor=Video.Descriptor.M3U,
+                drm=drm,
+                extra=playlist,
+                # video track args
+                **(dict(
+                    range_=Video.Range.DV if any(
+                        codec.split(".")[0] in ("dva1", "dvav", "dvhe", "dvh1")
+                        for codec in playlist.stream_info.codecs.lower().split(",")
+                    ) else Video.Range.from_m3u_range_tag(playlist.stream_info.video_range),
+                    width=playlist.stream_info.resolution[0],
+                    height=playlist.stream_info.resolution[1],
+                    fps=playlist.stream_info.frame_rate
+                ) if primary_track_type is Video else {})
+            ))
+
+        for media in self.manifest.media:
+            url = media.uri
+            if not url:
+                continue
+
+            if not re.match("^https?://", url):
+                url = media.base_uri + url
+
+            if media.type == "AUDIO":
+                if session_drm:
+                    drm = session_drm
+                else:
+                    # keys may be in the invariant playlist instead, annoying...
+                    res = self.session.get(url, **args)
+                    if not res.ok:
+                        raise requests.ConnectionError(
+                            "Failed to request an invariant M3U(8) document.",
+                            response=res
+                        )
+
+                    invariant_playlist = m3u8.loads(res.text, url)
+                    drm = HLS.get_drm(invariant_playlist.keys)
+            else:
+                drm = None
+
+            if media.type == "AUDIO":
+                track_type = Audio
+                codec = audio_codecs_by_group_id.get(media.group_id)
+            else:
+                track_type = Subtitle
+                codec = Subtitle.Codec.WebVTT  # assuming WebVTT, codec info isn't shown
+
+            tracks.add(track_type(
+                id_=md5(str(media).encode()).hexdigest()[0:6],  # 6 chars only for filename length
+                url=url,
+                codec=codec,
+                language=media.language or language,  # HLS media may not have language info, fallback if needed
+                is_original_lang=language and is_close_match(media.language, [language]),
+                descriptor=Audio.Descriptor.M3U,
+                drm=drm,
+                extra=media,
+                # audio track args
+                **(dict(
+                    bitrate=0,  # TODO: M3U doesn't seem to state bitrate?
+                    channels=media.channels,
+                    descriptive="public.accessibility.describes-video" in (media.characteristics or ""),
+                ) if track_type is Audio else dict(
+                    forced=media.forced == "YES",
+                    sdh="public.accessibility.describes-music-and-sound" in (media.characteristics or ""),
+                ) if track_type is Subtitle else {})
+            ))
+
+        return tracks
+
+    @staticmethod
+    def get_drm(keys: list[Union[m3u8.model.SessionKey, m3u8.model.Key]]) -> list[DRM_T]:
+        drm = []
+
+        for key in keys:
+            if not key:
+                continue
+            # TODO: Add checks for Merlin, FairPlay, PlayReady, maybe more.
+            if key.method.startswith("AES"):
+                drm.append(ClearKey.from_m3u_key(key))
+            elif key.method == "ISO-23001-7":
+                drm.append(Widevine(PSSH.new(key_ids=[key.uri.split(",")[-1]], system_id=PSSH.SystemId.Widevine)))
+            elif key.keyformat and key.keyformat.lower() == WidevineCdm.urn:
+                drm.append(Widevine(
+                    pssh=PSSH(key.uri.split(",")[-1]),
+                    **key._extra_params  # noqa
+                ))
+
+        return drm
+
+
+__ALL__ = (HLS,)

devine/core/proxies/__init__.py

@@ -0,0 +1,3 @@
+from .basic import Basic
+from .hola import Hola
+from .nordvpn import NordVPN

devine/core/proxies/basic.py

@@ -0,0 +1,30 @@
+import random
+from typing import Optional
+
+from devine.core.proxies.proxy import Proxy
+
+
+class Basic(Proxy):
+    def __init__(self, **countries):
+        """Basic Proxy Service using Proxies specified in the config."""
+        self.countries = countries
+
+    def __repr__(self) -> str:
+        countries = len(self.countries)
+        servers = len(self.countries.values())
+
+        return f"{countries} Countr{['ies', 'y'][countries == 1]} ({servers} Server{['s', ''][servers == 1]})"
+
+    def get_proxy(self, query: str) -> Optional[str]:
+        """Get a proxy URI from the config."""
+        servers = self.countries.get(query)
+        if not servers:
+            return
+
+        proxy = random.choice(servers)
+
+        if "://" not in proxy:
+            # TODO: Improve the test for a valid URI
+            raise ValueError(f"The proxy '{proxy}' is not a valid proxy URI supported by Python-Requests.")
+
+        return proxy

devine/core/proxies/hola.py

@@ -0,0 +1,69 @@
+import random
+import re
+import subprocess
+from typing import Optional
+
+from devine.core.proxies.proxy import Proxy
+from devine.core.utilities import get_binary_path
+
+
+class Hola(Proxy):
+    def __init__(self):
+        """
+        Proxy Service using Hola's direct connections via the hola-proxy project.
+        https://github.com/Snawoot/hola-proxy
+        """
+        self.binary = get_binary_path("hola-proxy")
+        if not self.binary:
+            raise EnvironmentError("hola-proxy executable not found but is required for the Hola proxy provider.")
+
+        self.countries = self.get_countries()
+
+    def __repr__(self) -> str:
+        countries = len(self.countries)
+
+        return f"{countries} Countr{['ies', 'y'][countries == 1]}"
+
+    def get_proxy(self, query: str) -> Optional[str]:
+        """
+        Get an HTTP proxy URI for a Datacenter ('direct') or Residential ('lum') Hola server.
+
+        TODO: - Add ability to select 'lum' proxies (residential proxies).
+              - Return and use Proxy Authorization
+        """
+        query = query.lower()
+
+        p = subprocess.check_output([
+            self.binary,
+            "-country", query,
+            "-list-proxies"
+        ], stderr=subprocess.STDOUT).decode()
+
+        if "Transaction error: temporary ban detected." in p:
+            raise ConnectionError("Hola banned your IP temporarily from it's services. Try change your IP.")
+
+        username, password, proxy_authorization = re.search(
+            r"Login: (.*)\nPassword: (.*)\nProxy-Authorization: (.*)", p
+        ).groups()
+
+        servers = re.findall(r"(zagent.*)", p)
+        proxies = []
+        for server in servers:
+            host, ip_address, direct, peer, hola, trial, trial_peer, vendor = server.split(",")
+            proxies.append(f"http://{username}:{password}@{ip_address}:{peer}")
+
+        proxy = random.choice(proxies)
+        return proxy
+
+    def get_countries(self) -> list[dict[str, str]]:
+        """Get a list of available Countries."""
+        p = subprocess.check_output([
+            self.binary,
+            "-list-countries"
+        ]).decode("utf8")
+
+        return [
+            {code: name}
+            for country in p.splitlines()
+            for (code, name) in [country.split(" - ", maxsplit=1)]
+        ]

devine/core/proxies/nordvpn.py

@@ -0,0 +1,138 @@
+import json
+import re
+from typing import Optional
+
+import requests
+
+from devine.core.proxies.proxy import Proxy
+
+
+class NordVPN(Proxy):
+    def __init__(self, username: str, password: str, server_map: Optional[dict[str, int]] = None):
+        """
+        Proxy Service using NordVPN Service Credentials.
+
+        A username and password must be provided. These are Service Credentials, not your Login Credentials.
+        The Service Credentials can be found here: https://my.nordaccount.com/dashboard/nordvpn/
+        """
+        if not username:
+            raise ValueError("No Username was provided to the NordVPN Proxy Service.")
+        if not password:
+            raise ValueError("No Password was provided to the NordVPN Proxy Service.")
+        if not re.match(r"^[a-z0-9]{48}$", username + password, re.IGNORECASE) or "@" in username:
+            raise ValueError(
+                "The Username and Password must be NordVPN Service Credentials, not your Login Credentials. "
+                "The Service Credentials can be found here: https://my.nordaccount.com/dashboard/nordvpn/"
+            )
+
+        if server_map is not None and not isinstance(server_map, dict):
+            raise TypeError(f"Expected server_map to be a dict mapping a region to a server ID, not '{server_map!r}'.")
+
+        self.username = username
+        self.password = password
+        self.server_map = server_map or {}
+
+        self.countries = self.get_countries()
+
+    def __repr__(self) -> str:
+        countries = len(self.countries)
+        servers = sum(x["servers_count"] for x in self.countries)
+
+        return f"{countries} Countr{['ies', 'y'][countries == 1]} ({servers} Server{['s', ''][servers == 1]})"
+
+    def get_proxy(self, query: str) -> Optional[str]:
+        """
+        Get an HTTP(SSL) proxy URI for a NordVPN server.
+
+        HTTP proxies under port 80 were disabled on the 15th of Feb, 2021:
+        https://nordvpn.com/blog/removing-http-proxies
+        """
+        query = query.lower()
+        if re.match(r"^[a-z]{2}\d+$", query):
+            # country and nordvpn server id, e.g., us1, fr1234
+            hostname = f"{query}.nordvpn.com"
+        else:
+            if query.isdigit():
+                # country id
+                country = self.get_country(by_id=int(query))
+            elif re.match(r"^[a-z]+$", query):
+                # country code
+                country = self.get_country(by_code=query)
+            else:
+                raise ValueError(f"The query provided is unsupported and unrecognized: {query}")
+            if not country:
+                # NordVPN doesnt have servers in this region
+                return
+
+            server_mapping = self.server_map.get(country["code"].lower())
+            if server_mapping:
+                # country was set to a specific server ID in config
+                hostname = f"{country['code'].lower()}{server_mapping}.nordvpn.com"
+            else:
+                # get the recommended server ID
+                recommended_servers = self.get_recommended_servers(country["id"])
+                if not recommended_servers:
+                    raise ValueError(
+                        f"The NordVPN Country {query} currently has no recommended servers. "
+                        "Try again later. If the issue persists, double-check the query."
+                    )
+                hostname = recommended_servers[0]["hostname"]
+
+        if hostname.startswith("gb"):
+            # NordVPN uses the alpha2 of 'GB' in API responses, but 'UK' in the hostname
+            hostname = f"gb{hostname[2:]}"
+
+        return f"https://{self.username}:{self.password}@{hostname}:89"
+
+    def get_country(
+        self,
+        by_id: Optional[int] = None,
+        by_code: Optional[str] = None
+    ) -> Optional[dict]:
+        """Search for a Country and it's metadata."""
+        if all(x is None for x in (by_id, by_code)):
+            raise ValueError("At least one search query must be made.")
+
+        for country in self.countries:
+            if all([
+                by_id is None or country["id"] == int(by_id),
+                by_code is None or country["code"] == by_code.upper()
+            ]):
+                return country
+
+    @staticmethod
+    def get_recommended_servers(country_id: int) -> list[dict]:
+        """
+        Get the list of recommended Servers for a Country.
+
+        Note: There may not always be more than one recommended server.
+        """
+        res = requests.get(
+            url="https://nordvpn.com/wp-admin/admin-ajax.php",
+            params={
+                "action": "servers_recommendations",
+                "filters": json.dumps({"country_id": country_id})
+            }
+        )
+        if not res.ok:
+            raise ValueError(f"Failed to get a list of NordVPN countries [{res.status_code}]")
+
+        try:
+            return res.json()
+        except json.JSONDecodeError:
+            raise ValueError("Could not decode list of NordVPN countries, not JSON data.")
+
+    @staticmethod
+    def get_countries() -> list[dict]:
+        """Get a list of available Countries and their metadata."""
+        res = requests.get(
+            url="https://nordvpn.com/wp-admin/admin-ajax.php",
+            params={"action": "servers_countries"}
+        )
+        if not res.ok:
+            raise ValueError(f"Failed to get a list of NordVPN countries [{res.status_code}]")
+
+        try:
+            return res.json()
+        except json.JSONDecodeError:
+            raise ValueError("Could not decode list of NordVPN countries, not JSON data.")

devine/core/proxies/proxy.py

@@ -0,0 +1,31 @@
+from abc import abstractmethod
+from typing import Optional
+
+
+class Proxy:
+    @abstractmethod
+    def __init__(self, **kwargs):
+        """
+        The constructor initializes the Service using passed configuration data.
+
+        Any authorization or pre-fetching of data should be done here.
+        """
+
+    @abstractmethod
+    def __repr__(self) -> str:
+        """Return a string denoting a list of Countries and Servers (if possible)."""
+        countries = ...
+        servers = ...
+        return f"{countries} Countr{['ies', 'y'][countries == 1]} ({servers} Server{['s', ''][servers == 1]})"
+
+    @abstractmethod
+    def get_proxy(self, query: str) -> Optional[str]:
+        """
+        Get a Proxy URI from the Proxy Service.
+
+        Only return None if the query was accepted, but no proxy could be returned.
+        Otherwise, please use exceptions to denote any errors with the call or query.
+
+        The returned Proxy URI must be a string supported by Python-Requests:
+        '{scheme}://[{user}:{pass}@]{host}:{port}'
+        """

devine/core/service.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+import base64
+import logging
+from abc import ABCMeta, abstractmethod
+from http.cookiejar import MozillaCookieJar, CookieJar
+from typing import Optional, Union
+from urllib.parse import urlparse
+
+import click
+import requests
+from requests.adapters import Retry, HTTPAdapter
+
+from devine.core.config import config
+from devine.core.constants import AnyTrack
+from devine.core.titles import Titles_T, Title_T
+from devine.core.tracks import Chapter, Tracks
+from devine.core.utilities import get_ip_info
+from devine.core.cacher import Cacher
+from devine.core.credential import Credential
+
+
+class Service(metaclass=ABCMeta):
+    """The Service Base Class."""
+
+    # Abstract class variables
+    ALIASES: tuple[str, ...] = ()  # list of aliases for the service; alternatives to the service tag.
+    GEOFENCE: tuple[str, ...] = ()  # list of ip regions required to use the service. empty list == no specific region.
+
+    def __init__(self, ctx: click.Context):
+        self.config = ctx.obj.config
+
+        assert ctx.parent is not None
+        assert ctx.parent.parent is not None
+
+        self.log = logging.getLogger(self.__class__.__name__)
+        self.session = self.get_session()
+        self.cache = Cacher(self.__class__.__name__)
+
+        self.proxy = ctx.parent.params["proxy"]
+        if not self.proxy and self.GEOFENCE:
+            # no explicit proxy, let's get one to GEOFENCE if needed
+            current_region = get_ip_info(self.session)["country"].lower()
+            if not any([x.lower() == current_region for x in self.GEOFENCE]):
+                requested_proxy = self.GEOFENCE[0]  # first is likely main region
+                self.log.info(f"Current IP region is blocked by the service, getting Proxy to {requested_proxy}")
+                # current region is not in any of the service's supported regions
+                for proxy_provider in ctx.obj.proxy_providers:
+                    self.proxy = proxy_provider.get_proxy(requested_proxy)
+                    if self.proxy:
+                        self.log.info(f" + {self.proxy} (from {proxy_provider.__class__.__name__})")
+                        break
+        if self.proxy:
+            self.session.proxies.update({"all": self.proxy})
+            proxy_parse = urlparse(self.proxy)
+            if proxy_parse.username and proxy_parse.password:
+                self.session.headers.update({
+                    "Proxy-Authorization": base64.b64encode(
+                        f"{proxy_parse.username}:{proxy_parse.password}".encode("utf8")
+                    ).decode()
+                })
+
+    # Optional Abstract functions
+    # The following functions may be implemented by the Service.
+    # Otherwise, the base service code (if any) of the function will be executed on call.
+    # The functions will be executed in shown order.
+
+    def get_session(self) -> requests.Session:
+        """
+        Creates a Python-requests Session, adds common headers
+        from config, cookies, retry handler, and a proxy if available.
+        :returns: Prepared Python-requests Session
+        """
+        session = requests.Session()
+        session.headers.update(config.headers)
+        session.mount("https://", HTTPAdapter(
+            max_retries=Retry(
+                total=15,
+                backoff_factor=0.2,
+                status_forcelist=[429, 500, 502, 503, 504]
+            )
+        ))
+        session.mount("http://", session.adapters["https://"])
+        return session
+
+    def authenticate(self, cookies: Optional[MozillaCookieJar] = None, credential: Optional[Credential] = None) -> None:
+        """
+        Authenticate the Service with Cookies and/or Credentials (Email/Username and Password).
+
+        This is effectively a login() function. Any API calls or object initializations
+        needing to be made, should be made here. This will be run before any of the
+        following abstract functions.
+
+        You should avoid storing or using the Credential outside this function.
+        Make any calls you need for any Cookies, Tokens, or such, then use those.
+
+        The Cookie jar should also not be stored outside this function. However, you may load
+        the Cookie jar into the service session.
+        """
+        if cookies is not None:
+            if not isinstance(cookies, CookieJar):
+                raise TypeError(f"Expected cookies to be a {MozillaCookieJar}, not {cookies!r}.")
+            self.session.cookies.update(cookies)
+
+    def get_widevine_service_certificate(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Union[bytes, str]:
+        """
+        Get the Widevine Service Certificate used for Privacy Mode.
+
+        :param challenge: The service challenge, providing this to a License endpoint should return the
+            privacy certificate that the service uses.
+        :param title: The current `Title` from get_titles that is being executed. This is provided in
+            case it has data needed to be used, e.g. for a HTTP request.
+        :param track: The current `Track` needing decryption. Provided for same reason as `title`.
+        :return: The Service Privacy Certificate as Bytes or a Base64 string. Don't Base64 Encode or
+            Decode the data, return as is to reduce unnecessary computations.
+        """
+
+    def get_widevine_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[Union[bytes, str]]:
+        """
+        Get a Widevine License message by sending a License Request (challenge).
+
+        This License message contains the encrypted Content Decryption Keys and will be
+        read by the Cdm and decrypted.
+
+        This is a very important request to get correct. A bad, unexpected, or missing
+        value in the request can cause your key to be detected and promptly banned,
+        revoked, disabled, or downgraded.
+
+        :param challenge: The license challenge from the Widevine CDM.
+        :param title: The current `Title` from get_titles that is being executed. This is provided in
+            case it has data needed to be used, e.g. for a HTTP request.
+        :param track: The current `Track` needing decryption. Provided for same reason as `title`.
+        :return: The License response as Bytes or a Base64 string. Don't Base64 Encode or
+            Decode the data, return as is to reduce unnecessary computations.
+        """
+
+    # Required Abstract functions
+    # The following functions *must* be implemented by the Service.
+    # The functions will be executed in shown order.
+
+    @abstractmethod
+    def get_titles(self) -> Titles_T:
+        """
+        Get Titles for the provided title ID.
+
+        Return a Movies, Series, or Album objects containing Movie, Episode, or Song title objects respectively.
+        The returned data must be for the given title ID, or a spawn of the title ID.
+
+        At least one object is expected to be returned, or it will presume an invalid Title ID was
+        provided.
+
+        You can use the `data` dictionary class instance attribute of each Title to store data you may need later on.
+        This can be useful to store information on each title that will be required like any sub-asset IDs, or such.
+        """
+
+    @abstractmethod
+    def get_tracks(self, title: Title_T) -> Tracks:
+        """
+        Get Track objects of the Title.
+
+        Return a Tracks object, which itself can contain Video, Audio, Subtitle or even Chapters.
+        Tracks.videos, Tracks.audio, Tracks.subtitles, and Track.chapters should be a List of Track objects.
+
+        Each Track in the Tracks should represent a Video/Audio Stream/Representation/Adaptation or
+        a Subtitle file.
+
+        While one Track should only hold information for one stream/downloadable, try to get as many
+        unique Track objects per stream type so Stream selection by the root code can give you more
+        options in terms of Resolution, Bitrate, Codecs, Language, e.t.c.
+
+        No decision making or filtering of which Tracks get returned should happen here. It can be
+        considered an error to filter for e.g. resolution, codec, and such. All filtering based on
+        arguments will be done by the root code automatically when needed.
+
+        Make sure you correctly mark which Tracks are encrypted or not, and by which DRM System
+        via its `drm` property.
+
+        If you are able to obtain the Track's KID (Key ID) as a 32 char (16 bit) HEX string, provide
+        it to the Track's `kid` variable as it will speed up the decryption process later on. It may
+        or may not be needed, that depends on the service. Generally if you can provide it, without
+        downloading any of the Track's stream data, then do.
+
+        :param title: The current `Title` from get_titles that is being executed.
+        :return: Tracks object containing Video, Audio, Subtitles, and Chapters, if available.
+        """
+
+    @abstractmethod
+    def get_chapters(self, title: Title_T) -> list[Chapter]:
+        """
+        Get Chapter objects of the Title.
+
+        Return a list of Chapter objects. This will be run after get_tracks. If there's anything
+        from the get_tracks that may be needed, e.g. "device_id" or a-like, store it in the class
+        via `self` and re-use the value in get_chapters.
+
+        How it's used is generally the same as get_titles. These are only separated as to reduce
+        function complexity and keep them focused on simple tasks.
+
+        You do not need to sort or order the chapters in any way. However, you do need to filter
+        and alter them as needed by the service. No modification is made after get_chapters is
+        ran. So that means ensure that the Chapter objects returned have consistent Chapter Titles
+        and Chapter Numbers.
+
+        :param title: The current `Title` from get_titles that is being executed.
+        :return: List of Chapter objects, if available, empty list otherwise.
+        """
+
+
+__ALL__ = (Service,)

devine/core/services.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import click
+
+from devine.core.config import config
+from devine.core.service import Service
+from devine.core.utilities import import_module_by_path
+
+_SERVICES = sorted(
+    (
+        path
+        for path in config.directories.services.glob("*/__init__.py")
+    ),
+    key=lambda x: x.parent.stem
+)
+
+_MODULES = {
+    path.parent.stem: getattr(import_module_by_path(path), path.parent.stem)
+    for path in _SERVICES
+}
+
+_ALIASES = {
+    tag: getattr(module, "ALIASES")
+    for tag, module in _MODULES.items()
+}
+
+
+class Services(click.MultiCommand):
+    """Lazy-loaded command group of project services."""
+
+    # Click-specific methods
+
+    def list_commands(self, ctx: click.Context) -> list[str]:
+        """Returns a list of all available Services as command names for Click."""
+        return Services.get_tags()
+
+    def get_command(self, ctx: click.Context, name: str) -> click.Command:
+        """Load the Service and return the Click CLI method."""
+        tag = Services.get_tag(name)
+        service = Services.load(tag)
+
+        if hasattr(service, "cli"):
+            return service.cli
+
+        raise click.ClickException(f"Service '{tag}' has no 'cli' method configured.")
+
+    # Methods intended to be used anywhere
+
+    @staticmethod
+    def get_tags() -> list[str]:
+        """Returns a list of service tags from all available Services."""
+        return [x.parent.stem for x in _SERVICES]
+
+    @staticmethod
+    def get_path(name: str) -> Path:
+        """Get the directory path of a command."""
+        tag = Services.get_tag(name)
+        for service in _SERVICES:
+            if service.parent.stem == tag:
+                return service.parent
+        raise click.ClickException(f"Unable to find service by the name '{name}'")
+
+    @staticmethod
+    def get_tag(value: str) -> str:
+        """
+        Get the Service Tag (e.g. DSNP, not DisneyPlus/Disney+, etc.) by an Alias.
+        Input value can be of any case-sensitivity.
+        Original input value is returned if it did not match a service tag.
+        """
+        original_value = value
+        value = value.lower()
+        for path in _SERVICES:
+            tag = path.parent.stem
+            if value in (tag.lower(), *_ALIASES.get(tag, [])):
+                return tag
+        return original_value
+
+    @staticmethod
+    def load(tag: str) -> Service:
+        """Load a Service module by Service tag."""
+        module = _MODULES.get(tag)
+        if not module:
+            raise click.ClickException(f"Unable to find Service by the tag '{tag}'")
+        return module
+
+
+__ALL__ = (Services,)

devine/core/titles/__init__.py

@@ -0,0 +1,9 @@
+from typing import Union
+
+from .episode import Episode, Series
+from .movie import Movie, Movies
+from .song import Song, Album
+
+
+Title_T = Union[Movie, Episode, Song]
+Titles_T = Union[Movies, Series, Album]

devine/core/titles/episode.py

@@ -0,0 +1,195 @@
+import re
+from abc import ABC
+from collections import Counter
+from typing import Any, Optional, Union, Iterable
+
+from langcodes import Language
+from pymediainfo import MediaInfo
+from sortedcontainers import SortedKeyList
+
+from devine.core.config import config
+from devine.core.constants import AUDIO_CODEC_MAP, DYNAMIC_RANGE_MAP, VIDEO_CODEC_MAP
+from devine.core.titles.title import Title
+from devine.core.utilities import sanitize_filename
+
+
+class Episode(Title):
+    def __init__(
+        self,
+        id_: Any,
+        service: type,
+        title: str,
+        season: Union[int, str],
+        number: Union[int, str],
+        name: Optional[str] = None,
+        year: Optional[Union[int, str]] = None,
+        language: Optional[Union[str, Language]] = None,
+        data: Optional[Any] = None,
+    ) -> None:
+        super().__init__(id_, service, language, data)
+
+        if not title:
+            raise ValueError("Episode title must be provided")
+        if not isinstance(title, str):
+            raise TypeError(f"Expected title to be a str, not {title!r}")
+
+        if season != 0 and not season:
+            raise ValueError("Episode season must be provided")
+        if isinstance(season, str) and season.isdigit():
+            season = int(season)
+        elif not isinstance(season, int):
+            raise TypeError(f"Expected season to be an int, not {season!r}")
+
+        if number != 0 and not number:
+            raise ValueError("Episode number must be provided")
+        if isinstance(number, str) and number.isdigit():
+            number = int(number)
+        elif not isinstance(number, int):
+            raise TypeError(f"Expected number to be an int, not {number!r}")
+
+        if name is not None and not isinstance(name, str):
+            raise TypeError(f"Expected name to be a str, not {name!r}")
+
+        if year is not None:
+            if isinstance(year, str) and year.isdigit():
+                year = int(year)
+            elif not isinstance(year, int):
+                raise TypeError(f"Expected year to be an int, not {year!r}")
+
+        title = title.strip()
+
+        if name is not None:
+            name = name.strip()
+            # ignore episode names that are the episode number or title name
+            if re.match(r"Episode ?#?\d+", name, re.IGNORECASE):
+                name = None
+            elif name.lower() == title.lower():
+                name = None
+
+        if year is not None and year <= 0:
+            raise ValueError(f"Episode year cannot be {year}")
+
+        self.title = title
+        self.season = season
+        self.number = number
+        self.name = name
+        self.year = year
+
+    def __str__(self) -> str:
+        return "{title} S{season:02}E{number:02} {name}".format(
+            title=self.title,
+            season=self.season,
+            number=self.number,
+            name=self.name or ""
+        ).strip()
+
+    def get_filename(self, media_info: MediaInfo, folder: bool = False, show_service: bool = True) -> str:
+        primary_video_track = next(iter(media_info.video_tracks), None)
+        primary_audio_track = next(iter(media_info.audio_tracks), None)
+        unique_audio_languages = len({
+            x.language.split("-")[0]
+            for x in media_info.audio_tracks
+            if x.language
+        })
+
+        # Title SXXEXX Name (or Title SXX if folder)
+        if folder:
+            name = f"{self.title} S{self.season:02}"
+        else:
+            name = "{title} S{season:02}E{number:02} {name}".format(
+                title=self.title.replace("$", "S"),  # e.g., Arli$$
+                season=self.season,
+                number=self.number,
+                name=self.name or ""
+            ).strip()
+
+        # MULTi
+        if unique_audio_languages > 1:
+            name += " MULTi"
+
+        # Resolution
+        if primary_video_track:
+            resolution = primary_video_track.height
+            aspect_ratio = [
+                int(float(plane))
+                for plane in primary_video_track.other_display_aspect_ratio[0].split(":")
+            ]
+            if len(aspect_ratio) == 1:
+                # e.g., aspect ratio of 2 (2.00:1) would end up as `(2.0,)`, add 1
+                aspect_ratio.append(1)
+            if aspect_ratio[0] / aspect_ratio[1] not in (16 / 9, 4 / 3):
+                # We want the resolution represented in a 4:3 or 16:9 canvas.
+                # If it's not 4:3 or 16:9, calculate as if it's inside a 16:9 canvas,
+                # otherwise the track's height value is fine.
+                # We are assuming this title is some weird aspect ratio so most
+                # likely a movie or HD source, so it's most likely widescreen so
+                # 16:9 canvas makes the most sense.
+                resolution = int(primary_video_track.width * (9 / 16))
+            name += f" {resolution}p"
+
+        # Service
+        if show_service:
+            name += f" {self.service.__name__}"
+
+        # 'WEB-DL'
+        name += " WEB-DL"
+
+        # Audio Codec + Channels (+ feature)
+        if primary_audio_track:
+            codec = primary_audio_track.format
+            channel_layout = primary_audio_track.channel_layout or primary_audio_track.channellayout_original
+            channels = float(sum(
+                {"LFE": 0.1}.get(position.upper(), 1)
+                for position in channel_layout.split(" ")
+            ))
+            features = primary_audio_track.format_additionalfeatures or ""
+            name += f" {AUDIO_CODEC_MAP.get(codec, codec)}{channels:.1f}"
+            if "JOC" in features:
+                name += " Atmos"
+
+        # Video (dynamic range + hfr +) Codec
+        if primary_video_track:
+            codec = primary_video_track.format
+            hdr_format = primary_video_track.hdr_format_commercial
+            trc = primary_video_track.transfer_characteristics or primary_video_track.transfer_characteristics_original
+            frame_rate = float(primary_video_track.frame_rate)
+            if hdr_format:
+                name += f" {DYNAMIC_RANGE_MAP.get(hdr_format)} "
+            elif trc and "HLG" in trc:
+                name += " HLG"
+            if frame_rate > 30:
+                name += " HFR"
+            name += f" {VIDEO_CODEC_MAP.get(codec, codec)}"
+
+        if config.tag:
+            name += f"-{config.tag}"
+
+        return sanitize_filename(name)
+
+
+class Series(SortedKeyList, ABC):
+    def __init__(self, iterable: Optional[Iterable] = None):
+        super().__init__(
+            iterable,
+            key=lambda x: (x.season, x.number, x.year or 0)
+        )
+
+    def __str__(self) -> str:
+        if not self:
+            return super().__str__()
+
+        lines = [
+            f"Series: {self[0].title} ({self[0].year or '?'})",
+            f"Episodes: ({len(self)})",
+            *[
+                f"├─ S{season:02}: {episodes} episodes"
+                for season, episodes in Counter(x.season for x in self).items()
+            ]
+        ]
+        last_line = lines.pop(-1)
+        lines.append(last_line.replace("├", "└"))
+
+        return "\n".join(lines)
+
+
+__ALL__ = (Episode, Series)

devine/core/titles/movie.py

@@ -0,0 +1,155 @@
+from abc import ABC
+from typing import Any, Optional, Union, Iterable
+
+from langcodes import Language
+from pymediainfo import MediaInfo
+from sortedcontainers import SortedKeyList
+
+from devine.core.config import config
+from devine.core.constants import AUDIO_CODEC_MAP, DYNAMIC_RANGE_MAP, VIDEO_CODEC_MAP
+from devine.core.titles.title import Title
+from devine.core.utilities import sanitize_filename
+
+
+class Movie(Title):
+    def __init__(
+        self,
+        id_: Any,
+        service: type,
+        name: str,
+        year: Optional[Union[int, str]] = None,
+        language: Optional[Union[str, Language]] = None,
+        data: Optional[Any] = None,
+    ) -> None:
+        super().__init__(id_, service, language, data)
+
+        if not name:
+            raise ValueError("Movie name must be provided")
+        if not isinstance(name, str):
+            raise TypeError(f"Expected name to be a str, not {name!r}")
+
+        if year is not None:
+            if isinstance(year, str) and year.isdigit():
+                year = int(year)
+            elif not isinstance(year, int):
+                raise TypeError(f"Expected year to be an int, not {year!r}")
+
+        name = name.strip()
+
+        if year is not None and year <= 0:
+            raise ValueError(f"Movie year cannot be {year}")
+
+        self.name = name
+        self.year = year
+
+    def __str__(self) -> str:
+        if self.year:
+            return f"{self.name} ({self.year})"
+        return self.name
+
+    def get_filename(self, media_info: MediaInfo, folder: bool = False, show_service: bool = True) -> str:
+        primary_video_track = next(iter(media_info.video_tracks), None)
+        primary_audio_track = next(iter(media_info.audio_tracks), None)
+        unique_audio_languages = len({
+            x.language.split("-")[0]
+            for x in media_info.audio_tracks
+            if x.language
+        })
+
+        # Name (Year)
+        name = str(self).replace("$", "S")  # e.g., Arli$$
+
+        # MULTi
+        if unique_audio_languages > 1:
+            name += " MULTi"
+
+        # Resolution
+        if primary_video_track:
+            resolution = primary_video_track.height
+            aspect_ratio = [
+                int(float(plane))
+                for plane in primary_video_track.other_display_aspect_ratio[0].split(":")
+            ]
+            if len(aspect_ratio) == 1:
+                # e.g., aspect ratio of 2 (2.00:1) would end up as `(2.0,)`, add 1
+                aspect_ratio.append(1)
+            if aspect_ratio[0] / aspect_ratio[1] not in (16 / 9, 4 / 3):
+                # We want the resolution represented in a 4:3 or 16:9 canvas.
+                # If it's not 4:3 or 16:9, calculate as if it's inside a 16:9 canvas,
+                # otherwise the track's height value is fine.
+                # We are assuming this title is some weird aspect ratio so most
+                # likely a movie or HD source, so it's most likely widescreen so
+                # 16:9 canvas makes the most sense.
+                resolution = int(primary_video_track.width * (9 / 16))
+            name += f" {resolution}p"
+
+        # Service
+        if show_service:
+            name += f" {self.service.__name__}"
+
+        # 'WEB-DL'
+        name += " WEB-DL"
+
+        # Audio Codec + Channels (+ feature)
+        if primary_audio_track:
+            codec = primary_audio_track.format
+            channel_layout = primary_audio_track.channel_layout or primary_audio_track.channellayout_original
+            channels = float(sum(
+                {"LFE": 0.1}.get(position.upper(), 1)
+                for position in channel_layout.split(" ")
+            ))
+            features = primary_audio_track.format_additionalfeatures or ""
+            name += f" {AUDIO_CODEC_MAP.get(codec, codec)}{channels:.1f}"
+            if "JOC" in features:
+                name += " Atmos"
+
+        # Video (dynamic range + hfr +) Codec
+        if primary_video_track:
+            codec = primary_video_track.format
+            hdr_format = primary_video_track.hdr_format_commercial
+            trc = primary_video_track.transfer_characteristics or primary_video_track.transfer_characteristics_original
+            frame_rate = float(primary_video_track.frame_rate)
+            if hdr_format:
+                name += f" {DYNAMIC_RANGE_MAP.get(hdr_format)} "
+            elif trc and "HLG" in trc:
+                name += " HLG"
+            if frame_rate > 30:
+                name += " HFR"
+            name += f" {VIDEO_CODEC_MAP.get(codec, codec)}"
+
+        if config.tag:
+            name += f"-{config.tag}"
+
+        return sanitize_filename(name)
+
+
+class Movies(SortedKeyList, ABC):
+    def __init__(self, iterable: Optional[Iterable] = None):
+        super().__init__(
+            iterable,
+            key=lambda x: x.year or 0
+        )
+
+    def __str__(self) -> str:
+        if not self:
+            return super().__str__()
+
+        if len(self) > 1:
+            lines = [
+                f"Movies: ({len(self)})",
+                *[
+                    f"├─ {movie.name} ({movie.year or '?'})"
+                    for movie in self
+                ]
+            ]
+            last_line = lines.pop(-1)
+            lines.append(last_line.replace("├", "└"))
+        else:
+            lines = [
+                f"Movie: {self[0].name} ({self[0].year or '?'})"
+            ]
+
+        return "\n".join(lines)
+
+
+__ALL__ = (Movie, Movies)

devine/core/titles/song.py

@@ -0,0 +1,148 @@
+from abc import ABC
+from typing import Any, Optional, Union, Iterable
+
+from langcodes import Language
+from pymediainfo import MediaInfo
+from sortedcontainers import SortedKeyList
+
+from devine.core.config import config
+from devine.core.constants import AUDIO_CODEC_MAP
+from devine.core.titles.title import Title
+from devine.core.utilities import sanitize_filename
+
+
+class Song(Title):
+    def __init__(
+        self,
+        id_: Any,
+        service: type,
+        name: str,
+        artist: str,
+        album: str,
+        track: int,
+        disc: int,
+        year: int,
+        language: Optional[Union[str, Language]] = None,
+        data: Optional[Any] = None,
+    ) -> None:
+        super().__init__(id_, service, language, data)
+
+        if not name:
+            raise ValueError("Song name must be provided")
+        if not isinstance(name, str):
+            raise TypeError(f"Expected name to be a str, not {name!r}")
+
+        if not artist:
+            raise ValueError("Song artist must be provided")
+        if not isinstance(artist, str):
+            raise TypeError(f"Expected artist to be a str, not {artist!r}")
+
+        if not album:
+            raise ValueError("Song album must be provided")
+        if not isinstance(album, str):
+            raise TypeError(f"Expected album to be a str, not {name!r}")
+
+        if not track:
+            raise ValueError("Song track must be provided")
+        if not isinstance(track, int):
+            raise TypeError(f"Expected track to be an int, not {track!r}")
+
+        if not disc:
+            raise ValueError("Song disc must be provided")
+        if not isinstance(disc, int):
+            raise TypeError(f"Expected disc to be an int, not {disc!r}")
+
+        if not year:
+            raise ValueError("Song year must be provided")
+        if not isinstance(year, int):
+            raise TypeError(f"Expected year to be an int, not {year!r}")
+
+        name = name.strip()
+        artist = artist.strip()
+        album = album.strip()
+
+        if track <= 0:
+            raise ValueError(f"Song track cannot be {track}")
+        if disc <= 0:
+            raise ValueError(f"Song disc cannot be {disc}")
+        if year <= 0:
+            raise ValueError(f"Song year cannot be {year}")
+
+        self.name = name
+        self.artist = artist
+        self.album = album
+        self.track = track
+        self.disc = disc
+        self.year = year
+
+    def __str__(self) -> str:
+        return "{artist} - {album} ({year}) / {track:02}. {name}".format(
+            artist=self.artist,
+            album=self.album,
+            year=self.year,
+            track=self.track,
+            name=self.name
+        ).strip()
+
+    def get_filename(self, media_info: MediaInfo, folder: bool = False, show_service: bool = True) -> str:
+        audio_track = next(iter(media_info.audio_tracks), None)
+        codec = audio_track.format
+        channel_layout = audio_track.channel_layout or audio_track.channellayout_original
+        channels = float(sum(
+            {"LFE": 0.1}.get(position.upper(), 1)
+            for position in channel_layout.split(" ")
+        ))
+        features = audio_track.format_additionalfeatures or ""
+
+        if folder:
+            # Artist - Album (Year)
+            name = str(self).split(" / ")[0]
+        else:
+            # NN. Song Name
+            name = str(self).split(" / ")[1]
+
+        # Service
+        if show_service:
+            name += f" {self.service.__name__}"
+
+        # 'WEB-DL'
+        name += " WEB-DL"
+
+        # Audio Codec + Channels (+ feature)
+        name += f" {AUDIO_CODEC_MAP.get(codec, codec)}{channels:.1f}"
+        if "JOC" in features:
+            name += " Atmos"
+
+        if config.tag:
+            name += f"-{config.tag}"
+
+        return sanitize_filename(name, " ")
+
+
+class Album(SortedKeyList, ABC):
+    def __init__(self, iterable: Optional[Iterable] = None):
+        super().__init__(
+            iterable,
+            key=lambda x: (x.album, x.disc, x.track, x.year or 0)
+        )
+
+    def __str__(self) -> str:
+        if not self:
+            return super().__str__()
+
+        lines = [
+            f"Album: {self[0].album} ({self[0].year or '?'})",
+            f"Artist: {self[0].artist}",
+            f"Tracks: ({len(self)})",
+            *[
+                f"├─ {song.track:02}. {song.name}"
+                for song in self
+            ]
+        ]
+        last_line = lines.pop(-1)
+        lines.append(last_line.replace("├", "└"))
+
+        return "\n".join(lines)
+
+
+__ALL__ = (Song, Album)

devine/core/titles/title.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from abc import abstractmethod
+from typing import Optional, Union, Any
+
+from langcodes import Language
+from pymediainfo import MediaInfo
+
+from devine.core.tracks import Tracks
+
+
+class Title:
+    def __init__(
+        self,
+        id_: Any,
+        service: type,
+        language: Optional[Union[str, Language]] = None,
+        data: Optional[Any] = None
+    ) -> None:
+        """
+        Media Title from a Service.
+
+        Parameters:
+            id_: An identifier for this specific title. It must be unique. Can be of any
+                value.
+            service: Service class that this title is from.
+            language: The original recorded language for the title. If that information
+                is not available, this should not be set to anything.
+            data: Arbitrary storage for the title. Often used to store extra metadata
+                information, IDs, URIs, and so on.
+        """
+        if not id_:  # includes 0, false, and similar values, this is intended
+            raise ValueError("A unique ID must be provided")
+        if hasattr(id_, "__len__") and len(id_) < 4:
+            raise ValueError("The unique ID is not large enough, clash likely.")
+
+        if not service:
+            raise ValueError("Service class must be provided")
+        if not isinstance(service, type):
+            raise TypeError(f"Expected service to be a Class (type), not {service!r}")
+
+        if language is not None:
+            if isinstance(language, str):
+                language = Language.get(language)
+            elif not isinstance(language, Language):
+                raise TypeError(f"Expected language to be a {Language} or str, not {language!r}")
+
+        self.id = id_
+        self.service = service
+        self.language = language
+        self.data = data
+
+        self.tracks = Tracks()
+
+    def __eq__(self, other: Title) -> bool:
+        return self.id == other.id
+
+    @abstractmethod
+    def get_filename(self, media_info: MediaInfo, folder: bool = False, show_service: bool = True) -> str:
+        """
+        Get a Filename for this Title with the provided Media Info.
+        All filenames should be sanitized with the sanitize_filename() utility function.
+
+        Parameters:
+            media_info: MediaInfo object of the file this name will be used for.
+            folder: This filename will be used as a folder name. Some changes may want to
+                be made if this is the case.
+            show_service: Show the service tag (e.g., iT, NF) in the filename.
+        """
+
+
+__ALL__ = (Title,)

devine/core/tracks/__init__.py

@@ -0,0 +1,6 @@
+from .audio import Audio
+from .track import Track
+from .chapter import Chapter
+from .subtitle import Subtitle
+from .tracks import Tracks
+from .video import Video

devine/core/tracks/audio.py

@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+import math
+from enum import Enum
+from typing import Any, Optional, Union
+
+from devine.core.tracks.track import Track
+
+
+class Audio(Track):
+    class Codec(str, Enum):
+        AAC = "AAC"    # https://wikipedia.org/wiki/Advanced_Audio_Coding
+        AC3 = "DD"     # https://wikipedia.org/wiki/Dolby_Digital
+        EC3 = "DD+"    # https://wikipedia.org/wiki/Dolby_Digital_Plus
+        OPUS = "OPUS"  # https://wikipedia.org/wiki/Opus_(audio_format)
+        OGG = "VORB"  # https://wikipedia.org/wiki/Vorbis
+        DTS = "DTS"  # https://en.wikipedia.org/wiki/DTS_(company)#DTS_Digital_Surround
+        ALAC = "ALAC"  # https://en.wikipedia.org/wiki/Apple_Lossless_Audio_Codec
+
+        @property
+        def extension(self) -> str:
+            return self.name.lower()
+
+        @staticmethod
+        def from_mime(mime: str) -> Audio.Codec:
+            mime = mime.lower().strip().split(".")[0]
+            if mime == "mp4a":
+                return Audio.Codec.AAC
+            if mime == "ac-3":
+                return Audio.Codec.AC3
+            if mime == "ec-3":
+                return Audio.Codec.EC3
+            if mime == "opus":
+                return Audio.Codec.OPUS
+            if mime == "dtsc":
+                return Audio.Codec.DTS
+            if mime == "alac":
+                return Audio.Codec.ALAC
+            raise ValueError(f"The MIME '{mime}' is not a supported Audio Codec")
+
+        @staticmethod
+        def from_codecs(codecs: str) -> Audio.Codec:
+            for codec in codecs.lower().split(","):
+                mime = codec.strip().split(".")[0]
+                try:
+                    return Audio.Codec.from_mime(mime)
+                except ValueError:
+                    pass
+            raise ValueError(f"No MIME types matched any supported Audio Codecs in '{codecs}'")
+
+        @staticmethod
+        def from_netflix_profile(profile: str) -> Audio.Codec:
+            profile = profile.lower().strip()
+            if profile.startswith("heaac"):
+                return Audio.Codec.AAC
+            if profile.startswith("dd-"):
+                return Audio.Codec.AC3
+            if profile.startswith("ddplus"):
+                return Audio.Codec.EC3
+            if profile.startswith("playready-oggvorbis"):
+                return Audio.Codec.OGG
+            raise ValueError(f"The Content Profile '{profile}' is not a supported Audio Codec")
+
+    def __init__(self, *args: Any, codec: Audio.Codec, bitrate: Union[str, int, float],
+                 channels: Optional[Union[str, int, float]] = None, joc: int = 0, descriptive: bool = False,
+                 **kwargs: Any):
+        super().__init__(*args, **kwargs)
+        # required
+        self.codec = codec
+        self.bitrate = int(math.ceil(float(bitrate))) if bitrate else None
+        self.channels = self.parse_channels(channels) if channels else None
+        # optional
+        self.joc = joc
+        self.descriptive = bool(descriptive)
+
+    @staticmethod
+    def parse_channels(channels: Union[str, float]) -> str:
+        """
+        Converts a string to a float-like string which represents audio channels.
+        It does not handle values that are incorrect/out of bounds or e.g. 6.0->5.1, as that
+        isn't what this is intended for.
+        E.g. "3" -> "3.0", "2.1" -> "2.1", ".1" -> "0.1".
+        """
+        # TODO: Support all possible DASH channel configurations (https://datatracker.ietf.org/doc/html/rfc8216)
+        if channels.upper() == "A000":
+            return "2.0"
+        if channels.upper() == "F801":
+            return "5.1"
+
+        if str(channels).isdigit():
+            # This is to avoid incorrectly transforming channels=6 to 6.0, for example
+            return f"{channels}ch"
+
+        try:
+            return str(float(channels))
+        except ValueError:
+            return str(channels)
+
+    def get_track_name(self) -> Optional[str]:
+        """Return the base Track Name."""
+        track_name = super().get_track_name() or ""
+        flag = self.descriptive and "Descriptive"
+        if flag:
+            if track_name:
+                flag = f" ({flag})"
+            track_name += flag
+        return track_name or None
+
+    def __str__(self) -> str:
+        return " | ".join(filter(bool, [
+            "AUD",
+            f"[{self.codec.value}]",
+            (self.channels or "2.0?") + (f" (JOC {self.joc})" if self.joc else ""),
+            f"{self.bitrate // 1000 if self.bitrate else '?'} kb/s",
+            str(self.language),
+            self.get_track_name(),
+            self.edition
+        ]))
+
+
+__ALL__ = (Audio,)

devine/core/tracks/chapter.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Optional, Union
+
+
+class Chapter:
+    line_1 = re.compile(r"^CHAPTER(?P<number>\d+)=(?P<timecode>[\d\\.]+)$")
+    line_2 = re.compile(r"^CHAPTER(?P<number>\d+)NAME=(?P<title>[\d\\.]+)$")
+
+    def __init__(self, number: int, timecode: str, title: Optional[str] = None):
+        self.id = f"chapter-{number}"
+        self.number = number
+        self.timecode = timecode
+        self.title = title
+
+        if "." not in self.timecode:
+            self.timecode += ".000"
+
+    def __bool__(self) -> bool:
+        return self.number and self.number >= 0 and self.timecode
+
+    def __repr__(self) -> str:
+        """
+        OGM-based Simple Chapter Format intended for use with MKVToolNix.
+
+        This format is not officially part of the Matroska spec. This was a format
+        designed for OGM tools that MKVToolNix has since re-used. More Information:
+        https://mkvtoolnix.download/doc/mkvmerge.html#mkvmerge.chapters.simple
+        """
+        return "CHAPTER{num}={time}\nCHAPTER{num}NAME={name}".format(
+            num=f"{self.number:02}",
+            time=self.timecode,
+            name=self.title or ""
+        )
+
+    def __str__(self) -> str:
+        return " | ".join(filter(bool, [
+            "CHP",
+            f"[{self.number:02}]",
+            self.timecode,
+            self.title
+        ]))
+
+    @property
+    def named(self) -> bool:
+        """Check if Chapter is named."""
+        return bool(self.title)
+
+    @classmethod
+    def loads(cls, data: str) -> Chapter:
+        """Load chapter data from a string."""
+        lines = [x.strip() for x in data.strip().splitlines(keepends=False)]
+        if len(lines) > 2:
+            return cls.loads("\n".join(lines))
+        one, two = lines
+
+        one_m = cls.line_1.match(one)
+        two_m = cls.line_2.match(two)
+        if not one_m or not two_m:
+            raise SyntaxError(f"An unexpected syntax error near:\n{one}\n{two}")
+
+        one_str, timecode = one_m.groups()
+        two_str, title = two_m.groups()
+        one_num, two_num = int(one_str.lstrip("0")), int(two_str.lstrip("0"))
+
+        if one_num != two_num:
+            raise SyntaxError(f"The chapter numbers ({one_num},{two_num}) does not match.")
+        if not timecode:
+            raise SyntaxError("The timecode is missing.")
+        if not title:
+            title = None
+
+        return cls(number=one_num, timecode=timecode, title=title)
+
+    @classmethod
+    def load(cls, path: Union[Path, str]) -> Chapter:
+        """Load chapter data from a file."""
+        if isinstance(path, str):
+            path = Path(path)
+        return cls.loads(path.read_text(encoding="utf8"))
+
+    def dumps(self) -> str:
+        """Return chapter data as a string."""
+        return repr(self)
+
+    def dump(self, path: Union[Path, str]) -> int:
+        """Write chapter data to a file."""
+        if isinstance(path, str):
+            path = Path(path)
+        return path.write_text(self.dumps(), encoding="utf8")
+
+
+__ALL__ = (Chapter,)

devine/core/tracks/subtitle.py

@@ -0,0 +1,399 @@
+from __future__ import annotations
+
+import subprocess
+from collections import defaultdict
+from enum import Enum
+from io import BytesIO
+from pathlib import Path
+from typing import Any, Iterable, Optional
+
+import pycaption
+from construct import Container
+from pycaption import Caption, CaptionList, CaptionNode, WebVTTReader
+from pycaption.geometry import Layout
+from pymp4.parser import MP4
+from subtitle_filter import Subtitles
+
+from devine.core.tracks.track import Track
+from devine.core.utilities import get_binary_path
+
+
+class Subtitle(Track):
+    class Codec(str, Enum):
+        SubRip = "SRT"                # https://wikipedia.org/wiki/SubRip
+        SubStationAlpha = "SSA"       # https://wikipedia.org/wiki/SubStation_Alpha
+        SubStationAlphav4 = "ASS"     # https://wikipedia.org/wiki/SubStation_Alpha#Advanced_SubStation_Alpha=
+        TimedTextMarkupLang = "TTML"  # https://wikipedia.org/wiki/Timed_Text_Markup_Language
+        WebVTT = "VTT"                # https://wikipedia.org/wiki/WebVTT
+        # MPEG-DASH box-encapsulated subtitle formats
+        fTTML = "STPP"  # https://www.w3.org/TR/2018/REC-ttml-imsc1.0.1-20180424
+        fVTT = "WVTT"   # https://www.w3.org/TR/webvtt1
+
+        @property
+        def extension(self) -> str:
+            return self.value.lower()
+
+        @staticmethod
+        def from_mime(mime: str) -> Subtitle.Codec:
+            mime = mime.lower().strip().split(".")[0]
+            if mime == "srt":
+                return Subtitle.Codec.SubRip
+            elif mime == "ssa":
+                return Subtitle.Codec.SubStationAlpha
+            elif mime == "ass":
+                return Subtitle.Codec.SubStationAlphav4
+            elif mime == "ttml":
+                return Subtitle.Codec.TimedTextMarkupLang
+            elif mime == "vtt":
+                return Subtitle.Codec.WebVTT
+            elif mime == "stpp":
+                return Subtitle.Codec.fTTML
+            elif mime == "wvtt":
+                return Subtitle.Codec.fVTT
+            raise ValueError(f"The MIME '{mime}' is not a supported Subtitle Codec")
+
+        @staticmethod
+        def from_codecs(codecs: str) -> Subtitle.Codec:
+            for codec in codecs.lower().split(","):
+                mime = codec.strip().split(".")[0]
+                try:
+                    return Subtitle.Codec.from_mime(mime)
+                except ValueError:
+                    pass
+            raise ValueError(f"No MIME types matched any supported Subtitle Codecs in '{codecs}'")
+
+        @staticmethod
+        def from_netflix_profile(profile: str) -> Subtitle.Codec:
+            profile = profile.lower().strip()
+            if profile.startswith("webvtt"):
+                return Subtitle.Codec.WebVTT
+            if profile.startswith("dfxp"):
+                return Subtitle.Codec.TimedTextMarkupLang
+            raise ValueError(f"The Content Profile '{profile}' is not a supported Subtitle Codec")
+
+    def __init__(self, *args: Any, codec: Subtitle.Codec, cc: bool = False, sdh: bool = False, forced: bool = False,
+                 **kwargs: Any):
+        """
+        Information on Subtitle Types:
+            https://bit.ly/2Oe4fLC (3PlayMedia Blog on SUB vs CC vs SDH).
+            However, I wouldn't pay much attention to the claims about SDH needing to
+            be in the original source language. It's logically not true.
+
+            CC == Closed Captions. Source: Basically every site.
+            SDH = Subtitles for the Deaf or Hard-of-Hearing. Source: Basically every site.
+            HOH = Exact same as SDH. Is a term used in the UK. Source: https://bit.ly/2PGJatz (ICO UK)
+
+            More in-depth information, examples, and stuff to look for can be found in the Parameter
+            explanation list below.
+
+        Parameters:
+            cc: Closed Caption.
+                - Intended as if you couldn't hear the audio at all.
+                - Can have Sound as well as Dialogue, but doesn't have to.
+                - Original source would be from an EIA-CC encoded stream. Typically all
+                  upper-case characters.
+                Indicators of it being CC without knowing original source:
+                  - Extracted with CCExtractor, or
+                  - >>> (or similar) being used at the start of some or all lines, or
+                  - All text is uppercase or at least the majority, or
+                  - Subtitles are Scrolling-text style (one line appears, oldest line
+                    then disappears).
+                Just because you downloaded it as a SRT or VTT or such, doesn't mean it
+                 isn't from an EIA-CC stream. And I wouldn't take the streaming services
+                 (CC) as gospel either as they tend to get it wrong too.
+            sdh: Deaf or Hard-of-Hearing. Also known as HOH in the UK (EU?).
+                 - Intended as if you couldn't hear the audio at all.
+                 - MUST have Sound as well as Dialogue to be considered SDH.
+                 - It has no "syntax" or "format" but is not transmitted using archaic
+                   forms like EIA-CC streams, would be intended for transmission via
+                   SubRip (SRT), WebVTT (VTT), TTML, etc.
+                 If you can see important audio/sound transcriptions and not just dialogue
+                  and it doesn't have the indicators of CC, then it's most likely SDH.
+                 If it doesn't have important audio/sounds transcriptions it might just be
+                  regular subtitling (you wouldn't mark as CC or SDH). This would be the
+                  case for most translation subtitles. Like Anime for example.
+            forced: Typically used if there's important information at some point in time
+                     like watching Dubbed content and an important Sign or Letter is shown
+                     or someone talking in a different language.
+                    Forced tracks are recommended by the Matroska Spec to be played if
+                     the player's current playback audio language matches a subtitle
+                     marked as "forced".
+                    However, that doesn't mean every player works like this but there is
+                     no other way to reliably work with Forced subtitles where multiple
+                     forced subtitles may be in the output file. Just know what to expect
+                     with "forced" subtitles.
+        """
+        super().__init__(*args, **kwargs)
+        self.codec = codec
+        self.cc = bool(cc)
+        self.sdh = bool(sdh)
+        if self.cc and self.sdh:
+            raise ValueError("A text track cannot be both CC and SDH.")
+        self.forced = bool(forced)
+        if (self.cc or self.sdh) and self.forced:
+            raise ValueError("A text track cannot be CC/SDH as well as Forced.")
+
+    def get_track_name(self) -> Optional[str]:
+        """Return the base Track Name."""
+        track_name = super().get_track_name() or ""
+        flag = self.cc and "CC" or self.sdh and "SDH" or self.forced and "Forced"
+        if flag:
+            if track_name:
+                flag = f" ({flag})"
+            track_name += flag
+        return track_name or None
+
+    @staticmethod
+    def parse(data: bytes, codec: Subtitle.Codec) -> pycaption.CaptionSet:
+        # TODO: Use an "enum" for subtitle codecs
+        if not isinstance(data, bytes):
+            raise ValueError(f"Subtitle data must be parsed as bytes data, not {type(data).__name__}")
+        try:
+            if codec == Subtitle.Codec.fTTML:
+                captions: dict[str, pycaption.CaptionList] = defaultdict(pycaption.CaptionList)
+                for segment in (
+                    Subtitle.parse(box.data, Subtitle.Codec.TimedTextMarkupLang)
+                    for box in MP4.parse_stream(BytesIO(data))
+                    if box.type == b"mdat"
+                ):
+                    for lang in segment.get_languages():
+                        captions[lang].extend(segment.get_captions(lang))
+                captions: pycaption.CaptionSet = pycaption.CaptionSet(captions)
+                return captions
+            if codec == Subtitle.Codec.TimedTextMarkupLang:
+                text = data.decode("utf8").replace("tt:", "")
+                return pycaption.DFXPReader().read(text)
+            if codec == Subtitle.Codec.fVTT:
+                caption_lists: dict[str, pycaption.CaptionList] = defaultdict(pycaption.CaptionList)
+                caption_list, language = Subtitle.merge_segmented_wvtt(data)
+                caption_lists[language] = caption_list
+                caption_set: pycaption.CaptionSet = pycaption.CaptionSet(caption_lists)
+                return caption_set
+            if codec == Subtitle.Codec.WebVTT:
+                text = data.decode("utf8").replace("\r", "").replace("\n\n\n", "\n \n\n").replace("\n\n<", "\n<")
+                captions: pycaption.CaptionSet = pycaption.WebVTTReader().read(text)
+                return captions
+        except pycaption.exceptions.CaptionReadSyntaxError:
+            raise SyntaxError(f"A syntax error has occurred when reading the \"{codec}\" subtitle")
+        except pycaption.exceptions.CaptionReadNoCaptions:
+            return pycaption.CaptionSet({"en": []})
+
+        raise ValueError(f"Unknown Subtitle Format \"{codec}\"...")
+
+    @staticmethod
+    def merge_same_cues(caption_set: pycaption.CaptionSet):
+        """Merge captions with the same timecodes and text as one in-place."""
+        for lang in caption_set.get_languages():
+            captions = caption_set.get_captions(lang)
+            last_caption = None
+            concurrent_captions = pycaption.CaptionList()
+            merged_captions = pycaption.CaptionList()
+            for caption in captions:
+                if last_caption:
+                    if (caption.start, caption.end) == (last_caption.start, last_caption.end):
+                        if caption.get_text() != last_caption.get_text():
+                            concurrent_captions.append(caption)
+                        last_caption = caption
+                        continue
+                    else:
+                        merged_captions.append(pycaption.base.merge(concurrent_captions))
+                concurrent_captions = [caption]
+                last_caption = caption
+
+            if concurrent_captions:
+                merged_captions.append(pycaption.base.merge(concurrent_captions))
+            if merged_captions:
+                caption_set.set_captions(lang, merged_captions)
+
+    @staticmethod
+    def merge_segmented_wvtt(data: bytes, period_start: float = 0.) -> tuple[CaptionList, Optional[str]]:
+        """
+        Convert Segmented DASH WebVTT cues into a pycaption Caption List.
+        Also returns an ISO 639-2 alpha-3 language code if available.
+
+        Code ported originally by xhlove to Python from shaka-player.
+        Has since been improved upon by rlaphoenix using pymp4 and
+        pycaption functions.
+        """
+        captions = CaptionList()
+
+        # init:
+        saw_wvtt_box = False
+        timescale = None
+        language = None
+
+        # media:
+        # > tfhd
+        default_duration = None
+        # > tfdt
+        saw_tfdt_box = False
+        base_time = 0
+        # > trun
+        saw_trun_box = False
+        samples = []
+
+        def flatten_boxes(box: Container) -> Iterable[Container]:
+            for child in box:
+                if hasattr(child, "children"):
+                    yield from flatten_boxes(child.children)
+                    del child["children"]
+                if hasattr(child, "entries"):
+                    yield from flatten_boxes(child.entries)
+                    del child["entries"]
+                # some boxes (mainly within 'entries') uses format not type
+                child["type"] = child.get("type") or child.get("format")
+                yield child
+
+        for box in flatten_boxes(MP4.parse_stream(BytesIO(data))):
+            # init
+            if box.type == b"mdhd":
+                timescale = box.timescale
+                language = box.language
+
+            if box.type == b"wvtt":
+                saw_wvtt_box = True
+
+            # media
+            if box.type == b"styp":
+                # essentially the start of each segment
+                # media var resets
+                # > tfhd
+                default_duration = None
+                # > tfdt
+                saw_tfdt_box = False
+                base_time = 0
+                # > trun
+                saw_trun_box = False
+                samples = []
+
+            if box.type == b"tfhd":
+                if box.flags.default_sample_duration_present:
+                    default_duration = box.default_sample_duration
+
+            if box.type == b"tfdt":
+                saw_tfdt_box = True
+                base_time = box.baseMediaDecodeTime
+
+            if box.type == b"trun":
+                saw_trun_box = True
+                samples = box.sample_info
+
+            if box.type == b"mdat":
+                if not timescale:
+                    raise ValueError("Timescale was not found in the Segmented WebVTT.")
+                if not saw_wvtt_box:
+                    raise ValueError("The WVTT box was not found in the Segmented WebVTT.")
+                if not saw_tfdt_box:
+                    raise ValueError("The TFDT box was not found in the Segmented WebVTT.")
+                if not saw_trun_box:
+                    raise ValueError("The TRUN box was not found in the Segmented WebVTT.")
+
+                vttc_boxes = MP4.parse_stream(BytesIO(box.data))
+                current_time = base_time + period_start
+
+                for sample, vttc_box in zip(samples, vttc_boxes):
+                    duration = sample.sample_duration or default_duration
+                    if sample.sample_composition_time_offsets:
+                        current_time += sample.sample_composition_time_offsets
+
+                    start_time = current_time
+                    end_time = current_time + (duration or 0)
+                    current_time = end_time
+
+                    if vttc_box.type == b"vtte":
+                        # vtte is a vttc that's empty, skip
+                        continue
+
+                    layout: Optional[Layout] = None
+                    nodes: list[CaptionNode] = []
+
+                    for cue_box in MP4.parse_stream(BytesIO(vttc_box.data)):
+                        if cue_box.type == b"vsid":
+                            # this is a V(?) Source ID box, we don't care
+                            continue
+                        cue_data = cue_box.data.decode("utf8")
+                        if cue_box.type == b"sttg":
+                            layout = Layout(webvtt_positioning=cue_data)
+                        elif cue_box.type == b"payl":
+                            nodes.extend([
+                                node
+                                for line in cue_data.split("\n")
+                                for node in [
+                                    CaptionNode.create_text(WebVTTReader()._decode(line)),
+                                    CaptionNode.create_break()
+                                ]
+                            ])
+                            nodes.pop()
+
+                    if nodes:
+                        caption = Caption(
+                            start=start_time * timescale,  # as microseconds
+                            end=end_time * timescale,
+                            nodes=nodes,
+                            layout_info=layout
+                        )
+                        p_caption = captions[-1] if captions else None
+                        if p_caption and caption.start == p_caption.end and str(caption.nodes) == str(p_caption.nodes):
+                            # it's a duplicate, but lets take its end time
+                            p_caption.end = caption.end
+                            continue
+                        captions.append(caption)
+
+        return captions, language
+
+    def strip_hearing_impaired(self) -> None:
+        """
+        Strip captions for hearing impaired (SDH).
+        It uses SubtitleEdit if available, otherwise filter-subs.
+        """
+        if not self.path or not self.path.exists():
+            raise ValueError("You must download the subtitle track first.")
+
+        executable = get_binary_path("SubtitleEdit")
+        if executable:
+            subprocess.run([
+                executable,
+                "/Convert", self.path, "srt",
+                "/overwrite",
+                "/RemoveTextForHI"
+            ], check=True)
+            # Remove UTF-8 Byte Order Marks
+            self.path.write_text(
+                self.path.read_text(encoding="utf-8-sig"),
+                encoding="utf8"
+            )
+        else:
+            sub = Subtitles(self.path)
+            sub.filter(
+                rm_fonts=True,
+                rm_ast=True,
+                rm_music=True,
+                rm_effects=True,
+                rm_names=True,
+                rm_author=True
+            )
+            sub.save()
+
+    def download(self, *args, **kwargs) -> Path:
+        save_path = super().download(*args, **kwargs)
+        if self.codec not in (Subtitle.Codec.SubRip, Subtitle.Codec.SubStationAlphav4):
+            caption_set = self.parse(save_path.read_bytes(), self.codec)
+            self.merge_same_cues(caption_set)
+            srt = pycaption.SRTWriter().write(caption_set)
+            # NowTV sometimes has this, when it isn't, causing mux problems
+            srt = srt.replace("MULTI-LANGUAGE SRT\n", "")
+            save_path.write_text(srt, encoding="utf8")
+            self.codec = Subtitle.Codec.SubRip
+            self.move(self.path.with_suffix(".srt"))
+        return save_path
+
+    def __str__(self) -> str:
+        return " | ".join(filter(bool, [
+            "SUB",
+            f"[{self.codec.value}]",
+            str(self.language),
+            self.get_track_name()
+        ]))
+
+
+__ALL__ = (Subtitle,)

devine/core/tracks/track.py

@@ -0,0 +1,335 @@
+import asyncio
+import logging
+import re
+import subprocess
+from enum import Enum
+from pathlib import Path
+from typing import Any, Callable, Iterable, Optional, Union
+from urllib.parse import urljoin
+
+import m3u8
+import requests
+from langcodes import Language
+
+from devine.core.constants import TERRITORY_MAP
+from devine.core.downloaders import aria2c
+from devine.core.drm import DRM_T
+from devine.core.utilities import get_binary_path
+
+
+class Track:
+    class DRM(Enum):
+        pass
+
+    class Descriptor(Enum):
+        URL = 1  # Direct URL, nothing fancy
+        M3U = 2  # https://en.wikipedia.org/wiki/M3U (and M3U8)
+        MPD = 3  # https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP
+
+    def __init__(
+        self,
+        id_: str,
+        url: Union[str, list[str]],
+        language: Union[Language, str],
+        is_original_lang: bool = False,
+        descriptor: Descriptor = Descriptor.URL,
+        needs_proxy: bool = False,
+        needs_repack: bool = False,
+        drm: Optional[Iterable[DRM_T]] = None,
+        edition: Optional[str] = None,
+        extra: Optional[Any] = None
+    ) -> None:
+        self.id = id_
+        self.url = url
+        # required basic metadata
+        self.language = Language.get(language)
+        self.is_original_lang = bool(is_original_lang)
+        # optional io metadata
+        self.descriptor = descriptor
+        self.needs_proxy = bool(needs_proxy)
+        self.needs_repack = bool(needs_repack)
+        # drm
+        self.drm = drm
+        # extra data
+        self.edition: str = edition
+        self.extra: Any = extra or {}  # allow anything for extra, but default to a dict
+
+        # events
+        self.OnSegmentFilter: Optional[Callable] = None
+        self.OnDownloaded: Optional[Callable] = None
+        self.OnDecrypted: Optional[Callable] = None
+        self.OnRepacked: Optional[Callable] = None
+
+        # should only be set internally
+        self.path: Optional[Path] = None
+
+    def __repr__(self) -> str:
+        return "{name}({items})".format(
+            name=self.__class__.__name__,
+            items=", ".join([f"{k}={repr(v)}" for k, v in self.__dict__.items()])
+        )
+
+    def __eq__(self, other: object) -> bool:
+        return isinstance(other, Track) and self.id == other.id
+
+    def get_track_name(self) -> Optional[str]:
+        """Return the base Track Name. This may be enhanced in sub-classes."""
+        if (self.language.language or "").lower() == (self.language.territory or "").lower():
+            self.language.territory = None  # e.g. en-en, de-DE
+        if self.language.territory == "US":
+            self.language.territory = None
+        reduced = self.language.simplify_script()
+        extra_parts = []
+        if reduced.script is not None:
+            extra_parts.append(reduced.script_name(max_distance=25))
+        if reduced.territory is not None:
+            territory = reduced.territory_name(max_distance=25)
+            extra_parts.append(TERRITORY_MAP.get(territory, territory))
+        return ", ".join(extra_parts) or None
+
+    def get_init_segment(self, session: Optional[requests.Session] = None) -> bytes:
+        """
+        Get the Track's Initial Segment Data Stream.
+        If the Track URL is not detected to be an init segment, it will download
+        up to the first 20,000 (20KB) bytes only.
+        """
+        if not session:
+            session = requests.Session()
+
+        url = None
+        is_init_stream = False
+
+        if self.descriptor == self.Descriptor.M3U:
+            master = m3u8.loads(session.get(self.url).text, uri=self.url)
+            for segment in master.segments:
+                if not segment.init_section:
+                    continue
+                # skip any segment that would be skipped from the download
+                # as we cant consider these a true initial segment
+                if callable(self.OnSegmentFilter) and self.OnSegmentFilter(segment):
+                    continue
+                url = ("" if re.match("^https?://", segment.init_section.uri) else segment.init_section.base_uri)
+                url += segment.init_section.uri
+                is_init_stream = True
+                break
+
+        if not url:
+            url = self.url
+
+        if isinstance(url, list):
+            url = url[0]
+            is_init_stream = True
+
+        if is_init_stream:
+            return session.get(url).content
+
+        # likely a full single-file download, get first 20k bytes
+        with session.get(url, stream=True) as s:
+            # assuming enough to contain the pssh/kid
+            for chunk in s.iter_content(20000):
+                # we only want the first chunk
+                return chunk
+
+    def download(self, out: Path, name_template: str = "{type}_{id}", headers: Optional[dict] = None,
+                 proxy: Optional[str] = None) -> Path:
+        """
+        Download the Track and apply any necessary post-edits like Subtitle conversion.
+
+        Parameters:
+            out: Output Directory Path for the downloaded track.
+            name_template: Override the default filename template.
+                Must contain both `{type}` and `{id}` variables.
+            headers: Headers to use when downloading.
+            proxy: Proxy to use when downloading.
+
+        Returns:
+            Where the file was saved, as a Path object.
+        """
+        if out.is_file():
+            raise ValueError("Path must be to a directory and not a file")
+
+        log = logging.getLogger("download")
+
+        out.mkdir(parents=True, exist_ok=True)
+
+        file_name = name_template.format(
+            type=self.__class__.__name__,
+            id=self.id
+        )
+
+        # we must use .mp4 on tracks:
+        # - as shaka-packager expects mp4 input and mp4 output
+        # - and mkvtoolnix would try to parse the file in raw-bitstream
+        save_path = (out / file_name).with_suffix(".mp4")
+        if self.__class__.__name__ == "Subtitle":
+            save_path = save_path.with_suffix(f".{self.codec.extension}")
+
+        # these would be files like .decrypted, .repack and such.
+        # we cannot trust that these files were not interrupted while writing to disc
+        # lets just delete them before re-attempting a download
+        for existing_file in save_path.parent.glob(f"{save_path.stem}.*{save_path.suffix}"):
+            existing_file.unlink()
+        save_path.with_suffix(".srt").unlink(missing_ok=True)
+
+        if self.descriptor == self.Descriptor.M3U:
+            master = m3u8.loads(
+                requests.get(
+                    self.url,
+                    headers=headers,
+                    proxies={"all": proxy} if self.needs_proxy and proxy else None
+                ).text,
+                uri=self.url
+            )
+
+            if not master.segments:
+                raise ValueError("Track URI (an M3U8) has no segments...")
+
+            if all(segment.uri == master.segments[0].uri for segment in master.segments):
+                # all segments use the same file, presumably an EXT-X-BYTERANGE M3U (FUNI)
+                # TODO: This might be a risky way to deal with these kinds of Playlists
+                #       What if there's an init section, or one segment is reusing a byte-range
+                segment = master.segments[0]
+                if not re.match("^https?://", segment.uri):
+                    segment.uri = urljoin(segment.base_uri, segment.uri)
+                self.url = segment.uri
+                self.descriptor = self.Descriptor.URL
+            else:
+                has_init = False
+                segments = []
+                for segment in master.segments:
+                    # merge base uri with uri where needed in both normal and init segments
+                    if not re.match("^https?://", segment.uri):
+                        segment.uri = segment.base_uri + segment.uri
+                    if segment.init_section and not re.match("^https?://", segment.init_section.uri):
+                        segment.init_section.uri = segment.init_section.base_uri + segment.init_section.uri
+
+                    if segment.discontinuity:
+                        has_init = False
+
+                    # skip segments we don't want to download (e.g., bumpers, dub cards)
+                    if callable(self.OnSegmentFilter) and self.OnSegmentFilter(segment):
+                        continue
+
+                    if segment.init_section and not has_init:
+                        segments.append(segment.init_section.uri)
+                        has_init = True
+                    segments.append(segment.uri)
+                self.url = list(dict.fromkeys(segments))
+
+        is_segmented = isinstance(self.url, list) and len(self.url) > 1
+        segments_dir = save_path.with_name(save_path.name + "_segments")
+
+        attempts = 1
+        while True:
+            try:
+                asyncio.run(aria2c(
+                    self.url,
+                    [save_path, segments_dir][is_segmented],
+                    headers,
+                    proxy if self.needs_proxy else None
+                ))
+                break
+            except subprocess.CalledProcessError:
+                log.info(f" - Download attempt {attempts} failed, {['retrying', 'stopping'][attempts == 3]}...")
+                if attempts == 3:
+                    raise
+                attempts += 1
+
+        if is_segmented:
+            # merge the segments together
+            with open(save_path, "wb") as f:
+                for file in sorted(segments_dir.iterdir()):
+                    data = file.read_bytes()
+                    # Apple TV+ needs this done to fix audio decryption
+                    data = re.sub(b"(tfhd\x00\x02\x00\x1a\x00\x00\x00\x01\x00\x00\x00)\x02", b"\\g<1>\x01", data)
+                    f.write(data)
+                    file.unlink()  # delete, we don't need it anymore
+            segments_dir.rmdir()
+
+        self.path = save_path
+
+        if self.path.stat().st_size <= 3:  # Empty UTF-8 BOM == 3 bytes
+            raise IOError(
+                "Download failed, the downloaded file is empty. "
+                f"This {'was' if self.needs_proxy else 'was not'} downloaded with a proxy." +
+                (
+                    " Perhaps you need to set `needs_proxy` as True to use the proxy for this track."
+                    if not self.needs_proxy else ""
+                )
+            )
+
+        return self.path
+
+    def delete(self) -> None:
+        if self.path:
+            self.path.unlink()
+            self.path = None
+
+    def repackage(self) -> None:
+        if not self.path or not self.path.exists():
+            raise ValueError("Cannot repackage a Track that has not been downloaded.")
+
+        executable = get_binary_path("ffmpeg")
+        if not executable:
+            raise EnvironmentError("FFmpeg executable \"ffmpeg\" was not found but is required for this call.")
+
+        repacked_path = self.path.with_suffix(f".repack{self.path.suffix}")
+
+        def _ffmpeg(extra_args: list[str] = None):
+            subprocess.run(
+                [
+                    executable, "-hide_banner",
+                    "-loglevel", "error",
+                    "-i", self.path,
+                    *(extra_args or []),
+                    # Following are very important!
+                    "-map_metadata", "-1",  # don't transfer metadata to output file
+                    "-fflags", "bitexact",  # only have minimal tag data, reproducible mux
+                    "-codec", "copy",
+                    str(repacked_path)
+                ],
+                check=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE
+            )
+
+        try:
+            _ffmpeg()
+        except subprocess.CalledProcessError as e:
+            if b"Malformed AAC bitstream detected" in e.stderr:
+                # e.g., TruTV's dodgy encodes
+                _ffmpeg(["-y", "-bsf:a", "aac_adtstoasc"])
+            else:
+                raise
+
+        self.swap(repacked_path)
+
+    def move(self, target: Union[str, Path]) -> bool:
+        """
+        Move the Track's file from current location, to target location.
+        This will overwrite anything at the target path.
+        """
+        if not self.path:
+            return False
+        target = Path(target)
+        ok = self.path.rename(target).resolve() == target.resolve()
+        if ok:
+            self.path = target
+        return ok
+
+    def swap(self, target: Union[str, Path]) -> bool:
+        """
+        Swaps the Track's file with the Target file. The current Track's file is deleted.
+        Returns False if the Track is not yet downloaded, or the target path does not exist.
+        """
+        target = Path(target)
+        if not target.exists() or not self.path:
+            return False
+        self.path.unlink()
+        ok = target.rename(self.path) == self.path
+        if not ok:
+            return False
+        return self.move(target)
+
+
+__ALL__ = (Track,)

devine/core/tracks/tracks.py

@@ -0,0 +1,354 @@
+from __future__ import annotations
+
+import logging
+import subprocess
+from pathlib import Path
+from typing import Callable, Iterator, Optional, Sequence, Union
+
+from Cryptodome.Random import get_random_bytes
+from langcodes import Language, closest_supported_match
+
+from devine.core.config import config
+from devine.core.constants import LANGUAGE_MAX_DISTANCE, LANGUAGE_MUX_MAP, AnyTrack, TrackT
+from devine.core.tracks.audio import Audio
+from devine.core.tracks.track import Track
+from devine.core.tracks.chapter import Chapter
+from devine.core.tracks.subtitle import Subtitle
+from devine.core.tracks.video import Video
+from devine.core.utilities import sanitize_filename, is_close_match
+from devine.core.utils.collections import as_list, flatten
+
+
+class Tracks:
+    """
+    Video, Audio, Subtitle, and Chapter Track Store.
+    It provides convenience functions for listing, sorting, and selecting tracks.
+    """
+
+    TRACK_ORDER_MAP = {
+        Video: 0,
+        Audio: 1,
+        Subtitle: 2,
+        Chapter: 3
+    }
+
+    def __init__(self, *args: Union[Tracks, list[Track], Track]):
+        self.videos: list[Video] = []
+        self.audio: list[Audio] = []
+        self.subtitles: list[Subtitle] = []
+        self.chapters: list[Chapter] = []
+
+        if args:
+            self.add(args)
+
+    def __iter__(self) -> Iterator[AnyTrack]:
+        return iter(as_list(self.videos, self.audio, self.subtitles))
+
+    def __len__(self) -> int:
+        return len(self.videos) + len(self.audio) + len(self.subtitles)
+
+    def __repr__(self) -> str:
+        return "{name}({items})".format(
+            name=self.__class__.__name__,
+            items=", ".join([f"{k}={repr(v)}" for k, v in self.__dict__.items()])
+        )
+
+    def __str__(self) -> str:
+        rep = {
+            Video: [],
+            Audio: [],
+            Subtitle: [],
+            Chapter: []
+        }
+        tracks = [*list(self), *self.chapters]
+
+        for track in sorted(tracks, key=lambda t: self.TRACK_ORDER_MAP[type(t)]):
+            if not rep[type(track)]:
+                count = sum(type(x) is type(track) for x in tracks)
+                rep[type(track)].append("{count} {type} Track{plural}{colon}".format(
+                    count=count,
+                    type=track.__class__.__name__,
+                    plural="s" if count != 1 else "",
+                    colon=":" if count > 0 else ""
+                ))
+            rep[type(track)].append(str(track))
+
+        for type_ in list(rep):
+            if not rep[type_]:
+                del rep[type_]
+                continue
+            rep[type_] = "\n".join(
+                [rep[type_][0]] +
+                [f"├─ {x}" for x in rep[type_][1:-1]] +
+                [f"└─ {rep[type_][-1]}"]
+            )
+        rep = "\n".join(list(rep.values()))
+
+        return rep
+
+    def exists(self, by_id: Optional[str] = None, by_url: Optional[Union[str, list[str]]] = None) -> bool:
+        """Check if a track already exists by various methods."""
+        if by_id:  # recommended
+            return any(x.id == by_id for x in self)
+        if by_url:
+            return any(x.url == by_url for x in self)
+        return False
+
+    def add(
+        self,
+        tracks: Union[Tracks, Sequence[Union[AnyTrack, Chapter]], Track, Chapter],
+        warn_only: bool = False
+    ) -> None:
+        """Add a provided track to its appropriate array and ensuring it's not a duplicate."""
+        if isinstance(tracks, Tracks):
+            tracks = [*list(tracks), *tracks.chapters]
+
+        duplicates = 0
+        for track in flatten(tracks):
+            if self.exists(by_id=track.id):
+                if not warn_only:
+                    raise ValueError(
+                        "One or more of the provided Tracks is a duplicate. "
+                        "Track IDs must be unique but accurate using static values. The "
+                        "value should stay the same no matter when you request the same "
+                        "content. Use a value that has relation to the track content "
+                        "itself and is static or permanent and not random/RNG data that "
+                        "wont change each refresh or conflict in edge cases."
+                    )
+                duplicates += 1
+                continue
+
+            if isinstance(track, Video):
+                self.videos.append(track)
+            elif isinstance(track, Audio):
+                self.audio.append(track)
+            elif isinstance(track, Subtitle):
+                self.subtitles.append(track)
+            elif isinstance(track, Chapter):
+                self.chapters.append(track)
+            else:
+                raise ValueError("Track type was not set or is invalid.")
+
+        log = logging.getLogger("Tracks")
+
+        if duplicates:
+            log.warning(f" - Found and skipped {duplicates} duplicate tracks...")
+
+    def print(self, level: int = logging.INFO) -> None:
+        """Print the __str__ to log at a specified level."""
+        log = logging.getLogger("Tracks")
+        for line in str(self).splitlines(keepends=False):
+            log.log(level, line)
+
+    def sort_videos(self, by_language: Optional[Sequence[Union[str, Language]]] = None) -> None:
+        """Sort video tracks by bitrate, and optionally language."""
+        if not self.videos:
+            return
+        # bitrate
+        self.videos.sort(
+            key=lambda x: float(x.bitrate or 0.0),
+            reverse=True
+        )
+        # language
+        for language in reversed(by_language or []):
+            if str(language) == "all":
+                language = next((x.language for x in self.videos if x.is_original_lang), "")
+            if not language:
+                continue
+            self.videos.sort(key=lambda x: str(x.language))
+            self.videos.sort(key=lambda x: not is_close_match(language, [x.language]))
+
+    def sort_audio(self, by_language: Optional[Sequence[Union[str, Language]]] = None) -> None:
+        """Sort audio tracks by bitrate, descriptive, and optionally language."""
+        if not self.audio:
+            return
+        # bitrate
+        self.audio.sort(
+            key=lambda x: float(x.bitrate or 0.0),
+            reverse=True
+        )
+        # descriptive
+        self.audio.sort(key=lambda x: str(x.language) if x.descriptive else "")
+        # language
+        for language in reversed(by_language or []):
+            if str(language) == "all":
+                language = next((x.language for x in self.audio if x.is_original_lang), "")
+            if not language:
+                continue
+            self.audio.sort(key=lambda x: str(x.language))
+            self.audio.sort(key=lambda x: not is_close_match(language, [x.language]))
+
+    def sort_subtitles(self, by_language: Optional[Sequence[Union[str, Language]]] = None) -> None:
+        """
+        Sort subtitle tracks by various track attributes to a common P2P standard.
+        You may optionally provide a sequence of languages to prioritize to the top.
+
+        Section Order:
+          - by_language groups prioritized to top, and ascending alphabetically
+          - then rest ascending alphabetically after the prioritized groups
+          (Each section ascending alphabetically, but separated)
+
+        Language Group Order:
+          - Forced
+          - Normal
+          - Hard of Hearing (SDH/CC)
+          (Least to most captions expected in the subtitle)
+        """
+        if not self.subtitles:
+            return
+        # language groups
+        self.subtitles.sort(key=lambda x: str(x.language))
+        self.subtitles.sort(key=lambda x: x.sdh or x.cc)
+        self.subtitles.sort(key=lambda x: x.forced, reverse=True)
+        # sections
+        for language in reversed(by_language or []):
+            if str(language) == "all":
+                language = next((x.language for x in self.subtitles if x.is_original_lang), "")
+            if not language:
+                continue
+            self.subtitles.sort(key=lambda x: is_close_match(language, [x.language]), reverse=True)
+
+    def sort_chapters(self) -> None:
+        """Sort chapter tracks by chapter number."""
+        if not self.chapters:
+            return
+        # number
+        self.chapters.sort(key=lambda x: x.number)
+
+    def select_video(self, x: Callable[[Video], bool]) -> None:
+        self.videos = list(filter(x, self.videos))
+
+    def select_audio(self, x: Callable[[Audio], bool]) -> None:
+        self.audio = list(filter(x, self.audio))
+
+    def select_subtitles(self, x: Callable[[Subtitle], bool]) -> None:
+        self.subtitles = list(filter(x, self.subtitles))
+
+    def with_resolution(self, resolution: int) -> None:
+        if resolution:
+            # Note: Do not merge these list comprehensions. They must be done separately so the results
+            # from the 16:9 canvas check is only used if there's no exact height resolution match.
+            videos_quality = [x for x in self.videos if x.height == resolution]
+            if not videos_quality:
+                videos_quality = [x for x in self.videos if int(x.width * (9 / 16)) == resolution]
+            self.videos = videos_quality
+
+    def export_chapters(self, to_file: Optional[Union[Path, str]] = None) -> str:
+        """Export all chapters in order to a string or file."""
+        self.sort_chapters()
+        data = "\n".join(map(repr, self.chapters))
+        if to_file:
+            to_file = Path(to_file)
+            to_file.parent.mkdir(parents=True, exist_ok=True)
+            to_file.write_text(data, encoding="utf8")
+        return data
+
+    @staticmethod
+    def select_per_language(tracks: list[TrackT], languages: list[str]) -> list[TrackT]:
+        """
+        Enumerates and return the first Track per language.
+        You should sort the list so the wanted track is closer to the start of the list.
+        """
+        tracks_ = []
+        for language in languages:
+            match = closest_supported_match(language, [str(x.language) for x in tracks], LANGUAGE_MAX_DISTANCE)
+            if match:
+                tracks_.append(next(x for x in tracks if str(x.language) == match))
+        return tracks_
+
+    def mux(self, title: str, delete: bool = True) -> tuple[Path, int]:
+        """
+        Takes the Video, Audio and Subtitle Tracks, and muxes them into an MKV file.
+        It will attempt to detect Forced/Default tracks, and will try to parse the language codes of the Tracks
+        """
+        cl = [
+            "mkvmerge",
+            "--no-date",  # remove dates from the output for security
+        ]
+
+        if config.muxing.get("set_title", True):
+            cl.extend(["--title", title])
+
+        for i, vt in enumerate(self.videos):
+            if not vt.path or not vt.path.exists():
+                raise ValueError("Video Track must be downloaded before muxing...")
+            cl.extend([
+                "--language", "0:{}".format(LANGUAGE_MUX_MAP.get(
+                    str(vt.language), str(vt.language)
+                )),
+                "--default-track", f"0:{i == 0}",
+                "--original-flag", f"0:{vt.is_original_lang}",
+                "--compression", "0:none",  # disable extra compression
+                "(", str(vt.path), ")"
+            ])
+
+        for i, at in enumerate(self.audio):
+            if not at.path or not at.path.exists():
+                raise ValueError("Audio Track must be downloaded before muxing...")
+            cl.extend([
+                "--track-name", f"0:{at.get_track_name() or ''}",
+                "--language", "0:{}".format(LANGUAGE_MUX_MAP.get(
+                    str(at.language), str(at.language)
+                )),
+                "--default-track", f"0:{i == 0}",
+                "--visual-impaired-flag", f"0:{at.descriptive}",
+                "--original-flag", f"0:{at.is_original_lang}",
+                "--compression", "0:none",  # disable extra compression
+                "(", str(at.path), ")"
+            ])
+
+        for st in self.subtitles:
+            if not st.path or not st.path.exists():
+                raise ValueError("Text Track must be downloaded before muxing...")
+            default = bool(self.audio and is_close_match(st.language, [self.audio[0].language]) and st.forced)
+            cl.extend([
+                "--track-name", f"0:{st.get_track_name() or ''}",
+                "--language", "0:{}".format(LANGUAGE_MUX_MAP.get(
+                    str(st.language), str(st.language)
+                )),
+                "--sub-charset", "0:UTF-8",
+                "--forced-track", f"0:{st.forced}",
+                "--default-track", f"0:{default}",
+                "--hearing-impaired-flag", f"0:{st.sdh}",
+                "--original-flag", f"0:{st.is_original_lang}",
+                "--compression", "0:none",  # disable extra compression (probably zlib)
+                "(", str(st.path), ")"
+            ])
+
+        if self.chapters:
+            chapters_path = config.directories.temp / config.filenames.chapters.format(
+                title=sanitize_filename(title),
+                random=get_random_bytes(16).hex()
+            )
+            self.export_chapters(chapters_path)
+            cl.extend(["--chapters", str(chapters_path)])
+        else:
+            chapters_path = None
+
+        output_path = (
+            self.videos[0].path.with_suffix(".muxed.mkv") if self.videos else
+            self.audio[0].path.with_suffix(".muxed.mka") if self.audio else
+            self.subtitles[0].path.with_suffix(".muxed.mks") if self.subtitles else
+            chapters_path.with_suffix(".muxed.mkv") if self.chapters else
+            None
+        )
+        if not output_path:
+            raise ValueError("No tracks provided, at least one track must be provided.")
+
+        # let potential failures go to caller, caller should handle
+        try:
+            p = subprocess.run([
+                *cl,
+                "--output", str(output_path)
+            ])
+            return output_path, p.returncode
+        finally:
+            if chapters_path:
+                # regardless of delete param, we delete as it's a file we made during muxing
+                chapters_path.unlink()
+            if delete:
+                for track in self:
+                    track.delete()
+
+
+__ALL__ = (Tracks,)

devine/core/tracks/video.py

@@ -0,0 +1,333 @@
+from __future__ import annotations
+
+import logging
+import math
+import re
+import subprocess
+from enum import Enum
+from pathlib import Path
+from typing import Any, Optional, Union
+
+from langcodes import Language
+
+from devine.core.config import config
+from devine.core.tracks.track import Track
+from devine.core.tracks.subtitle import Subtitle
+from devine.core.utilities import get_binary_path, get_boxes, FPS
+
+
+class Video(Track):
+    class Codec(str, Enum):
+        AVC = "H.264"
+        HEVC = "H.265"
+        VC1 = "VC-1"
+        VP8 = "VP8"
+        VP9 = "VP9"
+        AV1 = "AV1"
+
+        @property
+        def extension(self) -> str:
+            return self.value.lower().replace(".", "").replace("-", "")
+
+        @staticmethod
+        def from_mime(mime: str) -> Video.Codec:
+            mime = mime.lower().strip().split(".")[0]
+            if mime in (
+                "avc1", "avc2", "avc3",
+                "dva1", "dvav",  # Dolby Vision
+            ):
+                return Video.Codec.AVC
+            if mime in (
+                "hev1", "hev2", "hev3", "hvc1", "hvc2", "hvc3",
+                "dvh1", "dvhe",  # Dolby Vision
+                "lhv1", "lhe1",  # Layered
+            ):
+                return Video.Codec.HEVC
+            if mime == "vc-1":
+                return Video.Codec.VC1
+            if mime in ("vp08", "vp8"):
+                return Video.Codec.VP8
+            if mime in ("vp09", "vp9"):
+                return Video.Codec.VP9
+            if mime == "av01":
+                return Video.Codec.AV1
+            raise ValueError(f"The MIME '{mime}' is not a supported Video Codec")
+
+        @staticmethod
+        def from_codecs(codecs: str) -> Video.Codec:
+            for codec in codecs.lower().split(","):
+                codec = codec.strip()
+                mime = codec.split(".")[0]
+                try:
+                    return Video.Codec.from_mime(mime)
+                except ValueError:
+                    pass
+            raise ValueError(f"No MIME types matched any supported Video Codecs in '{codecs}'")
+
+        @staticmethod
+        def from_netflix_profile(profile: str) -> Video.Codec:
+            profile = profile.lower().strip()
+            if profile.startswith("playready-h264"):
+                return Video.Codec.AVC
+            if profile.startswith("hevc"):
+                return Video.Codec.HEVC
+            if profile.startswith("vp9"):
+                return Video.Codec.VP9
+            if profile.startswith("av1"):
+                return Video.Codec.AV1
+            raise ValueError(f"The Content Profile '{profile}' is not a supported Video Codec")
+
+    class Range(str, Enum):
+        SDR = "SDR"        # No Dynamic Range
+        HLG = "HLG"        # https://en.wikipedia.org/wiki/Hybrid_log%E2%80%93gamma
+        HDR10 = "HDR10"    # https://en.wikipedia.org/wiki/HDR10
+        HDR10P = "HDR10+"  # https://en.wikipedia.org/wiki/HDR10%2B
+        DV = "DV"          # https://en.wikipedia.org/wiki/Dolby_Vision
+
+        @staticmethod
+        def from_cicp(primaries: int, transfer: int, matrix: int) -> Video.Range:
+            """
+            ISO/IEC 23001-8 Coding-independent code points to Video Range.
+            Sources for Code points:
+            https://www.itu.int/rec/dologin_pub.asp?lang=e&id=T-REC-H.Sup19-201903-S!!PDF-E&type=items
+            """
+            class Primaries(Enum):
+                Unspecified = 0
+                BT_709 = 1
+                BT_601_625 = 5
+                BT_601_525 = 6
+                BT_2020 = 9  # BT.2100 shares the same CP
+
+            class Transfer(Enum):
+                Unspecified = 0
+                SDR_BT_709 = 1
+                SDR_BT_601_625 = 5
+                SDR_BT_601_525 = 6
+                SDR_BT_2020 = 14
+                SDR_BT_2100 = 15
+                PQ = 16
+                HLG = 18
+
+            class Matrix(Enum):
+                RGB = 0
+                YCbCr_BT_709 = 1
+                YCbCr_BT_601_625 = 5
+                YCbCr_BT_601_525 = 6
+                YCbCr_BT_2020 = 9  # YCbCr BT.2100 shares the same CP
+
+            primaries = Primaries(primaries)
+            transfer = Transfer(transfer)
+            matrix = Matrix(matrix)
+
+            # primaries and matrix does not strictly correlate to a range
+
+            if (primaries, transfer, matrix) == (0, 0, 0):
+                return Video.Range.SDR
+
+            if primaries in (Primaries.BT_601_525, Primaries.BT_601_625):
+                return Video.Range.SDR
+
+            if transfer == Transfer.PQ:
+                return Video.Range.HDR10
+            elif transfer == Transfer.HLG:
+                return Video.Range.HLG
+            else:
+                return Video.Range.SDR
+
+        @staticmethod
+        def from_m3u_range_tag(tag: str) -> Video.Range:
+            tag = (tag or "").upper().replace('"', '').strip()
+            if not tag or tag == "SDR":
+                return Video.Range.SDR
+            elif tag == "PQ":
+                return Video.Range.HDR10  # technically could be any PQ-transfer range
+            elif tag == "HLG":
+                return Video.Range.HLG
+            # for some reason there's no Dolby Vision info tag
+            raise ValueError(f"The M3U Range Tag '{tag}' is not a supported Video Range")
+
+    def __init__(self, *args: Any, codec: Video.Codec, range_: Video.Range, bitrate: Union[str, int, float],
+                 width: int, height: int, fps: Optional[Union[str, int, float]] = None, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        # required
+        self.codec = codec
+        self.range = range_ or Video.Range.SDR
+        self.bitrate = int(math.ceil(float(bitrate))) if bitrate else None
+        self.width = int(width)
+        self.height = int(height)
+        # optional
+        self.fps = FPS.parse(str(fps)) if fps else None
+
+    def __str__(self) -> str:
+        fps = f"{self.fps:.3f}" if self.fps else "Unknown"
+        return " | ".join(filter(bool, [
+            "VID",
+            f"[{self.codec.value}, {self.range.name}]",
+            str(self.language),
+            f"{self.width}x{self.height} @ {self.bitrate // 1000 if self.bitrate else '?'} kb/s, {fps} FPS",
+            self.edition
+        ]))
+
+    def change_color_range(self, range_: int) -> None:
+        """Change the Video's Color Range to Limited (0) or Full (1)."""
+        if not self.path or not self.path.exists():
+            raise ValueError("Cannot repackage a Track that has not been downloaded.")
+
+        executable = get_binary_path("ffmpeg")
+        if not executable:
+            raise EnvironmentError("FFmpeg executable \"ffmpeg\" was not found but is required for this call.")
+
+        filter_key = {
+            Video.Codec.AVC: "h264_metadata",
+            Video.Codec.HEVC: "hevc_metadata"
+        }[self.codec]
+
+        changed_path = self.path.with_suffix(f".range{range_}{self.path.suffix}")
+        subprocess.run([
+            executable, "-hide_banner",
+            "-loglevel", "panic",
+            "-i", self.path,
+            "-codec", "copy",
+            "-bsf:v", f"{filter_key}=video_full_range_flag={range_}",
+            str(changed_path)
+        ], check=True)
+
+        self.swap(changed_path)
+
+    def ccextractor(
+        self, track_id: Any, out_path: Union[Path, str], language: Language, original: bool = False
+    ) -> Optional[Subtitle]:
+        """Return a TextTrack object representing CC track extracted by CCExtractor."""
+        if not self.path:
+            raise ValueError("You must download the track first.")
+
+        executable = get_binary_path("ccextractor", "ccextractorwin", "ccextractorwinfull")
+        if not executable:
+            raise EnvironmentError("ccextractor executable was not found.")
+
+        out_path = Path(out_path)
+
+        try:
+            subprocess.run([
+                executable,
+                "-trim", "-noru", "-ru1",
+                self.path, "-o", out_path
+            ], check=True)
+        except subprocess.CalledProcessError as e:
+            out_path.unlink(missing_ok=True)
+            if not e.returncode == 10:  # No captions found
+                raise
+
+        if out_path.exists():
+            if out_path.stat().st_size <= 3:
+                # An empty UTF-8 file with BOM is 3 bytes.
+                # If the subtitle file is empty, mkvmerge will fail to mux.
+                out_path.unlink()
+                return None
+            cc_track = Subtitle(
+                id_=track_id,
+                url="",  # doesn't need to be downloaded
+                codec=Subtitle.Codec.SubRip,
+                language=language,
+                is_original_lang=original,
+                cc=True
+            )
+            cc_track.path = out_path
+            return cc_track
+
+        return None
+
+    def extract_c608(self) -> list[Subtitle]:
+        """
+        Extract Apple-Style c608 box (CEA-608) subtitle using ccextractor.
+
+        This isn't much more than a wrapper to the track.ccextractor function.
+        All this does, is actually check if a c608 box exists and only if so
+        does it actually call ccextractor.
+
+        Even though there is a possibility of more than one c608 box, only one
+        can actually be extracted. Not only that but it's very possible this
+        needs to be done before any decryption as the decryption may destroy
+        some of the metadata.
+
+        TODO: Need a test file with more than one c608 box to add support for
+              more than one CEA-608 extraction.
+        """
+        if not self.path:
+            raise ValueError("You must download the track first.")
+        with self.path.open("rb") as f:
+            # assuming 20KB is enough to contain the c608 box.
+            # ffprobe will fail, so a c608 box check must be done.
+            c608_count = len(list(get_boxes(f.read(20000), b"c608")))
+        if c608_count > 0:
+            # TODO: Figure out the real language, it might be different
+            #       CEA-608 boxes doesnt seem to carry language information :(
+            # TODO: Figure out if the CC language is original lang or not.
+            #       Will need to figure out above first to do so.
+            track_id = f"ccextractor-{self.id}"
+            cc_lang = self.language
+            cc_track = self.ccextractor(
+                track_id=track_id,
+                out_path=config.directories.temp / config.filenames.subtitle.format(
+                    id=track_id,
+                    language=cc_lang
+                ),
+                language=cc_lang,
+                original=False
+            )
+            if not cc_track:
+                return []
+            return [cc_track]
+        return []
+
+    def remove_eia_cc(self) -> bool:
+        """
+        Remove EIA-CC data from Bitstream while keeping SEI data.
+
+        This works by removing all NAL Unit's with the Type of 6 from the bistream
+        and then re-adding SEI data (effectively a new NAL Unit with just the SEI data).
+        Only bitstreams with x264 encoding information is currently supported due to the
+        obscurity on the MDAT mp4 box structure. Therefore, we need to use hacky regex.
+        """
+        if not self.path or not self.path.exists():
+            raise ValueError("Cannot clean a Track that has not been downloaded.")
+
+        executable = get_binary_path("ffmpeg")
+        if not executable:
+            raise EnvironmentError("FFmpeg executable \"ffmpeg\" was not found but is required for this call.")
+
+        log = logging.getLogger("x264-clean")
+        log.info("Removing EIA-CC from Video Track with FFMPEG")
+
+        with open(self.path, "rb") as f:
+            file = f.read(60000)
+
+        x264 = re.search(br"(.{16})(x264)", file)
+        if not x264:
+            log.info(" - No x264 encode settings were found, unsupported...")
+            return False
+
+        uuid = x264.group(1).hex()
+        i = file.index(b"x264")
+        encoding_settings = file[i: i + file[i:].index(b"\x00")].replace(b":", br"\\:").replace(b",", br"\,").decode()
+
+        cleaned_path = self.path.with_suffix(f".cleaned{self.path.suffix}")
+        subprocess.run([
+            executable, "-hide_banner",
+            "-loglevel", "panic",
+            "-i", self.path,
+            "-map_metadata", "-1",
+            "-fflags", "bitexact",
+            "-bsf:v", f"filter_units=remove_types=6,h264_metadata=sei_user_data={uuid}+{encoding_settings}",
+            "-codec", "copy",
+            str(cleaned_path)
+        ], check=True)
+
+        log.info(" + Removed")
+
+        self.swap(cleaned_path)
+
+        return True
+
+
+__ALL__ = (Video,)

devine/core/utilities.py

@@ -0,0 +1,205 @@
+import ast
+import contextlib
+import importlib.util
+import re
+import shutil
+import sys
+from urllib.parse import urlparse
+
+import pproxy
+import requests
+import unicodedata
+from pathlib import Path
+from types import ModuleType
+from typing import Optional, Union, Sequence, AsyncIterator
+
+from langcodes import Language, closest_match
+from pymp4.parser import Box
+from unidecode import unidecode
+
+from devine.core.config import config
+from devine.core.constants import LANGUAGE_MAX_DISTANCE
+
+
+def import_module_by_path(path: Path) -> ModuleType:
+    """Import a Python file by Path as a Module."""
+    if not path:
+        raise ValueError("Path must be provided")
+    if not isinstance(path, Path):
+        raise TypeError(f"Expected path to be a {Path}, not {path!r}")
+    if not path.exists():
+        raise ValueError("Path does not exist")
+
+    # compute package hierarchy for relative import support
+    if path.is_relative_to(config.directories.core_dir):
+        name = []
+        _path = path.parent
+        while _path.stem != config.directories.core_dir.stem:
+            name.append(_path.stem)
+            _path = _path.parent
+        name = ".".join([config.directories.core_dir.stem] + name[::-1])
+    else:
+        # is outside the src package
+        if str(path.parent.parent) not in sys.path:
+            sys.path.insert(1, str(path.parent.parent))
+        name = path.parent.stem
+
+    spec = importlib.util.spec_from_file_location(name, path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+
+    return module
+
+
+def get_binary_path(*names: str) -> Optional[Path]:
+    """Find the path of the first found binary name."""
+    for name in names:
+        path = shutil.which(name)
+        if path:
+            return Path(path)
+    return None
+
+
+def sanitize_filename(filename: str, spacer: str = ".") -> str:
+    """
+    Sanitize a string to be filename safe.
+
+    The spacer is safer to be a '.' for older DDL and p2p sharing spaces.
+    This includes web-served content via direct links and such.
+    """
+    # replace all non-ASCII characters with ASCII equivalents
+    filename = unidecode(filename)
+
+    # remove or replace further characters as needed
+    filename = "".join(c for c in filename if unicodedata.category(c) != "Mn")  # hidden characters
+    filename = filename.\
+        replace("/", " & ").\
+        replace(";", " & ")  # e.g. multi-episode filenames
+    filename = re.sub(rf"[:; ]", spacer, filename)  # structural chars to (spacer)
+    filename = re.sub(r"[\\*!?¿,'\"()<>|$#]", "", filename)  # not filename safe chars
+    filename = re.sub(rf"[{spacer}]{{2,}}", spacer, filename)  # remove extra neighbouring (spacer)s
+
+    return filename
+
+
+def is_close_match(language: Union[str, Language], languages: Sequence[Union[str, Language, None]]) -> bool:
+    """Check if a language is a close match to any of the provided languages."""
+    languages = [x for x in languages if x]
+    if not languages:
+        return False
+    return closest_match(language, list(map(str, languages)))[1] <= LANGUAGE_MAX_DISTANCE
+
+
+def get_boxes(data: bytes, box_type: bytes, as_bytes: bool = False) -> Box:
+    """Scan a byte array for a wanted box, then parse and yield each find."""
+    # using slicing to get to the wanted box is done because parsing the entire box and recursively
+    # scanning through each box and its children often wouldn't scan far enough to reach the wanted box.
+    # since it doesnt care what child box the wanted box is from, this works fine.
+    if not isinstance(data, (bytes, bytearray)):
+        raise ValueError("data must be bytes")
+    while True:
+        try:
+            index = data.index(box_type)
+        except ValueError:
+            break
+        if index < 0:
+            break
+        if index > 4:
+            index -= 4  # size is before box type and is 4 bytes long
+        data = data[index:]
+        try:
+            box = Box.parse(data)
+        except IOError:
+            # TODO: Does this miss any data we may need?
+            break
+        if as_bytes:
+            box = Box.build(box)
+        yield box
+
+
+def ap_case(text: str, keep_spaces: bool = False, stop_words: tuple[str] = None) -> str:
+    """
+    Convert a string to title case using AP/APA style.
+    Based on https://github.com/words/ap-style-title-case
+
+    Parameters:
+        text: The text string to title case with AP/APA style.
+        keep_spaces: To keep the original whitespace, or to just use a normal space.
+            This would only be needed if you have special whitespace between words.
+        stop_words: Override the default stop words with your own ones.
+    """
+    if not text:
+        return ""
+
+    if not stop_words:
+        stop_words = ("a", "an", "and", "at", "but", "by", "for", "in", "nor",
+                      "of", "on", "or", "so", "the", "to", "up", "yet")
+
+    splitter = re.compile(r"(\s+|[-‑–—])")
+    words = splitter.split(text)
+
+    return "".join([
+        [" ", word][keep_spaces] if re.match(r"\s+", word) else
+        word if splitter.match(word) else
+        word.lower() if i != 0 and i != len(words) - 1 and word.lower() in stop_words else
+        word.capitalize()
+        for i, word in enumerate(words)
+    ])
+
+
+def get_ip_info(session: Optional[requests.Session] = None) -> dict:
+    """
+    Use ipinfo.io to get IP location information.
+
+    If you provide a Requests Session with a Proxy, that proxies IP information
+    is what will be returned.
+    """
+    return (session or requests.Session()).get("https://ipinfo.io/json").json()
+
+
+@contextlib.asynccontextmanager
+async def start_pproxy(proxy: str) -> AsyncIterator[str]:
+    proxy = urlparse(proxy)
+
+    scheme = {
+        "https": "http+ssl",
+        "socks5h": "socks"
+    }.get(proxy.scheme, proxy.scheme)
+
+    remote_server = f"{scheme}://{proxy.hostname}"
+    if proxy.port:
+        remote_server += f":{proxy.port}"
+    if proxy.username or proxy.password:
+        remote_server += "#"
+    if proxy.username:
+        remote_server += proxy.username
+    if proxy.password:
+        remote_server += f":{proxy.password}"
+
+    server = pproxy.Server("http://localhost:0")  # random port
+    remote = pproxy.Connection(remote_server)
+    handler = await server.start_server({"rserver": [remote]})
+
+    try:
+        port = handler.sockets[0].getsockname()[1]
+        yield f"http://localhost:{port}"
+    finally:
+        handler.close()
+        await handler.wait_closed()
+
+
+class FPS(ast.NodeVisitor):
+    def visit_BinOp(self, node: ast.BinOp) -> float:
+        if isinstance(node.op, ast.Div):
+            return self.visit(node.left) / self.visit(node.right)
+        raise ValueError(f"Invalid operation: {node.op}")
+
+    def visit_Num(self, node: ast.Num) -> complex:
+        return node.n
+
+    def visit_Expr(self, node: ast.Expr) -> float:
+        return self.visit(node.value)
+
+    @classmethod
+    def parse(cls, expr: str) -> float:
+        return cls().visit(ast.parse(expr).body[0])

devine/core/utils/atomicsql.py

@@ -0,0 +1,105 @@
+"""
+AtomicSQL - Race-condition and Threading safe SQL Database Interface.
+Copyright (C) 2020-2023 rlaphoenix
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import os
+import sqlite3
+import time
+from threading import Lock
+from typing import Any, Callable, Union
+
+import pymysql.cursors
+
+Connections = Union[sqlite3.Connection, pymysql.connections.Connection]
+Cursors = Union[sqlite3.Cursor, pymysql.cursors.Cursor]
+
+
+class AtomicSQL:
+    """
+    Race-condition and Threading safe SQL Database Interface.
+    """
+
+    def __init__(self) -> None:
+        self.master_lock = Lock()  # prevents race condition
+        self.db: dict[bytes, Connections] = {}  # used to hold the database connections and commit changes and such
+        self.cursor: dict[bytes, Cursors] = {}  # used to execute queries and receive results
+        self.session_lock: dict[bytes, Lock] = {}  # like master_lock, but per-session
+
+    def load(self, connection: Connections) -> bytes:
+        """
+        Store SQL Connection object and return a reference ticket.
+        :param connection: SQLite3 or pymysql Connection object.
+        :returns: Session ID in which the database connection is referenced with.
+        """
+        self.master_lock.acquire()
+        try:
+            # obtain a unique cryptographically random session_id
+            session_id = None
+            while not session_id or session_id in self.db:
+                session_id = os.urandom(16)
+            self.db[session_id] = connection
+            self.cursor[session_id] = self.db[session_id].cursor()
+            self.session_lock[session_id] = Lock()
+            return session_id
+        finally:
+            self.master_lock.release()
+
+    def safe_execute(self, session_id: bytes, action: Callable) -> Any:
+        """
+        Execute code on the Database Connection in a race-condition safe way.
+        :param session_id: Database Connection's Session ID.
+        :param action: Function or lambda in which to execute, it's provided `db` and `cursor` arguments.
+        :returns: Whatever `action` returns.
+        """
+        if session_id not in self.db:
+            raise ValueError(f"Session ID {session_id!r} is invalid.")
+        self.master_lock.acquire()
+        self.session_lock[session_id].acquire()
+        try:
+            failures = 0
+            while True:
+                try:
+                    action(
+                        db=self.db[session_id],
+                        cursor=self.cursor[session_id]
+                    )
+                    break
+                except sqlite3.OperationalError as e:
+                    failures += 1
+                    delay = 3 * failures
+                    print(f"AtomicSQL.safe_execute failed, {e}, retrying in {delay} seconds...")
+                    time.sleep(delay)
+                if failures == 10:
+                    raise ValueError("AtomicSQL.safe_execute failed too many time's. Aborting.")
+            return self.cursor[session_id]
+        finally:
+            self.session_lock[session_id].release()
+            self.master_lock.release()
+
+    def commit(self, session_id: bytes) -> bool:
+        """
+        Commit changes to the Database Connection immediately.
+        This isn't necessary to be run every time you make changes, just ensure it's run
+        at least before termination.
+        :param session_id: Database Connection's Session ID.
+        :returns: True if it committed.
+        """
+        self.safe_execute(
+            session_id,
+            lambda db, cursor: db.commit()
+        )
+        return True  # todo ; actually check if db.commit worked

devine/core/utils/click_types.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import re
+from typing import Optional, Union
+
+import click
+from pywidevine.cdm import Cdm as WidevineCdm
+
+
+class ContextData:
+    def __init__(self, config: dict, cdm: WidevineCdm, proxy_providers: list, profile: Optional[str] = None):
+        self.config = config
+        self.cdm = cdm
+        self.proxy_providers = proxy_providers
+        self.profile = profile
+
+
+class SeasonRange(click.ParamType):
+    name = "ep_range"
+
+    MIN_EPISODE = 0
+    MAX_EPISODE = 999
+
+    def parse_tokens(self, *tokens: str) -> list[str]:
+        """
+        Parse multiple tokens or ranged tokens as '{s}x{e}' strings.
+
+        Supports exclusioning by putting a `-` before the token.
+
+        Example:
+            >>> sr = SeasonRange()
+            >>> sr.parse_tokens("S01E01")
+            ["1x1"]
+            >>> sr.parse_tokens("S02E01", "S02E03-S02E05")
+            ["2x1", "2x3", "2x4", "2x5"]
+            >>> sr.parse_tokens("S01-S05", "-S03", "-S02E01")
+            ["1x0", "1x1", ..., "2x0", (...), "2x2", (...), "4x0", ..., "5x0", ...]
+        """
+        if len(tokens) == 0:
+            return []
+        computed: list = []
+        exclusions: list = []
+        for token in tokens:
+            exclude = token.startswith("-")
+            if exclude:
+                token = token[1:]
+            parsed = [
+                re.match(r"^S(?P<season>\d+)(E(?P<episode>\d+))?$", x, re.IGNORECASE)
+                for x in re.split(r"[:-]", token)
+            ]
+            if len(parsed) > 2:
+                self.fail(f"Invalid token, only a left and right range is acceptable: {token}")
+            if len(parsed) == 1:
+                parsed.append(parsed[0])
+            if any(x is None for x in parsed):
+                self.fail(f"Invalid token, syntax error occurred: {token}")
+            from_season, from_episode = [
+                int(v) if v is not None else self.MIN_EPISODE
+                for k, v in parsed[0].groupdict().items() if parsed[0]  # type: ignore[union-attr]
+            ]
+            to_season, to_episode = [
+                int(v) if v is not None else self.MAX_EPISODE
+                for k, v in parsed[1].groupdict().items() if parsed[1]  # type: ignore[union-attr]
+            ]
+            if from_season > to_season:
+                self.fail(f"Invalid range, left side season cannot be bigger than right side season: {token}")
+            if from_season == to_season and from_episode > to_episode:
+                self.fail(f"Invalid range, left side episode cannot be bigger than right side episode: {token}")
+            for s in range(from_season, to_season + 1):
+                for e in range(
+                    from_episode if s == from_season else 0,
+                    (self.MAX_EPISODE if s < to_season else to_episode) + 1
+                ):
+                    (computed if not exclude else exclusions).append(f"{s}x{e}")
+        for exclusion in exclusions:
+            if exclusion in computed:
+                computed.remove(exclusion)
+        return list(set(computed))
+
+    def convert(
+        self, value: str, param: Optional[click.Parameter] = None, ctx: Optional[click.Context] = None
+    ) -> list[str]:
+        return self.parse_tokens(*re.split(r"\s*[,;]\s*", value))
+
+
+class LanguageRange(click.ParamType):
+    name = "lang_range"
+
+    def convert(
+        self, value: Union[str, list], param: Optional[click.Parameter] = None, ctx: Optional[click.Context] = None
+    ) -> list[str]:
+        if isinstance(value, list):
+            return value
+        if not value:
+            return []
+        return re.split(r"\s*[,;]\s*", value)
+
+
+class Quality(click.ParamType):
+    name = "quality"
+
+    def convert(self, value: str, param: Optional[click.Parameter] = None, ctx: Optional[click.Context] = None) -> int:
+        try:
+            return int(value.lower().rstrip("p"))
+        except TypeError:
+            self.fail(
+                f"expected string for int() conversion, got {value!r} of type {type(value).__name__}",
+                param,
+                ctx
+            )
+        except ValueError:
+            self.fail(f"{value!r} is not a valid integer", param, ctx)
+
+
+SEASON_RANGE = SeasonRange()
+LANGUAGE_RANGE = LanguageRange()
+QUALITY = Quality()

devine/core/utils/collections.py

@@ -0,0 +1,51 @@
+import itertools
+from typing import Any, Iterable, Iterator, Sequence, Tuple, Type, Union
+
+
+def as_lists(*args: Any) -> Iterator[Any]:
+    """Converts any input objects to list objects."""
+    for item in args:
+        yield item if isinstance(item, list) else [item]
+
+
+def as_list(*args: Any) -> list:
+    """
+    Convert any input objects to a single merged list object.
+
+    Example:
+        >>> as_list('foo', ['buzz', 'bizz'], 'bazz', 'bozz', ['bar'], ['bur'])
+        ['foo', 'buzz', 'bizz', 'bazz', 'bozz', 'bar', 'bur']
+    """
+    return list(itertools.chain.from_iterable(as_lists(*args)))
+
+
+def flatten(items: Any, ignore_types: Union[Type, Tuple[Type, ...]] = str) -> Iterator:
+    """
+    Flattens items recursively.
+
+    Example:
+    >>> list(flatten(["foo", [["bar", ["buzz", [""]], "bee"]]]))
+    ['foo', 'bar', 'buzz', '', 'bee']
+    >>> list(flatten("foo"))
+    ['foo']
+    >>> list(flatten({1}, set))
+    [{1}]
+    """
+    if isinstance(items, (Iterable, Sequence)) and not isinstance(items, ignore_types):
+        for i in items:
+            yield from flatten(i, ignore_types)
+    else:
+        yield items
+
+
+def merge_dict(source: dict, destination: dict) -> None:
+    """Recursively merge Source into Destination in-place."""
+    if not source:
+        return
+    for key, value in source.items():
+        if isinstance(value, dict):
+            # get node or create one
+            node = destination.setdefault(key, {})
+            merge_dict(value, node)
+        else:
+            destination[key] = value

devine/core/utils/sslciphers.py

@@ -0,0 +1,77 @@
+import ssl
+from typing import Optional
+
+from requests.adapters import HTTPAdapter
+
+
+class SSLCiphers(HTTPAdapter):
+    """
+    Custom HTTP Adapter to change the TLS Cipher set and security requirements.
+
+    Security Level may optionally be provided. A level above 0 must be used at all times.
+    A list of Security Levels and their security is listed below. Usually 2 is used by default.
+    Do not set the Security level via @SECLEVEL in the cipher list.
+
+    Level 0:
+        Everything is permitted. This retains compatibility with previous versions of OpenSSL.
+
+    Level 1:
+        The security level corresponds to a minimum of 80 bits of security. Any parameters
+        offering below 80 bits of security are excluded. As a result RSA, DSA and DH keys
+        shorter than 1024 bits and ECC keys shorter than 160 bits are prohibited. All export
+        cipher suites are prohibited since they all offer less than 80 bits of security. SSL
+        version 2 is prohibited. Any cipher suite using MD5 for the MAC is also prohibited.
+
+    Level 2:
+        Security level set to 112 bits of security. As a result RSA, DSA and DH keys shorter
+        than 2048 bits and ECC keys shorter than 224 bits are prohibited. In addition to the
+        level 1 exclusions any cipher suite using RC4 is also prohibited. SSL version 3 is
+        also not allowed. Compression is disabled.
+
+    Level 3:
+        Security level set to 128 bits of security. As a result RSA, DSA and DH keys shorter
+        than 3072 bits and ECC keys shorter than 256 bits are prohibited. In addition to the
+        level 2 exclusions cipher suites not offering forward secrecy are prohibited. TLS
+        versions below 1.1 are not permitted. Session tickets are disabled.
+
+    Level 4:
+        Security level set to 192 bits of security. As a result RSA, DSA and DH keys shorter
+        than 7680 bits and ECC keys shorter than 384 bits are prohibited. Cipher suites using
+        SHA1 for the MAC are prohibited. TLS versions below 1.2 are not permitted.
+
+    Level 5:
+        Security level set to 256 bits of security. As a result RSA, DSA and DH keys shorter
+        than 15360 bits and ECC keys shorter than 512 bits are prohibited.
+    """
+
+    def __init__(self, cipher_list: Optional[str] = None, security_level: int = 0, *args, **kwargs):
+        if cipher_list:
+            if not isinstance(cipher_list, str):
+                raise TypeError(f"Expected cipher_list to be a str, not {cipher_list!r}")
+            if "@SECLEVEL" in cipher_list:
+                raise ValueError("You must not specify the Security Level manually in the cipher list.")
+        if not isinstance(security_level, int):
+            raise TypeError(f"Expected security_level to be an int, not {security_level!r}")
+        if security_level not in range(6):
+            raise ValueError(f"The security_level must be a value between 0 and 5, not {security_level}")
+
+        if not cipher_list:
+            # cpython's default cipher list differs to Python-requests cipher list
+            cipher_list = "DEFAULT"
+
+        cipher_list += f":@SECLEVEL={security_level}"
+
+        ctx = ssl.create_default_context()
+        ctx.check_hostname = False  # For some reason this is needed to avoid a verification error
+        ctx.set_ciphers(cipher_list)
+
+        self._ssl_context = ctx
+        super().__init__(*args, **kwargs)
+
+    def init_poolmanager(self, *args, **kwargs):
+        kwargs["ssl_context"] = self._ssl_context
+        return super().init_poolmanager(*args, **kwargs)
+
+    def proxy_manager_for(self, *args, **kwargs):
+        kwargs["ssl_context"] = self._ssl_context
+        return super().proxy_manager_for(*args, **kwargs)

devine/core/utils/subprocess.py

@@ -0,0 +1,31 @@
+import json
+import subprocess
+from pathlib import Path
+from typing import Union
+
+
+def ffprobe(uri: Union[bytes, Path]) -> dict:
+    """Use ffprobe on the provided data to get stream information."""
+    args = [
+        "ffprobe",
+        "-v", "quiet",
+        "-of", "json",
+        "-show_streams"
+    ]
+    if isinstance(uri, Path):
+        args.extend([
+            "-f", "lavfi",
+            "-i", "movie={}[out+subcc]".format(str(uri).replace("\\", '/').replace(":", "\\\\:"))
+        ])
+    elif isinstance(uri, bytes):
+        args.append("pipe:")
+    try:
+        ff = subprocess.run(
+            args,
+            input=uri if isinstance(uri, bytes) else None,
+            check=True,
+            capture_output=True
+        )
+    except subprocess.CalledProcessError:
+        return {}
+    return json.loads(ff.stdout.decode("utf8"))

devine/core/utils/xml.py

@@ -0,0 +1,24 @@
+from typing import Union
+
+from lxml import etree
+from lxml.etree import ElementTree
+
+
+def load_xml(xml: Union[str, bytes]) -> ElementTree:
+    """Safely parse XML data to an ElementTree, without namespaces in tags."""
+    if not isinstance(xml, bytes):
+        xml = xml.encode("utf8")
+    root = etree.fromstring(xml)
+    for elem in root.getiterator():
+        if not hasattr(elem.tag, "find"):
+            # e.g. comment elements
+            continue
+        elem.tag = etree.QName(elem).localname
+        for name, value in elem.attrib.items():
+            local_name = etree.QName(name).localname
+            if local_name == name:
+                continue
+            del elem.attrib[name]
+            elem.attrib[local_name] = value
+    etree.cleanup_namespaces(root)
+    return root

devine/core/vault.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from abc import ABCMeta, abstractmethod
+from typing import Iterator, Optional, Union
+from uuid import UUID
+
+
+class Vault(metaclass=ABCMeta):
+    def __init__(self, name: str):
+        self.name = name
+
+    def __str__(self) -> str:
+        return f"{self.name} {type(self).__name__}"
+
+    @abstractmethod
+    def get_key(self, kid: Union[UUID, str], service: str) -> Optional[str]:
+        """
+        Get Key from Vault by KID (Key ID) and Service.
+
+        It does not get Key by PSSH as the PSSH can be different depending on it's implementation,
+        or even how it was crafted. Some PSSH values may also actually be a CENC Header rather
+        than a PSSH MP4 Box too, which makes the value even more confusingly different.
+
+        However, the KID never changes unless the video file itself has changed too, meaning the
+        key for the presumed-matching KID wouldn't work, further proving matching by KID is
+        superior.
+        """
+
+    @abstractmethod
+    def get_keys(self, service: str) -> Iterator[tuple[str, str]]:
+        """Get All Keys from Vault by Service."""
+
+    @abstractmethod
+    def add_key(self, service: str, kid: Union[UUID, str], key: str, commit: bool = False) -> bool:
+        """Add KID:KEY to the Vault."""
+
+    @abstractmethod
+    def add_keys(self, service: str, kid_keys: dict[Union[UUID, str], str], commit: bool = False) -> int:
+        """
+        Add Multiple Content Keys with Key IDs for Service to the Vault.
+        Pre-existing Content Keys are ignored/skipped.
+        Raises PermissionError if the user has no permission to create the table.
+        """
+
+    @abstractmethod
+    def get_services(self) -> Iterator[str]:
+        """Get a list of Service Tags from Vault."""
+
+
+__ALL__ = (Vault,)

devine/core/vaults.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from typing import Iterator, Optional, Union, Any
+from uuid import UUID
+
+from devine.core.vault import Vault
+from devine.core.config import config
+from devine.core.utilities import import_module_by_path
+
+_VAULTS = sorted(
+    (
+        path
+        for path in config.directories.vaults.glob("*.py")
+        if path.stem.lower() != "__init__"
+    ),
+    key=lambda x: x.stem
+)
+
+_MODULES = {
+    path.stem: getattr(import_module_by_path(path), path.stem)
+    for path in _VAULTS
+}
+
+
+class Vaults:
+    """Keeps hold of Key Vaults with convenience functions, e.g. searching all vaults."""
+
+    def __init__(self, service: Optional[str] = None):
+        self.service = service or ""
+        self.vaults = []
+
+    def __iter__(self) -> Iterator[Vault]:
+        return iter(self.vaults)
+
+    def __len__(self) -> int:
+        return len(self.vaults)
+
+    def load(self, type_: str, **kwargs: Any) -> None:
+        """Load a Vault into the vaults list."""
+        module = _MODULES.get(type_)
+        if not module:
+            raise ValueError(f"Unable to find vault command by the name '{type_}'.")
+        vault = module(**kwargs)
+        self.vaults.append(vault)
+
+    def get_key(self, kid: Union[UUID, str]) -> tuple[Optional[str], Optional[Vault]]:
+        """Get Key from the first Vault it can by KID (Key ID) and Service."""
+        for vault in self.vaults:
+            key = vault.get_key(kid, self.service)
+            if key and key.count("0") != len(key):
+                return key, vault
+        return None, None
+
+    def add_key(self, kid: Union[UUID, str], key: str, excluding: Optional[Vault] = None) -> int:
+        """Add a KID:KEY to all Vaults, optionally with an exclusion."""
+        success = 0
+        for vault in self.vaults:
+            if vault != excluding:
+                try:
+                    success += vault.add_key(self.service, kid, key, commit=True)
+                except (PermissionError, NotImplementedError):
+                    pass
+        return success
+
+    def add_keys(self, kid_keys: dict[Union[UUID, str], str]) -> int:
+        """
+        Add multiple KID:KEYs to all Vaults. Duplicate Content Keys are skipped.
+        PermissionErrors when the user cannot create Tables are absorbed and ignored.
+        """
+        success = 0
+        for vault in self.vaults:
+            try:
+                success += bool(vault.add_keys(self.service, kid_keys, commit=True))
+            except (PermissionError, NotImplementedError):
+                pass
+        return success
+
+
+__ALL__ = (Vaults,)

devine/vaults/MySQL.py

@@ -0,0 +1,225 @@
+from __future__ import annotations
+
+from typing import Iterator, Optional, Union
+from uuid import UUID
+
+import pymysql
+from pymysql.cursors import DictCursor
+
+from devine.core.services import Services
+from devine.core.utils.atomicsql import AtomicSQL
+from devine.core.vault import Vault
+
+
+class MySQL(Vault):
+    """Key Vault using a remotely-accessed mysql database connection."""
+
+    def __init__(self, name: str, host: str, database: str, username: str, **kwargs):
+        """
+        All extra arguments provided via **kwargs will be sent to pymysql.connect.
+        This can be used to provide more specific connection information.
+        """
+        super().__init__(name)
+        self.slug = f"{host}:{database}:{username}"
+        self.con = pymysql.connect(
+            host=host,
+            db=database,
+            user=username,
+            cursorclass=DictCursor,
+            **kwargs
+        )
+        self.adb = AtomicSQL()
+        self.ticket = self.adb.load(self.con)
+
+        self.permissions = self.get_permissions()
+        if not self.has_permission("SELECT"):
+            raise PermissionError(f"MySQL vault {self.slug} has no SELECT permission.")
+
+    def get_key(self, kid: Union[UUID, str], service: str) -> Optional[str]:
+        if not self.has_table(service):
+            # no table, no key, simple
+            return None
+
+        if isinstance(kid, UUID):
+            kid = kid.hex
+
+        c = self.adb.safe_execute(
+            self.ticket,
+            lambda db, cursor: cursor.execute(
+                # TODO: SQL injection risk
+                f"SELECT `id`, `key_` FROM `{service}` WHERE `kid`=%s AND `key_`!=%s",
+                [kid, "0" * 32]
+            )
+        ).fetchone()
+        if not c:
+            return None
+
+        return c["key_"]
+
+    def get_keys(self, service: str) -> Iterator[tuple[str, str]]:
+        if not self.has_table(service):
+            # no table, no keys, simple
+            return None
+
+        c = self.adb.safe_execute(
+            self.ticket,
+            lambda db, cursor: cursor.execute(
+                # TODO: SQL injection risk
+                f"SELECT `kid`, `key_` FROM `{service}` WHERE `key_`!=%s",
+                ["0" * 32]
+            )
+        )
+
+        for row in c.fetchall():
+            yield row["kid"], row["key_"]
+
+    def add_key(self, service: str, kid: Union[UUID, str], key: str, commit: bool = False) -> bool:
+        if not key or key.count("0") == len(key):
+            raise ValueError("You cannot add a NULL Content Key to a Vault.")
+
+        if not self.has_permission("INSERT", table=service):
+            raise PermissionError(f"MySQL vault {self.slug} has no INSERT permission.")
+
+        if not self.has_table(service):
+            try:
+                self.create_table(service, commit)
+            except PermissionError:
+                return False
+
+        if isinstance(kid, UUID):
+            kid = kid.hex
+
+        if self.adb.safe_execute(
+            self.ticket,
+            lambda db, cursor: cursor.execute(
+                # TODO: SQL injection risk
+                f"SELECT `id` FROM `{service}` WHERE `kid`=%s AND `key_`=%s",
+                [kid, key]
+            )
+        ).fetchone():
+            # table already has this exact KID:KEY stored
+            return True
+
+        self.adb.safe_execute(
+            self.ticket,
+            lambda db, cursor: cursor.execute(
+                # TODO: SQL injection risk
+                f"INSERT INTO `{service}` (kid, key_) VALUES (%s, %s)",
+                (kid, key)
+            )
+        )
+
+        if commit:
+            self.commit()
+
+        return True
+
+    def add_keys(self, service: str, kid_keys: dict[Union[UUID, str], str], commit: bool = False) -> int:
+        for kid, key in kid_keys.items():
+            if not key or key.count("0") == len(key):
+                raise ValueError("You cannot add a NULL Content Key to a Vault.")
+
+        if not self.has_permission("INSERT", table=service):
+            raise PermissionError(f"MySQL vault {self.slug} has no INSERT permission.")
+
+        if not self.has_table(service):
+            try:
+                self.create_table(service, commit)
+            except PermissionError:
+                return 0
+
+        if not isinstance(kid_keys, dict):
+            raise ValueError(f"The kid_keys provided is not a dictionary, {kid_keys!r}")
+        if not all(isinstance(kid, (str, UUID)) and isinstance(key_, str) for kid, key_ in kid_keys.items()):
+            raise ValueError("Expecting dict with Key of str/UUID and value of str.")
+
+        if any(isinstance(kid, UUID) for kid, key_ in kid_keys.items()):
+            kid_keys = {
+                kid.hex if isinstance(kid, UUID) else kid: key_
+                for kid, key_ in kid_keys.items()
+            }
+
+        c = self.adb.safe_execute(
+            self.ticket,
+            lambda db, cursor: cursor.executemany(
+                # TODO: SQL injection risk
+                f"INSERT IGNORE INTO `{service}` (kid, key_) VALUES (%s, %s)",
+                kid_keys.items()
+            )
+        )
+
+        if commit:
+            self.commit()
+
+        return c.rowcount
+
+    def get_services(self) -> Iterator[str]:
+        c = self.adb.safe_execute(
+            self.ticket,
+            lambda db, cursor: cursor.execute("SHOW TABLES")
+        )
+        for table in c.fetchall():
+            # each entry has a key named `Tables_in_<db name>`
+            yield Services.get_tag(list(table.values())[0])
+
+    def has_table(self, name: str) -> bool:
+        """Check if the Vault has a Table with the specified name."""
+        return list(self.adb.safe_execute(
+            self.ticket,
+            lambda db, cursor: cursor.execute(
+                "SELECT count(TABLE_NAME) FROM information_schema.TABLES WHERE TABLE_SCHEMA=%s AND TABLE_NAME=%s",
+                [self.con.db, name]
+            )
+        ).fetchone().values())[0] == 1
+
+    def create_table(self, name: str, commit: bool = False):
+        """Create a Table with the specified name if not yet created."""
+        if self.has_table(name):
+            return
+
+        if not self.has_permission("CREATE"):
+            raise PermissionError(f"MySQL vault {self.slug} has no CREATE permission.")
+
+        self.adb.safe_execute(
+            self.ticket,
+            lambda db, cursor: cursor.execute(
+                # TODO: SQL injection risk
+                f"""
+                CREATE TABLE IF NOT EXISTS {name} (
+                  id          int AUTO_INCREMENT PRIMARY KEY,
+                  kid         VARCHAR(255) NOT NULL,
+                  key_        VARCHAR(255) NOT NULL,
+                  UNIQUE(kid, key_)
+                );
+                """
+            )
+        )
+
+        if commit:
+            self.commit()
+
+    def get_permissions(self) -> list:
+        """Get and parse Grants to a more easily usable list tuple array."""
+        with self.con.cursor() as c:
+            c.execute("SHOW GRANTS")
+            grants = c.fetchall()
+            grants = [next(iter(x.values())) for x in grants]
+        grants = [tuple(x[6:].split(" TO ")[0].split(" ON ")) for x in list(grants)]
+        grants = [(
+            list(map(str.strip, perms.replace("ALL PRIVILEGES", "*").split(","))),
+            location.replace("`", "").split(".")
+        ) for perms, location in grants]
+        return grants
+
+    def has_permission(self, operation: str, database: Optional[str] = None, table: Optional[str] = None) -> bool:
+        """Check if the current connection has a specific permission."""
+        grants = [x for x in self.permissions if x[0] == ["*"] or operation.upper() in x[0]]
+        if grants and database:
+            grants = [x for x in grants if x[1][0] in (database, "*")]
+        if grants and table:
+            grants = [x for x in grants if x[1][1] in (table, "*")]
+        return bool(grants)
+
+    def commit(self):
+        """Commit any changes made that has not been written to db."""
+        self.adb.commit(self.ticket)

devine/vaults/SQLite.py

@@ -0,0 +1,173 @@
+from __future__ import annotations
+
+import sqlite3
+from pathlib import Path
+from typing import Iterator, Optional, Union
+from uuid import UUID
+
+from devine.core.services import Services
+from devine.core.utils.atomicsql import AtomicSQL
+from devine.core.vault import Vault
+
+
+class SQLite(Vault):
+    """Key Vault using a locally-accessed sqlite DB file."""
+
+    def __init__(self, name: str, path: Union[str, Path]):
+        super().__init__(name)
+        self.path = Path(path).expanduser()
+        # TODO: Use a DictCursor or such to get fetches as dict?
+        self.con = sqlite3.connect(self.path)
+        self.adb = AtomicSQL()
+        self.ticket = self.adb.load(self.con)
+
+    def get_key(self, kid: Union[UUID, str], service: str) -> Optional[str]:
+        if not self.has_table(service):
+            # no table, no key, simple
+            return None
+
+        if isinstance(kid, UUID):
+            kid = kid.hex
+
+        c = self.adb.safe_execute(
+            self.ticket,
+            lambda db, cursor: cursor.execute(
+                # TODO: SQL injection risk
+                f"SELECT `id`, `key_` FROM `{service}` WHERE `kid`=? AND `key_`!=?",
+                [kid, "0" * 32]
+            )
+        ).fetchone()
+        if not c:
+            return None
+
+        return c[1]  # `key_`
+
+    def get_keys(self, service: str) -> Iterator[tuple[str, str]]:
+        if not self.has_table(service):
+            # no table, no keys, simple
+            return None
+        c = self.adb.safe_execute(
+            self.ticket,
+            lambda db, cursor: cursor.execute(
+                # TODO: SQL injection risk
+                f"SELECT `kid`, `key_` FROM `{service}` WHERE `key_`!=?",
+                ["0" * 32]
+            )
+        )
+        for (kid, key_) in c.fetchall():
+            yield kid, key_
+
+    def add_key(self, service: str, kid: Union[UUID, str], key: str, commit: bool = False) -> bool:
+        if not key or key.count("0") == len(key):
+            raise ValueError("You cannot add a NULL Content Key to a Vault.")
+
+        if not self.has_table(service):
+            self.create_table(service, commit)
+
+        if isinstance(kid, UUID):
+            kid = kid.hex
+
+        if self.adb.safe_execute(
+            self.ticket,
+            lambda db, cursor: cursor.execute(
+                # TODO: SQL injection risk
+                f"SELECT `id` FROM `{service}` WHERE `kid`=? AND `key_`=?",
+                [kid, key]
+            )
+        ).fetchone():
+            # table already has this exact KID:KEY stored
+            return True
+
+        self.adb.safe_execute(
+            self.ticket,
+            lambda db, cursor: cursor.execute(
+                # TODO: SQL injection risk
+                f"INSERT INTO `{service}` (kid, key_) VALUES (?, ?)",
+                (kid, key)
+            )
+        )
+
+        if commit:
+            self.commit()
+
+        return True
+
+    def add_keys(self, service: str, kid_keys: dict[Union[UUID, str], str], commit: bool = False) -> int:
+        for kid, key in kid_keys.items():
+            if not key or key.count("0") == len(key):
+                raise ValueError("You cannot add a NULL Content Key to a Vault.")
+
+        if not self.has_table(service):
+            self.create_table(service, commit)
+
+        if not isinstance(kid_keys, dict):
+            raise ValueError(f"The kid_keys provided is not a dictionary, {kid_keys!r}")
+        if not all(isinstance(kid, (str, UUID)) and isinstance(key_, str) for kid, key_ in kid_keys.items()):
+            raise ValueError("Expecting dict with Key of str/UUID and value of str.")
+
+        if any(isinstance(kid, UUID) for kid, key_ in kid_keys.items()):
+            kid_keys = {
+                kid.hex if isinstance(kid, UUID) else kid: key_
+                for kid, key_ in kid_keys.items()
+            }
+
+        c = self.adb.safe_execute(
+            self.ticket,
+            lambda db, cursor: cursor.executemany(
+                # TODO: SQL injection risk
+                f"INSERT OR IGNORE INTO `{service}` (kid, key_) VALUES (?, ?)",
+                kid_keys.items()
+            )
+        )
+
+        if commit:
+            self.commit()
+
+        return c.rowcount
+
+    def get_services(self) -> Iterator[str]:
+        c = self.adb.safe_execute(
+            self.ticket,
+            lambda db, cursor: cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+        )
+        for (name,) in c.fetchall():
+            if name != "sqlite_sequence":
+                yield Services.get_tag(name)
+
+    def has_table(self, name: str) -> bool:
+        """Check if the Vault has a Table with the specified name."""
+        return self.adb.safe_execute(
+            self.ticket,
+            lambda db, cursor: cursor.execute(
+                "SELECT count(name) FROM sqlite_master WHERE type='table' AND name=?",
+                [name]
+            )
+        ).fetchone()[0] == 1
+
+    def create_table(self, name: str, commit: bool = False):
+        """Create a Table with the specified name if not yet created."""
+        if self.has_table(name):
+            return
+
+        self.adb.safe_execute(
+            self.ticket,
+            lambda db, cursor: cursor.execute(
+                # TODO: SQL injection risk
+                f"""
+                CREATE TABLE IF NOT EXISTS {name} (
+                  "id"        INTEGER NOT NULL UNIQUE,
+                  "kid"       TEXT NOT NULL COLLATE NOCASE,
+                  "key_"      TEXT NOT NULL COLLATE NOCASE,
+                  PRIMARY KEY("id" AUTOINCREMENT),
+                  UNIQUE("kid", "key_")
+                );
+                """
+            )
+        )
+
+        if commit:
+            self.commit()
+
+    def commit(self):
+        """Commit any changes made that has not been written to db."""
+        self.adb.commit(self.ticket)