Compare commits

...

10 Commits

Author SHA1 Message Date
aa60603c76 Upted version to 3.3.4
Some checks failed
ci / lint (push) Has been cancelled
ci / build (3.10) (push) Has been cancelled
ci / build (3.11) (push) Has been cancelled
ci / build (3.9) (push) Has been cancelled
2025-09-27 20:15:37 +02:00
0a7ac3fdd2 Fixed missing segment error for curl_impersonate downloader 2025-09-27 20:03:42 +02:00
393b12c7f0 Fixed missing segment error for aria2c downloader 2025-09-27 20:00:51 +02:00
8ea3b5b26c Fixed missing segment errors 2025-09-27 19:53:23 +02:00
retouching
09eda16882 fix(dl): delete old file after repackage (#114)
* fix(dl): delete old file after repackage

* fix(dl): using original_path instead of self.path in repackage method
2024-06-03 16:57:26 +01:00
rlaphoenix
a95d32de9e chore: Add config to gitignore 2024-05-17 02:29:46 +01:00
rlaphoenix
221cd145c4 refactor(dl): Make Widevine CDM config optional
With this change you no longer have to define/configure a CDM to load. This is something that isn't necessary for a lot of services.

Note: It's also now less hand-holdy in terms of correct config formatting/values. I.e. if you define a cdm by profile for a service slightly incorrectly, say a typo on the service or profile name, it will no longer warn you.
2024-05-17 01:52:45 +01:00
rlaphoenix
0310646cb2 fix(Subtitle): Skip merging segmented WebVTT if only 1 segment 2024-05-17 01:42:44 +01:00
rlaphoenix
3426fc145f fix(HLS): Decrypt AES-encrypted segments separately
We cannot merge all the encrypted AES-128-CBC (ClearKey) segments and then decrypt them in one go because each segment should be padded to a 16-byte boundary in CBC mode.

Since it uses PKCS#5 or #7 style (cant remember which) then the merged file has a 15 in 16 chance to fail the boundary check. And in the 1 in 16 odds that it passes the boundary check, it will not decrypt properly as each segment's padding will be treated as actual data, and not padding.
2024-05-17 01:15:37 +01:00
rlaphoenix
e57d755837 fix(clearkey): Do not pad data before decryption
This is seemingly unnecessary and simply incorrect at least for two sources (VGTV, and TRUTV).

Without this change it is not possible to correctly merge all segments without at least some problem in the resulting file.
2024-05-17 01:00:11 +01:00
11 changed files with 75 additions and 30 deletions

2
.gitignore vendored
View File

@@ -1,4 +1,6 @@
# devine
devine.yaml
devine.yml
*.mkv
*.mp4
*.exe

View File

@@ -178,9 +178,10 @@ class dl:
except ValueError as e:
self.log.error(f"Failed to load Widevine CDM, {e}")
sys.exit(1)
self.log.info(
f"Loaded {self.cdm.__class__.__name__} Widevine CDM: {self.cdm.system_id} (L{self.cdm.security_level})"
)
if self.cdm:
self.log.info(
f"Loaded {self.cdm.__class__.__name__} Widevine CDM: {self.cdm.system_id} (L{self.cdm.security_level})"
)
with console.status("Loading Key Vaults...", spinner="dots"):
self.vaults = Vaults(self.service)
@@ -936,21 +937,21 @@ class dl:
return Credential.loads(credentials) # type: ignore
@staticmethod
def get_cdm(service: str, profile: Optional[str] = None) -> WidevineCdm:
def get_cdm(service: str, profile: Optional[str] = None) -> Optional[WidevineCdm]:
"""
Get CDM for a specified service (either Local or Remote CDM).
Raises a ValueError if there's a problem getting a CDM.
"""
cdm_name = config.cdm.get(service) or config.cdm.get("default")
if not cdm_name:
raise ValueError("A CDM to use wasn't listed in the config")
return None
if isinstance(cdm_name, dict):
if not profile:
raise ValueError("CDM config is mapped for profiles, but no profile was chosen")
return None
cdm_name = cdm_name.get(profile) or config.cdm.get("default")
if not cdm_name:
raise ValueError(f"A CDM to use was not mapped for the profile {profile}")
return None
cdm_api = next(iter(x for x in config.remote_cdm if x["name"] == cdm_name), None)
if cdm_api:

View File

@@ -1 +1 @@
__version__ = "3.3.3"
__version__ = "3.3.4"

View File

@@ -220,6 +220,7 @@ def download(
for dl in stopped_downloads:
if dl["status"] == "error":
error_code = int(dl.get("errorCode", 0))
used_uri = next(
uri["uri"]
for file in dl["files"]
@@ -227,6 +228,11 @@ def download(
for uri in file["uris"]
if uri["status"] == "used"
)
if error_code == 404:
yield dict(downloaded=f"[yellow]Skipped missing segment: {used_uri}")
continue # dont raise, safely ignore
error = f"Download Error (#{dl['gid']}): {dl['errorMessage']} ({dl['errorCode']}), {used_uri}"
error_pretty = "\n ".join(textwrap.wrap(
error,

View File

@@ -7,6 +7,8 @@ from pathlib import Path
from typing import Any, Generator, MutableMapping, Optional, Union
from curl_cffi.requests import Session
from curl_cffi.requests.exceptions import HTTPError
from rich import filesize
from devine.core.config import config
@@ -80,6 +82,10 @@ def download(
try:
stream = session.get(url, stream=True, **kwargs)
if stream.status_code == 404:
yield dict(downloaded=f"[yellow]Skipped missing segment (404): {url}")
break
stream.raise_for_status()
try:
@@ -118,6 +124,11 @@ def download(
)
break
except Exception as e:
if isinstance(e, requests.HTTPError) and e.response.status_code == 404:
# Safe skip, dont bubble up
yield dict(downloaded=f"[yellow]Skipped missing segment: {url}")
break
save_path.unlink(missing_ok=True)
if DOWNLOAD_CANCELLED.is_set() or attempts == MAX_ATTEMPTS:
raise e

View File

@@ -91,6 +91,11 @@ def download(
try:
stream = session.get(url, stream=True, **kwargs)
if stream.status_code == 404:
# Skip missing segments gracefully
yield dict(downloaded=f"[yellow]Segment missing (404 skipped)")
break
stream.raise_for_status()
if not segmented:
@@ -139,6 +144,11 @@ def download(
DOWNLOAD_SIZES.clear()
break
except Exception as e:
if isinstance(e, requests.HTTPError) and e.response.status_code == 404:
# Safe skip, dont bubble up
yield dict(downloaded=f"[yellow]Skipped missing segment: {url}")
break
save_path.unlink(missing_ok=True)
if DOWNLOAD_CANCELLED.is_set() or attempts == MAX_ATTEMPTS:
raise e

View File

@@ -7,7 +7,7 @@ from typing import Optional, Union
from urllib.parse import urljoin
from Cryptodome.Cipher import AES
from Cryptodome.Util.Padding import pad, unpad
from Cryptodome.Util.Padding import unpad
from m3u8.model import Key
from requests import Session
@@ -43,7 +43,7 @@ class ClearKey:
decrypted = AES. \
new(self.key, AES.MODE_CBC, self.iv). \
decrypt(pad(path.read_bytes(), AES.block_size))
decrypt(path.read_bytes())
try:
decrypted = unpad(decrypted, AES.block_size)

View File

@@ -387,15 +387,27 @@ class HLS:
elif len(files) != range_len:
raise ValueError(f"Missing {range_len - len(files)} segment files for {segment_range}...")
merge(
to=merged_path,
via=files,
delete=True,
include_map_data=True
)
drm.decrypt(merged_path)
merged_path.rename(decrypted_path)
if isinstance(drm, Widevine):
# with widevine we can merge all segments and decrypt once
merge(
to=merged_path,
via=files,
delete=True,
include_map_data=True
)
drm.decrypt(merged_path)
merged_path.rename(decrypted_path)
else:
# with other drm we must decrypt separately and then merge them
# for aes this is because each segment likely has 16-byte padding
for file in files:
drm.decrypt(file)
merge(
to=merged_path,
via=files,
delete=True,
include_map_data=True
)
events.emit(
events.Types.TRACK_DECRYPTED,

View File

@@ -206,17 +206,19 @@ class Subtitle(Track):
elif self.codec == Subtitle.Codec.WebVTT:
text = self.path.read_text("utf8")
if self.descriptor == Track.Descriptor.DASH:
text = merge_segmented_webvtt(
text,
segment_durations=self.data["dash"]["segment_durations"],
timescale=self.data["dash"]["timescale"]
)
if len(self.data["dash"]["segment_durations"]) > 1:
text = merge_segmented_webvtt(
text,
segment_durations=self.data["dash"]["segment_durations"],
timescale=self.data["dash"]["timescale"]
)
elif self.descriptor == Track.Descriptor.HLS:
text = merge_segmented_webvtt(
text,
segment_durations=self.data["hls"]["segment_durations"],
timescale=1 # ?
)
if len(self.data["hls"]["segment_durations"]) > 1:
text = merge_segmented_webvtt(
text,
segment_durations=self.data["hls"]["segment_durations"],
timescale=1 # ?
)
caption_set = pycaption.WebVTTReader().read(text)
Subtitle.merge_same_cues(caption_set)
subtitle_text = pycaption.WebVTTWriter().write(caption_set)

View File

@@ -542,6 +542,7 @@ class Track:
else:
raise
original_path.unlink()
self.path = output_path

View File

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
[tool.poetry]
name = "devine"
version = "3.3.3"
version = "3.3.4"
description = "Modular Movie, TV, and Music Archival Software."
license = "GPL-3.0-only"
authors = ["rlaphoenix <rlaphoenix@pm.me>"]