From daff602a319e7c37b79611ced2d44ae2b566fb3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Sun, 19 Dec 2021 15:10:16 +0100 Subject: [PATCH] rssVideos: Work correctly with merged files --- config/scripts/rssVideos | 92 +++++++++++++++++++++++++++++----------- 1 file changed, 67 insertions(+), 25 deletions(-) diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index bfc5180..0f740da 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -26,12 +26,13 @@ from xml.dom import minidom import coloredlogs import configargparse -import yt_dlp as youtube_dl +import yt_dlp log = logging.getLogger(__name__) # TODO Lockfile, or a way to parallel watch and download + def configure_logging(args: configargparse.Namespace) -> None: # Configure logging if args.verbosity: @@ -44,7 +45,25 @@ def configure_logging(args: configargparse.Namespace) -> None: logger=log, ) -def format_duration(duration: int) -> int: + +class SaveInfoPP(yt_dlp.postprocessor.common.PostProcessor): + """ + yt_dlp.process_ie_result() doesn't return a completely updated info dict, + notably the extension is still the one before it realizes the files cannot + be merged. So we use this PostProcessor to catch the info dict in its final + form and save it. + """ + + def __init__(self, rvelement: "RVElement") -> None: + self.rvelement = rvelement + super().__init__() + + def run(self, info: dict) -> tuple[list, dict]: + self.rvelement.ytdl_infos = info + return [], info + + +def format_duration(duration: int) -> str: return time.strftime("%H:%M:%S", time.gmtime(duration)) @@ -108,8 +127,8 @@ class RVElement: if not self.parent.args.research and cache.is_researched: self.__dict__["ytdl_infos"] = cache.__dict__["ytdl_infos"] log.debug(f"From cache: {self}") - if cache.was_downloaded: - self.was_downloaded = True + # if cache.was_downloaded: + # self.was_downloaded = True if cache.watched: self.watched = True @@ -135,10 +154,10 @@ class RVElement: def ytdl_infos(self) -> typing.Optional[dict]: log.info(f"Researching: {self}") try: - infos = self.parent.ytdl_dry.extract_info(self.link) + infos = self.parent.ytdl_dry.extract_info(self.link, download=False) except KeyboardInterrupt as e: raise e - except youtube_dl.utils.DownloadError as e: + except yt_dlp.utils.DownloadError as e: # TODO Still raise in case of temporary network issue log.warning(e) infos = None @@ -147,7 +166,7 @@ class RVElement: if ( infos and "thumbnails" in infos - and isinstance(infos["thumbnails"], youtube_dl.utils.LazyList) + and isinstance(infos["thumbnails"], yt_dlp.utils.LazyList) ): infos["thumbnails"] = infos["thumbnails"].exhaust() # Save database once it's been computed @@ -169,7 +188,6 @@ class RVElement: @property def filepath(self) -> str: assert self.is_video - # TODO This doesn't change the extension to mkv when the formats are incomaptible return self.parent.ytdl_dry.prepare_filename(self.ytdl_infos) @property @@ -181,7 +199,9 @@ class RVElement: assert self.is_video log.info(f"Downloading: {self}") if not self.parent.args.dryrun: - self.parent.ytdl.process_ie_result(self.ytdl_infos, True, {}) + with yt_dlp.YoutubeDL(self.parent.ytdl_opts) as ydl: + ydl.add_post_processor(SaveInfoPP(self)) + ydl.process_ie_result(self.ytdl_infos, download=True) self.was_downloaded = True self.parent.save() @@ -220,7 +240,9 @@ class RVElement: if args.link and not re.search(args.link, self.link): log.debug(f"Link not matching {args.link}: {self}") return False - if args.creator and (not self.creator or not re.search(args.creator, self.creator)): + if args.creator and ( + not self.creator or not re.search(args.creator, self.creator) + ): log.debug(f"Creator not matching {args.creator}: {self}") return False @@ -253,7 +275,9 @@ class RVElement: duration = int(dur) if not comparator(self.duration, duration * multiplier): - log.debug(f"Duration {self.duration} not matching {args.duration}: {self}") + log.debug( + f"Duration {self.duration} not matching {args.duration}: {self}" + ) return False return True @@ -382,16 +406,12 @@ class RVDatabase: @property def ytdl_dry_opts(self) -> dict: opts = self.ytdl_opts.copy() - opts.update({"simulate": True, "quiet": True}) + opts.update({"quiet": True}) return opts @property - def ytdl(self) -> youtube_dl.YoutubeDL: - return youtube_dl.YoutubeDL(self.ytdl_opts) - - @property - def ytdl_dry(self) -> youtube_dl.YoutubeDL: - return youtube_dl.YoutubeDL(self.ytdl_dry_opts) + def ytdl_dry(self) -> yt_dlp.YoutubeDL: + return yt_dlp.YoutubeDL(self.ytdl_dry_opts) def filter(self, args: configargparse.Namespace) -> typing.Iterable[RVElement]: elements: typing.Iterable[RVElement] @@ -401,7 +421,7 @@ class RVDatabase: elif args.order == "title": elements = sorted(self.elements, key=lambda el: el.title) elif args.order == "creator": - elements = sorted(self.elements, key=lambda el: el.creator or '') + elements = sorted(self.elements, key=lambda el: el.creator or "") elif args.order == "link": elements = sorted(self.elements, key=lambda el: el.link) elif args.order == "random": @@ -416,9 +436,13 @@ class RVDatabase: # Expensive sort if args.order == "short": - elements = sorted(elements, key=lambda el: el.duration if el.is_video else 0) + elements = sorted( + elements, key=lambda el: el.duration if el.is_video else 0 + ) elif args.order == "short": - elements = sorted(elements, key=lambda el: el.duration if el.is_video else 0, reverse=True) + elements = sorted( + elements, key=lambda el: el.duration if el.is_video else 0, reverse=True + ) return elements @@ -486,7 +510,12 @@ def get_args() -> configargparse.Namespace: parser.add("--title", help="Regex to filter by title") parser.add("--link", help="Regex to filter by link") parser.add("--duration", help="Comparative to filter by duration") - parser.add("--seen", choices=("seen","unseen","any"), default="unseen", help="Only include seen/unseen/any videos") + parser.add( + "--seen", + choices=("seen", "unseen", "any"), + default="unseen", + help="Only include seen/unseen/any videos", + ) # TODO Envrionment variables parser.add( "--max-duration", @@ -515,7 +544,16 @@ def get_args() -> configargparse.Namespace: parser.add( "action", nargs="?", - choices=("download", "list", "watch", "binge", "clean", "seen", "unseen", "duration"), + choices=( + "download", + "list", + "watch", + "binge", + "clean", + "seen", + "unseen", + "duration", + ), default="download", ) @@ -566,9 +604,13 @@ def main() -> None: if args.action == "watch": break elif args.action == "seen": - element.watched = True + if not element.watched: + log.info(f"Maked as seen: {element}") + element.watched = True elif args.action == "unseen": - element.watched = False + if element.watched: + log.info(f"Maked as unseen: {element}") + element.watched = False elif args.action == "duration": duration += element.duration else: