From a0d2d3082df6ec7eecf8a469eed8fbd80412528b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Tue, 30 Apr 2019 08:22:27 +0200 Subject: [PATCH] rssVideos --- config/git/gitignore | 1 + config/pythonstartup.py | 3 - config/scripts/rssVideos | 182 +++++++++++++++++++++++++++++++++++++++ config/shell/shenv | 1 + config/tmux.conf | 4 + config/vim/pluginconfig | 2 +- 6 files changed, 189 insertions(+), 4 deletions(-) create mode 100755 config/scripts/rssVideos diff --git a/config/git/gitignore b/config/git/gitignore index 05df6db..97ef313 100644 --- a/config/git/gitignore +++ b/config/git/gitignore @@ -2,3 +2,4 @@ *.swo *.ycm_extra_conf.py tags +.mypy_cache diff --git a/config/pythonstartup.py b/config/pythonstartup.py index 4e8f34b..b2837ad 100644 --- a/config/pythonstartup.py +++ b/config/pythonstartup.py @@ -1,10 +1,7 @@ import rlcompleter -import readline import sys import os -print("HELLO") - # From https://github.com/python/cpython/blob/v3.7.0b5/Lib/site.py#L436 # Changing the history file def register_readline(): diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos new file mode 100755 index 0000000..610896d --- /dev/null +++ b/config/scripts/rssVideos @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 + +""" +Script that download videos that are linked as an article +in a RSS feed. +The common use case would be a feed from an RSS aggregator +with the unread items (non-video links are ignored). +""" + +# TODO Distribute this correclty, in the meanwhile please do +# pip install --user youtube-dl ConfigArgParse progressbar2 + +# TODO Allow to specify youtube_dl options (e.g. subtitles) +# TODO Restrict quality (it's not that I don't like 8GB 4K videos but...) + +from typing import Dict, Set +import urllib.request +import urllib.parse +import os +from xml.dom import minidom +import youtube_dl +import configargparse + + +if __name__ == "__main__": + + defaultConfigPath = os.path.join(os.path.expanduser( + os.getenv('XDG_CONFIG_PATH', '~/.config/')), 'rssVideos') + + + parser = configargparse.ArgParser(description="Download videos linked in " + + "a RSS feed (e.g. an unread feed from " + + "an RSS aggregator", + default_config_files=[defaultConfigPath]) + parser.add('-c', '--config', required=False, is_config_file=True, + help='Configuration file') + parser.add('--feed', help='URL of the RSS feed (must be public for now)', + env_var='RSS_VIDEOS_FEED', required=True) + parser.add('--videos', help='Directory to store videos', + env_var='RSS_VIDEOS_VIDEO_DIR', required=True) + # TODO This feature might require additional documentation and an on/off switc + parser.add('--track', help='Directory where download videos are maked (so they are not downloaded twice)', + env_var='RSS_VIDEOS_TRACK', required=False, default='.rssVideos') + + args = parser.parse_args() + args.videos = os.path.realpath(os.path.expanduser(args.videos)) + args.track = os.path.expanduser(args.track) + if not os.path.isabs(args.track): + args.track = os.path.realpath(os.path.join(args.videos, args.track)) + + os.makedirs(args.videos, exist_ok=True) + os.makedirs(args.track, exist_ok=True) + + # Read the feed XML, get the links + print("→ Retrieveing RSS feed") + + links: Set[str] = set() + with urllib.request.urlopen(args.feed) as request: + with minidom.parse(request) as xmldoc: + for item in xmldoc.getElementsByTagName('item'): + try: + linkNode = item.getElementsByTagName('link')[0] + link: str = linkNode.childNodes[0].data + links.add(link) + except BaseException as e: + print("Error while getting link from item:", e) + continue + + # Filter out non-video links and store video download info + # and associated filename + print(f"→ Getting infos on {len(links)} unread articles") + + videosInfos: Dict[str, str] = {} + + ydl_opts = { + "simulate": True, + "quiet": True + } + with youtube_dl.YoutubeDL(ydl_opts) as ydl: + for link in links: + print(f"Researching {link}...") + try: + infos = ydl.extract_info(link) + filepath = ydl.prepare_filename(infos) + filename, extension = os.path.splitext(filepath) + videosInfos[filename] = infos + except BaseException as e: + print(e) + continue + + # Read the directory content, delete everything that's not a + # video on the download list or already downloaded + print(f"→ Deciding on what to do for {len(videosInfos)} videos") + + # Getting information on the video directory + + videosDownloaded: Set[str] = set() + videosPartiallyDownloaded: Set[str] = set() + + for filepath in os.listdir(args.videos): + fullpath = os.path.join(args.videos, filepath) + if not os.path.isfile(fullpath): + continue + filename, extension = os.path.splitext(filepath) + + for onlineFilename in videosInfos.keys(): + # Full name already there: completly downloaded → remove from the download list + if filename == onlineFilename: + videosDownloaded.add(onlineFilename) + break + # Partial name already there: not completly downloaded → keep on the download list + elif filename.startswith(onlineFilename): + videosPartiallyDownloaded.add(onlineFilename) + break + # Unrelated filename: delete + else: + print(f"Deleting: {filename}") + os.unlink(fullpath) + + # Getting informations on the tracking directory + + # Videos that were once downloaded using this tool + videosTracked: Set[str] = set() + + for filepath in os.listdir(args.track): + fullpath = os.path.join(args.track, filepath) + if not os.path.isfile(fullpath): + continue + # Here filename is a filepath as no extension + + if filepath in videosInfos: + videosTracked.add(filepath) + else: + os.unlink(fullpath) + + # Deciding for the rest based on the informations + + + def markTracked(filename): + markerPath = os.path.join(args.track, onlineFilename) + open(markerPath, 'a').close() + + + videosToDownload: Set[str] = set() + videosReads: Set[str] = set() + for onlineFilename in videosInfos.keys(): + # If the video was once downloaded but manually deleted, + # the marker should be left + if onlineFilename in videosTracked: + print(f"Should be marked as read: {onlineFilename}") + # TODO Automatically do that one day maybe? + # Need to login to the FreshRSS API and keep track of + # the item id along the process + videosReads.add(onlineFilename) + elif onlineFilename in videosDownloaded: + markTracked(onlineFilename) + print(f"Already downloaded: {onlineFilename}") + else: + if onlineFilename in videosPartiallyDownloaded: + print(f"Will be continued: {onlineFilename}") + else: + print(f"Will be downloaded: {onlineFilename}") + videosToDownload.add(onlineFilename) + + # Download the missing videos + print(f"→ Downloading {len(videosToDownload)} videos") + + os.chdir(args.videos) + + # TODO Progressbar one day maybe? + # We have all the info we need to make a reliable one + ydl_opts = { + } + with youtube_dl.YoutubeDL(ydl_opts) as ydl: + for onlineFilename in videosToDownload: + infos = videosInfos[onlineFilename] + + # Really download + ydl.process_ie_result(infos, True, {}) + + markTracked(onlineFilename) + diff --git a/config/shell/shenv b/config/shell/shenv index 2488868..8f1fa8b 100644 --- a/config/shell/shenv +++ b/config/shell/shenv @@ -53,6 +53,7 @@ direnv GNUPGHOME "$HOME/.config/gnupg" direnv GRADLE_USER_HOME "$HOME/.cache/gradle" export INPUTRC="$HOME/.config/inputrc" export LESSHISTFILE="$HOME/.cache/lesshst" +direnv MIX_ARCHIVES "$HOME/.cache/mix/archives" direnv MONO_GAC_PREFIX "$HOME/.cache/mono" export NODE_REPL_HISTORY="$HOME/.cache/node_repl_history" direnv npm_config_cache "$HOME/.cache/npm" diff --git a/config/tmux.conf b/config/tmux.conf index b761cf2..9c9f7d9 100644 --- a/config/tmux.conf +++ b/config/tmux.conf @@ -9,6 +9,10 @@ bind-key -n M-7 select-window -t 7 bind-key -n M-8 select-window -t 8 bind-key -n M-9 select-window -t 9 +set -g mouse on +# https://superuser.com/a/1007721 +bind -n WheelUpPane if-shell -F -t = "#{mouse_any_flag}" "send-keys -M" "if -Ft= '#{pane_in_mode}' 'send-keys -M' 'copy-mode -e; send-keys -M'" + # List of plugins set -g @plugin 'tmux-plugins/tpm' set -g @plugin 'tmux-plugins/tmux-sensible' diff --git a/config/vim/pluginconfig b/config/vim/pluginconfig index c9595bc..907872a 100644 --- a/config/vim/pluginconfig +++ b/config/vim/pluginconfig @@ -5,7 +5,7 @@ nmap :ALEFix let g:ale_sign_error = '×' let g:ale_sign_warning = '!' let g:ale_completion_enabled = 1 -let g:ale_fixers = ['shfmt', 'uncrustify', 'remove_trailing_lines', 'trim_whitespace', 'phpcbf'] +let g:ale_fixers = ['autopep8', 'shfmt', 'uncrustify', 'remove_trailing_lines', 'trim_whitespace', 'phpcbf'] let g:ale_php_phpcs_standard = '/srv/http/machines/ruleset.xml' " For PHP, install https://pear.php.net/package/PHP_CodeSniffer