From e4502a30c59c8ae698dae3e6cb1c0727835ff802 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Tue, 9 Nov 2021 13:59:25 +0100 Subject: [PATCH 01/34] automatrop: Various fixes --- config/automatrop/host_vars/curacao.geoffrey.frogeye.fr | 3 +++ config/automatrop/host_vars/pindakaas.geoffrey.frogeye.fr | 3 +++ config/automatrop/roles/dotfiles/tasks/main.yml | 3 ++- config/automatrop/roles/ecryptfs_automount/meta/main.yml | 2 -- config/automatrop/roles/software/tasks/main.yml | 3 --- config/i3/config.j2 | 2 +- config/shell/.gitignore | 1 + 7 files changed, 10 insertions(+), 7 deletions(-) delete mode 100644 config/automatrop/roles/ecryptfs_automount/meta/main.yml diff --git a/config/automatrop/host_vars/curacao.geoffrey.frogeye.fr b/config/automatrop/host_vars/curacao.geoffrey.frogeye.fr index 8fa5d1b..2b55a1b 100644 --- a/config/automatrop/host_vars/curacao.geoffrey.frogeye.fr +++ b/config/automatrop/host_vars/curacao.geoffrey.frogeye.fr @@ -14,3 +14,6 @@ has_forge_access: yes extensions: - g - gh +x11_screens: + - HDMI-0 + - eDP-1-1 diff --git a/config/automatrop/host_vars/pindakaas.geoffrey.frogeye.fr b/config/automatrop/host_vars/pindakaas.geoffrey.frogeye.fr index e2c5bb5..06b923d 100644 --- a/config/automatrop/host_vars/pindakaas.geoffrey.frogeye.fr +++ b/config/automatrop/host_vars/pindakaas.geoffrey.frogeye.fr @@ -9,3 +9,6 @@ encrypt_home_stacked_fs: yes extensions: - g - gh +x11_screens: + - DP-1 + - eDP-1 diff --git a/config/automatrop/roles/dotfiles/tasks/main.yml b/config/automatrop/roles/dotfiles/tasks/main.yml index 8d55342..0498403 100644 --- a/config/automatrop/roles/dotfiles/tasks/main.yml +++ b/config/automatrop/roles/dotfiles/tasks/main.yml @@ -14,8 +14,9 @@ git: repo: "{% if has_forge_access %}git@git.frogeye.fr:{% else %}https://git.frogeye.fr/{% endif %}geoffrey/dotfiles.git" dest: "{{ ansible_user_dir }}/.dotfiles" - update: "{{ not has_forge_access }}" + update: yes notify: install dotfiles + tags: dotfiles_repo # TODO Put actual dotfiles in a subdirectory of the repo, so we don't have to put everything in config - name: Register as Ansible collection diff --git a/config/automatrop/roles/ecryptfs_automount/meta/main.yml b/config/automatrop/roles/ecryptfs_automount/meta/main.yml deleted file mode 100644 index a3df829..0000000 --- a/config/automatrop/roles/ecryptfs_automount/meta/main.yml +++ /dev/null @@ -1,2 +0,0 @@ -dependencies: - - role: system diff --git a/config/automatrop/roles/software/tasks/main.yml b/config/automatrop/roles/software/tasks/main.yml index 09143f5..9a9907c 100644 --- a/config/automatrop/roles/software/tasks/main.yml +++ b/config/automatrop/roles/software/tasks/main.yml @@ -179,9 +179,6 @@ notify: "software changed" tags: softwarelist -- debug: - msg: "{{ packages }}" - - name: Install packages (Arch-based) aur: name: "{{ packages }}" diff --git a/config/i3/config.j2 b/config/i3/config.j2 index 26fd963..6eee325 100644 --- a/config/i3/config.j2 +++ b/config/i3/config.j2 @@ -149,7 +149,7 @@ set $WS9 9 set $WS10 10 # Workspace output -{% set screens = ["HDMI-0", "eDP-1-1"] %} +{% set screens = x11_screens | default(['DEFAULT']) %} {% for i in range(1, 11) %} workspace "$WS{{ i }}" output {{ screens[(i - 1) % (screens | length)] }} {% endfor %} diff --git a/config/shell/.gitignore b/config/shell/.gitignore index d3e4561..3cb575e 100644 --- a/config/shell/.gitignore +++ b/config/shell/.gitignore @@ -1,2 +1,3 @@ +extrc *.zwc .zcompdump From 57f428f4ad9219964aab7e9e764dcea3997a6eb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Thu, 11 Nov 2021 21:52:05 +0100 Subject: [PATCH 02/34] Fix to accomodate Feline API change --- .../automatrop/roles/vim/templates/plugins/feline.j2 | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/config/automatrop/roles/vim/templates/plugins/feline.j2 b/config/automatrop/roles/vim/templates/plugins/feline.j2 index 33c3dc8..58799e8 100644 --- a/config/automatrop/roles/vim/templates/plugins/feline.j2 +++ b/config/automatrop/roles/vim/templates/plugins/feline.j2 @@ -28,8 +28,8 @@ require('feline').setup({ base0F = base16_colors.base0F, }, components = { - left = { - active = { + active = { + { { provider = function() return string.format(' %d ', vim.fn.line('$')) end, -- If you can, make it depend on the actual bar size @@ -95,10 +95,8 @@ require('feline').setup({ provider='', hl = { bg = 'base01', fg = 'base02' }, }, - } - }, - right = { - active = { + }, + { { provider='', hl = { bg = 'base03', fg = 'base01', }, From 02441867aadbbe0fed32c78493eee589f8721673 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Thu, 11 Nov 2021 21:53:40 +0100 Subject: [PATCH 03/34] Trust mah kayh --- config/automatrop/roles/gnupg/tasks/main.yml | 1 + .../roles/software/templates/snippets/pm_dev_common.j2 | 2 ++ .../roles/software/templates/snippets/pm_dev_python.j2 | 1 + 3 files changed, 4 insertions(+) diff --git a/config/automatrop/roles/gnupg/tasks/main.yml b/config/automatrop/roles/gnupg/tasks/main.yml index ed4c7aa..619119b 100644 --- a/config/automatrop/roles/gnupg/tasks/main.yml +++ b/config/automatrop/roles/gnupg/tasks/main.yml @@ -48,3 +48,4 @@ - name: Install Geoffrey Frogeye's key gpg_key: fpr: 4FBA930D314A03215E2CDB0A8312C8CAC1BAC289 + trust: 5 diff --git a/config/automatrop/roles/software/templates/snippets/pm_dev_common.j2 b/config/automatrop/roles/software/templates/snippets/pm_dev_common.j2 index ec52f2e..1f6adfa 100644 --- a/config/automatrop/roles/software/templates/snippets/pm_dev_common.j2 +++ b/config/automatrop/roles/software/templates/snippets/pm_dev_common.j2 @@ -11,4 +11,6 @@ highlight {% endif %} {# For nvim's :Telescope live_grep #} ripgrep +{# Offline documentation #} +zeal {# EOF #} diff --git a/config/automatrop/roles/software/templates/snippets/pm_dev_python.j2 b/config/automatrop/roles/software/templates/snippets/pm_dev_python.j2 index 9186694..aa0d59f 100644 --- a/config/automatrop/roles/software/templates/snippets/pm_dev_python.j2 +++ b/config/automatrop/roles/software/templates/snippets/pm_dev_python.j2 @@ -8,3 +8,4 @@ python-lsp-server python-mypy-ls python-lsp-black {% endif %} +ipython From c8397580ec75428ec8c4d5665b74ae43d800c44a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Thu, 11 Nov 2021 21:54:17 +0100 Subject: [PATCH 04/34] More dev here please --- config/automatrop/host_vars/pindakaas.geoffrey.frogeye.fr | 1 + 1 file changed, 1 insertion(+) diff --git a/config/automatrop/host_vars/pindakaas.geoffrey.frogeye.fr b/config/automatrop/host_vars/pindakaas.geoffrey.frogeye.fr index 06b923d..fccb9b1 100644 --- a/config/automatrop/host_vars/pindakaas.geoffrey.frogeye.fr +++ b/config/automatrop/host_vars/pindakaas.geoffrey.frogeye.fr @@ -4,6 +4,7 @@ dev_stuffs: - shell - network - ansible + - python has_battery: yes encrypt_home_stacked_fs: yes extensions: From d065fa620766627bc1654f9f2658ca035b445a19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Thu, 2 Dec 2021 10:44:49 +0100 Subject: [PATCH 05/34] Add office computer --- config/automatrop/.gitignore | 1 + config/automatrop/host_vars/gho.geoffrey.frogeye.fr | 12 ++++++++++++ config/automatrop/hosts | 1 + config/scripts/automatrop | 8 +++++++- 4 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 config/automatrop/.gitignore create mode 100644 config/automatrop/host_vars/gho.geoffrey.frogeye.fr diff --git a/config/automatrop/.gitignore b/config/automatrop/.gitignore new file mode 100644 index 0000000..801d1f7 --- /dev/null +++ b/config/automatrop/.gitignore @@ -0,0 +1 @@ +self_hostname diff --git a/config/automatrop/host_vars/gho.geoffrey.frogeye.fr b/config/automatrop/host_vars/gho.geoffrey.frogeye.fr new file mode 100644 index 0000000..3aebfe0 --- /dev/null +++ b/config/automatrop/host_vars/gho.geoffrey.frogeye.fr @@ -0,0 +1,12 @@ +root_access: no +display_server: "x11" +dev_stuffs: + - shell + - network + - ansible + - python +extensions: + - gh +x11_screens: + - HDMI-1 + - HDMI-2 diff --git a/config/automatrop/hosts b/config/automatrop/hosts index 724017c..bc23e54 100644 --- a/config/automatrop/hosts +++ b/config/automatrop/hosts @@ -1,3 +1,4 @@ curacao.geoffrey.frogeye.fr # triffle.geoffrey.frogeye.fr pindakaas.geoffrey.frogeye.fr +gho.geoffrey.frogeye.fr ansible_host=localhost ansible_port=2222 diff --git a/config/scripts/automatrop b/config/scripts/automatrop index 2482dfb..d463b74 100755 --- a/config/scripts/automatrop +++ b/config/scripts/automatrop @@ -1,4 +1,10 @@ #!/usr/bin/env bash cd ~/.dotfiles/config/automatrop -ansible-playbook --diff playbooks/default.yml --limit $HOSTNAME --connection local "$@" +if [ -f ~/.config/automatrop/self_name ] +then + hostname=$(cat ~/.config/automatrop/self_name) +else + hostname="$HOSTNAME" +fi +ansible-playbook --diff playbooks/default.yml --limit $hostname --connection local "$@" From 9c82a364ed3ed5778d1e7ab6ef6d4647f424cf1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Thu, 2 Dec 2021 10:53:32 +0100 Subject: [PATCH 06/34] Improvements to smtpdummy --- config/scripts/smtpdummy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/config/scripts/smtpdummy b/config/scripts/smtpdummy index 4beab74..8738765 100755 --- a/config/scripts/smtpdummy +++ b/config/scripts/smtpdummy @@ -88,7 +88,7 @@ XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X""" body = f"\n\n{args.body}" text = f"""Date: {now_email} -From: {getattr(args, 'from')} +From: {args.me} <{getattr(args, 'from')}> Subject: {args.subject} To: {args.to} Message-ID: {mid} @@ -108,7 +108,9 @@ Input arguments: .""" # Transmission setup - cmd = ["ssh", args.origin] + cmd = [] + if args.origin != "localhost": + cmd += ["ssh", args.origin] if args.security == "plain": cmd += ["socat", "-", f"tcp:{args.destination}:{args.port}"] elif args.security == "ssl": From c4a0a4daef8b578b10db3f7f63700779cb342f1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Thu, 2 Dec 2021 10:57:09 +0100 Subject: [PATCH 07/34] Few changes junest --- config/xinitrc | 2 +- xsession | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/xinitrc b/config/xinitrc index 03e7cbe..30d939c 100755 --- a/config/xinitrc +++ b/config/xinitrc @@ -30,7 +30,7 @@ then fi # If we have junest installed DE try them before all others -sessions_dir_junest="$HOME/.local/share/xsessions" +sessions_dir_junest="$HOME/.junest/usr/share/xsessions" if [ -d "$sessions_dir_junest" ] then sessions_dirs="$sessions_dir_junest $sessions_dirs" diff --git a/xsession b/xsession index 91b527a..db89fac 100755 --- a/xsession +++ b/xsession @@ -6,7 +6,7 @@ # Sourced by display managers # -. ~/.xprofile +[ -f ~/.xprofile ] && . ~/.xprofile if [ -f ~/.config/override_dm_choice ] then . ~/.config/xinitrc From 2250f520fed3960a62043c606398c96f8d7bedbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Thu, 2 Dec 2021 16:55:55 +0100 Subject: [PATCH 08/34] Make ghautomatrop work with Junest --- config/automatrop/.gitignore | 2 +- .../roles/desktop_environment/tasks/main.yml | 1 + config/automatrop/roles/facts/tasks/main.yml | 10 +++- .../roles/software/handlers/main.yml | 6 ++ .../automatrop/roles/software/tasks/main.yml | 60 ++++++++++++++----- .../templates/snippets/pm_data_management.j2 | 2 +- .../snippets/pm_terminal_essentials.j2 | 2 + config/automatrop/roles/vim/tasks/main.yml | 3 - config/automatrop/self_name | 1 + 9 files changed, 65 insertions(+), 22 deletions(-) create mode 100644 config/automatrop/self_name diff --git a/config/automatrop/.gitignore b/config/automatrop/.gitignore index 801d1f7..206c8c2 100644 --- a/config/automatrop/.gitignore +++ b/config/automatrop/.gitignore @@ -1 +1 @@ -self_hostname +self_name diff --git a/config/automatrop/roles/desktop_environment/tasks/main.yml b/config/automatrop/roles/desktop_environment/tasks/main.yml index 54cf0b6..781be24 100644 --- a/config/automatrop/roles/desktop_environment/tasks/main.yml +++ b/config/automatrop/roles/desktop_environment/tasks/main.yml @@ -167,6 +167,7 @@ loop: - pulseaudio - mpd + when: has_systemd # TODO bar (might change bar in the future, so...) # TODO highlight (there IS a template but the colors look different from vim and mostly the same from when there's no config) diff --git a/config/automatrop/roles/facts/tasks/main.yml b/config/automatrop/roles/facts/tasks/main.yml index 4f06a62..fd07ad9 100644 --- a/config/automatrop/roles/facts/tasks/main.yml +++ b/config/automatrop/roles/facts/tasks/main.yml @@ -4,9 +4,17 @@ arch: "{{ ansible_lsb.id == 'Arch' }}" manjaro: "{{ ansible_lsb.id == 'Manjaro' or ansible_lsb.id == 'Manjaro-ARM' }}" termux: "{{ ansible_distribution == 'OtherLinux' and ansible_python.executable == '/data/data/com.termux/files/usr/bin/python' }}" - debian_based: "{{ ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' }}" debian: "{{ ansible_distribution == 'Debian' }}" ubuntu: "{{ ansible_distribution == 'Ubuntu' }}" + junest: "{{ ansible_distribution == 'Archlinux' and ansible_is_chroot }}" # TODO Check if /etc/junest exists + tags: + - always + +- name: Set composed facts + set_fact: + debian_based: "{{ debian or ubuntu }}" + can_chown: "{{ not junest }}" + has_systemd: "{{ not junest }}" tags: - always # TODO Make this a real Ansible fact maybe? diff --git a/config/automatrop/roles/software/handlers/main.yml b/config/automatrop/roles/software/handlers/main.yml index 66f7ee5..2f759f8 100644 --- a/config/automatrop/roles/software/handlers/main.yml +++ b/config/automatrop/roles/software/handlers/main.yml @@ -15,3 +15,9 @@ listen: "software changed" when: root_access when: arch_based + +- name: update pacman cache + pacman: + update_cache: yes + become: yes + when: arch_based diff --git a/config/automatrop/roles/software/tasks/main.yml b/config/automatrop/roles/software/tasks/main.yml index 9a9907c..9ef01ef 100644 --- a/config/automatrop/roles/software/tasks/main.yml +++ b/config/automatrop/roles/software/tasks/main.yml @@ -59,6 +59,30 @@ # Arch configuration +# TODO Patch sudo-fake so it allows using -u so `become` works + +- name: Enable multilib repo + lineinfile: + path: /etc/pacman.conf + regexp: '^#?\s*\[multilib\]$' + line: '[multilib]' + become: yes + when: arch_based and ansible_architecture == "x86_64" + notify: udpate pacman cache + +- name: Configure multilib repo + lineinfile: + path: /etc/pacman.conf + regexp: '^#?\s*Include\s*=\s*/etc/pacman.d/mirrorlist' + line: 'Include = /etc/pacman.d/mirrorlist' + insertafter: '^\[multilib\]$' + become: yes + when: arch_based and ansible_architecture == "x86_64" + notify: udpate pacman cache + +- name: Update cache if needed + meta: flush_handlers + - name: Install ccache pacman: name: ccache @@ -90,7 +114,6 @@ replace: "CFLAGS=\\1\\2" become: yes when: arch_based - tags: g - name: Change -march to native from makepkg CFLAGS replace: @@ -99,7 +122,6 @@ replace: "CFLAGS=\\1-march=native\\2\\3" become: yes when: arch_based - tags: g - name: Set makepkg MAKEFLAGS replace: @@ -140,24 +162,30 @@ # Install alternative package managers +- name: List packages from base-devel + command: pacman -Sqg base-devel + register: base_devel_packages + changed_when: no + check_mode: no + - name: Install dependencies for AUR helpers pacman: - name: - - fakeroot - - base-devel + name: "{{ (base_devel_packages.stdout | split('\n') | reject('eq', 'sudo')) + ['fakeroot'] }}" become: yes - when: arch_based and root_access + when: arch_based +# Do not install sudo because maybe sudo-fake is installed (otherwise it conflicts) +# It should already be installed already anyway - name: Install AUR package manager (Arch) aur: name: yay-bin - when: arch and root_access + when: arch - name: Install AUR package manager (Manjaro) pacman: name: yay become: yes - when: manjaro and root_access + when: manjaro # Not sure if regular Manjaro has yay in its community packages, # but Manjaro-ARM sure does @@ -172,13 +200,6 @@ packages: "{{ query('template', 'package_manager.j2')[0].split('\n')[:-1]|sort|unique }}" tags: softwarelist -- name: Check if list of packages changed - copy: - content: "{% for package in packages %}{{ package }}\n{% endfor %}" - dest: "{{ ansible_user_dir }}/.cache/automatrop/package_manager" - notify: "software changed" - tags: softwarelist - - name: Install packages (Arch-based) aur: name: "{{ packages }}" @@ -189,7 +210,14 @@ use: yay notify: "software changed" tags: softwarelist - when: arch_based and root_access + when: arch_based + +- name: Check if list of packages changed + copy: + content: "{% for package in packages %}{{ package }}\n{% endfor %}" + dest: "{{ ansible_user_dir }}/.cache/automatrop/package_manager" + notify: "software changed" + tags: softwarelist # translate-shell # $ curl -L git.io/trans > ~/.local/bin/trans diff --git a/config/automatrop/roles/software/templates/snippets/pm_data_management.j2 b/config/automatrop/roles/software/templates/snippets/pm_data_management.j2 index 75763f7..d6c2768 100644 --- a/config/automatrop/roles/software/templates/snippets/pm_data_management.j2 +++ b/config/automatrop/roles/software/templates/snippets/pm_data_management.j2 @@ -5,7 +5,7 @@ rsync borg syncthing {% if arch_based %} -{% if ansible_architecture == 'x86_64' %} +{% if ansible_architecture == 'x86_64' and can_chown %} freefilesync-bin {# Not worth the compilation if you can't have the binaries #} {% endif %} diff --git a/config/automatrop/roles/software/templates/snippets/pm_terminal_essentials.j2 b/config/automatrop/roles/software/templates/snippets/pm_terminal_essentials.j2 index 9350e26..69a99cd 100644 --- a/config/automatrop/roles/software/templates/snippets/pm_terminal_essentials.j2 +++ b/config/automatrop/roles/software/templates/snippets/pm_terminal_essentials.j2 @@ -1,7 +1,9 @@ moreutils man visidata +{% if can_chown or not arch_based %} insect +{% endif %} translate-shell gnupg {# Editor #} diff --git a/config/automatrop/roles/vim/tasks/main.yml b/config/automatrop/roles/vim/tasks/main.yml index 0b13399..fb63c5f 100644 --- a/config/automatrop/roles/vim/tasks/main.yml +++ b/config/automatrop/roles/vim/tasks/main.yml @@ -3,7 +3,6 @@ vim_variants: - vim - nvim - tags: g # TODO vim-minimal for bsh # TODO Select those in a clever way @@ -25,7 +24,6 @@ src: loader.j2 dest: "{{ ansible_user_dir }}/.config/vim/loader.vim" mode: "u=rw,g=r,o=r" - tags: g - name: Install theme template: @@ -54,4 +52,3 @@ loop: "{{ vim_variants }}" loop_control: loop_var: variant - tags: g diff --git a/config/automatrop/self_name b/config/automatrop/self_name new file mode 100644 index 0000000..90e6c19 --- /dev/null +++ b/config/automatrop/self_name @@ -0,0 +1 @@ +gho.geoffrey.frogeye.fr From 1b6cc39dada2b22931d4d79ebf6af2eb32149a21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Thu, 2 Dec 2021 16:56:27 +0100 Subject: [PATCH 09/34] Make alacritty run zsh regardless of default shell --- config/i3/config.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/i3/config.j2 b/config/i3/config.j2 index 6eee325..30c1d5c 100644 --- a/config/i3/config.j2 +++ b/config/i3/config.j2 @@ -50,7 +50,7 @@ bindsym $mod+Shift+d exec --no-startup-id rofi -modi drun -show drun # Start Applications # bindsym $mod+Return exec urxvtc -bindsym $mod+Return exec alacritty +bindsym $mod+Return exec alacritty -e zsh bindsym $mod+Shift+Return exec urxvt bindsym $mod+p exec thunar bindsym $mod+m exec qutebrowser --override-restore --backend=webengine From a01b369bc87448c062bc71b72046638fbf28dff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Fri, 3 Dec 2021 08:14:01 +0100 Subject: [PATCH 10/34] I thought it was ignored --- config/automatrop/self_name | 1 - 1 file changed, 1 deletion(-) delete mode 100644 config/automatrop/self_name diff --git a/config/automatrop/self_name b/config/automatrop/self_name deleted file mode 100644 index 90e6c19..0000000 --- a/config/automatrop/self_name +++ /dev/null @@ -1 +0,0 @@ -gho.geoffrey.frogeye.fr From 814b3165848c7def8e19406ef419b2dd7f751503 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Fri, 10 Dec 2021 22:59:39 +0100 Subject: [PATCH 11/34] Refactored rssVideos - Has a cache for yt-dlp research (save time on reruns) - Simplified logic for cleanup / continue downloading - Using OOP / functional programming (?) - Removed tracking logic (unused) --- config/scripts/rssVideos | 449 +++++++++++++++++++++------------------ 1 file changed, 242 insertions(+), 207 deletions(-) diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index fc5f80a..b88439c 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -1,5 +1,6 @@ #!/usr/bin/env python3 + """ Script that download videos that are linked as an article in a RSS feed. @@ -8,17 +9,235 @@ with the unread items (non-video links are ignored). """ # TODO Distribute this correclty, in the meanwhile please do -# pip install --user yt-dlp ConfigArgParse +# pip install --user coloredlogs ConfigArgParse yt-dlp -# TODO Better logging (youtube-dl allow to pass loggers) - -import sys -import urllib.request -import urllib.parse +import enum +import functools +import logging import os +import pickle +import sys +import typing +import urllib.parse +import urllib.request from xml.dom import minidom -import yt_dlp as youtube_dl + +import coloredlogs import configargparse +import yt_dlp as youtube_dl + +log = logging.getLogger(__name__) + + +def configure_logging(args: configargparse.Namespace) -> None: + # Configure logging + if args.verbosity: + coloredlogs.install( + level=args.verbosity, + ) + else: + coloredlogs.install( + fmt="%(message)s", + logger=log, + ) + + +class RVElement: + title: str + link: str + # creator: str + # description: str + # date: datetime.datetime + guid: int + + parent: "RVDatabase" + + def __init__(self, parent: "RVDatabase", item: minidom.Element) -> None: + def get_data(tag_name: str) -> str: + nodes = item.getElementsByTagName(tag_name) + if len(nodes) != 1: + raise RuntimeError(f"Exepected 1 tag `{tag_name}`, got {len(nodes)}.") + children = nodes[0].childNodes + if len(children) != 1: + raise RuntimeError( + f"Exepected 1 children for tag `{tag_name}`, got {len(children)}." + ) + return children[0].data + + self.title = get_data("title") + self.link = get_data("link") + # self.creator = get_data("dc:creator") + # self.description = get_data("description") + # self.date = get_data("pubDate") + self.guid = int(get_data("guid")) + + self.parent = parent + + def read_cache(self, cache: "RVElement") -> None: + if "ytdl_infos" in cache.__dict__: + self.__dict__["ytdl_infos"] = cache.__dict__["ytdl_infos"] + log.debug(f"From cache: {self}") + + def __str__(self) -> str: + return f"{self.title} – {self.link}" + + @property + def downloaded(self) -> bool: + if "ytdl_infos" not in self.__dict__: + return False + return os.path.isfile(self.filepath) + + @functools.cached_property + def ytdl_infos(self) -> typing.Optional[dict]: + log.info(f"Researching: {self}") + try: + infos = self.parent.ytdl_dry.extract_info(self.link) + except BaseException as e: + # TODO Still raise in case of temporary network issue + log.warn(e) + infos = None + # Apparently that thing is transformed from a LazyList + # somewhere in the normal yt_dlp process + if ( + infos + and "thumbnails" in infos + and isinstance(infos["thumbnails"], youtube_dl.utils.LazyList) + ): + infos["thumbnails"] = infos["thumbnails"].exhaust() + # Save database once it's been computed + self.__dict__["ytdl_infos"] = infos + self.parent.save() + return infos + + @property + def skip(self) -> bool: + assert self.is_video + assert self.ytdl_infos + if ( + self.parent.args.max_duration > 0 + and self.ytdl_infos["duration"] > self.parent.args.max_duration + ): + return True + return False + + @property + def is_video(self) -> bool: + # Duration might be missing in playlists and stuff + return self.ytdl_infos is not None and "duration" in self.ytdl_infos + + @property + def filepath(self) -> str: + assert self.is_video + return self.parent.ytdl_dry.prepare_filename(self.ytdl_infos) + + @property + def filename(self) -> str: + assert self.is_video + return os.path.splitext(self.filepath)[0] + + def download(self) -> None: + assert self.is_video + log.info(f"Downloading: {self}") + if self.parent.args.dryrun: + return + self.parent.ytdl.process_ie_result(self.ytdl_infos, True, {}) + + def act(self) -> None: + if not self.is_video: + log.debug(f"Not a video: {self}") + return + if self.downloaded: + log.debug(f"Already downloaded: {self}") + return + if self.skip: + log.debug(f"Skipped: {self}") + return + self.download() + + +class RVDatabase: + SAVE_FILE = ".cache.p" + + args: configargparse.Namespace + elements: list[RVElement] + + def __init__(self, args: configargparse.Namespace) -> None: + self.args = args + + def save(self) -> None: + if self.args.dryrun: + return + with open(self.SAVE_FILE, "wb") as save_file: + pickle.dump(self, save_file) + + @classmethod + def load(cls) -> typing.Optional["RVDatabase"]: + try: + with open(cls.SAVE_FILE, "rb") as save_file: + return pickle.load(save_file) + except (TypeError, AttributeError, EOFError): + log.warn("Corrupt / outdated cache, it will be rebuilt.") + except FileNotFoundError: + pass + return None + + def read_cache(self, cache: "RVDatabase") -> None: + cache_els = dict() + for cache_el in cache.elements: + cache_els[cache_el.guid] = cache_el + for el in self.elements: + if el.guid in cache_els: + el.read_cache(cache_els[el.guid]) + + def read_feed(self) -> None: + log.info("Fetching RSS feed") + self.elements = list() + with urllib.request.urlopen(self.args.feed) as request: + with minidom.parse(request) as xmldoc: + for item in xmldoc.getElementsByTagName("item"): + element = RVElement(self, item) + self.elements.insert(0, element) + log.debug(f"Known: {element}") + + def clean(self) -> None: + filenames = set() + for element in self.elements: + if element.is_video: + filenames.add(element.filename) + for file in os.listdir(): + if file == RVDatabase.SAVE_FILE: + continue + if not os.path.isfile(file): + continue + for filename in filenames: + if file.startswith(filename): + break + else: + log.info(f"Removing: {file}") + if not self.args.dryrun: + os.unlink(file) + + def act_all(self) -> None: + for element in self.elements: + element.act() + + @property + def ytdl_opts(self) -> dict: + return {"format": self.args.format, "allsubtitles": self.args.subtitles} + + @property + def ytdl_dry_opts(self) -> dict: + opts = self.ytdl_opts.copy() + opts.update({"simulate": True, "quiet": True}) + return opts + + @property + def ytdl(self) -> youtube_dl.YoutubeDL: + return youtube_dl.YoutubeDL(self.ytdl_opts) + + @property + def ytdl_dry(self) -> youtube_dl.YoutubeDL: + return youtube_dl.YoutubeDL(self.ytdl_dry_opts) def get_args() -> configargparse.Namespace: @@ -32,6 +251,13 @@ def get_args() -> configargparse.Namespace: + "an RSS aggregator", default_config_files=[defaultConfigPath], ) + parser.add_argument( + "-v", + "--verbosity", + choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + default=None, + help="Verbosity of log messages", + ) parser.add( "-c", "--config", required=False, is_config_file=True, help="Configuration file" ) @@ -55,15 +281,6 @@ def get_args() -> configargparse.Namespace: const=True, default=False, ) - # TODO This feature might require additional documentation and an on/off switch - parser.add( - "--track", - help="Directory where download videos are marked " - + "to not download them after deletion.", - env_var="RSS_VIDEOS_TRACK", - required=False, - default=".rssVideos", - ) parser.add( "--max-duration", help="Skip video longer than this amount of seconds", @@ -87,207 +304,25 @@ def get_args() -> configargparse.Namespace: args = parser.parse_args() args.videos = os.path.realpath(os.path.expanduser(args.videos)) - args.track = os.path.expanduser(args.track) - if not os.path.isabs(args.track): - args.track = os.path.realpath(os.path.join(args.videos, args.track)) return args -def get_links(args: configargparse.Namespace) -> list[str]: - """ - Read the feed XML, get the links - """ - links = list() - with urllib.request.urlopen(args.feed) as request: - with minidom.parse(request) as xmldoc: - for item in xmldoc.getElementsByTagName("item"): - try: - linkNode = item.getElementsByTagName("link")[0] - link: str = linkNode.childNodes[0].data - if link not in links: - links.append(link) - except BaseException as e: - print("Error while getting link from item:", e) - continue - return links - - -def get_video_infos( - args: configargparse.Namespace, ydl_opts: dict, links: list[str] -) -> dict[str, dict]: - """ - Filter out non-video links and store video download info - and associated filename - """ - videosInfos = dict() - - dry_ydl_opts = ydl_opts.copy() - dry_ydl_opts.update({"simulate": True, "quiet": True}) - with youtube_dl.YoutubeDL(dry_ydl_opts) as ydl: - for link in links: - print(f"Researching {link}...") - try: - infos = ydl.extract_info(link) - if args.max_duration > 0 and infos["duration"] > args.max_duration: - print( - f"{infos['title']}: Skipping as longer than max duration: " - f"{infos['duration']} > {args.max_duration}" - ) - continue - filepath = ydl.prepare_filename(infos) - filename, extension = os.path.splitext(filepath) - videosInfos[filename] = infos - print(f"{infos['title']}: Added") - - except BaseException as e: - print(e) - continue - - return videosInfos - - -def get_downloaded_videos( - args: configargparse.Namespace, videosInfos: dict[str, dict] -) -> tuple[set[str], set[str]]: - videosDownloaded = set() - videosPartiallyDownloaded = set() - """ - Read the directory content, delete everything that's not a - video on the download list or already downloaded - """ - - for filepath in os.listdir(args.videos): - fullpath = os.path.join(args.videos, filepath) - if not os.path.isfile(fullpath): - continue - filename, extension = os.path.splitext(filepath) - - for onlineFilename in videosInfos.keys(): - # Full name already there: completly downloaded - # → remove from the download list - if filename == onlineFilename: - videosDownloaded.add(onlineFilename) - break - elif filename.startswith(onlineFilename): - # Subtitle file - # → ignore - if filename.endswith(".vtt"): - break - - # Partial name already there: not completly downloaded - # → keep on the download list - videosPartiallyDownloaded.add(onlineFilename) - break - # Unrelated filename: delete - else: - print(f"Deleting: {filename}") - os.unlink(fullpath) - - return videosDownloaded, videosPartiallyDownloaded - - -def get_tracked_videos(args: configargparse.Namespace, known: set[str]) -> set[str]: - """ - Return videos previously downloaded (=tracked) amongst the unread videos. - This is stored in the tracking directory as empty extension-less files. - Other tracking markers (e.g. for now read videos) are deleted. - """ - - videosTracked = set() - - for filepath in os.listdir(args.track): - fullpath = os.path.join(args.track, filepath) - if not os.path.isfile(fullpath): - continue - # Here filename is a filepath as no extension - - if filepath in known: - videosTracked.add(filepath) - else: - os.unlink(fullpath) - - return videosTracked - - def main() -> None: - args = get_args() + configure_logging(args) os.makedirs(args.videos, exist_ok=True) - os.makedirs(args.track, exist_ok=True) - ydl_opts = {"format": args.format, "allsubtitles": args.subtitles} - - print("→ Retrieveing RSS feed") - links = get_links(args) - # Oldest first - links = links[::-1] - - print(f"→ Getting infos on {len(links)} unread articles") - videosInfos = get_video_infos(args, ydl_opts, links) - - print(f"→ Deciding on what to do for {len(videosInfos)} videos") - videosDownloaded, videosPartiallyDownloaded = get_downloaded_videos( - args, videosInfos - ) - videosTracked = get_tracked_videos(args, set(videosInfos.keys())) - - # Deciding for the rest based on the informations - - def markTracked(filename: str) -> None: - markerPath = os.path.join(args.track, onlineFilename) - open(markerPath, "a").close() - - videosToDownload: set[str] = set() - videosReads: set[str] = set() - for onlineFilename in videosInfos.keys(): - # If the video was once downloaded but manually deleted, - # the marker should be left - if onlineFilename in videosTracked: - print(f"Should be marked as read: {onlineFilename}") - # TODO Automatically do that one day maybe? - # Need to login to the FreshRSS API and keep track of - # the item id along the process - videosReads.add(onlineFilename) - elif onlineFilename in videosDownloaded: - markTracked(onlineFilename) - print(f"Already downloaded: {onlineFilename}") - else: - if onlineFilename in videosPartiallyDownloaded: - print(f"Will be continued: {onlineFilename}") - else: - print(f"Will be downloaded: {onlineFilename}") - videosToDownload.add(onlineFilename) - - # Download the missing videos - print(f"→ Downloading {len(videosToDownload)} videos") - os.chdir(args.videos) - exit_code = 0 - with youtube_dl.YoutubeDL(ydl_opts) as ydl: - for onlineFilename, infos in videosInfos.items(): - if onlineFilename not in videosToDownload: - continue - - # Really download - if args.dryrun: - print(f"Would download {onlineFilename}") - else: - # Apparently that thing is transformed from a LazyList - # somewhere in the normal yt_dlp process - if isinstance(infos["thumbnails"], youtube_dl.utils.LazyList): - infos["thumbnails"] = infos["thumbnails"].exhaust() - try: - ydl.process_ie_result(infos, True, {}) - - markTracked(onlineFilename) - except BaseException as e: - print(e) - exit_code = 1 - continue - - sys.exit(exit_code) + database = RVDatabase(args) + database.read_feed() + cache = RVDatabase.load() + if cache: + database.read_cache(cache) + database.clean() + database.act_all() + database.save() if __name__ == "__main__": From 7d9fa984a61a357fa63eb2cbf79610bf0b9f02be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Fri, 10 Dec 2021 23:13:29 +0100 Subject: [PATCH 12/34] rssVideos: Add list command Then you know which video you should watch next :) --- config/scripts/rssVideos | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index b88439c..c36b861 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -41,6 +41,9 @@ def configure_logging(args: configargparse.Namespace) -> None: logger=log, ) +class RVCommand(enum.Enum): + download = "download" + list = "list" class RVElement: title: str @@ -302,6 +305,15 @@ def get_args() -> configargparse.Namespace: action="store_true", ) + parser.set_defaults(subcommand=RVCommand.download) + subparsers = parser.add_subparsers(title="subcommand") + + sc_download = subparsers.add_parser("download") + sc_download.set_defaults(subcommand=RVCommand.download) + + sc_list = subparsers.add_parser("list") + sc_list.set_defaults(subcommand=RVCommand.list) + args = parser.parse_args() args.videos = os.path.realpath(os.path.expanduser(args.videos)) @@ -315,14 +327,22 @@ def main() -> None: os.makedirs(args.videos, exist_ok=True) os.chdir(args.videos) - database = RVDatabase(args) - database.read_feed() - cache = RVDatabase.load() - if cache: - database.read_cache(cache) - database.clean() - database.act_all() - database.save() + if args.subcommand == RVCommand.download: + database = RVDatabase(args) + database.read_feed() + cache = RVDatabase.load() + if cache: + database.read_cache(cache) + database.clean() + database.act_all() + database.save() + + elif args.subcommand == RVCommand.list: + cache = RVDatabase.load() + if not cache: + raise FileNotFoundError("This command doesn't work without a cache yet.") + for element in cache.elements: + print(element) if __name__ == "__main__": From 8e74f0616430788cd9b503bff5d3abc5ba094d19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Fri, 10 Dec 2021 23:14:25 +0100 Subject: [PATCH 13/34] Gentle arguments for scrot For some reason -se didn't work anymore... --- config/i3/config.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/i3/config.j2 b/config/i3/config.j2 index 30c1d5c..eb7c452 100644 --- a/config/i3/config.j2 +++ b/config/i3/config.j2 @@ -74,9 +74,9 @@ bindsym XF86MonBrightnessDown exec xbacklight -dec 5 -time 0 bindsym XF86MonBrightnessUp exec xbacklight -inc 5 -time 0 # Screenshots -bindsym Print exec scrot -ue 'mv $f ~/Screenshots/ && optipng ~/Screenshots/$f' -bindsym $mod+Print exec scrot -e 'mv $f ~/Screenshots/ && optipng ~/Screenshots/$f' -bindsym Ctrl+Print exec sleep 1 && scrot -se 'mv $f ~/Screenshots/ && optipng ~/Screenshots/$f' +bindsym Print exec scrot --focused --exec 'mv $f ~/Screenshots/ && optipng ~/Screenshots/$f' +bindsym $mod+Print exec scrot --exec 'mv $f ~/Screenshots/ && optipng ~/Screenshots/$f' +bindsym Ctrl+Print exec sleep 1 && scrot --select --exec 'mv $f ~/Screenshots/ && optipng ~/Screenshots/$f' focus_follows_mouse no mouse_warping output From 9493edc1fd4ead7fb15e18e45571ddca47880920 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Sun, 12 Dec 2021 13:40:24 +0100 Subject: [PATCH 14/34] rssVideos: Don't download already downloaded videos Because the good extension is not the one expected :/ --- config/scripts/rssVideos | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index c36b861..a59b323 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -41,10 +41,12 @@ def configure_logging(args: configargparse.Namespace) -> None: logger=log, ) + class RVCommand(enum.Enum): download = "download" list = "list" + class RVElement: title: str link: str @@ -54,6 +56,7 @@ class RVElement: guid: int parent: "RVDatabase" + was_downloaded: bool def __init__(self, parent: "RVDatabase", item: minidom.Element) -> None: def get_data(tag_name: str) -> str: @@ -75,11 +78,14 @@ class RVElement: self.guid = int(get_data("guid")) self.parent = parent + self.was_downloaded = False def read_cache(self, cache: "RVElement") -> None: if "ytdl_infos" in cache.__dict__: self.__dict__["ytdl_infos"] = cache.__dict__["ytdl_infos"] log.debug(f"From cache: {self}") + if cache.was_downloaded: + self.was_downloaded = True def __str__(self) -> str: return f"{self.title} – {self.link}" @@ -131,6 +137,7 @@ class RVElement: @property def filepath(self) -> str: assert self.is_video + # TODO This doesn't change the extension to mkv when the formats are incomaptible return self.parent.ytdl_dry.prepare_filename(self.ytdl_infos) @property @@ -141,16 +148,20 @@ class RVElement: def download(self) -> None: assert self.is_video log.info(f"Downloading: {self}") - if self.parent.args.dryrun: - return - self.parent.ytdl.process_ie_result(self.ytdl_infos, True, {}) + if not self.parent.args.dryrun: + self.parent.ytdl.process_ie_result(self.ytdl_infos, True, {}) + self.was_downloaded = True + self.parent.save() def act(self) -> None: if not self.is_video: log.debug(f"Not a video: {self}") return if self.downloaded: - log.debug(f"Already downloaded: {self}") + log.debug(f"Currently downloaded: {self}") + return + if self.was_downloaded: + log.debug(f"Downloaded previously: {self}") return if self.skip: log.debug(f"Skipped: {self}") @@ -168,6 +179,7 @@ class RVDatabase: self.args = args def save(self) -> None: + log.debug("Saving cache") if self.args.dryrun: return with open(self.SAVE_FILE, "wb") as save_file: @@ -205,7 +217,7 @@ class RVDatabase: def clean(self) -> None: filenames = set() for element in self.elements: - if element.is_video: + if element.is_video and not element.skip: filenames.add(element.filename) for file in os.listdir(): if file == RVDatabase.SAVE_FILE: From 6a6f5401e69213fb4128acb2f15ba006e01c052c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Sun, 12 Dec 2021 14:27:08 +0100 Subject: [PATCH 15/34] rssVideos: Show creator Even if it's not always present for all RSS feeds --- config/scripts/rssVideos | 84 +++++++++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 32 deletions(-) diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index a59b323..74c5787 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -48,38 +48,55 @@ class RVCommand(enum.Enum): class RVElement: - title: str - link: str - # creator: str - # description: str - # date: datetime.datetime - guid: int - parent: "RVDatabase" + item: minidom.Element was_downloaded: bool def __init__(self, parent: "RVDatabase", item: minidom.Element) -> None: - def get_data(tag_name: str) -> str: - nodes = item.getElementsByTagName(tag_name) - if len(nodes) != 1: - raise RuntimeError(f"Exepected 1 tag `{tag_name}`, got {len(nodes)}.") - children = nodes[0].childNodes - if len(children) != 1: - raise RuntimeError( - f"Exepected 1 children for tag `{tag_name}`, got {len(children)}." - ) - return children[0].data - - self.title = get_data("title") - self.link = get_data("link") - # self.creator = get_data("dc:creator") - # self.description = get_data("description") - # self.date = get_data("pubDate") - self.guid = int(get_data("guid")) - self.parent = parent + self.item = item self.was_downloaded = False + def get_tag_data(self, tag_name: str) -> str: + nodes = self.item.getElementsByTagName(tag_name) + if len(nodes) != 1: + raise KeyError(f"Exepected 1 tag `{tag_name}`, got {len(nodes)}.") + children = nodes[0].childNodes + if len(children) != 1: + raise KeyError( + f"Exepected 1 children for tag `{tag_name}`, got {len(children)}." + ) + return children[0].data + + @property + def title(self) -> str: + return self.get_tag_data("title") + + @property + def link(self) -> str: + return self.get_tag_data("link") + + @property + def creator(self) -> typing.Optional[str]: + try: + return self.get_tag_data("dc:creator") + except KeyError: + return None + + @property + def description(self) -> str: + # TODO Testing + return self.get_tag_data("description") + + @property + def date(self) -> str: + # TODO datetime format + return self.get_tag_data("pubDate") + + @property + def guid(self) -> int: + return int(self.get_tag_data("guid")) + def read_cache(self, cache: "RVElement") -> None: if "ytdl_infos" in cache.__dict__: self.__dict__["ytdl_infos"] = cache.__dict__["ytdl_infos"] @@ -88,7 +105,7 @@ class RVElement: self.was_downloaded = True def __str__(self) -> str: - return f"{self.title} – {self.link}" + return f"{self.guid}: {self.creator} – {self.title} – {self.link}" @property def downloaded(self) -> bool: @@ -204,15 +221,18 @@ class RVDatabase: if el.guid in cache_els: el.read_cache(cache_els[el.guid]) + @functools.cached_property + def feed_xml(self) -> minidom.Document: + with urllib.request.urlopen(self.args.feed) as request: + return minidom.parse(request) + def read_feed(self) -> None: log.info("Fetching RSS feed") self.elements = list() - with urllib.request.urlopen(self.args.feed) as request: - with minidom.parse(request) as xmldoc: - for item in xmldoc.getElementsByTagName("item"): - element = RVElement(self, item) - self.elements.insert(0, element) - log.debug(f"Known: {element}") + for item in self.feed_xml.getElementsByTagName("item"): + element = RVElement(self, item) + self.elements.insert(0, element) + log.debug(f"Known: {element}") def clean(self) -> None: filenames = set() From 7f0e24a29d5f2bfa83c5d4d1910b464d8d81a94b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Sun, 12 Dec 2021 14:52:21 +0100 Subject: [PATCH 16/34] rssVideos: Slightly better error handling Makes it actually quit on Ctrl+C --- config/scripts/requirements.txt | 3 ++- config/scripts/rssVideos | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/scripts/requirements.txt b/config/scripts/requirements.txt index 33c1a34..d364bdb 100644 --- a/config/scripts/requirements.txt +++ b/config/scripts/requirements.txt @@ -1,3 +1,4 @@ coloredlogs>=10.0<11 progressbar2>=3.47.0<4 -youtube-dl>=2021.6.6 +yt-dlp>=2021.10.22 +ConfigArgParse>=1.5<2 diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index 74c5787..7e8687b 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -8,9 +8,6 @@ The common use case would be a feed from an RSS aggregator with the unread items (non-video links are ignored). """ -# TODO Distribute this correclty, in the meanwhile please do -# pip install --user coloredlogs ConfigArgParse yt-dlp - import enum import functools import logging @@ -118,7 +115,9 @@ class RVElement: log.info(f"Researching: {self}") try: infos = self.parent.ytdl_dry.extract_info(self.link) - except BaseException as e: + except KeyboardInterrupt as e: + raise e + except youtube_dl.utils.DownloadError as e: # TODO Still raise in case of temporary network issue log.warn(e) infos = None From 76df5d4d8075f38a3e8aaa916585bff2c623f9d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Wed, 15 Dec 2021 21:59:45 +0100 Subject: [PATCH 17/34] Upgrade rofi config --- .../roles/desktop_environment/tasks/main.yml | 20 +++++++++++++------ config/rofi/.gitignore | 2 -- config/rofi/config | 4 ---- config/rofi/config.rasi | 6 ++++++ 4 files changed, 20 insertions(+), 12 deletions(-) create mode 100644 config/rofi/config.rasi diff --git a/config/automatrop/roles/desktop_environment/tasks/main.yml b/config/automatrop/roles/desktop_environment/tasks/main.yml index 781be24..2e80813 100644 --- a/config/automatrop/roles/desktop_environment/tasks/main.yml +++ b/config/automatrop/roles/desktop_environment/tasks/main.yml @@ -6,6 +6,7 @@ with_items: - ".config/Xresources" - ".config/rofi" + - ".local/share/rofi/themes" - ".local/bin" - ".local/share/fonts" - ".config/qutebrowser" @@ -108,17 +109,24 @@ - color when: display_server == 'x11' -- name: Set base16 theme for rofi +- name: Set base16 theme for rofi < 1.4 copy: - content: "{{ base16_schemes['schemes'][base16_scheme]['rofi']['themes']['base16-' + base16_scheme + '.' + item] }}" - dest: "{{ ansible_env.HOME }}/.config/rofi/theme.{{ item }}" + content: "{{ base16_schemes['schemes'][base16_scheme]['rofi']['themes']['base16-' + base16_scheme + '.config'] }}" + dest: "{{ ansible_env.HOME }}/.config/rofi/theme.config" mode: "u=rw,g=r,o=r" - with_items: - - rasi - - config tags: - color +- name: Set base16 theme for rofi >= 1.4 + copy: + content: "{{ base16_schemes['schemes'][base16_scheme]['rofi']['themes']['base16-' + base16_scheme + '.rasi'] }}" + dest: "{{ ansible_env.HOME }}/.local/share/rofi/themes/current.rasi" + mode: "u=rw,g=r,o=r" + tags: + - color + - g + when: no + - name: Configure Dunst template: src: "{{ ansible_env.HOME }}/.config/dunst/dunstrc.j2" diff --git a/config/rofi/.gitignore b/config/rofi/.gitignore index c1955d4..83193ec 100644 --- a/config/rofi/.gitignore +++ b/config/rofi/.gitignore @@ -1,3 +1 @@ theme.config -theme.rasi - diff --git a/config/rofi/config b/config/rofi/config index 64a7e14..fc193ca 100644 --- a/config/rofi/config +++ b/config/rofi/config @@ -1,8 +1,4 @@ #include "theme.config" rofi.theme: theme -rofi.cycle: true -rofi.case-sensitive: false -rofi.scroll-method: 0 -rofi.show-match: true rofi.lazy-grab: false rofi.matching: regex diff --git a/config/rofi/config.rasi b/config/rofi/config.rasi new file mode 100644 index 0000000..af02ce7 --- /dev/null +++ b/config/rofi/config.rasi @@ -0,0 +1,6 @@ +configuration { + theme: "current"; + lazy-grab: false; + matching: "regex"; +} + From 9100edac1ee706e29621e9f3bf71df070db9f530 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Fri, 17 Dec 2021 22:13:27 +0100 Subject: [PATCH 18/34] videoQuota: Support filters --- config/scripts/videoQuota | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/scripts/videoQuota b/config/scripts/videoQuota index 58af75e..1313ba8 100755 --- a/config/scripts/videoQuota +++ b/config/scripts/videoQuota @@ -33,6 +33,7 @@ audio_br_bi = 128000 quota_by = int(sys.argv[1]) in_file = sys.argv[2] out_file = sys.argv[3] +filters = sys.argv[4:] quota_bi = quota_by * 8 duration = duration_file(in_file) @@ -44,6 +45,7 @@ cmd = [ "ffmpeg", "-i", in_file, +] + filters + [ "-b:v", str(video_br_bi), "-b:a", @@ -51,4 +53,5 @@ cmd = [ out_file, ] +print(' '.join(cmd)) subprocess.run(cmd, check=True) From f11338a04a956c71b6c38b78128fe3a71915d268 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Fri, 17 Dec 2021 22:13:46 +0100 Subject: [PATCH 19/34] rssVideos: Support list filters --- config/scripts/rssVideos | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index 7e8687b..87c8ab7 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -13,6 +13,7 @@ import functools import logging import os import pickle +import re import sys import typing import urllib.parse @@ -184,6 +185,19 @@ class RVElement: return self.download() + def matches_search(self, args: configargparse.Namespace) -> bool: + if not self.is_video: + return False + if args.title and not re.search(args.title, self.title): + return False + if args.creator and not re.search(args.creator, self.creator): + return False + if args.guid and not re.search(args.guid, str(self.guid)): + return False + if args.link and not re.search(args.link, self.link): + return False + return True + class RVDatabase: SAVE_FILE = ".cache.p" @@ -344,6 +358,10 @@ def get_args() -> configargparse.Namespace: sc_list = subparsers.add_parser("list") sc_list.set_defaults(subcommand=RVCommand.list) + sc_list.add("--guid", help="Regex to filter guid") + sc_list.add("--creator", help="Regex to filter creator") + sc_list.add("--title", help="Regex to filter titles") + sc_list.add("--link", help="Regex to filter link") args = parser.parse_args() args.videos = os.path.realpath(os.path.expanduser(args.videos)) @@ -373,6 +391,8 @@ def main() -> None: if not cache: raise FileNotFoundError("This command doesn't work without a cache yet.") for element in cache.elements: + if not element.matches_search(args): + continue print(element) From 7aeecb1bff9b2c0aeaa42eef1558ae94ab5f5518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Fri, 17 Dec 2021 22:41:47 +0100 Subject: [PATCH 20/34] =?UTF-8?q?videoQuota:=20=E2=9C=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/scripts/videoQuota | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/config/scripts/videoQuota b/config/scripts/videoQuota index 1313ba8..03e6c76 100755 --- a/config/scripts/videoQuota +++ b/config/scripts/videoQuota @@ -41,17 +41,21 @@ tot_br_bi = quota_bi / duration video_br_bi = int(tot_br_bi - audio_br_bi) assert video_br_bi > 0, "Not even enough space for audio" -cmd = [ - "ffmpeg", - "-i", - in_file, -] + filters + [ - "-b:v", - str(video_br_bi), - "-b:a", - str(audio_br_bi), - out_file, -] +cmd = ( + [ + "ffmpeg", + "-i", + in_file, + ] + + filters + + [ + "-b:v", + str(video_br_bi), + "-b:a", + str(audio_br_bi), + out_file, + ] +) -print(' '.join(cmd)) +print(" ".join(cmd)) subprocess.run(cmd, check=True) From 7423a9320388f5e8e72bf5078373850c42fcde61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Fri, 17 Dec 2021 22:42:35 +0100 Subject: [PATCH 21/34] rssVideos: Filter by duration --- config/scripts/rssVideos | 53 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index 87c8ab7..532b26e 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -136,12 +136,17 @@ class RVElement: return infos @property - def skip(self) -> bool: + def duration(self) -> int: assert self.is_video assert self.ytdl_infos + return self.ytdl_infos["duration"] + + @property + def skip(self) -> bool: + assert self.is_video if ( self.parent.args.max_duration > 0 - and self.ytdl_infos["duration"] > self.parent.args.max_duration + and self.duration > self.parent.args.max_duration ): return True return False @@ -185,6 +190,17 @@ class RVElement: return self.download() + MATCHES_DURATION_MULTIPLIERS = {"s": 1, "m": 60, "h": 3600, None: 1} + + MATCHES_DURATION_COMPARATORS = { + "<": int.__lt__, + "-": int.__lt__, + ">": int.__gt__, + "+": int.__gt__, + "=": int.__eq__, + None: int.__le__, + } + def matches_search(self, args: configargparse.Namespace) -> bool: if not self.is_video: return False @@ -196,6 +212,32 @@ class RVElement: return False if args.link and not re.search(args.link, self.link): return False + if args.duration: + dur = args.duration + + mult_index = dur[-1].lower() + if mult_index.isdigit(): + mult_index = None + else: + dur = dur[:-1] + try: + multiplier = self.MATCHES_DURATION_MULTIPLIERS[mult_index] + except IndexError: + raise ValueError(f"Unknown duration multiplier: {mult_index}") + + comp_index = dur[0] + if comp_index.isdigit(): + comp_index = None + else: + dur = dur[1:] + try: + comparator = self.MATCHES_DURATION_COMPARATORS[comp_index] + except IndexError: + raise ValueError(f"Unknown duration comparator: {comp_index}") + + duration = int(dur) + if not comparator(self.duration, duration * multiplier): + return False return True @@ -359,9 +401,10 @@ def get_args() -> configargparse.Namespace: sc_list = subparsers.add_parser("list") sc_list.set_defaults(subcommand=RVCommand.list) sc_list.add("--guid", help="Regex to filter guid") - sc_list.add("--creator", help="Regex to filter creator") - sc_list.add("--title", help="Regex to filter titles") - sc_list.add("--link", help="Regex to filter link") + sc_list.add("--creator", help="Regex to filter by creator") + sc_list.add("--title", help="Regex to filter by title") + sc_list.add("--link", help="Regex to filter by link") + sc_list.add("--duration", help="Comparative to filter by duration") args = parser.parse_args() args.videos = os.path.realpath(os.path.expanduser(args.videos)) From 5b195bd141a7a0eca0f9bbf3e875da4b7223c7ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Fri, 17 Dec 2021 23:16:32 +0100 Subject: [PATCH 22/34] rssVideos: Add watch --- config/scripts/rssVideos | 55 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index 532b26e..b58efd3 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -13,7 +13,9 @@ import functools import logging import os import pickle +import random import re +import subprocess import sys import typing import urllib.parse @@ -43,17 +45,20 @@ def configure_logging(args: configargparse.Namespace) -> None: class RVCommand(enum.Enum): download = "download" list = "list" + watch = "watch" class RVElement: parent: "RVDatabase" item: minidom.Element was_downloaded: bool + watched: bool def __init__(self, parent: "RVDatabase", item: minidom.Element) -> None: self.parent = parent self.item = item self.was_downloaded = False + self.watched = False def get_tag_data(self, tag_name: str) -> str: nodes = self.item.getElementsByTagName(tag_name) @@ -101,6 +106,8 @@ class RVElement: log.debug(f"From cache: {self}") if cache.was_downloaded: self.was_downloaded = True + if cache.watched: + self.watched = True def __str__(self) -> str: return f"{self.guid}: {self.creator} – {self.title} – {self.link}" @@ -204,6 +211,8 @@ class RVElement: def matches_search(self, args: configargparse.Namespace) -> bool: if not self.is_video: return False + if self.watched: + return False if args.title and not re.search(args.title, self.title): return False if args.creator and not re.search(args.creator, self.creator): @@ -240,6 +249,16 @@ class RVElement: return False return True + def watch(self) -> None: + if not self.downloaded: + self.download() + + proc = subprocess.run(['mpv', self.filepath]) + proc.check_returncode() + + self.watched = True + self.parent.save() + class RVDatabase: SAVE_FILE = ".cache.p" @@ -400,11 +419,20 @@ def get_args() -> configargparse.Namespace: sc_list = subparsers.add_parser("list") sc_list.set_defaults(subcommand=RVCommand.list) - sc_list.add("--guid", help="Regex to filter guid") - sc_list.add("--creator", help="Regex to filter by creator") - sc_list.add("--title", help="Regex to filter by title") - sc_list.add("--link", help="Regex to filter by link") - sc_list.add("--duration", help="Comparative to filter by duration") + + sc_watch = subparsers.add_parser("watch") + sc_watch.set_defaults(subcommand=RVCommand.watch) + sc_watch.add("order", choices=("old", "new", "random"), nargs="?", default="old", help="Watch X first") + + # TODO Command to watch multiple + + # Common arguments for filtering + for sc in (sc_list, sc_watch): + sc.add("--guid", help="Regex to filter guid") + sc.add("--creator", help="Regex to filter by creator") + sc.add("--title", help="Regex to filter by title") + sc.add("--link", help="Regex to filter by link") + sc.add("--duration", help="Comparative to filter by duration") args = parser.parse_args() args.videos = os.path.realpath(os.path.expanduser(args.videos)) @@ -419,6 +447,8 @@ def main() -> None: os.makedirs(args.videos, exist_ok=True) os.chdir(args.videos) + # TODO Abstract a bit + if args.subcommand == RVCommand.download: database = RVDatabase(args) database.read_feed() @@ -438,6 +468,21 @@ def main() -> None: continue print(element) + elif args.subcommand == RVCommand.watch: + cache = RVDatabase.load() + if not cache: + raise FileNotFoundError("This command doesn't work without a cache yet.") + elements = cache.elements.copy() + if args.order == "new": + elements = reversed(elements) + elif args.order == "random": + random.shuffle(elements) + for element in elements: + if not element.matches_search(args): + continue + element.watch() + break + if __name__ == "__main__": main() From 07af9360fa74507bbcb12bcbdacf193cf8ad7f50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Sat, 18 Dec 2021 11:27:24 +0100 Subject: [PATCH 23/34] rssVideos: Abstract a bit, add binge --- config/scripts/rssVideos | 117 ++++++++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 46 deletions(-) diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index b58efd3..261707a 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -20,6 +20,7 @@ import sys import typing import urllib.parse import urllib.request +import urllib.error from xml.dom import minidom import coloredlogs @@ -46,6 +47,7 @@ class RVCommand(enum.Enum): download = "download" list = "list" watch = "watch" + binge = "binge" class RVElement: @@ -100,7 +102,7 @@ class RVElement: def guid(self) -> int: return int(self.get_tag_data("guid")) - def read_cache(self, cache: "RVElement") -> None: + def salvage_cache(self, cache: "RVElement") -> None: if "ytdl_infos" in cache.__dict__: self.__dict__["ytdl_infos"] = cache.__dict__["ytdl_infos"] log.debug(f"From cache: {self}") @@ -127,7 +129,7 @@ class RVElement: raise e except youtube_dl.utils.DownloadError as e: # TODO Still raise in case of temporary network issue - log.warn(e) + log.warning(e) infos = None # Apparently that thing is transformed from a LazyList # somewhere in the normal yt_dlp process @@ -208,19 +210,19 @@ class RVElement: None: int.__le__, } - def matches_search(self, args: configargparse.Namespace) -> bool: - if not self.is_video: - return False + def matches_filter(self, args: configargparse.Namespace) -> bool: if self.watched: return False if args.title and not re.search(args.title, self.title): return False - if args.creator and not re.search(args.creator, self.creator): - return False if args.guid and not re.search(args.guid, str(self.guid)): return False if args.link and not re.search(args.link, self.link): return False + if args.creator and self.creator and not re.search(args.creator, self.creator): + return False + if not self.is_video: + return False if args.duration: dur = args.duration @@ -253,8 +255,11 @@ class RVElement: if not self.downloaded: self.download() - proc = subprocess.run(['mpv', self.filepath]) - proc.check_returncode() + cmd = ["mpv", self.filepath] + log.debug(f"Running {cmd}") + if not self.parent.args.dryrun: + proc = subprocess.run(cmd) + proc.check_returncode() self.watched = True self.parent.save() @@ -282,27 +287,33 @@ class RVDatabase: with open(cls.SAVE_FILE, "rb") as save_file: return pickle.load(save_file) except (TypeError, AttributeError, EOFError): - log.warn("Corrupt / outdated cache, it will be rebuilt.") + log.warning("Corrupt / outdated cache, it will be rebuilt.") except FileNotFoundError: pass return None - def read_cache(self, cache: "RVDatabase") -> None: + def salvage_cache(self, cache: "RVDatabase") -> None: + log.debug(f"Salvaging cache") cache_els = dict() for cache_el in cache.elements: cache_els[cache_el.guid] = cache_el for el in self.elements: if el.guid in cache_els: - el.read_cache(cache_els[el.guid]) + el.salvage_cache(cache_els[el.guid]) + + def import_cache(self, cache: "RVDatabase") -> None: + log.debug(f"Importing cache") + self.feed_xml = cache.feed_xml + self.read_feed() @functools.cached_property def feed_xml(self) -> minidom.Document: + log.info("Fetching RSS feed") with urllib.request.urlopen(self.args.feed) as request: return minidom.parse(request) def read_feed(self) -> None: - log.info("Fetching RSS feed") - self.elements = list() + self.elements = [] for item in self.feed_xml.getElementsByTagName("item"): element = RVElement(self, item) self.elements.insert(0, element) @@ -348,6 +359,18 @@ class RVDatabase: def ytdl_dry(self) -> youtube_dl.YoutubeDL: return youtube_dl.YoutubeDL(self.ytdl_dry_opts) + def filter(self, args: configargparse.Namespace) -> typing.Iterable[RVElement]: + elements: typing.Iterable[RVElement] + if args.order == "old": + elements = self.elements + elif args.order == "new": + elements = reversed(self.elements) + elif args.order == "random": + elements_random = self.elements.copy() + random.shuffle(elements_random) + elements = elements_random + return filter(lambda el: el.matches_filter(args), elements) + def get_args() -> configargparse.Namespace: defaultConfigPath = os.path.join( @@ -422,17 +445,25 @@ def get_args() -> configargparse.Namespace: sc_watch = subparsers.add_parser("watch") sc_watch.set_defaults(subcommand=RVCommand.watch) - sc_watch.add("order", choices=("old", "new", "random"), nargs="?", default="old", help="Watch X first") - # TODO Command to watch multiple + sc_binge = subparsers.add_parser("binge") + sc_binge.set_defaults(subcommand=RVCommand.binge) # Common arguments for filtering - for sc in (sc_list, sc_watch): + for sc in (sc_list, sc_watch, sc_binge): + sc.add( + "order", + choices=("old", "new", "random"), + nargs="?", + default="old", + help="Sorting mechanism", + ) sc.add("--guid", help="Regex to filter guid") sc.add("--creator", help="Regex to filter by creator") sc.add("--title", help="Regex to filter by title") sc.add("--link", help="Regex to filter by link") sc.add("--duration", help="Comparative to filter by duration") + # TODO Allow to ask args = parser.parse_args() args.videos = os.path.realpath(os.path.expanduser(args.videos)) @@ -447,41 +478,35 @@ def main() -> None: os.makedirs(args.videos, exist_ok=True) os.chdir(args.videos) - # TODO Abstract a bit + database = RVDatabase(args) + cache = RVDatabase.load() + try: + database.read_feed() + except urllib.error.URLError as err: + if args.subcommand == RVCommand.download or not cache: + raise err + else: + log.warning("Cannot fetch RSS feed, using cached feed.", err) + database.import_cache(cache) + if cache: + database.salvage_cache(cache) + + log.debug(f"Running subcommand") if args.subcommand == RVCommand.download: - database = RVDatabase(args) - database.read_feed() - cache = RVDatabase.load() - if cache: - database.read_cache(cache) database.clean() database.act_all() - database.save() - elif args.subcommand == RVCommand.list: - cache = RVDatabase.load() - if not cache: - raise FileNotFoundError("This command doesn't work without a cache yet.") - for element in cache.elements: - if not element.matches_search(args): - continue - print(element) + elif args.subcommand in (RVCommand.list, RVCommand.watch, RVCommand.binge): + for element in database.filter(args): + if args.subcommand == RVCommand.list: + print(element) + elif args.subcommand in (RVCommand.watch, RVCommand.binge): + element.watch() + if args.subcommand == RVCommand.watch: + break - elif args.subcommand == RVCommand.watch: - cache = RVDatabase.load() - if not cache: - raise FileNotFoundError("This command doesn't work without a cache yet.") - elements = cache.elements.copy() - if args.order == "new": - elements = reversed(elements) - elif args.order == "random": - random.shuffle(elements) - for element in elements: - if not element.matches_search(args): - continue - element.watch() - break + database.save() if __name__ == "__main__": From 2dce725ee5965daa18cf6e9512a46231595de482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Sat, 18 Dec 2021 11:56:28 +0100 Subject: [PATCH 24/34] rssVideos: Abstract with download process as well --- config/scripts/rssVideos | 126 ++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 75 deletions(-) diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index 261707a..45eef2b 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -43,13 +43,6 @@ def configure_logging(args: configargparse.Namespace) -> None: ) -class RVCommand(enum.Enum): - download = "download" - list = "list" - watch = "watch" - binge = "binge" - - class RVElement: parent: "RVDatabase" item: minidom.Element @@ -150,16 +143,6 @@ class RVElement: assert self.ytdl_infos return self.ytdl_infos["duration"] - @property - def skip(self) -> bool: - assert self.is_video - if ( - self.parent.args.max_duration > 0 - and self.duration > self.parent.args.max_duration - ): - return True - return False - @property def is_video(self) -> bool: # Duration might be missing in playlists and stuff @@ -194,9 +177,6 @@ class RVElement: if self.was_downloaded: log.debug(f"Downloaded previously: {self}") return - if self.skip: - log.debug(f"Skipped: {self}") - return self.download() MATCHES_DURATION_MULTIPLIERS = {"s": 1, "m": 60, "h": 3600, None: 1} @@ -320,9 +300,10 @@ class RVDatabase: log.debug(f"Known: {element}") def clean(self) -> None: + log.debug("Cleaning") filenames = set() for element in self.elements: - if element.is_video and not element.skip: + if element.is_video: filenames.add(element.filename) for file in os.listdir(): if file == RVDatabase.SAVE_FILE: @@ -337,10 +318,6 @@ class RVDatabase: if not self.args.dryrun: os.unlink(file) - def act_all(self) -> None: - for element in self.elements: - element.act() - @property def ytdl_opts(self) -> dict: return {"format": self.args.format, "allsubtitles": self.args.subtitles} @@ -383,6 +360,8 @@ def get_args() -> configargparse.Namespace: + "an RSS aggregator", default_config_files=[defaultConfigPath], ) + + # Runtime settings parser.add_argument( "-v", "--verbosity", @@ -393,6 +372,16 @@ def get_args() -> configargparse.Namespace: parser.add( "-c", "--config", required=False, is_config_file=True, help="Configuration file" ) + parser.add( + "-n", + "--dryrun", + help="Only pretend to do actions", + action="store_const", + const=True, + default=False, + ) + + # Input/Output parser.add( "--feed", help="URL of the RSS feed (must be public for now)", @@ -405,21 +394,30 @@ def get_args() -> configargparse.Namespace: env_var="RSS_VIDEOS_VIDEO_DIR", required=True, ) + + # Which videos parser.add( - "-n", - "--dryrun", - help="Do not download the videos", - action="store_const", - const=True, - default=False, + "--order", + choices=("old", "new", "random"), + default="old", + help="Sorting mechanism", ) + parser.add("--guid", help="Regex to filter guid") + parser.add("--creator", help="Regex to filter by creator") + parser.add("--title", help="Regex to filter by title") + parser.add("--link", help="Regex to filter by link") + parser.add("--duration", help="Comparative to filter by duration") + # TODO Envrionment variables parser.add( "--max-duration", - help="Skip video longer than this amount of seconds", + help="(Deprecated, use --duration instead)", env_var="RSS_VIDEOS_MAX_DURATION", type=int, default=0, ) + # TODO Allow to ask + + # How to download parser.add( "--format", help="Use this format to download videos." @@ -434,39 +432,17 @@ def get_args() -> configargparse.Namespace: action="store_true", ) - parser.set_defaults(subcommand=RVCommand.download) - subparsers = parser.add_subparsers(title="subcommand") - - sc_download = subparsers.add_parser("download") - sc_download.set_defaults(subcommand=RVCommand.download) - - sc_list = subparsers.add_parser("list") - sc_list.set_defaults(subcommand=RVCommand.list) - - sc_watch = subparsers.add_parser("watch") - sc_watch.set_defaults(subcommand=RVCommand.watch) - - sc_binge = subparsers.add_parser("binge") - sc_binge.set_defaults(subcommand=RVCommand.binge) - - # Common arguments for filtering - for sc in (sc_list, sc_watch, sc_binge): - sc.add( - "order", - choices=("old", "new", "random"), - nargs="?", - default="old", - help="Sorting mechanism", - ) - sc.add("--guid", help="Regex to filter guid") - sc.add("--creator", help="Regex to filter by creator") - sc.add("--title", help="Regex to filter by title") - sc.add("--link", help="Regex to filter by link") - sc.add("--duration", help="Comparative to filter by duration") - # TODO Allow to ask + parser.add( + "action", + nargs="?", + choices=("download", "list", "watch", "binge"), + default="download", + ) args = parser.parse_args() args.videos = os.path.realpath(os.path.expanduser(args.videos)) + if not args.duration and args.max_duration: + args.duration = str(args.max_duration) return args @@ -483,7 +459,7 @@ def main() -> None: try: database.read_feed() except urllib.error.URLError as err: - if args.subcommand == RVCommand.download or not cache: + if args.action == "download" or not cache: raise err else: log.warning("Cannot fetch RSS feed, using cached feed.", err) @@ -491,20 +467,20 @@ def main() -> None: if cache: database.salvage_cache(cache) - log.debug(f"Running subcommand") - - if args.subcommand == RVCommand.download: + if args.action == "download": + # TODO Clean on watch? / cache import with missing video / all researched database.clean() - database.act_all() - elif args.subcommand in (RVCommand.list, RVCommand.watch, RVCommand.binge): - for element in database.filter(args): - if args.subcommand == RVCommand.list: - print(element) - elif args.subcommand in (RVCommand.watch, RVCommand.binge): - element.watch() - if args.subcommand == RVCommand.watch: - break + log.debug(f"Running action") + for element in database.filter(args): + if args.action == "download": + element.act() + elif args.action == "list": + print(element) + elif args.action in ("watch", "binge"): + element.watch() + if args.action == "watch": + break database.save() From 1948fc0af2e41b7732686693886e37bc2e756dfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Sat, 18 Dec 2021 12:44:43 +0100 Subject: [PATCH 25/34] rssVideos: Cleverer cleaning --- config/scripts/rssVideos | 88 +++++++++++++++++++++++++++++----------- 1 file changed, 65 insertions(+), 23 deletions(-) diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index 45eef2b..71ba5a6 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -29,6 +29,7 @@ import yt_dlp as youtube_dl log = logging.getLogger(__name__) +# TODO Lockfile, or a way to parallel watch and download def configure_logging(args: configargparse.Namespace) -> None: # Configure logging @@ -95,8 +96,12 @@ class RVElement: def guid(self) -> int: return int(self.get_tag_data("guid")) + @property + def is_researched(self) -> bool: + return "ytdl_infos" in self.__dict__ + def salvage_cache(self, cache: "RVElement") -> None: - if "ytdl_infos" in cache.__dict__: + if cache.is_researched: self.__dict__["ytdl_infos"] = cache.__dict__["ytdl_infos"] log.debug(f"From cache: {self}") if cache.was_downloaded: @@ -109,7 +114,7 @@ class RVElement: @property def downloaded(self) -> bool: - if "ytdl_infos" not in self.__dict__: + if not self.is_researched: return False return os.path.isfile(self.filepath) @@ -167,10 +172,8 @@ class RVElement: self.was_downloaded = True self.parent.save() - def act(self) -> None: - if not self.is_video: - log.debug(f"Not a video: {self}") - return + def preload(self) -> None: + assert self.is_video if self.downloaded: log.debug(f"Currently downloaded: {self}") return @@ -192,16 +195,22 @@ class RVElement: def matches_filter(self, args: configargparse.Namespace) -> bool: if self.watched: + log.debug(f"Already watched: {self}") return False if args.title and not re.search(args.title, self.title): + log.debug(f"Title not matching {args.title}: {self}") return False if args.guid and not re.search(args.guid, str(self.guid)): + log.debug(f"Guid not matching {args.guid}: {self}") return False if args.link and not re.search(args.link, self.link): + log.debug(f"Link not matching {args.link}: {self}") return False - if args.creator and self.creator and not re.search(args.creator, self.creator): + if args.creator and (not self.creator or not re.search(args.creator, self.creator)): + log.debug(f"Creator not matching {args.creator}: {self}") return False if not self.is_video: + log.debug(f"Not a video: {self}") return False if args.duration: dur = args.duration @@ -228,6 +237,7 @@ class RVElement: duration = int(dur) if not comparator(self.duration, duration * multiplier): + log.debug(f"Duration {self.duration} not matching {args.duration}: {self}") return False return True @@ -244,6 +254,15 @@ class RVElement: self.watched = True self.parent.save() + def clean(self) -> None: + assert self.is_video + log.info(f"Removing gone video: {self.filename}*") + for file in os.listdir(): + if file.startswith(self.filename): + log.debug(f"Removing file: {file}") + if not self.parent.args.dryrun: + os.unlink(file) + class RVDatabase: SAVE_FILE = ".cache.p" @@ -281,6 +300,16 @@ class RVDatabase: if el.guid in cache_els: el.salvage_cache(cache_els[el.guid]) + def clean_cache(self, cache: "RVDatabase") -> None: + log.debug(f"Cleaning cache") + self_els = dict() + for self_el in self.elements: + self_els[self_el.guid] = self_el + for el in cache.elements: + if el.guid not in self_els: + if el.is_researched and el.is_video: + el.clean() + def import_cache(self, cache: "RVDatabase") -> None: log.debug(f"Importing cache") self.feed_xml = cache.feed_xml @@ -314,10 +343,21 @@ class RVDatabase: if file.startswith(filename): break else: - log.info(f"Removing: {file}") + log.info(f"Removing unknown file: {file}") if not self.args.dryrun: os.unlink(file) + @property + def all_researched(self) -> bool: + for element in self.elements: + if not element.is_researched: + return False + return True + + def attempt_clean(self) -> None: + if self.all_researched: + self.clean() + @property def ytdl_opts(self) -> dict: return {"format": self.args.format, "allsubtitles": self.args.subtitles} @@ -435,7 +475,7 @@ def get_args() -> configargparse.Namespace: parser.add( "action", nargs="?", - choices=("download", "list", "watch", "binge"), + choices=("download", "list", "watch", "binge", "clean"), default="download", ) @@ -466,22 +506,24 @@ def main() -> None: database.import_cache(cache) if cache: database.salvage_cache(cache) - - if args.action == "download": - # TODO Clean on watch? / cache import with missing video / all researched - database.clean() + database.clean_cache(cache) + database.save() log.debug(f"Running action") - for element in database.filter(args): - if args.action == "download": - element.act() - elif args.action == "list": - print(element) - elif args.action in ("watch", "binge"): - element.watch() - if args.action == "watch": - break - + if args.action == "clean": + database.clean() + else: + database.attempt_clean() + for element in database.filter(args): + if args.action == "download": + element.preload() + elif args.action == "list": + print(element) + elif args.action in ("watch", "binge"): + element.watch() + if args.action == "watch": + break + database.attempt_clean() database.save() From f4c81e346a0182d21fde04cfc52613ebccdf959e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Sat, 18 Dec 2021 22:23:48 +0100 Subject: [PATCH 26/34] rssVideos: Add --seen flag --- config/scripts/rssVideos | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index 71ba5a6..ad5943f 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -194,8 +194,8 @@ class RVElement: } def matches_filter(self, args: configargparse.Namespace) -> bool: - if self.watched: - log.debug(f"Already watched: {self}") + if args.seen != "any" and (args.seen == "seen") != self.watched: + log.debug(f"Not {args.seen}: {self}") return False if args.title and not re.search(args.title, self.title): log.debug(f"Title not matching {args.title}: {self}") @@ -447,6 +447,7 @@ def get_args() -> configargparse.Namespace: parser.add("--title", help="Regex to filter by title") parser.add("--link", help="Regex to filter by link") parser.add("--duration", help="Comparative to filter by duration") + parser.add("--seen", choices=("seen","unseen","any"), default="unseen", help="Only include seen/unseen/any videos") # TODO Envrionment variables parser.add( "--max-duration", From 48905556680eaa64d428a29e00c3c42c6bebc1a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Sun, 19 Dec 2021 10:59:02 +0100 Subject: [PATCH 27/34] rssVideos: Can toggle seen/unseen video state --- config/scripts/rssVideos | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index ad5943f..03495b9 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -476,7 +476,7 @@ def get_args() -> configargparse.Namespace: parser.add( "action", nargs="?", - choices=("download", "list", "watch", "binge", "clean"), + choices=("download", "list", "watch", "binge", "clean", "seen", "unseen"), default="download", ) @@ -522,6 +522,10 @@ def main() -> None: print(element) elif args.action in ("watch", "binge"): element.watch() + elif args.action == "seen": + element.watched = True + elif args.action == "unseen": + element.watched = False if args.action == "watch": break database.attempt_clean() From 9684586eecd64f1f1a0f4897dc6cbfebc5f867d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Sun, 19 Dec 2021 11:45:41 +0100 Subject: [PATCH 28/34] rssVideos: More sort orders and duration command --- config/scripts/rssVideos | 67 ++++++++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index 03495b9..bfc5180 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -17,6 +17,7 @@ import random import re import subprocess import sys +import time import typing import urllib.parse import urllib.request @@ -43,6 +44,9 @@ def configure_logging(args: configargparse.Namespace) -> None: logger=log, ) +def format_duration(duration: int) -> int: + return time.strftime("%H:%M:%S", time.gmtime(duration)) + class RVElement: parent: "RVDatabase" @@ -101,7 +105,7 @@ class RVElement: return "ytdl_infos" in self.__dict__ def salvage_cache(self, cache: "RVElement") -> None: - if cache.is_researched: + if not self.parent.args.research and cache.is_researched: self.__dict__["ytdl_infos"] = cache.__dict__["ytdl_infos"] log.debug(f"From cache: {self}") if cache.was_downloaded: @@ -110,7 +114,16 @@ class RVElement: self.watched = True def __str__(self) -> str: - return f"{self.guid}: {self.creator} – {self.title} – {self.link}" + str = f"{self.guid}: {self.creator if self.creator else '?'} – {self.title}" + if self.is_researched: + if self.is_video: + str += f" ({format_duration(self.duration)})" + else: + str += " (N/A)" + else: + str += " (?)" + str += f" – {self.link}" + return str @property def downloaded(self) -> bool: @@ -194,6 +207,7 @@ class RVElement: } def matches_filter(self, args: configargparse.Namespace) -> bool: + # Inexpensive filters if args.seen != "any" and (args.seen == "seen") != self.watched: log.debug(f"Not {args.seen}: {self}") return False @@ -209,6 +223,8 @@ class RVElement: if args.creator and (not self.creator or not re.search(args.creator, self.creator)): log.debug(f"Creator not matching {args.creator}: {self}") return False + + # Expensive filters if not self.is_video: log.debug(f"Not a video: {self}") return False @@ -239,6 +255,7 @@ class RVElement: if not comparator(self.duration, duration * multiplier): log.debug(f"Duration {self.duration} not matching {args.duration}: {self}") return False + return True def watch(self) -> None: @@ -378,15 +395,32 @@ class RVDatabase: def filter(self, args: configargparse.Namespace) -> typing.Iterable[RVElement]: elements: typing.Iterable[RVElement] - if args.order == "old": - elements = self.elements - elif args.order == "new": + # Inexpensive sort + if args.order == "new": elements = reversed(self.elements) + elif args.order == "title": + elements = sorted(self.elements, key=lambda el: el.title) + elif args.order == "creator": + elements = sorted(self.elements, key=lambda el: el.creator or '') + elif args.order == "link": + elements = sorted(self.elements, key=lambda el: el.link) elif args.order == "random": elements_random = self.elements.copy() random.shuffle(elements_random) elements = elements_random - return filter(lambda el: el.matches_filter(args), elements) + else: + elements = self.elements + + # Possibly expensive filtering + elements = filter(lambda el: el.matches_filter(args), elements) + + # Expensive sort + if args.order == "short": + elements = sorted(elements, key=lambda el: el.duration if el.is_video else 0) + elif args.order == "short": + elements = sorted(elements, key=lambda el: el.duration if el.is_video else 0, reverse=True) + + return elements def get_args() -> configargparse.Namespace: @@ -428,6 +462,11 @@ def get_args() -> configargparse.Namespace: env_var="RSS_VIDEOS_FEED", required=True, ) + parser.add( + "--research", + help="Fetch video info again", + action="store_true", + ) parser.add( "--videos", help="Directory to store videos", @@ -438,7 +477,7 @@ def get_args() -> configargparse.Namespace: # Which videos parser.add( "--order", - choices=("old", "new", "random"), + choices=("old", "new", "title", "creator", "link", "short", "long", "random"), default="old", help="Sorting mechanism", ) @@ -476,7 +515,7 @@ def get_args() -> configargparse.Namespace: parser.add( "action", nargs="?", - choices=("download", "list", "watch", "binge", "clean", "seen", "unseen"), + choices=("download", "list", "watch", "binge", "clean", "seen", "unseen", "duration"), default="download", ) @@ -515,6 +554,8 @@ def main() -> None: database.clean() else: database.attempt_clean() + if args.action == "duration": + duration = 0 for element in database.filter(args): if args.action == "download": element.preload() @@ -522,12 +563,18 @@ def main() -> None: print(element) elif args.action in ("watch", "binge"): element.watch() + if args.action == "watch": + break elif args.action == "seen": element.watched = True elif args.action == "unseen": element.watched = False - if args.action == "watch": - break + elif args.action == "duration": + duration += element.duration + else: + raise NotImplementedError(f"Unimplemented action: {args.action}") + if args.action == "duration": + print(format_duration(duration)) database.attempt_clean() database.save() From daff602a319e7c37b79611ced2d44ae2b566fb3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Sun, 19 Dec 2021 15:10:16 +0100 Subject: [PATCH 29/34] rssVideos: Work correctly with merged files --- config/scripts/rssVideos | 92 +++++++++++++++++++++++++++++----------- 1 file changed, 67 insertions(+), 25 deletions(-) diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index bfc5180..0f740da 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -26,12 +26,13 @@ from xml.dom import minidom import coloredlogs import configargparse -import yt_dlp as youtube_dl +import yt_dlp log = logging.getLogger(__name__) # TODO Lockfile, or a way to parallel watch and download + def configure_logging(args: configargparse.Namespace) -> None: # Configure logging if args.verbosity: @@ -44,7 +45,25 @@ def configure_logging(args: configargparse.Namespace) -> None: logger=log, ) -def format_duration(duration: int) -> int: + +class SaveInfoPP(yt_dlp.postprocessor.common.PostProcessor): + """ + yt_dlp.process_ie_result() doesn't return a completely updated info dict, + notably the extension is still the one before it realizes the files cannot + be merged. So we use this PostProcessor to catch the info dict in its final + form and save it. + """ + + def __init__(self, rvelement: "RVElement") -> None: + self.rvelement = rvelement + super().__init__() + + def run(self, info: dict) -> tuple[list, dict]: + self.rvelement.ytdl_infos = info + return [], info + + +def format_duration(duration: int) -> str: return time.strftime("%H:%M:%S", time.gmtime(duration)) @@ -108,8 +127,8 @@ class RVElement: if not self.parent.args.research and cache.is_researched: self.__dict__["ytdl_infos"] = cache.__dict__["ytdl_infos"] log.debug(f"From cache: {self}") - if cache.was_downloaded: - self.was_downloaded = True + # if cache.was_downloaded: + # self.was_downloaded = True if cache.watched: self.watched = True @@ -135,10 +154,10 @@ class RVElement: def ytdl_infos(self) -> typing.Optional[dict]: log.info(f"Researching: {self}") try: - infos = self.parent.ytdl_dry.extract_info(self.link) + infos = self.parent.ytdl_dry.extract_info(self.link, download=False) except KeyboardInterrupt as e: raise e - except youtube_dl.utils.DownloadError as e: + except yt_dlp.utils.DownloadError as e: # TODO Still raise in case of temporary network issue log.warning(e) infos = None @@ -147,7 +166,7 @@ class RVElement: if ( infos and "thumbnails" in infos - and isinstance(infos["thumbnails"], youtube_dl.utils.LazyList) + and isinstance(infos["thumbnails"], yt_dlp.utils.LazyList) ): infos["thumbnails"] = infos["thumbnails"].exhaust() # Save database once it's been computed @@ -169,7 +188,6 @@ class RVElement: @property def filepath(self) -> str: assert self.is_video - # TODO This doesn't change the extension to mkv when the formats are incomaptible return self.parent.ytdl_dry.prepare_filename(self.ytdl_infos) @property @@ -181,7 +199,9 @@ class RVElement: assert self.is_video log.info(f"Downloading: {self}") if not self.parent.args.dryrun: - self.parent.ytdl.process_ie_result(self.ytdl_infos, True, {}) + with yt_dlp.YoutubeDL(self.parent.ytdl_opts) as ydl: + ydl.add_post_processor(SaveInfoPP(self)) + ydl.process_ie_result(self.ytdl_infos, download=True) self.was_downloaded = True self.parent.save() @@ -220,7 +240,9 @@ class RVElement: if args.link and not re.search(args.link, self.link): log.debug(f"Link not matching {args.link}: {self}") return False - if args.creator and (not self.creator or not re.search(args.creator, self.creator)): + if args.creator and ( + not self.creator or not re.search(args.creator, self.creator) + ): log.debug(f"Creator not matching {args.creator}: {self}") return False @@ -253,7 +275,9 @@ class RVElement: duration = int(dur) if not comparator(self.duration, duration * multiplier): - log.debug(f"Duration {self.duration} not matching {args.duration}: {self}") + log.debug( + f"Duration {self.duration} not matching {args.duration}: {self}" + ) return False return True @@ -382,16 +406,12 @@ class RVDatabase: @property def ytdl_dry_opts(self) -> dict: opts = self.ytdl_opts.copy() - opts.update({"simulate": True, "quiet": True}) + opts.update({"quiet": True}) return opts @property - def ytdl(self) -> youtube_dl.YoutubeDL: - return youtube_dl.YoutubeDL(self.ytdl_opts) - - @property - def ytdl_dry(self) -> youtube_dl.YoutubeDL: - return youtube_dl.YoutubeDL(self.ytdl_dry_opts) + def ytdl_dry(self) -> yt_dlp.YoutubeDL: + return yt_dlp.YoutubeDL(self.ytdl_dry_opts) def filter(self, args: configargparse.Namespace) -> typing.Iterable[RVElement]: elements: typing.Iterable[RVElement] @@ -401,7 +421,7 @@ class RVDatabase: elif args.order == "title": elements = sorted(self.elements, key=lambda el: el.title) elif args.order == "creator": - elements = sorted(self.elements, key=lambda el: el.creator or '') + elements = sorted(self.elements, key=lambda el: el.creator or "") elif args.order == "link": elements = sorted(self.elements, key=lambda el: el.link) elif args.order == "random": @@ -416,9 +436,13 @@ class RVDatabase: # Expensive sort if args.order == "short": - elements = sorted(elements, key=lambda el: el.duration if el.is_video else 0) + elements = sorted( + elements, key=lambda el: el.duration if el.is_video else 0 + ) elif args.order == "short": - elements = sorted(elements, key=lambda el: el.duration if el.is_video else 0, reverse=True) + elements = sorted( + elements, key=lambda el: el.duration if el.is_video else 0, reverse=True + ) return elements @@ -486,7 +510,12 @@ def get_args() -> configargparse.Namespace: parser.add("--title", help="Regex to filter by title") parser.add("--link", help="Regex to filter by link") parser.add("--duration", help="Comparative to filter by duration") - parser.add("--seen", choices=("seen","unseen","any"), default="unseen", help="Only include seen/unseen/any videos") + parser.add( + "--seen", + choices=("seen", "unseen", "any"), + default="unseen", + help="Only include seen/unseen/any videos", + ) # TODO Envrionment variables parser.add( "--max-duration", @@ -515,7 +544,16 @@ def get_args() -> configargparse.Namespace: parser.add( "action", nargs="?", - choices=("download", "list", "watch", "binge", "clean", "seen", "unseen", "duration"), + choices=( + "download", + "list", + "watch", + "binge", + "clean", + "seen", + "unseen", + "duration", + ), default="download", ) @@ -566,9 +604,13 @@ def main() -> None: if args.action == "watch": break elif args.action == "seen": - element.watched = True + if not element.watched: + log.info(f"Maked as seen: {element}") + element.watched = True elif args.action == "unseen": - element.watched = False + if element.watched: + log.info(f"Maked as unseen: {element}") + element.watched = False elif args.action == "duration": duration += element.duration else: From 00a9da6afc078f9dc1e6569051e74a7f22d60d1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Sun, 19 Dec 2021 22:29:16 +0100 Subject: [PATCH 30/34] rssVideos: Allow skipping feed fetching For dev speed --- config/scripts/rssVideos | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index 0f740da..93a1723 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -31,7 +31,7 @@ import yt_dlp log = logging.getLogger(__name__) # TODO Lockfile, or a way to parallel watch and download - +# TODO Save ytdl infos and view info separately def configure_logging(args: configargparse.Namespace) -> None: # Configure logging @@ -491,6 +491,12 @@ def get_args() -> configargparse.Namespace: help="Fetch video info again", action="store_true", ) + parser.add( + "--no-refresh", + dest="refresh", + help="Don't fetch feed", + action="store_false", + ) parser.add( "--videos", help="Directory to store videos", @@ -574,14 +580,22 @@ def main() -> None: database = RVDatabase(args) cache = RVDatabase.load() - try: - database.read_feed() - except urllib.error.URLError as err: - if args.action == "download" or not cache: - raise err - else: - log.warning("Cannot fetch RSS feed, using cached feed.", err) + feed_fetched = False + if args.refresh: + try: + database.read_feed() + feed_fetched = True + except urllib.error.URLError as err: + if args.action == "download": + raise RuntimeError("Couldn't fetch feed, refusing to download") + # This is a quirky failsafe in case of no internet connection, + # so the script doesn't go noting that no element is a video. + if not feed_fetched: + if cache: + log.warning("Using cached feed.") database.import_cache(cache) + else: + raise FileNotFoundError("Feed not fetched and no cached feed.") if cache: database.salvage_cache(cache) database.clean_cache(cache) From 5b7926df8fc61cf46c7c06f1344fc1bc4aec716e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Sun, 19 Dec 2021 23:13:41 +0100 Subject: [PATCH 31/34] rssVideos: --total-duration Controleld binging --- config/scripts/rssVideos | 116 +++++++++++++++++++++++---------------- 1 file changed, 68 insertions(+), 48 deletions(-) diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index 93a1723..0954ea4 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -62,6 +62,45 @@ class SaveInfoPP(yt_dlp.postprocessor.common.PostProcessor): self.rvelement.ytdl_infos = info return [], info +def parse_duration(string: str) -> int: + DURATION_MULTIPLIERS = {"s": 1, "m": 60, "h": 3600, "": 1} + + mult_index = string[-1].lower() + if mult_index.isdigit(): + mult_index = "" + else: + string = string[:-1] + try: + multiplier = DURATION_MULTIPLIERS[mult_index] + except IndexError: + raise ValueError(f"Unknown duration multiplier: {mult_index}") + + return int(string) * multiplier + + +def compare_duration(compstr: str) -> typing.Callable[[int], bool]: + DURATION_COMPARATORS = { + "<": int.__lt__, + "-": int.__lt__, + ">": int.__gt__, + "+": int.__gt__, + "=": int.__eq__, + "": int.__le__, + } + + comp_index = compstr[0] + if comp_index.isdigit(): + comp_index = "" + else: + compstr = compstr[1:] + try: + comparator = DURATION_COMPARATORS[comp_index] + except IndexError: + raise ValueError(f"Unknown duration comparator: {comp_index}") + + duration = parse_duration(compstr) + + return lambda d: comparator(d, duration) def format_duration(duration: int) -> str: return time.strftime("%H:%M:%S", time.gmtime(duration)) @@ -152,6 +191,7 @@ class RVElement: @functools.cached_property def ytdl_infos(self) -> typing.Optional[dict]: + # TODO Sanitize according to documentation log.info(f"Researching: {self}") try: infos = self.parent.ytdl_dry.extract_info(self.link, download=False) @@ -215,17 +255,6 @@ class RVElement: return self.download() - MATCHES_DURATION_MULTIPLIERS = {"s": 1, "m": 60, "h": 3600, None: 1} - - MATCHES_DURATION_COMPARATORS = { - "<": int.__lt__, - "-": int.__lt__, - ">": int.__gt__, - "+": int.__gt__, - "=": int.__eq__, - None: int.__le__, - } - def matches_filter(self, args: configargparse.Namespace) -> bool: # Inexpensive filters if args.seen != "any" and (args.seen == "seen") != self.watched: @@ -250,35 +279,11 @@ class RVElement: if not self.is_video: log.debug(f"Not a video: {self}") return False - if args.duration: - dur = args.duration - - mult_index = dur[-1].lower() - if mult_index.isdigit(): - mult_index = None - else: - dur = dur[:-1] - try: - multiplier = self.MATCHES_DURATION_MULTIPLIERS[mult_index] - except IndexError: - raise ValueError(f"Unknown duration multiplier: {mult_index}") - - comp_index = dur[0] - if comp_index.isdigit(): - comp_index = None - else: - dur = dur[1:] - try: - comparator = self.MATCHES_DURATION_COMPARATORS[comp_index] - except IndexError: - raise ValueError(f"Unknown duration comparator: {comp_index}") - - duration = int(dur) - if not comparator(self.duration, duration * multiplier): - log.debug( - f"Duration {self.duration} not matching {args.duration}: {self}" - ) - return False + if args.duration and not compare_duration(args.duration)(self.duration): + log.debug( + f"Duration {self.duration} not matching {args.duration}: {self}" + ) + return False return True @@ -439,11 +444,26 @@ class RVDatabase: elements = sorted( elements, key=lambda el: el.duration if el.is_video else 0 ) - elif args.order == "short": + elif args.order == "long": elements = sorted( elements, key=lambda el: el.duration if el.is_video else 0, reverse=True ) + # Post sorting filtering + if args.total_duration: + rem = parse_duration(args.total_duration) + old_els = list(elements) + elements = list() + while rem > 0: + for el in old_els: + if el.duration < rem: + elements.append(el) + rem -= el.duration + old_els.remove(el) + break + else: + break + return elements @@ -522,6 +542,10 @@ def get_args() -> configargparse.Namespace: default="unseen", help="Only include seen/unseen/any videos", ) + parser.add( + "--total-duration", + help="Use videos that fit under the total given", + ) # TODO Envrionment variables parser.add( "--max-duration", @@ -558,7 +582,6 @@ def get_args() -> configargparse.Namespace: "clean", "seen", "unseen", - "duration", ), default="download", ) @@ -606,8 +629,7 @@ def main() -> None: database.clean() else: database.attempt_clean() - if args.action == "duration": - duration = 0 + duration = 0 for element in database.filter(args): if args.action == "download": element.preload() @@ -625,12 +647,10 @@ def main() -> None: if element.watched: log.info(f"Maked as unseen: {element}") element.watched = False - elif args.action == "duration": - duration += element.duration else: raise NotImplementedError(f"Unimplemented action: {args.action}") - if args.action == "duration": - print(format_duration(duration)) + duration += element.duration if element.is_video else 0 + log.info(f"Total duration: {format_duration(duration)}") database.attempt_clean() database.save() From 105bd9461c8f1e2337616bddd1f1db432212741d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Mon, 20 Dec 2021 18:57:13 +0100 Subject: [PATCH 32/34] rssVideos: Better sanitization of ytdl info --- config/scripts/rssVideos | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index 0954ea4..35e8284 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -51,7 +51,7 @@ class SaveInfoPP(yt_dlp.postprocessor.common.PostProcessor): yt_dlp.process_ie_result() doesn't return a completely updated info dict, notably the extension is still the one before it realizes the files cannot be merged. So we use this PostProcessor to catch the info dict in its final - form and save it. + form and save what we need from it (it's not serializable in this state). """ def __init__(self, rvelement: "RVElement") -> None: @@ -59,7 +59,7 @@ class SaveInfoPP(yt_dlp.postprocessor.common.PostProcessor): super().__init__() def run(self, info: dict) -> tuple[list, dict]: - self.rvelement.ytdl_infos = info + self.rvelement.update_post_download(info) return [], info def parse_duration(string: str) -> int: @@ -109,13 +109,13 @@ def format_duration(duration: int) -> str: class RVElement: parent: "RVDatabase" item: minidom.Element - was_downloaded: bool + downloaded_filepath: typing.Optional[str] watched: bool def __init__(self, parent: "RVDatabase", item: minidom.Element) -> None: self.parent = parent self.item = item - self.was_downloaded = False + self.downloaded_filepath = None self.watched = False def get_tag_data(self, tag_name: str) -> str: @@ -166,8 +166,8 @@ class RVElement: if not self.parent.args.research and cache.is_researched: self.__dict__["ytdl_infos"] = cache.__dict__["ytdl_infos"] log.debug(f"From cache: {self}") - # if cache.was_downloaded: - # self.was_downloaded = True + if cache.downloaded_filepath: + self.downloaded_filepath = cache.downloaded_filepath if cache.watched: self.watched = True @@ -191,7 +191,6 @@ class RVElement: @functools.cached_property def ytdl_infos(self) -> typing.Optional[dict]: - # TODO Sanitize according to documentation log.info(f"Researching: {self}") try: infos = self.parent.ytdl_dry.extract_info(self.link, download=False) @@ -201,14 +200,8 @@ class RVElement: # TODO Still raise in case of temporary network issue log.warning(e) infos = None - # Apparently that thing is transformed from a LazyList - # somewhere in the normal yt_dlp process - if ( - infos - and "thumbnails" in infos - and isinstance(infos["thumbnails"], yt_dlp.utils.LazyList) - ): - infos["thumbnails"] = infos["thumbnails"].exhaust() + if infos: + infos = self.parent.ytdl_dry.sanitize_info(infos) # Save database once it's been computed self.__dict__["ytdl_infos"] = infos self.parent.save() @@ -228,6 +221,8 @@ class RVElement: @property def filepath(self) -> str: assert self.is_video + if self.downloaded_filepath: + return self.downloaded_filepath return self.parent.ytdl_dry.prepare_filename(self.ytdl_infos) @property @@ -242,9 +237,15 @@ class RVElement: with yt_dlp.YoutubeDL(self.parent.ytdl_opts) as ydl: ydl.add_post_processor(SaveInfoPP(self)) ydl.process_ie_result(self.ytdl_infos, download=True) - self.was_downloaded = True self.parent.save() + def update_post_download(self, info: dict) -> None: + self.downloaded_filepath = self.parent.ytdl_dry.prepare_filename(info) + + @property + def was_downloaded(self) -> bool: + return self.downloaded_filepath is not None + def preload(self) -> None: assert self.is_video if self.downloaded: @@ -628,7 +629,6 @@ def main() -> None: if args.action == "clean": database.clean() else: - database.attempt_clean() duration = 0 for element in database.filter(args): if args.action == "download": From 84a2906be9749ac1937f52d24d9f5fc80d2447ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Tue, 21 Dec 2021 17:25:47 +0100 Subject: [PATCH 33/34] rofi: Fix theme definition for newer version I was right the first time? --- .../roles/desktop_environment/tasks/main.yml | 20 ++++++------------- config/rofi/.gitignore | 1 + config/rofi/config.rasi | 3 +-- 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/config/automatrop/roles/desktop_environment/tasks/main.yml b/config/automatrop/roles/desktop_environment/tasks/main.yml index 2e80813..31a7f92 100644 --- a/config/automatrop/roles/desktop_environment/tasks/main.yml +++ b/config/automatrop/roles/desktop_environment/tasks/main.yml @@ -6,7 +6,6 @@ with_items: - ".config/Xresources" - ".config/rofi" - - ".local/share/rofi/themes" - ".local/bin" - ".local/share/fonts" - ".config/qutebrowser" @@ -109,23 +108,16 @@ - color when: display_server == 'x11' -- name: Set base16 theme for rofi < 1.4 +- name: Set base16 theme for rofi copy: - content: "{{ base16_schemes['schemes'][base16_scheme]['rofi']['themes']['base16-' + base16_scheme + '.config'] }}" - dest: "{{ ansible_env.HOME }}/.config/rofi/theme.config" + content: "{{ base16_schemes['schemes'][base16_scheme]['rofi']['themes']['base16-' + base16_scheme + '.' + item] }}" + dest: "{{ ansible_env.HOME }}/.config/rofi/theme.{{ item }}" mode: "u=rw,g=r,o=r" tags: - color - -- name: Set base16 theme for rofi >= 1.4 - copy: - content: "{{ base16_schemes['schemes'][base16_scheme]['rofi']['themes']['base16-' + base16_scheme + '.rasi'] }}" - dest: "{{ ansible_env.HOME }}/.local/share/rofi/themes/current.rasi" - mode: "u=rw,g=r,o=r" - tags: - - color - - g - when: no + loop: + - config + - rasi - name: Configure Dunst template: diff --git a/config/rofi/.gitignore b/config/rofi/.gitignore index 83193ec..ba58f6e 100644 --- a/config/rofi/.gitignore +++ b/config/rofi/.gitignore @@ -1 +1,2 @@ theme.config +theme.rasi diff --git a/config/rofi/config.rasi b/config/rofi/config.rasi index af02ce7..9843b5f 100644 --- a/config/rofi/config.rasi +++ b/config/rofi/config.rasi @@ -1,6 +1,5 @@ configuration { - theme: "current"; lazy-grab: false; matching: "regex"; } - +@theme "theme" From 7292e8ea88f209b38b0cf5fe48d54a265614a13b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Sun, 26 Dec 2021 15:06:40 +0100 Subject: [PATCH 34/34] rssVideos: Make --research about downloading videos --- config/scripts/rssVideos | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/scripts/rssVideos b/config/scripts/rssVideos index 35e8284..109b85c 100755 --- a/config/scripts/rssVideos +++ b/config/scripts/rssVideos @@ -163,7 +163,7 @@ class RVElement: return "ytdl_infos" in self.__dict__ def salvage_cache(self, cache: "RVElement") -> None: - if not self.parent.args.research and cache.is_researched: + if cache.is_researched: self.__dict__["ytdl_infos"] = cache.__dict__["ytdl_infos"] log.debug(f"From cache: {self}") if cache.downloaded_filepath: @@ -233,6 +233,8 @@ class RVElement: def download(self) -> None: assert self.is_video log.info(f"Downloading: {self}") + if self.parent.args.research: + del self.ytdl_infos if not self.parent.args.dryrun: with yt_dlp.YoutubeDL(self.parent.ytdl_opts) as ydl: ydl.add_post_processor(SaveInfoPP(self))