|
|
@ -8,7 +8,7 @@ The common use case would be a feed from an RSS aggregator
|
|
|
|
with the unread items (non-video links are ignored).
|
|
|
|
with the unread items (non-video links are ignored).
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
import enum
|
|
|
|
import datetime
|
|
|
|
import functools
|
|
|
|
import functools
|
|
|
|
import logging
|
|
|
|
import logging
|
|
|
|
import os
|
|
|
|
import os
|
|
|
@ -17,12 +17,8 @@ import random
|
|
|
|
import requests
|
|
|
|
import requests
|
|
|
|
import re
|
|
|
|
import re
|
|
|
|
import subprocess
|
|
|
|
import subprocess
|
|
|
|
import sys
|
|
|
|
|
|
|
|
import time
|
|
|
|
import time
|
|
|
|
import typing
|
|
|
|
import typing
|
|
|
|
import urllib.parse
|
|
|
|
|
|
|
|
import urllib.request
|
|
|
|
|
|
|
|
import urllib.error
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import coloredlogs
|
|
|
|
import coloredlogs
|
|
|
|
import configargparse
|
|
|
|
import configargparse
|
|
|
@ -31,7 +27,6 @@ import yt_dlp
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
|
|
|
# TODO Lockfile, or a way to parallel watch and download
|
|
|
|
# TODO Lockfile, or a way to parallel watch and download
|
|
|
|
# TODO Save ytdl infos and view info separately
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def configure_logging(args: configargparse.Namespace) -> None:
|
|
|
|
def configure_logging(args: configargparse.Namespace) -> None:
|
|
|
@ -113,13 +108,15 @@ class RVElement:
|
|
|
|
parent: "RVDatabase"
|
|
|
|
parent: "RVDatabase"
|
|
|
|
item: dict
|
|
|
|
item: dict
|
|
|
|
downloaded_filepath: typing.Optional[str]
|
|
|
|
downloaded_filepath: typing.Optional[str]
|
|
|
|
watched: bool
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def __init__(self, parent: "RVDatabase", item: dict) -> None:
|
|
|
|
def __init__(self, parent: "RVDatabase", item: dict) -> None:
|
|
|
|
self.parent = parent
|
|
|
|
self.parent = parent
|
|
|
|
self.item = item
|
|
|
|
self.item = item
|
|
|
|
self.downloaded_filepath = None
|
|
|
|
self.downloaded_filepath = None
|
|
|
|
self.watched = False
|
|
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
|
|
|
def id(self) -> str:
|
|
|
|
|
|
|
|
return self.item["id"]
|
|
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
@property
|
|
|
|
def title(self) -> str:
|
|
|
|
def title(self) -> str:
|
|
|
@ -134,8 +131,8 @@ class RVElement:
|
|
|
|
return self.item["origin"]["title"]
|
|
|
|
return self.item["origin"]["title"]
|
|
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
@property
|
|
|
|
def guid(self) -> int:
|
|
|
|
def date(self) -> datetime.datetime:
|
|
|
|
return int(self.item["timestampUsec"])
|
|
|
|
return datetime.datetime.fromtimestamp(self.item["published"])
|
|
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
@property
|
|
|
|
def is_researched(self) -> bool:
|
|
|
|
def is_researched(self) -> bool:
|
|
|
@ -147,19 +144,21 @@ class RVElement:
|
|
|
|
log.debug(f"From cache: {self}")
|
|
|
|
log.debug(f"From cache: {self}")
|
|
|
|
if cache.downloaded_filepath:
|
|
|
|
if cache.downloaded_filepath:
|
|
|
|
self.downloaded_filepath = cache.downloaded_filepath
|
|
|
|
self.downloaded_filepath = cache.downloaded_filepath
|
|
|
|
if cache.watched:
|
|
|
|
|
|
|
|
self.watched = True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def __str__(self) -> str:
|
|
|
|
def __str__(self) -> str:
|
|
|
|
str = f"{self.guid}: {self.creator if self.creator else '?'} – {self.title}"
|
|
|
|
str = f"{self.date.strftime('%y-%m-%d %H:%M')} ("
|
|
|
|
if self.is_researched:
|
|
|
|
if self.is_researched:
|
|
|
|
if self.is_video:
|
|
|
|
if self.is_video:
|
|
|
|
str += f" ({format_duration(self.duration)})"
|
|
|
|
str += format_duration(self.duration)
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
str += " (N/A)"
|
|
|
|
str += "--:--:--"
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
str += " (?)"
|
|
|
|
str += "??:??:??"
|
|
|
|
str += f" – {self.link}"
|
|
|
|
str += (
|
|
|
|
|
|
|
|
f") {self.creator if self.creator else '?'} "
|
|
|
|
|
|
|
|
f"– {self.title} "
|
|
|
|
|
|
|
|
f"– {self.link}"
|
|
|
|
|
|
|
|
)
|
|
|
|
return str
|
|
|
|
return str
|
|
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
@property
|
|
|
@ -237,6 +236,12 @@ class RVElement:
|
|
|
|
return
|
|
|
|
return
|
|
|
|
self.download()
|
|
|
|
self.download()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
|
|
|
def watched(self) -> bool:
|
|
|
|
|
|
|
|
if not self.is_researched:
|
|
|
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
return self.was_downloaded and not self.downloaded
|
|
|
|
|
|
|
|
|
|
|
|
def matches_filter(self, args: configargparse.Namespace) -> bool:
|
|
|
|
def matches_filter(self, args: configargparse.Namespace) -> bool:
|
|
|
|
# Inexpensive filters
|
|
|
|
# Inexpensive filters
|
|
|
|
if args.seen != "any" and (args.seen == "seen") != self.watched:
|
|
|
|
if args.seen != "any" and (args.seen == "seen") != self.watched:
|
|
|
@ -245,9 +250,6 @@ class RVElement:
|
|
|
|
if args.title and not re.search(args.title, self.title):
|
|
|
|
if args.title and not re.search(args.title, self.title):
|
|
|
|
log.debug(f"Title not matching {args.title}: {self}")
|
|
|
|
log.debug(f"Title not matching {args.title}: {self}")
|
|
|
|
return False
|
|
|
|
return False
|
|
|
|
if args.guid and not re.search(args.guid, str(self.guid)):
|
|
|
|
|
|
|
|
log.debug(f"Guid not matching {args.guid}: {self}")
|
|
|
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
if args.link and not re.search(args.link, self.link):
|
|
|
|
if args.link and not re.search(args.link, self.link):
|
|
|
|
log.debug(f"Link not matching {args.link}: {self}")
|
|
|
|
log.debug(f"Link not matching {args.link}: {self}")
|
|
|
|
return False
|
|
|
|
return False
|
|
|
@ -277,8 +279,8 @@ class RVElement:
|
|
|
|
proc = subprocess.run(cmd)
|
|
|
|
proc = subprocess.run(cmd)
|
|
|
|
proc.check_returncode()
|
|
|
|
proc.check_returncode()
|
|
|
|
|
|
|
|
|
|
|
|
self.watched = True
|
|
|
|
self.clean()
|
|
|
|
self.parent.save()
|
|
|
|
self.try_mark_read()
|
|
|
|
|
|
|
|
|
|
|
|
def clean(self) -> None:
|
|
|
|
def clean(self) -> None:
|
|
|
|
assert self.is_video
|
|
|
|
assert self.is_video
|
|
|
@ -289,6 +291,32 @@ class RVElement:
|
|
|
|
if not self.parent.args.dryrun:
|
|
|
|
if not self.parent.args.dryrun:
|
|
|
|
os.unlink(file)
|
|
|
|
os.unlink(file)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def mark_read(self) -> None:
|
|
|
|
|
|
|
|
log.debug(f"Marking {self} read")
|
|
|
|
|
|
|
|
if self.parent.args.dryrun:
|
|
|
|
|
|
|
|
return
|
|
|
|
|
|
|
|
r = requests.post(
|
|
|
|
|
|
|
|
f"{self.parent.args.url}/reader/api/0/edit-tag",
|
|
|
|
|
|
|
|
data={
|
|
|
|
|
|
|
|
"i": self.id,
|
|
|
|
|
|
|
|
"a": "user/-/state/com.google/read",
|
|
|
|
|
|
|
|
"ac": "edit",
|
|
|
|
|
|
|
|
"token": self.parent.feed_token,
|
|
|
|
|
|
|
|
},
|
|
|
|
|
|
|
|
headers=self.parent.auth_headers,
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
r.raise_for_status()
|
|
|
|
|
|
|
|
if r.text.strip() != "OK":
|
|
|
|
|
|
|
|
raise RuntimeError(f"Couldn't mark {self} as read: {r.text}")
|
|
|
|
|
|
|
|
log.info(f"Marked {self} as read")
|
|
|
|
|
|
|
|
self.parent.elements.remove(self)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def try_mark_read(self) -> None:
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
self.mark_read()
|
|
|
|
|
|
|
|
except requests.ConnectionError:
|
|
|
|
|
|
|
|
log.warning(f"Couldn't mark {self} as read")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class RVDatabase:
|
|
|
|
class RVDatabase:
|
|
|
|
SAVE_FILE = ".cache.p"
|
|
|
|
SAVE_FILE = ".cache.p"
|
|
|
@ -322,26 +350,26 @@ class RVDatabase:
|
|
|
|
self.auth_headers = cache.auth_headers
|
|
|
|
self.auth_headers = cache.auth_headers
|
|
|
|
|
|
|
|
|
|
|
|
def salvage_cache(self, cache: "RVDatabase") -> None:
|
|
|
|
def salvage_cache(self, cache: "RVDatabase") -> None:
|
|
|
|
log.debug(f"Salvaging cache")
|
|
|
|
log.debug("Salvaging cache")
|
|
|
|
cache_els = dict()
|
|
|
|
cache_els = dict()
|
|
|
|
for cache_el in cache.elements:
|
|
|
|
for cache_el in cache.elements:
|
|
|
|
cache_els[cache_el.guid] = cache_el
|
|
|
|
cache_els[cache_el.id] = cache_el
|
|
|
|
for el in self.elements:
|
|
|
|
for el in self.elements:
|
|
|
|
if el.guid in cache_els:
|
|
|
|
if el.id in cache_els:
|
|
|
|
el.salvage_cache(cache_els[el.guid])
|
|
|
|
el.salvage_cache(cache_els[el.id])
|
|
|
|
|
|
|
|
|
|
|
|
def clean_cache(self, cache: "RVDatabase") -> None:
|
|
|
|
def clean_cache(self, cache: "RVDatabase") -> None:
|
|
|
|
log.debug(f"Cleaning cache")
|
|
|
|
log.debug("Cleaning cache")
|
|
|
|
self_els = dict()
|
|
|
|
self_els = dict()
|
|
|
|
for self_el in self.elements:
|
|
|
|
for self_el in self.elements:
|
|
|
|
self_els[self_el.guid] = self_el
|
|
|
|
self_els[self_el.id] = self_el
|
|
|
|
for el in cache.elements:
|
|
|
|
for el in cache.elements:
|
|
|
|
if el.guid not in self_els:
|
|
|
|
if el.id not in self_els:
|
|
|
|
if el.is_researched and el.is_video:
|
|
|
|
if el.is_researched and el.is_video:
|
|
|
|
el.clean()
|
|
|
|
el.clean()
|
|
|
|
|
|
|
|
|
|
|
|
def import_cache(self, cache: "RVDatabase") -> None:
|
|
|
|
def import_cache(self, cache: "RVDatabase") -> None:
|
|
|
|
log.debug(f"Importing cache")
|
|
|
|
log.debug("Importing cache")
|
|
|
|
self.build_list([element.item for element in cache.elements])
|
|
|
|
self.build_list([element.item for element in cache.elements])
|
|
|
|
|
|
|
|
|
|
|
|
@functools.cached_property
|
|
|
|
@functools.cached_property
|
|
|
@ -483,6 +511,20 @@ class RVDatabase:
|
|
|
|
|
|
|
|
|
|
|
|
return elements
|
|
|
|
return elements
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@functools.cached_property
|
|
|
|
|
|
|
|
def feed_token(self) -> str:
|
|
|
|
|
|
|
|
r = requests.get(
|
|
|
|
|
|
|
|
f"{self.args.url}/reader/api/0/token",
|
|
|
|
|
|
|
|
headers=self.auth_headers,
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
r.raise_for_status()
|
|
|
|
|
|
|
|
return r.text.strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def try_mark_watched_read(self) -> None:
|
|
|
|
|
|
|
|
for element in self.elements:
|
|
|
|
|
|
|
|
if element.watched:
|
|
|
|
|
|
|
|
element.try_mark_read()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_args() -> configargparse.Namespace:
|
|
|
|
def get_args() -> configargparse.Namespace:
|
|
|
|
defaultConfigPath = os.path.join(
|
|
|
|
defaultConfigPath = os.path.join(
|
|
|
@ -558,11 +600,11 @@ def get_args() -> configargparse.Namespace:
|
|
|
|
default="old",
|
|
|
|
default="old",
|
|
|
|
help="Sorting mechanism",
|
|
|
|
help="Sorting mechanism",
|
|
|
|
)
|
|
|
|
)
|
|
|
|
parser.add("--guid", help="Regex to filter guid")
|
|
|
|
|
|
|
|
parser.add("--creator", help="Regex to filter by creator")
|
|
|
|
parser.add("--creator", help="Regex to filter by creator")
|
|
|
|
parser.add("--title", help="Regex to filter by title")
|
|
|
|
parser.add("--title", help="Regex to filter by title")
|
|
|
|
parser.add("--link", help="Regex to filter by link")
|
|
|
|
parser.add("--link", help="Regex to filter by link")
|
|
|
|
parser.add("--duration", help="Comparative to filter by duration")
|
|
|
|
parser.add("--duration", help="Comparative to filter by duration")
|
|
|
|
|
|
|
|
# TODO Date selector
|
|
|
|
parser.add(
|
|
|
|
parser.add(
|
|
|
|
"--seen",
|
|
|
|
"--seen",
|
|
|
|
choices=("seen", "unseen", "any"),
|
|
|
|
choices=("seen", "unseen", "any"),
|
|
|
@ -600,8 +642,6 @@ def get_args() -> configargparse.Namespace:
|
|
|
|
"watch",
|
|
|
|
"watch",
|
|
|
|
"binge",
|
|
|
|
"binge",
|
|
|
|
"clean",
|
|
|
|
"clean",
|
|
|
|
"seen",
|
|
|
|
|
|
|
|
"unseen",
|
|
|
|
|
|
|
|
),
|
|
|
|
),
|
|
|
|
default="download",
|
|
|
|
default="download",
|
|
|
|
)
|
|
|
|
)
|
|
|
@ -614,13 +654,7 @@ def get_args() -> configargparse.Namespace:
|
|
|
|
return args
|
|
|
|
return args
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main() -> None:
|
|
|
|
def get_database(args: configargparse.Namespace) -> RVDatabase:
|
|
|
|
args = get_args()
|
|
|
|
|
|
|
|
configure_logging(args)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
os.makedirs(args.videos, exist_ok=True)
|
|
|
|
|
|
|
|
os.chdir(args.videos)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
database = RVDatabase(args)
|
|
|
|
database = RVDatabase(args)
|
|
|
|
cache = RVDatabase.load()
|
|
|
|
cache = RVDatabase.load()
|
|
|
|
feed_fetched = False
|
|
|
|
feed_fetched = False
|
|
|
@ -635,6 +669,7 @@ def main() -> None:
|
|
|
|
raise RuntimeError("Couldn't fetch feed, refusing to download")
|
|
|
|
raise RuntimeError("Couldn't fetch feed, refusing to download")
|
|
|
|
# This is a quirky failsafe in case of no internet connection,
|
|
|
|
# This is a quirky failsafe in case of no internet connection,
|
|
|
|
# so the script doesn't go noting that no element is a video.
|
|
|
|
# so the script doesn't go noting that no element is a video.
|
|
|
|
|
|
|
|
log.warning(f"Couldn't fetch feed: {err}")
|
|
|
|
if not feed_fetched:
|
|
|
|
if not feed_fetched:
|
|
|
|
if cache:
|
|
|
|
if cache:
|
|
|
|
log.warning("Using cached feed.")
|
|
|
|
log.warning("Using cached feed.")
|
|
|
@ -646,12 +681,25 @@ def main() -> None:
|
|
|
|
database.clean_cache(cache)
|
|
|
|
database.clean_cache(cache)
|
|
|
|
database.save()
|
|
|
|
database.save()
|
|
|
|
|
|
|
|
|
|
|
|
log.debug(f"Running action")
|
|
|
|
return database
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main() -> None:
|
|
|
|
|
|
|
|
args = get_args()
|
|
|
|
|
|
|
|
configure_logging(args)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
os.makedirs(args.videos, exist_ok=True)
|
|
|
|
|
|
|
|
os.chdir(args.videos)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
database = get_database(args)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
log.debug("Running action")
|
|
|
|
if args.action == "clean":
|
|
|
|
if args.action == "clean":
|
|
|
|
database.clean()
|
|
|
|
database.clean()
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
duration = 0
|
|
|
|
duration = 0
|
|
|
|
for element in database.filter(args):
|
|
|
|
for element in database.filter(args):
|
|
|
|
|
|
|
|
duration += element.duration if element.is_video else 0
|
|
|
|
if args.action == "download":
|
|
|
|
if args.action == "download":
|
|
|
|
element.preload()
|
|
|
|
element.preload()
|
|
|
|
elif args.action == "list":
|
|
|
|
elif args.action == "list":
|
|
|
@ -660,19 +708,11 @@ def main() -> None:
|
|
|
|
element.watch()
|
|
|
|
element.watch()
|
|
|
|
if args.action == "watch":
|
|
|
|
if args.action == "watch":
|
|
|
|
break
|
|
|
|
break
|
|
|
|
elif args.action == "seen":
|
|
|
|
|
|
|
|
if not element.watched:
|
|
|
|
|
|
|
|
log.info(f"Maked as seen: {element}")
|
|
|
|
|
|
|
|
element.watched = True
|
|
|
|
|
|
|
|
elif args.action == "unseen":
|
|
|
|
|
|
|
|
if element.watched:
|
|
|
|
|
|
|
|
log.info(f"Maked as unseen: {element}")
|
|
|
|
|
|
|
|
element.watched = False
|
|
|
|
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
raise NotImplementedError(f"Unimplemented action: {args.action}")
|
|
|
|
raise NotImplementedError(f"Unimplemented action: {args.action}")
|
|
|
|
duration += element.duration if element.is_video else 0
|
|
|
|
|
|
|
|
log.info(f"Total duration: {format_duration(duration)}")
|
|
|
|
log.info(f"Total duration: {format_duration(duration)}")
|
|
|
|
database.attempt_clean()
|
|
|
|
database.attempt_clean()
|
|
|
|
|
|
|
|
database.try_mark_watched_read()
|
|
|
|
database.save()
|
|
|
|
database.save()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|