diff --git a/scripts/compressPictureMovies b/scripts/compressPictureMovies index 1a929ba..da74b42 100755 --- a/scripts/compressPictureMovies +++ b/scripts/compressPictureMovies @@ -4,20 +4,98 @@ import os import shutil import subprocess import sys +import logging +import coloredlogs +import progressbar +import time +import hashlib +import tempfile +import json +import statistics +import datetime + +coloredlogs.install(level='DEBUG', fmt='%(levelname)s %(message)s') +log = logging.getLogger() # Constants PICTURES_FOLDER = os.path.join(os.path.expanduser("~"), "Images") -ORIGNAL_FOLDER = os.path.join(PICTURES_FOLDER, ".Originaux") -MOVIE_EXTENSIONS = ["mov", "avi", "mp4"] -OUTPUT_EXTENSION = "mp4" -OUTPUT_FFMPEG_PARAMETERS = ["-codec:v", "libx265", "-crf", "28", "-preset:v", "slower", "-codec:a", "libfdk_aac", "-movflags", "+faststart", "-vbr", "5"] -OUTPUT_METADATA_FIELD = ["episode_id"] +ORIGINAL_FOLDER = os.path.join(os.path.expanduser("~"), ".ImagesOriginaux") +MOVIE_EXTENSIONS = ["mov", "avi", "mp4", "3gp", "webm", "mkv"] +OUTPUT_EXTENSION = "webm" +OUTPUT_FFMPEG_PARAMETERS = ["-c:v", "libvpx-vp9", "-crf", "30", "-b:v", "0"] +# OUTPUT_FFMPEG_PARAMETERS = ["-c:v", "libaom-av1", "-crf", "30", "-strict", "experimental", "-c:a", "libopus"] +DURATION_MAX_DEV = 1 +def videoMetadata(filename): + assert os.path.isfile(filename) + cmd = ["ffmpeg", "-i", filename, "-f", "ffmetadata", "-"] + p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + p.check_returncode() + metadataRaw = p.stdout + data = dict() + for metadataLine in metadataRaw.split(b'\n'): + # Skip empty lines + if not len(metadataLine): + continue + # Skip comments + if metadataLine.startswith(b';'): + continue + # Parse key-value + metadataLineSplit = metadataLine.split(b'=') + if len(metadataLineSplit) != 2: + log.warning("Unparsed metadata line: `{}`".format(metadataLine)) + continue + key, val = metadataLineSplit + key = key.decode().lower() + val = val.decode() + data[key] = val + return data + +def videoInfos(filename): + assert os.path.isfile(filename) + cmd = ["ffprobe", filename, "-print_format", "json", "-show_streams"] + p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + p.check_returncode() + infosRaw = p.stdout + infos = json.loads(infosRaw) + return infos + +from pprint import pprint +def streamDuration(stream): + if "duration" in stream: + return float(stream["duration"]) + elif "sample_rate" in stream and "nb_frames" in stream: + return int(stream["nb_frames"]) / int(stream["sample_rate"]) + elif "tags" in stream and "DURATION" in stream["tags"]: + durRaw = stream["tags"]["DURATION"] + durSplit = durRaw.split(":") + assert len(durSplit) == 3 + durSplitFloat = [float(a) for a in durSplit] + hours, minutes, seconds = durSplitFloat + return (hours * 60 + minutes) * 60 + seconds + else: + raise KeyError("Can't find duration information in stream") + +def videoDuration(filename): + # TODO Doesn't work with VP8 / webm + infos = videoInfos(filename) + durations = [streamDuration(stream) for stream in infos["streams"]] + dev = statistics.stdev(durations) + assert dev <= DURATION_MAX_DEV, "Too much deviation ({} s)".format(dev) + return sum(durations)/len(durations) + + +todos = set() +totalSize = 0 +totalDuration = 0 + # Walk folders +log.info("Listing files in {}".format(PICTURES_FOLDER)) +allVideos = list() for root, dirs, files in os.walk(PICTURES_FOLDER): # If folder is in ORIGINAL_FOLDER, skip it - if root.startswith(ORIGNAL_FOLDER): + if root.startswith(ORIGINAL_FOLDER): continue # Iterate over files for inputName in files: @@ -27,137 +105,109 @@ for root, dirs, files in os.walk(PICTURES_FOLDER): if inputExt not in MOVIE_EXTENSIONS: continue - # Generates all needed filepaths - ## Found file - inputFull = os.path.join(root, inputName) - inputRel = os.path.relpath(inputFull, PICTURES_FOLDER) - ## Original file - originalFull = os.path.join(ORIGNAL_FOLDER, inputRel) - originalRel = inputRel - ## Compressed file - outputFull = os.path.join(root, inputNameBase + "." + OUTPUT_EXTENSION) + allVideos.append((root, inputName)) - # If the extension is the same of the output one - if inputExt == OUTPUT_EXTENSION: - # Read the metadata of the video - metadataRaw = subprocess.run(["ffmpeg", "-i", inputFull, "-f", "ffmetadata", "-"], stdout=subprocess.PIPE).stdout - # If it has the field with the original file - originalRel = None - wantedPattern = OUTPUT_METADATA_FIELD.encode() + b"=" - for metadataLine in metadataRaw.split('\n'): - if metadataLine.startswith(wantedPattern): - originalRel = metadataLine[len(wantedPattern)+1:] - break - if originalRel: - # If the original file does not exists, warn about it - originalFull = os.path.join(ORIGNAL_FOLDER, originalRel) - if not os.path.isfile(originalFull): - print("WARN {inputRel} states to have {originalRel} as original but this file doesn't exist".format(inputRel=inputRel, originalRel=originalRel)) - # If the original is not aligned with the compressed, warn about it (TODO move it automatically) - if inputRel != originalRel: - print("WARN {inputRel} is not aligned with original {originalRel}".format(inputRel=inputRel, originalRel=originalRel)) - # Skip file - continue - # Initiate a conversion in a temporary file - # If the temporary file does not have the same caracteristics as the original - # Warn about it - # Delete it - # Skip file - # Move the original to the corresponding original folder - # Move the converted file in place of the original +log.info("Analyzing videos") +for root, inputName in progressbar.progressbar(allVideos): + inputNameBase, inputExt = os.path.splitext(inputName) + inputExt = inputExt[1:].lower() -# TODO Iterate over the orignal folder to find non-matching compressed videos not found in the above pass + # Generates all needed filepaths + ## Found file + inputFull = os.path.join(root, inputName) + inputRel = os.path.relpath(inputFull, PICTURES_FOLDER) + ## Original file + originalFull = os.path.join(ORIGINAL_FOLDER, inputRel) + originalRel = inputRel + assert not os.path.isfile(originalFull), originalFile + " exists" -sys.exit(0) + ## Compressed file + outputFull = os.path.join(root, inputNameBase + "." + OUTPUT_EXTENSION) -# Constants -SOURCE_FOLDER = os.path.join(os.path.expanduser("~"), "Musique") -OUTPUT_FOLDER = os.path.join(os.path.expanduser("~"), ".MusiqueCompressed") -CONVERSIONS = {"flac": "m4a"} -FORBIDDEN_EXTENSIONS = ["jpg", "pdf", "ffs_db"] -FORGIVEN_FILENAMES = ["cover.jpg"] -IGNORED_EMPTY_FOLDER = [".stfolder"] + # If the extension is the same of the output one + if inputExt == OUTPUT_EXTENSION: + # Read the metadata of the video + meta = videoMetadata(inputFull) - - -# Listing files -sourceFiles = dict() -for root, dirs, files in os.walk(SOURCE_FOLDER): - for f in files: - fullPath = os.path.join(root, f) - path = os.path.relpath(fullPath, SOURCE_FOLDER) - sourceFiles[path] = os.path.getctime(fullPath) - -outputFiles = dict() -for root, dirs, files in os.walk(OUTPUT_FOLDER): - for f in files: - fullPath = os.path.join(root, f) - path = os.path.relpath(fullPath, OUTPUT_FOLDER) - outputFiles[path] = os.path.getctime(fullPath) - -# Sorting files -remainingConversions = dict() -extraFiles = list(outputFiles.keys()) - -def convertPath(path): - filename, extension = os.path.splitext(path) - extension = extension[1:].lower() - # If the extension isn't allowed - if extension in FORBIDDEN_EXTENSIONS: - basename = os.path.basename(path) - # And the filename is not an exception - if basename not in FORGIVEN_FILENAMES: - # This file shouldn't be copied nor converted - return False - # If this needs a conversion - elif extension in CONVERSIONS: - extension = CONVERSIONS[extension] - return filename + "." + extension - # In all other case, this is a simple copy - return path - -for sourceFile in sourceFiles: - outputFile = convertPath(sourceFile) - # If the file should not be converted, do nothing - if outputFile == False: - continue - # If the file already has something as an output - elif outputFile in outputFiles: - extraFiles.remove(outputFile) - # If the output file is newer than the source file, do not initiate a conversion - if outputFiles[outputFile] >= sourceFiles[sourceFile]: + # If it has the field with the original file + if 'original' in meta: + # Skip file continue - # If the file needs to be converted, do it - remainingConversions[sourceFile] = outputFile - -# Converting files -for sourceFile in remainingConversions: - outputFile = remainingConversions[sourceFile] - - # Creating folder if it doesn't exists - fullOutputFile = os.path.join(OUTPUT_FOLDER, outputFile) - fullOutputDir = os.path.dirname(fullOutputFile) - os.makedirs(fullOutputDir, exist_ok=True) - - # Converting - fullSourceFile = os.path.join(SOURCE_FOLDER, sourceFile) - print(fullSourceFile, "→", fullOutputFile) - if sourceFile == outputFile: - # shutil.copy(fullSourceFile, fullOutputFile) - os.link(fullSourceFile, fullOutputFile) else: - subprocess.run(["ffmpeg", "-y", "-i", fullSourceFile, "-codec:a", "libfdk_aac", "-cutoff", "18000", "-movflags", "+faststart", "-vbr", "5", fullOutputFile]) - -# Removing extra files -for extraFile in extraFiles: - fullExtraFile = os.path.join(OUTPUT_FOLDER, extraFile) - os.remove(fullExtraFile) - -# Removing empty dirs -for root, dirs, files in os.walk(OUTPUT_FOLDER): - if not dirs and not files: - dirBasename = os.path.basename(root) - if dirBasename not in IGNORED_EMPTY_FOLDER: - os.rmdir(root) + assert not os.path.isfile(outputFull), outputFull + " exists" + size = os.stat(inputFull).st_size + try: + duration = videoDuration(inputFull) + except Exception as e: + log.warning("Can't determine duration of {}, skipping".format(inputFull)) + log.debug(e, exc_info=True) + continue + + todo = (inputFull, originalFull, outputFull, size, duration) + + totalDuration += duration + totalSize += size + todos.add(todo) + +log.info("Converting {} videos ({})".format(len(todos), datetime.timedelta(seconds=totalDuration))) + +# From https://stackoverflow.com/a/3431838 +def sha256(fname): + hash_sha256 = hashlib.sha256() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(131072), b""): + hash_sha256.update(chunk) + return hash_sha256.hexdigest() + +# Progress bar things +totalDataSize = progressbar.widgets.DataSize() +totalDataSize.variable = 'max_value' +barWidgets = [progressbar.widgets.DataSize(), ' of ', totalDataSize, ' ', progressbar.widgets.Bar(), ' ', progressbar.widgets.FileTransferSpeed(), ' ', progressbar.widgets.AdaptiveETA()] +bar = progressbar.DataTransferBar(max_value=totalSize, widgets=barWidgets) +bar.start() +processedSize = 0 + + +for inputFull, originalFull, outputFull, size, duration in todos: + tmpfile = tempfile.mkstemp(prefix="compressPictureMovies", suffix="."+OUTPUT_EXTENSION)[1] + try: + # Calculate the sum of the original file + checksum = sha256(inputFull) + + # Initiate a conversion in a temporary file + originalRel = os.path.relpath(originalFull, ORIGINAL_FOLDER) + originalContent = "{} {}".format(originalRel, checksum) + metadataCmd = ["-metadata", 'original="{}"'.format(originalContent)] + cmd = ["ffmpeg", "-hide_banner", "-y", "-i", inputFull] + OUTPUT_FFMPEG_PARAMETERS + metadataCmd + [tmpfile] + p = subprocess.run(cmd) + p.check_returncode() + + # Verify the durartion of the new file + newDuration = videoDuration(tmpfile) + dev = statistics.stdev((duration, newDuration)) + assert dev < DURATION_MAX_DEV, "Too much deviation in duration" + + # Move the original to the corresponding original folder + originalDir = os.path.dirname(originalFull) + os.makedirs(originalDir, exist_ok=True) + shutil.move(inputFull, originalFull) + + # Move the converted file in place of the original + shutil.move(tmpfile, outputFull) + except Exception as e: + log.error("Couldn't process file {}".format(inputFull)) + log.error(e, exc_info=True) + try: + os.unlink(tmpfile) + except Exception: + pass + # Progress bar things + processedSize += size + bar.update(processedSize) +bar.finish() + + +# TODO Iterate over the already compressed videos to assert the originals are +# in their correct place, else move them diff --git a/scripts/musiqueBof b/scripts/musiqueBof new file mode 100755 index 0000000..b0f9658 --- /dev/null +++ b/scripts/musiqueBof @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +import sys +import os +import shutil +import logging +import coloredlogs + +coloredlogs.install(level='DEBUG', fmt='%(levelname)s %(message)s') +log = logging.getLogger() + +MUSICS_FOLDER = os.path.join(os.path.expanduser("~"), "Musique") +BOF_FOLDER = os.path.join(os.path.expanduser("~"), ".MusiqueBof") + +for f in sys.argv[1:]: + src = os.path.realpath(f) + if not os.path.isfile(src): + log.error("{} does not exists".format(src)) + continue + + srcBase = None + if src.startswith(MUSICS_FOLDER): + srcBase = MUSICS_FOLDER + dstBase = BOF_FOLDER + elif src.startswith(BOF_FOLDER): + srcBase = BOF_FOLDER + dstBase = MUSIC_FOLDER + else: + log.error("{} not in any music folder".format(src)) + continue + + common = os.path.relpath(src, srcBase) + dst = os.path.join(dstBase, common) + dstFolder = os.path.dirname(dst) + + log.info("{} → {}".format(src, dst)) + os.makedirs(dstFolder, exist_ok=True) + shutil.move(src, dst) + diff --git a/scripts/replayGain b/scripts/replayGain index 5e0b7d9..c6178ce 100755 --- a/scripts/replayGain +++ b/scripts/replayGain @@ -4,94 +4,65 @@ # which is usually -89.0 dB import os -import subprocess import coloredlogs import logging -import progressbar +import r128gain +import sys coloredlogs.install(level='DEBUG', fmt='%(levelname)s %(message)s') log = logging.getLogger() +# TODO Remove debug # Constants -FORCE = False # TODO UX cli argument -SOURCE_FOLDER = os.path.join(os.path.expanduser("~"), "Musique") -GAIN_COMMANDS = {("flac",): ["metaflac", "--add-replay-gain"], - ("mp3",): ["mp3gain", "-a", "-k"] + (["-s", "r"] if FORCE else []), - ("m4a", "mp4"): ["aacgain", "-a", "-k"] + (["-s", "r"] if FORCE else []), - ("ogg",): ["vorbisgain", "--album"] + ([] if FORCE else ["--fast"]), - } +FORCE = '-f' in sys.argv +if FORCE: + sys.argv.remove('-f') +SOURCE_FOLDER = os.path.realpath(sys.argv[1]) if len(sys.argv) >= 2 else os.path.join(os.path.expanduser("~"), "Musique") -# Since metaflac ALWAYS recalculate the tags, we need to specificaly verify -# that the tags are present on every track of an album to skip it -def isFlacTagged(f): - # TODO PERF Run metaflac --show-tag for the whole album and compare the - # output with the number of files tagged - for tag in ["REPLAYGAIN_TRACK_GAIN", "REPLAYGAIN_ALBUM_GAIN"]: - cmd = ["metaflac", "--show-tag", tag, f] - proc = subprocess.run(cmd, stdout=subprocess.PIPE) - res = len(proc.stdout.strip()) > 0 - if not res: - return False - return True - -# TODO UX Do the same thing with other formats so the output is not -# inconsistent +def isMusic(f): + ext = os.path.splitext(f)[1][1:].lower() + return ext in r128gain.AUDIO_EXTENSIONS # Get album paths +log.info("Listing albums and tracks") albums = set() +singleFiles = set() for root, dirs, files in os.walk(SOURCE_FOLDER): relRoot = os.path.relpath(root, SOURCE_FOLDER) - # See if it's an album (only 2 components in the path) head, tail = os.path.split(relRoot) + # 1 component in the path: save files path as single if not len(head): - continue + for f in files: + if isMusic(f): + fullPath = os.path.join(root, f) + singleFiles.add(fullPath) head, tail = os.path.split(head) if len(head): continue + # 2 components in the path: save album path albums.add(root) -cmds = list() +log.info("Processing single files") +# r128gain.process(list(singleFiles), album_gain=False, skip_tagged=not FORCE, report=True) for album in albums: albumName = os.path.relpath(album, SOURCE_FOLDER) log.info("Processing album {}".format(albumName)) - musicFiles = dict() + musicFiles = set() for root, dirs, files in os.walk(album): for f in files: - ext = os.path.splitext(f)[1][1:].lower() + if isMusic(f): + fullPath = os.path.join(root, f) + musicFiles.add(fullPath) - for exts in GAIN_COMMANDS.keys(): - if ext in exts: - if exts not in musicFiles.keys(): - musicFiles[exts] = set() - fullPath = os.path.join(root, f) - musicFiles[exts].add(fullPath) - - if len(musicFiles) >= 2: - log.warn("Different extensions for album {}. AlbumGain won't be on par.".format(albumName)) - - for exts, files in musicFiles.items(): - - if exts == ("flac",) and not FORCE: - allTagged = True - for f in files: - if not isFlacTagged(f): - allTaged = False - break - if allTagged: - log.debug("Already tagged (for flac only)!") - break - - cmd = GAIN_COMMANDS[exts] + list(files) - log.debug("Registering command: `{}`".format(" ".join(cmd))) - cmds.append(cmd) - -logging.info("Executing commands") -for cmd in progressbar.progressbar(cmds): - subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + # print(musicFiles) + if not len(musicFiles): + continue + r128gain.process(list(musicFiles), album_gain=True, skip_tagged=not FORCE, report=True) + print("==============================") diff --git a/scripts/tagCreatorPhotos b/scripts/tagCreatorPhotos new file mode 100755 index 0000000..0002db7 --- /dev/null +++ b/scripts/tagCreatorPhotos @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +import sys +import os +import piexif + +assert len(sys.argv) >= 3, "Usage {} CREATOR FILENAMES...".format(sys.argv[0]) +creator = sys.argv[1] +filenames = sys.argv[2:] + +for filename in filenames: + assert os.path.isfile(filename) + exifDict = piexif.load(filename) + exifDict['0th'][piexif.ImageIFD.Copyright] = creator.encode() + exifBytes = piexif.dump(exifDict) + piexif.insert(exifBytes, filename) + diff --git a/vimpcrc b/vimpcrc index 97f442c..d3f547c 100644 --- a/vimpcrc +++ b/vimpcrc @@ -5,3 +5,4 @@ map ° D:browseA:shuffle:play:playlist set songformat {%a - %b: %t}|{%f}$E$R $H[$H%l$H]$H set libraryformat %n \| {%t}|{%f}$E$R $H[$H%l$H]$H set ignorecase +set sort library diff --git a/vimrc b/vimrc index 3f62987..733c99c 100644 --- a/vimrc +++ b/vimrc @@ -41,15 +41,30 @@ Plug 'tomtom/tcomment_vim' " Plug 'tomlion/vim-solidity' " Plug 'godlygeek/tabular' " Plug 'jrozner/vim-antlr' -Plug 'maralla/completor.vim' +" +" Plug 'maralla/completor.vim' +if has('nvim') + Plug 'Shougo/deoplete.nvim', { 'do': ':UpdateRemotePlugins' } +else + Plug 'Shougo/deoplete.nvim' + Plug 'roxma/nvim-yarp' + Plug 'roxma/vim-hug-neovim-rpc' +endif +Plug 'zchee/deoplete-jedi' + Plug 'python-mode/python-mode', { 'branch': 'develop' } Plug 'junegunn/fzf', {'do': './install --bin'} Plug 'junegunn/fzf.vim' Plug 'ervandew/supertab' Plug 'dpelle/vim-LanguageTool' +Plug 'terryma/vim-smooth-scroll' call plug#end() +""" COMPLETOR """ + +let g:deoplete#enable_at_startup = 1 + """ UNDOTREE """ nmap :UndotreeToggle @@ -69,7 +84,7 @@ let g:airline#extensions#tabline#enabled = 1 let g:airline_section_a = airline#section#create(['mode']) let g:airline_section_b = airline#section#create(['branch', 'hunks']) let g:airline_section_z = airline#section#create(['%B', '@', '%l', ':', '%c']) -let g:airline_theme = 'base16' +let g:airline_theme = 'base16_monokai' """ AUTOFORMAT """ nmap :Autoformat @@ -203,7 +218,14 @@ vmap nmap o nmap :bp nmap :bn -nmap kkkkkkkkkkkkkkkkkkkkk -nmap jjjjjjjjjjjjjjjjjjjjj +if has('nvim') + " nmap 20k + " nmap 20j + noremap :call smooth_scroll#up(20, 5, 1) + noremap :call smooth_scroll#down(20, 5, 1) +else + nmap kkkkkkkkkkkkkkkkkkkkk + nmap jjjjjjjjjjjjjjjjjjjjj +endif