Python is the new bash

scripts/compressPictureMovies

@@ -4,20 +4,98 @@ import os
 import shutil
 import subprocess
 import sys
+import logging
+import coloredlogs
+import progressbar
+import time
+import hashlib
+import tempfile
+import json
+import statistics
+import datetime
+
+coloredlogs.install(level='DEBUG', fmt='%(levelname)s %(message)s')
+log = logging.getLogger()
 
 # Constants
 PICTURES_FOLDER = os.path.join(os.path.expanduser("~"), "Images")
-ORIGNAL_FOLDER = os.path.join(PICTURES_FOLDER, ".Originaux")
+ORIGINAL_FOLDER = os.path.join(os.path.expanduser("~"), ".ImagesOriginaux")
-MOVIE_EXTENSIONS = ["mov", "avi", "mp4"]
+MOVIE_EXTENSIONS = ["mov", "avi", "mp4", "3gp", "webm", "mkv"]
-OUTPUT_EXTENSION = "mp4"
+OUTPUT_EXTENSION = "webm"
-OUTPUT_FFMPEG_PARAMETERS = ["-codec:v", "libx265", "-crf", "28", "-preset:v", "slower", "-codec:a", "libfdk_aac", "-movflags", "+faststart", "-vbr", "5"]
+OUTPUT_FFMPEG_PARAMETERS = ["-c:v", "libvpx-vp9", "-crf", "30", "-b:v", "0"]
-OUTPUT_METADATA_FIELD = ["episode_id"]
+# OUTPUT_FFMPEG_PARAMETERS = ["-c:v", "libaom-av1", "-crf", "30", "-strict", "experimental", "-c:a", "libopus"]
+DURATION_MAX_DEV = 1
 
 
+def videoMetadata(filename):
+    assert os.path.isfile(filename)
+    cmd = ["ffmpeg", "-i", filename, "-f", "ffmetadata", "-"]
+    p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+    p.check_returncode()
+    metadataRaw = p.stdout
+    data = dict()
+    for metadataLine in metadataRaw.split(b'\n'):
+        # Skip empty lines
+        if not len(metadataLine):
+            continue
+        # Skip comments
+        if metadataLine.startswith(b';'):
+            continue
+        # Parse key-value
+        metadataLineSplit = metadataLine.split(b'=')
+        if len(metadataLineSplit) != 2:
+            log.warning("Unparsed metadata line: `{}`".format(metadataLine))
+            continue
+        key, val = metadataLineSplit
+        key = key.decode().lower()
+        val = val.decode()
+        data[key] = val
+    return data
+
+def videoInfos(filename):
+    assert os.path.isfile(filename)
+    cmd = ["ffprobe", filename, "-print_format", "json", "-show_streams"]
+    p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+    p.check_returncode()
+    infosRaw = p.stdout
+    infos = json.loads(infosRaw)
+    return infos
+
+from pprint import pprint
+def streamDuration(stream):
+    if "duration" in stream:
+        return float(stream["duration"])
+    elif "sample_rate" in stream and "nb_frames" in stream:
+        return int(stream["nb_frames"]) / int(stream["sample_rate"])
+    elif "tags" in stream and "DURATION" in stream["tags"]:
+        durRaw = stream["tags"]["DURATION"]
+        durSplit = durRaw.split(":")
+        assert len(durSplit) == 3
+        durSplitFloat = [float(a) for a in durSplit]
+        hours, minutes, seconds = durSplitFloat
+        return (hours * 60 + minutes) * 60 + seconds
+    else:
+        raise KeyError("Can't find duration information in stream")
+
+def videoDuration(filename):
+    # TODO Doesn't work with VP8 / webm
+    infos = videoInfos(filename)
+    durations = [streamDuration(stream) for stream in infos["streams"]]
+    dev = statistics.stdev(durations)
+    assert dev <= DURATION_MAX_DEV, "Too much deviation ({} s)".format(dev)
+    return sum(durations)/len(durations)
+
+
+todos = set()
+totalSize = 0
+totalDuration = 0
+
 # Walk folders
+log.info("Listing files in {}".format(PICTURES_FOLDER))
+allVideos = list()
 for root, dirs, files in os.walk(PICTURES_FOLDER):
     # If folder is in ORIGINAL_FOLDER, skip it
-    if root.startswith(ORIGNAL_FOLDER):
+    if root.startswith(ORIGINAL_FOLDER):
         continue
     # Iterate over files
     for inputName in files:
@@ -27,137 +105,109 @@ for root, dirs, files in os.walk(PICTURES_FOLDER):
         if inputExt not in MOVIE_EXTENSIONS:
             continue
 
-        # Generates all needed filepaths
+        allVideos.append((root, inputName))
-        ## Found file
-        inputFull = os.path.join(root, inputName)
-        inputRel = os.path.relpath(inputFull, PICTURES_FOLDER)
-        ## Original file
-        originalFull = os.path.join(ORIGNAL_FOLDER, inputRel)
-        originalRel = inputRel
-        ## Compressed file
-        outputFull = os.path.join(root, inputNameBase + "." + OUTPUT_EXTENSION)
 
-        # If the extension is the same of the output one
+log.info("Analyzing videos")
-        if inputExt == OUTPUT_EXTENSION:
+for root, inputName in progressbar.progressbar(allVideos):
-            # Read the metadata of the video
+    inputNameBase, inputExt = os.path.splitext(inputName)
-            metadataRaw = subprocess.run(["ffmpeg", "-i", inputFull, "-f", "ffmetadata", "-"], stdout=subprocess.PIPE).stdout
+    inputExt = inputExt[1:].lower()
-            # If it has the field with the original file 
-            originalRel = None
-            wantedPattern = OUTPUT_METADATA_FIELD.encode() + b"="
-            for metadataLine in metadataRaw.split('\n'):
-                if metadataLine.startswith(wantedPattern):
-                    originalRel = metadataLine[len(wantedPattern)+1:]
-                    break
-            if originalRel:
-                # If the original file does not exists, warn about it
-                originalFull = os.path.join(ORIGNAL_FOLDER, originalRel)
-                if not os.path.isfile(originalFull):
-                    print("WARN {inputRel} states to have {originalRel} as original but this file doesn't exist".format(inputRel=inputRel, originalRel=originalRel))
-                # If the original is not aligned with the compressed, warn about it (TODO move it automatically)
-                if inputRel != originalRel:
-                    print("WARN {inputRel} is not aligned with original {originalRel}".format(inputRel=inputRel, originalRel=originalRel))
-                # Skip file
-                continue
-            # Initiate a conversion in a temporary file
-            # If the temporary file does not have the same caracteristics as the original
-                # Warn about it
-                # Delete it
-                # Skip file
-            # Move the original to the corresponding original folder
-            # Move the converted file in place of the original
 
-# TODO Iterate over the orignal folder to find non-matching compressed videos not found in the above pass
+    # Generates all needed filepaths
+    ## Found file
+    inputFull = os.path.join(root, inputName)
+    inputRel = os.path.relpath(inputFull, PICTURES_FOLDER)
+    ## Original file
+    originalFull = os.path.join(ORIGINAL_FOLDER, inputRel)
+    originalRel = inputRel
+    assert not os.path.isfile(originalFull), originalFile + " exists"
 
-sys.exit(0)
+    ## Compressed file
+    outputFull = os.path.join(root, inputNameBase + "." + OUTPUT_EXTENSION)
 
-# Constants
+    # If the extension is the same of the output one
-SOURCE_FOLDER = os.path.join(os.path.expanduser("~"), "Musique")
+    if inputExt == OUTPUT_EXTENSION:
-OUTPUT_FOLDER = os.path.join(os.path.expanduser("~"), ".MusiqueCompressed")
+        # Read the metadata of the video
-CONVERSIONS = {"flac": "m4a"}
+        meta = videoMetadata(inputFull)
-FORBIDDEN_EXTENSIONS = ["jpg", "pdf", "ffs_db"]
-FORGIVEN_FILENAMES = ["cover.jpg"]
-IGNORED_EMPTY_FOLDER = [".stfolder"]
 
-
+        # If it has the field with the original file
-
+        if 'original' in meta:
-# Listing files
+            # Skip file
-sourceFiles = dict()
-for root, dirs, files in os.walk(SOURCE_FOLDER):
-    for f in files:
-        fullPath = os.path.join(root, f)
-        path = os.path.relpath(fullPath, SOURCE_FOLDER)
-        sourceFiles[path] = os.path.getctime(fullPath)
-
-outputFiles = dict()
-for root, dirs, files in os.walk(OUTPUT_FOLDER):
-    for f in files:
-        fullPath = os.path.join(root, f)
-        path = os.path.relpath(fullPath, OUTPUT_FOLDER)
-        outputFiles[path] = os.path.getctime(fullPath)
-
-# Sorting files
-remainingConversions = dict()
-extraFiles = list(outputFiles.keys())
-
-def convertPath(path):
-    filename, extension = os.path.splitext(path)
-    extension = extension[1:].lower()
-    # If the extension isn't allowed
-    if extension in FORBIDDEN_EXTENSIONS:
-        basename = os.path.basename(path)
-        # And the filename is not an exception
-        if basename not in FORGIVEN_FILENAMES:
-            # This file shouldn't be copied nor converted
-            return False
-    # If this needs a conversion
-    elif extension in CONVERSIONS:
-        extension = CONVERSIONS[extension]
-        return filename + "." + extension
-    # In all other case, this is a simple copy
-    return path
-
-for sourceFile in sourceFiles:
-    outputFile = convertPath(sourceFile)
-    # If the file should not be converted, do nothing
-    if outputFile == False:
-        continue
-    # If the file already has something as an output
-    elif outputFile in outputFiles:
-        extraFiles.remove(outputFile)
-        # If the output file is newer than the source file, do not initiate a conversion
-        if outputFiles[outputFile] >= sourceFiles[sourceFile]:
             continue
-    # If the file needs to be converted, do it
-    remainingConversions[sourceFile] = outputFile
-
-# Converting files
-for sourceFile in remainingConversions:
-    outputFile = remainingConversions[sourceFile]
-
-    # Creating folder if it doesn't exists
-    fullOutputFile = os.path.join(OUTPUT_FOLDER, outputFile)
-    fullOutputDir = os.path.dirname(fullOutputFile)
-    os.makedirs(fullOutputDir, exist_ok=True)
-
-    # Converting
-    fullSourceFile = os.path.join(SOURCE_FOLDER, sourceFile)
-    print(fullSourceFile, "→", fullOutputFile)
-    if sourceFile == outputFile:
-        # shutil.copy(fullSourceFile, fullOutputFile)
-        os.link(fullSourceFile, fullOutputFile)
     else:
-        subprocess.run(["ffmpeg", "-y", "-i", fullSourceFile, "-codec:a", "libfdk_aac", "-cutoff", "18000", "-movflags", "+faststart", "-vbr", "5", fullOutputFile])
+        assert not os.path.isfile(outputFull), outputFull + " exists"
-
-# Removing extra files
-for extraFile in extraFiles:
-    fullExtraFile = os.path.join(OUTPUT_FOLDER, extraFile)
-    os.remove(fullExtraFile)
-
-# Removing empty dirs
-for root, dirs, files in os.walk(OUTPUT_FOLDER):
-    if not dirs and not files:
-        dirBasename = os.path.basename(root)
-        if dirBasename not in IGNORED_EMPTY_FOLDER:
-            os.rmdir(root)
 
 
+    size = os.stat(inputFull).st_size
+    try:
+        duration = videoDuration(inputFull)
+    except Exception as e:
+        log.warning("Can't determine duration of {}, skipping".format(inputFull))
+        log.debug(e, exc_info=True)
+        continue
+
+    todo = (inputFull, originalFull, outputFull, size, duration)
+
+    totalDuration += duration
+    totalSize += size
+    todos.add(todo)
+
+log.info("Converting {} videos ({})".format(len(todos), datetime.timedelta(seconds=totalDuration)))
+
+# From https://stackoverflow.com/a/3431838
+def sha256(fname):
+    hash_sha256 = hashlib.sha256()
+    with open(fname, "rb") as f:
+        for chunk in iter(lambda: f.read(131072), b""):
+            hash_sha256.update(chunk)
+    return hash_sha256.hexdigest()
+
+# Progress bar things
+totalDataSize = progressbar.widgets.DataSize()
+totalDataSize.variable = 'max_value'
+barWidgets = [progressbar.widgets.DataSize(), ' of ', totalDataSize, ' ', progressbar.widgets.Bar(), ' ', progressbar.widgets.FileTransferSpeed(), ' ', progressbar.widgets.AdaptiveETA()]
+bar = progressbar.DataTransferBar(max_value=totalSize, widgets=barWidgets)
+bar.start()
+processedSize = 0
+
+
+for inputFull, originalFull, outputFull, size, duration in todos:
+    tmpfile = tempfile.mkstemp(prefix="compressPictureMovies", suffix="."+OUTPUT_EXTENSION)[1]
+    try:
+        # Calculate the sum of the original file
+        checksum = sha256(inputFull)
+
+        # Initiate a conversion in a temporary file
+        originalRel = os.path.relpath(originalFull, ORIGINAL_FOLDER)
+        originalContent = "{} {}".format(originalRel, checksum)
+        metadataCmd = ["-metadata", 'original="{}"'.format(originalContent)]
+        cmd = ["ffmpeg", "-hide_banner", "-y", "-i", inputFull] + OUTPUT_FFMPEG_PARAMETERS + metadataCmd + [tmpfile]
+        p = subprocess.run(cmd)
+        p.check_returncode()
+
+        # Verify the durartion of the new file
+        newDuration = videoDuration(tmpfile)
+        dev = statistics.stdev((duration, newDuration))
+        assert dev < DURATION_MAX_DEV, "Too much deviation in duration"
+
+        # Move the original to the corresponding original folder
+        originalDir = os.path.dirname(originalFull)
+        os.makedirs(originalDir, exist_ok=True)
+        shutil.move(inputFull, originalFull)
+
+        # Move the converted file in place of the original
+        shutil.move(tmpfile, outputFull)
+    except Exception as e:
+        log.error("Couldn't process file {}".format(inputFull))
+        log.error(e, exc_info=True)
+        try:
+            os.unlink(tmpfile)
+        except Exception:
+            pass
+    # Progress bar things
+    processedSize += size
+    bar.update(processedSize)
+bar.finish()
+
+
+# TODO Iterate over the already compressed videos to assert the originals are
+# in their correct place, else move them

scripts/musiqueBof

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import shutil
+import logging
+import coloredlogs
+
+coloredlogs.install(level='DEBUG', fmt='%(levelname)s %(message)s')
+log = logging.getLogger()
+
+MUSICS_FOLDER = os.path.join(os.path.expanduser("~"), "Musique")
+BOF_FOLDER = os.path.join(os.path.expanduser("~"), ".MusiqueBof")
+
+for f in sys.argv[1:]:
+    src = os.path.realpath(f)
+    if not os.path.isfile(src):
+        log.error("{} does not exists".format(src))
+        continue
+
+    srcBase = None
+    if src.startswith(MUSICS_FOLDER):
+        srcBase = MUSICS_FOLDER
+        dstBase = BOF_FOLDER
+    elif src.startswith(BOF_FOLDER):
+        srcBase = BOF_FOLDER
+        dstBase = MUSIC_FOLDER
+    else:
+        log.error("{} not in any music folder".format(src))
+        continue
+
+    common = os.path.relpath(src, srcBase)
+    dst = os.path.join(dstBase, common)
+    dstFolder = os.path.dirname(dst)
+
+    log.info("{} → {}".format(src, dst))
+    os.makedirs(dstFolder, exist_ok=True)
+    shutil.move(src, dst)
+

scripts/replayGain

@@ -4,94 +4,65 @@
 # which is usually -89.0 dB
 
 import os
-import subprocess
 import coloredlogs
 import logging
-import progressbar
+import r128gain
+import sys
 
 coloredlogs.install(level='DEBUG', fmt='%(levelname)s %(message)s')
 log = logging.getLogger()
 
+# TODO Remove debug
 
 # Constants
-FORCE = False # TODO UX cli argument
+FORCE = '-f' in sys.argv
-SOURCE_FOLDER = os.path.join(os.path.expanduser("~"), "Musique")
+if FORCE:
-GAIN_COMMANDS = {("flac",): ["metaflac", "--add-replay-gain"],
+    sys.argv.remove('-f')
-                 ("mp3",): ["mp3gain", "-a", "-k"] + (["-s", "r"] if FORCE else []),
+SOURCE_FOLDER = os.path.realpath(sys.argv[1]) if len(sys.argv) >= 2 else os.path.join(os.path.expanduser("~"), "Musique")
-                 ("m4a", "mp4"): ["aacgain", "-a", "-k"] + (["-s", "r"] if FORCE else []),
-                 ("ogg",): ["vorbisgain", "--album"] + ([] if FORCE else ["--fast"]),
-                 }
 
-# Since metaflac ALWAYS recalculate the tags, we need to specificaly verify
+def isMusic(f):
-# that the tags are present on every track of an album to skip it
+    ext = os.path.splitext(f)[1][1:].lower()
-def isFlacTagged(f):
+    return ext in r128gain.AUDIO_EXTENSIONS
-    # TODO PERF Run metaflac --show-tag for the whole album and compare the
-    # output with the number of files tagged
-    for tag in ["REPLAYGAIN_TRACK_GAIN", "REPLAYGAIN_ALBUM_GAIN"]:
-        cmd = ["metaflac", "--show-tag", tag, f]
-        proc = subprocess.run(cmd, stdout=subprocess.PIPE)
-        res = len(proc.stdout.strip()) > 0
-        if not res:
-            return False
-    return True
-
-# TODO UX Do the same thing with other formats so the output is not
-# inconsistent
 
 # Get album paths
+log.info("Listing albums and tracks")
 albums = set()
+singleFiles = set()
 for root, dirs, files in os.walk(SOURCE_FOLDER):
 
     relRoot = os.path.relpath(root, SOURCE_FOLDER)
 
-    # See if it's an album (only 2 components in the path)
     head, tail = os.path.split(relRoot)
+    # 1 component in the path: save files path as single
     if not len(head):
-        continue
+        for f in files:
+            if isMusic(f):
+                fullPath = os.path.join(root, f)
+                singleFiles.add(fullPath)
     head, tail = os.path.split(head)
     if len(head):
         continue
+    # 2 components in the path: save album path
     albums.add(root)
 
-cmds = list()
+log.info("Processing single files")
+# r128gain.process(list(singleFiles), album_gain=False, skip_tagged=not FORCE, report=True)
 for album in albums:
     albumName = os.path.relpath(album, SOURCE_FOLDER)
     log.info("Processing album {}".format(albumName))
 
-    musicFiles = dict()
+    musicFiles = set()
     for root, dirs, files in os.walk(album):
         for f in files:
-            ext = os.path.splitext(f)[1][1:].lower()
+            if isMusic(f):
+                fullPath = os.path.join(root, f)
+                musicFiles.add(fullPath)
 
-            for exts in GAIN_COMMANDS.keys():
+    # print(musicFiles)
-                if ext in exts:
+    if not len(musicFiles):
-                    if exts not in musicFiles.keys():
+        continue
-                        musicFiles[exts] = set()
+    r128gain.process(list(musicFiles), album_gain=True, skip_tagged=not FORCE, report=True)
-                    fullPath = os.path.join(root, f)
+    print("==============================")
-                    musicFiles[exts].add(fullPath)
-
-    if len(musicFiles) >= 2:
-        log.warn("Different extensions for album {}. AlbumGain won't be on par.".format(albumName))
-
-    for exts, files in musicFiles.items():
-
-        if exts == ("flac",) and not FORCE:
-            allTagged = True
-            for f in files:
-                if not isFlacTagged(f):
-                    allTaged = False
-                    break
-            if allTagged:
-                log.debug("Already tagged (for flac only)!")
-                break
-
-        cmd = GAIN_COMMANDS[exts] + list(files)
-        log.debug("Registering command: `{}`".format(" ".join(cmd)))
-        cmds.append(cmd)
-
-logging.info("Executing commands")
-for cmd in progressbar.progressbar(cmds):
-    subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
 
 
 

scripts/tagCreatorPhotos

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import piexif
+
+assert len(sys.argv) >= 3, "Usage {} CREATOR FILENAMES...".format(sys.argv[0])
+creator = sys.argv[1]
+filenames = sys.argv[2:]
+
+for filename in filenames:
+    assert os.path.isfile(filename)
+    exifDict = piexif.load(filename)
+    exifDict['0th'][piexif.ImageIFD.Copyright] = creator.encode()
+    exifBytes = piexif.dump(exifDict)
+    piexif.insert(exifBytes, filename)
+

vimpcrc

@@ -5,3 +5,4 @@ map ° D:browse<C-M>A:shuffle<C-M>:play<C-M>:playlist<C-M>
 set songformat {%a - %b: %t}|{%f}$E$R $H[$H%l$H]$H
 set libraryformat %n \| {%t}|{%f}$E$R $H[$H%l$H]$H
 set ignorecase
+set sort library

vimrc

@@ -41,15 +41,30 @@ Plug 'tomtom/tcomment_vim'
 " Plug 'tomlion/vim-solidity'
 " Plug 'godlygeek/tabular'
 " Plug 'jrozner/vim-antlr'
-Plug 'maralla/completor.vim'
+"
+" Plug 'maralla/completor.vim'
+if has('nvim')
+  Plug 'Shougo/deoplete.nvim', { 'do': ':UpdateRemotePlugins' }
+else
+  Plug 'Shougo/deoplete.nvim'
+  Plug 'roxma/nvim-yarp'
+  Plug 'roxma/vim-hug-neovim-rpc'
+endif
+Plug 'zchee/deoplete-jedi'
+
 Plug 'python-mode/python-mode', { 'branch': 'develop' }
 Plug 'junegunn/fzf', {'do': './install --bin'}
 Plug 'junegunn/fzf.vim'
 Plug 'ervandew/supertab'
 Plug 'dpelle/vim-LanguageTool'
+Plug 'terryma/vim-smooth-scroll'
 
 call plug#end()
 
+""" COMPLETOR """
+
+let g:deoplete#enable_at_startup = 1
+
 """ UNDOTREE """
 
 nmap <F7> :UndotreeToggle<CR>
@@ -69,7 +84,7 @@ let g:airline#extensions#tabline#enabled = 1
 let g:airline_section_a = airline#section#create(['mode'])
 let g:airline_section_b = airline#section#create(['branch', 'hunks'])
 let g:airline_section_z = airline#section#create(['%B', '@', '%l', ':', '%c'])
-let g:airline_theme = 'base16'
+let g:airline_theme = 'base16_monokai'
 
 """ AUTOFORMAT """
 nmap <F3> :Autoformat<CR>
@@ -203,7 +218,14 @@ vmap <Enter> <Esc>
 nmap <Enter> o<Esc>
 nmap <C-H> :bp<CR>
 nmap <C-L> :bn<CR>
-nmap <C-K> kkkkkkkkkkkkkkkkkkkkk
+if has('nvim')
-nmap <C-J> jjjjjjjjjjjjjjjjjjjjj
+    " nmap <C-K> 20k
+    " nmap <C-J> 20j
+    noremap <silent> <C-K> :call smooth_scroll#up(20, 5, 1)<CR>
+    noremap <silent> <C-J> :call smooth_scroll#down(20, 5, 1)<CR>
+else
+    nmap <C-K> kkkkkkkkkkkkkkkkkkkkk
+    nmap <C-J> jjjjjjjjjjjjjjjjjjjjj
+endif