Compress all the way

scripts/compressPictureMovies

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+
+import os
+import shutil
+import subprocess
+import sys
+
+# Constants
+PICTURES_FOLDER = os.path.join(os.path.expanduser("~"), "Images")
+ORIGNAL_FOLDER = os.path.join(PICTURES_FOLDER, ".Originaux")
+MOVIE_EXTENSIONS = ["mov", "avi", "mp4"]
+OUTPUT_EXTENSION = "mp4"
+OUTPUT_FFMPEG_PARAMETERS = ["-codec:v", "libx265", "-crf", "28", "-preset:v", "slower", "-codec:a", "libfdk_aac", "-movflags", "+faststart", "-vbr", "5"]
+OUTPUT_METADATA_FIELD = ["episode_id"]
+
+
+# Walk folders
+for root, dirs, files in os.walk(PICTURES_FOLDER):
+    # If folder is in ORIGINAL_FOLDER, skip it
+    if root.startswith(ORIGNAL_FOLDER):
+        continue
+    # Iterate over files
+    for inputName in files:
+        # If the file is not a video, skip it
+        inputNameBase, inputExt = os.path.splitext(inputName)
+        inputExt = inputExt[1:].lower()
+        if inputExt not in MOVIE_EXTENSIONS:
+            continue
+
+        # Generates all needed filepaths
+        ## Found file
+        inputFull = os.path.join(root, inputName)
+        inputRel = os.path.relpath(inputFull, PICTURES_FOLDER)
+        ## Original file
+        originalFull = os.path.join(ORIGNAL_FOLDER, inputRel)
+        originalRel = inputRel
+        ## Compressed file
+        outputFull = os.path.join(root, inputNameBase + "." + OUTPUT_EXTENSION)
+
+        # If the extension is the same of the output one
+        if inputExt == OUTPUT_EXTENSION:
+            # Read the metadata of the video
+            metadataRaw = subprocess.run(["ffmpeg", "-i", inputFull, "-f", "ffmetadata", "-"], stdout=subprocess.PIPE).stdout
+            # If it has the field with the original file 
+            originalRel = None
+            wantedPattern = OUTPUT_METADATA_FIELD.encode() + b"="
+            for metadataLine in metadataRaw.split('\n'):
+                if metadataLine.startswith(wantedPattern):
+                    originalRel = metadataLine[len(wantedPattern)+1:]
+                    break
+            if originalRel:
+                # If the original file does not exists, warn about it
+                originalFull = os.path.join(ORIGNAL_FOLDER, originalRel)
+                if not os.path.isfile(originalFull):
+                    print("WARN {inputRel} states to have {originalRel} as original but this file doesn't exist".format(inputRel=inputRel, originalRel=originalRel))
+                # If the original is not aligned with the compressed, warn about it (TODO move it automatically)
+                if inputRel != originalRel:
+                    print("WARN {inputRel} is not aligned with original {originalRel}".format(inputRel=inputRel, originalRel=originalRel))
+                # Skip file
+                continue
+            # Initiate a conversion in a temporary file
+            # If the temporary file does not have the same caracteristics as the original
+                # Warn about it
+                # Delete it
+                # Skip file
+            # Move the original to the corresponding original folder
+            # Move the converted file in place of the original
+
+# TODO Iterate over the orignal folder to find non-matching compressed videos not found in the above pass
+
+sys.exit(0)
+
+# Constants
+SOURCE_FOLDER = os.path.join(os.path.expanduser("~"), "Musique")
+OUTPUT_FOLDER = os.path.join(os.path.expanduser("~"), ".MusiqueCompressed")
+CONVERSIONS = {"flac": "m4a"}
+FORBIDDEN_EXTENSIONS = ["jpg", "pdf", "ffs_db"]
+FORGIVEN_FILENAMES = ["cover.jpg"]
+IGNORED_EMPTY_FOLDER = [".stfolder"]
+
+
+
+# Listing files
+sourceFiles = dict()
+for root, dirs, files in os.walk(SOURCE_FOLDER):
+    for f in files:
+        fullPath = os.path.join(root, f)
+        path = os.path.relpath(fullPath, SOURCE_FOLDER)
+        sourceFiles[path] = os.path.getctime(fullPath)
+
+outputFiles = dict()
+for root, dirs, files in os.walk(OUTPUT_FOLDER):
+    for f in files:
+        fullPath = os.path.join(root, f)
+        path = os.path.relpath(fullPath, OUTPUT_FOLDER)
+        outputFiles[path] = os.path.getctime(fullPath)
+
+# Sorting files
+remainingConversions = dict()
+extraFiles = list(outputFiles.keys())
+
+def convertPath(path):
+    filename, extension = os.path.splitext(path)
+    extension = extension[1:].lower()
+    # If the extension isn't allowed
+    if extension in FORBIDDEN_EXTENSIONS:
+        basename = os.path.basename(path)
+        # And the filename is not an exception
+        if basename not in FORGIVEN_FILENAMES:
+            # This file shouldn't be copied nor converted
+            return False
+    # If this needs a conversion
+    elif extension in CONVERSIONS:
+        extension = CONVERSIONS[extension]
+        return filename + "." + extension
+    # In all other case, this is a simple copy
+    return path
+
+for sourceFile in sourceFiles:
+    outputFile = convertPath(sourceFile)
+    # If the file should not be converted, do nothing
+    if outputFile == False:
+        continue
+    # If the file already has something as an output
+    elif outputFile in outputFiles:
+        extraFiles.remove(outputFile)
+        # If the output file is newer than the source file, do not initiate a conversion
+        if outputFiles[outputFile] >= sourceFiles[sourceFile]:
+            continue
+    # If the file needs to be converted, do it
+    remainingConversions[sourceFile] = outputFile
+
+# Converting files
+for sourceFile in remainingConversions:
+    outputFile = remainingConversions[sourceFile]
+
+    # Creating folder if it doesn't exists
+    fullOutputFile = os.path.join(OUTPUT_FOLDER, outputFile)
+    fullOutputDir = os.path.dirname(fullOutputFile)
+    os.makedirs(fullOutputDir, exist_ok=True)
+
+    # Converting
+    fullSourceFile = os.path.join(SOURCE_FOLDER, sourceFile)
+    print(fullSourceFile, "→", fullOutputFile)
+    if sourceFile == outputFile:
+        # shutil.copy(fullSourceFile, fullOutputFile)
+        os.link(fullSourceFile, fullOutputFile)
+    else:
+        subprocess.run(["ffmpeg", "-y", "-i", fullSourceFile, "-codec:a", "libfdk_aac", "-cutoff", "18000", "-movflags", "+faststart", "-vbr", "5", fullOutputFile])
+
+# Removing extra files
+for extraFile in extraFiles:
+    fullExtraFile = os.path.join(OUTPUT_FOLDER, extraFile)
+    os.remove(fullExtraFile)
+
+# Removing empty dirs
+for root, dirs, files in os.walk(OUTPUT_FOLDER):
+    if not dirs and not files:
+        dirBasename = os.path.basename(root)
+        if dirBasename not in IGNORED_EMPTY_FOLDER:
+            os.rmdir(root)
+
+

scripts/install-termux

@@ -31,6 +31,8 @@ apt upgrade
 # (needed for install-prefs)
 apt install coreutils
 apt install grep
+# Used by some of my termux scripts
+apt install jq
 
 # Config
 touch ~/.hushlogin

scripts/md2html

@@ -114,7 +114,7 @@ if (latex) {
 // Conversion
 htmlString = marked(markdownString, {
     renderer: renderer,
-    breaks: true
+    breaks: false
 });
 // fullHtmlString = htmlString;
 fullHtmlString = template.replace('%BODY%', () => { return htmlString });

scripts/updateCompressedMusic

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+
+import os
+import shutil
+import subprocess
+
+# Constants
+SOURCE_FOLDER = os.path.join(os.path.expanduser("~"), "Musique")
+OUTPUT_FOLDER = os.path.join(os.path.expanduser("~"), ".MusiqueCompressed")
+CONVERSIONS = {"flac": "m4a"}
+FORBIDDEN_EXTENSIONS = ["jpg", "pdf", "ffs_db"]
+FORGIVEN_FILENAMES = ["cover.jpg"]
+IGNORED_EMPTY_FOLDER = [".stfolder"]
+
+# Listing files
+sourceFiles = dict()
+for root, dirs, files in os.walk(SOURCE_FOLDER):
+    for f in files:
+        fullPath = os.path.join(root, f)
+        path = os.path.relpath(fullPath, SOURCE_FOLDER)
+        sourceFiles[path] = os.path.getctime(fullPath)
+
+outputFiles = dict()
+for root, dirs, files in os.walk(OUTPUT_FOLDER):
+    for f in files:
+        fullPath = os.path.join(root, f)
+        path = os.path.relpath(fullPath, OUTPUT_FOLDER)
+        outputFiles[path] = os.path.getctime(fullPath)
+
+# Sorting files
+remainingConversions = dict()
+extraFiles = list(outputFiles.keys())
+
+def convertPath(path):
+    filename, extension = os.path.splitext(path)
+    extension = extension[1:].lower()
+    # If the extension isn't allowed
+    if extension in FORBIDDEN_EXTENSIONS:
+        basename = os.path.basename(path)
+        # And the filename is not an exception
+        if basename not in FORGIVEN_FILENAMES:
+            # This file shouldn't be copied nor converted
+            return False
+    # If this needs a conversion
+    elif extension in CONVERSIONS:
+        extension = CONVERSIONS[extension]
+        return filename + "." + extension
+    # In all other case, this is a simple copy
+    return path
+
+for sourceFile in sourceFiles:
+    outputFile = convertPath(sourceFile)
+    # If the file should not be converted, do nothing
+    if outputFile == False:
+        continue
+    # If the file already has something as an output
+    elif outputFile in outputFiles:
+        extraFiles.remove(outputFile)
+        # If the output file is newer than the source file, do not initiate a conversion
+        if outputFiles[outputFile] >= sourceFiles[sourceFile]:
+            continue
+    # If the file needs to be converted, do it
+    remainingConversions[sourceFile] = outputFile
+
+# Converting files
+for sourceFile in remainingConversions:
+    outputFile = remainingConversions[sourceFile]
+
+    # Creating folder if it doesn't exists
+    fullOutputFile = os.path.join(OUTPUT_FOLDER, outputFile)
+    fullOutputDir = os.path.dirname(fullOutputFile)
+    os.makedirs(fullOutputDir, exist_ok=True)
+
+    # Converting
+    fullSourceFile = os.path.join(SOURCE_FOLDER, sourceFile)
+    print(fullSourceFile, "→", fullOutputFile)
+    if sourceFile == outputFile:
+        # shutil.copy(fullSourceFile, fullOutputFile)
+        os.link(fullSourceFile, fullOutputFile)
+    else:
+        subprocess.run(["ffmpeg", "-y", "-i", fullSourceFile, "-codec:a", "libfdk_aac", "-cutoff", "18000", "-movflags", "+faststart", "-vbr", "5", fullOutputFile])
+
+# Removing extra files
+for extraFile in extraFiles:
+    fullExtraFile = os.path.join(OUTPUT_FOLDER, extraFile)
+    os.remove(fullExtraFile)
+
+# Removing empty dirs
+for root, dirs, files in os.walk(OUTPUT_FOLDER):
+    if not dirs and not files:
+        dirBasename = os.path.basename(root)
+        if dirBasename not in IGNORED_EMPTY_FOLDER:
+            os.rmdir(root)
+
+

scripts/updatedate

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+echo ssh "$1" date --set="'$(date -R)'"

scripts/vidcmp

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+
+# Compresses video using FFMPEG using
+# FFMPEG's reasonable default settings
+
+import os
+import sys
+import subprocess
+
+files = sys.argv[1:]
+
+remove = False
+if '-r' in files:
+    files.remove('-r')
+    remove = True
+
+for f in files:
+    print(os.path.splitext(f))
+
+