From c4a6f653a3738283623a55217f6a59b34497b7e0 Mon Sep 17 00:00:00 2001 From: Geoffrey Frogeye Date: Sun, 24 Jun 2018 18:28:37 +0200 Subject: [PATCH] Compress all the way --- scripts/compressPictureMovies | 163 ++++++++++++++++++++++++++++++++++ scripts/install-termux | 2 + scripts/md2html | 2 +- scripts/updateCompressedMusic | 95 ++++++++++++++++++++ scripts/updatedate | 3 + scripts/vidcmp | 20 +++++ 6 files changed, 284 insertions(+), 1 deletion(-) create mode 100755 scripts/compressPictureMovies create mode 100755 scripts/updateCompressedMusic create mode 100755 scripts/updatedate create mode 100755 scripts/vidcmp diff --git a/scripts/compressPictureMovies b/scripts/compressPictureMovies new file mode 100755 index 0000000..1a929ba --- /dev/null +++ b/scripts/compressPictureMovies @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 + +import os +import shutil +import subprocess +import sys + +# Constants +PICTURES_FOLDER = os.path.join(os.path.expanduser("~"), "Images") +ORIGNAL_FOLDER = os.path.join(PICTURES_FOLDER, ".Originaux") +MOVIE_EXTENSIONS = ["mov", "avi", "mp4"] +OUTPUT_EXTENSION = "mp4" +OUTPUT_FFMPEG_PARAMETERS = ["-codec:v", "libx265", "-crf", "28", "-preset:v", "slower", "-codec:a", "libfdk_aac", "-movflags", "+faststart", "-vbr", "5"] +OUTPUT_METADATA_FIELD = ["episode_id"] + + +# Walk folders +for root, dirs, files in os.walk(PICTURES_FOLDER): + # If folder is in ORIGINAL_FOLDER, skip it + if root.startswith(ORIGNAL_FOLDER): + continue + # Iterate over files + for inputName in files: + # If the file is not a video, skip it + inputNameBase, inputExt = os.path.splitext(inputName) + inputExt = inputExt[1:].lower() + if inputExt not in MOVIE_EXTENSIONS: + continue + + # Generates all needed filepaths + ## Found file + inputFull = os.path.join(root, inputName) + inputRel = os.path.relpath(inputFull, PICTURES_FOLDER) + ## Original file + originalFull = os.path.join(ORIGNAL_FOLDER, inputRel) + originalRel = inputRel + ## Compressed file + outputFull = os.path.join(root, inputNameBase + "." + OUTPUT_EXTENSION) + + # If the extension is the same of the output one + if inputExt == OUTPUT_EXTENSION: + # Read the metadata of the video + metadataRaw = subprocess.run(["ffmpeg", "-i", inputFull, "-f", "ffmetadata", "-"], stdout=subprocess.PIPE).stdout + # If it has the field with the original file + originalRel = None + wantedPattern = OUTPUT_METADATA_FIELD.encode() + b"=" + for metadataLine in metadataRaw.split('\n'): + if metadataLine.startswith(wantedPattern): + originalRel = metadataLine[len(wantedPattern)+1:] + break + if originalRel: + # If the original file does not exists, warn about it + originalFull = os.path.join(ORIGNAL_FOLDER, originalRel) + if not os.path.isfile(originalFull): + print("WARN {inputRel} states to have {originalRel} as original but this file doesn't exist".format(inputRel=inputRel, originalRel=originalRel)) + # If the original is not aligned with the compressed, warn about it (TODO move it automatically) + if inputRel != originalRel: + print("WARN {inputRel} is not aligned with original {originalRel}".format(inputRel=inputRel, originalRel=originalRel)) + # Skip file + continue + # Initiate a conversion in a temporary file + # If the temporary file does not have the same caracteristics as the original + # Warn about it + # Delete it + # Skip file + # Move the original to the corresponding original folder + # Move the converted file in place of the original + +# TODO Iterate over the orignal folder to find non-matching compressed videos not found in the above pass + +sys.exit(0) + +# Constants +SOURCE_FOLDER = os.path.join(os.path.expanduser("~"), "Musique") +OUTPUT_FOLDER = os.path.join(os.path.expanduser("~"), ".MusiqueCompressed") +CONVERSIONS = {"flac": "m4a"} +FORBIDDEN_EXTENSIONS = ["jpg", "pdf", "ffs_db"] +FORGIVEN_FILENAMES = ["cover.jpg"] +IGNORED_EMPTY_FOLDER = [".stfolder"] + + + +# Listing files +sourceFiles = dict() +for root, dirs, files in os.walk(SOURCE_FOLDER): + for f in files: + fullPath = os.path.join(root, f) + path = os.path.relpath(fullPath, SOURCE_FOLDER) + sourceFiles[path] = os.path.getctime(fullPath) + +outputFiles = dict() +for root, dirs, files in os.walk(OUTPUT_FOLDER): + for f in files: + fullPath = os.path.join(root, f) + path = os.path.relpath(fullPath, OUTPUT_FOLDER) + outputFiles[path] = os.path.getctime(fullPath) + +# Sorting files +remainingConversions = dict() +extraFiles = list(outputFiles.keys()) + +def convertPath(path): + filename, extension = os.path.splitext(path) + extension = extension[1:].lower() + # If the extension isn't allowed + if extension in FORBIDDEN_EXTENSIONS: + basename = os.path.basename(path) + # And the filename is not an exception + if basename not in FORGIVEN_FILENAMES: + # This file shouldn't be copied nor converted + return False + # If this needs a conversion + elif extension in CONVERSIONS: + extension = CONVERSIONS[extension] + return filename + "." + extension + # In all other case, this is a simple copy + return path + +for sourceFile in sourceFiles: + outputFile = convertPath(sourceFile) + # If the file should not be converted, do nothing + if outputFile == False: + continue + # If the file already has something as an output + elif outputFile in outputFiles: + extraFiles.remove(outputFile) + # If the output file is newer than the source file, do not initiate a conversion + if outputFiles[outputFile] >= sourceFiles[sourceFile]: + continue + # If the file needs to be converted, do it + remainingConversions[sourceFile] = outputFile + +# Converting files +for sourceFile in remainingConversions: + outputFile = remainingConversions[sourceFile] + + # Creating folder if it doesn't exists + fullOutputFile = os.path.join(OUTPUT_FOLDER, outputFile) + fullOutputDir = os.path.dirname(fullOutputFile) + os.makedirs(fullOutputDir, exist_ok=True) + + # Converting + fullSourceFile = os.path.join(SOURCE_FOLDER, sourceFile) + print(fullSourceFile, "→", fullOutputFile) + if sourceFile == outputFile: + # shutil.copy(fullSourceFile, fullOutputFile) + os.link(fullSourceFile, fullOutputFile) + else: + subprocess.run(["ffmpeg", "-y", "-i", fullSourceFile, "-codec:a", "libfdk_aac", "-cutoff", "18000", "-movflags", "+faststart", "-vbr", "5", fullOutputFile]) + +# Removing extra files +for extraFile in extraFiles: + fullExtraFile = os.path.join(OUTPUT_FOLDER, extraFile) + os.remove(fullExtraFile) + +# Removing empty dirs +for root, dirs, files in os.walk(OUTPUT_FOLDER): + if not dirs and not files: + dirBasename = os.path.basename(root) + if dirBasename not in IGNORED_EMPTY_FOLDER: + os.rmdir(root) + + diff --git a/scripts/install-termux b/scripts/install-termux index 51e3437..f066717 100755 --- a/scripts/install-termux +++ b/scripts/install-termux @@ -31,6 +31,8 @@ apt upgrade # (needed for install-prefs) apt install coreutils apt install grep +# Used by some of my termux scripts +apt install jq # Config touch ~/.hushlogin diff --git a/scripts/md2html b/scripts/md2html index 9564de2..ca33436 100755 --- a/scripts/md2html +++ b/scripts/md2html @@ -114,7 +114,7 @@ if (latex) { // Conversion htmlString = marked(markdownString, { renderer: renderer, - breaks: true + breaks: false }); // fullHtmlString = htmlString; fullHtmlString = template.replace('%BODY%', () => { return htmlString }); diff --git a/scripts/updateCompressedMusic b/scripts/updateCompressedMusic new file mode 100755 index 0000000..e4aefce --- /dev/null +++ b/scripts/updateCompressedMusic @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 + +import os +import shutil +import subprocess + +# Constants +SOURCE_FOLDER = os.path.join(os.path.expanduser("~"), "Musique") +OUTPUT_FOLDER = os.path.join(os.path.expanduser("~"), ".MusiqueCompressed") +CONVERSIONS = {"flac": "m4a"} +FORBIDDEN_EXTENSIONS = ["jpg", "pdf", "ffs_db"] +FORGIVEN_FILENAMES = ["cover.jpg"] +IGNORED_EMPTY_FOLDER = [".stfolder"] + +# Listing files +sourceFiles = dict() +for root, dirs, files in os.walk(SOURCE_FOLDER): + for f in files: + fullPath = os.path.join(root, f) + path = os.path.relpath(fullPath, SOURCE_FOLDER) + sourceFiles[path] = os.path.getctime(fullPath) + +outputFiles = dict() +for root, dirs, files in os.walk(OUTPUT_FOLDER): + for f in files: + fullPath = os.path.join(root, f) + path = os.path.relpath(fullPath, OUTPUT_FOLDER) + outputFiles[path] = os.path.getctime(fullPath) + +# Sorting files +remainingConversions = dict() +extraFiles = list(outputFiles.keys()) + +def convertPath(path): + filename, extension = os.path.splitext(path) + extension = extension[1:].lower() + # If the extension isn't allowed + if extension in FORBIDDEN_EXTENSIONS: + basename = os.path.basename(path) + # And the filename is not an exception + if basename not in FORGIVEN_FILENAMES: + # This file shouldn't be copied nor converted + return False + # If this needs a conversion + elif extension in CONVERSIONS: + extension = CONVERSIONS[extension] + return filename + "." + extension + # In all other case, this is a simple copy + return path + +for sourceFile in sourceFiles: + outputFile = convertPath(sourceFile) + # If the file should not be converted, do nothing + if outputFile == False: + continue + # If the file already has something as an output + elif outputFile in outputFiles: + extraFiles.remove(outputFile) + # If the output file is newer than the source file, do not initiate a conversion + if outputFiles[outputFile] >= sourceFiles[sourceFile]: + continue + # If the file needs to be converted, do it + remainingConversions[sourceFile] = outputFile + +# Converting files +for sourceFile in remainingConversions: + outputFile = remainingConversions[sourceFile] + + # Creating folder if it doesn't exists + fullOutputFile = os.path.join(OUTPUT_FOLDER, outputFile) + fullOutputDir = os.path.dirname(fullOutputFile) + os.makedirs(fullOutputDir, exist_ok=True) + + # Converting + fullSourceFile = os.path.join(SOURCE_FOLDER, sourceFile) + print(fullSourceFile, "→", fullOutputFile) + if sourceFile == outputFile: + # shutil.copy(fullSourceFile, fullOutputFile) + os.link(fullSourceFile, fullOutputFile) + else: + subprocess.run(["ffmpeg", "-y", "-i", fullSourceFile, "-codec:a", "libfdk_aac", "-cutoff", "18000", "-movflags", "+faststart", "-vbr", "5", fullOutputFile]) + +# Removing extra files +for extraFile in extraFiles: + fullExtraFile = os.path.join(OUTPUT_FOLDER, extraFile) + os.remove(fullExtraFile) + +# Removing empty dirs +for root, dirs, files in os.walk(OUTPUT_FOLDER): + if not dirs and not files: + dirBasename = os.path.basename(root) + if dirBasename not in IGNORED_EMPTY_FOLDER: + os.rmdir(root) + + diff --git a/scripts/updatedate b/scripts/updatedate new file mode 100755 index 0000000..7339128 --- /dev/null +++ b/scripts/updatedate @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +echo ssh "$1" date --set="'$(date -R)'" diff --git a/scripts/vidcmp b/scripts/vidcmp new file mode 100755 index 0000000..f8e27eb --- /dev/null +++ b/scripts/vidcmp @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +# Compresses video using FFMPEG using +# FFMPEG's reasonable default settings + +import os +import sys +import subprocess + +files = sys.argv[1:] + +remove = False +if '-r' in files: + files.remove('-r') + remove = True + +for f in files: + print(os.path.splitext(f)) + +