Compress all the way

This commit is contained in:
Geoffrey Frogeye 2018-06-24 18:28:37 +02:00
parent 81a0bc59c5
commit c4a6f653a3
6 changed files with 284 additions and 1 deletions

163
scripts/compressPictureMovies Executable file
View file

@ -0,0 +1,163 @@
#!/usr/bin/env python3
import os
import shutil
import subprocess
import sys
# Constants
PICTURES_FOLDER = os.path.join(os.path.expanduser("~"), "Images")
ORIGNAL_FOLDER = os.path.join(PICTURES_FOLDER, ".Originaux")
MOVIE_EXTENSIONS = ["mov", "avi", "mp4"]
OUTPUT_EXTENSION = "mp4"
OUTPUT_FFMPEG_PARAMETERS = ["-codec:v", "libx265", "-crf", "28", "-preset:v", "slower", "-codec:a", "libfdk_aac", "-movflags", "+faststart", "-vbr", "5"]
OUTPUT_METADATA_FIELD = ["episode_id"]
# Walk folders
for root, dirs, files in os.walk(PICTURES_FOLDER):
# If folder is in ORIGINAL_FOLDER, skip it
if root.startswith(ORIGNAL_FOLDER):
continue
# Iterate over files
for inputName in files:
# If the file is not a video, skip it
inputNameBase, inputExt = os.path.splitext(inputName)
inputExt = inputExt[1:].lower()
if inputExt not in MOVIE_EXTENSIONS:
continue
# Generates all needed filepaths
## Found file
inputFull = os.path.join(root, inputName)
inputRel = os.path.relpath(inputFull, PICTURES_FOLDER)
## Original file
originalFull = os.path.join(ORIGNAL_FOLDER, inputRel)
originalRel = inputRel
## Compressed file
outputFull = os.path.join(root, inputNameBase + "." + OUTPUT_EXTENSION)
# If the extension is the same of the output one
if inputExt == OUTPUT_EXTENSION:
# Read the metadata of the video
metadataRaw = subprocess.run(["ffmpeg", "-i", inputFull, "-f", "ffmetadata", "-"], stdout=subprocess.PIPE).stdout
# If it has the field with the original file
originalRel = None
wantedPattern = OUTPUT_METADATA_FIELD.encode() + b"="
for metadataLine in metadataRaw.split('\n'):
if metadataLine.startswith(wantedPattern):
originalRel = metadataLine[len(wantedPattern)+1:]
break
if originalRel:
# If the original file does not exists, warn about it
originalFull = os.path.join(ORIGNAL_FOLDER, originalRel)
if not os.path.isfile(originalFull):
print("WARN {inputRel} states to have {originalRel} as original but this file doesn't exist".format(inputRel=inputRel, originalRel=originalRel))
# If the original is not aligned with the compressed, warn about it (TODO move it automatically)
if inputRel != originalRel:
print("WARN {inputRel} is not aligned with original {originalRel}".format(inputRel=inputRel, originalRel=originalRel))
# Skip file
continue
# Initiate a conversion in a temporary file
# If the temporary file does not have the same caracteristics as the original
# Warn about it
# Delete it
# Skip file
# Move the original to the corresponding original folder
# Move the converted file in place of the original
# TODO Iterate over the orignal folder to find non-matching compressed videos not found in the above pass
sys.exit(0)
# Constants
SOURCE_FOLDER = os.path.join(os.path.expanduser("~"), "Musique")
OUTPUT_FOLDER = os.path.join(os.path.expanduser("~"), ".MusiqueCompressed")
CONVERSIONS = {"flac": "m4a"}
FORBIDDEN_EXTENSIONS = ["jpg", "pdf", "ffs_db"]
FORGIVEN_FILENAMES = ["cover.jpg"]
IGNORED_EMPTY_FOLDER = [".stfolder"]
# Listing files
sourceFiles = dict()
for root, dirs, files in os.walk(SOURCE_FOLDER):
for f in files:
fullPath = os.path.join(root, f)
path = os.path.relpath(fullPath, SOURCE_FOLDER)
sourceFiles[path] = os.path.getctime(fullPath)
outputFiles = dict()
for root, dirs, files in os.walk(OUTPUT_FOLDER):
for f in files:
fullPath = os.path.join(root, f)
path = os.path.relpath(fullPath, OUTPUT_FOLDER)
outputFiles[path] = os.path.getctime(fullPath)
# Sorting files
remainingConversions = dict()
extraFiles = list(outputFiles.keys())
def convertPath(path):
filename, extension = os.path.splitext(path)
extension = extension[1:].lower()
# If the extension isn't allowed
if extension in FORBIDDEN_EXTENSIONS:
basename = os.path.basename(path)
# And the filename is not an exception
if basename not in FORGIVEN_FILENAMES:
# This file shouldn't be copied nor converted
return False
# If this needs a conversion
elif extension in CONVERSIONS:
extension = CONVERSIONS[extension]
return filename + "." + extension
# In all other case, this is a simple copy
return path
for sourceFile in sourceFiles:
outputFile = convertPath(sourceFile)
# If the file should not be converted, do nothing
if outputFile == False:
continue
# If the file already has something as an output
elif outputFile in outputFiles:
extraFiles.remove(outputFile)
# If the output file is newer than the source file, do not initiate a conversion
if outputFiles[outputFile] >= sourceFiles[sourceFile]:
continue
# If the file needs to be converted, do it
remainingConversions[sourceFile] = outputFile
# Converting files
for sourceFile in remainingConversions:
outputFile = remainingConversions[sourceFile]
# Creating folder if it doesn't exists
fullOutputFile = os.path.join(OUTPUT_FOLDER, outputFile)
fullOutputDir = os.path.dirname(fullOutputFile)
os.makedirs(fullOutputDir, exist_ok=True)
# Converting
fullSourceFile = os.path.join(SOURCE_FOLDER, sourceFile)
print(fullSourceFile, "→", fullOutputFile)
if sourceFile == outputFile:
# shutil.copy(fullSourceFile, fullOutputFile)
os.link(fullSourceFile, fullOutputFile)
else:
subprocess.run(["ffmpeg", "-y", "-i", fullSourceFile, "-codec:a", "libfdk_aac", "-cutoff", "18000", "-movflags", "+faststart", "-vbr", "5", fullOutputFile])
# Removing extra files
for extraFile in extraFiles:
fullExtraFile = os.path.join(OUTPUT_FOLDER, extraFile)
os.remove(fullExtraFile)
# Removing empty dirs
for root, dirs, files in os.walk(OUTPUT_FOLDER):
if not dirs and not files:
dirBasename = os.path.basename(root)
if dirBasename not in IGNORED_EMPTY_FOLDER:
os.rmdir(root)

View file

@ -31,6 +31,8 @@ apt upgrade
# (needed for install-prefs) # (needed for install-prefs)
apt install coreutils apt install coreutils
apt install grep apt install grep
# Used by some of my termux scripts
apt install jq
# Config # Config
touch ~/.hushlogin touch ~/.hushlogin

View file

@ -114,7 +114,7 @@ if (latex) {
// Conversion // Conversion
htmlString = marked(markdownString, { htmlString = marked(markdownString, {
renderer: renderer, renderer: renderer,
breaks: true breaks: false
}); });
// fullHtmlString = htmlString; // fullHtmlString = htmlString;
fullHtmlString = template.replace('%BODY%', () => { return htmlString }); fullHtmlString = template.replace('%BODY%', () => { return htmlString });

95
scripts/updateCompressedMusic Executable file
View file

@ -0,0 +1,95 @@
#!/usr/bin/env python3
import os
import shutil
import subprocess
# Constants
SOURCE_FOLDER = os.path.join(os.path.expanduser("~"), "Musique")
OUTPUT_FOLDER = os.path.join(os.path.expanduser("~"), ".MusiqueCompressed")
CONVERSIONS = {"flac": "m4a"}
FORBIDDEN_EXTENSIONS = ["jpg", "pdf", "ffs_db"]
FORGIVEN_FILENAMES = ["cover.jpg"]
IGNORED_EMPTY_FOLDER = [".stfolder"]
# Listing files
sourceFiles = dict()
for root, dirs, files in os.walk(SOURCE_FOLDER):
for f in files:
fullPath = os.path.join(root, f)
path = os.path.relpath(fullPath, SOURCE_FOLDER)
sourceFiles[path] = os.path.getctime(fullPath)
outputFiles = dict()
for root, dirs, files in os.walk(OUTPUT_FOLDER):
for f in files:
fullPath = os.path.join(root, f)
path = os.path.relpath(fullPath, OUTPUT_FOLDER)
outputFiles[path] = os.path.getctime(fullPath)
# Sorting files
remainingConversions = dict()
extraFiles = list(outputFiles.keys())
def convertPath(path):
filename, extension = os.path.splitext(path)
extension = extension[1:].lower()
# If the extension isn't allowed
if extension in FORBIDDEN_EXTENSIONS:
basename = os.path.basename(path)
# And the filename is not an exception
if basename not in FORGIVEN_FILENAMES:
# This file shouldn't be copied nor converted
return False
# If this needs a conversion
elif extension in CONVERSIONS:
extension = CONVERSIONS[extension]
return filename + "." + extension
# In all other case, this is a simple copy
return path
for sourceFile in sourceFiles:
outputFile = convertPath(sourceFile)
# If the file should not be converted, do nothing
if outputFile == False:
continue
# If the file already has something as an output
elif outputFile in outputFiles:
extraFiles.remove(outputFile)
# If the output file is newer than the source file, do not initiate a conversion
if outputFiles[outputFile] >= sourceFiles[sourceFile]:
continue
# If the file needs to be converted, do it
remainingConversions[sourceFile] = outputFile
# Converting files
for sourceFile in remainingConversions:
outputFile = remainingConversions[sourceFile]
# Creating folder if it doesn't exists
fullOutputFile = os.path.join(OUTPUT_FOLDER, outputFile)
fullOutputDir = os.path.dirname(fullOutputFile)
os.makedirs(fullOutputDir, exist_ok=True)
# Converting
fullSourceFile = os.path.join(SOURCE_FOLDER, sourceFile)
print(fullSourceFile, "→", fullOutputFile)
if sourceFile == outputFile:
# shutil.copy(fullSourceFile, fullOutputFile)
os.link(fullSourceFile, fullOutputFile)
else:
subprocess.run(["ffmpeg", "-y", "-i", fullSourceFile, "-codec:a", "libfdk_aac", "-cutoff", "18000", "-movflags", "+faststart", "-vbr", "5", fullOutputFile])
# Removing extra files
for extraFile in extraFiles:
fullExtraFile = os.path.join(OUTPUT_FOLDER, extraFile)
os.remove(fullExtraFile)
# Removing empty dirs
for root, dirs, files in os.walk(OUTPUT_FOLDER):
if not dirs and not files:
dirBasename = os.path.basename(root)
if dirBasename not in IGNORED_EMPTY_FOLDER:
os.rmdir(root)

3
scripts/updatedate Executable file
View file

@ -0,0 +1,3 @@
#!/usr/bin/env bash
echo ssh "$1" date --set="'$(date -R)'"

20
scripts/vidcmp Executable file
View file

@ -0,0 +1,20 @@
#!/usr/bin/env python3
# Compresses video using FFMPEG using
# FFMPEG's reasonable default settings
import os
import sys
import subprocess
files = sys.argv[1:]
remove = False
if '-r' in files:
files.remove('-r')
remove = True
for f in files:
print(os.path.splitext(f))