From 054d282ba624d5e6c08e1782eb9f1c0b0e7d4184 Mon Sep 17 00:00:00 2001 From: Sofus Rose Date: Tue, 6 Dec 2016 23:06:56 -0500 Subject: [PATCH] Even more modularization. Working on using locals vs. globals in the new internal functions as best as at all possible. --- convmlv.sh | 44 +- src/core/develop.sh | 1264 ++++++++++++++-------------------- src/core/parsing.sh | 3 + src/core/proc.sh | 247 +++++++ src/documentation/TODO.sh | 4 + src/documentation/help.sh | 2 + src/helpers/error.sh | 50 +- src/helpers/format.sh | 32 + src/helpers/platform.sh | 2 + src/helpers/utility.sh | 41 +- src/imgProcessing/imgMath.sh | 30 + src/programs/califrame.sh | 4 + src/programs/shotSettings.sh | 21 + 13 files changed, 959 insertions(+), 785 deletions(-) create mode 100644 src/core/proc.sh create mode 100644 src/imgProcessing/imgMath.sh diff --git a/convmlv.sh b/convmlv.sh index 90bb351..2d2d14e 100755 --- a/convmlv.sh +++ b/convmlv.sh @@ -1,7 +1,6 @@ #!/bin/bash -VERSION="2.1.0a1" #Version string. -INPUT_ARGS=$(echo "$@") #The original input argument string. +#desc: Main file - uses convmlv components as needed. #~ The MIT License (MIT) @@ -25,6 +24,9 @@ INPUT_ARGS=$(echo "$@") #The original input argument string. #~ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE #~ SOFTWARE. +VERSION="2.1.0a2" #Version string. +INPUT_ARGS=$(echo "$@") #The original input argument string. + if [ $# == 0 ]; then #No given arguments. echo -e "\033[0;31m\033[1mNo arguments given.\033[0m\n\tType 'convmlv -h/--help' to see help page, or 'convmlv -v/--version' for current version string." fi @@ -214,6 +216,8 @@ source "$SRC_PATH/helpers/utility.sh" source "$SRC_PATH/core/parsing.sh" source "$SRC_PATH/helpers/error.sh" source "$SRC_PATH/core/develop.sh" +source "$SRC_PATH/core/proc.sh" +source "$SRC_PATH/imgProcessing/imgMath.sh" #OPTION PARSING FROM CONFIG AND CLI - same as at bottom of develop(). @@ -259,7 +263,41 @@ case "$PROGRAM" in ;; develop) checkDeps #Check static dependencies. - develop #Do the development step, using the globals that exist. + + ARG_INC=0 + for ARG in "${FILE_ARGS_ARRAY[@]}"; do #Go through FILE_ARGS_ARRAY array, copied from parsed $@ because $@ is going to be changing on 'set --' + #Check the argument + fReturn=$(checkArg "$ARG") + if [[ $fReturn = "false" ]]; then continue; fi + + #Evaluate local configuration file for file-specific blocks. + parseConf "$LCONFIG" true + + #Do the development step, using the globals that exist. + develop "$ARG" + + #RESET ARGS & REPARSE OPTIONS - same as in convmlv.sh. + #Big parse/reparse, making sure global, local, command line options all override each other correctly. + set -- $INPUT_ARGS #Reset the argument input for reparsing. + setDefaults #Hard set/reset all the lovely globals. + OPTIND=1 #Reset argument parsing. + + parseConf "$GCONFIG" false #Parse global config file. + + parseArgs "$@" #First, parse all cli args. We only need the -C flag, but that forces us to just parse everything. + shift $((OPTIND-1)) #Shift past all of the options to the file arguments. + + parseConf "$LCONFIG" false #Parse local config file. + set -- $INPUT_ARGS #Reset $@ for cli option reparsing. + OPTIND=1 #To reset argument parsing, we must set OPTIND to 1. + + parseArgs "$@" #Reparse cli to overwrite local config options. + shift $((OPTIND-1)) #Shift past all of the options to the file arguments. + OPTIND=1 #Reset argument index. + + #Decrement the arguments that are left. + let ARG_INC++ + done ;; esac diff --git a/src/core/develop.sh b/src/core/develop.sh index 0941c73..4f6278d 100644 --- a/src/core/develop.sh +++ b/src/core/develop.sh @@ -1,5 +1,8 @@ setRange() { #FRAMES must be set at this point. + #usage: setRange + #desc: Using FRAMES, sets the variables pertaining to frame range. + if [[ $isFR == true ]]; then #Ensure that FRAME_RANGE is set with $FRAMES. FRAME_RANGE="1-${FRAMES}" FRAME_START="1" @@ -22,787 +25,536 @@ setRange() { } develop() { - trap "rm -rf ${TMP}; exit 1" INT #TMP will be removed if you CTRL+C. - for ARG in "${FILE_ARGS_ARRAY[@]}"; do #Go through FILE_ARGS_ARRAY array, copied from parsed $@ because $@ is going to be changing on 'set --' - ARG=$(readlinkF "$ARG") #Use platform-independent readline. + #Usage: develop arg -- globals are treated as a large, passed in data structure. + #Desc: Develops footage according to the current configuration of global variables. - #The Very Basics - BASE="$(basename "$ARG")" - EXT="${BASE##*.}" - if [[ "${EXT}" == ".${BASE}" ]]; then EXT=""; fi #This means the input is a folder, which has no extension. - DIRNAME=$(dirname "$ARG") - TRUNC_ARG="${BASE%.*}" - SCALE=`echo "($(echo "${PROXY_SCALE}" | sed 's/%//') / 100) * 2" | bc -l` #Get scale as factor for halved video, *2 for 50% - setBL=true - #Evaluate local configuration file for file-specific blocks. - parseConf "$LCONFIG" true - - #Check that ARG exists and works. - local action=$(checkArg) - if [[ $action = "skip" ]]; then - let ARGNUM--; continue - fi - - #Color Management - The Color LUT is chosen + applied. + local ARG=$(readlinkF "$1") #ARG is the MLV, RAW, or DNG sequence to be processed. + + local DIRNAME=$(dirname "$ARG") #DIRNAME is the name of the folder containing the ARG. + +#The Very Basics + local BASE="$(basename "$ARG")" #BASE is the name of the file. + local TRUNC_ARG="${BASE%.*}" #TRUNC_ARG is the name of the file, sans extension. + + local EXT="${BASE##*.}" #EXT is the file's extension. + if [[ "${EXT}" == ".${BASE}" ]]; then EXT=""; fi #If the input is a folder, it will have no extension. + + local SCALE=`echo "($(echo "${PROXY_SCALE}" | sed 's/%//') / 100) * 2" | bc -l` #Get scale as factor for halved video, *2 for 50% + setBL=true + +#Color Management - The Color LUT is chosen + applied. - #We define what "STANDARD" means. Gamma 2.2 if it's not specifically defined. - if [[ $COLOR_GAMUT != "xyz" ]]; then - - #~ if [[ $COLOR_GAMUT == "aces" ]]; then #List of Linear Only gamuts. XYZ excluded, in that that's default output; no filtering needed. - #~ COLOR_GAMMA="lin" - #~ fi - - if [[ $COLOR_GAMMA == "STANDARD" ]]; then - if [[ $COLOR_GAMUT == "argb" || $COLOR_GAMUT == "ssg3c" ]]; then #List of gamuts with Gamma 2.2 . - COLOR_GAMMA="y2*2" - else - COLOR_GAMMA=$COLOR_GAMUT - fi + #We define what "STANDARD" means. Gamma 2.2 if it's not specifically defined. + if [[ $COLOR_GAMUT != "xyz" ]]; then + + #This block defines what a "STANDARD" gamma means for each gamut. + if [[ $COLOR_GAMMA == "STANDARD" ]]; then + if [[ $COLOR_GAMUT == "argb" || $COLOR_GAMUT == "ssg3c" ]]; then #List of gamuts with Gamma 2.2 . + COLOR_GAMMA="y2*2" + elif [[ $COLOR_GAMUT == "aces" ]]; then #ACES is linear. + COLOR_GAMMA="lin" + else + COLOR_GAMMA=$COLOR_GAMUT #Otherwise, "standard" is the name of the gamut. fi - - for source in "${COLOR_LUTS[@]}"; do - colorName="${source}/lin_xyz--${COLOR_GAMMA}_${COLOR_GAMUT}.cube" - if [[ -f $colorName ]]; then - COLOR_VF="lut3d=$colorName" - colorDesc="Color Management LUT" - FFMPEG_FILTERS=true - fi - done - - if [[ $COLOR_VF == "" ]]; then - echo -e "\033[0;31m\033[1mSpecified LUT not found! Is color-ext loaded?.\033[0m\n" - fi - - fi #COLOR_VF is nothing if the gamut is xyz - it'll pass directly out of dcraw/IM, without LUT application. - - #Construct the FFMPEG filters. - FINAL_SCALE="scale=trunc(iw/2)*${SCALE}:trunc(ih/2)*${SCALE}" - if [[ $FFMPEG_FILTERS == true ]]; then - V_FILTERS="-vf $(joinArgs , ${COLOR_VF} ${LUTS[@]} ${HQ_NOISE} ${TEMP_NOISE} ${REM_NOISE} ${SHARP} ${DESHAKE})" - V_FILTERS_PROX="-vf $(joinArgs , ${COLOR_VF} ${LUTS[@]} ${HQ_NOISE} ${TEMP_NOISE} ${REM_NOISE} ${SHARP} ${DESHAKE} ${FINAL_SCALE})" #Proxy filter set adds the FINAL_SCALE component. - - #Created formatted array of filters, FILTER_ARR. - compFilters=() - declare -a compFilters=("${colorDesc}" "${sharpDesc}" "${hqDesc}" "${tempDesc}" "${remDesc}" "${deshakeDesc}" "${lutDesc}") - for v in "${compFilters[@]}"; do if test "$v"; then FILTER_ARR+=("$v"); fi; done - else - V_FILTERS_PROX="-vf ${FINAL_SCALE}" fi - #List remaining files to process. - remFiles=${@:`echo "$# - ($ARGNUM - 1)" | bc`:$#} - remArr=$(echo $remFiles) - - list="" - for item in $remArr; do - itemBase=$(basename $item) - itemExt=".${itemBase##*.}" #Dot must be in here. - if [[ "${itemExt}" == ".${itemBase}" ]]; then itemExt=""; fi #This means the input is a folder, which has no extension. - itemDir=$(dirname "$item") - - if [ -z "${list}" ]; then - if [[ $itemBase == $(basename $ARG) ]]; then - list="${itemDir}/\033[1m\033[32m${itemBase%.*}\033[0m${itemExt}" - else - list="${itemDir}/\033[1m${itemBase%.*}\033[0m${itemExt}" - fi - else - list="${list}, ${itemDir}/\033[1m${itemBase%.*}\033[0m${itemExt}" + for lutSource in "${COLOR_LUTS[@]}"; do + colorName="${lutSource}/lin_xyz--${COLOR_GAMMA}_${COLOR_GAMUT}.cube" + #~ colorName="${lutSource}/lin_srgb--${COLOR_GAMMA}_${COLOR_GAMUT}.cube" + if [[ -f $colorName ]]; then + COLOR_VF="lut3d=$colorName" + colorDesc="Color Management LUT" + FFMPEG_FILTERS=true fi done - if [ $ARGNUM == 1 ]; then - echo -e "\n\033[1m${ARGNUM} File Left to Process:\033[0m ${list}\n" - else - echo -e "\n\033[1m${ARGNUM} Files Left to Process:\033[0m ${list}\n" + if [[ $COLOR_VF == "" ]]; then + error "Specified LUT not found! Is color-ext loaded?\n" fi - - #PREPARATION - - #Establish Basic Directory Structure. - OUTDIR=$(readlinkF "$OUTDIR") + + fi #COLOR_VF is nothing if the gamut is xyz - it'll pass directly out of dcraw/IM, without LUT application. + +#Construct the FFMPEG filters. + FINAL_SCALE="scale=trunc(iw/2)*${SCALE}:trunc(ih/2)*${SCALE}" + if [[ $FFMPEG_FILTERS == true ]]; then + V_FILTERS="-vf $(joinArgs , ${COLOR_VF} ${LUTS[@]} ${HQ_NOISE} ${TEMP_NOISE} ${REM_NOISE} ${SHARP} ${DESHAKE})" + V_FILTERS_PROX="-vf $(joinArgs , ${COLOR_VF} ${LUTS[@]} ${HQ_NOISE} ${TEMP_NOISE} ${REM_NOISE} ${SHARP} ${DESHAKE} ${FINAL_SCALE})" #Proxy filter set adds the FINAL_SCALE component. - if [ $OUTDIR != $PWD ] && [ $isOutGen == false ]; then - mkdir -p $OUTDIR #NO RISKS. WE REMEMBER THE LUT.py. RIP ad-hoc HALD LUT implementation :'( . - isOutGen=true + #Created formatted array of filters, FILTER_ARR. + compFilters=() + declare -a compFilters=("${colorDesc}" "${sharpDesc}" "${hqDesc}" "${tempDesc}" "${remDesc}" "${deshakeDesc}" "${lutDesc}") + for v in "${compFilters[@]}"; do if test "$v"; then FILTER_ARR+=("$v"); fi; done + else + V_FILTERS_PROX="-vf ${FINAL_SCALE}" + fi + +#List remaining files to process. + local remFiles=("${FILE_ARGS_ARRAY[@]:$ARG_INC:$ARGNUM}") #Make a new array from slice. + + #Assemble list of files left to process. + list="" + for item in "${remFiles[@]}"; do + local itemBase=$(basename $item) + local itemExt=".${itemBase##*.}" #Dot must be in here. + if [[ "${itemExt}" == ".${itemBase}" ]]; then itemExt=""; fi #This means the input is a folder, which has no extension. + + local itemDir=$(dirname "$item") + + if [ -z "${list}" ]; then + list=$(formArg "$ARG" "selected") + #~ if [[ $itemBase == $(basename $ARG) ]]; then + #~ list="${itemDir}/$(selected "${itemBase%.*}")${itemExt}" + #~ else + #~ list="${itemDir}/$(bold ${itemBase%.*})${itemExt}" + #~ fi + else + list="${list}, $(formArg "$ARG" "deselected")" fi + done + + if [[ ${#remFiles[@]} == 1 ]]; then + echo -e "\n$(bold "${#remFiles[@]} File Left to Process:") ${list}\n" + else + echo -e "\n$(bold "${#remFiles[@]} Files Left to Process:") ${list}\n" + fi + +#PREPARATION + +#Establish Basic Directory Structure. + OUTDIR=$(readlinkF "$OUTDIR") + + if [ $OUTDIR != $PWD ] && [ $isOutGen == false ]; then + mkdir -p $OUTDIR #NO RISKS. WE REMEMBER THE LUT.py. RIP ad-hoc HALD LUT implementation :'( . + isOutGen=true + fi + + local FILE="${OUTDIR}/${TRUNC_ARG}" + local TMP="${FILE}/tmp_${TRUNC_ARG}" + +#DNG argument, reused or not. Also, create FILE and TMP. + DEVELOP=true + if [[ ( -d $ARG ) && ( ( `basename ${ARG} | cut -c1-3` == "dng" && -f "${ARG}/../settings.txt" ) || ( `basename ${ARG}` == $TRUNC_ARG && -f "${ARG}/settings.txt" ) ) ]]; then #If we're reusing a dng sequence, copy over before we delete the original. + echo -e "\033[1m${TRUNC_ARG}:\033[0m Moving DNGs from previous run...\n" #Use prespecified DNG sequence. + + #User may specify either the dng_ or the trunc_arg folder; must account for both. + if [[ `folderName ${ARG}` == $TRUNC_ARG && -d "${ARG}/dng_${TRUNC_ARG}" ]]; then #Accounts for the trunc_arg folder. + ARG="${ARG}/dng_${TRUNC_ARG}" #Set arg to the dng argument. + elif [[ `folderName ${ARG}` == $TRUNC_ARG && `echo "$(basename ${ARG})" | cut -c 1-3` == "dng" ]]; then #Accounts for the dng_ folder. + TRUNC_ARG=`echo $TRUNC_ARG | cut -c5-${#TRUNC_ARG}` + else + echo -e "\033[0;31m\033[1mCannot reuse - DNG folder does not exist! Skipping argument.\033[0m" + return 1 + fi + + DNG_LOC=${OUTDIR}/tmp_reused + mkdir -p ${OUTDIR}/tmp_reused + + find $ARG -iname "*.dng" | xargs -I {} mv {} $DNG_LOC #Moving DNGs to temporary location. + + reuseSet "$ARG" + #~ dngSet "$DNG_LOC" + #~ FPS=`cat ${ARG}/../settings.txt | grep "FPS" | cut -d $" " -f2` #Grab FPS from previous run. + #~ FRAMES=`cat ${ARG}/../settings.txt | grep "Frames" | cut -d $" " -f2` #Grab FRAMES from previous run. + #~ KELVIN=`cat ${ARG}/../settings.txt | grep "WBKelvin" | cut -d $" " -f2` + cp "${ARG}/../settings.txt" $DNG_LOC + + oldARG="${ARG}" + DIRNAME=$(dirname "$oldARG") + BASE="$(basename "$oldARG")" + EXT="" FILE="${OUTDIR}/${TRUNC_ARG}" - TMP="${FILE}/tmp_${TRUNC_ARG}" + TMP="${FILE}/tmp_${TRUNC_ARG}" #Remove dng_ from ARG by redefining basic constants. Ready to go! + ARG="${FILE}/dng_${TRUNC_ARG}" #Careful. This won't exist till later. - #DNG argument, reused or not. Also, create FILE and TMP. - DEVELOP=true - if [[ ( -d $ARG ) && ( ( `basename ${ARG} | cut -c1-3` == "dng" && -f "${ARG}/../settings.txt" ) || ( `basename ${ARG}` == $TRUNC_ARG && -f "${ARG}/settings.txt" ) ) ]]; then #If we're reusing a dng sequence, copy over before we delete the original. - echo -e "\033[1m${TRUNC_ARG}:\033[0m Moving DNGs from previous run...\n" #Use prespecified DNG sequence. - - #User may specify either the dng_ or the trunc_arg folder; must account for both. - if [[ `folderName ${ARG}` == $TRUNC_ARG && -d "${ARG}/dng_${TRUNC_ARG}" ]]; then #Accounts for the trunc_arg folder. - ARG="${ARG}/dng_${TRUNC_ARG}" #Set arg to the dng argument. - elif [[ `folderName ${ARG}` == $TRUNC_ARG && `echo "$(basename ${ARG})" | cut -c 1-3` == "dng" ]]; then #Accounts for the dng_ folder. - TRUNC_ARG=`echo $TRUNC_ARG | cut -c5-${#TRUNC_ARG}` - else - echo -e "\033[0;31m\033[1mCannot reuse - DNG folder does not exist! Skipping argument.\033[0m" - continue - fi - - DNG_LOC=${OUTDIR}/tmp_reused - mkdir -p ${OUTDIR}/tmp_reused - - find $ARG -iname "*.dng" | xargs -I {} mv {} $DNG_LOC #Copying DNGs to temporary location. - - dngSet "$DNG_LOC" - FPS=`cat ${ARG}/../settings.txt | grep "FPS" | cut -d $" " -f2` #Grab FPS from previous run. - FRAMES=`cat ${ARG}/../settings.txt | grep "Frames" | cut -d $" " -f2` #Grab FRAMES from previous run. - KELVIN=`cat ${ARG}/../settings.txt | grep "WBKelvin" | cut -d $" " -f2` - cp "${ARG}/../settings.txt" $DNG_LOC - - oldARG="${ARG}" - DIRNAME=$(dirname "$oldARG") - BASE="$(basename "$oldARG")" - EXT="" - - FILE="${OUTDIR}/${TRUNC_ARG}" - TMP="${FILE}/tmp_${TRUNC_ARG}" #Remove dng_ from ARG by redefining basic constants. Ready to go! - ARG="${FILE}/dng_${TRUNC_ARG}" #Careful. This won't exist till later. - - - dngLocClean() { - find $DNG_LOC -iname "*.dng" | xargs -I {} mv {} $oldARG - rm -rf $DNG_LOC - } - - mkdirS $FILE dngLocClean - mkdirS $TMP #Make the folders. - - find $DNG_LOC -iname "*.dng" | xargs -I {} mv {} $TMP #Moving files to where they need to go. - cp "${DNG_LOC}/settings.txt" $FILE - - setBL=false - DEVELOP=false - rm -r $DNG_LOC - elif [ -d $ARG ]; then #If it's a DNG sequence, but not a reused one. - mkdirS $FILE - mkdirS $TMP - - echo -e "\033[1m${TRUNC_ARG}:\033[0m Using specified folder of RAW sequences...\n" #Use prespecified DNG sequence. - - FPS=24 #Set it to a safe default. - FRAMES=$(find ${ARG} -name "*.dng" | wc -l) - + + dngLocClean() { + find $DNG_LOC -iname "*.dng" | xargs -I {} mv {} $oldARG + rm -rf $DNG_LOC + } + + mkdirS $FILE dngLocClean; if [ $? -eq 1 ]; then return 1; fi + mkdirS $TMP; if [ $? -eq 1 ]; then return 1; fi #Make the folders. + + find $DNG_LOC -iname "*.dng" | xargs -I {} mv {} $TMP #Moving files to where they need to go. + cp "${DNG_LOC}/settings.txt" $FILE + + setBL=false + DEVELOP=false + rm -r $DNG_LOC + elif [ -d $ARG ]; then #If it's a DNG sequence, but not a reused one. + mkdirS $FILE; if [ $? -eq 1 ]; then return 1; fi + mkdirS $TMP; if [ $? -eq 1 ]; then return 1; fi + + echo -e "\033[1m${TRUNC_ARG}:\033[0m Using specified folder of RAW sequences...\n" #Use prespecified DNG sequence. + + FPS=24 #Set it to a safe default. + FRAMES=$(find ${ARG} -name "*.dng" | wc -l) + + setRange + + i=1 + for dng in $ARG/*.dng; do + ln -s $dng $(printf "${TMP}/${TRUNC_ARG}_%06d.dng" $i) #Since we're not touching the DNGs, we can link them from wherever to TMP!!! :) Super duper fast. + let i++ + if [[ i -gt $FRAME_END ]]; then break; fi + done + + dngSet + + DEVELOP=false #We're not developing DNG's; we already have them! + else + mkdirS $FILE; if [ $? -eq 1 ]; then return 1; fi + mkdirS $TMP; if [ $? -eq 1 ]; then return 1; fi + fi + +#Darkframe Averaging + if [[ $useDF == true ]]; then + echo -e "\033[1m${TRUNC_ARG}:\033[0m Creating darkframe for subtraction...\n" + + avgFrame="${TMP}/avg.darkframe" #The path to the averaged darkframe file. + + #~ There was a bug - retest this. + darkBase="$(basename "$DARKFRAME")" + darkExt="${darkBase##*.}" + + if [ $darkExt != 'darkframe' ]; then + $MLV_DUMP -o "${avgFrame}" -a $DARKFRAME >/dev/null 2>/dev/null + else + cp $DARKFRAME $avgFrame #Copy the preaveraged frame if the extension is .darkframe. + fi + + RES_DARK=`echo "$(${MLV_DUMP} -v -m ${avgFrame})" | grep "Res" | sed 's/[[:alpha:] ]*: //'` + + DARK_PROC="-s ${avgFrame}" + fi + +#Develop sequence if needed. + if [ $DEVELOP == true ]; then + echo -e "\033[1m${TRUNC_ARG}:\033[0m Dumping to DNG Sequence...\n" + + if [ ! $DARKFRAME == "" ] && [ ! $CHROMA_SMOOTH == "--no-cs" ]; then #Just to let the user know that certain features are impossible with RAW. + rawStat="*Skipping Darkframe subtraction and Chroma Smoothing for RAW file ${TRUNC_ARG}." + elif [ ! $DARKFRAME == "" ]; then + rawStat="*Skipping Darkframe subtraction for RAW file ${TRUNC_ARG}." + elif [ ! $CHROMA_SMOOTH == "--no-cs" ]; then + rawStat="*Skipping Chroma Smoothing for RAW file ${TRUNC_ARG}." + else + rawStat="\c" + fi + + #IF extension is RAW, we want to convert to MLV. All the interesting features are MLV-only, because of mlv_dump's amazingness. + + if [ $EXT == "MLV" ] || [ $EXT == "mlv" ]; then + # Read the header. + mlvSet setRange - i=1 - for dng in $ARG/*.dng; do - ln -s $dng $(printf "${TMP}/${TRUNC_ARG}_%06d.dng" $i) #Since we're not touching the DNGs, we can link them from wherever to TMP!!! :) Super duper fast. - let i++ - if [[ i -gt $FRAME_END ]]; then break; fi - done - - dngSet - - DEVELOP=false #We're not developing DNG's; we already have them! - else - mkdirS $FILE - mkdirS $TMP - fi - - #Darkframe Averaging - if [[ $useDF == true ]]; then - echo -e "\033[1m${TRUNC_ARG}:\033[0m Creating darkframe for subtraction...\n" - - avgFrame="${TMP}/avg.darkframe" #The path to the averaged darkframe file. - - #~ There was a bug - retest this. - darkBase="$(basename "$DARKFRAME")" - darkExt="${darkBase##*.}" - - if [ $darkExt != 'darkframe' ]; then - $MLV_DUMP -o "${avgFrame}" -a $DARKFRAME >/dev/null 2>/dev/null - else - cp $DARKFRAME $avgFrame #Copy the preaveraged frame if the extension is .darkframe. + # Error checking: Darkframe resolution must match resolution. + if [[ (! -z $RES_DARK) && $RES_DARK != $RES_IN ]]; then + invOption "Darkframe Resolution doesn't match MLV Resolution! Use another darkframe!" fi - RES_DARK=`echo "$(${MLV_DUMP} -v -m ${avgFrame})" | grep "Res" | sed 's/[[:alpha:] ]*: //'` - - DARK_PROC="-s ${avgFrame}" - fi - - #Develop sequence if needed. - if [ $DEVELOP == true ]; then - echo -e "\033[1m${TRUNC_ARG}:\033[0m Dumping to DNG Sequence...\n" - - if [ ! $DARKFRAME == "" ] && [ ! $CHROMA_SMOOTH == "--no-cs" ]; then #Just to let the user know that certain features are impossible with RAW. - rawStat="*Skipping Darkframe subtraction and Chroma Smoothing for RAW file ${TRUNC_ARG}." - elif [ ! $DARKFRAME == "" ]; then - rawStat="*Skipping Darkframe subtraction for RAW file ${TRUNC_ARG}." - elif [ ! $CHROMA_SMOOTH == "--no-cs" ]; then - rawStat="*Skipping Chroma Smoothing for RAW file ${TRUNC_ARG}." - else - rawStat="\c" - fi - - #IF extension is RAW, we want to convert to MLV. All the interesting features are MLV-only, because of mlv_dump's amazingness. - - if [ $EXT == "MLV" ] || [ $EXT == "mlv" ]; then - # Read the header. - mlvSet - setRange - - # Error checking: Darkframe resolution must match resolution. - if [[ (! -z $RES_DARK) && $RES_DARK != $RES_IN ]]; then - invOption "Darkframe Resolution doesn't match MLV Resolution! Use another darkframe!" - fi - - #Dual ISO might want to do the chroma smoothing. In which case, don't do it now! - if [ $DUAL_ISO == true ]; then - smooth="--no-cs" - else - smooth=$CHROMA_SMOOTH - fi - - #Create new MLV with adequate number of frames, if needed. - REAL_MLV=$ARG - REAL_FRAMES=$FRAMES - if [ $isFR == false ]; then - REAL_MLV="${TMP}/newer.mlv" - $MLV_DUMP $ARG -o ${REAL_MLV} -f ${FRAME_RANGE} >/dev/null 2>/dev/null - REAL_FRAMES=`${MLV_DUMP} ${REAL_MLV} | awk '/Processed/ { print $2; }'` - fi - - fileRanges=(`echo $($SRANGE $REAL_FRAMES $THREADS)`) #Get an array of frame ranges from the amount of frames and threads. I used a python script for this. - #Looks like this: 0-1 2-2 3-4 5-5 6-7 8-8 9-10. Put that in an array. - - devDNG() { #Takes n arguments: 1{}, the frame range 2$MLV_DUMP 3$REAL_MLV 4$DARK_PROC 5$no_data 6$smooth 7$TMP 8$FRAME_END 9$TRUNC_ARG 10$FRAME_START - range=$1 - firstFrame=false - if [[ $range == "0-0" ]]; then #mlv_dump can't handle 0-0, so we develop 0-1. - range="0-1" - firstFrame=true - fi - - tmpOut=${7}/${range} #Each output will number from 0, so give each its own folder. - mkdir -p $tmpOut - - start=$(echo "$range" | cut -d'-' -f1) - end=$(echo "$range" | cut -d'-' -f2) #Get start and end frames from the frame range - - $2 $3 $4 -o "${tmpOut}/${9}_" -f ${range} $6 --dng --batch | { #mlv_dump command. Uses frame range. - lastCur=0 - while IFS= read -r line; do - output=$(echo $line | grep -Po 'V.*A' | cut -d':' -f2 | cut -d$' ' -f1) #Hacked my way to the important bit. - if [[ $output == "" ]]; then continue; fi #If there's no important bit, don't print. - - cur=$(echo "$output" | cut -d'/' -f1) #Current frame. - if [[ $cur == $lastCur ]] || [[ $cur -gt $end ]] || [[ $cur -lt $start ]]; then continue; fi #Turns out, it goes through all the frames, even if cutting the frame range. So, clamp it! - - lastCur=$cur #It likes to repeat itself. - echo -e "\033[2K\rMLV to DNG: Frame $(echo "${cur} + ${10}" | bc)/${8}\c" #Print out beautiful progress bar, in parallel! - done - - } #Progress Bar - if [[ $firstFrame == true ]]; then #If 0-0. - rm $(printf "${tmpOut}/${9}_%06d.dng" 1) 2>/dev/null #Remove frame #1, leaving us only with frame #0. - mv $tmpOut "${7}/0-0" #Move back to 0-0, as if that's how it was developed all along. - fi - } - - export -f devDNG #Export to run in subshell. - - for range in "${fileRanges[@]}"; do echo $range; done | #For each frame range, assign a thread. - xargs -I {} -P $THREADS -n 1 \ - bash -c "devDNG '{}' '$MLV_DUMP' '$REAL_MLV' '$DARK_PROC' 'no_data' '$smooth' '$TMP' '$FRAME_END' '$TRUNC_ARG' '$FRAME_START'" - - #Since devDNG must run in a subshell, globals don't follow. Must pass *everything* in. - echo -e "\033[2K\rMLV to DNG: Frame ${FRAME_END}/${FRAME_END}\c" #Ensure it looks right at the end. - echo -e "\n" - - count=$FRAME_START - for range in "${fileRanges[@]}"; do #Go through the subfolders sequentially - tmpOut=${TMP}/${range} #Use temporary folder. It will be named the same as the frame range. - for dng in ${tmpOut}/*.dng; do - if [ $count -gt $FRAME_END ]; then echo "ERROR! Count greater than end!"; fi - mv $dng $(printf "${TMP}/${TRUNC_ARG}_%06d.dng" $count) #Move dngs out sequentially, numbering them properly. - let count++ - done - rm -r $tmpOut #Remove the now empty subfolder - done - - elif [ $EXT == "RAW" ] || [ $EXT == "raw" ]; then - rawSet - setRange - - echo -e $rawStat - FPS=`$RAW_DUMP $ARG "${TMP}/${TRUNC_ARG}_" | awk '/FPS/ { print $3; }'` #Run the dump while awking for the FPS. - fi - - #~ BLACK_LEVEL=$(exiftool -BlackLevel -s -s -s ${TMP}/${TRUNC_ARG}_$(printf "%06d" $(echo "$FRAME_START" | bc)).dng) - fi - - setRange #Just to be sure the frame range was set, in case the input isn't MLV. - - BLACK_LEVEL=$(exiftool -BlackLevel -s -s -s ${TMP}/${TRUNC_ARG}_$(printf "%06d" $(echo "$FRAME_START" | bc)).dng) #Use the first DNG to get the correct black level. - - prntSet > $FILE/settings.txt - sed -i -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" $FILE/settings.txt #Strip escape sequences. - - #Create badpixels file. - if [ $isBP == true ] && [ $DEVELOP == true ]; then - echo -e "\033[1m${TRUNC_ARG}:\033[0m Generating badpixels file...\n" - - bad_name="badpixels_${TRUNC_ARG}.txt" - gen_bad="${TMP}/${bad_name}" - touch $gen_bad - - if [ $EXT == "MLV" ] || [ $EXT == "mlv" ]; then - $MLV_BP -o $gen_bad $ARG - elif [ $EXT == "RAW" ] || [ $EXT == "raw" ]; then - $MLV_BP -o $gen_bad $ARG - fi - - if [[ ! -z $BADPIXEL_PATH ]]; then - if [ -f "${gen_bad}" ]; then - echo -e "\033[1m${TRUNC_ARG}:\033[0m Concatenating with specified badpixels file...\n" - mv "${gen_bad}" "${TMP}/bp_gen" - cp $BADPIXEL_PATH "${TMP}/bp_imp" - - { cat "${TMP}/bp_gen" && cat "${TMP}/bp_imp"; } > "${gen_bad}" #Combine specified file with the generated file. - else - cp $BADPIXEL_PATH "${gen_bad}" - fi - fi - - BADPIXELS="-P ${gen_bad}" - elif [[ ! -z $BADPIXEL_PATH ]]; then - echo -e "\033[1m${TRUNC_ARG}:\033[0m Using specified badpixels file...\n" - - bad_name="badpixels_${TRUNC_ARG}.txt" - gen_bad="${TMP}/${bad_name}" - - cp $BADPIXEL_PATH "${gen_bad}" - BADPIXELS="-P ${gen_bad}" - fi - - #Dual ISO Conversion - if [ $DUAL_ISO == true ]; then - echo -e "\033[1m${TRUNC_ARG}:\033[0m Combining Dual ISO...\n" - - #Original DNGs will be moved here. - oldFiles="${TMP}/orig_dng" - mkdirS $oldFiles - - inc_iso() { #6 args: 1{} 2$CR_HDR 3$TMP 4$FRAME_END 5$oldFiles 6$CHROMA_SMOOTH. {} is a path. Progress is thread safe. Experiment gone right :). - count=$(echo "$(echo $(echo $1 | rev | cut -d "_" -f 1 | rev | cut -d "." -f 1 | grep "[0-9]") | bc) + 1" | bc) #Get count from filename. - - $2 $1 $6 >/dev/null 2>/dev/null #The LQ option, --mean23, is completely unusable in my opinion. - - name=$(basename "$1") - mv "${3}/${name%.*}.dng" $5 #Move away original dngs. - mv "${3}/${name%.*}.DNG" "${3}/${name%.*}.dng" #Rename *.DNG to *.dng. - - echo -e "\033[2K\rDual ISO Development: Frame ${count}/${4}\c" - } - - export -f inc_iso #Must expose function to subprocess. - - #~ echo "${CR_HDR} ${TMP}/${TRUNC_ARG}_$(printf "%06d" $FRAME_START).dng" - if [[ $(${CR_HDR} "${TMP}/${TRUNC_ARG}_$(printf "%06d" $FRAME_START).dng") == *"ISO blending didn't work"* ]]; then - invOption "The input wasn't shot Dual ISO!" - fi - - find $TMP -maxdepth 1 -name "*.dng" -print0 | sort -z | xargs -0 -I {} -P $THREADS -n 1 \ - bash -c "inc_iso '{}' '$CR_HDR' '$TMP' '$FRAME_END' '$oldFiles' '$CHROMA_SMOOTH'" - - BLACK_LEVEL=$(exiftool -BlackLevel -s -s -s ${TMP}/${TRUNC_ARG}_$(printf "%06d" $(echo "$FRAME_START" | bc)).dng) #Use the first DNG to get the new correct black level. - - echo -e "\n" - fi - - if [ $setBL == true ]; then - echo -e "BlackLevel: ${BLACK_LEVEL}" >> $FILE/settings.txt #Black level must now be set. - fi - - normToOne() { - wBal=$1 - - max=0.0 - for mult in $wBal; do - if [ $(echo " $mult > $max" | bc) -eq 1 ]; then - max=$mult - fi - done - - for mult in $wBal; do - echo -e "$(echo "scale=6; x=${mult} / ${max}; if(x<1) print 0; x" | bc -l) \c" #BC is bae. - done - } - - getGreen() { - wBal=$1 - - i=0 - for mult in $wBal; do - if [ $i -eq 1 ]; then - echo -e "${mult}" - fi - let i++ - done - } - - #Get White Balance correction factor. - if [ $GEN_WHITE == true ]; then - echo -e "\033[1m${TRUNC_ARG}:\033[0m Generating WB...\n" - - #Calculate n, the distance between samples. - frameLen=$(echo "$FRAME_END - $FRAME_START + 1" | bc) #Offset by one to avoid division by 0 errors later. min value must be 1. - - #A point of improvement, this. - if [[ $WHITE_SPD -gt $frameLen ]]; then - WHITE_SPD=$frameLen - fi - n=`echo "${frameLen} / ${WHITE_SPD}" | bc` - - toBal="${TMP}/toBal" - mkdirS $toBal - - #Develop every nth file for averaging. - local i=0 - local t=0 - for file in $TMP/*.dng; do - if [ `echo "(${i}+1) % ${n}" | bc` -eq 0 ]; then - $DCRAW -q 0 $BADPIXELS -r 1 1 1 1 -g $GAMMA -k $BLACK_LEVEL $SATPOINT -o 0 -T "${file}" - name=$(basename "$file") - mv "$TMP/${name%.*}.tiff" $toBal #TIFF MOVEMENT. We use TIFFs here because it's easy for dcraw and Python. - let t++ - fi - echo -e "\033[2K\rWB Development: Sample ${t}/$(echo "${frameLen} / $n" | bc) (Frame: $(echo "${i} + 1" | bc)/${FRAME_END})\c" - let i++ - done - echo "" - - #Calculate + store result into a form dcraw likes. - echo -e "Calculating Auto White Balance..." - BALANCE=`$BAL $toBal` - - elif [ $CAMERA_WB == true ]; then - echo -e "\033[1m${TRUNC_ARG}:\033[0m Retrieving Camera White Balance..." - - for file in $TMP/*.dng; do - #dcraw a single file verbosely, to get the camera multiplier with awk. - BALANCE=`$DCRAW -T -w -v -c ${file} 2>&1 | awk '/multipliers/ { print $2, $3, $4 }'` - break - done - - else #Something must always be set. - echo -e "\033[1m${TRUNC_ARG}:\033[0m Ignoring White Balance..." - - BALANCE="1.000000 1.000000 1.000000" - fi - - #Finally, set the white balance after determining it. - if [ $isScale = false ]; then - BALANCE=$(normToOne "$BALANCE") - fi - green=$(getGreen "$BALANCE") - WHITE="-r ${BALANCE} ${green}" - echo -e "Correction Factor (RGBG): ${BALANCE} ${green}\n" - - #Move .wav. - SOUND_PATH="${TMP}/${TRUNC_ARG}_.wav" - - if [ ! -f $SOUND_PATH ]; then - echo -e "*Not moving .wav, because it doesn't exist.\n" - else - echo -e "*Moving .wav.\n" - cp $SOUND_PATH $FILE - fi - - #DEFINE PROCESSING FUNCTIONS - dcrawOpt() { #Find, develop, and splay raw DNG data as ppm, ready to be processed. - find "${TMP}" -maxdepth 1 -iname "*.dng" -print0 | sort -z | tr -d "\n" | xargs -0 \ - $DCRAW -c -q $DEMO_MODE $FOUR_COLOR -k $BLACK_LEVEL $SATPOINT $BADPIXELS $WHITE -H $HIGHLIGHT_MODE -g $GAMMA $WAVE_NOISE -o $SPACE $DEPTH - } #Is prepared to pipe all the files in TMP outwards. - - dcrawImg() { #Find and splay image sequence data as ppm, ready to be processed by ffmpeg. Not working well. - find "${SEQ}" -maxdepth 1 -iname "*.${IMG_FMT}" -print0 | sort -z | xargs -0 -I {} convert '{}' -set colorspace sRGB -colorspace RGB ppm:- - } #Finds all images, prints to stdout, without any operations, using convert. ppm conversion is inevitably slow, however... - - mov_main() { - ffmpeg -f image2pipe -vcodec ppm -r $FPS -i pipe:0 \ - -loglevel panic -stats $SOUND -vcodec prores_ks -pix_fmt rgb48be -n -r $FPS -profile:v 4444 -alpha_bits 0 -vendor ap4h $V_FILTERS $SOUND_ACTION "${VID}_hq.mov" - } #-loglevel panic -stats - - mov_prox() { - ffmpeg -f image2pipe -vcodec ppm -r $FPS -i pipe:0 \ - -loglevel panic -stats $SOUND -c:v libx264 -n -r $FPS -preset fast $V_FILTERS_PROX -crf 23 -c:a mp3 "${VID}_lq.mp4" - } #The option -vf "scale=trunc(iw/2)*2:trunc(ih/2)*2" fixes when x264 is unhappy about non-2 divisible dimensions. - - mov_main_img() { - ffmpeg -start_number $FRAME_START -loglevel panic -stats -f image2 -i ${SEQ}/${TRUNC_ARG}_%06d.${IMG_FMT} $SOUND -vcodec prores_ks \ - -pix_fmt rgb48le -n -r $FPS -profile:v 4444 -alpha_bits 0 -vendor ap4h $V_FILTERS $SOUND_ACTION "${VID}_hq.mov" - } - - mov_prox_img() { - ffmpeg -start_number $FRAME_START -loglevel panic -stats -f image2 -i ${SEQ}/${TRUNC_ARG}_%06d.${IMG_FMT} $V_FILTERS_PROX $SOUND -c:v libx264 \ - -n -r $FPS -preset veryfast -crf 21 -c:a mp3 -b:a 320k "${VID}_lq.mp4" - } - - - runSim() { - # Command: cat $PIPE | cmd1 & cmdOrig | tee $PIPE | cmd2 - - # cat $PIPE | cmd1 - gives output of pipe live. Pipes it into cmd1. Nothing yet; just setup. - # & - runs the next part in the background. - # cmdOrig | tee $PIPE | cmd2 - cmdOrig pipes into the tee, which splits it back into the previous pipe, piping on to cmd2! - - # End Result: Output of cmdOrig is piped into cmd1 and cmd2, which execute, both printing to stdout. - - cmdOrig=$1 - cmd1=$2 - cmd2=$3 - - #~ echo $cmdOrig $cmd1 $cmd2 - #~ echo $($cmdOrig) - - PIPE="${TMP}/pipe_vid" # $(date +%s%N | cut -b1-13)" - mkfifo $PIPE 2>/dev/null - - cat $PIPE | $cmd1 & $cmdOrig | tee $PIPE | $cmd2 #The magic of simultaneous execution ^_^ - #~ cat $PIPE | tr 'e' 'a' & echo 'hello' | tee $PIPE | tr 'e' 'o' #The magic of simultaneous execution ^_^ - } - - img_par() { #Takes 22 arguments: {} 2$DEMO_MODE 3$FOUR_COLOR 4$BADPIXELS 5$WHITE 6$HIGHLIGHT_MODE 7$GAMMA 8$WAVE_NOISE 9$DEPTH 10$SEQ 11$TRUNC_ARG 12$IMG_FMT 13$FRAME_END 14$DEPTH_OUT 15$COMPRESS 16$isJPG 17$PROXY_SCALE 18$PROXY 19$BLACK_LEVEL 20$SPACE 21$SATPOINT 22$DCRAW 23$FFMPEG_FILTERS - count=$(echo $(echo $1 | rev | cut -d "_" -f 1 | rev | cut -d "." -f 1 | grep "[0-9]") | bc) #Instead of count from file, count from name! - DCRAW=${22} - - DPXHACK="" - if [[ ${12^^} == "DPX" && ( ${23} == false ) ]]; then DPXHACK="-colorspace sRGB"; else DPXHACK=""; fi - #Trust me, I've tried everything else; but this sRGB transform works. Must be an IM bug. Keep an eye on it! - #The sRGB curve is only applied if going to DPX while DPX is the target image format. Aka. At the end; not in the middle. - - if [ ${16} == true ]; then - $DCRAW -c -q $2 $3 $4 $5 -H $6 -k ${19} ${21} -g $7 $8 -o ${20} $9 $1 | \ - tee >(convert ${14} - -set colorspace RGB ${DPXHACK} ${15} $(printf "${10}/${11}_%06d.${12}" ${count})) | \ - convert - -set colorspace XYZ -quality 80 -colorspace sRGB -resize ${17} $(printf "${18}/${11}_%06d.jpg" ${count}) - #JPGs don't get ffmpeg filters applied. They simply can't handle it. - echo -e "\033[2K\rDNG to ${12^^}/JPG: Frame ${count^^}/${13}\c" - else - $DCRAW -c -q $2 $3 $4 $5 -H $6 -k ${19} ${21} -g $7 $8 -o ${20} $9 $1 | \ - convert ${14} - -set colorspace RGB ${DPXHACK} ${15} $(printf "${10}/${11}_%06d.${12}" ${count}) - echo -e "\033[2K\rDNG to ${12^^}: Frame ${count^^}/${13}\c" - fi - } - #~ See http://www.imagemagick.org/discourse-server/viewtopic.php?t=21161 - - export -f img_par - - - #PROCESSING - - #IMAGE PROCESSING - if [ $IMAGES == true ] ; then - echo -e "\033[1m${TRUNC_ARG}:\033[0m Processing Image Sequence from Frame ${FRAME_START} to ${FRAME_END}...\n" - - #Define Image Directories, Create SEQ directory - SEQ="${FILE}/${IMG_FMT}_${TRUNC_ARG}" - PROXY="${FILE}/proxy_${TRUNC_ARG}" - - mkdirS $SEQ - - if [ $isJPG == true ]; then - mkdirS $PROXY - fi - - #Define hardcoded compression based on IMG_FMT - if [ $isCOMPRESS == true ]; then - if [ $IMG_FMT == "exr" ]; then - COMPRESS="-compress piz" - elif [ $IMG_FMT == "tiff" ]; then - COMPRESS="-compress zip" - elif [ $IMG_FMT == "png" ]; then - COMPRESS="-quality 0" - elif [ $IMG_FMT == "dpx" ]; then - COMPRESS="-compress rle" - fi - fi - - #Convert all the actual DNGs to IMG_FMT, in parallel. - find "${TMP}" -maxdepth 1 -name '*.dng' -print0 | sort -z | xargs -0 -I {} -P $THREADS -n 1 \ - bash -c "img_par '{}' '$DEMO_MODE' '$FOUR_COLOR' '$BADPIXELS' '$WHITE' '$HIGHLIGHT_MODE' '$GAMMA' '$WAVE_NOISE' '$DEPTH' \ - '$SEQ' '$TRUNC_ARG' '$IMG_FMT' '$FRAME_END' '$DEPTH_OUT' '$COMPRESS' '$isJPG' '$PROXY_SCALE' '$PROXY' '$BLACK_LEVEL' '$SPACE' '$SATPOINT' '$DCRAW' '$FFMPEG_FILTERS'" - - # Removed | cut -d '' -f $FRAME_RANGE , as this happens when creating the DNGs in the first place. - - if [ $isJPG == true ]; then #Make it print "Frame $FRAMES / $FRAMES" as the last output :). - echo -e "\033[2K\rDNG to ${IMG_FMT^^}/JPG: Frame ${FRAME_END}/${FRAME_END}\c" - else - echo -e "\033[2K\rDNG to ${IMG_FMT^^}: Frame ${FRAME_END}/${FRAME_END}\c" - fi - - echo -e "\n" - - tConvert() { #Arguments: 1$inFolder 2$outFolder 3$fromFMT 4$toFMT - inFolder=$1 - outFolder=$2 - fromFMT=$3 - toFMT=$4 - iccProf=$5 - - if [[ ! -z $iccProf ]]; then iccProf="+profile icm -profile $iccProf"; fi - - conv_par() { # Arguments: 1${} 2$TRUNC_ARG 3$outFolder 4$fromFMT 5$iccProf 6$toFMT 7$DEPTH_OUT 8$compress 9$FRAME_END 10$IMG_FMT - count=$(echo $(echo $1 | rev | cut -d "_" -f 1 | rev | cut -d "." -f 1 | grep "[0-9]") | bc) #Get count from filename. - - echo -e "\033[2K\rMiddle-step: ${4^^} to ${6^^}, Frame ${count^^}/${9}\c" - - DPXHACK="" - if [[ ${6^^} == "DPX" && ${10^^} == ${6^^} ]]; then DPXHACK="-colorspace sRGB"; else DPXHACK=""; fi - #Trust me, I've tried everything else; but this sRGB transform works. Must be an IM bug. Keep an eye on it! - #The sRGB curve is only applied if going to DPX while DPX is the target image format. Aka. At the end; not in the middle. - - convert ${7} ${1} ${5} $8 -set colorspace RGB ${DPXHACK} "${3}/$(printf "${2}_%06d" ${count}).${6}" - } - - export -f conv_par - - compress="" - if [[ ${IMG_FMT^^} == ${toFMT^^} ]]; then compress=${COMPRESS}; fi - - find $inFolder -iname "*.${fromFMT}" -print0 | sort -z | xargs -0 -I {} -P $THREADS -n 1 \ - bash -c "conv_par '{}' '$TRUNC_ARG' '$outFolder' '$fromFMT' '$iccProf' '$toFMT' '$DEPTH_OUT' '$compress' '$FRAME_END' '$IMG_FMT'" - - echo "" - } - - #FFMPEG Filter Application: Temporal Denoising, 3D LUTs, Deshake, hqdn Denoising, removegrain denoising, unsharp so far. - #See construction of $V_FILTERS in PREPARATION. - if [[ $FFMPEG_FILTERS == true ]]; then - tmpFiltered="${TMP}/filtered" - tmpUnfiltered="${TMP}/unfiltered" - mkdir $tmpFiltered - mkdir $tmpUnfiltered - - #Give correct output. - echo -e "\033[1mApplying Filters:\033[0m $(joinArgs ", " "${FILTER_ARR[@]}")...\n" - - applyFilters() { #Ideally, this would be all we need. But alas, ffmpeg + exr is broken. - IO=$1 - FMT=$2 - - if [[ -z $FMT ]]; then FMT="${IMG_FMT}"; fi - - ffmpeg -start_number $FRAME_START -f image2 -i "${IO}/${TRUNC_ARG}_%06d.${FMT}" -loglevel panic -stats $V_FILTERS \ - -pix_fmt rgb48be -start_number $FRAME_START "${tmpFiltered}/${TRUNC_ARG}_%06d.${FMT}" - - tConvert "$tmpFiltered" "$IO" "$FMT" "$FMT" # "/home/sofus/subhome/src/convmlv/color/lin_xyz--srgb_srgb.icc" - profile application didn't work... - } - - if [[ $IMG_FMT == "exr" ]]; then - echo -e "Note: EXR filtering lags due to middle-step conversion (ffmpeg has no EXR encoder).\n" - - img_res=$(identify ${SEQ}/${TRUNC_ARG}_$(printf "%06d" $(echo "$FRAME_START" | bc)).${IMG_FMT} | cut -d$' ' -f3) - - tConvert "$SEQ" "$tmpUnfiltered" "$IMG_FMT" "dpx" - - ffmpeg -start_number $FRAME_START -f image2 -vcodec dpx -s "${img_res}" -r $FPS -loglevel panic -stats -i "${tmpUnfiltered}/${TRUNC_ARG}_%06d.dpx" \ - $V_FILTERS -pix_fmt rgb48be -vcodec dpx -n -r $FPS -start_number $FRAME_START ${tmpFiltered}/${TRUNC_ARG}_%06d.dpx - - tConvert "$tmpFiltered" "$SEQ" "dpx" "$IMG_FMT" - else - applyFilters "$SEQ" - fi - - echo "" - fi - fi - - #MOVIE PROCESSING - VID="${FILE}/${TRUNC_ARG}" - - SOUND="-i ${TMP}/${TRUNC_ARG}_.wav" - SOUND_ACTION="-c:a mp3 -b:a 320k" - if [ ! -f $SOUND_PATH ]; then - SOUND="" - SOUND_ACTION="" - fi - - if [[ $MOVIE == true && $IMAGES == false && $isH264 == true ]]; then - echo -e "\033[1m${TRUNC_ARG}:\033[0m Encoding to ProRes/H.264..." - runSim dcrawOpt mov_main mov_prox - echo "" - elif [[ $MOVIE == true && $IMAGES == false && $isH264 == false ]]; then - echo -e "\033[1m${TRUNC_ARG}:\033[0m Encoding to ProRes..." - dcrawOpt | mov_main - echo "" - elif [[ $MOVIE == false && $IMAGES == false && $isH264 == true ]]; then - echo -e "\033[1m${TRUNC_ARG}:\033[0m Encoding to H.264..." - dcrawOpt | mov_prox - echo "" - elif [[ $IMAGES == true ]]; then - V_FILTERS="" #Only needed if reading from images. - V_FILTERS_PROX="-vf $FINAL_SCALE" - - if [[ $IMG_FMT == "dpx" ]]; then - V_FILTERS="-vf lut3d=${COLOR_LUTS[0]}/srgb--lin.cube" - V_FILTERS_PROX="-vf lut3d=${COLOR_LUTS[0]}/srgb--lin.cube,$FINAL_SCALE" #We apply a sRGB to Linear 1D LUT if reading DPX. Because it's fucking broken. - fi - - #Use images if available, as opposed to developing the files again. - if [[ $MOVIE == true && $isH264 == true ]]; then - echo -e "\033[1m${TRUNC_ARG}:\033[0m Encoding to ProRes/H.264..." - mov_main_img & - mov_prox_img & - wait - - echo "" - elif [[ $MOVIE == true && $isH264 == false ]]; then - echo -e "\033[1m${TRUNC_ARG}:\033[0m Encoding to ProRes..." - mov_main_img - echo "" - elif [[ $MOVIE == false && $isH264 == true ]]; then - echo -e "\033[1m${TRUNC_ARG}:\033[0m Encoding to H.264..." - mov_prox_img - echo "" - fi - fi - - #Potentially move DNGs. - if [ $KEEP_DNGS == true ]; then - echo -e "\033[1mMoving DNGs...\033[0m" - DNG="${FILE}/dng_${TRUNC_ARG}" - mkdirS $DNG - + #Dual ISO might want to do the chroma smoothing. In which case, don't do it now! if [ $DUAL_ISO == true ]; then - oldFiles="${TMP}/orig_dng" - find $oldFiles -name "*.dng" | xargs -I '{}' mv {} $DNG #Preserve the original, unprocessed DNGs. + smooth="--no-cs" else - find $TMP -name "*.dng" | xargs -I '{}' mv {} $DNG + smooth=$CHROMA_SMOOTH + fi + + #Create new MLV with adequate number of frames, if needed. + REAL_MLV=$ARG + REAL_FRAMES=$FRAMES + if [ $isFR == false ]; then + REAL_MLV="${TMP}/newer.mlv" + $MLV_DUMP $ARG -o ${REAL_MLV} -f ${FRAME_RANGE} >/dev/null 2>/dev/null + REAL_FRAMES=`${MLV_DUMP} ${REAL_MLV} | awk '/Processed/ { print $2; }'` + fi + + fileRanges=(`echo $($SRANGE $REAL_FRAMES $THREADS)`) #Get an array of frame ranges from the amount of frames and threads. I used a python script for this. + #Looks like this: 0-1 2-2 3-4 5-5 6-7 8-8 9-10. Put that in an array. + + for range in "${fileRanges[@]}"; do echo $range; done | #For each frame range, assign a thread. + xargs -I {} -P $THREADS -n 1 \ + bash -c "dng_par '{}' '$MLV_DUMP' '$REAL_MLV' '$DARK_PROC' 'no_data' '$smooth' '$TMP' '$FRAME_END' '$TRUNC_ARG' '$FRAME_START'" + + #Since dng_par must run in a subshell, globals don't follow. Must pass *everything* in. + echo -e "\033[2K\rMLV to DNG: Frame ${FRAME_END}/${FRAME_END}\c" #Ensure it looks right at the end. + echo -e "\n" + + count=$FRAME_START + for range in "${fileRanges[@]}"; do #Go through the subfolders sequentially + tmpOut=${TMP}/${range} #Use temporary folder. It will be named the same as the frame range. + for dng in ${tmpOut}/*.dng; do + if [ $count -gt $FRAME_END ]; then echo "ERROR! Count greater than end!"; fi + mv $dng $(printf "${TMP}/${TRUNC_ARG}_%06d.dng" $count) #Move dngs out sequentially, numbering them properly. + let count++ + done + rm -r $tmpOut #Remove the now empty subfolder + done + + elif [ $EXT == "RAW" ] || [ $EXT == "raw" ]; then + rawSet + setRange + + echo -e $rawStat + FPS=`$RAW_DUMP $ARG "${TMP}/${TRUNC_ARG}_" | awk '/FPS/ { print $3; }'` #Run the dump while awking for the FPS. + fi + + #~ BLACK_LEVEL=$(exiftool -BlackLevel -s -s -s ${TMP}/${TRUNC_ARG}_$(printf "%06d" $(echo "$FRAME_START" | bc)).dng) + fi + + setRange #Just to be sure the frame range was set, in case the input isn't MLV. + + BLACK_LEVEL=$(exiftool -BlackLevel -s -s -s ${TMP}/${TRUNC_ARG}_$(printf "%06d" $(echo "$FRAME_START" | bc)).dng) #Use the first DNG to get the correct black level. + + prntSet > $FILE/settings.txt + sed -i -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" $FILE/settings.txt #Strip escape sequences. + +#Create badpixels file. + if [ $isBP == true ] && [ $DEVELOP == true ]; then + echo -e "\033[1m${TRUNC_ARG}:\033[0m Generating badpixels file...\n" + + bad_name="badpixels_${TRUNC_ARG}.txt" + gen_bad="${TMP}/${bad_name}" + touch $gen_bad + + if [ $EXT == "MLV" ] || [ $EXT == "mlv" ]; then + $MLV_BP -o $gen_bad $ARG + elif [ $EXT == "RAW" ] || [ $EXT == "raw" ]; then + $MLV_BP -o $gen_bad $ARG + fi + + if [[ ! -z $BADPIXEL_PATH ]]; then + if [ -f "${gen_bad}" ]; then + echo -e "\033[1m${TRUNC_ARG}:\033[0m Concatenating with specified badpixels file...\n" + mv "${gen_bad}" "${TMP}/bp_gen" + cp $BADPIXEL_PATH "${TMP}/bp_imp" + + { cat "${TMP}/bp_gen" && cat "${TMP}/bp_imp"; } > "${gen_bad}" #Combine specified file with the generated file. + else + cp $BADPIXEL_PATH "${gen_bad}" fi fi - echo -e "\n\033[1mCleaning Up.\033[0m\n\n" + BADPIXELS="-P ${gen_bad}" + elif [[ ! -z $BADPIXEL_PATH ]]; then + echo -e "\033[1m${TRUNC_ARG}:\033[0m Using specified badpixels file...\n" - #Delete tmp - rm -rf $TMP + bad_name="badpixels_${TRUNC_ARG}.txt" + gen_bad="${TMP}/${bad_name}" - #RESET ARGS & REPARSE OPTIONS - same as in convmlv.sh. - #Big parse/reparse, making sure global, local, command line options all override each other correctly. - set -- $INPUT_ARGS #Reset the argument input for reparsing. - setDefaults #Hard set/reset all the lovely globals. - OPTIND=1 #Reset argument parsing. - - parseConf "$GCONFIG" false #Parse global config file. + cp $BADPIXEL_PATH "${gen_bad}" + BADPIXELS="-P ${gen_bad}" + fi - parseArgs "$@" #First, parse all cli args. We only need the -C flag, but that forces us to just parse everything. - shift $((OPTIND-1)) #Shift past all of the options to the file arguments. - - parseConf "$LCONFIG" false #Parse local config file. - set -- $INPUT_ARGS #Reset $@ for cli option reparsing. - OPTIND=1 #To reset argument parsing, we must set OPTIND to 1. - - parseArgs "$@" #Reparse cli to overwrite local config options. - shift $((OPTIND-1)) #Shift past all of the options to the file arguments. - OPTIND=1 #Reset argument index. +#Dual ISO Conversion + if [ $DUAL_ISO == true ]; then + echo -e "\033[1m${TRUNC_ARG}:\033[0m Combining Dual ISO...\n" - let ARGNUM-- - done + #Original DNGs will be moved here. + oldFiles="${TMP}/orig_dng" + mkdirS $oldFiles; if [ $? -eq 1 ]; then return 1; fi + + if [[ $(${CR_HDR} "${TMP}/${TRUNC_ARG}_$(printf "%06d" $FRAME_START).dng") == *"ISO blending didn't work"* ]]; then + invOption "The input wasn't shot Dual ISO!" + fi + + find $TMP -maxdepth 1 -name "*.dng" -print0 | sort -z | xargs -0 -I {} -P $THREADS -n 1 \ + bash -c "iso_par '{}' '$CR_HDR' '$TMP' '$FRAME_END' '$oldFiles' '$CHROMA_SMOOTH'" + + BLACK_LEVEL=$(exiftool -BlackLevel -s -s -s ${TMP}/${TRUNC_ARG}_$(printf "%06d" $(echo "$FRAME_START" | bc)).dng) #Use the first DNG to get the new correct black level. + + echo -e "\n" + fi + + if [ $setBL == true ]; then + echo -e "BlackLevel: ${BLACK_LEVEL}" >> $FILE/settings.txt #Black level must now be set. + fi + +#Get White Balance correction factor. + if [ $GEN_WHITE == true ]; then + echo -e "\033[1m${TRUNC_ARG}:\033[0m Generating WB...\n" + + #Calculate n, the distance between samples. + frameLen=$(echo "$FRAME_END - $FRAME_START + 1" | bc) #Offset by one to avoid division by 0 errors later. min value must be 1. + + #A point of improvement, this. + if [[ $WHITE_SPD -gt $frameLen ]]; then + WHITE_SPD=$frameLen + fi + n=`echo "${frameLen} / ${WHITE_SPD}" | bc` + + toBal="${TMP}/toBal" + mkdirS $toBal; if [ $? -eq 1 ]; then return 1; fi + + #Develop every nth file for averaging. + local i=0 + local t=0 + for file in $TMP/*.dng; do + if [ `echo "(${i}+1) % ${n}" | bc` -eq 0 ]; then + $DCRAW -q 0 $BADPIXELS -r 1 1 1 1 -g $GAMMA -k $BLACK_LEVEL $SATPOINT -o 0 -T "${file}" + name=$(basename "$file") + mv "$TMP/${name%.*}.tiff" $toBal #TIFF MOVEMENT. We use TIFFs here because it's easy for dcraw and Python. + let t++ + fi + echo -e "\033[2K\rWB Development: Sample ${t}/$(echo "${frameLen} / $n" | bc) (Frame: $(echo "${i} + 1" | bc)/${FRAME_END})\c" + let i++ + done + echo "" + + #Calculate + store result into a form dcraw likes. + echo -e "Calculating Auto White Balance..." + BALANCE=`$BAL $toBal` + + elif [ $CAMERA_WB == true ]; then + echo -e "\033[1m${TRUNC_ARG}:\033[0m Retrieving Camera White Balance..." + + for file in $TMP/*.dng; do + #dcraw a single file verbosely, to get the camera multiplier with awk. + BALANCE=`$DCRAW -T -w -v -c ${file} 2>&1 | awk '/multipliers/ { print $2, $3, $4 }'` + break + done + + else #Something must always be set. + echo -e "\033[1m${TRUNC_ARG}:\033[0m Ignoring White Balance..." + + BALANCE="1.000000 1.000000 1.000000" + fi + + #Finally, set the white balance after determining it. + if [ $isScale = false ]; then + BALANCE=$(normToOne "$BALANCE") + fi + green=$(getGreen "$BALANCE") + WHITE="-r ${BALANCE} ${green}" + echo -e "Correction Factor (RGBG): ${BALANCE} ${green}\n" + +#Move .wav. + SOUND_PATH="${TMP}/${TRUNC_ARG}_.wav" + + if [ ! -f $SOUND_PATH ]; then + echo -e "*Not moving .wav, because it doesn't exist.\n" + else + echo -e "*Moving .wav.\n" + cp $SOUND_PATH $FILE + fi + +#PROCESSING + +#IMAGE PROCESSING + if [ $IMAGES == true ] ; then + echo -e "$(bold ${TRUNC_ARG}:) Processing Image Sequence from Frame ${FRAME_START} to ${FRAME_END}...\n" + +#Define Image Directories, Create SEQ directory + local SEQ="${FILE}/${IMG_FMT}_${TRUNC_ARG}" + local PROXY="${FILE}/proxy_${TRUNC_ARG}" + + mkdirS $SEQ; if [ $? -eq 1 ]; then return 1; fi + + if [ $isJPG == true ]; then + mkdirS $PROXY; if [ $? -eq 1 ]; then return 1; fi + fi + +#Run the parallel image processing + iProc + +#FFMPEG Filter Application: Temporal Denoising, 3D LUTs, Deshake, hqdn Denoising, removegrain denoising, unsharp so far. +#See construction of $V_FILTERS in PREPARATION. + if [[ $FFMPEG_FILTERS == true ]]; then + tmpFiltered="${TMP}/filtered" + tmpUnfiltered="${TMP}/unfiltered" + mkdir $tmpFiltered + mkdir $tmpUnfiltered + + #Give correct output. + echo -e "\033[1mApplying Filters:\033[0m $(joinArgs ", " "${FILTER_ARR[@]}")...\n" + + if [[ $IMG_FMT == "exr" ]]; then + echo -e "Note: EXR filtering lags due to middle-step conversion (ffmpeg has no EXR encoder).\n" + + img_res=$(identify ${SEQ}/${TRUNC_ARG}_$(printf "%06d" $(echo "$FRAME_START" | bc)).${IMG_FMT} | cut -d$' ' -f3) + + tConvert "$SEQ" "$tmpUnfiltered" "$IMG_FMT" "dpx" + + ffmpeg -start_number $FRAME_START -f image2 -vcodec dpx -s "${img_res}" -r $FPS -loglevel panic -stats -i "${tmpUnfiltered}/${TRUNC_ARG}_%06d.dpx" \ + $V_FILTERS -pix_fmt rgb48be -vcodec dpx -n -r $FPS -start_number $FRAME_START ${tmpFiltered}/${TRUNC_ARG}_%06d.dpx + + tConvert "$tmpFiltered" "$SEQ" "dpx" "$IMG_FMT" + else + applyFilters "$SEQ" + fi + + echo "" + fi + fi + +#MOVIE PROCESSING + VID="${FILE}/${TRUNC_ARG}" + + SOUND="-i ${TMP}/${TRUNC_ARG}_.wav" + SOUND_ACTION="-c:a mp3 -b:a 320k" + if [ ! -f $SOUND_PATH ]; then + SOUND="" + SOUND_ACTION="" + fi + + if [[ $MOVIE == true && $IMAGES == false && $isH264 == true ]]; then + echo -e "\033[1m${TRUNC_ARG}:\033[0m Encoding to ProRes/H.264..." + runSim dcrawOpt mov_main mov_prox + echo "" + elif [[ $MOVIE == true && $IMAGES == false && $isH264 == false ]]; then + echo -e "\033[1m${TRUNC_ARG}:\033[0m Encoding to ProRes..." + dcrawOpt | mov_main + echo "" + elif [[ $MOVIE == false && $IMAGES == false && $isH264 == true ]]; then + echo -e "\033[1m${TRUNC_ARG}:\033[0m Encoding to H.264..." + dcrawOpt | mov_prox + echo "" + elif [[ $IMAGES == true ]]; then + V_FILTERS="" #Only needed if reading from images. + V_FILTERS_PROX="-vf $FINAL_SCALE" + + if [[ $IMG_FMT == "dpx" ]]; then + V_FILTERS="-vf lut3d=${COLOR_LUTS[0]}/srgb--lin.cube" + V_FILTERS_PROX="-vf lut3d=${COLOR_LUTS[0]}/srgb--lin.cube,$FINAL_SCALE" #We apply a sRGB to Linear 1D LUT if reading DPX. Because it's fucking broken. + fi + + #Use images if available, as opposed to developing the files again. + if [[ $MOVIE == true && $isH264 == true ]]; then + echo -e "\033[1m${TRUNC_ARG}:\033[0m Encoding to ProRes/H.264..." + mov_main_img & + mov_prox_img & + wait + + echo "" + elif [[ $MOVIE == true && $isH264 == false ]]; then + echo -e "\033[1m${TRUNC_ARG}:\033[0m Encoding to ProRes..." + mov_main_img + echo "" + elif [[ $MOVIE == false && $isH264 == true ]]; then + echo -e "\033[1m${TRUNC_ARG}:\033[0m Encoding to H.264..." + mov_prox_img + echo "" + fi + fi + +#Potentially move DNGs. + if [ $KEEP_DNGS == true ]; then + echo -e "\033[1mMoving DNGs...\033[0m" + DNG="${FILE}/dng_${TRUNC_ARG}" + mkdirS $DNG; if [ $? -eq 1 ]; then return 1; fi + + if [ $DUAL_ISO == true ]; then + oldFiles="${TMP}/orig_dng" + find $oldFiles -name "*.dng" | xargs -I '{}' mv {} $DNG #Preserve the original, unprocessed DNGs. + else + find $TMP -name "*.dng" | xargs -I '{}' mv {} $DNG + fi + fi + + echo -e "\n\033[1mCleaning Up.\033[0m\n\n" + +#Delete tmp + rm -rf $TMP } diff --git a/src/core/parsing.sh b/src/core/parsing.sh index e7f99f8..2317114 100644 --- a/src/core/parsing.sh +++ b/src/core/parsing.sh @@ -1,3 +1,6 @@ +#!/bin/bash + +#desc: All config and command line parsing happens here. parseConf() { file=$1 #The File to Parse diff --git a/src/core/proc.sh b/src/core/proc.sh new file mode 100644 index 0000000..e6a6349 --- /dev/null +++ b/src/core/proc.sh @@ -0,0 +1,247 @@ +#!/bin/bash + +#desc: The bulk image processing operations; from dcraw, to ffmpeg, to parallel functions. + +dcrawOpt() { + #usage: dcrawOpt | >>ppm data + #desc: Find, develop, and splay raw DNG data as ppm, ready to be processed. + + find "${TMP}" -maxdepth 1 -iname "*.dng" -print0 | sort -z | tr -d "\n" | xargs -0 \ + $DCRAW -c -q $DEMO_MODE $FOUR_COLOR -k $BLACK_LEVEL $SATPOINT $BADPIXELS $WHITE -H $HIGHLIGHT_MODE -g $GAMMA $WAVE_NOISE -o $SPACE $DEPTH +} #Is prepared to pipe all the files in TMP outwards. + +dcrawImg() { # + #usage: dcrawImg | >>ppm data + #desc: Find and splay image sequence data as ppm, ready to be processed by ffmpeg. + #note: Not working well. IM has trouble keeping itself linear. Kinda slow b/c ppm conversion, even if it worked... + + find "${SEQ}" -maxdepth 1 -iname "*.${IMG_FMT}" -print0 | sort -z | xargs -0 -I {} convert '{}' -set colorspace sRGB -colorspace RGB ppm:- +} #Finds all images, prints to stdout, without any operations, using convert. ppm conversion is inevitably slow, however... + +mov_main() { + #usage: >>ppm data | mov_main + #desc: Creates the primary, high quality movie MOV file from input ppm data. + + ffmpeg -f image2pipe -vcodec ppm -r $FPS -i pipe:0 \ + -loglevel panic -stats $SOUND -vcodec prores_ks -pix_fmt rgb48be -n -r $FPS -profile:v 4444 -alpha_bits 0 -vendor ap4h $V_FILTERS $SOUND_ACTION "${VID}_hq.mov" +} #-loglevel panic -stats + +mov_prox() { + #usage: >>ppm data | mov_prox + #desc: Creates the lower quality movie MP4 file from input ppm data. + + ffmpeg -f image2pipe -vcodec ppm -r $FPS -i pipe:0 \ + -loglevel panic -stats $SOUND -c:v libx264 -n -r $FPS -preset fast $V_FILTERS_PROX -crf 23 -c:a mp3 "${VID}_lq.mp4" +} #The option -vf "scale=trunc(iw/2)*2:trunc(ih/2)*2" fixes when x264 is unhappy about non-2 divisible dimensions. + +mov_main_img() { + #usage: mov_main_img + #desc: Creates the primary, high quality movie MOV file from the image sequence. + + ffmpeg -start_number $FRAME_START -loglevel panic -stats -f image2 -i ${SEQ}/${TRUNC_ARG}_%06d.${IMG_FMT} $SOUND -vcodec prores_ks \ + -pix_fmt rgb48le -n -r $FPS -profile:v 4444 -alpha_bits 0 -vendor ap4h $V_FILTERS $SOUND_ACTION "${VID}_hq.mov" +} + +mov_prox_img() { + #usage: mov_prox_img + #desc: Creates the lower quality movie MP4 file from the image sequence. + + ffmpeg -start_number $FRAME_START -loglevel panic -stats -f image2 -i ${SEQ}/${TRUNC_ARG}_%06d.${IMG_FMT} $V_FILTERS_PROX $SOUND -c:v libx264 \ + -n -r $FPS -preset veryfast -crf 21 -c:a mp3 -b:a 320k "${VID}_lq.mp4" +} + +#PARALLEL FUNCTIONS + +dng_par() { + #usage: dng_par 1${} (frame range) 2$MLV_DUMP 3$REAL_MLV 4$DARK_PROC 5$no_data 6$smooth 7$TMP 8$FRAME_END 9$TRUNC_ARG 10$FRAME_START + #arglength: Takes 10 args. + #desc: Called by xargs; is capable of dumping DNGs with the mlv_dump backend in parallel by being run in a subshell. + #note: Unfortunately, this means the entire relevant environment needs to be passed along too. + + range=$1 + firstFrame=false + if [[ $range == "0-0" ]]; then #mlv_dump can't handle 0-0, so we develop 0-1. + range="0-1" + firstFrame=true + fi + + tmpOut=${7}/${range} #Each output will number from 0, so give each its own folder. + mkdir -p $tmpOut + + start=$(echo "$range" | cut -d'-' -f1) + end=$(echo "$range" | cut -d'-' -f2) #Get start and end frames from the frame range + + $2 $3 $4 -o "${tmpOut}/${9}_" -f ${range} $6 --dng --batch | { #mlv_dump command. Uses frame range. + lastCur=0 + while IFS= read -r line; do + output=$(echo $line | grep -Po 'V.*A' | cut -d':' -f2 | cut -d$' ' -f1) #Hacked my way to the important bit. + if [[ $output == "" ]]; then continue; fi #If there's no important bit, don't print. + + cur=$(echo "$output" | cut -d'/' -f1) #Current frame. + if [[ $cur == $lastCur ]] || [[ $cur -gt $end ]] || [[ $cur -lt $start ]]; then continue; fi #Turns out, it goes through all the frames, even if cutting the frame range. So, clamp it! + + lastCur=$cur #It likes to repeat itself. + echo -e "\033[2K\rMLV to DNG: Frame $(echo "${cur} + ${10}" | bc)/${8}\c" #Print out beautiful progress bar, in parallel! + done + + } #Progress Bar + if [[ $firstFrame == true ]]; then #If 0-0. + rm $(printf "${tmpOut}/${9}_%06d.dng" 1) 2>/dev/null #Remove frame #1, leaving us only with frame #0. + mv $tmpOut "${7}/0-0" #Move back to 0-0, as if that's how it was developed all along. + fi +} + +export -f dng_par #Export to run in subshell. + +iso_par() { + #usage: inc_iso 1${} 2$CR_HDR 3$TMP 4$FRAME_END 5$oldFiles 6$CHROMA_SMOOTH. + #arglength: Takes 6 args. + #desc: Called by xargs; is capable of processing Dual ISO DNGs in parallel by being run in a subshell. + #note: Progress bar is thread safe. Experiment gone right :). + #note: Unfortunately, this means the entire relevant environment needs to be passed along too. + + count=$(echo "$(echo $(echo $1 | rev | cut -d "_" -f 1 | rev | cut -d "." -f 1 | grep "[0-9]") | bc) + 1" | bc) #Get count from filename. + + $2 $1 $6 >/dev/null 2>/dev/null #The LQ option, --mean23, is completely unusable in my opinion. + + name=$(basename "$1") + mv "${3}/${name%.*}.dng" $5 #Move away original dngs. + mv "${3}/${name%.*}.DNG" "${3}/${name%.*}.dng" #Rename *.DNG to *.dng. + + echo -e "\033[2K\rDual ISO Development: Frame ${count}/${4}\c" +} + +export -f iso_par #Must expose function to subprocess. + +img_par() { + #usage: img_par 1${} 2$DEMO_MODE 3$FOUR_COLOR 4$BADPIXELS 5$WHITE 6$HIGHLIGHT_MODE 7$GAMMA 8$WAVE_NOISE 9$DEPTH 10$SEQ 11$TRUNC_ARG 12$IMG_FMT 13$FRAME_END 14$DEPTH_OUT 15$COMPRESS 16$isJPG 17$PROXY_SCALE 18$PROXY 19$BLACK_LEVEL 20$SPACE 21$SATPOINT 22$DCRAW 23$FFMPEG_FILTERS + #arglength: Takes 22 arguments. + #desc: Called by xargs; is capable of developing images in parallel by being run in a subshell. + #note: Unfortunately, this means the entire relevantenvironment needs to be passed along too. + + count=$(echo $(echo $1 | rev | cut -d "_" -f 1 | rev | cut -d "." -f 1 | grep "[0-9]") | bc) #Instead of count from file, count from name! + DCRAW=${22} + + DPXHACK="" + if [[ ${12^^} == "DPX" && ( ${23} == false ) ]]; then DPXHACK="-colorspace sRGB"; else DPXHACK=""; fi + #Trust me, I've tried everything else; but this sRGB transform works. Must be an IM bug. Keep an eye on it! + #The sRGB curve is only applied if going to DPX while DPX is the target image format. Aka. At the end; not in the middle. + + if [ ${16} == true ]; then + $DCRAW -c -q $2 $3 $4 $5 -H $6 -k ${19} ${21} -g $7 $8 -o ${20} $9 $1 | \ + tee >(convert ${14} - -set colorspace RGB ${DPXHACK} ${15} $(printf "${10}/${11}_%06d.${12}" ${count})) | \ + convert - -set colorspace XYZ -quality 80 -colorspace sRGB -resize ${17} $(printf "${18}/${11}_%06d.jpg" ${count}) + #JPGs don't get ffmpeg filters applied. They simply can't handle it. + echo -e "\033[2K\rDNG to ${12^^}/JPG: Frame ${count^^}/${13}\c" + else + $DCRAW -c -q $2 $3 $4 $5 -H $6 -k ${19} ${21} -g $7 $8 -o ${20} $9 $1 | \ + convert ${14} - -set colorspace RGB ${DPXHACK} ${15} $(printf "${10}/${11}_%06d.${12}" ${count}) + echo -e "\033[2K\rDNG to ${12^^}: Frame ${count^^}/${13}\c" + fi +} + +export -f img_par #Need to export the function for it to be used in child functions. + +#~ See http://www.imagemagick.org/discourse-server/viewtopic.php?t=21161 + + +conv_par() { + #usage: conv_par 1${} 2$TRUNC_ARG 3$outFolder 4$fromFMT 5$iccProf 6$toFMT 7$DEPTH_OUT 8$compress 9$FRAME_END 10$IMG_FMT + #arglength: Takes 10 arguments. + #desc: Called by xargs; is capable of converting images in parallel by being run in a subshell. Automatically applies the DPX hack. + #note: Unfortunately, this means the entire relevant environment needs to be passed along too. + + count=$(echo $(echo $1 | rev | cut -d "_" -f 1 | rev | cut -d "." -f 1 | grep "[0-9]") | bc) #Get count from filename. + + echo -e "\033[2K\rMiddle-step: ${4^^} to ${6^^}, Frame ${count^^}/${9}\c" + + DPXHACK="" + if [[ ${6^^} == "DPX" && ${10^^} == ${6^^} ]]; then DPXHACK="-colorspace sRGB"; else DPXHACK=""; fi + #Trust me, I've tried everything else; but this sRGB transform works. Must be an IM bug. Keep an eye on it! + #The sRGB curve is only applied if going to DPX while DPX is the target image format. Aka. At the end; not in the middle. + + convert ${7} ${1} ${5} $8 -set colorspace RGB ${DPXHACK} "${3}/$(printf "${2}_%06d" ${count}).${6}" +} + +export -f conv_par + + +#PRACTICAL PARALLEL FUNCTIONS + +iProc() { + #usage: iProc + #desc: Develop the sequence from the current global settings. + +#Define hardcoded compression based on IMG_FMT + local COMPRESS="" + if [ $isCOMPRESS == true ]; then + case ${IMG_FMT} in + exr) + COMPRESS="-compress piz" + ;; + tiff) + COMPRESS="-compress zip" + ;; + png) + COMPRESS="-quality 0" + ;; + dpx) + COMPRESS="-compress rle" + ;; + esac + fi + +#Convert all the actual DNGs to IMG_FMT, in parallel. + find "${TMP}" -maxdepth 1 -name '*.dng' -print0 | sort -z | xargs -0 -I {} -P $THREADS -n 1 \ + bash -c "img_par '{}' '$DEMO_MODE' '$FOUR_COLOR' '$BADPIXELS' '$WHITE' '$HIGHLIGHT_MODE' '$GAMMA' '$WAVE_NOISE' '$DEPTH' \ + '$SEQ' '$TRUNC_ARG' '$IMG_FMT' '$FRAME_END' '$DEPTH_OUT' '$COMPRESS' '$isJPG' '$PROXY_SCALE' '$PROXY' '$BLACK_LEVEL' '$SPACE' '$SATPOINT' '$DCRAW' '$FFMPEG_FILTERS'" + + # Removed | cut -d '' -f $FRAME_RANGE , as this happens when creating the DNGs in the first place. + + if [ $isJPG == true ]; then #Make it print "Frame $FRAMES / $FRAMES" as the last output. + echo -e "\033[2K\rDNG to ${IMG_FMT^^}/JPG: Frame ${FRAME_END}/${FRAME_END}\c" + else + echo -e "\033[2K\rDNG to ${IMG_FMT^^}: Frame ${FRAME_END}/${FRAME_END}\c" + fi + + echo -e "\n" +} + +tConvert() { + #usage: tConvert 1$inFolder 2$outFolder 3$fromFMT 4$toFMT + #desc: Convert an image sequence from fromFMT to toFMT, with i/o inFolder and outFolder. + #note: optional icc profile. Wouldn't suggest using it right now... + + inFolder=$1 + outFolder=$2 + fromFMT=$3 + toFMT=$4 + iccProf=$5 + + if [[ ! -z $iccProf ]]; then iccProf="+profile icm -profile $iccProf"; fi + + compress="" + if [[ ${IMG_FMT^^} == ${toFMT^^} ]]; then compress=${COMPRESS}; fi + + find $inFolder -iname "*.${fromFMT}" -print0 | sort -z | xargs -0 -I {} -P $THREADS -n 1 \ + bash -c "conv_par '{}' '$TRUNC_ARG' '$outFolder' '$fromFMT' '$iccProf' '$toFMT' '$DEPTH_OUT' '$compress' '$FRAME_END' '$IMG_FMT'" + + echo "" +} + +applyFilters() { + #usage: applyFilters IO + #desc: Applies all ffmpeg filters to the given IO sequence of FMT format. + #note: If FMT isn't defined, IMG_FMT is automatically used. + #note: Ideally, this would be all we need. But alas, ffmpeg + exr is broken. So there's a custom middle step for that scenario. + + IO=$1 + FMT=$2 + + if [[ -z $FMT ]]; then FMT="${IMG_FMT}"; fi + + ffmpeg -start_number $FRAME_START -f image2 -i "${IO}/${TRUNC_ARG}_%06d.${FMT}" -loglevel panic -stats $V_FILTERS \ + -pix_fmt rgb48be -start_number $FRAME_START "${tmpFiltered}/${TRUNC_ARG}_%06d.${FMT}" + + tConvert "$tmpFiltered" "$IO" "$FMT" "$FMT" # "/home/sofus/subhome/src/convmlv/color/lin_xyz--srgb_srgb.icc" - profile application didn't work... +} diff --git a/src/documentation/TODO.sh b/src/documentation/TODO.sh index 6142e73..a15513e 100755 --- a/src/documentation/TODO.sh +++ b/src/documentation/TODO.sh @@ -1,3 +1,7 @@ +#!/bin/bash + +#desc: Kinda standalone TODO list. + less -R << EOF TODO: diff --git a/src/documentation/help.sh b/src/documentation/help.sh index 23288a9..c5af9e2 100644 --- a/src/documentation/help.sh +++ b/src/documentation/help.sh @@ -1,5 +1,7 @@ #!/bin/bash +#desc: The help function lives here, as well as helper functions thereof. + help() { less -R << EOF Usage: diff --git a/src/helpers/error.sh b/src/helpers/error.sh index b156bd8..8c6c959 100644 --- a/src/helpers/error.sh +++ b/src/helpers/error.sh @@ -1,3 +1,6 @@ +#!/bin/bash + +#desc: Error checking and consequence functions. invOption() { str=$1 @@ -13,50 +16,55 @@ invOption() { } checkArg() { - #stderr is for printing; stdout is for return values. + #usage: checkArg arg + #desc: Checks the argument to see if it's a valid MLV, RAW, or DNG sequence. + #return: 'false' if not valid, 'true' if it is. + #print (stderr): Error stuff. - argBase="$(basename "$ARG")" - argExt="${argBase##*.}" - argTrunc="${argBase%.*}" + local arg="$1" + + local argBase="$(basename "$arg")" + local argExt="${argBase##*.}" + local argTrunc="${argBase%.*}" local cont #Argument Checks - if [ ! -f $ARG ] && [ ! -d $ARG ]; then - nFound "File" "${ARG}" "Skipping File" - echo "skip" >&1; return + if [ ! -f $arg ] && [ ! -d $arg ]; then + nFound "File" "${arg}" "Skipping File" + echo "false" >&1; return fi - if [[ ! -d $ARG && ! ( $argExt == "MLV" || $argExt == "mlv" || $argExt == "RAW" || $argExt == "raw" ) ]]; then - echo -e "\033[0;31m\033[1mFile ${ARG} has invalid extension!\033[0m\n" >&2 - echo "skip" >&1; return + if [[ ! -d $arg && ! ( $argExt == "MLV" || $argExt == "mlv" || $argExt == "RAW" || $argExt == "raw" ) ]]; then + error "File ${arg} has invalid extension!\n" >&2 + echo "false" >&1; return fi - if [[ ( ( ! -f $ARG ) && $(ls -1 ${ARG%/}/*.[Dd][Nn][Gg] 2>/dev/null | wc -l) == 0 ) && ( `folderName ${ARG}` != $argTrunc ) ]]; then - echo -e "\033[0;31m\033[1mFolder ${ARG} contains no DNG files!\033[0m\n" >&2 - echo "skip" >&1; return + if [[ ( ( ! -f $arg ) && $(ls -1 ${arg%/}/*.[Dd][Nn][Gg] 2>/dev/null | wc -l) == 0 ) && ( `folderName ${arg}` != $argTrunc ) ]]; then + error "Folder ${arg} contains no DNG files!" >&2 + echo "false" >&1; return fi - if [ ! -d $ARG ] && [[ $(echo $(wc -c ${ARG} | xargs | cut -d " " -f1) / 1000 | bc) -lt 1000 ]]; then #Check that the file is not too small. + if [ ! -d $arg ] && [[ $(echo $(wc -c ${arg} | xargs | cut -d " " -f1) / 1000 | bc) -lt 1000 ]]; then #Check that the file is not too small. cont=false while true; do #xargs easily trims the cut statement, which has a leading whitespace on Mac. - read -p "${ARG} is unusually small at $(echo "$(echo "$(wc -c ${ARG})" | xargs | cut -d$' ' -f1) / 1000" | bc)KB. Continue, skip, remove, or quit? [c/s/r/q] " csr + read -p "${arg} is unusually small at $(echo "$(echo "$(wc -c ${arg})" | xargs | cut -d$' ' -f1) / 1000" | bc)KB. Continue, skip, remove, or quit? [c/s/r/q] " csr case $csr in - [Cc]* ) "\n\033[0;31m\033[1mContinuing.\033[0m\n"; break + [Cc]* ) error "\nContinuing.\n"; break ;; - [Ss]* ) echo -e "\n\033[0;31m\033[1mSkipping.\033[0m\n"; cont=true; break + [Ss]* ) error "\nSkipping.\n"; cont=true; break ;; - [Rr]* ) echo -e "\n\033[0;31m\033[1mRemoving ${ARG}.\033[0m\n"; cont=true; rm $ARG; break + [Rr]* ) error "\nRemoving ${arg}.\n"; cont=true; rm $arg; break ;; - [Qq]* ) echo -e "\n\033[0;31m\033[1mQuitting.\033[0m\n"; isExit=true; break + [Qq]* ) error "\nQuitting.\n"; isExit=true; break ;; - * ) echo -e "\033[0;31m\033[1mPlease answer continue, skip, or remove.\033[0m\n" + * ) error "Please answer continue, skip, or remove.\n" ;; esac done if [ $cont == true ]; then - echo "skip" >&1; return + echo "false" >&1; return fi fi } diff --git a/src/helpers/format.sh b/src/helpers/format.sh index 35c9a80..f8fb757 100644 --- a/src/helpers/format.sh +++ b/src/helpers/format.sh @@ -1,7 +1,39 @@ +#!/bin/bash + +#desc: Text formatting functions. + bold() { echo -e "\033[1m${1}\033[0m" } +selected() { + echo -e "$(bold "\033[32m${1}\033[0m")" +} + +error() { + echo -e "$(bold "\033[31m${1}\033[0m")" +} + +formArg() { + #usage: formArg item selection + #desc: Formats ARG values prettily; param can be "selected" or "deselected". + #return: Formatted path. + local item="$1" + local selection="$2" + + local itemBase=$(basename $item) + local itemExt=".${itemBase##*.}" #Dot must be in here. + if [[ "${itemExt}" == ".${itemBase}" ]]; then itemExt=""; fi #This means the input is a folder, which has no extension. + + local itemDir=$(dirname "$item") + + if [[ $selection == "selected" ]]; then + echo -e "${itemDir}/$(selected "${itemBase%.*}")${itemExt}" + elif [[ $selection == "deselected" ]]; then + echo -e "${itemDir}/$(bold ${itemBase%.*})${itemExt}" + fi +} + cVal() { #usage: cVal value #desc: Formats config file values, as bolded grey. diff --git a/src/helpers/platform.sh b/src/helpers/platform.sh index 059f65d..b739c5d 100644 --- a/src/helpers/platform.sh +++ b/src/helpers/platform.sh @@ -1,5 +1,7 @@ #!/bin/bash +#desc: Platform-specific functions. + getThreads() { local threads=4 #4 threads by default diff --git a/src/helpers/utility.sh b/src/helpers/utility.sh index 78269cb..10601e3 100644 --- a/src/helpers/utility.sh +++ b/src/helpers/utility.sh @@ -1,3 +1,7 @@ +#!/bin/bash + +#desc: Portable utility functions that could live in their own library. + nFound() { #Prints: ${type} ${name} not found! ${exec_instr}.\n\t${down_instr} to stderr. type=$1 name="$2" @@ -12,6 +16,11 @@ nFound() { #Prints: ${type} ${name} not found! ${exec_instr}.\n\t${down_instr} t } mkdirS() { + #usage: mkdirS path; if [ $? -eq 1 ]; then return 1; fi + #desc: A function that allows the user to decide whether to overwrite an existing directory. + #note: The user must use return codes to provide the return from develop(). + #return: Exit code 1 denotes that we must continue. + path=$1 cleanup=$2 cont=false @@ -22,11 +31,11 @@ mkdirS() { case $ynq in [Yy]* ) echo -e ""; rm -rf $path; mkdir -p $path >/dev/null 2>/dev/null; break ;; - [Nn]* ) echo -e "\n\033[0;31m\033[1mDirectory ${path} won't be created.\033[0m\n"; cont=true; `$cleanup`; break + [Nn]* ) error "\nDirectory ${path} won't be created.\n"; cont=true; `$cleanup`; break ;; - [Qq]* ) echo -e "\n\033[0;31m\033[1mHalting execution. Directory ${path} won't be created.\033[0m\n"; `$cleanup`; exit 1; + [Qq]* ) error "\nHalting execution. Directory ${path} won't be created.\n"; `$cleanup`; exit 1; ;; - * ) echo -e "\033[0;31m\033[1mPlease answer yes or no.\033[0m\n" + * ) error "Please answer yes or no, or quit.\n" ;; esac done @@ -35,8 +44,7 @@ mkdirS() { fi if [ $cont == true ]; then - let ARGNUM-- - continue + return 1 fi } @@ -50,3 +58,26 @@ joinArgs() { #Joins the arguments of the input array using commas. local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}" } + +runSim() { + # Command: cat $PIPE | cmd1 & cmdOrig | tee $PIPE | cmd2 + + # cat $PIPE | cmd1 - gives output of pipe live. Pipes it into cmd1. Nothing yet; just setup. + # & - runs the next part in the background. + # cmdOrig | tee $PIPE | cmd2 - cmdOrig pipes into the tee, which splits it back into the previous pipe, piping on to cmd2! + + # End Result: Output of cmdOrig is piped into cmd1 and cmd2, which execute, both printing to stdout. + + cmdOrig=$1 + cmd1=$2 + cmd2=$3 + + #~ echo $cmdOrig $cmd1 $cmd2 + #~ echo $($cmdOrig) + + PIPE="${TMP}/pipe_vid" # $(date +%s%N | cut -b1-13)" + mkfifo $PIPE 2>/dev/null + + cat $PIPE | $cmd1 & $cmdOrig | tee $PIPE | $cmd2 #The magic of simultaneous execution ^_^ + #~ cat $PIPE | tr 'e' 'a' & echo 'hello' | tee $PIPE | tr 'e' 'o' #The magic of simultaneous execution ^_^ + } diff --git a/src/imgProcessing/imgMath.sh b/src/imgProcessing/imgMath.sh new file mode 100644 index 0000000..7dbcdb7 --- /dev/null +++ b/src/imgProcessing/imgMath.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +#desc: Math for bash regarding image operations. So that we don't have to hop over to python! + +normToOne() { + wBal=$1 + + max=0.0 + for mult in $wBal; do + if [ $(echo " $mult > $max" | bc) -eq 1 ]; then + max=$mult + fi + done + + for mult in $wBal; do + echo -e "$(echo "scale=6; x=${mult} / ${max}; if(x<1) print 0; x" | bc -l) \c" #BC is bae. + done +} + +getGreen() { + wBal=$1 + + i=0 + for mult in $wBal; do + if [ $i -eq 1 ]; then + echo -e "${mult}" + fi + let i++ + done +} diff --git a/src/programs/califrame.sh b/src/programs/califrame.sh index eada7a8..90c0e23 100644 --- a/src/programs/califrame.sh +++ b/src/programs/califrame.sh @@ -1,3 +1,7 @@ +#!/bin/bash + +#desc: Calibration frame code. + mkDarkframe() { echo -e "\n\033[1m\033[0;32m\033[1mAveraging Darkframe File\033[0m: ${ARG}" $MLV_DUMP -o $DARK_OUT ${FILE_ARGS_ARRAY[0]} 2>/dev/null 1>/dev/null diff --git a/src/programs/shotSettings.sh b/src/programs/shotSettings.sh index dc25f32..8ffe87a 100644 --- a/src/programs/shotSettings.sh +++ b/src/programs/shotSettings.sh @@ -1,3 +1,7 @@ +#!/bin/bash + +#desc: Functions dealing with getting and printing footage (ARG) settings. + prntSet() { cat << EOF $(bold CameraName): ${CAM_NAME} @@ -74,6 +78,23 @@ dngSet() { #Set as many options as the RAW spec will allow. Grey out the rest. rm $dataDNG } +reuseSet() { + local dng_loc="$1" + + FPS=`cat ${dng_loc}/../settings.txt | grep "FPS" | cut -d $" " -f2` + + CAM_NAME=`cat ${dng_loc}/../settings.txt | grep "CameraName" | cut -d $" " -f2` + FRAMES=`cat ${dng_loc}/../settings.txt | grep "Frames" | cut -d $" " -f2` #Grab FRAMES from previous run. + RES_IN=`cat ${dng_loc}/../settings.txt | grep "Resolution" | cut -d $" " -f2` + ISO=`cat ${dng_loc}/../settings.txt | grep "ISO" | cut -d $" " -f2` + APERTURE=`cat ${dng_loc}/../settings.txt | grep "Aperture" | cut -d $" " -f2` + LEN_FOCAL=`cat ${dng_loc}/../settings.txt | grep "FocalLength" | cut -d $" " -f2` + SHUTTER=`cat ${dng_loc}/../settings.txt | grep "ShutterSpeed" | cut -d $" " -f2` + REC_DATE=`cat ${dng_loc}/../settings.txt | grep "RecordingDate" | cut -d $" " -f2` + REC_TIME=`cat ${dng_loc}/../settings.txt | grep "RecordingTime" | cut -d $" " -f2` + KELVIN=`cat ${dng_loc}/../settings.txt | grep "WBKelvin" | cut -d $" " -f2` +} + printFileSettings() { ARG="${FILE_ARGS_ARRAY[0]}" checkArg