#!/bin/bash

# lld -- lossless decimator/de-duplicator

# This script takes a compatible all-I-frame video file and losslessly deduplicates to a new video file.
# The new file will have the same fps as the old file, though. Changing fps is left to the user --
#   it's not simple for e.g. ProRes to do this and remain "legal".
#
# You may want to tweak the configuration of mpdecimate below. For me, "hi" is disabled, because even between my
# duplicate frames there typically existed one 8x8 block that would exceed a high value, for some reason.
# lo and frac may need tweaking for your source material: https://ffmpeg.org/ffmpeg-filters.html#mpdecimate
# After running, you can investigate the output of mpdecimate in the temp directory file mpdecimate.txt.
#
# This script assumes no audio associated with the footage -- it would likely need to be modified to handle
# footage with audio
#
# max frame number in file is 99,999,999 frames (~38 days at 29.97fps)
#
# Development/discussion thread here: https://forum.videohelp.com/threads/383274-de-duplicating-decimating-ProRes-losslessly#post2483571

echo -e "\nlld version 1.0\n"

if [ -z $1 ]; then echo -e "Usage: ddpr file [expected_fps [debug]]\n  where expected_fps is the actual frame rate (used for info purposes only -- will not affect processing)\n  note: lld will create file.tempdir in current directory\n  If you specify \"debug\", lld will leave the temporary directory behind for evaluating logs, etc.\n"; exit; fi

fname=$1
newfps=$2
debug=$3

if [ ! -f "$fname" ]; then
  echo "$fname does not exist. Doing nothing."
  exit 1
fi

if [ -e "$fname".tempdir ]; then
  echo "$fname".tempdir already exists. Doing nothing.
  exit 1
fi

if [ -e "$fname.dedup.mov" ]; then
  echo "$fname.dedup.mov already exists. Doing nothing."
  exit 1
fi

fext=`echo $fname | sed -r 's/.*(\.[^.]*)$/\1/'`
fextcnt=`echo -n $fext | wc -c`

if [ $fextcnt -le 1 ]; then
  echo "Couldn't detect filename extension. Doing nothing."
  exit 1
fi

mkdir "$fname".tempdir
mkdir "$fname".tempdir/frames

cd "$fname".tempdir

fps=`ffprobe -v 0 -of compact=p=0 -select_streams 0 -show_entries stream=r_frame_rate ../$fname | sed 's/^r_frame_rate=/scale=15;/g' | bc` 

echo "$fps fps detected. Detecting duplicates..."

flen=`echo "scale=15;1/$fps" | bc`
flenoffset=`echo "scale=15;.1*$flen" | bc`

ffmpeg -i ../$fname -vf mpdecimate=max=1:hi=999999999:lo=64*3:frac=0.4 -loglevel debug -f null - > mpdecimate.txt 2>&1

fcnt=`cat mpdecimate.txt | grep "frames successfully decoded" | sed -r 's/^(.*) frames .*$/\1/'`

cat mpdecimate.txt | grep Parsed | grep keep | sed -r 's/^.*pts_time:(.*) drop.*/\1/' > ts.txt

if [ ! -f ts.txt ]; then
  echo "ts.txt not created. Exiting."
  exit 1
fi

res=`cat ts.txt | wc -l`

if [ $res -le "0" ]; then
 echo "ts.txt didn't generate correctly. Exiting."
 exit 1
fi

# The offset version is necessary because apparently ffmpeg doesn't find the nearest frame to the timestamp, but the next frame.
# As a result, occasional rounding errors mean that the wrong frame would be targeted when extracting them from the original file.
# The offset is a 10% backwards shift in the timestamp to guarantee that the next frame found will be the correct frame.
cat ts.txt | awk "{print \$1-$flenoffset}" | bc > ts2.txt

if [ ! -f ts2.txt ]; then
  echo "ts2.txt not created. Exiting."
  exit 1
fi

res=`cat ts2.txt | wc -l`

if [ $res -le "0" ]; then
 echo "ts2.txt didn't generate correctly. Exiting."
 exit 1
fi

newfr=`echo "scale=3;$fps*($res/$fcnt)" | bc`

echo "Detected $res good frames out of $fcnt total frames = detected actual frame rate of $newfr fps"

if [ ! -z $newfps ]; then
  fpserr=`echo "scale=3;100*($newfr/$newfps - 1)" | bc`

  fpserrfirstchar=`echo $fpserr | sed -r 's/^(.).*$/\1/g'`
  if [ $fpserrfirstchar == "." ]; then
    fpserr=`echo 0$fpserr`
  fi
  fpserrfirstchars=`echo $fpserr | sed -r 's/^(..).*$/\1/g'`
  if [ $fpserrfirstchars == "-." ]; then
    fpserr=`echo $fpserr | sed -r 's/^.(.*)$/\1/g'`
    fpserr=`echo "-0"$fpserr`
  fi

  echo "$fpserr% error from expected $newfps fps."
fi

echo "Generating segment video files..."

# doing -ss after -i because before is not accurate in this case for some reason (first few seconds work fine, then starts going off the rails)
cat ts2.txt | awk "{printf \"ffmpeg -i ../$1 -ss %s -t 0$flen -vcodec copy -acodec copy frames/%08d$fext\n\",\$1,f;f++}" > ffmpeg_frame_extraction_commands.txt

if [ ! -f ffmpeg_frame_extraction_commands.txt ]; then
  echo "ffmpeg_frame_extraction_commands.txt not created. Exiting."
  exit 1
fi

res=`cat ffmpeg_frame_extraction_commands.txt | wc -l`

if [ $res -le "0" ]; then
 echo "ffmpeg_frame_extraction_commands.txt didn't generate correctly. Exiting."
 exit 1
fi

source ffmpeg_frame_extraction_commands.txt > ffmpeg_frame_extraction_results.log 2>&1

res=`/bin/ls -1 frames/*$fext | wc -l`

if [ $res -le "0" ]; then
  echo "No segment files generated. Exiting."
  exit 1
fi

echo "Generating segment logfile for concatenation and concatenating..."

/bin/ls -1 frames/*$fext | sed -r "s/(.*)/file '\1'/" > segment_files_list.txt

if [ ! -f segment_files_list.txt ]; then
  echo "segment_files_list.txt not created. Exiting."
  exit 1
fi

res=`cat segment_files_list.txt | wc -l`

if [ $res -le "0" ]; then
 echo "segment_files_list.txt didn't generate correctly. Exiting."
 exit 1
fi

ffmpeg -f concat -i segment_files_list.txt -c copy ../$fname.dedup$fext > ffmpeg_concatenation_results.log 2>&1

if [ -z $debug ]; then
 echo "Deleting temporary directory. (Use e.g. \"lld file.mov 18 debug\" to prevent this.)"
 echo "Removing temporary directory: $fname.tempdir"
 cd ..
 rm -rf "$fname".tempdir
else
 echo "Temporary directory $fname.tempdir left behind."
fi

echo -e "Done. Resulting filename: $fname.dedup$fext\n"