2022-04-20 20:24:55 +00:00
use std ::borrow ::{ Borrow , Cow } ;
use std ::cmp ::{ Ordering , Reverse } ;
use std ::collections ::{ BTreeSet , HashSet } ;
use std ::convert ::TryInto ;
use std ::ffi ::OsString ;
use std ::fs ::File ;
use std ::io ::Write ;
use std ::path ::{ Path , PathBuf } ;
use std ::process ::{ exit , Command , Stdio } ;
use std ::sync ::atomic ::{ self , AtomicBool , AtomicU64 , AtomicUsize } ;
use std ::sync ::{ mpsc , Arc } ;
2022-05-21 08:55:43 +00:00
use std ::thread ::available_parallelism ;
2022-04-20 20:24:55 +00:00
use std ::{ cmp , fs , iter , thread } ;
use ansi_term ::{ Color , Style } ;
2021-11-17 08:57:08 +00:00
use anyhow ::{ bail , ensure , Context } ;
2021-09-03 19:01:11 +00:00
use crossbeam_utils ;
2022-02-22 20:17:37 +00:00
use ffmpeg ::format ::Pixel ;
2021-10-04 16:22:08 +00:00
use itertools ::Itertools ;
2022-01-02 05:49:47 +00:00
use rand ::prelude ::SliceRandom ;
use rand ::thread_rng ;
2021-09-03 19:01:11 +00:00
use tokio ::io ::{ AsyncBufReadExt , BufReader } ;
2022-01-07 23:03:39 +00:00
use tokio ::process ::ChildStderr ;
2021-09-03 19:01:11 +00:00
2022-04-20 20:24:55 +00:00
use crate ::broker ::{ Broker , EncoderCrash } ;
use crate ::chunk ::Chunk ;
use crate ::concat ::{ self , ConcatMethod } ;
use crate ::ffmpeg ::{ compose_ffmpeg_pipe , num_frames } ;
2022-05-11 17:52:24 +00:00
use crate ::grain ::{ create_film_grain_file , TransferFunction } ;
2022-04-20 20:24:55 +00:00
use crate ::parse ::valid_params ;
use crate ::progress_bar ::{
finish_progress_bar , inc_bar , inc_mp_bar , init_multi_progress_bar , init_progress_bar ,
reset_bar_at , reset_mp_bar_at , update_mp_chunk , update_mp_msg , update_progress_bar_estimates ,
} ;
use crate ::scene_detect ::av_scenechange_detect ;
use crate ::scenes ::{ Scene , ZoneOptions } ;
use crate ::split ::{ extra_splits , segment , write_scenes_to_file } ;
use crate ::vapoursynth ::{ create_vs_file , is_ffms2_installed , is_lsmash_installed } ;
use crate ::vmaf ::{ self , validate_libvmaf } ;
use crate ::{
create_dir , determine_workers , finish_multi_progress_bar , get_done , init_done , into_vec ,
read_chunk_queue , save_chunk_queue , ChunkMethod , ChunkOrdering , DashMap , DoneJson , Encoder ,
Input , ScenecutMethod , SplitMethod , TargetQuality , Verbosity ,
} ;
2021-10-30 11:49:26 +00:00
pub struct PixelFormat {
pub format : Pixel ,
pub bit_depth : usize ,
}
2021-11-20 18:43:13 +00:00
pub enum InputPixelFormat {
VapourSynth { bit_depth : usize } ,
FFmpeg { format : Pixel } ,
}
2021-11-25 19:05:24 +00:00
#[ allow(clippy::struct_excessive_bools) ]
2021-10-05 17:08:18 +00:00
pub struct EncodeArgs {
2021-09-03 19:01:11 +00:00
pub frames : usize ,
2021-10-04 16:22:08 +00:00
pub input : Input ,
2021-09-03 19:01:11 +00:00
pub temp : String ,
pub output_file : String ,
2021-12-25 07:34:06 +00:00
pub vs_script : Option < PathBuf > ,
2021-09-03 19:01:11 +00:00
pub chunk_method : ChunkMethod ,
2022-01-02 05:49:47 +00:00
pub chunk_order : ChunkOrdering ,
2021-10-05 17:08:18 +00:00
pub scenes : Option < PathBuf > ,
2021-09-03 19:01:11 +00:00
pub split_method : SplitMethod ,
2021-12-09 17:48:37 +00:00
pub sc_pix_format : Option < Pixel > ,
2021-09-14 21:04:15 +00:00
pub sc_method : ScenecutMethod ,
2022-03-02 05:33:31 +00:00
pub sc_only : bool ,
2021-09-14 21:04:15 +00:00
pub sc_downscale_height : Option < usize > ,
2021-09-03 19:01:11 +00:00
pub extra_splits_len : Option < usize > ,
pub min_scene_len : usize ,
2021-11-29 00:38:01 +00:00
pub max_tries : usize ,
2021-09-03 19:01:11 +00:00
pub passes : u8 ,
pub video_params : Vec < String > ,
pub encoder : Encoder ,
pub workers : usize ,
2021-11-15 22:15:26 +00:00
pub set_thread_affinity : Option < usize > ,
2022-01-01 21:13:03 +00:00
pub photon_noise : Option < u8 > ,
2022-03-28 15:23:08 +00:00
pub zones : Option < PathBuf > ,
2021-09-03 19:01:11 +00:00
// FFmpeg params
2021-10-30 11:49:26 +00:00
pub ffmpeg_filter_args : Vec < String > ,
2021-09-03 19:01:11 +00:00
pub audio_params : Vec < String > ,
2021-11-20 18:43:13 +00:00
pub input_pix_format : InputPixelFormat ,
pub output_pix_format : PixelFormat ,
2021-09-03 19:01:11 +00:00
pub verbosity : Verbosity ,
2021-11-18 20:06:50 +00:00
pub log_file : PathBuf ,
2021-09-03 19:01:11 +00:00
pub resume : bool ,
pub keep : bool ,
pub force : bool ,
pub vmaf : bool ,
pub vmaf_path : Option < PathBuf > ,
2021-10-05 17:08:18 +00:00
pub vmaf_res : String ,
2021-09-03 19:01:11 +00:00
pub concat : ConcatMethod ,
2021-10-05 17:08:18 +00:00
pub target_quality : Option < f64 > ,
2021-09-03 19:01:11 +00:00
pub probes : u32 ,
pub probe_slow : bool ,
pub min_q : Option < u32 > ,
pub max_q : Option < u32 > ,
pub probing_rate : u32 ,
2021-10-04 16:22:08 +00:00
pub vmaf_threads : Option < usize > ,
2021-09-03 19:01:11 +00:00
pub vmaf_filter : Option < String > ,
}
2021-10-05 17:08:18 +00:00
impl EncodeArgs {
2021-09-03 19:01:11 +00:00
/// Initialize logging routines and create temporary directories
pub fn initialize ( & mut self ) -> anyhow ::Result < ( ) > {
2022-02-22 20:17:37 +00:00
ffmpeg ::init ( ) ? ;
ffmpeg ::util ::log ::set_level ( ffmpeg ::util ::log ::level ::Level ::Fatal ) ;
2021-10-01 20:48:40 +00:00
2021-11-18 00:07:42 +00:00
if ! self . resume & & Path ::new ( & self . temp ) . is_dir ( ) {
fs ::remove_dir_all ( & self . temp )
. with_context ( | | format! ( " Failed to remove temporary directory {:?} " , & self . temp ) ) ? ;
}
create_dir! ( Path ::new ( & self . temp ) ) ? ;
create_dir! ( Path ::new ( & self . temp ) . join ( " split " ) ) ? ;
create_dir! ( Path ::new ( & self . temp ) . join ( " encode " ) ) ? ;
2022-01-18 23:20:01 +00:00
debug! ( " temporary directory: {} " , & self . temp ) ;
2021-11-18 00:07:42 +00:00
2022-01-02 19:28:01 +00:00
let done_path = Path ::new ( & self . temp ) . join ( " done.json " ) ;
let done_json_exists = done_path . exists ( ) ;
2021-11-17 08:57:08 +00:00
let chunks_json_exists = Path ::new ( & self . temp ) . join ( " chunks.json " ) . exists ( ) ;
2021-09-03 19:01:11 +00:00
2021-11-17 08:57:08 +00:00
if self . resume {
match ( done_json_exists , chunks_json_exists ) {
// both files exist, so there is no problem
( true , true ) = > { }
( false , true ) = > {
2021-11-19 14:59:12 +00:00
info! (
2021-11-17 08:57:08 +00:00
" resume was set but done.json does not exist in temporary directory {:?} " ,
& self . temp
) ;
self . resume = false ;
}
( true , false ) = > {
2021-11-19 14:59:12 +00:00
info! (
2021-11-17 08:57:08 +00:00
" resume was set but chunks.json does not exist in temporary directory {:?} " ,
& self . temp
) ;
self . resume = false ;
}
( false , false ) = > {
2021-11-19 14:59:12 +00:00
info! (
2021-11-17 08:57:08 +00:00
" resume was set but neither chunks.json nor done.json exist in temporary directory {:?} " ,
& self . temp
) ;
self . resume = false ;
}
}
}
2021-09-03 19:01:11 +00:00
2022-01-02 19:28:01 +00:00
if self . resume & & done_json_exists {
let done =
fs ::read_to_string ( done_path ) . with_context ( | | " Failed to read contents of done.json " ) ? ;
let done : DoneJson =
serde_json ::from_str ( & done ) . with_context ( | | " Failed to parse done.json " ) ? ;
self . frames = done . frames . load ( atomic ::Ordering ::Relaxed ) ;
// frames need to be recalculated in this case
if self . frames = = 0 {
self . frames = self . input . frames ( ) ? ;
done . frames . store ( self . frames , atomic ::Ordering ::Relaxed ) ;
}
init_done ( done ) ;
} else {
init_done ( DoneJson {
frames : AtomicUsize ::new ( 0 ) ,
done : DashMap ::new ( ) ,
audio_done : AtomicBool ::new ( false ) ,
} ) ;
let mut done_file = fs ::File ::create ( & done_path ) . unwrap ( ) ;
done_file . write_all ( serde_json ::to_string ( get_done ( ) ) ? . as_bytes ( ) ) ? ;
} ;
2021-09-03 19:01:11 +00:00
Ok ( ( ) )
}
2021-11-19 00:57:10 +00:00
fn read_queue_files ( source_path : & Path ) -> anyhow ::Result < Vec < PathBuf > > {
2021-09-03 19:01:11 +00:00
let mut queue_files = fs ::read_dir ( & source_path )
2021-11-19 00:57:10 +00:00
. with_context ( | | {
format! (
" Failed to read queue files from source path {:?} " ,
source_path
)
} ) ?
2021-09-03 19:01:11 +00:00
. map ( | res | res . map ( | e | e . path ( ) ) )
2021-11-19 00:57:10 +00:00
. collect ::< Result < Vec < _ > , _ > > ( ) ? ;
2021-10-04 16:22:08 +00:00
queue_files . retain ( | file | {
file . is_file ( ) & & matches! ( file . extension ( ) . map ( | ext | ext = = " mkv " ) , Some ( true ) )
} ) ;
2021-10-05 17:08:18 +00:00
concat ::sort_files_by_filename ( & mut queue_files ) ;
2021-09-03 19:01:11 +00:00
2021-11-19 00:57:10 +00:00
Ok ( queue_files )
2021-09-03 19:01:11 +00:00
}
2021-11-25 18:22:32 +00:00
/// Returns the number of frames encoded if crashed, to reset the progress bar.
2021-09-03 19:01:11 +00:00
pub fn create_pipes (
& self ,
2021-10-04 16:22:08 +00:00
chunk : & Chunk ,
2022-03-28 15:23:08 +00:00
encoder : Encoder ,
passes : u8 ,
2021-09-03 19:01:11 +00:00
current_pass : u8 ,
worker_id : usize ,
2022-01-19 01:42:41 +00:00
padding : usize ,
2021-12-14 23:17:07 +00:00
tpl_crash_workaround : bool ,
2021-11-25 18:22:32 +00:00
) -> Result < ( ) , ( EncoderCrash , u64 ) > {
2022-01-19 01:42:41 +00:00
update_mp_chunk ( worker_id , chunk . index , padding ) ;
2021-10-04 16:22:08 +00:00
let fpf_file = Path ::new ( & chunk . temp )
2021-09-03 19:01:11 +00:00
. join ( " split " )
2021-10-04 16:22:08 +00:00
. join ( format! ( " {} _fpf " , chunk . name ( ) ) ) ;
2021-09-03 19:01:11 +00:00
2022-03-28 15:23:08 +00:00
let mut video_params = chunk
. overrides
. as_ref ( )
. map_or_else ( | | self . video_params . clone ( ) , | ovr | ovr . video_params . clone ( ) ) ;
2021-12-14 23:17:07 +00:00
if tpl_crash_workaround {
// In aomenc for duplicate arguments, whichever is specified last takes precedence.
video_params . push ( " --enable-tpl-model=0 " . to_string ( ) ) ;
}
2022-03-28 15:23:08 +00:00
let mut enc_cmd = if passes = = 1 {
encoder . compose_1_1_pass ( video_params , chunk . output ( ) )
2021-09-03 19:01:11 +00:00
} else if current_pass = = 1 {
2022-03-28 15:23:08 +00:00
encoder . compose_1_2_pass ( video_params , fpf_file . to_str ( ) . unwrap ( ) )
2021-09-03 19:01:11 +00:00
} else {
2022-03-28 15:23:08 +00:00
encoder . compose_2_2_pass ( video_params , fpf_file . to_str ( ) . unwrap ( ) , chunk . output ( ) )
2021-09-03 19:01:11 +00:00
} ;
2021-11-11 02:52:32 +00:00
if let Some ( per_shot_target_quality_cq ) = chunk . tq_cq {
2022-03-28 15:23:08 +00:00
enc_cmd = encoder . man_command ( enc_cmd , per_shot_target_quality_cq as usize ) ;
2021-09-03 19:01:11 +00:00
}
let rt = tokio ::runtime ::Builder ::new_current_thread ( )
. enable_io ( )
. build ( )
. unwrap ( ) ;
2022-01-07 23:03:39 +00:00
let ( source_pipe_stderr , ffmpeg_pipe_stderr , enc_output , enc_stderr , frame ) =
rt . block_on ( async {
let mut source_pipe = if let [ source , args @ .. ] = & * chunk . source {
tokio ::process ::Command ::new ( source )
2021-10-13 02:21:55 +00:00
. args ( args )
. stdout ( Stdio ::piped ( ) )
. stderr ( Stdio ::piped ( ) )
. spawn ( )
. unwrap ( )
} else {
unreachable! ( )
} ;
2021-10-04 16:22:08 +00:00
2022-01-07 23:03:39 +00:00
let source_pipe_stdout : Stdio = source_pipe . stdout . take ( ) . unwrap ( ) . try_into ( ) . unwrap ( ) ;
2021-09-03 19:01:11 +00:00
2022-01-07 23:03:39 +00:00
let source_pipe_stderr = source_pipe . stderr . take ( ) . unwrap ( ) ;
2021-11-20 23:25:13 +00:00
2022-01-07 23:03:39 +00:00
// converts the pixel format
let create_ffmpeg_pipe = | pipe_from : Stdio , source_pipe_stderr : ChildStderr | {
let ffmpeg_pipe = compose_ffmpeg_pipe (
self . ffmpeg_filter_args . as_slice ( ) ,
self . output_pix_format . format ,
) ;
2021-09-03 19:01:11 +00:00
2022-01-07 23:03:39 +00:00
let mut ffmpeg_pipe = if let [ ffmpeg , args @ .. ] = & * ffmpeg_pipe {
tokio ::process ::Command ::new ( ffmpeg )
. args ( args )
. stdin ( pipe_from )
. stdout ( Stdio ::piped ( ) )
. stderr ( Stdio ::piped ( ) )
. spawn ( )
. unwrap ( )
} else {
unreachable! ( )
} ;
let ffmpeg_pipe_stdout : Stdio = ffmpeg_pipe . stdout . take ( ) . unwrap ( ) . try_into ( ) . unwrap ( ) ;
let ffmpeg_pipe_stderr = ffmpeg_pipe . stderr . take ( ) . unwrap ( ) ;
(
ffmpeg_pipe_stdout ,
source_pipe_stderr ,
Some ( ffmpeg_pipe_stderr ) ,
)
} ;
2021-11-20 23:25:13 +00:00
2022-01-07 23:03:39 +00:00
let ( y4m_pipe , source_pipe_stderr , mut ffmpeg_pipe_stderr ) =
if self . ffmpeg_filter_args . is_empty ( ) {
match & self . input_pix_format {
InputPixelFormat ::FFmpeg { format } = > {
if self . output_pix_format . format = = * format {
( source_pipe_stdout , source_pipe_stderr , None )
} else {
create_ffmpeg_pipe ( source_pipe_stdout , source_pipe_stderr )
}
}
InputPixelFormat ::VapourSynth { bit_depth } = > {
if self . output_pix_format . bit_depth = = * bit_depth {
( source_pipe_stdout , source_pipe_stderr , None )
} else {
create_ffmpeg_pipe ( source_pipe_stdout , source_pipe_stderr )
}
}
}
} else {
create_ffmpeg_pipe ( source_pipe_stdout , source_pipe_stderr )
} ;
let mut source_reader = BufReader ::new ( source_pipe_stderr ) . lines ( ) ;
let ffmpeg_reader = ffmpeg_pipe_stderr
. take ( )
. map ( | stderr | BufReader ::new ( stderr ) . lines ( ) ) ;
let pipe_stderr = Arc ::new ( parking_lot ::Mutex ::new ( String ::with_capacity ( 128 ) ) ) ;
let p_stdr2 = Arc ::clone ( & pipe_stderr ) ;
let ffmpeg_stderr = if ffmpeg_reader . is_some ( ) {
Some ( Arc ::new ( parking_lot ::Mutex ::new ( String ::with_capacity (
128 ,
) ) ) )
} else {
None
} ;
2021-11-20 23:25:13 +00:00
2022-01-07 23:03:39 +00:00
let f_stdr2 = ffmpeg_stderr . as_ref ( ) . map ( Arc ::clone ) ;
2021-11-20 23:25:13 +00:00
2022-01-07 23:03:39 +00:00
tokio ::spawn ( async move {
while let Some ( line ) = source_reader . next_line ( ) . await . unwrap ( ) {
p_stdr2 . lock ( ) . push_str ( & line ) ;
p_stdr2 . lock ( ) . push ( '\n' ) ;
}
} ) ;
if let Some ( mut ffmpeg_reader ) = ffmpeg_reader {
let f_stdr2 = f_stdr2 . unwrap ( ) ;
tokio ::spawn ( async move {
while let Some ( line ) = ffmpeg_reader . next_line ( ) . await . unwrap ( ) {
f_stdr2 . lock ( ) . push_str ( & line ) ;
f_stdr2 . lock ( ) . push ( '\n' ) ;
}
} ) ;
}
2021-09-03 19:01:11 +00:00
2022-01-07 23:03:39 +00:00
let mut enc_pipe = if let [ encoder , args @ .. ] = & * enc_cmd {
tokio ::process ::Command ::new ( encoder )
. args ( args )
. stdin ( y4m_pipe )
. stdout ( Stdio ::piped ( ) )
. stderr ( Stdio ::piped ( ) )
. spawn ( )
. unwrap ( )
} else {
unreachable! ( )
} ;
2021-09-03 19:01:11 +00:00
2022-01-07 23:03:39 +00:00
let mut frame = 0 ;
2021-09-03 19:01:11 +00:00
2022-01-07 23:03:39 +00:00
let mut reader = BufReader ::new ( enc_pipe . stderr . take ( ) . unwrap ( ) ) ;
2021-09-03 19:01:11 +00:00
2022-01-07 23:03:39 +00:00
let mut buf = Vec ::with_capacity ( 128 ) ;
let mut enc_stderr = String ::with_capacity ( 128 ) ;
2021-09-03 19:01:11 +00:00
2022-01-07 23:03:39 +00:00
while let Ok ( read ) = reader . read_until ( b '\r' , & mut buf ) . await {
if read = = 0 {
break ;
2021-09-03 19:01:11 +00:00
}
2022-01-07 23:03:39 +00:00
if let Ok ( line ) = simdutf8 ::basic ::from_utf8_mut ( & mut buf ) {
if self . verbosity = = Verbosity ::Verbose & & ! line . contains ( '\n' ) {
update_mp_msg ( worker_id , ( * line ) . to_string ( ) ) ;
}
// This needs to be done before parse_encoded_frames, as it potentially
// mutates the string
enc_stderr . push_str ( line ) ;
enc_stderr . push ( '\n' ) ;
2022-03-28 15:23:08 +00:00
if let Some ( new ) = encoder . parse_encoded_frames ( line ) {
2022-01-07 23:03:39 +00:00
if new > frame {
if self . verbosity = = Verbosity ::Normal {
inc_bar ( ( new - frame ) as u64 ) ;
} else if self . verbosity = = Verbosity ::Verbose {
inc_mp_bar ( ( new - frame ) as u64 ) ;
}
frame = new ;
2021-09-03 19:01:11 +00:00
}
}
}
2022-01-07 23:03:39 +00:00
buf . clear ( ) ;
}
2021-09-03 19:01:11 +00:00
2022-01-07 23:03:39 +00:00
let enc_output = enc_pipe . wait_with_output ( ) . await . unwrap ( ) ;
2021-09-03 19:01:11 +00:00
2022-01-07 23:03:39 +00:00
let source_pipe_stderr = pipe_stderr . lock ( ) . clone ( ) ;
let ffmpeg_pipe_stderr = ffmpeg_stderr . map ( | x | x . lock ( ) . clone ( ) ) ;
(
source_pipe_stderr ,
ffmpeg_pipe_stderr ,
enc_output ,
enc_stderr ,
frame ,
)
} ) ;
2021-09-03 19:01:11 +00:00
2021-11-11 02:52:32 +00:00
if ! enc_output . status . success ( ) {
2021-11-25 18:22:32 +00:00
return Err ( (
EncoderCrash {
exit_status : enc_output . status ,
2022-01-07 23:03:39 +00:00
source_pipe_stderr : source_pipe_stderr . into ( ) ,
ffmpeg_pipe_stderr : ffmpeg_pipe_stderr . map ( Into ::into ) ,
2021-11-25 18:22:32 +00:00
stderr : enc_stderr . into ( ) ,
stdout : enc_output . stdout . into ( ) ,
} ,
frame ,
) ) ;
2021-09-03 19:01:11 +00:00
}
2022-04-17 03:51:23 +00:00
if current_pass = = passes {
let encoded_frames = num_frames ( chunk . output ( ) . as_ref ( ) ) . unwrap ( ) ;
if encoded_frames ! = chunk . frames {
return Err ( (
EncoderCrash {
exit_status : enc_output . status ,
source_pipe_stderr : source_pipe_stderr . into ( ) ,
ffmpeg_pipe_stderr : ffmpeg_pipe_stderr . map ( Into ::into ) ,
stderr : enc_stderr . into ( ) ,
stdout : format ! (
" FRAME MISMATCH: chunk {}: {}/{} (actual/expected frames) " ,
chunk . index , encoded_frames , chunk . frames
)
. into ( ) ,
} ,
frame ,
) ) ;
}
}
2021-09-03 19:01:11 +00:00
Ok ( ( ) )
}
2021-10-05 17:08:18 +00:00
fn validate_encoder_params ( & self ) {
let video_params : Vec < & str > = self
2021-09-03 19:01:11 +00:00
. video_params
. iter ( )
. filter_map ( | param | {
2021-10-12 21:51:21 +00:00
if param . starts_with ( '-' ) & & [ Encoder ::aom , Encoder ::vpx ] . contains ( & self . encoder ) {
// These encoders require args to be passed using an equal sign,
// e.g. `--cq-level=30`
2021-09-03 19:01:11 +00:00
param . split ( '=' ) . next ( )
} else {
2021-10-12 21:51:21 +00:00
// The other encoders use a space, so we don't need to do extra splitting,
// e.g. `--crf 30`
2021-09-03 19:01:11 +00:00
None
}
} )
. collect ( ) ;
2021-10-05 17:08:18 +00:00
let help_text = {
let [ cmd , arg ] = self . encoder . help_command ( ) ;
String ::from_utf8 ( Command ::new ( cmd ) . arg ( arg ) . output ( ) . unwrap ( ) . stdout ) . unwrap ( )
} ;
2021-12-10 07:23:24 +00:00
let valid_params = valid_params ( & help_text , self . encoder ) ;
2021-10-05 17:08:18 +00:00
let invalid_params = invalid_params ( & video_params , & valid_params ) ;
2021-09-03 19:01:11 +00:00
for wrong_param in & invalid_params {
eprintln! (
" '{}' isn't a valid parameter for {} " ,
wrong_param , self . encoder ,
) ;
if let Some ( suggestion ) = suggest_fix ( wrong_param , & valid_params ) {
eprintln! ( " \t Did you mean ' {} '? " , suggestion ) ;
}
}
if ! invalid_params . is_empty ( ) {
println! ( " \n To continue anyway, run av1an with '--force' " ) ;
2021-10-30 11:49:26 +00:00
exit ( 1 ) ;
2021-09-03 19:01:11 +00:00
}
}
pub fn startup_check ( & mut self ) -> anyhow ::Result < ( ) > {
2021-10-09 11:07:21 +00:00
if self . concat = = ConcatMethod ::Ivf
& & ! matches! (
self . encoder ,
Encoder ::rav1e | Encoder ::aom | Encoder ::svt_av1 | Encoder ::vpx
)
2021-09-03 19:01:11 +00:00
{
bail! ( " .ivf only supports VP8, VP9, and AV1 " ) ;
}
2021-11-29 00:38:01 +00:00
ensure! ( self . max_tries > 0 ) ;
2021-09-03 19:01:11 +00:00
ensure! (
2021-10-05 17:08:18 +00:00
self . input . as_path ( ) . exists ( ) ,
2021-09-03 19:01:11 +00:00
" Input file {:?} does not exist! " ,
self . input
) ;
2021-11-11 02:52:32 +00:00
if self . target_quality . is_some ( ) | | self . vmaf {
validate_libvmaf ( ) ? ;
}
2021-09-03 19:01:11 +00:00
if which ::which ( " ffmpeg " ) . is_err ( ) {
2021-10-04 16:22:08 +00:00
bail! ( " FFmpeg not found. Is it installed in system path? " ) ;
}
2021-10-05 17:08:18 +00:00
if self . concat = = ConcatMethod ::MKVMerge & & which ::which ( " mkvmerge " ) . is_err ( ) {
bail! ( " mkvmerge not found, but `--concat mkvmerge` was specified. Is it installed in system path? " ) ;
2021-09-03 19:01:11 +00:00
}
2021-11-07 12:20:12 +00:00
if self . encoder = = Encoder ::x265 & & self . concat ! = ConcatMethod ::MKVMerge {
bail! ( " mkvmerge is required for concatenating x265, as x265 outputs raw HEVC bitstream files without the timestamps correctly set, which FFmpeg cannot concatenate \
properly into a mkv file . Specify mkvmerge as the concatenation method by setting ` - - concat mkvmerge ` . " );
}
2021-11-23 14:01:43 +00:00
if self . chunk_method = = ChunkMethod ::LSMASH {
ensure! (
is_lsmash_installed ( ) ,
" LSMASH is not installed, but it was specified as the chunk method "
) ;
}
if self . chunk_method = = ChunkMethod ::FFMS2 {
ensure! (
is_ffms2_installed ( ) ,
" FFMS2 is not installed, but it was specified as the chunk method "
) ;
}
2022-01-28 15:16:09 +00:00
if self . chunk_method = = ChunkMethod ::Select {
warn! ( " It is not recommended to use the \" select \" chunk method, as it is very slow " ) ;
}
2021-11-23 14:01:43 +00:00
2021-10-05 17:08:18 +00:00
if let Some ( vmaf_path ) = & self . vmaf_path {
ensure! ( vmaf_path . exists ( ) ) ;
2021-09-03 19:01:11 +00:00
}
if self . probes < 4 {
println! ( " Target quality with less than 4 probes is experimental and not recommended " ) ;
}
let ( min , max ) = self . encoder . get_default_cq_range ( ) ;
match self . min_q {
None = > {
self . min_q = Some ( min as u32 ) ;
}
2021-10-09 11:07:21 +00:00
Some ( min_q ) = > ensure! ( min_q > 1 ) ,
2021-09-03 19:01:11 +00:00
}
if self . max_q . is_none ( ) {
self . max_q = Some ( max as u32 ) ;
}
let encoder_bin = self . encoder . bin ( ) ;
2021-10-08 23:47:08 +00:00
if which ::which ( encoder_bin ) . is_err ( ) {
2021-09-03 19:01:11 +00:00
bail! (
" Encoder {} not found. Is it installed in the system path? " ,
encoder_bin
) ;
}
if self . video_params . is_empty ( ) {
2022-01-03 19:47:31 +00:00
self . video_params = self
. encoder
. get_default_arguments ( self . input . calculate_tiles ( ) ) ;
2021-09-03 19:01:11 +00:00
}
2022-01-01 21:13:03 +00:00
if let Some ( strength ) = self . photon_noise {
if strength > 64 {
bail! ( " Valid strength values for photon noise are 0-64 " ) ;
}
if self . encoder ! = Encoder ::aom {
bail! ( " Photon noise synth is only supported with aomenc " ) ;
}
}
2021-10-01 20:48:40 +00:00
if self . encoder = = Encoder ::aom
& & self . concat ! = ConcatMethod ::MKVMerge
& & self
. video_params
. iter ( )
. any ( | param | param = = " --enable-keyframe-filtering=2 " )
{
bail! (
" keyframe filtering mode 2 currently only works when using mkvmerge as the concat method "
) ;
}
2021-12-22 05:09:25 +00:00
if matches! ( self . encoder , Encoder ::aom | Encoder ::vpx )
& & self . passes ! = 1
& & self . video_params . iter ( ) . any ( | param | param = = " --rt " )
{
// --rt must be used with 1-pass mode
self . passes = 1 ;
}
2021-10-05 17:08:18 +00:00
if ! self . force {
self . validate_encoder_params ( ) ;
2022-01-16 04:28:37 +00:00
self . check_rate_control ( ) ;
2021-10-05 17:08:18 +00:00
}
2021-09-03 19:01:11 +00:00
Ok ( ( ) )
}
2022-01-16 04:28:37 +00:00
/// Warns if rate control was not specified in encoder arguments
fn check_rate_control ( & self ) {
if self . encoder = = Encoder ::aom {
if ! self
. video_params
. iter ( )
. any ( | f | Self ::check_aom_encoder_mode ( f ) )
{
2022-01-18 23:20:01 +00:00
warn! ( " [WARN] --end-usage was not specified " ) ;
2022-01-16 04:28:37 +00:00
}
if ! self . video_params . iter ( ) . any ( | f | Self ::check_aom_rate ( f ) ) {
2022-01-18 23:20:01 +00:00
warn! ( " [WARN] --cq-level or --target-bitrate was not specified " ) ;
2022-01-16 04:28:37 +00:00
}
}
}
fn check_aom_encoder_mode ( s : & str ) -> bool {
const END_USAGE : & str = " --end-usage= " ;
if s . len ( ) < = END_USAGE . len ( ) | | ! s . starts_with ( END_USAGE ) {
return false ;
}
s . as_bytes ( ) [ END_USAGE . len ( ) .. ]
. iter ( )
. all ( | & b | ( b as char ) . is_ascii_alphabetic ( ) )
}
fn check_aom_rate ( s : & str ) -> bool {
const CQ_LEVEL : & str = " --cq-level= " ;
const TARGET_BITRATE : & str = " --target-bitrate= " ;
if s . len ( ) < = CQ_LEVEL . len ( ) | | ! ( s . starts_with ( TARGET_BITRATE ) | | s . starts_with ( CQ_LEVEL ) ) {
return false ;
}
if s . starts_with ( CQ_LEVEL ) {
s . as_bytes ( ) [ CQ_LEVEL . len ( ) .. ]
. iter ( )
. all ( | & b | ( b as char ) . is_ascii_digit ( ) )
} else {
s . as_bytes ( ) [ TARGET_BITRATE . len ( ) .. ]
. iter ( )
. all ( | & b | ( b as char ) . is_ascii_digit ( ) )
}
}
2022-03-28 15:23:08 +00:00
fn create_encoding_queue ( & mut self , scenes : & [ Scene ] ) -> anyhow ::Result < Vec < Chunk > > {
2021-12-25 07:34:06 +00:00
let mut chunks = match & self . input {
Input ::Video ( _ ) = > match self . chunk_method {
ChunkMethod ::FFMS2 | ChunkMethod ::LSMASH = > {
let vs_script = self . vs_script . as_ref ( ) . unwrap ( ) . as_path ( ) ;
2022-03-28 15:23:08 +00:00
self . create_video_queue_vs ( scenes , vs_script )
2021-12-25 07:34:06 +00:00
}
2022-03-28 15:23:08 +00:00
ChunkMethod ::Hybrid = > self . create_video_queue_hybrid ( scenes ) ? ,
ChunkMethod ::Select = > self . create_video_queue_select ( scenes ) ,
ChunkMethod ::Segment = > self . create_video_queue_segment ( scenes ) ? ,
2021-12-25 07:34:06 +00:00
} ,
2022-03-28 15:23:08 +00:00
Input ::VapourSynth ( vs_script ) = > self . create_video_queue_vs ( scenes , vs_script . as_path ( ) ) ,
2021-12-29 20:30:10 +00:00
} ;
2021-09-03 19:01:11 +00:00
2022-01-02 05:49:47 +00:00
match self . chunk_order {
ChunkOrdering ::LongestFirst = > {
chunks . sort_unstable_by_key ( | chunk | Reverse ( chunk . frames ) ) ;
}
ChunkOrdering ::ShortestFirst = > {
chunks . sort_unstable_by_key ( | chunk | chunk . frames ) ;
}
ChunkOrdering ::Sequential = > {
// Already in order
}
ChunkOrdering ::Random = > {
chunks . shuffle ( & mut thread_rng ( ) ) ;
}
}
2021-09-03 19:01:11 +00:00
2021-11-19 00:57:10 +00:00
Ok ( chunks )
2021-09-03 19:01:11 +00:00
}
2022-03-28 15:23:08 +00:00
fn calc_split_locations ( & self ) -> anyhow ::Result < ( Vec < Scene > , usize ) > {
2022-04-03 20:23:47 +00:00
let zones = self . parse_zones ( ) ? ;
2021-12-25 07:34:06 +00:00
Ok ( match self . split_method {
2021-09-03 19:01:11 +00:00
SplitMethod ::AvScenechange = > av_scenechange_detect (
& self . input ,
2021-10-30 11:49:26 +00:00
self . encoder ,
2021-09-03 19:01:11 +00:00
self . frames ,
self . min_scene_len ,
self . verbosity ,
2021-12-09 17:48:37 +00:00
self . sc_pix_format ,
2021-09-14 21:04:15 +00:00
self . sc_method ,
self . sc_downscale_height ,
2022-04-03 20:23:47 +00:00
& zones ,
2021-12-25 07:34:06 +00:00
) ? ,
2022-04-03 20:23:47 +00:00
SplitMethod ::None = > {
let mut scenes = Vec ::with_capacity ( 2 * zones . len ( ) + 1 ) ;
let mut frames_processed = 0 ;
for zone in zones {
let end_frame = zone . end_frame ;
if end_frame > frames_processed {
scenes . push ( Scene {
start_frame : frames_processed ,
end_frame : zone . start_frame ,
zone_overrides : None ,
} ) ;
}
scenes . push ( zone ) ;
frames_processed + = end_frame ;
}
if self . frames > frames_processed {
scenes . push ( Scene {
start_frame : frames_processed ,
end_frame : self . frames ,
zone_overrides : None ,
} ) ;
}
( scenes , self . input . frames ( ) ? )
}
2021-12-25 07:34:06 +00:00
} )
2021-09-03 19:01:11 +00:00
}
2022-03-28 15:23:08 +00:00
fn parse_zones ( & self ) -> anyhow ::Result < Vec < Scene > > {
let mut zones = Vec ::new ( ) ;
if let Some ( ref zones_file ) = self . zones {
let input = fs ::read_to_string ( & zones_file ) ? ;
for zone_line in input . lines ( ) . map ( str ::trim ) . filter ( | line | ! line . is_empty ( ) ) {
zones . push ( Scene ::parse_from_zone ( zone_line , self ) ? ) ;
}
zones . sort_unstable_by_key ( | zone | zone . start_frame ) ;
let mut segments = BTreeSet ::new ( ) ;
for zone in & zones {
if segments . contains ( & zone . start_frame ) {
bail! ( " Zones file contains overlapping zones " ) ;
}
segments . extend ( zone . start_frame .. zone . end_frame ) ;
}
}
Ok ( zones )
}
2021-09-03 19:01:11 +00:00
// If we are not resuming, then do scene detection. Otherwise: get scenes from
// scenes.json and return that.
2022-03-28 15:23:08 +00:00
fn split_routine ( & mut self ) -> anyhow ::Result < Vec < Scene > > {
2021-12-29 20:30:10 +00:00
let scene_file = self . scenes . as_ref ( ) . map_or_else (
| | Cow ::Owned ( Path ::new ( & self . temp ) . join ( " scenes.json " ) ) ,
| path | Cow ::Borrowed ( path . as_path ( ) ) ,
) ;
2021-09-03 19:01:11 +00:00
2022-01-18 23:20:01 +00:00
let used_existing_cuts ;
2021-12-25 07:34:06 +00:00
let ( mut scenes , frames ) = if ( self . scenes . is_some ( ) & & scene_file . exists ( ) ) | | self . resume {
2022-01-18 23:20:01 +00:00
used_existing_cuts = true ;
2021-12-25 07:34:06 +00:00
crate ::split ::read_scenes_from_file ( scene_file . as_ref ( ) ) ?
2021-09-03 19:01:11 +00:00
} else {
2022-01-18 23:20:01 +00:00
used_existing_cuts = false ;
2022-03-28 15:23:08 +00:00
self . frames = self . input . frames ( ) ? ;
2021-12-10 07:23:24 +00:00
self . calc_split_locations ( ) ?
2021-09-03 19:01:11 +00:00
} ;
2021-12-25 07:34:06 +00:00
self . frames = frames ;
get_done ( )
. frames
. store ( self . frames , atomic ::Ordering ::SeqCst ) ;
2022-04-03 20:23:47 +00:00
let scenes_before = scenes . len ( ) ;
2022-01-18 23:20:01 +00:00
if ! used_existing_cuts {
if let Some ( split_len ) = self . extra_splits_len {
scenes = extra_splits ( & scenes , self . frames , split_len ) ;
2022-04-03 20:23:47 +00:00
let scenes_after = scenes . len ( ) ;
2022-01-18 23:20:01 +00:00
info! (
" scenecut: found {} scene(s) [with extra_splits ({} frames): {} scene(s)] " ,
scenes_before , split_len , scenes_after
) ;
} else {
info! ( " scenecut: found {} scene(s) " , scenes_before ) ;
}
2021-09-07 18:42:00 +00:00
}
2021-09-03 19:01:11 +00:00
2021-11-19 00:57:10 +00:00
write_scenes_to_file ( & scenes , self . frames , scene_file ) ? ;
2021-09-03 19:01:11 +00:00
2021-11-19 00:57:10 +00:00
Ok ( scenes )
2021-09-03 19:01:11 +00:00
}
fn create_select_chunk (
& self ,
index : usize ,
2021-10-04 16:22:08 +00:00
src_path : & Path ,
2021-09-03 19:01:11 +00:00
frame_start : usize ,
mut frame_end : usize ,
2022-03-28 15:23:08 +00:00
overrides : Option < ZoneOptions > ,
2021-09-03 19:01:11 +00:00
) -> Chunk {
assert! (
2021-10-09 11:07:21 +00:00
frame_start < frame_end ,
2021-09-03 19:01:11 +00:00
" Can't make a chunk with <= 0 frames! "
) ;
let frames = frame_end - frame_start ;
frame_end - = 1 ;
2021-10-04 16:22:08 +00:00
let ffmpeg_gen_cmd : Vec < OsString > = into_vec! [
2021-09-03 19:01:11 +00:00
" ffmpeg " ,
" -y " ,
" -hide_banner " ,
" -loglevel " ,
" error " ,
" -i " ,
2021-10-04 16:22:08 +00:00
src_path ,
2021-09-03 19:01:11 +00:00
" -vf " ,
format! (
" select=between(n \\ ,{} \\ ,{}),setpts=PTS-STARTPTS " ,
frame_start , frame_end
) ,
" -pix_fmt " ,
2021-11-20 18:43:13 +00:00
self . output_pix_format . format . descriptor ( ) . unwrap ( ) . name ( ) ,
2021-09-03 19:01:11 +00:00
" -strict " ,
" -1 " ,
" -f " ,
" yuv4mpegpipe " ,
" - " ,
] ;
2021-10-04 16:22:08 +00:00
let output_ext = self . encoder . output_extension ( ) ;
2021-09-03 19:01:11 +00:00
Chunk {
temp : self . temp . clone ( ) ,
index ,
2021-10-04 16:22:08 +00:00
source : ffmpeg_gen_cmd ,
output_ext : output_ext . to_owned ( ) ,
2021-09-03 19:01:11 +00:00
frames ,
2022-03-28 15:23:08 +00:00
overrides ,
2021-09-03 19:01:11 +00:00
.. Chunk ::default ( )
}
}
2022-03-28 15:23:08 +00:00
fn create_vs_chunk ( & self , index : usize , vs_script : & Path , scene : & Scene ) -> Chunk {
let frames = scene . end_frame - scene . start_frame ;
2021-09-03 19:01:11 +00:00
// the frame end boundary is actually a frame that should be included in the next chunk
2022-03-28 15:23:08 +00:00
let frame_end = scene . end_frame - 1 ;
2021-09-03 19:01:11 +00:00
2021-10-04 16:22:08 +00:00
let vspipe_cmd_gen : Vec < OsString > = into_vec! [
" vspipe " ,
2021-09-03 19:01:11 +00:00
vs_script ,
2021-10-04 16:22:08 +00:00
" -y " ,
" - " ,
" -s " ,
2022-03-28 15:23:08 +00:00
format! ( " {} " , scene . start_frame ) ,
2021-10-04 16:22:08 +00:00
" -e " ,
2021-12-02 03:00:00 +00:00
format! ( " {} " , frame_end ) ,
2021-09-03 19:01:11 +00:00
] ;
let output_ext = self . encoder . output_extension ( ) ;
Chunk {
temp : self . temp . clone ( ) ,
index ,
2021-10-04 16:22:08 +00:00
source : vspipe_cmd_gen ,
2021-09-03 19:01:11 +00:00
output_ext : output_ext . to_owned ( ) ,
frames ,
2022-03-28 15:23:08 +00:00
overrides : scene . zone_overrides . clone ( ) ,
2021-09-03 19:01:11 +00:00
.. Chunk ::default ( )
}
}
2022-03-28 15:23:08 +00:00
fn create_video_queue_vs ( & self , scenes : & [ Scene ] , vs_script : & Path ) -> Vec < Chunk > {
let chunk_queue : Vec < Chunk > = scenes
. iter ( )
2021-09-03 19:01:11 +00:00
. enumerate ( )
2022-03-28 15:23:08 +00:00
. map ( | ( index , scene ) | self . create_vs_chunk ( index , vs_script , scene ) )
2021-09-03 19:01:11 +00:00
. collect ( ) ;
2021-12-29 20:30:10 +00:00
chunk_queue
2021-09-03 19:01:11 +00:00
}
2022-03-28 15:23:08 +00:00
fn create_video_queue_select ( & self , scenes : & [ Scene ] ) -> Vec < Chunk > {
2021-10-04 16:22:08 +00:00
let input = self . input . as_video_path ( ) ;
2022-03-28 15:23:08 +00:00
let chunk_queue : Vec < Chunk > = scenes
. iter ( )
2021-09-03 19:01:11 +00:00
. enumerate ( )
2022-03-28 15:23:08 +00:00
. map ( | ( index , scene ) | {
self . create_select_chunk (
index ,
input ,
scene . start_frame ,
scene . end_frame ,
scene . zone_overrides . clone ( ) ,
)
2021-09-03 19:01:11 +00:00
} )
. collect ( ) ;
chunk_queue
}
2022-03-28 15:23:08 +00:00
fn create_video_queue_segment ( & self , scenes : & [ Scene ] ) -> anyhow ::Result < Vec < Chunk > > {
2021-10-04 16:22:08 +00:00
let input = self . input . as_video_path ( ) ;
2022-01-18 23:20:01 +00:00
debug! ( " Splitting video " ) ;
2022-03-28 15:23:08 +00:00
segment (
input ,
& self . temp ,
& scenes
. iter ( )
. skip ( 1 )
. map ( | scene | scene . start_frame )
. collect ::< Vec < usize > > ( ) ,
) ;
2022-01-18 23:20:01 +00:00
debug! ( " Splitting done " ) ;
2021-09-03 19:01:11 +00:00
let source_path = Path ::new ( & self . temp ) . join ( " split " ) ;
2021-11-19 00:57:10 +00:00
let queue_files = Self ::read_queue_files ( & source_path ) ? ;
2021-09-03 19:01:11 +00:00
assert! (
! queue_files . is_empty ( ) ,
" Error: No files found in temp/split, probably splitting not working "
) ;
let chunk_queue : Vec < Chunk > = queue_files
. iter ( )
. enumerate ( )
2022-03-28 15:23:08 +00:00
. map ( | ( index , file ) | {
self . create_chunk_from_segment (
index ,
file . as_path ( ) . to_str ( ) . unwrap ( ) ,
scenes [ index ] . zone_overrides . clone ( ) ,
)
} )
2021-09-03 19:01:11 +00:00
. collect ( ) ;
2021-11-19 00:57:10 +00:00
Ok ( chunk_queue )
2021-09-03 19:01:11 +00:00
}
2022-03-28 15:23:08 +00:00
fn create_video_queue_hybrid ( & self , scenes : & [ Scene ] ) -> anyhow ::Result < Vec < Chunk > > {
2021-10-04 16:22:08 +00:00
let input = self . input . as_video_path ( ) ;
2022-02-22 20:17:37 +00:00
let keyframes = crate ::ffmpeg ::get_keyframes ( input ) . unwrap ( ) ;
2021-10-04 16:22:08 +00:00
2021-09-03 19:01:11 +00:00
let to_split : Vec < usize > = keyframes
. iter ( )
2022-03-28 15:23:08 +00:00
. filter ( | kf | scenes . iter ( ) . any ( | scene | scene . start_frame = = * * kf ) )
2021-09-03 19:01:11 +00:00
. copied ( )
. collect ( ) ;
2022-01-18 23:20:01 +00:00
debug! ( " Segmenting video " ) ;
2021-10-04 16:22:08 +00:00
segment ( input , & self . temp , & to_split [ 1 .. ] ) ;
2022-01-18 23:20:01 +00:00
debug! ( " Segment done " ) ;
2021-09-03 19:01:11 +00:00
let source_path = Path ::new ( & self . temp ) . join ( " split " ) ;
2021-11-19 00:57:10 +00:00
let queue_files = Self ::read_queue_files ( & source_path ) ? ;
2021-09-03 19:01:11 +00:00
2021-10-09 11:07:21 +00:00
let kf_list = to_split
2021-09-03 19:01:11 +00:00
. iter ( )
2021-10-09 11:07:21 +00:00
. copied ( )
. chain ( iter ::once ( self . frames ) )
. tuple_windows ( ) ;
2021-09-03 19:01:11 +00:00
2022-03-28 15:23:08 +00:00
let mut segments = Vec ::with_capacity ( scenes . len ( ) ) ;
2021-10-09 11:07:21 +00:00
for ( file , ( x , y ) ) in queue_files . iter ( ) . zip ( kf_list ) {
2022-03-28 15:23:08 +00:00
for s in scenes {
let s0 = s . start_frame ;
let s1 = s . end_frame ;
2021-10-09 11:07:21 +00:00
if s0 > = x & & s1 < = y & & s0 < s1 {
2022-03-28 15:23:08 +00:00
segments . push ( ( file . as_path ( ) , ( s0 - x , s1 - x , s ) ) ) ;
2021-09-03 19:01:11 +00:00
}
}
}
let chunk_queue : Vec < Chunk > = segments
. iter ( )
. enumerate ( )
2022-03-28 15:23:08 +00:00
. map ( | ( index , & ( file , ( start , end , scene ) ) ) | {
self . create_select_chunk ( index , file , start , end , scene . zone_overrides . clone ( ) )
} )
2021-09-03 19:01:11 +00:00
. collect ( ) ;
2021-11-19 00:57:10 +00:00
Ok ( chunk_queue )
2021-09-03 19:01:11 +00:00
}
2022-03-28 15:23:08 +00:00
fn create_chunk_from_segment (
& self ,
index : usize ,
file : & str ,
overrides : Option < ZoneOptions > ,
) -> Chunk {
2021-10-04 16:22:08 +00:00
let ffmpeg_gen_cmd : Vec < OsString > = into_vec! [
" ffmpeg " ,
" -y " ,
" -hide_banner " ,
" -loglevel " ,
" error " ,
" -i " ,
2021-09-03 19:01:11 +00:00
file . to_owned ( ) ,
2021-10-04 16:22:08 +00:00
" -strict " ,
" -1 " ,
" -pix_fmt " ,
2021-11-20 18:43:13 +00:00
self . output_pix_format . format . descriptor ( ) . unwrap ( ) . name ( ) ,
2021-10-04 16:22:08 +00:00
" -f " ,
" yuv4mpegpipe " ,
" - " ,
2021-09-03 19:01:11 +00:00
] ;
2021-10-04 16:22:08 +00:00
let output_ext = self . encoder . output_extension ( ) ;
2021-09-03 19:01:11 +00:00
Chunk {
temp : self . temp . clone ( ) ,
2021-10-30 11:49:26 +00:00
frames : self . frames ,
2021-10-04 16:22:08 +00:00
source : ffmpeg_gen_cmd ,
output_ext : output_ext . to_owned ( ) ,
2021-09-03 19:01:11 +00:00
index ,
2022-03-28 15:23:08 +00:00
overrides ,
2021-09-03 19:01:11 +00:00
.. Chunk ::default ( )
}
}
2022-01-03 21:25:39 +00:00
/// Returns unfinished chunks and number of total chunks
2022-03-28 15:23:08 +00:00
fn load_or_gen_chunk_queue ( & mut self , splits : & [ Scene ] ) -> anyhow ::Result < ( Vec < Chunk > , usize ) > {
2021-09-03 19:01:11 +00:00
if self . resume {
2021-11-17 08:57:08 +00:00
let mut chunks = read_chunk_queue ( self . temp . as_ref ( ) ) ? ;
2022-01-11 06:17:46 +00:00
let num_chunks = chunks . len ( ) ;
2021-09-03 19:01:11 +00:00
let done = get_done ( ) ;
// only keep the chunks that are not done
chunks . retain ( | chunk | ! done . done . contains_key ( & chunk . name ( ) ) ) ;
2022-01-03 21:25:39 +00:00
Ok ( ( chunks , num_chunks ) )
2021-09-03 19:01:11 +00:00
} else {
2021-11-19 00:57:10 +00:00
let chunks = self . create_encoding_queue ( splits ) ? ;
2022-01-03 21:25:39 +00:00
let num_chunks = chunks . len ( ) ;
2022-01-11 06:17:46 +00:00
save_chunk_queue ( & self . temp , & chunks ) ? ;
2022-01-03 21:25:39 +00:00
Ok ( ( chunks , num_chunks ) )
2021-09-03 19:01:11 +00:00
}
}
2021-10-09 11:07:21 +00:00
pub fn encode_file ( & mut self ) -> anyhow ::Result < ( ) > {
2022-01-02 19:28:01 +00:00
let initial_frames = get_done ( )
. done
. iter ( )
. map ( | ref_multi | ref_multi . frames )
. sum ::< usize > ( ) ;
2021-09-03 19:01:11 +00:00
2021-12-25 07:34:06 +00:00
let vspipe_cache =
// Technically we should check if the vapoursynth cache file exists rather than !self.resume,
// but the code still works if we are resuming and the cache file doesn't exist (as it gets
// generated when vspipe is first called), so it's not worth adding all the extra complexity.
if ( self . input . is_vapoursynth ( )
| | ( self . input . is_video ( )
& & matches! ( self . chunk_method , ChunkMethod ::LSMASH | ChunkMethod ::FFMS2 ) ) )
& & ! self . resume
{
self . vs_script = Some ( match & self . input {
Input ::VapourSynth ( path ) = > path . clone ( ) ,
2021-12-29 20:30:10 +00:00
Input ::Video ( path ) = > create_vs_file ( & self . temp , path , self . chunk_method ) ? ,
2021-12-25 07:34:06 +00:00
} ) ;
let vs_script = self . vs_script . clone ( ) . unwrap ( ) ;
Some ( {
thread ::spawn ( move | | {
Command ::new ( " vspipe " )
. arg ( " -i " )
. arg ( vs_script )
. args ( [ " -i " , " - " ] )
. stdout ( Stdio ::piped ( ) )
. stderr ( Stdio ::piped ( ) )
. spawn ( )
. unwrap ( )
. wait ( )
. unwrap ( )
} )
} )
} else {
None
} ;
2022-05-11 17:52:24 +00:00
let res = self . input . resolution ( ) ? ;
let fps = self . input . frame_rate ( ) ? ;
let format = self . input . pixel_format ( ) ? ;
2022-05-16 18:26:14 +00:00
let tfc = self
. input
. transfer_function_params_adjusted ( & self . video_params ) ? ;
2022-05-11 17:52:24 +00:00
info! (
" Input: {}x{} @ {:.3} fps, {}, {} " ,
res . 0 ,
res . 1 ,
fps ,
format ,
match tfc {
TransferFunction ::SMPTE2084 = > " HDR " ,
TransferFunction ::BT1886 = > " SDR " ,
}
) ;
2021-12-02 00:27:08 +00:00
let splits = self . split_routine ( ) ? ;
2022-03-02 05:33:31 +00:00
if self . sc_only {
debug! ( " scene detection only " ) ;
if let Err ( e ) = fs ::remove_dir_all ( & self . temp ) {
warn! ( " Failed to delete temp directory: {} " , e ) ;
}
exit ( 0 ) ;
}
2022-03-28 15:23:08 +00:00
let ( mut chunk_queue , total_chunks ) = self . load_or_gen_chunk_queue ( & splits ) ? ;
2021-11-17 08:57:08 +00:00
if self . resume {
let chunks_done = get_done ( ) . done . len ( ) ;
info! (
" encoding resumed with {}/{} chunks completed ({} remaining) " ,
chunks_done ,
chunk_queue . len ( ) + chunks_done ,
chunk_queue . len ( )
) ;
}
2021-09-03 19:01:11 +00:00
2021-12-25 07:34:06 +00:00
if let Some ( vspipe_cache ) = vspipe_cache {
vspipe_cache . join ( ) . unwrap ( ) ;
}
2022-03-28 15:23:08 +00:00
let mut grain_table = None ;
2022-01-01 21:13:03 +00:00
if let Some ( strength ) = self . photon_noise {
2022-03-28 15:23:08 +00:00
let table = Path ::new ( & self . temp ) . join ( " grain.tbl " ) ;
if ! table . exists ( ) {
2022-01-01 21:13:03 +00:00
debug! (
" Generating grain table at ISO {} " ,
u32 ::from ( strength ) * 100
) ;
let ( width , height ) = self . input . resolution ( ) ? ;
2022-05-16 18:26:14 +00:00
let transfer = self
. input
. transfer_function_params_adjusted ( & self . video_params ) ? ;
2022-03-28 15:23:08 +00:00
create_film_grain_file ( & table , strength , width , height , transfer ) ? ;
2022-01-01 21:13:03 +00:00
} else {
debug! ( " Using existing grain table " ) ;
}
// We should not use a grain table together with aom's grain generation
self
. video_params
. retain ( | param | ! param . starts_with ( " --denoise-noise-level= " ) ) ;
2022-03-28 15:23:08 +00:00
self
. video_params
. push ( format! ( " --film-grain-table= {} " , table . to_str ( ) . unwrap ( ) ) ) ;
grain_table = Some ( table ) ;
}
for chunk in & mut chunk_queue {
// Also apply grain tables to zone overrides
if let Some ( strength ) = chunk . overrides . as_ref ( ) . and_then ( | ovr | ovr . photon_noise ) {
let grain_table = if Some ( strength ) = = self . photon_noise {
// We can reuse the existing photon noise table from the main encode
grain_table . clone ( ) . unwrap ( )
} else {
let grain_table = Path ::new ( & self . temp ) . join ( & format! ( " chunk {} -grain.tbl " , chunk . index ) ) ;
debug! (
" Generating grain table at ISO {} " ,
u32 ::from ( strength ) * 100
) ;
let ( width , height ) = self . input . resolution ( ) ? ;
2022-05-16 18:26:14 +00:00
let transfer = self
. input
. transfer_function_params_adjusted ( & self . video_params ) ? ;
2022-03-28 15:23:08 +00:00
create_film_grain_file ( & grain_table , strength , width , height , transfer ) ? ;
grain_table
} ;
// We should not use a grain table together with aom's grain generation
let overrides = chunk . overrides . as_mut ( ) . unwrap ( ) ;
overrides
. video_params
. retain ( | param | ! param . starts_with ( " --denoise-noise-level= " ) ) ;
overrides . video_params . push ( format! (
" --film-grain-table={} " ,
grain_table . to_str ( ) . unwrap ( )
) ) ;
}
2022-01-01 21:13:03 +00:00
}
2021-10-09 11:07:21 +00:00
crossbeam_utils ::thread ::scope ( | s | -> anyhow ::Result < ( ) > {
2021-10-04 16:22:08 +00:00
// vapoursynth audio is currently unsupported
2021-12-09 17:47:43 +00:00
let audio_size_bytes = Arc ::new ( AtomicU64 ::new ( 0 ) ) ;
2021-10-04 16:22:08 +00:00
let audio_thread = if self . input . is_video ( )
& & ( ! self . resume | | ! get_done ( ) . audio_done . load ( atomic ::Ordering ::SeqCst ) )
{
let input = self . input . as_video_path ( ) ;
2021-09-03 19:01:11 +00:00
let temp = self . temp . as_str ( ) ;
let audio_params = self . audio_params . as_slice ( ) ;
2021-12-09 17:47:43 +00:00
let audio_size_ref = Arc ::clone ( & audio_size_bytes ) ;
2021-09-03 19:01:11 +00:00
Some ( s . spawn ( move | _ | {
2022-02-22 20:17:37 +00:00
let audio_output = crate ::ffmpeg ::encode_audio ( input , temp , audio_params ) ;
2021-09-03 19:01:11 +00:00
get_done ( ) . audio_done . store ( true , atomic ::Ordering ::SeqCst ) ;
let progress_file = Path ::new ( temp ) . join ( " done.json " ) ;
let mut progress_file = File ::create ( & progress_file ) . unwrap ( ) ;
progress_file
. write_all ( serde_json ::to_string ( get_done ( ) ) . unwrap ( ) . as_bytes ( ) )
. unwrap ( ) ;
2021-12-09 17:47:43 +00:00
if let Some ( ref audio_output ) = audio_output {
audio_size_ref . store (
audio_output . metadata ( ) . unwrap ( ) . len ( ) ,
atomic ::Ordering ::SeqCst ,
) ;
}
audio_output . is_some ( )
2021-09-03 19:01:11 +00:00
} ) )
} else {
None
} ;
if self . workers = = 0 {
self . workers = determine_workers ( self . encoder ) as usize ;
}
self . workers = cmp ::min ( self . workers , chunk_queue . len ( ) ) ;
2021-11-18 05:01:46 +00:00
2022-01-18 23:20:01 +00:00
if atty ::is ( atty ::Stream ::Stderr ) {
eprintln! (
" {}{} {} {}{} {} {}{} {} \n {}: {} " ,
Color ::Green . bold ( ) . paint ( " Q " ) ,
Color ::Green . paint ( " ueue " ) ,
Color ::Green . bold ( ) . paint ( format! ( " {} " , chunk_queue . len ( ) ) ) ,
Color ::Blue . bold ( ) . paint ( " W " ) ,
Color ::Blue . paint ( " orkers " ) ,
Color ::Blue . bold ( ) . paint ( format! ( " {} " , self . workers ) ) ,
Color ::Purple . bold ( ) . paint ( " P " ) ,
Color ::Purple . paint ( " asses " ) ,
Color ::Purple . bold ( ) . paint ( format! ( " {} " , self . passes ) ) ,
Style ::default ( ) . bold ( ) . paint ( " Params " ) ,
Style ::default ( ) . dimmed ( ) . paint ( self . video_params . join ( " " ) )
) ;
} else {
eprintln! (
" Queue {} Workers {} Passes {} \n Params: {} " ,
chunk_queue . len ( ) ,
self . workers ,
self . passes ,
self . video_params . join ( " " )
) ;
}
2021-09-03 19:01:11 +00:00
if self . verbosity = = Verbosity ::Normal {
2021-11-25 18:47:42 +00:00
init_progress_bar ( self . frames as u64 ) ;
2021-11-25 19:26:11 +00:00
reset_bar_at ( initial_frames as u64 ) ;
2021-09-03 19:01:11 +00:00
} else if self . verbosity = = Verbosity ::Verbose {
2022-01-19 01:42:41 +00:00
init_multi_progress_bar ( self . frames as u64 , self . workers , total_chunks ) ;
2021-11-25 19:26:11 +00:00
reset_mp_bar_at ( initial_frames as u64 ) ;
2021-09-03 19:01:11 +00:00
}
2021-12-09 17:47:43 +00:00
if ! get_done ( ) . done . is_empty ( ) {
2022-01-02 19:28:01 +00:00
let frame_rate = self . input . frame_rate ( ) ? ;
2021-12-09 17:47:43 +00:00
update_progress_bar_estimates (
frame_rate ,
self . frames ,
self . verbosity ,
audio_size_bytes . load ( atomic ::Ordering ::SeqCst ) ,
) ;
}
2021-09-03 19:01:11 +00:00
let broker = Broker {
chunk_queue ,
2022-01-19 01:42:41 +00:00
total_chunks ,
2021-09-03 19:01:11 +00:00
project : self ,
target_quality : if self . target_quality . is_some ( ) {
Some ( TargetQuality ::new ( self ) )
} else {
None
} ,
2021-11-29 00:38:01 +00:00
max_tries : self . max_tries ,
2021-09-03 19:01:11 +00:00
} ;
2021-12-09 17:47:43 +00:00
let audio_size_ref = Arc ::clone ( & audio_size_bytes ) ;
2021-09-03 19:01:11 +00:00
let ( tx , rx ) = mpsc ::channel ( ) ;
let handle = s . spawn ( | _ | {
2021-12-09 17:47:43 +00:00
broker . encoding_loop ( tx , self . set_thread_affinity , audio_size_ref ) ;
2021-09-03 19:01:11 +00:00
} ) ;
// Queue::encoding_loop only sends a message if there was an error (meaning a chunk crashed)
// more than MAX_TRIES. So, we have to explicitly exit the program if that happens.
2021-12-09 15:33:48 +00:00
while rx . recv ( ) . is_ok ( ) {
2021-10-30 11:49:26 +00:00
exit ( 1 ) ;
2021-09-03 19:01:11 +00:00
}
handle . join ( ) . unwrap ( ) ;
if self . verbosity = = Verbosity ::Normal {
finish_progress_bar ( ) ;
} else if self . verbosity = = Verbosity ::Verbose {
finish_multi_progress_bar ( ) ;
}
// TODO add explicit parameter to concatenation functions to control whether audio is also muxed in
let _audio_output_exists =
audio_thread . map_or ( false , | audio_thread | audio_thread . join ( ) . unwrap ( ) ) ;
2022-01-18 23:20:01 +00:00
debug! ( " encoding finished, concatenating with {} " , self . concat ) ;
2021-09-03 19:01:11 +00:00
2021-10-04 16:22:08 +00:00
match self . concat {
2021-09-03 19:01:11 +00:00
ConcatMethod ::Ivf = > {
2021-10-04 16:22:08 +00:00
concat ::ivf (
& Path ::new ( & self . temp ) . join ( " encode " ) ,
self . output_file . as_ref ( ) ,
2021-10-09 11:07:21 +00:00
) ? ;
2021-09-03 19:01:11 +00:00
}
ConcatMethod ::MKVMerge = > {
2022-01-01 00:04:08 +00:00
concat ::mkvmerge (
self . temp . as_ref ( ) ,
self . output_file . as_ref ( ) ,
self . encoder ,
2022-01-03 21:25:39 +00:00
total_chunks ,
2022-01-01 00:04:08 +00:00
) ? ;
2021-09-03 19:01:11 +00:00
}
ConcatMethod ::FFmpeg = > {
2021-11-19 01:31:19 +00:00
concat ::ffmpeg ( self . temp . as_ref ( ) , self . output_file . as_ref ( ) ) ? ;
2021-09-03 19:01:11 +00:00
}
}
2021-10-04 16:22:08 +00:00
if self . vmaf {
2021-11-11 02:52:32 +00:00
if let Err ( e ) = vmaf ::plot (
2021-10-04 16:22:08 +00:00
self . output_file . as_ref ( ) ,
& self . input ,
self . vmaf_path . as_deref ( ) ,
2021-10-05 17:08:18 +00:00
self . vmaf_res . as_str ( ) ,
2021-10-04 16:22:08 +00:00
1 ,
2021-10-09 11:07:21 +00:00
match self . vmaf_filter . as_deref ( ) {
Some ( filter ) = > Some ( filter ) ,
2021-10-04 16:22:08 +00:00
None = > None ,
} ,
2022-05-21 08:55:43 +00:00
self . vmaf_threads . unwrap_or_else ( | | {
available_parallelism ( )
. expect ( " Unrecoverable: Failed to get thread count " )
. get ( )
} ) ,
2021-11-11 02:52:32 +00:00
) {
2022-01-18 23:20:01 +00:00
error! ( " VMAF calculation failed with error: {} " , e ) ;
2021-11-11 02:52:32 +00:00
}
2021-09-03 19:01:11 +00:00
}
2021-10-04 16:22:08 +00:00
if ! Path ::new ( & self . output_file ) . exists ( ) {
2021-09-03 19:01:11 +00:00
warn! (
2021-10-04 16:22:08 +00:00
" Concatenation failed for unknown reasons! Temp folder will not be deleted: {} " ,
& self . temp
2021-09-03 19:01:11 +00:00
) ;
2021-10-04 16:22:08 +00:00
} else if ! self . keep {
if let Err ( e ) = fs ::remove_dir_all ( & self . temp ) {
2021-09-03 19:01:11 +00:00
warn! ( " Failed to delete temp directory: {} " , e ) ;
}
}
2021-10-09 11:07:21 +00:00
Ok ( ( ) )
2021-09-03 19:01:11 +00:00
} )
2021-10-09 11:07:21 +00:00
. unwrap ( ) ? ;
Ok ( ( ) )
2021-09-03 19:01:11 +00:00
}
}
2022-03-28 15:23:08 +00:00
#[ must_use ]
pub ( crate ) fn invalid_params < ' a > (
params : & ' a [ & ' a str ] ,
valid_options : & ' a HashSet < Cow < ' a , str > > ,
) -> Vec < & ' a str > {
params
. iter ( )
. filter ( | param | ! valid_options . contains ( Borrow ::< str > ::borrow ( & * * param ) ) )
. copied ( )
. collect ( )
}
#[ must_use ]
pub ( crate ) fn suggest_fix < ' a > (
wrong_arg : & str ,
arg_dictionary : & ' a HashSet < Cow < ' a , str > > ,
) -> Option < & ' a str > {
// Minimum threshold to consider a suggestion similar enough that it could be a typo
const MIN_THRESHOLD : f64 = 0.75 ;
arg_dictionary
. iter ( )
. map ( | arg | ( arg , strsim ::jaro_winkler ( arg , wrong_arg ) ) )
. max_by ( | ( _ , a ) , ( _ , b ) | a . partial_cmp ( b ) . unwrap_or ( Ordering ::Less ) )
. and_then ( | ( suggestion , score ) | {
if score > MIN_THRESHOLD {
Some ( suggestion . borrow ( ) )
} else {
None
}
} )
}