VFR aware Y4M decoder and encoders

cli/encode-server/encode.go

@@ -52,7 +52,13 @@ func encodeFromReader(reader io.ReadCloser, job *Job, w http.ResponseWriter) {
 	job.Status.Read.Store(0)
 	job.Status.Processed.Store(0)
 
-	decoder, err := y4m.NewDecoder(reader, nil)
+	settings := make(map[string]any)
+
+	if len(job.Config.Timecodes) > 0 {
+		settings["timecodes"] = job.Config.Timecodes
+	}
+
+	decoder, err := y4m.NewDecoder(reader, settings)
 	if err != nil {
 		w.Header().Set("x-encoder-error", "")
 		w.Header().Set("x-decoder-error", err.Error())
@@ -61,7 +67,12 @@ func encodeFromReader(reader io.ReadCloser, job *Job, w http.ResponseWriter) {
 	}
 
 	//check decoder frame properties match
-	if decoder.Properties() != job.Config.Properties {
+	decProps := decoder.Properties()
+	if decProps.Width != job.Config.Properties.Width ||
+		decProps.Height != job.Config.Properties.Height ||
+		decProps.ColorSpace != job.Config.Properties.ColorSpace ||
+		decProps.FullColorRange != job.Config.Properties.FullColorRange ||
+		decProps.TimeBase() != job.Config.Properties.TimeBase() {
 		w.Header().Set("x-encoder-error", "")
 		w.Header().Set("x-decoder-error", "mismatched config properties")
 		w.WriteHeader(http.StatusBadRequest)

cli/encode-server/main.go

@@ -6,6 +6,7 @@ import (
 	encode_utils "git.gammaspectra.live/S.O.N.G/Ignite/cli/encode-utils"
 	"git.gammaspectra.live/S.O.N.G/Ignite/encoder/libaom"
 	"git.gammaspectra.live/S.O.N.G/Ignite/encoder/libx264"
+	"git.gammaspectra.live/S.O.N.G/Ignite/utilities"
 	"github.com/shirou/gopsutil/cpu"
 	"github.com/shirou/gopsutil/host"
 	"github.com/shirou/gopsutil/load"
@@ -124,6 +125,15 @@ func main() {
 			return
 		}
 
+		if cfg.TimecodesV1 != "" {
+			cfg.Timecodes, err = utilities.ParseTimecodesV1(strings.NewReader(cfg.TimecodesV1))
+			if err != nil {
+				w.WriteHeader(http.StatusBadRequest)
+				_, _ = w.Write([]byte(err.Error()))
+				return
+			}
+		}
+
 		if job := func() *Job {
 			JobsMutex.Lock()
 			defer JobsMutex.Unlock()
@@ -361,7 +371,7 @@ func main() {
 	})
 
 	s := http.Server{
-		ReadTimeout:  time.Second * 10,
+		ReadTimeout:  0,
 		IdleTimeout:  time.Second * 60,
 		WriteTimeout: 0,
 		Addr:         *listenAddr,

cli/encode-utils/config.go

@@ -1,6 +1,9 @@
 package encode_utils
 
-import "git.gammaspectra.live/S.O.N.G/Ignite/frame"
+import (
+	"git.gammaspectra.live/S.O.N.G/Ignite/frame"
+	"git.gammaspectra.live/S.O.N.G/Ignite/utilities"
+)
 
 type JobConfig struct {
 	Encoder struct {
@@ -9,6 +12,9 @@ type JobConfig struct {
 	} `json:"encoder" yaml:"encoder"`
 
 	Properties frame.StreamProperties `json:"properties" yaml:"properties"`
+
+	TimecodesV1 string              `json:"timecodes_v1" yaml:"timecodes_v1"`
+	Timecodes   utilities.Timecodes `json:"timecodes" yaml:"timecodes"`
 }
 
 const (

cli/timecodes/timecodes.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"git.gammaspectra.live/S.O.N.G/Ignite/utilities"
+	"os"
+)
+
+func main() {
+	inputPath := flag.String("input", "", "Input timecodes file")
+	inputFormat := flag.String("format", "v1", "Input format. Supported: v1")
+
+	flag.Parse()
+
+	f, err := os.Open(*inputPath)
+	if err != nil {
+		panic(err)
+	}
+	defer f.Close()
+
+	switch *inputFormat {
+	case "v1":
+		tc, err := utilities.ParseTimecodesV1(f)
+		if err != nil {
+			panic(err)
+		}
+		for _, e := range tc {
+			_, _ = fmt.Printf("%d\n", e)
+		}
+	default:
+		panic("unsupported format")
+	}
+}

decoder/y4m/y4m.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"git.gammaspectra.live/S.O.N.G/Ignite/color"
 	"git.gammaspectra.live/S.O.N.G/Ignite/frame"
+	"git.gammaspectra.live/S.O.N.G/Ignite/utilities"
 	"github.com/ulikunitz/xz"
 	"io"
 	"runtime"
@@ -25,6 +26,7 @@ type Decoder struct {
 	frameStartOffset int64
 
 	frameCounter int
+	timecodes    utilities.Timecodes
 
 	bufPool sync.Pool
 }
@@ -76,9 +78,42 @@ func NewDecoder(reader io.Reader, settings map[string]any) (*Decoder, error) {
 		}
 	}
 
+	if t, ok := settings["timecodes"]; ok {
+		if tc, ok := t.(utilities.Timecodes); ok {
+			s.timecodes = tc
+		}
+	}
+
+	// Flagged VFR
+	if s.properties.FrameRate.Numerator == 0 && s.properties.FrameRate.Denominator == 0 && len(s.timecodes) == 0 {
+		return nil, errors.New("Y4M indicates VFR but timecodes is not set")
+	}
+
+	if s.properties.PixelAspectRatio.Numerator == 0 && s.properties.PixelAspectRatio.Denominator == 0 {
+		// Assume 1:1
+		s.properties.PixelAspectRatio = utilities.NewRatio(1, 1)
+	}
+
+	s.properties.VFR = s.IsVFR()
+
 	return s, nil
 }
 
+func (s *Decoder) IsVFR() bool {
+	return len(s.timecodes) != 0
+}
+
+// FramePTS Returns -1 if not found
+func (s *Decoder) FramePTS(n int) int64 {
+	if s.IsVFR() {
+		if n >= len(s.timecodes) {
+			return -1
+		}
+		return s.timecodes[n]
+	}
+	return int64(n)
+}
+
 func (s *Decoder) Properties() frame.StreamProperties {
 	return s.properties
 }
@@ -175,14 +210,19 @@ func (s *Decoder) GetFrame() (parameters map[Parameter][]string, frameObject fra
 		return nil, nil, err
 	}
 
+	pts := s.FramePTS(s.frameCounter)
+	if pts == -1 {
+		return nil, nil, fmt.Errorf("frame %d PTS could not be calculated", s.frameCounter)
+	}
+
 	if s.properties.ColorSpace.BitDepth > 8 {
 		//it's copied below
 		defer s.bufPool.Put(buf)
-		if frameObject, err = frame.NewUint16FrameFromBytes(s.properties.FrameProperties(), int64(s.frameCounter), buf); err != nil {
+		if frameObject, err = frame.NewUint16FrameFromBytes(s.properties.FrameProperties(), pts, buf); err != nil {
 			return nil, nil, err
 		}
 	} else {
-		if f8, err := frame.NewUint8FrameFromBytes(s.properties.FrameProperties(), int64(s.frameCounter), buf); err != nil {
+		if f8, err := frame.NewUint8FrameFromBytes(s.properties.FrameProperties(), pts, buf); err != nil {
 			s.bufPool.Put(buf)
 			return nil, nil, err
 		} else {

encoder/libaom/libaom.go

@@ -131,10 +131,10 @@ func NewEncoder(w io.Writer, properties frame.StreamProperties, settings map[str
 	 * \ref RECOMMENDED method is to set the timebase to that of the parent
 	 * container or multimedia framework (ex: 1/1000 for ms, as in FLV).
 	 */
-	reciprocalFrameRate := properties.FrameRate.Reciprocal()
+	timeBase := properties.TimeBase()
 
-	e.cfg.g_timebase.num = C.int(reciprocalFrameRate.Numerator)
-	e.cfg.g_timebase.den = C.int(reciprocalFrameRate.Denominator)
+	e.cfg.g_timebase.num = C.int(timeBase.Numerator)
+	e.cfg.g_timebase.den = C.int(timeBase.Denominator)
 
 	// boolean settings
 
@@ -279,7 +279,7 @@ func NewEncoder(w io.Writer, properties frame.StreamProperties, settings map[str
 	}
 
 	var err error
-	if e.w, err = obuwriter.NewWriter(w, properties.Width, properties.Height, 0x31305641, reciprocalFrameRate); err != nil {
+	if e.w, err = obuwriter.NewWriter(w, properties.Width, properties.Height, 0x31305641, timeBase); err != nil {
 		return nil, err
 	}
 

encoder/libx264/libx264.go

@@ -102,11 +102,19 @@ func NewEncoder(w io.Writer, properties frame.StreamProperties, settings map[str
 
 	e.params.i_width = C.int(properties.Width)
 	e.params.i_height = C.int(properties.Height)
-	e.params.b_vfr_input = 0
+
+	if properties.VFR {
+		e.params.b_vfr_input = 1
+	} else {
+		e.params.b_vfr_input = 0
+	}
 	e.params.b_repeat_headers = 1
 	e.params.b_annexb = 1
-	e.params.i_fps_num = C.uint32_t(properties.FrameRate.Numerator)
-	e.params.i_fps_den = C.uint32_t(properties.FrameRate.Denominator)
+
+	timeBase := properties.TimeBase()
+
+	e.params.i_timebase_num = C.uint32_t(timeBase.Numerator)
+	e.params.i_timebase_den = C.uint32_t(timeBase.Denominator)
 	if properties.FullColorRange {
 		e.params.vui.b_fullrange = 1
 	} else {

frame/frame.go

@@ -11,7 +11,7 @@ type AllowedFrameTypes interface {
 
 type Frame interface {
 	Properties() Properties
-	// PTS usually frame number
+	// PTS usually frame number, but can differ on VFR
 	PTS() int64
 
 	// Get16 get a pixel sample in 16-bit depth

frame/stream.go

@@ -32,6 +32,16 @@ type StreamProperties struct {
 	ColorSpace     color.Space     `json:"colorspace" yaml:"colorspace"`
 	FrameRate      utilities.Ratio `json:"framerate" yaml:"framerate"`
 	FullColorRange bool            `json:"fullrange" yaml:"fullrange"`
+	VFR            bool            `json:"vfr" yaml:"vfr"`
+}
+
+func (p StreamProperties) TimeBase() utilities.Ratio {
+	timeBase := p.FrameRate.Reciprocal()
+	if p.VFR {
+		timeBase = utilities.NewRatio(1, 1000)
+	}
+
+	return timeBase
 }
 
 func (p StreamProperties) FrameProperties() Properties {

go.mod

@@ -3,6 +3,7 @@ module git.gammaspectra.live/S.O.N.G/Ignite
 go 1.21
 
 require (
+	github.com/nethruster/go-fraction v0.0.0-20221224165113-1b5f693330ad
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/stretchr/testify v1.8.1
 	github.com/ulikunitz/xz v0.5.11

go.sum

@@ -3,6 +3,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/nethruster/go-fraction v0.0.0-20221224165113-1b5f693330ad h1:HtuO+7iVoOXFaZouOdlIgT8gjN3lhk8Gfa2Eah34qSw=
+github.com/nethruster/go-fraction v0.0.0-20221224165113-1b5f693330ad/go.mod h1:pyvrvZatpiWIxRlZpeEU0DD6SKOxgLMqZV/AU4Yz6V8=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=

utilities/ratio.go

@@ -3,6 +3,7 @@ package utilities
 import (
 	"fmt"
 	"gopkg.in/yaml.v3"
+	"math"
 )
 
 type Ratio struct {
@@ -36,3 +37,36 @@ func (r Ratio) String() string {
 func (r Ratio) Reciprocal() Ratio {
 	return Ratio{Numerator: r.Denominator, Denominator: r.Numerator}
 }
+
+const NTSCRatio = float64(1001) / 1000
+
+const ratioEpsilon = 1.0e-9
+
+func NewRatio(n, d int) Ratio {
+	return Ratio{
+		Numerator:   n,
+		Denominator: d,
+	}
+}
+
+// FPSToRatio Attempt common FPS conversions to ratios
+func FPSToRatio(fps float64) Ratio {
+	// common NTSC
+	ntscValue := fps * NTSCRatio
+
+	// Number is whole NTSC
+	if math.Abs(ntscValue-math.Round(ntscValue)) < ratioEpsilon {
+		return NewRatio(int(math.Round(ntscValue))*1000, 1001)
+	}
+
+	// Number is whole other
+	if math.Abs(fps-math.Round(fps)) < ratioEpsilon {
+		return NewRatio(int(math.Round(fps)), 1)
+	}
+
+	// Get a few decimals of precision
+	return Ratio{
+		Numerator:   int(fps * 1000),
+		Denominator: 1000,
+	}
+}

utilities/timecodes.go

@@ -0,0 +1,99 @@
+package utilities
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/nethruster/go-fraction"
+	"io"
+	"strings"
+)
+
+// Timecodes Entries in milliseconds
+type Timecodes []int64
+
+const timecodesv1Header = "# timecode format v1"
+
+func ParseTimecodesV1(reader io.Reader) (Timecodes, error) {
+	var tc Timecodes
+
+	scanner := bufio.NewScanner(reader)
+
+	// Read header
+	if !scanner.Scan() {
+		return nil, io.ErrUnexpectedEOF
+	}
+	switch strings.TrimSpace(scanner.Text()) {
+	case timecodesv1Header:
+	default:
+		return nil, fmt.Errorf("unexpected line: %s", strings.TrimSpace(scanner.Text()))
+	}
+	// Read assume line
+
+	if !scanner.Scan() {
+		return nil, io.ErrUnexpectedEOF
+	}
+
+	var fps float64
+
+	var from, to int64
+
+	_, err := fmt.Sscanf(strings.ToLower(strings.TrimSpace(scanner.Text())), "assume %f", &fps)
+	if err != nil {
+		return nil, err
+	}
+
+	assumed := FPSToRatio(fps)
+	assumedFraction, err := fraction.New(assumed.Denominator, assumed.Numerator)
+	if err != nil {
+		return nil, err
+	}
+	if assumed.Denominator == 1000 {
+		assumedFraction, err = fraction.FromFloat64(1 / fps)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	running, err := fraction.New(0, 1000)
+	if err != nil {
+		return nil, err
+	}
+
+	var currentFrame int64
+
+	for scanner.Scan() {
+		line := strings.ReplaceAll(strings.TrimSpace(scanner.Text()), " ", "")
+		if line == "" || line[0] == '#' {
+			continue
+		}
+
+		_, err := fmt.Sscanf(strings.ReplaceAll(strings.TrimSpace(scanner.Text()), " ", ""), "%d,%d,%f", &from, &to, &fps)
+		if err != nil {
+			return nil, err
+		}
+
+		ratio := FPSToRatio(fps)
+		frac, err := fraction.New(ratio.Denominator, ratio.Numerator)
+		if err != nil {
+			return nil, err
+		}
+		if ratio.Denominator == 1000 {
+			frac, err = fraction.FromFloat64(1 / fps)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		for ; currentFrame < from; currentFrame++ {
+			tc = append(tc, (running.Numerator()*1000)/running.Denominator())
+			running = running.Add(assumedFraction)
+		}
+
+		for ; currentFrame <= to; currentFrame++ {
+			tc = append(tc, (running.Numerator()*1000)/running.Denominator())
+			running = running.Add(frac)
+		}
+	}
+
+	return tc, nil
+}

utilities/timecodes_test.go

@@ -0,0 +1,88 @@
+package utilities
+
+import (
+	"bytes"
+	"runtime"
+	"testing"
+)
+
+var tcTest = []byte(`# timecode format v1
+Assume 23.976023976024
+380,382,17.982017982018
+439,443,29.970029970030
+488,490,17.982017982018
+539,543,29.970029970030
+608,610,17.982017982018
+775,779,29.970029970030
+1284,1288,29.970029970030
+1481,1485,29.970029970030
+1726,1728,17.982017982018
+1985,1989,29.970029970030
+2034,2036,17.982017982018
+2153,2157,29.970029970030
+2226,2230,29.970029970030
+2419,2421,17.982017982018
+2486,2490,29.970029970030
+2891,2893,17.982017982018
+5818,5822,29.970029970030
+6407,6409,17.982017982018
+6458,6462,29.970029970030
+6715,6719,29.970029970030
+7276,7280,29.970029970030
+8873,8877,29.970029970030
+9066,9070,29.970029970030
+10083,10087,29.970029970030
+12096,12100,29.970029970030
+13077,13079,17.982017982018
+13392,13396,29.970029970030
+14213,14215,17.982017982018
+14228,14232,29.970029970030
+15941,15945,29.970029970030
+16894,16898,29.970029970030
+17079,17081,17.982017982018
+17214,17218,29.970029970030
+17255,17257,17.982017982018
+17462,17466,29.970029970030
+17691,17695,29.970029970030
+19804,19808,29.970029970030
+23665,23669,29.970029970030
+23890,23894,29.970029970030
+24411,24415,29.970029970030
+25572,25576,29.970029970030
+25637,25639,17.982017982018
+25744,25748,29.970029970030
+25809,25811,17.982017982018
+26100,26104,29.970029970030
+27533,27537,29.970029970030
+28610,28614,29.970029970030
+29099,29101,17.982017982018
+29258,29262,29.970029970030
+29619,29623,29.970029970030
+29776,29778,17.982017982018
+29995,29999,29.970029970030
+30164,30166,17.982017982018
+30191,30195,29.970029970030
+30752,30754,17.982017982018
+31995,31997,17.982017982018
+32546,32550,29.970029970030
+34243,34247,29.970029970030
+35580,35584,29.970029970030
+37581,37585,29.970029970030
+38778,38782,29.970029970030
+39735,39739,29.970029970030
+39956,39960,29.970029970030
+40121,40125,29.970029970030
+40322,40324,17.982017982018
+40349,40351,17.982017982018
+40372,40376,29.970029970030
+40677,47463,59.940059940059
+`)
+
+func TestParseTimecodesV1(t *testing.T) {
+	tc, err := ParseTimecodesV1(bytes.NewReader(tcTest))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	runtime.KeepAlive(tc)
+}